Skip to content

phsl / new-api

Go to a project
Commit 057f71c2 by CaIon
Options

fix(logs): isolate privileged metadata

parent 73afad58
Showing with 883 additions and 221 deletions
.gitignore
......@@ -8,6 +8,8 @@ upload
*.db
build
*.db-journal
*.db-shm
*.db-wal
logs
web/dist
web/node_modules
......
controller/channel-test.go
......@@ -559,7 +559,7 @@ func settleTestQuota(info *relaycommon.RelayInfo, priceData hosttypes.PriceData,
return common.QuotaFromFloat(priceData.ModelPrice * common.QuotaPerUnit), nil
}
func buildTestLogOther(c *gin.Context, info *relaycommon.RelayInfo, priceData hosttypes.PriceData, usage *dto.Usage, tieredResult *billingexpr.TieredResult) map[string]interface{} {
func buildTestLogOther(c *gin.Context, info *relaycommon.RelayInfo, priceData hosttypes.PriceData, usage *dto.Usage, tieredResult *billingexpr.TieredResult) *model.LogOther {
other := service.GenerateTextOtherInfo(c, info, priceData.ModelRatio, priceData.GroupRatioInfo.GroupRatio, priceData.CompletionRatio,
usage.PromptTokensDetails.CachedTokens, priceData.CacheRatio, priceData.ModelPrice, priceData.GroupRatioInfo.GroupSpecialRatio)
if tieredResult != nil {
......
controller/channel_test_internal_test.go
......@@ -279,10 +279,11 @@ func TestBuildTestLogOtherInjectsTieredInfo(t *testing.T) {
RequestRules: requestRules,
})
require.Equal(t, "tiered_expr", other["billing_mode"])
require.Equal(t, "base", other["matched_tier"])
require.Equal(t, requestRules, other["request_rules"])
require.NotEmpty(t, other["expr_b64"])
fields := other.Snapshot()
require.Equal(t, "tiered_expr", fields["billing_mode"])
require.Equal(t, "base", fields["matched_tier"])
require.Equal(t, requestRules, fields["request_rules"])
require.NotEmpty(t, fields["expr_b64"])
}
func TestResolveChannelTestUserIDUsesRequestUser(t *testing.T) {
......
controller/log.go
......@@ -29,6 +29,8 @@ func GetAllLogs(c *gin.Context) {
}
if c.GetInt("role") < common.RoleRootUser {
model.FormatAdminLogs(logs)
} else {
model.FormatRootLogs(logs)
}
pageInfo.SetTotal(int(total))
pageInfo.SetItems(logs)
......
controller/relay.go
......@@ -411,41 +411,21 @@ func processChannelError(c *gin.Context, channelError types.ChannelError, err *t
modelName := c.GetString("original_model")
tokenId := c.GetInt("token_id")
userGroup := c.GetString("group")
channelId := c.GetInt("channel_id")
other := make(map[string]interface{})
other := model.NewLogOther()
if c.Request != nil && c.Request.URL != nil {
other["request_path"] = c.Request.URL.Path
}
other["error_type"] = err.GetErrorType()
other["error_code"] = err.GetErrorCode()
other["status_code"] = err.StatusCode
other["channel_id"] = channelId
other["channel_name"] = c.GetString("channel_name")
other["channel_type"] = c.GetInt("channel_type")
adminInfo := make(map[string]interface{})
adminInfo["use_channel"] = c.GetStringSlice("use_channel")
if relayInfo != nil {
if diagnostics := relayInfo.ConversionDiagnostics(); len(diagnostics) > 0 {
adminInfo["conversion_diagnostics"] = diagnostics
}
if relayInfo.ConversionDiagnosticsTruncated() {
adminInfo["conversion_diagnostics_truncated"] = true
}
}
isMultiKey := common.GetContextKeyBool(c, constant.ContextKeyChannelIsMultiKey)
if isMultiKey {
adminInfo["is_multi_key"] = true
adminInfo["multi_key_index"] = common.GetContextKeyInt(c, constant.ContextKeyChannelMultiKeyIndex)
other.SetPublic("request_path", c.Request.URL.Path)
}
service.AppendChannelAffinityAdminInfo(c, adminInfo)
other["admin_info"] = adminInfo
other.SetPublic("error_type", err.GetErrorType())
other.SetPublic("error_code", err.GetErrorCode())
other.SetPublic("status_code", err.StatusCode)
service.AppendRelayLogAdminInfo(c, relayInfo, other)
service.AppendTaskPluginContextAuditInfo(c, other)
startTime := common.GetContextKeyTime(c, constant.ContextKeyRequestStartTime)
if startTime.IsZero() {
startTime = time.Now()
}
useTimeSeconds := int(time.Since(startTime).Seconds())
model.RecordErrorLog(c, userId, channelId, modelName, tokenName, err.MaskSensitiveErrorWithStatusCode(), tokenId, useTimeSeconds, common.GetContextKeyBool(c, constant.ContextKeyIsStream), userGroup, other)
model.RecordErrorLog(c, userId, channelError.ChannelId, modelName, tokenName, err.MaskSensitiveErrorWithStatusCode(), tokenId, useTimeSeconds, common.GetContextKeyBool(c, constant.ContextKeyIsStream), userGroup, other)
}
}
......
controller/relay_error_log_test.go 0 → 100644
package controller
import (
"errors"
"net/http"
"net/http/httptest"
"testing"
"time"
"github.com/QuantumNous/new-api/common"
"github.com/QuantumNous/new-api/constant"
"github.com/QuantumNous/new-api/model"
"github.com/QuantumNous/new-api/relaykit/types"
"github.com/gin-gonic/gin"
"github.com/glebarez/sqlite"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"gorm.io/gorm"
)
func TestProcessChannelErrorUsesSnapshotWithoutLeakingChannelMetadata(t *testing.T) {
gin.SetMode(gin.TestMode)
previousDB, previousLogDB := model.DB, model.LOG_DB
previousRedisEnabled := common.RedisEnabled
previousMainDatabaseType := common.MainDatabaseType()
previousLogDatabaseType := common.LogDatabaseType()
previousErrorLogEnabled := constant.ErrorLogEnabled
database, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
require.NoError(t, err)
sqlDB, err := database.DB()
require.NoError(t, err)
sqlDB.SetMaxOpenConns(1)
require.NoError(t, database.AutoMigrate(&model.User{}, &model.Log{}))
model.DB, model.LOG_DB = database, database
common.RedisEnabled = false
common.SetDatabaseTypes(common.DatabaseTypeSQLite, common.DatabaseTypeSQLite)
constant.ErrorLogEnabled = true
t.Cleanup(func() {
model.DB, model.LOG_DB = previousDB, previousLogDB
common.RedisEnabled = previousRedisEnabled
common.SetDatabaseTypes(previousMainDatabaseType, previousLogDatabaseType)
constant.ErrorLogEnabled = previousErrorLogEnabled
require.NoError(t, sqlDB.Close())
})
require.NoError(t, database.Create(&model.User{Id: 7, Username: "log-owner", Group: "default"}).Error)
recorder := httptest.NewRecorder()
ctx, _ := gin.CreateTestContext(recorder)
ctx.Request = httptest.NewRequest(http.MethodPost, "/v1/chat/completions", nil)
ctx.Set("id", 7)
ctx.Set("username", "log-owner")
ctx.Set("token_name", "test-token")
ctx.Set("token_id", 11)
ctx.Set("original_model", "gpt-test")
ctx.Set("group", "default")
ctx.Set("channel_id", 202)
ctx.Set("channel_name", "mutable-context-channel")
ctx.Set("channel_type", 9)
ctx.Set("use_channel", []string{"101"})
common.SetContextKey(ctx, constant.ContextKeyRequestStartTime, time.Now().Add(-time.Second))
channelSnapshot := types.ChannelError{
ChannelId: 101,
ChannelType: 1,
ChannelName: "snapshot-channel",
AutoBan: false,
}
apiErr := types.NewOpenAIError(errors.New("upstream failed"), types.ErrorCodeBadResponseStatusCode, http.StatusBadGateway)
processChannelError(ctx, channelSnapshot, apiErr, nil)
var stored model.Log
require.NoError(t, database.First(&stored).Error)
assert.Equal(t, channelSnapshot.ChannelId, stored.ChannelId)
storedOther, err := common.StrToMap(stored.Other)
require.NoError(t, err)
assert.Equal(t, float64(http.StatusBadGateway), storedOther["status_code"])
for _, key := range []string{"channel_id", "channel_name", "channel_type"} {
assert.NotContains(t, storedOther, key)
}
adminInfo, ok := storedOther["admin_info"].(map[string]interface{})
require.True(t, ok)
assert.Equal(t, []interface{}{"101"}, adminInfo["use_channel"])
logs, total, err := model.GetUserLogs(7, model.LogTypeError, 0, 0, "", "", 0, 10, "", "", "")
require.NoError(t, err)
require.Equal(t, int64(1), total)
require.Len(t, logs, 1)
assert.Equal(t, channelSnapshot.ChannelId, logs[0].ChannelId)
assert.Empty(t, logs[0].ChannelName)
userOther, err := common.StrToMap(logs[0].Other)
require.NoError(t, err)
assert.NotContains(t, userOther, "admin_info")
for _, key := range []string{"channel_id", "channel_name", "channel_type"} {
assert.NotContains(t, userOther, key)
}
}
model/log.go
......@@ -116,19 +116,7 @@ func assignDisplayLogIds(logs []*Log, startIdx int) {
func formatUserLogs(logs []*Log, startIdx int) {
for i := range logs {
logs[i].ChannelName = ""
var otherMap map[string]interface{}
otherMap, _ = common.StrToMap(logs[i].Other)
if otherMap != nil {
// Remove admin-only debug fields.
delete(otherMap, "admin_info")
// Remove diagnostics reserved for root.
delete(otherMap, "root_info")
// Remove operation-audit details (operator/route info), admin-only.
delete(otherMap, "audit_info")
// delete(otherMap, "reject_reason")
// delete(otherMap, "stream_status")
}
logs[i].Other = common.MapToJsonStr(otherMap)
logs[i].Other = formatLogOtherJSON(logs[i].Other, logOtherVisibilityUser)
}
assignDisplayLogIds(logs, startIdx)
}
......@@ -137,12 +125,15 @@ func formatUserLogs(logs []*Log, startIdx int) {
// admin_info. Root callers must not pass their results through this formatter.
func FormatAdminLogs(logs []*Log) {
for i := range logs {
otherMap, _ := common.StrToMap(logs[i].Other)
if otherMap == nil {
continue
}
delete(otherMap, "root_info")
logs[i].Other = common.MapToJsonStr(otherMap)
logs[i].Other = formatLogOtherJSON(logs[i].Other, logOtherVisibilityAdmin)
}
}
// FormatRootLogs normalizes legacy metadata into the current scoped shape
// without removing root-only diagnostics.
func FormatRootLogs(logs []*Log) {
for i := range logs {
logs[i].Other = formatLogOtherJSON(logs[i].Other, logOtherVisibilityRoot)
}
}
......@@ -188,10 +179,9 @@ func RecordLogWithAdminInfo(userId int, logType int, content string, adminInfo m
Content: content,
}
if len(adminInfo) > 0 {
other := map[string]interface{}{
"admin_info": adminInfo,
}
log.Other = common.MapToJsonStr(other)
other := NewLogOther()
other.MergeAdmin(adminInfo)
log.Other = other.JSONString()
}
if err := createLog(log); err != nil {
common.SysLog("failed to record log: " + err.Error())
......@@ -216,11 +206,9 @@ func buildOpField(action string, params map[string]interface{}) map[string]inter
// content 为英文兜底文本（用于导出）；action+params 供前端本地化渲染。
// extra 可携带 login_method、user_agent 等附加信息（普通用户可见）。
func RecordLoginLog(userId int, username string, content string, ip string, action string, params map[string]interface{}, extra map[string]interface{}) {
other := map[string]interface{}{}
for k, v := range extra {
other[k] = v
}
other["op"] = buildOpField(action, params)
other := NewLogOther()
other.MergePublic(extra)
other.SetPublic("op", buildOpField(action, params))
log := &Log{
UserId: userId,
Username: username,
......@@ -228,7 +216,7 @@ func RecordLoginLog(userId int, username string, content string, ip string, acti
Type: LogTypeLogin,
Content: content,
Ip: ip,
Other: common.MapToJsonStr(other),
Other: other.JSONString(),
}
if err := createLog(log); err != nil {
common.SysLog("failed to record login log: " + err.Error())
......@@ -243,15 +231,10 @@ func RecordLoginLog(userId int, username string, content string, ip string, acti
// auditInfo 存放路由/方法/结果等中间件兜底信息（写入 Other.audit_info，普通用户查询时剥离）。
func RecordOperationAuditLog(logUserId int, content string, ip string, action string, params map[string]interface{}, adminInfo map[string]interface{}, auditInfo map[string]interface{}) {
username, _ := GetUsernameById(logUserId, false)
other := map[string]interface{}{
"op": buildOpField(action, params),
}
if len(adminInfo) > 0 {
other["admin_info"] = adminInfo
}
if len(auditInfo) > 0 {
other["audit_info"] = auditInfo
}
other := NewLogOther()
other.SetPublic("op", buildOpField(action, params))
other.MergeAdmin(adminInfo)
other.MergeAudit(auditInfo)
log := &Log{
UserId: logUserId,
Username: username,
......@@ -259,7 +242,7 @@ func RecordOperationAuditLog(logUserId int, content string, ip string, action st
Type: LogTypeManage,
Content: content,
Ip: ip,
Other: common.MapToJsonStr(other),
Other: other.JSONString(),
}
if err := createLog(log); err != nil {
common.SysLog("failed to record operation audit log: " + err.Error())
......@@ -268,17 +251,15 @@ func RecordOperationAuditLog(logUserId int, content string, ip string, action st
func RecordTopupLog(userId int, content string, callerIp string, paymentMethod string, callbackPaymentMethod string) {
username, _ := GetUsernameById(userId, false)
adminInfo := map[string]interface{}{
other := NewLogOther()
other.MergeAdmin(map[string]interface{}{
"server_ip": common.GetIp(),
"node_name": common.NodeName,
"caller_ip": callerIp,
"payment_method": paymentMethod,
"callback_payment_method": callbackPaymentMethod,
"version": common.Version,
}
other := map[string]interface{}{
"admin_info": adminInfo,
}
})
log := &Log{
UserId: userId,
Username: username,
......@@ -286,7 +267,7 @@ func RecordTopupLog(userId int, content string, callerIp string, paymentMethod s
Type: LogTypeTopup,
Content: content,
Ip: callerIp,
Other: common.MapToJsonStr(other),
Other: other.JSONString(),
}
err := createLog(log)
if err != nil {
......@@ -295,12 +276,12 @@ func RecordTopupLog(userId int, content string, callerIp string, paymentMethod s
}
func RecordErrorLog(c *gin.Context, userId int, channelId int, modelName string, tokenName string, content string, tokenId int, useTimeSeconds int,
isStream bool, group string, other map[string]interface{}) {
isStream bool, group string, other *LogOther) {
logger.LogInfo(c, fmt.Sprintf("record error log: userId=%d, channelId=%d, modelName=%s, tokenName=%s, content=%s", userId, channelId, modelName, tokenName, common.LocalLogPreview(content)))
username := c.GetString("username")
requestId := c.GetString(common.RequestIdKey)
upstreamRequestId := c.GetString(common.UpstreamRequestIdKey)
otherStr := common.MapToJsonStr(other)
otherStr := other.JSONString()
// 判断是否需要记录 IP
needRecordIp := false
if settingMap, err := GetUserSetting(userId, false); err == nil {
......@@ -341,18 +322,18 @@ func RecordErrorLog(c *gin.Context, userId int, channelId int, modelName string,
}
type RecordConsumeLogParams struct {
ChannelId int `json:"channel_id"`
PromptTokens int `json:"prompt_tokens"`
CompletionTokens int `json:"completion_tokens"`
ModelName string `json:"model_name"`
TokenName string `json:"token_name"`
Quota int `json:"quota"`
Content string `json:"content"`
TokenId int `json:"token_id"`
UseTimeSeconds int `json:"use_time_seconds"`
IsStream bool `json:"is_stream"`
Group string `json:"group"`
Other map[string]interface{} `json:"other"`
ChannelId int `json:"channel_id"`
PromptTokens int `json:"prompt_tokens"`
CompletionTokens int `json:"completion_tokens"`
ModelName string `json:"model_name"`
TokenName string `json:"token_name"`
Quota int `json:"quota"`
Content string `json:"content"`
TokenId int `json:"token_id"`
UseTimeSeconds int `json:"use_time_seconds"`
IsStream bool `json:"is_stream"`
Group string `json:"group"`
Other *LogOther `json:"other"`
}
func RecordConsumeLog(c *gin.Context, userId int, params RecordConsumeLogParams) {
......@@ -364,7 +345,7 @@ func RecordConsumeLog(c *gin.Context, userId int, params RecordConsumeLogParams)
requestId := c.GetString(common.RequestIdKey)
upstreamRequestId := c.GetString(common.UpstreamRequestIdKey)
createdAt := common.GetTimestamp()
otherStr := common.MapToJsonStr(params.Other)
otherStr := params.Other.JSONString()
// 判断是否需要记录 IP
needRecordIp := false
if settingMap, err := GetUserSetting(userId, false); err == nil {
......@@ -427,7 +408,7 @@ type RecordTaskBillingLogParams struct {
Quota int
TokenId int
Group string
Other map[string]interface{}
Other *LogOther
NodeName string // 任务发起节点；为空时回退当前节点
}
......@@ -455,7 +436,7 @@ func RecordTaskBillingLog(params RecordTaskBillingLogParams) {
ChannelId: params.ChannelId,
TokenId: params.TokenId,
Group: params.Group,
Other: common.MapToJsonStr(params.Other),
Other: params.Other.JSONString(),
}
err := createLog(log)
if err != nil {
......
model/log_format_test.go
......@@ -75,9 +75,158 @@ func TestTaskPluginLogVisibilityIsRoleSeparated(t *testing.T) {
})
t.Run("root", func(t *testing.T) {
parsed, err := common.StrToMap(other)
logs := []*Log{{Other: other}}
FormatRootLogs(logs)
parsed, err := common.StrToMap(logs[0].Other)
require.NoError(t, err)
assert.Contains(t, parsed, "admin_info")
assert.Contains(t, parsed, "root_info")
})
}
func TestLegacyLogOtherVisibilityIsRoleSeparated(t *testing.T) {
other := common.MapToJsonStr(map[string]interface{}{
"request_path": "/v1/chat/completions",
"channel_id": 202,
"channel_name": "legacy-secret-channel",
"channel_type": 1,
"reject_reason": "legacy-policy-rejection",
"admin_info": map[string]interface{}{
"existing_admin_field": "preserved",
},
"root_info": map[string]interface{}{
"upstream_request_id": "upstream-private",
},
"audit_info": map[string]interface{}{
"method": "POST",
},
})
t.Run("user", func(t *testing.T) {
logs := []*Log{{
Id: 99,
ChannelId: 77,
ChannelName: "resolved-secret-channel",
Other: other,
}}
formatUserLogs(logs, 10)
assert.Equal(t, 11, logs[0].Id)
assert.Equal(t, 77, logs[0].ChannelId)
assert.Empty(t, logs[0].ChannelName)
parsed, err := common.StrToMap(logs[0].Other)
require.NoError(t, err)
assert.Equal(t, "/v1/chat/completions", parsed["request_path"])
for _, key := range []string{
"channel_id",
"channel_name",
"channel_type",
"reject_reason",
"admin_info",
"root_info",
"audit_info",
} {
assert.NotContains(t, parsed, key)
}
})
t.Run("admin", func(t *testing.T) {
logs := []*Log{{Other: other}}
FormatAdminLogs(logs)
parsed, err := common.StrToMap(logs[0].Other)
require.NoError(t, err)
assert.Equal(t, "legacy-secret-channel", parsed["channel_name"])
assert.NotContains(t, parsed, "reject_reason")
assert.NotContains(t, parsed, "root_info")
assert.Contains(t, parsed, "audit_info")
adminInfo, ok := parsed["admin_info"].(map[string]interface{})
require.True(t, ok)
assert.Equal(t, "preserved", adminInfo["existing_admin_field"])
assert.Equal(t, "legacy-policy-rejection", adminInfo["reject_reason"])
})
t.Run("root", func(t *testing.T) {
logs := []*Log{{Other: other}}
FormatRootLogs(logs)
parsed, err := common.StrToMap(logs[0].Other)
require.NoError(t, err)
assert.Equal(t, "legacy-secret-channel", parsed["channel_name"])
assert.NotContains(t, parsed, "reject_reason")
assert.Contains(t, parsed, "root_info")
assert.Contains(t, parsed, "audit_info")
adminInfo, ok := parsed["admin_info"].(map[string]interface{})
require.True(t, ok)
assert.Equal(t, "preserved", adminInfo["existing_admin_field"])
assert.Equal(t, "legacy-policy-rejection", adminInfo["reject_reason"])
})
}
func TestLegacyRejectReasonDoesNotOverrideScopedValue(t *testing.T) {
other := common.MapToJsonStr(map[string]interface{}{
"reject_reason": "legacy-value",
"admin_info": map[string]interface{}{
"reject_reason": "scoped-value",
},
})
logs := []*Log{{Other: other}}
FormatRootLogs(logs)
parsed, err := common.StrToMap(logs[0].Other)
require.NoError(t, err)
assert.NotContains(t, parsed, "reject_reason")
adminInfo, ok := parsed["admin_info"].(map[string]interface{})
require.True(t, ok)
assert.Equal(t, "scoped-value", adminInfo["reject_reason"])
}
func TestLegacyRejectReasonHandlesNullAdminInfo(t *testing.T) {
logs := []*Log{{Other: `{"reject_reason":"legacy-value","admin_info":null}`}}
FormatAdminLogs(logs)
parsed, err := common.StrToMap(logs[0].Other)
require.NoError(t, err)
assert.NotContains(t, parsed, "reject_reason")
adminInfo, ok := parsed["admin_info"].(map[string]interface{})
require.True(t, ok)
assert.Equal(t, "legacy-value", adminInfo["reject_reason"])
}
func TestLogFormattingPreservesLargeIntegerLexemes(t *testing.T) {
const other = `{"public_id":9007199254740993,"admin_info":{"admin_id":9007199254740995},"root_info":{"generation":18446744073709551615}}`
t.Run("user", func(t *testing.T) {
logs := []*Log{{Other: other}}
formatUserLogs(logs, 0)
assert.Contains(t, logs[0].Other, `"public_id":9007199254740993`)
assert.NotContains(t, logs[0].Other, "admin_id")
assert.NotContains(t, logs[0].Other, "generation")
})
t.Run("admin", func(t *testing.T) {
logs := []*Log{{Other: other}}
FormatAdminLogs(logs)
assert.Contains(t, logs[0].Other, `"public_id":9007199254740993`)
assert.Contains(t, logs[0].Other, `"admin_id":9007199254740995`)
assert.NotContains(t, logs[0].Other, "generation")
})
t.Run("root", func(t *testing.T) {
logs := []*Log{{Other: other}}
FormatRootLogs(logs)
assert.Equal(t, other, logs[0].Other)
})
}
model/log_other.go 0 → 100644
package model
import (
"encoding/json"
"maps"
"github.com/QuantumNous/new-api/common"
)
const (
logOtherAdminInfoKey = "admin_info"
logOtherRootInfoKey = "root_info"
logOtherAuditInfoKey = "audit_info"
)
type logOtherVisibility int
const (
logOtherVisibilityUser logOtherVisibility = iota
logOtherVisibilityAdmin
logOtherVisibilityRoot
)
// LogOther separates usage-log metadata by the audience allowed to see it.
// Its maps stay private so callers cannot accidentally place privileged fields
// in the user-visible top level.
type LogOther struct {
public map[string]any
adminInfo map[string]any
rootInfo map[string]any
auditInfo map[string]any
}
func NewLogOther() *LogOther {
return &LogOther{}
}
func isReservedLogOtherKey(key string) bool {
switch key {
case logOtherAdminInfoKey, logOtherRootInfoKey, logOtherAuditInfoKey,
"channel_id", "channel_name", "channel_type", "reject_reason":
return true
default:
return false
}
}
// SetPublic records metadata that log owners may receive from self/token log APIs.
// It rejects role-scoped and legacy-sensitive keys so new writers cannot recreate
// the historical channel/reject-reason leak.
func (o *LogOther) SetPublic(key string, value any) bool {
if o == nil || key == "" || isReservedLogOtherKey(key) {
return false
}
if o.public == nil {
o.public = make(map[string]any)
}
o.public[key] = value
return true
}
func (o *LogOther) MergePublic(values map[string]any) {
for key, value := range values {
o.SetPublic(key, value)
}
}
func (o *LogOther) SetAdmin(key string, value any) bool {
if o == nil || key == "" {
return false
}
if o.adminInfo == nil {
o.adminInfo = make(map[string]any)
}
o.adminInfo[key] = value
return true
}
func (o *LogOther) MergeAdmin(values map[string]any) {
for key, value := range values {
o.SetAdmin(key, value)
}
}
func (o *LogOther) SetRoot(key string, value any) bool {
if o == nil || key == "" {
return false
}
if o.rootInfo == nil {
o.rootInfo = make(map[string]any)
}
o.rootInfo[key] = value
return true
}
func (o *LogOther) MergeRoot(values map[string]any) {
for key, value := range values {
o.SetRoot(key, value)
}
}
func (o *LogOther) SetAudit(key string, value any) bool {
if o == nil || key == "" {
return false
}
if o.auditInfo == nil {
o.auditInfo = make(map[string]any)
}
o.auditInfo[key] = value
return true
}
func (o *LogOther) MergeAudit(values map[string]any) {
for key, value := range values {
o.SetAudit(key, value)
}
}
func copyLogOtherMap(values map[string]any) map[string]any {
if len(values) == 0 {
return nil
}
copyValues := make(map[string]any, len(values))
maps.Copy(copyValues, values)
return copyValues
}
func (o *LogOther) normalizeLegacyAdminFields() {
if o == nil || o.public == nil {
return
}
if rejectReason, ok := o.public["reject_reason"]; ok {
if _, exists := o.adminInfo["reject_reason"]; !exists {
o.SetAdmin("reject_reason", rejectReason)
}
delete(o.public, "reject_reason")
}
}
func (o *LogOther) toMap(visibility logOtherVisibility) map[string]any {
result := make(map[string]any)
if o == nil {
return result
}
for key, value := range o.public {
if visibility == logOtherVisibilityUser {
switch key {
case "channel_id", "channel_name", "channel_type", "reject_reason":
continue
}
}
result[key] = value
}
if visibility >= logOtherVisibilityAdmin {
if adminInfo := copyLogOtherMap(o.adminInfo); len(adminInfo) > 0 {
result[logOtherAdminInfoKey] = adminInfo
}
if auditInfo := copyLogOtherMap(o.auditInfo); len(auditInfo) > 0 {
result[logOtherAuditInfoKey] = auditInfo
}
}
if visibility == logOtherVisibilityRoot {
if rootInfo := copyLogOtherMap(o.rootInfo); len(rootInfo) > 0 {
result[logOtherRootInfoKey] = rootInfo
}
}
return result
}
func (o *LogOther) jsonString(visibility logOtherVisibility) string {
if o == nil {
return ""
}
o.normalizeLegacyAdminFields()
data, err := common.Marshal(o.toMap(visibility))
if err != nil {
common.SysError("failed to marshal log other: " + err.Error())
return ""
}
return string(data)
}
// JSONString returns the complete stored representation, including all
// privileged scopes. API projections must use the role-specific formatter.
func (o *LogOther) JSONString() string {
return o.jsonString(logOtherVisibilityRoot)
}
// Snapshot returns a detached top-level view for tests and read-only
// inspection. Mutating it cannot add or replace fields in LogOther.
func (o *LogOther) Snapshot() map[string]any {
if o == nil {
return nil
}
return o.toMap(logOtherVisibilityRoot)
}
func (o *LogOther) MarshalJSON() ([]byte, error) {
return common.Marshal(o.toMap(logOtherVisibilityRoot))
}
func normalizeLegacyRejectReason(values map[string]json.RawMessage) bool {
rejectReason, ok := values["reject_reason"]
if !ok {
return false
}
adminInfo := make(map[string]json.RawMessage)
if rawAdminInfo, exists := values[logOtherAdminInfoKey]; exists {
_ = common.Unmarshal(rawAdminInfo, &adminInfo)
}
if adminInfo == nil {
adminInfo = make(map[string]json.RawMessage)
}
if _, exists := adminInfo["reject_reason"]; !exists {
adminInfo["reject_reason"] = rejectReason
}
encodedAdminInfo, err := common.Marshal(adminInfo)
if err != nil {
return false
}
values[logOtherAdminInfoKey] = encodedAdminInfo
delete(values, "reject_reason")
return true
}
// formatLogOtherJSON applies the role projection while keeping untouched JSON
// values as RawMessage. This preserves integers larger than JavaScript's safe
// range instead of round-tripping them through float64.
func formatLogOtherJSON(value string, visibility logOtherVisibility) string {
if value == "" {
return ""
}
var values map[string]json.RawMessage
if err := common.UnmarshalJsonStr(value, &values); err != nil {
if visibility == logOtherVisibilityRoot {
return value
}
return "{}"
}
changed := false
if visibility == logOtherVisibilityUser {
for _, key := range []string{
logOtherAdminInfoKey,
logOtherRootInfoKey,
logOtherAuditInfoKey,
"channel_id",
"channel_name",
"channel_type",
"reject_reason",
} {
if _, exists := values[key]; exists {
delete(values, key)
changed = true
}
}
} else {
changed = normalizeLegacyRejectReason(values)
if visibility == logOtherVisibilityAdmin {
if _, exists := values[logOtherRootInfoKey]; exists {
delete(values, logOtherRootInfoKey)
changed = true
}
}
}
if visibility == logOtherVisibilityRoot && !changed {
return value
}
formatted, err := common.Marshal(values)
if err != nil {
if visibility == logOtherVisibilityRoot {
return value
}
return "{}"
}
return string(formatted)
}
model/log_other_test.go 0 → 100644
package model
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestLogOtherScopesAndMerges(t *testing.T) {
var other LogOther
assert.True(t, other.SetPublic("request_path", "/v1/chat/completions"))
other.MergePublic(map[string]interface{}{
"zero": 0,
})
assert.True(t, other.SetAdmin("use_channel", []string{"channel-a"}))
other.MergeAdmin(map[string]interface{}{
"rejected": false,
})
assert.True(t, other.SetRoot("upstream_request_id", "upstream-private"))
other.MergeRoot(map[string]interface{}{
"generation": 0,
})
assert.True(t, other.SetAudit("method", "POST"))
other.MergeAudit(map[string]interface{}{
"success": false,
})
require.JSONEq(t, `{
"request_path": "/v1/chat/completions",
"zero": 0,
"admin_info": {
"use_channel": ["channel-a"],
"rejected": false
},
"root_info": {
"upstream_request_id": "upstream-private",
"generation": 0
},
"audit_info": {
"method": "POST",
"success": false
}
}`, other.JSONString())
}
func TestLogOtherRejectsSensitivePublicFields(t *testing.T) {
other := NewLogOther()
for _, key := range []string{
"admin_info",
"root_info",
"audit_info",
"channel_id",
"channel_name",
"channel_type",
"reject_reason",
} {
assert.False(t, other.SetPublic(key, "must-not-leak"), key)
}
other.MergePublic(map[string]interface{}{
"request_path": "/v1/responses",
"channel_name": "still-must-not-leak",
"admin_info": map[string]interface{}{"secret": true},
})
require.JSONEq(t, `{"request_path":"/v1/responses"}`, other.JSONString())
require.JSONEq(t, `{}`, NewLogOther().JSONString())
}
relay/convert_request_error_test.go
......@@ -57,7 +57,7 @@ func TestOptInSafeToolLossRejectedAsBadRequestWithAdminDiagnostics(t *testing.T)
assert.True(t, hasHostDiagnosticCode(diagnostics, "unsupported_hosted_tool"))
other := service.GenerateTextOtherInfo(c, info, 1, 1, 1, 0, 0, 0, 1)
adminInfo, ok := other["admin_info"].(map[string]interface{})
adminInfo, ok := other.Snapshot()["admin_info"].(map[string]interface{})
require.True(t, ok)
require.Contains(t, adminInfo, "conversion_diagnostics")
}
......
service/billing_usage.go
package service
import "github.com/QuantumNous/new-api/relaykit/dto"
import (
"github.com/QuantumNous/new-api/model"
"github.com/QuantumNous/new-api/relaykit/dto"
)
const (
usageBillingPathLocal = "local"
......@@ -50,16 +53,11 @@ func usageBillingPathForLog(isLocalCountTokens bool, usage *dto.Usage) string {
return usageBillingPathUpstream
}
func appendUsageBillingPathForLog(other map[string]interface{}, isLocalCountTokens bool, usage *dto.Usage) {
func appendUsageBillingPathForLog(other *model.LogOther, isLocalCountTokens bool, usage *dto.Usage) {
if other == nil {
return
}
adminInfo, ok := other["admin_info"].(map[string]interface{})
if !ok || adminInfo == nil {
adminInfo = make(map[string]interface{})
other["admin_info"] = adminInfo
}
adminInfo["usage_billing_path"] = usageBillingPathForLog(isLocalCountTokens, usage)
other.SetAdmin("usage_billing_path", usageBillingPathForLog(isLocalCountTokens, usage))
}
func usageFromBillingUsage(usage *dto.Usage) (*dto.Usage, bool) {
......
service/channel_affinity.go
......@@ -10,6 +10,7 @@ import (
"time"
"github.com/QuantumNous/new-api/common"
"github.com/QuantumNous/new-api/model"
"github.com/QuantumNous/new-api/pkg/cachex"
"github.com/QuantumNous/new-api/relaykit/dto"
"github.com/QuantumNous/new-api/relaykit/types"
......@@ -699,15 +700,15 @@ func MarkChannelAffinityUsed(c *gin.Context, selectedGroup string, channelID int
c.Set(ginKeyChannelAffinityLogInfo, info)
}
func AppendChannelAffinityAdminInfo(c *gin.Context, adminInfo map[string]interface{}) {
if c == nil || adminInfo == nil {
func AppendChannelAffinityAdminInfo(c *gin.Context, other *model.LogOther) {
if c == nil || other == nil {
return
}
anyInfo, ok := c.Get(ginKeyChannelAffinityLogInfo)
if !ok || anyInfo == nil {
return
}
adminInfo["channel_affinity"] = anyInfo
other.SetAdmin("channel_affinity", anyInfo)
}
func RecordChannelAffinity(c *gin.Context, channelID int) {
......
service/midjourney.go
......@@ -117,6 +117,9 @@ func RefundMidjourneyQuota(ctx context.Context, task *model.Midjourney, reason s
billingChannelId := task.GetBillingChannelId()
model.UpdateUserUsedQuota(task.UserId, -quota)
model.UpdateChannelUsedQuota(billingChannelId, -quota)
other := model.NewLogOther()
other.SetPublic("task_id", task.MjId)
other.SetPublic("reason", reason)
model.RecordTaskBillingLog(model.RecordTaskBillingLogParams{
UserId: task.UserId,
LogType: model.LogTypeRefund,
......@@ -125,10 +128,7 @@ func RefundMidjourneyQuota(ctx context.Context, task *model.Midjourney, reason s
ModelName: CovertMjpActionToModelName(task.Action),
Quota: quota,
TokenId: task.TokenId,
Other: map[string]interface{}{
"task_id": task.MjId,
"reason": reason,
},
Other: other,
})
task.Quota = 0
......
service/quota_saturation_test.go
......@@ -6,6 +6,7 @@ import (
"testing"
"github.com/QuantumNous/new-api/common"
"github.com/QuantumNous/new-api/model"
relaycommon "github.com/QuantumNous/new-api/relay/common"
"github.com/QuantumNous/new-api/relaykit/dto"
"github.com/QuantumNous/new-api/relaykit/types"
......@@ -33,10 +34,11 @@ func TestAttachQuotaSaturationNestsUnderAdminInfo(t *testing.T) {
},
}
other := map[string]interface{}{"model_price": 0.004}
other := model.NewLogOther()
other.SetPublic("model_price", 0.004)
attachQuotaSaturation(ctx, relayInfo, other)
adminInfo, ok := other["admin_info"].(map[string]interface{})
adminInfo, ok := other.Snapshot()["admin_info"].(map[string]interface{})
require.True(t, ok, "admin_info should be created")
sat, ok := adminInfo["quota_saturation"].(map[string]interface{})
require.True(t, ok, "quota_saturation should be nested under admin_info")
......@@ -76,12 +78,11 @@ func TestAttachQuotaSaturationPreservesExistingAdminInfo(t *testing.T) {
relayInfo := &relaycommon.RelayInfo{
QuotaClamp: &common.QuotaClamp{Op: "QuotaFromFloat", Kind: common.QuotaClampUnderflow, Clamped: common.MinQuota},
}
other := map[string]interface{}{
"admin_info": map[string]interface{}{"admin_username": "root"},
}
other := model.NewLogOther()
other.SetAdmin("admin_username", "root")
attachQuotaSaturation(ctx, relayInfo, other)
adminInfo := other["admin_info"].(map[string]interface{})
adminInfo := other.Snapshot()["admin_info"].(map[string]interface{})
require.Equal(t, "root", adminInfo["admin_username"], "existing admin_info fields preserved")
require.NotNil(t, adminInfo["quota_saturation"])
}
......@@ -93,10 +94,11 @@ func TestAttachQuotaSaturationNoClampNoMarker(t *testing.T) {
ctx, _ := gin.CreateTestContext(nil)
relayInfo := &relaycommon.RelayInfo{QuotaClamp: nil}
other := map[string]interface{}{"model_price": 0.004}
other := model.NewLogOther()
other.SetPublic("model_price", 0.004)
attachQuotaSaturation(ctx, relayInfo, other)
_, hasAdmin := other["admin_info"]
_, hasAdmin := other.Snapshot()["admin_info"]
require.False(t, hasAdmin, "no admin_info should be added when there is no clamp")
}
......
service/task_billing.go
......@@ -42,27 +42,27 @@ func LogTaskConsumption(c *gin.Context, info *relaycommon.RelayInfo, task *model
logContent = fmt.Sprintf("%s, 计算参数：%s", logContent, strings.Join(contents, ", "))
}
}
other := make(map[string]interface{})
other["is_task"] = true
other["request_path"] = c.Request.URL.Path
other["model_price"] = info.PriceData.ModelPrice
other := model.NewLogOther()
other.SetPublic("is_task", true)
other.SetPublic("request_path", c.Request.URL.Path)
other.SetPublic("model_price", info.PriceData.ModelPrice)
if info.PriceData.ModelRatio > 0 {
other["model_ratio"] = info.PriceData.ModelRatio
other.SetPublic("model_ratio", info.PriceData.ModelRatio)
}
other["group_ratio"] = info.PriceData.GroupRatioInfo.GroupRatio
other.SetPublic("group_ratio", info.PriceData.GroupRatioInfo.GroupRatio)
if info.PriceData.GroupRatioInfo.HasSpecialRatio {
other["user_group_ratio"] = info.PriceData.GroupRatioInfo.GroupSpecialRatio
other.SetPublic("user_group_ratio", info.PriceData.GroupRatioInfo.GroupSpecialRatio)
}
if info.IsModelMapped {
other["is_model_mapped"] = true
other["upstream_model_name"] = info.UpstreamModelName
other.SetPublic("is_model_mapped", true)
other.SetPublic("upstream_model_name", info.UpstreamModelName)
}
if snap := info.TieredBillingSnapshot; snap != nil {
other["billing_mode"] = "tiered_expr"
other["expr_b64"] = base64.StdEncoding.EncodeToString([]byte(snap.ExprString))
other["matched_tier"] = snap.EstimatedTier
other.SetPublic("billing_mode", "tiered_expr")
other.SetPublic("expr_b64", base64.StdEncoding.EncodeToString([]byte(snap.ExprString)))
other.SetPublic("matched_tier", snap.EstimatedTier)
if len(snap.UsageFacts) > 0 {
other["usage_facts"] = snap.UsageFacts
other.SetPublic("usage_facts", snap.UsageFacts)
}
}
appendTaskLogInfo(task, other)
......@@ -134,43 +134,43 @@ func taskAdjustTokenQuota(ctx context.Context, task *model.Task, delta int) {
}
// taskBillingOther 从 task 的 BillingContext 构建日志 Other 字段。
func taskBillingOther(task *model.Task) map[string]interface{} {
other := make(map[string]interface{})
func taskBillingOther(task *model.Task) *model.LogOther {
other := model.NewLogOther()
if bc := task.PrivateData.BillingContext; bc != nil {
other["model_price"] = bc.ModelPrice
other.SetPublic("model_price", bc.ModelPrice)
if bc.ModelRatio > 0 {
other["model_ratio"] = bc.ModelRatio
other.SetPublic("model_ratio", bc.ModelRatio)
}
other["group_ratio"] = bc.GroupRatio
other.SetPublic("group_ratio", bc.GroupRatio)
if priceData := taskBillingContextPriceData(bc); priceData != nil {
for k, v := range priceData.OtherRatios() {
other[k] = v
other.SetPublic(k, v)
}
}
if snap := bc.TieredSnapshot; snap != nil {
other["billing_mode"] = "tiered_expr"
other["expr_b64"] = base64.StdEncoding.EncodeToString([]byte(snap.ExprString))
other["matched_tier"] = snap.EstimatedTier
other.SetPublic("billing_mode", "tiered_expr")
other.SetPublic("expr_b64", base64.StdEncoding.EncodeToString([]byte(snap.ExprString)))
other.SetPublic("matched_tier", snap.EstimatedTier)
if len(snap.UsageFacts) > 0 {
other["usage_facts"] = snap.UsageFacts
other.SetPublic("usage_facts", snap.UsageFacts)
}
}
}
props := task.Properties
if props.UpstreamModelName != "" && props.UpstreamModelName != props.OriginModelName {
other["is_model_mapped"] = true
other["upstream_model_name"] = props.UpstreamModelName
other.SetPublic("is_model_mapped", true)
other.SetPublic("upstream_model_name", props.UpstreamModelName)
}
appendTaskLogInfo(task, other)
return other
}
func appendTaskLogInfo(task *model.Task, other map[string]interface{}) {
func appendTaskLogInfo(task *model.Task, other *model.LogOther) {
if task == nil || other == nil {
return
}
if task.TaskID != "" {
other["task_id"] = task.TaskID
other.SetPublic("task_id", task.TaskID)
}
if task.PrivateData.Execution != nil {
AppendTaskPluginAuditInfo(other, task.PrivateData.Execution.TaskPlugin)
......@@ -178,16 +178,11 @@ func appendTaskLogInfo(task *model.Task, other map[string]interface{}) {
if task.PrivateData.UpstreamTaskID == "" && task.PrivateData.NodeName == "" {
return
}
rootInfo, ok := other["root_info"].(map[string]interface{})
if !ok || rootInfo == nil {
rootInfo = map[string]interface{}{}
other["root_info"] = rootInfo
}
if task.PrivateData.UpstreamTaskID != "" {
rootInfo["upstream_task_id"] = task.PrivateData.UpstreamTaskID
other.SetRoot("upstream_task_id", task.PrivateData.UpstreamTaskID)
}
if task.PrivateData.NodeName != "" {
rootInfo["node_name"] = task.PrivateData.NodeName
other.SetRoot("node_name", task.PrivateData.NodeName)
}
}
......@@ -234,8 +229,8 @@ func RefundTaskQuota(ctx context.Context, task *model.Task, reason string) bool
// 4. 记录日志
other := taskBillingOther(task)
other["task_id"] = task.TaskID
other["reason"] = reason
other.SetPublic("task_id", task.TaskID)
other.SetPublic("reason", reason)
model.RecordTaskBillingLog(model.RecordTaskBillingLogParams{
UserId: task.UserId,
LogType: model.LogTypeRefund,
......@@ -310,9 +305,9 @@ func RecalculateTaskQuota(ctx context.Context, task *model.Task, actualQuota int
logQuota = -quotaDelta
}
other := taskBillingOther(task)
other["task_id"] = task.TaskID
other["pre_consumed_quota"] = preConsumedQuota
other["actual_quota"] = actualQuota
other.SetPublic("task_id", task.TaskID)
other.SetPublic("pre_consumed_quota", preConsumedQuota)
other.SetPublic("actual_quota", actualQuota)
for _, clamp := range clamps {
attachQuotaSaturationToOther(other, clamp)
}
......
service/task_billing_test.go
......@@ -163,6 +163,13 @@ func makeTask(userId, channelId, quota, tokenId int, billingSource string, subsc
}
}
func taskBillingOtherMap(t *testing.T, other *model.LogOther) map[string]interface{} {
t.Helper()
var values map[string]interface{}
require.NoError(t, common.UnmarshalJsonStr(other.JSONString(), &values))
return values
}
func TestPriceDataOtherRatiosFilterAndSnapshot(t *testing.T) {
priceData := types.PriceData{}
......@@ -227,7 +234,7 @@ func TestTaskBillingOtherFiltersHistoricalOtherRatios(t *testing.T) {
"inf": math.Inf(1),
}
other := taskBillingOther(task)
other := taskBillingOtherMap(t, taskBillingOther(task))
assert.Equal(t, 2.0, other["seconds"])
assert.Equal(t, 1.0, other["identity"])
......@@ -253,7 +260,7 @@ func TestTaskBillingOtherIncludesTieredSnapshotAndKeepsUsageFactsNested(t *testi
},
}
other := taskBillingOther(task)
other := taskBillingOtherMap(t, taskBillingOther(task))
assert.Equal(t, "tiered_expr", other["billing_mode"])
assert.Equal(t, base64.StdEncoding.EncodeToString([]byte(expression)), other["expr_b64"])
......@@ -262,7 +269,7 @@ func TestTaskBillingOtherIncludesTieredSnapshotAndKeepsUsageFactsNested(t *testi
require.True(t, ok)
assert.Equal(t, map[string]any{
"resolution": "720P",
"seconds": 5,
"seconds": float64(5),
}, facts)
assert.NotContains(t, other, "resolution")
assert.NotContains(t, other, "seconds")
......@@ -277,7 +284,7 @@ func TestTaskBillingOtherOmitsEmptyUsageFacts(t *testing.T) {
UsageFacts: map[string]any{},
}
other := taskBillingOther(task)
other := taskBillingOtherMap(t, taskBillingOther(task))
assert.Equal(t, "tiered_expr", other["billing_mode"])
assert.Equal(t, base64.StdEncoding.EncodeToString([]byte(expression)), other["expr_b64"])
......@@ -401,7 +408,7 @@ func TestTaskBillingOtherSeparatesPluginAndRootDiagnostics(t *testing.T) {
},
}
other := taskBillingOther(task)
other := taskBillingOtherMap(t, taskBillingOther(task))
assert.Equal(t, "task_public", other["task_id"])
adminInfo, ok := other["admin_info"].(map[string]interface{})
......@@ -421,7 +428,7 @@ func TestTaskBillingOtherSeparatesPluginAndRootDiagnostics(t *testing.T) {
assert.Equal(t, "node-a", rootInfo["node_name"])
runtimeInfo, ok := rootInfo["task_plugin"].(map[string]interface{})
require.True(t, ok)
assert.Equal(t, uint64(42), runtimeInfo["generation"])
assert.Equal(t, float64(42), runtimeInfo["generation"])
assert.NotContains(t, runtimeInfo, "author")
}
......
service/task_plugin_audit.go
......@@ -51,15 +51,10 @@ func TaskExecutionSnapshotFromContext(ctx *gin.Context) *model.TaskExecutionSnap
// AppendTaskPluginAuditInfo writes role-separated, credential-free plugin
// provenance into a usage log.
func AppendTaskPluginAuditInfo(other map[string]interface{}, snapshot *model.TaskPluginSnapshot) {
func AppendTaskPluginAuditInfo(other *model.LogOther, snapshot *model.TaskPluginSnapshot) {
if other == nil || snapshot == nil || snapshot.Key == "" {
return
}
adminInfo, ok := other["admin_info"].(map[string]interface{})
if !ok || adminInfo == nil {
adminInfo = map[string]interface{}{}
other["admin_info"] = adminInfo
}
taskPlugin := map[string]interface{}{
"key": snapshot.Key,
"name": snapshot.Name,
......@@ -72,24 +67,18 @@ func AppendTaskPluginAuditInfo(other map[string]interface{}, snapshot *model.Tas
}
taskPlugin["author"] = author
}
adminInfo["task_plugin"] = taskPlugin
rootInfo, ok := other["root_info"].(map[string]interface{})
if !ok || rootInfo == nil {
rootInfo = map[string]interface{}{}
other["root_info"] = rootInfo
}
rootInfo["task_plugin"] = map[string]interface{}{
other.SetAdmin("task_plugin", taskPlugin)
other.SetRoot("task_plugin", map[string]interface{}{
"key": snapshot.Key,
"version": snapshot.Version,
"api_version": snapshot.APIVersion,
"generation": snapshot.Generation,
}
})
}
// AppendTaskPluginContextAuditInfo is used before a task row exists, such as
// an upstream submission error log.
func AppendTaskPluginContextAuditInfo(ctx *gin.Context, other map[string]interface{}) {
func AppendTaskPluginContextAuditInfo(ctx *gin.Context, other *model.LogOther) {
execution := TaskExecutionSnapshotFromContext(ctx)
if execution == nil {
return
......
service/text_quota.go
......@@ -31,11 +31,11 @@ type ToolSurchargeItem struct {
Price float64 `json:"price"`
}
func appendToolSurchargeLogInfo(other map[string]interface{}, items []ToolSurchargeItem) {
func appendToolSurchargeLogInfo(other *model.LogOther, items []ToolSurchargeItem) {
if len(items) == 0 {
return
}
other["tool_surcharges"] = items
other.SetPublic("tool_surcharges", items)
}
type textQuotaSummary struct {
......@@ -463,7 +463,7 @@ func PostTextConsumeQuota(ctx *gin.Context, relayInfo *relaycommon.RelayInfo, us
}
logContent := strings.Join(extraContent, ", ")
var other map[string]interface{}
var other *model.LogOther
if summary.IsClaudeUsageSemantic {
other = GenerateClaudeOtherInfo(ctx, relayInfo,
summary.ModelRatio, summary.GroupRatio, summary.CompletionRatio,
......@@ -472,50 +472,50 @@ func PostTextConsumeQuota(ctx *gin.Context, relayInfo *relaycommon.RelayInfo, us
summary.CacheCreationTokens5m, summary.CacheCreationRatio5m,
summary.CacheCreationTokens1h, summary.CacheCreationRatio1h,
summary.ModelPrice, relayInfo.PriceData.GroupRatioInfo.GroupSpecialRatio)
other["usage_semantic"] = "anthropic"
other.SetPublic("usage_semantic", "anthropic")
} else {
other = GenerateTextOtherInfo(ctx, relayInfo, summary.ModelRatio, summary.GroupRatio, summary.CompletionRatio, summary.CacheTokens, summary.CacheRatio, summary.ModelPrice, relayInfo.PriceData.GroupRatioInfo.GroupSpecialRatio)
}
appendUsageBillingPathForLog(other, common.GetContextKeyBool(ctx, constant.ContextKeyLocalCountTokens), originUsage)
if adminRejectReason != "" {
other["reject_reason"] = adminRejectReason
other.SetAdmin("reject_reason", adminRejectReason)
}
if summary.ImageTokens != 0 {
other["image"] = true
other["image_ratio"] = summary.ImageRatio
other["image_output"] = summary.ImageTokens
other.SetPublic("image", true)
other.SetPublic("image_ratio", summary.ImageRatio)
other.SetPublic("image_output", summary.ImageTokens)
}
appendToolSurchargeLogInfo(other, summary.ToolSurchargeItems)
if summary.AudioInputPrice > 0 && summary.AudioTokens > 0 {
other["audio_input_seperate_price"] = true
other["audio_input_token_count"] = summary.AudioTokens
other["audio_input_price"] = summary.AudioInputPrice
other.SetPublic("audio_input_seperate_price", true)
other.SetPublic("audio_input_token_count", summary.AudioTokens)
other.SetPublic("audio_input_price", summary.AudioInputPrice)
}
if summary.CacheCreationTokens > 0 {
other["cache_creation_tokens"] = summary.CacheCreationTokens
other["cache_creation_ratio"] = summary.CacheCreationRatio
other.SetPublic("cache_creation_tokens", summary.CacheCreationTokens)
other.SetPublic("cache_creation_ratio", summary.CacheCreationRatio)
}
if summary.CacheCreationTokens5m > 0 {
other["cache_creation_tokens_5m"] = summary.CacheCreationTokens5m
other["cache_creation_ratio_5m"] = summary.CacheCreationRatio5m
other.SetPublic("cache_creation_tokens_5m", summary.CacheCreationTokens5m)
other.SetPublic("cache_creation_ratio_5m", summary.CacheCreationRatio5m)
}
if summary.CacheCreationTokens1h > 0 {
other["cache_creation_tokens_1h"] = summary.CacheCreationTokens1h
other["cache_creation_ratio_1h"] = summary.CacheCreationRatio1h
other.SetPublic("cache_creation_tokens_1h", summary.CacheCreationTokens1h)
other.SetPublic("cache_creation_ratio_1h", summary.CacheCreationRatio1h)
}
cacheWriteTokens := cacheWriteTokensTotal(summary)
if cacheWriteTokens > 0 {
// cache_write_tokens: normalized cache creation total for UI display.
// If split 5m/1h values are present, this is their sum; otherwise it falls back
// to cache_creation_tokens.
other["cache_write_tokens"] = cacheWriteTokens
other.SetPublic("cache_write_tokens", cacheWriteTokens)
}
if relayInfo.GetFinalRequestRelayFormat() != types.RelayFormatClaude && billingUsage != nil && billingUsage.UsageSource != "" && billingUsage.InputTokens > 0 {
// input_tokens_total: explicit normalized total input used by the usage log UI.
// Only write this field when upstream/current conversion has already provided a
// reliable total input value and tagged the usage source. Do not infer it from
// prompt/cache fields here, otherwise old upstream payloads may be double-counted.
other["input_tokens_total"] = billingUsage.InputTokens
other.SetPublic("input_tokens_total", billingUsage.InputTokens)
}
if tieredBillingApplied {
InjectTieredBillingInfo(other, relayInfo, tieredResult)
......
service/text_quota_test.go
......@@ -8,6 +8,7 @@ import (
"github.com/QuantumNous/new-api/common"
"github.com/QuantumNous/new-api/constant"
"github.com/QuantumNous/new-api/model"
"github.com/QuantumNous/new-api/pkg/billingexpr"
relaycommon "github.com/QuantumNous/new-api/relay/common"
relayconstant "github.com/QuantumNous/new-api/relay/constant"
......@@ -445,20 +446,21 @@ func TestUsageBillingPathForLog(t *testing.T) {
}
func TestAppendUsageBillingPathForLogWritesAdminInfo(t *testing.T) {
other := map[string]interface{}{
"admin_info": map[string]interface{}{},
}
other := model.NewLogOther()
appendUsageBillingPathForLog(other, true, &dto.Usage{
BillingUsage: dto.NewClaudeMessagesBillingUsage(&dto.ClaudeUsage{InputTokens: 1}),
})
adminInfo, ok := other["admin_info"].(map[string]interface{})
var values map[string]interface{}
require.NoError(t, common.UnmarshalJsonStr(other.JSONString(), &values))
adminInfo, ok := values["admin_info"].(map[string]interface{})
require.True(t, ok)
require.Equal(t, usageBillingPathAnthropic, adminInfo["usage_billing_path"])
other = map[string]interface{}{}
other = model.NewLogOther()
appendUsageBillingPathForLog(other, true, nil)
adminInfo, ok = other["admin_info"].(map[string]interface{})
require.NoError(t, common.UnmarshalJsonStr(other.JSONString(), &values))
adminInfo, ok = values["admin_info"].(map[string]interface{})
require.True(t, ok)
require.Equal(t, usageBillingPathLocal, adminInfo["usage_billing_path"])
}
......@@ -1118,9 +1120,9 @@ func TestCalculateTextToolCallSurchargeGeminiFunctionCall(t *testing.T) {
assert.Equal(t, 2, summary.ToolSurchargeItems[0].Count)
assert.Equal(t, 5.0, summary.ToolSurchargeItems[0].Price)
other := map[string]interface{}{}
other := model.NewLogOther()
appendToolSurchargeLogInfo(other, summary.ToolSurchargeItems)
assert.Equal(t, summary.ToolSurchargeItems, other["tool_surcharges"])
assert.Equal(t, summary.ToolSurchargeItems, other.Snapshot()["tool_surcharges"])
}
func TestCalculateTextToolCallSurchargeImageGenerationDefaultPrice(t *testing.T) {
......@@ -1214,15 +1216,16 @@ func TestAppendToolSurchargeLogInfoWritesOnlyStructuredFields(t *testing.T) {
{Name: dto.BuildInToolWebSearch, Count: 2, Price: 10},
{Name: dto.BuildInToolImageGeneration, Count: 1, Price: 150},
}
other := map[string]interface{}{}
other := model.NewLogOther()
appendToolSurchargeLogInfo(other, items)
assert.Equal(t, items, other["tool_surcharges"])
assert.NotContains(t, other, "web_search")
assert.NotContains(t, other, "web_search_call_count")
assert.NotContains(t, other, "web_search_price")
assert.NotContains(t, other, "file_search")
assert.NotContains(t, other, "image_generation_call")
assert.NotContains(t, other, "image_generation_call_price")
fields := other.Snapshot()
assert.Equal(t, items, fields["tool_surcharges"])
assert.NotContains(t, fields, "web_search")
assert.NotContains(t, fields, "web_search_call_count")
assert.NotContains(t, fields, "web_search_price")
assert.NotContains(t, fields, "file_search")
assert.NotContains(t, fields, "image_generation_call")
assert.NotContains(t, fields, "image_generation_call_price")
}
service/violation_fee.go
......@@ -134,7 +134,8 @@ func ChargeViolationFeeIfNeeded(ctx *gin.Context, relayInfo *relaycommon.RelayIn
tokenName := ctx.GetString("token_name")
oai := apiErr.ToOpenAIError()
other := map[string]any{
other := model.NewLogOther()
other.MergePublic(map[string]interface{}{
"violation_fee": true,
"violation_fee_code": string(types.ErrorCodeViolationFeeGrokCSAM),
"fee_quota": feeQuota,
......@@ -144,7 +145,7 @@ func ChargeViolationFeeIfNeeded(ctx *gin.Context, relayInfo *relaycommon.RelayIn
"upstream_error_type": oai.Type,
"upstream_error_code": fmt.Sprintf("%v", oai.Code),
"violation_fee_marker": CSAMViolationMarker,
}
})
model.RecordConsumeLog(ctx, relayInfo.UserId, model.RecordConsumeLogParams{
ChannelId: relayInfo.ChannelId,
......
web/src/features/usage-logs/components/__tests__/reject-reason.test.tsx 0 → 100644
/*
Copyright (C) 2023-2026 QuantumNous
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
For commercial licensing, please contact support@quantumnous.com
*/
import { QueryClient, QueryClientProvider } from '@tanstack/react-query'
import { render, screen } from '@testing-library/react'
import { afterEach, describe, expect, test } from 'vitest'
import type { UsageLog } from '../../data/schema'
import type { LogOtherData } from '../../types'
import { DetailsDialog } from '../dialogs/details-dialog'
const queryClients: QueryClient[] = []
function makeLog(other: LogOtherData): UsageLog {
return {
id: 1,
user_id: 1,
created_at: 1,
type: 5,
content: 'request rejected',
username: 'user',
token_name: 'token',
model_name: 'gpt-test',
quota: 0,
prompt_tokens: 0,
completion_tokens: 0,
use_time: 0,
is_stream: false,
channel: 1,
channel_name: 'channel',
token_id: 1,
group: 'default',
ip: '',
other: JSON.stringify(other),
request_id: 'req-1',
upstream_request_id: '',
}
}
function renderDetails(isAdmin: boolean): void {
const queryClient = new QueryClient({
defaultOptions: { queries: { retry: false } },
})
const freshAt = Date.now() + 60_000
queryClient.setQueryData(['status'], {}, { updatedAt: freshAt })
queryClients.push(queryClient)
render(
<QueryClientProvider client={queryClient}>
<DetailsDialog
log={makeLog({
admin_info: {
reject_reason: 'blocked by channel policy',
},
})}
isAdmin={isAdmin}
isRoot={false}
open
onOpenChange={() => undefined}
/>
</QueryClientProvider>
)
}
afterEach(() => {
for (const queryClient of queryClients) {
queryClient.clear()
}
queryClients.length = 0
})
describe('usage log reject reason', () => {
test('shows the nested admin reject reason to admins', () => {
renderDetails(true)
expect(screen.getByText('Reject Reason')).toBeInTheDocument()
expect(screen.getByText('blocked by channel policy')).toBeInTheDocument()
})
test('hides the nested admin reject reason from non-admin users', () => {
renderDetails(false)
expect(screen.queryByText('Reject Reason')).toBeNull()
expect(screen.queryByText('blocked by channel policy')).toBeNull()
})
})
web/src/features/usage-logs/components/dialogs/details-dialog.tsx
......@@ -841,13 +841,13 @@ export function DetailsDialog(props: DetailsDialogProps) {
)}
{/* Reject reason (admin only) */}
{props.isAdmin && other?.reject_reason && (
{props.isAdmin && adminInfo?.reject_reason && (
<DetailSection
icon={<AlertTriangle className='size-3.5' aria-hidden='true' />}
label={t('Reject Reason')}
variant='danger'
>
<p className='text-xs wrap-break-word'>{other.reject_reason}</p>
<p className='text-xs wrap-break-word'>{adminInfo.reject_reason}</p>
</DetailSection>
)}
......
web/src/features/usage-logs/types.ts
......@@ -142,6 +142,8 @@ export interface LogOtherData {
original: number
clamped: number
}
// Reject / intercept reason (admin only)
reject_reason?: string
task_plugin?: TaskPluginInfo
}
root_info?: {
......@@ -236,8 +238,6 @@ export interface LogOtherData {
violation_fee_code?: string
violation_fee_marker?: string
fee_quota?: number
// Reject / intercept reason (admin)
reject_reason?: string
// Task-related fields (for refund logs, type=6)
is_task?: boolean
task_id?: string
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment