Skip to content

phsl / new-api

Go to a project
Commit 208d86e9 by CaIon
Options

feat(performance): add recent success rates to model summary and enhance metrics retrieval

parent a37ce3d6
Showing with 119 additions and 13 deletions
model/perf_metric.go
...@@ -68,6 +68,16 @@ type PerfMetricSummary struct { ...@@ -68,6 +68,16 @@ type PerfMetricSummary struct {
GenerationMs int64 `json:"generation_ms"` GenerationMs int64 `json:"generation_ms"`
} }
type PerfMetricSummaryBucket struct {
ModelName string `json:"model_name"`
BucketTs int64 `json:"bucket_ts"`
RequestCount int64 `json:"request_count"`
SuccessCount int64 `json:"success_count"`
TotalLatencyMs int64 `json:"total_latency_ms"`
OutputTokens int64 `json:"output_tokens"`
GenerationMs int64 `json:"generation_ms"`
}
func GetPerfMetricsSummaryAll(startTs int64, endTs int64, groups []string) ([]PerfMetricSummary, error) { func GetPerfMetricsSummaryAll(startTs int64, endTs int64, groups []string) ([]PerfMetricSummary, error) {
var summaries []PerfMetricSummary var summaries []PerfMetricSummary
query := DB.Model(&PerfMetric{}). query := DB.Model(&PerfMetric{}).
...@@ -86,6 +96,25 @@ func GetPerfMetricsSummaryAll(startTs int64, endTs int64, groups []string) ([]Pe ...@@ -86,6 +96,25 @@ func GetPerfMetricsSummaryAll(startTs int64, endTs int64, groups []string) ([]Pe
return summaries, err return summaries, err
} }
func GetPerfMetricsSummaryBucketsAll(startTs int64, endTs int64, groups []string) ([]PerfMetricSummaryBucket, error) {
var summaries []PerfMetricSummaryBucket
query := DB.Model(&PerfMetric{}).
Select("model_name, bucket_ts, SUM(request_count) as request_count, SUM(success_count) as success_count, SUM(total_latency_ms) as total_latency_ms, SUM(output_tokens) as output_tokens, SUM(generation_ms) as generation_ms").
Where("bucket_ts >= ? AND bucket_ts <= ?", startTs, endTs)
if groups != nil {
if len(groups) == 0 {
return summaries, nil
}
query = query.Where(commonGroupCol+" IN ?", groups)
}
err := query.
Group("model_name, bucket_ts").
Having("SUM(request_count) > 0").
Order("bucket_ts ASC").
Find(&summaries).Error
return summaries, err
}
func DeletePerfMetricsBefore(cutoffTs int64) error { func DeletePerfMetricsBefore(cutoffTs int64) error {
if cutoffTs <= 0 { if cutoffTs <= 0 {
return nil return nil
......
pkg/perf_metrics/metrics.go
...@@ -133,20 +133,23 @@ func QuerySummaryAll(hours int, groups []string) (SummaryAllResult, error) { ...@@ -133,20 +133,23 @@ func QuerySummaryAll(hours int, groups []string) (SummaryAllResult, error) {
startTs := endTs - int64(hours)*3600 startTs := endTs - int64(hours)*3600
allowedGroups := allowedGroupSet(groups) allowedGroups := allowedGroupSet(groups)
rows, err := model.GetPerfMetricsSummaryAll(startTs, endTs, groups) rows, err := model.GetPerfMetricsSummaryBucketsAll(startTs, endTs, groups)
if err != nil { if err != nil {
return SummaryAllResult{}, err return SummaryAllResult{}, err
} }
totals := map[string]counters{} totals := map[string]counters{}
modelBuckets := map[string]map[int64]counters{}
for _, row := range rows { for _, row := range rows {
totals[row.ModelName] = counters{ value := counters{
requestCount: row.RequestCount, requestCount: row.RequestCount,
successCount: row.SuccessCount, successCount: row.SuccessCount,
totalLatencyMs: row.TotalLatencyMs, totalLatencyMs: row.TotalLatencyMs,
outputTokens: row.OutputTokens, outputTokens: row.OutputTokens,
generationMs: row.GenerationMs, generationMs: row.GenerationMs,
} }
mergeModelTotals(totals, row.ModelName, value)
mergeModelBucket(modelBuckets, row.ModelName, row.BucketTs, value)
} }
hotBuckets.Range(func(key, value any) bool { hotBuckets.Range(func(key, value any) bool {
...@@ -163,13 +166,8 @@ func QuerySummaryAll(hours int, groups []string) (SummaryAllResult, error) { ...@@ -163,13 +166,8 @@ func QuerySummaryAll(hours int, groups []string) (SummaryAllResult, error) {
if snap.requestCount == 0 { if snap.requestCount == 0 {
return true return true
} }
cur := totals[k.model] mergeModelTotals(totals, k.model, snap)
cur.requestCount += snap.requestCount mergeModelBucket(modelBuckets, k.model, k.bucketTs, snap)
cur.successCount += snap.successCount
cur.totalLatencyMs += snap.totalLatencyMs
cur.outputTokens += snap.outputTokens
cur.generationMs += snap.generationMs
totals[k.model] = cur
return true return true
}) })
...@@ -189,6 +187,7 @@ func QuerySummaryAll(hours int, groups []string) (SummaryAllResult, error) { ...@@ -189,6 +187,7 @@ func QuerySummaryAll(hours int, groups []string) (SummaryAllResult, error) {
AvgLatencyMs: avgLatency, AvgLatencyMs: avgLatency,
SuccessRate: math.Round(successRate*100) / 100, SuccessRate: math.Round(successRate*100) / 100,
AvgTps: math.Round(avgTps*100) / 100, AvgTps: math.Round(avgTps*100) / 100,
RecentSuccessRates: recentSuccessRates(modelBuckets[name], 3),
RequestCount: total.requestCount, RequestCount: total.requestCount,
}) })
} }
...@@ -199,6 +198,60 @@ func QuerySummaryAll(hours int, groups []string) (SummaryAllResult, error) { ...@@ -199,6 +198,60 @@ func QuerySummaryAll(hours int, groups []string) (SummaryAllResult, error) {
return SummaryAllResult{Models: models}, nil return SummaryAllResult{Models: models}, nil
} }
func mergeModelTotals(totals map[string]counters, modelName string, value counters) {
if value.requestCount == 0 {
return
}
current := totals[modelName]
current.requestCount += value.requestCount
current.successCount += value.successCount
current.totalLatencyMs += value.totalLatencyMs
current.ttftSumMs += value.ttftSumMs
current.ttftCount += value.ttftCount
current.outputTokens += value.outputTokens
current.generationMs += value.generationMs
totals[modelName] = current
}
func mergeModelBucket(modelBuckets map[string]map[int64]counters, modelName string, bucketTs int64, value counters) {
if value.requestCount == 0 {
return
}
if _, ok := modelBuckets[modelName]; !ok {
modelBuckets[modelName] = map[int64]counters{}
}
current := modelBuckets[modelName][bucketTs]
current.requestCount += value.requestCount
current.successCount += value.successCount
current.totalLatencyMs += value.totalLatencyMs
current.ttftSumMs += value.ttftSumMs
current.ttftCount += value.ttftCount
current.outputTokens += value.outputTokens
current.generationMs += value.generationMs
modelBuckets[modelName][bucketTs] = current
}
func recentSuccessRates(buckets map[int64]counters, limit int) []float64 {
if len(buckets) == 0 || limit <= 0 {
return nil
}
timestamps := make([]int64, 0, len(buckets))
for ts := range buckets {
timestamps = append(timestamps, ts)
}
sort.Slice(timestamps, func(i, j int) bool {
return timestamps[i] < timestamps[j]
})
if len(timestamps) > limit {
timestamps = timestamps[len(timestamps)-limit:]
}
rates := make([]float64, 0, len(timestamps))
for _, ts := range timestamps {
rates = append(rates, math.Round(successRate(buckets[ts])*100)/100)
}
return rates
}
func allowedGroupSet(groups []string) map[string]struct{} { func allowedGroupSet(groups []string) map[string]struct{} {
if groups == nil { if groups == nil {
return nil return nil
......
pkg/perf_metrics/types.go
...@@ -52,6 +52,7 @@ type ModelSummary struct { ...@@ -52,6 +52,7 @@ type ModelSummary struct {
AvgLatencyMs int64 `json:"avg_latency_ms"` AvgLatencyMs int64 `json:"avg_latency_ms"`
SuccessRate float64 `json:"success_rate"` SuccessRate float64 `json:"success_rate"`
AvgTps float64 `json:"avg_tps"` AvgTps float64 `json:"avg_tps"`
RecentSuccessRates []float64 `json:"recent_success_rates,omitempty"`
RequestCount int64 `json:"-"` RequestCount int64 `json:"-"`
} }
......
web/default/src/features/performance-metrics/types.ts
...@@ -48,6 +48,7 @@ export type PerfModelSummary = { ...@@ -48,6 +48,7 @@ export type PerfModelSummary = {
avg_latency_ms: number avg_latency_ms: number
success_rate: number success_rate: number
avg_tps: number avg_tps: number
recent_success_rates?: number[]
request_count?: number request_count?: number
} }
......
web/default/src/features/pricing/components/model-perf-badge.tsx
...@@ -29,6 +29,7 @@ export type ModelPerfBadgeData = { ...@@ -29,6 +29,7 @@ export type ModelPerfBadgeData = {
avg_latency_ms: number avg_latency_ms: number
success_rate: number success_rate: number
avg_tps: number avg_tps: number
recent_success_rates?: number[]
} }
export interface ModelPerfBadgeProps extends React.HTMLAttributes<HTMLDivElement> { export interface ModelPerfBadgeProps extends React.HTMLAttributes<HTMLDivElement> {
...@@ -50,7 +51,15 @@ export const ModelPerfBadge = memo(function ModelPerfBadge( ...@@ -50,7 +51,15 @@ export const ModelPerfBadge = memo(function ModelPerfBadge(
const { avg_latency_ms, avg_tps, success_rate } = props.perf const { avg_latency_ms, avg_tps, success_rate } = props.perf
const statusColor = getSuccessRateDotClass(success_rate) const recentRates =
props.perf.recent_success_rates?.filter((rate) => Number.isFinite(rate)) ??
[]
const statusRates =
recentRates.length > 0 ? recentRates.slice(-3) : [success_rate]
const statusBars = [
...Array(Math.max(0, 3 - statusRates.length)).fill(null),
...statusRates,
].slice(-3)
return ( return (
<div <div
...@@ -83,9 +92,22 @@ export const ModelPerfBadge = memo(function ModelPerfBadge( ...@@ -83,9 +92,22 @@ export const ModelPerfBadge = memo(function ModelPerfBadge(
{t('Status short')} {t('Status short')}
</div> </div>
<div className='flex h-4 items-center justify-end gap-0.5'> <div className='flex h-4 items-center justify-end gap-0.5'>
<span className='bg-muted-foreground/10 h-2 w-1 rounded-full' /> {statusBars.map((rate, index) => (
<span className='bg-muted-foreground/15 h-2.5 w-1 rounded-full' /> <span
<span className={cn('h-3 w-1 rounded-full', statusColor)} /> key={`${index}-${rate ?? 'empty'}`}
className={cn(
'w-1 rounded-full',
index === 0 && 'h-2',
index === 1 && 'h-2.5',
index === 2 && 'h-3',
rate == null
? index === 0
? 'bg-muted-foreground/10'
: 'bg-muted-foreground/15'
: getSuccessRateDotClass(rate)
)}
/>
))}
</div> </div>
</div> </div>
</div> </div>
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment