chaos/chaos-api
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: collect model performance metrics (#4635)

Browse Source
This commit is contained in:
Calcium-Ion
2026-05-06 13:55:23 +08:00
committed by GitHub
parent 8b2b03d276
commit 9acf5fecae
26 changed files with 1078 additions and 120 deletions
Download Patch File Download Diff File Expand all files Collapse all files
web/default/src/features/pricing/api.ts
Vendored
+43
View File
@@ -10,3 +10,46 @@ export async function getPricing(): Promise<PricingData> {
const res = await api.get('/api/pricing')
return res.data
}
export type PerformanceSeriesPoint = {
ts: number
avg_ttft_ms: number
avg_latency_ms: number
success_rate: number
count: number
success_count: number
ttft_count: number
}
export type PerformanceGroup = {
group: string
avg_ttft_ms: number
avg_latency_ms: number
success_rate: number
request_count: number
success_count: number
ttft_count: number
series: PerformanceSeriesPoint[]
}
export type PerformanceMetricsData = {
success: boolean
message?: string
data: {
model_name: string
series_schema?: string
groups: PerformanceGroup[]
}
}
export async function getPerfMetrics(
modelName: string,
hours = 24
): Promise<PerformanceMetricsData> {
const params = new URLSearchParams({
model: modelName,
hours: String(hours),
})
const res = await api.get(`/api/perf-metrics?${params.toString()}`)
return res.data
}
web/default/src/features/pricing/components/model-details-charts.tsx
Vendored
+7
View File
@@ -14,6 +14,13 @@ function formatHourLabel(iso: string): string {
function formatDayLabel(date: string): string {
const parsed = new Date(date)
if (date.includes('T')) {
return parsed.toLocaleString(undefined, {
month: 'short',
day: 'numeric',
hour: '2-digit',
})
}
return parsed.toLocaleDateString(undefined, {
month: 'short',
day: 'numeric',
web/default/src/features/pricing/components/model-details-performance.tsx
Vendored
+133 -110
View File
@@ -1,8 +1,8 @@
import { useMemo } from 'react'
import { useQuery } from '@tanstack/react-query'
import {
Activity,
AlertTriangle,
Gauge,
HeartPulse,
Timer,
TrendingUp,
@@ -18,22 +18,14 @@ import {
TableRow,
} from '@/components/ui/table'
import { GroupBadge } from '@/components/group-badge'
import { getPerfMetrics, type PerformanceGroup } from '../api'
import {
aggregateUptime,
buildGroupPerformance,
buildLatencyTimeSeries,
buildUptimeSeries,
formatLatency,
formatThroughput,
formatUptimePct,
type UptimeDayPoint,
} from '../lib/mock-stats'
import type { PricingModel } from '../types'
import {
LatencyTrendChart,
ThroughputBarChart,
UptimeBarChart,
} from './model-details-charts'
import { LatencyTrendChart, UptimeBarChart } from './model-details-charts'
import { UptimeSparkline } from './model-details-uptime-sparkline'
const COMPACT_NUMBER = new Intl.NumberFormat(undefined, {
@@ -74,33 +66,102 @@ function StatCard(props: {
)
}
type PerformanceRow = {
group: string
avg_ttft_ms: number
avg_latency_ms: number
success_rate: number
request_count: number
}
function toLatencySeries(groups: PerformanceGroup[]) {
return groups.flatMap((group) =>
group.series
.filter((point) => point.ttft_count > 0 && point.avg_ttft_ms > 0)
.map((point) => ({
timestamp: new Date(point.ts * 1000).toISOString(),
group: group.group,
ttft_ms: point.avg_ttft_ms,
}))
)
}
function toUptimeSeries(groups: PerformanceGroup[]): UptimeDayPoint[] {
const byTs = new Map<number, { count: number; success: number }>()
for (const group of groups) {
for (const point of group.series) {
const current = byTs.get(point.ts) ?? { count: 0, success: 0 }
current.count += point.count
current.success += point.success_count
byTs.set(point.ts, current)
}
}
return Array.from(byTs.entries())
.sort(([a], [b]) => a - b)
.map(([ts, value]) => {
const uptime = value.count > 0 ? (value.success / value.count) * 100 : 0
return {
date: new Date(ts * 1000).toISOString(),
uptime_pct: Math.round(uptime * 100) / 100,
incidents: value.success < value.count ? 1 : 0,
outage_minutes: 0,
}
})
}
function toGroupUptimeSeries(group: PerformanceGroup): UptimeDayPoint[] {
return group.series.map((point) => ({
date: new Date(point.ts * 1000).toISOString(),
uptime_pct: Math.round(point.success_rate * 100) / 100,
incidents: point.success_count < point.count ? 1 : 0,
outage_minutes: 0,
}))
}
function weightedAverage(
rows: PerformanceRow[],
field: 'avg_ttft_ms' | 'avg_latency_ms'
): number {
let total = 0
let count = 0
for (const row of rows) {
if (row[field] <= 0 || row.request_count <= 0) continue
total += row[field] * row.request_count
count += row.request_count
}
return count > 0 ? Math.round(total / count) : 0
}
export function ModelDetailsPerformance(props: { model: PricingModel }) {
const { t } = useTranslation()
const performances = useMemo(
() => buildGroupPerformance(props.model),
[props.model]
)
const latencySeries = useMemo(
() => buildLatencyTimeSeries(props.model),
[props.model]
)
const uptimeSeries = useMemo(
() => buildUptimeSeries(props.model),
[props.model]
)
const aggregated = useMemo(
() => aggregateUptime(uptimeSeries),
[uptimeSeries]
const metricsQuery = useQuery({
queryKey: ['perf-metrics', props.model.model_name],
queryFn: () => getPerfMetrics(props.model.model_name, 24),
staleTime: 60 * 1000,
})
const groups = metricsQuery.data?.data.groups ?? []
const performances = useMemo<PerformanceRow[]>(
() =>
groups.map((group) => ({
group: group.group,
avg_ttft_ms: group.avg_ttft_ms,
avg_latency_ms: group.avg_latency_ms,
success_rate: group.success_rate,
request_count: group.request_count,
})),
[groups]
)
const latencySeries = useMemo(() => toLatencySeries(groups), [groups])
const uptimeSeries = useMemo(() => toUptimeSeries(groups), [groups])
const uptimeByGroup = useMemo<Record<string, UptimeDayPoint[]>>(() => {
const map: Record<string, UptimeDayPoint[]> = {}
for (const perf of performances) {
map[perf.group] = buildUptimeSeries(props.model, perf.group)
for (const group of groups) {
map[group.group] = toGroupUptimeSeries(group)
}
return map
}, [performances, props.model])
}, [groups])
if (performances.length === 0) {
if (metricsQuery.isLoading || performances.length === 0) {
return (
<div className='text-muted-foreground rounded-lg border p-6 text-center text-sm'>
{t('Performance data is not yet available for this model.')}
@@ -108,18 +169,22 @@ export function ModelDetailsPerformance(props: { model: PricingModel }) {
)
}
const bestTtft = Math.min(...performances.map((p) => p.ttft_p50_ms))
const bestThroughput = Math.max(...performances.map((p) => p.throughput_tps))
const totalRequests = performances.reduce(
(s, p) => s + p.request_volume_24h,
0
)
const intent =
aggregated.uptime_pct >= 99.9
? 'success'
: aggregated.uptime_pct >= 99
? 'default'
: 'warning'
const ttftValues = performances
.map((p) => p.avg_ttft_ms)
.filter((value) => value > 0)
const bestTtft = ttftValues.length > 0 ? Math.min(...ttftValues) : 0
const avgLatency = weightedAverage(performances, 'avg_latency_ms')
const totalRequests = performances.reduce((s, p) => s + p.request_count, 0)
const totalSuccess = groups.reduce((s, p) => s + p.success_count, 0)
const successRate =
totalRequests > 0 ? (totalSuccess / totalRequests) * 100 : 0
const incidentCount = uptimeSeries.reduce((s, p) => s + p.incidents, 0)
let intent: 'default' | 'warning' | 'success' = 'warning'
if (successRate >= 99.9) {
intent = 'success'
} else if (successRate >= 99) {
intent = 'default'
}
const headerCellClass =
'text-muted-foreground py-2 text-[10px] font-medium tracking-wider uppercase'
@@ -134,21 +199,21 @@ export function ModelDetailsPerformance(props: { model: PricingModel }) {
hint={t('Lowest median first-token latency')}
/>
<StatCard
icon={Gauge}
label={t('Peak throughput')}
value={formatThroughput(bestThroughput)}
icon={Timer}
label={t('Average latency')}
value={formatLatency(avgLatency)}
hint={t('Across all groups')}
/>
<StatCard
icon={HeartPulse}
label={t('Uptime (30d)')}
value={formatUptimePct(aggregated.uptime_pct)}
label={t('Success rate')}
value={formatUptimePct(successRate)}
hint={
aggregated.incidents > 0
? t('{{count}} incidents in the last 30 days', {
count: aggregated.incidents,
incidentCount > 0
? t('{{count}} incidents in the last 24 hours', {
count: incidentCount,
})
: t('No incidents in the last 30 days')
: t('No incidents in the last 24 hours')
}
intent={intent}
/>
@@ -164,9 +229,7 @@ export function ModelDetailsPerformance(props: { model: PricingModel }) {
<SectionHeader
icon={Activity}
title={t('Per-group performance')}
description={t(
'TTFT percentiles, throughput, and 30-day uptime by group'
)}
description={t('Average latency, TTFT, and success rate by group')}
/>
<div className='overflow-x-auto rounded-lg border'>
<Table className='text-sm'>
@@ -174,31 +237,24 @@ export function ModelDetailsPerformance(props: { model: PricingModel }) {
<TableRow className='hover:bg-transparent'>
<TableHead className={headerCellClass}>{t('Group')}</TableHead>
<TableHead className={`${headerCellClass} text-right`}>
{t('TTFT P50')}
{t('Average TTFT')}
</TableHead>
<TableHead className={`${headerCellClass} text-right`}>
{t('TTFT P95')}
</TableHead>
<TableHead className={`${headerCellClass} text-right`}>
{t('TTFT P99')}
</TableHead>
<TableHead className={`${headerCellClass} text-right`}>
{t('Throughput')}
{t('Average latency')}
</TableHead>
<TableHead
className={`${headerCellClass} min-w-[160px] text-left`}
>
{t('Uptime (30d)')}
{t('Success rate')}
</TableHead>
<TableHead className={`${headerCellClass} text-right`}>
{t('Requests / 24h')}
{t('Request Count')}
</TableHead>
</TableRow>
</TableHeader>
<TableBody>
{performances.map((perf) => {
const isBestTtft = perf.ttft_p50_ms === bestTtft
const isBestTput = perf.throughput_tps === bestThroughput
const isBestTtft = perf.avg_ttft_ms === bestTtft
return (
<TableRow key={perf.group}>
<TableCell className='py-2.5'>
@@ -210,23 +266,10 @@ export function ModelDetailsPerformance(props: { model: PricingModel }) {
isBestTtft && 'text-emerald-600 dark:text-emerald-400'
)}
>
{formatLatency(perf.ttft_p50_ms)}
{formatLatency(perf.avg_ttft_ms)}
</TableCell>
<TableCell className='text-muted-foreground py-2.5 text-right font-mono'>
{formatLatency(perf.ttft_p95_ms)}
</TableCell>
<TableCell className='text-muted-foreground py-2.5 text-right font-mono'>
{formatLatency(perf.ttft_p99_ms)}
</TableCell>
<TableCell
className={cn(
'py-2.5 text-right font-mono',
isBestTput &&
perf.throughput_tps > 0 &&
'text-emerald-600 dark:text-emerald-400'
)}
>
{formatThroughput(perf.throughput_tps)}
{formatLatency(perf.avg_latency_ms)}
</TableCell>
<TableCell className='py-2.5'>
<UptimeSparkline
@@ -235,7 +278,7 @@ export function ModelDetailsPerformance(props: { model: PricingModel }) {
/>
</TableCell>
<TableCell className='text-muted-foreground py-2.5 text-right font-mono'>
{COMPACT_NUMBER.format(perf.request_volume_24h)}
{COMPACT_NUMBER.format(perf.request_count)}
</TableCell>
</TableRow>
)
@@ -249,45 +292,31 @@ export function ModelDetailsPerformance(props: { model: PricingModel }) {
<SectionHeader
icon={Timer}
title={t('Latency trend (last 24h)')}
description={t(
'Median time-to-first-token (TTFT) sampled hourly per group'
)}
description={t('Average time-to-first-token (TTFT) by group')}
/>
<LatencyTrendChart series={latencySeries} />
</section>
{bestThroughput > 0 && (
<section>
<SectionHeader
icon={Gauge}
title={t('Throughput by group')}
description={t('Average tokens per second sustained per group')}
/>
<ThroughputBarChart rows={performances} />
</section>
)}
<section>
<SectionHeader
icon={HeartPulse}
title={t('Uptime (last 30 days)')}
title={t('Availability (last 24h)')}
description={
aggregated.incidents > 0
incidentCount > 0
? t(
'Daily uptime; {{incidents}} incidents totalling {{minutes}} minutes',
'Request success rate; {{incidents}} incident buckets in the last 24 hours',
{
incidents: aggregated.incidents,
minutes: aggregated.outage_minutes,
incidents: incidentCount,
}
)
: t('Daily uptime over the last 30 days')
: t('Request success rate sampled over the last 24 hours')
}
accent={
aggregated.incidents > 0 ? (
incidentCount > 0 ? (
<span className='inline-flex items-center gap-1 text-amber-600 dark:text-amber-400'>
<AlertTriangle className='size-3.5' />
{t('{{count}} incidents', {
count: aggregated.incidents,
count: incidentCount,
})}
</span>
) : null
@@ -295,12 +324,6 @@ export function ModelDetailsPerformance(props: { model: PricingModel }) {
/>
<UptimeBarChart series={uptimeSeries} />
</section>
<p className='text-muted-foreground/60 text-[11px] leading-relaxed'>
{t(
'Performance metrics shown here are simulated for preview purposes and will be replaced with live observability data once the backend integration is complete.'
)}
</p>
</div>
)
}
web/default/src/features/pricing/components/model-details.tsx
Vendored
-8
View File
@@ -41,7 +41,6 @@ import {
isDynamicPricingModel,
} from '../lib/dynamic-price'
import { parseTags } from '../lib/filters'
import { buildUptimeSeries } from '../lib/mock-stats'
import {
getAvailableGroups,
isTokenBasedModel,
@@ -57,7 +56,6 @@ import { ModelDetailsCapabilities } from './model-details-capabilities'
import { ModalitiesMatrix } from './model-details-modalities'
import { ModelDetailsPerformance } from './model-details-performance'
import { ModelDetailsQuickStats } from './model-details-quick-stats'
import { UptimeStatusRow } from './model-details-uptime-sparkline'
// ----------------------------------------------------------------------------
// Local UI helpers
@@ -782,10 +780,6 @@ export function ModelDetailsContent(props: ModelDetailsContentProps) {
const { t } = useTranslation()
const showRechargePrice = props.showRechargePrice ?? false
const metadata = useMemo(() => inferModelMetadata(props.model), [props.model])
const uptimeSeries = useMemo(
() => buildUptimeSeries(props.model),
[props.model]
)
const isDynamic =
props.model.billing_mode === 'tiered_expr' &&
@@ -797,8 +791,6 @@ export function ModelDetailsContent(props: ModelDetailsContentProps) {
<ModelDetailsQuickStats metadata={metadata} />
<UptimeStatusRow series={uptimeSeries} />
<Tabs defaultValue='overview' className='gap-4'>
<TabsList className='bg-muted/60 h-auto w-full justify-start gap-1 overflow-x-auto rounded-lg p-1'>
{TAB_VALUES.map((value) => {
web/default/src/features/system-settings/maintenance/config.ts
Vendored
+4
View File
@@ -75,6 +75,10 @@ export const DEFAULT_MAINTENANCE_SETTINGS: MaintenanceSettings = {
'performance_setting.monitor_cpu_threshold': 90,
'performance_setting.monitor_memory_threshold': 90,
'performance_setting.monitor_disk_threshold': 95,
'perf_metrics_setting.enabled': true,
'perf_metrics_setting.flush_interval': 5,
'perf_metrics_setting.bucket_time': 'hour',
'perf_metrics_setting.retention_days': 0,
}
const toBoolean = (value: unknown, fallback: boolean): boolean => {
web/default/src/features/system-settings/maintenance/performance-section.tsx
Vendored
+96
View File
@@ -59,6 +59,10 @@ const perfSchema = z.object({
.number()
.min(0)
.max(100),
'perf_metrics_setting.enabled': z.boolean(),
'perf_metrics_setting.flush_interval': z.coerce.number().min(1),
'perf_metrics_setting.bucket_time': z.enum(['minute', '5min', 'hour']),
'perf_metrics_setting.retention_days': z.coerce.number().min(0),
})
type PerfFormValues = z.infer<typeof perfSchema>
@@ -248,6 +252,7 @@ export function PerformanceSection(props: Props) {
const diskEnabled = form.watch('performance_setting.disk_cache_enabled')
const monitorEnabled = form.watch('performance_setting.monitor_enabled')
const perfMetricsEnabled = form.watch('perf_metrics_setting.enabled')
const maxCacheSizeMb = form.watch(
'performance_setting.disk_cache_max_size_mb'
)
@@ -452,6 +457,97 @@ export function PerformanceSection(props: Props) {
/>
</div>
<Separator />
<div>
<h4 className='font-medium'>{t('Model performance metrics')}</h4>
<p className='text-muted-foreground mt-1 text-xs'>
{t(
'Collect relay latency and success-rate metrics for the model square.'
)}
</p>
</div>
<div className='grid grid-cols-1 gap-4 md:grid-cols-4'>
<FormField
control={form.control}
name='perf_metrics_setting.enabled'
render={({ field }) => (
<FormItem className='flex items-center gap-2'>
<FormControl>
<Switch
checked={field.value}
onCheckedChange={field.onChange}
/>
</FormControl>
<FormLabel>{t('Enable model performance metrics')}</FormLabel>
</FormItem>
)}
/>
<FormField
control={form.control}
name='perf_metrics_setting.flush_interval'
render={({ field }) => (
<FormItem>
<FormLabel>{t('Flush interval (minutes)')}</FormLabel>
<FormControl>
<Input
type='number'
min={1}
{...field}
disabled={!perfMetricsEnabled}
/>
</FormControl>
</FormItem>
)}
/>
<FormField
control={form.control}
name='perf_metrics_setting.bucket_time'
render={({ field }) => (
<FormItem>
<FormLabel>{t('Aggregation bucket')}</FormLabel>
<Select
value={field.value}
onValueChange={field.onChange}
disabled={!perfMetricsEnabled}
>
<FormControl>
<SelectTrigger>
<SelectValue />
</SelectTrigger>
</FormControl>
<SelectContent>
<SelectItem value='minute'>{t('1 minute')}</SelectItem>
<SelectItem value='5min'>{t('5 minutes')}</SelectItem>
<SelectItem value='hour'>{t('1 hour')}</SelectItem>
</SelectContent>
</Select>
</FormItem>
)}
/>
<FormField
control={form.control}
name='perf_metrics_setting.retention_days'
render={({ field }) => (
<FormItem>
<FormLabel>{t('Retention days')}</FormLabel>
<FormControl>
<Input
type='number'
min={0}
{...field}
disabled={!perfMetricsEnabled}
/>
</FormControl>
<FormDescription>
{t('0 means data is kept permanently')}
</FormDescription>
</FormItem>
)}
/>
</div>
<Button type='submit' disabled={updateOption.isPending}>
{updateOption.isPending ? t('Saving...') : t('Save Changes')}
</Button>
web/default/src/features/system-settings/maintenance/section-registry.tsx
Vendored
+8
View File
@@ -102,6 +102,14 @@ const MAINTENANCE_SECTIONS = [
settings['performance_setting.monitor_memory_threshold'] ?? 90,
'performance_setting.monitor_disk_threshold':
settings['performance_setting.monitor_disk_threshold'] ?? 95,
'perf_metrics_setting.enabled':
settings['perf_metrics_setting.enabled'] ?? true,
'perf_metrics_setting.flush_interval':
settings['perf_metrics_setting.flush_interval'] ?? 5,
'perf_metrics_setting.bucket_time':
settings['perf_metrics_setting.bucket_time'] ?? 'hour',
'perf_metrics_setting.retention_days':
settings['perf_metrics_setting.retention_days'] ?? 0,
}}
/>
),
web/default/src/features/system-settings/types.ts
Vendored
+4
View File
@@ -254,6 +254,10 @@ export type MaintenanceSettings = {
'performance_setting.monitor_cpu_threshold': number
'performance_setting.monitor_memory_threshold': number
'performance_setting.monitor_disk_threshold': number
'perf_metrics_setting.enabled': boolean
'perf_metrics_setting.flush_interval': number
'perf_metrics_setting.bucket_time': 'hour' | 'minute' | '5min'
'perf_metrics_setting.retention_days': number
}
export type RequestLimitsSettings = {