feat: add system instance info panel (#5716)

* feat: add system instance reporting

* feat: show system instance resources

* fix: update translations for heartbeat messages in Russian and Vietnamese

controller/system_info.go

+package controller
+
+import (
+	"net/http"
+
+	"github.com/QuantumNous/new-api/common"
+	"github.com/QuantumNous/new-api/model"
+
+	"github.com/gin-gonic/gin"
+)
+
+func ListSystemInstances(c *gin.Context) {
+	instances, err := model.ListSystemInstances()
+	if err != nil {
+		common.ApiError(c, err)
+		return
+	}
+
+	now := common.GetTimestamp()
+	responses := make([]model.SystemInstanceResponse, 0, len(instances))
+	for _, instance := range instances {
+		responses = append(responses, instance.ToResponse(now))
+	}
+
+	c.JSON(http.StatusOK, gin.H{
+		"success": true,
+		"message": "",
+		"data":    responses,
+	})
+}

main.go

@@ -117,6 +117,10 @@ func main() {
 	// Subscription quota reset task (daily/weekly/monthly/custom)
 	// Subscription quota reset task (daily/weekly/monthly/custom)
 	service.StartSubscriptionQuotaResetTask()
 	service.StartSubscriptionQuotaResetTask()
 
 
+	// Report this process as a system instance so the System Info page can show
+	// all currently alive nodes in multi-instance deployments.
+	service.StartSystemInstanceReporter()
+
 	// Wire task polling adaptor factory (breaks service -> relay import cycle).
 	// Wire task polling adaptor factory (breaks service -> relay import cycle).
 	// Must run before the system task runner starts: the async_task_poll handler
 	// Must run before the system task runner starts: the async_task_poll handler
 	// calls service.RunTaskPollingOnce, which needs this factory set.
 	// calls service.RunTaskPollingOnce, which needs this factory set.

model/main.go

@@ -294,6 +294,7 @@ func migrateDB() error {
 		&CustomOAuthProvider{},
 		&CustomOAuthProvider{},
 		&UserOAuthBinding{},
 		&UserOAuthBinding{},
 		&PerfMetric{},
 		&PerfMetric{},
+		&SystemInstance{},
 		&SystemTask{},
 		&SystemTask{},
 		&SystemTaskLock{},
 		&SystemTaskLock{},
 	)
 	)
@@ -345,6 +346,7 @@ func migrateDBFast() error {
 		{&CustomOAuthProvider{}, "CustomOAuthProvider"},
 		{&CustomOAuthProvider{}, "CustomOAuthProvider"},
 		{&UserOAuthBinding{}, "UserOAuthBinding"},
 		{&UserOAuthBinding{}, "UserOAuthBinding"},
 		{&PerfMetric{}, "PerfMetric"},
 		{&PerfMetric{}, "PerfMetric"},
+		{&SystemInstance{}, "SystemInstance"},
 		{&SystemTask{}, "SystemTask"},
 		{&SystemTask{}, "SystemTask"},
 		{&SystemTaskLock{}, "SystemTaskLock"},
 		{&SystemTaskLock{}, "SystemTaskLock"},
 	}
 	}

model/system_instance.go

+package model
+
+import (
+	"github.com/QuantumNous/new-api/common"
+
+	"gorm.io/gorm"
+	"gorm.io/gorm/clause"
+)
+
+const (
+	SystemInstanceStatusOnline = "online"
+	SystemInstanceStatusStale  = "stale"
+
+	SystemInstanceStaleAfterSeconds int64 = 90
+)
+
+type SystemInstance struct {
+	NodeName   string `json:"node_name" gorm:"type:varchar(128);primaryKey"`
+	Info       string `json:"info" gorm:"type:text"`
+	StartedAt  int64  `json:"started_at" gorm:"bigint;index"`
+	LastSeenAt int64  `json:"last_seen_at" gorm:"bigint;index"`
+	CreatedAt  int64  `json:"created_at" gorm:"bigint;index"`
+	UpdatedAt  int64  `json:"updated_at" gorm:"bigint;index"`
+}
+
+type SystemInstanceResponse struct {
+	NodeName          string `json:"node_name"`
+	Status            string `json:"status"`
+	StaleAfterSeconds int64  `json:"stale_after_seconds"`
+	StartedAt         int64  `json:"started_at"`
+	LastSeenAt        int64  `json:"last_seen_at"`
+	Info              any    `json:"info"`
+}
+
+func (instance *SystemInstance) BeforeCreate(_ *gorm.DB) error {
+	now := common.GetTimestamp()
+	if instance.CreatedAt == 0 {
+		instance.CreatedAt = now
+	}
+	if instance.UpdatedAt == 0 {
+		instance.UpdatedAt = now
+	}
+	return nil
+}
+
+func UpsertSystemInstance(nodeName string, info any, startedAt int64, lastSeenAt int64) error {
+	infoText, err := marshalSystemInstanceInfo(info)
+	if err != nil {
+		return err
+	}
+	if lastSeenAt == 0 {
+		lastSeenAt = common.GetTimestamp()
+	}
+	instance := &SystemInstance{
+		NodeName:   nodeName,
+		Info:       infoText,
+		StartedAt:  startedAt,
+		LastSeenAt: lastSeenAt,
+		UpdatedAt:  lastSeenAt,
+	}
+	return DB.Clauses(clause.OnConflict{
+		Columns: []clause.Column{{Name: "node_name"}},
+		DoUpdates: clause.AssignmentColumns([]string{
+			"info",
+			"started_at",
+			"last_seen_at",
+			"updated_at",
+		}),
+	}).Create(instance).Error
+}
+
+func ListSystemInstances() ([]*SystemInstance, error) {
+	var instances []*SystemInstance
+	err := DB.Order("last_seen_at desc").Find(&instances).Error
+	return instances, err
+}
+
+func (instance *SystemInstance) ToResponse(now int64) SystemInstanceResponse {
+	status := SystemInstanceStatusOnline
+	if now-instance.LastSeenAt > SystemInstanceStaleAfterSeconds {
+		status = SystemInstanceStatusStale
+	}
+	return SystemInstanceResponse{
+		NodeName:          instance.NodeName,
+		Status:            status,
+		StaleAfterSeconds: SystemInstanceStaleAfterSeconds,
+		StartedAt:         instance.StartedAt,
+		LastSeenAt:        instance.LastSeenAt,
+		Info:              decodeSystemInstanceInfo(instance.Info),
+	}
+}
+
+func marshalSystemInstanceInfo(v any) (string, error) {
+	if v == nil {
+		return "", nil
+	}
+	data, err := common.Marshal(v)
+	if err != nil {
+		return "", err
+	}
+	return string(data), nil
+}
+
+func decodeSystemInstanceInfo(data string) any {
+	if data == "" {
+		return nil
+	}
+	var value any
+	if err := common.UnmarshalJsonStr(data, &value); err != nil {
+		return data
+	}
+	return value
+}

model/task_cas_test.go

@@ -48,6 +48,7 @@ func TestMain(m *testing.M) {
 		&UserSubscription{},
 		&UserSubscription{},
 		&UserOAuthBinding{},
 		&UserOAuthBinding{},
 		&PerfMetric{},
 		&PerfMetric{},
+		&SystemInstance{},
 		&SystemTask{},
 		&SystemTask{},
 		&SystemTaskLock{},
 		&SystemTaskLock{},
 	); err != nil {
 	); err != nil {
@@ -73,6 +74,7 @@ func truncateTables(t *testing.T) {
 		DB.Exec("DELETE FROM user_subscriptions")
 		DB.Exec("DELETE FROM user_subscriptions")
 		DB.Exec("DELETE FROM user_oauth_bindings")
 		DB.Exec("DELETE FROM user_oauth_bindings")
 		DB.Exec("DELETE FROM perf_metrics")
 		DB.Exec("DELETE FROM perf_metrics")
+		DB.Exec("DELETE FROM system_instances")
 		DB.Exec("DELETE FROM system_task_locks")
 		DB.Exec("DELETE FROM system_task_locks")
 		DB.Exec("DELETE FROM system_tasks")
 		DB.Exec("DELETE FROM system_tasks")
 	})
 	})

router/api-router.go

@@ -322,6 +322,11 @@ func SetApiRouter(router *gin.Engine) {
 			systemTaskRoute.GET("/current", controller.GetCurrentSystemTask)
 			systemTaskRoute.GET("/current", controller.GetCurrentSystemTask)
 			systemTaskRoute.GET("/:task_id", controller.GetSystemTask)
 			systemTaskRoute.GET("/:task_id", controller.GetSystemTask)
 		}
 		}
+		systemInfoRoute := apiRouter.Group("/system-info")
+		systemInfoRoute.Use(middleware.RootAuth())
+		{
+			systemInfoRoute.GET("/instances", controller.ListSystemInstances)
+		}
 
 
 		dataRoute := apiRouter.Group("/data")
 		dataRoute := apiRouter.Group("/data")
 		dataRoute.GET("/", middleware.AdminAuth(), controller.GetAllQuotaDates)
 		dataRoute.GET("/", middleware.AdminAuth(), controller.GetAllQuotaDates)

service/system_instance.go

+package service
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"runtime"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/QuantumNous/new-api/common"
+	"github.com/QuantumNous/new-api/logger"
+	"github.com/QuantumNous/new-api/model"
+
+	"github.com/bytedance/gopkg/util/gopool"
+)
+
+const systemInstanceReportInterval = 30 * time.Second
+
+var systemInstanceReporterOnce sync.Once
+
+type SystemInstanceInfo struct {
+	SchemaVersion int                       `json:"schema_version"`
+	Node          common.NodeIdentity       `json:"node"`
+	Role          SystemInstanceRoleInfo    `json:"role"`
+	Runtime       SystemInstanceRuntimeInfo `json:"runtime"`
+	Host          SystemInstanceHostInfo    `json:"host"`
+	Resources     SystemInstanceResources   `json:"resources,omitempty"`
+	Extra         map[string]any            `json:"extra,omitempty"`
+}
+
+type SystemInstanceRoleInfo struct {
+	IsMaster bool `json:"is_master"`
+}
+
+type SystemInstanceRuntimeInfo struct {
+	Version   string `json:"version"`
+	GOOS      string `json:"goos"`
+	GOARCH    string `json:"goarch"`
+	StartedAt int64  `json:"started_at"`
+}
+
+type SystemInstanceHostInfo struct {
+	Hostname string `json:"hostname"`
+}
+
+type SystemInstanceResources struct {
+	CPU     SystemInstanceResourceUsage  `json:"cpu"`
+	Memory  SystemInstanceResourceUsage  `json:"memory"`
+	Storage SystemInstanceStorageMetrics `json:"storage"`
+}
+
+type SystemInstanceResourceUsage struct {
+	UsagePercent float64 `json:"usage_percent"`
+}
+
+type SystemInstanceStorageMetrics struct {
+	TotalBytes  uint64  `json:"total_bytes"`
+	UsedBytes   uint64  `json:"used_bytes"`
+	FreeBytes   uint64  `json:"free_bytes"`
+	UsedPercent float64 `json:"used_percent"`
+}
+
+func StartSystemInstanceReporter() {
+	systemInstanceReporterOnce.Do(func() {
+		gopool.Go(func() {
+			reportSystemInstanceWithLog()
+
+			ticker := time.NewTicker(systemInstanceReportInterval)
+			defer ticker.Stop()
+			for range ticker.C {
+				reportSystemInstanceWithLog()
+			}
+		})
+	})
+}
+
+func ReportCurrentSystemInstance() error {
+	identity := common.GetNodeIdentity()
+	hostname, hostnameErr := os.Hostname()
+	if strings.TrimSpace(identity.Name) == "" {
+		if hostnameErr != nil || strings.TrimSpace(hostname) == "" {
+			return fmt.Errorf("system instance node name is empty")
+		}
+		identity.Name = hostname
+		identity.Source = common.NodeNameSourceHostname
+		identity.ManuallyConfigured = false
+		identity.ShouldConfigureManually = true
+	}
+	systemStatus := common.GetSystemStatus()
+	diskInfo := common.GetDiskSpaceInfo()
+	info := SystemInstanceInfo{
+		SchemaVersion: 1,
+		Node:          identity,
+		Role: SystemInstanceRoleInfo{
+			IsMaster: common.IsMasterNode,
+		},
+		Runtime: SystemInstanceRuntimeInfo{
+			Version:   common.Version,
+			GOOS:      runtime.GOOS,
+			GOARCH:    runtime.GOARCH,
+			StartedAt: common.StartTime,
+		},
+		Host: SystemInstanceHostInfo{
+			Hostname: hostname,
+		},
+		Resources: SystemInstanceResources{
+			CPU: SystemInstanceResourceUsage{
+				UsagePercent: systemStatus.CPUUsage,
+			},
+			Memory: SystemInstanceResourceUsage{
+				UsagePercent: systemStatus.MemoryUsage,
+			},
+			Storage: SystemInstanceStorageMetrics{
+				TotalBytes:  diskInfo.Total,
+				UsedBytes:   diskInfo.Used,
+				FreeBytes:   diskInfo.Free,
+				UsedPercent: diskInfo.UsedPercent,
+			},
+		},
+	}
+	return model.UpsertSystemInstance(identity.Name, info, common.StartTime, common.GetTimestamp())
+}
+
+func reportSystemInstanceWithLog() {
+	if err := ReportCurrentSystemInstance(); err != nil {
+		logger.LogWarn(context.Background(), fmt.Sprintf("system instance report failed: %v", err))
+	}
+}

web/default/src/features/system-info/api.ts

+/*
+Copyright (C) 2023-2026 QuantumNous
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+For commercial licensing, please contact support@quantumnous.com
+*/
+import { api } from '@/lib/api'
+import type { SystemInstanceListResponse } from './types'
+
+export async function listSystemInstances() {
+  const res = await api.get<SystemInstanceListResponse>(
+    '/api/system-info/instances'
+  )
+  return res.data
+}

web/default/src/features/system-info/components/system-tasks-panel.tsx

@@ -19,7 +19,7 @@ For commercial licensing, please contact support@quantumnous.com
 import { useQuery } from '@tanstack/react-query'
 import { useQuery } from '@tanstack/react-query'
 import { ListChecks, RefreshCw } from 'lucide-react'
 import { ListChecks, RefreshCw } from 'lucide-react'
 import { useTranslation } from 'react-i18next'
 import { useTranslation } from 'react-i18next'
-import { formatTimestampToDate } from '@/lib/format'
+import { formatTimestampRelative, formatTimestampToDate } from '@/lib/format'
 import { cn } from '@/lib/utils'
 import { cn } from '@/lib/utils'
 import { ErrorState } from '@/components/error-state'
 import { ErrorState } from '@/components/error-state'
 import { Badge } from '@/components/ui/badge'
 import { Badge } from '@/components/ui/badge'
@@ -98,7 +98,7 @@ type SystemTasksTableProps = {
 }
 }
 
 
 function SystemTasksTable(props: SystemTasksTableProps) {
 function SystemTasksTable(props: SystemTasksTableProps) {
-  const { t } = useTranslation()
+  const { t, i18n } = useTranslation()
 
 
   return (
   return (
     <div className='overflow-x-auto rounded-md border'>
     <div className='overflow-x-auto rounded-md border'>
@@ -172,8 +172,15 @@ function SystemTasksTable(props: SystemTasksTableProps) {
                 <TableCell className='text-muted-foreground max-w-[280px] truncate py-3 font-mono text-xs align-middle'>
                 <TableCell className='text-muted-foreground max-w-[280px] truncate py-3 font-mono text-xs align-middle'>
                   {task.locked_by || '-'}
                   {task.locked_by || '-'}
                 </TableCell>
                 </TableCell>
-                <TableCell className='text-muted-foreground py-3 text-xs whitespace-nowrap align-middle'>
-                  {formatTimestampToDate(task.updated_at)}
+                <TableCell
+                  className='text-muted-foreground py-3 text-xs whitespace-nowrap align-middle'
+                  title={formatTimestampToDate(task.updated_at)}
+                >
+                  {formatTimestampRelative(
+                    task.updated_at,
+                    'seconds',
+                    i18n.language
+                  )}
                 </TableCell>
                 </TableCell>
                 <TableCell
                 <TableCell
                   className='text-destructive max-w-[220px] truncate py-3 pr-4 text-xs align-middle'
                   className='text-destructive max-w-[220px] truncate py-3 pr-4 text-xs align-middle'

web/default/src/features/system-info/index.tsx

@@ -18,6 +18,8 @@ For commercial licensing, please contact support@quantumnous.com
 */
 */
 import { useTranslation } from 'react-i18next'
 import { useTranslation } from 'react-i18next'
 import { SectionPageLayout } from '@/components/layout'
 import { SectionPageLayout } from '@/components/layout'
+import { Badge } from '@/components/ui/badge'
+import { SystemInstancesPanel } from './components/system-instances-panel'
 import { SystemTasksPanel } from './components/system-tasks-panel'
 import { SystemTasksPanel } from './components/system-tasks-panel'
 
 
 export function SystemInfo() {
 export function SystemInfo() {
@@ -25,9 +27,19 @@ export function SystemInfo() {
 
 
   return (
   return (
     <SectionPageLayout>
     <SectionPageLayout>
-      <SectionPageLayout.Title>{t('System Info')}</SectionPageLayout.Title>
+      <SectionPageLayout.Title>
+        <span className='inline-flex min-w-0 items-center gap-2'>
+          <span className='truncate'>{t('System Info')}</span>
+          <Badge variant='outline' className='shrink-0'>
+            Root
+          </Badge>
+        </span>
+      </SectionPageLayout.Title>
       <SectionPageLayout.Content>
       <SectionPageLayout.Content>
-        <SystemTasksPanel />
+        <div className='space-y-4'>
+          <SystemInstancesPanel />
+          <SystemTasksPanel />
+        </div>
       </SectionPageLayout.Content>
       </SectionPageLayout.Content>
     </SectionPageLayout>
     </SectionPageLayout>
   )
   )

web/default/src/features/system-info/types.ts

+/*
+Copyright (C) 2023-2026 QuantumNous
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+For commercial licensing, please contact support@quantumnous.com
+*/
+export type SystemInstanceStatus = 'online' | 'stale'
+
+export type SystemInstanceInfo = {
+  schema_version?: number
+  node?: {
+    name?: string
+    source?: string
+    manually_configured?: boolean
+    should_configure_manually?: boolean
+    [key: string]: unknown
+  }
+  role?: {
+    is_master?: boolean
+    [key: string]: unknown
+  }
+  runtime?: {
+    version?: string
+    goos?: string
+    goarch?: string
+    started_at?: number
+    [key: string]: unknown
+  }
+  host?: {
+    hostname?: string
+    [key: string]: unknown
+  }
+  resources?: {
+    cpu?: {
+      usage_percent?: number
+      [key: string]: unknown
+    }
+    memory?: {
+      usage_percent?: number
+      [key: string]: unknown
+    }
+    storage?: {
+      total_bytes?: number
+      used_bytes?: number
+      free_bytes?: number
+      used_percent?: number
+      [key: string]: unknown
+    }
+    [key: string]: unknown
+  }
+  [key: string]: unknown
+}
+
+export type SystemInstance = {
+  node_name: string
+  status: SystemInstanceStatus
+  stale_after_seconds: number
+  started_at: number
+  last_seen_at: number
+  info?: SystemInstanceInfo
+}
+
+export type SystemInstanceListResponse = {
+  success: boolean
+  message: string
+  data?: SystemInstance[]
+}

web/default/src/i18n/static-keys.ts

@@ -45,6 +45,12 @@ export const STATIC_I18N_KEYS = [
   'Routing Reliability',
   'Routing Reliability',
   'Maintenance',
   'Maintenance',
 
 
+  // System info
+  'online',
+  'stale',
+  'Master instances run scheduled background tasks.',
+  'Worker instances do not run master-only background tasks.',
+
   // Pricing constants
   // Pricing constants
   'Name',
   'Name',
   'Price: Low to High',
   'Price: Low to High',

web/default/src/lib/format.ts

@@ -145,6 +145,46 @@ export function formatTimestampToDate(
   return dayjs(ms).format('YYYY-MM-DD HH:mm:ss')
   return dayjs(ms).format('YYYY-MM-DD HH:mm:ss')
 }
 }
 
 
+/**
+ * Format timestamp as relative time, e.g. "30 seconds ago".
+ * @param timestamp - Timestamp in seconds or milliseconds
+ * @param unit - Unit of the timestamp ('seconds' or 'milliseconds')
+ * @param locales - Locale passed to Intl.RelativeTimeFormat
+ */
+export function formatTimestampRelative(
+  timestamp?: number,
+  unit: 'seconds' | 'milliseconds' = 'seconds',
+  locales?: Intl.LocalesArgument
+): string {
+  if (!timestamp || timestamp === -1 || timestamp === 0) {
+    return '-'
+  }
+
+  const ms = unit === 'seconds' ? timestamp * 1000 : timestamp
+  const diffSeconds = Math.round((ms - Date.now()) / 1000)
+  const absSeconds = Math.abs(diffSeconds)
+  const formatter = new Intl.RelativeTimeFormat(locales, {
+    numeric: 'always',
+  })
+
+  if (absSeconds < 60) {
+    return formatter.format(diffSeconds, 'second')
+  }
+  if (absSeconds < 3600) {
+    return formatter.format(Math.round(diffSeconds / 60), 'minute')
+  }
+  if (absSeconds < 86400) {
+    return formatter.format(Math.round(diffSeconds / 3600), 'hour')
+  }
+  if (absSeconds < 2592000) {
+    return formatter.format(Math.round(diffSeconds / 86400), 'day')
+  }
+  if (absSeconds < 31536000) {
+    return formatter.format(Math.round(diffSeconds / 2592000), 'month')
+  }
+  return formatter.format(Math.round(diffSeconds / 31536000), 'year')
+}
+
 /** Format a Date object to YYYY-MM-DD HH:mm:ss */
 /** Format a Date object to YYYY-MM-DD HH:mm:ss */
 export function formatDateTimeStr(date: Date): string {
 export function formatDateTimeStr(date: Date): string {
   return dayjs(date).format('YYYY-MM-DD HH:mm:ss')
   return dayjs(date).format('YYYY-MM-DD HH:mm:ss')