Commit 17317ba4 by DigHuang Committed by GitHub

chore(helm): move charts and release workflow to fastgpt-pro (#7463)

* feat(sandbox): pause OpenSandbox instance on stop instead of deleting

* chore(helm): remove charts and release workflow moved to fastgpt-pro

* revert: pause opensandbox
parent 33f2f108
name: Release helm chart
on:
push:
tags:
- 'v*.*.*'
workflow_dispatch:
jobs:
validate-version:
runs-on: ubuntu-24.04
outputs:
version: ${{ steps.version.outputs.version }}
steps:
- name: Validate release version
id: version
env:
VERSION: ${{ github.ref_name }}
REF_TYPE: ${{ github.ref_type }}
CURRENT_REF: ${{ github.ref }}
run: |
if [[ "$REF_TYPE" != "tag" || ! "$VERSION" =~ ^v ]]; then
echo "::error::Release workflow must run on a tag starting with v. Current ref: ${CURRENT_REF}"
exit 1
fi
echo "version=${VERSION}" >> "$GITHUB_OUTPUT"
helm:
needs: validate-version
permissions:
packages: write
contents: read
attestations: write
id-token: write
runs-on: ubuntu-24.04
steps:
- name: Checkout
uses: actions/checkout@v4
with:
fetch-tags: true
fetch-depth: 0
- name: Release Helm
run: |
echo ${{ secrets.GITHUB_TOKEN }} | helm registry login ghcr.io -u ${{ github.repository_owner }} --password-stdin
export APP_VERSION=${{ needs.validate-version.outputs.version }}
export HELM_VERSION=${{ needs.validate-version.outputs.version }}
export HELM_REPO=ghcr.io/${{ github.repository_owner }}
helm dependency update deploy/helm/fastgpt
helm package deploy/helm/fastgpt --version ${HELM_VERSION}-helm --app-version ${APP_VERSION} -d bin
helm push bin/fastgpt-${HELM_VERSION}-helm.tgz oci://${HELM_REPO}
......@@ -279,7 +279,7 @@ services:
# runtime=docker 模式需要挂载 Docker socket
# 配置 docker.host_ip 为宿主机 LAN IP(容器内访问宿主机服务用)
opensandbox-server:
image: sandbox-registry.cn-zhangjiakou.cr.aliyuncs.com/opensandbox/server:v0.2.1
image: sandbox-registry.cn-zhangjiakou.cr.aliyuncs.com/opensandbox/server:v0.2.2
container_name: fastgpt-opensandbox-server
restart: always
ports:
......@@ -340,7 +340,7 @@ services:
profiles:
- prepull
opensandbox-egress-image:
image: sandbox-registry.cn-zhangjiakou.cr.aliyuncs.com/opensandbox/egress:v1.1.4
image: sandbox-registry.cn-zhangjiakou.cr.aliyuncs.com/opensandbox/egress:v1.1.5
profiles:
- prepull
# 卷管理微服务:负责幂等创建/删除 Docker named volume 或 k8s PVC
......@@ -446,7 +446,9 @@ configs:
execd_image = "sandbox-registry.cn-zhangjiakou.cr.aliyuncs.com/opensandbox/execd:v1.0.21"
[egress]
image = "sandbox-registry.cn-zhangjiakou.cr.aliyuncs.com/opensandbox/egress:v1.1.4"
image = "sandbox-registry.cn-zhangjiakou.cr.aliyuncs.com/opensandbox/egress:v1.1.5"
mode = "dns+nft"
disable_ipv6 = true
[docker]
network_mode = "bridge"
......
......@@ -279,7 +279,7 @@ services:
# runtime=docker 模式需要挂载 Docker socket
# 配置 docker.host_ip 为宿主机 LAN IP(容器内访问宿主机服务用)
opensandbox-server:
image: opensandbox/server:v0.2.1
image: opensandbox/server:v0.2.2
container_name: fastgpt-opensandbox-server
restart: always
ports:
......@@ -340,7 +340,7 @@ services:
profiles:
- prepull
opensandbox-egress-image:
image: opensandbox/egress:v1.1.4
image: opensandbox/egress:v1.1.5
profiles:
- prepull
# 卷管理微服务:负责幂等创建/删除 Docker named volume 或 k8s PVC
......@@ -446,7 +446,9 @@ configs:
execd_image = "opensandbox/execd:v1.0.21"
[egress]
image = "opensandbox/egress:v1.1.4"
image = "opensandbox/egress:v1.1.5"
mode = "dns+nft"
disable_ipv6 = true
[docker]
network_mode = "bridge"
......
# Patterns to ignore when building packages.
# This supports shell glob matching, relative path matching, and
# negation (prefixed with !). Only one pattern per line.
.DS_Store
# Common VCS dirs
.git/
.gitignore
.bzr/
.bzrignore
.hg/
.hgignore
.svn/
# Common backup files
*.swp
*.bak
*.tmp
*.orig
*~
# Various IDEs
.project
.idea/
*.tmproj
.vscode/
dependencies:
- name: mongodb
repository: oci://registry-1.docker.io/bitnamicharts
version: 15.0.1
- name: postgresql
repository: oci://registry-1.docker.io/bitnamicharts
version: 15.0.0
digest: sha256:eaee52dc30a5f0e8e65e30a99240707c8493fa2773a4afcfb54d286fb4fa6306
generated: "2024-03-19T13:47:47.7518066+08:00"
apiVersion: v2
name: fastgpt
description: A Helm chart for FastGPT
# A chart can be either an 'application' or a 'library' chart.
#
# Application charts are a collection of templates that can be packaged into versioned archives
# to be deployed.
#
# Library charts provide useful utilities or functions for the chart developer. They're included as
# a dependency of application charts to inject those utilities and functions into the rendering
# pipeline. Library charts do not define any templates and therefore cannot be deployed.
type: application
# This is the chart version. This version number should be incremented each time you make changes
# to the chart and its templates, including the app version.
# Versions are expected to follow Semantic Versioning (https://semver.org/)
version: 0.1.0
# This is the version number of the application being deployed. This version number should be
# incremented each time you make changes to the application. Versions are not expected to
# follow Semantic Versioning. They should reflect the version the application is using.
# It is recommended to use it with quotes.
appVersion: "v4.14.12"
dependencies:
- name: mongodb
version: 15.0.1
repository: oci://registry-1.docker.io/bitnamicharts
- name: postgresql
version: 15.0.0
repository: oci://registry-1.docker.io/bitnamicharts
# fastgpt
![Version: 0.1.0](https://img.shields.io/badge/Version-0.1.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v4.6.6](https://img.shields.io/badge/AppVersion-v4.6.6-informational?style=flat-square)
A Helm chart for FastGPT
## Requirements
| Repository | Name | Version |
|------------|------|---------|
| oci://registry-1.docker.io/bitnamicharts | mongodb | 15.0.1 |
| oci://registry-1.docker.io/bitnamicharts | postgresql | 15.0.0 |
## Values
| Key | Type | Default | Description |
|-----|------|---------|-------------|
| affinity | object | `{}` | |
| autoscaling.enabled | bool | `false` | |
| autoscaling.maxReplicas | int | `100` | |
| autoscaling.minReplicas | int | `1` | |
| autoscaling.targetCPUUtilizationPercentage | int | `80` | |
| fullnameOverride | string | `""` | |
| image.pullPolicy | string | `"IfNotPresent"` | |
| image.repository | string | `"ghcr.io/labring/fastgpt"` | |
| image.tag | string | `""` | |
| imagePullSecrets | list | `[]` | |
| ingress.annotations | object | `{}` | |
| ingress.className | string | `""` | |
| ingress.enabled | bool | `false` | |
| ingress.hosts[0].host | string | `"chart-example.local"` | |
| ingress.hosts[0].paths[0].path | string | `"/"` | |
| ingress.hosts[0].paths[0].pathType | string | `"ImplementationSpecific"` | |
| ingress.tls | list | `[]` | |
| livenessProbe.httpGet.path | string | `"/"` | |
| livenessProbe.httpGet.port | string | `"http"` | |
| mongodb.architecture | string | `"replicaset"` | |
| mongodb.auth.rootPassword | string | `"123456"` | |
| mongodb.auth.rootUser | string | `"root"` | |
| mongodb.enabled | bool | `true` | Enable or disable the built-in MangoDB |
| nameOverride | string | `""` | |
| nodeSelector | object | `{}` | |
| podAnnotations | object | `{}` | |
| podLabels | object | `{}` | |
| podSecurityContext | object | `{}` | |
| postgresql.enabled | bool | `true` | Enable or disable the built-in PostgreSQL |
| postgresql.global.postgresql.auth.database | string | `"postgres"` | The default database of PostgreSQL |
| postgresql.global.postgresql.auth.postgresPassword | string | `"postgres"` | The password of PostgreSQL, default username is `postgres` |
| postgresql.image.repository | string | `"linuxsuren/pgvector"` | The PostgreSQL image which include the pgvector extension. See also the source code from https://github.com/LinuxSuRen/pgvector-docker |
| postgresql.image.tag | string | `"v0.0.1"` | |
| readinessProbe.httpGet.path | string | `"/"` | |
| readinessProbe.httpGet.port | string | `"http"` | |
| replicaCount | int | `1` | |
| resources | object | `{}` | |
| securityContext | object | `{}` | |
| service.port | int | `3000` | |
| service.type | string | `"ClusterIP"` | |
| serviceAccount.annotations | object | `{}` | |
| serviceAccount.automount | bool | `true` | |
| serviceAccount.create | bool | `true` | |
| serviceAccount.name | string | `""` | |
| tolerations | list | `[]` | |
| volumeMounts | list | `[]` | |
| volumes | list | `[]` | |
----------------------------------------------
Autogenerated from chart metadata using [helm-docs v1.13.1](https://github.com/norwoodj/helm-docs/releases/v1.13.1)
1. Get the application URL by running these commands:
{{- if .Values.ingress.enabled }}
{{- range $host := .Values.ingress.hosts }}
{{- range .paths }}
http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ .path }}
{{- end }}
{{- end }}
{{- else if contains "NodePort" .Values.service.type }}
export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "fastgpt.fullname" . }})
export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}")
echo http://$NODE_IP:$NODE_PORT
{{- else if contains "LoadBalancer" .Values.service.type }}
NOTE: It may take a few minutes for the LoadBalancer IP to be available.
You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "fastgpt.fullname" . }}'
export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "fastgpt.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}")
echo http://$SERVICE_IP:{{ .Values.service.port }}
{{- else if contains "ClusterIP" .Values.service.type }}
export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "fastgpt.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}")
export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}")
echo "Visit http://127.0.0.1:8080 to use your application"
kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT
{{- end }}
{{/*
Expand the name of the chart.
*/}}
{{- define "fastgpt.name" -}}
{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
{{- end }}
{{/*
Create a default fully qualified app name.
We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
If release name contains chart name it will be used as a full name.
*/}}
{{- define "fastgpt.fullname" -}}
{{- if .Values.fullnameOverride }}
{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
{{- else }}
{{- $name := default .Chart.Name .Values.nameOverride }}
{{- if contains $name .Release.Name }}
{{- .Release.Name | trunc 63 | trimSuffix "-" }}
{{- else }}
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
{{- end }}
{{- end }}
{{- end }}
{{/*
Create chart name and version as used by the chart label.
*/}}
{{- define "fastgpt.chart" -}}
{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
{{- end }}
{{/*
Common labels
*/}}
{{- define "fastgpt.labels" -}}
helm.sh/chart: {{ include "fastgpt.chart" . }}
{{ include "fastgpt.selectorLabels" . }}
{{- if .Chart.AppVersion }}
app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
{{- end }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
{{- end }}
{{/*
Selector labels
*/}}
{{- define "fastgpt.selectorLabels" -}}
app.kubernetes.io/name: {{ include "fastgpt.name" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
{{- end }}
{{/*
Create the name of the service account to use
*/}}
{{- define "fastgpt.serviceAccountName" -}}
{{- if .Values.serviceAccount.create }}
{{- default (include "fastgpt.fullname" .) .Values.serviceAccount.name }}
{{- else }}
{{- default "default" .Values.serviceAccount.name }}
{{- end }}
{{- end }}
apiVersion: v1
data:
config.json: |
{
"systemEnv": {
"openapiPrefix": "fastgpt",
"vectorMaxProcess": 15,
"qaMaxProcess": 15,
"vlmMaxProcess": 15,
"hnswEfSearch": 100
},
"llmModels": [
{
"model": "gpt-3.5-turbo",
"name": "gpt-3.5-turbo",
"maxContext": 16000,
"maxResponse": 4000,
"quoteMaxToken": 13000,
"maxTemperature": 1.2,
"charsPointsPrice": 0,
"censor": false,
"vision": false,
"toolChoice": true,
"functionCall": false,
"defaultSystemChatPrompt": "",
"defaultConfig": {}
},
{
"model": "gpt-3.5-turbo-16k",
"name": "gpt-3.5-turbo-16k",
"maxContext": 16000,
"maxResponse": 16000,
"quoteMaxToken": 13000,
"maxTemperature": 1.2,
"charsPointsPrice": 0,
"censor": false,
"vision": false,
"toolChoice": true,
"functionCall": false,
"defaultSystemChatPrompt": "",
"defaultConfig": {}
},
{
"model": "gpt-4-0125-preview",
"name": "gpt-4-turbo",
"maxContext": 125000,
"maxResponse": 4000,
"quoteMaxToken": 100000,
"maxTemperature": 1.2,
"charsPointsPrice": 0,
"censor": false,
"vision": false,
"toolChoice": true,
"functionCall": false,
"defaultSystemChatPrompt": "",
"defaultConfig": {}
},
{
"model": "gpt-4-vision-preview",
"name": "gpt-4-vision",
"maxContext": 128000,
"maxResponse": 4000,
"quoteMaxToken": 100000,
"maxTemperature": 1.2,
"charsPointsPrice": 0,
"censor": false,
"vision": true,
"toolChoice": true,
"functionCall": false,
"defaultSystemChatPrompt": "",
"defaultConfig": {}
}
],
"vectorModels": [
{
"model": "text-embedding-3-large",
"name": "Embedding-2",
"avatar": "/imgs/model/openai.svg",
"charsPointsPrice": 0,
"defaultToken": 512,
"maxToken": 3000,
"weight": 100,
"dbConfig": {},
"queryConfig": {},
"defaultConfig": {
"dimensions": 1024
}
},
{
"model": "text-embedding-3-small",
"name": "Embedding-2",
"avatar": "/imgs/model/openai.svg",
"charsPointsPrice": 0,
"defaultToken": 512,
"maxToken": 3000,
"weight": 100,
"dbConfig": {},
"queryConfig": {}
},
{
"model": "text-embedding-ada-002",
"name": "Embedding-2",
"avatar": "/imgs/model/openai.svg",
"charsPointsPrice": 0,
"defaultToken": 512,
"maxToken": 3000,
"weight": 100,
"dbConfig": {},
"queryConfig": {}
}
],
"reRankModels": [],
"audioSpeechModels": [
{
"model": "tts-1",
"name": "OpenAI TTS1",
"charsPointsPrice": 0,
"voices": [
{
"label": "Alloy",
"value": "alloy",
"bufferId": "openai-Alloy"
},
{
"label": "Echo",
"value": "echo",
"bufferId": "openai-Echo"
},
{
"label": "Fable",
"value": "fable",
"bufferId": "openai-Fable"
},
{
"label": "Onyx",
"value": "onyx",
"bufferId": "openai-Onyx"
},
{
"label": "Nova",
"value": "nova",
"bufferId": "openai-Nova"
},
{
"label": "Shimmer",
"value": "shimmer",
"bufferId": "openai-Shimmer"
}
]
}
],
"whisperModel": {
"model": "whisper-1",
"name": "Whisper1",
"charsPointsPrice": 0
}
}
kind: ConfigMap
metadata:
labels:
{{ include "fastgpt.labels" . | nindent 4 }}
name: {{ include "fastgpt.fullname" . }}-config
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ include "fastgpt.fullname" . }}
labels:
{{- include "fastgpt.labels" . | nindent 4 }}
spec:
{{- if not .Values.autoscaling.enabled }}
replicas: {{ .Values.replicaCount }}
{{- end }}
selector:
matchLabels:
{{- include "fastgpt.selectorLabels" . | nindent 6 }}
template:
metadata:
{{- with .Values.podAnnotations }}
annotations:
{{- toYaml . | nindent 8 }}
{{- end }}
labels:
{{- include "fastgpt.labels" . | nindent 8 }}
{{- with .Values.podLabels }}
{{- toYaml . | nindent 8 }}
{{- end }}
spec:
{{- with .Values.imagePullSecrets }}
imagePullSecrets:
{{- toYaml . | nindent 8 }}
{{- end }}
serviceAccountName: {{ include "fastgpt.serviceAccountName" . }}
securityContext:
{{- toYaml .Values.podSecurityContext | nindent 8 }}
containers:
- name: {{ .Chart.Name }}
securityContext:
{{- toYaml .Values.securityContext | nindent 12 }}
image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
imagePullPolicy: {{ .Values.image.pullPolicy }}
envFrom:
- secretRef:
name: {{ include "fastgpt.fullname" . }}-env
ports:
- name: http
containerPort: {{ .Values.service.port }}
protocol: TCP
resources:
{{- toYaml .Values.resources | nindent 12 }}
volumeMounts:
- mountPath: /app/data/config.json
name: config
subPath: config.json
volumes:
- name: config
configMap:
name: {{ include "fastgpt.fullname" . }}-config
{{- with .Values.nodeSelector }}
nodeSelector:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.affinity }}
affinity:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.tolerations }}
tolerations:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- if .Values.autoscaling.enabled }}
apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
name: {{ include "fastgpt.fullname" . }}
labels:
{{- include "fastgpt.labels" . | nindent 4 }}
spec:
scaleTargetRef:
apiVersion: apps/v1
kind: Deployment
name: {{ include "fastgpt.fullname" . }}
minReplicas: {{ .Values.autoscaling.minReplicas }}
maxReplicas: {{ .Values.autoscaling.maxReplicas }}
metrics:
{{- if .Values.autoscaling.targetCPUUtilizationPercentage }}
- type: Resource
resource:
name: cpu
target:
type: Utilization
averageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }}
{{- end }}
{{- if .Values.autoscaling.targetMemoryUtilizationPercentage }}
- type: Resource
resource:
name: memory
target:
type: Utilization
averageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }}
{{- end }}
{{- end }}
{{- if .Values.ingress.enabled -}}
{{- $fullName := include "fastgpt.fullname" . -}}
{{- $svcPort := .Values.service.port -}}
{{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }}
{{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }}
{{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}}
{{- end }}
{{- end }}
{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}}
apiVersion: networking.k8s.io/v1
{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}}
apiVersion: networking.k8s.io/v1beta1
{{- else -}}
apiVersion: extensions/v1beta1
{{- end }}
kind: Ingress
metadata:
name: {{ $fullName }}
labels:
{{- include "fastgpt.labels" . | nindent 4 }}
{{- with .Values.ingress.annotations }}
annotations:
{{- toYaml . | nindent 4 }}
{{- end }}
spec:
{{- if and .Values.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }}
ingressClassName: {{ .Values.ingress.className }}
{{- end }}
{{- if .Values.ingress.tls }}
tls:
{{- range .Values.ingress.tls }}
- hosts:
{{- range .hosts }}
- {{ . | quote }}
{{- end }}
secretName: {{ .secretName }}
{{- end }}
{{- end }}
rules:
{{- range .Values.ingress.hosts }}
- host: {{ .host | quote }}
http:
paths:
{{- range .paths }}
- path: {{ .path }}
{{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }}
pathType: {{ .pathType }}
{{- end }}
backend:
{{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }}
service:
name: {{ $fullName }}
port:
number: {{ $svcPort }}
{{- else }}
serviceName: {{ $fullName }}
servicePort: {{ $svcPort }}
{{- end }}
{{- end }}
{{- end }}
{{- end }}
apiVersion: v1
stringData:
DEFAULT_ROOT_PSW: "1234"
OPENAI_BASE_URL: "https://api.openai.com/v1"
CHAT_API_KEY: "sk-xxxx"
DB_MAX_LINK: "5"
SYSTEM_MAX_STRING_LENGTH_M: "{{ .Values.system.maxStringLengthM }}"
ROOT_KEY: "root_key"
FILE_TOKEN_KEY: "filetoken"
AES256_SECRET_KEY: "fastgptsecret"
INVOKE_TOKEN_SECRET: "fastgpt_invoke_token_secret_32_chars_min"
MONGODB_URI: "mongodb://{{ .Values.mongodb.auth.rootUser }}:{{ .Values.mongodb.auth.rootPassword }}@{{ include "fastgpt.fullname" . }}-mongodb-headless:27017/fastgpt?authSource=admin"
PG_URL: "postgresql://postgres:{{ .Values.postgresql.auth.rootPassword }}@{{ include "fastgpt.fullname" . }}-postgresql:5432/{{ .Values.postgresql.global.postgresql.auth.database }}"
kind: Secret
type: Opaque
metadata:
labels:
{{ include "fastgpt.labels" . | nindent 4 }}
name: {{ include "fastgpt.fullname" . }}-env
apiVersion: v1
kind: Service
metadata:
name: {{ include "fastgpt.fullname" . }}
labels:
{{- include "fastgpt.labels" . | nindent 4 }}
spec:
type: {{ .Values.service.type }}
ports:
- port: {{ .Values.service.port }}
{{- if .Values.service.nodePort }}
nodePort: {{ .Values.service.nodePort }}
{{- end}}
targetPort: http
protocol: TCP
name: http
selector:
{{- include "fastgpt.selectorLabels" . | nindent 4 }}
{{- if .Values.serviceAccount.create -}}
apiVersion: v1
kind: ServiceAccount
metadata:
name: {{ include "fastgpt.serviceAccountName" . }}
labels:
{{- include "fastgpt.labels" . | nindent 4 }}
{{- with .Values.serviceAccount.annotations }}
annotations:
{{- toYaml . | nindent 4 }}
{{- end }}
automountServiceAccountToken: {{ .Values.serviceAccount.automount }}
{{- end }}
apiVersion: v1
kind: Pod
metadata:
name: "{{ include "fastgpt.fullname" . }}-test-connection"
labels:
{{- include "fastgpt.labels" . | nindent 4 }}
annotations:
"helm.sh/hook": test
spec:
containers:
- name: wget
image: busybox
command: ['wget']
args: ['{{ include "fastgpt.fullname" . }}:{{ .Values.service.port }}']
restartPolicy: Never
# Default values for fastgpt.
# This is a YAML-formatted file.
# Declare variables to be passed into your templates.
replicaCount: 1
image:
repository: ghcr.io/labring/fastgpt
pullPolicy: IfNotPresent
# Overrides the image tag whose default is the chart appVersion.
tag: ""
imagePullSecrets: []
nameOverride: ""
fullnameOverride: ""
serviceAccount:
# Specifies whether a service account should be created
create: true
# Automatically mount a ServiceAccount's API credentials?
automount: true
# Annotations to add to the service account
annotations: {}
# The name of the service account to use.
# If not set and create is true, a name is generated using the fullname template
name: ""
podAnnotations: {}
podLabels: {}
podSecurityContext:
{}
# fsGroup: 2000
securityContext:
{}
# capabilities:
# drop:
# - ALL
# readOnlyRootFilesystem: true
# runAsNonRoot: true
# runAsUser: 1000
service:
type: ClusterIP
port: 3000
# nodePort: 31440
ingress:
enabled: false
className: ""
annotations:
{}
# kubernetes.io/ingress.class: nginx
# kubernetes.io/tls-acme: "true"
hosts:
- host: chart-example.local
paths:
- path: /
pathType: ImplementationSpecific
tls: []
# - secretName: chart-example-tls
# hosts:
# - chart-example.local
resources:
{}
# We usually recommend not to specify default resources and to leave this as a conscious
# choice for the user. This also increases chances charts run on environments with little
# resources, such as Minikube. If you do want to specify resources, uncomment the following
# lines, adjust them as necessary, and remove the curly braces after 'resources:'.
# limits:
# cpu: 100m
# memory: 128Mi
# requests:
# cpu: 100m
# memory: 128Mi
livenessProbe:
httpGet:
path: /
port: http
readinessProbe:
httpGet:
path: /
port: http
autoscaling:
enabled: false
minReplicas: 1
maxReplicas: 100
targetCPUUtilizationPercentage: 80
# targetMemoryUtilizationPercentage: 80
system:
# Maximum character length for synchronous system string operations, in M characters.
# 1 means 1,000,000 characters. Valid range: 1 to 100.
maxStringLengthM: 100
# Additional volumes on the output Deployment definition.
volumes: []
# - name: foo
# secret:
# secretName: mysecret
# optional: false
# Additional volumeMounts on the output Deployment definition.
volumeMounts: []
# - name: foo
# mountPath: "/etc/foo"
# readOnly: true
nodeSelector: {}
tolerations: []
affinity: {}
mongodb:
# -- Enable or disable the built-in MangoDB
enabled: true
architecture: replicaset
auth:
rootUser: root
rootPassword: "123456"
postgresql:
# -- Enable or disable the built-in PostgreSQL
enabled: true
image:
# registry: 172.11.0.6:30002
# -- The PostgreSQL image which include the pgvector extension. See also the source code from https://github.com/LinuxSuRen/pgvector-docker
repository: linuxsuren/pgvector
tag: v0.0.1
global:
postgresql:
auth:
# -- The password of PostgreSQL, default username is `postgres`
postgresPassword: postgres
# -- The default database of PostgreSQL
database: postgres
# Patterns to ignore when building packages.
# This supports shell glob matching, relative path matching, and
# negation (prefixed with !). Only one pattern per line.
.DS_Store
# Common VCS dirs
.git/
.gitignore
.bzr/
.bzrignore
.hg/
.hgignore
.svn/
# Common backup files
*.swp
*.bak
*.tmp
*.orig
*~
# Various IDEs
.project
.idea/
*.tmproj
.vscode/
# Changelog
All notable changes to the OpenSandbox Helm Chart will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
## [0.1.0] - Initial Release
### Added
#### Core Features
- OpenSandbox Kubernetes Controller Helm chart
- **OpenSandbox Server deployment with FastAPI control plane for SDK integration**
- Support for deploying controller with configurable replicas and resources
- BatchSandbox and Pool CRD definitions
- RBAC resources (ClusterRole, ClusterRoleBinding, ServiceAccount)
- Leader election configuration for high availability
#### Server Features
- Server Deployment with configurable replicas and resources
- Server Service with ClusterIP/NodePort/LoadBalancer support
- ConfigMap-based configuration management
- Optional Ingress support for external access
- Health probes for liveness and readiness checks
- In-cluster Kubernetes configuration
- API key authentication support (optional)
- SDK-compatible REST API on port 8080
#### Pool Management
- Default agent-pool with execd and task-executor sidecar
- Pool template support for creating pre-warmed Pod pools
- Configurable Pool capacity (bufferMin, bufferMax, poolMin, poolMax)
- SDK-compatible Pool configuration with execd on port 44772
#### Multiple Values Files
- `values.yaml` - Default configuration with agent-pool enabled
- `values-e2e.yaml` - End-to-end testing with minimal resources (2-5 pods)
- Use `--set` or custom values files for production/development overrides
#### Templates
- `deployment.yaml` - Controller manager deployment
- **`server-deployment.yaml` - Server deployment**
- **`server-service.yaml` - Server service**
- **`server-configmap.yaml` - Server configuration**
- **`server-ingress.yaml` - Server ingress (optional)**
- `pools.yaml` - Dynamic Pool resource generation from values
- `serviceaccount.yaml` - Service account for controller
- `clusterrole.yaml` - RBAC cluster role
- `clusterrolebinding.yaml` - RBAC cluster role binding
- `leader-election-role.yaml` - Leader election RBAC
- `leader-election-rolebinding.yaml` - Leader election binding
- `metrics-service.yaml` - Metrics service endpoint
- `metrics-rbac.yaml` - Metrics RBAC resources
- `servicemonitor.yaml` - Prometheus ServiceMonitor (optional)
- `extra-roles.yaml` - User management roles (viewer, editor)
- `poddisruptionbudget.yaml` - High availability Pod disruption budget
- `NOTES.txt` - Post-installation guidance
- `_helpers.tpl` - Template helper functions
#### Scripts
- `scripts/install.sh` - Interactive installation wizard with environment selection
- `scripts/uninstall.sh` - Safe uninstallation with resource cleanup
- `scripts/e2e-test.sh` - End-to-end validation (Install → Server → Pool → SDK → Uninstall)
- `scripts/README.md` - Comprehensive script documentation and troubleshooting guide
#### Configuration Options
- `nameOverride` and `fullnameOverride` for custom resource naming
- **`server.enabled` - Enable/disable server deployment (default: true)**
- **`server.service.type` - Service type (ClusterIP/NodePort/LoadBalancer)**
- **`server.service.nodePort` - NodePort value (optional)**
- **`server.ingress.enabled` - Enable Ingress for external access**
- **`server.config.server.apiKey` - Optional API key authentication**
- `healthProbePort` - Configurable health check port (default: 8081)
- `healthProbes.liveness` - Liveness probe timing configuration
- `healthProbes.readiness` - Readiness probe timing configuration
- `podDisruptionBudget.enabled` - Optional PDB for HA deployments
- `namespaceOverride` - Custom namespace (default: opensandbox)
#### Documentation
- Comprehensive README.md with installation and configuration guide
- examples/README.md with usage scenarios and best practices
- examples/pool-agent-production.yaml with production-ready Pool configuration
- examples/DIRECTORY_STRUCTURE.md explaining file organization
- Example YAML files for Pool and BatchSandbox resources
### Configuration Defaults
- Controller image: `opensandbox/controller:dev`
- **Server image: `opensandbox/server:v0.1.0`**
- Task executor image: `opensandbox/task-executor:dev`
- Image pull policy: `Never` (for local development)
- Namespace: `opensandbox`
- Controller replicas: 1 (3 in production values)
- **Server replicas: 1**
- **Server enabled: true (required for SDK usage)**
- Default Pool enabled: `agent-pool` with 2-5 pods (E2E) or 10-100 pods (default)
### Notes
- This is the initial release of the Helm chart
- All templates have been tested with `helm lint` and E2E validation
- Chart supports Kubernetes 1.19+
apiVersion: v2
name: opensandbox-controller
description: A Helm chart for deploying OpenSandbox Kubernetes Controller
type: application
version: 0.1.0
appVersion: "0.0.1"
keywords:
- opensandbox
- sandbox
- kubernetes
- operator
- controller
home: https://github.com/alibaba/OpenSandbox
maintainers:
- name: OpenSandbox Team
url: https://github.com/alibaba/OpenSandbox
icon: https://avatars.githubusercontent.com/u/1961952
# Helm Chart Makefile for OpenSandbox Controller
CHART_NAME := opensandbox-controller
RELEASE_NAME := opensandbox-controller
NAMESPACE := opensandbox
VALUES_FILE := values.yaml
# Helm commands
HELM := helm
KUBECTL := kubectl
.PHONY: help
help: ## Display this help
@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m<target>\033[0m\n\nTargets:\n"} /^[a-zA-Z_-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 }' $(MAKEFILE_LIST)
.PHONY: lint
lint: ## Lint the Helm chart
$(HELM) lint .
.PHONY: template
template: ## Render chart templates to stdout
$(HELM) template $(RELEASE_NAME) . -f $(VALUES_FILE)
.PHONY: template-debug
template-debug: ## Render chart templates with debug output
$(HELM) template $(RELEASE_NAME) . -f $(VALUES_FILE) --debug
.PHONY: dry-run
dry-run: ## Perform a dry-run installation
$(HELM) install $(RELEASE_NAME) . \
-f $(VALUES_FILE) \
--namespace $(NAMESPACE) \
--create-namespace \
--dry-run --debug
.PHONY: install
install: ## Install the chart
$(HELM) install $(RELEASE_NAME) . \
-f $(VALUES_FILE) \
--namespace $(NAMESPACE) \
--create-namespace
.PHONY: install-e2e
install-e2e: ## Install the chart with e2e test values
$(HELM) install $(RELEASE_NAME) . \
-f values-e2e.yaml \
--namespace $(NAMESPACE) \
--create-namespace
.PHONY: upgrade
upgrade: ## Upgrade the chart
$(HELM) upgrade $(RELEASE_NAME) . \
-f $(VALUES_FILE) \
--namespace $(NAMESPACE)
.PHONY: upgrade-e2e
upgrade-e2e: ## Upgrade with e2e test values
$(HELM) upgrade $(RELEASE_NAME) . \
-f values-e2e.yaml \
--namespace $(NAMESPACE)
.PHONY: uninstall
uninstall: ## Uninstall the chart
$(HELM) uninstall $(RELEASE_NAME) --namespace $(NAMESPACE)
.PHONY: status
status: ## Show release status
$(HELM) status $(RELEASE_NAME) --namespace $(NAMESPACE)
.PHONY: list
list: ## List all releases
$(HELM) list --namespace $(NAMESPACE)
.PHONY: get-values
get-values: ## Get values for the release
$(HELM) get values $(RELEASE_NAME) --namespace $(NAMESPACE)
.PHONY: get-all
get-all: ## Get all information about the release
$(HELM) get all $(RELEASE_NAME) --namespace $(NAMESPACE)
.PHONY: package
package: ## Package the chart into an archive
$(HELM) package .
.PHONY: verify-install
verify-install: ## Verify the installation
@echo "Checking deployment..."
$(KUBECTL) get deployment -n $(NAMESPACE)
@echo "\nChecking pods..."
$(KUBECTL) get pods -n $(NAMESPACE)
@echo "\nChecking CRDs..."
$(KUBECTL) get crds | grep sandbox.opensandbox.io
.PHONY: logs
logs: ## Show controller logs
$(KUBECTL) logs -n $(NAMESPACE) -l control-plane=controller-manager -f
.PHONY: clean-crds
clean-crds: ## Delete CRDs (use with caution!)
$(KUBECTL) delete crd batchsandboxes.sandbox.opensandbox.io
$(KUBECTL) delete crd pools.sandbox.opensandbox.io
.PHONY: test-connection
test-connection: ## Test if kubectl can connect to the cluster
$(KUBECTL) cluster-info
$(KUBECTL) get nodes
.PHONY: create-namespace
create-namespace: ## Create the namespace
$(KUBECTL) create namespace $(NAMESPACE) --dry-run=client -o yaml | $(KUBECTL) apply -f -
.PHONY: delete-namespace
delete-namespace: ## Delete the namespace
$(KUBECTL) delete namespace $(NAMESPACE)
# Advanced targets
.PHONY: diff
diff: ## Show diff between current release and chart
@command -v helm-diff >/dev/null 2>&1 || { echo "helm-diff plugin required. Install: helm plugin install https://github.com/databus23/helm-diff"; exit 1; }
$(HELM) diff upgrade $(RELEASE_NAME) . -f $(VALUES_FILE) --namespace $(NAMESPACE)
.PHONY: history
history: ## Show release history
$(HELM) history $(RELEASE_NAME) --namespace $(NAMESPACE)
.PHONY: rollback
rollback: ## Rollback to previous release
$(HELM) rollback $(RELEASE_NAME) --namespace $(NAMESPACE)
.PHONY: rollback-to
rollback-to: ## Rollback to specific revision (usage: make rollback-to REVISION=2)
@if [ -z "$(REVISION)" ]; then echo "Please specify REVISION=<number>"; exit 1; fi
$(HELM) rollback $(RELEASE_NAME) $(REVISION) --namespace $(NAMESPACE)
---
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
annotations:
controller-gen.kubebuilder.io/version: v0.18.0
name: batchsandboxes.sandbox.opensandbox.io
spec:
group: sandbox.opensandbox.io
names:
kind: BatchSandbox
listKind: BatchSandboxList
plural: batchsandboxes
shortNames:
- bsbx
singular: batchsandbox
scope: Namespaced
versions:
- additionalPrinterColumns:
- description: The desired number of pods.
jsonPath: .spec.replicas
name: DESIRED
type: integer
- description: The number of currently all pods.
jsonPath: .status.replicas
name: TOTAL
type: integer
- description: The number of currently all allocated pods.
jsonPath: .status.allocated
name: ALLOCATED
type: integer
- description: The number of currently all ready pods.
jsonPath: .status.ready
name: Ready
type: integer
- description: The number of currently all running tasks.
jsonPath: .status.taskRunning
name: TASK_RUNNING
priority: 1
type: integer
- description: The number of currently all succeed tasks.
jsonPath: .status.taskSucceed
name: TASK_SUCCEED
priority: 1
type: integer
- description: The number of currently all failed tasks.
jsonPath: .status.taskFailed
name: TASK_FAILED
priority: 1
type: integer
- description: The number of currently all unknown tasks.
jsonPath: .status.taskUnknown
name: TASK_UNKNOWN
priority: 1
type: integer
- description: sandbox expire time
jsonPath: .spec.expireTime
name: EXPIRE
type: string
- description: CreationTimestamp is a timestamp representing the server time when
this object was created. It is not guaranteed to be set in happens-before
order across separate operations. Clients may not set this value. It is represented
in RFC3339 form and is in UTC.
jsonPath: .metadata.creationTimestamp
name: AGE
type: date
name: v1alpha1
schema:
openAPIV3Schema:
description: BatchSandbox is the Schema for the batchsandboxes API.
properties:
apiVersion:
description: |-
APIVersion defines the versioned schema of this representation of an object.
Servers should convert recognized schemas to the latest internal value, and
may reject unrecognized values.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
type: string
kind:
description: |-
Kind is a string value representing the REST resource this object represents.
Servers may infer this from the endpoint the client submits requests to.
Cannot be updated.
In CamelCase.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
type: string
metadata:
type: object
spec:
description: BatchSandboxSpec defines the desired state of BatchSandbox.
properties:
expireTime:
description: |-
ExpireTime - Absolute time when the batch-sandbox is deleted.
If a time in the past is provided, the batch-sandbox will be deleted immediately.
format: date-time
type: string
poolRef:
description: |-
PoolRef references the Pool resource name for pooled sandbox creation.
Mutually exclusive with Template - use PoolRef for pool-based allocation or Template for direct sandbox creation.
type: string
replicas:
default: 1
description: Replicas is the number of desired replicas.
format: int32
minimum: 0
type: integer
shardPatches:
description: ShardPatches indicates patching to the Template for BatchSandbox.
x-kubernetes-preserve-unknown-fields: true
shardTaskPatches:
description: ShardTaskPatches indicates patching to the TaskTemplate
for individual Task.
x-kubernetes-preserve-unknown-fields: true
taskResourcePolicyWhenCompleted:
default: Retain
description: |-
TaskResourcePolicyWhenCompleted specifies how resources should be handled once a task reaches a completed state (SUCCEEDED or FAILED).
- Retain: Keep the resources until the BatchSandbox is deleted.
- Release: Free the resources immediately when the task completes.
type: string
taskTemplate:
description: |-
Task is a custom task spec that is automatically dispatched after the sandbox is successfully created.
The Sandbox is responsible for managing the lifecycle of the task.
x-kubernetes-preserve-unknown-fields: true
template:
description: Template describes the pods that will be created.
x-kubernetes-preserve-unknown-fields: true
required:
- replicas
type: object
status:
description: BatchSandboxStatus defines the observed state of BatchSandbox.
properties:
allocated:
description: "\tAllocated is the number of actual scheduled Pod"
format: int32
type: integer
observedGeneration:
description: |-
ObservedGeneration is the most recent generation observed for this BatchSandbox. It corresponds to the
BatchSandbox's generation, which is updated on mutation by the API Server.
format: int64
type: integer
ready:
description: "\tReady is the number of actual Ready Pod"
format: int32
type: integer
replicas:
description: Replicas is the number of actual Pods
format: int32
type: integer
taskFailed:
description: TaskFailed is the number of Failed task
format: int32
type: integer
taskPending:
description: TaskPending is the number of Pending task which is unassigned
format: int32
type: integer
taskRunning:
description: TaskRunning is the number of Running task
format: int32
type: integer
taskSucceed:
description: TaskSucceed is the number of Succeed task
format: int32
type: integer
taskUnknown:
description: TaskUnknown is the number of Unknown task
format: int32
type: integer
required:
- allocated
- ready
- replicas
- taskFailed
- taskPending
- taskRunning
- taskSucceed
- taskUnknown
type: object
type: object
served: true
storage: true
subresources:
status: {}
---
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
annotations:
controller-gen.kubebuilder.io/version: v0.18.0
name: pools.sandbox.opensandbox.io
spec:
group: sandbox.opensandbox.io
names:
kind: Pool
listKind: PoolList
plural: pools
singular: pool
scope: Namespaced
versions:
- additionalPrinterColumns:
- description: The number of all nodes in pool.
jsonPath: .status.total
name: TOTAL
type: integer
- description: The number of allocated nodes in pool.
jsonPath: .status.allocated
name: ALLOCATED
type: integer
- description: The number of available nodes in pool.
jsonPath: .status.available
name: AVAILABLE
type: integer
name: v1alpha1
schema:
openAPIV3Schema:
description: Pool is the Schema for the pools API.
properties:
apiVersion:
description: |-
APIVersion defines the versioned schema of this representation of an object.
Servers should convert recognized schemas to the latest internal value, and
may reject unrecognized values.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
type: string
kind:
description: |-
Kind is a string value representing the REST resource this object represents.
Servers may infer this from the endpoint the client submits requests to.
Cannot be updated.
In CamelCase.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
type: string
metadata:
type: object
spec:
description: PoolSpec defines the desired state of Pool.
properties:
capacitySpec:
description: CapacitySpec controls the size of the resource pool.
properties:
bufferMax:
description: BufferMax is the maximum number of nodes kept in
the warm buffer.
format: int32
minimum: 0
type: integer
bufferMin:
description: BufferMin is the minimum number of nodes that must
remain in the buffer.
format: int32
minimum: 0
type: integer
poolMax:
description: PoolMax is the maximum total number of nodes allowed
in the entire pool.
format: int32
minimum: 0
type: integer
poolMin:
description: PoolMin is the minimum total size of the pool.
format: int32
minimum: 0
type: integer
required:
- bufferMax
- bufferMin
- poolMax
- poolMin
type: object
template:
description: Pod Template used to create pre-warmed nodes in the pool.
x-kubernetes-preserve-unknown-fields: true
required:
- capacitySpec
type: object
status:
description: PoolStatus defines the observed state of Pool.
properties:
allocated:
description: Allocated is the number of nodes currently allocated
to sandboxes.
format: int32
type: integer
available:
description: Available is the number of nodes currently available
in the pool.
format: int32
type: integer
observedGeneration:
description: |-
ObservedGeneration is the most recent generation observed for this BatchSandbox. It corresponds to the
BatchSandbox's generation, which is updated on mutation by the API Server.
format: int64
type: integer
revision:
description: Revision is the latest version of pool
type: string
total:
description: Total is the total number of nodes in the pool.
format: int32
type: integer
required:
- allocated
- available
- revision
- total
type: object
type: object
served: true
storage: true
subresources:
status: {}
# Examples 目录结构
本文档说明 `examples/` 目录的组织结构和设计原则。
## 📂 目录结构
```
examples/
├── README.md # 主文档:快速开始、最佳实践
├── DIRECTORY_STRUCTURE.md # 本文档:目录结构说明
├── pool-examples.md # 详细的 Pool 配置说明
│
├── pool-agent-production.yaml # 🌟 生产级 Agent Pool(推荐)
├── pool-sdk-compatible.yaml # SDK 基础 Pool(execd only)
├── pool-sdk-with-tasks.yaml # SDK 完整 Pool(execd + task-executor)
│
├── batchsandbox-basic.yaml # Non-pooled 模式示例
└── batchsandbox-with-tasks.yaml # Pooled 批量任务示例
```
## 📝 文件分类
### Pool 配置文件(3 个)
| 文件 | 类型 | SDK 支持 | 自定义 entrypoint | 推荐度 |
|------|------|----------|-----------------|--------|
| `pool-agent-production.yaml` | 生产级 | ✅ | ✅ | ⭐⭐⭐⭐⭐ |
| `pool-sdk-with-tasks.yaml` | 完整功能 | ✅ | ✅ | ⭐⭐⭐⭐ |
| `pool-sdk-compatible.yaml` | 基础功能 | ✅ | ❌ | ⭐⭐⭐ |
**选择建议**:
- **生产环境**:使用 `pool-agent-production.yaml`(包含详细注释和最佳实践)
- **快速测试**:使用 `pool-sdk-with-tasks.yaml`(更简洁)
- **特殊需求**:如果不需要自定义 entrypoint,使用 `pool-sdk-compatible.yaml`
### BatchSandbox 配置文件(2 个)
| 文件 | 模式 | 依赖 | 用途 |
|------|------|------|------|
| `batchsandbox-basic.yaml` | Non-pooled | 无 | 演示直接创建 Pod |
| `batchsandbox-with-tasks.yaml` | Pooled | Pool | 演示批量异构任务 |
**使用说明**:
- **SDK 场景**:通常不需要手动创建 BatchSandbox(SDK 自动创建)
- **kubectl 场景**:用于批量任务执行(RL 训练、压力测试等)
### 文档文件(3 个)
| 文件 | 内容 | 面向用户 |
|------|------|---------|
| `README.md` | 快速开始、最佳实践、故障排查 | 所有用户 |
| `pool-examples.md` | Pool 详细配置说明 | 高级用户 |
| `DIRECTORY_STRUCTURE.md` | 目录结构说明 | 开发者 |
## 🎯 设计原则
### 1. 简化选择
**问题**:之前有太多相似的示例(pool-basic, pool-with-execd, pool-with-task-executor...),用户不知道选哪个。
**解决**:
- 保留 3 个 Pool 示例,明确分类和推荐度
- 删除误导性示例(pool-basic 无 execd,pool-with-task-executor 无 execd)
- 突出推荐 `pool-agent-production.yaml`
### 2. 面向实际场景
**问题**:示例文件缺乏实际使用场景说明。
**解决**:
- 明确标注适用场景(Agent 服务、RL 训练、压力测试等)
- 提供完整的 SDK 使用代码示例
- 包含部署、验证、监控的完整流程
### 3. 最佳实践优先
**问题**:缺少生产级配置参考。
**解决**:
- 创建 `pool-agent-production.yaml` 包含:
- 详细的配置注释
- 容量规划建议
- 安全最佳实践
- 监控和调试指南
### 4. 纠正常见误区
**问题**:用户容易误解 Pool 的使用方式。
**解决**:
- 在 README.md 中突出"常见误区"章节
- 明确说明:
- Pool 是 Pod 池,不是 Sandbox 池
- 不需要预创建 BatchSandbox
- SDK 每次 create() 创建新 BatchSandbox
## 🔄 文件变更历史
### 删除的文件(4 个)
| 文件 | 删除原因 |
|------|---------|
| `pool-basic.yaml` | 无 execd,SDK 无法使用,误导性 |
| `pool-with-task-executor.yaml` | 只有 task-executor 无 execd,不完整 |
| `pool-with-execd.yaml` | execd 启动方式不标准,已被 pool-sdk-compatible.yaml 替代 |
| `batchsandbox-pooled.yaml` | 依赖不存在的 basic-pool,无效 |
### 新增的文件(1 个)
| 文件 | 内容 |
|------|------|
| `pool-agent-production.yaml` | 生产级 Agent Pool 配置,包含详细注释和最佳实践 |
### 修改的文件(3 个)
| 文件 | 修改内容 |
|------|---------|
| `README.md` | 重写,添加场景分类、常见误区、容量规划等 |
| `batchsandbox-with-tasks.yaml` | 修改 poolRef 为 agent-pool,添加注释 |
| `batchsandbox-basic.yaml` | 添加详细注释说明 |
## 📊 使用场景映射
### 场景 A:多 Agent 并发使用
```
用户需求: Agent 服务、Code Interpreter、动态工作流
↓
推荐配置: pool-agent-production.yaml
↓
使用方式: SDK 动态创建 sandbox
↓
流程: Helm 部署 Pool → SDK.create() → SDK.kill()
```
### 场景 B:批量任务执行
```
用户需求: RL 训练、压力测试、批量数据处理
↓
推荐配置: pool-agent-production.yaml + batchsandbox-with-tasks.yaml
↓
使用方式: kubectl 创建 BatchSandbox
↓
流程: kubectl apply pool → kubectl apply batchsandbox → 自动清理
```
### 场景 C:测试和开发
```
用户需求: 测试特定镜像、验证功能
↓
推荐配置: batchsandbox-basic.yaml (non-pooled)
↓
使用方式: kubectl 直接创建
↓
流程: kubectl apply → kubectl delete
```
## 🔗 相关资源
- **深度分析**:`/data/home/cz/sandbox-test/pool-analysis/`
- Pool 使用指南
- 架构流程图
- 验证测试脚本
- **Helm 配置**:`/data/home/cz/OpenSandbox/kubernetes/helm-chart/`
- `values.yaml` - 默认配置
- `values-e2e.yaml` - E2E 测试配置
- 生产/开发环境配置:使用 `--set` 或自定义 values 文件
- **主文档**:`/data/home/cz/OpenSandbox/kubernetes/README.md`
- Kubernetes 部署完整指南
## 💡 维护建议
### 添加新示例时
1. **明确场景**:每个示例应对应明确的使用场景
2. **完整注释**:包含配置说明、使用方式、注意事项
3. **验证测试**:确保示例可以正常运行
4. **更新文档**:同步更新 README.md 和本文档
### 修改现有示例时
1. **保持兼容**:避免破坏性变更
2. **版本说明**:在注释中说明版本要求
3. **测试验证**:修改后进行完整测试
4. **文档同步**:更新相关文档
### 删除示例时
1. **评估影响**:确认没有外部依赖
2. **提供替代**:在文档中说明替代方案
3. **迁移指南**:如果有用户使用,提供迁移步骤
# ==============================================================================
# BatchSandbox - Non-pooled 模式(直接创建 Pod)
# ==============================================================================
#
# 用途:演示不使用 Pool 直接创建 BatchSandbox
#
# 使用场景:
# - 需要使用特定镜像或资源配置(Pool 无法满足)
# - 一次性任务,不需要预热优化
# - 测试和开发环境
#
# 特点:
# - 可自由指定镜像、资源、配置
# - 创建速度较慢(需要拉取镜像、启动 Pod)
# - 不复用预热资源
#
# 注意:
# - SDK 使用场景通常不需要预创建 BatchSandbox(动态创建即可)
# - 此示例主要用于演示 kubectl 直接创建
#
# ==============================================================================
apiVersion: sandbox.opensandbox.io/v1alpha1
kind: BatchSandbox
metadata:
name: basic-batch-sandbox
namespace: default
labels:
mode: non-pooled
annotations:
description: "Non-pooled 模式示例"
spec:
# 创建3个沙箱副本
replicas: 3
# 不使用资源池,直接创建Pod
# poolRef: "" # 留空表示 non-pooled 模式
# TTL:3600秒(1小时)后自动清理
ttlSecondsAfterFinished: 3600
# 沙箱模板
template:
spec:
containers:
- name: sandbox-container
image: ubuntu:22.04
command: ["sleep", "infinity"]
resources:
requests:
cpu: "100m"
memory: "128Mi"
limits:
cpu: "500m"
memory: "256Mi"
# ==============================================================================
# BatchSandbox with Heterogeneous Tasks(带异构任务的批量沙箱)
# ==============================================================================
#
# 用途:演示如何使用 Pool 批量执行异构任务(每个 sandbox 执行不同任务)
#
# 使用场景:
# - RL 训练:批量创建训练环境,每个环境执行不同策略
# - 压力测试:批量执行不同的测试用例
# - 数据处理:并行处理多个数据分片
#
# 注意:
# - 此示例主要用于 kubectl 直接创建的批量任务场景
# - SDK 使用场景通常不需要预创建 BatchSandbox(动态创建即可)
# - 需要先创建 pool-agent-production.yaml
#
# 相关文档:
# - /data/home/cz/sandbox-test/pool-analysis/opensandbox_pool_usage_guide.md
#
# ==============================================================================
apiVersion: sandbox.opensandbox.io/v1alpha1
kind: BatchSandbox
metadata:
name: task-batch-sandbox
namespace: default
labels:
use-case: batch-training
annotations:
description: "批量异构任务示例"
spec:
# 创建3个沙箱副本,每个执行不同的任务
replicas: 3
# 使用包含 execd + task-executor 的 Pool
# 注意:需要先创建 pool-agent-production.yaml
poolRef: agent-pool
# TTL:3600秒(1小时)后自动清理
ttlSecondsAfterFinished: 3600
# 默认任务模板(如果shardTaskPatches没有覆盖,则使用此模板)
taskTemplate:
spec:
process:
command: ["echo", "Default task message"]
# 异构任务:为每个沙箱自定义不同的任务
shardTaskPatches:
# 第1个沙箱的任务
- spec:
process:
command: ["bash", "-c"]
args:
- |
echo "Task 1: Running Python script"
python3 -c "
import time
print('Task 1 started')
time.sleep(2)
print('Task 1 completed')
"
# 第2个沙箱的任务
- spec:
process:
command: ["bash", "-c"]
args:
- |
echo "Task 2: Running shell commands"
date
uname -a
sleep 1
echo "Task 2 completed"
# 第3个沙箱的任务
- spec:
process:
command: ["bash", "-c"]
args:
- |
echo "Task 3: System info check"
cat /etc/os-release
df -h
free -h
echo "Task 3 completed"
# Pool示例 - 包含Task Executor Sidecar
## 基本Pool(不包含任务执行)
```yaml
apiVersion: sandbox.opensandbox.io/v1alpha1
kind: Pool
metadata:
name: basic-pool
namespace: default
spec:
template:
spec:
containers:
- name: sandbox-container
image: ubuntu:22.04
command: ["sleep", "infinity"]
resources:
requests:
cpu: "100m"
memory: "128Mi"
limits:
cpu: "500m"
memory: "256Mi"
capacitySpec:
bufferMax: 10
bufferMin: 2
poolMax: 20
poolMin: 5
```
## Pool with Task Executor(支持任务执行)
**重要提示**:
- Task Executor作为sidecar容器运行在Pool的Pod中
- 必须启用`shareProcessNamespace: true`以共享进程命名空间
- Task Executor需要`SYS_PTRACE`权限来注入进程
```yaml
apiVersion: sandbox.opensandbox.io/v1alpha1
kind: Pool
metadata:
name: task-enabled-pool
namespace: default
spec:
template:
spec:
# 必需:共享进程命名空间,允许task-executor访问sandbox容器的进程
shareProcessNamespace: true
containers:
# 主容器:沙箱环境
- name: sandbox-container
image: ubuntu:22.04
command: ["sleep", "infinity"]
resources:
requests:
cpu: "100m"
memory: "128Mi"
limits:
cpu: "500m"
memory: "256Mi"
# Sidecar:Task Executor(用于任务注入)
- name: task-executor
# 使用Helm values中配置的镜像
# {{ .Values.taskExecutor.image.repository }}:{{ .Values.taskExecutor.image.tag }}
image: opensandbox.io/task-executor:v0.0.1
imagePullPolicy: IfNotPresent
resources:
requests:
cpu: "100m"
memory: "128Mi"
limits:
cpu: "500m"
memory: "256Mi"
securityContext:
# 必需:需要ptrace权限来注入进程到sandbox容器
capabilities:
add: ["SYS_PTRACE"]
capacitySpec:
bufferMax: 10
bufferMin: 2
poolMax: 20
poolMin: 5
```
## BatchSandbox with Tasks(使用Pool执行任务)
创建使用上述Pool的BatchSandbox,并执行异构任务:
```yaml
apiVersion: sandbox.opensandbox.io/v1alpha1
kind: BatchSandbox
metadata:
name: task-batch-sandbox
namespace: default
spec:
# 副本数量
replicas: 3
# 引用包含task-executor的Pool
poolRef: task-enabled-pool
# TTL:3600秒后自动清理
ttlSecondsAfterFinished: 3600
# 默认任务模板(所有沙箱共享)
taskTemplate:
spec:
process:
command: ["echo", "Default task"]
# 异构任务:为每个沙箱自定义不同的任务
shardTaskPatches:
- spec:
process:
command: ["python3", "-c", "print('Task for sandbox 0')"]
- spec:
process:
command: ["bash", "-c", "echo 'Task for sandbox 1' && sleep 5"]
- spec:
process:
command: ["node", "-e", "console.log('Task for sandbox 2')"]
```
## 镜像配置说明
### 方式1:使用Helm Values配置
在`values.yaml`中配置task-executor镜像:
```yaml
taskExecutor:
image:
repository: your-registry/opensandbox-task-executor
tag: "v1.0.0"
pullPolicy: IfNotPresent
```
然后在Pool YAML中引用:
```yaml
image: your-registry/opensandbox-task-executor:v1.0.0
```
### 方式2:使用环境变量(ConfigMap)
创建ConfigMap存储镜像信息:
```yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: opensandbox-images
namespace: default
data:
taskExecutorImage: "your-registry/opensandbox-task-executor:v1.0.0"
```
在应用层读取ConfigMap并创建Pool。
### 方式3:使用Kustomize替换
使用Kustomize的镜像替换功能:
```yaml
# kustomization.yaml
images:
- name: opensandbox.io/task-executor
newName: your-registry/opensandbox-task-executor
newTag: v1.0.0
```
## 验证Task Executor
创建资源后,验证task-executor是否正常运行:
```bash
# 查看Pool状态
kubectl get pools task-enabled-pool
# 查看Pool创建的Pod
kubectl get pods -l pool=task-enabled-pool
# 检查Pod中是否有task-executor容器
kubectl get pods -l pool=task-enabled-pool -o jsonpath='{.items[0].spec.containers[*].name}'
# 输出应包含: sandbox-container task-executor
# 查看task-executor日志
kubectl logs <pod-name> -c task-executor
# 查看BatchSandbox任务状态
kubectl get batchsandbox task-batch-sandbox -o wide
# 应显示: TASK_RUNNING, TASK_SUCCEED, TASK_FAILED 等状态
```
## 故障排查
### Task Executor无法启动
```bash
# 检查容器状态
kubectl describe pod <pod-name>
# 检查权限问题
kubectl get pod <pod-name> -o jsonpath='{.spec.containers[1].securityContext}'
# 应显示: {"capabilities":{"add":["SYS_PTRACE"]}}
# 检查进程命名空间共享
kubectl get pod <pod-name> -o jsonpath='{.spec.shareProcessNamespace}'
# 应显示: true
```
### 任务执行失败
```bash
# 查看任务状态
kubectl describe batchsandbox task-batch-sandbox
# 查看task-executor日志
kubectl logs <pod-name> -c task-executor -f
# 查看sandbox容器日志
kubectl logs <pod-name> -c sandbox-container
```
## 性能考虑
- **资源配置**:根据任务复杂度调整task-executor的资源限制
- **并发控制**:Pool的`bufferMax`和`poolMax`控制并发沙箱数量
- **任务超时**:在taskTemplate中配置超时时间防止任务卡死
- **清理策略**:使用`ttlSecondsAfterFinished`自动清理完成的沙箱
## 最佳实践
1. **镜像版本管理**:controller和task-executor镜像版本保持一致
2. **资源限制**:task-executor通常需要更多CPU用于进程注入
3. **安全配置**:只在需要时启用`SYS_PTRACE`权限
4. **任务设计**:将长时间运行的任务拆分为多个短任务
5. **监控告警**:监控任务失败率和执行时间
# ==============================================================================
# Pool for kubectl-only scenarios(纯 kubectl 场景 Pool)
# ==============================================================================
#
# 用途:纯 kubectl 场景,不使用 SDK
#
# 适用场景:
# - RL 训练批量任务
# - 压力测试
# - 批量数据处理
# - 不需要 SDK 动态交互的场景
#
# 特点:
# - 没有 execd(不支持 SDK)
# - 包含 task-executor(支持 taskTemplate)
# - 配置简洁,资源占用少
#
# ==============================================================================
apiVersion: sandbox.opensandbox.io/v1alpha1
kind: Pool
metadata:
name: kubectl-pool
namespace: default
labels:
app: opensandbox
component: kubectl-pool
annotations:
description: "纯 kubectl 场景 Pool,不包含 execd"
spec:
template:
metadata:
labels:
pool: kubectl-pool
sdk-compatible: "false"
spec:
# ========================================
# 必需:共享进程命名空间(task-executor 需要)
# ========================================
shareProcessNamespace: true
# ========================================
# 主容器:Sandbox 环境
# ========================================
containers:
- name: sandbox-container
image: nginx:latest
imagePullPolicy: IfNotPresent
# 直接运行业务命令,不需要 bootstrap.sh
command: ["sleep", "infinity"]
# 资源配置
resources:
requests:
cpu: "100m"
memory: "128Mi"
limits:
cpu: "500m"
memory: "256Mi"
# ========================================
# Sidecar:Task Executor
# ========================================
- name: task-executor
image: opensandbox/task-executor:dev
imagePullPolicy: Never
ports:
- containerPort: 5758
name: task-executor
protocol: TCP
resources:
requests:
cpu: "100m"
memory: "128Mi"
limits:
cpu: "500m"
memory: "256Mi"
# 安全上下文
securityContext:
capabilities:
add: ["SYS_PTRACE"]
# ========================================
# Pool 容量配置
# ========================================
capacitySpec:
bufferMin: 2
bufferMax: 5
poolMin: 2
poolMax: 10
apiVersion: sandbox.opensandbox.io/v1alpha1
kind: Pool
metadata:
name: sdk-pool
namespace: default
spec:
template:
spec:
# Init container: 安装execd
initContainers:
- name: execd-installer
image: opensandbox/execd:v1.0.21
command: ["/bin/sh", "-c"]
args:
- |
cp ./execd /opt/opensandbox/bin/execd && \
cp ./bootstrap.sh /opt/opensandbox/bin/bootstrap.sh && \
chmod +x /opt/opensandbox/bin/execd && \
chmod +x /opt/opensandbox/bin/bootstrap.sh
volumeMounts:
- name: opensandbox-bin
mountPath: /opt/opensandbox/bin
# 主容器:带execd
containers:
- name: sandbox-container
image: nginx:latest
command:
- /opt/opensandbox/bin/bootstrap.sh
- nginx
- -g
- daemon off; # nginx前台运行
env:
- name: EXECD
value: /opt/opensandbox/bin/execd
ports:
- containerPort: 80
name: http
- containerPort: 44772
name: execd
protocol: TCP
resources:
requests:
cpu: "100m"
memory: "128Mi"
limits:
cpu: "500m"
memory: "256Mi"
volumeMounts:
- name: opensandbox-bin
mountPath: /opt/opensandbox/bin
# 共享卷
volumes:
- name: opensandbox-bin
emptyDir: {}
capacitySpec:
bufferMax: 10
bufferMin: 2
poolMax: 20
poolMin: 5
apiVersion: sandbox.opensandbox.io/v1alpha1
kind: Pool
metadata:
name: sdk-pool-with-tasks
namespace: default
spec:
template:
spec:
shareProcessNamespace: true # task-executor需要
# Init container: 安装execd
initContainers:
- name: execd-installer
image: opensandbox/execd:v1.0.21
command: ["/bin/sh", "-c"]
args:
- |
cp ./execd /opt/opensandbox/bin/execd && \
cp ./bootstrap.sh /opt/opensandbox/bin/bootstrap.sh && \
chmod +x /opt/opensandbox/bin/execd && \
chmod +x /opt/opensandbox/bin/bootstrap.sh
volumeMounts:
- name: opensandbox-bin
mountPath: /opt/opensandbox/bin
containers:
# 主容器:带execd
- name: sandbox-container
image: nginx:latest
command:
- /opt/opensandbox/bin/bootstrap.sh
- sleep
- infinity
env:
- name: EXECD
value: /opt/opensandbox/bin/execd
ports:
- containerPort: 44772
name: execd
protocol: TCP
resources:
requests:
cpu: "100m"
memory: "128Mi"
limits:
cpu: "500m"
memory: "256Mi"
volumeMounts:
- name: opensandbox-bin
mountPath: /opt/opensandbox/bin
# task-executor sidecar: 支持自定义entrypoint
- name: task-executor
image: opensandbox/task-executor:dev
imagePullPolicy: Never # 使用本地镜像
securityContext:
capabilities:
add: ["SYS_PTRACE"]
resources:
requests:
cpu: "100m"
memory: "128Mi"
limits:
cpu: "500m"
memory: "256Mi"
volumes:
- name: opensandbox-bin
emptyDir: {}
capacitySpec:
bufferMax: 10
bufferMin: 2
poolMax: 20
poolMin: 5
# OpenSandbox Helm Chart Scripts
This directory contains utility scripts for OpenSandbox Controller deployment and testing.
## Script List
### 1. install.sh - Installation Script
Interactive installation of OpenSandbox Controller to Kubernetes cluster.
**Features:**
- Automatic detection of sudo privilege requirements
- Validation of dependency tools (helm, kubectl)
- Cluster connection verification
- Support for multiple deployment environments:
- Default configuration (values.yaml)
- E2E testing (values-e2e.yaml)
- Custom configuration (via --set or custom values file)
- Helm Chart validation
- Display verification commands after deployment
**Usage:**
```bash
cd scripts
./install.sh
```
**Environment Variables:**
- `IMAGE_REPO` - Override controller image repository
- `IMAGE_TAG` - Override controller image tag
- `SERVER_IMAGE_REPO` - Override server image repository
- `SERVER_IMAGE_TAG` - Override server image tag
**Example:**
```bash
# Using custom images
IMAGE_REPO=myregistry.com/controller \
IMAGE_TAG=v1.0.0 \
SERVER_IMAGE_REPO=myregistry.com/server \
SERVER_IMAGE_TAG=v0.2.1 \
./install.sh
```
### 2. uninstall.sh - Uninstallation Script
Uninstall OpenSandbox Controller and clean up related resources.
**Features:**
- Check running BatchSandbox and Pool resources
- Display Controller and Server deployment status
- Optional CRD deletion
- Optional namespace deletion
- Post-uninstall cleanup verification
**Usage:**
```bash
cd scripts
./uninstall.sh
```
**Environment Variables:**
- `RELEASE_NAME` - Release name (default: opensandbox-controller)
- `NAMESPACE` - Namespace (default: opensandbox)
**Example:**
```bash
# Uninstall specific release
RELEASE_NAME=my-release NAMESPACE=my-namespace ./uninstall.sh
```
### 3. e2e-test.sh - End-to-End Test Script
Execute complete end-to-end test workflow.
**Test Workflow:**
1. Helm Install (using values-e2e.yaml)
2. Verify Controller and Server deployment
3. Verify Pool deployment
4. Verify SDK calls
5. Helm Uninstall
**Features:**
- Automatic Server port-forward setup
- Server API health check validation
- Pool Pod execd process verification
- SDK integration test execution
- Automatic resource cleanup (including port-forward processes)
**Usage:**
```bash
cd scripts
./e2e-test.sh [VALUES_FILE]
# Using default values-e2e.yaml
./e2e-test.sh
# Using custom values file
./e2e-test.sh custom-values.yaml
```
**Prerequisites:**
- Required Docker images must be loaded:
- opensandbox/controller:dev
- opensandbox/server:v0.2.1
- opensandbox/task-executor:dev
- opensandbox/execd:v1.0.21
- nginx:latest
- Python SDK installed (using uv)
- Cluster has sufficient resources to run test Pods
## General Instructions
### Sudo Privileges
All scripts automatically detect whether sudo privileges are required to execute kubectl and helm commands.
### Script Paths
Scripts use relative paths to locate the Chart directory and can be invoked from any location:
```bash
# From chart root directory
./scripts/install.sh
# From scripts directory
cd scripts
./install.sh
# From other directory
/path/to/opensandbox-controller/scripts/install.sh
```
### Colored Output
Scripts use ANSI color codes to enhance readability:
- 🟢 Green - Success messages
- 🟡 Yellow - Warnings and step titles
- 🔴 Red - Error messages
### Error Handling
All scripts use `set -e`, exiting immediately on errors. The e2e-test.sh uses trap to ensure cleanup functions execute on exit.
## Troubleshooting
### install.sh
**Issue: Cannot connect to Kubernetes cluster**
```bash
# Check kubeconfig
kubectl cluster-info
# Check context
kubectl config current-context
```
**Issue: Chart validation fails**
```bash
# Manual validation
helm lint ../
```
### uninstall.sh
**Issue: Resources are still running**
```bash
# View all BatchSandbox
kubectl get batchsandboxes -A
# View all Pool
kubectl get pools -A
# Delete all resources
kubectl delete batchsandboxes --all -A
kubectl delete pools --all -A
```
### e2e-test.sh
**Issue: Port-forward fails**
```bash
# Check if any process is using port 8080
lsof -i :8080
# Manual port-forward test
kubectl port-forward -n opensandbox svc/opensandbox-controller-server 8080:8080
```
**Issue: SDK test fails**
```bash
# Check Server logs
kubectl logs -n opensandbox -l app.kubernetes.io/component=server
# Check Pool Pod logs
kubectl logs -n opensandbox -l pool=agent-pool
# Test Server API
curl http://localhost:8080/health
```
**Issue: Image not found**
```bash
# Check if images are loaded
docker images | grep opensandbox
# Reload images
docker load -i /path/to/image.tar
```
## Development Guide
### Modifying Scripts
After modifying scripts, ensure:
1. Maintain consistent error handling
2. Update related documentation
3. Use meaningful colored output
4. Add appropriate validation steps
### Adding New Scripts
New scripts should follow these conventions:
- Use `#!/bin/bash` shebang
- Use `set -e` to enable exit-on-error
- Implement automatic sudo detection
- Add colored output for readability
- Add documentation in this README
## License
Apache License 2.0
#!/bin/bash
# OpenSandbox Controller Deployment Script
set -e
# Check if sudo is required
USE_SUDO=false
if ! kubectl get nodes &> /dev/null 2>&1; then
if sudo kubectl get nodes &> /dev/null 2>&1; then
echo "Detected that sudo permissions are required, will use sudo to execute commands"
USE_SUDO=true
else
echo "Error: Unable to access Kubernetes cluster"
exit 1
fi
fi
# Define command functions
kubectl_cmd() {
if [ "$USE_SUDO" = true ]; then
sudo kubectl "$@"
else
kubectl "$@"
fi
}
helm_cmd() {
if [ "$USE_SUDO" = true ]; then
sudo helm "$@"
else
helm "$@"
fi
}
# Color output
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
RED='\033[0;31m'
NC='\033[0m' # No Color
echo -e "${GREEN}======================================${NC}"
echo -e "${GREEN}OpenSandbox Controller Helm Deployment${NC}"
echo -e "${GREEN}======================================${NC}"
echo ""
# Check dependencies
echo -e "${YELLOW}[1/6] Checking dependencies...${NC}"
if ! command -v helm &> /dev/null; then
echo -e "${RED}Error: helm command not found${NC}"
echo "Please install Helm 3.0+: https://helm.sh/docs/intro/install/"
exit 1
fi
if ! command -v kubectl &> /dev/null; then
echo -e "${RED}Error: kubectl command not found${NC}"
echo "Please install kubectl: https://kubernetes.io/docs/tasks/tools/"
exit 1
fi
echo -e "${GREEN}✓ Helm version: $(helm version --short)${NC}"
echo -e "${GREEN}✓ Kubectl version: $(kubectl version --client --short 2>/dev/null || kubectl version --client)${NC}"
echo ""
# Check cluster connection
echo -e "${YELLOW}[2/6] Checking Kubernetes cluster connection...${NC}"
if ! kubectl_cmd cluster-info &> /dev/null; then
echo -e "${RED}Error: Unable to connect to Kubernetes cluster${NC}"
echo "Please check your ~/.kube/config configuration"
exit 1
fi
echo -e "${GREEN}✓ Cluster connection successful${NC}"
kubectl_cmd cluster-info | head -2
echo ""
# Configuration parameters
# Get the parent directory of the script directory (chart root directory)
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
CHART_DIR="$(dirname "$SCRIPT_DIR")"
RELEASE_NAME="opensandbox-controller"
NAMESPACE="opensandbox"
echo -e "${YELLOW}[3/6] Configuration parameters${NC}"
echo "Chart directory: $CHART_DIR"
echo "Release name: $RELEASE_NAME"
echo "Namespace: $NAMESPACE"
if [ -n "$IMAGE_REPO" ] || [ -n "$IMAGE_TAG" ]; then
echo "Image override: ${IMAGE_REPO:-<from values>}:${IMAGE_TAG:-<from values>}"
else
echo "Image configuration: Using configuration from values file"
fi
echo ""
# Select deployment environment
echo -e "${YELLOW}[4/6] Select deployment environment${NC}"
echo "1) Default configuration (values.yaml)"
echo "2) End-to-end testing (values-e2e.yaml)"
echo "3) Custom (custom values)"
read -p "Please select [1-3]: " env_choice
case $env_choice in
1)
VALUES_FILE="$CHART_DIR/values.yaml"
echo -e "${GREEN}✓ Using default configuration${NC}"
;;
2)
VALUES_FILE="$CHART_DIR/values-e2e.yaml"
echo -e "${GREEN}✓ E2E test configuration selected${NC}"
;;
3)
read -p "Please enter values file path: " custom_values
VALUES_FILE="$custom_values"
echo -e "${GREEN}✓ Using custom configuration: $VALUES_FILE${NC}"
;;
*)
echo -e "${RED}Invalid selection, using default configuration${NC}"
VALUES_FILE="$CHART_DIR/values.yaml"
;;
esac
echo ""
# Validate Chart
echo -e "${YELLOW}[5/6] Validating Helm Chart...${NC}"
if ! helm lint "$CHART_DIR" &> /dev/null; then
echo -e "${RED}Error: Chart validation failed${NC}"
helm lint "$CHART_DIR"
exit 1
fi
echo -e "${GREEN}✓ Chart validation passed${NC}"
echo ""
# Confirm deployment
echo -e "${YELLOW}[6/6] Preparing for deployment${NC}"
echo "The following operations will be performed:"
echo " - Create namespace: $NAMESPACE"
echo " - Install CRDs: BatchSandbox, Pool"
echo " - Deploy Controller Manager"
echo " - Deploy Server (FastAPI control plane)"
echo " - Deploy RBAC resources"
echo " - Deploy Metrics service"
echo " - Create default Pool (agent-pool)"
echo ""
read -p "Confirm deployment? [y/N]: " confirm
if [[ ! $confirm =~ ^[Yy]$ ]]; then
echo -e "${YELLOW}Deployment cancelled${NC}"
exit 0
fi
# Execute deployment
echo ""
echo -e "${GREEN}Starting deployment...${NC}"
echo ""
# If environment variables are set, override image configuration in values file
EXTRA_ARGS=""
if [ -n "$IMAGE_REPO" ]; then
EXTRA_ARGS="$EXTRA_ARGS --set controllerManager.image.repository=$IMAGE_REPO"
fi
if [ -n "$IMAGE_TAG" ]; then
EXTRA_ARGS="$EXTRA_ARGS --set controllerManager.image.tag=$IMAGE_TAG"
fi
if [ -n "$SERVER_IMAGE_REPO" ]; then
EXTRA_ARGS="$EXTRA_ARGS --set server.image.repository=$SERVER_IMAGE_REPO"
fi
if [ -n "$SERVER_IMAGE_TAG" ]; then
EXTRA_ARGS="$EXTRA_ARGS --set server.image.tag=$SERVER_IMAGE_TAG"
fi
helm_cmd upgrade --install "$RELEASE_NAME" "$CHART_DIR" \
--namespace "$NAMESPACE" \
--create-namespace \
-f "$VALUES_FILE" \
$EXTRA_ARGS \
--wait \
--timeout 5m
echo ""
echo -e "${GREEN}======================================${NC}"
echo -e "${GREEN}✓ Deployment completed!${NC}"
echo -e "${GREEN}======================================${NC}"
echo ""
# Display deployment information
echo -e "${YELLOW}Deployment information:${NC}"
helm_cmd status "$RELEASE_NAME" -n "$NAMESPACE"
echo ""
echo -e "${YELLOW}Verify deployment:${NC}"
echo "1. Check Pod status:"
echo " $([ "$USE_SUDO" = true ] && echo "sudo ")kubectl get pods -n $NAMESPACE"
echo ""
echo "2. View Controller logs:"
echo " $([ "$USE_SUDO" = true ] && echo "sudo ")kubectl logs -n $NAMESPACE -l control-plane=controller-manager -f"
echo ""
echo "3. View Server logs:"
echo " $([ "$USE_SUDO" = true ] && echo "sudo ")kubectl logs -n $NAMESPACE -l app.kubernetes.io/component=server -f"
echo ""
echo "4. Access Server API (Port Forward):"
echo " $([ "$USE_SUDO" = true ] && echo "sudo ")kubectl port-forward -n $NAMESPACE svc/$RELEASE_NAME-server 8080:8080"
echo " curl http://localhost:8080/health"
echo ""
echo "5. View CRDs:"
echo " $([ "$USE_SUDO" = true ] && echo "sudo ")kubectl get crds | grep sandbox.opensandbox.io"
echo ""
echo "6. Check Pool status:"
echo " $([ "$USE_SUDO" = true ] && echo "sudo ")kubectl get pools -n $NAMESPACE"
echo ""
echo -e "${GREEN}Thank you for using OpenSandbox Controller!${NC}"
#!/bin/bash
# OpenSandbox Controller Uninstall Script
set -e
# Check if sudo is required
USE_SUDO=false
if ! kubectl get nodes &> /dev/null 2>&1; then
if sudo kubectl get nodes &> /dev/null 2>&1; then
echo "Detected sudo privileges required, will use sudo to execute commands"
USE_SUDO=true
else
echo "Error: Unable to access Kubernetes cluster"
exit 1
fi
fi
# Define command functions
kubectl_cmd() {
if [ "$USE_SUDO" = true ]; then
sudo kubectl "$@"
else
kubectl "$@"
fi
}
helm_cmd() {
if [ "$USE_SUDO" = true ]; then
sudo helm "$@"
else
helm "$@"
fi
}
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
RED='\033[0;31m'
NC='\033[0m'
echo -e "${YELLOW}======================================${NC}"
echo -e "${YELLOW}OpenSandbox Controller Uninstall${NC}"
echo -e "${YELLOW}======================================${NC}"
echo ""
RELEASE_NAME="${RELEASE_NAME:-opensandbox-controller}"
NAMESPACE="${NAMESPACE:-opensandbox}"
# Check if already installed
if ! helm_cmd list -n "$NAMESPACE" | grep -q "$RELEASE_NAME"; then
echo -e "${RED}Release not found: $RELEASE_NAME${NC}"
echo "Currently installed releases:"
helm_cmd list -A
exit 1
fi
echo "About to uninstall:"
echo " Release: $RELEASE_NAME"
echo " Namespace: $NAMESPACE"
echo ""
# Show current resources
echo -e "${YELLOW}Current resources:${NC}"
echo "Controller:"
kubectl_cmd get deployment -n "$NAMESPACE" -l control-plane=controller-manager 2>/dev/null || echo " Controller not found"
echo ""
echo "Server:"
kubectl_cmd get deployment -n "$NAMESPACE" -l app.kubernetes.io/component=server 2>/dev/null || echo " Server not found"
echo ""
# Check if there are running resources
echo -e "${YELLOW}Checking custom resources...${NC}"
BATCHSANDBOXES=$(kubectl_cmd get batchsandboxes -A --no-headers 2>/dev/null | wc -l)
POOLS=$(kubectl_cmd get pools -A --no-headers 2>/dev/null | wc -l)
if [ "$BATCHSANDBOXES" -gt 0 ] || [ "$POOLS" -gt 0 ]; then
echo -e "${RED}Warning: Running resources detected!${NC}"
echo " BatchSandboxes: $BATCHSANDBOXES"
echo " Pools: $POOLS"
echo ""
# Show Pool details
if [ "$POOLS" -gt 0 ]; then
echo "Pool details:"
kubectl_cmd get pools -A
echo ""
fi
echo "Recommended to delete these resources first:"
echo " $([ "$USE_SUDO" = true ] && echo "sudo ")kubectl delete batchsandboxes --all -A"
echo " $([ "$USE_SUDO" = true ] && echo "sudo ")kubectl delete pools --all -A"
echo ""
read -p "Continue with uninstall? [y/N]: " force_continue
if [[ ! $force_continue =~ ^[Yy]$ ]]; then
echo -e "${YELLOW}Uninstall cancelled${NC}"
exit 0
fi
fi
# Confirm uninstall
read -p "Confirm uninstall of $RELEASE_NAME (including Controller and Server)? [y/N]: " confirm
if [[ ! $confirm =~ ^[Yy]$ ]]; then
echo -e "${YELLOW}Uninstall cancelled${NC}"
exit 0
fi
echo ""
echo -e "${GREEN}Starting uninstall...${NC}"
# Uninstall Helm release
helm_cmd uninstall "$RELEASE_NAME" -n "$NAMESPACE"
echo -e "${GREEN}✓ Helm release uninstalled${NC}"
echo ""
# Wait for Pod termination
echo "Waiting for Pods to terminate..."
sleep 5
# Ask whether to delete CRDs
read -p "Delete CRDs? (This will delete all BatchSandbox and Pool resources) [y/N]: " delete_crds
if [[ $delete_crds =~ ^[Yy]$ ]]; then
echo "Deleting CRDs..."
kubectl_cmd delete crd batchsandboxes.sandbox.opensandbox.io 2>/dev/null || echo " CRD batchsandboxes does not exist"
kubectl_cmd delete crd pools.sandbox.opensandbox.io 2>/dev/null || echo " CRD pools does not exist"
echo -e "${GREEN}✓ CRDs deleted${NC}"
else
echo -e "${YELLOW}⊗ CRDs retained${NC}"
fi
echo ""
# Ask whether to delete namespace
read -p "Delete namespace $NAMESPACE? [y/N]: " delete_ns
if [[ $delete_ns =~ ^[Yy]$ ]]; then
echo "Deleting namespace..."
kubectl_cmd delete namespace "$NAMESPACE" 2>/dev/null || echo " Namespace does not exist"
echo -e "${GREEN}✓ Namespace deleted${NC}"
else
echo -e "${YELLOW}⊗ Namespace retained${NC}"
fi
echo ""
echo -e "${GREEN}======================================${NC}"
echo -e "${GREEN}✓ Uninstall completed${NC}"
echo -e "${GREEN}======================================${NC}"
echo ""
# Verify uninstall
echo -e "${YELLOW}Verifying uninstall:${NC}"
echo "Checking Helm releases:"
helm_cmd list -n "$NAMESPACE" 2>/dev/null || echo " Namespace does not exist"
echo ""
echo "Checking CRDs:"
kubectl_cmd get crds | grep sandbox.opensandbox.io || echo " No OpenSandbox CRDs found"
echo ""
Thank you for installing {{ .Chart.Name }}!
Your release is named {{ .Release.Name }}.
To learn more about the release, try:
$ helm status {{ .Release.Name }}
$ helm get all {{ .Release.Name }}
===============================================================================
OpenSandbox Kubernetes Controller has been deployed to namespace: {{ include "opensandbox-controller.namespace" . }}
Note: Both the controller and user resources (Pool, BatchSandbox) use the same namespace.
1. Check the controller status:
kubectl get deployment -n {{ include "opensandbox-controller.namespace" . }} {{ .Values.namePrefix }}controller-manager
2. View controller logs:
kubectl logs -n {{ include "opensandbox-controller.namespace" . }} -l control-plane=controller-manager -f
3. Verify CRDs are installed:
kubectl get crds | grep sandbox.opensandbox.io
You should see:
- batchsandboxes.sandbox.opensandbox.io
- pools.sandbox.opensandbox.io
4. Pool Resources:
{{- if .Values.pools }}
{{- $enabledPools := list }}
{{- range .Values.pools }}
{{- if .enabled }}
{{- $enabledPools = append $enabledPools .name }}
{{- end }}
{{- end }}
{{- if $enabledPools }}
✅ The following Pools have been deployed:
{{- range $enabledPools }}
- {{ . }}
{{- end }}
Check Pool status:
kubectl get pools -n {{ include "opensandbox-controller.namespace" . }}
Check Pool details:
kubectl describe pool agent-pool -n {{ include "opensandbox-controller.namespace" . }}
Monitor Pool capacity:
kubectl get pool agent-pool -n {{ include "opensandbox-controller.namespace" . }} -o jsonpath='{.status}'
{{- else }}
⚠️ All Pools are disabled in values.yaml.
To enable the default agent-pool:
helm upgrade {{ .Release.Name }} opensandbox-controller --set pools[0].enabled=true
{{- end }}
{{- else }}
No Pools configured.
{{- end }}
To disable Pool auto-deployment:
helm upgrade {{ .Release.Name }} opensandbox-controller --set pools[0].enabled=false
Create additional Pool manually:
# Basic Pool without task executor
cat <<EOF | kubectl apply -f -
apiVersion: sandbox.opensandbox.io/v1alpha1
kind: Pool
metadata:
name: my-first-pool
spec:
minBufferSize: 2
maxBufferSize: 5
capacity: 10
sandboxTemplate:
spec:
image: ubuntu:latest
command: ["sleep", "infinity"]
EOF
# Pool with task executor (for task execution features)
# Note: Requires task-executor image
cat <<EOF | kubectl apply -f -
apiVersion: sandbox.opensandbox.io/v1alpha1
kind: Pool
metadata:
name: task-enabled-pool
spec:
template:
spec:
shareProcessNamespace: true
containers:
- name: sandbox-container
image: ubuntu:latest
command: ["sleep", "infinity"]
- name: task-executor
image: {{ .Values.taskExecutor.image.repository }}:{{ .Values.taskExecutor.image.tag }}
securityContext:
capabilities:
add: ["SYS_PTRACE"]
capacitySpec:
bufferMax: 5
bufferMin: 2
poolMax: 10
poolMin: 2
EOF
5. Create your first BatchSandbox:
cat <<EOF | kubectl apply -f -
apiVersion: sandbox.opensandbox.io/v1alpha1
kind: BatchSandbox
metadata:
name: my-first-batchsandbox
spec:
replicas: 3
ttlSecondsAfterFinished: 3600
sandboxTemplate:
spec:
image: ubuntu:latest
command: ["sleep", "infinity"]
EOF
6. Check your resources:
kubectl get pools
kubectl get batchsandboxes
kubectl get pods
{{- if .Values.metrics.enabled }}
7. Access metrics (requires port-forward):
kubectl port-forward -n {{ include "opensandbox-controller.namespace" . }} svc/{{ .Values.namePrefix }}controller-manager-metrics-service {{ .Values.metrics.service.port }}:{{ .Values.metrics.service.port }}
Then visit: https://localhost:{{ .Values.metrics.service.port }}/metrics
{{- end }}
===============================================================================
For more information:
- Documentation: https://github.com/alibaba/OpenSandbox
- Issues: https://github.com/alibaba/OpenSandbox/issues
{{ if .Values.extraRoles.batchsandboxEditor.enabled }}
Note: Additional ClusterRoles for user management have been created:
- {{ .Values.namePrefix }}batchsandbox-editor-role
- {{ .Values.namePrefix }}batchsandbox-viewer-role
- {{ .Values.namePrefix }}pool-editor-role
- {{ .Values.namePrefix }}pool-viewer-role
You can bind these roles to users/groups as needed.
{{- end }}
{{/*
扩展chart名称
*/}}
{{- define "opensandbox-controller.name" -}}
{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
{{- end }}
{{/*
创建完整的限定名称
*/}}
{{- define "opensandbox-controller.fullname" -}}
{{- if .Values.fullnameOverride }}
{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
{{- else }}
{{- $name := default .Chart.Name .Values.nameOverride }}
{{- if contains $name .Release.Name }}
{{- .Release.Name | trunc 63 | trimSuffix "-" }}
{{- else }}
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
{{- end }}
{{- end }}
{{- end }}
{{/*
Chart标签
*/}}
{{- define "opensandbox-controller.chart" -}}
{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
{{- end }}
{{/*
通用标签
*/}}
{{- define "opensandbox-controller.labels" -}}
helm.sh/chart: {{ include "opensandbox-controller.chart" . }}
{{ include "opensandbox-controller.selectorLabels" . }}
{{- if .Chart.AppVersion }}
app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
{{- end }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
{{- with .Values.labels }}
{{ toYaml . }}
{{- end }}
{{- end }}
{{/*
选择器标签
*/}}
{{- define "opensandbox-controller.selectorLabels" -}}
app.kubernetes.io/name: {{ include "opensandbox-controller.name" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
control-plane: controller-manager
{{- end }}
{{/*
创建ServiceAccount名称
*/}}
{{- define "opensandbox-controller.serviceAccountName" -}}
{{- if .Values.rbac.serviceAccount.create }}
{{- default (printf "%scontroller-manager" .Values.namePrefix) .Values.rbac.serviceAccount.name }}
{{- else }}
{{- default "default" .Values.rbac.serviceAccount.name }}
{{- end }}
{{- end }}
{{/*
命名空间
*/}}
{{- define "opensandbox-controller.namespace" -}}
{{- if .Values.namespaceOverride }}
{{- .Values.namespaceOverride }}
{{- else }}
{{- printf "%ssystem" .Values.namePrefix }}
{{- end }}
{{- end }}
{{/*
Controller镜像
*/}}
{{- define "opensandbox-controller.controllerImage" -}}
{{- printf "%s:%s" .Values.controllerManager.image.repository (.Values.controllerManager.image.tag | default .Chart.AppVersion) }}
{{- end }}
{{/*
Task Executor镜像
*/}}
{{- define "opensandbox-controller.taskExecutorImage" -}}
{{- printf "%s:%s" .Values.taskExecutor.image.repository (.Values.taskExecutor.image.tag | default .Chart.AppVersion) }}
{{- end }}
{{- if .Values.rbac.create }}
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: {{ .Values.namePrefix }}manager-role
labels:
{{- include "opensandbox-controller.labels" . | nindent 4 }}
rules:
- apiGroups:
- ""
resources:
- events
- pods
verbs:
- create
- delete
- get
- list
- patch
- update
- watch
- apiGroups:
- ""
resources:
- pods/status
verbs:
- get
- patch
- update
- apiGroups:
- sandbox.opensandbox.io
resources:
- batchsandboxes
- pools
verbs:
- create
- delete
- get
- list
- patch
- update
- watch
- apiGroups:
- sandbox.opensandbox.io
resources:
- batchsandboxes/finalizers
- pools/finalizers
verbs:
- update
- apiGroups:
- sandbox.opensandbox.io
resources:
- batchsandboxes/status
- pools/status
verbs:
- get
- patch
- update
{{- end }}
{{- if .Values.rbac.create }}
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: {{ .Values.namePrefix }}manager-rolebinding
labels:
{{- include "opensandbox-controller.labels" . | nindent 4 }}
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: {{ .Values.namePrefix }}manager-role
subjects:
- kind: ServiceAccount
name: {{ include "opensandbox-controller.serviceAccountName" . }}
namespace: {{ include "opensandbox-controller.namespace" . }}
{{- end }}
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ .Values.namePrefix }}controller-manager
namespace: {{ include "opensandbox-controller.namespace" . }}
labels:
{{- include "opensandbox-controller.labels" . | nindent 4 }}
spec:
replicas: {{ .Values.controllerManager.replicas }}
selector:
matchLabels:
{{- include "opensandbox-controller.selectorLabels" . | nindent 6 }}
template:
metadata:
annotations:
{{- with .Values.podAnnotations }}
{{- toYaml . | nindent 8 }}
{{- end }}
labels:
{{- include "opensandbox-controller.selectorLabels" . | nindent 8 }}
{{- with .Values.podLabels }}
{{- toYaml . | nindent 8 }}
{{- end }}
spec:
{{- with .Values.imagePullSecrets }}
imagePullSecrets:
{{- toYaml . | nindent 8 }}
{{- end }}
serviceAccountName: {{ include "opensandbox-controller.serviceAccountName" . }}
securityContext:
{{- toYaml .Values.controllerManager.securityContext | nindent 8 }}
terminationGracePeriodSeconds: {{ .Values.terminationGracePeriodSeconds }}
containers:
- name: manager
image: {{ include "opensandbox-controller.controllerImage" . }}
imagePullPolicy: {{ .Values.controllerManager.image.pullPolicy }}
command:
- /workspace/server
args:
{{- if .Values.controllerManager.leaderElect }}
- --leader-elect
{{- end }}
- --health-probe-bind-address={{ .Values.controllerManager.healthProbeBindAddress }}
- --zap-log-level={{ .Values.controllerManager.logLevel }}
{{- with .Values.controllerManager.extraArgs }}
{{- toYaml . | nindent 8 }}
{{- end }}
securityContext:
{{- toYaml .Values.controllerManager.containerSecurityContext | nindent 10 }}
{{- with .Values.controllerManager.env }}
env:
{{- toYaml . | nindent 10 }}
{{- end }}
livenessProbe:
httpGet:
path: /healthz
port: {{ .Values.controllerManager.healthProbePort | default 8081 }}
initialDelaySeconds: {{ .Values.healthProbes.liveness.initialDelaySeconds | default 15 }}
periodSeconds: {{ .Values.healthProbes.liveness.periodSeconds | default 20 }}
timeoutSeconds: {{ .Values.healthProbes.liveness.timeoutSeconds | default 1 }}
failureThreshold: {{ .Values.healthProbes.liveness.failureThreshold | default 3 }}
readinessProbe:
httpGet:
path: /readyz
port: {{ .Values.controllerManager.healthProbePort | default 8081 }}
initialDelaySeconds: {{ .Values.healthProbes.readiness.initialDelaySeconds | default 5 }}
periodSeconds: {{ .Values.healthProbes.readiness.periodSeconds | default 10 }}
timeoutSeconds: {{ .Values.healthProbes.readiness.timeoutSeconds | default 1 }}
failureThreshold: {{ .Values.healthProbes.readiness.failureThreshold | default 3 }}
resources:
{{- toYaml .Values.controllerManager.resources | nindent 10 }}
{{- with .Values.controllerManager.nodeSelector }}
nodeSelector:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.controllerManager.affinity }}
affinity:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.controllerManager.tolerations }}
tolerations:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- if .Values.rbac.create }}
{{- if .Values.extraRoles.batchsandboxEditor.enabled }}
---
# This role is provided to allow the cluster admin to help manage permissions for users.
# Grants permissions to create, update, and delete BatchSandbox resources.
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: {{ .Values.namePrefix }}batchsandbox-editor-role
labels:
{{- include "opensandbox-controller.labels" . | nindent 4 }}
rules:
- apiGroups:
- sandbox.opensandbox.io
resources:
- batchsandboxes
verbs:
- create
- delete
- get
- list
- patch
- update
- watch
- apiGroups:
- sandbox.opensandbox.io
resources:
- batchsandboxes/status
verbs:
- get
{{- end }}
{{- if .Values.extraRoles.batchsandboxViewer.enabled }}
---
# This role is provided to allow the cluster admin to help manage permissions for users.
# Grants read-only permissions for BatchSandbox resources.
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: {{ .Values.namePrefix }}batchsandbox-viewer-role
labels:
{{- include "opensandbox-controller.labels" . | nindent 4 }}
rules:
- apiGroups:
- sandbox.opensandbox.io
resources:
- batchsandboxes
verbs:
- get
- list
- watch
- apiGroups:
- sandbox.opensandbox.io
resources:
- batchsandboxes/status
verbs:
- get
{{- end }}
{{- if .Values.extraRoles.poolEditor.enabled }}
---
# This role is provided to allow the cluster admin to help manage permissions for users.
# Grants permissions to create, update, and delete Pool resources.
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: {{ .Values.namePrefix }}pool-editor-role
labels:
{{- include "opensandbox-controller.labels" . | nindent 4 }}
rules:
- apiGroups:
- sandbox.opensandbox.io
resources:
- pools
verbs:
- create
- delete
- get
- list
- patch
- update
- watch
- apiGroups:
- sandbox.opensandbox.io
resources:
- pools/status
verbs:
- get
{{- end }}
{{- if .Values.extraRoles.poolViewer.enabled }}
---
# This role is provided to allow the cluster admin to help manage permissions for users.
# Grants read-only permissions for Pool resources.
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: {{ .Values.namePrefix }}pool-viewer-role
labels:
{{- include "opensandbox-controller.labels" . | nindent 4 }}
rules:
- apiGroups:
- sandbox.opensandbox.io
resources:
- pools
verbs:
- get
- list
- watch
- apiGroups:
- sandbox.opensandbox.io
resources:
- pools/status
verbs:
- get
{{- end }}
{{- end }}
{{- if .Values.rbac.create }}
# permissions to do leader election.
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: {{ .Values.namePrefix }}leader-election-role
namespace: {{ include "opensandbox-controller.namespace" . }}
labels:
{{- include "opensandbox-controller.labels" . | nindent 4 }}
rules:
- apiGroups:
- ""
resources:
- configmaps
verbs:
- get
- list
- watch
- create
- update
- patch
- delete
- apiGroups:
- coordination.k8s.io
resources:
- leases
verbs:
- get
- list
- watch
- create
- update
- patch
- delete
- apiGroups:
- ""
resources:
- events
verbs:
- create
- patch
{{- end }}
{{- if .Values.rbac.create }}
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: {{ .Values.namePrefix }}leader-election-rolebinding
namespace: {{ include "opensandbox-controller.namespace" . }}
labels:
{{- include "opensandbox-controller.labels" . | nindent 4 }}
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: {{ .Values.namePrefix }}leader-election-role
subjects:
- kind: ServiceAccount
name: {{ include "opensandbox-controller.serviceAccountName" . }}
namespace: {{ include "opensandbox-controller.namespace" . }}
{{- end }}
{{- if and .Values.rbac.create .Values.metrics.enabled }}
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: {{ .Values.namePrefix }}metrics-auth-role
labels:
{{- include "opensandbox-controller.labels" . | nindent 4 }}
rules:
- apiGroups:
- authentication.k8s.io
resources:
- tokenreviews
verbs:
- create
- apiGroups:
- authorization.k8s.io
resources:
- subjectaccessreviews
verbs:
- create
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: {{ .Values.namePrefix }}metrics-auth-rolebinding
labels:
{{- include "opensandbox-controller.labels" . | nindent 4 }}
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: {{ .Values.namePrefix }}metrics-auth-role
subjects:
- kind: ServiceAccount
name: {{ include "opensandbox-controller.serviceAccountName" . }}
namespace: {{ include "opensandbox-controller.namespace" . }}
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: {{ .Values.namePrefix }}metrics-reader
labels:
{{- include "opensandbox-controller.labels" . | nindent 4 }}
rules:
- nonResourceURLs:
- /metrics
verbs:
- get
{{- end }}
{{- if .Values.metrics.enabled }}
apiVersion: v1
kind: Service
metadata:
name: {{ .Values.namePrefix }}controller-manager-metrics-service
namespace: {{ include "opensandbox-controller.namespace" . }}
labels:
{{- include "opensandbox-controller.labels" . | nindent 4 }}
control-plane: controller-manager
spec:
type: {{ .Values.metrics.service.type }}
ports:
- name: https
port: {{ .Values.metrics.service.port }}
protocol: TCP
targetPort: {{ .Values.metrics.service.port }}
selector:
{{- include "opensandbox-controller.selectorLabels" . | nindent 4 }}
{{- end }}
{{- if .Values.podDisruptionBudget.enabled }}
apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
name: {{ include "opensandbox-controller.fullname" . }}-controller
namespace: {{ include "opensandbox-controller.namespace" . }}
labels:
{{- include "opensandbox-controller.labels" . | nindent 4 }}
spec:
{{- if .Values.podDisruptionBudget.minAvailable }}
minAvailable: {{ .Values.podDisruptionBudget.minAvailable }}
{{- end }}
{{- if .Values.podDisruptionBudget.maxUnavailable }}
maxUnavailable: {{ .Values.podDisruptionBudget.maxUnavailable }}
{{- end }}
selector:
matchLabels:
control-plane: controller-manager
{{- include "opensandbox-controller.selectorLabels" . | nindent 6 }}
{{- end }}
{{- if .Values.pools }}
{{- range .Values.pools }}
{{- if .enabled }}
---
apiVersion: sandbox.opensandbox.io/v1alpha1
kind: Pool
metadata:
name: {{ .name }}
namespace: {{ include "opensandbox-controller.namespace" $ }}
labels:
{{- include "opensandbox-controller.labels" $ | nindent 4 }}
{{- with .labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
{{- with .annotations }}
annotations:
{{- toYaml . | nindent 4 }}
{{- end }}
spec:
{{- if .template }}
template:
{{- with .template.metadata }}
metadata:
{{- toYaml . | nindent 6 }}
{{- end }}
spec:
{{- if .template.spec.shareProcessNamespace }}
shareProcessNamespace: {{ .template.spec.shareProcessNamespace }}
{{- end }}
{{- if .template.spec.initContainers }}
initContainers:
{{- range .template.spec.initContainers }}
- name: {{ .name }}
image: {{ .image }}
{{- if .imagePullPolicy }}
imagePullPolicy: {{ .imagePullPolicy }}
{{- end }}
{{- if .command }}
command:
{{- toYaml .command | nindent 10 }}
{{- end }}
{{- if .args }}
args:
{{- toYaml .args | nindent 10 }}
{{- end }}
{{- if .env }}
env:
{{- toYaml .env | nindent 10 }}
{{- end }}
{{- if .volumeMounts }}
volumeMounts:
{{- toYaml .volumeMounts | nindent 10 }}
{{- end }}
{{- if .resources }}
resources:
{{- toYaml .resources | nindent 10 }}
{{- end }}
{{- if .securityContext }}
securityContext:
{{- toYaml .securityContext | nindent 10 }}
{{- end }}
{{- end }}
{{- end }}
containers:
{{- range .template.spec.containers }}
- name: {{ .name }}
image: {{ .image }}
{{- if .imagePullPolicy }}
imagePullPolicy: {{ .imagePullPolicy }}
{{- end }}
{{- if .command }}
command:
{{- toYaml .command | nindent 10 }}
{{- end }}
{{- if .args }}
args:
{{- toYaml .args | nindent 10 }}
{{- end }}
{{- if .env }}
env:
{{- toYaml .env | nindent 10 }}
{{- end }}
{{- if .ports }}
ports:
{{- toYaml .ports | nindent 10 }}
{{- end }}
{{- if .volumeMounts }}
volumeMounts:
{{- toYaml .volumeMounts | nindent 10 }}
{{- end }}
{{- if .resources }}
resources:
{{- toYaml .resources | nindent 10 }}
{{- end }}
{{- if .securityContext }}
securityContext:
{{- toYaml .securityContext | nindent 10 }}
{{- end }}
{{- if .livenessProbe }}
livenessProbe:
{{- toYaml .livenessProbe | nindent 10 }}
{{- end }}
{{- if .readinessProbe }}
readinessProbe:
{{- toYaml .readinessProbe | nindent 10 }}
{{- end }}
{{- end }}
{{- if .template.spec.volumes }}
volumes:
{{- toYaml .template.spec.volumes | nindent 6 }}
{{- end }}
{{- if .template.spec.nodeSelector }}
nodeSelector:
{{- toYaml .template.spec.nodeSelector | nindent 8 }}
{{- end }}
{{- if .template.spec.tolerations }}
tolerations:
{{- toYaml .template.spec.tolerations | nindent 8 }}
{{- end }}
{{- if .template.spec.affinity }}
affinity:
{{- toYaml .template.spec.affinity | nindent 8 }}
{{- end }}
{{- end }}
capacitySpec:
{{- if .capacitySpec.bufferMin }}
bufferMin: {{ .capacitySpec.bufferMin }}
{{- end }}
{{- if .capacitySpec.bufferMax }}
bufferMax: {{ .capacitySpec.bufferMax }}
{{- end }}
{{- if .capacitySpec.poolMin }}
poolMin: {{ .capacitySpec.poolMin }}
{{- end }}
{{- if .capacitySpec.poolMax }}
poolMax: {{ .capacitySpec.poolMax }}
{{- end }}
{{- end }}
{{- end }}
{{- end }}
{{- if .Values.server.enabled }}
apiVersion: v1
kind: ConfigMap
metadata:
name: {{ include "opensandbox-controller.fullname" . }}-server-config
namespace: {{ include "opensandbox-controller.namespace" . }}
labels:
{{- include "opensandbox-controller.labels" . | nindent 4 }}
app.kubernetes.io/component: server
data:
config.toml: |
# OpenSandbox Server Configuration
# Generated by Helm Chart
[server]
host = "{{ .Values.server.config.server.host }}"
port = {{ .Values.server.config.server.port }}
log_level = "{{ .Values.server.config.server.logLevel }}"
{{- if .Values.server.config.server.apiKey }}
api_key = "{{ .Values.server.config.server.apiKey }}"
{{- end }}
[runtime]
type = "{{ .Values.server.config.runtime.type }}"
execd_image = "{{ .Values.server.config.runtime.execdImage }}"
[kubernetes]
# Use in-cluster configuration (empty kubeconfig_path)
kubeconfig_path = ""
namespace = "{{ include "opensandbox-controller.namespace" . }}"
# 优先使用 Pool 模式(batchsandbox provider + Pool)
workload_provider = "{{ .Values.server.config.kubernetes.workloadProvider }}"
# BatchSandbox 模板文件(非 Pool 模式需要)
batchsandbox_template_file = "/etc/opensandbox/batchsandbox-template.yaml"
# BatchSandbox 模板(用于非 Pool 模式)
batchsandbox-template.yaml: |
# BatchSandbox CR template for non-pooled mode
# Pool mode does not use this template
metadata:
# Metadata will be merged with runtime-generated values
spec:
replicas: 1
template:
spec:
restartPolicy: Never
{{- if .Values.server.config.kubernetes.batchsandboxTemplate.tolerations }}
tolerations:
{{- toYaml .Values.server.config.kubernetes.batchsandboxTemplate.tolerations | nindent 12 }}
{{- else }}
tolerations:
- operator: "Exists"
{{- end }}
{{- if .Values.server.config.kubernetes.batchsandboxTemplate.nodeSelector }}
nodeSelector:
{{- toYaml .Values.server.config.kubernetes.batchsandboxTemplate.nodeSelector | nindent 12 }}
{{- end }}
{{- end }}
{{- if .Values.server.enabled }}
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ include "opensandbox-controller.fullname" . }}-server
namespace: {{ include "opensandbox-controller.namespace" . }}
labels:
{{- include "opensandbox-controller.labels" . | nindent 4 }}
app.kubernetes.io/component: server
spec:
replicas: {{ .Values.server.replicas }}
selector:
matchLabels:
{{- include "opensandbox-controller.selectorLabels" . | nindent 6 }}
app.kubernetes.io/component: server
template:
metadata:
annotations:
checksum/config: {{ include (print $.Template.BasePath "/server-configmap.yaml") . | sha256sum }}
{{- with .Values.server.podAnnotations }}
{{- toYaml . | nindent 8 }}
{{- end }}
labels:
{{- include "opensandbox-controller.selectorLabels" . | nindent 8 }}
app.kubernetes.io/component: server
{{- with .Values.server.podLabels }}
{{- toYaml . | nindent 8 }}
{{- end }}
spec:
{{- with .Values.imagePullSecrets }}
imagePullSecrets:
{{- toYaml . | nindent 8 }}
{{- end }}
serviceAccountName: {{ include "opensandbox-controller.serviceAccountName" . }}
securityContext:
{{- toYaml .Values.server.securityContext | nindent 8 }}
terminationGracePeriodSeconds: {{ .Values.server.terminationGracePeriodSeconds | default 10 }}
containers:
- name: server
image: "{{ .Values.server.image.repository }}:{{ .Values.server.image.tag }}"
imagePullPolicy: {{ .Values.server.image.pullPolicy }}
args:
- --config
- /etc/opensandbox/config.toml
{{- if .Values.server.reload }}
- --reload
{{- end }}
{{- with .Values.server.extraArgs }}
{{- toYaml . | nindent 8 }}
{{- end }}
ports:
- name: http
containerPort: {{ .Values.server.config.server.port }}
protocol: TCP
{{- with .Values.server.env }}
env:
{{- toYaml . | nindent 10 }}
{{- end }}
livenessProbe:
httpGet:
path: {{ .Values.server.healthProbes.liveness.path }}
port: http
initialDelaySeconds: {{ .Values.server.healthProbes.liveness.initialDelaySeconds }}
periodSeconds: {{ .Values.server.healthProbes.liveness.periodSeconds }}
timeoutSeconds: {{ .Values.server.healthProbes.liveness.timeoutSeconds }}
failureThreshold: {{ .Values.server.healthProbes.liveness.failureThreshold }}
readinessProbe:
httpGet:
path: {{ .Values.server.healthProbes.readiness.path }}
port: http
initialDelaySeconds: {{ .Values.server.healthProbes.readiness.initialDelaySeconds }}
periodSeconds: {{ .Values.server.healthProbes.readiness.periodSeconds }}
timeoutSeconds: {{ .Values.server.healthProbes.readiness.timeoutSeconds }}
failureThreshold: {{ .Values.server.healthProbes.readiness.failureThreshold }}
resources:
{{- toYaml .Values.server.resources | nindent 10 }}
volumeMounts:
- name: config
mountPath: /etc/opensandbox
readOnly: true
securityContext:
{{- toYaml .Values.server.containerSecurityContext | nindent 10 }}
volumes:
- name: config
configMap:
name: {{ include "opensandbox-controller.fullname" . }}-server-config
{{- with .Values.server.nodeSelector }}
nodeSelector:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.server.affinity }}
affinity:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.server.tolerations }}
tolerations:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- end }}
{{- if and .Values.server.enabled .Values.server.ingress.enabled }}
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: {{ include "opensandbox-controller.fullname" . }}-server
namespace: {{ include "opensandbox-controller.namespace" . }}
labels:
{{- include "opensandbox-controller.labels" . | nindent 4 }}
app.kubernetes.io/component: server
{{- with .Values.server.ingress.annotations }}
annotations:
{{- toYaml . | nindent 4 }}
{{- end }}
spec:
{{- if .Values.server.ingress.className }}
ingressClassName: {{ .Values.server.ingress.className }}
{{- end }}
{{- if .Values.server.ingress.tls }}
tls:
{{- range .Values.server.ingress.tls }}
- hosts:
{{- range .hosts }}
- {{ . | quote }}
{{- end }}
secretName: {{ .secretName }}
{{- end }}
{{- end }}
rules:
{{- range .Values.server.ingress.hosts }}
- host: {{ .host | quote }}
http:
paths:
{{- range .paths }}
- path: {{ .path }}
pathType: {{ .pathType }}
backend:
service:
name: {{ include "opensandbox-controller.fullname" $ }}-server
port:
number: {{ $.Values.server.service.port }}
{{- end }}
{{- end }}
{{- end }}
{{- if .Values.server.enabled }}
apiVersion: v1
kind: Service
metadata:
name: {{ include "opensandbox-controller.fullname" . }}-server
namespace: {{ include "opensandbox-controller.namespace" . }}
labels:
{{- include "opensandbox-controller.labels" . | nindent 4 }}
app.kubernetes.io/component: server
{{- with .Values.server.service.annotations }}
annotations:
{{- toYaml . | nindent 4 }}
{{- end }}
spec:
type: {{ .Values.server.service.type }}
{{- if and (eq .Values.server.service.type "NodePort") .Values.server.service.nodePort }}
ports:
- port: {{ .Values.server.service.port }}
targetPort: http
protocol: TCP
name: http
nodePort: {{ .Values.server.service.nodePort }}
{{- else }}
ports:
- port: {{ .Values.server.service.port }}
targetPort: http
protocol: TCP
name: http
{{- end }}
selector:
{{- include "opensandbox-controller.selectorLabels" . | nindent 4 }}
app.kubernetes.io/component: server
{{- end }}
{{- if .Values.rbac.serviceAccount.create -}}
apiVersion: v1
kind: ServiceAccount
metadata:
name: {{ include "opensandbox-controller.serviceAccountName" . }}
namespace: {{ include "opensandbox-controller.namespace" . }}
labels:
{{- include "opensandbox-controller.labels" . | nindent 4 }}
{{- with .Values.rbac.serviceAccount.annotations }}
annotations:
{{- toYaml . | nindent 4 }}
{{- end }}
{{- end }}
{{- if and .Values.metrics.enabled .Values.metrics.serviceMonitor.enabled }}
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
name: {{ .Values.namePrefix }}controller-manager-metrics-monitor
namespace: {{ include "opensandbox-controller.namespace" . }}
labels:
{{- include "opensandbox-controller.labels" . | nindent 4 }}
spec:
endpoints:
- path: /metrics
port: https
scheme: https
bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
tlsConfig:
insecureSkipVerify: true
interval: {{ .Values.metrics.serviceMonitor.interval }}
scrapeTimeout: {{ .Values.metrics.serviceMonitor.scrapeTimeout }}
selector:
matchLabels:
{{- include "opensandbox-controller.selectorLabels" . | nindent 6 }}
control-plane: controller-manager
{{- end }}
# OpenSandbox Helm Chart - E2E测试配置
# values-e2e.yaml
#
# 此配置文件专为端到端测试设计,使用最小资源配置
# Controller配置
controllerManager:
image:
repository: opensandbox/controller
tag: latest
pullPolicy: Never
replicas: 1
resources:
limits:
cpu: 1
memory: 1Gi
requests:
cpu: 200m
memory: 256Mi
# Task Executor镜像配置
taskExecutor:
image:
repository: opensandbox/task-executor
tag: latest
pullPolicy: Never
# Server配置 - E2E测试
server:
enabled: true
image:
repository: opensandbox/server
tag: latest
pullPolicy: Never
replicas: 1
resources:
limits:
cpu: 500m
memory: 512Mi
requests:
cpu: 100m
memory: 256Mi
# Server 安全上下文 - 镜像以 root 运行
securityContext: {}
containerSecurityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- "ALL"
# 使用 ClusterIP,通过 port-forward 访问
service:
type: ClusterIP
port: 8080
# Pool配置 - 最小配置用于E2E测试
pools:
- name: agent-pool
enabled: true
labels:
app: opensandbox
component: agent-pool
annotations:
description: "E2E Test Pool - Minimal Capacity"
template:
metadata:
labels:
pool: agent-pool
sdk-compatible: "true"
spec:
shareProcessNamespace: true
initContainers:
- name: execd-installer
image: opensandbox/execd:v1.0.21
imagePullPolicy: Never
command: ["/bin/sh", "-c"]
args:
- |
cp ./execd /opt/opensandbox/bin/execd && \
cp ./bootstrap.sh /opt/opensandbox/bin/bootstrap.sh && \
chmod +x /opt/opensandbox/bin/*
volumeMounts:
- name: opensandbox-bin
mountPath: /opt/opensandbox/bin
containers:
- name: sandbox-container
image: nginx:latest
imagePullPolicy: Never
command: ["/opt/opensandbox/bin/bootstrap.sh", "sleep", "infinity"]
env:
- name: EXECD
value: /opt/opensandbox/bin/execd
ports:
- containerPort: 44772
name: execd
resources:
requests:
cpu: "50m"
memory: "64Mi"
limits:
cpu: "200m"
memory: "128Mi"
volumeMounts:
- name: opensandbox-bin
mountPath: /opt/opensandbox/bin
- name: task-executor
image: opensandbox/task-executor:dev
imagePullPolicy: Never
resources:
requests:
cpu: "50m"
memory: "64Mi"
limits:
cpu: "200m"
memory: "128Mi"
securityContext:
capabilities:
add: ["SYS_PTRACE"]
volumes:
- name: opensandbox-bin
emptyDir: {}
# E2E测试容量配置:最小化以避免主机过载
capacitySpec:
bufferMin: 2 # 最小可用2个
bufferMax: 3 # 最多预热3个
poolMin: 2 # 最小总数2个
poolMax: 5 # 最大总数5个
# Metrics监控
metrics:
enabled: true
serviceMonitor:
enabled: false # 禁用 ServiceMonitor
......@@ -447,6 +447,8 @@ configs:
[egress]
image = "${{opensandbox-egress.image}}:${{opensandbox-egress.tag}}"
mode = "dns+nft"
disable_ipv6 = true
[docker]
network_mode = "bridge"
......
......@@ -8,9 +8,9 @@
"volume-manager": "v0.3.0-beta4",
"agent-sandbox-image": "v0.3.0-beta4",
"agent-sandbox-proxy": "v0.3.0-beta4",
"opensandbox-server": "v0.2.1",
"opensandbox-server": "v0.2.2",
"opensandbox-execd": "v1.0.21",
"opensandbox-egress": "v1.1.4",
"opensandbox-egress": "v1.1.5",
"aiproxy": "v0.6.5",
"aiproxy-pg": "0.8.0-pg15",
......
......@@ -4,25 +4,60 @@
* 只负责 OpenSandbox createConfig 映射,不连接远端实例。
*/
import { serviceEnv } from '../../../../../../env';
import type { SandboxRuntimeProfile } from './types';
import type { SandboxRuntimeCreateConfigInput, SandboxRuntimeProfile } from './types';
import { getSandboxSkillsRootPath, mergeStringRecord, normalizeEntrypoint } from './utils';
import { OPEN_SANDBOX_DEFAULT_ROOT_PATH, parseImageSpec } from '@fastgpt-sdk/sandbox-adapter';
const OPEN_SANDBOX_ENTRYPOINT = '/home/sandbox/entrypoint.sh';
const OPEN_SANDBOX_DOCKER_LOCAL_NETWORK_POLICY = {
const OPEN_SANDBOX_PROTECTED_NETWORK_DENY_RULES = [
{ action: 'deny' as const, target: 'localhost' },
{ action: 'deny' as const, target: '127.0.0.0/8' },
{ action: 'deny' as const, target: '::1/128' },
{ action: 'deny' as const, target: '10.0.0.0/8' },
{ action: 'deny' as const, target: '100.64.0.0/10' },
{ action: 'deny' as const, target: '169.254.0.0/16' },
{ action: 'deny' as const, target: '172.16.0.0/12' },
{ action: 'deny' as const, target: '192.168.0.0/16' },
{ action: 'deny' as const, target: '198.18.0.0/15' },
{ action: 'deny' as const, target: '224.0.0.0/4' },
{ action: 'deny' as const, target: 'fc00::/7' },
{ action: 'deny' as const, target: 'fe80::/10' },
{ action: 'deny' as const, target: '*.local' },
{ action: 'deny' as const, target: 'host.docker.internal' },
{ action: 'deny' as const, target: 'host.orb.internal' },
{ action: 'deny' as const, target: 'docker.orb.internal' },
{ action: 'deny' as const, target: 'gateway.orb.internal' },
{ action: 'deny' as const, target: 'proxyproxy.orb.internal' },
{ action: 'deny' as const, target: '*.orb.internal' },
{ action: 'deny' as const, target: '*.orb.local' }
];
const OPEN_SANDBOX_NETWORK_POLICY = {
defaultAction: 'allow' as const,
egress: [
{ action: 'deny' as const, target: 'localhost' },
{ action: 'deny' as const, target: 'host.docker.internal' },
{ action: 'deny' as const, target: 'host.orb.internal' },
{ action: 'deny' as const, target: 'docker.orb.internal' },
{ action: 'deny' as const, target: 'gateway.orb.internal' },
{ action: 'deny' as const, target: 'proxyproxy.orb.internal' },
{ action: 'deny' as const, target: '*.orb.internal' },
{ action: 'deny' as const, target: '*.orb.local' }
]
egress: OPEN_SANDBOX_PROTECTED_NETWORK_DENY_RULES
};
/** Keep internal network targets denied while allowing all unmatched public destinations. */
function buildOpenSandboxNetworkPolicy(
policy?: NonNullable<SandboxRuntimeCreateConfigInput['createConfig']>['networkPolicy']
) {
const protectedTargets = new Set(
OPEN_SANDBOX_PROTECTED_NETWORK_DENY_RULES.map(({ target }) => target)
);
return {
// Public egress is intentionally open. Create callers may add deny rules, but
// cannot replace the platform default with deny or override protected targets.
defaultAction: OPEN_SANDBOX_NETWORK_POLICY.defaultAction,
egress: [
...OPEN_SANDBOX_PROTECTED_NETWORK_DENY_RULES,
...(policy?.egress?.filter(
({ action, target }) => action === 'deny' && !protectedTargets.has(target)
) ?? [])
]
};
}
/**
* 构建 OpenSandbox 的 FastGPT 运行态 profile。
*
......@@ -71,13 +106,8 @@ export function buildOpenSandboxRuntimeProfile(): SandboxRuntimeProfile {
const metadata = mergeStringRecord(createConfig.metadata, input.metadata);
// volume 既可能来自 volume manager,也可能来自调用方透传的 createConfig;运行态 VM 配置优先。
const volumes = input.volumes ?? input.vmConfig?.volumes ?? createConfig.volumes;
// Docker 模式下默认拒绝常见宿主机别名;公网默认保持放行,私网 CIDR 需依赖部署网络边界。
const networkPolicy =
createConfig.networkPolicy ??
(!serviceEnv.AGENT_SANDBOX_OPENSANDBOX_DISABLE_NETWORK_POLICY &&
serviceEnv.AGENT_SANDBOX_OPENSANDBOX_RUNTIME === 'docker'
? OPEN_SANDBOX_DOCKER_LOCAL_NETWORK_POLICY
: undefined);
// Docker/Kubernetes 均启用 nftables 层的内部网段拒绝;未命中规则的公网地址保持放行。
const networkPolicy = buildOpenSandboxNetworkPolicy(createConfig.networkPolicy);
return {
...createConfig,
......
......@@ -120,12 +120,6 @@ export const serviceEnv = createEnv({
AGENT_SANDBOX_OPENSANDBOX_VOLUME_NAME_PREFIX: SandboxVolumeNameSchema.default(
'fastgpt-session'
).meta({ description: 'OpenSandbox persistent volume claimName prefix' }),
AGENT_SANDBOX_OPENSANDBOX_DISABLE_NETWORK_POLICY: BoolSchema.default(false).meta({
description:
'Disable the default outbound network policy for OpenSandbox Docker runtime. ' +
'Set to true when the OpenSandbox server uses a user-defined Docker network (not bridge), ' +
'as networkPolicy is only supported with network_mode=bridge.'
}),
AGENT_SANDBOX_SUSPEND_MINUTES: IntSchema.min(1).default(60).meta({
description: 'Agent sandbox 持续未活跃多少分钟后自动暂停'
}),
......
......@@ -28,10 +28,22 @@ const loadSandboxConfigModule = async () => {
return import('@fastgpt/service/core/ai/sandbox/infrastructure/provider/config');
};
const defaultOpenSandboxDockerNetworkPolicy = {
const defaultOpenSandboxNetworkPolicy = {
defaultAction: 'allow',
egress: [
{ action: 'deny', target: 'localhost' },
{ action: 'deny', target: '127.0.0.0/8' },
{ action: 'deny', target: '::1/128' },
{ action: 'deny', target: '10.0.0.0/8' },
{ action: 'deny', target: '100.64.0.0/10' },
{ action: 'deny', target: '169.254.0.0/16' },
{ action: 'deny', target: '172.16.0.0/12' },
{ action: 'deny', target: '192.168.0.0/16' },
{ action: 'deny', target: '198.18.0.0/15' },
{ action: 'deny', target: '224.0.0.0/4' },
{ action: 'deny', target: 'fc00::/7' },
{ action: 'deny', target: 'fe80::/10' },
{ action: 'deny', target: '*.local' },
{ action: 'deny', target: 'host.docker.internal' },
{ action: 'deny', target: 'host.orb.internal' },
{ action: 'deny', target: 'docker.orb.internal' },
......@@ -253,7 +265,7 @@ describe('sandbox provider config', () => {
metadata: { teamId: 'team-1' },
networkPolicy: {
defaultAction: 'allow',
egress: [{ action: 'deny', target: 'host.docker.internal' }]
egress: defaultOpenSandboxNetworkPolicy.egress
},
extensions: {
traceId: 'trace-1'
......@@ -283,7 +295,7 @@ describe('sandbox provider config', () => {
memoryMiB: 4096
},
readyTimeoutSeconds: 120,
networkPolicy: defaultOpenSandboxDockerNetworkPolicy
networkPolicy: defaultOpenSandboxNetworkPolicy
});
expect(
......@@ -305,6 +317,46 @@ describe('sandbox provider config', () => {
});
});
it('applies the protected network policy for kubernetes runtime as well', async () => {
vi.stubEnv('AGENT_SANDBOX_OPENSANDBOX_RUNTIME', 'kubernetes');
vi.stubEnv('AGENT_SANDBOX_OPENSANDBOX_IMAGE', 'default-opensandbox-image:stable');
vi.resetModules();
const { getSandboxRuntimeProfile } =
await import('@fastgpt/service/core/ai/sandbox/infrastructure/provider/runtimeProfile');
const profile = getSandboxRuntimeProfile('opensandbox');
expect(profile.buildConfig()?.networkPolicy).toEqual(defaultOpenSandboxNetworkPolicy);
});
it('does not allow a caller policy to open protected targets or close public egress', async () => {
vi.stubEnv('AGENT_SANDBOX_OPENSANDBOX_RUNTIME', 'docker');
vi.stubEnv('AGENT_SANDBOX_OPENSANDBOX_IMAGE', 'default-opensandbox-image:stable');
vi.resetModules();
const { getSandboxRuntimeProfile } =
await import('@fastgpt/service/core/ai/sandbox/infrastructure/provider/runtimeProfile');
const profile = getSandboxRuntimeProfile('opensandbox');
expect(
profile.buildConfig({
createConfig: {
networkPolicy: {
defaultAction: 'deny',
egress: [
{ action: 'allow', target: '10.0.0.1' },
{ action: 'deny', target: 'api.example.com' }
]
}
}
})?.networkPolicy
).toEqual({
defaultAction: 'allow',
egress: [
...defaultOpenSandboxNetworkPolicy.egress,
{ action: 'deny', target: 'api.example.com' }
]
});
});
it('validates sealosdevbox token requirement', async () => {
const { validateSandboxConfig } = await loadSandboxConfigModule();
......
Subproject commit a72dbb1b3614fbb57cbe2e9765a92c6a4ad2b16e
Subproject commit 97d9bb623b33ff23d76810a873c3c90168254e36
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment