Commit 17317ba4 by DigHuang Committed by GitHub

chore(helm): move charts and release workflow to fastgpt-pro (#7463)

* feat(sandbox): pause OpenSandbox instance on stop instead of deleting

* chore(helm): remove charts and release workflow moved to fastgpt-pro

* revert: pause opensandbox
parent 33f2f108
name: Release helm chart
on:
push:
tags:
- 'v*.*.*'
workflow_dispatch:
jobs:
validate-version:
runs-on: ubuntu-24.04
outputs:
version: ${{ steps.version.outputs.version }}
steps:
- name: Validate release version
id: version
env:
VERSION: ${{ github.ref_name }}
REF_TYPE: ${{ github.ref_type }}
CURRENT_REF: ${{ github.ref }}
run: |
if [[ "$REF_TYPE" != "tag" || ! "$VERSION" =~ ^v ]]; then
echo "::error::Release workflow must run on a tag starting with v. Current ref: ${CURRENT_REF}"
exit 1
fi
echo "version=${VERSION}" >> "$GITHUB_OUTPUT"
helm:
needs: validate-version
permissions:
packages: write
contents: read
attestations: write
id-token: write
runs-on: ubuntu-24.04
steps:
- name: Checkout
uses: actions/checkout@v4
with:
fetch-tags: true
fetch-depth: 0
- name: Release Helm
run: |
echo ${{ secrets.GITHUB_TOKEN }} | helm registry login ghcr.io -u ${{ github.repository_owner }} --password-stdin
export APP_VERSION=${{ needs.validate-version.outputs.version }}
export HELM_VERSION=${{ needs.validate-version.outputs.version }}
export HELM_REPO=ghcr.io/${{ github.repository_owner }}
helm dependency update deploy/helm/fastgpt
helm package deploy/helm/fastgpt --version ${HELM_VERSION}-helm --app-version ${APP_VERSION} -d bin
helm push bin/fastgpt-${HELM_VERSION}-helm.tgz oci://${HELM_REPO}
......@@ -279,7 +279,7 @@ services:
# runtime=docker 模式需要挂载 Docker socket
# 配置 docker.host_ip 为宿主机 LAN IP(容器内访问宿主机服务用)
opensandbox-server:
image: sandbox-registry.cn-zhangjiakou.cr.aliyuncs.com/opensandbox/server:v0.2.1
image: sandbox-registry.cn-zhangjiakou.cr.aliyuncs.com/opensandbox/server:v0.2.2
container_name: fastgpt-opensandbox-server
restart: always
ports:
......@@ -340,7 +340,7 @@ services:
profiles:
- prepull
opensandbox-egress-image:
image: sandbox-registry.cn-zhangjiakou.cr.aliyuncs.com/opensandbox/egress:v1.1.4
image: sandbox-registry.cn-zhangjiakou.cr.aliyuncs.com/opensandbox/egress:v1.1.5
profiles:
- prepull
# 卷管理微服务:负责幂等创建/删除 Docker named volume 或 k8s PVC
......@@ -446,7 +446,9 @@ configs:
execd_image = "sandbox-registry.cn-zhangjiakou.cr.aliyuncs.com/opensandbox/execd:v1.0.21"
[egress]
image = "sandbox-registry.cn-zhangjiakou.cr.aliyuncs.com/opensandbox/egress:v1.1.4"
image = "sandbox-registry.cn-zhangjiakou.cr.aliyuncs.com/opensandbox/egress:v1.1.5"
mode = "dns+nft"
disable_ipv6 = true
[docker]
network_mode = "bridge"
......
......@@ -279,7 +279,7 @@ services:
# runtime=docker 模式需要挂载 Docker socket
# 配置 docker.host_ip 为宿主机 LAN IP(容器内访问宿主机服务用)
opensandbox-server:
image: opensandbox/server:v0.2.1
image: opensandbox/server:v0.2.2
container_name: fastgpt-opensandbox-server
restart: always
ports:
......@@ -340,7 +340,7 @@ services:
profiles:
- prepull
opensandbox-egress-image:
image: opensandbox/egress:v1.1.4
image: opensandbox/egress:v1.1.5
profiles:
- prepull
# 卷管理微服务:负责幂等创建/删除 Docker named volume 或 k8s PVC
......@@ -446,7 +446,9 @@ configs:
execd_image = "opensandbox/execd:v1.0.21"
[egress]
image = "opensandbox/egress:v1.1.4"
image = "opensandbox/egress:v1.1.5"
mode = "dns+nft"
disable_ipv6 = true
[docker]
network_mode = "bridge"
......
# Patterns to ignore when building packages.
# This supports shell glob matching, relative path matching, and
# negation (prefixed with !). Only one pattern per line.
.DS_Store
# Common VCS dirs
.git/
.gitignore
.bzr/
.bzrignore
.hg/
.hgignore
.svn/
# Common backup files
*.swp
*.bak
*.tmp
*.orig
*~
# Various IDEs
.project
.idea/
*.tmproj
.vscode/
dependencies:
- name: mongodb
repository: oci://registry-1.docker.io/bitnamicharts
version: 15.0.1
- name: postgresql
repository: oci://registry-1.docker.io/bitnamicharts
version: 15.0.0
digest: sha256:eaee52dc30a5f0e8e65e30a99240707c8493fa2773a4afcfb54d286fb4fa6306
generated: "2024-03-19T13:47:47.7518066+08:00"
apiVersion: v2
name: fastgpt
description: A Helm chart for FastGPT
# A chart can be either an 'application' or a 'library' chart.
#
# Application charts are a collection of templates that can be packaged into versioned archives
# to be deployed.
#
# Library charts provide useful utilities or functions for the chart developer. They're included as
# a dependency of application charts to inject those utilities and functions into the rendering
# pipeline. Library charts do not define any templates and therefore cannot be deployed.
type: application
# This is the chart version. This version number should be incremented each time you make changes
# to the chart and its templates, including the app version.
# Versions are expected to follow Semantic Versioning (https://semver.org/)
version: 0.1.0
# This is the version number of the application being deployed. This version number should be
# incremented each time you make changes to the application. Versions are not expected to
# follow Semantic Versioning. They should reflect the version the application is using.
# It is recommended to use it with quotes.
appVersion: "v4.14.12"
dependencies:
- name: mongodb
version: 15.0.1
repository: oci://registry-1.docker.io/bitnamicharts
- name: postgresql
version: 15.0.0
repository: oci://registry-1.docker.io/bitnamicharts
# fastgpt
![Version: 0.1.0](https://img.shields.io/badge/Version-0.1.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v4.6.6](https://img.shields.io/badge/AppVersion-v4.6.6-informational?style=flat-square)
A Helm chart for FastGPT
## Requirements
| Repository | Name | Version |
|------------|------|---------|
| oci://registry-1.docker.io/bitnamicharts | mongodb | 15.0.1 |
| oci://registry-1.docker.io/bitnamicharts | postgresql | 15.0.0 |
## Values
| Key | Type | Default | Description |
|-----|------|---------|-------------|
| affinity | object | `{}` | |
| autoscaling.enabled | bool | `false` | |
| autoscaling.maxReplicas | int | `100` | |
| autoscaling.minReplicas | int | `1` | |
| autoscaling.targetCPUUtilizationPercentage | int | `80` | |
| fullnameOverride | string | `""` | |
| image.pullPolicy | string | `"IfNotPresent"` | |
| image.repository | string | `"ghcr.io/labring/fastgpt"` | |
| image.tag | string | `""` | |
| imagePullSecrets | list | `[]` | |
| ingress.annotations | object | `{}` | |
| ingress.className | string | `""` | |
| ingress.enabled | bool | `false` | |
| ingress.hosts[0].host | string | `"chart-example.local"` | |
| ingress.hosts[0].paths[0].path | string | `"/"` | |
| ingress.hosts[0].paths[0].pathType | string | `"ImplementationSpecific"` | |
| ingress.tls | list | `[]` | |
| livenessProbe.httpGet.path | string | `"/"` | |
| livenessProbe.httpGet.port | string | `"http"` | |
| mongodb.architecture | string | `"replicaset"` | |
| mongodb.auth.rootPassword | string | `"123456"` | |
| mongodb.auth.rootUser | string | `"root"` | |
| mongodb.enabled | bool | `true` | Enable or disable the built-in MangoDB |
| nameOverride | string | `""` | |
| nodeSelector | object | `{}` | |
| podAnnotations | object | `{}` | |
| podLabels | object | `{}` | |
| podSecurityContext | object | `{}` | |
| postgresql.enabled | bool | `true` | Enable or disable the built-in PostgreSQL |
| postgresql.global.postgresql.auth.database | string | `"postgres"` | The default database of PostgreSQL |
| postgresql.global.postgresql.auth.postgresPassword | string | `"postgres"` | The password of PostgreSQL, default username is `postgres` |
| postgresql.image.repository | string | `"linuxsuren/pgvector"` | The PostgreSQL image which include the pgvector extension. See also the source code from https://github.com/LinuxSuRen/pgvector-docker |
| postgresql.image.tag | string | `"v0.0.1"` | |
| readinessProbe.httpGet.path | string | `"/"` | |
| readinessProbe.httpGet.port | string | `"http"` | |
| replicaCount | int | `1` | |
| resources | object | `{}` | |
| securityContext | object | `{}` | |
| service.port | int | `3000` | |
| service.type | string | `"ClusterIP"` | |
| serviceAccount.annotations | object | `{}` | |
| serviceAccount.automount | bool | `true` | |
| serviceAccount.create | bool | `true` | |
| serviceAccount.name | string | `""` | |
| tolerations | list | `[]` | |
| volumeMounts | list | `[]` | |
| volumes | list | `[]` | |
----------------------------------------------
Autogenerated from chart metadata using [helm-docs v1.13.1](https://github.com/norwoodj/helm-docs/releases/v1.13.1)
1. Get the application URL by running these commands:
{{- if .Values.ingress.enabled }}
{{- range $host := .Values.ingress.hosts }}
{{- range .paths }}
http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ .path }}
{{- end }}
{{- end }}
{{- else if contains "NodePort" .Values.service.type }}
export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "fastgpt.fullname" . }})
export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}")
echo http://$NODE_IP:$NODE_PORT
{{- else if contains "LoadBalancer" .Values.service.type }}
NOTE: It may take a few minutes for the LoadBalancer IP to be available.
You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "fastgpt.fullname" . }}'
export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "fastgpt.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}")
echo http://$SERVICE_IP:{{ .Values.service.port }}
{{- else if contains "ClusterIP" .Values.service.type }}
export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "fastgpt.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}")
export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}")
echo "Visit http://127.0.0.1:8080 to use your application"
kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT
{{- end }}
{{/*
Expand the name of the chart.
*/}}
{{- define "fastgpt.name" -}}
{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
{{- end }}
{{/*
Create a default fully qualified app name.
We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
If release name contains chart name it will be used as a full name.
*/}}
{{- define "fastgpt.fullname" -}}
{{- if .Values.fullnameOverride }}
{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
{{- else }}
{{- $name := default .Chart.Name .Values.nameOverride }}
{{- if contains $name .Release.Name }}
{{- .Release.Name | trunc 63 | trimSuffix "-" }}
{{- else }}
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
{{- end }}
{{- end }}
{{- end }}
{{/*
Create chart name and version as used by the chart label.
*/}}
{{- define "fastgpt.chart" -}}
{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
{{- end }}
{{/*
Common labels
*/}}
{{- define "fastgpt.labels" -}}
helm.sh/chart: {{ include "fastgpt.chart" . }}
{{ include "fastgpt.selectorLabels" . }}
{{- if .Chart.AppVersion }}
app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
{{- end }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
{{- end }}
{{/*
Selector labels
*/}}
{{- define "fastgpt.selectorLabels" -}}
app.kubernetes.io/name: {{ include "fastgpt.name" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
{{- end }}
{{/*
Create the name of the service account to use
*/}}
{{- define "fastgpt.serviceAccountName" -}}
{{- if .Values.serviceAccount.create }}
{{- default (include "fastgpt.fullname" .) .Values.serviceAccount.name }}
{{- else }}
{{- default "default" .Values.serviceAccount.name }}
{{- end }}
{{- end }}
apiVersion: v1
data:
config.json: |
{
"systemEnv": {
"openapiPrefix": "fastgpt",
"vectorMaxProcess": 15,
"qaMaxProcess": 15,
"vlmMaxProcess": 15,
"hnswEfSearch": 100
},
"llmModels": [
{
"model": "gpt-3.5-turbo",
"name": "gpt-3.5-turbo",
"maxContext": 16000,
"maxResponse": 4000,
"quoteMaxToken": 13000,
"maxTemperature": 1.2,
"charsPointsPrice": 0,
"censor": false,
"vision": false,
"toolChoice": true,
"functionCall": false,
"defaultSystemChatPrompt": "",
"defaultConfig": {}
},
{
"model": "gpt-3.5-turbo-16k",
"name": "gpt-3.5-turbo-16k",
"maxContext": 16000,
"maxResponse": 16000,
"quoteMaxToken": 13000,
"maxTemperature": 1.2,
"charsPointsPrice": 0,
"censor": false,
"vision": false,
"toolChoice": true,
"functionCall": false,
"defaultSystemChatPrompt": "",
"defaultConfig": {}
},
{
"model": "gpt-4-0125-preview",
"name": "gpt-4-turbo",
"maxContext": 125000,
"maxResponse": 4000,
"quoteMaxToken": 100000,
"maxTemperature": 1.2,
"charsPointsPrice": 0,
"censor": false,
"vision": false,
"toolChoice": true,
"functionCall": false,
"defaultSystemChatPrompt": "",
"defaultConfig": {}
},
{
"model": "gpt-4-vision-preview",
"name": "gpt-4-vision",
"maxContext": 128000,
"maxResponse": 4000,
"quoteMaxToken": 100000,
"maxTemperature": 1.2,
"charsPointsPrice": 0,
"censor": false,
"vision": true,
"toolChoice": true,
"functionCall": false,
"defaultSystemChatPrompt": "",
"defaultConfig": {}
}
],
"vectorModels": [
{
"model": "text-embedding-3-large",
"name": "Embedding-2",
"avatar": "/imgs/model/openai.svg",
"charsPointsPrice": 0,
"defaultToken": 512,
"maxToken": 3000,
"weight": 100,
"dbConfig": {},
"queryConfig": {},
"defaultConfig": {
"dimensions": 1024
}
},
{
"model": "text-embedding-3-small",
"name": "Embedding-2",
"avatar": "/imgs/model/openai.svg",
"charsPointsPrice": 0,
"defaultToken": 512,
"maxToken": 3000,
"weight": 100,
"dbConfig": {},
"queryConfig": {}
},
{
"model": "text-embedding-ada-002",
"name": "Embedding-2",
"avatar": "/imgs/model/openai.svg",
"charsPointsPrice": 0,
"defaultToken": 512,
"maxToken": 3000,
"weight": 100,
"dbConfig": {},
"queryConfig": {}
}
],
"reRankModels": [],
"audioSpeechModels": [
{
"model": "tts-1",
"name": "OpenAI TTS1",
"charsPointsPrice": 0,
"voices": [
{
"label": "Alloy",
"value": "alloy",
"bufferId": "openai-Alloy"
},
{
"label": "Echo",
"value": "echo",
"bufferId": "openai-Echo"
},
{
"label": "Fable",
"value": "fable",
"bufferId": "openai-Fable"
},
{
"label": "Onyx",
"value": "onyx",
"bufferId": "openai-Onyx"
},
{
"label": "Nova",
"value": "nova",
"bufferId": "openai-Nova"
},
{
"label": "Shimmer",
"value": "shimmer",
"bufferId": "openai-Shimmer"
}
]
}
],
"whisperModel": {
"model": "whisper-1",
"name": "Whisper1",
"charsPointsPrice": 0
}
}
kind: ConfigMap
metadata:
labels:
{{ include "fastgpt.labels" . | nindent 4 }}
name: {{ include "fastgpt.fullname" . }}-config
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ include "fastgpt.fullname" . }}
labels:
{{- include "fastgpt.labels" . | nindent 4 }}
spec:
{{- if not .Values.autoscaling.enabled }}
replicas: {{ .Values.replicaCount }}
{{- end }}
selector:
matchLabels:
{{- include "fastgpt.selectorLabels" . | nindent 6 }}
template:
metadata:
{{- with .Values.podAnnotations }}
annotations:
{{- toYaml . | nindent 8 }}
{{- end }}
labels:
{{- include "fastgpt.labels" . | nindent 8 }}
{{- with .Values.podLabels }}
{{- toYaml . | nindent 8 }}
{{- end }}
spec:
{{- with .Values.imagePullSecrets }}
imagePullSecrets:
{{- toYaml . | nindent 8 }}
{{- end }}
serviceAccountName: {{ include "fastgpt.serviceAccountName" . }}
securityContext:
{{- toYaml .Values.podSecurityContext | nindent 8 }}
containers:
- name: {{ .Chart.Name }}
securityContext:
{{- toYaml .Values.securityContext | nindent 12 }}
image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
imagePullPolicy: {{ .Values.image.pullPolicy }}
envFrom:
- secretRef:
name: {{ include "fastgpt.fullname" . }}-env
ports:
- name: http
containerPort: {{ .Values.service.port }}
protocol: TCP
resources:
{{- toYaml .Values.resources | nindent 12 }}
volumeMounts:
- mountPath: /app/data/config.json
name: config
subPath: config.json
volumes:
- name: config
configMap:
name: {{ include "fastgpt.fullname" . }}-config
{{- with .Values.nodeSelector }}
nodeSelector:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.affinity }}
affinity:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.tolerations }}
tolerations:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- if .Values.autoscaling.enabled }}
apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
name: {{ include "fastgpt.fullname" . }}
labels:
{{- include "fastgpt.labels" . | nindent 4 }}
spec:
scaleTargetRef:
apiVersion: apps/v1
kind: Deployment
name: {{ include "fastgpt.fullname" . }}
minReplicas: {{ .Values.autoscaling.minReplicas }}
maxReplicas: {{ .Values.autoscaling.maxReplicas }}
metrics:
{{- if .Values.autoscaling.targetCPUUtilizationPercentage }}
- type: Resource
resource:
name: cpu
target:
type: Utilization
averageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }}
{{- end }}
{{- if .Values.autoscaling.targetMemoryUtilizationPercentage }}
- type: Resource
resource:
name: memory
target:
type: Utilization
averageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }}
{{- end }}
{{- end }}
{{- if .Values.ingress.enabled -}}
{{- $fullName := include "fastgpt.fullname" . -}}
{{- $svcPort := .Values.service.port -}}
{{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }}
{{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }}
{{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}}
{{- end }}
{{- end }}
{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}}
apiVersion: networking.k8s.io/v1
{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}}
apiVersion: networking.k8s.io/v1beta1
{{- else -}}
apiVersion: extensions/v1beta1
{{- end }}
kind: Ingress
metadata:
name: {{ $fullName }}
labels:
{{- include "fastgpt.labels" . | nindent 4 }}
{{- with .Values.ingress.annotations }}
annotations:
{{- toYaml . | nindent 4 }}
{{- end }}
spec:
{{- if and .Values.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }}
ingressClassName: {{ .Values.ingress.className }}
{{- end }}
{{- if .Values.ingress.tls }}
tls:
{{- range .Values.ingress.tls }}
- hosts:
{{- range .hosts }}
- {{ . | quote }}
{{- end }}
secretName: {{ .secretName }}
{{- end }}
{{- end }}
rules:
{{- range .Values.ingress.hosts }}
- host: {{ .host | quote }}
http:
paths:
{{- range .paths }}
- path: {{ .path }}
{{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }}
pathType: {{ .pathType }}
{{- end }}
backend:
{{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }}
service:
name: {{ $fullName }}
port:
number: {{ $svcPort }}
{{- else }}
serviceName: {{ $fullName }}
servicePort: {{ $svcPort }}
{{- end }}
{{- end }}
{{- end }}
{{- end }}
apiVersion: v1
stringData:
DEFAULT_ROOT_PSW: "1234"
OPENAI_BASE_URL: "https://api.openai.com/v1"
CHAT_API_KEY: "sk-xxxx"
DB_MAX_LINK: "5"
SYSTEM_MAX_STRING_LENGTH_M: "{{ .Values.system.maxStringLengthM }}"
ROOT_KEY: "root_key"
FILE_TOKEN_KEY: "filetoken"
AES256_SECRET_KEY: "fastgptsecret"
INVOKE_TOKEN_SECRET: "fastgpt_invoke_token_secret_32_chars_min"
MONGODB_URI: "mongodb://{{ .Values.mongodb.auth.rootUser }}:{{ .Values.mongodb.auth.rootPassword }}@{{ include "fastgpt.fullname" . }}-mongodb-headless:27017/fastgpt?authSource=admin"
PG_URL: "postgresql://postgres:{{ .Values.postgresql.auth.rootPassword }}@{{ include "fastgpt.fullname" . }}-postgresql:5432/{{ .Values.postgresql.global.postgresql.auth.database }}"
kind: Secret
type: Opaque
metadata:
labels:
{{ include "fastgpt.labels" . | nindent 4 }}
name: {{ include "fastgpt.fullname" . }}-env
apiVersion: v1
kind: Service
metadata:
name: {{ include "fastgpt.fullname" . }}
labels:
{{- include "fastgpt.labels" . | nindent 4 }}
spec:
type: {{ .Values.service.type }}
ports:
- port: {{ .Values.service.port }}
{{- if .Values.service.nodePort }}
nodePort: {{ .Values.service.nodePort }}
{{- end}}
targetPort: http
protocol: TCP
name: http
selector:
{{- include "fastgpt.selectorLabels" . | nindent 4 }}
{{- if .Values.serviceAccount.create -}}
apiVersion: v1
kind: ServiceAccount
metadata:
name: {{ include "fastgpt.serviceAccountName" . }}
labels:
{{- include "fastgpt.labels" . | nindent 4 }}
{{- with .Values.serviceAccount.annotations }}
annotations:
{{- toYaml . | nindent 4 }}
{{- end }}
automountServiceAccountToken: {{ .Values.serviceAccount.automount }}
{{- end }}
apiVersion: v1
kind: Pod
metadata:
name: "{{ include "fastgpt.fullname" . }}-test-connection"
labels:
{{- include "fastgpt.labels" . | nindent 4 }}
annotations:
"helm.sh/hook": test
spec:
containers:
- name: wget
image: busybox
command: ['wget']
args: ['{{ include "fastgpt.fullname" . }}:{{ .Values.service.port }}']
restartPolicy: Never
# Default values for fastgpt.
# This is a YAML-formatted file.
# Declare variables to be passed into your templates.
replicaCount: 1
image:
repository: ghcr.io/labring/fastgpt
pullPolicy: IfNotPresent
# Overrides the image tag whose default is the chart appVersion.
tag: ""
imagePullSecrets: []
nameOverride: ""
fullnameOverride: ""
serviceAccount:
# Specifies whether a service account should be created
create: true
# Automatically mount a ServiceAccount's API credentials?
automount: true
# Annotations to add to the service account
annotations: {}
# The name of the service account to use.
# If not set and create is true, a name is generated using the fullname template
name: ""
podAnnotations: {}
podLabels: {}
podSecurityContext:
{}
# fsGroup: 2000
securityContext:
{}
# capabilities:
# drop:
# - ALL
# readOnlyRootFilesystem: true
# runAsNonRoot: true
# runAsUser: 1000
service:
type: ClusterIP
port: 3000
# nodePort: 31440
ingress:
enabled: false
className: ""
annotations:
{}
# kubernetes.io/ingress.class: nginx
# kubernetes.io/tls-acme: "true"
hosts:
- host: chart-example.local
paths:
- path: /
pathType: ImplementationSpecific
tls: []
# - secretName: chart-example-tls
# hosts:
# - chart-example.local
resources:
{}
# We usually recommend not to specify default resources and to leave this as a conscious
# choice for the user. This also increases chances charts run on environments with little
# resources, such as Minikube. If you do want to specify resources, uncomment the following
# lines, adjust them as necessary, and remove the curly braces after 'resources:'.
# limits:
# cpu: 100m
# memory: 128Mi
# requests:
# cpu: 100m
# memory: 128Mi
livenessProbe:
httpGet:
path: /
port: http
readinessProbe:
httpGet:
path: /
port: http
autoscaling:
enabled: false
minReplicas: 1
maxReplicas: 100
targetCPUUtilizationPercentage: 80
# targetMemoryUtilizationPercentage: 80
system:
# Maximum character length for synchronous system string operations, in M characters.
# 1 means 1,000,000 characters. Valid range: 1 to 100.
maxStringLengthM: 100
# Additional volumes on the output Deployment definition.
volumes: []
# - name: foo
# secret:
# secretName: mysecret
# optional: false
# Additional volumeMounts on the output Deployment definition.
volumeMounts: []
# - name: foo
# mountPath: "/etc/foo"
# readOnly: true
nodeSelector: {}
tolerations: []
affinity: {}
mongodb:
# -- Enable or disable the built-in MangoDB
enabled: true
architecture: replicaset
auth:
rootUser: root
rootPassword: "123456"
postgresql:
# -- Enable or disable the built-in PostgreSQL
enabled: true
image:
# registry: 172.11.0.6:30002
# -- The PostgreSQL image which include the pgvector extension. See also the source code from https://github.com/LinuxSuRen/pgvector-docker
repository: linuxsuren/pgvector
tag: v0.0.1
global:
postgresql:
auth:
# -- The password of PostgreSQL, default username is `postgres`
postgresPassword: postgres
# -- The default database of PostgreSQL
database: postgres
# Patterns to ignore when building packages.
# This supports shell glob matching, relative path matching, and
# negation (prefixed with !). Only one pattern per line.
.DS_Store
# Common VCS dirs
.git/
.gitignore
.bzr/
.bzrignore
.hg/
.hgignore
.svn/
# Common backup files
*.swp
*.bak
*.tmp
*.orig
*~
# Various IDEs
.project
.idea/
*.tmproj
.vscode/
# Changelog
All notable changes to the OpenSandbox Helm Chart will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
## [0.1.0] - Initial Release
### Added
#### Core Features
- OpenSandbox Kubernetes Controller Helm chart
- **OpenSandbox Server deployment with FastAPI control plane for SDK integration**
- Support for deploying controller with configurable replicas and resources
- BatchSandbox and Pool CRD definitions
- RBAC resources (ClusterRole, ClusterRoleBinding, ServiceAccount)
- Leader election configuration for high availability
#### Server Features
- Server Deployment with configurable replicas and resources
- Server Service with ClusterIP/NodePort/LoadBalancer support
- ConfigMap-based configuration management
- Optional Ingress support for external access
- Health probes for liveness and readiness checks
- In-cluster Kubernetes configuration
- API key authentication support (optional)
- SDK-compatible REST API on port 8080
#### Pool Management
- Default agent-pool with execd and task-executor sidecar
- Pool template support for creating pre-warmed Pod pools
- Configurable Pool capacity (bufferMin, bufferMax, poolMin, poolMax)
- SDK-compatible Pool configuration with execd on port 44772
#### Multiple Values Files
- `values.yaml` - Default configuration with agent-pool enabled
- `values-e2e.yaml` - End-to-end testing with minimal resources (2-5 pods)
- Use `--set` or custom values files for production/development overrides
#### Templates
- `deployment.yaml` - Controller manager deployment
- **`server-deployment.yaml` - Server deployment**
- **`server-service.yaml` - Server service**
- **`server-configmap.yaml` - Server configuration**
- **`server-ingress.yaml` - Server ingress (optional)**
- `pools.yaml` - Dynamic Pool resource generation from values
- `serviceaccount.yaml` - Service account for controller
- `clusterrole.yaml` - RBAC cluster role
- `clusterrolebinding.yaml` - RBAC cluster role binding
- `leader-election-role.yaml` - Leader election RBAC
- `leader-election-rolebinding.yaml` - Leader election binding
- `metrics-service.yaml` - Metrics service endpoint
- `metrics-rbac.yaml` - Metrics RBAC resources
- `servicemonitor.yaml` - Prometheus ServiceMonitor (optional)
- `extra-roles.yaml` - User management roles (viewer, editor)
- `poddisruptionbudget.yaml` - High availability Pod disruption budget
- `NOTES.txt` - Post-installation guidance
- `_helpers.tpl` - Template helper functions
#### Scripts
- `scripts/install.sh` - Interactive installation wizard with environment selection
- `scripts/uninstall.sh` - Safe uninstallation with resource cleanup
- `scripts/e2e-test.sh` - End-to-end validation (Install → Server → Pool → SDK → Uninstall)
- `scripts/README.md` - Comprehensive script documentation and troubleshooting guide
#### Configuration Options
- `nameOverride` and `fullnameOverride` for custom resource naming
- **`server.enabled` - Enable/disable server deployment (default: true)**
- **`server.service.type` - Service type (ClusterIP/NodePort/LoadBalancer)**
- **`server.service.nodePort` - NodePort value (optional)**
- **`server.ingress.enabled` - Enable Ingress for external access**
- **`server.config.server.apiKey` - Optional API key authentication**
- `healthProbePort` - Configurable health check port (default: 8081)
- `healthProbes.liveness` - Liveness probe timing configuration
- `healthProbes.readiness` - Readiness probe timing configuration
- `podDisruptionBudget.enabled` - Optional PDB for HA deployments
- `namespaceOverride` - Custom namespace (default: opensandbox)
#### Documentation
- Comprehensive README.md with installation and configuration guide
- examples/README.md with usage scenarios and best practices
- examples/pool-agent-production.yaml with production-ready Pool configuration
- examples/DIRECTORY_STRUCTURE.md explaining file organization
- Example YAML files for Pool and BatchSandbox resources
### Configuration Defaults
- Controller image: `opensandbox/controller:dev`
- **Server image: `opensandbox/server:v0.1.0`**
- Task executor image: `opensandbox/task-executor:dev`
- Image pull policy: `Never` (for local development)
- Namespace: `opensandbox`
- Controller replicas: 1 (3 in production values)
- **Server replicas: 1**
- **Server enabled: true (required for SDK usage)**
- Default Pool enabled: `agent-pool` with 2-5 pods (E2E) or 10-100 pods (default)
### Notes
- This is the initial release of the Helm chart
- All templates have been tested with `helm lint` and E2E validation
- Chart supports Kubernetes 1.19+
apiVersion: v2
name: opensandbox-controller
description: A Helm chart for deploying OpenSandbox Kubernetes Controller
type: application
version: 0.1.0
appVersion: "0.0.1"
keywords:
- opensandbox
- sandbox
- kubernetes
- operator
- controller
home: https://github.com/alibaba/OpenSandbox
maintainers:
- name: OpenSandbox Team
url: https://github.com/alibaba/OpenSandbox
icon: https://avatars.githubusercontent.com/u/1961952
# Helm Chart Makefile for OpenSandbox Controller
CHART_NAME := opensandbox-controller
RELEASE_NAME := opensandbox-controller
NAMESPACE := opensandbox
VALUES_FILE := values.yaml
# Helm commands
HELM := helm
KUBECTL := kubectl
.PHONY: help
help: ## Display this help
@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m<target>\033[0m\n\nTargets:\n"} /^[a-zA-Z_-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 }' $(MAKEFILE_LIST)
.PHONY: lint
lint: ## Lint the Helm chart
$(HELM) lint .
.PHONY: template
template: ## Render chart templates to stdout
$(HELM) template $(RELEASE_NAME) . -f $(VALUES_FILE)
.PHONY: template-debug
template-debug: ## Render chart templates with debug output
$(HELM) template $(RELEASE_NAME) . -f $(VALUES_FILE) --debug
.PHONY: dry-run
dry-run: ## Perform a dry-run installation
$(HELM) install $(RELEASE_NAME) . \
-f $(VALUES_FILE) \
--namespace $(NAMESPACE) \
--create-namespace \
--dry-run --debug
.PHONY: install
install: ## Install the chart
$(HELM) install $(RELEASE_NAME) . \
-f $(VALUES_FILE) \
--namespace $(NAMESPACE) \
--create-namespace
.PHONY: install-e2e
install-e2e: ## Install the chart with e2e test values
$(HELM) install $(RELEASE_NAME) . \
-f values-e2e.yaml \
--namespace $(NAMESPACE) \
--create-namespace
.PHONY: upgrade
upgrade: ## Upgrade the chart
$(HELM) upgrade $(RELEASE_NAME) . \
-f $(VALUES_FILE) \
--namespace $(NAMESPACE)
.PHONY: upgrade-e2e
upgrade-e2e: ## Upgrade with e2e test values
$(HELM) upgrade $(RELEASE_NAME) . \
-f values-e2e.yaml \
--namespace $(NAMESPACE)
.PHONY: uninstall
uninstall: ## Uninstall the chart
$(HELM) uninstall $(RELEASE_NAME) --namespace $(NAMESPACE)
.PHONY: status
status: ## Show release status
$(HELM) status $(RELEASE_NAME) --namespace $(NAMESPACE)
.PHONY: list
list: ## List all releases
$(HELM) list --namespace $(NAMESPACE)
.PHONY: get-values
get-values: ## Get values for the release
$(HELM) get values $(RELEASE_NAME) --namespace $(NAMESPACE)
.PHONY: get-all
get-all: ## Get all information about the release
$(HELM) get all $(RELEASE_NAME) --namespace $(NAMESPACE)
.PHONY: package
package: ## Package the chart into an archive
$(HELM) package .
.PHONY: verify-install
verify-install: ## Verify the installation
@echo "Checking deployment..."
$(KUBECTL) get deployment -n $(NAMESPACE)
@echo "\nChecking pods..."
$(KUBECTL) get pods -n $(NAMESPACE)
@echo "\nChecking CRDs..."
$(KUBECTL) get crds | grep sandbox.opensandbox.io
.PHONY: logs
logs: ## Show controller logs
$(KUBECTL) logs -n $(NAMESPACE) -l control-plane=controller-manager -f
.PHONY: clean-crds
clean-crds: ## Delete CRDs (use with caution!)
$(KUBECTL) delete crd batchsandboxes.sandbox.opensandbox.io
$(KUBECTL) delete crd pools.sandbox.opensandbox.io
.PHONY: test-connection
test-connection: ## Test if kubectl can connect to the cluster
$(KUBECTL) cluster-info
$(KUBECTL) get nodes
.PHONY: create-namespace
create-namespace: ## Create the namespace
$(KUBECTL) create namespace $(NAMESPACE) --dry-run=client -o yaml | $(KUBECTL) apply -f -
.PHONY: delete-namespace
delete-namespace: ## Delete the namespace
$(KUBECTL) delete namespace $(NAMESPACE)
# Advanced targets
.PHONY: diff
diff: ## Show diff between current release and chart
@command -v helm-diff >/dev/null 2>&1 || { echo "helm-diff plugin required. Install: helm plugin install https://github.com/databus23/helm-diff"; exit 1; }
$(HELM) diff upgrade $(RELEASE_NAME) . -f $(VALUES_FILE) --namespace $(NAMESPACE)
.PHONY: history
history: ## Show release history
$(HELM) history $(RELEASE_NAME) --namespace $(NAMESPACE)
.PHONY: rollback
rollback: ## Rollback to previous release
$(HELM) rollback $(RELEASE_NAME) --namespace $(NAMESPACE)
.PHONY: rollback-to
rollback-to: ## Rollback to specific revision (usage: make rollback-to REVISION=2)
@if [ -z "$(REVISION)" ]; then echo "Please specify REVISION=<number>"; exit 1; fi
$(HELM) rollback $(RELEASE_NAME) $(REVISION) --namespace $(NAMESPACE)
# OpenSandbox Controller Helm Chart
This Helm chart deploys the OpenSandbox Kubernetes Controller, which manages sandbox environments through custom resources.
## Prerequisites
- Kubernetes 1.19+
- Helm 3.0+
- Container runtime (Docker, containerd, etc.)
- **Three container images required**:
1. **Controller image**: The main controller manager
2. **Server image**: FastAPI control plane for SDK usage
3. **Task Executor image**: Sidecar container for task execution (optional but required for task features)
## Important: Image Requirements
OpenSandbox requires **three separate images**:
### 1. Controller Image
The main controller that manages BatchSandbox and Pool resources.
```bash
# Build controller image
make docker-build IMG=your-registry/opensandbox-controller:v1.0.0
docker push your-registry/opensandbox-controller:v1.0.0
```
### 2. Server Image
FastAPI control plane that exposes REST API for SDK usage. **This is the entry point for SDK clients**.
```bash
# Build server image (from server directory)
cd ../../../server
TAG=v1.0.0 ./build.sh
# Or manually:
docker build -t your-registry/opensandbox-server:v1.0.0 .
docker push your-registry/opensandbox-server:v1.0.0
```
**Note**: The server is **required for SDK usage**. If you only use `kubectl` to manage CRDs directly, you can disable it by setting `server.enabled=false`.
### 3. Task Executor Image
A sidecar container injected into Pool pods for task execution. **This is not deployed as a separate Deployment**, but configured in Pool resources.
```bash
# Build task-executor image
make docker-build-task-executor TASK_EXECUTOR_IMG=your-registry/opensandbox-task-executor:v1.0.0
docker push your-registry/opensandbox-task-executor:v1.0.0
```
**Note**: The task-executor image is only needed if you want to use task execution features. For basic sandbox management without tasks, only the controller and server images are required.
## Features
- **SDK Control Plane**: FastAPI server for Python SDK integration
- **Batch Sandbox Management**: Create and manage multiple identical sandbox environments
- **Resource Pooling**: Maintain pre-warmed resource pools for rapid provisioning
- **Task Orchestration**: Optional integrated task execution engine
- **High Availability**: Leader election support for multiple replicas
- **Metrics & Monitoring**: Prometheus metrics endpoint with optional ServiceMonitor
- **Flexible Access**: ClusterIP, NodePort, or Ingress support for server access
## Installation
### Quick Start
```bash
# Add the chart repository (if published)
helm repo add opensandbox https://charts.opensandbox.io
helm repo update
# Install the chart with all images
helm install opensandbox-controller opensandbox/opensandbox-controller \
--set controllerManager.image.repository=your-registry/opensandbox-controller \
--set controllerManager.image.tag=v1.0.0 \
--set server.image.repository=your-registry/opensandbox-server \
--set server.image.tag=v1.0.0 \
--set taskExecutor.image.repository=your-registry/opensandbox-task-executor \
--set taskExecutor.image.tag=v1.0.0
# Or install from local directory
helm install opensandbox-controller ./opensandbox-controller \
--set controllerManager.image.repository=your-registry/opensandbox-controller \
--set controllerManager.image.tag=v1.0.0 \
--set server.image.repository=your-registry/opensandbox-server \
--set server.image.tag=v1.0.0 \
--set taskExecutor.image.repository=your-registry/opensandbox-task-executor \
--set taskExecutor.image.tag=v1.0.0
```
### Custom Installation
```bash
# Install with custom values
helm install opensandbox-controller ./opensandbox-controller \
--set controllerManager.image.repository=your-registry/sandbox-controller \
--set controllerManager.image.tag=v1.0.0 \
--namespace opensandbox \
--create-namespace
# Install with values file
helm install opensandbox-controller ./opensandbox-controller \
-f custom-values.yaml
```
## Configuration
The following table lists the configurable parameters of the chart and their default values.
### Controller Manager Configuration
| Parameter | Description | Default |
|-----------|-------------|---------|
| `controllerManager.image.repository` | Controller image repository | `opensandbox/controller` |
| `controllerManager.image.tag` | Controller image tag | `dev` |
| `controllerManager.image.pullPolicy` | Image pull policy | `Never` |
| `controllerManager.replicas` | Number of controller replicas | `1` |
| `controllerManager.resources.limits.cpu` | CPU limit | `500m` |
| `controllerManager.resources.limits.memory` | Memory limit | `128Mi` |
| `controllerManager.resources.requests.cpu` | CPU request | `10m` |
| `controllerManager.resources.requests.memory` | Memory request | `64Mi` |
| `controllerManager.leaderElect` | Enable leader election | `true` |
| `controllerManager.logLevel` | Log verbosity level | `3` |
### Task Executor Configuration
**Important**: The task-executor is not deployed as a separate service. It is configured as a sidecar container in Pool resources. These settings provide the default image and resource configurations for reference when creating Pools.
| Parameter | Description | Default |
|-----------|-------------|---------|
| `taskExecutor.image.repository` | Task Executor image repository | `opensandbox/task-executor` |
| `taskExecutor.image.tag` | Task Executor image tag | `dev` |
| `taskExecutor.image.pullPolicy` | Image pull policy | `Never` |
| `taskExecutor.resources.limits.cpu` | Recommended CPU limit for sidecar | `500m` |
| `taskExecutor.resources.limits.memory` | Recommended memory limit for sidecar | `256Mi` |
| `taskExecutor.resources.requests.cpu` | Recommended CPU request for sidecar | `100m` |
| `taskExecutor.resources.requests.memory` | Recommended memory request for sidecar | `128Mi` |
### Server Configuration
**Important**: The server is a FastAPI control plane that exposes REST API for SDK usage. It is **required for SDK integration** but can be disabled if you only use `kubectl` to manage CRDs.
| Parameter | Description | Default |
|-----------|-------------|---------|
| `server.enabled` | Enable server deployment | `true` |
| `server.image.repository` | Server image repository | `opensandbox/server` |
| `server.image.tag` | Server image tag | `v0.2.1` |
| `server.image.pullPolicy` | Image pull policy | `Never` |
| `server.replicas` | Number of server replicas | `1` |
| `server.resources.limits.cpu` | CPU limit | `1` |
| `server.resources.limits.memory` | Memory limit | `512Mi` |
| `server.resources.requests.cpu` | CPU request | `100m` |
| `server.resources.requests.memory` | Memory request | `256Mi` |
| `server.config.server.host` | Server listen host | `0.0.0.0` |
| `server.config.server.port` | Server listen port | `8080` |
| `server.config.server.logLevel` | Log level (INFO/DEBUG/WARNING/ERROR) | `INFO` |
| `server.config.server.apiKey` | Optional API key for authentication | `""` |
| `server.config.runtime.type` | Runtime type (kubernetes/docker) | `kubernetes` |
| `server.config.runtime.execdImage` | execd image for non-pool mode | `opensandbox/execd:v1.0.21` |
| `server.config.kubernetes.workloadProvider` | Workload provider type | `batchsandbox` |
| `server.service.type` | Service type (ClusterIP/NodePort/LoadBalancer) | `ClusterIP` |
| `server.service.port` | Service port | `8080` |
| `server.service.nodePort` | NodePort (when type=NodePort) | `""` |
| `server.ingress.enabled` | Enable Ingress | `false` |
| `server.ingress.className` | Ingress class name | `""` |
| `server.ingress.hosts` | Ingress host configuration | `[]` |
### Namespace Configuration
| Parameter | Description | Default |
|-----------|-------------|---------|
| `namespaceOverride` | Override the default namespace name | `"opensandbox"` |
**Note**: Both the controller, server, and user resources (Pool, BatchSandbox) use the same namespace for simplicity.
The server automatically uses in-cluster Kubernetes configuration and reads the namespace from the Helm chart configuration.
### Accessing the Server
#### Option 1: Port Forward (Development)
```bash
# Forward local port to server
kubectl port-forward -n opensandbox svc/opensandbox-controller-server 8080:8080
# Test connection
curl http://localhost:8080/health
```
#### Option 2: NodePort (Local Development)
```bash
# Install with NodePort
helm install opensandbox-controller ./opensandbox-controller \
--set server.service.type=NodePort \
--set server.service.nodePort=30080
# Access via node IP
curl http://<node-ip>:30080/health
```
#### Option 3: Ingress (Production)
```bash
# Install with Ingress
helm install opensandbox-controller ./opensandbox-controller \
--set server.ingress.enabled=true \
--set server.ingress.className=nginx \
--set server.ingress.hosts[0].host=opensandbox.example.com \
--set server.ingress.hosts[0].paths[0].path=/ \
--set server.ingress.hosts[0].paths[0].pathType=Prefix
# Access via domain
curl https://opensandbox.example.com/health
```
### RBAC Configuration
| Parameter | Description | Default |
|-----------|-------------|---------|
| `rbac.create` | Create RBAC resources | `true` |
| `rbac.serviceAccount.create` | Create ServiceAccount | `true` |
| `rbac.serviceAccount.name` | ServiceAccount name (if not created) | `""` |
### Metrics Configuration
| Parameter | Description | Default |
|-----------|-------------|---------|
| `metrics.enabled` | Enable metrics service | `true` |
| `metrics.service.type` | Metrics service type | `ClusterIP` |
| `metrics.service.port` | Metrics service port | `8443` |
| `metrics.serviceMonitor.enabled` | Create ServiceMonitor (Prometheus Operator) | `false` |
| `metrics.serviceMonitor.interval` | Scrape interval | `30s` |
### CRD Configuration
| Parameter | Description | Default |
|-----------|-------------|---------|
| `crds.install` | Install CRDs | `true` |
### Extra Roles Configuration
| Parameter | Description | Default |
|-----------|-------------|---------|
| `extraRoles.batchsandboxEditor.enabled` | Create BatchSandbox editor role | `true` |
| `extraRoles.batchsandboxViewer.enabled` | Create BatchSandbox viewer role | `true` |
| `extraRoles.poolEditor.enabled` | Create Pool editor role | `true` |
| `extraRoles.poolViewer.enabled` | Create Pool viewer role | `true` |
## Usage Examples
### Example 1: Install with Custom Image
```bash
helm install opensandbox-controller ./opensandbox-controller \
--set controllerManager.image.repository=myregistry.com/sandbox-controller \
--set controllerManager.image.tag=latest
```
### Example 2: Install with High Availability
```bash
helm install opensandbox-controller ./opensandbox-controller \
--set controllerManager.replicas=3 \
--set controllerManager.resources.requests.cpu=100m \
--set controllerManager.resources.requests.memory=256Mi
```
### Example 3: Install with Prometheus Monitoring
```bash
helm install opensandbox-controller ./opensandbox-controller \
--set metrics.serviceMonitor.enabled=true
```
### Example 4: Install without CRDs (for upgrades)
```bash
helm upgrade opensandbox-controller ./opensandbox-controller \
--set crds.install=false
```
## Creating Resources
After installation, you can create OpenSandbox resources:
### Create a Pool
```yaml
apiVersion: sandbox.opensandbox.io/v1alpha1
kind: Pool
metadata:
name: example-pool
spec:
minBufferSize: 2
maxBufferSize: 5
capacity: 10
sandboxTemplate:
spec:
image: ubuntu:latest
command: ["sleep", "infinity"]
```
### Create a BatchSandbox
```yaml
apiVersion: sandbox.opensandbox.io/v1alpha1
kind: BatchSandbox
metadata:
name: example-batchsandbox
spec:
replicas: 3
ttlSecondsAfterFinished: 3600
sandboxTemplate:
spec:
image: ubuntu:latest
command: ["sleep", "infinity"]
```
## Using with SDK
The OpenSandbox Python SDK connects to the server to manage sandboxes. The server must be accessible from where you run the SDK.
### Access Methods
#### 1. Port Forward (Recommended for Development)
```bash
# Forward local port to server
kubectl port-forward -n opensandbox svc/opensandbox-controller-server 8080:8080
```
Then use SDK with `localhost:8080`:
```python
from opensandbox import Sandbox
from opensandbox.config import ConnectionConfig
sandbox = await Sandbox.create(
"ubuntu:latest",
entrypoint=["sleep", "infinity"],
connection_config=ConnectionConfig(domain="localhost:8080"),
extensions={"poolRef": "agent-pool"}
)
```
#### 2. In-Cluster Access
If running SDK inside the same Kubernetes cluster:
```python
sandbox = await Sandbox.create(
"ubuntu:latest",
entrypoint=["sleep", "infinity"],
connection_config=ConnectionConfig(
domain="opensandbox-controller-server.opensandbox.svc.cluster.local:8080"
),
extensions={"poolRef": "agent-pool"}
)
```
#### 3. NodePort / LoadBalancer / Ingress
For external access, configure the service type accordingly and use the appropriate domain.
### SDK Usage Examples
The OpenSandbox Python SDK supports two creation modes:
### Pooled Mode (Recommended)
Fast creation using pre-warmed pools. **Image must match the Pool's configuration**:
```python
from opensandbox import Sandbox
from opensandbox.config import ConnectionConfig
sandbox = await Sandbox.create(
"ubuntu:latest", # Must match Pool's image
entrypoint=["sleep", "infinity"],
connection_config=ConnectionConfig(domain="localhost:8080"), # Server address
extensions={"poolRef": "agent-pool"} # Reference to Pool name
)
```
**Important**: When using `poolRef`, the SDK's `image` parameter will be **ignored** - the Pool's pre-configured image is used instead. Only `entrypoint` and `env` can be customized.
### Non-pooled Mode
Direct creation with custom image and resources:
```python
sandbox = await Sandbox.create(
"python:3.11", # Any image
resource={"cpu": "1", "memory": "500Mi"},
connection_config=ConnectionConfig(domain="localhost:8080")
# No poolRef specified
)
```
### Connect to Existing Sandbox
```python
# List all sandboxes
from opensandbox import SandboxManager
manager = SandboxManager(connection_config=ConnectionConfig(domain="localhost:8080"))
sandboxes = await manager.list_sandbox_infos(SandboxFilter())
# Connect to existing
sandbox = await Sandbox.connect(
sandbox_id="<sandbox-id>",
connection_config=ConnectionConfig(domain="localhost:8080")
)
```
For detailed SDK integration guide including troubleshooting, see [examples/README.md](examples/README.md)
## Upgrading
```bash
# Upgrade to a new version
helm upgrade opensandbox-controller ./opensandbox-controller \
--set controllerManager.image.tag=v1.1.0
# Upgrade with new values
helm upgrade opensandbox-controller ./opensandbox-controller \
-f new-values.yaml
```
## Uninstalling
```bash
# Uninstall the release
helm uninstall opensandbox-controller
# Note: CRDs are not automatically deleted. To remove them:
kubectl delete crd batchsandboxes.sandbox.opensandbox.io
kubectl delete crd pools.sandbox.opensandbox.io
```
## Troubleshooting
### Check Controller Status
```bash
# Check deployment
kubectl get deployment -n opensandbox
# Check pods
kubectl get pods -n opensandbox
# Check logs
kubectl logs -n opensandbox -l control-plane=controller-manager
```
### Verify CRDs
```bash
# List CRDs
kubectl get crds | grep sandbox.opensandbox.io
# Describe CRD
kubectl describe crd batchsandboxes.sandbox.opensandbox.io
```
### Check RBAC
```bash
# Check ServiceAccount
kubectl get sa -n opensandbox
# Check ClusterRoles
kubectl get clusterrole | grep sandbox-k8s
# Check ClusterRoleBindings
kubectl get clusterrolebinding | grep sandbox-k8s
```
## Development
### Quick Start Scripts
The chart includes utility scripts in the `scripts/` directory:
- **`scripts/install.sh`** - Interactive installation wizard
- **`scripts/uninstall.sh`** - Safe uninstallation with cleanup
- **`scripts/e2e-test.sh`** - End-to-end validation
See [scripts/README.md](scripts/README.md) for detailed documentation.
### Linting the Chart
```bash
helm lint ./opensandbox-controller
```
### Testing the Chart
```bash
# Dry run
helm install opensandbox-controller ./opensandbox-controller --dry-run --debug
# Template rendering
helm template opensandbox-controller ./opensandbox-controller
```
### Package the Chart
```bash
helm package ./opensandbox-controller
```
## Contributing
Please refer to the main [OpenSandbox repository](https://github.com/alibaba/OpenSandbox) for contribution guidelines.
## License
Apache License 2.0
## Support
- Documentation: https://github.com/alibaba/OpenSandbox
- Issues: https://github.com/alibaba/OpenSandbox/issues
---
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
annotations:
controller-gen.kubebuilder.io/version: v0.18.0
name: batchsandboxes.sandbox.opensandbox.io
spec:
group: sandbox.opensandbox.io
names:
kind: BatchSandbox
listKind: BatchSandboxList
plural: batchsandboxes
shortNames:
- bsbx
singular: batchsandbox
scope: Namespaced
versions:
- additionalPrinterColumns:
- description: The desired number of pods.
jsonPath: .spec.replicas
name: DESIRED
type: integer
- description: The number of currently all pods.
jsonPath: .status.replicas
name: TOTAL
type: integer
- description: The number of currently all allocated pods.
jsonPath: .status.allocated
name: ALLOCATED
type: integer
- description: The number of currently all ready pods.
jsonPath: .status.ready
name: Ready
type: integer
- description: The number of currently all running tasks.
jsonPath: .status.taskRunning
name: TASK_RUNNING
priority: 1
type: integer
- description: The number of currently all succeed tasks.
jsonPath: .status.taskSucceed
name: TASK_SUCCEED
priority: 1
type: integer
- description: The number of currently all failed tasks.
jsonPath: .status.taskFailed
name: TASK_FAILED
priority: 1
type: integer
- description: The number of currently all unknown tasks.
jsonPath: .status.taskUnknown
name: TASK_UNKNOWN
priority: 1
type: integer
- description: sandbox expire time
jsonPath: .spec.expireTime
name: EXPIRE
type: string
- description: CreationTimestamp is a timestamp representing the server time when
this object was created. It is not guaranteed to be set in happens-before
order across separate operations. Clients may not set this value. It is represented
in RFC3339 form and is in UTC.
jsonPath: .metadata.creationTimestamp
name: AGE
type: date
name: v1alpha1
schema:
openAPIV3Schema:
description: BatchSandbox is the Schema for the batchsandboxes API.
properties:
apiVersion:
description: |-
APIVersion defines the versioned schema of this representation of an object.
Servers should convert recognized schemas to the latest internal value, and
may reject unrecognized values.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
type: string
kind:
description: |-
Kind is a string value representing the REST resource this object represents.
Servers may infer this from the endpoint the client submits requests to.
Cannot be updated.
In CamelCase.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
type: string
metadata:
type: object
spec:
description: BatchSandboxSpec defines the desired state of BatchSandbox.
properties:
expireTime:
description: |-
ExpireTime - Absolute time when the batch-sandbox is deleted.
If a time in the past is provided, the batch-sandbox will be deleted immediately.
format: date-time
type: string
poolRef:
description: |-
PoolRef references the Pool resource name for pooled sandbox creation.
Mutually exclusive with Template - use PoolRef for pool-based allocation or Template for direct sandbox creation.
type: string
replicas:
default: 1
description: Replicas is the number of desired replicas.
format: int32
minimum: 0
type: integer
shardPatches:
description: ShardPatches indicates patching to the Template for BatchSandbox.
x-kubernetes-preserve-unknown-fields: true
shardTaskPatches:
description: ShardTaskPatches indicates patching to the TaskTemplate
for individual Task.
x-kubernetes-preserve-unknown-fields: true
taskResourcePolicyWhenCompleted:
default: Retain
description: |-
TaskResourcePolicyWhenCompleted specifies how resources should be handled once a task reaches a completed state (SUCCEEDED or FAILED).
- Retain: Keep the resources until the BatchSandbox is deleted.
- Release: Free the resources immediately when the task completes.
type: string
taskTemplate:
description: |-
Task is a custom task spec that is automatically dispatched after the sandbox is successfully created.
The Sandbox is responsible for managing the lifecycle of the task.
x-kubernetes-preserve-unknown-fields: true
template:
description: Template describes the pods that will be created.
x-kubernetes-preserve-unknown-fields: true
required:
- replicas
type: object
status:
description: BatchSandboxStatus defines the observed state of BatchSandbox.
properties:
allocated:
description: "\tAllocated is the number of actual scheduled Pod"
format: int32
type: integer
observedGeneration:
description: |-
ObservedGeneration is the most recent generation observed for this BatchSandbox. It corresponds to the
BatchSandbox's generation, which is updated on mutation by the API Server.
format: int64
type: integer
ready:
description: "\tReady is the number of actual Ready Pod"
format: int32
type: integer
replicas:
description: Replicas is the number of actual Pods
format: int32
type: integer
taskFailed:
description: TaskFailed is the number of Failed task
format: int32
type: integer
taskPending:
description: TaskPending is the number of Pending task which is unassigned
format: int32
type: integer
taskRunning:
description: TaskRunning is the number of Running task
format: int32
type: integer
taskSucceed:
description: TaskSucceed is the number of Succeed task
format: int32
type: integer
taskUnknown:
description: TaskUnknown is the number of Unknown task
format: int32
type: integer
required:
- allocated
- ready
- replicas
- taskFailed
- taskPending
- taskRunning
- taskSucceed
- taskUnknown
type: object
type: object
served: true
storage: true
subresources:
status: {}
---
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
annotations:
controller-gen.kubebuilder.io/version: v0.18.0
name: pools.sandbox.opensandbox.io
spec:
group: sandbox.opensandbox.io
names:
kind: Pool
listKind: PoolList
plural: pools
singular: pool
scope: Namespaced
versions:
- additionalPrinterColumns:
- description: The number of all nodes in pool.
jsonPath: .status.total
name: TOTAL
type: integer
- description: The number of allocated nodes in pool.
jsonPath: .status.allocated
name: ALLOCATED
type: integer
- description: The number of available nodes in pool.
jsonPath: .status.available
name: AVAILABLE
type: integer
name: v1alpha1
schema:
openAPIV3Schema:
description: Pool is the Schema for the pools API.
properties:
apiVersion:
description: |-
APIVersion defines the versioned schema of this representation of an object.
Servers should convert recognized schemas to the latest internal value, and
may reject unrecognized values.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
type: string
kind:
description: |-
Kind is a string value representing the REST resource this object represents.
Servers may infer this from the endpoint the client submits requests to.
Cannot be updated.
In CamelCase.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
type: string
metadata:
type: object
spec:
description: PoolSpec defines the desired state of Pool.
properties:
capacitySpec:
description: CapacitySpec controls the size of the resource pool.
properties:
bufferMax:
description: BufferMax is the maximum number of nodes kept in
the warm buffer.
format: int32
minimum: 0
type: integer
bufferMin:
description: BufferMin is the minimum number of nodes that must
remain in the buffer.
format: int32
minimum: 0
type: integer
poolMax:
description: PoolMax is the maximum total number of nodes allowed
in the entire pool.
format: int32
minimum: 0
type: integer
poolMin:
description: PoolMin is the minimum total size of the pool.
format: int32
minimum: 0
type: integer
required:
- bufferMax
- bufferMin
- poolMax
- poolMin
type: object
template:
description: Pod Template used to create pre-warmed nodes in the pool.
x-kubernetes-preserve-unknown-fields: true
required:
- capacitySpec
type: object
status:
description: PoolStatus defines the observed state of Pool.
properties:
allocated:
description: Allocated is the number of nodes currently allocated
to sandboxes.
format: int32
type: integer
available:
description: Available is the number of nodes currently available
in the pool.
format: int32
type: integer
observedGeneration:
description: |-
ObservedGeneration is the most recent generation observed for this BatchSandbox. It corresponds to the
BatchSandbox's generation, which is updated on mutation by the API Server.
format: int64
type: integer
revision:
description: Revision is the latest version of pool
type: string
total:
description: Total is the total number of nodes in the pool.
format: int32
type: integer
required:
- allocated
- available
- revision
- total
type: object
type: object
served: true
storage: true
subresources:
status: {}
# Examples 目录结构
本文档说明 `examples/` 目录的组织结构和设计原则。
## 📂 目录结构
```
examples/
├── README.md # 主文档:快速开始、最佳实践
├── DIRECTORY_STRUCTURE.md # 本文档:目录结构说明
├── pool-examples.md # 详细的 Pool 配置说明
│
├── pool-agent-production.yaml # 🌟 生产级 Agent Pool(推荐)
├── pool-sdk-compatible.yaml # SDK 基础 Pool(execd only)
├── pool-sdk-with-tasks.yaml # SDK 完整 Pool(execd + task-executor)
│
├── batchsandbox-basic.yaml # Non-pooled 模式示例
└── batchsandbox-with-tasks.yaml # Pooled 批量任务示例
```
## 📝 文件分类
### Pool 配置文件(3 个)
| 文件 | 类型 | SDK 支持 | 自定义 entrypoint | 推荐度 |
|------|------|----------|-----------------|--------|
| `pool-agent-production.yaml` | 生产级 | ✅ | ✅ | ⭐⭐⭐⭐⭐ |
| `pool-sdk-with-tasks.yaml` | 完整功能 | ✅ | ✅ | ⭐⭐⭐⭐ |
| `pool-sdk-compatible.yaml` | 基础功能 | ✅ | ❌ | ⭐⭐⭐ |
**选择建议**:
- **生产环境**:使用 `pool-agent-production.yaml`(包含详细注释和最佳实践)
- **快速测试**:使用 `pool-sdk-with-tasks.yaml`(更简洁)
- **特殊需求**:如果不需要自定义 entrypoint,使用 `pool-sdk-compatible.yaml`
### BatchSandbox 配置文件(2 个)
| 文件 | 模式 | 依赖 | 用途 |
|------|------|------|------|
| `batchsandbox-basic.yaml` | Non-pooled | 无 | 演示直接创建 Pod |
| `batchsandbox-with-tasks.yaml` | Pooled | Pool | 演示批量异构任务 |
**使用说明**:
- **SDK 场景**:通常不需要手动创建 BatchSandbox(SDK 自动创建)
- **kubectl 场景**:用于批量任务执行(RL 训练、压力测试等)
### 文档文件(3 个)
| 文件 | 内容 | 面向用户 |
|------|------|---------|
| `README.md` | 快速开始、最佳实践、故障排查 | 所有用户 |
| `pool-examples.md` | Pool 详细配置说明 | 高级用户 |
| `DIRECTORY_STRUCTURE.md` | 目录结构说明 | 开发者 |
## 🎯 设计原则
### 1. 简化选择
**问题**:之前有太多相似的示例(pool-basic, pool-with-execd, pool-with-task-executor...),用户不知道选哪个。
**解决**:
- 保留 3 个 Pool 示例,明确分类和推荐度
- 删除误导性示例(pool-basic 无 execd,pool-with-task-executor 无 execd)
- 突出推荐 `pool-agent-production.yaml`
### 2. 面向实际场景
**问题**:示例文件缺乏实际使用场景说明。
**解决**:
- 明确标注适用场景(Agent 服务、RL 训练、压力测试等)
- 提供完整的 SDK 使用代码示例
- 包含部署、验证、监控的完整流程
### 3. 最佳实践优先
**问题**:缺少生产级配置参考。
**解决**:
- 创建 `pool-agent-production.yaml` 包含:
- 详细的配置注释
- 容量规划建议
- 安全最佳实践
- 监控和调试指南
### 4. 纠正常见误区
**问题**:用户容易误解 Pool 的使用方式。
**解决**:
- 在 README.md 中突出"常见误区"章节
- 明确说明:
- Pool 是 Pod 池,不是 Sandbox 池
- 不需要预创建 BatchSandbox
- SDK 每次 create() 创建新 BatchSandbox
## 🔄 文件变更历史
### 删除的文件(4 个)
| 文件 | 删除原因 |
|------|---------|
| `pool-basic.yaml` | 无 execd,SDK 无法使用,误导性 |
| `pool-with-task-executor.yaml` | 只有 task-executor 无 execd,不完整 |
| `pool-with-execd.yaml` | execd 启动方式不标准,已被 pool-sdk-compatible.yaml 替代 |
| `batchsandbox-pooled.yaml` | 依赖不存在的 basic-pool,无效 |
### 新增的文件(1 个)
| 文件 | 内容 |
|------|------|
| `pool-agent-production.yaml` | 生产级 Agent Pool 配置,包含详细注释和最佳实践 |
### 修改的文件(3 个)
| 文件 | 修改内容 |
|------|---------|
| `README.md` | 重写,添加场景分类、常见误区、容量规划等 |
| `batchsandbox-with-tasks.yaml` | 修改 poolRef 为 agent-pool,添加注释 |
| `batchsandbox-basic.yaml` | 添加详细注释说明 |
## 📊 使用场景映射
### 场景 A:多 Agent 并发使用
```
用户需求: Agent 服务、Code Interpreter、动态工作流
↓
推荐配置: pool-agent-production.yaml
↓
使用方式: SDK 动态创建 sandbox
↓
流程: Helm 部署 Pool → SDK.create() → SDK.kill()
```
### 场景 B:批量任务执行
```
用户需求: RL 训练、压力测试、批量数据处理
↓
推荐配置: pool-agent-production.yaml + batchsandbox-with-tasks.yaml
↓
使用方式: kubectl 创建 BatchSandbox
↓
流程: kubectl apply pool → kubectl apply batchsandbox → 自动清理
```
### 场景 C:测试和开发
```
用户需求: 测试特定镜像、验证功能
↓
推荐配置: batchsandbox-basic.yaml (non-pooled)
↓
使用方式: kubectl 直接创建
↓
流程: kubectl apply → kubectl delete
```
## 🔗 相关资源
- **深度分析**:`/data/home/cz/sandbox-test/pool-analysis/`
- Pool 使用指南
- 架构流程图
- 验证测试脚本
- **Helm 配置**:`/data/home/cz/OpenSandbox/kubernetes/helm-chart/`
- `values.yaml` - 默认配置
- `values-e2e.yaml` - E2E 测试配置
- 生产/开发环境配置:使用 `--set` 或自定义 values 文件
- **主文档**:`/data/home/cz/OpenSandbox/kubernetes/README.md`
- Kubernetes 部署完整指南
## 💡 维护建议
### 添加新示例时
1. **明确场景**:每个示例应对应明确的使用场景
2. **完整注释**:包含配置说明、使用方式、注意事项
3. **验证测试**:确保示例可以正常运行
4. **更新文档**:同步更新 README.md 和本文档
### 修改现有示例时
1. **保持兼容**:避免破坏性变更
2. **版本说明**:在注释中说明版本要求
3. **测试验证**:修改后进行完整测试
4. **文档同步**:更新相关文档
### 删除示例时
1. **评估影响**:确认没有外部依赖
2. **提供替代**:在文档中说明替代方案
3. **迁移指南**:如果有用户使用,提供迁移步骤
# OpenSandbox Examples
This directory contains various usage examples and best practices for the OpenSandbox Kubernetes Controller.
## 🎉 Important Update
**Starting from version v0.2.0, the Helm chart deploys an agent-pool by default** without manual creation!
```bash
# Default installation automatically creates agent-pool
helm install opensandbox opensandbox-controller
# View the automatically created Pool
kubectl get pools -n opensandbox
```
If you don't need the Pool, you can disable it:
```bash
helm install opensandbox opensandbox-controller --set pools[0].enabled=false
```
## 📁 File List
### Pool Examples
| File | SDK Compatible | Custom Entrypoint | Purpose |
|------|----------------|-------------------|---------|
| `pool-sdk-compatible.yaml` | ✅ Supported | ❌ Not Supported | SDK Basic Mode (execd only)|
| `pool-sdk-with-tasks.yaml` | ✅ Supported | ✅ Supported | SDK Complete Mode (execd + task-executor)|
| **`pool-agent-production.yaml`** | ✅ Supported | ✅ Supported | **🌟 Production-Grade Agent Pool (Recommended)** |
### BatchSandbox Examples
| File | Mode | Purpose |
|------|------|---------|
| `batchsandbox-basic.yaml` | Non-pooled | Direct Pod creation without using Pool |
| `batchsandbox-with-tasks.yaml` | Pooled | Batch heterogeneous task example |
### Documentation
| File | Content |
|------|---------|
| `README.md` | This document |
| `pool-examples.md` | Detailed Pool configuration guide |
## 🎯 Core Concepts
### ❌ Common Misconceptions
> **Misconception 1**: Pool is a pre-created pool of Sandboxes that Agents can reuse
> **Correct**: Pool is a **Pod pool**, not a Sandbox pool
> **Misconception 2**: Need to pre-create BatchSandbox during Helm deployment for Agent use
> **Correct**: SDK creates a **new BatchSandbox** with each create() call, no pre-creation needed
> **Misconception 3**: Pool without execd can work with SDK
> **Correct**: SDK **requires** Pool to contain execd (port 44772)
### ✅ Correct Understanding
```
During Helm deployment:
└─> Only create Pool (long-running, maintains pre-warmed Pods)
During SDK runtime:
├─> Agent-1: SDK.create() → Creates BatchSandbox-1 (allocates Pod-1)
├─> Agent-2: SDK.create() → Creates BatchSandbox-2 (allocates Pod-2)
└─> Agent-1: SDK.kill() → Deletes BatchSandbox-1 (Pod-1 returns to Pool)
Next request:
└─> Agent-3: SDK.create() → Creates BatchSandbox-3 (reuses Pod-1) ← Fast!
```
**Key Points**:
- ✅ Pool maintains **Pods** (pre-warmed containers)
- ✅ Each SDK.create() creates a **new BatchSandbox**
- ✅ Pods are reused, BatchSandboxes are not
- ❌ Don't pre-create BatchSandboxes
## 🚀 Quick Start
### Scenario A: Multi-Agent Concurrent Usage (Recommended)
**Use Cases**: Agent services, Code Interpreter, dynamic workflows
```bash
# 1. Install Helm chart (automatically creates agent-pool)
helm install opensandbox opensandbox-controller
# 2. Verify deployment
kubectl get deployment -n opensandbox
kubectl get pool -n opensandbox
# 3. Check Pool status
kubectl get pool agent-pool -n opensandbox -o jsonpath='{.status}' | jq
# Example output:
# {
# "total": 10, # Total Pods
# "allocated": 0, # Allocated
# "available": 10 # Available
# }
# 4. View Pool Pods
kubectl get pods -l pool=agent-pool -n opensandbox
```
**SDK Usage**:
```python
from opensandbox import Sandbox
from opensandbox.config import ConnectionConfig
from datetime import timedelta
async def handle_agent_request(agent_id: str, task: str):
"""Create a new sandbox for each Agent request"""
# SDK.create() will allocate a Pod from agent-pool
sandbox = await Sandbox.create(
"nginx:latest", # Will be ignored, uses image from Pool
entrypoint=["/bin/sh", "-c", "sleep infinity"],
env={"AGENT_ID": agent_id},
timeout=timedelta(hours=1),
connection_config=ConnectionConfig(domain="<server-ip>:8088"),
extensions={"poolRef": "agent-pool"} # Use default agent-pool
)
try:
# Use sandbox
result = await sandbox.commands.run(task)
return result
finally:
# Delete BatchSandbox, Pod returns to Pool
await sandbox.kill()
```
### Scenario B: Custom Pool Capacity (High Concurrency Scenarios)
If you need higher concurrency capacity, override default capacity parameters using `--set`:
```bash
# Use configuration optimized for multiple Agents (bufferMin: 50, poolMax: 300)
helm install opensandbox opensandbox-controller \
--set pools[0].capacitySpec.bufferMin=50 \
--set pools[0].capacitySpec.bufferMax=100 \
--set pools[0].capacitySpec.poolMin=50 \
--set pools[0].capacitySpec.poolMax=300
```
### Scenario C: Batch Task Execution (kubectl)
**Use Cases**: RL training, stress testing, batch data processing
```bash
# 1. Ensure Pool is deployed (automatically created by default)
kubectl get pool agent-pool -n opensandbox
# 2. Create BatchSandbox to execute batch tasks
kubectl apply -f batchsandbox-with-tasks.yaml
# 3. View task execution status
kubectl get batchsandbox task-batch-sandbox -n opensandbox -o wide
# 4. View task logs
POD_NAME=$(kubectl get pods -l batchsandbox=task-batch-sandbox -n opensandbox -o jsonpath='{.items[0].metadata.name}')
kubectl logs $POD_NAME -c sandbox-container -n opensandbox
kubectl logs $POD_NAME -c task-executor -n opensandbox
# 5. Automatic cleanup after task completion (ttlSecondsAfterFinished)
```
### Scenario D: Non-pooled Direct Creation
**Use Cases**: Testing environments, special image requirements
```bash
# Directly create BatchSandbox (without using Pool)
kubectl apply -f batchsandbox-basic.yaml
# View created Pods
kubectl get pods -l batchsandbox=basic-batch-sandbox -n opensandbox
```
## 📊 Pool Configuration Type Comparison
### Type 1: SDK Pool (Basic) - pool-sdk-compatible.yaml
```yaml
# ✅ SDK compatible - contains execd only
# ❌ Does not support custom entrypoint
initContainers:
- name: execd-installer
image: opensandbox/execd:v1.0.21
containers:
- name: sandbox-container
command: ["/opt/opensandbox/bin/bootstrap.sh", "nginx", "-g", "daemon off;"]
ports:
- containerPort: 44772
name: execd
```
**SDK Usage**:
```python
sandbox = await Sandbox.create(
"nginx:latest",
# ❌ Cannot pass entrypoint
env={"VAR": "value"}, # ✅ Can pass environment variables
extensions={"poolRef": "sdk-pool"}
)
```
### Type 2: Task Pool (Complete) - pool-agent-production.yaml (Recommended)
```yaml
# ✅ SDK compatible - contains execd + task-executor
# ✅ Supports custom entrypoint
spec:
shareProcessNamespace: true # Required by task-executor
initContainers:
- name: execd-installer
image: opensandbox/execd:v1.0.21
containers:
- name: sandbox-container
command: ["/opt/opensandbox/bin/bootstrap.sh", "sleep", "infinity"]
ports:
- containerPort: 44772
name: execd
- name: task-executor # Add task-executor sidecar
image: opensandbox/task-executor:dev
securityContext:
capabilities:
add: ["SYS_PTRACE"]
```
**SDK Usage** (with custom entrypoint):
```python
sandbox = await Sandbox.create(
"nginx:latest",
entrypoint=["/bin/sh", "-c", "custom command"], # ✅ Can customize
env={"VAR": "value"},
extensions={"poolRef": "agent-pool"}
)
```
## 🔍 Monitoring and Debugging
### Monitor Pool Utilization
```bash
# Real-time monitoring
watch kubectl get pool agent-pool -o jsonpath='{.status}' | jq
# View detailed information
kubectl describe pool agent-pool
# View Pool Pod list
kubectl get pods -l pool=agent-pool -o wide
```
**Optimization Recommendations**:
- If `available` is frequently 0 → Increase `bufferMax`
- If `available` is always close to `total` → Decrease `bufferMin`
- If `total` frequently reaches `poolMax` → Increase `poolMax` or optimize Agent usage
### Verify Pool Configuration
```bash
# Check if Pod contains execd
kubectl exec -it <pod-name> -c sandbox-container -- ps aux | grep execd
# Check execd port
kubectl exec -it <pod-name> -c sandbox-container -- nc -zv localhost 44772
# Check task-executor (if present)
kubectl get pods -l pool=agent-pool -o jsonpath='{.items[0].spec.containers[*].name}'
# Output should include: sandbox-container task-executor
```
### View BatchSandbox Status
```bash
# List all BatchSandboxes
kubectl get batchsandboxes
# View detailed status
kubectl describe batchsandbox <name>
# View task execution statistics
kubectl get batchsandbox -o custom-columns=\
NAME:.metadata.name,\
REPLICAS:.spec.replicas,\
RUNNING:.status.taskRunning,\
SUCCEED:.status.taskSucceed,\
FAILED:.status.taskFailed
```
## 🛠️ Troubleshooting
### Pool Pod Fails to Start
```bash
# View Pod events
kubectl describe pod <pod-name>
# View container logs
kubectl logs <pod-name> -c sandbox-container
kubectl logs <pod-name> -c task-executor # If present
# Check image pull
kubectl describe pod <pod-name> | grep -A 5 Events
```
### SDK Sandbox Creation Timeout
**Symptom**: SDK error `Health check timeout`
**Possible Causes**:
1. Pool doesn't have execd → Use `pool-agent-production.yaml`
2. execd not started → Check execd process in Pod
3. Network issues → Check network connectivity between Server and Pod
**Troubleshooting Steps**:
```bash
# 1. Confirm Pool contains execd
kubectl get pool agent-pool -o yaml | grep -A 10 initContainers
# 2. Check execd process
kubectl exec -it <pod-name> -c sandbox-container -- ps aux | grep execd
# 3. Check execd port
kubectl exec -it <pod-name> -c sandbox-container -- nc -zv localhost 44772
# 4. View Server logs
kubectl logs -l app=opensandbox-server -n opensandbox
```
### task-executor Permission Issues
```bash
# Check security context
kubectl get pod <pod-name> -o yaml | grep -A 10 securityContext
# Should contain:
# capabilities:
# add: ["SYS_PTRACE"]
# Check process namespace sharing
kubectl get pod <pod-name> -o jsonpath='{.spec.shareProcessNamespace}'
# Should output: true
```
## 📦 Capacity Planning Recommendations
Plan Pool capacity based on concurrent Agent count:
| Concurrent Agents | bufferMin | bufferMax | poolMin | poolMax | Description |
|------------------|-----------|-----------|---------|---------|-------------|
| 1-10 | 2 | 5 | 2 | 20 | Small-scale testing |
| 10-50 | 10 | 20 | 10 | 100 | Small to medium applications |
| 50-200 | 50 | 100 | 50 | 300 | Medium to large applications |
| 200+ | 100 | 200 | 100 | 500 | Large-scale production |
**Parameter Descriptions**:
- `bufferMin`: Minimum buffer, ensures fast response
- `bufferMax`: Maximum buffer, controls pre-warming cost
- `poolMin`: Minimum capacity during low traffic periods
- `poolMax`: Maximum capacity during peak periods
**Cost Optimization**:
- Low traffic periods: Pool scales down to `poolMin`, saving resources
- Peak periods: Pool expands to `poolMax`, ensuring response speed
- Buffer zone: `bufferMin` ensures fast response, `bufferMax` avoids excessive pre-warming
## 📚 Related Documentation
### In-depth Analysis Documents
- **Pool Usage Guide**: `/data/home/cz/sandbox-test/pool-analysis/opensandbox_pool_usage_guide.md`
- **Architecture Diagrams**: `/data/home/cz/sandbox-test/pool-analysis/pool_architecture.txt`
- **Verification Test Scripts**: `/data/home/cz/sandbox-test/pool-analysis/test_pool_behavior.py`
### Helm Deployment Configuration
- **Main values configuration**: `/data/home/cz/OpenSandbox/kubernetes/helm-chart/values.yaml`
- **E2E test configuration**: `/data/home/cz/OpenSandbox/kubernetes/helm-chart/values-e2e.yaml`
- **Main README**: `/data/home/cz/OpenSandbox/kubernetes/README.md`
### API Reference
```bash
# View Pool CRD definition
kubectl explain pool
kubectl explain pool.spec
kubectl explain pool.spec.capacitySpec
# View BatchSandbox CRD definition
kubectl explain batchsandbox
kubectl explain batchsandbox.spec
kubectl explain batchsandbox.spec.taskTemplate
```
## 🧹 Resource Cleanup
```bash
# Delete BatchSandbox
kubectl delete batchsandbox --all -n default
# Delete Pool (automatically cleans up related Pods)
kubectl delete pool --all -n default
# Delete all resources in namespace
kubectl delete all --all -n default
```
## 💡 Best Practices Summary
1. **Pool is a Pod pool, not a Sandbox pool**
2. **SDK creates a new BatchSandbox with each create() call**
3. **No need to pre-create BatchSandboxes for reuse**
4. **Only create Pool during Helm deployment**
5. **Dynamically create/delete BatchSandboxes at runtime**
6. **Configure Pool capacity parameters appropriately to optimize cost and performance**
7. **Use `pool-agent-production.yaml` as production environment template**
## 🔗 More Examples
- **Detailed Pool configuration guide**: See `pool-examples.md`
- **SDK integration examples**: See "Quick Start" section in this document
- **Helm Chart configuration**: See `values.yaml` and `values-e2e.yaml`
# ==============================================================================
# BatchSandbox - Non-pooled 模式(直接创建 Pod)
# ==============================================================================
#
# 用途:演示不使用 Pool 直接创建 BatchSandbox
#
# 使用场景:
# - 需要使用特定镜像或资源配置(Pool 无法满足)
# - 一次性任务,不需要预热优化
# - 测试和开发环境
#
# 特点:
# - 可自由指定镜像、资源、配置
# - 创建速度较慢(需要拉取镜像、启动 Pod)
# - 不复用预热资源
#
# 注意:
# - SDK 使用场景通常不需要预创建 BatchSandbox(动态创建即可)
# - 此示例主要用于演示 kubectl 直接创建
#
# ==============================================================================
apiVersion: sandbox.opensandbox.io/v1alpha1
kind: BatchSandbox
metadata:
name: basic-batch-sandbox
namespace: default
labels:
mode: non-pooled
annotations:
description: "Non-pooled 模式示例"
spec:
# 创建3个沙箱副本
replicas: 3
# 不使用资源池,直接创建Pod
# poolRef: "" # 留空表示 non-pooled 模式
# TTL:3600秒(1小时)后自动清理
ttlSecondsAfterFinished: 3600
# 沙箱模板
template:
spec:
containers:
- name: sandbox-container
image: ubuntu:22.04
command: ["sleep", "infinity"]
resources:
requests:
cpu: "100m"
memory: "128Mi"
limits:
cpu: "500m"
memory: "256Mi"
# ==============================================================================
# BatchSandbox with Heterogeneous Tasks(带异构任务的批量沙箱)
# ==============================================================================
#
# 用途:演示如何使用 Pool 批量执行异构任务(每个 sandbox 执行不同任务)
#
# 使用场景:
# - RL 训练:批量创建训练环境,每个环境执行不同策略
# - 压力测试:批量执行不同的测试用例
# - 数据处理:并行处理多个数据分片
#
# 注意:
# - 此示例主要用于 kubectl 直接创建的批量任务场景
# - SDK 使用场景通常不需要预创建 BatchSandbox(动态创建即可)
# - 需要先创建 pool-agent-production.yaml
#
# 相关文档:
# - /data/home/cz/sandbox-test/pool-analysis/opensandbox_pool_usage_guide.md
#
# ==============================================================================
apiVersion: sandbox.opensandbox.io/v1alpha1
kind: BatchSandbox
metadata:
name: task-batch-sandbox
namespace: default
labels:
use-case: batch-training
annotations:
description: "批量异构任务示例"
spec:
# 创建3个沙箱副本,每个执行不同的任务
replicas: 3
# 使用包含 execd + task-executor 的 Pool
# 注意:需要先创建 pool-agent-production.yaml
poolRef: agent-pool
# TTL:3600秒(1小时)后自动清理
ttlSecondsAfterFinished: 3600
# 默认任务模板(如果shardTaskPatches没有覆盖,则使用此模板)
taskTemplate:
spec:
process:
command: ["echo", "Default task message"]
# 异构任务:为每个沙箱自定义不同的任务
shardTaskPatches:
# 第1个沙箱的任务
- spec:
process:
command: ["bash", "-c"]
args:
- |
echo "Task 1: Running Python script"
python3 -c "
import time
print('Task 1 started')
time.sleep(2)
print('Task 1 completed')
"
# 第2个沙箱的任务
- spec:
process:
command: ["bash", "-c"]
args:
- |
echo "Task 2: Running shell commands"
date
uname -a
sleep 1
echo "Task 2 completed"
# 第3个沙箱的任务
- spec:
process:
command: ["bash", "-c"]
args:
- |
echo "Task 3: System info check"
cat /etc/os-release
df -h
free -h
echo "Task 3 completed"
# ==============================================================================
# OpenSandbox Agent Pool - 生产级配置
# ==============================================================================
#
# 用途:为多 Agent 并发场景提供预热的沙箱资源池(SDK 模式)
#
# 注意:此配置适用于 SDK 场景(需要 execd)
# 如果只使用 kubectl 管理 BatchSandbox,可以移除 execd 相关配置
#
# 架构说明:
# Pool 维护预热的 Pod 池(不是 Sandbox 池)
# - 每个 Pod 包含 execd(SDK 通信)+ task-executor(任务注入)
# - SDK.create() 从 Pool 分配 Pod,创建新的 BatchSandbox
# - SDK.kill() 删除 BatchSandbox,Pod 返回 Pool
#
# 使用模式:
# 1. Helm 部署时创建此 Pool(长期运行)
# 2. Agent 运行时通过 SDK 动态创建/删除 BatchSandbox
# 3. 不需要预创建 BatchSandbox
#
# 容量规划建议:
# | 并发 Agent 数 | bufferMin | bufferMax | poolMin | poolMax |
# |--------------|-----------|-----------|---------|---------|
# | 1-10 | 2 | 5 | 2 | 20 |
# | 10-50 | 10 | 20 | 10 | 100 |
# | 50-200 | 50 | 100 | 50 | 300 |
# | 200+ | 100 | 200 | 100 | 500 |
#
# 相关文档:
# - /data/home/cz/sandbox-test/pool-analysis/opensandbox_pool_usage_guide.md
# - /data/home/cz/OpenSandbox/kubernetes/helm-chart/values.yaml (查看 pools 配置)
#
# ==============================================================================
apiVersion: sandbox.opensandbox.io/v1alpha1
kind: Pool
metadata:
name: agent-pool
namespace: default # 使用与 Helm chart namespaceOverride 一致的命名空间
labels:
app: opensandbox
component: agent-pool
annotations:
description: "生产级 Agent Pool,支持 SDK 动态创建 sandbox"
spec:
template:
metadata:
labels:
pool: agent-pool
sdk-compatible: "true"
spec:
# ========================================
# 必需:共享进程命名空间
# ========================================
# task-executor 需要访问 sandbox 容器的进程树
shareProcessNamespace: true
# ========================================
# Init Container:安装 execd(仅 SDK 场景需要)
# ========================================
# 注意:如果只使用 kubectl 管理 BatchSandbox(不使用 SDK),
# 可以移除此 init container 和相关的 execd 配置。
#
# execd 的作用:
# - 提供 SDK 与 Pod 的通信接口(44772 端口)
# - 执行 SDK 发送的命令(commands.run(), files.read_file() 等)
#
# 使用场景:
# ✅ 需要:SDK 动态创建 sandbox(Agent、Code Interpreter 等)
# ❌ 不需要:纯 kubectl 批量任务(RL 训练、压力测试等)
initContainers:
- name: execd-installer
image: opensandbox/execd:v1.0.21
imagePullPolicy: IfNotPresent
command: ["/bin/sh", "-c"]
args:
- |
# 复制 execd 二进制和启动脚本
cp ./execd /opt/opensandbox/bin/execd && \
cp ./bootstrap.sh /opt/opensandbox/bin/bootstrap.sh && \
chmod +x /opt/opensandbox/bin/execd && \
chmod +x /opt/opensandbox/bin/bootstrap.sh && \
echo "execd installed successfully"
volumeMounts:
- name: opensandbox-bin
mountPath: /opt/opensandbox/bin
# ========================================
# 主容器:Sandbox 环境
# ========================================
containers:
- name: sandbox-container
# 镜像说明:
# - Pool 中的镜像由 Pool 预定义,SDK 指定的镜像会被忽略
# - 根据 Agent 需求选择合适的基础镜像(nginx, ubuntu, python, etc.)
# - 确保镜像包含 /bin/sh 用于执行 bootstrap.sh
image: nginx:latest
imagePullPolicy: IfNotPresent
# 启动命令:使用 bootstrap.sh 启动 execd
# bootstrap.sh 会:
# 1. 后台启动 execd(监听 44772 端口)
# 2. 执行用户指定的命令(这里是 sleep infinity)
command: ["/opt/opensandbox/bin/bootstrap.sh", "sleep", "infinity"]
# 环境变量
env:
- name: EXECD
value: /opt/opensandbox/bin/execd
# 可添加其他环境变量:
# - name: CUSTOM_VAR
# value: "custom-value"
# 端口配置
ports:
- containerPort: 44772
name: execd
protocol: TCP
# 可暴露其他端口(如应用端口):
# - containerPort: 8080
# name: app
# protocol: TCP
# 资源配置
# 根据 Agent 任务复杂度调整
resources:
requests:
cpu: "100m" # 最小 CPU(保证调度)
memory: "128Mi" # 最小内存
limits:
cpu: "500m" # 最大 CPU(防止资源抢占)
memory: "256Mi" # 最大内存
# 卷挂载
volumeMounts:
- name: opensandbox-bin
mountPath: /opt/opensandbox/bin
readOnly: true
# 健康检查(可选)
# livenessProbe:
# tcpSocket:
# port: execd
# initialDelaySeconds: 10
# periodSeconds: 30
# readinessProbe:
# tcpSocket:
# port: execd
# initialDelaySeconds: 5
# periodSeconds: 10
# ========================================
# Sidecar:Task Executor
# ========================================
- name: task-executor
# 镜像说明:
# - 使用 Helm chart 中配置的 task-executor 镜像
# - 版本应与 controller 版本保持一致
# - 开发环境可用:opensandbox/task-executor:dev
# - 生产环境建议:your-registry/opensandbox-task-executor:v1.0.0
image: opensandbox/task-executor:dev
imagePullPolicy: Never # 开发环境使用本地镜像;生产环境改为 IfNotPresent
# 端口配置
ports:
- containerPort: 5758
name: task-executor
protocol: TCP
# 资源配置
# task-executor 需要更多 CPU 用于进程注入
resources:
requests:
cpu: "100m"
memory: "128Mi"
limits:
cpu: "500m" # 可能需要更多 CPU
memory: "256Mi"
# 安全上下文
# 必需:SYS_PTRACE 权限用于注入进程到 sandbox 容器
securityContext:
capabilities:
add: ["SYS_PTRACE"]
# 生产环境建议添加其他安全设置:
# runAsNonRoot: true
# runAsUser: 1000
# allowPrivilegeEscalation: false
# ========================================
# 卷配置
# ========================================
volumes:
- name: opensandbox-bin
emptyDir: {}
# 可添加其他卷(如配置文件、数据持久化):
# - name: config
# configMap:
# name: agent-config
# - name: data
# persistentVolumeClaim:
# claimName: agent-data-pvc
# ========================================
# 调度配置(可选)
# ========================================
# 节点选择器
# nodeSelector:
# workload-type: agent
# zone: production
# 容忍度(允许调度到特定污点的节点)
# tolerations:
# - key: "workload"
# operator: "Equal"
# value: "agent"
# effect: "NoSchedule"
# 亲和性(控制 Pod 分布)
# affinity:
# podAntiAffinity:
# preferredDuringSchedulingIgnoredDuringExecution:
# - weight: 100
# podAffinityTerm:
# labelSelector:
# matchExpressions:
# - key: pool
# operator: In
# values:
# - agent-pool
# topologyKey: kubernetes.io/hostname
# ========================================
# Pool 容量配置
# ========================================
capacitySpec:
# bufferMin: 最小缓冲 - Pool 保证至少有这么多可用 Pod
# - 保证快速响应,避免 Agent 等待
# - 根据并发 Agent 峰值设置
bufferMin: 10
# bufferMax: 最大缓冲 - Pool 最多预热这么多 Pod
# - 控制预热成本,避免资源浪费
# - 通常设为 bufferMin 的 2-5 倍
bufferMax: 50
# poolMin: Pool 最小容量 - 即使没人用,也保持这么多 Pod
# - 低峰期保底容量,避免冷启动
# - 通常与 bufferMin 相同或稍大
poolMin: 10
# poolMax: Pool 最大容量 - 高峰期最多这么多 Pod
# - 限制最大资源使用,避免集群资源耗尽
# - 根据集群资源和业务峰值设置
poolMax: 200
# 容量规划示例:
# 场景:200 个并发 Agent,平均每个 Agent 会话 5 分钟
# bufferMin: 50 (保证 50 个 Agent 立即可用)
# bufferMax: 100 (预热 100 个,覆盖短期突发)
# poolMin: 50 (低峰期保持 50 个)
# poolMax: 300 (高峰期最多 300 个)
# ==============================================================================
# SDK 使用示例
# ==============================================================================
#
# Python SDK 使用此 Pool:
#
# ```python
# import asyncio
# from datetime import timedelta
# from opensandbox import Sandbox
# from opensandbox.config import ConnectionConfig
#
# async def create_agent_sandbox(agent_id: str):
# """为 Agent 创建 sandbox"""
# sandbox = await Sandbox.create(
# "nginx:latest", # 镜像会被忽略,使用 Pool 中的镜像
# entrypoint=["/bin/sh", "-c", "sleep infinity"], # 可自定义
# env={"AGENT_ID": agent_id}, # 可传递环境变量
# timeout=timedelta(hours=1),
# connection_config=ConnectionConfig(domain="<server-ip>:8088"),
# extensions={"poolRef": "agent-pool"} # 指定 Pool 名称
# )
# return sandbox
#
# async def handle_agent_request(agent_id: str, task: str):
# """处理单个 Agent 请求"""
# sandbox = await create_agent_sandbox(agent_id)
# try:
# result = await sandbox.commands.run(task)
# return result
# finally:
# await sandbox.kill() # Pod 返回 Pool
# ```
#
# ==============================================================================
# 部署和监控
# ==============================================================================
#
# 1. 部署 Pool:
# kubectl apply -f pool-agent-production.yaml
#
# 2. 验证 Pool 状态:
# kubectl get pool agent-pool -n default
# kubectl get pool agent-pool -n default -o jsonpath='{.status}' | jq
#
# 3. 查看 Pool 的 Pod:
# kubectl get pods -l pool=agent-pool -n default
#
# 4. 监控 Pool 使用率:
# watch kubectl get pool agent-pool -n default -o jsonpath='{.status}'
# # 输出示例:
# # {
# # "total": 50, # 总 Pod 数
# # "allocated": 30, # 已分配
# # "available": 20 # 可用
# # }
#
# 5. 优化建议:
# - 如果 available 经常为 0 → 增加 bufferMax
# - 如果 available 总是接近 total → 减少 bufferMin
# - 如果 total 经常达到 poolMax → 增加 poolMax 或优化 Agent 使用
#
# ==============================================================================
# 故障排查
# ==============================================================================
#
# Pool Pod 无法启动:
# kubectl describe pod -l pool=agent-pool -n default
# kubectl logs -l pool=agent-pool -n default -c sandbox-container
# kubectl logs -l pool=agent-pool -n default -c task-executor
#
# execd 连接失败:
# kubectl exec -it <pod-name> -n default -c sandbox-container -- ps aux | grep execd
# kubectl exec -it <pod-name> -n default -c sandbox-container -- nc -zv localhost 44772
#
# task-executor 权限问题:
# kubectl get pod <pod-name> -n default -o yaml | grep -A 10 securityContext
#
# ==============================================================================
# Pool示例 - 包含Task Executor Sidecar
## 基本Pool(不包含任务执行)
```yaml
apiVersion: sandbox.opensandbox.io/v1alpha1
kind: Pool
metadata:
name: basic-pool
namespace: default
spec:
template:
spec:
containers:
- name: sandbox-container
image: ubuntu:22.04
command: ["sleep", "infinity"]
resources:
requests:
cpu: "100m"
memory: "128Mi"
limits:
cpu: "500m"
memory: "256Mi"
capacitySpec:
bufferMax: 10
bufferMin: 2
poolMax: 20
poolMin: 5
```
## Pool with Task Executor(支持任务执行)
**重要提示**:
- Task Executor作为sidecar容器运行在Pool的Pod中
- 必须启用`shareProcessNamespace: true`以共享进程命名空间
- Task Executor需要`SYS_PTRACE`权限来注入进程
```yaml
apiVersion: sandbox.opensandbox.io/v1alpha1
kind: Pool
metadata:
name: task-enabled-pool
namespace: default
spec:
template:
spec:
# 必需:共享进程命名空间,允许task-executor访问sandbox容器的进程
shareProcessNamespace: true
containers:
# 主容器:沙箱环境
- name: sandbox-container
image: ubuntu:22.04
command: ["sleep", "infinity"]
resources:
requests:
cpu: "100m"
memory: "128Mi"
limits:
cpu: "500m"
memory: "256Mi"
# Sidecar:Task Executor(用于任务注入)
- name: task-executor
# 使用Helm values中配置的镜像
# {{ .Values.taskExecutor.image.repository }}:{{ .Values.taskExecutor.image.tag }}
image: opensandbox.io/task-executor:v0.0.1
imagePullPolicy: IfNotPresent
resources:
requests:
cpu: "100m"
memory: "128Mi"
limits:
cpu: "500m"
memory: "256Mi"
securityContext:
# 必需:需要ptrace权限来注入进程到sandbox容器
capabilities:
add: ["SYS_PTRACE"]
capacitySpec:
bufferMax: 10
bufferMin: 2
poolMax: 20
poolMin: 5
```
## BatchSandbox with Tasks(使用Pool执行任务)
创建使用上述Pool的BatchSandbox,并执行异构任务:
```yaml
apiVersion: sandbox.opensandbox.io/v1alpha1
kind: BatchSandbox
metadata:
name: task-batch-sandbox
namespace: default
spec:
# 副本数量
replicas: 3
# 引用包含task-executor的Pool
poolRef: task-enabled-pool
# TTL:3600秒后自动清理
ttlSecondsAfterFinished: 3600
# 默认任务模板(所有沙箱共享)
taskTemplate:
spec:
process:
command: ["echo", "Default task"]
# 异构任务:为每个沙箱自定义不同的任务
shardTaskPatches:
- spec:
process:
command: ["python3", "-c", "print('Task for sandbox 0')"]
- spec:
process:
command: ["bash", "-c", "echo 'Task for sandbox 1' && sleep 5"]
- spec:
process:
command: ["node", "-e", "console.log('Task for sandbox 2')"]
```
## 镜像配置说明
### 方式1:使用Helm Values配置
在`values.yaml`中配置task-executor镜像:
```yaml
taskExecutor:
image:
repository: your-registry/opensandbox-task-executor
tag: "v1.0.0"
pullPolicy: IfNotPresent
```
然后在Pool YAML中引用:
```yaml
image: your-registry/opensandbox-task-executor:v1.0.0
```
### 方式2:使用环境变量(ConfigMap)
创建ConfigMap存储镜像信息:
```yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: opensandbox-images
namespace: default
data:
taskExecutorImage: "your-registry/opensandbox-task-executor:v1.0.0"
```
在应用层读取ConfigMap并创建Pool。
### 方式3:使用Kustomize替换
使用Kustomize的镜像替换功能:
```yaml
# kustomization.yaml
images:
- name: opensandbox.io/task-executor
newName: your-registry/opensandbox-task-executor
newTag: v1.0.0
```
## 验证Task Executor
创建资源后,验证task-executor是否正常运行:
```bash
# 查看Pool状态
kubectl get pools task-enabled-pool
# 查看Pool创建的Pod
kubectl get pods -l pool=task-enabled-pool
# 检查Pod中是否有task-executor容器
kubectl get pods -l pool=task-enabled-pool -o jsonpath='{.items[0].spec.containers[*].name}'
# 输出应包含: sandbox-container task-executor
# 查看task-executor日志
kubectl logs <pod-name> -c task-executor
# 查看BatchSandbox任务状态
kubectl get batchsandbox task-batch-sandbox -o wide
# 应显示: TASK_RUNNING, TASK_SUCCEED, TASK_FAILED 等状态
```
## 故障排查
### Task Executor无法启动
```bash
# 检查容器状态
kubectl describe pod <pod-name>
# 检查权限问题
kubectl get pod <pod-name> -o jsonpath='{.spec.containers[1].securityContext}'
# 应显示: {"capabilities":{"add":["SYS_PTRACE"]}}
# 检查进程命名空间共享
kubectl get pod <pod-name> -o jsonpath='{.spec.shareProcessNamespace}'
# 应显示: true
```
### 任务执行失败
```bash
# 查看任务状态
kubectl describe batchsandbox task-batch-sandbox
# 查看task-executor日志
kubectl logs <pod-name> -c task-executor -f
# 查看sandbox容器日志
kubectl logs <pod-name> -c sandbox-container
```
## 性能考虑
- **资源配置**:根据任务复杂度调整task-executor的资源限制
- **并发控制**:Pool的`bufferMax`和`poolMax`控制并发沙箱数量
- **任务超时**:在taskTemplate中配置超时时间防止任务卡死
- **清理策略**:使用`ttlSecondsAfterFinished`自动清理完成的沙箱
## 最佳实践
1. **镜像版本管理**:controller和task-executor镜像版本保持一致
2. **资源限制**:task-executor通常需要更多CPU用于进程注入
3. **安全配置**:只在需要时启用`SYS_PTRACE`权限
4. **任务设计**:将长时间运行的任务拆分为多个短任务
5. **监控告警**:监控任务失败率和执行时间
# ==============================================================================
# Pool for kubectl-only scenarios(纯 kubectl 场景 Pool)
# ==============================================================================
#
# 用途:纯 kubectl 场景,不使用 SDK
#
# 适用场景:
# - RL 训练批量任务
# - 压力测试
# - 批量数据处理
# - 不需要 SDK 动态交互的场景
#
# 特点:
# - 没有 execd(不支持 SDK)
# - 包含 task-executor(支持 taskTemplate)
# - 配置简洁,资源占用少
#
# ==============================================================================
apiVersion: sandbox.opensandbox.io/v1alpha1
kind: Pool
metadata:
name: kubectl-pool
namespace: default
labels:
app: opensandbox
component: kubectl-pool
annotations:
description: "纯 kubectl 场景 Pool,不包含 execd"
spec:
template:
metadata:
labels:
pool: kubectl-pool
sdk-compatible: "false"
spec:
# ========================================
# 必需:共享进程命名空间(task-executor 需要)
# ========================================
shareProcessNamespace: true
# ========================================
# 主容器:Sandbox 环境
# ========================================
containers:
- name: sandbox-container
image: nginx:latest
imagePullPolicy: IfNotPresent
# 直接运行业务命令,不需要 bootstrap.sh
command: ["sleep", "infinity"]
# 资源配置
resources:
requests:
cpu: "100m"
memory: "128Mi"
limits:
cpu: "500m"
memory: "256Mi"
# ========================================
# Sidecar:Task Executor
# ========================================
- name: task-executor
image: opensandbox/task-executor:dev
imagePullPolicy: Never
ports:
- containerPort: 5758
name: task-executor
protocol: TCP
resources:
requests:
cpu: "100m"
memory: "128Mi"
limits:
cpu: "500m"
memory: "256Mi"
# 安全上下文
securityContext:
capabilities:
add: ["SYS_PTRACE"]
# ========================================
# Pool 容量配置
# ========================================
capacitySpec:
bufferMin: 2
bufferMax: 5
poolMin: 2
poolMax: 10
apiVersion: sandbox.opensandbox.io/v1alpha1
kind: Pool
metadata:
name: sdk-pool
namespace: default
spec:
template:
spec:
# Init container: 安装execd
initContainers:
- name: execd-installer
image: opensandbox/execd:v1.0.21
command: ["/bin/sh", "-c"]
args:
- |
cp ./execd /opt/opensandbox/bin/execd && \
cp ./bootstrap.sh /opt/opensandbox/bin/bootstrap.sh && \
chmod +x /opt/opensandbox/bin/execd && \
chmod +x /opt/opensandbox/bin/bootstrap.sh
volumeMounts:
- name: opensandbox-bin
mountPath: /opt/opensandbox/bin
# 主容器:带execd
containers:
- name: sandbox-container
image: nginx:latest
command:
- /opt/opensandbox/bin/bootstrap.sh
- nginx
- -g
- daemon off; # nginx前台运行
env:
- name: EXECD
value: /opt/opensandbox/bin/execd
ports:
- containerPort: 80
name: http
- containerPort: 44772
name: execd
protocol: TCP
resources:
requests:
cpu: "100m"
memory: "128Mi"
limits:
cpu: "500m"
memory: "256Mi"
volumeMounts:
- name: opensandbox-bin
mountPath: /opt/opensandbox/bin
# 共享卷
volumes:
- name: opensandbox-bin
emptyDir: {}
capacitySpec:
bufferMax: 10
bufferMin: 2
poolMax: 20
poolMin: 5
apiVersion: sandbox.opensandbox.io/v1alpha1
kind: Pool
metadata:
name: sdk-pool-with-tasks
namespace: default
spec:
template:
spec:
shareProcessNamespace: true # task-executor需要
# Init container: 安装execd
initContainers:
- name: execd-installer
image: opensandbox/execd:v1.0.21
command: ["/bin/sh", "-c"]
args:
- |
cp ./execd /opt/opensandbox/bin/execd && \
cp ./bootstrap.sh /opt/opensandbox/bin/bootstrap.sh && \
chmod +x /opt/opensandbox/bin/execd && \
chmod +x /opt/opensandbox/bin/bootstrap.sh
volumeMounts:
- name: opensandbox-bin
mountPath: /opt/opensandbox/bin
containers:
# 主容器:带execd
- name: sandbox-container
image: nginx:latest
command:
- /opt/opensandbox/bin/bootstrap.sh
- sleep
- infinity
env:
- name: EXECD
value: /opt/opensandbox/bin/execd
ports:
- containerPort: 44772
name: execd
protocol: TCP
resources:
requests:
cpu: "100m"
memory: "128Mi"
limits:
cpu: "500m"
memory: "256Mi"
volumeMounts:
- name: opensandbox-bin
mountPath: /opt/opensandbox/bin
# task-executor sidecar: 支持自定义entrypoint
- name: task-executor
image: opensandbox/task-executor:dev
imagePullPolicy: Never # 使用本地镜像
securityContext:
capabilities:
add: ["SYS_PTRACE"]
resources:
requests:
cpu: "100m"
memory: "128Mi"
limits:
cpu: "500m"
memory: "256Mi"
volumes:
- name: opensandbox-bin
emptyDir: {}
capacitySpec:
bufferMax: 10
bufferMin: 2
poolMax: 20
poolMin: 5
# OpenSandbox Helm Chart Scripts
This directory contains utility scripts for OpenSandbox Controller deployment and testing.
## Script List
### 1. install.sh - Installation Script
Interactive installation of OpenSandbox Controller to Kubernetes cluster.
**Features:**
- Automatic detection of sudo privilege requirements
- Validation of dependency tools (helm, kubectl)
- Cluster connection verification
- Support for multiple deployment environments:
- Default configuration (values.yaml)
- E2E testing (values-e2e.yaml)
- Custom configuration (via --set or custom values file)
- Helm Chart validation
- Display verification commands after deployment
**Usage:**
```bash
cd scripts
./install.sh
```
**Environment Variables:**
- `IMAGE_REPO` - Override controller image repository
- `IMAGE_TAG` - Override controller image tag
- `SERVER_IMAGE_REPO` - Override server image repository
- `SERVER_IMAGE_TAG` - Override server image tag
**Example:**
```bash
# Using custom images
IMAGE_REPO=myregistry.com/controller \
IMAGE_TAG=v1.0.0 \
SERVER_IMAGE_REPO=myregistry.com/server \
SERVER_IMAGE_TAG=v0.2.1 \
./install.sh
```
### 2. uninstall.sh - Uninstallation Script
Uninstall OpenSandbox Controller and clean up related resources.
**Features:**
- Check running BatchSandbox and Pool resources
- Display Controller and Server deployment status
- Optional CRD deletion
- Optional namespace deletion
- Post-uninstall cleanup verification
**Usage:**
```bash
cd scripts
./uninstall.sh
```
**Environment Variables:**
- `RELEASE_NAME` - Release name (default: opensandbox-controller)
- `NAMESPACE` - Namespace (default: opensandbox)
**Example:**
```bash
# Uninstall specific release
RELEASE_NAME=my-release NAMESPACE=my-namespace ./uninstall.sh
```
### 3. e2e-test.sh - End-to-End Test Script
Execute complete end-to-end test workflow.
**Test Workflow:**
1. Helm Install (using values-e2e.yaml)
2. Verify Controller and Server deployment
3. Verify Pool deployment
4. Verify SDK calls
5. Helm Uninstall
**Features:**
- Automatic Server port-forward setup
- Server API health check validation
- Pool Pod execd process verification
- SDK integration test execution
- Automatic resource cleanup (including port-forward processes)
**Usage:**
```bash
cd scripts
./e2e-test.sh [VALUES_FILE]
# Using default values-e2e.yaml
./e2e-test.sh
# Using custom values file
./e2e-test.sh custom-values.yaml
```
**Prerequisites:**
- Required Docker images must be loaded:
- opensandbox/controller:dev
- opensandbox/server:v0.2.1
- opensandbox/task-executor:dev
- opensandbox/execd:v1.0.21
- nginx:latest
- Python SDK installed (using uv)
- Cluster has sufficient resources to run test Pods
## General Instructions
### Sudo Privileges
All scripts automatically detect whether sudo privileges are required to execute kubectl and helm commands.
### Script Paths
Scripts use relative paths to locate the Chart directory and can be invoked from any location:
```bash
# From chart root directory
./scripts/install.sh
# From scripts directory
cd scripts
./install.sh
# From other directory
/path/to/opensandbox-controller/scripts/install.sh
```
### Colored Output
Scripts use ANSI color codes to enhance readability:
- 🟢 Green - Success messages
- 🟡 Yellow - Warnings and step titles
- 🔴 Red - Error messages
### Error Handling
All scripts use `set -e`, exiting immediately on errors. The e2e-test.sh uses trap to ensure cleanup functions execute on exit.
## Troubleshooting
### install.sh
**Issue: Cannot connect to Kubernetes cluster**
```bash
# Check kubeconfig
kubectl cluster-info
# Check context
kubectl config current-context
```
**Issue: Chart validation fails**
```bash
# Manual validation
helm lint ../
```
### uninstall.sh
**Issue: Resources are still running**
```bash
# View all BatchSandbox
kubectl get batchsandboxes -A
# View all Pool
kubectl get pools -A
# Delete all resources
kubectl delete batchsandboxes --all -A
kubectl delete pools --all -A
```
### e2e-test.sh
**Issue: Port-forward fails**
```bash
# Check if any process is using port 8080
lsof -i :8080
# Manual port-forward test
kubectl port-forward -n opensandbox svc/opensandbox-controller-server 8080:8080
```
**Issue: SDK test fails**
```bash
# Check Server logs
kubectl logs -n opensandbox -l app.kubernetes.io/component=server
# Check Pool Pod logs
kubectl logs -n opensandbox -l pool=agent-pool
# Test Server API
curl http://localhost:8080/health
```
**Issue: Image not found**
```bash
# Check if images are loaded
docker images | grep opensandbox
# Reload images
docker load -i /path/to/image.tar
```
## Development Guide
### Modifying Scripts
After modifying scripts, ensure:
1. Maintain consistent error handling
2. Update related documentation
3. Use meaningful colored output
4. Add appropriate validation steps
### Adding New Scripts
New scripts should follow these conventions:
- Use `#!/bin/bash` shebang
- Use `set -e` to enable exit-on-error
- Implement automatic sudo detection
- Add colored output for readability
- Add documentation in this README
## License
Apache License 2.0
#!/bin/bash
# OpenSandbox Helm Chart End-to-End Test Script
set -x
set -e
# Get the parent directory of the script's directory (chart root directory)
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
CHART_DIR="$(dirname "$SCRIPT_DIR")"
NAMESPACE="opensandbox"
RELEASE_NAME="opensandbox-e2e-test"
VALUES_FILE="${1:-values-e2e.yaml}"
# Cleanup function: ensure temporary resources are cleaned up
cleanup() {
# Clean up any test resources that might be left over
if command -v kubectl &> /dev/null || command -v minikube &> /dev/null; then
kubectl delete configmap sdk-test-script -n "$NAMESPACE" --ignore-not-found=true > /dev/null 2>&1 || true
kubectl delete job sdk-test-job -n "$NAMESPACE" --ignore-not-found=true > /dev/null 2>&1 || true
kubectl delete pod server-health-check -n "$NAMESPACE" --ignore-not-found=true > /dev/null 2>&1 || true
fi
}
# Register cleanup function to ensure execution on script exit
trap cleanup EXIT INT TERM
# Check kubectl availability and detect minikube
USE_SUDO=false
USE_MINIKUBE=false
# First check if kubectl is available directly
if kubectl get nodes &> /dev/null 2>&1; then
echo "Detected kubectl access to Kubernetes cluster"
elif sudo kubectl get nodes &> /dev/null 2>&1; then
echo "Detected sudo privileges required, will use sudo for commands"
USE_SUDO=true
# If kubectl is not available, check for minikube
elif command -v minikube &> /dev/null && minikube kubectl -- get nodes &> /dev/null 2>&1; then
echo "Detected minikube cluster, will use 'minikube kubectl --' for commands"
USE_MINIKUBE=true
elif command -v minikube &> /dev/null && sudo minikube kubectl -- get nodes &> /dev/null 2>&1; then
echo "Detected minikube cluster with sudo, will use 'sudo minikube kubectl --' for commands"
USE_MINIKUBE=true
USE_SUDO=true
else
echo "Error: Unable to access Kubernetes cluster"
echo "Please ensure kubectl or minikube is properly configured"
exit 1
fi
# Define command functions
kubectl_cmd() {
if [ "$USE_MINIKUBE" = true ]; then
if [ "$USE_SUDO" = true ]; then
sudo minikube kubectl -- "$@"
else
minikube kubectl -- "$@"
fi
else
if [ "$USE_SUDO" = true ]; then
sudo kubectl "$@"
else
kubectl "$@"
fi
fi
}
helm_cmd() {
if [ "$USE_SUDO" = true ]; then
sudo helm "$@"
else
helm "$@"
fi
}
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
RED='\033[0;31m'
NC='\033[0m'
echo -e "${GREEN}==========================================${NC}"
echo -e "${GREEN}OpenSandbox Helm Chart E2E Validation${NC}"
echo -e "${GREEN}==========================================${NC}"
echo ""
echo "Test Coverage:"
echo " 1. Helm Install (using ${VALUES_FILE})"
echo " 2. Server Deployment Verification"
echo " 3. Pool Deployment Verification"
echo " 4. SDK Integration Verification"
echo " 5. Helm Uninstall"
echo ""
echo "Environment Info:"
echo " Chart: ${CHART_DIR}"
echo " Values: ${VALUES_FILE}"
echo " Release: ${RELEASE_NAME}"
echo " Namespace: ${NAMESPACE}"
echo ""
# ==========================================
# Stage 1: Helm Install
# ==========================================
echo -e "${GREEN}==========================================${NC}"
echo -e "${GREEN}Stage 1: Helm Install${NC}"
echo -e "${GREEN}==========================================${NC}"
echo ""
echo -e "${YELLOW}[1.1] Checking for existing release...${NC}"
if helm_cmd list -n "$NAMESPACE" 2>/dev/null | grep -q "$RELEASE_NAME"; then
echo " Release already exists, uninstalling first..."
helm_cmd uninstall "$RELEASE_NAME" -n "$NAMESPACE" 2>/dev/null || true
sleep 5
fi
echo -e "${GREEN}✓ Check completed${NC}"
echo ""
echo -e "${YELLOW}[1.2] Installing Helm chart (using ${VALUES_FILE})...${NC}"
helm_cmd install "$RELEASE_NAME" "$CHART_DIR" \
--values "$CHART_DIR/$VALUES_FILE" \
--namespace "$NAMESPACE" \
--create-namespace \
--wait \
--timeout 3m 2>&1 | tail -5
echo -e "${GREEN}✓ Helm chart installed successfully${NC}"
echo ""
echo -e "${YELLOW}[1.3] Waiting for Controller to be ready...${NC}"
kubectl_cmd wait --for=condition=available \
deployment/opensandbox-controller-manager \
-n "$NAMESPACE" \
--timeout=120s 2>/dev/null
echo -e "${GREEN}✓ Controller is ready${NC}"
echo ""
echo -e "${YELLOW}[1.4] Checking deployment status...${NC}"
kubectl_cmd get deployment -n "$NAMESPACE"
echo ""
echo -e "${GREEN}✅ Stage 1 Complete: Helm Install Successful${NC}"
echo ""
# ==========================================
# Stage 2: Server Deployment Verification
# ==========================================
echo -e "${GREEN}==========================================${NC}"
echo -e "${GREEN}Stage 2: Server Deployment Verification${NC}"
echo -e "${GREEN}==========================================${NC}"
echo ""
echo -e "${YELLOW}[2.1] Checking Server Service...${NC}"
SERVER_SERVICE_NAME=$(kubectl_cmd get svc -n "$NAMESPACE" -l app.kubernetes.io/component=server -o jsonpath='{.items[0].metadata.name}' 2>/dev/null)
if [ -z "$SERVER_SERVICE_NAME" ]; then
echo -e "${RED}❌ Server Service does not exist${NC}"
exit 1
fi
echo " Server Service: ${SERVER_SERVICE_NAME}"
kubectl_cmd get svc "$SERVER_SERVICE_NAME" -n "$NAMESPACE"
echo ""
echo -e "${YELLOW}[2.2] Waiting for Server Pod to be ready...${NC}"
SERVER_DEPLOYMENT_NAME=$(kubectl_cmd get deployment -n "$NAMESPACE" -l app.kubernetes.io/component=server -o jsonpath='{.items[0].metadata.name}' 2>/dev/null)
if [ -z "$SERVER_DEPLOYMENT_NAME" ]; then
echo -e "${RED}❌ Server Deployment does not exist${NC}"
exit 1
fi
echo " Server Deployment: ${SERVER_DEPLOYMENT_NAME}"
kubectl_cmd wait --for=condition=available \
deployment/"$SERVER_DEPLOYMENT_NAME" \
-n "$NAMESPACE" \
--timeout=120s 2>/dev/null
echo -e "${GREEN}✓ Server Deployment is ready${NC}"
echo ""
echo -e "${YELLOW}[2.3] Checking Server Pod status...${NC}"
kubectl_cmd get pods -n "$NAMESPACE" -l app.kubernetes.io/component=server
echo ""
echo -e "${YELLOW}[2.4] Testing Server API (from within cluster)...${NC}"
# Create a simple test pod to check server health from inside the cluster
cat <<EOF | kubectl_cmd apply -f - > /dev/null
apiVersion: v1
kind: Pod
metadata:
name: server-health-check
namespace: $NAMESPACE
spec:
restartPolicy: Never
containers:
- name: curl
image: curlimages/curl:latest
command: ['curl', '-s', 'http://${SERVER_SERVICE_NAME}:8080/health']
EOF
# Wait for pod to complete
kubectl_cmd wait --for=condition=Ready pod/server-health-check -n "$NAMESPACE" --timeout=30s 2>/dev/null || true
sleep 2
HEALTH_RESPONSE=$(kubectl_cmd logs server-health-check -n "$NAMESPACE" 2>/dev/null || echo "")
kubectl_cmd delete pod server-health-check -n "$NAMESPACE" --ignore-not-found=true > /dev/null 2>&1
if [ -n "$HEALTH_RESPONSE" ]; then
echo -e "${GREEN}✓ Server API responding normally: $HEALTH_RESPONSE${NC}"
else
echo -e "${RED}❌ Server API not responding${NC}"
exit 1
fi
echo ""
echo -e "${GREEN}✅ Stage 2 Complete: Server Deployment Verified${NC}"
echo ""
# ==========================================
# Stage 3: Pool Deployment Verification
# ==========================================
echo -e "${GREEN}==========================================${NC}"
echo -e "${GREEN}Stage 3: Pool Deployment Verification${NC}"
echo -e "${GREEN}==========================================${NC}"
echo ""
echo -e "${YELLOW}[3.1] Checking Pool resources...${NC}"
POOL_COUNT=$(kubectl_cmd get pool -n "$NAMESPACE" --no-headers 2>/dev/null | wc -l)
echo " Pool count: ${POOL_COUNT}"
if [ "$POOL_COUNT" -eq 0 ]; then
echo -e "${RED}❌ No Pool resources found${NC}"
exit 1
fi
kubectl_cmd get pool -n "$NAMESPACE"
echo ""
echo -e "${YELLOW}[3.2] Checking agent-pool status...${NC}"
if ! kubectl_cmd get pool agent-pool -n "$NAMESPACE" --no-headers 2>/dev/null | grep -q agent-pool; then
echo -e "${RED}❌ agent-pool does not exist${NC}"
exit 1
fi
echo -e "${GREEN}✓ agent-pool exists${NC}"
echo ""
echo -e "${YELLOW}[3.3] Viewing Pool detailed status...${NC}"
kubectl_cmd get pool agent-pool -n "$NAMESPACE" -o jsonpath='{.status}' 2>/dev/null | jq '.' 2>/dev/null || echo " (jq not installed, skipping JSON formatting)"
echo ""
echo -e "${YELLOW}[3.4] Waiting for Pool Pods to be ready (up to 180 seconds)...${NC}"
TIMEOUT=180
ELAPSED=0
READY=false
while [ $ELAPSED -lt $TIMEOUT ]; do
AVAILABLE=$(kubectl_cmd get pool agent-pool -n "$NAMESPACE" -o jsonpath='{.status.available}' 2>/dev/null || echo "")
if [ -n "$AVAILABLE" ] && [ "$AVAILABLE" -gt 0 ]; then
echo -e "${GREEN}✓ Pool has ${AVAILABLE} available Pods${NC}"
READY=true
break
fi
echo " Waiting... (${ELAPSED}s/${TIMEOUT}s)"
sleep 5
ELAPSED=$((ELAPSED + 5))
done
if [ "$READY" = false ]; then
echo -e "${RED}❌ Pool Pods not ready, timed out${NC}"
echo ""
echo "Viewing Pool events:"
kubectl_cmd describe pool agent-pool -n "$NAMESPACE" | tail -20
exit 1
fi
echo ""
echo -e "${YELLOW}[3.5] Viewing Pool Pods...${NC}"
kubectl_cmd get pods -l pool=agent-pool -n "$NAMESPACE"
echo ""
echo -e "${YELLOW}[3.6] Checking execd process in Pod...${NC}"
POD_NAME=$(kubectl_cmd get pods -l pool=agent-pool -n "$NAMESPACE" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null)
if [ -n "$POD_NAME" ]; then
echo " Checking Pod: ${POD_NAME}"
sleep 3
if kubectl_cmd exec -n "$NAMESPACE" "$POD_NAME" -c sandbox-container -- pgrep -f execd > /dev/null 2>&1; then
EXECD_PID=$(kubectl_cmd exec -n "$NAMESPACE" "$POD_NAME" -c sandbox-container -- pgrep -f execd 2>/dev/null)
echo -e "${GREEN}✓ execd process is running (PID: ${EXECD_PID})${NC}"
else
echo -e "${YELLOW}⚠️ execd process not found, checking container logs...${NC}"
kubectl_cmd logs -n "$NAMESPACE" "$POD_NAME" -c sandbox-container --tail=20 2>/dev/null || true
fi
else
echo -e "${YELLOW}⚠️ No agent-pool Pod found${NC}"
fi
echo ""
echo -e "${GREEN}✅ Stage 3 Complete: Pool Deployment Verified${NC}"
echo ""
# ==========================================
# Stage 4: SDK Integration Verification
# ==========================================
echo -e "${GREEN}==========================================${NC}"
echo -e "${GREEN}Stage 4: SDK Integration Verification${NC}"
echo -e "${GREEN}==========================================${NC}"
echo ""
echo -e "${YELLOW}[4.1] Creating SDK test script ConfigMap...${NC}"
SDK_TEST_SCRIPT=$(cat <<'EOF'
import asyncio
from datetime import timedelta
from opensandbox import Sandbox
from opensandbox.config import ConnectionConfig
async def main():
print("=" * 60)
print("SDK End-to-End Test")
print("=" * 60)
# Use internal service name for cluster-internal communication
config = ConnectionConfig(domain="SERVER_SERVICE_PLACEHOLDER:8080")
print("\n[Test 1] Creating sandbox (using agent-pool)...")
try:
sandbox = await Sandbox.create(
"nginx:latest",
entrypoint=["/bin/sh", "-c", "sleep infinity"],
env={"TEST": "e2e"},
timeout=timedelta(minutes=10),
ready_timeout=timedelta(minutes=5),
connection_config=config,
extensions={"poolRef": "agent-pool"}
)
print(f"✅ Sandbox created successfully: {sandbox.id}")
except Exception as e:
print(f"❌ Sandbox creation failed: {e}")
import traceback
traceback.print_exc()
return False
try:
print("\n[Test 2] Executing command...")
execution = await sandbox.commands.run("echo 'Hello from E2E test'")
if execution.logs.stdout:
print(f"✅ Command executed successfully: {execution.logs.stdout[0].text}")
else:
print("⚠️ Command executed successfully but no output")
print("\n[Test 3] File operations...")
from opensandbox.models import WriteEntry
await sandbox.files.write_files([
WriteEntry(path="/tmp/e2e.txt", data="E2E Test", mode=644)
])
print("✅ File written successfully")
content = await sandbox.files.read_file("/tmp/e2e.txt")
print(f"✅ File read successfully: {content}")
print("\n[Test 4] Cleaning up sandbox...")
await sandbox.kill()
print("✅ Sandbox cleaned up successfully")
print("\n" + "=" * 60)
print("✅ All SDK end-to-end tests passed!")
print("=" * 60)
return True
except Exception as e:
print(f"❌ Test failed: {e}")
import traceback
traceback.print_exc()
try:
await sandbox.kill()
except:
pass
return False
if __name__ == "__main__":
success = asyncio.run(main())
exit(0 if success else 1)
EOF
)
# Replace placeholder with actual service name
SDK_TEST_SCRIPT="${SDK_TEST_SCRIPT//SERVER_SERVICE_PLACEHOLDER/${SERVER_SERVICE_NAME}}"
# Create ConfigMap with test script
cat <<EOF | kubectl_cmd apply -f - > /dev/null
apiVersion: v1
kind: ConfigMap
metadata:
name: sdk-test-script
namespace: $NAMESPACE
data:
test.py: |
$(echo "$SDK_TEST_SCRIPT" | sed 's/^/ /')
EOF
echo -e "${GREEN}✓ SDK test script ConfigMap created${NC}"
echo ""
echo -e "${YELLOW}[4.2] Creating SDK test Job...${NC}"
cat <<EOF | kubectl_cmd apply -f - > /dev/null
apiVersion: batch/v1
kind: Job
metadata:
name: sdk-test-job
namespace: $NAMESPACE
spec:
ttlSecondsAfterFinished: 300
backoffLimit: 0
template:
spec:
restartPolicy: Never
containers:
- name: sdk-test
image: ghcr.io/astral-sh/uv:python3.12-bookworm
command:
- /bin/bash
- -c
- |
set -e
echo "Installing opensandbox SDK..."
uv pip install --system opensandbox
echo ""
echo "Running SDK tests..."
python /test/test.py
volumeMounts:
- name: test-script
mountPath: /test
volumes:
- name: test-script
configMap:
name: sdk-test-script
EOF
echo -e "${GREEN}✓ SDK test Job created${NC}"
echo ""
echo -e "${YELLOW}[4.3] Waiting for SDK test Job to complete (timeout: 5 minutes)...${NC}"
# Wait for job to start
sleep 5
# Get pod name
SDK_TEST_POD=""
for i in $(seq 1 30); do
SDK_TEST_POD=$(kubectl_cmd get pods -n "$NAMESPACE" -l job-name=sdk-test-job -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || echo "")
if [ -n "$SDK_TEST_POD" ]; then
echo " SDK test pod: $SDK_TEST_POD"
break
fi
sleep 2
done
if [ -z "$SDK_TEST_POD" ]; then
echo -e "${RED}❌ SDK test pod not found${NC}"
kubectl_cmd get jobs -n "$NAMESPACE" sdk-test-job
exit 1
fi
# Stream logs and wait for completion
echo ""
echo " Streaming test logs:"
echo " ---"
kubectl_cmd logs -f "$SDK_TEST_POD" -n "$NAMESPACE" 2>/dev/null || true
echo " ---"
echo ""
# Wait a bit for Job to update status after pod completion
echo " Waiting for Job status to update..."
sleep 5
# Wait for job to complete (up to 60 seconds)
for i in $(seq 1 60); do
JOB_STATUS=$(kubectl_cmd get job sdk-test-job -n "$NAMESPACE" -o jsonpath='{.status.conditions[?(@.type=="Complete")].status}' 2>/dev/null || echo "")
JOB_FAILED=$(kubectl_cmd get job sdk-test-job -n "$NAMESPACE" -o jsonpath='{.status.conditions[?(@.type=="Failed")].status}' 2>/dev/null || echo "")
if [ "$JOB_STATUS" = "True" ] || [ "$JOB_FAILED" = "True" ]; then
break
fi
sleep 1
done
# Check job status
JOB_STATUS=$(kubectl_cmd get job sdk-test-job -n "$NAMESPACE" -o jsonpath='{.status.conditions[?(@.type=="Complete")].status}' 2>/dev/null || echo "")
JOB_FAILED=$(kubectl_cmd get job sdk-test-job -n "$NAMESPACE" -o jsonpath='{.status.conditions[?(@.type=="Failed")].status}' 2>/dev/null || echo "")
if [ "$JOB_STATUS" = "True" ]; then
echo -e "${GREEN}✅ Stage 4 Complete: SDK Integration Verified${NC}"
elif [ "$JOB_FAILED" = "True" ]; then
echo -e "${RED}❌ Stage 4 Failed: SDK Integration Failed${NC}"
echo ""
echo -e "${YELLOW}Diagnostic Information:${NC}"
kubectl_cmd describe job sdk-test-job -n "$NAMESPACE" | tail -20
exit 1
else
echo -e "${RED}❌ Stage 4 Failed: SDK test job did not complete${NC}"
kubectl_cmd get job sdk-test-job -n "$NAMESPACE"
exit 1
fi
echo ""
# ==========================================
# Stage 5: Helm Uninstall
# ==========================================
echo -e "${GREEN}==========================================${NC}"
echo -e "${GREEN}Stage 5: Helm Uninstall${NC}"
echo -e "${GREEN}==========================================${NC}"
echo ""
echo -e "${YELLOW}[5.1] Cleaning up test resources...${NC}"
kubectl_cmd delete configmap sdk-test-script -n "$NAMESPACE" --ignore-not-found=true > /dev/null 2>&1
kubectl_cmd delete job sdk-test-job -n "$NAMESPACE" --ignore-not-found=true > /dev/null 2>&1
echo -e "${GREEN}✓ Test resources cleaned up${NC}"
echo ""
echo -e "${YELLOW}[5.2] Uninstalling Helm release...${NC}"
helm_cmd uninstall "$RELEASE_NAME" -n "$NAMESPACE"
echo -e "${GREEN}✓ Helm release uninstalled${NC}"
echo ""
echo "Waiting for resource cleanup..."
sleep 10
echo -e "${YELLOW}[5.3] Verifying resources are cleaned up...${NC}"
REMAINING_PODS=$(kubectl_cmd get pods -n "$NAMESPACE" --no-headers 2>/dev/null | wc -l)
REMAINING_POOLS=$(kubectl_cmd get pools -n "$NAMESPACE" --no-headers 2>/dev/null | wc -l)
echo " Remaining Pods: ${REMAINING_PODS}"
echo " Remaining Pools: ${REMAINING_POOLS}"
if [ "$REMAINING_PODS" -eq 0 ] && [ "$REMAINING_POOLS" -eq 0 ]; then
echo -e "${GREEN}✓ All resources cleaned up${NC}"
else
echo -e "${YELLOW}⚠️ Resources still remaining (Terminating)${NC}"
if [ "$REMAINING_PODS" -gt 0 ]; then
kubectl_cmd get pods -n "$NAMESPACE" 2>/dev/null || true
fi
fi
echo ""
echo -e "${GREEN}✅ Stage 5 Complete: Helm Uninstall Successful${NC}"
echo ""
# ==========================================
# Test Summary
# ==========================================
echo -e "${GREEN}==========================================${NC}"
echo -e "${GREEN}End-to-End Test Complete!${NC}"
echo -e "${GREEN}==========================================${NC}"
echo ""
echo "Test Results:"
echo -e " ${GREEN}✅ Stage 1: Helm Install - Success${NC}"
echo -e " ${GREEN}✅ Stage 2: Server Deployment Verification - Success${NC}"
echo -e " ${GREEN}✅ Stage 3: Pool Deployment Verification - Success${NC}"
echo -e " ${GREEN}✅ Stage 4: SDK Integration Verification - Success${NC}"
echo -e " ${GREEN}✅ Stage 5: Helm Uninstall - Success${NC}"
echo ""
echo -e "${GREEN}🎉 All tests passed!${NC}"
echo ""
#!/bin/bash
# OpenSandbox Controller Deployment Script
set -e
# Check if sudo is required
USE_SUDO=false
if ! kubectl get nodes &> /dev/null 2>&1; then
if sudo kubectl get nodes &> /dev/null 2>&1; then
echo "Detected that sudo permissions are required, will use sudo to execute commands"
USE_SUDO=true
else
echo "Error: Unable to access Kubernetes cluster"
exit 1
fi
fi
# Define command functions
kubectl_cmd() {
if [ "$USE_SUDO" = true ]; then
sudo kubectl "$@"
else
kubectl "$@"
fi
}
helm_cmd() {
if [ "$USE_SUDO" = true ]; then
sudo helm "$@"
else
helm "$@"
fi
}
# Color output
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
RED='\033[0;31m'
NC='\033[0m' # No Color
echo -e "${GREEN}======================================${NC}"
echo -e "${GREEN}OpenSandbox Controller Helm Deployment${NC}"
echo -e "${GREEN}======================================${NC}"
echo ""
# Check dependencies
echo -e "${YELLOW}[1/6] Checking dependencies...${NC}"
if ! command -v helm &> /dev/null; then
echo -e "${RED}Error: helm command not found${NC}"
echo "Please install Helm 3.0+: https://helm.sh/docs/intro/install/"
exit 1
fi
if ! command -v kubectl &> /dev/null; then
echo -e "${RED}Error: kubectl command not found${NC}"
echo "Please install kubectl: https://kubernetes.io/docs/tasks/tools/"
exit 1
fi
echo -e "${GREEN}✓ Helm version: $(helm version --short)${NC}"
echo -e "${GREEN}✓ Kubectl version: $(kubectl version --client --short 2>/dev/null || kubectl version --client)${NC}"
echo ""
# Check cluster connection
echo -e "${YELLOW}[2/6] Checking Kubernetes cluster connection...${NC}"
if ! kubectl_cmd cluster-info &> /dev/null; then
echo -e "${RED}Error: Unable to connect to Kubernetes cluster${NC}"
echo "Please check your ~/.kube/config configuration"
exit 1
fi
echo -e "${GREEN}✓ Cluster connection successful${NC}"
kubectl_cmd cluster-info | head -2
echo ""
# Configuration parameters
# Get the parent directory of the script directory (chart root directory)
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
CHART_DIR="$(dirname "$SCRIPT_DIR")"
RELEASE_NAME="opensandbox-controller"
NAMESPACE="opensandbox"
echo -e "${YELLOW}[3/6] Configuration parameters${NC}"
echo "Chart directory: $CHART_DIR"
echo "Release name: $RELEASE_NAME"
echo "Namespace: $NAMESPACE"
if [ -n "$IMAGE_REPO" ] || [ -n "$IMAGE_TAG" ]; then
echo "Image override: ${IMAGE_REPO:-<from values>}:${IMAGE_TAG:-<from values>}"
else
echo "Image configuration: Using configuration from values file"
fi
echo ""
# Select deployment environment
echo -e "${YELLOW}[4/6] Select deployment environment${NC}"
echo "1) Default configuration (values.yaml)"
echo "2) End-to-end testing (values-e2e.yaml)"
echo "3) Custom (custom values)"
read -p "Please select [1-3]: " env_choice
case $env_choice in
1)
VALUES_FILE="$CHART_DIR/values.yaml"
echo -e "${GREEN}✓ Using default configuration${NC}"
;;
2)
VALUES_FILE="$CHART_DIR/values-e2e.yaml"
echo -e "${GREEN}✓ E2E test configuration selected${NC}"
;;
3)
read -p "Please enter values file path: " custom_values
VALUES_FILE="$custom_values"
echo -e "${GREEN}✓ Using custom configuration: $VALUES_FILE${NC}"
;;
*)
echo -e "${RED}Invalid selection, using default configuration${NC}"
VALUES_FILE="$CHART_DIR/values.yaml"
;;
esac
echo ""
# Validate Chart
echo -e "${YELLOW}[5/6] Validating Helm Chart...${NC}"
if ! helm lint "$CHART_DIR" &> /dev/null; then
echo -e "${RED}Error: Chart validation failed${NC}"
helm lint "$CHART_DIR"
exit 1
fi
echo -e "${GREEN}✓ Chart validation passed${NC}"
echo ""
# Confirm deployment
echo -e "${YELLOW}[6/6] Preparing for deployment${NC}"
echo "The following operations will be performed:"
echo " - Create namespace: $NAMESPACE"
echo " - Install CRDs: BatchSandbox, Pool"
echo " - Deploy Controller Manager"
echo " - Deploy Server (FastAPI control plane)"
echo " - Deploy RBAC resources"
echo " - Deploy Metrics service"
echo " - Create default Pool (agent-pool)"
echo ""
read -p "Confirm deployment? [y/N]: " confirm
if [[ ! $confirm =~ ^[Yy]$ ]]; then
echo -e "${YELLOW}Deployment cancelled${NC}"
exit 0
fi
# Execute deployment
echo ""
echo -e "${GREEN}Starting deployment...${NC}"
echo ""
# If environment variables are set, override image configuration in values file
EXTRA_ARGS=""
if [ -n "$IMAGE_REPO" ]; then
EXTRA_ARGS="$EXTRA_ARGS --set controllerManager.image.repository=$IMAGE_REPO"
fi
if [ -n "$IMAGE_TAG" ]; then
EXTRA_ARGS="$EXTRA_ARGS --set controllerManager.image.tag=$IMAGE_TAG"
fi
if [ -n "$SERVER_IMAGE_REPO" ]; then
EXTRA_ARGS="$EXTRA_ARGS --set server.image.repository=$SERVER_IMAGE_REPO"
fi
if [ -n "$SERVER_IMAGE_TAG" ]; then
EXTRA_ARGS="$EXTRA_ARGS --set server.image.tag=$SERVER_IMAGE_TAG"
fi
helm_cmd upgrade --install "$RELEASE_NAME" "$CHART_DIR" \
--namespace "$NAMESPACE" \
--create-namespace \
-f "$VALUES_FILE" \
$EXTRA_ARGS \
--wait \
--timeout 5m
echo ""
echo -e "${GREEN}======================================${NC}"
echo -e "${GREEN}✓ Deployment completed!${NC}"
echo -e "${GREEN}======================================${NC}"
echo ""
# Display deployment information
echo -e "${YELLOW}Deployment information:${NC}"
helm_cmd status "$RELEASE_NAME" -n "$NAMESPACE"
echo ""
echo -e "${YELLOW}Verify deployment:${NC}"
echo "1. Check Pod status:"
echo " $([ "$USE_SUDO" = true ] && echo "sudo ")kubectl get pods -n $NAMESPACE"
echo ""
echo "2. View Controller logs:"
echo " $([ "$USE_SUDO" = true ] && echo "sudo ")kubectl logs -n $NAMESPACE -l control-plane=controller-manager -f"
echo ""
echo "3. View Server logs:"
echo " $([ "$USE_SUDO" = true ] && echo "sudo ")kubectl logs -n $NAMESPACE -l app.kubernetes.io/component=server -f"
echo ""
echo "4. Access Server API (Port Forward):"
echo " $([ "$USE_SUDO" = true ] && echo "sudo ")kubectl port-forward -n $NAMESPACE svc/$RELEASE_NAME-server 8080:8080"
echo " curl http://localhost:8080/health"
echo ""
echo "5. View CRDs:"
echo " $([ "$USE_SUDO" = true ] && echo "sudo ")kubectl get crds | grep sandbox.opensandbox.io"
echo ""
echo "6. Check Pool status:"
echo " $([ "$USE_SUDO" = true ] && echo "sudo ")kubectl get pools -n $NAMESPACE"
echo ""
echo -e "${GREEN}Thank you for using OpenSandbox Controller!${NC}"
#!/bin/bash
# OpenSandbox Controller Uninstall Script
set -e
# Check if sudo is required
USE_SUDO=false
if ! kubectl get nodes &> /dev/null 2>&1; then
if sudo kubectl get nodes &> /dev/null 2>&1; then
echo "Detected sudo privileges required, will use sudo to execute commands"
USE_SUDO=true
else
echo "Error: Unable to access Kubernetes cluster"
exit 1
fi
fi
# Define command functions
kubectl_cmd() {
if [ "$USE_SUDO" = true ]; then
sudo kubectl "$@"
else
kubectl "$@"
fi
}
helm_cmd() {
if [ "$USE_SUDO" = true ]; then
sudo helm "$@"
else
helm "$@"
fi
}
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
RED='\033[0;31m'
NC='\033[0m'
echo -e "${YELLOW}======================================${NC}"
echo -e "${YELLOW}OpenSandbox Controller Uninstall${NC}"
echo -e "${YELLOW}======================================${NC}"
echo ""
RELEASE_NAME="${RELEASE_NAME:-opensandbox-controller}"
NAMESPACE="${NAMESPACE:-opensandbox}"
# Check if already installed
if ! helm_cmd list -n "$NAMESPACE" | grep -q "$RELEASE_NAME"; then
echo -e "${RED}Release not found: $RELEASE_NAME${NC}"
echo "Currently installed releases:"
helm_cmd list -A
exit 1
fi
echo "About to uninstall:"
echo " Release: $RELEASE_NAME"
echo " Namespace: $NAMESPACE"
echo ""
# Show current resources
echo -e "${YELLOW}Current resources:${NC}"
echo "Controller:"
kubectl_cmd get deployment -n "$NAMESPACE" -l control-plane=controller-manager 2>/dev/null || echo " Controller not found"
echo ""
echo "Server:"
kubectl_cmd get deployment -n "$NAMESPACE" -l app.kubernetes.io/component=server 2>/dev/null || echo " Server not found"
echo ""
# Check if there are running resources
echo -e "${YELLOW}Checking custom resources...${NC}"
BATCHSANDBOXES=$(kubectl_cmd get batchsandboxes -A --no-headers 2>/dev/null | wc -l)
POOLS=$(kubectl_cmd get pools -A --no-headers 2>/dev/null | wc -l)
if [ "$BATCHSANDBOXES" -gt 0 ] || [ "$POOLS" -gt 0 ]; then
echo -e "${RED}Warning: Running resources detected!${NC}"
echo " BatchSandboxes: $BATCHSANDBOXES"
echo " Pools: $POOLS"
echo ""
# Show Pool details
if [ "$POOLS" -gt 0 ]; then
echo "Pool details:"
kubectl_cmd get pools -A
echo ""
fi
echo "Recommended to delete these resources first:"
echo " $([ "$USE_SUDO" = true ] && echo "sudo ")kubectl delete batchsandboxes --all -A"
echo " $([ "$USE_SUDO" = true ] && echo "sudo ")kubectl delete pools --all -A"
echo ""
read -p "Continue with uninstall? [y/N]: " force_continue
if [[ ! $force_continue =~ ^[Yy]$ ]]; then
echo -e "${YELLOW}Uninstall cancelled${NC}"
exit 0
fi
fi
# Confirm uninstall
read -p "Confirm uninstall of $RELEASE_NAME (including Controller and Server)? [y/N]: " confirm
if [[ ! $confirm =~ ^[Yy]$ ]]; then
echo -e "${YELLOW}Uninstall cancelled${NC}"
exit 0
fi
echo ""
echo -e "${GREEN}Starting uninstall...${NC}"
# Uninstall Helm release
helm_cmd uninstall "$RELEASE_NAME" -n "$NAMESPACE"
echo -e "${GREEN}✓ Helm release uninstalled${NC}"
echo ""
# Wait for Pod termination
echo "Waiting for Pods to terminate..."
sleep 5
# Ask whether to delete CRDs
read -p "Delete CRDs? (This will delete all BatchSandbox and Pool resources) [y/N]: " delete_crds
if [[ $delete_crds =~ ^[Yy]$ ]]; then
echo "Deleting CRDs..."
kubectl_cmd delete crd batchsandboxes.sandbox.opensandbox.io 2>/dev/null || echo " CRD batchsandboxes does not exist"
kubectl_cmd delete crd pools.sandbox.opensandbox.io 2>/dev/null || echo " CRD pools does not exist"
echo -e "${GREEN}✓ CRDs deleted${NC}"
else
echo -e "${YELLOW}⊗ CRDs retained${NC}"
fi
echo ""
# Ask whether to delete namespace
read -p "Delete namespace $NAMESPACE? [y/N]: " delete_ns
if [[ $delete_ns =~ ^[Yy]$ ]]; then
echo "Deleting namespace..."
kubectl_cmd delete namespace "$NAMESPACE" 2>/dev/null || echo " Namespace does not exist"
echo -e "${GREEN}✓ Namespace deleted${NC}"
else
echo -e "${YELLOW}⊗ Namespace retained${NC}"
fi
echo ""
echo -e "${GREEN}======================================${NC}"
echo -e "${GREEN}✓ Uninstall completed${NC}"
echo -e "${GREEN}======================================${NC}"
echo ""
# Verify uninstall
echo -e "${YELLOW}Verifying uninstall:${NC}"
echo "Checking Helm releases:"
helm_cmd list -n "$NAMESPACE" 2>/dev/null || echo " Namespace does not exist"
echo ""
echo "Checking CRDs:"
kubectl_cmd get crds | grep sandbox.opensandbox.io || echo " No OpenSandbox CRDs found"
echo ""
Thank you for installing {{ .Chart.Name }}!
Your release is named {{ .Release.Name }}.
To learn more about the release, try:
$ helm status {{ .Release.Name }}
$ helm get all {{ .Release.Name }}
===============================================================================
OpenSandbox Kubernetes Controller has been deployed to namespace: {{ include "opensandbox-controller.namespace" . }}
Note: Both the controller and user resources (Pool, BatchSandbox) use the same namespace.
1. Check the controller status:
kubectl get deployment -n {{ include "opensandbox-controller.namespace" . }} {{ .Values.namePrefix }}controller-manager
2. View controller logs:
kubectl logs -n {{ include "opensandbox-controller.namespace" . }} -l control-plane=controller-manager -f
3. Verify CRDs are installed:
kubectl get crds | grep sandbox.opensandbox.io
You should see:
- batchsandboxes.sandbox.opensandbox.io
- pools.sandbox.opensandbox.io
4. Pool Resources:
{{- if .Values.pools }}
{{- $enabledPools := list }}
{{- range .Values.pools }}
{{- if .enabled }}
{{- $enabledPools = append $enabledPools .name }}
{{- end }}
{{- end }}
{{- if $enabledPools }}
✅ The following Pools have been deployed:
{{- range $enabledPools }}
- {{ . }}
{{- end }}
Check Pool status:
kubectl get pools -n {{ include "opensandbox-controller.namespace" . }}
Check Pool details:
kubectl describe pool agent-pool -n {{ include "opensandbox-controller.namespace" . }}
Monitor Pool capacity:
kubectl get pool agent-pool -n {{ include "opensandbox-controller.namespace" . }} -o jsonpath='{.status}'
{{- else }}
⚠️ All Pools are disabled in values.yaml.
To enable the default agent-pool:
helm upgrade {{ .Release.Name }} opensandbox-controller --set pools[0].enabled=true
{{- end }}
{{- else }}
No Pools configured.
{{- end }}
To disable Pool auto-deployment:
helm upgrade {{ .Release.Name }} opensandbox-controller --set pools[0].enabled=false
Create additional Pool manually:
# Basic Pool without task executor
cat <<EOF | kubectl apply -f -
apiVersion: sandbox.opensandbox.io/v1alpha1
kind: Pool
metadata:
name: my-first-pool
spec:
minBufferSize: 2
maxBufferSize: 5
capacity: 10
sandboxTemplate:
spec:
image: ubuntu:latest
command: ["sleep", "infinity"]
EOF
# Pool with task executor (for task execution features)
# Note: Requires task-executor image
cat <<EOF | kubectl apply -f -
apiVersion: sandbox.opensandbox.io/v1alpha1
kind: Pool
metadata:
name: task-enabled-pool
spec:
template:
spec:
shareProcessNamespace: true
containers:
- name: sandbox-container
image: ubuntu:latest
command: ["sleep", "infinity"]
- name: task-executor
image: {{ .Values.taskExecutor.image.repository }}:{{ .Values.taskExecutor.image.tag }}
securityContext:
capabilities:
add: ["SYS_PTRACE"]
capacitySpec:
bufferMax: 5
bufferMin: 2
poolMax: 10
poolMin: 2
EOF
5. Create your first BatchSandbox:
cat <<EOF | kubectl apply -f -
apiVersion: sandbox.opensandbox.io/v1alpha1
kind: BatchSandbox
metadata:
name: my-first-batchsandbox
spec:
replicas: 3
ttlSecondsAfterFinished: 3600
sandboxTemplate:
spec:
image: ubuntu:latest
command: ["sleep", "infinity"]
EOF
6. Check your resources:
kubectl get pools
kubectl get batchsandboxes
kubectl get pods
{{- if .Values.metrics.enabled }}
7. Access metrics (requires port-forward):
kubectl port-forward -n {{ include "opensandbox-controller.namespace" . }} svc/{{ .Values.namePrefix }}controller-manager-metrics-service {{ .Values.metrics.service.port }}:{{ .Values.metrics.service.port }}
Then visit: https://localhost:{{ .Values.metrics.service.port }}/metrics
{{- end }}
===============================================================================
For more information:
- Documentation: https://github.com/alibaba/OpenSandbox
- Issues: https://github.com/alibaba/OpenSandbox/issues
{{ if .Values.extraRoles.batchsandboxEditor.enabled }}
Note: Additional ClusterRoles for user management have been created:
- {{ .Values.namePrefix }}batchsandbox-editor-role
- {{ .Values.namePrefix }}batchsandbox-viewer-role
- {{ .Values.namePrefix }}pool-editor-role
- {{ .Values.namePrefix }}pool-viewer-role
You can bind these roles to users/groups as needed.
{{- end }}
{{/*
扩展chart名称
*/}}
{{- define "opensandbox-controller.name" -}}
{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
{{- end }}
{{/*
创建完整的限定名称
*/}}
{{- define "opensandbox-controller.fullname" -}}
{{- if .Values.fullnameOverride }}
{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
{{- else }}
{{- $name := default .Chart.Name .Values.nameOverride }}
{{- if contains $name .Release.Name }}
{{- .Release.Name | trunc 63 | trimSuffix "-" }}
{{- else }}
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
{{- end }}
{{- end }}
{{- end }}
{{/*
Chart标签
*/}}
{{- define "opensandbox-controller.chart" -}}
{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
{{- end }}
{{/*
通用标签
*/}}
{{- define "opensandbox-controller.labels" -}}
helm.sh/chart: {{ include "opensandbox-controller.chart" . }}
{{ include "opensandbox-controller.selectorLabels" . }}
{{- if .Chart.AppVersion }}
app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
{{- end }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
{{- with .Values.labels }}
{{ toYaml . }}
{{- end }}
{{- end }}
{{/*
选择器标签
*/}}
{{- define "opensandbox-controller.selectorLabels" -}}
app.kubernetes.io/name: {{ include "opensandbox-controller.name" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
control-plane: controller-manager
{{- end }}
{{/*
创建ServiceAccount名称
*/}}
{{- define "opensandbox-controller.serviceAccountName" -}}
{{- if .Values.rbac.serviceAccount.create }}
{{- default (printf "%scontroller-manager" .Values.namePrefix) .Values.rbac.serviceAccount.name }}
{{- else }}
{{- default "default" .Values.rbac.serviceAccount.name }}
{{- end }}
{{- end }}
{{/*
命名空间
*/}}
{{- define "opensandbox-controller.namespace" -}}
{{- if .Values.namespaceOverride }}
{{- .Values.namespaceOverride }}
{{- else }}
{{- printf "%ssystem" .Values.namePrefix }}
{{- end }}
{{- end }}
{{/*
Controller镜像
*/}}
{{- define "opensandbox-controller.controllerImage" -}}
{{- printf "%s:%s" .Values.controllerManager.image.repository (.Values.controllerManager.image.tag | default .Chart.AppVersion) }}
{{- end }}
{{/*
Task Executor镜像
*/}}
{{- define "opensandbox-controller.taskExecutorImage" -}}
{{- printf "%s:%s" .Values.taskExecutor.image.repository (.Values.taskExecutor.image.tag | default .Chart.AppVersion) }}
{{- end }}
{{- if .Values.rbac.create }}
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: {{ .Values.namePrefix }}manager-role
labels:
{{- include "opensandbox-controller.labels" . | nindent 4 }}
rules:
- apiGroups:
- ""
resources:
- events
- pods
verbs:
- create
- delete
- get
- list
- patch
- update
- watch
- apiGroups:
- ""
resources:
- pods/status
verbs:
- get
- patch
- update
- apiGroups:
- sandbox.opensandbox.io
resources:
- batchsandboxes
- pools
verbs:
- create
- delete
- get
- list
- patch
- update
- watch
- apiGroups:
- sandbox.opensandbox.io
resources:
- batchsandboxes/finalizers
- pools/finalizers
verbs:
- update
- apiGroups:
- sandbox.opensandbox.io
resources:
- batchsandboxes/status
- pools/status
verbs:
- get
- patch
- update
{{- end }}
{{- if .Values.rbac.create }}
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: {{ .Values.namePrefix }}manager-rolebinding
labels:
{{- include "opensandbox-controller.labels" . | nindent 4 }}
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: {{ .Values.namePrefix }}manager-role
subjects:
- kind: ServiceAccount
name: {{ include "opensandbox-controller.serviceAccountName" . }}
namespace: {{ include "opensandbox-controller.namespace" . }}
{{- end }}
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ .Values.namePrefix }}controller-manager
namespace: {{ include "opensandbox-controller.namespace" . }}
labels:
{{- include "opensandbox-controller.labels" . | nindent 4 }}
spec:
replicas: {{ .Values.controllerManager.replicas }}
selector:
matchLabels:
{{- include "opensandbox-controller.selectorLabels" . | nindent 6 }}
template:
metadata:
annotations:
{{- with .Values.podAnnotations }}
{{- toYaml . | nindent 8 }}
{{- end }}
labels:
{{- include "opensandbox-controller.selectorLabels" . | nindent 8 }}
{{- with .Values.podLabels }}
{{- toYaml . | nindent 8 }}
{{- end }}
spec:
{{- with .Values.imagePullSecrets }}
imagePullSecrets:
{{- toYaml . | nindent 8 }}
{{- end }}
serviceAccountName: {{ include "opensandbox-controller.serviceAccountName" . }}
securityContext:
{{- toYaml .Values.controllerManager.securityContext | nindent 8 }}
terminationGracePeriodSeconds: {{ .Values.terminationGracePeriodSeconds }}
containers:
- name: manager
image: {{ include "opensandbox-controller.controllerImage" . }}
imagePullPolicy: {{ .Values.controllerManager.image.pullPolicy }}
command:
- /workspace/server
args:
{{- if .Values.controllerManager.leaderElect }}
- --leader-elect
{{- end }}
- --health-probe-bind-address={{ .Values.controllerManager.healthProbeBindAddress }}
- --zap-log-level={{ .Values.controllerManager.logLevel }}
{{- with .Values.controllerManager.extraArgs }}
{{- toYaml . | nindent 8 }}
{{- end }}
securityContext:
{{- toYaml .Values.controllerManager.containerSecurityContext | nindent 10 }}
{{- with .Values.controllerManager.env }}
env:
{{- toYaml . | nindent 10 }}
{{- end }}
livenessProbe:
httpGet:
path: /healthz
port: {{ .Values.controllerManager.healthProbePort | default 8081 }}
initialDelaySeconds: {{ .Values.healthProbes.liveness.initialDelaySeconds | default 15 }}
periodSeconds: {{ .Values.healthProbes.liveness.periodSeconds | default 20 }}
timeoutSeconds: {{ .Values.healthProbes.liveness.timeoutSeconds | default 1 }}
failureThreshold: {{ .Values.healthProbes.liveness.failureThreshold | default 3 }}
readinessProbe:
httpGet:
path: /readyz
port: {{ .Values.controllerManager.healthProbePort | default 8081 }}
initialDelaySeconds: {{ .Values.healthProbes.readiness.initialDelaySeconds | default 5 }}
periodSeconds: {{ .Values.healthProbes.readiness.periodSeconds | default 10 }}
timeoutSeconds: {{ .Values.healthProbes.readiness.timeoutSeconds | default 1 }}
failureThreshold: {{ .Values.healthProbes.readiness.failureThreshold | default 3 }}
resources:
{{- toYaml .Values.controllerManager.resources | nindent 10 }}
{{- with .Values.controllerManager.nodeSelector }}
nodeSelector:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.controllerManager.affinity }}
affinity:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.controllerManager.tolerations }}
tolerations:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- if .Values.rbac.create }}
{{- if .Values.extraRoles.batchsandboxEditor.enabled }}
---
# This role is provided to allow the cluster admin to help manage permissions for users.
# Grants permissions to create, update, and delete BatchSandbox resources.
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: {{ .Values.namePrefix }}batchsandbox-editor-role
labels:
{{- include "opensandbox-controller.labels" . | nindent 4 }}
rules:
- apiGroups:
- sandbox.opensandbox.io
resources:
- batchsandboxes
verbs:
- create
- delete
- get
- list
- patch
- update
- watch
- apiGroups:
- sandbox.opensandbox.io
resources:
- batchsandboxes/status
verbs:
- get
{{- end }}
{{- if .Values.extraRoles.batchsandboxViewer.enabled }}
---
# This role is provided to allow the cluster admin to help manage permissions for users.
# Grants read-only permissions for BatchSandbox resources.
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: {{ .Values.namePrefix }}batchsandbox-viewer-role
labels:
{{- include "opensandbox-controller.labels" . | nindent 4 }}
rules:
- apiGroups:
- sandbox.opensandbox.io
resources:
- batchsandboxes
verbs:
- get
- list
- watch
- apiGroups:
- sandbox.opensandbox.io
resources:
- batchsandboxes/status
verbs:
- get
{{- end }}
{{- if .Values.extraRoles.poolEditor.enabled }}
---
# This role is provided to allow the cluster admin to help manage permissions for users.
# Grants permissions to create, update, and delete Pool resources.
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: {{ .Values.namePrefix }}pool-editor-role
labels:
{{- include "opensandbox-controller.labels" . | nindent 4 }}
rules:
- apiGroups:
- sandbox.opensandbox.io
resources:
- pools
verbs:
- create
- delete
- get
- list
- patch
- update
- watch
- apiGroups:
- sandbox.opensandbox.io
resources:
- pools/status
verbs:
- get
{{- end }}
{{- if .Values.extraRoles.poolViewer.enabled }}
---
# This role is provided to allow the cluster admin to help manage permissions for users.
# Grants read-only permissions for Pool resources.
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: {{ .Values.namePrefix }}pool-viewer-role
labels:
{{- include "opensandbox-controller.labels" . | nindent 4 }}
rules:
- apiGroups:
- sandbox.opensandbox.io
resources:
- pools
verbs:
- get
- list
- watch
- apiGroups:
- sandbox.opensandbox.io
resources:
- pools/status
verbs:
- get
{{- end }}
{{- end }}
{{- if .Values.rbac.create }}
# permissions to do leader election.
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: {{ .Values.namePrefix }}leader-election-role
namespace: {{ include "opensandbox-controller.namespace" . }}
labels:
{{- include "opensandbox-controller.labels" . | nindent 4 }}
rules:
- apiGroups:
- ""
resources:
- configmaps
verbs:
- get
- list
- watch
- create
- update
- patch
- delete
- apiGroups:
- coordination.k8s.io
resources:
- leases
verbs:
- get
- list
- watch
- create
- update
- patch
- delete
- apiGroups:
- ""
resources:
- events
verbs:
- create
- patch
{{- end }}
{{- if .Values.rbac.create }}
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: {{ .Values.namePrefix }}leader-election-rolebinding
namespace: {{ include "opensandbox-controller.namespace" . }}
labels:
{{- include "opensandbox-controller.labels" . | nindent 4 }}
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: {{ .Values.namePrefix }}leader-election-role
subjects:
- kind: ServiceAccount
name: {{ include "opensandbox-controller.serviceAccountName" . }}
namespace: {{ include "opensandbox-controller.namespace" . }}
{{- end }}
{{- if and .Values.rbac.create .Values.metrics.enabled }}
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: {{ .Values.namePrefix }}metrics-auth-role
labels:
{{- include "opensandbox-controller.labels" . | nindent 4 }}
rules:
- apiGroups:
- authentication.k8s.io
resources:
- tokenreviews
verbs:
- create
- apiGroups:
- authorization.k8s.io
resources:
- subjectaccessreviews
verbs:
- create
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: {{ .Values.namePrefix }}metrics-auth-rolebinding
labels:
{{- include "opensandbox-controller.labels" . | nindent 4 }}
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: {{ .Values.namePrefix }}metrics-auth-role
subjects:
- kind: ServiceAccount
name: {{ include "opensandbox-controller.serviceAccountName" . }}
namespace: {{ include "opensandbox-controller.namespace" . }}
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: {{ .Values.namePrefix }}metrics-reader
labels:
{{- include "opensandbox-controller.labels" . | nindent 4 }}
rules:
- nonResourceURLs:
- /metrics
verbs:
- get
{{- end }}
{{- if .Values.metrics.enabled }}
apiVersion: v1
kind: Service
metadata:
name: {{ .Values.namePrefix }}controller-manager-metrics-service
namespace: {{ include "opensandbox-controller.namespace" . }}
labels:
{{- include "opensandbox-controller.labels" . | nindent 4 }}
control-plane: controller-manager
spec:
type: {{ .Values.metrics.service.type }}
ports:
- name: https
port: {{ .Values.metrics.service.port }}
protocol: TCP
targetPort: {{ .Values.metrics.service.port }}
selector:
{{- include "opensandbox-controller.selectorLabels" . | nindent 4 }}
{{- end }}
{{- if .Values.podDisruptionBudget.enabled }}
apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
name: {{ include "opensandbox-controller.fullname" . }}-controller
namespace: {{ include "opensandbox-controller.namespace" . }}
labels:
{{- include "opensandbox-controller.labels" . | nindent 4 }}
spec:
{{- if .Values.podDisruptionBudget.minAvailable }}
minAvailable: {{ .Values.podDisruptionBudget.minAvailable }}
{{- end }}
{{- if .Values.podDisruptionBudget.maxUnavailable }}
maxUnavailable: {{ .Values.podDisruptionBudget.maxUnavailable }}
{{- end }}
selector:
matchLabels:
control-plane: controller-manager
{{- include "opensandbox-controller.selectorLabels" . | nindent 6 }}
{{- end }}
{{- if .Values.pools }}
{{- range .Values.pools }}
{{- if .enabled }}
---
apiVersion: sandbox.opensandbox.io/v1alpha1
kind: Pool
metadata:
name: {{ .name }}
namespace: {{ include "opensandbox-controller.namespace" $ }}
labels:
{{- include "opensandbox-controller.labels" $ | nindent 4 }}
{{- with .labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
{{- with .annotations }}
annotations:
{{- toYaml . | nindent 4 }}
{{- end }}
spec:
{{- if .template }}
template:
{{- with .template.metadata }}
metadata:
{{- toYaml . | nindent 6 }}
{{- end }}
spec:
{{- if .template.spec.shareProcessNamespace }}
shareProcessNamespace: {{ .template.spec.shareProcessNamespace }}
{{- end }}
{{- if .template.spec.initContainers }}
initContainers:
{{- range .template.spec.initContainers }}
- name: {{ .name }}
image: {{ .image }}
{{- if .imagePullPolicy }}
imagePullPolicy: {{ .imagePullPolicy }}
{{- end }}
{{- if .command }}
command:
{{- toYaml .command | nindent 10 }}
{{- end }}
{{- if .args }}
args:
{{- toYaml .args | nindent 10 }}
{{- end }}
{{- if .env }}
env:
{{- toYaml .env | nindent 10 }}
{{- end }}
{{- if .volumeMounts }}
volumeMounts:
{{- toYaml .volumeMounts | nindent 10 }}
{{- end }}
{{- if .resources }}
resources:
{{- toYaml .resources | nindent 10 }}
{{- end }}
{{- if .securityContext }}
securityContext:
{{- toYaml .securityContext | nindent 10 }}
{{- end }}
{{- end }}
{{- end }}
containers:
{{- range .template.spec.containers }}
- name: {{ .name }}
image: {{ .image }}
{{- if .imagePullPolicy }}
imagePullPolicy: {{ .imagePullPolicy }}
{{- end }}
{{- if .command }}
command:
{{- toYaml .command | nindent 10 }}
{{- end }}
{{- if .args }}
args:
{{- toYaml .args | nindent 10 }}
{{- end }}
{{- if .env }}
env:
{{- toYaml .env | nindent 10 }}
{{- end }}
{{- if .ports }}
ports:
{{- toYaml .ports | nindent 10 }}
{{- end }}
{{- if .volumeMounts }}
volumeMounts:
{{- toYaml .volumeMounts | nindent 10 }}
{{- end }}
{{- if .resources }}
resources:
{{- toYaml .resources | nindent 10 }}
{{- end }}
{{- if .securityContext }}
securityContext:
{{- toYaml .securityContext | nindent 10 }}
{{- end }}
{{- if .livenessProbe }}
livenessProbe:
{{- toYaml .livenessProbe | nindent 10 }}
{{- end }}
{{- if .readinessProbe }}
readinessProbe:
{{- toYaml .readinessProbe | nindent 10 }}
{{- end }}
{{- end }}
{{- if .template.spec.volumes }}
volumes:
{{- toYaml .template.spec.volumes | nindent 6 }}
{{- end }}
{{- if .template.spec.nodeSelector }}
nodeSelector:
{{- toYaml .template.spec.nodeSelector | nindent 8 }}
{{- end }}
{{- if .template.spec.tolerations }}
tolerations:
{{- toYaml .template.spec.tolerations | nindent 8 }}
{{- end }}
{{- if .template.spec.affinity }}
affinity:
{{- toYaml .template.spec.affinity | nindent 8 }}
{{- end }}
{{- end }}
capacitySpec:
{{- if .capacitySpec.bufferMin }}
bufferMin: {{ .capacitySpec.bufferMin }}
{{- end }}
{{- if .capacitySpec.bufferMax }}
bufferMax: {{ .capacitySpec.bufferMax }}
{{- end }}
{{- if .capacitySpec.poolMin }}
poolMin: {{ .capacitySpec.poolMin }}
{{- end }}
{{- if .capacitySpec.poolMax }}
poolMax: {{ .capacitySpec.poolMax }}
{{- end }}
{{- end }}
{{- end }}
{{- end }}
{{- if .Values.server.enabled }}
apiVersion: v1
kind: ConfigMap
metadata:
name: {{ include "opensandbox-controller.fullname" . }}-server-config
namespace: {{ include "opensandbox-controller.namespace" . }}
labels:
{{- include "opensandbox-controller.labels" . | nindent 4 }}
app.kubernetes.io/component: server
data:
config.toml: |
# OpenSandbox Server Configuration
# Generated by Helm Chart
[server]
host = "{{ .Values.server.config.server.host }}"
port = {{ .Values.server.config.server.port }}
log_level = "{{ .Values.server.config.server.logLevel }}"
{{- if .Values.server.config.server.apiKey }}
api_key = "{{ .Values.server.config.server.apiKey }}"
{{- end }}
[runtime]
type = "{{ .Values.server.config.runtime.type }}"
execd_image = "{{ .Values.server.config.runtime.execdImage }}"
[kubernetes]
# Use in-cluster configuration (empty kubeconfig_path)
kubeconfig_path = ""
namespace = "{{ include "opensandbox-controller.namespace" . }}"
# 优先使用 Pool 模式(batchsandbox provider + Pool)
workload_provider = "{{ .Values.server.config.kubernetes.workloadProvider }}"
# BatchSandbox 模板文件(非 Pool 模式需要)
batchsandbox_template_file = "/etc/opensandbox/batchsandbox-template.yaml"
# BatchSandbox 模板(用于非 Pool 模式)
batchsandbox-template.yaml: |
# BatchSandbox CR template for non-pooled mode
# Pool mode does not use this template
metadata:
# Metadata will be merged with runtime-generated values
spec:
replicas: 1
template:
spec:
restartPolicy: Never
{{- if .Values.server.config.kubernetes.batchsandboxTemplate.tolerations }}
tolerations:
{{- toYaml .Values.server.config.kubernetes.batchsandboxTemplate.tolerations | nindent 12 }}
{{- else }}
tolerations:
- operator: "Exists"
{{- end }}
{{- if .Values.server.config.kubernetes.batchsandboxTemplate.nodeSelector }}
nodeSelector:
{{- toYaml .Values.server.config.kubernetes.batchsandboxTemplate.nodeSelector | nindent 12 }}
{{- end }}
{{- end }}
{{- if .Values.server.enabled }}
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ include "opensandbox-controller.fullname" . }}-server
namespace: {{ include "opensandbox-controller.namespace" . }}
labels:
{{- include "opensandbox-controller.labels" . | nindent 4 }}
app.kubernetes.io/component: server
spec:
replicas: {{ .Values.server.replicas }}
selector:
matchLabels:
{{- include "opensandbox-controller.selectorLabels" . | nindent 6 }}
app.kubernetes.io/component: server
template:
metadata:
annotations:
checksum/config: {{ include (print $.Template.BasePath "/server-configmap.yaml") . | sha256sum }}
{{- with .Values.server.podAnnotations }}
{{- toYaml . | nindent 8 }}
{{- end }}
labels:
{{- include "opensandbox-controller.selectorLabels" . | nindent 8 }}
app.kubernetes.io/component: server
{{- with .Values.server.podLabels }}
{{- toYaml . | nindent 8 }}
{{- end }}
spec:
{{- with .Values.imagePullSecrets }}
imagePullSecrets:
{{- toYaml . | nindent 8 }}
{{- end }}
serviceAccountName: {{ include "opensandbox-controller.serviceAccountName" . }}
securityContext:
{{- toYaml .Values.server.securityContext | nindent 8 }}
terminationGracePeriodSeconds: {{ .Values.server.terminationGracePeriodSeconds | default 10 }}
containers:
- name: server
image: "{{ .Values.server.image.repository }}:{{ .Values.server.image.tag }}"
imagePullPolicy: {{ .Values.server.image.pullPolicy }}
args:
- --config
- /etc/opensandbox/config.toml
{{- if .Values.server.reload }}
- --reload
{{- end }}
{{- with .Values.server.extraArgs }}
{{- toYaml . | nindent 8 }}
{{- end }}
ports:
- name: http
containerPort: {{ .Values.server.config.server.port }}
protocol: TCP
{{- with .Values.server.env }}
env:
{{- toYaml . | nindent 10 }}
{{- end }}
livenessProbe:
httpGet:
path: {{ .Values.server.healthProbes.liveness.path }}
port: http
initialDelaySeconds: {{ .Values.server.healthProbes.liveness.initialDelaySeconds }}
periodSeconds: {{ .Values.server.healthProbes.liveness.periodSeconds }}
timeoutSeconds: {{ .Values.server.healthProbes.liveness.timeoutSeconds }}
failureThreshold: {{ .Values.server.healthProbes.liveness.failureThreshold }}
readinessProbe:
httpGet:
path: {{ .Values.server.healthProbes.readiness.path }}
port: http
initialDelaySeconds: {{ .Values.server.healthProbes.readiness.initialDelaySeconds }}
periodSeconds: {{ .Values.server.healthProbes.readiness.periodSeconds }}
timeoutSeconds: {{ .Values.server.healthProbes.readiness.timeoutSeconds }}
failureThreshold: {{ .Values.server.healthProbes.readiness.failureThreshold }}
resources:
{{- toYaml .Values.server.resources | nindent 10 }}
volumeMounts:
- name: config
mountPath: /etc/opensandbox
readOnly: true
securityContext:
{{- toYaml .Values.server.containerSecurityContext | nindent 10 }}
volumes:
- name: config
configMap:
name: {{ include "opensandbox-controller.fullname" . }}-server-config
{{- with .Values.server.nodeSelector }}
nodeSelector:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.server.affinity }}
affinity:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.server.tolerations }}
tolerations:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- end }}
{{- if and .Values.server.enabled .Values.server.ingress.enabled }}
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: {{ include "opensandbox-controller.fullname" . }}-server
namespace: {{ include "opensandbox-controller.namespace" . }}
labels:
{{- include "opensandbox-controller.labels" . | nindent 4 }}
app.kubernetes.io/component: server
{{- with .Values.server.ingress.annotations }}
annotations:
{{- toYaml . | nindent 4 }}
{{- end }}
spec:
{{- if .Values.server.ingress.className }}
ingressClassName: {{ .Values.server.ingress.className }}
{{- end }}
{{- if .Values.server.ingress.tls }}
tls:
{{- range .Values.server.ingress.tls }}
- hosts:
{{- range .hosts }}
- {{ . | quote }}
{{- end }}
secretName: {{ .secretName }}
{{- end }}
{{- end }}
rules:
{{- range .Values.server.ingress.hosts }}
- host: {{ .host | quote }}
http:
paths:
{{- range .paths }}
- path: {{ .path }}
pathType: {{ .pathType }}
backend:
service:
name: {{ include "opensandbox-controller.fullname" $ }}-server
port:
number: {{ $.Values.server.service.port }}
{{- end }}
{{- end }}
{{- end }}
{{- if .Values.server.enabled }}
apiVersion: v1
kind: Service
metadata:
name: {{ include "opensandbox-controller.fullname" . }}-server
namespace: {{ include "opensandbox-controller.namespace" . }}
labels:
{{- include "opensandbox-controller.labels" . | nindent 4 }}
app.kubernetes.io/component: server
{{- with .Values.server.service.annotations }}
annotations:
{{- toYaml . | nindent 4 }}
{{- end }}
spec:
type: {{ .Values.server.service.type }}
{{- if and (eq .Values.server.service.type "NodePort") .Values.server.service.nodePort }}
ports:
- port: {{ .Values.server.service.port }}
targetPort: http
protocol: TCP
name: http
nodePort: {{ .Values.server.service.nodePort }}
{{- else }}
ports:
- port: {{ .Values.server.service.port }}
targetPort: http
protocol: TCP
name: http
{{- end }}
selector:
{{- include "opensandbox-controller.selectorLabels" . | nindent 4 }}
app.kubernetes.io/component: server
{{- end }}
{{- if .Values.rbac.serviceAccount.create -}}
apiVersion: v1
kind: ServiceAccount
metadata:
name: {{ include "opensandbox-controller.serviceAccountName" . }}
namespace: {{ include "opensandbox-controller.namespace" . }}
labels:
{{- include "opensandbox-controller.labels" . | nindent 4 }}
{{- with .Values.rbac.serviceAccount.annotations }}
annotations:
{{- toYaml . | nindent 4 }}
{{- end }}
{{- end }}
{{- if and .Values.metrics.enabled .Values.metrics.serviceMonitor.enabled }}
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
name: {{ .Values.namePrefix }}controller-manager-metrics-monitor
namespace: {{ include "opensandbox-controller.namespace" . }}
labels:
{{- include "opensandbox-controller.labels" . | nindent 4 }}
spec:
endpoints:
- path: /metrics
port: https
scheme: https
bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
tlsConfig:
insecureSkipVerify: true
interval: {{ .Values.metrics.serviceMonitor.interval }}
scrapeTimeout: {{ .Values.metrics.serviceMonitor.scrapeTimeout }}
selector:
matchLabels:
{{- include "opensandbox-controller.selectorLabels" . | nindent 6 }}
control-plane: controller-manager
{{- end }}
# OpenSandbox Helm Chart - E2E测试配置
# values-e2e.yaml
#
# 此配置文件专为端到端测试设计,使用最小资源配置
# Controller配置
controllerManager:
image:
repository: opensandbox/controller
tag: latest
pullPolicy: Never
replicas: 1
resources:
limits:
cpu: 1
memory: 1Gi
requests:
cpu: 200m
memory: 256Mi
# Task Executor镜像配置
taskExecutor:
image:
repository: opensandbox/task-executor
tag: latest
pullPolicy: Never
# Server配置 - E2E测试
server:
enabled: true
image:
repository: opensandbox/server
tag: latest
pullPolicy: Never
replicas: 1
resources:
limits:
cpu: 500m
memory: 512Mi
requests:
cpu: 100m
memory: 256Mi
# Server 安全上下文 - 镜像以 root 运行
securityContext: {}
containerSecurityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- "ALL"
# 使用 ClusterIP,通过 port-forward 访问
service:
type: ClusterIP
port: 8080
# Pool配置 - 最小配置用于E2E测试
pools:
- name: agent-pool
enabled: true
labels:
app: opensandbox
component: agent-pool
annotations:
description: "E2E Test Pool - Minimal Capacity"
template:
metadata:
labels:
pool: agent-pool
sdk-compatible: "true"
spec:
shareProcessNamespace: true
initContainers:
- name: execd-installer
image: opensandbox/execd:v1.0.21
imagePullPolicy: Never
command: ["/bin/sh", "-c"]
args:
- |
cp ./execd /opt/opensandbox/bin/execd && \
cp ./bootstrap.sh /opt/opensandbox/bin/bootstrap.sh && \
chmod +x /opt/opensandbox/bin/*
volumeMounts:
- name: opensandbox-bin
mountPath: /opt/opensandbox/bin
containers:
- name: sandbox-container
image: nginx:latest
imagePullPolicy: Never
command: ["/opt/opensandbox/bin/bootstrap.sh", "sleep", "infinity"]
env:
- name: EXECD
value: /opt/opensandbox/bin/execd
ports:
- containerPort: 44772
name: execd
resources:
requests:
cpu: "50m"
memory: "64Mi"
limits:
cpu: "200m"
memory: "128Mi"
volumeMounts:
- name: opensandbox-bin
mountPath: /opt/opensandbox/bin
- name: task-executor
image: opensandbox/task-executor:dev
imagePullPolicy: Never
resources:
requests:
cpu: "50m"
memory: "64Mi"
limits:
cpu: "200m"
memory: "128Mi"
securityContext:
capabilities:
add: ["SYS_PTRACE"]
volumes:
- name: opensandbox-bin
emptyDir: {}
# E2E测试容量配置:最小化以避免主机过载
capacitySpec:
bufferMin: 2 # 最小可用2个
bufferMax: 3 # 最多预热3个
poolMin: 2 # 最小总数2个
poolMax: 5 # 最大总数5个
# Metrics监控
metrics:
enabled: true
serviceMonitor:
enabled: false # 禁用 ServiceMonitor
# Default values for opensandbox-controller.
# This is a YAML-formatted file.
# Declare variables to be passed into your templates.
# Namespace configuration
# Controller and user resources use this namespace uniformly
namespaceOverride: "opensandbox"
# Name override configuration (optional)
# Used to customize the resource names generated by the Chart
nameOverride: ""
fullnameOverride: ""
# Controller image configuration
controllerManager:
image:
repository: opensandbox/controller
tag: "dev"
pullPolicy: Never
# Number of replicas
replicas: 1
# Resource limits
resources:
limits:
cpu: 500m
memory: 128Mi
requests:
cpu: 10m
memory: 64Mi
# Enable leader election (high availability)
leaderElect: true
# Health check address
healthProbeBindAddress: ":8081"
# Health check port (extracted from healthProbeBindAddress, or configured separately)
healthProbePort: 8081
# Log level
logLevel: 3
# Security context
securityContext:
runAsNonRoot: true
seccompProfile:
type: RuntimeDefault
containerSecurityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- "ALL"
# Node selector
nodeSelector: {}
# Tolerations
tolerations: []
# Affinity
affinity: {}
# Environment variables
env: []
# - name: EXAMPLE_ENV
# value: "example-value"
# Additional container arguments
extraArgs: []
# - --some-arg=value
# Task Executor image configuration
# Note: task-executor is not an independently deployed service, but a sidecar container
# injected into Pods created by Pool for executing tasks.
# Only image information is configured here for users to use when creating Pool resources.
taskExecutor:
image:
repository: opensandbox/task-executor
tag: "dev"
pullPolicy: Never
# Task Executor default resource configuration (for Pool template reference)
resources:
limits:
cpu: 500m
memory: 256Mi
requests:
cpu: 100m
memory: 128Mi
# Task Executor required security context
securityContext:
capabilities:
add: ["SYS_PTRACE"] # Requires ptrace permission to inject processes
# OpenSandbox Server configuration
# Server is the FastAPI control plane that exposes REST API for SDK usage
# SDK creates and manages Sandbox through Server
server:
# Enable Server (required for SDK scenarios)
enabled: true
# Server image configuration
image:
repository: opensandbox/server
tag: "v0.2.1"
pullPolicy: Never
# Number of replicas
replicas: 1
# Resource limits
resources:
limits:
cpu: 1
memory: 512Mi
requests:
cpu: 100m
memory: 256Mi
# Security context
# ⚠️ Security Warning: Server image currently runs as root user (uid=0)
# This is a known security concern and does not comply with Kubernetes best practices
# Production recommendations:
# 1. Use NetworkPolicy to restrict Server's network access
# 2. Use PodSecurityPolicy/PodSecurityStandard to restrict Pod permissions
# 3. Regularly audit Server logs and behavior
#
# TODO: Update Server Dockerfile to support non-root user (Issue: #xxx)
# Planned to be fixed in next version
securityContext: {} # Must be empty to allow root user execution
containerSecurityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- "ALL"
# Node selector
nodeSelector: {}
# Tolerations
tolerations: []
# Affinity
affinity: {}
# Environment variables
env: []
# Additional container arguments
extraArgs: []
# Development mode (enable hot reload)
reload: false
# Pod annotations
podAnnotations: {}
# Pod labels
podLabels: {}
# Termination grace period (seconds)
terminationGracePeriodSeconds: 10
# Server configuration (written to ConfigMap)
config:
server:
host: "0.0.0.0"
port: 8080
logLevel: "INFO"
# API Key authentication (optional)
apiKey: ""
runtime:
type: "kubernetes"
execdImage: "opensandbox/execd:v1.0.21"
kubernetes:
# workloadProvider: batchsandbox (use Pool) or agent-sandbox
workloadProvider: "batchsandbox"
# BatchSandbox template configuration (for non-Pool mode)
batchsandboxTemplate:
# BatchSandbox labels (optional)
labels: {}
# Pod labels (optional)
podLabels: {}
# Node tolerations
tolerations:
- operator: "Exists"
# Node selector (optional)
nodeSelector: {}
# Service configuration
service:
# Service type: ClusterIP, NodePort, LoadBalancer
type: ClusterIP
port: 8080
# NodePort (effective when type=NodePort)
# Set to specific port number (30000-32767) or leave empty for automatic allocation
# Example: 30080
nodePort: null
annotations: {}
# Ingress configuration (optional)
ingress:
enabled: false
className: ""
annotations: {}
# kubernetes.io/ingress.class: nginx
# cert-manager.io/cluster-issuer: letsencrypt-prod
hosts:
- host: opensandbox.example.com
paths:
- path: /
pathType: Prefix
tls: []
# - secretName: opensandbox-tls
# hosts:
# - opensandbox.example.com
# Health probe configuration
healthProbes:
liveness:
path: /health
initialDelaySeconds: 15
periodSeconds: 20
timeoutSeconds: 5
failureThreshold: 3
readiness:
path: /health
initialDelaySeconds: 5
periodSeconds: 10
timeoutSeconds: 5
failureThreshold: 3
# Metrics configuration
metrics:
enabled: true
service:
type: ClusterIP
port: 8443
# Enable ServiceMonitor (requires Prometheus Operator)
serviceMonitor:
enabled: false
interval: 30s
scrapeTimeout: 10s
# RBAC configuration
rbac:
# Create RBAC resources
create: true
# ServiceAccount configuration
serviceAccount:
# Create ServiceAccount
create: true
# Use existing ServiceAccount
name: ""
# ServiceAccount annotations
annotations: {}
# CRD configuration
crds:
# Whether to install CRD (set to true for first installation, can be set to false for upgrades)
install: true
# CRD annotations
annotations: {}
# Image pull secrets
imagePullSecrets: []
# - name: regcred
# Global labels
labels: {}
# Global annotations
annotations: {}
# Name prefix, used to generate resource names and namespace (namePrefix + "system")
namePrefix: "opensandbox-"
# Pod annotations
podAnnotations:
kubectl.kubernetes.io/default-container: manager
# Pod labels
podLabels: {}
# Termination grace period (seconds)
terminationGracePeriodSeconds: 10
# Pod Disruption Budget (optional)
# Used for high availability deployment to prevent too many Pods from being unavailable simultaneously
podDisruptionBudget:
enabled: false
minAvailable: 1
# maxUnavailable: 1 # Mutually exclusive with minAvailable, choose one
# Health probe configuration
healthProbes:
liveness:
initialDelaySeconds: 15
periodSeconds: 20
timeoutSeconds: 1
failureThreshold: 3
readiness:
initialDelaySeconds: 5
periodSeconds: 10
timeoutSeconds: 1
failureThreshold: 3
# Extra user roles (optional)
extraRoles:
# Create BatchSandbox admin role
batchsandboxAdmin:
enabled: true
# Create BatchSandbox editor role
batchsandboxEditor:
enabled: true
# Create BatchSandbox viewer role
batchsandboxViewer:
enabled: true
# Create Pool admin role
poolAdmin:
enabled: true
# Create Pool editor role
poolEditor:
enabled: true
# Create Pool viewer role
poolViewer:
enabled: true
# Pool resource configuration
# Automatically create Pool resources during Helm deployment
# Default: Enable a general Agent Pool suitable for most SDK scenarios
pools:
# Default Agent Pool - SDK scenario (with execd + task-executor)
- name: agent-pool
enabled: true
labels:
app: opensandbox
component: agent-pool
annotations:
description: "Default Agent Pool with execd and task-executor for SDK usage"
template:
metadata:
labels:
pool: agent-pool
sdk-compatible: "true"
spec:
# Required: Share process namespace (required by task-executor)
shareProcessNamespace: true
# Init Container: Install execd (required for SDK scenario)
initContainers:
- name: execd-installer
image: opensandbox/execd:v1.0.21
imagePullPolicy: Never
command: ["/bin/sh", "-c"]
args:
- |
cp ./execd /opt/opensandbox/bin/execd && \
cp ./bootstrap.sh /opt/opensandbox/bin/bootstrap.sh && \
chmod +x /opt/opensandbox/bin/execd && \
chmod +x /opt/opensandbox/bin/bootstrap.sh
volumeMounts:
- name: opensandbox-bin
mountPath: /opt/opensandbox/bin
containers:
# Main container: Sandbox environment
- name: sandbox-container
image: nginx:latest
imagePullPolicy: Never
command: ["/opt/opensandbox/bin/bootstrap.sh", "sleep", "infinity"]
env:
- name: EXECD
value: /opt/opensandbox/bin/execd
ports:
- containerPort: 44772
name: execd
protocol: TCP
resources:
requests:
cpu: "100m"
memory: "128Mi"
limits:
cpu: "500m"
memory: "256Mi"
volumeMounts:
- name: opensandbox-bin
mountPath: /opt/opensandbox/bin
# Sidecar: Task Executor
- name: task-executor
image: opensandbox/task-executor:dev
imagePullPolicy: Never
ports:
- containerPort: 5758
name: task-executor
protocol: TCP
resources:
requests:
cpu: "100m"
memory: "128Mi"
limits:
cpu: "500m"
memory: "256Mi"
securityContext:
capabilities:
add: ["SYS_PTRACE"]
volumes:
- name: opensandbox-bin
emptyDir: {}
# Capacity configuration: Suitable for small to medium scale deployments (10-50 concurrent Agents)
capacitySpec:
bufferMin: 10 # Minimum available Pods
bufferMax: 20 # Maximum pre-warmed Pods
poolMin: 10 # Minimum total Pods
poolMax: 100 # Maximum total Pods
# Notes:
# - If Pool is not needed, set pools[0].enabled=false
# - If custom configuration is needed, override or add more Pools in values
# - Capacity configuration can be adjusted according to actual concurrency requirements:
# | Concurrent Agents | bufferMin | bufferMax | poolMin | poolMax |
# |------------------|-----------|-----------|---------|---------|
# | 1-10 | 2 | 5 | 2 | 20 |
# | 10-50 | 10 | 20 | 10 | 100 |
# | 50-200 | 50 | 100 | 50 | 300 |
# | 200+ | 100 | 200 | 100 | 500 |
......@@ -447,6 +447,8 @@ configs:
[egress]
image = "${{opensandbox-egress.image}}:${{opensandbox-egress.tag}}"
mode = "dns+nft"
disable_ipv6 = true
[docker]
network_mode = "bridge"
......
......@@ -8,9 +8,9 @@
"volume-manager": "v0.3.0-beta4",
"agent-sandbox-image": "v0.3.0-beta4",
"agent-sandbox-proxy": "v0.3.0-beta4",
"opensandbox-server": "v0.2.1",
"opensandbox-server": "v0.2.2",
"opensandbox-execd": "v1.0.21",
"opensandbox-egress": "v1.1.4",
"opensandbox-egress": "v1.1.5",
"aiproxy": "v0.6.5",
"aiproxy-pg": "0.8.0-pg15",
......
......@@ -4,25 +4,60 @@
* 只负责 OpenSandbox createConfig 映射,不连接远端实例。
*/
import { serviceEnv } from '../../../../../../env';
import type { SandboxRuntimeProfile } from './types';
import type { SandboxRuntimeCreateConfigInput, SandboxRuntimeProfile } from './types';
import { getSandboxSkillsRootPath, mergeStringRecord, normalizeEntrypoint } from './utils';
import { OPEN_SANDBOX_DEFAULT_ROOT_PATH, parseImageSpec } from '@fastgpt-sdk/sandbox-adapter';
const OPEN_SANDBOX_ENTRYPOINT = '/home/sandbox/entrypoint.sh';
const OPEN_SANDBOX_DOCKER_LOCAL_NETWORK_POLICY = {
const OPEN_SANDBOX_PROTECTED_NETWORK_DENY_RULES = [
{ action: 'deny' as const, target: 'localhost' },
{ action: 'deny' as const, target: '127.0.0.0/8' },
{ action: 'deny' as const, target: '::1/128' },
{ action: 'deny' as const, target: '10.0.0.0/8' },
{ action: 'deny' as const, target: '100.64.0.0/10' },
{ action: 'deny' as const, target: '169.254.0.0/16' },
{ action: 'deny' as const, target: '172.16.0.0/12' },
{ action: 'deny' as const, target: '192.168.0.0/16' },
{ action: 'deny' as const, target: '198.18.0.0/15' },
{ action: 'deny' as const, target: '224.0.0.0/4' },
{ action: 'deny' as const, target: 'fc00::/7' },
{ action: 'deny' as const, target: 'fe80::/10' },
{ action: 'deny' as const, target: '*.local' },
{ action: 'deny' as const, target: 'host.docker.internal' },
{ action: 'deny' as const, target: 'host.orb.internal' },
{ action: 'deny' as const, target: 'docker.orb.internal' },
{ action: 'deny' as const, target: 'gateway.orb.internal' },
{ action: 'deny' as const, target: 'proxyproxy.orb.internal' },
{ action: 'deny' as const, target: '*.orb.internal' },
{ action: 'deny' as const, target: '*.orb.local' }
];
const OPEN_SANDBOX_NETWORK_POLICY = {
defaultAction: 'allow' as const,
egress: [
{ action: 'deny' as const, target: 'localhost' },
{ action: 'deny' as const, target: 'host.docker.internal' },
{ action: 'deny' as const, target: 'host.orb.internal' },
{ action: 'deny' as const, target: 'docker.orb.internal' },
{ action: 'deny' as const, target: 'gateway.orb.internal' },
{ action: 'deny' as const, target: 'proxyproxy.orb.internal' },
{ action: 'deny' as const, target: '*.orb.internal' },
{ action: 'deny' as const, target: '*.orb.local' }
]
egress: OPEN_SANDBOX_PROTECTED_NETWORK_DENY_RULES
};
/** Keep internal network targets denied while allowing all unmatched public destinations. */
function buildOpenSandboxNetworkPolicy(
policy?: NonNullable<SandboxRuntimeCreateConfigInput['createConfig']>['networkPolicy']
) {
const protectedTargets = new Set(
OPEN_SANDBOX_PROTECTED_NETWORK_DENY_RULES.map(({ target }) => target)
);
return {
// Public egress is intentionally open. Create callers may add deny rules, but
// cannot replace the platform default with deny or override protected targets.
defaultAction: OPEN_SANDBOX_NETWORK_POLICY.defaultAction,
egress: [
...OPEN_SANDBOX_PROTECTED_NETWORK_DENY_RULES,
...(policy?.egress?.filter(
({ action, target }) => action === 'deny' && !protectedTargets.has(target)
) ?? [])
]
};
}
/**
* 构建 OpenSandbox 的 FastGPT 运行态 profile。
*
......@@ -71,13 +106,8 @@ export function buildOpenSandboxRuntimeProfile(): SandboxRuntimeProfile {
const metadata = mergeStringRecord(createConfig.metadata, input.metadata);
// volume 既可能来自 volume manager,也可能来自调用方透传的 createConfig;运行态 VM 配置优先。
const volumes = input.volumes ?? input.vmConfig?.volumes ?? createConfig.volumes;
// Docker 模式下默认拒绝常见宿主机别名;公网默认保持放行,私网 CIDR 需依赖部署网络边界。
const networkPolicy =
createConfig.networkPolicy ??
(!serviceEnv.AGENT_SANDBOX_OPENSANDBOX_DISABLE_NETWORK_POLICY &&
serviceEnv.AGENT_SANDBOX_OPENSANDBOX_RUNTIME === 'docker'
? OPEN_SANDBOX_DOCKER_LOCAL_NETWORK_POLICY
: undefined);
// Docker/Kubernetes 均启用 nftables 层的内部网段拒绝;未命中规则的公网地址保持放行。
const networkPolicy = buildOpenSandboxNetworkPolicy(createConfig.networkPolicy);
return {
...createConfig,
......
......@@ -120,12 +120,6 @@ export const serviceEnv = createEnv({
AGENT_SANDBOX_OPENSANDBOX_VOLUME_NAME_PREFIX: SandboxVolumeNameSchema.default(
'fastgpt-session'
).meta({ description: 'OpenSandbox persistent volume claimName prefix' }),
AGENT_SANDBOX_OPENSANDBOX_DISABLE_NETWORK_POLICY: BoolSchema.default(false).meta({
description:
'Disable the default outbound network policy for OpenSandbox Docker runtime. ' +
'Set to true when the OpenSandbox server uses a user-defined Docker network (not bridge), ' +
'as networkPolicy is only supported with network_mode=bridge.'
}),
AGENT_SANDBOX_SUSPEND_MINUTES: IntSchema.min(1).default(60).meta({
description: 'Agent sandbox 持续未活跃多少分钟后自动暂停'
}),
......
......@@ -28,10 +28,22 @@ const loadSandboxConfigModule = async () => {
return import('@fastgpt/service/core/ai/sandbox/infrastructure/provider/config');
};
const defaultOpenSandboxDockerNetworkPolicy = {
const defaultOpenSandboxNetworkPolicy = {
defaultAction: 'allow',
egress: [
{ action: 'deny', target: 'localhost' },
{ action: 'deny', target: '127.0.0.0/8' },
{ action: 'deny', target: '::1/128' },
{ action: 'deny', target: '10.0.0.0/8' },
{ action: 'deny', target: '100.64.0.0/10' },
{ action: 'deny', target: '169.254.0.0/16' },
{ action: 'deny', target: '172.16.0.0/12' },
{ action: 'deny', target: '192.168.0.0/16' },
{ action: 'deny', target: '198.18.0.0/15' },
{ action: 'deny', target: '224.0.0.0/4' },
{ action: 'deny', target: 'fc00::/7' },
{ action: 'deny', target: 'fe80::/10' },
{ action: 'deny', target: '*.local' },
{ action: 'deny', target: 'host.docker.internal' },
{ action: 'deny', target: 'host.orb.internal' },
{ action: 'deny', target: 'docker.orb.internal' },
......@@ -253,7 +265,7 @@ describe('sandbox provider config', () => {
metadata: { teamId: 'team-1' },
networkPolicy: {
defaultAction: 'allow',
egress: [{ action: 'deny', target: 'host.docker.internal' }]
egress: defaultOpenSandboxNetworkPolicy.egress
},
extensions: {
traceId: 'trace-1'
......@@ -283,7 +295,7 @@ describe('sandbox provider config', () => {
memoryMiB: 4096
},
readyTimeoutSeconds: 120,
networkPolicy: defaultOpenSandboxDockerNetworkPolicy
networkPolicy: defaultOpenSandboxNetworkPolicy
});
expect(
......@@ -305,6 +317,46 @@ describe('sandbox provider config', () => {
});
});
it('applies the protected network policy for kubernetes runtime as well', async () => {
vi.stubEnv('AGENT_SANDBOX_OPENSANDBOX_RUNTIME', 'kubernetes');
vi.stubEnv('AGENT_SANDBOX_OPENSANDBOX_IMAGE', 'default-opensandbox-image:stable');
vi.resetModules();
const { getSandboxRuntimeProfile } =
await import('@fastgpt/service/core/ai/sandbox/infrastructure/provider/runtimeProfile');
const profile = getSandboxRuntimeProfile('opensandbox');
expect(profile.buildConfig()?.networkPolicy).toEqual(defaultOpenSandboxNetworkPolicy);
});
it('does not allow a caller policy to open protected targets or close public egress', async () => {
vi.stubEnv('AGENT_SANDBOX_OPENSANDBOX_RUNTIME', 'docker');
vi.stubEnv('AGENT_SANDBOX_OPENSANDBOX_IMAGE', 'default-opensandbox-image:stable');
vi.resetModules();
const { getSandboxRuntimeProfile } =
await import('@fastgpt/service/core/ai/sandbox/infrastructure/provider/runtimeProfile');
const profile = getSandboxRuntimeProfile('opensandbox');
expect(
profile.buildConfig({
createConfig: {
networkPolicy: {
defaultAction: 'deny',
egress: [
{ action: 'allow', target: '10.0.0.1' },
{ action: 'deny', target: 'api.example.com' }
]
}
}
})?.networkPolicy
).toEqual({
defaultAction: 'allow',
egress: [
...defaultOpenSandboxNetworkPolicy.egress,
{ action: 'deny', target: 'api.example.com' }
]
});
});
it('validates sealosdevbox token requirement', async () => {
const { validateSandboxConfig } = await loadSandboxConfigModule();
......
Subproject commit a72dbb1b3614fbb57cbe2e9765a92c6a4ad2b16e
Subproject commit 97d9bb623b33ff23d76810a873c3c90168254e36
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment