Commit 289da0f7 by Ryo Committed by GitHub

chore: bump pro submodule for hydration stability (#6808)

* sandbox-sync-agent

* refactor: host pro as submodule

* chore: checkpoint host pro restructure

* refactor workspace test layout and startup init

* chore: update next turbopack setup

* chore: snapshot current work before actions fix

* chore: update pro submodule

* chore: point pro submodule url to upstream https

* fix: Dockerfile

* chore: update pro submodule

* ci: support private pro submodule token and skip fork jobs

* fix(ci): build sdk workspace deps before code-sandbox bundle

* fix(app): exclude vitest configs from production typecheck

* fix(app-image): build sdk packages before next build

* fix(ci): align dockerfiles with workspace sdk build flow

* chore(docker): upgrade node20 docker images to node24

* fix(ci): read admin coverage output path in pro test workflow

* fix(app-image): include next-i18next config and locale assets

* chore: update pro submodule

* chore: do not specify branch for submodule

* chore: remove most ts-nocheck sign

* chore: update pro submodule

* chore: remove sandbox-agent-sync package

* chore: do not modify "pushData" file logic

* fix: health check

* chore: restore dev axios proxy state

* fix: test-fastgpt report workflow

* fix: use valid vitest coverage action inputs
parent e32410b9
name: Build fastgpt-sso-service images
on:
workflow_dispatch:
push:
tags:
- 'v*'
permissions:
contents: read
packages: write
attestations: write
id-token: write
jobs:
build-fastgpt-sso-service-images:
runs-on: buildjet-2vcpu-ubuntu-2204
steps:
- name: Checkout
uses: actions/checkout@v3
with:
fetch-depth: 1
- name: Update submodules
env:
PRO_SUBMODULE_TOKEN: ${{ secrets.PRO_SUBMODULE_TOKEN }}
run: |
if [ -f .gitmodules ]; then
if [ -n "${PRO_SUBMODULE_TOKEN}" ]; then
git config --global url."https://x-access-token:${PRO_SUBMODULE_TOKEN}@github.com/".insteadOf "https://github.com/"
fi
git submodule update --init --recursive
fi
- name: Install Dependencies
run: |
sudo apt update && sudo apt install -y nodejs npm
- name: Set up QEMU
uses: docker/setup-qemu-action@v2
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
with:
driver-opts: network=host
- name: Cache Docker layers
uses: actions/cache@v4
with:
path: /tmp/.buildx-cache
key: ${{ runner.os }}-buildx-${{ github.sha }}
restore-keys: |
${{ runner.os }}-buildx-
- name: Login to GitHub Container Registry
uses: docker/login-action@v2
with:
registry: ghcr.io
username: labring
password: ${{ secrets.GITHUB_TOKEN }}
- name: Login to Ali Hub
uses: docker/login-action@v2
with:
registry: registry.cn-hangzhou.aliyuncs.com
username: ${{ secrets.FASTGPT_ALI_IMAGE_USER }}
password: ${{ secrets.FASTGPT_ALI_IMAGE_PSW }}
- name: Set image tags
run: |
if [[ "${{ github.ref_name }}" == "main" ]]; then
echo "Git_Latest=ghcr.io/labring/fastgpt-sso-service:latest" >> $GITHUB_ENV
echo "Git_Tag=ghcr.io/labring/fastgpt-sso-service:latest" >> $GITHUB_ENV
echo "Ali_Latest=${{ secrets.FASTGPT_ALI_IMAGE_PREFIX }}/fastgpt-sso-service:latest" >> $GITHUB_ENV
echo "Ali_Tag=${{ secrets.FASTGPT_ALI_IMAGE_PREFIX }}/fastgpt-sso-service:latest" >> $GITHUB_ENV
else
echo "Git_Tag=ghcr.io/labring/fastgpt-sso-service:${{ github.ref_name }}" >> $GITHUB_ENV
echo "Git_Latest=ghcr.io/labring/fastgpt-sso-service:latest" >> $GITHUB_ENV
echo "Ali_Tag=${{ secrets.FASTGPT_ALI_IMAGE_PREFIX }}/fastgpt-sso-service:${{ github.ref_name }}" >> $GITHUB_ENV
echo "Ali_Latest=${{ secrets.FASTGPT_ALI_IMAGE_PREFIX }}/fastgpt-sso-service:latest" >> $GITHUB_ENV
fi
- name: Build and publish image
run: |
docker buildx build \
-f pro/sso/Dockerfile \
--build-arg name=sso \
--platform linux/amd64,linux/arm64 \
--label "org.opencontainers.image.source=https://github.com/labring/FastGPT" \
--label "org.opencontainers.image.description=fastgpt-sso-service image" \
--push \
--cache-from=type=local,src=/tmp/.buildx-cache \
--cache-to=type=local,dest=/tmp/.buildx-cache \
-t ${Git_Tag} \
-t ${Git_Latest} \
-t ${Ali_Tag} \
-t ${Ali_Latest} \
.
name: 'FastGPT-Pro-Test'
on:
pull_request:
workflow_dispatch:
concurrency:
group: 'fastgpt-pro-test-${{ github.event.pull_request.number || github.ref }}'
cancel-in-progress: true
jobs:
test:
if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository || secrets.PRO_SUBMODULE_TOKEN != '' }}
runs-on: ubuntu-latest
permissions:
contents: read
pull-requests: write
steps:
- uses: actions/checkout@v4
with:
ref: ${{ github.event.pull_request.head.ref || github.ref }}
repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }}
- name: Update submodules
env:
PRO_SUBMODULE_TOKEN: ${{ secrets.PRO_SUBMODULE_TOKEN }}
run: |
if [ -f .gitmodules ]; then
if [ -n "${PRO_SUBMODULE_TOKEN}" ]; then
git config --global url."https://x-access-token:${PRO_SUBMODULE_TOKEN}@github.com/".insteadOf "https://github.com/"
fi
git submodule update --init --recursive
fi
- name: Install system deps for node-canvas
run: |
sudo apt-get update
sudo apt-get install -y libcairo2-dev libpango1.0-dev libjpeg-dev libgif-dev librsvg2-dev
- uses: pnpm/action-setup@v4
with:
version: 10.33.2
- uses: actions/setup-node@v4
with:
node-version: '20'
cache: 'pnpm'
- name: Install Deps
run: pnpm install --frozen-lockfile
- name: Test
run: pnpm test:admin
- name: Report Coverage
if: always() && hashFiles('pro/admin/coverage/coverage-summary.json') != ''
uses: davelosert/vitest-coverage-report-action@v2
with:
json-final-path: pro/admin/coverage/coverage-final.json
json-summary-path: pro/admin/coverage/coverage-summary.json
name: Preview Admin Image - Build & Push
on:
pull_request_target:
types: [opened, synchronize, reopened]
workflow_dispatch:
concurrency:
group: 'preview-admin-build-${{ github.head_ref }}'
cancel-in-progress: true
permissions:
contents: read
packages: write
pull-requests: write
issues: write
jobs:
build-and-push:
if: ${{ github.event.pull_request.head.repo.full_name == github.repository || secrets.PRO_SUBMODULE_TOKEN != '' }}
runs-on: ubuntu-24.04
steps:
- name: Checkout PR code
uses: actions/checkout@v4
with:
ref: refs/pull/${{ github.event.pull_request.number }}/head
fetch-depth: 0
- name: Update submodules
env:
PRO_SUBMODULE_TOKEN: ${{ secrets.PRO_SUBMODULE_TOKEN }}
run: |
if [ -f .gitmodules ]; then
if [ -n "${PRO_SUBMODULE_TOKEN}" ]; then
git config --global url."https://x-access-token:${PRO_SUBMODULE_TOKEN}@github.com/".insteadOf "https://github.com/"
fi
git submodule update --init --recursive
fi
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Login to Aliyun Container Registry
uses: docker/login-action@v3
with:
registry: registry.cn-hangzhou.aliyuncs.com
username: ${{ secrets.FASTGPT_ALI_IMAGE_USER }}
password: ${{ secrets.FASTGPT_ALI_IMAGE_PSW }}
- name: Build and push Docker image
uses: docker/build-push-action@v6
with:
context: .
file: pro/admin/Dockerfile
platforms: linux/amd64
push: true
tags: ${{ secrets.FASTGPT_ALI_IMAGE_PREFIX }}/fastgpt-pro-pr:${{ github.event.pull_request.head.sha }}
labels: |
org.opencontainers.image.source=https://github.com/${{ github.repository_owner }}/FastGPT
org.opencontainers.image.description=fastgpt-pro admin image
- name: Add PR comment on success
if: success()
uses: actions/github-script@v7
with:
script: |
const prNumber = ${{ github.event.pull_request.number }};
const marker = '<!-- fastgpt-admin-preview -->';
const { data: comments } = await github.rest.issues.listComments({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: prNumber,
});
const existingComment = comments.find(comment =>
comment.body.includes(marker)
);
const commentBody = `${marker}
✅ **Admin Preview Image Ready!**
\`\`\`
${{ secrets.FASTGPT_ALI_IMAGE_PREFIX }}/fastgpt-pro-pr:${{ github.event.pull_request.head.sha }}
\`\`\`
`;
if (existingComment) {
await github.rest.issues.updateComment({
owner: context.repo.owner,
repo: context.repo.repo,
comment_id: existingComment.id,
body: commentBody
});
} else {
await github.rest.issues.createComment({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: prNumber,
body: commentBody
});
}
name: 'FastGPT-Test'
name: "FastGPT-Test"
on:
pull_request:
workflow_dispatch:
# Only one build per PR branch at a time
# Only one build per PR branch at a time
concurrency:
group: 'test-fastgpt-${{ github.head_ref }}'
group: "test-fastgpt-${{ github.event.pull_request.number || github.ref }}"
cancel-in-progress: true
permissions:
......@@ -20,18 +20,53 @@ jobs:
steps:
- uses: actions/checkout@v4
with:
ref: ${{ github.event.pull_request.head.ref }}
repository: ${{ github.event.pull_request.head.repo.full_name }}
ref: ${{ github.event.pull_request.head.ref || github.ref }}
repository: ${{ github.event.pull_request.head.repo.full_name ||
github.repository }}
- uses: pnpm/action-setup@v4
with:
version: 9
- name: 'Install Deps'
run: pnpm install
- name: 'Test'
run: pnpm run test
- name: 'Report Coverage'
version: 10.33.2
- uses: actions/setup-node@v4
with:
node-version: "24"
cache: "pnpm"
- name: "Install Deps"
run: pnpm install --frozen-lockfile
- name: "Test Global"
run: pnpm test:global
- name: "Test Service"
run: pnpm test:service
- name: "Test App"
run: pnpm test:app
- name: "Report Coverage (Global)"
# Set if: always() to also generate the report if tests are failing
# Only works if you set `reportOnFailure: true` in your vite config as specified above
if: always()
if: always() && hashFiles('packages/global/coverage/coverage-summary.json') != ''
uses: davelosert/vitest-coverage-report-action@v2
with:
name: global
json-final-path: packages/global/coverage/coverage-final.json
json-summary-path: packages/global/coverage/coverage-summary.json
- name: "Report Coverage (Service)"
if: always() && hashFiles('packages/service/coverage/coverage-summary.json') != ''
uses: davelosert/vitest-coverage-report-action@v2
with:
name: service
json-final-path: packages/service/coverage/coverage-final.json
json-summary-path: packages/service/coverage/coverage-summary.json
- name: "Report Coverage (App)"
if: always() && hashFiles('projects/app/coverage/coverage-summary.json') != ''
uses: davelosert/vitest-coverage-report-action@v2
with:
name: app
json-final-path: projects/app/coverage/coverage-final.json
json-summary-path: projects/app/coverage/coverage-summary.json
......@@ -20,7 +20,7 @@ jobs:
- uses: pnpm/action-setup@v4
with:
version: 9
version: 10.33.2
- uses: actions/setup-node@v4
with:
......
......@@ -2,6 +2,7 @@
node_modules/
# next.js
.next/
.turbo/
out/
# production
build/
......@@ -40,6 +41,9 @@ coverage
document/.source
projects/app/worker/
pro/admin/worker/
# Agent
.codex
.turbo
[submodule "pro"]
path = pro
url = https://github.com/labring/fastgpt-pro.git
......@@ -3,8 +3,8 @@
"editor.mouseWheelZoom": true,
"editor.defaultFormatter": "esbenp.prettier-vscode",
"prettier.prettierPath": "node_modules/prettier",
"typescript.preferences.includePackageJsonAutoImports": "on",
"typescript.tsdk": "node_modules/typescript/lib",
"js/ts.preferences.includePackageJsonAutoImports": "on",
"js/ts.tsdk.path": "node_modules/typescript/lib",
"i18n-ally.localesPaths": [
"packages/web/i18n",
],
......
......@@ -10,10 +10,16 @@ ifndef name
$(error name is not defined)
endif
filePath=./projects/$(name)/Dockerfile
projectDir=$(or $(wildcard ./projects/$(name)),$(wildcard ./pro/$(name)))
ifeq ($(strip $(projectDir)),)
$(error Unknown project name '$(name)'; expected ./projects/$(name) or ./pro/$(name))
endif
filePath=$(projectDir)/Dockerfile
dev:
pnpm --prefix ./projects/$(name) dev
pnpm --prefix $(projectDir) dev
build:
ifeq ($(proxy), taobao)
......
......@@ -60,6 +60,20 @@ docker compose up -d
- **商业版**
如果你需要更完整的功能,或深度的服务支持,可以选择我们的[商业版](https://doc.fastgpt.io/introduction/commercial)。我们除了提供完整的软件外,还提供相应的场景落地辅导,具体可提交[商业咨询](https://fael3z0zfze.feishu.cn/share/base/form/shrcnjJWtKqjOI9NbQTzhNyzljc)
## 📁 仓库结构
- `projects/app`: 开源主应用
- `pro/admin`: 商业版后台
- `pro/sso`: 商业版 SSO 服务
统一在仓库根目录执行 `pnpm i`,开发时使用:
```bash
make dev name=app
make dev name=admin
make dev name=sso
```
## 💡 核心功能
| | |
......
......@@ -5,6 +5,8 @@ Since FastGPT is managed in the same way as monorepo, it is recommended to insta
monorepo Project Name:
- app: main project
- admin: pro admin project
- sso: pro sso service
-......
## Dev
......@@ -21,6 +23,8 @@ pnpm dev
# Make cmd
make dev name=app
make dev name=admin
make dev name=sso
```
Note: If the Node version is >= 20, you need to pass the `--no-node-snapshot` parameter to Node when running `pnpm i`
......@@ -108,11 +112,20 @@ Please fill the AuditEventEnum and audit function is added to the ts, and on the
```sh
# Docker cmd: Build image, not proxy
docker build -f ./projects/app/Dockerfile -t registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt:v4.8.1 . --build-arg name=app
# Docker cmd: Build pro admin image, not proxy
docker build -f ./pro/admin/Dockerfile -t registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-admin:v4.8.1 . --build-arg name=admin
# Docker cmd: Build pro sso image, not proxy
docker build -f ./pro/sso/Dockerfile -t registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-sso-service:v4.8.1 . --build-arg name=sso
# Make cmd: Build image, not proxy
make build name=app image=registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt:v4.8.1
make build name=admin image=registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-admin:v4.8.1
make build name=sso image=registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-sso-service:v4.8.1
# Docker cmd: Build image with proxy
docker build -f ./projects/app/Dockerfile -t registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt:v4.8.1 . --build-arg name=app --build-arg proxy=taobao
# Docker cmd: Build pro admin image with proxy
docker build -f ./pro/admin/Dockerfile -t registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-admin:v4.8.1 . --build-arg name=admin --build-arg proxy=taobao
# Make cmd: Build image with proxy
make build name=app image=registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt:v4.8.1 proxy=taobao
make build name=admin image=registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-admin:v4.8.1 proxy=taobao
```
FROM node:20-alpine AS base
FROM node:24-alpine AS base
FROM base AS builder
RUN apk add --no-cache \
......
{
"name": "fast",
"name": "@fastgpt/document",
"version": "0.0.0",
"private": true,
"scripts": {
......
......@@ -3,6 +3,9 @@
"version": "4.0",
"private": true,
"scripts": {
"dev:app": "turbo run dev --filter=@fastgpt/app",
"dev:admin": "turbo run dev --filter=@fastgpt/admin",
"dev": "turbo run dev --filter=@fastgpt/app",
"prepare": "husky install",
"gen:theme-typings": "chakra-cli tokens packages/web/styles/theme.ts --out node_modules/.pnpm/node_modules/@chakra-ui/styled-system/dist/theming.types.d.ts",
"gen:deploy": "node deploy/init.mjs",
......@@ -10,36 +13,42 @@
"initIcon": "node ./scripts/icon/init.js && prettier --config \"./.prettierrc.js\" --write \"packages/web/components/common/Icon/constants.ts\"",
"previewIcon": "node ./scripts/icon/index.js",
"create:i18n": "node ./scripts/i18n/index.js",
"clean:unused:pro": "node --experimental-strip-types ./pro/scripts/cleanup-unused.ts",
"clean:unused:pro:write": "node --experimental-strip-types ./pro/scripts/cleanup-unused.ts --write",
"lint": "eslint \"**/*.{ts,tsx}\" --fix --ignore-path .eslintignore",
"test": "vitest run --coverage",
"test:vector": "vitest run --config test/integrationTest/vectorDB/vitest.config.mts"
"test": "pnpm test:workspace",
"test:all": "pnpm test:workspace && pnpm test:vector",
"test:repo": "vitest run --config vitest.config.mts --coverage --passWithNoTests",
"test:workspace": "turbo run test --filter=@fastgpt/app --filter=@fastgpt/admin --filter=@fastgpt/global --filter=@fastgpt/service",
"test:app": "turbo run test --filter=@fastgpt/app",
"test:admin": "turbo run test --filter=@fastgpt/admin",
"test:global": "turbo run test --filter=@fastgpt/global",
"test:service": "turbo run test --filter=@fastgpt/service",
"test:service:integration": "turbo run test:integration --filter=@fastgpt/service",
"test:vector": "turbo run test:integration --filter=@fastgpt/service",
"build:sdks": "pnpm -r --filter @fastgpt-sdk/storage --filter @fastgpt-sdk/logger --filter @fastgpt-sdk/otel build",
"predev": "pnpm run build:sdks"
},
"devDependencies": {
"@chakra-ui/cli": "^2.4.1",
"@typescript-eslint/eslint-plugin": "^6.21.0",
"@typescript-eslint/parser": "^6.21.0",
"@vitest/coverage-v8": "^3.0.9",
"@typescript-eslint/eslint-plugin": "catalog:",
"@typescript-eslint/parser": "catalog:",
"@vitest/coverage-v8": "catalog:",
"eslint": "catalog:",
"eslint-config-next": "catalog:",
"husky": "^8.0.3",
"i18next": "catalog:",
"js-yaml": "catalog:",
"lint-staged": "^13.3.0",
"mongodb-memory-server": "^10.1.4",
"mongodb-memory-server": "catalog:",
"next-i18next": "catalog:",
"prettier": "3.2.4",
"react-i18next": "catalog:",
"typescript": "^5.1.3",
"vitest": "^3.0.9",
"zhlint": "^0.7.4"
"turbo": "2.9.6",
"typescript": "catalog:",
"vitest": "catalog:"
},
"lint-staged": {
"./**/**/*.{ts,tsx,scss}": [
"prettier --config ./.prettierrc.js --write --ignore-unknown"
],
"./**/**/*.{ts,tsx}": [
"eslint --fix --ignore-path .eslintignore"
],
"./document/**/**/*.mdx": [
"pnpm -C ./document run format-doc",
"pnpm -C ./document run initDocTime",
......@@ -47,10 +56,18 @@
"pnpm -C ./document run checkDocRefs",
"pnpm -C ./document run removeInvalidImg",
"git add ."
],
"**/*.{ts,tsx}": [
"prettier --config ./.prettierrc.js --write",
"eslint --fix --ignore-path .eslintignore"
],
"**/*.scss": [
"prettier --config ./.prettierrc.js --write"
]
},
"engines": {
"node": ">=20",
"pnpm": "9.x"
}
"pnpm": "10.x"
},
"packageManager": "pnpm@10.33.2"
}
......@@ -21,6 +21,7 @@ export const APIFileServerSchema = z
})
.meta({ description: 'API 服务器配置' });
export type APIFileServerType = z.infer<typeof APIFileServerSchema>;
export type APIFileServer = APIFileServerType;
export const FeishuServerSchema = z
.object({
appId: z.string(),
......@@ -29,6 +30,7 @@ export const FeishuServerSchema = z
})
.meta({ description: '飞书服务器配置' });
export type FeishuServerType = z.infer<typeof FeishuServerSchema>;
export type FeishuServer = FeishuServerType;
export const YuqueServerSchema = z
.object({
userId: z.string(),
......@@ -37,6 +39,7 @@ export const YuqueServerSchema = z
})
.meta({ description: '语雀服务器配置' });
export type YuqueServerType = z.infer<typeof YuqueServerSchema>;
export type YuqueServer = YuqueServerType;
export const ApiDatasetServerSchema = z
.object({
......@@ -53,10 +56,12 @@ export const ApiFileReadContentResponseSchema = z.object({
rawText: z.string()
});
export type ApiFileReadContentResponseType = z.infer<typeof ApiFileReadContentResponseSchema>;
export type ApiFileReadContentResponse = ApiFileReadContentResponseType;
export const APIFileReadResponseSchema = z.object({
url: z.string()
});
export type APIFileReadResponseType = z.infer<typeof APIFileReadResponseSchema>;
export type APIFileReadResponse = APIFileReadResponseType;
export type ApiDatasetDetailResponse = APIFileItemType;
import { AdminInformPath } from './inform';
import { AdminLoginPath } from './login';
import { AdminInformPath } from './inform';
import type { OpenAPIPath } from '../../../type';
export const AdminUserPath: OpenAPIPath = {
......
......@@ -30,7 +30,6 @@ export type PaginationResponseType<T = any> = {
total: number;
list: T[];
};
export type PaginationResponse<T = any> = PaginationResponseType<T>;
/* 按 cursor 分页 */
......@@ -88,3 +87,6 @@ export type LinkedListResponse<T = {}, A = any> = {
hasMorePrev: boolean;
hasMoreNext: boolean;
};
// Backward-compatible alias for older callers that still import PaginationResponse.
export type PaginationResponse<T = any> = PaginationResponseType<T>;
{
"name": "@fastgpt/global",
"version": "1.0.0",
"scripts": {
"test": "vitest run -c vitest.config.ts",
"test:watch": "vitest -c vitest.config.ts"
},
"engines": {
"node": ">=20",
"pnpm": "9.x"
"pnpm": "10.x"
},
"dependencies": {
"@fastgpt-sdk/plugin": "0.6.0",
......@@ -16,7 +20,7 @@
"js-yaml": "catalog:",
"jschardet": "3.1.1",
"json5": "catalog:",
"nanoid": "^5.1.3",
"nanoid": "catalog:",
"next": "catalog:",
"openai": "4.104.0",
"openapi-types": "^12.1.3",
......@@ -27,7 +31,7 @@
},
"devDependencies": {
"@types/lodash": "catalog:",
"@types/js-yaml": "^4.0.9",
"@types/node": "20.14.0"
"@types/js-yaml": "catalog:",
"@types/node": "catalog:"
}
}
export * from '@fastgpt-sdk/plugin';
export {
FastGPTPluginClient,
RunToolWithStream,
ToolDetailSchema,
ToolSimpleSchema,
ToolTagsNameMap
} from '@fastgpt-sdk/plugin';
export type {
AIProxyChannelsType,
I18nStringStrictType,
ToolDetailType,
ToolSimpleType
} from '@fastgpt-sdk/plugin';
......@@ -123,6 +123,8 @@ export type OutLinkEditType<T extends OutlinkAppType = undefined> = {
app?: T;
};
export type OutLinkSchema<T extends OutlinkAppType = undefined> = OutLinkSchemaType<T>;
export const PlaygroundVisibilityConfigSchema = z.object({
showRunningStatus: z.boolean(),
showSkillReferences: z.boolean().optional().default(true),
......
import type { CollaboratorIdType, CollaboratorItemType } from './collaborator';
import { ManageRoleVal, OwnerRoleVal } from './constant';
import type { RoleValueType } from './type';
import { type PermissionValueType } from './type';
const OwnerRoleVal = ~0 >>> 0;
const ManageRoleVal = 0b001;
/**
* Sum the permission value.
* If no permission value is provided, return undefined to fallback to default value.
......
{
"extends": "../tsconfig.json",
"compilerOptions": {
"baseUrl": "..",
"paths": {
"@/*": ["../../packages/*"],
"@fastgpt-sdk/logger": ["../../sdk/logger/src/index.ts"],
"@fastgpt-sdk/storage": ["../../sdk/storage/src/index.ts"],
"@fastgpt-sdk/otel": ["../../sdk/otel/src/index.ts"],
"@fastgpt-sdk/otel/logger": ["../../sdk/otel/src/logger-entry.ts"],
"@fastgpt-sdk/otel/metrics": ["../../sdk/otel/src/metrics-entry.ts"],
"@fastgpt-sdk/otel/tracing": ["../../sdk/otel/src/tracing-entry.ts"],
"@fastgpt/*": ["../../packages/*"],
"@test/*": ["../../test/*"]
}
},
"include": ["**/*.test.ts", "../**/*.ts", "../**/*.tsx"],
"exclude": ["node_modules"]
}
import { resolve } from 'node:path';
import { defineConfig } from 'vitest/config';
export default defineConfig({
resolve: {
alias: {
'@': resolve('..'),
'@fastgpt-sdk/logger': resolve('../../sdk/logger/src/index.ts'),
'@fastgpt-sdk/storage': resolve('../../sdk/storage/src/index.ts'),
'@fastgpt-sdk/otel/logger': resolve('../../sdk/otel/src/logger-entry.ts'),
'@fastgpt-sdk/otel/metrics': resolve('../../sdk/otel/src/metrics-entry.ts'),
'@fastgpt-sdk/otel/tracing': resolve('../../sdk/otel/src/tracing-entry.ts'),
'@fastgpt-sdk/otel': resolve('../../sdk/otel/src/index.ts'),
'@fastgpt': resolve('..'),
'@test': resolve('../../test')
}
},
test: {
coverage: {
enabled: true,
reporter: ['text', 'text-summary', 'html', 'json-summary', 'json'],
reportOnFailure: true,
include: ['common/**/*.ts', 'core/**/*.ts', 'support/**/*.ts', 'openapi/**/*.ts'],
exclude: [
'**/node_modules/**',
'**/*.spec.ts',
'**/*/*.d.ts',
'**/test/**',
'**/*.test.ts',
'**/*/constants.ts',
'**/*/*.const.ts',
'**/*/type.ts',
'**/*/types.ts',
'**/*/type/*',
'**/*/schema.ts',
'**/*/*.schema.ts',
'openapi/**/*',
'core/workflow/template/**/*'
],
cleanOnRerun: false
},
outputFile: 'test-results.json',
include: ['test/**/*.test.ts']
}
});
......@@ -3,10 +3,11 @@ import { ProxyAgent } from 'proxy-agent';
import { isDevEnv } from '@fastgpt/global/common/system/constants';
export function createProxyAxios(config?: AxiosRequestConfig) {
const agent = new ProxyAgent();
if (isDevEnv) {
return _.create(config);
}
const agent = new ProxyAgent();
return _.create({
proxy: false,
......
......@@ -3,6 +3,7 @@ import {
type Processor,
Queue,
type QueueOptions,
UnrecoverableError,
Worker,
type WorkerOptions
} from 'bullmq';
......@@ -150,4 +151,5 @@ export function getWorker<DataType, ReturnType = void>(
return newWorker;
}
export * from 'bullmq';
export { Queue, UnrecoverableError, Worker, delay };
export type { ConnectionOptions, Job, Processor, QueueOptions, WorkerOptions } from 'bullmq';
import fs from 'node:fs';
import path from 'node:path';
import { fileURLToPath } from 'node:url';
import type { LocationName } from './type';
export const dbPath = path.join(process.cwd(), 'data/GeoLite2-City.mmdb');
const dbFileName = 'GeoLite2-City.mmdb';
const currentDir = path.dirname(fileURLToPath(import.meta.url));
const dbPathCandidates = [
path.resolve(currentDir, '../../../../projects/app/data', dbFileName),
path.resolve(process.cwd(), 'data', dbFileName),
path.resolve(process.cwd(), '../../projects/app/data', dbFileName)
];
export const dbPath = dbPathCandidates.find((item) => fs.existsSync(item)) ?? dbPathCandidates[0];
export const privateOrOtherLocationName: LocationName = {
city: undefined,
......
import { isTestEnv } from '@fastgpt/global/common/system/constants';
import { getLogger, LogCategories } from '../logger';
import type { Model } from 'mongoose';
import type {
AnyBulkWriteOperation,
ClientSession,
Model,
Mongoose as MongooseType,
PipelineStage
} from 'mongoose';
import mongoose, { Mongoose } from 'mongoose';
const logger = getLogger(LogCategories.INFRA.MONGO);
export default mongoose;
export * from 'mongoose';
export { Schema, Types } from 'mongoose';
export type {
AnyBulkWriteOperation,
ClientSession,
Model,
MongooseType as Mongoose,
PipelineStage
};
export const MONGO_URL = process.env.MONGODB_URI as string;
export const MONGO_LOG_URL = (process.env.MONGODB_LOG_URI ?? process.env.MONGODB_URI) as string;
......
import { getLogger, LogCategories } from '../logger';
import Redis from 'ioredis';
import type { RedisOptions } from 'ioredis';
const logger = getLogger(LogCategories.INFRA.REDIS);
......@@ -41,9 +42,60 @@ const REDIS_BASE_OPTION = {
enableOfflineQueue: true
};
export const newQueueRedisConnection = () => {
const redis = new Redis(REDIS_URL, {
const getRedisConnectionOptions = (): RedisOptions => {
if (REDIS_URL.startsWith('/')) {
return {
...REDIS_BASE_OPTION,
path: REDIS_URL
};
}
const normalizedRedisUrl = REDIS_URL.includes('://') ? REDIS_URL : `redis://${REDIS_URL}`;
try {
const redisUrl = new URL(normalizedRedisUrl);
const protocol = redisUrl.protocol.toLowerCase();
if (protocol !== 'redis:' && protocol !== 'rediss:') {
logger.warn('Unsupported Redis URL protocol, fallback to defaults', {
protocol,
redisUrl: REDIS_URL
});
return {
...REDIS_BASE_OPTION
};
}
const dbFromPath = redisUrl.pathname.replace(/^\//, '');
const parsedDb = dbFromPath ? Number(dbFromPath) : undefined;
const db = Number.isFinite(parsedDb) ? parsedDb : undefined;
const options: RedisOptions = {
...REDIS_BASE_OPTION,
host: redisUrl.hostname || 'localhost',
port: redisUrl.port ? Number(redisUrl.port) : 6379
};
if (redisUrl.username) options.username = decodeURIComponent(redisUrl.username);
if (redisUrl.password) options.password = decodeURIComponent(redisUrl.password);
if (db !== undefined) options.db = db;
if (protocol === 'rediss:') options.tls = {};
return options;
} catch (error) {
logger.warn('Failed to parse REDIS_URL with WHATWG URL API, fallback to defaults', {
redisUrl: REDIS_URL,
error: String(error)
});
return {
...REDIS_BASE_OPTION
};
}
};
export const newQueueRedisConnection = () => {
const redis = new Redis({
...getRedisConnectionOptions(),
// Limit retries for queue operations
maxRetriesPerRequest: 3
});
......@@ -51,8 +103,8 @@ export const newQueueRedisConnection = () => {
};
export const newWorkerRedisConnection = () => {
const redis = new Redis(REDIS_URL, {
...REDIS_BASE_OPTION,
const redis = new Redis({
...getRedisConnectionOptions(),
// BullMQ requires maxRetriesPerRequest: null for blocking operations
maxRetriesPerRequest: null
});
......@@ -63,8 +115,8 @@ export const FASTGPT_REDIS_PREFIX = 'fastgpt:';
export const getGlobalRedisConnection = () => {
if (global.redisClient) return global.redisClient;
global.redisClient = new Redis(REDIS_URL, {
...REDIS_BASE_OPTION,
global.redisClient = new Redis({
...getRedisConnectionOptions(),
keyPrefix: FASTGPT_REDIS_PREFIX,
maxRetriesPerRequest: 3
});
......
const emptyModule = new Proxy(function emptyModule() {}, {
get() {
return emptyModule;
},
apply() {
return undefined;
},
construct() {
return {};
}
});
module.exports = emptyModule;
module.exports.default = emptyModule;
import { inspect } from 'node:util';
type ErrTextGetter = (error: any, def?: string) => string;
type SerializedInitializationError = {
message: string;
name?: string;
code?: string;
stage?: string;
step?: string;
stack?: string;
cause?: SerializedInitializationError;
details: string;
};
type InitializationLogger = {
error: (message: string, payload?: Record<string, unknown>) => void;
info?: (message: string, payload?: Record<string, unknown>) => void;
};
const getObjectMessage = (error: Record<string, any>, fallback: string) => {
const message =
(typeof error.message === 'string' && error.message) ||
(typeof error.msg === 'string' && error.msg) ||
(typeof error.error === 'string' && error.error) ||
(typeof error.code === 'string' && error.code);
return message || fallback;
};
export const serializeInitializationError = (
error: unknown,
depth = 0
): SerializedInitializationError => {
const fallback = 'Unknown initialization error';
if (depth > 5) {
return {
message: 'Max initialization error depth reached',
details: 'Max initialization error depth reached'
};
}
if (error instanceof Error) {
const err = error as Error & {
code?: string;
stage?: string;
step?: string;
cause?: unknown;
};
return {
message: err.message || fallback,
name: err.name,
code: err.code,
stage: err.stage,
step: err.step,
stack: err.stack,
cause: err.cause ? serializeInitializationError(err.cause, depth + 1) : undefined,
details: inspect(error, { depth: 6, breakLength: 120 })
};
}
if (typeof error === 'string') {
return {
message: error || fallback,
details: error || fallback
};
}
if (error && typeof error === 'object') {
const err = error as Record<string, any>;
return {
message: getObjectMessage(err, fallback),
code: typeof err.code === 'string' ? err.code : undefined,
stage: typeof err.stage === 'string' ? err.stage : undefined,
step: typeof err.step === 'string' ? err.step : undefined,
cause: err.cause ? serializeInitializationError(err.cause, depth + 1) : undefined,
details: inspect(error, { depth: 6, breakLength: 120 })
};
}
return {
message: fallback,
details: String(error)
};
};
export const createInitializationError = (
error: unknown,
{
stage,
step,
getErrText
}: {
stage?: string;
step?: string;
getErrText?: ErrTextGetter;
} = {}
) => {
const fallback = 'Unknown initialization error';
const errorText =
getErrText?.(error, fallback) || serializeInitializationError(error).message || fallback;
const labels = [stage, step].filter(Boolean).join(' / ');
const wrappedError = new Error(labels ? `[${labels}]: ${errorText}` : errorText, {
cause: error
});
wrappedError.name = 'SystemInitializationError';
return Object.assign(wrappedError, {
stage,
step
});
};
export const runInitializationStep = async <T>({
step,
action,
stage,
logger,
getErrText,
meta
}: {
step: string;
action: () => Promise<T> | T;
stage?: string;
logger?: InitializationLogger;
getErrText?: ErrTextGetter;
meta?: Record<string, unknown>;
}) => {
try {
return await action();
} catch (error) {
const logPayload = {
step,
stage,
...meta,
...getInitializationErrorLog(error)
};
console.error('System initialization step failed', logPayload);
logger?.error(`System initialization step failed: ${step}`, logPayload);
throw createInitializationError(error, {
stage,
step,
getErrText
});
}
};
export const runBackgroundInitializationStep = ({
step,
action,
stage,
logger,
getErrText,
meta
}: {
step: string;
action: () => Promise<unknown> | unknown;
stage?: string;
logger?: InitializationLogger;
getErrText?: ErrTextGetter;
meta?: Record<string, unknown>;
}) => {
try {
const task = action();
logger?.info?.('System background initialization step started', {
step,
stage,
...meta
});
void Promise.resolve(task).catch((error) => {
const logPayload = {
step,
stage,
...meta,
...getInitializationErrorLog(error)
};
console.error('System background initialization step failed', logPayload);
logger?.error(`System background initialization step failed: ${step}`, logPayload);
});
} catch (error) {
const logPayload = {
step,
stage,
...meta,
...getInitializationErrorLog(error)
};
console.error('System background initialization step failed', logPayload);
logger?.error(`System background initialization step failed: ${step}`, logPayload);
throw createInitializationError(error, {
stage,
step,
getErrText
});
}
};
export const getInitializationErrorLog = (error: unknown) => {
const serialized = serializeInitializationError(error);
return {
errorMessage: serialized.message,
errorName: serialized.name,
errorCode: serialized.code,
errorStage: serialized.stage,
errorStep: serialized.step,
errorStack: serialized.stack,
errorCause: serialized.cause,
errorDetails: serialized.details
};
};
......@@ -55,6 +55,8 @@ SandboxInstanceSchema.index(
{
unique: true,
partialFilterExpression: {
// Keep the index compatible with Mongo-compatible backends that do not
// support `$ne: null` inside partial indexes.
appId: { $exists: true },
userId: { $exists: true },
chatId: { $exists: true }
......
......@@ -2,28 +2,34 @@
"name": "@fastgpt/service",
"version": "1.0.0",
"type": "module",
"scripts": {
"test": "vitest run -c vitest.config.ts",
"test:watch": "vitest -c vitest.config.ts",
"test:integration": "vitest run -c vitest.integration.config.ts",
"test:integration:watch": "vitest -c vitest.integration.config.ts"
},
"engines": {
"node": ">=20",
"pnpm": "9.x"
"pnpm": "10.x"
},
"dependencies": {
"@apidevtools/json-schema-ref-parser": "^11.7.2",
"@fastgpt-sdk/otel": "catalog:",
"@fastgpt-sdk/otel": "workspace:*",
"@fastgpt-sdk/sandbox-adapter": "^0.0.36",
"@fastgpt-sdk/storage": "catalog:",
"@fastgpt-sdk/storage": "workspace:*",
"@fastgpt/global": "workspace:*",
"@mariozechner/pi-agent-core": "^0.67.3",
"@mariozechner/pi-ai": "^0.67.3",
"@maxmind/geoip2-node": "^6.3.4",
"@modelcontextprotocol/sdk": "catalog:",
"@node-rs/jieba": "2.0.1",
"@node-rs/jieba": "catalog:",
"@opentelemetry/api": "^1.9.0",
"@t3-oss/env-core": "0.13.10",
"@xmldom/xmldom": "^0.8.10",
"@zilliz/milvus2-sdk-node": "2.4.10",
"axios": "catalog:",
"bullmq": "^5.52.2",
"chalk": "^5.3.0",
"chalk": "catalog:",
"cheerio": "1.0.0-rc.12",
"cookie": "^0.7.1",
"date-fns": "catalog:",
......@@ -42,14 +48,14 @@
"json5": "catalog:",
"jsonpath-plus": "^10.3.0",
"jsonrepair": "^3.0.0",
"jsonwebtoken": "^9.0.2",
"jsonwebtoken": "catalog:",
"jszip": "^3.10.1",
"lodash": "catalog:",
"mammoth": "^1.11.0",
"mime": "catalog:",
"mime-types": "catalog:",
"minio": "catalog:",
"mongoose": "^8.10.1",
"mongoose": "catalog:",
"multer": "2.1.0",
"mysql2": "^3.11.3",
"next": "catalog:",
......@@ -64,7 +70,7 @@
"pino-opentelemetry-transport": "^1.0.1",
"proxy-agent": "catalog:",
"proxy-from-env": "^1.1.0",
"request-ip": "^3.3.0",
"request-ip": "catalog:",
"tiktoken": "1.0.17",
"tunnel": "^0.0.6",
"turndown": "^7.1.2",
......@@ -76,7 +82,7 @@
"@types/async-retry": "^1.4.9",
"@types/cookie": "^0.5.2",
"@types/decompress": "^4.2.7",
"@types/jsonwebtoken": "^9.0.3",
"@types/jsonwebtoken": "catalog:",
"@types/lodash": "catalog:",
"@types/mime-types": "catalog:",
"@types/multer": "^1.4.10",
......@@ -84,7 +90,7 @@
"@types/papaparse": "5.3.7",
"@types/pg": "^8.6.6",
"@types/proxy-from-env": "^1.0.4",
"@types/request-ip": "^0.0.37",
"@types/request-ip": "catalog:",
"@types/tunnel": "^0.0.4",
"@types/turndown": "^5.0.4"
}
......
import { formatVectors } from '@fastgpt/service/core/ai/embedding/index';
import { describe, expect, it, vi } from 'vitest';
describe('formatVectors function test', () => {
// Helper function to create a normalized vector (L2 norm = 1)
const createNormalizedVector = (length: number): number[] => {
const vector = Array.from({ length }, (_, i) => (i + 1) / length);
const norm = Math.sqrt(vector.reduce((sum, val) => sum + val * val, 0));
return vector.map((val) => val / norm);
};
// Helper function to create an unnormalized vector
const createUnnormalizedVector = (length: number): number[] => {
return Array.from({ length }, (_, i) => (i + 1) * 10);
};
// Helper function to calculate L2 norm
const calculateNorm = (vector: number[]): number => {
return Math.sqrt(vector.reduce((sum, val) => sum + val * val, 0));
};
// Helper function to check if vector is normalized (L2 norm H 1)
const isNormalized = (vector: number[]): boolean => {
const norm = calculateNorm(vector);
return Math.abs(norm - 1) < 1e-10;
};
describe('1536 dimension vectors', () => {
it('should handle normalized 1536-dim vector with normalization=true', () => {
const inputVector = createNormalizedVector(1536);
const result = formatVectors(inputVector, true);
expect(result).toHaveLength(1536);
expect(isNormalized(result)).toBe(true);
// Since input is already normalized, result should be very similar
expect(result).toEqual(
expect.arrayContaining(inputVector.map((val) => expect.closeTo(val, 10)))
);
});
it('should handle normalized 1536-dim vector with normalization=false', () => {
const inputVector = createNormalizedVector(1536);
const result = formatVectors(inputVector, false);
expect(result).toHaveLength(1536);
expect(result).toEqual(inputVector);
expect(isNormalized(result)).toBe(true);
});
it('should handle unnormalized 1536-dim vector with normalization=true', () => {
const inputVector = createUnnormalizedVector(1536);
const result = formatVectors(inputVector, true);
expect(result).toHaveLength(1536);
expect(isNormalized(result)).toBe(true);
// Result should be different from input (normalized)
expect(result).not.toEqual(inputVector);
});
it('should handle unnormalized 1536-dim vector with normalization=false', () => {
const inputVector = createUnnormalizedVector(1536);
const result = formatVectors(inputVector, false);
expect(result).toHaveLength(1536);
expect(result).toEqual(inputVector);
expect(isNormalized(result)).toBe(false);
});
});
describe('Greater than 1536 dimension vectors', () => {
it('should handle normalized >1536-dim vector with normalization=true', () => {
const inputVector = createNormalizedVector(2048);
const result = formatVectors(inputVector, true);
expect(result).toHaveLength(1536);
expect(isNormalized(result)).toBe(true);
// Should be truncated to first 1536 elements and then normalized
expect(result).toEqual(
expect.arrayContaining(inputVector.slice(0, 1536).map((val) => expect.any(Number)))
);
});
it('should handle normalized >1536-dim vector with normalization=false', () => {
const inputVector = createNormalizedVector(2048);
const result = formatVectors(inputVector, true); // Always normalized for >1536 dims
expect(result).toHaveLength(1536);
expect(isNormalized(result)).toBe(true);
// Should be truncated and normalized regardless of normalization flag
});
it('should handle unnormalized >1536-dim vector with normalization=true', () => {
const inputVector = createUnnormalizedVector(2048);
const result = formatVectors(inputVector, true);
expect(result).toHaveLength(1536);
expect(isNormalized(result)).toBe(true);
// Should be truncated to first 1536 elements and then normalized
});
it('should handle unnormalized >1536-dim vector with normalization=false', () => {
const inputVector = createUnnormalizedVector(2048);
const result = formatVectors(inputVector, false); // Always normalized for >1536 dims
expect(result).toHaveLength(1536);
expect(isNormalized(result)).toBe(true);
// Should be truncated and normalized regardless of normalization flag
});
});
describe('Less than 1536 dimension vectors', () => {
it('should handle normalized <1536-dim vector with normalization=true', () => {
const inputVector = createNormalizedVector(512);
const result = formatVectors(inputVector, true);
expect(result).toHaveLength(1536);
expect(isNormalized(result)).toBe(true);
// First 512 elements should match input, rest should be 0
expect(result.slice(0, 512)).toEqual(
expect.arrayContaining(inputVector.map((val) => expect.any(Number)))
);
expect(result.slice(512)).toEqual(new Array(1024).fill(0));
});
it('should handle normalized <1536-dim vector with normalization=false', () => {
const inputVector = createNormalizedVector(512);
const result = formatVectors(inputVector, false);
expect(result).toHaveLength(1536);
// First 512 elements should match input exactly, rest should be 0
expect(result.slice(0, 512)).toEqual(inputVector);
expect(result.slice(512)).toEqual(new Array(1024).fill(0));
// The result remains normalized because adding zeros doesn't change the L2 norm
expect(isNormalized(result)).toBe(true);
});
it('should handle unnormalized <1536-dim vector with normalization=true', () => {
const inputVector = createUnnormalizedVector(512);
const result = formatVectors(inputVector, true);
expect(result).toHaveLength(1536);
expect(isNormalized(result)).toBe(true);
// Should be padded with zeros and then normalized
expect(result.slice(512)).toEqual(new Array(1024).fill(0));
});
it('should handle unnormalized <1536-dim vector with normalization=false', () => {
const inputVector = createUnnormalizedVector(512);
const result = formatVectors(inputVector, false);
expect(result).toHaveLength(1536);
// First 512 elements should match input exactly, rest should be 0
expect(result.slice(0, 512)).toEqual(inputVector);
expect(result.slice(512)).toEqual(new Array(1024).fill(0));
expect(isNormalized(result)).toBe(false);
});
it('should demonstrate that padding preserves normalization status', () => {
// Create a vector that becomes unnormalized after some scaling
const baseVector = [3, 4]; // norm = 5, not normalized
const result = formatVectors(baseVector, false);
expect(result).toHaveLength(1536);
expect(result[0]).toBe(3);
expect(result[1]).toBe(4);
expect(result.slice(2)).toEqual(new Array(1534).fill(0));
expect(isNormalized(result)).toBe(false);
expect(calculateNorm(result)).toBeCloseTo(5, 10);
});
});
describe('Edge cases', () => {
it('should handle zero vector', () => {
const inputVector = new Array(1536).fill(0);
const result = formatVectors(inputVector, true);
expect(result).toHaveLength(1536);
expect(result).toEqual(inputVector); // Zero vector remains zero after normalization
});
it('should handle single element vector', () => {
const inputVector = [5.0];
const result = formatVectors(inputVector, true);
expect(result).toHaveLength(1536);
expect(result[0]).toBeCloseTo(1.0, 10); // Normalized single element should be 1
expect(result.slice(1)).toEqual(new Array(1535).fill(0));
});
it('should handle exactly 1536 dimension vector', () => {
const inputVector = createNormalizedVector(1536);
const result = formatVectors(inputVector, true);
expect(result).toHaveLength(1536);
expect(isNormalized(result)).toBe(true);
});
it('should handle vector with negative values', () => {
const inputVector = [-1, -2, -3];
const result = formatVectors(inputVector, true);
expect(result).toHaveLength(1536);
expect(isNormalized(result)).toBe(true);
expect(result[0]).toBeLessThan(0); // Should preserve negative values
expect(result[1]).toBeLessThan(0);
expect(result[2]).toBeLessThan(0);
});
});
});
......@@ -6,7 +6,7 @@ import {
generateSimilarVector,
generateOrthogonalVector,
mockGetVectorsByText
} from '../../../../../mocks/core/ai/embedding';
} from '@test/mocks/core/ai/embedding';
describe('useTextCosine', () => {
beforeEach(() => {
......
import { describe, expect, it } from 'vitest';
import { MongoSandboxInstance } from '@fastgpt/service/core/ai/sandbox/schema';
describe('MongoSandboxInstance schema indexes', () => {
it('uses a Mongo-compatible partial index for chat sandbox uniqueness', () => {
const indexes = MongoSandboxInstance.schema.indexes();
const targetIndex = indexes.find(
([keys]) => keys.appId === 1 && keys.userId === 1 && keys.chatId === 1
);
expect(targetIndex).toBeDefined();
expect(targetIndex?.[1]).toMatchObject({
unique: true,
partialFilterExpression: {
appId: { $exists: true },
userId: { $exists: true },
chatId: { $exists: true }
}
});
expect(JSON.stringify(targetIndex?.[1] ?? {})).not.toContain('$ne');
});
});
# Service Integration Tests
- `vectorDB/`: real vector database integration tests and local compose files
# 向量数据库集成测试
对 FastGPT 各向量库控制器(PGVector、后续 Oceanbase/Milvus)做真实环境下的集成测试,保证向量相关操作兼容和稳定。采用**工厂模式**:同一套数据集(fixtures)和同一套用例(factory)驱动 n 个向量库测试。
对 FastGPT 各向量库控制器(PGVector、后续 Oceanbase/Milvus)做真实环境下的集成测试,保证向量相关操作兼容和稳定。采用**工厂模式**:同一套数据集和同一套用例驱动多个向量库测试。
## 环境变量
测试环境变量由 **test/.env.test.local** 提供(不提交到 git)。请复制模板并填写:
```bash
cp test/.env.test.template test/.env.test.local
cp test/.env.example test/.env.test.local
# 编辑 test/.env.test.local,填入 PG_URL 等
```
......@@ -18,7 +18,6 @@ cp test/.env.test.template test/.env.test.local
| `PG_URL` | PostgreSQL + pgvector 连接串 | PgVectorCtrl |
| `OCEANBASE_URL` | Oceanbase 连接串(后续) | ObVectorCtrl |
| `MILVUS_ADDRESS` | Milvus 地址(后续) | MilvusCtrl |
| `OPENGAUSS_URL` | openGauss DataVec 连接串 | OpenGaussVectorCtrl |
未设置对应环境变量时,该驱动的集成测试会**整体跳过**,不会报错。
......@@ -27,17 +26,20 @@ cp test/.env.test.template test/.env.test.local
在项目根目录执行:
```bash
# 仅运行单元测试(未配置 .env.test.local 或未设 PG_URL 时,vectorDB 集成测试会跳过)
# 仅运行 workspace 默认测试(不包含 integration)
pnpm test
# 运行所有向量库测试(包含 vectorDB 集成测试与相关单元测试)
# 运行 service integration tests
pnpm test:service:integration
# 运行当前这组 vectorDB 集成测试
pnpm test:vector
```
## 结构说明
- **fixtures.ts**:统一测试数据(`TEST_TEAM_ID`、`TEST_DATASET_ID`、`TEST_COLLECTION_ID`、1536 维 `TEST_VECTORS`),所有向量库共用。
- **factory.ts**:工厂函数 `runVectorDBTests(driver)`,同一套用例(init、insert、getVectorCount、embRecall、getVectorDataByTime、delete)供各驱动复用。
- **integration.test.ts**:注册各驱动(PG、后续 Oceanbase/Milvus),按 `driver.envKey` 决定是否跳过;每个驱动执行同一套 `runVectorDBTests(driver)`。
- `testData.ts`:统一测试数据(1536 维 `TEST_VECTORS` 等),所有向量库共用。
- `testSuites.ts`:工厂函数 `createVectorDBTestSuite(vectorCtrl)`,同一套用例供各驱动复用。
- `*/index.integration.test.ts`:各向量库入口,按环境变量决定是否跳过。
新增向量库时:在 `integration.test.ts` 的 `drivers` 数组中增加一项(`name`、`envKey`、`createCtrl`),无需改 fixtures 或 factory。
新增向量库时:新增一个 `*/index.integration.test.ts`,复用 `testData.ts` 和 `testSuites.ts` 即可。
import { beforeAll, describe, expect, test } from 'vitest';
import type { VectorControllerType } from '@fastgpt/service/common/vectorDB/type';
import { createTestIds, QUERY_VECTOR, TEST_COLLECTION_IDS, TEST_VECTORS } from './testData';
const insertTestVectors = async (
vectorCtrl: VectorControllerType,
teamId: string,
datasetId: string
) => {
const insertIds: string[] = [];
await Promise.all(
TEST_VECTORS.map(async (vector, index) => {
const { insertIds: ids } = await vectorCtrl.insert({
teamId,
datasetId,
collectionId: TEST_COLLECTION_IDS[index],
vectors: [vector]
});
insertIds.push(ids[0]);
})
);
await new Promise((resolve) => setTimeout(resolve, 500));
return insertIds;
};
const cleanupTestVectors = async (
vectorCtrl: VectorControllerType,
teamId: string,
datasetId: string
) => {
try {
await vectorCtrl.delete({
teamId,
datasetIds: [datasetId]
});
} catch (error) {
// Ignore cleanup errors
}
};
export const createVectorDBTestSuite = (vectorCtrl: VectorControllerType) => {
describe.sequential('vectorDB integration', () => {
beforeAll(async () => {
await vectorCtrl.init();
});
test('insert and count', async () => {
const { teamId, datasetId } = createTestIds();
const insertIds = await insertTestVectors(vectorCtrl, teamId, datasetId);
expect(insertIds).toHaveLength(TEST_VECTORS.length);
const count = await vectorCtrl.getVectorCount({ teamId, datasetId });
expect(count).toBe(TEST_VECTORS.length);
const collectionCount = await vectorCtrl.getVectorCount({
teamId,
datasetId,
collectionId: TEST_COLLECTION_IDS[0]
});
expect(collectionCount).toBe(1);
await cleanupTestVectors(vectorCtrl, teamId, datasetId);
});
test('embRecall returns results', async () => {
const { teamId, datasetId } = createTestIds();
await insertTestVectors(vectorCtrl, teamId, datasetId);
const { results } = await vectorCtrl.embRecall({
teamId,
datasetIds: [datasetId],
vector: QUERY_VECTOR,
limit: 3,
forbidCollectionIdList: []
});
expect(results.length).toBeGreaterThan(0);
expect(results.every((item) => TEST_COLLECTION_IDS.includes(item.collectionId))).toBe(true);
await cleanupTestVectors(vectorCtrl, teamId, datasetId);
});
test('embRecall respects forbidCollectionIdList', async () => {
const { teamId, datasetId } = createTestIds();
await insertTestVectors(vectorCtrl, teamId, datasetId);
const { results } = await vectorCtrl.embRecall({
teamId,
datasetIds: [datasetId],
vector: QUERY_VECTOR,
limit: 10,
forbidCollectionIdList: [TEST_COLLECTION_IDS[0]]
});
expect(results.length).toBeGreaterThan(0);
expect(results.every((item) => item.collectionId !== TEST_COLLECTION_IDS[0])).toBe(true);
await cleanupTestVectors(vectorCtrl, teamId, datasetId);
});
test('embRecall respects filterCollectionIdList', async () => {
const { teamId, datasetId } = createTestIds();
await insertTestVectors(vectorCtrl, teamId, datasetId);
const { results } = await vectorCtrl.embRecall({
teamId,
datasetIds: [datasetId],
vector: QUERY_VECTOR,
limit: 10,
forbidCollectionIdList: [],
filterCollectionIdList: [TEST_COLLECTION_IDS[1]]
});
expect(results.length).toBeGreaterThan(0);
expect(results.every((item) => item.collectionId === TEST_COLLECTION_IDS[1])).toBe(true);
await cleanupTestVectors(vectorCtrl, teamId, datasetId);
});
test('getVectorDataByTime returns data', async () => {
const { teamId, datasetId } = createTestIds();
const insertIds = await insertTestVectors(vectorCtrl, teamId, datasetId);
await new Promise((resolve) => setTimeout(resolve, 500));
const start = new Date(0);
const end = new Date(Date.now() + 600_000);
const data = await vectorCtrl.getVectorDataByTime(start, end);
const matchedIds = data
.filter((item) => item.teamId === teamId && item.datasetId === datasetId)
.map((item) => item.id);
expect(matchedIds.length).toBeGreaterThan(0);
expect(matchedIds).toEqual(expect.arrayContaining(insertIds));
await cleanupTestVectors(vectorCtrl, teamId, datasetId);
});
test('delete by idList removes vectors', async () => {
const { teamId, datasetId } = createTestIds();
const insertIds = await insertTestVectors(vectorCtrl, teamId, datasetId);
await vectorCtrl.delete({
teamId,
idList: insertIds.slice(0, 2)
});
const count = await vectorCtrl.getVectorCount({ teamId, datasetId });
expect(count).toBe(TEST_VECTORS.length - 2);
await cleanupTestVectors(vectorCtrl, teamId, datasetId);
});
});
};
{
"extends": "../tsconfig.json",
"compilerOptions": {
"baseUrl": "..",
"paths": {
"@/*": ["../../projects/app/src/*"],
"@fastgpt-sdk/logger": ["../../sdk/logger/src/index.ts"],
"@fastgpt-sdk/storage": ["../../sdk/storage/src/index.ts"],
"@fastgpt-sdk/otel": ["../../sdk/otel/src/index.ts"],
"@fastgpt-sdk/otel/logger": ["../../sdk/otel/src/logger-entry.ts"],
"@fastgpt-sdk/otel/metrics": ["../../sdk/otel/src/metrics-entry.ts"],
"@fastgpt-sdk/otel/tracing": ["../../sdk/otel/src/tracing-entry.ts"],
"@fastgpt/*": ["../../packages/*"],
"@test/*": ["../../test/*"]
}
},
"include": ["**/*.test.ts", "../**/*.ts", "../**/*.tsx", "../../test/**/*.ts"],
"exclude": ["node_modules"]
}
import { resolve } from 'node:path';
import { configDefaults, defineConfig } from 'vitest/config';
export default defineConfig({
resolve: {
alias: {
'@': resolve('../../projects/app/src'),
'@fastgpt-sdk/logger': resolve('../../sdk/logger/src/index.ts'),
'@fastgpt-sdk/storage': resolve('../../sdk/storage/src/index.ts'),
'@fastgpt-sdk/otel/logger': resolve('../../sdk/otel/src/logger-entry.ts'),
'@fastgpt-sdk/otel/metrics': resolve('../../sdk/otel/src/metrics-entry.ts'),
'@fastgpt-sdk/otel/tracing': resolve('../../sdk/otel/src/tracing-entry.ts'),
'@fastgpt-sdk/otel': resolve('../../sdk/otel/src/index.ts'),
'@fastgpt': resolve('..'),
'@test': resolve('../../test')
}
},
test: {
env: {
FILE_TOKEN_KEY:
process.env.FILE_TOKEN_KEY ??
'bfd697e7e798f75deaf2d31210bc93a2e41ad4eed9e7831071d77821b7b97cff'
},
coverage: {
enabled: true,
reporter: ['text', 'text-summary', 'html', 'json-summary', 'json'],
reportOnFailure: true,
include: ['common/**/*.ts', 'core/**/*.ts', 'support/**/*.ts', 'worker/**/*.ts'],
exclude: [
'**/node_modules/**',
'**/*.spec.ts',
'**/*/*.d.ts',
'**/test/**',
'**/*.test.ts',
'**/*/constants.ts',
'**/*/*.const.ts',
'**/*/type.ts',
'**/*/types.ts',
'**/*/type/*',
'**/*/schema.ts',
'**/*/*.schema.ts'
],
cleanOnRerun: false
},
outputFile: 'test-results.json',
setupFiles: '../../test/setup.ts',
globalSetup: '../../test/globalSetup.ts',
fileParallelism: false,
maxConcurrency: 10,
pool: 'threads',
testTimeout: 20000,
hookTimeout: 30000,
exclude: [...configDefaults.exclude, 'test/integrations/**/*.test.ts'],
reporters: ['github-actions', 'default'],
include: ['test/**/*.test.ts']
}
});
import { resolve } from 'node:path';
import { defineConfig } from 'vitest/config';
export default defineConfig({
resolve: {
alias: {
'@': resolve('../../projects/app/src'),
'@fastgpt': resolve('..'),
'@test': resolve('../../test')
}
},
test: {
env: {
FILE_TOKEN_KEY:
process.env.FILE_TOKEN_KEY ??
'bfd697e7e798f75deaf2d31210bc93a2e41ad4eed9e7831071d77821b7b97cff'
},
coverage: {
enabled: false
},
outputFile: 'test-results.integration.json',
setupFiles: '../../test/setup.ts',
globalSetup: '../../test/globalSetup.ts',
fileParallelism: false,
maxConcurrency: 10,
pool: 'threads',
testTimeout: 20000,
hookTimeout: 30000,
reporters: ['github-actions', 'default'],
include: ['test/integrations/**/*.integration.test.ts']
}
});
import { Worker } from 'worker_threads';
import type { Worker as NodeWorker } from 'worker_threads';
import path from 'path';
import { getLogger, LogCategories } from '../common/logger';
......@@ -20,13 +20,20 @@ export const getSafeEnv = () => {
};
};
export const getWorker = (name: `${WorkerNameEnum}`) => {
const workerPath = path.join(process.cwd(), 'worker', `${name}.js`);
const createNodeWorker = (workerPath: string) => {
const nodeRequire = eval('require') as (id: string) => typeof import('worker_threads');
const { Worker } = nodeRequire('worker_threads');
return new Worker(workerPath, {
env: getSafeEnv()
});
};
export const getWorker = (name: `${WorkerNameEnum}`) => {
const workerPath = path.join(process.cwd(), 'worker', `${name}.js`);
return createNodeWorker(workerPath);
};
export const runWorker = <T = any>(name: WorkerNameEnum, params?: Record<string, any>) => {
const logger = getLogger(LogCategories.INFRA.WORKER);
return new Promise<T>((resolve, reject) => {
......@@ -60,7 +67,7 @@ export const runWorker = <T = any>(name: WorkerNameEnum, params?: Record<string,
type WorkerRunTaskType<T> = { data: T; resolve: (e: any) => void; reject: (e: any) => void };
type WorkerQueueItem = {
id: string;
worker: Worker;
worker: NodeWorker;
status: 'running' | 'idle';
taskTime: number;
timeoutId?: NodeJS.Timeout;
......
......@@ -2,14 +2,29 @@ import { Box, Tbody } from '@chakra-ui/react';
import React, { type ReactElement, type ReactNode, useState } from 'react';
import {
DragDropContext,
Draggable,
Droppable,
type DraggableChildrenFn,
type DraggableProvided,
type DraggableStateSnapshot,
type DragStart,
type DropResult,
type DroppableProvided,
type DroppableStateSnapshot
type DroppableStateSnapshot,
type Omit
} from 'react-beautiful-dnd';
export * from 'react-beautiful-dnd';
export { Draggable };
export type {
DraggableChildrenFn,
DraggableProvided,
DraggableStateSnapshot,
DragStart,
DropResult,
DroppableProvided,
DroppableStateSnapshot,
Omit
};
type Props<T = any> = {
onDragEndCb: (result: T[]) => void;
......
......@@ -200,18 +200,6 @@
"log_update_dataset": "【{{name}}】Updated [{{datasetType}}] named [{{datasetName}}]",
"log_update_dataset_collaborator": "[{{name}}] Updated the collaborator named [{{datasetName}}] to: Organization: [{{orgList}}], Group: [{{groupList}}], Member [{{tmbList}}]; permissions updated to: [{{readPermission}}], [{{writePermission}}], [{{managePermission}}]",
"log_update_publish_app": "【{{name}}】【{{operationName}}】【{{appType}}】 named [{{appName}}】",
"log_create_skill": "[{{name}}] Created a skill named [{{skillName}}]",
"log_create_skill_folder": "[{{name}}] Created a skill folder named [{{folderName}}]",
"log_delete_skill": "[{{name}}] Deleted the skill named [{{skillName}}]",
"log_deploy_skill": "[{{name}}] Deployed the skill named [{{skillName}}]",
"log_import_skill": "[{{name}}] Imported a skill named [{{skillName}}]",
"log_export_skill": "[{{name}}] Exported a skill named [{{skillName}}]",
"log_copy_skill": "[{{name}}] Copied a skill named [{{skillName}}]",
"log_move_skill": "[{{name}}] Moved a skill named [{{skillName}}] to [{{targetFolderName}}]",
"log_update_skill_collaborator": "[{{name}}] Updated collaborators of [{{skillType}}] [{{skillName}}] to: Organizations: [{{orgList}}], Groups: [{{groupList}}], Members: [{{tmbList}}]; Permission: [{{permission}}]",
"log_delete_skill_collaborator": "[{{name}}] Deleted collaborator [{{itemName}}]: [{{itemValueName}}] from [{{skillType}}] [{{skillName}}]",
"log_transfer_skill_ownership": "[{{name}}] Transferred ownership of skill [{{skillName}}] to [{{newOwnerName}}]",
"log_update_skill": "[{{name}}] Updated the skill named [{{skillName}}]",
"log_user": "Operator",
"login": "Log in",
"manage_member": "Managing members",
......@@ -250,12 +238,15 @@
"search_test": "Search Test",
"set_invoice_header": "Set up invoice header",
"set_name_avatar": "Team avatar",
"skill.folder": "Folder",
"skill.skill": "Skill",
"sync_immediately": "Synchronize now",
"sync_member_failed": "Synchronization of members failed",
"sync_member_success": "Synchronize members successfully",
"total_team_members": "Total {{amount}} members",
"transfer_app_ownership": "Transfer app ownership",
"transfer_dataset_ownership": "Transfer dataset ownership",
"transfer_skill_ownership": "Transfer skill ownership",
"transfer_ownership": "Transfer ownership",
"transfer_team_ownership": "Transfer Team",
"transfer_success": "Transfer successful",
......@@ -277,21 +268,33 @@
"update_dataset": "Update the knowledge base",
"update_dataset_collaborator": "Knowledge Base Permission Changes",
"update_publish_app": "Application update",
"create_skill": "Create skill",
"create_skill_folder": "Create skill folder",
"delete_skill": "Delete skill",
"deploy_skill": "Deploy skill",
"import_skill": "Import skill",
"export_skill": "Export skill",
"copy_skill": "Copy skill",
"move_skill": "Move skill",
"update_skill_collaborator": "Update skill collaborator",
"delete_skill_collaborator": "Delete skill collaborator",
"transfer_skill_ownership": "Transfer skill ownership",
"update_skill": "Update skill",
"used_times_limit": "Limit",
"user_name": "username",
"user_team_invite_member": "Invite members",
"user_team_leave_team": "Leave the team",
"user_team_leave_team_failed": "Failure to leave the team"
"user_team_leave_team_failed": "Failure to leave the team",
"waiting": "To be accepted",
"copy_skill": "Copy skill",
"create_skill": "Create skill",
"create_skill_folder": "Create skill folder",
"delete_skill": "Delete skill",
"delete_skill_collaborator": "Skill permission deletion",
"deploy_skill": "Deploy skill",
"export_skill": "Export skill",
"import_skill": "Import skill",
"log_copy_skill": "[{{name}}] Copied [{{skillType}}] named [{{skillName}}]",
"log_create_skill": "[{{name}}] Created [{{skillType}}] named [{{skillName}}]",
"log_create_skill_folder": "[{{name}}] Created a skill folder named [{{folderName}}]",
"log_delete_skill": "[{{name}}] Deleted [{{skillType}}] named [{{skillName}}]",
"log_delete_skill_collaborator": "[{{name}}] Deleted the [{{itemName}}] permission named [{{itemValueName}}] from [{{skillType}}] named [{{skillName}}]",
"log_deploy_skill": "[{{name}}] Deployed [{{skillType}}] named [{{skillName}}]",
"log_export_skill": "[{{name}}] Exported [{{skillType}}] named [{{skillName}}]",
"log_import_skill": "[{{name}}] Imported [{{skillType}}] named [{{skillName}}]",
"log_move_skill": "[{{name}}] Moved [{{skillType}}] named [{{skillName}}] to [{{targetFolderName}}]",
"log_transfer_skill_ownership": "[{{name}}] Transferred ownership of [{{skillType}}] named [{{skillName}}] from [{{oldOwnerName}}] to [{{newOwnerName}}]",
"log_update_skill": "[{{name}}] Updated [{{skillType}}] named [{{skillName}}]",
"log_update_skill_collaborator": "[{{name}}] Updated collaborators for [{{skillType}}] named [{{skillName}}] to: Organization: [{{orgList}}], Group: [{{groupList}}], Member [{{tmbList}}]; permissions updated to: [{{permission}}]",
"move_skill": "Move skill"
}
......@@ -56,8 +56,6 @@
"create_invitation_link": "创建邀请链接",
"create_invoice": "开发票",
"create_org": "创建部门",
"create_skill": "创建技能",
"create_skill_folder": "创建技能文件夹",
"create_sub_org": "创建子部门",
"dataset.api_file": "API 知识库",
"dataset.common_dataset": "知识库",
......@@ -66,8 +64,6 @@
"dataset.folder_dataset": "文件夹",
"dataset.website_dataset": "网站同步",
"dataset.yuque_dataset": "语雀知识库",
"skill.folder": "技能文件夹",
"skill.skill": "技能",
"delete": "删除",
"delete_api_key": "删除api密钥",
"delete_app": "删除工作台应用",
......@@ -83,7 +79,6 @@
"delete_from_team": "移出团队",
"delete_group": "删除群组",
"delete_org": "删除部门",
"delete_skill": "删除技能",
"department": "部门",
"edit_info": "编辑信息",
"edit_member": "编辑用户",
......@@ -163,8 +158,6 @@
"log_create_group": "【{{name}}】创建了群组【{{groupName}}】",
"log_create_invitation_link": "【{{name}}】创建了邀请链接【{{link}}】",
"log_create_invoice": "【{{name}}】进行了开发票操作",
"log_create_skill": "【{{name}}】创建了名为【{{skillName}}】的技能",
"log_create_skill_folder": "【{{name}}】创建了名为【{{folderName}}】的技能文件夹",
"log_delete_api_key": "【{{name}}】删除了名为【{{keyName}}】的api密钥",
"log_delete_app": "【{{name}}】将名为【{{appName}}】的【{{appType}}】删除",
"log_delete_app_collaborator": "【{{name}}】将名为【{{appName}}】的【{{appType}}】中名为【{{itemValueName}}】的【{{itemName}}】权限删除",
......@@ -175,7 +168,6 @@
"log_delete_dataset_collaborator": "【{{name}}】将名为【{{datasetName}}】的【{{datasetType}}】中名为【itemValueName】的【itemName】权限删除",
"log_delete_department": "【{{name}}】删除了部门【{{departmentName}}】",
"log_delete_evaluation": "【{{name}}】删除了名为【{{appName}}】的【{{appType}}】的评测数据",
"log_delete_skill": "【{{name}}】删除了名为【{{skillName}}】的技能",
"log_delete_group": "【{{name}}】删除了群组【{{groupName}}】",
"log_details": "详情",
"log_export_app_chat_log": "【{{name}}】导出了名为【{{appName}}】的【{{appType}}】的聊天记录",
......@@ -206,15 +198,6 @@
"log_update_dataset": "【{{name}}】更新了名为【{{datasetName}}】的【{{datasetType}}】",
"log_update_dataset_collaborator": "【{{name}}】将名为【{{datasetName}}】的【{{datasetType}}】的合作者更新为:组织:【{{orgList}}】,群组:【{{groupList}}】,成员【{{tmbList}}】;权限更新为:【{{readPermission}}】,【{{writePermission}}】,【{{managePermission}}】",
"log_update_publish_app": "【{{name}}】【{{operationName}}】名为【{{appName}}】的【{{appType}}】",
"log_update_skill": "【{{name}}】更新了名为【{{skillName}}】的技能",
"log_deploy_skill": "【{{name}}】发布了名为【{{skillName}}】的技能",
"log_import_skill": "【{{name}}】导入了名为【{{skillName}}】的技能",
"log_export_skill": "【{{name}}】导出了名为【{{skillName}}】的技能",
"log_copy_skill": "【{{name}}】复制了名为【{{skillName}}】的技能",
"log_move_skill": "【{{name}}】将名为【{{skillName}}】的技能移动到【{{targetFolderName}}】",
"log_update_skill_collaborator": "【{{name}}】将名为【{{skillName}}】的【{{skillType}}】的协作者更新为:组织:【{{orgList}}】,群组:【{{groupList}}】,成员【{{tmbList}}】;权限更新为:【{{permission}}】",
"log_delete_skill_collaborator": "【{{name}}】删除了名为【{{skillName}}】的【{{skillType}}】的协作者【{{itemName}}】:【{{itemValueName}}】",
"log_transfer_skill_ownership": "【{{name}}】将名为【{{skillName}}】的技能所有权转移给了【{{newOwnerName}}】",
"log_user": "操作人员",
"login": "登录",
"manage_member": "管理成员",
......@@ -253,12 +236,15 @@
"search_test": "搜索测试",
"set_invoice_header": "设置发票抬头",
"set_name_avatar": "团队头像 & 团队名",
"skill.folder": "文件夹",
"skill.skill": "技能",
"sync_immediately": "立即同步",
"sync_member_failed": "同步成员失败",
"sync_member_success": "同步成员成功",
"total_team_members": "共 {{amount}} 名成员",
"transfer_app_ownership": "转移应用所有权",
"transfer_dataset_ownership": "转移知识库所有权",
"transfer_skill_ownership": "转移技能所有权",
"transfer_ownership": "转让所有者",
"transfer_team_ownership": "转让团队",
"transfer_success": "转让成功",
......@@ -281,17 +267,32 @@
"update_dataset_collaborator": "知识库权限更改",
"update_publish_app": "应用更新",
"update_skill": "更新技能",
"deploy_skill": "发布技能",
"import_skill": "导入技能",
"export_skill": "导出技能",
"copy_skill": "复制技能",
"move_skill": "移动技能",
"update_skill_collaborator": "更新技能协作者",
"delete_skill_collaborator": "删除技能协作者",
"transfer_skill_ownership": "转移技能所有权",
"used_times_limit": "有效人数",
"user_name": "用户名",
"user_team_invite_member": "邀请成员",
"user_team_leave_team": "离开团队",
"user_team_leave_team_failed": "离开团队失败"
"user_team_leave_team_failed": "离开团队失败",
"waiting": "待接受",
"copy_skill": "复制技能",
"create_skill": "创建技能",
"create_skill_folder": "创建技能文件夹",
"delete_skill": "删除技能",
"delete_skill_collaborator": "技能权限删除",
"deploy_skill": "部署技能",
"export_skill": "导出技能",
"import_skill": "导入技能",
"log_copy_skill": "【{{name}}】复制了名为【{{skillName}}】的【{{skillType}}】",
"log_create_skill": "【{{name}}】创建了名为【{{skillName}}】的【{{skillType}}】",
"log_create_skill_folder": "【{{name}}】创建了名为【{{folderName}}】的技能文件夹",
"log_delete_skill": "【{{name}}】删除了名为【{{skillName}}】的【{{skillType}}】",
"log_delete_skill_collaborator": "【{{name}}】将名为【{{skillName}}】的【{{skillType}}】中名为【{{itemValueName}}】的【{{itemName}}】权限删除",
"log_deploy_skill": "【{{name}}】部署了名为【{{skillName}}】的【{{skillType}}】",
"log_export_skill": "【{{name}}】导出了名为【{{skillName}}】的【{{skillType}}】",
"log_import_skill": "【{{name}}】导入了名为【{{skillName}}】的【{{skillType}}】",
"log_move_skill": "【{{name}}】将名为【{{skillName}}】的【{{skillType}}】移动到【{{targetFolderName}}】",
"log_transfer_skill_ownership": "【{{name}}】将名为【{{skillName}}】的【{{skillType}}】的所有权从【{{oldOwnerName}}】转移到【{{newOwnerName}}】",
"log_update_skill": "【{{name}}】更新了名为【{{skillName}}】的【{{skillType}}】",
"log_update_skill_collaborator": "【{{name}}】将名为【{{skillName}}】的【{{skillType}}】的合作者更新为:组织:【{{orgList}}】,群组:【{{groupList}}】,成员【{{tmbList}}】;权限更新为:【{{permission}}】",
"move_skill": "移动技能"
}
......@@ -196,18 +196,6 @@
"log_update_dataset": "【{{name}}】更新了名為【{{datasetName}}】的【{{datasetType}}】",
"log_update_dataset_collaborator": "【{{name}}】將名為【{{datasetName}}】的【{{datasetType}}】的合作者更新為:組織:【{{orgList}}】,群組:【{{groupList}}】,成員【{{tmbList}}】;權限更新為:【{{readPermission}}】,【{{writePermission}}】,【{{managePermission}}】",
"log_update_publish_app": "【{{name}}】【{{operationName}}】名為【{{appName}}】的【{{appType}}】",
"log_create_skill": "【{{name}}】建立了名為【{{skillName}}】的技能",
"log_create_skill_folder": "【{{name}}】建立了名為【{{folderName}}】的技能資料夾",
"log_delete_skill": "【{{name}}】刪除了名為【{{skillName}}】的技能",
"log_deploy_skill": "【{{name}}】發布了名為【{{skillName}}】的技能",
"log_import_skill": "【{{name}}】匯入了名為【{{skillName}}】的技能",
"log_export_skill": "【{{name}}】匯出了名為【{{skillName}}】的技能",
"log_copy_skill": "【{{name}}】複製了名為【{{skillName}}】的技能",
"log_move_skill": "【{{name}}】將名為【{{skillName}}】的技能移動到【{{targetFolderName}}】",
"log_update_skill_collaborator": "【{{name}}】將名為【{{skillName}}】的【{{skillType}}】的協作者更新為:組織:【{{orgList}}】,群組:【{{groupList}}】,成員【{{tmbList}}】;權限更新為:【{{permission}}】",
"log_delete_skill_collaborator": "【{{name}}】刪除了名為【{{skillName}}】的【{{skillType}}】的協作者【{{itemName}}】:【{{itemValueName}}】",
"log_transfer_skill_ownership": "【{{name}}】將名為【{{skillName}}】的技能所有權轉移給了【{{newOwnerName}}】",
"log_update_skill": "【{{name}}】更新了名為【{{skillName}}】的技能",
"log_user": "操作人員",
"login": "登入",
"manage_member": "管理成員",
......@@ -246,12 +234,15 @@
"search_test": "搜索測試",
"set_invoice_header": "設置發票抬頭",
"set_name_avatar": "團隊頭像",
"skill.folder": "資料夾",
"skill.skill": "技能",
"sync_immediately": "立即同步",
"sync_member_failed": "同步成員失敗",
"sync_member_success": "同步成員成功",
"total_team_members": "共 {{amount}} 名成員",
"transfer_app_ownership": "轉移應用程式所有權",
"transfer_dataset_ownership": "轉移知識庫所有權",
"transfer_skill_ownership": "轉移技能所有權",
"transfer_ownership": "轉讓所有者",
"transfer_team_ownership": "轉讓團隊",
"transfer_success": "轉讓成功",
......@@ -272,22 +263,34 @@
"update_data": "更新數據",
"update_dataset": "更新知識庫",
"update_dataset_collaborator": "知識庫權限更改",
"update_publish_app": "應用更新",
"create_skill": "建立技能",
"create_skill_folder": "建立技能資料夾",
"delete_skill": "刪除技能",
"deploy_skill": "發布技能",
"import_skill": "匯入技能",
"export_skill": "匯出技能",
"copy_skill": "複製技能",
"move_skill": "移動技能",
"update_skill_collaborator": "更新技能協作者",
"delete_skill_collaborator": "刪除技能協作者",
"transfer_skill_ownership": "轉移技能所有權",
"update_skill": "更新技能",
"update_skill_collaborator": "技能權限更改",
"update_publish_app": "應用更新",
"used_times_limit": "有效人數",
"user_name": "使用者名稱",
"user_team_invite_member": "邀請成員",
"user_team_leave_team": "離開團隊",
"user_team_leave_team_failed": "離開團隊失敗"
"user_team_leave_team_failed": "離開團隊失敗",
"waiting": "待接受",
"copy_skill": "複製技能",
"create_skill": "建立技能",
"create_skill_folder": "建立技能資料夾",
"delete_skill": "刪除技能",
"delete_skill_collaborator": "技能權限刪除",
"deploy_skill": "部署技能",
"export_skill": "導出技能",
"import_skill": "導入技能",
"log_copy_skill": "【{{name}}】複製了名為【{{skillName}}】的【{{skillType}}】",
"log_create_skill": "【{{name}}】建立了名為【{{skillName}}】的【{{skillType}}】",
"log_create_skill_folder": "【{{name}}】建立了名為【{{folderName}}】的技能資料夾",
"log_delete_skill": "【{{name}}】刪除了名為【{{skillName}}】的【{{skillType}}】",
"log_delete_skill_collaborator": "【{{name}}】將名為【{{skillName}}】的【{{skillType}}】中名為【{{itemValueName}}】的【{{itemName}}】權限刪除",
"log_deploy_skill": "【{{name}}】部署了名為【{{skillName}}】的【{{skillType}}】",
"log_export_skill": "【{{name}}】導出了名為【{{skillName}}】的【{{skillType}}】",
"log_import_skill": "【{{name}}】導入了名為【{{skillName}}】的【{{skillType}}】",
"log_move_skill": "【{{name}}】將名為【{{skillName}}】的【{{skillType}}】移動到【{{targetFolderName}}】",
"log_transfer_skill_ownership": "【{{name}}】將名為【{{skillName}}】的【{{skillType}}】的所有權從【{{oldOwnerName}}】轉移到【{{newOwnerName}}】",
"log_update_skill": "【{{name}}】更新了名為【{{skillName}}】的【{{skillType}}】",
"log_update_skill_collaborator": "【{{name}}】將名為【{{skillName}}】的【{{skillType}}】的合作者更新為:組織:【{{orgList}}】,群組:【{{groupList}}】,成員【{{tmbList}}】;權限更新為:【{{permission}}】",
"move_skill": "移動技能"
}
......@@ -3,7 +3,7 @@
"version": "1.0.0",
"engines": {
"node": ">=20",
"pnpm": "9.x"
"pnpm": "10.x"
},
"dependencies": {
"@chakra-ui/anatomy": "catalog:",
......@@ -25,8 +25,8 @@
"@lexical/text": "0.12.6",
"@lexical/utils": "0.12.6",
"@monaco-editor/react": "^4.7.0",
"@tanstack/react-query": "^4.24.10",
"ahooks": "^3.9.5",
"@tanstack/react-query": "catalog:",
"ahooks": "catalog:",
"axios": "catalog:",
"date-fns": "catalog:",
"dayjs": "catalog:",
......@@ -41,13 +41,13 @@
"react-beautiful-dnd": "^13.1.1",
"react-day-picker": "^9.14.0",
"react-dom": "catalog:",
"react-hook-form": "7.43.1",
"react-hook-form": "catalog:",
"react-i18next": "catalog:",
"react-markdown": "^9.0.1",
"react-markdown": "catalog:",
"react-photo-view": "^1.2.6",
"recharts": "^2.15.0",
"recharts": "catalog:",
"rehype-external-links": "^3.0.0",
"remark-gfm": "^4.0.1",
"remark-gfm": "catalog:",
"use-context-selector": "^1.4.4",
"zustand": "^4.3.5"
},
......
# 目录说明
该目录为 FastGPT 辅助子项目,非必须。
- model 私有化模型
\ No newline at end of file
FROM pytorch/pytorch:2.4.0-cuda11.8-cudnn9-runtime
# please download the model from https://huggingface.co/vikp/surya_det3
# and https://huggingface.co/vikp/surya_rec2, and put it in the directory vikp/
COPY ./vikp ./vikp
COPY requirements.txt .
RUN python3 -m pip install -r requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple
RUN python3 -m pip uninstall opencv-python -y
RUN python3 -m pip install opencv-python-headless -i https://pypi.tuna.tsinghua.edu.cn/simple
COPY app.py Dockerfile ./
ENTRYPOINT python3 app.py
\ No newline at end of file
# 接入Surya OCR文本检测
## 源码部署
### 1. 安装环境
- Python 3.9+
- CUDA 11.8
- 科学上网环境
### 2. 安装依赖
```bash
pip install -r requirements.txt
```
### 3. 下载模型
代码首次运行时会自动从huggingface下载模型,可跳过以下步骤。
也可以手动下载模型,在对应代码目录下clone模型
```sh
mkdir vikp && cd vikp
git lfs install
git clone https://huggingface.co/vikp/surya_det3
# 镜像下载 https://hf-mirror.com/vikp/surya_det3
git clone https://huggingface.co/vikp/surya_rec2
# 镜像下载 https://hf-mirror.com/vikp/surya_rec2
```
最终手动下载的目录结构如下:
```
vikp/surya_det3
vikp/surya_rec2
app.py
Dockerfile
requirements.txt
```
### 4. 运行代码
```bash
python app.py
```
对应请求地址为
`http://0.0.0.0:7230/v1/ocr/text`
### 5. 测试
```python
curl --location --request POST 'http://localhost:7230/v1/ocr/text' \
--header 'Authorization: Bearer your_access_token' \
--header 'User-Agent: Apifox/1.0.0 (https://apifox.com)' \
--header 'Content-Type: application/json' \
--data-raw '{
"images":[
"/9j/4AAQSkZJRgABAQAAAQABAAD/4gHYSUNDX1BST0ZJTEUAAQEAAAHIAAAAAAQwAABtbnRyUkdCIFhZWiAH4AABAAEAAAAAAABhY3NwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAA9tYAAQAAAADTLQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAlkZXNjAAAA8AAAACRyWFlaAAABFAAAABRnWFlaAAABKAAAABRiWFlaAAABPAAAABR3dHB0AAABUAAAABRyVFJDAAABZAAAAChnVFJDAAABZAAAAChiVFJDAAABZAAAAChjcHJ0AAABjAAAADxtbHVjAAAAAAAAAAEAAAAMZW5VUwAAAAgAAAAcAHMAUgBHAEJYWVogAAAAAAAAb6IAADj1AAADkFhZWiAAAAAAAABimQAAt4UAABjaWFlaIAAAAAAAACSgAAAPhAAAts9YWVogAAAAAAAA9tYAAQAAAADTLXBhcmEAAAAAAAQAAAACZmYAAPKnAAANWQAAE9AAAApbAAAAAAAAAABtbHVjAAAAAAAAAAEAAAAMZW5VUwAAACAAAAAcAEcAbwBvAGcAbABlACAASQBuAGMALgAgADIAMAAxADb/2wBDAAYEBQYFBAYGBQYHBwYIChAKCgkJChQODwwQFxQYGBcUFhYaHSUfGhsjHBYWICwgIyYnKSopGR8tMC0oMCUoKSj/2wBDAQcHBwoIChMKChMoGhYaKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCj/wAARCAK4Be4DASIAAhEBAxEB/8QAHQABAAEFAQEBAAAAAAAAAAAAAAECBQYHCAQDCf/EAGYQAAEDAwEGAwMECRAGBAoJBQABAgMEBREGBxIUITFhE0FRCBVxIjKBsxYYN0JVVpGU0hcjNTZSVHJ0dZKTobGy0dMzYnOClcEkV6K0JzhDU2N2g8LD8CU0RIWWo+Hj8SYoZWeE/8QAGwEBAQACAwEAAAAAAAAAAAAAAAEFBgMEBwL/xAAxEQEAAQIFAQcDAwQDAAAAAAAAARESAhZjouEDBAUhMUFxoRM0UUJhgSIykbEUI8P/2gAMAwEAAhEDEQA/AOVAAAAAAE4GAIBVuqN1QKQVbqjdUCkFW6o3VApBVuqN1QKQVbqjdUCkFW6o3VApBVuqN1QKQVbqjdUCkFW6o3VApBVuqN1QKQVbqjdUCkFW6o3VApBVuqN1QKQVbqjdUCkFW6o3VApBVuqN1QKQVbqjdUCkFW6o3VApBVuqN1QKQVbqjdUCkFW6o3VApBVuqN1QKQVbqjdUCkFW6o3VApBVuqN1QKQVbqjdUCkFW6o3VApBVuqN1QKQVbqjdUCkFW6o3VApBVuqN1QKQVbqjdUCkFW6o3VApBVuqN1QKQVbqjdUCkFW6o3VApBVuqN1QKQVbqjdUCkFW6o3VApBVuqN1QKQVbqjdUCkFW6o3VApBVuqN1QKQVbqjdUCkFW6o3VApBVuqN1QKQVbqjdUCkFW6o3VApBVuqN1QKQVbqjdUCkFW6o3VApBVuqN1QKQVbqjdUCkFW6o3VApBVuqN1QKQVbqjdUCkFW6o3VApBVuqN1QKQVbqjdUCkFW6o3VApBVuqN1QKQVbqjdUCkFW6o3VApBVuqN1QKQVbqjdUCkFW6o3VApBVuqN1QKQVbqjdUCkFW6o3VApBVuqN1QKQVbqjdUCkFW6o3VApBVuqN1QKQVbqjdUCkFW6o3VApBVuqN1QKQVbqjdUCkFW6o3VApBVuqN1QKQVbqjdUCkFW6o3VApBVuqN1QKQVbqjdUCkFW6o3VApBVuqN1QKQVbqjdUCkFW6o3VApBVuqN1QKQVbqkboEAnAAgAAAAAAKkQCEQqRpUiH0azIHzRpWjD7MjPs2IDypGpUkR7Ww9j6Ng7AW/wifBLklP2Kkpl9ALX4I8EunDL6Dhl9ALX4I8EunDL6Dhl9ALX4I8EunDL6Dhl9ALX4I8EunDL6Dhl9ALX4I8EunDL6Dhl9ALX4I8EunDL6Dhl9ALX4I8EunDL6Dhl9ALX4I8EunDL6Dhl9ALX4I8EunDL6Dhl9ALX4I8EunDL6Dhl9ALX4I8EunDL6Dhl9ALX4I8EunDL6Dhl9ALX4I8EunDL6Dhl9ALX4I8EunDL6Dhl9ALX4I8EunDL6Dhl9ALX4I8EunDL6Dhl9ALX4I8EunDL6Dhl9ALX4I8EunDL6Dhl9ALX4I8EunDL6Dhl9ALX4I8EunDL6Dhl9ALX4I8EunDL6Dhl9ALX4I8EunDL6Dhl9ALX4I8EunDL6Dhl9ALX4I8EunDL6Dhl9ALX4I8EunDL6Dhl9ALX4I8EunDL6Dhl9ALX4I8EunDL6Dhl9ALX4I8EunDL6Dhl9ALX4I8EunDL6Dhl9ALX4I8EunDL6Dhl9ALX4I8EunDL6Dhl9ALX4I8EunDL6Dhl9ALX4I8EunDL6Dhl9ALX4I8EunDL6Dhl9ALX4I8EunDL6Dhl9ALX4I8EunDL6Dhl9ALX4I8EunDL6Dhl9ALX4I8EunDL6Dhl9ALX4I8EunDL6Dhl9ALX4I8EunDL6Dhl9ALX4I8EunDL6Dhl9ALX4I8EunDL6Dhl9ALX4I8EunDL6Dhl9ALX4I8EunDL6Dhl9ALX4I8EunDL6Dhl9ALX4I8EunDL6Dhl9ALX4I8EunDL6Dhl9ALX4I8EunDL6Dhl9ALX4I8EunDL6Dhl9ALX4I8EunDL6Dhl9ALX4I8EunDL6Dhl9ALX4I8EunDL6Dhl9ALX4I8EunDL6Dhl9ALX4I8EunDL6Dhl9ALX4I8EunDL6Dhl9ALX4I8EunDL6Dhl9ALX4I8EunDL6Dhl9ALX4I8EunDL6Dhl9ALX4I8EunDL6Dhl9ALX4I8EunDL6Dhl9ALX4I8EunDL6Dhl9ALX4I8EunDL6Dhl9ALX4I8EunDL6Dhl9ALX4I8EunDL6Dhl9ALX4I8EunDL6Dhl9ALX4I8EunDL6Dhl9ALX4I8EunDL6Dhl9ALX4I8EunDL6Dhl9ALX4I8EunDL6Dhl9ALX4I8EunDL6Dhl9ALX4I8EunDL6Dhl9ALX4I8EunDL6Dhl9ALX4I8EunDL6Dhl9ALX4I8EunDL6Dhl9ALX4I8EunDL6Dhl9ALX4I8EunDL6Dhl9ALX4I8EunDL6Dhl9ALX4I8EunDL6Dhl9ALX4I8EunDL6Dhl9ALX4I8EunDL6Dhl9ALX4I8EunDL6Dhl9ALX4I8EunDL6Dhl9ALX4I8EunDL6Dhl9ALX4I8EunDL6Dhl9ALX4I8EunDL6Dhl9ALV4QWIuq06+hStP2AtSxFKxqXN0HY+boQLcrClWnufEfF8YHlVClUPs5uChUA+YJUgCUK2oUJ1PqxAK2NyemNhTE09kMecAIosnqigz5H3p4M45H6LeRYiqTL87Y6XsehlJ2P0KXqMFtKvz6bR9itKNfQ/QIYFqXPz+4JfQcEvofoCBaXPz+4JfQcEvofoABaXPz/AOCX0HBL6H6AAWlz8/8Agl9BwS+h+gJCi0ufn/wS+g4JfQ/QAILS5+f/AAS+g4JfQ/QAC0ufn/wS+g4JfQ/QAC0ufn/wS+g4JfQ/QAgtpc4A4JfQcEvofoABYXPz/wCCX0HBL6H6AEYFhc4A4JfQcEvod/kiwufn/wAEvoOCX0P0AAsLn5/8EvoOCX0O/wAkWFz8/wDgl9BwS+h3+BYXOAOCX0HBL6H6AAWlz8/+CX0HBL6H6AEC0ucAcEvoOCX0O/xgWFzgDgl9BwS+h+gAJaXPz/4JfQcEvod/gtpc4A4JfQcEvod/4IFpc4B4JfQcEvod/Ei1LnAHBL6Dgl9Dv4kWlzgDgl9BwS+h3+QLVucA8EvoOCX0O/yBalzgHgl9BwS+h3+BatzgDgl9BwS+h38CWlzgHgl9BwS+h38C2pc4B4JfQcEvod/YAtW5wDwS+g4JfQ79wTgWJe4B4JfQcEvod/YIFpc4C4JfQcEvod+E5Fhe4C4JfQcEvod+gWFzgLgl9BwS+h36BYXOAuCX0HBL6HfoFhc4C4JfQcEvod+gWFzgLgl9BwS+h36BYXuAuCX0HBL6HfoFhc4C4JfQcEvod+gWFzgLgl9BwS+h36BYXuAuCX0HBL6HfoFhc4C4JfQcEvod+gWFzgLgl9BwS+h36BYtzgLgl9BwS+h39gCwucA8EvoOCX0O/sAWJe4B4JfQcEvod/AWlzgHgl9BwS+h3+BatzgDgl9BwS+h3+MC0ucAcEvoOCX0P0ACC0ufn/wS+g4JfQ/QJCpOgtLn598EvoOCX0P0GRSoWlz89+CX0HBL6H6EYyQqEtWr8+OCX0HBL6H6CK30KRaVfn7wS+g4JfQ/QPBJKFX598EvoOCX0P0EQkUKvz64JfQcEvofoKBQq/Prgl9BwS+h+gxKChV+fHBL6Dgl9D9CCRRX57cEvoOCX0P0KJQiVfnpwS+g4JfQ/Q1CUUD88eCX0HBL6H6HkhX538EvoOCX0P0PIA/PHgl9BwS+h+hypkoVMAfnpwS+g4JfQ/QkAfntwS+g4JfQ/QkAfntwS+g4JfQ/QkAfntwS+g4JfQ/QkAfntwS+g4JfQ/QkAfntwS+g4JfQ/QkAfntwS+g4JfQ/QkAfntwS+g4JfQ/QkAfntwS+g4JfQ/QkAfntwS+g4JfQ/QkAfntwS+g4JfQ/QkAfntwS+g4JfQ/QkAfntwS+g4JfQ/QkAfntwS+g4JfQ/QkAfntwS+g4JfQ/QkAfntwS+g4JfQ/QkAfntwS+g4JfQ/QkAfntwS+g4JfQ/QkAfntwS+g4JfQ/QkAfntwS+g4JfQ/QkAfntwS+g4JfQ/QkAfntwS+g4JfQ/QkAfntwS+g4JfQ/QkAfnotGvoUOpOx+hwA/Ot9J2PNJTdj9HQB+bEsGPI8kkWDuz2hm72x7UCevD/8AeIziKohwBZpGHnc3BcpmHjlaB5XIUH1ch81AlvU9ESZU+DD1QoB6oW9C5UsecHjp29C8UcfQD2UkGccj9A8HB1DDnHI7y8iwkoBJGD6SgCcEYCUPIgnyGAIBOBgCBgkCBGASQURgEgCCcAACCogCBgkFSiASBUogEgVKIBIFRAJAqUQCQBAJGAIBOBgCATgYAgDAwAAwMAABgoAYBAAAAAFQIJAVAJARAJBBAJAVCgkARgY5kgIgYBIEEFRGAoR5lRAEDCEhAiMZGCojzAjAwSAqMDBICIwRgnBIEYBJAEYJwSAKScKSPIKpGCrAwEojHcYGCcAojBIwSBGBgkBUDAJCKcKTgkAQSAAAGAoSAAJCAiqkKkUpQlCCtFJwUFaKFMFKtRStAqAfFUwD649SlW46AUgAgEgACUBJFCSAgEoVFJJBOSclJKAVZKkUoySihVQIQqAjBGCojAFCtKF5H1IVMgfMEqmCAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADX+3xu/smvrfXwPr4zi6qhxnkdrbcW72y29p/sfr4zjquiwq8gMYqI8Kpbpml7rGdS0ztAt0icz4r1PRKnM87uoFTD2QJ0PIw9sHkBcaVuVQvtCzoWajTmhkFvbzQC+26LODuPHI4ntjOh2x5FhJRgEhS1RAJQFEAnHIj0IAJwRgAgA8iiASMAQME4CIBGATgBEYGCSAtEYGCQUojAwSAiASAIBIAgEgogEgggEjAEAYJwBAJwAIBIAgEjAEAnAKIABAAGAAAwUAMDBAABQIwSAiME4AIGCCQUQCQBAJAEAkYAgEkYFSgCcECpQBJGBUATgjAqAAwKiCRgYFQIJwMCoAYGAIJGCcAQCcACASRgABgkVKIBIIIGCQFRgkDAqIJGBgEAJAEEgkLRAJwMEEoAAJRSpFKUJQgrRSpFKEUlFKKyFQIpURXzVqKU4wfXBCtIPkShKtwQBIAAEgEAAASCCQJQkpAVWSilIArySUISigSQSSBTgoVvofTAA+IPorclCoqAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAMJ20pvbM7yn+x+uYch3CPGTr7bGqLs5u7fNfC+uYcl3JnNQMTrWdSyVKdTIa5vNSxVSdQLVMh5ndT1znkf1AqjPdAnQ8MZ7qfyAu1H1QyK3J0MeouqGR27qgGUWtOh2n5HGFr+9Oz/IsJJgAFQxzGASgFOAT5ACATgEEEFSDkUUgqVCPMABgYAAnAwBCkEqQAwAAJIwSAIwMEgojAwSAIwQTkkVFIKgKinAKgBBBUBUqpBIFRAJGAiATgYLUQCcDBKkIBOBgogE4GBVUAnAwEQCcDAqqASCVEAYJwKiATgjBapQAAKGBgAFDBCkjBBGASCiASAUQCQCiASAUQCQBAJAEAkAQCQBAJAKIBIC0QMEgFEAkAoEYJBChgYJwMCqIBOBgVEAnBIqqknBIFRAwSAAJwAIBIIIJAwAJAAlCUIySQVIVIp80KkCqycFKFSKBGClW+hWMAfFUJQ+ioUKmCCAAAAAAABRCSABJJAAqySilOSQKskoUgCsghCoCCFQqIwB81b6FJ9VQpVoFAJVMEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAUOka3pzU+TpFcnogR9nPa3z5nydI5enJCjBIGH7Wkzs+u3wi+tYcrXNOanVe1n7n12+EX1rDle6dVCwxO4JzUsFUnUyG4dVMfq+qgWuc8b+p7Jzxv6gVRnup/I8MZ7qfyAu9F1QyO29UMcouqGR23qgGWWv707QRDi+1/enaPkWElAyAAH0AAPIjBICIBIwFQgJAEAeY8yoBCcBECIBOOQwFQCVQgAAAUAAAwRgkARgYJAEYGCQURgYJBBGASAIGCQURgEgCASAIBIIiASAKSSRgKgE4AEAkAQQVEAAME4AgDAAAAIAYAVAJAEE4GAAwMAFEYGCQBGBgkARgYJAEYGCQBGCcAAMEYJBBGASCiFQYJBBBOAAIwSMAABgnARAJIwFATgAQMEgCMDBIAjBJOBgCACcAQCQBGCcAAAARQEjAEEkgAhJBIEopUilCEkH0QqPmikooFWCFQnJIHyc30KT7KhSqZA+YJVMEBQAAAAAJIAEkoU5JAqGSknIFRKKUopIFWSSnJIAhSoAUqmShWn0UgD5ArVuSjGOoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAC4ROa4ABeR8nTInzeZ8nPc7qoH2dKjenNT5Oe53mUgIYJCEgCQAMR2s/c+u3wi+tYcrXTqp1TtZ+59dvhF9aw5WunVQrFbh1Ux+r6qZBcOqmP1fVQLXOeN/U9k5439QKoz3U/keGM91P5AXei6oZHbeqGOUXVDI7b1QDLLX96donF1r+9O0QIwME+YKiBgkAQCojAAAAQCQBAJwMAQCcDAEISBgCAMABggkAAAEBgAKYIJARBOAAGCCQBGASAIBIKIBIAgEgVEAkAQCRgVEAnAwKiATgYFRAJwRgABgnAEAnAwBAGCcCogE4GAIAAADAwABOBgVEAAABgAAAAAwTgCATggABgnAEAYAAAnARAJwMBUAnAwBAJwAIBIBRGCcAAQMEgBgYAIoAAgCRgKgE4JAjAwSAIBIAjBIAAAYAAkEDAAAE5IAEk5KSQK0UlFKMk5CqxghFJAhUKHNx0PoMAfEFatyUqmAIAAAAACckACQMjIEoSUkgVICknIFWSSklAKiMDJIEEKmSogD5qhSfUpVvoBQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEKqJ1UCQqonVT5OlVfmnxcqquVXIR9nzc8NPi5yuXmpAAEkdSpACEhEJADAJAAnGSprPUDDtrX3Prt8IvrWHKt06qdXbXERNnd2+EX1rDlG6dVCsVuHVTH6vqpkFw6qY/V9VAtc5439T2Tnjf1AqjPdT+R4Yz3U/kBd6Lqhkdt6oY5RdUMjtvVAMstf3p2icXWv707SCSgEgogE+YwBHkCcchjoQQB5k/QBAAxyKAJIIJAAAKApRAAAAABgYJAEYGCQBGBgkARgYJAEYBIApBUAKSSQBSCoAUgqAFIKgBSCoAUgkYAgEgCATgYAgE4GAIBOBgCATgYAgE4GAIBOBgCASMAQCcAIgAkKgE4GAlEAnBAUAJCIBIwBAJwAqASAIBOBgCATgkCkE4GAIBOCQIGCQBGBgkAQMEgCMDBIAgkAAAAAAwAAwSBAwSCBgAAAAAAAKAABQAJyFQSMkASSUk5AqQlFKckooFaKSUIpOQJVCFTJKLkkD5K3BSfVUKVaBQCVQgAAAAAAEkEoAJyQAKiUKSQJyVIpQhIFYKSUAKQVAChURShUVD64IA+QKnN9CkAAAAAAAAAAAAAAAAAAAABDno0CSHORPM+TpFXpyKQit0i+XIoXmvMACClT6tjz1IkhXq0D5BEGOeCpACISgQnAAklGqvQrRnqBQiZ6FbWepWiehOAIRuCcABWH7Xvud3f4RfWsOT7p1U6w2vfc7u/wi+tYcn3TqoGK3Dqpj9X1UyC4dVMfq+qgWuc8b+p7Jzxv6gVRnup/I8MZ7qfyAu9F1QyO29UMcouqGR23qgGWWv707SOLbX96dpAAAEPMlCCUAjyGexPkR5gMcwB5gEAABRgGKbVqmei2bakqaOeWnqIqGR8csT1Y9jkbyVFTmigZXgnB+f9v1xqx9wpmu1RfVasrUVFuEqoqZT/AFj9AAGCFQkKBSDizbXqnUNDtS1DTUV9utPTxzojIoayRjGpuN5IiLhDf3sz3GtumzNtRc6yprKjjJW+LUSukdhN3CZVVXAG1QVEFAEnhvl1orHaKq53SdtPRUrFklkVM4T4J1XywB7Qcn3D2jLwuuG11DSsTTkeY0oHoiPlZn56v8n+mOSdOfNV6U0Zqu06xskV0sdSk0DuT2LyfE7za9vkv/8AKZQgvgPnVVENJTS1FVNHBTxNV8ksjka1jUTKqqryRE9SxfZxpL8aLF/xCH9IDIQY79nGkvxosX/EIf0jS3tLa7RlBYfsN1UiSLLN4/uq4c8Ybu7/AIbunXGe5R0WDlX2b9eVj9W3JNYaqnWj4FfD96XFfD8TxGfN8R2N7GenPGToj7ONJ/jRYv8AiEP6QGRAx37OdJ/jRYv+IQ/pF5t1fR3OkZV22qp6ulfncmp5EkY7C4XDkVUXCoqEHpANOX32g9K2a93C11Vvvb6iiqJKaR0cMStVzHK1VTMiLjKeiFG4waO+2W0h+DL/AP0EP+aPtltIfgy//wBBD/mgbxBo77ZbSH4Mv/8AQQ/5o+2W0h+DL/8A0EP+aBvEGjvtltIfgy//ANBD/mj7ZbSH4Mv/APQQ/wCaBvEGjvtltIfgy/8A9BD/AJpk+z3bFYNd391otNHdYalIXT71VHG1m61URUy17lz8pPIDZRBIAAwXbVqxNHbPblXRSbldM3haTnz8V6KmU/gpvO/3TkrRu0XWFvudFS02qqylppJWRufVyeNFE1VRFcqPyiIic+QHd4MIbtU0MjURdUW5VROqydf6i52DXGmdQ1/BWS9UdbV7iyeFE/Lt1Oq/1oBkgMH2j7S7Js/loI75FXSOrUe6Pho2vxubuc5cn7pDDftkNF/vW9/m8f8AmAbqBpX7ZDRf71vf5vH/AJg+2Q0X+9b3+bx/5gG6gaV+2Q0X+9b3+bx/5hs3ROp6HWOnKa9Wpk7KSoV6MSdqNf8AJcrVyiKqdUXzAvoBiO0jXts2f2qmuF4p62eGom8BraVjXOR26rsrvObywigZcDR32y2kPwZf/wCgh/zR9stpD8GX/wDoIf8ANA3iDR32y2kPwZf/AOgh/wA0fbLaQ/Bl/wD6CH/NA3iDR32y2kPwZf8A+gh/zR9stpD8GX/+gh/zQN4g0d9stpD8GX/+gh/zTN9me02z7Q33FtlpbhAtCkaycWxjc7+9jG6937lfQDOgAABznqz2ia+xapu9pj0/SzMoauWmSR1S5FejHq3Kpu8s4LT9s7cfxapPzp36IHUQOXftnbj+LVJ+dO/RM62O7ZavaBqqa0VFngomR0r6nxI5leqq1zG4wqJ+6/qA3QDSG1rbZWaF1hJZaey09YxkMcvivnVi/KTphEUwz7Z24/i1SfnTv0QOogcu/bO3H8WqT86d+iPtnbj+LVJ+dO/RA6iBy79s7cfxapPzp36J0/C5XxMeqYVzUXHoBWBgYFQAOQfaJ2j1N21q232C4TQ0FpzH4tNKrfEmX57stXmiYRqfBfUDr4HMPs16jvdyvVbV6k1ZULaaOLcbBXVmfFld0xvrnDURV5eatOifsjsn4Ztv50z/ABAuoLV9kdk/DNt/Omf4j7I7H+Gbb+dM/wAQLqC1fZHY/wAM2386Z/iXSN7JGNfG5rmORFa5q5RUXzQgkEgCMDBqLbftXrtndztlLRW2mrG1cLpXOme5qtVHYwmDWn2zl5/F+3f0rwUdU4Byt9s5efxft39K8fbOXn8X7d/SvA6pUjB56eoWW2x1KtRHPiSTd8sqmcHL/wBs5efxft39K8DqnAOVvtnLz+L9u/pXj7Zy8/i/bv6V4HVINF7Ittlx1zrKKy1dppKWJ8MkviRSOV2WpnHM3oCgAAAACgAAAkgAAAAAAAAAAAAAAlAQSgEoSilJOQKkUqRShFJRQKyFIyVAUqh81TB9SFQD5ArVpQAAAAAACckACSSABOSSklAKhkgZArySUZKkUCSFQEgQUq3JUFA+Spgg+qpkoc3AFIAAAAAAAAAAAFDpETpzArKXPRO6nyc9XeZARLnq7shAJRMgQETPQ+jWepWiIgHzaxfM+iNRCSUCoJRCRgCh8aO7L6nxcxWLzPUF59QPKjVU+jWInU+it9CACEkEgAAoAEZJAw/a99zu7/CL61hyfdOqnWG177nd3+EX1rDk+6dVAxW4dVMfq+qmQXDqpj9X1UC1znjf1PZOeN/UCqM91P5HhjPdT+QF3ouqGR23qhjlF1QyO29UAyy1/enaWORxba/vTtLyAAAASABAC9AAAAAABAw/bF9y3VP8ny/3SzbftWXPRuhGXGxztp619ZHA17o2vTCo5VTDkVOjTU2lto2o9c7P9oEF/mp5YaS0ucxY4UYu87Kc8dkUDQFt/ZGl/wBqz+1D9Ij826FquradrXKxyyNRHJ5LnqZnU6u1NZdo8Fbf7nW11fZ69UeyaVXJ8hyte1qLyRFTeTljqFdY7QdrGndCXiC2Xtle6pmp0qWrTwte3dVzmplVcnPLFMY+2N0T/wCZvH5sz9Mxrb3s81HtA1Rar1pWkirLa61xMbKtQxmVV8j0wjlRcbr2rnuaz/UF2g/giD88i/SAxTadfaTU2vLxeLckqUlXKj40lbuuxuonNEVfQ25sQ2v6b0TodtpvEdwdVJUyS5gha5u67GOauT0NGahs1bp69VVqusSRVtK7clYjkciLhF6pyXkpk+kNlurNXWdLnYqCKejWR0W+6ojYu8mMphyovmB0Z9sbon/zN4/Nm/pmR6C2u6c1vfHWqzR3BtUkLplWeFrW7rVRF5o5efNDkbW2zvUmiqamqNRUUdPFUvWONWzskyqJlfmquDOPZP8Auoy/ydN/eYEdilE0Uc8L4p42SRSNVr2PTLXIvJUVF6oVgDlTbpsTdZW1GodHwOfbEy+qoW83U/q9nqz1Tq34dNUbO9bXTQuoI7nan7zFw2opnL8idn7le/ovkv0ov6BKiKmF5oc/6v8AZ6pLrr2CutVRHQWCoVZa2nYmHRuRekSdER3fk3n1TCBWd3XU1FrvYtf7lYEkmSottTF4G7mRkvhrmNUT77mnTrlMdTjH7F7/APgO6/mkn+B+gtktNDY7VTW2000dLRU7dyOJickT/mq9VVeaqap1lt7s2ltT3Cy1VouE01HJ4bpI3M3XLhF5ZXPmEcofYvf/AMB3X80k/wADyXC1XG2tYtxoKukR+Uas8Lo97HXGU5nUP2zVg/AV1/nR/wCJq7bntSt20WjtENuoKukWikke5Z1au9vI1Exhf9UK1ZQW+suMro7fSVFVI1N5WQROeqJ6qiJ0Pd9i9/8AwHdfzST/AAMt2Ia+o9nuo6643CkqKqOopFp0ZArUVF32uyuV6fJN0/bNWD8BXX+dH/iBzT9i9/8AwHdfzST/AAOy/Z3paii2R2WCsglp52un3o5WKxyZneqZRefQxK2+0hYq640tJHZLm19RK2JrldHhFcqJlefc3qEDj/XGxbXV01pf7hQ2iOSkq7hUTwvWrhbvMdI5zVwrspyVOp2AAOJP1B9oX4Ej/PYP0x+oPtC/Akf57B+mdtgDiT9QfaF+BI/z2D9MfqD7QvwJH+ewfpnbYBVxJ+oPtC/Akf57B+meW6bGNb2qgmrblbaWlpIW70k0tfA1rU7qrzrLaNtHsOg6FZLrUJLXObmGhhVFlk9Fx963/WXl6ZXkcebSdpF919X+Jc5vBoI3ZgoYlVI4+6/unf6y/RhOQVhaphVTqby9ku1V0mu6u6MpZVt8NHJC+oxhiPc5io3PmuEVcIWzY1sZr9ZyRXS9JLQ6eRco7GJKrszPRv8Arfkz5de2W00FjtkFutNLFSUUDd2OKNMIn+K+qrzUD3EYJNQe0prO6aV0fFTWinmY65udTvr2/Ngbjm1F8nuTOF9EXHNORGjfaO10zVuseBt82/abVvQxuavyZZc/LenqnJGp2TPmYxpbZjrDVNpbc7FZn1VC57mNlWeKNHKnXCPciqnlnpnPoYjTJEtRElS6RsCvTxHRtRzkbnmqIqoirjyyh1HZvaD0VYrHS2y02S9x01JCkUMaxxInJOWV8Tqq81XHmqhXOWrNLXjSVxZQagpG0lY+NJUiSaORd1VVEVdxy46L1NreyVbKqo19W3GONeDpaJ0csi9Ec9zd1vxXdcv0Go9U32t1NqCuvFzfv1VXIsjvRqdEanZEwidkN47KNsOi9BaQp7VHbL1LWOXxqudsUWJJV64zJndRERE7J6qoGfe0Hs1vmv6mxvsT6JraNkzZeIlVnzlZjGEX9ypofWWxjUukNPVN5vE9rSkgVqKkc7nPcrnI1Eam6mV5/kydj6T1BS6m01Q3ujZLDS1cayMbOiI5qIqpzwqp5epy17SW0uDVVyisFkl8S00EqvlnavyaibGMp6tbzRF81VV6YUI1LpmyVmpL9RWe2Na6sq5PDj31w1PNVVfJERFVfgbT+1y1t/52z/nLv0C8eyZpVavUVZqaqbu09C1aamVfvpnp8rHwZn+eh1VLIyKN8kr2sjYiuc5y4RETqqqFfnPqK0VNgvtdaa5Y1qqOV0Mixuy3eTrhcIdnezfH4exqwZ5K5ah3/wCfIcgbQLhFdtdahr6Z6SU9TcJ5InouUcxZF3V/Jg7V2KUa0OynTELkwrqNs2P4aq//AN4IzbBqb2jNHXvWelrbR6dpG1VRDWeM9qysjw3ccmcuVE6qhtkAcSfqD7QvwJH+ewfpj9QfaF+BI/z2D9M7bAHEn6g+0L8CR/nsH6Y/UH2hfgSP89g/TO2wBxJ+oPtC/Akf57B+mP1B9oX4Ej/PYP0ztsAq4k/UH2hfgSP89g/TN1ezZoHUWiptQO1JQtpUq2wJDidkm9urJvfNVcfOTqbxAAYAA/Pzaj90vVf8q1X1rjbGj/Z3ZqLS1qvC6mdTrXUzKjwuA39zeTOM+ImfjhDU+1L7peq/5VqvrXHamyH7l2lv5Oh/uoBy9th2Pt2dWKiuLb2tx4ip4fw1pfC3fkudnO+7PzehcPZK+6bV/wAmS/WRGyPa/wD2jWb+UU+qea39kn7ptX/Jkv1kQHh9qX7rFR/FIf7FGx/Y63aJp6rubr4tv8CqWm8NKXxd7DGuznfb+6xjHkT7U33WKj+KQ/2KbY9kL7n92/lR/wBVEFWH7V1n42u/4d/+6W7Uns4R2XTt0ui6pdKlDSy1Ph8Bu7+4xXYz4i4zjqdSGD7brnDatlepJZpGs8akfTMRV5udIm4iJ6/OX8ihHBzWq5yNb1VcIfpQxu6xrfRMH5z6apVrdR2qkRMrPVxRIn8J6J/zP0ZBIAcqbadtt8nuNfp2xU9RZYIHugqJZOVTIqclRMfMb8OaphcpnAGV7f8AbHBbKWq01pWp8S5yIsVXVxO+TTp98xq+b/JVT5vx6cqFwslnuV+r20Vnoqiuq3IrkihYrnYTqvLy7n0tF0uumL2yrt089BcqZ6tzjdc1U6tc1eqeSoqfEKtYOu9le3azagihoNVcPartyaky8qedfVFX5i9l5ei+RutfBSPxF8NI8b29yxj1z6AfmyDrnalt3s1gimoNJ+Bdbrzas6JmnhX1ynz17Jy7+Ryxebtc9R3eStulTNW19Q5EVzuaqvk1ETonoicvQC3IiqqIiKqr0RD9E9Hxvh0lZIpmOjkZQwNcxyYVqpG1FRU8lNLbBNjCWdKfUeradHXNcPpKKRMpTej3p+79E+9+PToAIAAFHLHtiftj07/FJP76GM7CNl9r2iU15kulbW0y0T4msSmVvyt9HZzvIv7lDJvbE/bHp3+KSf30NebLNqNx2dQ3GO20NJVpWujc9ahXfJ3EdjGFT90Fby+1m0z+Gbz+WL9A5NVMKqG+/tmtQ/gO0/lk/SNCKuVyB+jVD+wFP/FW/wBw/OQ/Ruh/YCn/AIq3+4fnIB1fB7NOmpIY3reLwiuai9YvT+AYNtp2NWbQejmXe23C4VE7qpkG5UKzdw5HKq8moufkn0i9pfUMcbGJZLUqNRE6yfpGL7Sdst217p1touFtoaaFs7Z9+BX72WoqY5qvL5QHr9lv7rNN/FJ/7p2ccY+y391mm/ik/wDdOzgAAAAADQXtHbRdTaL1BaaXTtcymhqKV0kjXQRyZcj1TOXIvkai/V42hfhmL8zh/RMu9sL9tdh/iTvrFNbbHdKUOtNd0dlustTFSzRyvc6nc1r0VrFVMK5FTqnoBe/1eNoX4Zi/M4f0R+rxtC/DMX5nD+ibr+1q0d+Eb/8A08P+UPtatHfhG/8A9PD/AJQGlP1eNoX4Zi/M4f0R+rxtC/DMX5nD+ibr+1q0d+Eb/wD08P8AlD7WrR34Rv8A/Tw/5QG29LVc1fpi0VlU7fqKijhlkciImXOYiquE6c1LoeW1UMVstdHQQOe6GlhZAxXqiuVrWo1M488IWbaBquj0XpStvVeqOSFu7FFnCzSr81ifFfyIir5AebaDr+w6EoG1F8qHeNKi+BSxJvSzY9E8k7qqIc16r9onVdzkljscdLZ6VVVGKxiSzY7ud8n8jUNV6q1FctU3you16qHT1cy816NY3ya1PJqeSGy9k+w+56ypIrrd53WuzSc4lRuZp09WovJG/wCsv0IqcwMLq9pWtarPi6pvKf7OqfH/AHVQ+1u2pa4t8rZINUXR7m+VRMs6fkflFOn7fsA0DSwtZPQVda5Ewr56t6Kvf5CtT+o8F79nXRtbTSNtrq+2zr8x7Jlla1e7X5VU+lPiBgWhfaRr4JoqbWVDHVU6rh1ZSN3JGp6qz5rvo3fpOlLDebdqC1QXKzVcdXRTplksa8l9UVOqKnmi80OHdqOzS87Pa6JlwVlTb51VIK2JFRj1Tq1U+9d2/Iq8y7bB9oc+idVw09VM5bFXvSKqjVfkxuXkkqeipyz6pn0QDtwplkZDE+SV7WRsRXOc5cI1E6qq+SFRz57WWsKi3Wug0zQSrGte1Z6tW8lWJFw1ufRXI7P8HuoFO0b2i6a31MtBoulirpWKrXV1Rnwc/wCo1FRXfFVROyoaWu+17Xd0kV02pKyFFXk2lVIETt8hEX8pYNDaXrtY6no7LbN1s1Qq70j/AJsbETLnL2RPyrhPM6p097PGjLfTNS6trLtUYTefLM6Juf8AVaxUVE+Kr8QOYI9oWsmPVzdV37Pe4Sqn5FcZRp7bpruzysWS6MuUDesNbE16L/vJh/8AWdIT7DdnssSsSweGuMI5lXOip/2/7TT+13YGlgtNRetI1FRVUlO1ZKijnw6RjE6uY5ETeRPNFTOEzlQNubJdslo149tvnj923xG54Z7ssmx1WN3n67q8/jhVNqZPzWoqqehrIKujlfDUwPbJFIxcOY5Fyip3RTvzZbqlNZaFtd5XcSomj3KhrejZWruu5eSKqZTsqAZcigpyVZAYKVbkqGAPkqYIPqqZKFaBSAAAAAnIyQAJBBIE5JKScgSSUkgVZJyUgCsEEgCkqwRgChWlOD64KVTIHzBUqFKgAFVE6qfN0noB9Ch0iJ05nzVVXqQEqlVVeqkAlGqoEEoir0PokaJ1K8YQD5tj9T6IiICcBUAqGACDBIAAAAAABCoSAKSCohUAgEKMgSRkEAYhtd+55dvhF9aw5RunVTq7a79zy7fCL61hyjdOqgYrcOqmP1fVTILh1Ux+r6qBa5zxv6nsnPG/qBVGe6n8jwxnup/IC70XVDI7b1Qxyi6oZHbeqAZZa/vTtLyOLbX96dpeQAABE+YIQkKhegC9AAAABAABoX2wKlGaLslLnnLcPFx67sbk/wDfNa7GIP8AwV7VJ1TrQRMRf92VV/5F89sK5+LqSwWtF5U1I+oVO8j93/4R9tlFv4f2bdc1rkw6q8dEX1ayJqJ/WrgNAWz9kaX/AGrP7UNre1FYm2nadJWRM3YbpTsqeXTfTLHJ/wBlF/3jVNs/ZGl/2rP7UOzduuzKo2hw2d1FWUtFLQvk8WWfOPDciZxhOaorU646rzA+vs3X333sptzHv3p7c99E/n5NXLP+w5qfQbPNW7HdJWnZzbq+n+ymluK1jmSORXMjYxzUVPkpvKvPKZyvkhsL33avwnQ/nDP8QOINub0k2tamVvTicfSjWov9h0j7LDN3ZRCv7qsmX+tE/wCRy1tPqm1u0fU88b0fG65VG45q5RzUkVEVF+CIdU+zpWW637I7MyorqSKaR08j2Pma1UzK9Eyir6IgGLe2H+13Tv8AGpP7iGvvZP8Auoy/ydN/eYZx7XNfR1mn9PtpKunnc2qkVUikRyp8hOuDAvZZqYKTabJJVTxQx+75U3pHo1M7zOWVA7KLFqrVti0pFTyahuMVCyoVWxLI1y7ypjPRF9ULvS1VPVxrJSTxTxou6rono5EX0ynxNIe1Rp286htmn2WO2VdwfDNMsiU0SvViK1uM46dFAzb9WLQH4y0v9HJ+iP1YtA/jLS/zJP0TkFdmutUTK6WvGP4q/wDwMRA/SenmjqII5oXI+KRqPY5PNFTKKcK7dvuuam/jKf3Gnbum/wBrtr/isX9xDiLbt91zU38ZT+40DJdn+wu46z0lRX6mvNJTRVSyIkUkTnObuvczqn8HJZNrGyus2dU1umrLlT1qVr3sakUbm7u6iLzz/CM92UbcrJo3QdtsVdbLlPUUqyq6SHc3F3pXPTGXIvRyGNbdtqVs2iUdohtlDW0rqKSR71qN35W8jUTG6q+gGK7K9A1O0O9VVuo62GjfT061CvlYrkVN5rccv4RsC/8As6XSz2K43OS/UUjKKmkqXMbC9FcjGq5UT44MR2Ha9oNn2o6643OlqqmKopFp2tp93KLvtdlcqnL5JtfVPtEafvGmLvbILRdWS1tHNTMe/wAPda57Fairh3TmBzrpX9s9n/jkP99D9Fz86NK/tns/8ch/vofouAAAQAAA5r2u7fqqmra2x6OhWnlgkfBNcJ2orkc1VRUjb0Tmnzl/InU6UPzw11+3fUP8o1H1rgryRx3TUd5RrEq7ldKuT/Wlllcv9anS2yb2f6a3+DdNctjqqtMOZbmrvRRr/wCkX79eyfJ/hGS+zJp21UWzW23qnoom3Sv8bx6lUy9yNme1Goq9Ew1OSfE2+BTGxsbGsjajWNREa1qYRE9EKgABa9T2Wg1DYa213aBs9HURq17F6p5oqL5Ki4VF9ULoUTf6F/8ABUD83KSF1TVQwMVEdK9rEVeiKq4Nk7Stjt30DYIrtcrhQVMMlQ2nRlPv7yKrXOzzanL5KmurdM2nuFLNJncjla92OuEVFN77etrGm9caMp7XZON4qOtZUL48KMbuox6LzyvPLkA1vsu2c3HaJVV8FsrKSldRsY961G9hyOVUTG6i+hZNb6bqdI6prrHWzQz1FIrEfJDnddvMa5MZRF6OQz/2e9f2XQVxvM1+4ncq4o2R+BHvrlquVc809TEtreoKLVW0O73q1+LwdU6NY/Fbuu+TExq5T4tUDcN/s+o7l7Nmm59PVs7aOmpnvr6GLks8W+75WU5qjeaq3oqc/I1Hsw2eXbX95bTUDFhoInJxVa9vyIm+ierl8m/2JzOuthKZ2RaaRenDL/fcaR1BtuuejtYXezaYtllSw0dU+CGDh1YiK3k9UVjk6uRy5wvUDae1LZ/RQ7EKywWCldi2xtqqdrU3nvexcvcuOrnNV/0qcfz095jiWmniuDY8/wCie16Jn4KdlbQNX3q37DXaoo/Dt95kpqWdEa1JEiWSSNFTDkVF+S5U5oaO0Ftn11dtb6ft1feGSUlXXwQTM4SFu8x0jUcmUZlOSr0AxHZ7ss1JrG6QRR2+po7ark8atqIlYxjfPdz853ZPpwnM7moaWKhoqekpm7kEEbYo2+jWphE/Ih9wBhu0baJZtn8NBJfI6x7axz2x8NG1+FbjOcuT90hhH2yGi/3tevzdn+YfD2ntI37VlFp9mnrdLXOp5J1lSNzU3EcjMdVT0U0H+o7r78Wqr+fH+kB0H9shov8Ae16/N2f5g+2Q0X+9r1+bs/zDnz9R3X34tVX8+P8ASKZNkOvI43Pk05UsY1Fc5zpI0RETqq/KA6F+2Q0X+9r1+bs/zB9shov97Xr83Z/mHHxl1i2a6wv1rhuVosVTU0M2fDlRzWo7Cqi4yqL1RQOk/tkNF/va9fm7P8wfbIaL/e16/N2f5hz5+o7r78Wqr+fH+kP1Hdffi1Vfz4/0gOg/tkNF/va9fm7P8wfbIaL/AHtevzdn+Yc+fqO6+/Fqq/nx/pD9R3X34tVX8+P9IDrfZxtJsu0FbgljirWcD4fi8TG1md/exjDlz8xTNTRHsv6O1BpN2pV1FbJaHiUpvB8RzV393xd7GFXpvJ+U3uB+fm1L7peq/wCVar61xj7K2qY1GsqZ2tRMIiSKiIZBtS+6Xqv+Var61x2PsmtFtm2Z6Yklt9G+R1vhVznQtVVXdTmq4A4WmqZ52o2aaWRqLlEe9VNy+yT902r/AJMl+siM+9rW30VHom0PpKSngetwRFdFE1qqnhv5ckMB9kn7ptX/ACZL9ZEB4vam+6xUfxSH+xTbHshfc/u38qP+qiNT+1N91io/ikP9ilOx/bGmzvT9XbFsa3Hx6panxOL8Ldyxrd3G47Pzc5z5gZ97Vkl8sl1s13tF2uVHSVUTqaVlNVPjYkjF3kXCKiZVHL/MOdbperpdkal1uVbW7q5bxM75ML23lU39Xe0nRXCJsVfoaGqja7eRk1c16IvrhYevNTyRe0DY4Xo+LZzb2PTo5tSxF+pAxv2d9AXO+61tt7npJYrNbpUqVqJG4bJI3mxrM/O+VhVx0RO6HZhzWntQsRERNIKiJyREuX/7R0LYbh72sduuPheFxlNHUeHvb25vtR2M4TOM9cAe81Ltf2N0mvrvQXOlqmW6ta5I6yXc3vGiROS483p0RV8l59ELztY2mU+zqKhfV2msrW1m8kckTmtYjm4y1yrzRcLnp6+hpq7+05dZWqlo09RUy+Tqmd039TUYB0FobRVk0TakobFSJHvY8Wd/ypZl9Xu8/h0TyRDDdtmz7SOore+43muo7Fc2txHcXvaxH46Neiqm+n9aeS+Rzlfttmu7w1zHXp1FEv3lFG2HHwcnyv6zAK6tq7hUrPXVM9VO7rJNIr3L9K8wFxpm0ddPTx1MFUyJ6tSeBVVkmPNuURcfFC4Lqi+usCWRbvXLaEXKUnjO8P4Y9O3TPMyHSOynWOqXsWgs80FM7/7VWIsMePVFXm5P4KKbWX2Yp/c2U1HF7267nDr4H8Hezvf72PoA54t1LxtdBTLPBTpK9GLNO7djZnzcvkh2RsW2Y6W03Qw3WhrKS/XRyfshG5skca+kSIqo349fh0OZNYbLtXaUe9blaJ5KZvPiqVFmix6qrfm/7yIYrbLnX2qoSotdbVUU6f8AlKeV0bvytVFA/R8HEVi25a8tLWsW7Nrom9GVsLZF/nJhy/lM4tftO3WNqJddO0VSvmtNO6H+pyPA6lBrnZJtPj2jLXLTWapoY6NG78skrXsVzs4ai4RVXCKvT+0sntO6hu2nNGWyqsVwqKGokuDY3yQu3Vc3w3rj4ZRPyAZLtH2W2LaBW0dTe57hFJSxujYlLIxqKirnnvNUw/7W3Rn79vv5xH/lnOn6qeufxouf9KP1U9c/jRc/6UDov7W3Rn79vv5xH/lmsttGgNDbOoLfFA+811yq3K5Kd1ZGxGxJ1cqpEvVcInrz9DAf1U9c/jRc/wClMcvt7ud/r1rb1Wz1tWrUZ4szt526nRPgB1hsk2xO11eItPxaclpWx07nS1CVaSNjY1MZVNxOqq1PpKftbdGfv2+/nEf+Wcs6c1PetNPnfYblUUD50RJHQO3VciZwi/lUvf6qeufxouf9KBsPbRsz0xs7t1HUU1Dfq9lU5Y0qHVsTI4nomURyJEqqqplU6dF58jEtiGl9L6z1HUWbUc1fS1UrPEonU0zWteqZVzF3mrlcc0+C9jG71rzVN8t0lBd75W1lHIqK6GZ+81VRcov5ULDQVlTb62CsoZpIKqB6SRSxrhzHIuUVFA7Y0Jsa05onUDLxaam6SVTI3Ro2olY5mHJheSMRf6zZZwR+qnrn8aLn/SnS3sxahu2o9GXOqvtwqK6ojuDo2STO3la3w2Lj4ZVfygbhAAAAAcp+2F+2uw/xJ31imLezF91+2/7Cf6txlPthftrsP8Sd9Ypq/ZTq2LRGtKW+VFLJVxwxyMWJj0aq7zVb1X4gd+A54+2etn4t1v5y3/AfbPWz8W6385b/AIAdDg54+2etn4t1v5y3/AvGkfaCt+pNTW2zRWGqgkrZmwtldO1Uaq+apgDd5yv7Xeo31GoLVp2J36xSQ8XKiL1kflG5+DU/7anVBwpt8q31m13Ub3uzuTNhTsjGNbj+oCjYlo9mtNoFDQVTVdb4EWqq0TzjZj5P+85Wt+Cqd2RsbHG1kbWsY1Ea1rUwiInREQ5q9jihYsmp69yfLakEDV9EXfc7+xv5DpcAAAMZ2k6Yi1foq6WeRjXSzRK6ncvLcmbzYufLnjPZVOSf1CNof4Dj/PYP0ztwAWLQsVyg0bZoL7EsV0hpY4qhqvR677U3VXKKqLnGevmcze15Tys1/aqh294MttaxiqnLLZZFVE/nJ+U62NebbNnjNoOmGwU72RXajcstHI9cNVVRN5juzsJz8lRFA5j9njUtBpjaVTVF2kZDSVUL6RZ38mxK5UVrlXyTLUTPlnPQ7ea5r2o5io5qplFRcoqH5zX6y3LT9zmt95o5qOsiXDo5W4+lF6KnoqclMk0btQ1bpFkcNpu0rqNnSkqU8WJE9ERebU/gqgHexDmte1WvRHNVMKiplFQ5z0r7TNNJuRapskkLui1FA/fb/McqKif7ym4tJ7QtK6r3W2S9Us07v/s73eHL/MdhV+jKAaFvPs1Xia710tru1rioHzvdTxyeJvMjVyq1q4bjKJhDcWw/Q9z0BpmstV1q6Wq8SrWoidTq7DUVjWqi7yJ5tz9JsMlAJyTkpJQCtFJyU5AFZGAhIFCtyUKh9cEKgHyBUrcFIAAACckACQEAAnJAAqJKSQJJRSkkCpCSlCcgVYIA6gUnymVW4x5n3U+FR0QD4KqqQSVIxVAoKmsVT6tYieRVgChrEQq6E4JwBSTgnBIEICQAAAAAAAAAAIAkEZKVUCpVKVUgAAUq9GrhV5+g+Uv+qn9YRKuROqlOXO6Jup6qVI1G9E5+qkhWG7W2omzy7r1X9a5r/tWHKd06qdXbXPueXb4RfWsOUbp1UDFbh1Ux+r6qZBcOqmP1fVQLXOeN/U9k5439QKoz3U/keGM91P5AXei6oZHbeqGOUXVDI7b1QDLLX96dpeRxba/vTtLyAAACSPMeYAL0AXoAAAAIAgAwPX2ynTGuKvjLxBUx3DcSNKqnmVrkanRMLlvmvkWjUOlKXRWwO/2Oimknhgoql3iyIiOdvK53PHLllE+g2oYdti+5bqn+T5f7oHB1s/ZGl/2rP7UP0Q1N+1u7fxSb+4p+d9s/ZGl/2rP7UP0hc1r2q17Uc1yYVFTKKgH5qg/Rf7HrL+B7d+bM/wADVHtJVVo0zs8lgo7dQRXC6SJTROZTsRzWdZHJy9E3f95AOPwXTTNhuGpbxDa7PAs9bK17mMzjO61XLz+CHo0de36Y1TQ3N1NHUNppf16nmYipIzo5qovnjPwXC+QFjB+hVloNNXm00lyt1ttstJVRNlielMzm1Uz6cl9UPb9j1l/A9u/Nmf4Aak9kT7mtx/laX6mE3efCjo6WhiWKipoaeNV3lZDGjEVfXCefJD7gUTf6J/8ABU/Nc/Sib/RP/gqfmuB+jWm/2u2v+Kxf3ENC7Rdgl61Tra7Xqlu1uhgrJUkbHIj95qbqJzwmPI31pv8Aa7a/4rF/cQsu0DX1i0JbeJvVSnjvRVgpI8LLMvZPJO68gOWdfbFa3RGnpbteL/bNxFRkULEfvzPXo1qKnxXsiKalMv2j67u+0C/cZclVsLFVlLRxqqshaq9E9XLyyvVfhhEuuvdnk2i9D6dr7qj2Xe6SyvlhXl4EaNarWKn7rmqr6Zx5cwtuzDQVbtCvNVbrfV09LJT061Dnzo5UVN5rcJhF5/KM4l2B1cV+hssurLC27TRLNHSq5/iOYnmiY/8AnC+ilXsrsrX6o1E21SwxV62h6QPnarmNf4jN1XInNUya61vTamsetKp+pZKqK/tl8dahXrvOXPyZGOTy5clTpjHLGANxWf2b79Q3ehq5L1a3Mp52Sua1smVRrkXCcux1EaO2E7Zmap8Gw6meyK+I3EFRya2rx5Y6I/t0Xyx0N4gAAAAAA/PDXX7d9Q/yjUfWuP0PPzw11+3fUP8AKNR9a4DsX2cfuMad+FR/3iU2Ua19nH7jGnfhUf8AeJTZQAAACib/AEL/AOCpWUTqiQSKvJEav9gH5uUsLqmphgYqI+V6MRV6Iqrg2LtG2P33QViiu12rbZPTyVDadG00kjnbytc5F+UxEx8lfM1/a5GQ3OkllXdjZMxzl9ERyKp0T7Rm0TSuq9CU1Bp+7srKtlfHM6NsUjcMRkiKuXNROrk/KBqHZns6uu0OqroLPU0NO+jY171qnvaio5VRMbrXehZdaabq9I6mrbHcZYJaqkVqPfAqqxd5iOTCqiL0cnkbP9mfWNh0hc77LqO4Noo6iGJsSuje/eVHOVfmovqhhm2m80GoNpt7ulnqEqaCodEsUqNVqOxExq8lRF6oqdAOhLJrKHQ/s3We5K9vGvpXQ0carzfM5zsLj0T5y9kORnvdI9z3uVz3LlXKuVVfU3ZrLQ9yvGxTSOpbdNNUQ26icyoo85SNiyOVZWJ/e7Ii9EUy7YDsdt9Xpx961nbW1L65EWkppspuRdd9URU5u8uyJ6gaIuuu9UXe0vtdyvlbU296NR1PI/LFRqorUx2VE/IWGhq6igraesopXw1VPI2WKRi4VjmrlFTuiodV7ddnWkrDstvNys9jpaSuhWDw5mK7LczMavVfRVT6Tm3Z7R09x15pyirYmzUtRcaeKWN3R7HSNRUX4ooF1/VU1z+NFz/pTqX2cr5c9QbOkrb1WzVtXxkrPFmdl26iNwn9Zdv1ItBfizRflf8A4mT6dsFr03buAsdHHR0m+snhR5xvL1Xn8ALoAABpr2nNapp3Rnuajl3bleEWJd1ebIE+ev0/N+l3obduNbT26gqK2ulbDS08bpZZHdGtamVVfoOBtpurqjW+sq68T7zYXu8OmiX/AMlC35rfj5r3VQPJoTTVTq/VluslHlHVMiI+REz4cac3v+hqKv5EP0BtNvprTbKS30ESRUlLE2GJifetamENLey3oT3Lp1+pbhFivujcU6OTnHT5yi/76oi/BGm9ABb73erbYqPi7zXU9FTbyMSSeRGIrl6ImeqlwOENtTtSx6+uFHq2umrKmB6+A93yY1iXm1WNTk1FTHJPPOeaAd3oqKmU5oDSPszbQ/sisH2O3SbeutsjTwXOXnNTpyRe6t5Ivbd7m7gAAA/Pzaj90vVf8q1X1rjdOifaDs+ntI2e0T2W4TS0VLHA6Rj2I1ytTGUypo7aFL4+vtSy/u7nUu/LK5TZFj9njU14slvucNztEcVbTx1DGSPk3mte1HIi4YqZ5gU7bdrtu2haeobfQ22rpJKeqSoV8zmqipuObjl/CPr7JP3Tav8AkyX6yIxraXskvOz600twutbb6iKon8BraZz1cjt1XZXeanL5KmS+yT902r/kyX6yIDxe1N91io/ikP8AYprh9jnbpKC/tdvUz66Shc3d+Y5sbHoue6Od/N7mx/am+6xUfxSH+xS97MtLrqz2d9VUcLFfWU9ydWUyJzVZI4Y1wndW7zf94DBtkOzyLaLcK+hS9NttXTRNmYx1P4viszhyp8pMYVW/zjZ8nswyRxukk1bE1jUVznLQKiIief8ApDTGzPVUmjNa229MRz4YX7s8ber4nJhyfHC5TuiHWe3DWtHatktTW2+pbK68wpTUT2L89JW83J8Gby59cAcUTtY2eRkL1kjRyox6t3VcmeS48s+h+ielaOS3aXs9FP8A6Wmo4YX/AMJrERf7DhbZLYV1LtGsNtVu9C6pbLMn/o2fLd+VGqn0nfoGH7WdHs1voiutKIxKvHjUj3dGTN+bz8kXm1ezlOf7R7M+oZ91brebZRtXqkKPmcn0Yan9Z1gcS7XdXakotpmo6aj1BeKenirHtZFFWyMYxPRER2EQDc1g9mzTNGrX3i43C5PTq1qpBGv0Jl3/AGjZumtA6V0yrX2WxUVPM3pMrPElT/fdl39Zw79m+rPxnvv/ABCX9IfZvqz8Z77/AMQl/SA6Y9rW5TUWhrVBTTSQyTXBHK6NytVWtjflOXdyGmfZ4qq6t2v2GOasqZIm+M9zXSuVFxC/HLPrg19dr9eLwyNl3u1wr2RqqsbVVL5Uaq9VTeVcHmt1wrLZVtqrbV1FHUtRUbNTyujeiLyXDkVFA/SExTUuzvSWpXOfebDRTTO6zMasUi/F7MOX6VOIfs31Z+M99/4hL+kPs31Z+M99/wCIS/pAdKX32a9M1e860XK42569GvVs7E+hUR3/AGjCrn7Ml7iz7rv9uqfTiInw5/JvmoPs31Z+M99/4hL+kdH+yfeLreLXqKS73Kur1jmhbGtVO+Xc+S7ON5Vx5AbH2R6MboXRNHaXLG+tVVmq5I+aPld1wvmiIiNTshfdSactGpaOOlv1BDXU8cnisjlTKI7Cpn8ir+UuxqP2iNVat0hYaO4aYfTxUT3rDUzLD4ksTl+YqZy3dXmnNOuPUC93HZts5ttI+quNitFJTM5ulnduMb8VVcGjtoesNl1sZLSaP0lb7nW828VLG5tOxfVEyjn/ANSd1NN6g1FeNRVXEXy51dfKnzVnkVyN/gp0ROyFvpmwuqI21UkkcCuTffGxHuanmqNVUyvbKfECtGzXCuRsEG/UTyYZFBH1cq8mtan9SIdQ7H9gtHRUKXLXVMyqrpmfrdA5csgRfN+PnP7dE7r0xbZxtD2V6Eha+32a/wBTc1TD6+op4XSL6o39dwxOyfSqmefbK6O/B1//AKCH/NAzf9SfQn4sW/8Amr/iY3r7YZpm92CWHT9DT2i6M+XBPGi7rl/cvTn8lfVOadfVF2S+9UzdMrfFZLwiUfG7mE39zc38YzjOO/XzNR/bK6O/B1//AKCH/NA5bvtlumk78+hvNEtPW07kVYpmI5r08lTyc1fXopuTZ5q/ZZdY4aTV2lLfaq5cNWpYxzqd6+q81Vn05TuXzW+2HZlrW2cFfrFfpkbnwpmQQtlhX1Y7xeXw6L5opzte2WxlxkSxz1k1B1jdWQtikTsqNc5F+OefogHblv2a7ObjSsqrfYrRVUz+bZYHb7HfBUXBlWm9OWjTVHJS2GghoaeSTxXxxJhFdhEz+RE/Ifn/AGDUN409U8RY7nV0Eq/OWCVWo7+EiclT4nWXs46t1brC1XGs1LPBUUED2wU83gIySSTGXZVuEwiK3y6r15AbkAAAAAcp+2F+2uw/xJ31imntEaWr9Zahhs1pdA2rma97VncrWYa1VXKoi+SehuH2wv212H+JO+sUxb2Yvuv23/YT/VuAuf2uGtf3xZfzh/6A+1w1r++LL+cP/QOwgBx79rhrX98WX84f+gZFs82Eas09rey3evntTqWjqWzSJFO9XK1PRFYnM6gAA4K21RLDtW1O1c5Wte78uF/5nepxh7UNoW3bVampRitiuNPFUtXHJVRvhr/WzP0gbE9jiVFtep4uW82aBy/BWvT/AN06LORPZMv0du15WWqZ2626U2I+fWSNd5E/mq867AAAAD4V9XBb6GorKuRI6anjdLK9ejWtTKr+RDlWb2mdS+K/wbPZ0i3l3Uc2VVx5Z+X1A6xBiWyvUFy1Voa33u809PTVNZvvbFAjkajEcrWr8pVXmiZ+lDWeqfaHg09qS6WebTU8slDUyU6yJWI1H7rlRHY3OWU5/SBua/WC06gpOFvdupa6DybPGjt3uirzRe6GntV+zfpy4K+XT9bV2iZeaRu/X4U+hVRyfzl+Bm2yPaRTbRrdcKmChfQyUcrY3xOlSRVRyZR2cJ6OT6DPgOK9UbBtbWTfkpaSG7U7ee/QyZdj+A7Ds9kRTWFZSVVuq3QVtPPS1Ma/KjlYrHtXui80P0jLDq/SNk1fbXUV/oIqmNUVGSKmJIl9WO6ov/yuQOSdm+23UelKmGC51Et4s6fJdBUP3pGJ6sevNMei5T4dTsHTN9t+pbHSXa0TpNR1LN9jvNPVqp5Ki5RU9UODtpOlJdFayuFklk8ZkDkdFKqY8SNyZaq98Lhe6Kbr9j2+TcTfrC9+afcbWxtX712UY/Hxyz8gHTQIJUCSSklAKiclCE5ArBCKTnABUKHNJV/omSpQPiD6KmShyYAgE4IAEhAAAAAkgATklCkAVZJyUkgVZJyUZJyBVkolbv454JyMgUJGiFaIiDJGQJIGQAyAAAAAAAAAAABGQJBSrinOQK1VChVyAABQr06JzXsN1V+cuOyBEq9EXHVfRCMOd1VGp6J1KkRE6JgkKhGonROfqCQEAABh+1z7nl2+EX1rDlG6dVOrtrn3PLt8IvrWHKN06qFYrcOqmP1fVTILh1Ux+r6qBa5zxv6nsnPG/qBVGe6n8jwxnup/IC70XVDI7b1Qxyi6oZHbeqAZZa/vTtLyOLbX96dpJ0AjJJCEgCUAAhegC9AAAXqAAAAkw7bF9y3VP8ny/wB0zEw7bF9y3VP8ny/3QODrZ+yNL/tWf2ofpEfm7bP2Rpf9qz+1D9GrhXUttopqy4VENLSwt3pJpno1rU9VVQPrNLHBBJNO9scUbVe97lwjURMqqr5IcMbbNeP15rGWogc5LTSZgomLy+Rnm9U9XKmfhhPIy/bvtkfqpZLFpiWSOxIuJp8K11WvpjqjO3n5+hr/AGXaHrteaogt1Kx7KNjkfWVKJyhizzXP7peiJ5r2RQN2+yRpF0NNcdVVkKos6cJRq5OrEXMjk+Ko1uf9VxhntM6Bfp7VDtQW+DFpuj1dJupyiqF5uRfRHfOTvveh1tarfS2q20tvt8LYaSmjbFFG3o1qJhDzamsVBqWxVlou0KS0dUxWPTzT0ci+SouFRfVAOWvZy2pfY1Xs05fp8WWqf/0eaReVLKvqvkxy9fJF5+aqdcJzTKc0Pz/2j6IuehNRS225sV0TsvpqlE+RPHnk5O/qnkv0Ku09hm211hZDYNXzPktSYZTViornUyeTXeas9F6p05p0Dq4Hxo6qCtpYqmjmjnp5Wo+OWJyOa9q9FRU5KhXNLHDE+WZ7Y4mNVz3uXCNROaqq+SAUVs0VPRzzVEjIoY2K573uRrWoic1VV6Ifm0bb267VqnWt0ltlpmfFpymfhjW5Rapyf+Uf2/cp9K8+msm2i5OtLro231a2xr9x1WkLvCR3or8Yz9IHaW0G56ns+yWmr9GtgdVwUkT5lfHvvZF4abz406Kqcl5ovLJxfXVtz1BdlnrJ6q43Gpejd56rJJI5eiJ59kRDcexLba/TEEVi1WstRZkw2CpRN99Kn7lU6uZ26p5ZTCJurZzZNmtHdau86SqbZUVtW9ZEck7XOgR3VjGLzjTryxnnjphAMR2E7FfcElPqHVkTH3VER9NRrhW0y+Tner/ROjfj08Htj/sXpf8A20/91h0WnNORpP2mtHX7V9BYI9OW59c+mlmdKjZGN3UcjcfOVPRQNb+yD+3u8fyav1sZ0HtO0Ba9f2J1HcGpFWRoq0tY1uXwu/5tXzb59lRFTUvs2aA1PpLV1yrNQ2qSippaFYmPdLG7LvEYuPkuVeiKdFgfnveNJ37T+sfcEtLMl5ZK1sDYMqsiqvyHxqnVF6ov9iop3VoSG+U+krbFqueGe8tiRJ3xJyz5Iq9FdjGVTkq5wXGW10Et1guctJA+4QRuiiqHMRXsY7qiL5Jy/t9VPSyaJ8ixslY56JlWo5FVE+AH0AAAAAD88Ndft31D/KNR9a4/Q8/O3WkrJ9Y32WFyPjkr53NcnRUWRyooHZHs4/cY078Kj/vEpso1r7OP3GNO/Co/7xKbKAAAAW7UlS2j07dKp64ZBSyyKvojWKv/ACLia/29Xhtm2UX+VVxJUw8GxPVZV3F/7KuX6AOHrXQ1F0uVJQUUfiVVVMyCFmUTee5Ua1Mr05qhlWstmWqtHWplx1Bb46akfKkDXtqI5MvVFVEw1VXo1T3bAbY667WtPxo3LIJVqnr6JG1XJ/WiJ9Jv32tvuZUf8qRfVygczaJ0Rftaz1UOnKRlTJSta+VHTMj3UVVRPnKmeils1LYq/TV7qbReIUhr6ZWpLGj0eiZajk5oqovJUN7exz+zOpv4vD/eca99oj7smo/4cP1EYF+1DrvUuldE6Go7BdH0dNPanSSRpGx6Od40iZXeavkhjibZtoCIiJqOdET/ANBF+gb92d7OdMaz2Y6Sq9RW99VUQUSxRuSeSPDfEeuMNcnmql8/UH2e/gST89n/AEwOVtQbTdYaitE9rvN7lqqCfd8SJ0UbUduuRyc0ai9URTFrbXVNsuNLXUMqw1dNK2aGRERVY9q5RcLy5Kh01tr2TaO0xs0u93strfBX06w+HItTK/G9Mxq8nOVF5OU560Hb6a7a30/bq+NZKSrr4IJmI5W7zHSNRyZTmnJV6AZL+rPtA/GSf+gi/QNzezPrrUmrb3eoNRXSStigp2Pja6Njd1VdhV+SiGZfqD7PfwJJ+ez/AKZkei9nmmtF1NTUacoHUstQxI5FWeSTLUXKfOcuAMtANI7bdtNNpeOssWnHOl1Cn63JK5ioykymc80+U7C8sZTzXphQxP2pNozZVXRtnmRWtVH3GRi8lVObYfo5K7vhPJUNY7EdAya81fHDUMclno8TVr05ZbnlGi+rlTHwRV8jAle6qq1fUz/LlfmSaVVdzVebnYyq+q9VOodA7Ttmmz7S8FptdXW1kifrlRURUbmunlVObvlY5eSJ5IiAdAxRshiZFExrI2NRrWtTCNROiIhWaKq/aX0szKUtqvUy/wCuyJiL/wBtf7Cx13tPwoipQ6Wkcvk6atRuPoRi/wBoHSJpj2mNBfZLpVL3b4t662livVGpzlg6ub3Vvzk/3vU1lcPaX1PLlKG02inavnI2SRyf9pE/qMZuO3fX9Yjmsu0NMx3JWwUkX9rmqv8AWBgukr/W6X1FQ3m2P3amkkR6IvR6dHNXsqZRfid+6S1BRap05QXm2P3qarjR6IvVi9HNXuioqL8D87XuV73OdjLlyuEwn5DfHssa79036TS1wlxRXF2/Sq5eUc+Pm/ByJj4onqB1iUTysggkmlXEcbVe5fRETKlZg2269pYdl2oKnfRkstOtLFz5q6X5HLuiOVfoA4Vrqh9bXVFS/nJPI6Rfi5c/8z9FbDSLQWO3Uaphaemjix6brUT/AJHAmzu1uvWu7BbkbvNnrYken+ojkVy/Q1FU/QkDQ/tf/tGs38op9U81v7JP3Tav+TJfrIjY3tgORNE2VuflLccon/sn/wCJrn2Sfum1f8mS/WRAeL2pvusVH8Uh/sU2x7IX3P7t/Kj/AKqI1P7U33WKj+KQ/wBim1vZGeyLZ3eJJHNYxtye5znLhERIY8qqgag9obQb9IaylrKOHds10c6aBWp8mN/V8fbCrlOyp6Ka7rb1cK602+2VdVJLQ2/xOGicvKPfVFdj4qhu32jNq9BqSF2mNPtgqqCKVJJ65URyPe3okS+SJzy7z6Jy66BA6d9kfSLoaa46rq4seOnCUaqnVqLmRyfFUa3P+q46OOVNhO2qKw0tLpvVW6y1x/Ipa1jMeBlfmyInVuV+d1TzynNOpqSpgrKaKopJo56eVqPjkjcjmvReioqclQD6mN1+hdKXGsmq6/TlpqaqZ2/JLLSMc56+qqqc1MkPPX1tLbqSSquFTBS00aZfLNIjGNTuq8kAxz9TnRf4q2T8yj/wMa2gWvZvofT810u2mbHlEVsFOlJHvzv8mtTH5V8k5mObQPaHsdoZLS6Ui971yckncisp2L8er/ownc5i1dqi8atuz7jfqySqqF5NReTI2/uWN6NT4AW251fHXGpq/Agp0mkdIkMDEZHHlc7rWp0ROiHXGw/ZVZaXQFFU6mslFW3Ov/6U7i4GyOiY5E3GJlOXyURVT1cvoa22CbHKi+VlNqLVFM6KzRqklPTSJh1U5Oiqn/m/P/W+B1mBin6nOi/xVsn5lH/gP1OdF/irZPzKP/AyshzkY1XOVGtRMqqrhEQDiP2i7bbbRtNqaKz0VNRU0VPD+s08aMajlblVwnnzQ3B7H0Ct0hfZ/J9cjP5sbV/940JtjvsOpNpl+uVJI2WmfOkUT2rlHMjajEcnZd3P0nTPssUC0eyiGdyY42smqE7oipH/APDA2+W7UVmotQ2OttN0i8Wjq41jkb0XHkqL5Ki4VF9UQuIA5a9ozQOm9F6Ks/2PW5lPNJW7kk7nOfI9PDcuFcq9M+SYQ1dst2f1m0O61lDQVlPSPpofHc6ZFVFTeRMJj4m+vbB/adY/4+v1bjX/ALKd2t1o1ZeZbrcKShifQo1r6mZsSOXxGrhFcqZUC4fayX38PWz+ZJ/gaJudI6guVXRvcj308r4lcnRVaqplPyH6BfZrpX8ZbJ+fxfpHA+p5GS6lu0kT2vjfVzOa5q5RyK9cKigdyT/cZk/kBf8Au5wnaqN1xulHQxuax9TMyFrndEVzkTK/lO7J/uMyfyAv/dzh7SUjIdVWaWV7Y42VsLnPcuEaiPTKqvkgG6ftZL7+HrZ/Mk/wML2pbI7js9tFJX19ypKtlRP4DWwtciou6rsrn4HYH2a6V/GWyfn8X6RpL2q7/Zrvo+0RWq7W+ulZX7zmU1SyVzU8NyZVGquEA137OGlLNrDVF2t+oKNKqmbb1kYm+5isf4jERyK1UVFwqnXOktO2/Smn6Wz2iNzKOnRd3fXLnKqqqqq+aqqqcx+yD+3y7/ya762M60AAAAAAOU/bC/bXYf4k76xTFvZi+6/bf9hP9W4yn2wv212H+JO+sU0FHI+N29G9zHerVwoH6Ug/NzjKr98zf0ijjKr98zf0igfpGD83OMqv3zN/SKOMqv3zN/SKB+kZp32mdEy6n0ay52+PfuFnV026ifKkhVP1xE7phHfQvqXH2aXvk2Q2p0jnPcss/Ny5X/SuNorzTC9APzetdfU2u5UtfQSuhq6aRssUjerXNXKKdqbKtr1k1tQwQVU8NvvyNxLSSO3Ukd6xqvzkXrjqn9a622z7B55q2a9aFgjVkmXz21FRqo7zdF5YX9zyx5dcJzhWUtRRVUlNWQS09REu6+KVitc1fRUXmgH6SnzqZ4aWCSepljhhjTefJI5GtanqqryRD89rfrLU1tibFQaivFNE1MNjirZGtRPgi4PPd9R3u8tRt4vFxr2p0bU1L5UT6HKoG9/aF2w0V1tkmmNJ1KVFPMqcbWx/Mc1F/wBGxfNFXGXJyxyTOVNI6G0xW6w1RQ2a3NVZJ3/rj8coo0+c9eyJ+VcJ5n00Zou/ayuDaSw0Ek/PEk6puxRd3v6J8Oq+SKdkbItmdu2e2lWsVtVeKhqcVWYxnz3GejU/KvVfJEDNrVQU9qtlJb6GNI6WlibDExPJrUwn9SHHvtP6bks20mW4sY5KS7RtqGOxy8RERr259eSO/wB9DswxTaVom36801LariqxPR3iU9Q1MuhkToqeqc8KnmnpyVA5G2F6/ZoLVyzV2+torWJBVo1MqxM5bIiee6ufoVfPB2xabpQ3ihjrbVWQVlJImWywvR7V+lPPscF670DqDRFc6C90T2wK5WxVcab0MqeqO9ey4XsWC23S4WqZZbZXVVHKvV9PM6NfytVAP0fLZqC/2nTtC+svdwpqGnairvTPRqux5NTq5eyZU4Nk19rCSPw5NVX1zPRa+Xn/ANosNdW1VfOs9fUz1MyphZJpFe78q8wMr2vauj1tryvvFNG6Ojduw07X/O8NqYRV7rzXHlnBtr2PLRK65agvLm4gZCyjY5fvnOXfcifBGt/KhqrZ1sz1DrmsjS30r4LdvIktfM1UiYnnj92vZPpx1O1tEaXt+jtN0lmtTV8CBMukdjelevznu7qv/JPIC+kkACQEIcqJ1UCRnHUo3ld81OXqSjM83LkCd9V+an0lSNzzcuSU5E5AqTCElKKSAUjBUFQD5uaU4PqUqmQKAFTCgAAAAAAAAATkgATkkpAFQIAAkhABIIJAAAAAAABCrgCSMlO8RnIFSuKcqAvdQA5FO8qp8hM9/II3KfLXPbyCCuz81M/2DdVfnry9EKgFQiY6ckJQAIAAAQSAoAAMP2ufc8u3wi+tYco3Tqp1dtc+55dvhF9aw5RunVQMVuHVTH6vqpkFw6qY/V9VAtc5439T2Tnjf1AqjPdT+R4Yz3U/kBd6Lqhkdt6oY5RdUMjtvVAMstf3p2khxba/vTtLyABAAAQkgAvQDyAAAeYBAABJh22L7luqf5Pl/umYnju9tpLxbKq3XGFJqOpjWKaNXK3eavVMphU+gD86bZ+yNL/tWf2odNe1jpq91VBSX2krKmez0yIypokX5ELlXlKiJ1znCqvNOXkvLYcWxXZ9HI2RmnWI9qo5F4qfkqf75sCrpoaylmpquJk1PMxY5I3plr2qmFRU9FQDg3Zxs4v2va7w7VB4VCx2J66ZFSKPt/rO/wBVPpwnM7T0Fo+16J0/DarPFhrflSzOT5c7/N7l/wCXknIvdtoKS10ENFbqaKlpIW7scMTUa1qdkQ9AAkgkDGtf6NteuNPy2q7x8l+VDO1PlwP8nN/5p5ocUbR9AXnQV24S7Rb9NIq8PWRovhzJ2Xyd6tXmndMKvfh4L1Z7dfKB9DeKKCtpHKjlimYjm5Tmi/EDS/ssab1BbLBPc7pWVENorG5o7c/mi88+Nhebc9ExjKc18jKPaJptR1mzmrg0xF4rXu/6c2NV8V0CJlUYnnzxnzxlOeVNnNa1jUaxEa1EwiImERCQOBdlWiqjXesKa1RK6Olb+vVcyJ/o4kVM47rlETup3RTWO2U1hZZYqGBLU2HwEpVYisVmMKiovXPnnqeexaXs1huFzrbRQRUtTcpElqXRpjfcicuXROqry81VS9Acp7UvZ9uVDWz1+iGcbbnZfwLn/r0Pqjc/Pb6c97y59TR9zs1ztT1bdLdWUbkXGKiB0a/1oh+jhDmo5qtciKi9UUD82WTyxpiOV7U/1XKh9UrqtOlVOnwkU/ROey2uo/09topf4cDXf2oeZ2l9Pu+dY7UvxpI/8APz3941378qf6V3+JC3CsXrV1C/+0d/ifoMuktOL10/aF//AOKP9Ej7EdN/i9Z/zKL9ED89ZaiaVMSzSPT/AFnKp017Htl3Lbf73I3nLKyjid6I1N5/5d9n5DeH2I6b/F6z/mUX6Jc7fQUdtp/At1JT0kGVd4cEaRtyvVcImAPSAABwVtBZdNHa+vtqobhW0sVPVOWFsU7mYjd8pnRf3LmnepYbzo/Td7qnVN3sNrrKlyIjpp6VjpFREwiK5Uz/AFgcHP1dqSSF8Mmobw6J6Ycx1bKrXJ6Km8Wmkpp62qipqSGSeolcjI442q5zlXoiInVTvB+yrQz+umLanwjx/YXewaO05p6XxbJZLfRT43fGigakmPTexnH0gebZlYJdL6Cslnqd3iKanRJd3miPcqucievNy8zJwAAAAHPPtfLeZLNZo6elkWxxyulqJ280Sb5rEcnkmFdhV5KrjoY+FdSU9fRzUlbBHUU0zFZJFI1HNe1eqKigc1eyBpxzqu9akmYu4xiUMDlTkqrh8n5ERn85TMPa2+5lR/ypF9XKbV0rp226WssNqssHgUUTnOaxXK5cucrlyq816+fkiHvraKlrokirqaCpiRd5GTRo9EX1wvnzUDg3QWqNSaPpLnddM1badm9DBUo6FkmUdvq1flIuERWr09UMdu9yr79eKivuMz6q4Vcm/I/dTL3L2Tl9CH6A12k9P11BNQ1VmoH0kytWSJIGtR+6uUzhE6Keay6F0rY6plTatP22mqWLlkzIG77fg5eafQB8dldoqLDs60/ba1ix1UFI3xWO6sevylavwVcfQZUABrP2kvuM3/40/wD3iM5I2W/dL0p/KtL9a07s1Zp636qsFTZrwx76Go3fEax6sVd1yOTmndqGEWbYfoqz3eiuVFSVjaqjmZURK6pcqI9rkcmU8+aAbOAAA5D9qXTdRDtKjr6KmllbcqRkjvDYrv1xnyFTl/qoz8p14APzypdHamq8cLp28zZ/83RSu/saXik2U66q8eFpi5Nz/wCdj8L+8qHeoA4mo9g20Cox4lohp0Xzlq4v/dcpfqH2bNXTYWqr7NTN808WR7vyIzH9Z12AOZaH2YJlwtfqiNnq2GiV39avT+wyS3ezTpiHC191u9S5PJjo42r9G6q/1m9gBoXaFsF05TaGuUul6OoS807PHie+d71kRvNzN3OMqmccuuDl2w0txrLxRw2SGomuSyNdA2naqv30XKKmPTGc+WD9HDG9KaI0/pSoraiyW6KCorJXSSy9X/KdndRV6NTyanLkBc9OyXGWw2+S9wMgujoGLUxxuRzWyY+VhU5dTU3tSadv9+0lRSWWPiKGglfUVdOzKyO+Thr0TzRqK/KdeefJTdQA5E9k7Tz7jrupvMjM09rp1Rrv/SyZaifzfE/qNqe0lqnVOjqGy3PTNyWkppZX09S1YI5EVyojmL8tq45Nf0Np2PT9qsT651oooqTjZ1qZ0jTCOkVERVx5dOicsqq+alGqNN2nVVrW3X+jbWUe+knhue5vyk6LlqovmvmBwnrXXmotaupfsjuC1TaZF8JjYmRtaq4yuGomV5JzU3B7INhqXXy8398SpRx0/BMev3z3Oa9UT4IxM/wkNwU+xjZ9BKkjNNwK5PJ88r0/I56oZ3b6GlttFDR2+nhpqWFu7HDCxGtYnoiIBx17U33WKj+KQ/2KYzSQaibsgqKq31kyae96uiraaL5Py1ji3HPVOatXpheSKieanXWrtlWk9XXl10vlBLPWuY2NXtqJGJut6ckVEPbYdnumrHpq4WCht/8A9E17nOqIJZXP31c1GrzVcpyanTpjIHFWgNCXzXN0bS2WlcsLXIk9W9MRQJ6ud69k5qdWxbDdJN0R7gkp1fUL+uLc8IlR4uPnIvk3/U6Y78zZFotdDZrfDQ2qkhpKOJMMihajWp/+vfzPYBwRtI2c33QVwWO6QLLQPcqQV0SZilTy/gu/1V+jKczefskWu9R2S43OqrKhtkld4VLRuXLHPRflyoi9MfN5dV3s9EN93GhpLnQzUdxpoqqkmbuyQysRzXJ6KilFottHZ7ZTW62U7KeipmJHFEzo1qf1r8V5qBiG2So1bRaPnrdDzRMrKbMk7FhSSR8eOax5ymU64VFynTnyXiXUGorzqKp4i+XOrr5U6LPKrkb/AAU6J9B+iphVDst0XR3mpusdgo31k8iyqszVkYxy8/ksdlrefPkgHHOitnGqNYyMWzWuXhXLzq508OFO+8vX4NyvY6U2a7BLHpp8VdqB7bzdGKjmtc3FPEvZi/OXu78iG5mtRrUa1ERqJhEToiEgOnQwrbLS19Ts1vj7RV1VJX00PFRS00ro3p4ao5yIrVRebUcmO5mpTLGyWJ8crGvjeitc1yZRyL1RUA4Bh2ja0i+Zqq9r/CrZHf2qeW8a11Peqd1Pdb/dKqndydDJUvVjvi3OFO159l2h51y/S9rT+BCjP7MHyh2T6EhfvM0xb1X/AF2q5PyKqoBxTo/St31feIrbY6R88z1TffhUjib+6e7o1P8A5TKne+kbHDprTFss1Ku9FRQNh38Y31RPlO+lcr9J7LZbaG1UyU1soqajp05pFTxNjb+RERD1gAABoP2wf2nWP+Pr9W45hsljut9nkhsttrLhNG3feylhdIrW5xlUai4TJ3vrbRlk1rQ09JqGmfUQQSeLG1kro8OwqZy1U8lPDonZvpnRVdUVenqOSnnnj8KRz53yZblFxhyr5oBxd+p3rP8AFS+/mMv6JjM0UkEz4pmOjljcrXscmFaqclRU9T9KDWNZsM0HWVk9TUWud000jpHqlXKmXKuV5b3qoF6n+4zJ/IC/93ODIIpJ5o4YGOklkcjGMYmVc5VwiInmp+jDrTSOsS2hY14BabhFZvLnw93dxnr08zAaHYdoShraerprXO2eCRssarVyrhzVyi43vVAOSv1O9Z/ipffzGX9Et960vfrFBHPerNcbfDI7cY+qpnxtc7GcIrkTnhD9EjHdbaNsutaCCj1DTPqKeCXxmNZK6PDsKmctVPJVA5t9kH9vl3/k131sZ1oYborZrpjRdxnrtPUUtPUzReC9z53yZblFxhyr5ohmQAAAAABarxpux3uWOW82a23CWNu6x9XSslVqdcIrkXCFv+wHR34p6f8A+Gw/omSgDGvsB0d+Ken/APhsP6I+wHR34p6f/wCGw/omSgDGvsB0d+Ken/8AhsP6I+wHR34p6f8A+Gw/omSgDy2y3UNqo2Ulro6aipGKqthpomxsaqrlcNaiImVPUAALVfdPWa/w+Fe7XRV7MYTiIWvVvwVUyn0F1AGt6vYhs9qcqun2xu9YqqZv9SPwfe2bGtAW6VssOnKeR6fviWSdF/3XuVP6jYIA+FFSU1DTR01FTw01PGmGRQsRjWp2ROSH3AAAACieKOeJ8U8bJInphzHtRUcnoqKYTddkuhLo9XVWmqFrlXKrT70H1atM5AGsE2EbPUfvLZJFT9ytbPj+/kvdm2W6Is72vodNW/fauWunYs6ovqiyK4zQAQ1qMajWojWomEREwiISCHPRvUCSHORvVSjLn9EwhW2NE681Apy53zUwnqSkaZy7mpWAJTCeQIAE5JKScgVBFIGQK8kopQSigVEAkClUKVbjofTBGAPkCtWlOMAQAAAAAAAAAAABIAAACSMgCQQFUCSMlKuIAlVIUEKqN6qBJCqic1XBGXO6JhPVSUYic15r3CIyrvmpy9VCMTq7mvcqAU7AAAAAABUjQKQV7ociNTOAKAVoiKmUIVoFIAAw/a59zy7fCL61hyjdOqnV21z7nl2+EX1rDlG6dVAxW4dVMfq+qmQXDqpj9X1UC1znjf1PZOeN/UCqM91P5HhjPdT+QF3ouqGR23qhjlF1QyO29UAyy1/enaRxba/vTtRWoqcuoFIGMdQAJI8yQiF6AL0AUHmAAAAEgAAAABBJAAkgkAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAASBAJIAABc45AClXoi4TmvYpw93zlwnYra1qAU4e/shU2NG/EnJG8BUMkACcggASBkAAABOQQAKiSABVkkpyAKycFJKKAKVTJWQoHzVpSfVSlWgUAlUwQAAAAEoAIJBAEoSQQrsASMlGVAEqqqQClXIndfRAiohXInx9CMOd1+ShKNROiAQm8v+qn9ZLWonx9SUJCoQkEYAICQBGCQAIUISoROYFaJgkYJwAQlUyioRuZ6Lgld5OvMD4PXwlankpVvlM2X7qIi8lzzIRrvQCpVRSBuqRkDENrn3PLt8IvrWHKN06qdW7XF/wDB5dvhF9aw5SunVQMVuHVTH6vqpkFw6qY/V9VAtc5439T2Tnjf1AqjPdT+R4Yz3U/kBd6Lqhkdt6oY5RdUMjtvVAMstf3p2minFlr+9O0gK+S9UKVaqEFSKBSSVK3zQo8wAHkAAAAAACQCPMCQAAIJIAEkEgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAJAjBIyRkCchVyQAAAAAAAvMoVFyVgD5ElTm+hR5gVIpXk+YReYH0GCEUkCQQT5gAAAAAAlCCQJJQgASSilJIFeSclCKSigVEKgQkChUKVTB9CMAfIkqVCkAgQFKqBVkpVSM5ADOQR8SN/yamf7AiSN5PLmvog3VX56/QhUiInJOSBVO653zlwnohKNREwiYJAAZBIAAAAAAAAAAAQfJZmtmbGiK56rzRPJO5FS9zUbHFjxXrhM+Sea/QTBGyBqNTqq81XqqhHr8ggTmmUChUopOT5qpSsip5AfVSlVx5nwdM7ywfF7nO6qoH0nnTCtZ+U+cU7d9I3Za7yz998D5Keh8DZ6dqOTyyi+ihGJ7XPueXb4RfWsOUrp1U6k2oSOXZ5eoZl/XovCRV/dJ4rMKct3TqoVitw6qY/V9VMguHVTH6vqoFrnPG/qeyc8b+oFUZ7qfyPDGe6n8gLvRdUMjtvVDHKLqhkdt6oBllr+9O0ji21/enaQAAAVIuFJ5OQpCAFTCEFaLy5kOb5oBSAAAAAkjzJIAkAIAIJIAEkEgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAkCCRkgCSAAAAAAAAAAAAAAAAQqZJAHzciopB9SlzfQCkqRSlQB9MkofNFKkUCoBAAAAAAASAg6gSAAJJQpQkCpCSglAK0CoUK5E6qUukXyArcqJ1Pm53oUKqqQBKqCFVETKrgp3lVfkJ9KgVdOvQp3sr8lM/ElGebl3lJCKdzPzlz/YV/AAKEkDAEkEgCCQAAAAAAAAUuejeqgVFLnI1Oa4Pk+VV6cilI3OXK8u6geaSZVuTVTkiQqiZ+KZ/5H3ja50jXLnGfMoqoVYsc8SK58arlP3TV6p/Yv0HpjckjWvYqK1eaKEejm3pzQneRfMqTmnIpcxFCoU+TipWejlKHMX92BQ5D4SSMb1Xn6H0fEi/OkcpQkbGr8lvP1UD5JvPXKpus9PUucP+ib8Dwqiqoqap1PTq6NniOROTc4CMR2wNVNJXVW/NWCLe+Pjsx/zOWrp1U6e2m+NUbPLpVTruPd4S7rUwmPEaiIv5VU5hunVQrFbh1Ux+r6qZBcOqmP1fVQLXOeN/U9k5439QKoz3U/keGM91P5AXei6oZHbeqGOUXVDI7b1QDLLX96dpHFtr+9O0cASBgASAAIKkcUr0AFaojkKVTC9AiqhXlF6gfNAVObjoUgSQpj2tY9Uy2+BNFzWmGtSXMq3JHqxY8L03EVc5x/WYbwm2X8IaK/mVH6IG1EBqvhNsv4Q0V/MqP0TL9ERatipqpNaz2eadXp4C21Ho1G4572+ic8+gGS4INZ7SrtqZNdaX09pa7QWxblDUySyy0zZkzG1HJyX6U+k8GnbtrC17TanTmpL7TXSH3G+5Ruio2Q7r/FRidOa4wv5QNtkmjtnztpmstIW+/Q60oaVlWj8QutcblbuyOZ1wn7nP0l82e3fVce069aa1PeoLrFRUEdQ18VIyBEe9zcdEz0VfMDaoLTqbUFr0xZ5rne6uOlpIk5ucvNy+TWp1c5fRDRGitZa51Xq286l03RU1fQubwsNsnuaRpSsReTnRZ+cuPnKnm5E5AdGg0lq7aTr7SNuStv2mbHTRPduRt947z5HejWpzcvwNo6KuF1u2mKGuv9ubbbjO1XyUqOVfDTeXdznmiq3Cqi9FXAF8Brfa9eL7Q1+k7dpy7NtMt1rnU0tQ6mZOjW7uUXden/NCyOq9aab2h6Ptt21jHe6C7y1DJYm22Gn3Ujj3k5tyvNVTpjoBuMGnErNZ6k2i6wtlp1jHZKC0SU7YonW2Go3kkj3l5uwvJUX16l52TXe/1V/1faNR3lt3daaiCKGobTRwIqOYrl+Sz6Oqr0A2UAYZtU1v9gVgpbl7udcXVFYykbC2bw1y5r3IucL+4xjuBmYNVfqia1/6rrl+fs/QL7o3Vuo73d3Ut50VWWSlSJz0qpqpsjVciphuEanXK/kAzgGD7Qto9s0bPSUHD1N0vlZ/9WttIm9I/nhFX0TOfVVwuE5KY2u0bXNLCtZctmNcy3pzctPXNlmY3zXw0blfyIBtwFh0Xqu1ayscd1sc6yQOVWPY9N18T06te3yX/wDlDDbjtRrK681ts0Fper1JJRP8KoqknbT07H/uUe5FRy/k7ZQDaANUU21iutN2o6DaDpSr04yrekUNbxDainV6+TnoiI38q+q4TmbX69AABp7bLqm62fWumbXR6pZpq210E76mskpY52tViZblHp5ry5KnUDcINAfZFP8A9e9B/wAHp/8AE8NVrS8W++6aitu1Kn1I2tutPS1FJFbYIlSJzsOdvJlceXLHXqB0aAAAAAA89wWqS31K29InVqRO8BJc7iyYXd3sc8Zxk0ffrntSdq/T2nLne7JZ3Xzx9yW10yzLE2Jm8ufE816JhfpA3wDEI7xa9BWS3W/Vmp2y1Tmv3auvcjZJ8Oyq/RvInwwa905tMsrdrurpa7VMXuB9PTJQpJUqsG8jG7/htXki5znCAbxBbLHfrXf7e+usldBXUrXKxZIXZbvIiKqZ9eaflNV6c2s6u1Jao7lZNm0tXQyOc1srbvG1FVFwvJY0XqBucGhLneNplVrm1X+j0RX00VJC+Gpty3mN8NQ12cLjCIxyKvzsLnCehlH2d7Qf+qyf/jMX6AG0wYjsu1iuutLNvDrf7vcs8kKweN4uN1cZ3t1v9hcdYars+kLS+4X2sjp4kRdxmcySu/csb1cv/wArhAL6DSlPbtoWvGy6lZdqnSrYkR1nteV3ZEzneqU80cnLCouM5xj52V6B2hx3mrksOpYG2bVtMu5NQyuw2b/XhVfnNXrhMr8U5gbAB5bslatrrEtSwNuKwv4ZajPhpLurub+Oe7nGcc8GtfA2zfvvQ/5Kn9EDagNV+Btm/feh/wAlT+iY/rLUm1jSVNbp7lNpCRlfXR0ESQMncqSPRyors4+T8lc4yvYDegMV0lJqihttfUbQKqxN8L9cZLQK9sccaNVXK9ZETGOuemDVF/17c9T6rbPYdU6asNos9SrY47hdUT3i9OSuc2J2Vj9Ezhc559EDoEHNGsddakfqLSrvst0M/cq3LvW6ql8BvyU51P658z06czLvs/1J+OWyn85m/wAwDdILRpq+UN8t7ZaG5224yxta2ofQTJJG2TGVRMKuE9EXngu4AAAAfKrZLJSTMppUhncxyRyK3eRjsclx54XyNI7SJ9pGiNJ1F8l1rSVjIXxsWFtpiYq7zkb15+voBvMGotu+tLfZ6OOho9SyWnUtI+KuggRk3h1DUcv63IrGq1WrheS+aJnkpjG0rbLY71YLVarBqB9FUVtTF7xraeOdq0UTcOerHbiOcucImE5oi5xkDoQFk0rqOz6gsDLlZK7irazLOIkR7PmdVXfRF+lTXd81dBQ7bbZPLfEj00un1qXq2pVaZzlme1H4Rd1V6Jn4AbfBr2+7QoLLtTpdPXGstdFaXWta2Wpq5Ujd4iyK1rGuVyN5omcYyYxoLX9q/VI1/wC9dV0XurxaX3fxNxb4ONx+/wCDvO3cZxnd7ZA3SCzMu8N8sNfUaRuVsrqlrJI4JmTJLA2fdy1HqxV5ZVqqic8KYJ4G2b996H/JU/ogbUBqvwNs3770P+Sp/RMf1lqTaxpKmt09ym0hIyvro6CJIGTuVJHo5UV2cfJ+SucZXsBvQGNaIZq1lNVfZtLZpJ1engLbEk3Ubjnvb6Jzz6GKbTrtcLjrjSmjrBWz0lRPN7wr56d6tdHTR5+TlPJy7yfFE9QNoA1rV7a9HUtbU0slRXulppXQyblFI5Ee1cKmUT1MG2VbYrRb7HcI9S3C7VVU+4zyRPfDLOrYVxut3ueETn8nyA6DBYtG6ptesLOtzscsklIkrocyRqxd5uMphfihh9outdf9tN4dTV80Wn9P0baSWNsipFNVPVVXeToqtTKdlRPUDZoNQ02vNZ3e/XyjsVr08tNb7nLbY3VlW+J8rmZVMJ5rupnl3PPpC37V9OU1bTrTWW4xT1T6iNay4yyPhR2P1tHKmVamOWe4G5gYTsy1TddR0t8TUFBT0NXbK+Sic2BXOY7cRMuRXdUznma/v21OwV+1bTi0uqOH07QUk09U+KSRkc8rvktjciJ8rGEdzTHUDewMA15rm3w7JrxqSx1zJ4XU74aWeNVTMrl8NFTOFyjlz9BZNAbTtI2zRVlpLvq2GouMdKxamSokkkf4iplyK5UVVwqqnwQDbQLXpy/2rUlvWusdbFW0iPWPxY843kxlOfxQxTVFPtMffal2mKzS8doXd8Bta2ZZk+Sm9vbrVT529jtgDPwar4TbL+ENFfzKj9EcJtl/CGiv5lR+iBtQHntyVSW+lS4uidWpE1J1hzuLJhN7dzzxnOMmurvX6u0frVKqfx9QaTutS2Pw4YUWe2vdhrcI1PlR9P8A+fnBs0A1fsor6yr13tIhqquonhprlGyGOSRXNibh/JqKvJOSdANoAplkZFE+SRyNYxFc5y9EROqmg471r/U18u2utFR0z7BSwvpKGgr3zIlW1nN0rImKiOcrt5EVyp1x1TkG/hg09DrTXtHoebUV7ttqmZV0zW2+kttLUpUtqZHI1nitkyiMTnn1XGFXKZsltk2taMprBZ5pLRcW3GZ0fGVSVNU+CRybyNlci8k6oitRWphcr5gb9wCxaRTUjbfKmr5LS+t8VfDW2NkSPw8J13+e9ne7YwazsFVtC1dddUvtWq6K3UVtvVTbooZLcyVd2Nybq73wcic/QDdOSDQP2e6tqdlOm7hBdIYr3cdQJa5KpaZit3FWRqfIxjq1q+vIymtsW0+lo56h+vberYo3SKnuiPoiZA2qDC9jd8uOpNm1mu16lSavqUlWR6MRiLiV7W8kRET5KIWXbRtOotFWiWioZ45dRVLN2CFFRfAzy8WT0RM5RF6r2yqBs4Gj9CV+1C2aVoKemsluvUW6r21895SZ0yOVVzvby8ueERPJCuk2n61frqj0vNpm1SV0jmuqW0tYsvDRZTec9U5Nwi5wq56eqZDdoAAAAAAaXsGo9pWq67UDrBPpeChtt0nt7W1kU3iLuKmF+TlF5KnpzyBugGtdmOsrrcH6mp9aVNpgntFclH4tPmKJy4XPN68+acuhbtmOrIZtXa+ZdL/G+ljuSNo21FYisbHh3KPeXCN6dOQG2wfCirKWuh8aiqYamLO7vwyI9ufTKGrNT6gqdV7WLVo7T1ylpqW15uN3npZlY5yMVESDKL6uRHJ/rerQNtA0ZqHVGqtV7QKefZ7RTXPT9gkWOpWOtSlirJ1Tm1Xqvy2t+TyTKdfJUUvdNrfaHVUa1dLoi1T0qIqrNHfYXM5dflJy5AbYBh2zfVdXrDQ0V+qqBtA+ZZfDjbJ4iK1qq1HZVE6qimo6HUu0ao2UO1zNrChgpUjkelMttjVyubKsbW72MfKciflA6MVuSlUwpz/tA2k3NNN2fSFDc4Y9Y18cMVyrUekMdG9zcqxXpya9V5cunPouC737V+0HQ2lY6q9WvT60tKxkKSS18ss07sYROaZe9eq/SoG6Bktumam4V2nrfV3ikbR3CeFsk1O1cpE5Uzu8/QwXaXd7jW600rpCwVk9JVVM/H188Dt10dLHnkq+jlynxRPUDZyKVIpqtNpV8u1yulHo/RdVc0t9XJRS1M9bHTxJIxcL1znyXHXClioL7tJ1Zq68abWvs+mZbbHFJO+mgWpfiRu8iIrlwq4+AG8geCx0tTQ2mkpa6tfX1MMaMfVPYjHSqn3yonLJ7wAMQt2tG1u0y66QShcx9BRsq1qvFyj97c+Tu45fP658jGV2ia1RVxsuuS//AHgz9ADapJpbT22PUWo6Ba2ybOq6spUkWJZI7gzG8mMpzZ3Q2Zoy73K92h1VerHNY6pJXMSlmlSRytREw7KInXK/kAvpIMH1htT0ppC8e677WzQ1nhtl3GU75E3VzhctRU8gM4Bqep28aCfTSsZdKpHuYqIqUcqc8fwTVenNp1H+pTbaGfV90t2q6GeSoZK6KWdky77lbHKuF3mK1UTGeXLkuMAdWlKuX4GD7LNodu11bP8Ao0rXXSmhjdWxxxyNjY92eTVeiZTLV/8AnmeyTaToqN7mSaotDXtVUVFqW5RU+kDKQaR2u7RrNJ9in2Oaqp/k3qB1ZwdZj/o/Pe391ebOmc8jP/1S9E/jTZ/zpv8AiBly9CEVzuiY7qWuwajs2oY5n2O50lwZCqNkWnkR6MVeiLj4F1+HUJKEaiLlea+qlRCL69SQAIc5GtVzlRGomVVeiGmNVa9tX6sGiVodV0XuTwqvjvBuLOHz4TtzxcO3c5xje8+gVugGnNrGsq2j1PpCls2rqax2W6w1Es1xSGGpjVGtRzHIrkVFRV5clT53mWKu1PVQUVRNBtwoJ5Y43PZElqpUV7kTKN+noB0ADENkN4rtQbOLHdLvPxFfUxOdLLuNZvKj3J0aiInJE6IY9ftY7QYpbglq0HAykpFk/wCmVdxZuyNbn5aMTDsKiZQDaBJpjZ43W20G32vVF11U212uSVZYrdbKdG7yMkVqo97srhVavL5SKil9tN1r5Nv17tclZUOt0VmjmZTLIqxter2Irkb0zzXmBsoA1ily1ZpDXTKW7cRf9M3mqVtNVRQ5moJHc0Y9GphY+/oir2UNnAw7WVPrqa5RO0hXWKnoUhRJG18UjnrJvLlUVvLGN36clhWi2vY/ZbSC/wDsJv8AADZ5Q6RrfMtWn0u6WemZf5aWW6IipO+karY1XK43UXn0wYNqGvr7vtisGn7VWz09HbIH3K6rC9URyLhI4n+S5XC4XydnyA2U6RzuTeQbGq9eRq/bvrGO0bO4n2K6xw1F1qI6anraabPhtzvPkRzV6IjcKqfujGtVbULFR6p0VXWTVj6ygppVo7lTq+TdfG5mEneioiKreaqvNegG+msRvRCrBjum9a2DVMdaumrhFcn0jUdIyNHNVN7O6nNE67qmqtoGpNeaksTaW16Hv1pr4Z46iGpirW4y1fmvaiJvNVM8l5ZwvkBvZT5xsa16q35OeePJVNe6f1xqm43SipLjoC40EEr0ZLVuqWOZF6uxjODCtKbVbDa9oGuptTahmhgfXNgoqdzZpY2siRzHOajWq1u8qJ6ZwB0AiOTy/wAFJRyL15KYvo7X+mdZz1MOmrnxslM1HyokEke6irhPntTPTyMOj1ZtLqaqrgotJ2GpWmk8ORrLsxXRrjKI5EX5K4VFwvqBthUPm5DnXVWpdotXtF0xRRWKlpL5TJJVJSU1x34poMYd4yZ3WpyVEcvPK8vIy+m1jtJrHTtpdJ2Gdad6xTJFd2OWN6dWuwvJU9FA2s5pQqGKbO9Q3zUFBUVV7tlupot5Ep5KGtbUsl6o7LmrhMKiIZS57vPdb2QIomfhMNTmed0SzKniY3U57nr8T7Od1ROhDQMW2rK79T66ovTEX1rDlm6dVOpdq37QLr8IvrWHLV06qFYrcOqmP1fVTILh1Ux+r6qBa5zxv6nsnPG/qBVGe6n8jwxnup/IC70XVDI7b1Qxyi6oZHbeqAZZa/vTtHJxda/vTtIAAAJAAEL0AXoAATqABW1ScIqHzKkUDXWpNQbQ6O91cFj0TS3C2sciQ1L7lHGsiYTnuquU55T6DFLvtO1/aLxa7XXaFomXC5vcylgbcmPdIqYyvyVXdRM9VwnX0U3ZW1dPRUc9VWSshpoGLJJI9cNY1Eyqqvpg1PsspZ9Y6tue0W5QvZTSItDZIZEwrKZqqjpMeSuXP5XeSoB9fsp2q/8AV3Rf8Xh/SLlp3UO0SrvVJBe9E0lBbnvxNUsuUcixtwvNGouV54LrtTdqGn0jNXaSn3LjQyNqnQ+GjuJiYuXxc0ymU9Oa4x5l00XqSi1bpmhvVtd+sVLN5WKuVjenJzF7ouUA1xtXt9fdNrehaS03WS0Vj6atVlXHE2RzMMRVTdXkuUyn0ltsdpulm25VdPe77NfKl2lpJG1MsDYVa1ajCMw3lyVFXP8ArFw2mWiXUu2XStrp7lXWx8Ftqalaqhk3JmIqo3kvlnGPgqlvsdgl05tyq6Se83O8OfpWSXx7jL4kjUWoxuouE+T8nOPVVAxzZ5tJ0/a9jlqsTNYfY/f4FkV0q2ySqSNFne7Ct3FauWqnReWfoM12JVum63Ud8q6LVs2p9S10bJaqofQvpUbEzDURrVajUTLk6L6cuRg2g1fpTQ+gddxwukt8MdTbbujEy5Kd9VIrJMeaNfzX4ohsjZG9dS6p1XrZGKlDWysoLaqpjMEOUc9Oznc/iigeTaPqjZwzV9GzUkVRcr9aVVIqKKmlk3VciORVbhGOXoqc1659DCdWsu+qL1FqrTGnpdESUzkWbUF2qm0TZGqqIiSRKi7yKqomeeeSLnkbk1hb9WyV8M+jKqx0aOYqVLq6Bznvd97hW+SJ6mndra6/rY7boy73XT9ZUX6djG01FDI2RjGuR3iOVejEVv8AUvooFdst2odI6ofqLaBpuu1hOrkdT3agnSpZTt6p4cGE3fXPJE8vM2lpnazpPUV3htNHV1UN1mVWspKikkY9VRFVfJUTkir1LZFbNrUUbI2XrSaNaiNROGl6IerY9f7vqWG/P1KlDNXWm6S2+Oemh3WruIiOVM8+eV+hQLZtQu+nrncVs+pdC6rvkdC9HxzUVC90Kuc1FVWva9ueS4Xuimray96VtV2p2bOdAX2j1bSytkctVSSyPp4lTD3JF4rlVVRcIioifK69DO9td62iW6rp6e0yUVJaK+vit1Lwb8VlQ6Rqr856bsfNFTKYVFwvQjQFwn0JdrRY6rQdRan3ypWJ9yqLrFVTTyIiqrnq1uV+HJOa9wMS2i3bRV9qa1z9Baoj1fcY1dTPq6F6LK9rURF8NJkymERFwi464UznYxdbTbHUtqt+gtSWGuuDWrWTS0cnCpIxjlzvveqonVE5dVRD562vVTra43rTsGgZr1DZ6lI+MiukdNJDLjLXsVyIrV+CqnLmebY9e9odXqGutdzdTTWu2VLaaqZcJEfWQIrFc3EkaI2ReSIqrnqBvI017U0jIdDWWWVd2Nl7gc5fREjlVTcppz2oURdF2JFTKLfKfl/uSgXv9W/Z3+MTfzSo/wAsyDSGvtM6xqKiDTlzStlp2I+VqQyR7rVXCL8tqZL37pt37wpP6Fv+B9qajpaZyrTU0MKu5KsbEbn8gGoNiMLdRax1vrKt/X6p1xfbqV7k/wBFCxE5N9MtViL8O6m5jSuyOsi0jtC1foy6vZTTVVc65W5XOw2eOTybnq5ERvJPR3obpc5rGq56o1qJlVVcIiAYxprRdv05qO/Xa2y1DPfL2ST0uW+Ex7c/KaiJlFVVcq8/M1Rom9VuxmK4WHVtlr5bMtW+op7zRxeLG5rsf6T0XknfyxhMmeaL1xcNYX/VKWenpJLBbcQUVXh2aqfdyvys4VqKnknRzVzzPZsl1w3XemHVlRDFS3OnmfBWUjVX9acirjKLz5pjr5oqeQHmra7RW1zTU9mgvENXDNuyKyCRI6iNWqio5GvTeTpjO70VTOLfTJRUFNStkklbBE2JHyKiucjURMuVMc1xzNT7c9F6bp9IXDUlNBBZ73b0Sopq2lxA90qL8lq7uN5VXki9c45mwdAXCru2h7DcLkmK2qoYZZlwibznMRVdhOmeuO4F9lkZDE+SV7WRsRXOc5cI1E6qq+SGntptFJetZaev2nLvpCZ1shnjdBdqvMb1kTHzW5zhMr1Tng27W0sFdRVFJVxtlpp43RSxu6PY5MKi/FFU0PtAsOibFrrSOmrdpyyNqLpUI6qdUtdiODOERPlJ8py7yN7t6cwPdxWqv/8AUX9LIW25Wm+X266fkulds3oqW23OCve621L2SvRjubeaYXlnly545nm22aO0npe46MmodOs4ea5btVT0kavkqI0RFViNzzVfQ+3i7N/+qrU3/B3/AKYG8p7/AGmG01lzW4U8lDRxulnmhekqRtamVVd3K9PIwv8AVt2efjGz81n/AEDLNPaVsenaOopLLbKekpqhd6WNjcteuMc0Xsez3Na/wbRf0Df8AMH/AFbdnn4xs/NZ/wBAfq27PPxjZ+az/oGce5rX+DaL+gb/AID3Na/wbRf0Df8AACx2fVNo15Y7pHpC9r4rY1h4qOB6LTvc1d1yNeiZVOv0HPOufeNo2xaftV22g1T6iljcq3SSkY3gXStVN3dRcLlEblVXCI7PkdK6hqYtMaZulyt1sSaSmgdMlPTRo1ZVanJOX/zjJpBmlpqbaFs8ZqqOOqut8W51V1a5MtV0lOieF/BazDcdOS4AvWobFLFtI2W2vUde3UTl95LJPVU7USZPDRzcs5py5Y+GT66S05ZJtu+uqOWzW19JBS0axQOpWKyNVjaqq1uMJnsfW67JLy+r0lDbtTVMVJZn1iNrFdipgikREijj5KjsIm6qqqclXHLCJi2m9E3Wo2wavtkes73BU0tPSukrmKni1COjaqI/smcJ8AMx2DzMt2gdUTR00kkdNd657YKdmXuRrWruMb6rjCIWbSW0G0aA0o23UujddxWylV8rpqygb8neXKq52URE5me7HdI3DRem6233aoiqZ5rhNVNljcrt5rkbhXZRPlclyeDbnpzUeqNP0lBp+OnqKJs7Zq+kdULDJVMaqKkbXY3UTOVXK9Ub6Aaz2ea4qItUX/WF30lquvq7sqMo3UVAskUVKmN1rXKqZVcJlU5fJz5qZwzblQyXGS3s0frB1fGzxH0yUDVla3l8pW7+UTmnPHmVLtktGnLa6k1Jp+7adraaBUgopYN6OXdTkyKRqbqp0TKoiHt2HWeqfb7jrG9OjfeNSSJUuRjt5IYE/wBHGi9k/wCSLzQCNlkSR6Autv0jb71ZqmOSV1O7UNOjXLK9Mo7DerEXHl+Ux6o2X6lt1wo7/FUWvVWqVcqzz3xXpBTpyVvgRtXCYXPNenLCNN4mE6xbtCddmro6TTTbb4SZS5eN4viZXPzExjGP6wMEv2tNqlkvtitNbbtILVXmSSOmWNZ1aisRqrvLv8vnJ6nqumjdYa3uFHBr60aVZQMzmttr5m1kHJVasblVU+djkqKnYxPX7No6a+0El4k0ut0WoqPdy03jeEjt1m/4uUzjG7jHc2JBHti8ePx5dEeFvJv7qVOd3PPHLqBkmz6xX3T1DV0V+v7r3A2REopZY8Ssix0e775c/Hp154TDdT1erLzthn0zYNUusVHDaG16qlDFU7zvE3FT5WFTO8nn5dOZt00dqWr1BR+0NVSaVtlLcq1dPsa+Kon8FrY/GTLkX1yjUx3Aq1a3Xmi6vTVTV68fdaavvNNb5af3VBBlj1VVXeTK9GqnLHXqXn2h/wBhtI/+stH/AHZC1amt20XWFZp2G7actlDR2+709wklhr0e7DFVFTC9nKv0F19of9htI/8ArLR/3ZANpzxRzwyQzxskikarHseiK1zVTCoqL1RTnzX10t1p2l1djWXStgtsVFHOyarsrKjfe5ebeWPj9B0MaDuGpLxQV9TR1O0u6+PTyuhfuaQR7d5qqi4cjcKmU6pyAsnviw/jtob/APC6f4lp1PqmgtVofVWq/wCi7xVte1raSLTjY3ORVwq7y8uScy96GndVbO9sU8tW+ukkWpctTJT+A6VVhd8pY/vFX08jzaP1RXU2lLPBHtEuVEyOkiYlMzSizthRGp8hJN35aJ03vPqB0fbbZQWuN8dtoaWjY9d5zaeJsaOX1VERMnsLHorjHaaopbhdn3eWdnjNq30iUrnsdzaixp81URUT19S40tzoausq6SlrKeaqpFRtRCyRHPiVUym8ic0ynqB6wAB4r1VcFZq+qzjwKeSXPputVf8AkaO0Xs5q9oGzm2V2p9YalmSvZ4slKtSjouT13eTkXyRF+JvW50UNyttXQ1SOWnqoXwSI1cLuuaqLhfLkpoba/s0sOjtmd1utjmusFVS+CkKLXSKxu9Mxq/Jzjo5QLzfqnUFLt9r3aWt9FX1S2KJJGVc6xNazxl5oqIuVzjkWHZxcddRat10+12K0T1clwatZHLWKxsUm6uEYuPlJ15np1tdtm8urZam66zv9mvtNA2gqG21ZouTFzuqrYlzzX1xyQ1/pC6aJhv2p33PXuqqClkq2uo6imqJ2yVbMLl8qtjVVd8URQNl7Nq91v9n3U1bUNRksa3FytauUR/ysIi/FcFht+mo7hrPS2nqxEalToRIHrj5jnK/n8UcufoNm7Fa3SD9P1Vr0Rdaq501JL4k76pj0ejpVVeauY3Od1eifEtdyRE9pmzoiYRNOv+ukAs2yzQT7xdb/AHTaBbYbhLC6K10iVtN85kDEYsqNdnG/hFz673qfHZ1o3TVbtS2j0VXYbZPSUU1IlNDJTNcyFHMert1Mcs4TOPQzvVuzyTUV6luDdXantaPa1vDUFb4UTcJjKNx1XzNT6F2eSXHaLry3Jq3U9KtulpWrU09buzVO8165ldj5Spjl6ZUDoOx2S12GkfS2W30tBTvesjoqaJI2q5URFcqJ54RE+g1nqer1ZedsM+mbBql1io4bQ2vVUoYqned4m4qfKwqZ3k8/LpzNhaQsLtOWZtA+6XK6qj3P4i4TeLKufLe9E8jVOpavUFH7Q1VJpW2UtyrV0+xr4qifwWtj8ZMuRfXKNTHcCrVrdeaLq9NVNXrx91pq+801vlp/dUEGWPVVVd5Mr0aqcsdepefaH/YbSP8A6y0f92QtWprdtF1hWadhu2nLZQ0dvu9PcJJYa9HuwxVRUwvZyr9BdfaH/YbSP/rLR/3ZANhapu6WDTlyuzqeWpSjgfN4MSfKfupnCGkNikF9TbPqCr1bhLxW2ptVJH/5hr5GK2PthqImPLGDoM1VZP8AxkNR/wAiQ/32gYjsv1Xe7JFqiltej7jeoHX6rkWpp5WNa1yq1N3C+aYRfpPLsY1hfrVp66Q2/RNzukcl1qJXSwzMajHLu5YufNP+ZddB0W0yzJqKKxWmyxUM95qqhvvbxo5X7yp8pqN5KxURML8S1bGJ9orNPXRNN0mmZKVbpULKta+ZHJN8neRN1cbvTHmBlOwie4rsqv09rpme9Vr619PBOuG+NhFaxy8vvsIpgdLZ9Yt927NLhDbab33K+43aogldLVOj8RHPkkcnyW72N1MZzuonnz2fsOoLzZNn10bc7ZJFdOPqp20z0WNJFXCpuqqcmqqYRTFdH2DalPebxeKiltNluN2kRZq6selRLDC35kUMbVVqIn+t15Z5oB9W3O06s28WnT9ua1LTpmmmka2Lk19SiIz6dz5KIvq1TALpVaUl1JZW2a860kse9P7ym8Wd7uTcRtZhP3WcqbTtlgtum9u9nt9mpm09MlhnkVqc1c506qrlXzX/AJIieRYdjF91nQ6Ep4LFo6C6UDaioVlU66xwK5VldlNxWqqYXKAbH2Wuskuh3zaZrblWW+SSVUluD3Pl3k+SqfK545dDANhNZo2HZTamXup08y4Is++2rkhSX/SvxlHc+mMdsGSez457tlbnSs3JFq6tXNzndXxHZTPmWDYNonTF42TWqvuthttXWyLPvzTQNc92JnomVVPJERAMU0BZ/eOzXRtdqOogh0PZ31VfXI7LllnbO5sbXMRFVW/K8s5+Ui9UN/0Ng0vX0cFXR2ezT007EkjlZSxq17VTKKi4MI9nKnhq9ilsp6qJk0Eq1Mckb0y17VleioqeaKhddmemLxo26Xmzb7J9JbyVFre+XMsCuVVfDj9ynXP9uVwGPbBn1VLszv8ALaaRlTWR3KtdTUyuSNsj0RN1mV5NRVREz5Hs+ynar/1d0X/F4f0ifZt57PaiTyludS9P5yJ/yL/ta1c/SmmcW5vjX64v4S207ebnzO5I7Ho3OfjhPMDBrHtP1/fa+6Udq0LRVE9sl8Cqxc2I1knP5KOVcOVMLnCrjzLz9lO1X/q7ov8Ai8P6RlWy/STNGaPpLYrklrHZnrJs5WWd3N658/JE7IhYdVamumj9pdsku9T4mkLyxtGxVYiJRVSdFV2OjvVV9fJoF1sNw1vdbDdvedkobDd2t3aHxKltRE9yovN+5lURFwY1p/UGubftPtWm9XVVlqIK2klqUWgieipu5wiq7HmnobaNU6pmipvaB0xPUSsihjs1U98j3I1rWpvKqqq9EA9Fwte1l9fUuodQabjpFlcsLJKZyuazK7qKu71xg1xs6t+0SXV+uWWa82SCujrmJcHzwOVksmHYVibvJOp0PYbvQ361QXK01Damhn3ljlaiojkRytXr3RTWux77oe1H+VI/7HgZdoSk1hS8b9mtxtldvbnDcDErNz52/vZRM5+Tj4Kad2wUdFUbbaanrbfbK2mSxNckFfcVoIkd47/lI9Orua8vPKr5HQ9XUw0lLNU1UrIaeFjpJJHrhrGomVVV8kREOedqlxslZqybV1n1Doq5JS2laZLdclSpWVzXPk+QxFxvLlGpz81AtvuHT34qaM//ABjIYxtGtdoo9K1E1vsOnKGoR8aJPQ6jfWSty5M4iXrnzXy6mT2W601xtNJWS1Gx+iknjbI6mqKBEkiVU+a5N7kqFguk1NqqovOn567ZrZ4KV0LmXOlpfA8fPylSJ+V6Yw4Drw0FoHTWobzd9d1Fk1hU2OnZqatY6nipI5ke7eRd/Llz0VEx2N1WK/2i/wAMs1kuVJcIonbj300qSI1cZwuDSmzfRdVqhNU3iHVWoLPHU3+sc2C3VKRxvTeT5aphfldUz6IgGJw1UVt2IaLq66XEFLq9ss0u6q4a10yudhOfRFXCGVa72k6d1HM+G17UH2e0z0/gVFKyxyyufnO8qSKxHNyionLpjuY5ambmx7QTFcrsazjTLuar+uS9TIa7Ucmy6l1bpKSnkWWpdJVacWNuUek7t3wk9Fje5Vx58+2Q2joGu05btmlBVWGtfNp2ipn7tU+N6OVsauSR6tVqOzlHKvL4Gt6DWmzykrLy/R+mrpqW4XWR7qlaehfKku8uVYqyJlrOyJjsbW2faeTTWhbPZJWsV9NTNZMic2rIvOT4orlcYzVWzadHVzttN00rBb0kdw8TqWTLI8ruouOWcYzgDU1BpPX0FRcqPStZSaOhuUazwafnuqSVCt++dF8lVZ0XmmFToq+mUaG1PbdmdC+36g0RerJUu+VU3FrONZUu83OmbjPVVwmUTKll06zXurtqNzuttuun5quxU6W5a1IZOFfvKrlaxOquRVcir05J6oZVre6bU9I6Wrr5XXXTE1PSI1XRxU0m87ee1qYzhOrkA2To3WVi1lRzVWnK1auGFyMkVYXx7rlTOPlIn9RkJ4LD8u0Uk74o456iJk03hsRqK9WplT7xVtLLVy0sVTA+qiRFkhbIivYi9FVvVAPQAABz1s10ndr9W60qbbq26WSJmoauN0FI1qteuUXeXPnzRPoOhTnTQGiNO6gl1tdNRVlXS+BqGrh32Vy08aNRUXnzRM815gXfZNpSgus+0Cy6l3r1FDekV8lTlHSva1flruqnPmp4dlWzrSd6vuuuPslPPS0V5ko6VjnPxE1mUVEwvwPnsa1TpfSd511bmXLxIGVctVTORyzeJTxMVVf4iZReScueVVeRnPs80E8GzxtzrWK2svVXNcpc9V33YRfgqNRU+IFm2C0Es+y2/UFsqn2+V9xrIYKiNqOWBcNRrkReuORZ9B6Jt2mdtNdp9jpaqGfSzlq5ZXLvVD5J2I9y8+We3T48zKfZt57PqiTyludS9P5yJ/yPnNfrRHq/XmqaK31U9x03buBle6ZGwVCJ+uqxqYVUcjmYVeYHr2cXNj9T1en9H01NFoqwwcLJUbqudPVquVRj888JneXnlfiimDbOam7UXs3wVdmpaetZBPO+upJWqq1FLvu8VjFReTt3PPnyzjnguutqii/Ut0rcYbHJb9K1Mkdbcm2qoWGWjSVq/KbuoiObvSLnKeiYTqma1E9BobZukWiaKnuLKWlWqpaLiflTxKu8+RF5q7k5Xd+ieQF90tNaKjQ9DLpprGWd1Ii0zWfesx81fPKc0XPPKKaC0vOkuy/ZNariixWGsu0zqyoX5ivZUSOihd2e7+6by0LU2Gs0FS1Wk6eGmtM0LpI4Ymo3w3LneaqfukdlF7oaMt9+07T+y/S2W81kbLlU0tRLR0+6qvdIlVLuK3CcvlN6+mQM41XT3a9zagsNh2a0qQV1QvF3K5ythhmenJJURny3eqK1cpz81Uwuy6M1JbdVxspLrbtZ3XTUUbZbVcVmjbRrIm8xYXuXccuE5KvTlyyiYuFyi0zd9Y2Gm1s51DX1dhgrrhVVF1dSpFLutakaNVUTeXGVTr5lk01pvZ1PtG1hSV99p4rPTtpeAnW8IxJVdHmTEm98vDuXVcdAN2UetbrFpK93jUOlKy1T2yNX8L4zZlnw3PyHNTp05/4GtNjdPfk213qs1aqJd6+zJVviT/yDXys3Y+2GoiY8vibk0LFYafTcFFpavgr7ZTOdG2SKqSpw5V31ar0Vefyui+SoYZa//GTvX/q/H9a0DC9nus63Tly1tS0ulL7eWSairJVnoId9jF3kTdVfXln6UPNZNeV9Fte1Hck0ZqJ89woqdVoGwfr8bWIjd9zf3K9MmebC3NYzXznuRrW6orVVVXCInyeZGoJI7Zty0neY5GOor5bZrZ4rXZaqtXxWLnou9lET1AzDROpKnUlHUT1dhulldFIjEiuEe46RMZ3k7eRkiKWu23q33K43Kho6hJKq3SNiqo91U8Nzmo5OaphcovkXLIGotNf+M9qz+R4f/gm4TTmml/8A7ndV/wAjw/8AwTcaKBp72V/uaT/ylP8A2MNxmnPZX+5pP/KU/wDYw3GANEaj1RS6S9omurq6kuFVHJYGQIyig8Z6KsrVyqZ6fJXn3Q3uak07LHU+0PrCudIxsFvtdNQvkc5ERHPVr8Z/3V/IBkmidpVn1heqm1W+lulNWU8HEPZW03hfI3kT1VeqoYNsbvUOktkeqJavCe4LhWROjcuMubhWt+lzsfFS86RliuHtAayq6WRk0VLbqWmc9i7yI52HYyn8FfyGtKq3w6h233zRVHcqd+nbjcWXWuYx/wAp742K6SFF81V680Tpu56tUDoLQFxud30ZZ7jfoYYbjV07ZpY4Wq1rd7m3kqqqLuqmUz1yee46Q0VSUtTXV+nNPRwRNdNNNJQQ4a1EyrlXdLr75tcVbV29KyBtVRQJPNTovy44scnbvXHLyNK6m1NNtJp6irWC4UmzO2vR9XNHGvj3VyPREYxvXw84z8OeFwiBhWodPW65pY9Ue5qShoL7qalpaKkjp2xNShajmoqtaic5Fy5fgnkbL2a6a03Q3O56L1Dp+zTXe2vdNSVFRQxOfW0bnZZJvK35Tm53XfBO5h+1baVp68waPhtVHc4IrbeaapcySiWNEiYipusTzXphEMj1LqSzbSLnbItK+97ZrGkc+W2V01G6ONFRqudHI79w5GqnPPP1yqKG4LRY7VZGSss9sobeyVUWRtLTsiR6p0Vd1Eye8wvZnrebVLK623m2T2vUVrVrK6mcxfDyvRzHdMOwqomfhlOZly1lKldwfEwcXueJ4HiJ4m7+63euO4H2VMhFwnqhICKXNR7Fa5EVrkwqL5oc/wC0LTmjrDtd0THVWuz0FklgrH1jXxMZC/diduq/yXC4x3wb9qI3SQSMjkdG9zVRHt6tVU6oc73HQcVBtn0NRajulbqZ1cyqkqFuS78aqyJzmo1nPDcoi4yqcgrIdd0NLW33R920fctHsprPTStZRXOp8KNWSxojP1tEyiI1c4XGOR5K6s1FWUVRTOk2TMbNG6NXMqno5uUxlOXXmXLUNs0pUbabZpyewWKXjKGSrqXywfrrnplGNauUTOGKuMLyQwW30mlbLtF1vQ12g7jfaWGphbTR223JUNpW7iqqL8pN3KqnxwoG6dklPFZNntutPvCguNTbonNnW3zpM3Kuc5MLyXmnqiGK3rbDR1Fmr4U0jrSNZKeRm/JbERrctVMqu/yQzDZlS2NtmlrbBpmq062pk3ZaarpeHlcrM4VW5XlzXCnw21X2PT+zO+zvX9eqYHUcDUXm6SVFYmO6Iqu+DVA1jsd2nU2n9m9mtcumdVVr4Gy5norekkL8yvd8l2+meuOnVFPHbNpNNDtru98XTmp3Rz2qOnSkbQItSxUc1d5zN7k3l1z6G7tnFlk09oOxWqdEbPTUjGyonlIqZcn85VMMsv8A4ymoP5Ci/vsAyvRGtodWvrGU9kvttWma1yrc6RIEk3s8m/KXOMc/ihizNpGqKPUNjt2pNC+6YbrVNpY6hbrHNhy9V3Wt8k+BtU1ftg/bbs2/ltP7oHm13bdcWilvl9ZtEbRWqnSWpjpvdEL1jYmVbGjl5uXo1FXqpYbNFtDm0har7ftpdPZW3BI1ZFPaoHbqyL8hu9hOaphcY5Z7EbeNZ2yo1Na9FXGSpitiSx1l2dHC97pY0+UyFiInPeXGV6Jy58lQ9mstawa20pX2Gz6I1TXw1UXhsldRJBFG5ObHI5V5YVEXp5AbB0TZtT2njl1Pqdt+WVGcP/0FlN4Kpvb3zPnZy3r0x3NHXGHaJp+WsoZ4LJFfNa1roXyMkfJUozdx8jHyWRxtXqqqqZ/JuLYzqKp1BoiBLrvNvNtkfb69j/nJNGuMr3VN1V7qpg0endp902gXK/vp7NbJHNWjo56uTiOFp0VecTG5RXO6qrsdVTCdAMC2hWCtsV205UzUNQun7DdaW02+gZHvyVuE8WaVG9FdI5ERE8848i7bTdVUVfU6SWHZ/f7YlPe6eZzam0tiWqRFX9ZZ+7e7yb5mVbZqK72rSmhKeWvbeb3FqKnVKidiQtnlXxHNRWt5NblUTl5IWLapX7QJarRq3uzWSney+0zqNIKlzkknRV3Wvz0avmoG39B3ujulDcKuLS9y02yBUSRLhQpSulREVd5ET5yJz/KYrs2lrNca3uOup+IgskMS2+zwOVW+KxF+XM5vdU5Z+H3qKZxoms1RVU1SusKC3UUyPRIW0UqyI5uOarnouTXm3hr9KS0GsNPXeShv7VbQxW/CyRV7Fdnw/D8lTKrlP7VRQPBV19Ts9Zqy4aYuUV20ynis4RZXSzUFyc9GpGnVd1XPyqL6L5/OtmgJobLsL1dT3ejip73aFrIanxURXrM9uWOVe6va1F88GX+zpPa5tn3BwyOfd4amR92hqG7sralz1VVe1fgiIv8Aq+qKia72yz2r9V+G0pdI4LVd3UbdQMRF3Gujflm+5OSKrcJ25KoG8NklsiteznTkLYWMm93wulVGoiq5zUcufpVTFdS27TOntrdhuFBdKi1X26zLHVUFFF4ja5iovypWJncTe6vx6r5K5NkvclNE6RvyY2NzhqcsInoaL01rvZrY9TXbUCPvVXd7hIrnVNVSySOiYv8A5NmU5N/rxhM4QC1V2n9U2fXVZQaU1q66apujkfcpuAjxRU6dFklcrlYnTDGomcJyT5JdZZNMU+y272Oiu12stNb7o2jvFWyBJZ6lXOw+R6tVcMfzw7PRuML81dba0l2axutC6biu7vFucT7ilUk2JafKq9MKuFX4czPrBqrZZp+9XCstLLpBSV1I2knt3AvdTvRFVd5zVRVVcLjrjCr6gbp0lbbRadN0FJptsKWpsaOgdE7ebIi89/e++VeufMuimtdi9z0rwFxtWkai5vpoZlqfDrWPa2FJFXDI95PmpheXXnlepsOSdOjGqoRW9UamVJj5oedjXyORz+p7GMwgGJbV0/8AB/dfhF9aw5ZunVTqjaymNn12+EX1rDle6dVCsVuHVTH6vqpkFw6qY/V9VAtc5439T2Tnjf1AqjPdT+R4Yz3U/kBd6Lqhkdt6oY5RdUMjtvVAMstf3p2khxba/vTtIAAAJAAEL0AXoAAAAAADXO2rR+pNa2iitmn7nSUdCr3OroqhXtSdE3VYmWtVcIqLlOXl6FspdH7T0pYaVmubXbaWJiRsiorUxWsaiYRE3kRURENtADVabL9T1P7J7TtQS568GxKb+xylWyvRV30Dqu9WunklrNJVMTKqCoqJWrKyo+a9uERM5RMquE5I3zybURSrkqAa+otOXRm1XUOqKuKOWFttiorWxJcbyc3yI7l8ld9OuF5O+JZNOWrU962i3DVV/sSWSJLKtrp6Vatk75HLJv7yq3kidU/IbZKQNAaX0Lri46FsWjbm2KxaejjkfcZWSo+pn3p5HeCiJlGphWqq80wqc15tMt0no7UuhNT09Fp6sjuGiKl7lfS1kn67QclXMa/fIq+XfmnVxtIAYDrKp2hVV4ktmkrfaqGgVjV98Vc/iKmeqNiRMo5O6OReXry+2gtndJpmtqLvcK2ovWpKpMT3Kq+dj9yxv3je30dORnIA1zetQbR466spbTomglhbI9tPWy3Rm45uV3Xuj5OTKYXGS5bJtJ1WkNKvpbpURVF0rKqWurJIs7iyyKmUTPXCIhmgA13tosl9vVFpj7GKeKavor3T1m9Mv63E1jX/AC380VWoqplE546GP3Cw7RLjrbR1ZqNliqqCgrXSvfa0kasSKzGX+IvNPhzNyADTdvsO0S3a21jWacZYqWgr61srH3RJHLKiMxlnhryT48y9bI7HqO13rWFXquCmZVV9ZHKyWlVfBlRGKiqxFXeROnXmbKAAxfaHoq3a8ssFsu1RW08MNS2qa+ke1j99rXNTm5rkx8pfL0MoAGqv1FqL8ctb/wDFG/5ZeNKbM6XTd8gucWpNUV74kciQV9eksLt5qpzbuJnGcpz6megDFNd6BsOt6eFt6p3pUwc4Kunf4c0XPPyXf8lRUMQbsUp6iNKa76w1ZcbYip/0Kau/W3J6OTHNPhg20APBY7Rb7DaoLbZ6WOkooE3Y4o05J5r3VVXmqrzUwnVGyWyXm+S3u31t1sN3mystTa6lYllX1cmF5/DGeq5NigDVtHsXtMtfBV6nvV+1K+Bd6OK51aviRfXd6/RnHqhtFrUa1GtREaiYRE8iQB8qtZkpZlpGxuqEY7wmyKqNV2OSKqc0TJqbSuyp90tt6uG0Z7azUV7aiSOjVP8AoLEXLGxO54VFRFynLkic0yq7eAGitR6G2kvr7A9LlaL/AE1jrEq6WSpV0FTJjGGyLhWryROecmUrf9qDU56KtL19W3Vqf2obMAFi0dWX+ttkkuqbXTWyt8VWsggnSZFZhMOVyeec8uxfQAAAAGvdY2S4121nQV0paV8lvt6VvFTIqYi34sNzzzzXlyNhAAa70zYLnR7ZtZXqppXMtldTUrKedXNVJHMjajkxnKYVF6obEAFMm94bvD3d/C7u90z5ZNXTXHa5cXLDSWPTlmaqqi1FTVOnVE9Woz/mhtMAawsmyiKpuSXjaDc5NU3XdVrWVDEbSwIvVGRdP+XnhFPDX7LLnpqpkuOyy9y2mRy78lqqnLLSTL6Ii5Vq9+fZUNugCzaTqL3VaepZtTUVNQ3dyL40FPJvsauVxz7pzxlcepgfj7Z/3poj8tR+kbVAGidQaa2sX3UFgvFZDpFlTZZJJadsUk6Mcr0ai76LlV+anRUMh8fbP+9NEflqP0jaoAsmj3ahfZ0XV7Lay6eI7KW/f8Lc5Y+cqrnqXZKeFKlahIY0qFbuLLupvK3rjPXHY+oA8N8nrqa0Vc9ppGVtfHGroaZ0iRpK5Ojd5eSZ9VNQ6hotf69uOnKK86Tp7FbKG5w3Cep94xTuVI88kRq56OXyXnjobsAHwuEElTQVMEM76aWWNzGTRoiujVUVEcmeWU6mA/YDqL/rFv8A/RQ/omxQBpHTujL/AG/SW1Ogq4qmrq7k+oSjmlViPrMxORHclwiqqp6Fx0fs81NBpSzxT62vlulZSRNdRsjhVtOqNTMaLheSdOvkbdAFi0lZa6yUs8VyvtbenyP3my1TWNViYxupuonLzMX2iaLq5a5mq9FK2l1ZSImWou7HcI06wyp0XKJyVe3PoqbFAHwoJJ5aGnkq4Ep6l8bXSwo9HpG5U5t3k64Xlk+4AA0vq+1bQtoMUVgutjt1jsS1LJKqqStbO+WNjs4a1OaZwi4VOqJzQ3QAPmsMTlVVjYqr5q1DX2zfTtda9Xa7qrjQ+DTV9wbNSPduqkjEavNMdOvmbFAFLI2MzuNa3PXCYNf12n7pLt1t2oGUqraIrK6kfUb7eUqyPXd3c73RU54wbCAGsauo2wJVzJSUmilpt93hLItRvbueWcO64MYsOm9rFk1Jf73SQaQfVXp8T6hksk6sYsaORu4iKip85c5VTeoAxfQ8msZGVn2bw2SJ6KzhvdayYVOe9v76r/q4x3MjSnhSpWoSGNKhW7iy7qbyt64z1x2PqAPDfJ66mtFXPaaRlbXxxq6GmdIkaSuTo3eXkmfVTUOoaLX+vbjpyivOk6exWyhucNwnqfeMU7lSPPJEauejl8l546G7AANe2qxXKHbhfL3LSuba57VFTx1G8mHSI5qq3Gc+S+RsIADXuxOw3PT2nLrTXildSzzXaoqI2uc129G7d3XclXrhTYQA89xbVOt9U23viZWrE5IHSoqsbJhd1XY8s4yaybs71Zfv27a7rZKZ3zqK0RpSsXsr0TLk7KhtUAa+h0lV0W1SyXGhp8WKgsS21sjpUVzXI/5LcKu8vyUTmYps/wBGbSrHpqOip75a7VE2aV7aWakbUObvPVc76Owuc57ZN2ADAdjunbrpzQPuy+Rsir1qKh7ka9rkVHvVUX5KqnPPQw/ROwW00umKWHU61D7u1X+K6jrZEi5uXdwnL73GeXXJu4AYpsv0m/ROi6KxSVbax1O6RyzNj3EdvPV3TK+uOpatbXPaFFX1VJpTT9tqKR7ESCvmrEarFVvNVjVUzhc4+g2AAMR2U6Vl0ZoW3WaqmZPVxb8k8jM7qve5XLjPkmcZ88ZMDv8As917XbSanU1HfbM1se9FbkqYnSrSxL+5Zu7qO6oq5XOTdROANTroraTWL/0/aUkDf3NLa40/ry1Tx3vYtW321z01713qG4q5qujjlkRsHiInyVcznlEX6e5uQAYxs0ZfotE2uHVkLYrxDH4UuJEerkauGucqct5Woir3/IY3rnZdDrXXtDd71XOWzUtIkHAxZa6Z++5yo5yfeL8nl15eRsoAfGipaehpIaWigjp6aFqMjiiajWsanREROiGAbM7Dc7TrTX9ZcaR0FNca9k1LIrkVJWIj8qmF5dU6mxQB86iGKpgkgqI2SwytVj43tRzXtVMKiovJUVPI0xqrZxcdWXqa0x6e07pzTMUyK+upoY31dWxMKiMw1PDz0XOFT1cnJd1gDHIdDaThiZGzTNl3WNRqb1DEq4T1VW5UwnSmzyjg2ma0qrlpq3e5ahKT3f4lNE6PlHiTcbj5PyuvJMm2QBaEs9PabRXRaZt9BQ1Mkb3RtihbEx0u6qNV26nrjnheRhGlLdqDZ9svstDa9PrfLuqufWQJWshVj37z3OVzso7C4byXvzNnADRH2AajotmWi7ZwbJ7rSaiiuVXFHK3dhZvSqq72eaIjm5xnryye6o2ZX7XlVPedoFxdbq+NqttVJbZMtt67yKkiu++eqtTOF+lOSN3SAMM0AzWdFZ6+j1dwNdV0jtyiq4pN1axu7lFk5fJXOEzjPXkuMuxWpsu0bXaOg1BV02krE/lJSW+TxquVv7l0nREVPNPpapt0AYxS2Vmi9GPodE2iGeWnbvQ0j5vD8dyqm8rpF++VM819EToa/wBVW7aFtFoI7BdrBQadss00b6yZa9tTI9jHI7dajO6J19E5obnAEMa1jGsYiI1qYRE8kNOaS05Sv273q/6e47gWQzQ3GabKRSVbnoixRZRFVG7uVXKoi4RDcgAAADxXqidcrRWUUdVUUb6iJ0bainfuyRKqYRzV8lTqa4tOwjRlJKs1xir7xUOesr5K+qc7eevNXKjd1FVe+TagA1Zr3QlVdm2vS+m7fQ2fSlS7xLtUUrWRyOaxUVsSNRE+cvPPPpz6YdctXVet7errTonTNBLQNp2xU9dJWtjSD5OMeEvNd3y5+hsEAYfsy0nPo7Z/RWN1TG+ujY98k7Wq5vivcrlVEXGURVRPLOPLJarDs5datmN704+tbU3S7MqX1Vc5FTxZpUVEeqc1wibv5F9TYoA0vY02nWbS9Jp6XR9juVHS06Um++ubuzRtTdTLV9U65T6C46T09q27bSKfVGsKC32unt9C6ko6Slm8Rd5y83Kqcuiqnl5cuqm1wBiOkNFU+kmX+O2VMz6O5VDqqKjVESOmVzcK1nxX+pETHJVW07FtKyWXZzYaTUFrhiu9H4/+lYx74t6d70w5M45Ki8lNiACxXbR+m7xWurLtYbXW1bkRHTVFKyR6oiYRMqmTX+mdmNsi2kaxqLnpa2LYZm0nu1JKeJ0aKkeJdxn3vyuvJMm3QB4LLZrZY6V1NZrfSUFM56yOipomxtVyoiK7CJ1wic+xhlBY7lHtzul8fSubapbMymZUZTCyJI1VbjOeiL5GwgBqTQWzK/2ytr5tR6ihmoa+rqKyqtVHBiCofM1WuR7nfKVuF+bjCY6nnh2Z3p2n7tplaxsdBbquOu0zcXyb0tM9MuSNydd1q5bnuqonRE3IAMF2VaZu1jt92rNTyU8l9u1c+rqVp+cbUwjWtTsiJn6TNVTB9SFRFAxeh0fb6PXVw1ZFNVLca2mbSyRuc3wka3dwqJu5z8hPNfMyZFIVuABj2gNH2/Q9jfarTNVTU7p31CuqXNc/edjKZa1qY5ehk2T55Ja4CqRXJG5Y0a5+F3UcuEVfLK+RrLZ/s8VdLXyLXlLBV3HUFa+ruETXKrEw/MbEci5wmN5MLy3seRs1FKgNSXjQt9oZk07s6ht2mdOVMfiVtzjc6Spc7misairnOPPPnyVOi/W77HrdTaXtlPo+VLZfrTLxVHcXJl8sqom94q45o7CZ5csJhMcl2sRkDXOy7TGoKO+ag1LrVaNL5dFihSOkVVjjhjbhMZ6by88dk9cGxKeCKmgjhpomRQxpusjjajWtT0RE6IVJ1KgNdbZLBdL99iHumkdU8FfaerqMOa3w4m53nc1TOM+XM2KABCNajlcjURy9VxzU03V6cpLj7RFHerFxy1NCxy3mZ2Up2KsCMijYqpze5HIqoi4REz1NygClW+hQfUhUyB8jUVvs+s9R7UrFqHUdkpLNRWWGoYxGVjZ1ndIxzeW70Tmi8/Q2+qKhAGmrfstr77Fe71qup4HVtdWJUUdVSP31t7YstiRi+aKnVOWUxnCoebTum9p+k75fLjTs05epbtJHJPI+V8SqrGq1FRuERFVF5pzN3EdOgGK6LrNYVlRV/Zda7Zb4Gtb4HCTrK57lVc59ERMflMai0jfdV69Zetbspqe0WiZ3uu2QyeI2R3lPIvn0RUTGeXRMLvbQQkAa8tdgucO3K832Wlc20z2mOmjqN5uHSI9iq3Gc9EXyNhgAafj2Y1usa67XnaDUTw3R0zo7VFR1Lt23Rtd8iRiphFcuEdzT4oirhNwADVlOm1iyMSmSHTeoIo03WVb5H088jU6LInzc/A+i3zasnTSNiX/7xNngDB9DfZW+S7Ov1hstlWVPEjfRPR7pplzl0mOuOXNea5LE3Qmur1y1TtAqIIHcnU1lp20/L0SXCO/Kim1QgGnNoGzespdLaatWhY5nz0l9iuElRUzJI5i7rszP3lTewu6qonX0PlqLZ5tE1DJbJLnq60SOttWyup92g3d2VnzVXHVOxulQBhOlLdrihnq36jvttucawKlPFFS+CjZcphXKnPHVPpLHojQd3q9Sx6u2jVkdbfI0VKOih/8Aq9Ci/uU83d/LrlVwqbSKVTAGutd6XvEV4W5bPKG3Ud6u6JTXG6zO+VTxN5o9rOiuXGFXCryby809Vj2VWK36NuFhrkfcZLnl9wrpv9NUSrz3888Ki829cLz5qqqudo7B9WuyBr3ZlZtWadfW2TUFVT3Ky0jWJba/OJnM5p4b2/6qInNfVMKvlnLoz1KhQqAay2x2C53v7EvdVI+p4O+U9TPuqieHE3OXLleiGwVjU9aoMIB5Fhz1CUyZ6HrwUve1vVeYHybCjeiEqiN6qFe9/wA1MIS2LzdzUIw3ayqrs+u2E5Yi5/8AtWHK106qdX7XG42d3f4RfWsOULp1UKxW4dVMfq+qmQXDqpj9X1UC1znjf1PZOeN/UCqM91P5HhjPdT+QF3ouqGR23qhjlF1QyO29UAyy1/enaRxba/vTtLyAAACQABC9AF6AAAAAJQABgACSAABBJAAkgkAAAAAAAAAAAAAAAjBIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAJwBBOAMgBkgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAnBBIAAAAAAKVb6FRIHyBWqZKVTAEopUinzCKB9SCEUnIEoVFCKVgAAAAAAAAMFCt9CsAfIH0VuShUVFApVMhF8lJHUCQU80+BKLkCQAAAAAAACckACcggAHNz06kJlCoKmQJa8r5KfJUVCUdgCpUwfN8jW917H13sofNsTW/ED5K57+nyUKmRInXqfVEROhPIClGohIUhQMP2vfc7u/wi+tYcn3Tqp1ftd+53d/hF9aw5QunVQMVuHVTH6vqpkFw6qY/V9VAtc5439T2Tnjf1AqjPdT+R4Yz3U/kBd6Lqhkdt6oY5RdUMjtvVAMstf3p2kcW2v707SAAACQABC9AF6AAAAJIAAkAAAgAAgkgASQSAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAJwBBIIAkZIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAnAwAGAAAAAAAAAAAGE8wAKVb6FJ9UKXNRQKEUlHFKpgZArReZ9TzovND0AAAAAAAAAAAACpkAChzcFJ9SlW56AUlOPTkVKhABF8l6kkKRzQCoDIAAAAAAAAAEoQEAkhU9CQBSnIryQAJIAAEEkAYftd+53d/hF9aw5QunVTq/a79zu7/CL61hyhdOqgYrcOqmP1fVTILh1Ux+r6qBa5zxv6nsnPG/qBVGe6n8jwxnup/IC70XVDI7b1Qxyi6oZHbeqAZZa/vTtI4ttf3p2kAAAEgACF6AL0AAAeYBABgASAAAAAgkgASQMgSRgEgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACcAQSCAJGSAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAkBkAAAAAAAAkEgRgYJAEYGCQBGASAKVTPU+bmr5H2IwB5/Q9R8XM55T1PuBAJCARgEgCASAIBOCAAAAAACFTJSqYKwB8gVq30KMAMegRQnUKmQJBTnHUqAAAAAAAAAEkACSCSAAAAAADD9rv3O7v8IvrWHKF06qdX7Xfud3f4RfWsOULp1UDFbh1Ux+r6qZBcOqmP1fVQLXOeN/U9k5439QKoz3U/keGM91P5AXei6oZHbeqGOUXVDI7b1QDLLX96dpHFtr+9O0vIAR5kkAShKFJKAAPIAAAAACASAAAAAEEkAB5gkAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEgRgkZIAnJAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAJQAAAAAAAAACUJAgkAAAACDBKAAAAAJQAEQqwAgBUIKiAKQVEYAgEqQAGAAGCCSAAAAAAAQqZJAHzVuAfQpVvoBQpHToSACKSUqgRfUCoEEgAMkZAkAAAAAAAAAAYftd+53d/hF9aw5QunVTq/a79zu7/CL61hyhdOqgYrcOqmP1fVTILh1Ux+r6qBa5zxv6nsnPG/qBVGe6n8jwxnup/IC70XVDI7b1Qxyi6oZHbeqAZZa/vTtI4ttf3p2kAH0AfQA+gIPoCAPIDyAAAAAAAJAAAAAQSQAJIJAEEgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAnAAYAyQBOSAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAkYAAAAAABJBIEgAAAACE4ITqSAAAAAkAShAAkEEoBIAAAAAqEYJGAKQSpAAAABgACASpAAAAAABCtyUKnM+gVMgfIFStwUqBHQlFBAFQKUX1JQIkABQAAAAAAAGH7Xfud3f4RfWsOULp1U6v2u/c7u/wi+tYcoXTqoGK3Dqpj9X1UyC4dVMfq+qgWuc8b+p7Jzxv6gVRnup/I8MZ7qfyAu9F1QyO29UMcouqGR23qgGWWv707SOLbX96dpeQAAASQhJADyA8gAAAAAASAAAAAEEkACSCQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAkAMAgATkgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACQAgAAAAAAJQglAJAAAlCCQAAAAAAAABJBKAAABOSSkkCQAAGQAAVASBTgYJwAKQTggAMAAMEEqQAAAAAAClUyVAD5qhB9FTJSqYAoVMjp3QqIUCUXIKVT0JRfygSAAAAAAADD9rv3O7v8IvrWHKF06qdX7Xfud3f4RfWsOULp1UDFbh1Ux+r6qZBcOqmP1fVQLXOeN/U9k5439QKoz3U/keGM91P5AXei6oZHbeqGOUXVDI7b1QDLLX96dpeRxba/vTtLyABAACdQSAI8gAAAAAAASRkkAAAAIJToQAJIJAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAATgCCUAyABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABOSABIGBkAAAAAAkIEAEgAASRkATkjIADJJBIAAAAABICAASQAJJKSUAkAAAAACgACFJCoBSCcEABgACASQoAAAAAAAAFKtKVPoQqZA+YVMkqmABTnHUqBGMLyAkEIuSQAAAw/a79zu7/CL61hyhdOqnV+137nd3+EX1rDlC6dVAxW4dVMfq+qmQXDqpj9X1UC1znjf1PZOeN/UCqM91P5HhjPdT+QF3ouqGR23qhjlF1QyO29UAyy1/enaXkcW2v707S8gAAyABJAADyAAAAAABIObyTU80aW7ht+VdXby6PBzgBmjS3cJlXV28uj06EHOJIzRpbuDKurt5dGknOIGaNLdwZV1dvLo4HOSAZo0t3BlbV28ujQc5gZo0t3CZW1dvLowHOgGaNLdwZW1dvLosHOoGaNLdwZW1dvLooHOwLmfS3cGVtXby6JBzsSMz6W7gyvq7eXRAOdwMz6W7gyvq7eXRAOeAMz6W7hMr6u3l0ODngkZn0t3BlfV28uhgc8kjM+lu4Mr6u3l0KDnokZn0t3CZY1dvLoQHPYGZ9LdwZY1dvLoQHPYGZ9LdwZY1dvLoQHPgGZ9LdwZY1dvLoMHPoGZ9LdwmWdXby6CBz8BmfS3cGWdXby6BBz+BmfS3cGWdXby6ABz+SMz6W7gyzq7eW/waAAzPpbuDLOrt5b/BoAkZn0t3CZZ1dvLfwNAklzNpbuDLWrt5b9JwaBJGZtLdwZa1dvLfoU0EBmbS3cGWtXby34DQgGZtLdwZa1dvLfYNCAZm0t3BlrV28t9g0KBmbS3cJlvV+OW+gaFJGZtLdwZb1fjlvkGhwMzaW7hMt6vxy3wDRAGZtLdwZb1fjlvcGiATM+lu4Muavxy3uDRBIzPpbuDLmr8ct7A0SSXM2lu4Muavxy3qDRRIzNpbuEy5qfHLegNFgZm0t3BlzU+OW9AaLAzNpbuDLmp8ct6A1vs3/Zyf+LO/vNNkGd7v7Z/zOj9W2nywvbuyf8Tq/SrX4AAd10wAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAJwQSAAAAAkAEJAAEZIAnIIAAlCCQJBBIAAAMkkDIEgAATkgASCEJAEkACcggkCQQSAAAAZAAnBSqE5HUCMEFWCFAgKABAAAAAAAAAAAFKoVAD5grVMlC8gIVAi+pIAAjp8CQMP2u/c7u/wAIvrWHKF06qdX7Xfud3f4RfWsOULp1UDFbh1Ux+r6qZBcOqmP1fVQLXOeN/U9k5439QKoz3U/keGM91P5AXei6oZHbeqGOUXVDI7b1QDLLX96donF1r+9O0cACRgYAkAAQvQBegAAAAAAOcAAeTPXkgAIEkACQAVAlCCUCJAAEgAIkAFEggkASQSVAAACSCQBJBIQJIJAEkAqJAAAAEEgAIkABEgACQQSAJIARIAKgSQSAJIARJJAAkAASACoAAgkAFAkgBEkkEkRIIJAAAASQSECSCQBJACJAAAAAZZs2/Zyf+LO/vNNkGt9m37OT/wAWd/eabIN77g+zj3lpXfn3c+0AAM0w4AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAJwMAQTkYAAkYGQGAQqkATkEACQAAAAAAASMkEgEJIJAAAAhJBKAAAAAAE5BAyBIAAkZIAFQIQkAAQqonUCRnB83SehQqqvVQK3SInQiNyuVUXqUbqr0QlrFauQPqCU5oAIAAAgkAQCVIAAAAAAAVAAKVaUn0IVuQKCMEryAGHbXF/wDB5d07RfWsOUbp1U6v2u/c8u3wi+tYcoXTqoGK3Dqpj9X1UyC4dVMfq+qgWuc8b+p7Jzxv6gVRnup/I8MZ7qfyAu9F1QyO29UMcouqGR23qgGWWv707SOLbX96dpIA8x5jHMASAAIAXoAAAAAADnAAHkz15IIJCAAAkEH1ZBLJFJKyKR0ceN96NVUbnplfIsRM+STMR5vmSQAJJIJKgSQSQCSuCCaff8CKSTcar37jVXdanVVx0TuUFpNKpWPIJIAEgIiquETKn1lp5oUYs0MkaP5tV7VTPwLET5pMxE0fMAACSABIACJAAEgACQAVAAEAkgkIEkEgCSCQgSQSAAARIIJKgACCQAUSCCQiQQSAJIJKgACASQSAABUSSQAJJIBESAAAACJAAEgACQAEAABlmzb9nJ/4s7+802Qa32bfs5P/ABZ395psg3vuD7OPeWld+fdz7QAAzTDgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAATgCATglAIJBGQJGSlVAEqpAAAAAAgAEqCCQAAAAAASQAJJIIAqBGSQAAAZJIAEgIAAAAnIIAEgEOcjeoEhXInU+SyqvQoA+rpfRD5qqr1Ja1VPo1iIB80aq9D6IxPPmVIhIBECISEAIgJAFKgkAUglRgCAABAJAEAAAAAAAAKhQqYKwBhe137nd2+EX1rDlC6dVOsdr6f+Du7/APsvrWHJ106qBitw6qY/V9VMguHVTH6vqoFrnPG/qeyc8b+oFUZ7qfyPDGe6n8gLvRdUMjtvVDHKLqhkdt6oBllr+9O0ccji61/enaQAAASAAI8gF6AAAAAAA5wBBJ5M9eCSABIACL7aKKx1FKxa+4VUNU5VRYoqffTrywpfrTUabtlLcqRbnWSRVkXhPa6mVN1U6O+KczE7FWrbrzRVecJDK1zv4Oef9WS5a9oUoNVVzGJ+tyu8ZmOmHc1/ryZXodb6XR+tgwYaxNJ8/WJ8fP18fRiuv0fq9b6OPHipMVjy9Jjw8vTw9XpS16aWJ0iXWvWNq4V6Ui4RfTJ5rBFSs1rb46KV09LxDEa+Rm6rk5dUPZXf/R2z6gp+ktxqHVDk89xvJP8Akpa9HftptX8YZ/afWK3D1+lgjDETNszSvr6eMz6UTDdi6PVxzimY/qiK09PCvhEetWeO+yj30rUnoeC4jCNzBnw97p0znH0ngpZnUd81lUU6MbLCxzo1VqKiLvei8jwVMenI9QzTOqrm6pjqXSOYyFqpvI7KonPpyKqGrjr5dY1cG94U1O57d5MLhXeZk5639URdWYmf1Xfpxf4YzD0aYZm2kTGH9Nv6sP8Akfdqu8aIu8le6N74poUYrY2twirz6Ihb9MZqIUp4dOxXGRHKqzPc5qIndeiC1ftDvn+3g/tPXomtqrpcKe01sr57fHDJu06u3WuwiqiLjGefqdLBjnq9bo3zMziw08omv9U+da/5pLuY8MdLpdayIiMOKvrFP6Y8qU/xWGQ2qqs1PJcYKe30jqmKgmkqVp3uWNWpj9bRy9c+aoW6dKCen4uy6foq+lxlzWvf4sS+jmZz9KZQ+sVHXQ7/AIWmLbHvtVjt2uRN5q9UX5fQr4OWltFzrHWimtlRTwo6Galqd52c8+jlMlN+LDbiwxERX9ExT19cNPevn+zHRZhxXYcUzM0j++Jr6emKvtTy/dgdS5lRc3ObC2jjc9P1tqK5I08+XVTPKSohqqmKnpLtbFmkcjWNbZ1TK/Shh9PdLxW3yKrp5pZbmqbjHMaiu6Y6Yx0Nl0s1xoLDM64XCqqq98ngu4SFsy0zsZxhMc8dVX1T6el3XhjHOPFFaV86U/1ijxn8Umns7nemOcEYMM0rSlK1/wB4Z8I/PhX3W28XWKKr8Cj1HDQ+C1I5Gx29Xbz0+c7eRvr6cjENTPSeOKRb9LdHo5U3HwyMRiL5pvcvJOhlvH1/4R1F/wAMaWjWlbc6akpYX3Krnpq2JXOjqIWxuTC9FREObt0/U6WPFirT+f48PqTHw4Owx9Pq4MOGlf48fz4/TiflhJJBJrTZQAASCCQBJBIQJIJAEkElQAAAAEEgAIkABEgACQAAJIJKgACIEkElAkgkIEkEgAABIAKgACCQAUCSCQiQQSREgAAAXa2+4uFT3l7z4jK54fw9zHl15nJ0+n9SaViPdx9THZFaTPs+VkoYbjVuppqlKeR7V8FXJ8lz/Jqr5ZPLWUs1FUyU9TGsc0a4c1S9f/0r/wD5z/8AKL1rD3d7go/H4v3jup4PEbvjbmfv8eXp5/1nfjseHF0MWK6K4fGsT5/tP7/j8ujPa8WHrYcNs0xeFJjy/f2/P4YKSQSYxkQkgkIAADLNm37OT/xZ395psg1vs2/Zyf8Aizv7zTZBvfcH2ce8tK78+7n2gABmmHAAAAAAAAAAAAAAAAAAAAAAAAACcAQCQAwAMgBkgAAAAAAAAAAAAAAAkglOoAAAAAAAAAAASSUkgCSCQAAAEkACQQhS6RE7gVlKvRPifJz1X4FPUD6OkVenIo6lTY1XryPo1qNA+bWKp9GsRCoqRAIRCcEgAAAAAAnIIAEggZAKgJIAggnAwBAAAYIJAEAnAAgAAAABh21/7nV3+EX1rDk26dVOstr/ANzq7/CL61hybdOqgYrcOqmP1fVTILh1Ux+r6qBa5zxv6nsnPG/qBVGe6n8jwxnup/IC70XVDI7b1Qxyi6oZHbeqAZZa/vTtI4ttf3p2kAyBgASCMkoBC9AF6AAAAAAA5vJIB5M9eSAAJAARkFFpiarpIp23K0xpI1HIyWqRrk7KmOSl/vtokvL7c6a42aJ9PTsp5JErUcr8ffYx3UwAHf6faujgwTg+n4TSvj50/h0Op2Xq48cY/qeMVp4eVf5bB1bZlutyjWiudoZQ08LIIGPq0RUaieaY9c/1FnsFsfQa1tlO+ppJlSRsqyQyo5iImVXn68uhi4Prqds6ePqx1rKTWJ8/jyfPT7H1MHSno31ikx5fPmz1kFvseopbrV3uFZPHfI2no2+K5zVVfkuXomUXmTDT0cH2TutkjH0c9Ak8bWrzjRy53F9FT0+BgJ9IppYmvbHI9jXpuvRrlRHJ6L6nJh7wwx4RgiI8Z85rWYmJ8+PJ8T3finxnHMz4R6UpExMeXPmyGrutqg07NbbTDW79S9j5n1Ktwm75N3e569F2u8U88N2pLWtZTPY9iJ47Y95Fy1eq5Tz8jECTgwdr/wCzD1Mcf2+VKRT19Yn/AE5sfZP+rF08E/3edazXwp6TH7erYv2PO/Ep3/FP/wBT6OtNwhttdS27SjqV1XGkbpPeCSYTOeimtiTsR3h0o8sFPayP/N1Z7v6s+eOvvfP/AKLk73lp65yR7z6StY3dduuRVRHJ6pnyVDMNJNll0XL4cM1Q9bi5VSOsSmX/AEbeauXr8P8AA16fXxpfA8DxH+Dvb/h7y7u9jGcdM48zg7N2uOhjmaTOHxpFfz/FPh2O09knr4IisXeFZp+P5r8tjcNWfg2u/wCPNLRtEa5sNlSSN8bkgdlr5vFVPleb/Mww+kk0srI2SSPe2NMMRzlVGp6J6HP1u8cPU6WLp20rT8fmJ9MMODpd34un1cPUujwr+fxT1xSoJIBi2TSACgSQSAJIARJJAAkABEgAoAAgkEEhEggkCQQSECSCQAACJABUCSAQSACokAASAAJBBJUAAQSCCQAAKiSSCQgSQSQAABcbJXQ26rdUy0yVEjGr4KOX5LX+TlTzweWsqpq2pkqKqR0k0i5c5T4A5J6uKcEdOvhDijp4Yxzjp4pABxuRIACJAAGWbNv2cn/izv7zTZBrfZt+zk/8Wd/eabIN77g+zj3lpXfn3c+0AAM0w4AAAAAAAAAAAAAAAAAAABIEAnAAYAGQAyQAAAAAAAAAAAAAAAAABKEAAShBOAAAAAAAAAAAAEkEoAJIAEghVwfN0qJ05gfQodIidOZ8nOV3UhEVegFTnud1ITn0K0j9T6NaieQFDY18z6NaiEoSAJwMgASQABOSABJJCDIEgjJIAAAAABIIAABABGCCrBGAIBOCAAwABAJIAAADDtr/ANzq7/CL61hybdOqnWW1/wC51d/hF9aw5NunVQMVuHVTH6vqpkFw6qY/V9VAtc5439T2Tnjf1AqjPdT+R4Yz3U/kBd6Lqhkdt6oY5RdUMjtvVAMstf3p2kcW2v707SAAAB5koR5kgQvQBegAAAAAAObwAeTPXkgAASQSEAAAJIJAAAqJJIQFRJJAIJJIBUSAAJAARIIJKAAAkABEggkCQQSBIIJKgAABJAIiSSABJJACJJIAEgAASQSVAAERIIJKBJBIQJIJAEkACQAVAkgEEgAqJBBIEgAiJAAAABEggkCQQSBIIJCMs2bfs5P/ABZ395psg1vs2/Zyf+LO/vNNkG99wfZx7y0rvz7ufaAAGaYcAAAAAAAAAAAAAABgACcAABkgCSMgAAAAAAAAAAAAAAAAAAAAAAAAnABQMAASCFUACABOSN5AQrc9AKkXIPnzRStrsgSCHORvVT5ulX71APqqonXkfN0v7n8p8lyvVSWtVegBXKvUI1VPo2NPM+iJjoB82x+p9GpjoTgqAhEJAAAAAAAJyCABIIJQAAABOSABUCMkgAAAAAAAASMEEoBBGCrBAFIJwQAAAAgkAYbtf+51d/hF9aw5NunVTrLbB9zq7/CL61hybdOqgYrcOqmP1fVTILh1Ux+r6qBa5zxv6nsnPG/qBVGe6n8jwxnup/IC70XVDI7b1Qxyi6oZHbeqAZZa/vTtLyOLbX96dpAAAAxzJAAhegC9AAAAAAAc3EkEnkz18JIJCAAAkABAkgASACoEoQSgRIAAkABEgAokEEgCSCSoAAASQSAJIJCBJAAkkgBEgAoAAgkABEgAIkEEgSCCQBJACJAAQJIJKBJACJJIAEgACQAVAAEEgAASQCokkgkiJBBIAAACSCQgSQAJJIARlmzb9nJ/4s7+802Sa22bfs5P/Fnf3mmyTe+4Ps495aV3593PtAADNMOAAAAAAAAAE4AgYJADAGSAJyQAAAAAAAAAAAAAAAAAAAAAAAACcAQCcABgAABkgAAAAAAAAAFTKcz5SI5Pm9PU+oA8vVeZW1ir2PtuIi5wSEUtjROpWmEKcjIVUhUUIpKKBWCMgCcggASCEGQJAAAAAAABIIJyAAAAkgAVApKgAAAAAAEAAnIIGQGASFApwQVYIwBAJIAw3bB9zq7/AAi+tYcm3Tqp1ntg+51d/hF9aw5MunVQMVuHVTH6vqpkFw6qY/V9VAtc5439T2Tnjf1AqjPdT+R4Yz3U/kBd6Lqhkdt6oY5RdUMjtvVAMstf3p2khxba/vTtJOgDAAAkAAQvQBegAAAAAAJAAAAACCSABJBIAAAAAAAAAAAAAAAJwBBIGQGB0IAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAATgAAMkATkgAAAAAAAAAAAAAAAAAAAAAAAAAABgkBgAjIEjJAAAAAAAAAAAAAAAAAADqAgFKpggrIVAIJRSkBFaKVIp80JRQr6ApRSQJAAEggASCPpJAAAAAABJAAkAACSABP0klJOQJBBIAAAAABOQQAJIGSQKRgkAYZtg+51d/hF9aw5MunVTrPbB9zm7/CL61hyZdOqgYrcOqmP1fVTILh1Ux+r6qBa5zxv6nsnPG/qBVGe6n8jwxnup/IC70XVDI7b1Qxyi6oZHbeqAZZa/vTtI4ttf3p2l5AAABIAAhegC9AAAAAAASAAAAAEEkACSCQAAAAAAAAABOAIJwCFAnAyQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAwABIAYAyQBOSAAAAAAAAAAAAAAAAAAAAAAAAAAAGCcARgkZIAnJAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQqZIxgrwRgCgklU9CkCpFKkU+ZKKB9CSjJKKBUAAAAAkEISAAAAAACckACQQSAAAEjJAAkkhABIAAAAAAAGQAgGG7YU/8HN3+EX1rDku6dVOtdsP3Obv8IvrmHJV06qBitw6qY/V9VMguHVTH6vqoFrnPG/qeyc8b+oFUZ7qfyPDGe6n8gLvRdUMjtvVDHKLqhkdt6oBllr+9O0ji21/enaQAAASAAIXoAvQAAAAAAEgAAAABBJAAkgkAAAABOAIBJGQCE5IAAEZJAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABOAIJwBkBgEZAAAAAAAAAAAAAAAAAAAAAAAAAAAAACeQEYJGSAJyQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEoUqhIApIK15lKpgASilIArRSo+eSpFArBCKSAAAEgIAAAAAAAAAJyMkACQAAJIAFQKSUAkAAAAAAAGG7YPuc3f4RfXMOS7p1U602wfc5u/wi+tYcl3TqoGK3Dqpj9X1UyC4dVMfq+qgWuc8b+p7Jzxv6gVRnup/I8MZ7qfyAu9F1QyO29UMcouqGR23qgGWWv707SOLbX96dpAAABIAAhegC9AAAAAAASAAAAAEEkACQgwAQYAAnJAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAkCCcAZADJAAAAAAAAAAAAAAAAAAAAAAAAAAAAACcABgDJAE5IAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIVCkrGMgUEhUwQEVIpUinzJRQPpkkoyVZCpGQAGSSBkCQAAAAAAACckACQQSAJIAE5BAAqBGSQAAAw3bB9zm7/CL61hyXdOqnWm2D7nN3+EX1rDku6dVAxW4dVMfq+qmQXDqpj9X1UC1znjf1PZOeN/UCqM91P5HhjPbB5AXijXmhkVuXmhjdGvNDIbcvNAMuta/NO1Die2O6HbAAAASAAIXoAvQAAAAAAEgAACcACBgEATkhVAAISQhIAAAAQSAAAABQAAAAAAAAAAAAAAAAABGCQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABIEE4AyAGSAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEjIAZIADIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAShAAnqUq0lAoFAK8ZKVQBklFKSQK0UkoyVIoFQIySAyEAAkEZJAAAAAAAAAkEEoAAAAnJAAlCSEGQMN2w/c5u/wAIvrmHJdzXmp1ntkXGze8f+x+uYckXN3NQMauC81Mfquql9r15qWGqXqBbJzxv6nrnPI/qBUw9kC9DxMPXCoF2pF5oX+gdzQxymdzQvVE/oBl9tk6HcJwjb5cYO7gAAAkAAQvQBegAAAASEQAMDAyAJyQAAIJIAEkEgQhIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAE4ADAwMkZAkgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAwT0GQAyQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABK8yCcgUqmCk+hCoBQSgwQBWilSKfMlFArJKEUqQCQABKAgASAAAAAAAATkgATkEEgAABhW2h2Nml5X/AGP1zDkO4yc1OuNt7t3Zfe1/2P10Zx3Xy9QLNXO6ljqV6l1rH9SzVC9QPDOeV/U9Ey8zzu6gGdT0xKeZvU+0a8wLlA7oXWkkxgscLuh76eTGAMoop8Y5l+o6nGOZh1NPjHMutNVYxzAzekq8Y5l1p63uYPT1mPMuENd3AzeKv7nobX9zC46/ufZtw7gZj7w7j3h3MQ94dx7w7gZf7w7j3h3MQ94dx7w7gZf7w7j3h3MQ94dx7w7gZf7w7j3h3MQ94dx7w7gZf7w7j3h3MQ94dx7w7gZf7w7j3h3MQ94dx7w7gZf7w7j3h3MQ94dx7w7gZf7w7j3h3MQ94dx7w7gZf7w7j3h3MQ94dx7w7gZf7w7j3h3MQ94dx7w7gZf7w7j3h3MQ94dx7w7gZf7w7j3h3MQ94dx7w7gZf7w7j3h3MQ94dx7w7gZf7w7j3h3MQ94dx7w7gZf7w7j3h3MQ94dx7w7gZf7w7j3h3MQ94dx7w7gZf7w7j3h3MQ94dx7w7gZf7w7j3h3MQ94dx7w7gZf7w7j3h3MQ94dx7w7gZf7w7j3h3MQ94dx7w7gZf7w7j3h3MQ94dx7w7gZf7w7j3h3MQ94dx7w7gZf7w7j3h3MQ94dx7w7gZf7w7j3h3MQ94dx7w7gZf7w7j3h3MQ94dx7w7gZf7w7j3h3MQ94dx7w7gZf7w7j3h3MQ94dx7w7gZf7w7j3h3MQ94dx7w7gZf7w7j3h3MQ94dx7w7gZf7w7j3h3MQ94dx7w7gZf7w7j3h3MQ94dx7w7gZf7w7j3h3MQ94dx7w7gZf7w7j3h3MQ94dx7w7gZf7w7j3h3MQ94dx7w7gZf7w7j3h3MQ94dx7w7gZf7w7j3h3MQ94dx7w7gZf7w7j3h3MQ94dx7w7gZf7w7j3h3MQ94dx7w7gZf7w7j3h3MQ94dx7w7gZf7w7j3h3MQ94dx7w7gZf7w7j3h3MQ94dx7w7gZf7w7j3h3MQ94dx7w7gZf7w7j3h3MQ94dx7w7gZf7w7j3h3MQ94dx7w7gZf7w7j3h3MQ94dx7w7gZf7w7j3h3MQ94dx7w7gZf7w7j3h3MQ94dx7w7gZf7w7j3h3MQ94dx7w7gZf7w7j3h3MQ94dx7w7gZf7w7j3h3MQ94dx7w7gZf7w7j3h3MQ94dx7w7gZf7w7j3h3MQ94dx7w7gZf7w7j3h3MQ94dx7w7gZf7w7j3h3MQ94dx7w7gZf7w7j3h3MQ94dx7w7gZf7w7j3h3MQ94dx7w7gZf7w7j3h3MQ94dx7w7gZf7w7j3h3MQ94dx7w7gZf7w7j3h3MQ94dx7w7gZf7w7j3h3MQ94dx7w7gZf7w7j3h3MQ94dx7w7gZf7w7j3h3MQ94dx7w7gZf7w7j3h3MQ94dx7w7gZf7w7j3h3MQ94dx7w7gZf7w7j3h3MQ94dx7w7gZf7w7j3h3MQ94dx7w7gZf7w7j3h3MQ94dx7w7gZf7w7j3h3MQ94dx7w7gZf7w7j3h3MQ94dx7w7gZf7w7j3h3MQ94dx7w7gZf7w7j3h3MQ94dx7w7gZf7w7j3h3MQ94dx7w7gZf7w7j3h3MQ94dx7w7gZf7w7j3h3MQ94dx7w7gZf7w7j3h3MQ94dx7w7gZf7w7j3h3MQ94dx7w7gZf7w7j3h3MQ94dx7w7gZf7w7j3h3MQ94dx7w7gZf7w7j3h3MQ94dx7w7gZf7w7j3h3MQ94dx7w7gZf7w7j3h3MQ94dx7w7gZf7w7j3h3MQ94dx7w7gZf7w7j3h3MQ94dx7w7gZf7w7j3h3MQ94dx7w7gZf7w7ke8O5iPvDuPeHcDK31/c8s1d3MbdcO58JK/uBe6mt68y0VdXnPMt81d15luqKvOeYH0rKnOeZYqybKqfSpqc55lpqZs55geeqkyqlsnceieTJ4ZXAeeReZ8HdT6vU+S9QCH0ap8itqgemNx64pC3tXB9mPAu8M2PM90NTjzLEyQ+7Ju4GRxVWPM9cdZjzMZZUY8z7Nql9QMnbXdytK7uYylX3Kkq+4GTcf3HH9zGuL7ji+4GS8f3HH9zGuL7ji+4GS8f3HH9zGuL7ji+4GS8f3HH9zGuL7ji+4GS8f3HH9zGuL7ji+4GS8f3HH9zGuL7ji+4GS8f3HH9zGuL7ji+4GS8f3HH9zGuL7ji+4GS8f3HH9zGuL7ji+4GS8f3HH9zGuL7ji+4GS8f3HH9zGuL7ji+4GS8f3HH9zGuL7ji+4GS8f3HH9zGuL7ji+4GS8f3HH9zGuL7ji+4GS8f3HH9zGuL7ji+4GS8f3HH9zGuL7ji+4GS8f3HH9zGuL7ji+4GS8f3HH9zGuL7ji+4GS8f3HH9zGuL7ji+4GS8f3HH9zGuL7ji+4GS8f3HH9zGuL7ji+4GS8f3HH9zGuL7ji+4GS8f3HH9zGuL7ji+4GS8f3HH9zGuL7ji+4GS8f3HH9zGuL7ji+4GS8f3HH9zGuL7ji+4GS8f3HH9zGuL7ji+4GS8f3HH9zGuL7ji+4GS8f3HH9zGuL7ji+4GS8f3HH9zGuL7ji+4GS8f3HH9zGuL7ji+4GS8f3HH9zGuL7ji+4GS8f3HH9zGuL7ji+4GS8f3HH9zGuL7ji+4GS8f3HH9zGuL7ji+4GS8f3HH9zGuL7ji+4GS8f3HH9zGuL7ji+4GS8f3HH9zGuL7ji+4GS8f3HH9zGuL7ji+4GS8f3HH9zGuL7ji+4GS8f3HH9zGuL7ji+4GS8f3HH9zGuL7ji+4GS8f3HH9zGuL7ji+4GS8f3HH9zGuL7ji+4GS8f3HH9zGuL7ji+4GS8f3HH9zGuL7ji+4GS8f3HH9zGuL7ji+4GS8f3HH9zGuL7ji+4GS8f3HH9zGuL7ji+4GS8f3HH9zGuL7ji+4GS8f3HH9zGuL7ji+4GS8f3HH9zGuL7ji+4GS8f3HH9zGuL7ji+4GS8f3HH9zGuL7ji+4GS8f3HH9zGuL7ji+4GS8f3HH9zGuL7ji+4GS8f3HH9zGuL7ji+4GS8f3HH9zGuL7ji+4GS8f3HH9zGuL7ji+4GS8f3HH9zGuL7ji+4GS8f3HH9zGuL7ji+4GS8f3HH9zGuL7ji+4GS8f3HH9zGuL7ji+4GS8f3HH9zGuL7ji+4GS8f3HH9zGuL7ji+4GS8f3HH9zGuL7ji+4GS8f3HH9zGuL7ji+4GS8f3HH9zGuL7ji+4GS8f3HH9zGuL7ji+4GS8f3HH9zGuL7ji+4GS8f3HH9zGuL7ji+4GS8f3HH9zGuL7ji+4GS8f3HH9zGuL7ji+4GS8f3HH9zGuL7ji+4GS8f3HH9zGuL7ji+4GS8f3HH9zGuL7ji+4GS8f3HH9zGuL7ji+4GS8f3HH9zGuL7ji+4GS8f3HH9zGuL7ji+4GS8f3HH9zGuL7ji+4GS8f3HH9zGuL7ji+4GS8f3HH9zGuL7ji+4GS8f3HH9zGuL7ji+4GS8f3HH9zGuL7ji+4GS8f3HH9zGuL7ji+4GRrXdz5Pre5YFq+5Q6q7gXqSs7nklqs55lrdUr6nxfPnzA9k1RnzPDNLk+L5u58HyATK88sjskvefFygQ5T5lSqUgCUAAqRStHAAVtfg+jZAAPokvcrSbuABKTdypJwAHj9x4/cAB4/ceP3AAeP3Hj9wAHj9x4/cAB4/ceP3AAeP3Hj9wAHj9x4/cAB4/ceP3AAeP3Hj9wAHj9x4/cAB4/ceP3AAeP3Hj9wAHj9x4/cAB4/ceP3AAeP3Hj9wAHj9x4/cAB4/ceP3AAeP3Hj9wAHj9x4/cAB4/ceP3AAeP3Hj9wAHj9x4/cAB4/ceP3AAeP3Hj9wAHj9x4/cAB4/ceP3AAeP3Hj9wAHj9x4/cAB4/ceP3AAeP3Hj9wAHj9x4/cAB4/ceP3AAeP3Hj9wAHj9x4/cAB4/ceP3AAeP3Hj9wAHj9x4/cAB4/ceP3AAeP3Hj9wAHj9x4/cAB4/ceP3AAeP3Hj9wAHj9x4/cAB4/ceP3AAeP3Hj9wAHj9x4/cAB4/ceP3AAeP3Hj9wAHj9x4/cAB4/ceP3AAeP3Hj9wAHj9x4/cAB4/ceP3AAeP3Hj9wAHj9x4/cAB4/ceP3AAeP3Hj9wAHj9x4/cAB4/ceP3AAeP3Hj9wAHj9x4/cAB4/ceP3AAeP3Hj9wAHj9x4/cAB4/ceP3AAeP3Hj9wAHj9x4/cAB4/ceP3AAeP3Hj9wAHj9x4/cAB4/ceP3AAeP3Hj9wAHj9x4/cAB4/ceP3AAeP3Hj9wAHj9x4/cAB4/ceP3AAeP3Hj9wAHj9x4/cAB4/ceP3AAeP3Hj9wAHj9x4/cAB4/ceP3AAeP3Hj9wAHj9x4/cACFnIWbuABSs3coWXuAB83SHzc/IAFCqUKoAEEAAf/2Q=="
],
"sort": true
}'
```
## docker部署
### 镜像获取
**本地编译镜像:**
```bash
docker build -t surya_ocr:v0.1 .
```
**或拉取线上镜像:**
Todo:待发布
### docker-compose.yml示例
```yaml
version: '3'
services:
surya-ocr:
image: surya_ocr:v0.1
container_name: surya-ocr
# GPU运行环境,如果宿主机未安装,将deploy配置隐藏即可
deploy:
resources:
reservations:
devices:
- driver: nvidia
count: all
capabilities: [gpu]
ports:
- 7230:7230
environment:
- BATCH_SIZE=32
- ACCESS_TOKEN=YOUR_ACCESS_TOKEN
- LANGS='["zh","en"]'
```
**环境变量:**
```
BATCH_SIZE:根据实际内存/显存情况配置,每个batch约占用40MB的VRAM,cpu默认32,mps默认64,cuda默认512
ACCESS_TOKEN:服务的access_token
LANGS:支持的语言列表,默认["zh","en"]
```
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import base64
import io
import json
import logging
import os
from typing import List, Optional
import torch
import uvicorn
from fastapi import FastAPI, HTTPException, Security
from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer
from PIL import Image, ImageFile
from pydantic import BaseModel
from surya.model.detection.model import load_model as load_det_model
from surya.model.detection.model import load_processor as load_det_processor
from surya.model.recognition.model import load_model as load_rec_model
from surya.model.recognition.processor import load_processor as load_rec_processor
from surya.ocr import run_ocr
from surya.schema import OCRResult
import warnings
warnings.filterwarnings("ignore", category=FutureWarning, module="transformers")
app = FastAPI()
security = HTTPBearer()
env_bearer_token = None
# GPU显存回收
def torch_gc():
if torch.cuda.is_available(): # 检查是否可用CUDA
torch.cuda.empty_cache() # 清空CUDA缓存
torch.cuda.ipc_collect() # 收集CUDA内存碎片
class ImageReq(BaseModel):
images: List[str]
sorted: Optional[bool] = False
class Singleton(type):
def __call__(cls, *args, **kwargs):
if not hasattr(cls, '_instance'):
cls._instance = super().__call__(*args, **kwargs)
return cls._instance
class Surya(metaclass=Singleton):
def __init__(self):
self.langs = json.loads(os.getenv("LANGS", '["zh", "en"]'))
self.batch_size = os.getenv("BATCH_SIZE")
if self.batch_size is not None:
self.batch_size = int(self.batch_size)
self.det_processor, self.det_model = load_det_processor(
), load_det_model()
self.rec_model, self.rec_processor = load_rec_model(
), load_rec_processor()
def run(self, image: ImageFile.ImageFile) -> List[OCRResult]:
predictions = run_ocr([image], [self.langs], self.det_model,
self.det_processor, self.rec_model,
self.rec_processor, self.batch_size)
return predictions
class Chat(object):
def __init__(self):
self.surya = Surya()
def base64_to_image(base64_string: str) -> ImageFile.ImageFile:
image_data = base64.b64decode(base64_string)
image_stream = io.BytesIO(image_data)
image = Image.open(image_stream)
return image
def sort_text_by_bbox(original_data: List[dict]) -> str:
# 根据bbox进行排序,从左到右,从上到下。返回排序后的按行的字符串。
# 排序
lines, line = [], []
original_data.sort(key=lambda item: item["bbox"][1])
for item in original_data:
mid_h = (item["bbox"][1] + item["bbox"][3]) / 2
if len(line) == 0 or (mid_h >= line[0]["bbox"][1]
and mid_h <= line[0]["bbox"][3]):
line.append(item)
else:
lines.append(line)
line = [item]
lines.append(line)
for line in lines:
line.sort(key=lambda item: item["bbox"][0])
# 构建行字符串
string_result = ""
for line in lines:
for item in line:
string_result += item["text"] + " "
string_result += "\n"
return string_result
def query_ocr(self, image_base64: str,
sorted: bool) -> str:
if image_base64 is None or len(image_base64) == 0:
return ""
try:
image = Chat.base64_to_image(image_base64)
ocr_result = self.surya.run(image)
result = []
for text_line in ocr_result[0].text_lines:
result.append(text_line.text)
if sorted:
result = self.sort_text_lines(result)
# 将所有文本行合并成一个字符串,用换行符分隔
final_result = "\n".join(result)
torch_gc()
return final_result
except Exception as e:
logging.error(f"OCR 处理失败: {e}")
raise HTTPException(status_code=400, detail=f"OCR 处理失败: {str(e)}")
@staticmethod
def sort_text_lines(text_lines: List[str]) -> List[str]:
# 这里可以实现自定义的排序逻辑
# 目前只是简单地返回原始列表,因为我们没有位置信息来进行排序
return text_lines
@app.post('/v1/ocr/text')
async def handle_post_request(
image_req: ImageReq,
credentials: HTTPAuthorizationCredentials = Security(security)):
token = credentials.credentials
if env_bearer_token is not None and token != env_bearer_token:
raise HTTPException(status_code=401, detail="无效的令牌")
chat = Chat()
try:
results = []
for image_base64 in image_req.images:
results.append(chat.query_ocr(image_base64, image_req.sorted))
return {"error": None, "results": results}
except HTTPException as he:
raise he
except Exception as e:
logging.error(f"识别报错:{e}")
raise HTTPException(status_code=500, detail=f"识别出错: {str(e)}")
if __name__ == "__main__":
env_bearer_token = os.getenv("ACCESS_TOKEN")
try:
uvicorn.run(app, host='0.0.0.0', port=7230)
except Exception as e:
logging.error(f"API启动失败!报错:{e}")
surya-ocr==0.5.0
fastapi==0.104.1
uvicorn==0.17.6
\ No newline at end of file
FROM pytorch/pytorch:2.4.1-cuda12.4-cudnn9-devel
ENV DEBIAN_FRONTEND=noninteractive
ENV LANG=C.UTF-8
# 安装构建依赖 cv2 dependencies
RUN apt-get update && apt-get install ffmpeg libsm6 libxext6 -y
# 设置 pip 配置
RUN mkdir -p /root/.pip
COPY pip.conf /root/.pip/
# 创建模型文件夹
RUN mkdir -p /root/huggingface
# 复制依赖文件
COPY requirements.txt /root/
COPY api_mp.py /root/
# 导入huggingface的代理和huggingface模型位置
ENV HF_ENDPOINT=https://hf-mirror.com \
HF_DATASETS_CACHE=/root/huggingface \
HUGGINGFACE_HUB_CACHE=/root/huggingface \
HF_HOME=/root/huggingface
# 设置工作目录
WORKDIR /root
# 安装 Python 依赖
RUN pip3 install --no-cache-dir -r requirements.txt
# 删除不必要的工具和文件以减小镜像体积
RUN apt-get purge -y vim && apt-get autoremove -y && rm -rf /root/.pip /root/.cache/pip
# 设置容器启动命令
CMD ["python3", "api_mp.py"]
\ No newline at end of file
# 项目介绍
本项目实现了一个高效的 **PDF 转 Markdown 接口服务**,支持多进程并行处理多个 PDF 文件。通过高性能的接口设计,快速将 PDF 文档转换为 Markdown 格式文本。
- **简洁性:**项目无需修改代码,仅需调整文件路径即可使用,简单易用
- **易用性:**通过提供简洁的 API,开发者只需发送 HTTP 请求即可完成 PDF 转换
- **灵活性:**支持本地部署和 Docker 容器部署两种方式,便于快速上手和灵活集成
# 配置推荐
## 常规配置
24G显存的显卡两张,可以支持四个文件同时处理
## 最低配置
**不低于11G** 显存的显卡一张
并设置每张卡处理的进程数为1
```bash
export PROCESSES_PER_GPU="1"
```
## 单文件实测速率
| 显卡 | 中文PDF | 英文PDF | 扫描件 |
| ------------- | ------------ | ------------ | ------------ |
| **4090D 24G** | **0.75s/页** | **1.60s/页** | **3.26s/页** |
| **P40 24G** | **0.99s/页** | **2.22s/页** | **5.24s/页** |
## 多文件实测速率
中文PDF+英文PDF
| 显卡 | 串行处理 | 并行处理 | 提升效率 |
| ------------- | ------------ | ------------ | --------- |
| **4090D 24G** | **0.92s/页** | **0.62s/页** | **31.9%** |
| **P40 24G** | **1.22s/页** | **0.85s/页** | **30.5%** |
# 本地开发
## 基本流程
1. 克隆一个FastGPT的项目文件
```
git clone https://github.com/labring/FastGPT.git
```
2. 将主目录设置为 python下的pdf-marker文件
```
cd python/pdf-marker
```
3. 创建Anaconda并安装requirement.txt文件
安装的Anaconda版本:**conda 24.7.1**
```
conda create -n pdf-marker python=3.11
pip install -r requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple
conda activate pdf-marker
```
4. 执行主文件启动pdf2md服务
```
python api_mp.py
```
# 镜像打包和部署(推荐)
## 本地构建镜像
1. 在 `pdf-marker` 根目录下执行:
```bash
sudo docker build -t model_pdf -f Dockerfile .
```
2. 运行容器
```bash
sudo docker run --gpus all -itd -p 7231:7231 --name model_pdf_v1 -e PROCESSES_PER_GPU="2" model_pdf
```
## 快速构建镜像(推荐)
```dockerfile
docker pull crpi-h3snc261q1dosroc.cn-hangzhou.personal.cr.aliyuncs.com/marker11/marker_images:v0.2
docker run --gpus all -itd -p 7231:7232 --name model_pdf_v2 -e PROCESSES_PER_GPU="2" crpi-h3snc261q1dosroc.cn-hangzhou.personal.cr.aliyuncs.com/marker11/marker_images:v0.2
```
# 访问示例
marker v0.1:用Post方法访问端口为 `7321 ` 的 `v1/parse/file` 服务
marker v0.2:用Post方法访问端口为 `7321 ` 的 `v2/parse/file` 服务
- 访问方法
- v0.2
```
curl --location --request POST "http://localhost:7231/v2/parse/file" \
--header "Authorization: Bearer your_access_token" \
--form "file=@./file/chinese_test.pdf"
```
- v0.1
```
curl --location --request POST "http://localhost:7231/v1/parse/file" \
--header "Authorization: Bearer your_access_token" \
--form "file=@./file/chinese_test.pdf"
```
参数:file-->本地文件的地址
- 多文件测试数据
运行 `test` 文件下的 `test.py` 文件,修改里面的 `file_paths` 为自己仓库的 `url` 即可
# FQA
- 如果出现huggingface模型下载不下来?
可以选择在环境变量中加入huggingface镜像
```bash
export HF_ENDPOINT=https://hf-mirror.com
export HF_DATASETS_CACHE=/huggingface
export HUGGINGFACE_HUB_CACHE=/huggingface
export HF_HOME=/huggingface
```
也可以直接访问 [huggingface][https://huggingface.co] 来下载模型到 `/huggingface` 文件夹下
```
https://huggingface.co/vikp/surya_det3/tree/main
https://huggingface.co/vikp/surya_layout3/tree/main
https://huggingface.co/vikp/surya_order/tree/main
https://huggingface.co/vikp/surya_rec2/tree/main
https://huggingface.co/vikp/surya_tablerec/tree/main
https://huggingface.co/vikp/texify2/tree/main
```
\ No newline at end of file
import asyncio
import base64
import fitz
import torch.multiprocessing as mp
import shutil
import time
from contextlib import asynccontextmanager
from loguru import logger
from fastapi import HTTPException, FastAPI, UploadFile, File
import multiprocessing
from marker.output import save_markdown
from marker.convert import convert_single_pdf
from marker.models import load_all_models
import torch
from concurrent.futures import ProcessPoolExecutor
import os
app = FastAPI()
model_lst = None
model_refs = None
temp_dir = "./temp"
os.environ['PROCESSES_PER_GPU'] = str(2)
def worker_init(counter, lock):
global model_lst
num_gpus = torch.cuda.device_count()
processes_per_gpu = int(os.environ.get('PROCESSES_PER_GPU', 1))
with lock:
worker_id = counter.value
counter.value += 1
if num_gpus == 0:
device = 'cpu'
else:
device_id = worker_id // processes_per_gpu
if device_id >= num_gpus:
raise ValueError(f"Worker ID {worker_id} exceeds available GPUs ({num_gpus}).")
device = f'cuda:{device_id}'
model_lst = load_all_models(device=device, dtype=torch.float32)
print(f"Worker {worker_id}: Models loaded successfully on {device}!")
for model in model_lst:
if model is None:
continue
model.share_memory()
def process_file_with_multiprocessing(temp_file_path):
global model_lst
full_text, images, out_meta = convert_single_pdf(temp_file_path, model_lst, batch_multiplier=1)
fname = os.path.basename(temp_file_path)
subfolder_path = save_markdown(r'./result', fname, full_text, images, out_meta)
md_content_with_base64_images = embed_images_as_base64(full_text, subfolder_path)
return md_content_with_base64_images, out_meta
@asynccontextmanager
async def lifespan(app: FastAPI):
try:
mp.set_start_method('spawn')
except RuntimeError:
raise RuntimeError("Set start method to spawn twice. This may be a temporary issue with the script. Please try running it again.")
manager = multiprocessing.Manager()
worker_counter = manager.Value('i', 0)
worker_lock = manager.Lock()
global my_pool
gpu_count = torch.cuda.device_count()
my_pool = ProcessPoolExecutor(max_workers=gpu_count*int(os.environ.get('PROCESSES_PER_GPU', 1)), initializer=worker_init, initargs=(worker_counter, worker_lock))
yield
global temp_dir
if temp_dir and os.path.exists(temp_dir):
shutil.rmtree(temp_dir)
del model_lst
del model_refs
print("Application shutdown, cleaning up...")
app.router.lifespan_context = lifespan
@app.post("/v1/parse/file")
async def read_file(
file: UploadFile = File(...)):
try:
start_time = time.time()
global temp_dir
os.makedirs(temp_dir, exist_ok=True)
temp_file_path = os.path.join(temp_dir, file.filename)
with open(temp_file_path, "wb") as temp_file:
temp_file.write(await file.read())
pdf_document = fitz.open(temp_file_path)
total_pages = pdf_document.page_count
pdf_document.close()
global my_pool
loop = asyncio.get_event_loop()
md_content_with_base64_images, out_meta = await loop.run_in_executor(my_pool, process_file_with_multiprocessing, temp_file_path)
end_time = time.time()
duration = end_time - start_time
print(file.filename+"Total time:", duration)
return {
"success": True,
"message": "",
"data": {
"markdown": md_content_with_base64_images,
"page": total_pages,
"duration": duration
}
}
except Exception as e:
logger.exception(e)
raise HTTPException(status_code=500, detail=f"错误信息: {str(e)}")
finally:
if temp_file_path and os.path.exists(temp_file_path):
os.remove(temp_file_path)
def img_to_base64(img_path):
with open(img_path, "rb") as img_file:
return base64.b64encode(img_file.read()).decode('utf-8')
def embed_images_as_base64(md_content, image_dir):
lines = md_content.split('\n')
new_lines = []
for line in lines:
if line.startswith("![") and "](" in line and ")" in line:
start_idx = line.index("](") + 2
end_idx = line.index(")", start_idx)
img_rel_path = line[start_idx:end_idx]
img_name = os.path.basename(img_rel_path)
img_path = os.path.join(image_dir, img_name)
if os.path.exists(img_path):
img_base64 = img_to_base64(img_path)
new_line = f'{line[:start_idx]}data:image/png;base64,{img_base64}{line[end_idx:]}'
new_lines.append(new_line)
else:
new_lines.append(line)
else:
new_lines.append(line)
return '\n'.join(new_lines)
if __name__ == "__main__":
import uvicorn
uvicorn.run(app, host="0.0.0.0", port=7231)
[global]
time-out=60
index-url=https://pypi.tuna.tsinghua.edu.cn/simple
[install]
trusted-host=pypi.tuna.tsinghua.edu.cn
acres==0.1.0
aiofiles==24.1.0
annotated-types==0.7.0
anyio==4.6.2.post1
certifi==2024.8.30
charset-normalizer==3.4.0
ci-info==0.3.0
click==8.1.7
coloredlogs==15.0.1
configobj==5.0.9
configparser==7.1.0
dol==0.2.83
etelemetry==0.3.1
fastapi==0.115.5
filelock==3.16.1
filetype==1.2.0
flatbuffers==24.3.25
frontend==0.0.3
fsspec==2024.10.0
ftfy==6.3.1
h11==0.14.0
httplib2==0.22.0
huggingface-hub==0.26.2
humanfriendly==10.0
i2==0.1.36
idna==3.10
importlib_resources==6.4.5
isodate==0.6.1
itsdangerous==2.2.0
Jinja2==3.1.4
joblib==1.4.2
loguru==0.7.2
looseversion==1.3.0
lxml==5.3.0
marker-pdf==0.3.10
MarkupSafe==3.0.2
mpmath==1.3.0
networkx==3.4.2
nibabel==5.3.2
nipype==1.9.1
numpy==2.1.3
nvidia-cublas-cu12==12.4.5.8
nvidia-cuda-cupti-cu12==12.4.127
nvidia-cuda-nvrtc-cu12==12.4.127
nvidia-cuda-runtime-cu12==12.4.127
nvidia-cudnn-cu12==9.1.0.70
nvidia-cufft-cu12==11.2.1.3
nvidia-curand-cu12==10.3.5.147
nvidia-cusolver-cu12==11.6.1.9
nvidia-cusparse-cu12==12.3.1.170
nvidia-nccl-cu12==2.21.5
nvidia-nvjitlink-cu12==12.4.127
nvidia-nvtx-cu12==12.4.127
onnxruntime==1.20.1
opencv-python==4.10.0.84
opencv-python-headless==4.10.0.84
packaging==24.2
pandas==2.2.3
pathlib==1.0.1
pdftext==0.3.19
pillow==10.4.0
pip==24.3.1
protobuf==5.28.3
prov==2.0.1
puremagic==1.28
pydantic==2.10.0
pydantic_core==2.27.0
pydantic-settings==2.6.1
pydot==3.0.2
PyMuPDF==1.24.14
pyparsing==3.2.0
pypdfium2==4.30.0
python-dateutil==2.9.0.post0
python-dotenv==1.0.1
python-multipart==0.0.17
pytz==2024.2
pyxnat==1.6.2
PyYAML==6.0.2
RapidFuzz==3.10.1
rdflib==6.3.2
regex==2024.11.6
requests==2.32.3
safetensors==0.4.5
scikit-learn==1.5.2
scipy==1.14.1
setuptools==75.6.0
simplejson==3.19.3
six==1.16.0
sniffio==1.3.1
starlette==0.41.3
surya-ocr==0.6.13
sympy==1.13.1
tabled-pdf==0.1.4
tabulate==0.9.0
texify==0.2.1
threadpoolctl==3.5.0
tokenizers==0.20.3
torch==2.5.1
tqdm==4.67.0
traits==6.4.3
transformers==4.46.3
triton==3.1.0
typing_extensions==4.12.2
tzdata==2024.2
urllib3==2.2.3
uvicorn==0.32.1
wcwidth==0.2.13
wheel==0.45.0
import json
import os
from io import BytesIO
import requests
from multiprocessing import Process
def request_(file_path):
url = "http://127.0.0.1:7231/v1/parse/file"
response = requests.get(file_path)
if response.status_code == 200:
file_data = BytesIO(response.content)
pdf_name = os.path.basename(file_path)
files = {'file': (pdf_name, file_data, 'application/pdf')}
response = requests.post(url, files=files)
if response.status_code == 200:
print("Response JSON:", json.dumps(response.json(), indent=4, ensure_ascii=False))
else:
print(f"Request failed with status code: {response.status_code}")
print(response.text)
if __name__ == "__main__":
file_paths = ["https://objectstorageapi.bja.sealos.run/czrn86r1-yyh/english_test.pdf", "https://objectstorageapi.bja.sealos.run/czrn86r1-yyh/chinese_test.pdf",
"https://objectstorageapi.bja.sealos.run/czrn86r1-yyh/ocr_test.pdf","https://objectstorageapi.bja.sealos.run/czrn86r1-yyh/english_file/3649329.3658477.pdf"]
for file_path in file_paths:
p = Process(target=request_, args=(file_path))
p.start()
__pycache__
.pyc
.pyo
.pyd
.Python
env
venv
.venv
pip-log.txt
pip-delete-this-directory.txt
.tox
.coverage
.coverage.
.cache
nosetests.xml
coverage.xml
.cover
.log
.git
.mypy_cache
.pytest_cache
\ No newline at end of file
MINERU_TOKEN=官网申请的API 密钥
\ No newline at end of file
# ---- 基础镜像 ----
FROM python:3.12-slim
# ---- 工作目录 ----
WORKDIR /app
# ---- 复制代码 ----
COPY mineru_saas_api.py .
COPY requirements.txt .
# ---- 安装依赖 ----
RUN pip install --no-cache-dir -r requirements.txt
# ---- 环境变量(运行时注入)----
ENV MINERU_TOKEN="YOUR_TOKEN_WILL_BE_INJECTED"
# ---- 暴露端口 ----
EXPOSE 1234
# ---- 启动命令 ----
CMD ["uvicorn", "mineru_saas_api:app", "--host", "0.0.0.0", "--port", "1234"]
# **MinerU SaaS Wrapper For Fastgpt 详细部署文档**
**—— 为 FastGPT 提供稳定、高效、开箱即用的纯白嫖文档解析服务,转接服务用grok写的,文档也是,有不明白出问题了,`docker logs -f mineru-saas-wrapper` 查看日志,问他~**
---
> **适用人群**:FastGPT 开发者、后端工程师、DevOps、AI 应用集成者
> **目标**:在 **5 分钟内**完成从零到生产可用的 MinerU saas服务api的文档解析服务部署
---
## 一、项目概述
| 项目 | 说明 |
|------|------|
| **名称** | MinerU SaaS Wrapper for FastGPT |
| **框架** | FastAPI + Uvicorn |
| **核心功能** | 接收文件 → 调用 MinerU 官方 SaaS API → 轮询结果 → 返回内嵌图片的 Markdown → fasgpt读取解析内容转为知识库 |
| **部署方式** | Docker(推荐) / docker-compose |
| **接口路径** | `POST /v2/parse/file` |
---
## 二、前置条件
| **MinerU Token** | 在 [https://mineru.net](https://mineru.net) 注册并获取 SaaS Token |
> **获取 Token 步骤**:
> 1. 登录 MinerU 官网
> 2. 进入 **控制台 → API 密钥**
> 3. 创建新密钥(建议命名 `fastgpt-wrapper`)
> 4. 复制完整 Token(以 `eyJ...` 开头)
---
## 三、目录结构说明
```bash
mineru-saas-wrapper/
├── .dockerignore
├── Dockerfile
├── docker-compose.yml
├── mineru_saas_api.py # 主服务逻辑
├── requirements.txt # 依赖包
├── .env # (可选)环境变量文件
└── README.md
```
---
## 四、部署方式一:使用 `docker-compose`(推荐)
### 步骤 1:克隆项目
```bash
mkdir mineru-saas-wrapper
cd mineru-saas-wrapper
```
### 步骤 2:创建 `.env` 文件(推荐,防止 Token 泄露)
```bash
touch .env
```
编辑 `.env`:
```env
MINERU_TOKEN=官网申请的API 密钥
POLL_INTERVAL=3
POLL_TIMEOUT=600
PORT=1234
```
### 步骤 3:修改 `docker-compose.yml`
```yaml
services:
mineru-saas-wrapper:
build:
context: .
dockerfile: Dockerfile
container_name: mineru-saas-wrapper
restart: unless-stopped
ports:
- "1234:1234"
env_file:
- .env # 改为读取 .env 文件
healthcheck:
test: ["CMD", "curl", "-f", "http://localhost:1234/health"]
interval: 30s
timeout: 10s
retries: 3
start_period: 10s
logging:
driver: "json-file"
options:
max-size: "10m"
max-file: "3"
```
### 步骤 4:启动服务
```bash
docker-compose up -d --build
```
### 步骤 5:验证服务状态
```bash
# 查看容器状态
docker ps | grep mineru-saas-wrapper
# 查看健康检查
curl http://localhost:1234/health
# 预期输出:
{"status":"healthy"}
```
## 五、接口测试
### 1. 使用 `curl` 测试
```bash
curl -X POST "http://localhost:1234/v2/parse/file" \
-F "file=@./sample.pdf" | jq
```
### 2. 预期成功响应
```json
{
"success": true,
"message": "",
"markdown": "# 标题\n\n![](data:image/png;base64,iVBORw0KGgoAAA...) ...",
"pages": 8
}
```
### 查看详细日志
```bash
docker logs -f mineru-saas-wrapper
```
关键日志关键词:
- `Got upload url` → 上传成功
- `Polling ... -> done` → 解析完成
- `Parse finished, X pages` → 成功返回
---
## 九、FastGPT 集成指南
### 1. 在 FastGPT 中配置「文档解析」节点
| 字段 | 值 |
|------|---- |
| **解析服务地址** | `http://your-server-ip:1234/v2/parse/file` |
| **请求方式** | POST |
| **文件字段名** | `file` |
| **响应字段映射** | `markdown` → 内容,`pages` → 页数 |
### 2. FastGPT 示例配置(JSON)
```json
// 已使用 json5 进行解析,会自动去掉注释,无需手动去除
{
"feConfigs": {
"lafEnv": "https://laf.dev", // laf环境。 https://laf.run (杭州阿里云) ,或者私有化的laf环境。如果使用 Laf openapi 功能,需要最新版的 laf 。
"mcpServerProxyEndpoint": "" // mcp server 代理地址,例如: http://localhost:3005
},
"systemEnv": {
"datasetParseMaxProcess": 10, // 知识库文件解析最大线程数量
"vectorMaxProcess": 10, // 向量处理线程数量
"qaMaxProcess": 10, // 问答拆分线程数量
"vlmMaxProcess": 10, // 图片理解模型最大处理进程
"tokenWorkers": 30, // Token 计算线程保持数,会持续占用内存,不能设置太大。
"hnswEfSearch": 100, // 向量搜索参数,仅对 PG 和 OB 生效。越大,搜索越精确,但是速度越慢。设置为100,有99%+精度。
"hnswMaxScanTuples": 100000, // 向量搜索最大扫描数据量,仅对 PG生效。
"customPdfParse": {
"url": "http://your-server-ip:1234/v2/parse/file", // 自定义 PDF 解析服务地址
"key": "", // 自定义 PDF 解析服务密钥
"doc2xKey": "", // doc2x 服务密钥
"price": 0 // PDF 解析服务价格
}
}
}
```
---
**部署完成!**
现在你的 FastGPT 已拥有强大的 **MinerU 文档解析能力**,支持 PDF + 图片 → 完美 Markdown 内嵌渲染。
> 如有问题,欢迎提交 Issue 或查看日志排查。祝你解析愉快!
\ No newline at end of file
services:
mineru-saas-wrapper:
build:
context: .
dockerfile: Dockerfile
container_name: mineru-saas-wrapper
restart: unless-stopped
ports:
- "1234:1234"
environment:
# 你的 MinerU SaaS API Token(必须)
- MINERU_TOKEN=eyJ0eXBlIjoiSldUIiwiYWxnIjoiSFM1MTIifQ.eyJqdGkiOiIzODcwOTM0MyIsInJvbCI6IlJPTEVfUkVHSVNURVIiLCJpc3MiOiJPcGVuWExhYiIsImlhdCI6MTc2Mjc2MTEzMywiY2xpZW50SWQiOiJsa3pkeDU3bnZ5MjJqa3BxOXgydyIsInBob25lIjoiMTg1MjEzMzQ1MDEiLCJvcGVuSWQiOm51bGwsInV1aWQiOiI4OTI5YjgzNC05ZTY4LTRhOTctOTNiMi1hMGVkNDk5N2YzYmYiLCJlbWFpbCI6IiIsImV4cCI6MTc2Mzk3MDczM30.CadUrEtAc_B_04opSk4b5ykK60m-CbrXArZuhNGV35MKsX_SaWTbrMHd3ND309f9fgM10QTWHAszjP2Duamzwg
# 可选:自定义轮询间隔(秒)
- POLL_INTERVAL=3
# 可选:最大等待时间(秒)
- POLL_TIMEOUT=600
# 可选:如果你的网络在国外,可改为国内加速镜像源(可选)
# - MINERU_BASE=https://mineru.net
healthcheck:
test: ["CMD", "curl", "-f", "http://localhost:1234/health"]
interval: 30s
timeout: 10s
retries: 3
start_period: 10s
logging:
driver: "json-file"
options:
max-size: "10m"
max-file: "3"
# -*- coding: utf-8 -*-
import os
import io
import time
import zipfile
import base64
import tempfile
from pathlib import Path
from typing import List
import httpx
import uvicorn
from fastapi import FastAPI, File, UploadFile, HTTPException
from fastapi.responses import JSONResponse
from loguru import logger
# --------------------------------------------------------------
# 配置(全部走环境变量,Docker 里通过 -e 注入)
# --------------------------------------------------------------
MINERU_TOKEN = os.getenv("MINERU_TOKEN") # 必须
MINERU_BASE = os.getenv("MINERU_BASE", "https://mineru.net")
POLL_INTERVAL = int(os.getenv("POLL_INTERVAL", "3")) # 秒
POLL_TIMEOUT = int(os.getenv("POLL_TIMEOUT", "600")) # 秒
# --------------------------------------------------------------
app = FastAPI(title="MinerU SaaS Wrapper", version="1.0.0")
# ---------- 工具 ----------
def img_to_base64(img_bytes: bytes) -> str:
return base64.b64encode(img_bytes).decode("utf-8")
def embed_images(md: str, img_dir: Path) -> str:
"""把 markdown 中 ![xxx](relative_path) 替换为 data-uri"""
lines = md.splitlines()
out: List[str] = []
for line in lines:
if line.startswith("![") and "](" in line and ")" in line:
start = line.index("](") + 2
end = line.index(")", start)
rel = line[start:end]
img_path = img_dir / rel
if img_path.is_file():
b64 = img_to_base64(img_path.read_bytes())
new_line = f'![](data:image/png;base64,{b64})'
out.append(new_line)
continue
out.append(line)
return "\n".join(out)
# ---------- SaaS 调用 ----------
async def create_task(file_bytes: bytes, filename: str) -> str:
url = f"{MINERU_BASE}/api/v4/extract/task"
headers = {
"Authorization": f"Bearer {MINERU_TOKEN}",
"Content-Type": "application/json",
}
# 这里使用 VLM(默认),如需 pipeline 可改 model_version
payload = {
"url": "", # 必填但我们用 upload 方式,留空
"model_version": "vlm",
}
# SaaS 目前只接受 URL,我们先把文件上传到临时公开位置不可行 → 改用 **批量上传** 方式
# 下面改成 **批量文件上传**(一次只传一个文件),返回 task_id 列表
raise NotImplementedError("请看下方完整实现")
# --------------------------------------------------------------
# 下面是 **完整实现**(一次只处理一个文件,使用批量上传接口)
# --------------------------------------------------------------
async def _upload_and_create(file_bytes: bytes, filename: str) -> str:
"""
1. 调用 /api/v4/file-urls/batch 获取上传 URL(一次一个文件)
2. PUT 上传文件
3. 系统自动提交解析任务,返回 batch_id
4. 轮询 /api/v4/extract-results/batch/{batch_id} 取结果
"""
client = httpx.AsyncClient(timeout=60.0)
# ---- 1. 申请上传 URL ----
batch_url = f"{MINERU_BASE}/api/v4/file-urls/batch"
headers = {"Authorization": f"Bearer {MINERU_TOKEN}", "Content-Type": "application/json"}
batch_payload = {
"files": [{"name": filename}],
"model_version": "vlm"
}
r = await client.post(batch_url, headers=headers, json=batch_payload)
r.raise_for_status()
batch_resp = r.json()
if batch_resp.get("code") != 0:
raise HTTPException(status_code=500, detail=f"MinerU batch create fail: {batch_resp.get('msg')}")
batch_id = batch_resp["data"]["batch_id"]
upload_url = batch_resp["data"]["file_urls"][0]
logger.info(f"Got upload url for {filename}, batch_id={batch_id}")
# ---- 2. 上传文件 ----
put_r = await client.put(upload_url, content=file_bytes)
put_r.raise_for_status()
logger.info(f"File uploaded, status={put_r.status_code}")
# ---- 3. 轮询结果 ----
result_url = f"{MINERU_BASE}/api/v4/extract-results/batch/{batch_id}"
start = time.time()
while True:
if time.time() - start > POLL_TIMEOUT:
raise HTTPException(status_code=504, detail="MinerU SaaS timeout")
poll = await client.get(result_url, headers=headers)
poll.raise_for_status()
data = poll.json()
if data.get("code") != 0:
raise HTTPException(status_code=500, detail=data.get("msg"))
results = data["data"]["extract_result"]
# 只有一个文件
task = results[0]
state = task["state"]
logger.debug(f"Polling {batch_id} -> {state}")
if state == "done":
zip_url = task["full_zip_url"]
await client.aclose()
return zip_url
if state in ("failed",):
raise HTTPException(status_code=500, detail=task.get("err_msg", "MinerU parse failed"))
# pending / running / converting / waiting-file
await asyncio.sleep(POLL_INTERVAL)
# ---------- 主入口 ----------
import asyncio
@app.post("/v2/parse/file")
async def parse_file(file: UploadFile = File(...)):
"""
FastGPT 调用的统一入口
"""
if not MINERU_TOKEN:
raise HTTPException(status_code=500, detail="MINERU_TOKEN not set")
allowed = {".pdf", ".png", ".jpeg", ".jpg"}
ext = Path(file.filename).suffix.lower()
if ext not in allowed:
raise HTTPException(status_code=400,
detail=f"Unsupported file type {ext}. Allowed: {allowed}")
file_bytes = await file.read()
if not file_bytes:
raise HTTPException(status_code=400, detail="Empty file")
filename = Path(file.filename).name
start = time.time()
try:
# 1. 上传 + 提交任务 → 得到 zip_url
zip_url = await _upload_and_create(file_bytes, filename)
# 2. 下载 zip
async with httpx.AsyncClient() as client:
resp = await client.get(zip_url)
resp.raise_for_status()
zip_bytes = resp.content
# 3. 解压到临时目录
with tempfile.TemporaryDirectory() as tmp:
tmp_path = Path(tmp)
with zipfile.ZipFile(io.BytesIO(zip_bytes)) as z:
z.extractall(tmp_path)
# 4. 找 markdown(默认是和文件名同名的 .md)
md_files = list(tmp_path.rglob("*.md"))
if not md_files:
raise HTTPException(status_code=500, detail="No markdown in result zip")
md_path = md_files[0]
markdown = md_path.read_text(encoding="utf-8")
# 5. 嵌入图片(图片在同一级目录或子目录)
img_dir = md_path.parent
markdown_b64 = embed_images(markdown, img_dir)
# 6. 计算页数(zip 中通常有 page_*.png)
page_imgs = list(tmp_path.rglob("page_*.png")) + list(tmp_path.rglob("page_*.jpg"))
pages = len(page_imgs)
logger.info(f"Parse finished, {pages} pages, {time.time()-start:.1f}s")
return JSONResponse({
"success": True,
"message": "",
"markdown": markdown_b64,
"pages": pages
})
except Exception as e:
logger.exception(f"Parse error for {filename}")
raise HTTPException(status_code=500, detail=str(e))
# ---------- 健康检查 ----------
@app.get("/health")
async def health():
return {"status": "healthy"}
# --------------------------------------------------------------
if __name__ == "__main__":
port = int(os.getenv("PORT", "1234"))
host = os.getenv("HOST", "0.0.0.0")
logger.info(f"Starting MinerU SaaS wrapper on {host}:{port}")
uvicorn.run("mineru_saas_api:app", host=host, port=port, reload=False)
MISTRAL_API_KEY=
\ No newline at end of file
# PDF-Mistral 插件
此插件使用 Mistral 的 OCR API 将 PDF 文件转换为 Markdown 文本。它可以从 PDF 文档中提取文本内容和图像,并将它们作为带有嵌入式 base64 图像的 Markdown 返回。
## 功能特点
- 使用 Mistral OCR API 提取 PDF 文本
- Markdown 中的 base64 图像嵌入
- 完善的错误处理
- 支持多页 PDF
## 设置
### 前提条件
- Python 3.8+
- Mistral API 密钥([在此获取](https://mistral.ai/))
### 安装
1. 安装所需的软件包:
```bash
pip install -r requirements.txt
```
2. 通过创建/编辑 `.env` 文件设置环境变量:
```bash
# 在 .env 文件中
MISTRAL_API_KEY=你的-mistral-api-密钥
```
## 使用方法
### 启动服务器
使用以下命令运行服务器:
```bash
python api_mp.py
```
或者直接使用 uvicorn:
```bash
uvicorn api_mp:app --host 0.0.0.0 --port 7231
```
然后配置到FastGPT配置文件即可
```json
{
xxx
"systemEnv": {
xxx
"customPdfParse": {
"url": "http://localhost:7231/v1/parse/file", // 自定义 PDF 解析服务地址
}
}
}
```
### API 端点
#### 解析 PDF 文件
**端点**:`POST /v1/parse/file`
**请求**:
- 包含文件字段的多部分表单数据
**响应**:
```json
{
"pages": 5, // PDF 中的页数
"markdown": "...", // 带有嵌入式 base64 图像的 Markdown 内容
"duration": 10.5 // 处理时间(秒)
}
```
**错误响应**:
```json
{
"pages": 0,
"markdown": "",
"error": "错误信息"
}
```
### 使用示例
使用 curl:
```bash
curl -X POST -F "file=@path/to/your/document.pdf" http://localhost:7231/v1/parse/file
```
使用 JavaScript/Axios:
```javascript
const formData = new FormData();
formData.append('file', pdfFile);
const response = await axios.post('http://localhost:7231/v1/parse/file', formData, {
headers: {
'Content-Type': 'multipart/form-data'
}
});
if (response.data.error) {
console.error('错误:', response.data.error);
} else {
console.log('页数:', response.data.pages);
console.log('Markdown:', response.data.markdown);
}
```
## 限制
- PDF 文件必须可读且没有密码保护
- 最大文件大小取决于 Mistral API 限制(目前最大52.4M)
- Mistral API 有页面限制(最多最大1000页)
## 故障排除
### 常见错误
1. **"MISTRAL_API_KEY environment variable not set"(未设置 MISTRAL_API_KEY 环境变量)**
- 确保您已将 Mistral API 密钥添加到 `.env` 文件中
- 确保 `.env` 文件与脚本在同一目录中
2. **"Failed to process PDF file"(无法处理 PDF 文件)**
- PDF 可能已损坏或受密码保护
- 尝试使用其他 PDF 文件
3. **Mistral API 错误**
- 检查您的 Mistral API 密钥是否有效
- 确保您在 Mistral API 速率限制范围内
- 验证 PDF 是否在大小/页数限制范围内
## 许可证
MIT 许可证
\ No newline at end of file
import time
import base64
import fitz
import re
import json
from contextlib import asynccontextmanager
from loguru import logger
from fastapi import HTTPException, FastAPI, UploadFile, File
from fastapi.responses import JSONResponse
from mistralai import Mistral
import os
import shutil
from dotenv import load_dotenv
# Load environment variables from .env file
load_dotenv()
app = FastAPI()
temp_dir = "./temp"
# Initialize Mistral client with API key from environment variable
mistral_api_key = os.environ.get("MISTRAL_API_KEY", "")
if not mistral_api_key:
logger.warning("MISTRAL_API_KEY environment variable not set. PDF processing will fail.")
mistral_client = Mistral(api_key=mistral_api_key) if mistral_api_key else None
@asynccontextmanager
async def lifespan(app: FastAPI):
# Create temp directory if it doesn't exist
global temp_dir
if not os.path.exists(temp_dir):
os.makedirs(temp_dir)
print("Application startup, creating temp directory...")
yield
if temp_dir and os.path.exists(temp_dir):
shutil.rmtree(temp_dir)
print("Application shutdown, cleaning up...")
app.router.lifespan_context = lifespan
@app.post("/v1/parse/file")
async def read_file(
file: UploadFile = File(...)):
temp_file_path = None
try:
start_time = time.time()
global temp_dir
os.makedirs(temp_dir, exist_ok=True)
temp_file_path = os.path.join(temp_dir, file.filename)
with open(temp_file_path, "wb") as temp_file:
file_content = await file.read()
temp_file.write(file_content)
# Get page count using PyMuPDF
try:
pdf_document = fitz.open(temp_file_path)
total_pages = pdf_document.page_count
pdf_document.close()
except Exception as e:
logger.error(f"Failed to open PDF file: {str(e)}")
return {
"pages": 0,
"markdown": "",
"error": f"Failed to process PDF file: {str(e)}"
}
if mistral_client is None:
return {
"pages": 0,
"markdown": "",
"error": "MISTRAL_API_KEY environment variable not set."
}
# Step 1: Upload the file to Mistral's servers
logger.info(f"Uploading file {file.filename} to Mistral servers")
with open(temp_file_path, "rb") as f:
try:
uploaded_file = mistral_client.files.upload(
file={
"file_name": file.filename,
"content": f,
},
purpose="ocr"
)
except Exception as e:
error_msg = str(e)
# Try to parse Mistral API error format
try:
error_data = json.loads(error_msg)
if error_data.get("object") == "error":
error_msg = error_data.get("message", error_msg)
except:
pass
return {
"pages": 0,
"markdown": "",
"error": f"Mistral API upload error: {error_msg}"
}
# Step 2: Get a signed URL for the uploaded file
logger.info(f"Getting signed URL for file ID: {uploaded_file.id}")
try:
signed_url = mistral_client.files.get_signed_url(file_id=uploaded_file.id)
except Exception as e:
error_msg = str(e)
# Try to parse Mistral API error format
try:
error_data = json.loads(error_msg)
if error_data.get("object") == "error":
error_msg = error_data.get("message", error_msg)
except:
pass
return {
"pages": 0,
"markdown": "",
"error": f"Mistral API signed URL error: {error_msg}"
}
# Step 3: Process the file using the signed URL
logger.info("Processing file with OCR API")
try:
ocr_response = mistral_client.ocr.process(
model="mistral-ocr-latest",
document={
"type": "document_url",
"document_url": signed_url.url,
},
include_image_base64=True
)
except Exception as e:
error_msg = str(e)
# Try to parse Mistral API error format
try:
error_data = json.loads(error_msg)
if error_data.get("object") == "error":
error_msg = error_data.get("message", error_msg)
except:
pass
return {
"pages": 0,
"markdown": "",
"error": f"Mistral OCR processing error: {error_msg}"
}
# Combine all pages' markdown content
markdown_content = "\n".join(page.markdown for page in ocr_response.pages)
# Create a dictionary to map image filenames to their base64 data
image_map = {}
for page in ocr_response.pages:
for img in page.images:
# Extract the image filename from the image id
img_id = img.id
img_base64 = img.image_base64
# Print a sample of the first image base64 data for debugging
if len(image_map) == 0 and img_base64:
print("Sample image base64 prefix:", img_base64[:50] if len(img_base64) > 50 else img_base64)
print("Does base64 already include prefix?", img_base64.startswith("data:image/"))
# Ensure the base64 data is in the correct format for the upstream system
# If it doesn't already have the prefix, add it
if not img_base64.startswith("data:image/"):
# Assume it's a PNG if we can't determine the type
img_base64 = f"data:image/png;base64,{img_base64}"
# Add both potential formats to the map
image_map[f"{img_id}.jpeg"] = img_base64
image_map[f"{img_id}.png"] = img_base64
image_map[img_id] = img_base64
# Use regex to find all image references in the markdown content
# This will match patterns like ![any-text](any-filename.extension)
image_pattern = r'!\[(.*?)\]\((.*?)\)'
def replace_image_with_base64(match):
alt_text = match.group(1)
img_filename = match.group(2)
# Extract just the filename without path
img_filename_only = os.path.basename(img_filename)
# Check if we have base64 data for this image
if img_filename_only in image_map:
return f"![]({image_map[img_filename_only]})"
else:
# If we don't have base64 data, keep the original reference
logger.warning(f"No base64 data found for image: {img_filename_only}")
return match.group(0)
# Replace all image references with base64 data
markdown_content = re.sub(image_pattern, replace_image_with_base64, markdown_content)
# Clean up the uploaded file from Mistral's servers
try:
logger.info(f"Deleting uploaded file from Mistral servers: {uploaded_file.id}")
mistral_client.files.delete(file_id=uploaded_file.id)
except Exception as e:
logger.warning(f"Failed to delete uploaded file: {e}")
end_time = time.time()
duration = end_time - start_time
print(file.filename + " Total time:", duration)
# Return with format matching client expectations
return {
"pages": total_pages,
"markdown": markdown_content,
"duration": duration # Keep this for logging purposes
}
except Exception as e:
logger.exception(e)
return {
"pages": 0,
"markdown": "",
"error": f"Internal server error: {str(e)}"
}
finally:
if temp_file_path and os.path.exists(temp_file_path):
os.remove(temp_file_path)
if __name__ == "__main__":
import uvicorn
uvicorn.run(app, host="0.0.0.0", port=7231)
# 接入 bge-rerank 重排模型
## 不同模型推荐配置
推荐配置如下:
| 模型名 | 内存 | 显存 | 硬盘空间 | 启动命令 |
| ---------------- | ----- | ----- | -------- | ------------- |
| bge-reranker-base | >=4GB | >=4GB | >=8GB | python app.py |
| bge-reranker-large | >=8GB | >=8GB | >=8GB | python app.py |
| bge-reranker-v2-m3 | >=8GB | >=8GB | >=8GB | python app.py |
## 源码部署
### 1. 安装环境
- Python 3.9, 3.10
- CUDA 11.7
- 科学上网环境
### 2. 下载代码
3 个模型代码分别为:
1. [https://github.com/labring/FastGPT/tree/main/plugins/model/rerank-bge/bge-reranker-base](https://github.com/labring/FastGPT/tree/main/plugins/model/rerank-bge/bge-reranker-base)
2. [https://github.com/labring/FastGPT/tree/main/plugins/model/rerank-bge/bge-reranker-large](https://github.com/labring/FastGPT/tree/main/plugins/model/rerank-bge/bge-reranker-large)
3. [https://github.com/labring/FastGPT/tree/main/plugins/model/rerank-bge/bge-reranker-v2-m3](https://github.com/labring/FastGPT/tree/main/plugins/model/rerank-bge/bge-reranker-v2-m3)
### 3. 安装依赖
```sh
pip install -r requirements.txt
```
### 4. 下载模型
3个模型的 huggingface 仓库地址如下:
1. [https://huggingface.co/BAAI/bge-reranker-base](https://huggingface.co/BAAI/bge-reranker-base)
2. [https://huggingface.co/BAAI/bge-reranker-large](https://huggingface.co/BAAI/bge-reranker-large)
3. [https://huggingface.co/BAAI/bge-reranker-v2-m3](https://huggingface.co/BAAI/bge-reranker-v2-m3)
在对应代码目录下 clone 模型。目录结构:
```
bge-reranker-base/
app.py
Dockerfile
requirements.txt
```
### 5. 运行代码
```bash
python app.py
```
启动成功后应该会显示如下地址:
![](./rerank1.png)
> 这里的 `http://0.0.0.0:6006` 就是请求地址。
## docker 部署
**镜像名分别为:**
1. registry.cn-hangzhou.aliyuncs.com/fastgpt/bge-rerank-base:v0.1
2. registry.cn-hangzhou.aliyuncs.com/fastgpt/bge-rerank-large:v0.1
3. registry.cn-hangzhou.aliyuncs.com/fastgpt/bge-rerank-v2-m3:v0.1
**端口**
6006
**环境变量**
```
ACCESS_TOKEN=访问安全凭证,请求时,Authorization: Bearer ${ACCESS_TOKEN}
```
**运行命令示例**
```sh
# auth token 为mytoken
docker run -d --name reranker -p 6006:6006 -e ACCESS_TOKEN=mytoken --gpus all registry.cn-hangzhou.aliyuncs.com/fastgpt/bge-rerank-base:v0.1
```
**docker-compose.yml示例**
```
version: "3"
services:
reranker:
image: registry.cn-hangzhou.aliyuncs.com/fastgpt/bge-rerank-base:v0.1
container_name: reranker
# GPU运行环境,如果宿主机未安装,将deploy配置隐藏即可
deploy:
resources:
reservations:
devices:
- driver: nvidia
count: all
capabilities: [gpu]
ports:
- 6006:6006
environment:
- ACCESS_TOKEN=mytoken
```
## 接入 FastGPT
参考 [ReRank模型接入](/docs/self-host/config/model/intro/#rerank-模型接入)
FROM pytorch/pytorch:2.0.1-cuda11.7-cudnn8-runtime
# please download the model from https://huggingface.co/BAAI/bge-reranker-base and put it in the same directory as Dockerfile
COPY ./bge-reranker-base ./bge-reranker-base
COPY requirements.txt .
RUN python3 -m pip install -r requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple
COPY app.py Dockerfile .
ENTRYPOINT python3 app.py
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
@Time: 2023/11/7 22:45
@Author: zhidong
@File: reranker.py
@Desc:
"""
import os
import numpy as np
import logging
import uvicorn
import datetime
from fastapi import FastAPI, Security, HTTPException
from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials
from FlagEmbedding import FlagReranker
from pydantic import Field, BaseModel, validator
from typing import Optional, List
app = FastAPI()
security = HTTPBearer()
env_bearer_token = 'ACCESS_TOKEN'
class QADocs(BaseModel):
query: Optional[str]
documents: Optional[List[str]]
class Singleton(type):
def __call__(cls, *args, **kwargs):
if not hasattr(cls, '_instance'):
cls._instance = super().__call__(*args, **kwargs)
return cls._instance
RERANK_MODEL_PATH = os.path.join(os.path.dirname(__file__), "bge-reranker-base")
class ReRanker(metaclass=Singleton):
def __init__(self, model_path):
self.reranker = FlagReranker(model_path, use_fp16=False)
def compute_score(self, pairs: List[List[str]]):
if len(pairs) > 0:
result = self.reranker.compute_score(pairs, normalize=True)
if isinstance(result, float):
result = [result]
return result
else:
return None
class Chat(object):
def __init__(self, rerank_model_path: str = RERANK_MODEL_PATH):
self.reranker = ReRanker(rerank_model_path)
def fit_query_answer_rerank(self, query_docs: QADocs) -> List:
if query_docs is None or len(query_docs.documents) == 0:
return []
pair = [[query_docs.query, doc] for doc in query_docs.documents]
scores = self.reranker.compute_score(pair)
new_docs = []
for index, score in enumerate(scores):
new_docs.append({"index": index, "text": query_docs.documents[index], "score": score})
results = [{"index": documents["index"], "relevance_score": documents["score"]} for documents in list(sorted(new_docs, key=lambda x: x["score"], reverse=True))]
return results
@app.post('/v1/rerank')
async def handle_post_request(docs: QADocs, credentials: HTTPAuthorizationCredentials = Security(security)):
token = credentials.credentials
if env_bearer_token is not None and token != env_bearer_token:
raise HTTPException(status_code=401, detail="Invalid token")
chat = Chat()
try:
results = chat.fit_query_answer_rerank(docs)
return {"results": results}
except Exception as e:
print(f"报错:\n{e}")
return {"error": "重排出错"}
if __name__ == "__main__":
token = os.getenv("ACCESS_TOKEN")
if token is not None:
env_bearer_token = token
try:
uvicorn.run(app, host='0.0.0.0', port=6006)
except Exception as e:
print(f"API启动失败!\n报错:\n{e}")
FROM pytorch/pytorch:2.0.1-cuda11.7-cudnn8-runtime
# please download the model from https://huggingface.co/BAAI/bge-reranker-large and put it in the same directory as Dockerfile
COPY ./bge-reranker-large ./bge-reranker-large
COPY requirements.txt .
RUN python3 -m pip install -r requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple
COPY app.py Dockerfile .
ENTRYPOINT python3 app.py
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
@Time: 2023/11/7 22:45
@Author: zhidong
@File: reranker.py
@Desc:
"""
import os
import numpy as np
import logging
import uvicorn
import datetime
from fastapi import FastAPI, Security, HTTPException
from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials
from FlagEmbedding import FlagReranker
from pydantic import Field, BaseModel, validator
from typing import Optional, List
app = FastAPI()
security = HTTPBearer()
env_bearer_token = 'ACCESS_TOKEN'
class QADocs(BaseModel):
query: Optional[str]
documents: Optional[List[str]]
class Singleton(type):
def __call__(cls, *args, **kwargs):
if not hasattr(cls, '_instance'):
cls._instance = super().__call__(*args, **kwargs)
return cls._instance
RERANK_MODEL_PATH = os.path.join(os.path.dirname(__file__), "bge-reranker-large")
class ReRanker(metaclass=Singleton):
def __init__(self, model_path):
self.reranker = FlagReranker(model_path, use_fp16=False)
def compute_score(self, pairs: List[List[str]]):
if len(pairs) > 0:
result = self.reranker.compute_score(pairs, normalize=True)
if isinstance(result, float):
result = [result]
return result
else:
return None
class Chat(object):
def __init__(self, rerank_model_path: str = RERANK_MODEL_PATH):
self.reranker = ReRanker(rerank_model_path)
def fit_query_answer_rerank(self, query_docs: QADocs) -> List:
if query_docs is None or len(query_docs.documents) == 0:
return []
pair = [[query_docs.query, doc] for doc in query_docs.documents]
scores = self.reranker.compute_score(pair)
new_docs = []
for index, score in enumerate(scores):
new_docs.append({"index": index, "text": query_docs.documents[index], "score": score})
results = [{"index": documents["index"], "relevance_score": documents["score"]} for documents in list(sorted(new_docs, key=lambda x: x["score"], reverse=True))]
return results
@app.post('/v1/rerank')
async def handle_post_request(docs: QADocs, credentials: HTTPAuthorizationCredentials = Security(security)):
token = credentials.credentials
if env_bearer_token is not None and token != env_bearer_token:
raise HTTPException(status_code=401, detail="Invalid token")
chat = Chat()
try:
results = chat.fit_query_answer_rerank(docs)
return {"results": results}
except Exception as e:
print(f"报错:\n{e}")
return {"error": "重排出错"}
if __name__ == "__main__":
token = os.getenv("ACCESS_TOKEN")
if token is not None:
env_bearer_token = token
try:
uvicorn.run(app, host='0.0.0.0', port=6006)
except Exception as e:
print(f"API启动失败!\n报错:\n{e}")
FROM pytorch/pytorch:2.0.1-cuda11.7-cudnn8-runtime
# please download the model from https://huggingface.co/BAAI/bge-reranker-v2-m3 and put it in the same directory as Dockerfile
COPY ./bge-reranker-v2-m3 ./bge-reranker-v2-m3
COPY requirements.txt .
RUN python3 -m pip install -r requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple
COPY app.py Dockerfile .
ENTRYPOINT python3 app.py
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
@Time: 2023/11/7 22:45
@Author: zhidong
@File: reranker.py
@Desc:
"""
import os
import numpy as np
import logging
import uvicorn
import datetime
from fastapi import FastAPI, Security, HTTPException
from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials
from FlagEmbedding import FlagReranker
from pydantic import Field, BaseModel, validator
from typing import Optional, List
app = FastAPI()
security = HTTPBearer()
env_bearer_token = 'ACCESS_TOKEN'
class QADocs(BaseModel):
query: Optional[str]
documents: Optional[List[str]]
class Singleton(type):
def __call__(cls, *args, **kwargs):
if not hasattr(cls, '_instance'):
cls._instance = super().__call__(*args, **kwargs)
return cls._instance
RERANK_MODEL_PATH = os.path.join(os.path.dirname(__file__), "bge-reranker-v2-m3")
class ReRanker(metaclass=Singleton):
def __init__(self, model_path):
self.reranker = FlagReranker(model_path, use_fp16=False)
def compute_score(self, pairs: List[List[str]]):
if len(pairs) > 0:
result = self.reranker.compute_score(pairs, normalize=True)
if isinstance(result, float):
result = [result]
return result
else:
return None
class Chat(object):
def __init__(self, rerank_model_path: str = RERANK_MODEL_PATH):
self.reranker = ReRanker(rerank_model_path)
def fit_query_answer_rerank(self, query_docs: QADocs) -> List:
if query_docs is None or len(query_docs.documents) == 0:
return []
pair = [[query_docs.query, doc] for doc in query_docs.documents]
scores = self.reranker.compute_score(pair)
new_docs = []
for index, score in enumerate(scores):
new_docs.append({"index": index, "text": query_docs.documents[index], "score": score})
results = [{"index": documents["index"], "relevance_score": documents["score"]} for documents in list(sorted(new_docs, key=lambda x: x["score"], reverse=True))]
return results
@app.post('/v1/rerank')
async def handle_post_request(docs: QADocs, credentials: HTTPAuthorizationCredentials = Security(security)):
token = credentials.credentials
if env_bearer_token is not None and token != env_bearer_token:
raise HTTPException(status_code=401, detail="Invalid token")
chat = Chat()
try:
results = chat.fit_query_answer_rerank(docs)
return {"results": results}
except Exception as e:
print(f"报错:\n{e}")
return {"error": "重排出错"}
if __name__ == "__main__":
token = os.getenv("ACCESS_TOKEN")
if token is not None:
env_bearer_token = token
try:
uvicorn.run(app, host='0.0.0.0', port=6006)
except Exception as e:
print(f"API启动失败!\n报错:\n{e}")
#FROM yiminger/sensevoice:latest
FROM pytorch/pytorch:2.1.2-cuda12.1-cudnn8-runtime
COPY ./app /app
WORKDIR /app
#COPY main.py /app/main.py
RUN pip3 install -r requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple
CMD ["python3","main.py"]
Looking in indexes: https://download.pytorch.org/whl/cpu
Collecting torch
Downloading https://download.pytorch.org/whl/cpu/torch-2.3.1%2Bcpu-cp38-cp38-linux_x86_64.whl (190.4 MB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 190.4/190.4 MB 1.3 MB/s eta 0:00:00
Collecting torchaudio
Downloading https://download.pytorch.org/whl/cpu/torchaudio-2.3.1%2Bcpu-cp38-cp38-linux_x86_64.whl (1.7 MB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 1.7/1.7 MB 22.6 MB/s eta 0:00:00
Collecting typing-extensions>=4.8.0
Downloading https://download.pytorch.org/whl/typing_extensions-4.9.0-py3-none-any.whl (32 kB)
Collecting networkx
Downloading https://download.pytorch.org/whl/networkx-3.2.1-py3-none-any.whl (1.6 MB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 1.6/1.6 MB 22.8 MB/s eta 0:00:00
Collecting jinja2
Downloading https://download.pytorch.org/whl/Jinja2-3.1.3-py3-none-any.whl (133 kB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 133.2/133.2 kB 25.7 MB/s eta 0:00:00
Collecting fsspec
Downloading https://download.pytorch.org/whl/fsspec-2024.2.0-py3-none-any.whl (170 kB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 170.9/170.9 kB 24.7 MB/s eta 0:00:00
Collecting sympy
Downloading https://download.pytorch.org/whl/sympy-1.12-py3-none-any.whl (5.7 MB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 5.7/5.7 MB 22.4 MB/s eta 0:00:00
Collecting filelock
Downloading https://download.pytorch.org/whl/filelock-3.13.1-py3-none-any.whl (11 kB)
Collecting MarkupSafe>=2.0
Downloading https://download.pytorch.org/whl/MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (26 kB)
Collecting networkx
Downloading https://download.pytorch.org/whl/networkx-3.0-py3-none-any.whl (2.0 MB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 2.0/2.0 MB 22.7 MB/s eta 0:00:00
Collecting mpmath>=0.19
Downloading https://download.pytorch.org/whl/mpmath-1.3.0-py3-none-any.whl (536 kB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 536.2/536.2 kB 106.6 MB/s eta 0:00:00
Installing collected packages: mpmath, typing-extensions, sympy, networkx, MarkupSafe, fsspec, filelock, jinja2, torch, torchaudio
Successfully installed MarkupSafe-2.1.5 filelock-3.13.1 fsspec-2024.2.0 jinja2-3.1.3 mpmath-1.3.0 networkx-3.0 sympy-1.12 torch-2.3.1+cpu torchaudio-2.3.1+cpu typing-extensions-4.9.0
Revision:master,CreatedAt:1720157464
\ No newline at end of file
---
frameworks:
- Pytorch
license: Apache License 2.0
tasks:
- auto-speech-recognition
#model-type:
##如 gpt、phi、llama、chatglm、baichuan 等
#- gpt
#domain:
##如 nlp、cv、audio、multi-modal
#- nlp
#language:
##语言代码列表 https://help.aliyun.com/document_detail/215387.html?spm=a2c4g.11186623.0.0.9f8d7467kni6Aa
#- cn
#metrics:
##如 CIDEr、Blue、ROUGE 等
#- CIDEr
#tags:
##各种自定义,包括 pretrained、fine-tuned、instruction-tuned、RL-tuned 等训练方法和其他
#- pretrained
#tools:
##如 vllm、fastchat、llamacpp、AdaSeq 等
#- vllm
---
# Highlights
**SenseVoice**专注于高精度多语言语音识别、情感辨识和音频事件检测
- **多语言识别:** 采用超过40万小时数据训练,支持超过50种语言,识别效果上优于Whisper模型。
- **富文本识别:**
- 具备优秀的情感识别,能够在测试数据上达到和超过目前最佳情感识别模型的效果。
- 支持声音事件检测能力,支持音乐、掌声、笑声、哭声、咳嗽、喷嚏等多种常见人机交互事件进行检测。
- **高效推理:** SenseVoice-Small模型采用非自回归端到端框架,推理延迟极低,10s音频推理仅耗时70ms,15倍优于Whisper-Large。
- **微调定制:** 具备便捷的微调脚本与策略,方便用户根据业务场景修复长尾样本问题。
- **服务部署:** 具有完整的服务部署链路,支持多并发请求,支持客户端语言有,python、c++、html、java与c#等。
## <strong>[SenseVoice开源项目介绍]()</strong>
<strong>[SenseVoice]()</strong>开源模型是多语言音频理解模型,具有包括语音识别、语种识别、语音情感识别,声学事件检测能力。
[**github仓库**]()
| [**最新动态**]()
| [**环境安装**]()
# 模型结构图
SenseVoice多语言音频理解模型,支持语音识别、语种识别、语音情感识别、声学事件检测、逆文本正则化等能力,采用工业级数十万小时的标注音频进行模型训练,保证了模型的通用识别效果。模型可以被应用于中文、粤语、英语、日语、韩语音频识别,并输出带有情感和事件的富文本转写结果。
<p align="center">
<img src="fig/sensevoice.png" alt="SenseVoice模型结构" width="1500" />
</p>
SenseVoice-Small是基于非自回归端到端框架模型,为了指定任务,我们在语音特征前添加四个嵌入作为输入传递给编码器:
- LID:用于预测音频语种标签。
- SER:用于预测音频情感标签。
- AED:用于预测音频包含的事件标签。
- ITN:用于指定识别输出文本是否进行逆文本正则化。
# 用法
## 推理
### modelscope pipeline推理
```python
from modelscope.pipelines import pipeline
from modelscope.utils.constant import Tasks
inference_pipeline = pipeline(
task=Tasks.auto_speech_recognition,
model='iic/SenseVoiceSmall',
model_revision="master")
rec_result = inference_pipeline('https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/asr_example_zh.wav')
print(rec_result)
```
### 直接推理
```python
from model import SenseVoiceSmall
model_dir = "iic/SenseVoiceSmall"
m, kwargs = SenseVoiceSmall.from_pretrained(model=model_dir)
res = m.inference(
data_in="https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/asr_example_zh.wav",
language="auto", # "zn", "en", "yue", "ja", "ko", "nospeech"
use_itn=False,
**kwargs,
)
print(res)
```
### 使用funasr推理
```python
from funasr import AutoModel
model_dir = "iic/SenseVoiceSmall"
input_file = (
"https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/asr_example_zh.wav"
)
model = AutoModel(model=model_dir,
vad_model="fsmn-vad",
vad_kwargs={"max_single_segment_time": 30000},
trust_remote_code=True, device="cuda:0")
res = model.generate(
input=input_file,
cache={},
language="auto", # "zn", "en", "yue", "ja", "ko", "nospeech"
use_itn=False,
batch_size_s=0,
)
print(res)
```
funasr版本已经集成了vad模型,支持任意时长音频输入,`batch_size_s`单位为秒。
如果输入均为短音频,并且需要批量化推理,为了加快推理效率,可以移除vad模型,并设置`batch_size`
```python
model = AutoModel(model=model_dir, trust_remote_code=True, device="cuda:0")
res = model.generate(
input=input_file,
cache={},
language="auto", # "zn", "en", "yue", "ja", "ko", "nospeech"
use_itn=False,
batch_size=64,
)
```
更多详细用法,请参考 [文档](https://github.com/modelscope/FunASR/blob/main/docs/tutorial/README.md)
## 模型下载
SDK下载
```bash
#安装ModelScope
pip install modelscope
```
```python
#SDK模型下载
from modelscope import snapshot_download
model_dir = snapshot_download('iic/SenseVoiceSmall')
```
Git下载
```
#Git模型下载
git clone https://www.modelscope.cn/iic/SenseVoiceSmall.git
```
## 服务部署
Undo
# Performance
## 语音识别效果
我们在开源基准数据集(包括 AISHELL-1、AISHELL-2、Wenetspeech、Librispeech和Common Voice)上比较了SenseVoice与Whisper的多语言语音识别性能和推理效率。在中文和粤语识别效果上,SenseVoice-Small模型具有明显的效果优势。
<p align="center">
<img src="fig/asr_results.png" alt="SenseVoice模型在开源测试集上的表现" width="2500" />
</p>
## 情感识别效果
由于目前缺乏被广泛使用的情感识别测试指标和方法,我们在多个测试集的多种指标进行测试,并与近年来Benchmark上的多个结果进行了全面的对比。所选取的测试集同时包含中文/英文两种语言以及表演、影视剧、自然对话等多种风格的数据,在不进行目标数据微调的前提下,SenseVoice能够在测试数据上达到和超过目前最佳情感识别模型的效果。
<p align="center">
<img src="fig/ser_table.png" alt="SenseVoice模型SER效果1" width="1500" />
</p>
同时,我们还在测试集上对多个开源情感识别模型进行对比,结果表明,SenseVoice-Large模型可以在几乎所有数据上都达到了最佳效果,而SenseVoice-Small模型同样可以在多数数据集上取得超越其他开源模型的效果。
<p align="center">
<img src="fig/ser_figure.png" alt="SenseVoice模型SER效果2" width="500" />
</p>
## 事件检测效果
尽管SenseVoice只在语音数据上进行训练,它仍然可以作为事件检测模型进行单独使用。我们在环境音分类ESC-50数据集上与目前业内广泛使用的BEATS与PANN模型的效果进行了对比。SenseVoice模型能够在这些任务上取得较好的效果,但受限于训练数据与训练方式,其事件分类效果专业的事件检测模型相比仍然有一定的差距。
<p align="center">
<img src="fig/aed_figure.png" alt="SenseVoice模型AED效果" width="500" />
</p>
## 推理效率
SenseVoice-Small模型采用非自回归端到端架构,推理延迟极低。在参数量与Whisper-Small模型相当的情况下,比Whisper-Small模型推理速度快7倍,比Whisper-Large模型快17倍。同时SenseVoice-small模型在音频时长增加的情况下,推理耗时也无明显增加。
<p align="center">
<img src="fig/inference.png" alt="SenseVoice模型的推理效率" width="1500" />
</p>
<p style="color: lightgrey;">如果您是本模型的贡献者,我们邀请您根据<a href="https://modelscope.cn/docs/ModelScope%E6%A8%A1%E5%9E%8B%E6%8E%A5%E5%85%A5%E6%B5%81%E7%A8%8B%E6%A6%82%E8%A7%88" style="color: lightgrey; text-decoration: underline;">模型贡献文档</a>,及时完善模型卡片内容。</p>
<Nnet>
<Splice> 560 560
[ 0 ]
<AddShift> 560 560
<LearnRateCoef> 0 [ -8.311879 -8.600912 -9.615928 -10.43595 -11.21292 -11.88333 -12.36243 -12.63706 -12.8818 -12.83066 -12.89103 -12.95666 -13.19763 -13.40598 -13.49113 -13.5546 -13.55639 -13.51915 -13.68284 -13.53289 -13.42107 -13.65519 -13.50713 -13.75251 -13.76715 -13.87408 -13.73109 -13.70412 -13.56073 -13.53488 -13.54895 -13.56228 -13.59408 -13.62047 -13.64198 -13.66109 -13.62669 -13.58297 -13.57387 -13.4739 -13.53063 -13.48348 -13.61047 -13.64716 -13.71546 -13.79184 -13.90614 -14.03098 -14.18205 -14.35881 -14.48419 -14.60172 -14.70591 -14.83362 -14.92122 -15.00622 -15.05122 -15.03119 -14.99028 -14.92302 -14.86927 -14.82691 -14.7972 -14.76909 -14.71356 -14.61277 -14.51696 -14.42252 -14.36405 -14.30451 -14.23161 -14.19851 -14.16633 -14.15649 -14.10504 -13.99518 -13.79562 -13.3996 -12.7767 -11.71208 -8.311879 -8.600912 -9.615928 -10.43595 -11.21292 -11.88333 -12.36243 -12.63706 -12.8818 -12.83066 -12.89103 -12.95666 -13.19763 -13.40598 -13.49113 -13.5546 -13.55639 -13.51915 -13.68284 -13.53289 -13.42107 -13.65519 -13.50713 -13.75251 -13.76715 -13.87408 -13.73109 -13.70412 -13.56073 -13.53488 -13.54895 -13.56228 -13.59408 -13.62047 -13.64198 -13.66109 -13.62669 -13.58297 -13.57387 -13.4739 -13.53063 -13.48348 -13.61047 -13.64716 -13.71546 -13.79184 -13.90614 -14.03098 -14.18205 -14.35881 -14.48419 -14.60172 -14.70591 -14.83362 -14.92122 -15.00622 -15.05122 -15.03119 -14.99028 -14.92302 -14.86927 -14.82691 -14.7972 -14.76909 -14.71356 -14.61277 -14.51696 -14.42252 -14.36405 -14.30451 -14.23161 -14.19851 -14.16633 -14.15649 -14.10504 -13.99518 -13.79562 -13.3996 -12.7767 -11.71208 -8.311879 -8.600912 -9.615928 -10.43595 -11.21292 -11.88333 -12.36243 -12.63706 -12.8818 -12.83066 -12.89103 -12.95666 -13.19763 -13.40598 -13.49113 -13.5546 -13.55639 -13.51915 -13.68284 -13.53289 -13.42107 -13.65519 -13.50713 -13.75251 -13.76715 -13.87408 -13.73109 -13.70412 -13.56073 -13.53488 -13.54895 -13.56228 -13.59408 -13.62047 -13.64198 -13.66109 -13.62669 -13.58297 -13.57387 -13.4739 -13.53063 -13.48348 -13.61047 -13.64716 -13.71546 -13.79184 -13.90614 -14.03098 -14.18205 -14.35881 -14.48419 -14.60172 -14.70591 -14.83362 -14.92122 -15.00622 -15.05122 -15.03119 -14.99028 -14.92302 -14.86927 -14.82691 -14.7972 -14.76909 -14.71356 -14.61277 -14.51696 -14.42252 -14.36405 -14.30451 -14.23161 -14.19851 -14.16633 -14.15649 -14.10504 -13.99518 -13.79562 -13.3996 -12.7767 -11.71208 -8.311879 -8.600912 -9.615928 -10.43595 -11.21292 -11.88333 -12.36243 -12.63706 -12.8818 -12.83066 -12.89103 -12.95666 -13.19763 -13.40598 -13.49113 -13.5546 -13.55639 -13.51915 -13.68284 -13.53289 -13.42107 -13.65519 -13.50713 -13.75251 -13.76715 -13.87408 -13.73109 -13.70412 -13.56073 -13.53488 -13.54895 -13.56228 -13.59408 -13.62047 -13.64198 -13.66109 -13.62669 -13.58297 -13.57387 -13.4739 -13.53063 -13.48348 -13.61047 -13.64716 -13.71546 -13.79184 -13.90614 -14.03098 -14.18205 -14.35881 -14.48419 -14.60172 -14.70591 -14.83362 -14.92122 -15.00622 -15.05122 -15.03119 -14.99028 -14.92302 -14.86927 -14.82691 -14.7972 -14.76909 -14.71356 -14.61277 -14.51696 -14.42252 -14.36405 -14.30451 -14.23161 -14.19851 -14.16633 -14.15649 -14.10504 -13.99518 -13.79562 -13.3996 -12.7767 -11.71208 -8.311879 -8.600912 -9.615928 -10.43595 -11.21292 -11.88333 -12.36243 -12.63706 -12.8818 -12.83066 -12.89103 -12.95666 -13.19763 -13.40598 -13.49113 -13.5546 -13.55639 -13.51915 -13.68284 -13.53289 -13.42107 -13.65519 -13.50713 -13.75251 -13.76715 -13.87408 -13.73109 -13.70412 -13.56073 -13.53488 -13.54895 -13.56228 -13.59408 -13.62047 -13.64198 -13.66109 -13.62669 -13.58297 -13.57387 -13.4739 -13.53063 -13.48348 -13.61047 -13.64716 -13.71546 -13.79184 -13.90614 -14.03098 -14.18205 -14.35881 -14.48419 -14.60172 -14.70591 -14.83362 -14.92122 -15.00622 -15.05122 -15.03119 -14.99028 -14.92302 -14.86927 -14.82691 -14.7972 -14.76909 -14.71356 -14.61277 -14.51696 -14.42252 -14.36405 -14.30451 -14.23161 -14.19851 -14.16633 -14.15649 -14.10504 -13.99518 -13.79562 -13.3996 -12.7767 -11.71208 -8.311879 -8.600912 -9.615928 -10.43595 -11.21292 -11.88333 -12.36243 -12.63706 -12.8818 -12.83066 -12.89103 -12.95666 -13.19763 -13.40598 -13.49113 -13.5546 -13.55639 -13.51915 -13.68284 -13.53289 -13.42107 -13.65519 -13.50713 -13.75251 -13.76715 -13.87408 -13.73109 -13.70412 -13.56073 -13.53488 -13.54895 -13.56228 -13.59408 -13.62047 -13.64198 -13.66109 -13.62669 -13.58297 -13.57387 -13.4739 -13.53063 -13.48348 -13.61047 -13.64716 -13.71546 -13.79184 -13.90614 -14.03098 -14.18205 -14.35881 -14.48419 -14.60172 -14.70591 -14.83362 -14.92122 -15.00622 -15.05122 -15.03119 -14.99028 -14.92302 -14.86927 -14.82691 -14.7972 -14.76909 -14.71356 -14.61277 -14.51696 -14.42252 -14.36405 -14.30451 -14.23161 -14.19851 -14.16633 -14.15649 -14.10504 -13.99518 -13.79562 -13.3996 -12.7767 -11.71208 -8.311879 -8.600912 -9.615928 -10.43595 -11.21292 -11.88333 -12.36243 -12.63706 -12.8818 -12.83066 -12.89103 -12.95666 -13.19763 -13.40598 -13.49113 -13.5546 -13.55639 -13.51915 -13.68284 -13.53289 -13.42107 -13.65519 -13.50713 -13.75251 -13.76715 -13.87408 -13.73109 -13.70412 -13.56073 -13.53488 -13.54895 -13.56228 -13.59408 -13.62047 -13.64198 -13.66109 -13.62669 -13.58297 -13.57387 -13.4739 -13.53063 -13.48348 -13.61047 -13.64716 -13.71546 -13.79184 -13.90614 -14.03098 -14.18205 -14.35881 -14.48419 -14.60172 -14.70591 -14.83362 -14.92122 -15.00622 -15.05122 -15.03119 -14.99028 -14.92302 -14.86927 -14.82691 -14.7972 -14.76909 -14.71356 -14.61277 -14.51696 -14.42252 -14.36405 -14.30451 -14.23161 -14.19851 -14.16633 -14.15649 -14.10504 -13.99518 -13.79562 -13.3996 -12.7767 -11.71208 ]
<Rescale> 560 560
<LearnRateCoef> 0 [ 0.155775 0.154484 0.1527379 0.1518718 0.1506028 0.1489256 0.147067 0.1447061 0.1436307 0.1443568 0.1451849 0.1455157 0.1452821 0.1445717 0.1439195 0.1435867 0.1436018 0.1438781 0.1442086 0.1448844 0.1454756 0.145663 0.146268 0.1467386 0.1472724 0.147664 0.1480913 0.1483739 0.1488841 0.1493636 0.1497088 0.1500379 0.1502916 0.1505389 0.1506787 0.1507102 0.1505992 0.1505445 0.1505938 0.1508133 0.1509569 0.1512396 0.1514625 0.1516195 0.1516156 0.1515561 0.1514966 0.1513976 0.1512612 0.151076 0.1510596 0.1510431 0.151077 0.1511168 0.1511917 0.151023 0.1508045 0.1505885 0.1503493 0.1502373 0.1501726 0.1500762 0.1500065 0.1499782 0.150057 0.1502658 0.150469 0.1505335 0.1505505 0.1505328 0.1504275 0.1502438 0.1499674 0.1497118 0.1494661 0.1493102 0.1493681 0.1495501 0.1499738 0.1509654 0.155775 0.154484 0.1527379 0.1518718 0.1506028 0.1489256 0.147067 0.1447061 0.1436307 0.1443568 0.1451849 0.1455157 0.1452821 0.1445717 0.1439195 0.1435867 0.1436018 0.1438781 0.1442086 0.1448844 0.1454756 0.145663 0.146268 0.1467386 0.1472724 0.147664 0.1480913 0.1483739 0.1488841 0.1493636 0.1497088 0.1500379 0.1502916 0.1505389 0.1506787 0.1507102 0.1505992 0.1505445 0.1505938 0.1508133 0.1509569 0.1512396 0.1514625 0.1516195 0.1516156 0.1515561 0.1514966 0.1513976 0.1512612 0.151076 0.1510596 0.1510431 0.151077 0.1511168 0.1511917 0.151023 0.1508045 0.1505885 0.1503493 0.1502373 0.1501726 0.1500762 0.1500065 0.1499782 0.150057 0.1502658 0.150469 0.1505335 0.1505505 0.1505328 0.1504275 0.1502438 0.1499674 0.1497118 0.1494661 0.1493102 0.1493681 0.1495501 0.1499738 0.1509654 0.155775 0.154484 0.1527379 0.1518718 0.1506028 0.1489256 0.147067 0.1447061 0.1436307 0.1443568 0.1451849 0.1455157 0.1452821 0.1445717 0.1439195 0.1435867 0.1436018 0.1438781 0.1442086 0.1448844 0.1454756 0.145663 0.146268 0.1467386 0.1472724 0.147664 0.1480913 0.1483739 0.1488841 0.1493636 0.1497088 0.1500379 0.1502916 0.1505389 0.1506787 0.1507102 0.1505992 0.1505445 0.1505938 0.1508133 0.1509569 0.1512396 0.1514625 0.1516195 0.1516156 0.1515561 0.1514966 0.1513976 0.1512612 0.151076 0.1510596 0.1510431 0.151077 0.1511168 0.1511917 0.151023 0.1508045 0.1505885 0.1503493 0.1502373 0.1501726 0.1500762 0.1500065 0.1499782 0.150057 0.1502658 0.150469 0.1505335 0.1505505 0.1505328 0.1504275 0.1502438 0.1499674 0.1497118 0.1494661 0.1493102 0.1493681 0.1495501 0.1499738 0.1509654 0.155775 0.154484 0.1527379 0.1518718 0.1506028 0.1489256 0.147067 0.1447061 0.1436307 0.1443568 0.1451849 0.1455157 0.1452821 0.1445717 0.1439195 0.1435867 0.1436018 0.1438781 0.1442086 0.1448844 0.1454756 0.145663 0.146268 0.1467386 0.1472724 0.147664 0.1480913 0.1483739 0.1488841 0.1493636 0.1497088 0.1500379 0.1502916 0.1505389 0.1506787 0.1507102 0.1505992 0.1505445 0.1505938 0.1508133 0.1509569 0.1512396 0.1514625 0.1516195 0.1516156 0.1515561 0.1514966 0.1513976 0.1512612 0.151076 0.1510596 0.1510431 0.151077 0.1511168 0.1511917 0.151023 0.1508045 0.1505885 0.1503493 0.1502373 0.1501726 0.1500762 0.1500065 0.1499782 0.150057 0.1502658 0.150469 0.1505335 0.1505505 0.1505328 0.1504275 0.1502438 0.1499674 0.1497118 0.1494661 0.1493102 0.1493681 0.1495501 0.1499738 0.1509654 0.155775 0.154484 0.1527379 0.1518718 0.1506028 0.1489256 0.147067 0.1447061 0.1436307 0.1443568 0.1451849 0.1455157 0.1452821 0.1445717 0.1439195 0.1435867 0.1436018 0.1438781 0.1442086 0.1448844 0.1454756 0.145663 0.146268 0.1467386 0.1472724 0.147664 0.1480913 0.1483739 0.1488841 0.1493636 0.1497088 0.1500379 0.1502916 0.1505389 0.1506787 0.1507102 0.1505992 0.1505445 0.1505938 0.1508133 0.1509569 0.1512396 0.1514625 0.1516195 0.1516156 0.1515561 0.1514966 0.1513976 0.1512612 0.151076 0.1510596 0.1510431 0.151077 0.1511168 0.1511917 0.151023 0.1508045 0.1505885 0.1503493 0.1502373 0.1501726 0.1500762 0.1500065 0.1499782 0.150057 0.1502658 0.150469 0.1505335 0.1505505 0.1505328 0.1504275 0.1502438 0.1499674 0.1497118 0.1494661 0.1493102 0.1493681 0.1495501 0.1499738 0.1509654 0.155775 0.154484 0.1527379 0.1518718 0.1506028 0.1489256 0.147067 0.1447061 0.1436307 0.1443568 0.1451849 0.1455157 0.1452821 0.1445717 0.1439195 0.1435867 0.1436018 0.1438781 0.1442086 0.1448844 0.1454756 0.145663 0.146268 0.1467386 0.1472724 0.147664 0.1480913 0.1483739 0.1488841 0.1493636 0.1497088 0.1500379 0.1502916 0.1505389 0.1506787 0.1507102 0.1505992 0.1505445 0.1505938 0.1508133 0.1509569 0.1512396 0.1514625 0.1516195 0.1516156 0.1515561 0.1514966 0.1513976 0.1512612 0.151076 0.1510596 0.1510431 0.151077 0.1511168 0.1511917 0.151023 0.1508045 0.1505885 0.1503493 0.1502373 0.1501726 0.1500762 0.1500065 0.1499782 0.150057 0.1502658 0.150469 0.1505335 0.1505505 0.1505328 0.1504275 0.1502438 0.1499674 0.1497118 0.1494661 0.1493102 0.1493681 0.1495501 0.1499738 0.1509654 0.155775 0.154484 0.1527379 0.1518718 0.1506028 0.1489256 0.147067 0.1447061 0.1436307 0.1443568 0.1451849 0.1455157 0.1452821 0.1445717 0.1439195 0.1435867 0.1436018 0.1438781 0.1442086 0.1448844 0.1454756 0.145663 0.146268 0.1467386 0.1472724 0.147664 0.1480913 0.1483739 0.1488841 0.1493636 0.1497088 0.1500379 0.1502916 0.1505389 0.1506787 0.1507102 0.1505992 0.1505445 0.1505938 0.1508133 0.1509569 0.1512396 0.1514625 0.1516195 0.1516156 0.1515561 0.1514966 0.1513976 0.1512612 0.151076 0.1510596 0.1510431 0.151077 0.1511168 0.1511917 0.151023 0.1508045 0.1505885 0.1503493 0.1502373 0.1501726 0.1500762 0.1500065 0.1499782 0.150057 0.1502658 0.150469 0.1505335 0.1505505 0.1505328 0.1504275 0.1502438 0.1499674 0.1497118 0.1494661 0.1493102 0.1493681 0.1495501 0.1499738 0.1509654 ]
</Nnet>
encoder: SenseVoiceEncoderSmall
encoder_conf:
output_size: 512
attention_heads: 4
linear_units: 2048
num_blocks: 50
tp_blocks: 20
dropout_rate: 0.1
positional_dropout_rate: 0.1
attention_dropout_rate: 0.1
input_layer: pe
pos_enc_class: SinusoidalPositionEncoder
normalize_before: true
kernel_size: 11
sanm_shfit: 0
selfattention_layer_type: sanm
model: SenseVoiceSmall
model_conf:
length_normalized_loss: true
sos: 1
eos: 2
ignore_id: -1
tokenizer: SentencepiecesTokenizer
tokenizer_conf:
bpemodel: null
unk_symbol: <unk>
split_with_space: true
frontend: WavFrontend
frontend_conf:
fs: 16000
window: hamming
n_mels: 80
frame_length: 25
frame_shift: 10
lfr_m: 7
lfr_n: 6
cmvn_file: null
dataset: SenseVoiceCTCDataset
dataset_conf:
index_ds: IndexDSJsonl
batch_sampler: EspnetStyleBatchSampler
data_split_num: 32
batch_type: token
batch_size: 14000
max_token_length: 2000
min_token_length: 60
max_source_length: 2000
min_source_length: 60
max_target_length: 200
min_target_length: 0
shuffle: true
num_workers: 4
sos: ${model_conf.sos}
eos: ${model_conf.eos}
IndexDSJsonl: IndexDSJsonl
retry: 20
train_conf:
accum_grad: 1
grad_clip: 5
max_epoch: 20
keep_nbest_models: 10
avg_nbest_model: 10
log_interval: 100
resume: true
validate_interval: 10000
save_checkpoint_interval: 10000
optim: adamw
optim_conf:
lr: 0.00002
scheduler: warmuplr
scheduler_conf:
warmup_steps: 25000
specaug: SpecAugLFR
specaug_conf:
apply_time_warp: false
time_warp_window: 5
time_warp_mode: bicubic
apply_freq_mask: true
freq_mask_width_range:
- 0
- 30
lfr_rate: 6
num_freq_mask: 1
apply_time_mask: true
time_mask_width_range:
- 0
- 12
num_time_mask: 1
{
"framework": "pytorch",
"task" : "auto-speech-recognition",
"model": {"type" : "funasr"},
"pipeline": {"type":"funasr-pipeline"},
"model_name_in_hub": {
"ms":"",
"hf":""},
"file_path_metas": {
"init_param":"model.pt",
"config":"config.yaml",
"tokenizer_conf": {"bpemodel": "chn_jpn_yue_eng_ko_spectok.bpe.model"},
"frontend_conf":{"cmvn_file": "am.mvn"}}
}
\ No newline at end of file
Revision:master,CreatedAt:1707184291
\ No newline at end of file
---
tasks:
- voice-activity-detection
domain:
- audio
model-type:
- VAD model
frameworks:
- pytorch
backbone:
- fsmn
metrics:
- f1_score
license: Apache License 2.0
language:
- cn
tags:
- FunASR
- FSMN
- Alibaba
- Online
datasets:
train:
- 20,000 hour industrial Mandarin task
test:
- 20,000 hour industrial Mandarin task
widgets:
- task: voice-activity-detection
model_revision: v2.0.4
inputs:
- type: audio
name: input
title: 音频
examples:
- name: 1
title: 示例1
inputs:
- name: input
data: git://example/vad_example.wav
inferencespec:
cpu: 1 #CPU数量
memory: 4096
---
# FSMN-Monophone VAD 模型介绍
[//]: # (FSMN-Monophone VAD 模型)
## Highlight
- 16k中文通用VAD模型:可用于检测长语音片段中有效语音的起止时间点。
- 基于[Paraformer-large长音频模型](https://www.modelscope.cn/models/damo/speech_paraformer-large-vad-punc_asr_nat-zh-cn-16k-common-vocab8404-pytorch/summary)场景的使用
- 基于[FunASR框架](https://github.com/alibaba-damo-academy/FunASR),可进行ASR,VAD,[中文标点](https://www.modelscope.cn/models/damo/punc_ct-transformer_zh-cn-common-vocab272727-pytorch/summary)的自由组合
- 基于音频数据的有效语音片段起止时间点检测
## <strong>[FunASR开源项目介绍](https://github.com/alibaba-damo-academy/FunASR)</strong>
<strong>[FunASR](https://github.com/alibaba-damo-academy/FunASR)</strong>希望在语音识别的学术研究和工业应用之间架起一座桥梁。通过发布工业级语音识别模型的训练和微调,研究人员和开发人员可以更方便地进行语音识别模型的研究和生产,并推动语音识别生态的发展。让语音识别更有趣!
[**github仓库**](https://github.com/alibaba-damo-academy/FunASR)
| [**最新动态**](https://github.com/alibaba-damo-academy/FunASR#whats-new)
| [**环境安装**](https://github.com/alibaba-damo-academy/FunASR#installation)
| [**服务部署**](https://www.funasr.com)
| [**模型库**](https://github.com/alibaba-damo-academy/FunASR/tree/main/model_zoo)
| [**联系我们**](https://github.com/alibaba-damo-academy/FunASR#contact)
## 模型原理介绍
FSMN-Monophone VAD是达摩院语音团队提出的高效语音端点检测模型,用于检测输入音频中有效语音的起止时间点信息,并将检测出来的有效音频片段输入识别引擎进行识别,减少无效语音带来的识别错误。
<p align="center">
<img src="fig/struct.png" alt="VAD模型结构" width="500" />
FSMN-Monophone VAD模型结构如上图所示:模型结构层面,FSMN模型结构建模时可考虑上下文信息,训练和推理速度快,且时延可控;同时根据VAD模型size以及低时延的要求,对FSMN的网络结构、右看帧数进行了适配。在建模单元层面,speech信息比较丰富,仅用单类来表征学习能力有限,我们将单一speech类升级为Monophone。建模单元细分,可以避免参数平均,抽象学习能力增强,区分性更好。
## 基于ModelScope进行推理
- 推理支持音频格式如下:
- wav文件路径,例如:data/test/audios/vad_example.wav
- wav文件url,例如:https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/vad_example.wav
- wav二进制数据,格式bytes,例如:用户直接从文件里读出bytes数据或者是麦克风录出bytes数据。
- 已解析的audio音频,例如:audio, rate = soundfile.read("vad_example_zh.wav"),类型为numpy.ndarray或者torch.Tensor。
- wav.scp文件,需符合如下要求:
```sh
cat wav.scp
vad_example1 data/test/audios/vad_example1.wav
vad_example2 data/test/audios/vad_example2.wav
...
```
- 若输入格式wav文件url,api调用方式可参考如下范例:
```python
from modelscope.pipelines import pipeline
from modelscope.utils.constant import Tasks
inference_pipeline = pipeline(
task=Tasks.voice_activity_detection,
model='iic/speech_fsmn_vad_zh-cn-16k-common-pytorch',
model_revision="v2.0.4",
)
segments_result = inference_pipeline(input='https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/vad_example.wav')
print(segments_result)
```
- 输入音频为pcm格式,调用api时需要传入音频采样率参数fs,例如:
```python
segments_result = inference_pipeline(input='https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/vad_example.pcm', fs=16000)
```
- 若输入格式为文件wav.scp(注:文件名需要以.scp结尾),可添加 output_dir 参数将识别结果写入文件中,参考示例如下:
```python
inference_pipeline(input="wav.scp", output_dir='./output_dir')
```
识别结果输出路径结构如下:
```sh
tree output_dir/
output_dir/
└── 1best_recog
└── text
1 directory, 1 files
```
text:VAD检测语音起止时间点结果文件(单位:ms)
- 若输入音频为已解析的audio音频,api调用方式可参考如下范例:
```python
import soundfile
waveform, sample_rate = soundfile.read("vad_example_zh.wav")
segments_result = inference_pipeline(input=waveform)
print(segments_result)
```
- VAD常用参数调整说明(参考:vad.yaml文件):
- max_end_silence_time:尾部连续检测到多长时间静音进行尾点判停,参数范围500ms~6000ms,默认值800ms(该值过低容易出现语音提前截断的情况)。
- speech_noise_thres:speech的得分减去noise的得分大于此值则判断为speech,参数范围:(-1,1)
- 取值越趋于-1,噪音被误判定为语音的概率越大,FA越高
- 取值越趋于+1,语音被误判定为噪音的概率越大,Pmiss越高
- 通常情况下,该值会根据当前模型在长语音测试集上的效果取balance
## 基于FunASR进行推理
下面为快速上手教程,测试音频([中文](https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/vad_example.wav),[英文](https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/asr_example_en.wav))
### 可执行命令行
在命令行终端执行:
```shell
funasr ++model=paraformer-zh ++vad_model="fsmn-vad" ++punc_model="ct-punc" ++input=vad_example.wav
```
注:支持单条音频文件识别,也支持文件列表,列表为kaldi风格wav.scp:`wav_id wav_path`
### python示例
#### 非实时语音识别
```python
from funasr import AutoModel
# paraformer-zh is a multi-functional asr model
# use vad, punc, spk or not as you need
model = AutoModel(model="paraformer-zh", model_revision="v2.0.4",
vad_model="fsmn-vad", vad_model_revision="v2.0.4",
punc_model="ct-punc-c", punc_model_revision="v2.0.4",
# spk_model="cam++", spk_model_revision="v2.0.2",
)
res = model.generate(input=f"{model.model_path}/example/asr_example.wav",
batch_size_s=300,
hotword='魔搭')
print(res)
```
注:`model_hub`:表示模型仓库,`ms`为选择modelscope下载,`hf`为选择huggingface下载。
#### 实时语音识别
```python
from funasr import AutoModel
chunk_size = [0, 10, 5] #[0, 10, 5] 600ms, [0, 8, 4] 480ms
encoder_chunk_look_back = 4 #number of chunks to lookback for encoder self-attention
decoder_chunk_look_back = 1 #number of encoder chunks to lookback for decoder cross-attention
model = AutoModel(model="paraformer-zh-streaming", model_revision="v2.0.4")
import soundfile
import os
wav_file = os.path.join(model.model_path, "example/asr_example.wav")
speech, sample_rate = soundfile.read(wav_file)
chunk_stride = chunk_size[1] * 960 # 600ms
cache = {}
total_chunk_num = int(len((speech)-1)/chunk_stride+1)
for i in range(total_chunk_num):
speech_chunk = speech[i*chunk_stride:(i+1)*chunk_stride]
is_final = i == total_chunk_num - 1
res = model.generate(input=speech_chunk, cache=cache, is_final=is_final, chunk_size=chunk_size, encoder_chunk_look_back=encoder_chunk_look_back, decoder_chunk_look_back=decoder_chunk_look_back)
print(res)
```
注:`chunk_size`为流式延时配置,`[0,10,5]`表示上屏实时出字粒度为`10*60=600ms`,未来信息为`5*60=300ms`。每次推理输入为`600ms`(采样点数为`16000*0.6=960`),输出为对应文字,最后一个语音片段输入需要设置`is_final=True`来强制输出最后一个字。
#### 语音端点检测(非实时)
```python
from funasr import AutoModel
model = AutoModel(model="fsmn-vad", model_revision="v2.0.4")
wav_file = f"{model.model_path}/example/asr_example.wav"
res = model.generate(input=wav_file)
print(res)
```
#### 语音端点检测(实时)
```python
from funasr import AutoModel
chunk_size = 200 # ms
model = AutoModel(model="fsmn-vad", model_revision="v2.0.4")
import soundfile
wav_file = f"{model.model_path}/example/vad_example.wav"
speech, sample_rate = soundfile.read(wav_file)
chunk_stride = int(chunk_size * sample_rate / 1000)
cache = {}
total_chunk_num = int(len((speech)-1)/chunk_stride+1)
for i in range(total_chunk_num):
speech_chunk = speech[i*chunk_stride:(i+1)*chunk_stride]
is_final = i == total_chunk_num - 1
res = model.generate(input=speech_chunk, cache=cache, is_final=is_final, chunk_size=chunk_size)
if len(res[0]["value"]):
print(res)
```
#### 标点恢复
```python
from funasr import AutoModel
model = AutoModel(model="ct-punc", model_revision="v2.0.4")
res = model.generate(input="那今天的会就到这里吧 happy new year 明年见")
print(res)
```
#### 时间戳预测
```python
from funasr import AutoModel
model = AutoModel(model="fa-zh", model_revision="v2.0.4")
wav_file = f"{model.model_path}/example/asr_example.wav"
text_file = f"{model.model_path}/example/text.txt"
res = model.generate(input=(wav_file, text_file), data_type=("sound", "text"))
print(res)
```
更多详细用法([示例](https://github.com/alibaba-damo-academy/FunASR/tree/main/examples/industrial_data_pretraining))
## 微调
详细用法([示例](https://github.com/alibaba-damo-academy/FunASR/tree/main/examples/industrial_data_pretraining))
## 使用方式以及适用范围
运行范围
- 支持Linux-x86_64、Mac和Windows运行。
使用方式
- 直接推理:可以直接对长语音数据进行计算,有效语音片段的起止时间点信息(单位:ms)。
## 相关论文以及引用信息
```BibTeX
@inproceedings{zhang2018deep,
title={Deep-FSMN for large vocabulary continuous speech recognition},
author={Zhang, Shiliang and Lei, Ming and Yan, Zhijie and Dai, Lirong},
booktitle={2018 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)},
pages={5869--5873},
year={2018},
organization={IEEE}
}
```
<Nnet>
<Splice> 400 400
[ 0 ]
<AddShift> 400 400
<LearnRateCoef> 0 [ -8.311879 -8.600912 -9.615928 -10.43595 -11.21292 -11.88333 -12.36243 -12.63706 -12.8818 -12.83066 -12.89103 -12.95666 -13.19763 -13.40598 -13.49113 -13.5546 -13.55639 -13.51915 -13.68284 -13.53289 -13.42107 -13.65519 -13.50713 -13.75251 -13.76715 -13.87408 -13.73109 -13.70412 -13.56073 -13.53488 -13.54895 -13.56228 -13.59408 -13.62047 -13.64198 -13.66109 -13.62669 -13.58297 -13.57387 -13.4739 -13.53063 -13.48348 -13.61047 -13.64716 -13.71546 -13.79184 -13.90614 -14.03098 -14.18205 -14.35881 -14.48419 -14.60172 -14.70591 -14.83362 -14.92122 -15.00622 -15.05122 -15.03119 -14.99028 -14.92302 -14.86927 -14.82691 -14.7972 -14.76909 -14.71356 -14.61277 -14.51696 -14.42252 -14.36405 -14.30451 -14.23161 -14.19851 -14.16633 -14.15649 -14.10504 -13.99518 -13.79562 -13.3996 -12.7767 -11.71208 -8.311879 -8.600912 -9.615928 -10.43595 -11.21292 -11.88333 -12.36243 -12.63706 -12.8818 -12.83066 -12.89103 -12.95666 -13.19763 -13.40598 -13.49113 -13.5546 -13.55639 -13.51915 -13.68284 -13.53289 -13.42107 -13.65519 -13.50713 -13.75251 -13.76715 -13.87408 -13.73109 -13.70412 -13.56073 -13.53488 -13.54895 -13.56228 -13.59408 -13.62047 -13.64198 -13.66109 -13.62669 -13.58297 -13.57387 -13.4739 -13.53063 -13.48348 -13.61047 -13.64716 -13.71546 -13.79184 -13.90614 -14.03098 -14.18205 -14.35881 -14.48419 -14.60172 -14.70591 -14.83362 -14.92122 -15.00622 -15.05122 -15.03119 -14.99028 -14.92302 -14.86927 -14.82691 -14.7972 -14.76909 -14.71356 -14.61277 -14.51696 -14.42252 -14.36405 -14.30451 -14.23161 -14.19851 -14.16633 -14.15649 -14.10504 -13.99518 -13.79562 -13.3996 -12.7767 -11.71208 -8.311879 -8.600912 -9.615928 -10.43595 -11.21292 -11.88333 -12.36243 -12.63706 -12.8818 -12.83066 -12.89103 -12.95666 -13.19763 -13.40598 -13.49113 -13.5546 -13.55639 -13.51915 -13.68284 -13.53289 -13.42107 -13.65519 -13.50713 -13.75251 -13.76715 -13.87408 -13.73109 -13.70412 -13.56073 -13.53488 -13.54895 -13.56228 -13.59408 -13.62047 -13.64198 -13.66109 -13.62669 -13.58297 -13.57387 -13.4739 -13.53063 -13.48348 -13.61047 -13.64716 -13.71546 -13.79184 -13.90614 -14.03098 -14.18205 -14.35881 -14.48419 -14.60172 -14.70591 -14.83362 -14.92122 -15.00622 -15.05122 -15.03119 -14.99028 -14.92302 -14.86927 -14.82691 -14.7972 -14.76909 -14.71356 -14.61277 -14.51696 -14.42252 -14.36405 -14.30451 -14.23161 -14.19851 -14.16633 -14.15649 -14.10504 -13.99518 -13.79562 -13.3996 -12.7767 -11.71208 -8.311879 -8.600912 -9.615928 -10.43595 -11.21292 -11.88333 -12.36243 -12.63706 -12.8818 -12.83066 -12.89103 -12.95666 -13.19763 -13.40598 -13.49113 -13.5546 -13.55639 -13.51915 -13.68284 -13.53289 -13.42107 -13.65519 -13.50713 -13.75251 -13.76715 -13.87408 -13.73109 -13.70412 -13.56073 -13.53488 -13.54895 -13.56228 -13.59408 -13.62047 -13.64198 -13.66109 -13.62669 -13.58297 -13.57387 -13.4739 -13.53063 -13.48348 -13.61047 -13.64716 -13.71546 -13.79184 -13.90614 -14.03098 -14.18205 -14.35881 -14.48419 -14.60172 -14.70591 -14.83362 -14.92122 -15.00622 -15.05122 -15.03119 -14.99028 -14.92302 -14.86927 -14.82691 -14.7972 -14.76909 -14.71356 -14.61277 -14.51696 -14.42252 -14.36405 -14.30451 -14.23161 -14.19851 -14.16633 -14.15649 -14.10504 -13.99518 -13.79562 -13.3996 -12.7767 -11.71208 -8.311879 -8.600912 -9.615928 -10.43595 -11.21292 -11.88333 -12.36243 -12.63706 -12.8818 -12.83066 -12.89103 -12.95666 -13.19763 -13.40598 -13.49113 -13.5546 -13.55639 -13.51915 -13.68284 -13.53289 -13.42107 -13.65519 -13.50713 -13.75251 -13.76715 -13.87408 -13.73109 -13.70412 -13.56073 -13.53488 -13.54895 -13.56228 -13.59408 -13.62047 -13.64198 -13.66109 -13.62669 -13.58297 -13.57387 -13.4739 -13.53063 -13.48348 -13.61047 -13.64716 -13.71546 -13.79184 -13.90614 -14.03098 -14.18205 -14.35881 -14.48419 -14.60172 -14.70591 -14.83362 -14.92122 -15.00622 -15.05122 -15.03119 -14.99028 -14.92302 -14.86927 -14.82691 -14.7972 -14.76909 -14.71356 -14.61277 -14.51696 -14.42252 -14.36405 -14.30451 -14.23161 -14.19851 -14.16633 -14.15649 -14.10504 -13.99518 -13.79562 -13.3996 -12.7767 -11.71208 ]
<Rescale> 400 400
<LearnRateCoef> 0 [ 0.155775 0.154484 0.1527379 0.1518718 0.1506028 0.1489256 0.147067 0.1447061 0.1436307 0.1443568 0.1451849 0.1455157 0.1452821 0.1445717 0.1439195 0.1435867 0.1436018 0.1438781 0.1442086 0.1448844 0.1454756 0.145663 0.146268 0.1467386 0.1472724 0.147664 0.1480913 0.1483739 0.1488841 0.1493636 0.1497088 0.1500379 0.1502916 0.1505389 0.1506787 0.1507102 0.1505992 0.1505445 0.1505938 0.1508133 0.1509569 0.1512396 0.1514625 0.1516195 0.1516156 0.1515561 0.1514966 0.1513976 0.1512612 0.151076 0.1510596 0.1510431 0.151077 0.1511168 0.1511917 0.151023 0.1508045 0.1505885 0.1503493 0.1502373 0.1501726 0.1500762 0.1500065 0.1499782 0.150057 0.1502658 0.150469 0.1505335 0.1505505 0.1505328 0.1504275 0.1502438 0.1499674 0.1497118 0.1494661 0.1493102 0.1493681 0.1495501 0.1499738 0.1509654 0.155775 0.154484 0.1527379 0.1518718 0.1506028 0.1489256 0.147067 0.1447061 0.1436307 0.1443568 0.1451849 0.1455157 0.1452821 0.1445717 0.1439195 0.1435867 0.1436018 0.1438781 0.1442086 0.1448844 0.1454756 0.145663 0.146268 0.1467386 0.1472724 0.147664 0.1480913 0.1483739 0.1488841 0.1493636 0.1497088 0.1500379 0.1502916 0.1505389 0.1506787 0.1507102 0.1505992 0.1505445 0.1505938 0.1508133 0.1509569 0.1512396 0.1514625 0.1516195 0.1516156 0.1515561 0.1514966 0.1513976 0.1512612 0.151076 0.1510596 0.1510431 0.151077 0.1511168 0.1511917 0.151023 0.1508045 0.1505885 0.1503493 0.1502373 0.1501726 0.1500762 0.1500065 0.1499782 0.150057 0.1502658 0.150469 0.1505335 0.1505505 0.1505328 0.1504275 0.1502438 0.1499674 0.1497118 0.1494661 0.1493102 0.1493681 0.1495501 0.1499738 0.1509654 0.155775 0.154484 0.1527379 0.1518718 0.1506028 0.1489256 0.147067 0.1447061 0.1436307 0.1443568 0.1451849 0.1455157 0.1452821 0.1445717 0.1439195 0.1435867 0.1436018 0.1438781 0.1442086 0.1448844 0.1454756 0.145663 0.146268 0.1467386 0.1472724 0.147664 0.1480913 0.1483739 0.1488841 0.1493636 0.1497088 0.1500379 0.1502916 0.1505389 0.1506787 0.1507102 0.1505992 0.1505445 0.1505938 0.1508133 0.1509569 0.1512396 0.1514625 0.1516195 0.1516156 0.1515561 0.1514966 0.1513976 0.1512612 0.151076 0.1510596 0.1510431 0.151077 0.1511168 0.1511917 0.151023 0.1508045 0.1505885 0.1503493 0.1502373 0.1501726 0.1500762 0.1500065 0.1499782 0.150057 0.1502658 0.150469 0.1505335 0.1505505 0.1505328 0.1504275 0.1502438 0.1499674 0.1497118 0.1494661 0.1493102 0.1493681 0.1495501 0.1499738 0.1509654 0.155775 0.154484 0.1527379 0.1518718 0.1506028 0.1489256 0.147067 0.1447061 0.1436307 0.1443568 0.1451849 0.1455157 0.1452821 0.1445717 0.1439195 0.1435867 0.1436018 0.1438781 0.1442086 0.1448844 0.1454756 0.145663 0.146268 0.1467386 0.1472724 0.147664 0.1480913 0.1483739 0.1488841 0.1493636 0.1497088 0.1500379 0.1502916 0.1505389 0.1506787 0.1507102 0.1505992 0.1505445 0.1505938 0.1508133 0.1509569 0.1512396 0.1514625 0.1516195 0.1516156 0.1515561 0.1514966 0.1513976 0.1512612 0.151076 0.1510596 0.1510431 0.151077 0.1511168 0.1511917 0.151023 0.1508045 0.1505885 0.1503493 0.1502373 0.1501726 0.1500762 0.1500065 0.1499782 0.150057 0.1502658 0.150469 0.1505335 0.1505505 0.1505328 0.1504275 0.1502438 0.1499674 0.1497118 0.1494661 0.1493102 0.1493681 0.1495501 0.1499738 0.1509654 0.155775 0.154484 0.1527379 0.1518718 0.1506028 0.1489256 0.147067 0.1447061 0.1436307 0.1443568 0.1451849 0.1455157 0.1452821 0.1445717 0.1439195 0.1435867 0.1436018 0.1438781 0.1442086 0.1448844 0.1454756 0.145663 0.146268 0.1467386 0.1472724 0.147664 0.1480913 0.1483739 0.1488841 0.1493636 0.1497088 0.1500379 0.1502916 0.1505389 0.1506787 0.1507102 0.1505992 0.1505445 0.1505938 0.1508133 0.1509569 0.1512396 0.1514625 0.1516195 0.1516156 0.1515561 0.1514966 0.1513976 0.1512612 0.151076 0.1510596 0.1510431 0.151077 0.1511168 0.1511917 0.151023 0.1508045 0.1505885 0.1503493 0.1502373 0.1501726 0.1500762 0.1500065 0.1499782 0.150057 0.1502658 0.150469 0.1505335 0.1505505 0.1505328 0.1504275 0.1502438 0.1499674 0.1497118 0.1494661 0.1493102 0.1493681 0.1495501 0.1499738 0.1509654 ]
</Nnet>
\ No newline at end of file
frontend: WavFrontendOnline
frontend_conf:
fs: 16000
window: hamming
n_mels: 80
frame_length: 25
frame_shift: 10
dither: 0.0
lfr_m: 5
lfr_n: 1
model: FsmnVADStreaming
model_conf:
sample_rate: 16000
detect_mode: 1
snr_mode: 0
max_end_silence_time: 800
max_start_silence_time: 3000
do_start_point_detection: True
do_end_point_detection: True
window_size_ms: 200
sil_to_speech_time_thres: 150
speech_to_sil_time_thres: 150
speech_2_noise_ratio: 1.0
do_extend: 1
lookback_time_start_point: 200
lookahead_time_end_point: 100
max_single_segment_time: 60000
snr_thres: -100.0
noise_frame_num_used_for_snr: 100
decibel_thres: -100.0
speech_noise_thres: 0.6
fe_prior_thres: 0.0001
silence_pdf_num: 1
sil_pdf_ids: [0]
speech_noise_thresh_low: -0.1
speech_noise_thresh_high: 0.3
output_frame_probs: False
frame_in_ms: 10
frame_length_ms: 25
encoder: FSMN
encoder_conf:
input_dim: 400
input_affine_dim: 140
fsmn_layers: 4
linear_dim: 250
proj_dim: 128
lorder: 20
rorder: 0
lstride: 1
rstride: 0
output_affine_dim: 140
output_dim: 248
{
"framework": "pytorch",
"task" : "voice-activity-detection",
"pipeline": {"type":"funasr-pipeline"},
"model": {"type" : "funasr"},
"file_path_metas": {
"init_param":"model.pt",
"config":"config.yaml",
"frontend_conf":{"cmvn_file": "am.mvn"}},
"model_name_in_hub": {
"ms":"iic/speech_fsmn_vad_zh-cn-16k-common-pytorch",
"hf":""}
}
\ No newline at end of file
# -*- coding: utf-8 -*-
from fastapi import FastAPI, File, UploadFile, HTTPException
from pydantic import BaseModel, HttpUrl, ValidationError
from typing import List
from funasr import AutoModel
from funasr.utils.postprocess_utils import rich_transcription_postprocess
import uuid
import os
app = FastAPI()
# 数据验证模型
class UrlInput(BaseModel):
audio_urls: List[HttpUrl]
# 模型加载
model_dir = "iic/SenseVoiceSmall"
# 快速预测
# model = AutoModel(model=model_dir, trust_remote_code=True, device="cpu")
# 准确预测
model = AutoModel(
model=model_dir,
vad_model="fsmn-vad",
vad_kwargs={"max_single_segment_time": 30000},
trust_remote_code=True,
device="cuda:0",
)
@app.post("/upload-url/")
async def upload_url(data: UrlInput):
try:
results = []
for url in data.audio_urls:
res = model.generate(
input=str(url), # 将 URL 转换为字符串
cache={},
language=language,
use_itn=False,
batch_size=batch_size,
)
data = rich_transcription_postprocess(res[0]["text"])
results.append(data)
return {"message": "URL input processed successfully", "results": results}
except ValidationError as e:
raise HTTPException(status_code=400, detail=e.errors())
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))
@app.post("/v1/audio/transcriptions")
async def upload_file(file: UploadFile = File(...)):
try:
#for file in files:
if not file.content_type.startswith("audio/"):
raise HTTPException(status_code=400, detail="Invalid file type")
# 读取文件为 bytes
#audio_bytes = await file.read()
unique_filename = str(uuid.uuid4()) + ".mp3"
# 保存上传的音频文件
audio_file_path = os.path.join("/tmp", unique_filename)
with open(audio_file_path, "wb") as audio_file:
audio_file.write(await file.read())
# 直接将文件对象传递给模型
res = model.generate(
input=audio_file_path,
cache={},
language=language,
use_itn=True,
batch_size=batch_size,
merge_vad=True, #
merge_length_s=15,
)
data = rich_transcription_postprocess(res[0]["text"])
return {"message": "File inputs processed successfully", "text": data}
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))
if __name__ == "__main__":
batch_size = 60
language = "auto"
import uvicorn
uvicorn.run(app, host="0.0.0.0", port=8000)
import time
import torch
from torch import nn
import torch.nn.functional as F
from typing import Iterable, Optional
from funasr.register import tables
from funasr.models.ctc.ctc import CTC
from funasr.utils.datadir_writer import DatadirWriter
from funasr.models.paraformer.search import Hypothesis
from funasr.train_utils.device_funcs import force_gatherable
from funasr.losses.label_smoothing_loss import LabelSmoothingLoss
from funasr.metrics.compute_acc import compute_accuracy, th_accuracy
from funasr.utils.load_utils import load_audio_text_image_video, extract_fbank
class SinusoidalPositionEncoder(torch.nn.Module):
""" """
def __int__(self, d_model=80, dropout_rate=0.1):
pass
def encode(
self, positions: torch.Tensor = None, depth: int = None, dtype: torch.dtype = torch.float32
):
batch_size = positions.size(0)
positions = positions.type(dtype)
device = positions.device
log_timescale_increment = torch.log(torch.tensor([10000], dtype=dtype, device=device)) / (
depth / 2 - 1
)
inv_timescales = torch.exp(
torch.arange(depth / 2, device=device).type(dtype) * (-log_timescale_increment)
)
inv_timescales = torch.reshape(inv_timescales, [batch_size, -1])
scaled_time = torch.reshape(positions, [1, -1, 1]) * torch.reshape(
inv_timescales, [1, 1, -1]
)
encoding = torch.cat([torch.sin(scaled_time), torch.cos(scaled_time)], dim=2)
return encoding.type(dtype)
def forward(self, x):
batch_size, timesteps, input_dim = x.size()
positions = torch.arange(1, timesteps + 1, device=x.device)[None, :]
position_encoding = self.encode(positions, input_dim, x.dtype).to(x.device)
return x + position_encoding
class PositionwiseFeedForward(torch.nn.Module):
"""Positionwise feed forward layer.
Args:
idim (int): Input dimenstion.
hidden_units (int): The number of hidden units.
dropout_rate (float): Dropout rate.
"""
def __init__(self, idim, hidden_units, dropout_rate, activation=torch.nn.ReLU()):
"""Construct an PositionwiseFeedForward object."""
super(PositionwiseFeedForward, self).__init__()
self.w_1 = torch.nn.Linear(idim, hidden_units)
self.w_2 = torch.nn.Linear(hidden_units, idim)
self.dropout = torch.nn.Dropout(dropout_rate)
self.activation = activation
def forward(self, x):
"""Forward function."""
return self.w_2(self.dropout(self.activation(self.w_1(x))))
class MultiHeadedAttentionSANM(nn.Module):
"""Multi-Head Attention layer.
Args:
n_head (int): The number of heads.
n_feat (int): The number of features.
dropout_rate (float): Dropout rate.
"""
def __init__(
self,
n_head,
in_feat,
n_feat,
dropout_rate,
kernel_size,
sanm_shfit=0,
lora_list=None,
lora_rank=8,
lora_alpha=16,
lora_dropout=0.1,
):
"""Construct an MultiHeadedAttention object."""
super().__init__()
assert n_feat % n_head == 0
# We assume d_v always equals d_k
self.d_k = n_feat // n_head
self.h = n_head
# self.linear_q = nn.Linear(n_feat, n_feat)
# self.linear_k = nn.Linear(n_feat, n_feat)
# self.linear_v = nn.Linear(n_feat, n_feat)
self.linear_out = nn.Linear(n_feat, n_feat)
self.linear_q_k_v = nn.Linear(in_feat, n_feat * 3)
self.attn = None
self.dropout = nn.Dropout(p=dropout_rate)
self.fsmn_block = nn.Conv1d(
n_feat, n_feat, kernel_size, stride=1, padding=0, groups=n_feat, bias=False
)
# padding
left_padding = (kernel_size - 1) // 2
if sanm_shfit > 0:
left_padding = left_padding + sanm_shfit
right_padding = kernel_size - 1 - left_padding
self.pad_fn = nn.ConstantPad1d((left_padding, right_padding), 0.0)
def forward_fsmn(self, inputs, mask, mask_shfit_chunk=None):
b, t, d = inputs.size()
if mask is not None:
mask = torch.reshape(mask, (b, -1, 1))
if mask_shfit_chunk is not None:
mask = mask * mask_shfit_chunk
inputs = inputs * mask
x = inputs.transpose(1, 2)
x = self.pad_fn(x)
x = self.fsmn_block(x)
x = x.transpose(1, 2)
x += inputs
x = self.dropout(x)
if mask is not None:
x = x * mask
return x
def forward_qkv(self, x):
"""Transform query, key and value.
Args:
query (torch.Tensor): Query tensor (#batch, time1, size).
key (torch.Tensor): Key tensor (#batch, time2, size).
value (torch.Tensor): Value tensor (#batch, time2, size).
Returns:
torch.Tensor: Transformed query tensor (#batch, n_head, time1, d_k).
torch.Tensor: Transformed key tensor (#batch, n_head, time2, d_k).
torch.Tensor: Transformed value tensor (#batch, n_head, time2, d_k).
"""
b, t, d = x.size()
q_k_v = self.linear_q_k_v(x)
q, k, v = torch.split(q_k_v, int(self.h * self.d_k), dim=-1)
q_h = torch.reshape(q, (b, t, self.h, self.d_k)).transpose(
1, 2
) # (batch, head, time1, d_k)
k_h = torch.reshape(k, (b, t, self.h, self.d_k)).transpose(
1, 2
) # (batch, head, time2, d_k)
v_h = torch.reshape(v, (b, t, self.h, self.d_k)).transpose(
1, 2
) # (batch, head, time2, d_k)
return q_h, k_h, v_h, v
def forward_attention(self, value, scores, mask, mask_att_chunk_encoder=None):
"""Compute attention context vector.
Args:
value (torch.Tensor): Transformed value (#batch, n_head, time2, d_k).
scores (torch.Tensor): Attention score (#batch, n_head, time1, time2).
mask (torch.Tensor): Mask (#batch, 1, time2) or (#batch, time1, time2).
Returns:
torch.Tensor: Transformed value (#batch, time1, d_model)
weighted by the attention score (#batch, time1, time2).
"""
n_batch = value.size(0)
if mask is not None:
if mask_att_chunk_encoder is not None:
mask = mask * mask_att_chunk_encoder
mask = mask.unsqueeze(1).eq(0) # (batch, 1, *, time2)
min_value = -float(
"inf"
) # float(numpy.finfo(torch.tensor(0, dtype=scores.dtype).numpy().dtype).min)
scores = scores.masked_fill(mask, min_value)
self.attn = torch.softmax(scores, dim=-1).masked_fill(
mask, 0.0
) # (batch, head, time1, time2)
else:
self.attn = torch.softmax(scores, dim=-1) # (batch, head, time1, time2)
p_attn = self.dropout(self.attn)
x = torch.matmul(p_attn, value) # (batch, head, time1, d_k)
x = (
x.transpose(1, 2).contiguous().view(n_batch, -1, self.h * self.d_k)
) # (batch, time1, d_model)
return self.linear_out(x) # (batch, time1, d_model)
def forward(self, x, mask, mask_shfit_chunk=None, mask_att_chunk_encoder=None):
"""Compute scaled dot product attention.
Args:
query (torch.Tensor): Query tensor (#batch, time1, size).
key (torch.Tensor): Key tensor (#batch, time2, size).
value (torch.Tensor): Value tensor (#batch, time2, size).
mask (torch.Tensor): Mask tensor (#batch, 1, time2) or
(#batch, time1, time2).
Returns:
torch.Tensor: Output tensor (#batch, time1, d_model).
"""
q_h, k_h, v_h, v = self.forward_qkv(x)
fsmn_memory = self.forward_fsmn(v, mask, mask_shfit_chunk)
q_h = q_h * self.d_k ** (-0.5)
scores = torch.matmul(q_h, k_h.transpose(-2, -1))
att_outs = self.forward_attention(v_h, scores, mask, mask_att_chunk_encoder)
return att_outs + fsmn_memory
def forward_chunk(self, x, cache=None, chunk_size=None, look_back=0):
"""Compute scaled dot product attention.
Args:
query (torch.Tensor): Query tensor (#batch, time1, size).
key (torch.Tensor): Key tensor (#batch, time2, size).
value (torch.Tensor): Value tensor (#batch, time2, size).
mask (torch.Tensor): Mask tensor (#batch, 1, time2) or
(#batch, time1, time2).
Returns:
torch.Tensor: Output tensor (#batch, time1, d_model).
"""
q_h, k_h, v_h, v = self.forward_qkv(x)
if chunk_size is not None and look_back > 0 or look_back == -1:
if cache is not None:
k_h_stride = k_h[:, :, : -(chunk_size[2]), :]
v_h_stride = v_h[:, :, : -(chunk_size[2]), :]
k_h = torch.cat((cache["k"], k_h), dim=2)
v_h = torch.cat((cache["v"], v_h), dim=2)
cache["k"] = torch.cat((cache["k"], k_h_stride), dim=2)
cache["v"] = torch.cat((cache["v"], v_h_stride), dim=2)
if look_back != -1:
cache["k"] = cache["k"][:, :, -(look_back * chunk_size[1]) :, :]
cache["v"] = cache["v"][:, :, -(look_back * chunk_size[1]) :, :]
else:
cache_tmp = {
"k": k_h[:, :, : -(chunk_size[2]), :],
"v": v_h[:, :, : -(chunk_size[2]), :],
}
cache = cache_tmp
fsmn_memory = self.forward_fsmn(v, None)
q_h = q_h * self.d_k ** (-0.5)
scores = torch.matmul(q_h, k_h.transpose(-2, -1))
att_outs = self.forward_attention(v_h, scores, None)
return att_outs + fsmn_memory, cache
class LayerNorm(nn.LayerNorm):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
def forward(self, input):
output = F.layer_norm(
input.float(),
self.normalized_shape,
self.weight.float() if self.weight is not None else None,
self.bias.float() if self.bias is not None else None,
self.eps,
)
return output.type_as(input)
def sequence_mask(lengths, maxlen=None, dtype=torch.float32, device=None):
if maxlen is None:
maxlen = lengths.max()
row_vector = torch.arange(0, maxlen, 1).to(lengths.device)
matrix = torch.unsqueeze(lengths, dim=-1)
mask = row_vector < matrix
mask = mask.detach()
return mask.type(dtype).to(device) if device is not None else mask.type(dtype)
class EncoderLayerSANM(nn.Module):
def __init__(
self,
in_size,
size,
self_attn,
feed_forward,
dropout_rate,
normalize_before=True,
concat_after=False,
stochastic_depth_rate=0.0,
):
"""Construct an EncoderLayer object."""
super(EncoderLayerSANM, self).__init__()
self.self_attn = self_attn
self.feed_forward = feed_forward
self.norm1 = LayerNorm(in_size)
self.norm2 = LayerNorm(size)
self.dropout = nn.Dropout(dropout_rate)
self.in_size = in_size
self.size = size
self.normalize_before = normalize_before
self.concat_after = concat_after
if self.concat_after:
self.concat_linear = nn.Linear(size + size, size)
self.stochastic_depth_rate = stochastic_depth_rate
self.dropout_rate = dropout_rate
def forward(self, x, mask, cache=None, mask_shfit_chunk=None, mask_att_chunk_encoder=None):
"""Compute encoded features.
Args:
x_input (torch.Tensor): Input tensor (#batch, time, size).
mask (torch.Tensor): Mask tensor for the input (#batch, time).
cache (torch.Tensor): Cache tensor of the input (#batch, time - 1, size).
Returns:
torch.Tensor: Output tensor (#batch, time, size).
torch.Tensor: Mask tensor (#batch, time).
"""
skip_layer = False
# with stochastic depth, residual connection `x + f(x)` becomes
# `x <- x + 1 / (1 - p) * f(x)` at training time.
stoch_layer_coeff = 1.0
if self.training and self.stochastic_depth_rate > 0:
skip_layer = torch.rand(1).item() < self.stochastic_depth_rate
stoch_layer_coeff = 1.0 / (1 - self.stochastic_depth_rate)
if skip_layer:
if cache is not None:
x = torch.cat([cache, x], dim=1)
return x, mask
residual = x
if self.normalize_before:
x = self.norm1(x)
if self.concat_after:
x_concat = torch.cat(
(
x,
self.self_attn(
x,
mask,
mask_shfit_chunk=mask_shfit_chunk,
mask_att_chunk_encoder=mask_att_chunk_encoder,
),
),
dim=-1,
)
if self.in_size == self.size:
x = residual + stoch_layer_coeff * self.concat_linear(x_concat)
else:
x = stoch_layer_coeff * self.concat_linear(x_concat)
else:
if self.in_size == self.size:
x = residual + stoch_layer_coeff * self.dropout(
self.self_attn(
x,
mask,
mask_shfit_chunk=mask_shfit_chunk,
mask_att_chunk_encoder=mask_att_chunk_encoder,
)
)
else:
x = stoch_layer_coeff * self.dropout(
self.self_attn(
x,
mask,
mask_shfit_chunk=mask_shfit_chunk,
mask_att_chunk_encoder=mask_att_chunk_encoder,
)
)
if not self.normalize_before:
x = self.norm1(x)
residual = x
if self.normalize_before:
x = self.norm2(x)
x = residual + stoch_layer_coeff * self.dropout(self.feed_forward(x))
if not self.normalize_before:
x = self.norm2(x)
return x, mask, cache, mask_shfit_chunk, mask_att_chunk_encoder
def forward_chunk(self, x, cache=None, chunk_size=None, look_back=0):
"""Compute encoded features.
Args:
x_input (torch.Tensor): Input tensor (#batch, time, size).
mask (torch.Tensor): Mask tensor for the input (#batch, time).
cache (torch.Tensor): Cache tensor of the input (#batch, time - 1, size).
Returns:
torch.Tensor: Output tensor (#batch, time, size).
torch.Tensor: Mask tensor (#batch, time).
"""
residual = x
if self.normalize_before:
x = self.norm1(x)
if self.in_size == self.size:
attn, cache = self.self_attn.forward_chunk(x, cache, chunk_size, look_back)
x = residual + attn
else:
x, cache = self.self_attn.forward_chunk(x, cache, chunk_size, look_back)
if not self.normalize_before:
x = self.norm1(x)
residual = x
if self.normalize_before:
x = self.norm2(x)
x = residual + self.feed_forward(x)
if not self.normalize_before:
x = self.norm2(x)
return x, cache
@tables.register("encoder_classes", "SenseVoiceEncoderSmall")
class SenseVoiceEncoderSmall(nn.Module):
"""
Author: Speech Lab of DAMO Academy, Alibaba Group
SCAMA: Streaming chunk-aware multihead attention for online end-to-end speech recognition
https://arxiv.org/abs/2006.01713
"""
def __init__(
self,
input_size: int,
output_size: int = 256,
attention_heads: int = 4,
linear_units: int = 2048,
num_blocks: int = 6,
tp_blocks: int = 0,
dropout_rate: float = 0.1,
positional_dropout_rate: float = 0.1,
attention_dropout_rate: float = 0.0,
stochastic_depth_rate: float = 0.0,
input_layer: Optional[str] = "conv2d",
pos_enc_class=SinusoidalPositionEncoder,
normalize_before: bool = True,
concat_after: bool = False,
positionwise_layer_type: str = "linear",
positionwise_conv_kernel_size: int = 1,
padding_idx: int = -1,
kernel_size: int = 11,
sanm_shfit: int = 0,
selfattention_layer_type: str = "sanm",
**kwargs,
):
super().__init__()
self._output_size = output_size
self.embed = SinusoidalPositionEncoder()
self.normalize_before = normalize_before
positionwise_layer = PositionwiseFeedForward
positionwise_layer_args = (
output_size,
linear_units,
dropout_rate,
)
encoder_selfattn_layer = MultiHeadedAttentionSANM
encoder_selfattn_layer_args0 = (
attention_heads,
input_size,
output_size,
attention_dropout_rate,
kernel_size,
sanm_shfit,
)
encoder_selfattn_layer_args = (
attention_heads,
output_size,
output_size,
attention_dropout_rate,
kernel_size,
sanm_shfit,
)
self.encoders0 = nn.ModuleList(
[
EncoderLayerSANM(
input_size,
output_size,
encoder_selfattn_layer(*encoder_selfattn_layer_args0),
positionwise_layer(*positionwise_layer_args),
dropout_rate,
)
for i in range(1)
]
)
self.encoders = nn.ModuleList(
[
EncoderLayerSANM(
output_size,
output_size,
encoder_selfattn_layer(*encoder_selfattn_layer_args),
positionwise_layer(*positionwise_layer_args),
dropout_rate,
)
for i in range(num_blocks - 1)
]
)
self.tp_encoders = nn.ModuleList(
[
EncoderLayerSANM(
output_size,
output_size,
encoder_selfattn_layer(*encoder_selfattn_layer_args),
positionwise_layer(*positionwise_layer_args),
dropout_rate,
)
for i in range(tp_blocks)
]
)
self.after_norm = LayerNorm(output_size)
self.tp_norm = LayerNorm(output_size)
def output_size(self) -> int:
return self._output_size
def forward(
self,
xs_pad: torch.Tensor,
ilens: torch.Tensor,
):
"""Embed positions in tensor."""
masks = sequence_mask(ilens, device=ilens.device)[:, None, :]
xs_pad *= self.output_size() ** 0.5
xs_pad = self.embed(xs_pad)
# forward encoder1
for layer_idx, encoder_layer in enumerate(self.encoders0):
encoder_outs = encoder_layer(xs_pad, masks)
xs_pad, masks = encoder_outs[0], encoder_outs[1]
for layer_idx, encoder_layer in enumerate(self.encoders):
encoder_outs = encoder_layer(xs_pad, masks)
xs_pad, masks = encoder_outs[0], encoder_outs[1]
xs_pad = self.after_norm(xs_pad)
# forward encoder2
olens = masks.squeeze(1).sum(1).int()
for layer_idx, encoder_layer in enumerate(self.tp_encoders):
encoder_outs = encoder_layer(xs_pad, masks)
xs_pad, masks = encoder_outs[0], encoder_outs[1]
xs_pad = self.tp_norm(xs_pad)
return xs_pad, olens
@tables.register("model_classes", "SenseVoiceSmall")
class SenseVoiceSmall(nn.Module):
"""CTC-attention hybrid Encoder-Decoder model"""
def __init__(
self,
specaug: str = None,
specaug_conf: dict = None,
normalize: str = None,
normalize_conf: dict = None,
encoder: str = None,
encoder_conf: dict = None,
ctc_conf: dict = None,
input_size: int = 80,
vocab_size: int = -1,
ignore_id: int = -1,
blank_id: int = 0,
sos: int = 1,
eos: int = 2,
length_normalized_loss: bool = False,
**kwargs,
):
super().__init__()
if specaug is not None:
specaug_class = tables.specaug_classes.get(specaug)
specaug = specaug_class(**specaug_conf)
if normalize is not None:
normalize_class = tables.normalize_classes.get(normalize)
normalize = normalize_class(**normalize_conf)
encoder_class = tables.encoder_classes.get(encoder)
encoder = encoder_class(input_size=input_size, **encoder_conf)
encoder_output_size = encoder.output_size()
if ctc_conf is None:
ctc_conf = {}
ctc = CTC(odim=vocab_size, encoder_output_size=encoder_output_size, **ctc_conf)
self.blank_id = blank_id
self.sos = sos if sos is not None else vocab_size - 1
self.eos = eos if eos is not None else vocab_size - 1
self.vocab_size = vocab_size
self.ignore_id = ignore_id
self.specaug = specaug
self.normalize = normalize
self.encoder = encoder
self.error_calculator = None
self.ctc = ctc
self.length_normalized_loss = length_normalized_loss
self.encoder_output_size = encoder_output_size
self.lid_dict = {"auto": 0, "zh": 3, "en": 4, "yue": 7, "ja": 11, "ko": 12, "nospeech": 13}
self.lid_int_dict = {24884: 3, 24885: 4, 24888: 7, 24892: 11, 24896: 12, 24992: 13}
self.textnorm_dict = {"withitn": 14, "woitn": 15}
self.textnorm_int_dict = {25016: 14, 25017: 15}
self.embed = torch.nn.Embedding(7 + len(self.lid_dict) + len(self.textnorm_dict), input_size)
self.emo_dict = {"unk": 25009, "happy": 25001, "sad": 25002, "angry": 25003, "neutral": 25004}
self.criterion_att = LabelSmoothingLoss(
size=self.vocab_size,
padding_idx=self.ignore_id,
smoothing=kwargs.get("lsm_weight", 0.0),
normalize_length=self.length_normalized_loss,
)
@staticmethod
def from_pretrained(model:str=None, **kwargs):
from funasr import AutoModel
model, kwargs = AutoModel.build_model(model=model, trust_remote_code=True, **kwargs)
return model, kwargs
def forward(
self,
speech: torch.Tensor,
speech_lengths: torch.Tensor,
text: torch.Tensor,
text_lengths: torch.Tensor,
**kwargs,
):
"""Encoder + Decoder + Calc loss
Args:
speech: (Batch, Length, ...)
speech_lengths: (Batch, )
text: (Batch, Length)
text_lengths: (Batch,)
"""
# import pdb;
# pdb.set_trace()
if len(text_lengths.size()) > 1:
text_lengths = text_lengths[:, 0]
if len(speech_lengths.size()) > 1:
speech_lengths = speech_lengths[:, 0]
batch_size = speech.shape[0]
# 1. Encoder
encoder_out, encoder_out_lens = self.encode(speech, speech_lengths, text)
loss_ctc, cer_ctc = None, None
loss_rich, acc_rich = None, None
stats = dict()
loss_ctc, cer_ctc = self._calc_ctc_loss(
encoder_out[:, 4:, :], encoder_out_lens - 4, text[:, 4:], text_lengths - 4
)
loss_rich, acc_rich = self._calc_rich_ce_loss(
encoder_out[:, :4, :], text[:, :4]
)
loss = loss_ctc
# Collect total loss stats
stats["loss"] = torch.clone(loss.detach()) if loss_ctc is not None else None
stats["loss_rich"] = torch.clone(loss_rich.detach()) if loss_rich is not None else None
stats["acc_rich"] = acc_rich
# force_gatherable: to-device and to-tensor if scalar for DataParallel
if self.length_normalized_loss:
batch_size = int((text_lengths + 1).sum())
loss, stats, weight = force_gatherable((loss, stats, batch_size), loss.device)
return loss, stats, weight
def encode(
self,
speech: torch.Tensor,
speech_lengths: torch.Tensor,
text: torch.Tensor,
**kwargs,
):
"""Frontend + Encoder. Note that this method is used by asr_inference.py
Args:
speech: (Batch, Length, ...)
speech_lengths: (Batch, )
ind: int
"""
# Data augmentation
if self.specaug is not None and self.training:
speech, speech_lengths = self.specaug(speech, speech_lengths)
# Normalization for feature: e.g. Global-CMVN, Utterance-CMVN
if self.normalize is not None:
speech, speech_lengths = self.normalize(speech, speech_lengths)
lids = torch.LongTensor([[self.lid_int_dict[int(lid)] if torch.rand(1) > 0.2 and int(lid) in self.lid_int_dict else 0 ] for lid in text[:, 0]]).to(speech.device)
language_query = self.embed(lids)
styles = torch.LongTensor([[self.textnorm_int_dict[int(style)]] for style in text[:, 3]]).to(speech.device)
style_query = self.embed(styles)
speech = torch.cat((style_query, speech), dim=1)
speech_lengths += 1
event_emo_query = self.embed(torch.LongTensor([[1, 2]]).to(speech.device)).repeat(speech.size(0), 1, 1)
input_query = torch.cat((language_query, event_emo_query), dim=1)
speech = torch.cat((input_query, speech), dim=1)
speech_lengths += 3
encoder_out, encoder_out_lens = self.encoder(speech, speech_lengths)
return encoder_out, encoder_out_lens
def _calc_ctc_loss(
self,
encoder_out: torch.Tensor,
encoder_out_lens: torch.Tensor,
ys_pad: torch.Tensor,
ys_pad_lens: torch.Tensor,
):
# Calc CTC loss
loss_ctc = self.ctc(encoder_out, encoder_out_lens, ys_pad, ys_pad_lens)
# Calc CER using CTC
cer_ctc = None
if not self.training and self.error_calculator is not None:
ys_hat = self.ctc.argmax(encoder_out).data
cer_ctc = self.error_calculator(ys_hat.cpu(), ys_pad.cpu(), is_ctc=True)
return loss_ctc, cer_ctc
def _calc_rich_ce_loss(
self,
encoder_out: torch.Tensor,
ys_pad: torch.Tensor,
):
decoder_out = self.ctc.ctc_lo(encoder_out)
# 2. Compute attention loss
loss_rich = self.criterion_att(decoder_out, ys_pad.contiguous())
acc_rich = th_accuracy(
decoder_out.view(-1, self.vocab_size),
ys_pad.contiguous(),
ignore_label=self.ignore_id,
)
return loss_rich, acc_rich
def inference(
self,
data_in,
data_lengths=None,
key: list = ["wav_file_tmp_name"],
tokenizer=None,
frontend=None,
**kwargs,
):
meta_data = {}
if (
isinstance(data_in, torch.Tensor) and kwargs.get("data_type", "sound") == "fbank"
): # fbank
speech, speech_lengths = data_in, data_lengths
if len(speech.shape) < 3:
speech = speech[None, :, :]
if speech_lengths is None:
speech_lengths = speech.shape[1]
else:
# extract fbank feats
time1 = time.perf_counter()
audio_sample_list = load_audio_text_image_video(
data_in,
fs=frontend.fs,
audio_fs=kwargs.get("fs", 16000),
data_type=kwargs.get("data_type", "sound"),
tokenizer=tokenizer,
)
time2 = time.perf_counter()
meta_data["load_data"] = f"{time2 - time1:0.3f}"
speech, speech_lengths = extract_fbank(
audio_sample_list, data_type=kwargs.get("data_type", "sound"), frontend=frontend
)
time3 = time.perf_counter()
meta_data["extract_feat"] = f"{time3 - time2:0.3f}"
meta_data["batch_data_time"] = (
speech_lengths.sum().item() * frontend.frame_shift * frontend.lfr_n / 1000
)
speech = speech.to(device=kwargs["device"])
speech_lengths = speech_lengths.to(device=kwargs["device"])
language = kwargs.get("language", "auto")
language_query = self.embed(
torch.LongTensor(
[[self.lid_dict[language] if language in self.lid_dict else 0]]
).to(speech.device)
).repeat(speech.size(0), 1, 1)
use_itn = kwargs.get("use_itn", False)
textnorm = kwargs.get("text_norm", None)
if textnorm is None:
textnorm = "withitn" if use_itn else "woitn"
textnorm_query = self.embed(
torch.LongTensor([[self.textnorm_dict[textnorm]]]).to(speech.device)
).repeat(speech.size(0), 1, 1)
speech = torch.cat((textnorm_query, speech), dim=1)
speech_lengths += 1
event_emo_query = self.embed(torch.LongTensor([[1, 2]]).to(speech.device)).repeat(
speech.size(0), 1, 1
)
input_query = torch.cat((language_query, event_emo_query), dim=1)
speech = torch.cat((input_query, speech), dim=1)
speech_lengths += 3
# Encoder
encoder_out, encoder_out_lens = self.encoder(speech, speech_lengths)
if isinstance(encoder_out, tuple):
encoder_out = encoder_out[0]
# c. Passed the encoder result and the beam search
ctc_logits = self.ctc.log_softmax(encoder_out)
if kwargs.get("ban_emo_unk", False):
ctc_logits[:, :, self.emo_dict["unk"]] = -float("inf")
results = []
b, n, d = encoder_out.size()
if isinstance(key[0], (list, tuple)):
key = key[0]
if len(key) < b:
key = key * b
for i in range(b):
x = ctc_logits[i, : encoder_out_lens[i].item(), :]
yseq = x.argmax(dim=-1)
yseq = torch.unique_consecutive(yseq, dim=-1)
ibest_writer = None
if kwargs.get("output_dir") is not None:
if not hasattr(self, "writer"):
self.writer = DatadirWriter(kwargs.get("output_dir"))
ibest_writer = self.writer[f"1best_recog"]
mask = yseq != self.blank_id
token_int = yseq[mask].tolist()
# Change integer-ids to tokens
text = tokenizer.decode(token_int)
result_i = {"key": key[i], "text": text}
results.append(result_i)
if ibest_writer is not None:
ibest_writer["text"][key[i]] = text
return results, meta_data
def export(self, **kwargs):
from export_meta import export_rebuild_model
if "max_seq_len" not in kwargs:
kwargs["max_seq_len"] = 512
models = export_rebuild_model(model=self, **kwargs)
return models
from fastapi import FastAPI, File, UploadFile, HTTPException
from fastapi.responses import JSONResponse
from tempfile import NamedTemporaryFile
from funasr import AutoModel
from funasr.utils.postprocess_utils import rich_transcription_postprocess
import os
# 加载模型
model_dir = "./iic/SenseVoiceSmall"
model = AutoModel(
model=model_dir,
trust_remote_code=True,
remote_code="./model.py",
vad_model="fsmn-vad",
vad_kwargs={"max_single_segment_time": 30000},
device="cuda:0",
)
app = FastAPI()
@app.post("/v1/audio/transcriptions")
async def handler(file: UploadFile = File(...)):
if not file:
raise HTTPException(status_code=400, detail="No file was provided")
# 使用NamedTemporaryFile创建临时文件
with NamedTemporaryFile(delete=False) as temp_file:
# 将用户上传的文件写入临时文件
content = await file.read()
temp_file.write(content)
temp_file_path = temp_file.name
try:
# 开始运行模型
result = model.generate(
input=temp_file_path,
cache={},
language="auto",
use_itn=True,
batch_size_s=60,
merge_vad=True,
merge_length_s=15,
)
text = rich_transcription_postprocess(result[0]["text"])
# 返回包含结果的JSON响应
return JSONResponse(content={'text': text})
finally:
# 删除临时文件
os.unlink(temp_file_path)
if __name__ == "__main__":
import uvicorn
uvicorn.run(app, host="0.0.0.0", port=8000)
docker run -d -p 8000:8000 registry.cn-hangzhou.aliyuncs.com/luanshaotong/sensevoice:v0.1
FROM dockerhub.icu/pytorch/pytorch:2.1.0-cuda11.8-cudnn8-runtime
ENV DEBIAN_FRONTEND=noninteractive
WORKDIR /opt/CosyVoice
RUN chmod 777 /tmp && sed -i 's@//.*archive.ubuntu.com@//mirrors.ustc.edu.cn@g' /etc/apt/sources.list && apt-get update -y && apt-get -y install git unzip git-lfs
RUN git lfs install && git clone --recursive https://github.com/FunAudioLLM/CosyVoice.git
# here we use python==3.10 because we cannot find an image which have both python3.8 and torch2.0.1-cu118 installed
COPY ./requirements.txt CosyVoice
RUN cd CosyVoice && pip3 install -r requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple
RUN cd CosyVoice/runtime/python/grpc && python3 -m grpc_tools.protoc -I. --python_out=. --grpc_python_out=. cosyvoice.proto
COPY fastapi/server.py CosyVoice/runtime/python/fastapi/
import argparse
import logging
import requests
def saveResponse(path, response):
# 以二进制写入模式打开文件
with open(path, 'wb') as file:
# 将响应的二进制内容写入文件
file.write(response.content)
def main():
api = args.api_base
if args.mode == 'sft':
url = api + "/api/inference/sft"
payload={
'tts': args.tts_text,
'role': args.spk_id
}
response = requests.request("POST", url, data=payload)
saveResponse(args.tts_wav, response)
elif args.mode == 'zero_shot':
url = api + "/api/inference/zero-shot"
payload={
'tts': args.tts_text,
'prompt': args.prompt_text
}
files=[('audio', ('prompt_audio.wav', open(args.prompt_wav,'rb'), 'application/octet-stream'))]
response = requests.request("POST", url, data=payload, files=files)
saveResponse(args.tts_wav, response)
elif args.mode == 'cross_lingual':
url = api + "/api/inference/cross-lingual"
payload={
'tts': args.tts_text,
}
files=[('audio', ('prompt_audio.wav', open(args.prompt_wav,'rb'), 'application/octet-stream'))]
response = requests.request("POST", url, data=payload, files=files)
saveResponse(args.tts_wav, response)
else:
url = api + "/api/inference/instruct"
payload = {
'tts': args.tts_text,
'role': args.spk_id,
'instruct': args.instruct_text
}
response = requests.request("POST", url, data=payload)
saveResponse(args.tts_wav, response)
logging.info("Response save to {}", args.tts_wav)
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument('--api_base',
type=str,
default='http://127.0.0.1:50000')
parser.add_argument('--mode',
default='sft',
choices=['sft', 'zero_shot', 'cross_lingual', 'instruct'],
help='request mode')
parser.add_argument('--tts_text',
type=str,
default='你好,我是通义千问语音合成大模型,请问有什么可以帮您的吗?')
parser.add_argument('--spk_id',
type=str,
default='中文男')
parser.add_argument('--prompt_text',
type=str,
default='希望你以后能够做的比我还好呦。')
parser.add_argument('--prompt_wav',
type=str,
default='../../../zero_shot_prompt.wav')
parser.add_argument('--instruct_text',
type=str,
default='Theo \'Crimson\', is a fiery, passionate rebel leader. Fights with fervor for justice, but struggles with impulsiveness.')
parser.add_argument('--tts_wav',
type=str,
default='loushiming.mp3')
args = parser.parse_args()
prompt_sr, target_sr = 16000, 22050
main()
# Set inference model
# export MODEL_DIR=pretrained_models/CosyVoice-300M-Instruct
# For development
# fastapi dev --port 6006 fastapi_server.py
# For production deployment
# fastapi run --port 6006 fastapi_server.py
import os
import sys
import io,time
from fastapi import FastAPI, Request, Response, File, UploadFile, Form, Body
from fastapi.responses import HTMLResponse
from fastapi.middleware.cors import CORSMiddleware #引入 CORS中间件模块
from contextlib import asynccontextmanager
ROOT_DIR = os.path.dirname(os.path.abspath(__file__))
sys.path.append('{}/../../..'.format(ROOT_DIR))
sys.path.append('{}/../../../third_party/Matcha-TTS'.format(ROOT_DIR))
from cosyvoice.cli.cosyvoice import CosyVoice
from cosyvoice.utils.file_utils import load_wav
import numpy as np
import torch
import torchaudio
import logging
from pydantic import BaseModel
logging.getLogger('matplotlib').setLevel(logging.WARNING)
class LaunchFailed(Exception):
pass
@asynccontextmanager
async def lifespan(app: FastAPI):
model_dir = os.getenv("MODEL_DIR", "pretrained_models/CosyVoice-300M-SFT")
if model_dir:
logging.info("MODEL_DIR is {}", model_dir)
app.cosyvoice = CosyVoice(model_dir)
# sft usage
logging.info("Avaliable speakers {}", app.cosyvoice.list_avaliable_spks())
else:
raise LaunchFailed("MODEL_DIR environment must set")
yield
app = FastAPI(lifespan=lifespan)
#设置允许访问的域名
origins = ["*"] #"*",即为所有,也可以改为允许的特定ip。
app.add_middleware(
CORSMiddleware,
allow_origins=origins, #设置允许的origins来源
allow_credentials=True,
allow_methods=["*"], # 设置允许跨域的http方法,比如 get、post、put等。
allow_headers=["*"]) #允许跨域的headers,可以用来鉴别来源等作用。
def buildResponse(output):
buffer = io.BytesIO()
torchaudio.save(buffer, output, 22050, format="mp3")
buffer.seek(0)
return Response(content=buffer.read(-1), media_type="audio/mpeg")
@app.post("/api/inference/sft")
@app.get("/api/inference/sft")
async def sft(tts: str = Form(), role: str = Form()):
start = time.process_time()
output = app.cosyvoice.inference_sft(tts, role)
end = time.process_time()
logging.info("infer time is {} seconds", end-start)
return buildResponse(output['tts_speech'])
class SpeechRequest(BaseModel):
model: str
input: str
voice: str
@app.post("/v1/audio/speech")
async def sft(request: Request, speech_request: SpeechRequest):
# 解析请求体中的JSON数据
data = speech_request.dict()
start = time.process_time()
output = app.cosyvoice.inference_sft(data['input'], data['voice'])
end = time.process_time()
logging.info("infer time is {} seconds", end-start)
return buildResponse(output['tts_speech'])
@app.post("/api/inference/zero-shot")
async def zeroShot(tts: str = Form(), prompt: str = Form(), audio: UploadFile = File()):
start = time.process_time()
prompt_speech = load_wav(audio.file, 16000)
prompt_audio = (prompt_speech.numpy() * (2**15)).astype(np.int16).tobytes()
prompt_speech_16k = torch.from_numpy(np.array(np.frombuffer(prompt_audio, dtype=np.int16))).unsqueeze(dim=0)
prompt_speech_16k = prompt_speech_16k.float() / (2**15)
output = app.cosyvoice.inference_zero_shot(tts, prompt, prompt_speech_16k)
end = time.process_time()
logging.info("infer time is {} seconds", end-start)
return buildResponse(output['tts_speech'])
@app.post("/api/inference/cross-lingual")
async def crossLingual(tts: str = Form(), audio: UploadFile = File()):
start = time.process_time()
prompt_speech = load_wav(audio.file, 16000)
prompt_audio = (prompt_speech.numpy() * (2**15)).astype(np.int16).tobytes()
prompt_speech_16k = torch.from_numpy(np.array(np.frombuffer(prompt_audio, dtype=np.int16))).unsqueeze(dim=0)
prompt_speech_16k = prompt_speech_16k.float() / (2**15)
output = app.cosyvoice.inference_cross_lingual(tts, prompt_speech_16k)
end = time.process_time()
logging.info("infer time is {} seconds", end-start)
return buildResponse(output['tts_speech'])
@app.post("/api/inference/instruct")
@app.get("/api/inference/instruct")
async def instruct(tts: str = Form(), role: str = Form(), instruct: str = Form()):
start = time.process_time()
output = app.cosyvoice.inference_instruct(tts, role, instruct)
end = time.process_time()
logging.info("infer time is {} seconds", end-start)
return buildResponse(output['tts_speech'])
@app.get("/api/roles")
async def roles():
return {"roles": app.cosyvoice.list_avaliable_spks()}
@app.get("/", response_class=HTMLResponse)
async def root():
return """
<!DOCTYPE html>
<html lang=zh-cn>
<head>
<meta charset=utf-8>
<title>Api information</title>
</head>
<body>
Get the supported tones from the Roles API first, then enter the tones and textual content in the TTS API for synthesis. <a href='./docs'>Documents of API</a>
</body>
</html>
"""
# Copyright (c) 2024 Alibaba Inc (authors: Xiang Lyu)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import sys
ROOT_DIR = os.path.dirname(os.path.abspath(__file__))
sys.path.append('{}/../../..'.format(ROOT_DIR))
sys.path.append('{}/../../../third_party/Matcha-TTS'.format(ROOT_DIR))
import logging
import argparse
import torchaudio
import cosyvoice_pb2
import cosyvoice_pb2_grpc
import grpc
import torch
import numpy as np
from cosyvoice.utils.file_utils import load_wav
def main():
with grpc.insecure_channel("{}:{}".format(args.host, args.port)) as channel:
stub = cosyvoice_pb2_grpc.CosyVoiceStub(channel)
request = cosyvoice_pb2.Request()
if args.mode == 'sft':
logging.info('send sft request')
sft_request = cosyvoice_pb2.sftRequest()
sft_request.spk_id = args.spk_id
sft_request.tts_text = args.tts_text
request.sft_request.CopyFrom(sft_request)
elif args.mode == 'zero_shot':
logging.info('send zero_shot request')
zero_shot_request = cosyvoice_pb2.zeroshotRequest()
zero_shot_request.tts_text = args.tts_text
zero_shot_request.prompt_text = args.prompt_text
prompt_speech = load_wav(args.prompt_wav, 16000)
zero_shot_request.prompt_audio = (prompt_speech.numpy() * (2**15)).astype(np.int16).tobytes()
request.zero_shot_request.CopyFrom(zero_shot_request)
elif args.mode == 'cross_lingual':
logging.info('send cross_lingual request')
cross_lingual_request = cosyvoice_pb2.crosslingualRequest()
cross_lingual_request.tts_text = args.tts_text
prompt_speech = load_wav(args.prompt_wav, 16000)
cross_lingual_request.prompt_audio = (prompt_speech.numpy() * (2**15)).astype(np.int16).tobytes()
request.cross_lingual_request.CopyFrom(cross_lingual_request)
else:
logging.info('send instruct request')
instruct_request = cosyvoice_pb2.instructRequest()
instruct_request.tts_text = args.tts_text
instruct_request.spk_id = args.spk_id
instruct_request.instruct_text = args.instruct_text
request.instruct_request.CopyFrom(instruct_request)
response = stub.Inference(request)
logging.info('save response to {}'.format(args.tts_wav))
tts_speech = torch.from_numpy(np.array(np.frombuffer(response.tts_audio, dtype=np.int16))).unsqueeze(dim=0)
torchaudio.save(args.tts_wav, tts_speech, target_sr)
logging.info('get response')
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument('--host',
type=str,
default='0.0.0.0')
parser.add_argument('--port',
type=int,
default='50000')
parser.add_argument('--mode',
default='sft',
choices=['sft', 'zero_shot', 'cross_lingual', 'instruct'],
help='request mode')
parser.add_argument('--tts_text',
type=str,
default='你好,我是通义千问语音合成大模型,请问有什么可以帮您的吗?')
parser.add_argument('--spk_id',
type=str,
default='中文女')
parser.add_argument('--prompt_text',
type=str,
default='希望你以后能够做的比我还好呦。')
parser.add_argument('--prompt_wav',
type=str,
default='../../../zero_shot_prompt.wav')
parser.add_argument('--instruct_text',
type=str,
default='Theo \'Crimson\', is a fiery, passionate rebel leader. Fights with fervor for justice, but struggles with impulsiveness.')
parser.add_argument('--tts_wav',
type=str,
default='demo.wav')
args = parser.parse_args()
prompt_sr, target_sr = 16000, 22050
main()
syntax = "proto3";
package cosyvoice;
option go_package = "protos/";
service CosyVoice{
rpc Inference(Request) returns (Response) {}
}
message Request{
oneof RequestPayload {
sftRequest sft_request = 1;
zeroshotRequest zero_shot_request = 2;
crosslingualRequest cross_lingual_request = 3;
instructRequest instruct_request = 4;
}
}
message sftRequest{
string spk_id = 1;
string tts_text = 2;
}
message zeroshotRequest{
string tts_text = 1;
string prompt_text = 2;
bytes prompt_audio = 3;
}
message crosslingualRequest{
string tts_text = 1;
bytes prompt_audio = 2;
}
message instructRequest{
string tts_text = 1;
string spk_id = 2;
string instruct_text = 3;
}
message Response{
bytes tts_audio = 1;
}
\ No newline at end of file
# Copyright (c) 2024 Alibaba Inc (authors: Xiang Lyu)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import sys
ROOT_DIR = os.path.dirname(os.path.abspath(__file__))
sys.path.append('{}/../../..'.format(ROOT_DIR))
sys.path.append('{}/../../../third_party/Matcha-TTS'.format(ROOT_DIR))
from concurrent import futures
import argparse
import cosyvoice_pb2
import cosyvoice_pb2_grpc
import logging
logging.getLogger('matplotlib').setLevel(logging.WARNING)
import grpc
import torch
import numpy as np
from cosyvoice.cli.cosyvoice import CosyVoice
logging.basicConfig(level=logging.DEBUG,
format='%(asctime)s %(levelname)s %(message)s')
class CosyVoiceServiceImpl(cosyvoice_pb2_grpc.CosyVoiceServicer):
def __init__(self, args):
self.cosyvoice = CosyVoice(args.model_dir)
logging.info('grpc service initialized')
def Inference(self, request, context):
if request.HasField('sft_request'):
logging.info('get sft inference request')
model_output = self.cosyvoice.inference_sft(request.sft_request.tts_text, request.sft_request.spk_id)
elif request.HasField('zero_shot_request'):
logging.info('get zero_shot inference request')
prompt_speech_16k = torch.from_numpy(np.array(np.frombuffer(request.zero_shot_request.prompt_audio, dtype=np.int16))).unsqueeze(dim=0)
prompt_speech_16k = prompt_speech_16k.float() / (2**15)
model_output = self.cosyvoice.inference_zero_shot(request.zero_shot_request.tts_text, request.zero_shot_request.prompt_text, prompt_speech_16k)
elif request.HasField('cross_lingual_request'):
logging.info('get cross_lingual inference request')
prompt_speech_16k = torch.from_numpy(np.array(np.frombuffer(request.cross_lingual_request.prompt_audio, dtype=np.int16))).unsqueeze(dim=0)
prompt_speech_16k = prompt_speech_16k.float() / (2**15)
model_output = self.cosyvoice.inference_cross_lingual(request.cross_lingual_request.tts_text, prompt_speech_16k)
else:
logging.info('get instruct inference request')
model_output = self.cosyvoice.inference_instruct(request.instruct_request.tts_text, request.instruct_request.spk_id, request.instruct_request.instruct_text)
logging.info('send inference response')
response = cosyvoice_pb2.Response()
response.tts_audio = (model_output['tts_speech'].numpy() * (2 ** 15)).astype(np.int16).tobytes()
return response
def main():
grpcServer = grpc.server(futures.ThreadPoolExecutor(max_workers=args.max_conc), maximum_concurrent_rpcs=args.max_conc)
cosyvoice_pb2_grpc.add_CosyVoiceServicer_to_server(CosyVoiceServiceImpl(args), grpcServer)
grpcServer.add_insecure_port('0.0.0.0:{}'.format(args.port))
grpcServer.start()
logging.info("server listening on 0.0.0.0:{}".format(args.port))
grpcServer.wait_for_termination()
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('--port',
type=int,
default=50000)
parser.add_argument('--max_conc',
type=int,
default=4)
parser.add_argument('--model_dir',
type=str,
default='iic/CosyVoice-300M',
help='local path or modelscope repo id')
args = parser.parse_args()
main()
--extra-index-url https://download.pytorch.org/whl/cu118
conformer==0.3.2
deepspeed==0.15.1; sys_platform == 'linux'
diffusers==0.27.2
gdown==5.2.2
gradio==5.11.0
grpcio==1.57.0
grpcio-tools==1.57.0
hydra-core==1.3.2
HyperPyYAML==1.2.2
inflect==7.3.1
librosa==0.10.2
lightning==2.3.3
matplotlib==3.7.5
modelscope==1.15.0
networkx==3.1
omegaconf==2.3.0
onnxruntime-gpu; sys_platform == 'linux'
onnxruntime; sys_platform == 'darwin' or sys_platform == 'windows'
openai-whisper==20231117
protobuf==5.29.6
pydantic==2.7.0
rich==13.7.1
soundfile==0.12.1
tensorboard
wget==3.2
fastapi==0.111.0
fastapi-cli==0.0.4
WeTextProcessing==1.0.3
# 忽略 .git 目录及其内容
.git
.gitignore
*~
searxng-docker.service
caddy
srv
searxng/uwsgi.ini
.env
SPIDER/.env
# 忽略 node_modules 文件夹
SPIDER/node_modules/
# 忽略构建输出文件夹
SPIDER/dist/
# 忽略日志文件
*.log
# 忽略操作系统生成的文件
.DS_Store
Thumbs.db
# 忽略 IDE/编辑器生成的文件
.vscode/
.idea/
\ No newline at end of file
# By default listen on http://localhost
# To change this:
# * uncomment SEARXNG_HOSTNAME, and replace <host> by the SearXNG hostname
# * uncomment LETSENCRYPT_EMAIL, and replace <email> by your email (require to create a Let's Encrypt certificate)
# SEARXNG_HOSTNAME=<host>
# LETSENCRYPT_EMAIL=<email>
# Optional:
# If you run a very small or a very large instance, you might want to change the amount of used uwsgi workers and threads per worker
# More workers (= processes) means that more search requests can be handled at the same time, but it also causes more resource usage
SEARXNG_UWSGI_WORKERS=4
SEARXNG_UWSGI_THREADS=4
{
admin off
log {
output stderr
format filter {
# Preserves first 8 bits from IPv4 and 32 bits from IPv6
request>remote_ip ip_mask 8 32
request>client_ip ip_mask 8 32
# Remove identificable information
request>remote_port delete
request>headers delete
request>uri query {
delete url
delete h
delete q
}
}
}
}
{$SEARXNG_HOSTNAME}
tls {$SEARXNG_TLS}
encode zstd gzip
@api {
path /config
path /healthz
path /stats/errors
path /stats/checker
}
@search {
path /search
}
@imageproxy {
path /image_proxy
}
@static {
path /static/*
}
header {
# CSP (https://content-security-policy.com)
Content-Security-Policy "upgrade-insecure-requests; default-src 'none'; script-src 'self'; style-src 'self' 'unsafe-inline'; form-action 'self' https://github.com/searxng/searxng/issues/new; font-src 'self'; frame-ancestors 'self'; base-uri 'self'; connect-src 'self' https://overpass-api.de; img-src * data:; frame-src https://www.youtube-nocookie.com https://player.vimeo.com https://www.dailymotion.com https://www.deezer.com https://www.mixcloud.com https://w.soundcloud.com https://embed.spotify.com;"
# Disable some browser features
Permissions-Policy "accelerometer=(),camera=(),geolocation=(),gyroscope=(),magnetometer=(),microphone=(),payment=(),usb=()"
# Set referrer policy
Referrer-Policy "no-referrer"
# Force clients to use HTTPS
Strict-Transport-Security "max-age=31536000"
# Prevent MIME type sniffing from the declared Content-Type
X-Content-Type-Options "nosniff"
# X-Robots-Tag (comment to allow site indexing)
X-Robots-Tag "noindex, noarchive, nofollow"
# Remove "Server" header
-Server
}
header @api {
Access-Control-Allow-Methods "GET, OPTIONS"
Access-Control-Allow-Origin "*"
}
route {
# Cache policy
header Cache-Control "max-age=0, no-store"
header @search Cache-Control "max-age=5, private"
header @imageproxy Cache-Control "max-age=604800, public"
header @static Cache-Control "max-age=31536000, public, immutable"
}
# SearXNG (uWSGI)
reverse_proxy localhost:8080 {
header_up X-Forwarded-Port ""
header_up X-Real-IP ""
# https://github.com/searx/searx-docker/issues/24
header_up Connection "close"
}
FROM node:20.10.0-slim
WORKDIR /app
# 安装 Chrome 运行依赖
RUN apt-get update && apt-get install -y \
ca-certificates \
fonts-liberation \
libasound2 \
libatk-bridge2.0-0 \
libatk1.0-0 \
libc6 \
libcairo2 \
libcups2 \
libdbus-1-3 \
libexpat1 \
libfontconfig1 \
libgbm1 \
libgcc1 \
libglib2.0-0 \
libgtk-3-0 \
libnspr4 \
libnss3 \
libpango-1.0-0 \
libpangocairo-1.0-0 \
libstdc++6 \
libx11-6 \
libx11-xcb1 \
libxcb1 \
libxcomposite1 \
libxcursor1 \
libxdamage1 \
libxext6 \
libxfixes3 \
libxi6 \
libxrandr2 \
libxrender1 \
libxss1 \
libxtst6 \
lsb-release \
wget \
xdg-utils \
chromium \
&& rm -rf /var/lib/apt/lists/*
# 安装中文字体
RUN apt-get update && apt-get install -y fonts-wqy-microhei && fc-cache -f -v
COPY SPIDER/. .
RUN test -f package.json || (echo "package.json missing" && exit 1)
RUN test -f .env || (echo ".env file missing in SPIDER directory" && exit 1)
RUN npm run build
EXPOSE 3000
CMD ["npm", "start"]
\ No newline at end of file
# webcrawler
## docker版快速部署
## 代码版部署
0. 按照 https://github.com/searxng/searxng-docker 的方式处理docker
1. 参考SPIDER文件夹下的.env.example,添加.env文件
2. 进入SPIDER文件夹进行pnpm install
3. 回到根目录,运行docker compose up -d
## 代码版开发
1. 将docker-compose.yml中与SPIDER相关的部分注释掉(nodeapp)
2. .env文件中的URL参照注释修改
3. 注释掉启动puppteer部分里面指定浏览器地址的代码
4. pnpm run dev
## 测试样例:
Auth的Bear Token记得填,也就是.env里的ACCESS_TOKEN
### 读取单页面(content以HTML形式返回)
```
http://localhost:3000/api/read?queryUrl=<url>
```
返回结构
```json
{
"status": 200,
"data": {
"title": "something here",
"content": "something here"
}
}
{
"status": 400,
"error": {
"code": "MISSING_PARAM",
"message": "缺少必要参数: query"
}
}
```
### 搜索(content以HTML形式返回)
```
http://localhost:3000/api/search?query=<something>&pageCount=5&needDetails=true&engine=baidu
```
```json
{
"status": 200,
"data": {
"results": [
{
"title": "string",
"url": "string",
"snippet": "string",
"source": "string",
"crawlStatus": "string",
"score": 0,
"content": "string"
}
]
}
}
{
"status": 400,
"error": {
"code": "MISSING_PARAM",
"message": "缺少必要参数: query"
}
}
```
\ No newline at end of file
ACCESS_TOKEN=114514
DETECT_WEBSITE = zhuanlan.zhihu.com
STRATEGIES=[{"waitUntil":"networkidle0","timeout":5000},{"waitUntil":"networkidle2","timeout":10000},{"waitUntil":"load","timeout":15000}]
PORT=3000
MAX_CONCURRENCY=10
NODE_ENV=development
ENGINE = [
]
ENGINE_BAIDUURL=https://www.baidu.com/s
#ENGINE_SEARCHXNGURL=http://localhost:8080/search
ENGINE_SEARCHXNGURL=http://searxng:8080/search
#MONGODB_URI=mongodb://root:example@localhost:27017
MONGODB_URI=mongodb://root:example@mongodb:27017
BLACKLIST = [".gov.cn",".edu.cn"]
STD_TTL=3600
EXPIRE_AFTER_SECONDS=9000
#VALIDATE_PROXY=[{"ip":"","port":},{"ip":"","port":}]
\ No newline at end of file
This source diff could not be displayed because it is too large. You can view the blob instead.
{
"name": "spider",
"version": "1.0.0",
"description": "",
"main": "/dist/index.ts",
"scripts": {
"test": "echo \"Error: no test specified\" && exit 1",
"start": "ts-node src/index.ts",
"build": "webpack",
"dev": "ts-node-dev --respawn src/index.ts"
},
"keywords": [],
"author": "",
"license": "ISC",
"dependencies": {
"@types/node-fetch": "^2.6.12",
"assert": "^2.1.0",
"axios": "^1.13.2",
"body-parser": "^1.20.3",
"browserify-zlib": "^0.2.0",
"buffer": "^6.0.3",
"cheerio": "^1.0.0",
"crypto-browserify": "^3.12.1",
"dotenv": "^16.4.7",
"express": "^4.21.2",
"https-proxy-agent": "^7.0.6",
"jsdom": "^26.0.0",
"mongodb": "^6.13.1",
"node-cache": "^5.1.2",
"node-fetch": "^2.7.0",
"os-browserify": "^0.3.0",
"path-browserify": "^1.0.1",
"puppeteer": "^24.2.1",
"puppeteer-cluster": "^0.24.0",
"querystring-es3": "^0.2.1",
"random-useragent": "^0.5.0",
"spider": "file:",
"stream-browserify": "^3.0.0",
"stream-http": "^3.2.0",
"string_decoder": "^1.3.0",
"turndown": "^7.2.0",
"turndown-plugin-gfm": "^1.0.2",
"url": "^0.11.4",
"user-agents": "^1.1.454",
"util": "^0.12.5",
"vm-browserify": "^1.1.2"
},
"devDependencies": {
"@types/body-parser": "^1.19.5",
"@types/express": "^5.0.0",
"@types/jsdom": "^21.1.7",
"@types/node": "^22.13.4",
"@types/random-useragent": "^0.3.3",
"@types/user-agents": "^1.0.4",
"ts-loader": "^9.5.2",
"ts-node-dev": "^2.0.0",
"typescript": "^5.7.3",
"webpack": "^5.98.0",
"webpack-cli": "^6.0.1",
"webpack-node-externals": "^3.0.0"
}
}
import type { Request, Response } from 'express';
import fetch from 'node-fetch';
import dotenv from 'dotenv';
dotenv.config();
const userAgents = [
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3',
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0.3 Safari/605.1.15',
'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:89.0) Gecko/20100101 Firefox/89.0'
];
export const quickFetch = async (req: Request, res: Response): Promise<void> => {
const { url } = req.query;
if (!url) {
res.status(400).json({
status: 400,
error: {
code: 'MISSING_PARAM',
message: '缺少必要参数: url'
}
});
return;
}
try {
const response = await fetch(url as string, {
headers: {
'User-Agent': userAgents[Math.floor(Math.random() * userAgents.length)],
Referer: 'https://www.google.com/',
'Accept-Language': 'en-US,en;q=0.9',
Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
Connection: 'keep-alive',
'Cache-Control': 'no-cache'
}
});
if (!response.ok) {
throw new Error(`HTTP error! status: ${response.status}`);
}
const data = await response.text();
res.status(200).json({
status: 200,
data: {
content: data
}
});
} catch (error) {
console.error('Error fetching the page:', error);
res.status(500).json({
status: 500,
error: {
code: 'INTERNAL_SERVER_ERROR',
message: '发生错误'
}
});
}
};
export default { quickFetch };
import type { Request, Response } from 'express';
import puppeteer, { Page } from 'puppeteer';
import * as cheerio from 'cheerio';
import UserAgent from 'user-agents';
import { setupPage } from '../utils/setupPage'; // 导入 setupPage 模块
import dotenv from 'dotenv'; // 导入 dotenv 模块
import { URL } from 'url'; // 导入 URL 模块
import { handleSpecialWebsite } from '../specialHandlers'; // 导入 handleSpecialWebsite 模块
import fetch from 'node-fetch';
import { getCachedPage, updateCacheAsync } from '../utils/cacheUpdater'; // 导入缓存相关模块
dotenv.config(); // 加载环境变量
const detectWebsites = process.env.DETECT_WEBSITES?.split(',') || [];
const blacklistDomains = process.env.BLACKLIST ? JSON.parse(process.env.BLACKLIST) : [];
export const readPage = async (req: Request, res: Response): Promise<void> => {
const { queryUrl } = req.query;
console.log('-------');
console.log(queryUrl);
console.log('-------');
if (!queryUrl) {
res.status(400).json({
status: 400,
error: {
code: 'MISSING_PARAM',
message: '缺少必要参数: queryUrl'
}
});
return;
}
const urlDomain = new URL(queryUrl as string).hostname;
if (blacklistDomains.some((domain: string) => urlDomain.endsWith(domain))) {
res.status(403).json({
status: 403,
error: {
code: 'BLACKLISTED_DOMAIN',
message: '该域名受到保护中'
}
});
return;
}
try {
const response = await fetch(queryUrl as string, {
headers: {
'User-Agent': new UserAgent({
deviceCategory: 'desktop',
platform: 'Linux x86_64'
}).toString(),
Referer: 'https://www.google.com/',
'Accept-Language': 'en-US,en;q=0.9',
Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
Connection: 'keep-alive',
'Cache-Control': 'no-cache'
}
});
if (response.ok) {
const content = await response.text();
const $ = cheerio.load(content);
const cleanedContent = $('body').html();
res.status(200).json({
status: 200,
data: {
title: $('title').text(),
content: cleanedContent
}
});
await updateCacheAsync(queryUrl as string, cleanedContent || '');
console.log('Page read successfully');
return;
} else {
throw new Error(`HTTP error! status: ${response.status}`);
}
} catch (error) {
console.error('快速抓取页面时发生错误:', error);
}
try {
const browser = await puppeteer.launch({
ignoreDefaultArgs: ['--enable-automation'],
headless: true,
executablePath: '/usr/bin/chromium', // 明确指定 Chromium 路径
pipe: true,
args: [
'--no-sandbox',
'--disable-setuid-sandbox',
'--disable-dev-shm-usage',
'--disable-gpu'
// '--single-process'
]
});
const page = await browser.newPage();
// 检测是否需要特殊处理
if (
typeof queryUrl === 'string' &&
detectWebsites.some((website) => queryUrl.includes(website))
) {
await setupPage(page);
} else {
const userAgent = new UserAgent({ deviceCategory: 'desktop', platform: 'Linux x86_64' });
await page.setUserAgent(userAgent.toString());
}
const queryUrlSafe = new URL(queryUrl as string).toString();
await page.goto(queryUrlSafe, { waitUntil: 'load' });
await page.waitForSelector('body');
const title = await page.title();
let cleanedContent = await handleSpecialWebsite(page, queryUrl as string);
if (!cleanedContent) {
const content = await page.content();
const $ = cheerio.load(content);
cleanedContent = $('body').html();
}
await page.close();
await browser.close();
res.status(200).json({
status: 200,
data: {
title,
content: cleanedContent
}
});
await updateCacheAsync(queryUrl as string, cleanedContent || '');
console.log('Page read successfully');
} catch (error) {
console.error(error);
res.status(500).json({
status: 500,
error: {
code: 'INTERNAL_SERVER_ERROR',
message: '读取页面时发生内部服务器错误'
}
});
}
};
import type { Request, Response } from 'express';
import { Cluster } from 'puppeteer-cluster';
import dotenv from 'dotenv';
import { performDeepSearch } from '../utils/deepSearch';
import { fetchSearchResults as fetchBaiduResults } from '../engines/baiduEngine';
import { fetchSearchResults as fetchSearchxngResults } from '../engines/searchxngEngine';
dotenv.config();
const strategies = JSON.parse(process.env.STRATEGIES || '[]');
const detectWebsites = process.env.DETECT_WEBSITES?.split(',') || [];
const maxConcurrency = parseInt(process.env.MAX_CONCURRENCY || '10', 10);
export const search = async (req: Request, res: Response): Promise<void> => {
const {
query,
pageCount = 10,
needDetails = 'false',
engine = 'baidu',
categories = 'general'
} = req.query;
const needDetailsBool = needDetails === 'true';
if (!query) {
res.status(400).json({
status: 400,
error: {
code: 'MISSING_PARAM',
message: '缺少必要参数: query'
}
});
return;
}
let fetchSearchResults;
let searchUrlBase;
try {
if (engine === 'baidu') {
fetchSearchResults = fetchBaiduResults;
searchUrlBase = process.env.ENGINE_BAIDUURL;
} else if (engine === 'searchxng') {
fetchSearchResults = fetchSearchxngResults;
searchUrlBase = process.env.ENGINE_SEARCHXNGURL;
} else {
res.status(400).json({
status: 400,
error: {
code: 'INVALID_ENGINE',
message: '无效的搜索引擎'
}
});
return;
}
const { resultUrls, results } = await fetchSearchResults(
query as string,
Number(pageCount),
searchUrlBase || '',
categories as string
);
//如果返回值为空,返回空数组
if (results.size === 0) {
console.log('No results found');
res.status(200).json({
status: 200,
data: {
results: []
}
});
return;
}
if (!needDetailsBool) {
console.log('Need details is false');
results.forEach((value: any) => {
if (value.crawlStatus === 'Pending') {
value.crawlStatus = 'Success';
}
});
res.status(200).json({
status: 200,
data: {
results: Array.from(results.values())
}
});
} else {
console.log('Need details is true');
const clusterInstance = await Cluster.launch({
concurrency: Cluster.CONCURRENCY_CONTEXT,
maxConcurrency: maxConcurrency,
puppeteerOptions: {
ignoreDefaultArgs: ['--enable-automation'],
headless: 'true',
executablePath: '/usr/bin/chromium', // 明确指定 Chromium 路径
pipe: true,
args: [
'--no-sandbox',
'--disable-setuid-sandbox',
'--disable-dev-shm-usage',
'--disable-gpu'
]
}
});
const sortedResults = await performDeepSearch(
clusterInstance,
resultUrls,
results,
strategies,
detectWebsites,
Number(pageCount)
);
res.status(200).json({
status: 200,
data: {
results: sortedResults.slice(0, Number(pageCount))
}
});
}
} catch (error) {
res.status(500).json({
status: 500,
error: {
code: 'INTERNAL_SERVER_ERROR',
message: '发生错误'
}
});
}
};
export default { search };
import { URL } from 'url';
import { JSDOM } from 'jsdom';
import puppeteer from 'puppeteer';
import { setupPage } from '../utils/setupPage';
import { Cluster } from 'puppeteer-cluster';
async function randomWait(min: number, max: number) {
// 随机等待时间
const delay = Math.floor(Math.random() * (max - min + 1)) + min;
return new Promise((resolve) => setTimeout(resolve, delay));
}
export const fetchSearchResults = async (
query: string,
pageCount: number,
searchUrlBase: string,
categories: string
) => {
console.log(`Fetching Baidu search results for query: ${query}`);
// 如果 searchUrlBase 为空,返回空数组
if (!searchUrlBase) {
return { resultUrls: [], results: new Map() };
}
const resultUrls: string[] = [];
const results = new Map<string, any>();
const pagesToFetch = Math.ceil(pageCount / 10);
const browser = await puppeteer.launch({
ignoreDefaultArgs: ['--enable-automation'],
headless: true,
executablePath: '/usr/bin/chromium', // 明确指定 Chromium 路径
pipe: true,
args: [
'--no-sandbox',
'--disable-setuid-sandbox',
'--disable-dev-shm-usage',
'--disable-gpu'
// '--single-process'
]
});
const page = await browser.newPage();
await setupPage(page);
for (let i = 0; i < pagesToFetch; i++) {
const searchUrl = new URL(`${searchUrlBase}?wd=${encodeURIComponent(query)}&pn=${i * 10}`);
console.log(`Fetching page ${i + 1} from Baidu: ${searchUrl.toString()}`);
let retryCount = 0;
let success = false;
while (retryCount < 5 && !success) {
try {
console.time(`Page Load Time for page ${i + 1}`);
await page.goto(searchUrl.toString(), { waitUntil: 'load' });
console.timeEnd(`Page Load Time for page ${i + 1}`);
let content = await page.content();
let dom = new JSDOM(content);
let document = dom.window.document;
console.log(document.title);
// 如果是百度安全验证页面,重新设置页面并重新访问
if (document.title.includes('百度安全验证')) {
console.log('Detected Baidu security verification, retrying...');
await setupPage(page);
retryCount++;
//随机等待时间
await randomWait(1000, 3000);
continue;
}
// 解析搜索结果
console.time(`Link Retrieval Time for page ${i + 1}`);
const resultContainers = document.querySelectorAll('.result.c-container');
for (const result of resultContainers) {
if (resultUrls.length > pageCount + 5) {
break;
}
const titleElement = result.querySelector('h3 a');
const title = titleElement ? titleElement.textContent : '';
const url = titleElement ? titleElement.getAttribute('href') : '';
const contentElement = result.querySelector('[class^="content"]');
const content = contentElement ? contentElement.textContent : '';
if (url) {
resultUrls.push(url);
results.set(url, {
title,
url,
snippet: content,
source: 'baidu',
crawlStatus: 'Pending',
score: 0
});
}
}
console.timeEnd(`Link Retrieval Time for page ${i + 1}`);
success = true;
} catch (error) {
console.error(`Error fetching page ${i + 1}:`, error);
retryCount++;
}
}
}
await browser.close();
console.log('fetch all fake urls');
// 快速检索真实 URL
const urlsToProcessWithPuppeteer = [];
for (const url of resultUrls) {
try {
const response = await fetch(url, {
headers: {
'User-Agent':
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3',
Referer: 'https://www.google.com/',
'Accept-Language': 'en-US,en;q=0.9',
Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
Connection: 'keep-alive',
'Cache-Control': 'no-cache'
}
});
if (response.ok) {
const realUrl = response.url;
console.log('realurl:', realUrl);
const result = results.get(url);
if (result) {
result.url = realUrl;
result.crawlStatus = 'Success';
}
} else {
throw new Error(`HTTP error! status: ${response.status}`);
}
} catch (error) {
console.error(`Error fetching original URL for ${url}:`, error);
urlsToProcessWithPuppeteer.push(url);
}
}
console.log('pass quickfetch');
// 并发处理真实 URL
const cluster = await Cluster.launch({
concurrency: Cluster.CONCURRENCY_CONTEXT,
maxConcurrency: 10,
puppeteerOptions: {
ignoreDefaultArgs: ['--enable-automation'],
headless: 'true',
executablePath: '/usr/bin/chromium', // 明确指定 Chromium 路径
pipe: true,
args: ['--no-sandbox', '--disable-setuid-sandbox', '--disable-dev-shm-usage', '--disable-gpu']
}
});
let failedUrlCount = 0;
await cluster.task(async ({ page, data: url }) => {
let retryUrlCount = 0;
let urlSuccess = false;
while (retryUrlCount < 3 && !urlSuccess) {
console.log(`Fetching original URL for ${url}, attempt ${retryUrlCount + 1}`);
try {
await page.goto(url, { waitUntil: 'load' });
// 检查页面是否被分离
if (page.isClosed()) {
throw new Error('Page has been closed');
}
const realUrl = page.url(); // 获取真实 URL
const result = results.get(url);
if (result) {
result.url = realUrl;
result.crawlStatus = 'Success';
}
urlSuccess = true;
} catch (error) {
console.error(`Error fetching original URL, retrying...`, error);
retryUrlCount++;
await randomWait(1000, 3000);
}
}
if (!urlSuccess) {
failedUrlCount++;
}
});
for (const url of urlsToProcessWithPuppeteer) {
cluster.queue(url);
}
await cluster.idle();
await cluster.close();
console.log(`Number of URLs that failed to return a real URL: ${failedUrlCount}`);
// 过滤并返回前 pageCount 个结果
const filteredResults = Array.from(results.values()).slice(0, pageCount);
return {
resultUrls: filteredResults.map((result) => result.url),
results: new Map(filteredResults.map((result) => [result.url, result]))
};
};
import axios from 'axios';
import { URL } from 'url';
import dotenv from 'dotenv';
dotenv.config();
const blacklistDomains = process.env.BLACKLIST ? JSON.parse(process.env.BLACKLIST) : [];
export const fetchSearchResults = async (
query: string,
pageCount: number,
searchUrlBase: string,
categories: string
) => {
const MAX_PAGES = (pageCount / 10 + 1) * 2 + 1; // 最多搜索的页面数
//如果searchUrlBase为空,返回空数组,pagecount是需要搜索结果的数量
if (!searchUrlBase) {
return { resultUrls: [], results: new Map() };
}
const resultUrls: string[] = [];
const results = new Map<string, any>();
let fetchedResultsCount = 0;
let pageIndex = 0;
while (fetchedResultsCount < pageCount && pageIndex < MAX_PAGES) {
const searchUrl = new URL(
`${searchUrlBase}?q=${encodeURIComponent(query)}&pageno=${pageIndex + 1}&format=json&categories=${categories}`
);
console.log(`Fetching page ${pageIndex + 1} from SearchXNG: ${searchUrl.toString()}`);
const response = await axios.get(searchUrl.toString());
const jsonResults = response.data.results;
for (let index = 0; index < jsonResults.length; index++) {
const result = jsonResults[index];
const resultDomain = new URL(result.url).hostname;
if (
blacklistDomains.some((domain: string) => resultDomain.endsWith(domain)) ||
resultDomain.includes('zhihu')
) {
continue;
}
resultUrls.push(result.url);
results.set(result.url, {
title: result.title,
url: result.url,
snippet: result.content,
source: result.engine,
crawlStatus: 'Pending',
score: result.score
});
fetchedResultsCount++;
if (fetchedResultsCount >= pageCount) {
break;
}
}
pageIndex++;
if (jsonResults.length === 0) {
break; // 如果没有更多结果,退出循环
}
}
return { resultUrls, results };
};
import type { Application } from 'express';
import express from 'express';
import bodyParser from 'body-parser';
import searchRoutes from './routes/searchRoutes';
import readRoutes from './routes/readRoutes';
import quickfetchRoutes from './routes/quickfetchRoutes';
import dotenv from 'dotenv';
dotenv.config();
const app: Application = express();
app.use(bodyParser.json());
app.use('/api', searchRoutes);
app.use('/api', readRoutes);
app.use('/api', quickfetchRoutes);
const PORT = process.env.PORT || 3000;
app.listen(PORT, () => console.log(`Server running on port ${PORT}`));
import type { Request, Response, NextFunction } from 'express';
const authMiddleware = (req: Request, res: Response, next: NextFunction) => {
const bearerHeader = req.headers['authorization'];
if (bearerHeader) {
console.log('bearerHeader:' + bearerHeader);
const bearer = bearerHeader.split(' ');
const bearerToken = bearer[1];
if (bearerToken === process.env.ACCESS_TOKEN) {
next();
} else {
res.status(403).json({ message: 'Invalid token' });
}
} else {
res.status(401).json({ message: 'Bearer token not found' });
}
};
export default authMiddleware;
import express from 'express';
import { quickFetch } from '../controllers/quickfetchController';
import authMiddleware from '../middleware/authMiddleware';
const readRoutes = express.Router();
readRoutes.get('/quickFetch', authMiddleware, quickFetch);
export default readRoutes;
import express from 'express';
import { readPage } from '../controllers/readController';
import authMiddleware from '../middleware/authMiddleware';
const readRoutes = express.Router();
readRoutes.get('/read', authMiddleware, readPage);
export default readRoutes;
import express from 'express';
import searchController from '../controllers/searchController';
import authMiddleware from '../middleware/authMiddleware';
const searchRoutes = express.Router();
searchRoutes.get('/search', authMiddleware, searchController.search);
export default searchRoutes;
import type { Page } from 'puppeteer';
export const handleSpecialWebsite = async (page: Page, url: string): Promise<string | null> => {
if (url.includes('blog.csdn.net')) {
await page.waitForSelector('article');
const content = await page.$eval('article', (el) => el.innerHTML);
return content;
}
if (url.includes('zhuanlan.zhihu.com')) {
console.log('是知乎,需要点击按掉!');
console.log(await page.content());
if (
(await page.content()).includes(
'{"error":{"message":"您当前请求存在异常,暂时限制本次访问。如有疑问,您可以通过手机摇一摇或登录后私信知乎小管家反馈。","code":40362}}'
)
)
return null;
await page.waitForSelector('button[aria-label="关闭"]');
await page.click('button[aria-label="关闭"]'); // 使用 aria-label 选择按钮
await page.waitForSelector('article');
const content = await page.$eval('article', (el) => el.innerHTML);
return content;
}
// 可以添加更多特殊网站的处理逻辑
return null;
};
import NodeCache from 'node-cache';
import { MongoClient } from 'mongodb';
import crypto from 'crypto';
import dotenv from 'dotenv';
dotenv.config();
const cache = new NodeCache({ stdTTL: parseInt(process.env.STD_TTL || '3600') });
const mongoClient = new MongoClient(process.env.MONGODB_URI || 'mongodb://localhost:27017');
const dbName = 'pageCache';
const collectionName = 'pages';
const connectToMongo = async () => {
await mongoClient.connect();
return mongoClient.db(dbName);
};
const createTTLIndex = async () => {
try {
const db = await connectToMongo();
await db
.collection(collectionName)
.createIndex(
{ updatedAt: 1 },
{ expireAfterSeconds: parseInt(process.env.EXPIRE_AFTER_SECONDS || '9000') }
);
console.log('TTL index created successfully');
} catch (error) {
console.error('Error creating TTL index:', error);
}
};
const getPageHash = (content: string) => {
return crypto.createHash('md5').update(content).digest('hex');
};
export const getCachedPage = async (url: string) => {
const cachedPage = cache.get(url);
if (cachedPage) return cachedPage;
try {
const db = await connectToMongo();
const page = await db.collection(collectionName).findOne({ url });
if (page) cache.set(url, page);
return page;
} catch (error) {
console.error('Error getting cached page:', error);
throw error;
}
};
const savePageToCache = async (url: string, content: string) => {
const hash = getPageHash(content);
const page = { url, content, hash, updatedAt: new Date() };
cache.set(url, page); // 更新内存缓存
try {
const db = await connectToMongo();
await db.collection(collectionName).updateOne({ url }, { $set: page }, { upsert: true }); // 更新持久化缓存
} catch (error) {
console.error('Error saving page to cache:', error);
throw error;
}
};
export const updateCacheAsync = async (url: string, content: string) => {
await savePageToCache(url, content);
};
process.on('SIGINT', async () => {
await mongoClient.close();
process.exit(0);
});
// 在应用启动时创建 TTL 索引
createTTLIndex();
import type { Cluster } from 'puppeteer-cluster';
import * as cheerio from 'cheerio';
import UserAgent from 'user-agents';
import { setupPage } from './setupPage';
import { getCachedPage, updateCacheAsync } from './cacheUpdater';
import { handleSpecialWebsite } from '../specialHandlers';
import fetch from 'node-fetch';
interface CachedPage {
url: string;
content: string;
hash: string;
updatedAt: Date;
}
export const performDeepSearch = async (
clusterInstance: Cluster,
resultUrls: string[],
results: Map<string, any>,
strategies: any[],
detectWebsites: string[],
pageCount: number
) => {
const tasks = [];
await clusterInstance.task(async ({ page, data: { searchUrl } }) => {
try {
const cachedPage = (await getCachedPage(searchUrl)) as CachedPage | null;
if (cachedPage) {
const result = results.get(searchUrl);
if (result) {
result.content = cachedPage.content;
result.crawlStatus = 'Success';
}
return;
}
} catch (error) {
console.error(`从缓存获取页面 ${searchUrl} 时发生错误:`, error);
results.set(searchUrl, {
url: searchUrl,
error: (error as Error).message,
crawlStatus: 'Failed'
});
return;
}
try {
const response = await fetch(searchUrl, {
headers: {
'User-Agent': new UserAgent({
deviceCategory: 'desktop',
platform: 'Linux x86_64'
}).toString(),
Referer: 'https://www.google.com/',
'Accept-Language': 'en-US,en;q=0.9',
Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
Connection: 'keep-alive',
'Cache-Control': 'no-cache'
}
});
if (response.ok) {
const content = await response.text();
const $ = cheerio.load(content);
const cleanedContent = $('body').html() || '';
const result = results.get(searchUrl);
if (result) {
result.content = cleanedContent;
result.crawlStatus = 'Success';
}
await updateCacheAsync(searchUrl, cleanedContent || '');
return;
} else {
throw new Error(`HTTP error! status: ${response.status}`);
}
} catch (error) {
console.error(`快速抓取页面 ${searchUrl} 时发生错误:`, error);
}
try {
if (detectWebsites.some((website) => searchUrl.includes(website))) {
await setupPage(page);
} else {
const userAgent = new UserAgent({ deviceCategory: 'desktop', platform: 'Linux x86_64' });
await page.setUserAgent(userAgent.toString());
}
} catch (error) {
console.error(`访问页面 ${searchUrl} 设置用户代理时发生错误:`, error);
}
let pageLoaded = false;
let pageLoadError: Error | null = null;
for (const strategy of strategies) {
try {
await page.goto(searchUrl, { waitUntil: strategy.waitUntil, timeout: strategy.timeout });
pageLoaded = true;
break;
} catch (error: any) {
if (error.name === 'TimeoutError') {
pageLoadError = error;
continue;
} else {
pageLoadError = error;
throw error;
}
}
}
if (!pageLoaded) {
const result = results.get(searchUrl);
if (result) {
result.error = pageLoadError;
result.crawlStatus = 'Failed';
}
return;
}
try {
let cleanedContent = await handleSpecialWebsite(page, searchUrl);
if (!cleanedContent) {
const content = await page.content();
const $ = cheerio.load(content);
cleanedContent = $('body').html() || '';
}
const result = results.get(searchUrl);
if (result) {
result.content = cleanedContent;
result.crawlStatus = 'Success';
}
await updateCacheAsync(searchUrl, cleanedContent || '');
} catch (error) {
results.set(searchUrl, {
url: searchUrl,
error: (error as Error).message,
crawlStatus: 'Failed'
});
} finally {
await page.close().catch(() => {});
}
});
for (const url of resultUrls) {
if (tasks.length >= pageCount + 10) {
break;
}
tasks.push(clusterInstance.queue({ searchUrl: url }));
}
await Promise.all(tasks);
await clusterInstance.idle();
await clusterInstance.close();
return Array.from(results.values()).sort((a, b) => b.score - a.score);
};
import type { Page } from 'puppeteer';
import randomUseragent from 'random-useragent';
import dotenv from 'dotenv';
dotenv.config();
const getRandomUserAgent = () => {
return randomUseragent.getRandom();
};
const getRandomPlatform = () => {
const platforms = ['Win32', 'MacIntel', 'Linux x86_64'];
return platforms[Math.floor(Math.random() * platforms.length)];
};
//代理池
const validateproxy = process.env.VALIDATE_PROXY ? JSON.parse(process.env.VALIDATE_PROXY) : [];
const getRandomProxy = () => {
return validateproxy.length > 0
? validateproxy[Math.floor(Math.random() * validateproxy.length)]
: null;
};
const getRandomLanguages = () => {
const languages = [
['zh-CN', 'zh', 'en'],
['en-US', 'en', 'fr'],
['es-ES', 'es', 'en']
];
return languages[Math.floor(Math.random() * languages.length)];
};
export const setupPage = async (page: Page): Promise<void> => {
const proxy = getRandomProxy();
if (proxy) {
await page.authenticate({
username: proxy.ip,
password: proxy.port.toString()
});
}
await page.evaluateOnNewDocument(() => {
const newProto = (navigator as any).__proto__;
delete newProto.webdriver;
(navigator as any).__proto__ = newProto;
(window as any).chrome = {};
(window as any).chrome.app = {
InstallState: 'testt',
RunningState: 'estt',
getDetails: 'stte',
getIsInstalled: 'ttes'
};
(window as any).chrome.csi = function () {};
(window as any).chrome.loadTimes = function () {};
(window as any).chrome.runtime = function () {};
Object.defineProperty(navigator, 'userAgent', {
get: () => getRandomUserAgent()
});
Object.defineProperty(navigator, 'platform', {
get: () => getRandomPlatform()
});
Object.defineProperty(navigator, 'plugins', {
get: () => [
{
description: 'Shockwave Flash',
filename: 'pepflashplayer.dll',
length: 1,
name: 'Shockwave Flash'
}
]
});
Object.defineProperty(navigator, 'languages', {
get: () => getRandomLanguages()
});
const originalQuery = (window.navigator.permissions as any).query;
(window.navigator.permissions as any).query = (parameters: any) =>
parameters.name === 'notifications'
? Promise.resolve({ state: Notification.permission } as PermissionStatus)
: originalQuery(parameters);
});
};
{
"compilerOptions": {
/* Visit https://aka.ms/tsconfig to read more about this file */
/* Projects */
// "incremental": true, /* Save .tsbuildinfo files to allow for incremental compilation of projects. */
// "composite": true, /* Enable constraints that allow a TypeScript project to be used with project references. */
// "tsBuildInfoFile": "./.tsbuildinfo", /* Specify the path to .tsbuildinfo incremental compilation file. */
// "disableSourceOfProjectReferenceRedirect": true, /* Disable preferring source files instead of declaration files when referencing composite projects. */
// "disableSolutionSearching": true, /* Opt a project out of multi-project reference checking when editing. */
// "disableReferencedProjectLoad": true, /* Reduce the number of projects loaded automatically by TypeScript. */
"types": ["node"],
/* Language and Environment */
"target": "es6", /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */
// "lib": [], /* Specify a set of bundled library declaration files that describe the target runtime environment. */
// "jsx": "preserve", /* Specify what JSX code is generated. */
// "experimentalDecorators": true, /* Enable experimental support for legacy experimental decorators. */
// "emitDecoratorMetadata": true, /* Emit design-type metadata for decorated declarations in source files. */
// "jsxFactory": "", /* Specify the JSX factory function used when targeting React JSX emit, e.g. 'React.createElement' or 'h'. */
// "jsxFragmentFactory": "", /* Specify the JSX Fragment reference used for fragments when targeting React JSX emit e.g. 'React.Fragment' or 'Fragment'. */
// "jsxImportSource": "", /* Specify module specifier used to import the JSX factory functions when using 'jsx: react-jsx*'. */
// "reactNamespace": "", /* Specify the object invoked for 'createElement'. This only applies when targeting 'react' JSX emit. */
// "noLib": true, /* Disable including any library files, including the default lib.d.ts. */
// "useDefineForClassFields": true, /* Emit ECMAScript-standard-compliant class fields. */
// "moduleDetection": "auto", /* Control what method is used to detect module-format JS files. */
/* Modules */
//"module": "es6", /* Specify what module code is generated. */
"rootDir": "./src", /* Specify the root folder within your source files. */
"moduleResolution": "node", /* Specify how TypeScript looks up a file from a given module specifier. */
// "baseUrl": "./", /* Specify the base directory to resolve non-relative module names. */
// "paths": {}, /* Specify a set of entries that re-map imports to additional lookup locations. */
// "rootDirs": [], /* Allow multiple folders to be treated as one when resolving modules. */ /* Specify type package names to be included without being referenced in a source file. */
// "allowUmdGlobalAccess": true, /* Allow accessing UMD globals from modules. */
// "moduleSuffixes": [], /* List of file name suffixes to search when resolving a module. */
// "allowImportingTsExtensions": true, /* Allow imports to include TypeScript file extensions. Requires '--moduleResolution bundler' and either '--noEmit' or '--emitDeclarationOnly' to be set. */
// "rewriteRelativeImportExtensions": true, /* Rewrite '.ts', '.tsx', '.mts', and '.cts' file extensions in relative import paths to their JavaScript equivalent in output files. */
// "resolvePackageJsonExports": true, /* Use the package.json 'exports' field when resolving package imports. */
// "resolvePackageJsonImports": true, /* Use the package.json 'imports' field when resolving imports. */
// "customConditions": [], /* Conditions to set in addition to the resolver-specific defaults when resolving imports. */
// "noUncheckedSideEffectImports": true, /* Check side effect imports. */
// "resolveJsonModule": true, /* Enable importing .json files. */
// "allowArbitraryExtensions": true, /* Enable importing files with any extension, provided a declaration file is present. */
// "noResolve": true, /* Disallow 'import's, 'require's or '<reference>'s from expanding the number of files TypeScript should add to a project. */
/* JavaScript Support */
// "allowJs": true, /* Allow JavaScript files to be a part of your program. Use the 'checkJS' option to get errors from these files. */
// "checkJs": true, /* Enable error reporting in type-checked JavaScript files. */
// "maxNodeModuleJsDepth": 1, /* Specify the maximum folder depth used for checking JavaScript files from 'node_modules'. Only applicable with 'allowJs'. */
/* Emit */
// "declaration": true, /* Generate .d.ts files from TypeScript and JavaScript files in your project. */
// "declarationMap": true, /* Create sourcemaps for d.ts files. */
// "emitDeclarationOnly": true, /* Only output d.ts files and not JavaScript files. */
// "sourceMap": true, /* Create source map files for emitted JavaScript files. */
// "inlineSourceMap": true, /* Include sourcemap files inside the emitted JavaScript. */
// "noEmit": true, /* Disable emitting files from a compilation. */
// "outFile": "./", /* Specify a file that bundles all outputs into one JavaScript file. If 'declaration' is true, also designates a file that bundles all .d.ts output. */
"outDir": "./dist", /* Specify an output folder for all emitted files. */
// "removeComments": true, /* Disable emitting comments. */
// "importHelpers": true, /* Allow importing helper functions from tslib once per project, instead of including them per-file. */
// "downlevelIteration": true, /* Emit more compliant, but verbose and less performant JavaScript for iteration. */
// "sourceRoot": "", /* Specify the root path for debuggers to find the reference source code. */
// "mapRoot": "", /* Specify the location where debugger should locate map files instead of generated locations. */
// "inlineSources": true, /* Include source code in the sourcemaps inside the emitted JavaScript. */
// "emitBOM": true, /* Emit a UTF-8 Byte Order Mark (BOM) in the beginning of output files. */
// "newLine": "crlf", /* Set the newline character for emitting files. */
// "stripInternal": true, /* Disable emitting declarations that have '@internal' in their JSDoc comments. */
// "noEmitHelpers": true, /* Disable generating custom helper functions like '__extends' in compiled output. */
// "noEmitOnError": true, /* Disable emitting files if any type checking errors are reported. */
// "preserveConstEnums": true, /* Disable erasing 'const enum' declarations in generated code. */
// "declarationDir": "./", /* Specify the output directory for generated declaration files. */
/* Interop Constraints */
// "isolatedModules": true, /* Ensure that each file can be safely transpiled without relying on other imports. */
// "verbatimModuleSyntax": true, /* Do not transform or elide any imports or exports not marked as type-only, ensuring they are written in the output file's format based on the 'module' setting. */
// "isolatedDeclarations": true, /* Require sufficient annotation on exports so other tools can trivially generate declaration files. */
// "allowSyntheticDefaultImports": true, /* Allow 'import x from y' when a module doesn't have a default export. */
"esModuleInterop": true, /* Emit additional JavaScript to ease support for importing CommonJS modules. This enables 'allowSyntheticDefaultImports' for type compatibility. */
// "preserveSymlinks": true, /* Disable resolving symlinks to their realpath. This correlates to the same flag in node. */
"forceConsistentCasingInFileNames": true, /* Ensure that casing is correct in imports. */
/* Type Checking */
"typeRoots": ["./node_modules/@types"],
"strict": true, /* Enable all strict type-checking options. */
// "noImplicitAny": true, /* Enable error reporting for expressions and declarations with an implied 'any' type. */
// "strictNullChecks": true, /* When type checking, take into account 'null' and 'undefined'. */
// "strictFunctionTypes": true, /* When assigning functions, check to ensure parameters and the return values are subtype-compatible. */
// "strictBindCallApply": true, /* Check that the arguments for 'bind', 'call', and 'apply' methods match the original function. */
// "strictPropertyInitialization": true, /* Check for class properties that are declared but not set in the constructor. */
// "strictBuiltinIteratorReturn": true, /* Built-in iterators are instantiated with a 'TReturn' type of 'undefined' instead of 'any'. */
// "noImplicitThis": true, /* Enable error reporting when 'this' is given the type 'any'. */
// "useUnknownInCatchVariables": true, /* Default catch clause variables as 'unknown' instead of 'any'. */
// "alwaysStrict": true, /* Ensure 'use strict' is always emitted. */
// "noUnusedLocals": true, /* Enable error reporting when local variables aren't read. */
// "noUnusedParameters": true, /* Raise an error when a function parameter isn't read. */
// "exactOptionalPropertyTypes": true, /* Interpret optional property types as written, rather than adding 'undefined'. */
// "noImplicitReturns": true, /* Enable error reporting for codepaths that do not explicitly return in a function. */
// "noFallthroughCasesInSwitch": true, /* Enable error reporting for fallthrough cases in switch statements. */
// "noUncheckedIndexedAccess": true, /* Add 'undefined' to a type when accessed using an index. */
// "noImplicitOverride": true, /* Ensure overriding members in derived classes are marked with an override modifier. */
// "noPropertyAccessFromIndexSignature": true, /* Enforces using indexed accessors for keys declared using an indexed type. */
// "allowUnusedLabels": true, /* Disable error reporting for unused labels. */
// "allowUnreachableCode": true, /* Disable error reporting for unreachable code. */
/* Completeness */
// "skipDefaultLibCheck": true,
// /* Skip type checking .d.ts files that are included with TypeScript. */
"skipLibCheck": true/* Skip type checking all .d.ts files. */
},
"include": ["src/**/*.ts"],
"exclude": ["node_modules"]
}
// 引入path包
const path = require('path')
require('dotenv').config();
const mode = process.env.NODE_ENV || 'development'
const nodeExternals = require('webpack-node-externals');
module.exports = {
target: 'node', // 指定构建目标为 Node.js
externals: [nodeExternals()], // 排除 node_modules
// 指定入口文件
entry: "./src/index.ts",
// 指定打包文件所在目录
output: {
path: path.resolve(__dirname, 'dist'),
// 打包后文件的名称
filename: "bundle.js"
},
resolve: {
extensions: ['.ts', '.tsx', '.js', '.json'],
fallback: {
"zlib": require.resolve("browserify-zlib"),
"querystring": require.resolve("querystring-es3"),
"path": require.resolve("path-browserify"),
"crypto": require.resolve("crypto-browserify"),
"stream": require.resolve("stream-browserify"),
"os": require.resolve("os-browserify/browser"),
"http": require.resolve("stream-http"),
"net": false,
"string_decoder": require.resolve("string_decoder/"),
"url": require.resolve("url/"),
"buffer": require.resolve("buffer/"),
"util": require.resolve("util/"),
// 新增 assert 的 fallback
"assert": require.resolve("assert/"),
// 处理新出现的 vm 警告
"vm": require.resolve("vm-browserify"),
"fs": false
}
},
// 指定webpack打包的时候要使用的模块
module: {
// 指定要价在的规则
rules: [
{
// test指定的是规则生效的文件,意思是,用ts-loader来处理以ts为结尾的文件
test: /\.ts$/,
use: 'ts-loader',
exclude: /node_modules/
}
]
},
mode,
}
name: spider
version: "2.2"
services:
searxng:
container_name: searxng
image: docker.io/searxng/searxng:latest
platform: linux/amd64
restart: unless-stopped
networks:
- spider_net
ports:
- "8080:8080"
volumes:
- ./searxng:/etc/searxng:rw
environment:
- SEARXNG_BASE_URL=https://${SEARXNG_HOSTNAME:-localhost}/
- UWSGI_WORKERS=4 # UWSGI 工作进程数
- UWSGI_THREADS=4 # UWSGI 线程数
cap_drop:
- ALL
mongodb:
container_name: mongodb
image: mongo:4.4
restart: unless-stopped
networks:
- spider_net
ports:
- "27017:27017"
volumes:
- mongo-data:/data/db
environment:
MONGO_INITDB_ROOT_USERNAME: root # MongoDB 根用户名
MONGO_INITDB_ROOT_PASSWORD: example # MongoDB 根用户密码
nodeapp:
container_name: main
platform: linux/amd64
#build:
# context: .
image: gggaaallleee/webcrawler-test-new:latest
ports:
- "3000:3000"
networks:
- spider_net
depends_on:
- mongodb
logging:
driver: "json-file"
options:
max-size: "1m"
max-file: "1"
volumes:
- /dev/shm:/dev/shm
environment:
- ACCESS_TOKEN=webcrawler # 访问令牌
- DETECT_WEBSITE=zhuanlan.zhihu.com # 无法处理跳过的网站
- STRATEGIES=[{"waitUntil":"networkidle0","timeout":5000},{"waitUntil":"networkidle2","timeout":10000},{"waitUntil":"load","timeout":15000}] # 页面加载策略
- PORT=3000
- MAX_CONCURRENCY=10 # 最大并发数
- NODE_ENV=development
- ENGINE_BAIDUURL=https://www.baidu.com/s # 百度搜索引擎 URL
- ENGINE_SEARCHXNGURL=http://searxng:8080/search # Searxng 搜索引擎 URL
- MONGODB_URI=mongodb://root:example@mongodb:27017 # MongoDB 连接 URI
- BLACKLIST=[".gov.cn",".edu.cn"] # 受保护域名
- STD_TTL=3600 # 标准 TTL(秒)
- EXPIRE_AFTER_SECONDS=9000 # 过期时间(秒)
#- VALIDATE_PROXY=[{"ip":"","port":},{"ip":"","port":}] #代理池
deploy:
resources:
limits:
memory: 4G
cpus: '2.0'
networks:
spider_net:
volumes:
mongo-data:
\ No newline at end of file
# This configuration file updates the default configuration file
# See https://github.com/searxng/searxng/blob/master/searx/limiter.toml
[botdetection.ip_limit]
# activate link_token method in the ip_limit method
link_token = true
general:
debug: false
instance_name: "searxng"
privacypolicy_url: false
donation_url: false
contact_url: false
enable_metrics: true
open_metrics: ''
brand:
new_issue_url: https://github.com/searxng/searxng/issues/new
docs_url: https://docs.searxng.org/
public_instances: https://searx.space
wiki_url: https://github.com/searxng/searxng/wiki
issue_url: https://github.com/searxng/searxng/issues
search:
safe_search: 0
autocomplete: ""
autocomplete_min: 4
default_lang: "auto"
ban_time_on_fail: 5
max_ban_time_on_fail: 120
formats:
- html
server:
port: 8080
bind_address: "0.0.0.0"
base_url: false
limiter: false
public_instance: false
secret_key: "example"
image_proxy: false
http_protocol_version: "1.0"
method: "POST"
default_http_headers:
X-Content-Type-Options: nosniff
X-Download-Options: noopen
X-Robots-Tag: noindex, nofollow
Referrer-Policy: no-referrer
redis:
url: false
ui:
static_path: ""
static_use_hash: false
templates_path: ""
default_theme: simple
default_locale: ""
query_in_title: false
infinite_scroll: false
center_alignment: false
theme_args:
simple_style: auto
# 启用 cn 分类
enabled_categories: [cn,en, general, images,en]
# 或者定义分类显示顺序
categories_order: [cn, en,general, images]
outgoing:
request_timeout: 30.0
max_request_timeout: 40.0
pool_connections: 200
pool_maxsize: 50
enable_http2: false
retries: 5
engines:
- name: bing
engine: bing
disabled: false
categories: cn
#- name: bilibili
# engine: bilibili
# shortcut: bil
# disabled: false
# categories: cn
- name : baidu
engine : json_engine
paging : True
first_page_num : 0
search_url : https://www.baidu.com/s?tn=json&wd={query}&pn={pageno}&rn=50
url_query : url
title_query : title
content_query : abs
categories : cn
- name : 360search
engine: 360search
disabled: false
categories: cn
- name : sogou
disabled: false
categories: cn
- name: google
disabled: false
categories: en
- name: yahoo
disabled: false
categories: en
- name: duckduckgo
disabled: false
categories: en
search:
formats:
- html
- json
doi_resolvers:
oadoi.org: 'https://oadoi.org/'
doi.org: 'https://doi.org/'
doai.io: 'https://dissem.in/'
sci-hub.se: 'https://sci-hub.se/'
sci-hub.st: 'https://sci-hub.st/'
sci-hub.ru: 'https://sci-hub.ru/'
default_doi_resolver: 'oadoi.org'
name: spider
version: "0.0.1"
services:
caddy:
container_name: caddy
image: docker.io/library/caddy:2-alpine
network_mode: host
restart: unless-stopped
volumes:
- ./Caddyfile:/etc/caddy/Caddyfile:ro
- caddy-data:/data:rw
- caddy-config:/config:rw
environment:
- SEARXNG_HOSTNAME=${SEARXNG_HOSTNAME:-http://localhost}
- SEARXNG_TLS=${LETSENCRYPT_EMAIL:-internal}
cap_add:
- NET_BIND_SERVICE
cap_drop:
- ALL
logging:
driver: "json-file"
options:
max-size: "1m"
max-file: "1"
redis:
container_name: redis
image: docker.io/valkey/valkey:8-alpine
command: valkey-server --save 30 1 --loglevel warning
restart: unless-stopped
networks:
- searxng
volumes:
- valkey-data2:/data
cap_drop:
- ALL
cap_add:
- SETGID
- SETUID
- DAC_OVERRIDE
logging:
driver: "json-file"
options:
max-size: "1m"
max-file: "1"
searxng:
container_name: searxng
image: docker.io/searxng/searxng:latest
restart: unless-stopped
networks:
- searxng
ports:
- "127.0.0.1:8080:8080"
volumes:
- ./searxng:/etc/searxng:rw
environment:
- SEARXNG_BASE_URL=https://${SEARXNG_HOSTNAME:-localhost}/
- UWSGI_WORKERS=${SEARXNG_UWSGI_WORKERS:-4}
- UWSGI_THREADS=${SEARXNG_UWSGI_THREADS:-4}
env_file:
- .searchxng.env
cap_drop:
- ALL
cap_add:
- CHOWN
- SETGID
- SETUID
logging:
driver: "json-file"
options:
max-size: "1m"
max-file: "1"
mongodb:
container_name: mongodb
image: mongo:4.4
restart: unless-stopped
networks:
- searxng
ports:
- "27017:27017"
volumes:
- mongo-data:/data/db
environment:
MONGO_INITDB_ROOT_USERNAME: root
MONGO_INITDB_ROOT_PASSWORD: example
logging:
driver: "json-file"
options:
max-size: "1m"
max-file: "1"
nodeapp:
container_name: main
build:
context: .
ports:
- "3000:3000"
networks:
- searxng
depends_on:
- mongodb
logging:
driver: "json-file"
options:
max-size: "1m"
max-file: "1"
volumes:
- /dev/shm:/dev/shm
deploy:
resources:
limits:
memory: 4G
cpus: '2.0'
networks:
searxng:
volumes:
caddy-data:
caddy-config:
valkey-data2:
mongo-data:
\ No newline at end of file
[Unit]
Description=SearXNG service
Requires=docker.service
After=docker.service
[Service]
Restart=on-failure
Environment=SEARXNG_DOCKERCOMPOSEFILE=docker-compose.yaml
WorkingDirectory=/usr/local/searxng-docker
ExecStart=/usr/local/bin/docker compose -f ${SEARXNG_DOCKERCOMPOSEFILE} up --remove-orphans
ExecStop=/usr/local/bin/docker compose -f ${SEARXNG_DOCKERCOMPOSEFILE} down
[Install]
WantedBy=multi-user.target
# This configuration file updates the default configuration file
# See https://github.com/searxng/searxng/blob/master/searx/limiter.toml
[botdetection.ip_limit]
# activate link_token method in the ip_limit method
link_token = true
# see https://docs.searxng.org/admin/settings/settings.html#settings-use-default-settings
use_default_settings: true
server:
# base_url is defined in the SEARXNG_BASE_URL environment variable, see .env and docker-compose.yml
secret_key: "01042f00ae8bb522a9c03d3e7e1910318208a2c9fbdd23a6315577a9c98553a8" # change this!
limiter: false # can be disabled for a private instance
image_proxy: true
ui:
static_use_hash: true
# 启用 cn 分类
enabled_categories: [cn, general, images] # 按需添加其他分类
# 或者定义分类显示顺序
categories_order: [cn, general, images]
redis:
url: redis://redis:6379/0
engines:
- name: bing
disabled: false
categories: cn
#- name: bilibili
# engine: bilibili
# shortcut: bil
# disabled: false
# categories: cn
- name : baidu
engine : json_engine
paging : True
first_page_num : 0
search_url : https://www.baidu.com/s?tn=json&wd={query}&pn={pageno}&rn=50
url_query : url
title_query : title
content_query : abs
categories : cn
search:
formats:
- html
- json
This source diff could not be displayed because it is too large. You can view the blob instead.
packages:
- packages/*
- projects/*
- pro/admin
- pro/sso
- scripts/icon
- sdk/*
allowBuilds:
'@parcel/watcher': true
core-js: true
esbuild: true
mongodb-memory-server: true
msgpackr-extract: true
protobufjs: true
sharp: true
vue-demi: true
catalog:
'@chakra-ui/anatomy': ^2
'@chakra-ui/icons': ^2
'@chakra-ui/next-js': ^2
'@chakra-ui/react': ^2
'@chakra-ui/styled-system': ^2
'@chakra-ui/system': ^2
'@emotion/react': ^11
'@emotion/styled': ^11
'@fastgpt-sdk/logger': 0.1.2
'@fastgpt-sdk/otel': 0.1.2
'@fastgpt-sdk/storage': 0.6.17
'@modelcontextprotocol/sdk': ^1
'@types/lodash': ^4
'@types/mime-types': 3.0.1
'@types/node': ^20
'@types/react': ^18
'@types/react-dom': ^18
"@fastgpt-sdk/logger": 0.1.2
"@fastgpt-sdk/otel": 0.1.2
"@fastgpt-sdk/storage": 0.6.15
"@modelcontextprotocol/sdk": ^1
"@node-rs/jieba": 2.0.1
"@svgr/webpack": ^6.5.1
"@tanstack/react-query": ^4.24.10
"@types/js-yaml": ^4.0.9
"@types/jsonwebtoken": ^9.0.3
"@types/lodash": ^4
"@types/mime-types": ^3.0.1
"@types/node": ^20
"@types/react": ^18
"@types/react-dom": ^18
"@types/request-ip": ^0.0.38
"@typescript-eslint/eslint-plugin": ^6.21.0
"@typescript-eslint/parser": ^6.21.0
"@vitest/coverage-v8": ^4.1.5
ahooks: ^3.9.5
"@chakra-ui/anatomy": ^2
"@chakra-ui/icons": ^2
"@chakra-ui/next-js": ^2
"@chakra-ui/react": ^2
"@chakra-ui/styled-system": ^2
"@chakra-ui/system": ^2
"@emotion/react": ^11
"@emotion/styled": ^11
axios: 1.13.6
chalk: ^5.6.2
date-fns: ^3.6.0
dayjs: 1.11.19
eslint: ^8
eslint: ^8.57.0
eslint-config-next: 15.5.12
express: ^4
file-type: 21.3.0
i18next: 23.16.8
js-yaml: ^4.1.1
jsonwebtoken: ^9.0.3
json5: ^2.2.3
lodash: 4.17.23
mime: ^4.1.0
mime-types: 3.0.2
minio: 8.0.7
next: 16.2.1
mongodb-memory-server: ^10.1.4
mongoose: ^8.10.1
nanoid: ^5.1.3
next: 16.2.4
next-i18next: 15.4.2
next-rspack: 16.2.1
proxy-agent: ^6
react: ^18
react-dom: ^18
react-hook-form: 7.43.1
react-i18next: 14.1.2
react-markdown: ^9.0.1
recharts: ^2.15.0
remark-gfm: ^4.0.1
request-ip: ^3.3.0
tsdown: 0.21.4
tsx: ^4.20.6
typescript: ^5.9.3
vitest: ^4.1.5
zod: ^4
onlyBuiltDependencies:
- '@parcel/watcher'
- "@parcel/watcher"
- bufferutil
- canvas
- core-js
......@@ -71,7 +85,7 @@ onlyBuiltDependencies:
- vue-demi
overrides:
'@types/react': ^18
'@types/react-dom': ^18
"@types/react": ^18
"@types/react-dom": ^18
react: ^18
react-dom: ^18
Subproject commit 20bf396872d2b072018c7d2251cb4631d75500dd
# --------- install dependence -----------
FROM node:20.14.0-alpine AS maindeps
FROM node:24-alpine AS maindeps
WORKDIR /app
ARG proxy
RUN [ -z "$proxy" ] || sed -i 's/dl-cdn.alpinelinux.org/mirrors.ustc.edu.cn/g' /etc/apk/repositories
RUN apk add --no-cache libc6-compat && npm install -g pnpm@9
RUN apk add --no-cache libc6-compat && npm install -g pnpm@10.33.2
# copy packages and one project
COPY pnpm-lock.yaml pnpm-workspace.yaml .npmrc ./
......@@ -23,7 +23,7 @@ RUN if [ -z "$proxy" ]; then \
fi
# --------- builder -----------
FROM node:20.14.0-alpine AS builder
FROM node:24-alpine AS builder
WORKDIR /app
ARG proxy
......@@ -39,10 +39,13 @@ COPY --from=maindeps /app/projects/app/node_modules ./projects/app/node_modules
RUN [ -z "$proxy" ] || sed -i 's/dl-cdn.alpinelinux.org/mirrors.ustc.edu.cn/g' /etc/apk/repositories
RUN apk add --no-cache libc6-compat && npm install -g pnpm@9
RUN apk add --no-cache libc6-compat && npm install -g pnpm@10.33.2
ENV NODE_OPTIONS="--max-old-space-size=4096"
ENV NEXT_PUBLIC_BASE_URL=$base_url
# Build SDK workspace packages first so subpath exports like
# @fastgpt-sdk/otel/logger have generated dist type files.
RUN pnpm build:sdks
RUN pnpm --filter=app build
# Bundle server.ts into a single CJS file; only 'next' is kept external (already in standalone output)
......@@ -66,7 +69,7 @@ RUN rm -rf projects/app/.next/standalone/node_modules/.pnpm/@next+rspack-binding
projects/app/.next/standalone/node_modules/.pnpm/@img+sharp-libvips-linux-x64@*/
# --------- runner -----------
FROM node:20.14.0-alpine AS runner
FROM node:24-alpine AS runner
WORKDIR /app
ARG proxy
......@@ -85,13 +88,16 @@ RUN apk add --no-cache curl ca-certificates \
# copy running files
COPY --from=builder /app/projects/app/public /app/projects/app/public
COPY --from=builder /app/projects/app/next.config.ts /app/projects/app/next.config.ts
COPY --from=builder /app/projects/app/next-i18next.config.js /app/projects/app/next-i18next.config.js
COPY --from=builder --chown=nextjs:nodejs /app/projects/app/.next/standalone /app/
COPY --from=builder --chown=nextjs:nodejs /app/projects/app/.next/static /app/projects/app/.next/static
# next-i18next loads locale json files from ../../packages/web/i18n at runtime.
COPY --from=builder --chown=nextjs:nodejs /app/packages/web/i18n /app/packages/web/i18n
# copy server chunks
COPY --from=builder --chown=nextjs:nodejs /app/projects/app/.next/server/chunks /app/projects/app/.next/server/chunks
# copy worker
COPY --from=builder --chown=nextjs:nodejs /app/projects/app/worker /app/projects/app/worker
COPY --from=builder --chown=nextjs:nodejs /app/projects/app/server-proxy.js /app/projects/app/server-proxy.js
COPY --from=builder --chown=nextjs:nodejs /app/projects/app/worker /app/worker
# copy standload packages
COPY --from=maindeps /app/node_modules/tiktoken ./node_modules/tiktoken
......
import type { NextConfig } from 'next';
import path from 'path';
import withBundleAnalyzerInit from '@next/bundle-analyzer';
import withRspack from 'next-rspack';
const withBundleAnalyzer = withBundleAnalyzerInit({ enabled: process.env.ANALYZE === 'true' });
const basePath = process.env.NEXT_PUBLIC_BASE_URL || undefined;
const isDev = process.env.NODE_ENV === 'development';
const isWebpack = process.env.WEBPACK === '1';
const isRspack = isDev && !isWebpack;
const nextConfig: NextConfig = {
basePath: process.env.NEXT_PUBLIC_BASE_URL,
i18n: {
defaultLocale: 'en',
locales: ['en', 'zh-CN', 'zh-Hant'],
localeDetection: false
},
output: 'standalone',
// 关闭 strict mode,避免第三方库的双重渲染问题
reactStrictMode: !isDev,
productionBrowserSourceMaps: false,
async headers() {
return [
{
source: '/((?!chat\\/share$)(?!proxy\\/)(?!absproxy\\/).*)',
headers: [
const securityHeaders = [
{
key: 'X-Frame-Options',
value: 'DENY'
......@@ -45,143 +24,50 @@ const nextConfig: NextConfig = {
key: 'Permissions-Policy',
value: 'geolocation=(self), microphone=(self), camera=(self)'
}
]
}
];
},
webpack(config, { isServer }) {
config.ignoreWarnings = [
...(config.ignoreWarnings || []),
{
module: /@scalar\/api-reference-react/,
message: /autoprefixer/
},
{
module: /any-promise[\\/]register\.js$/,
message: /Critical dependency: the request of a dependency is an expression/
},
{
module: /bullmq[\\/]dist[\\/](cjs|esm)[\\/]classes[\\/]child-processor\.js$/,
message: /Critical dependency: the request of a dependency is an expression/
},
{
module: /@fastgpt-sdk[\\/]sandbox-adapter[\\/]/,
message: /Critical dependency/
}
];
];
Object.assign(config.resolve!.alias, {
'@mongodb-js/zstd': false,
'@aws-sdk/credential-providers': false,
'gcp-metadata': false,
snappy: false,
aws4: false,
'mongodb-client-encryption': false,
kerberos: false,
'supports-color': false,
'bson-ext': false,
'pg-native': false,
...(isDev &&
(() => {
// In dev, fastgpt-pro + FastGPT nested pnpm workspaces create two separate .pnpm stores,
// causing duplicate module instances (React, Lexical, etc.) and runtime errors like
// "Cannot read properties of null (reading 'useContext')" or
// "Unable to find an active editor state".
// Force all shared packages to resolve from this project's node_modules.
const resolve = (pkg: string) => {
try {
return path.dirname(require.resolve(`${pkg}/package.json`, { paths: [__dirname] }));
} catch {
return undefined;
}
};
const dups = [
'react',
'react-dom',
'lexical',
'@lexical/react',
'@lexical/code',
'@lexical/list',
'@lexical/markdown',
'@lexical/rich-text',
'@lexical/selection',
'@lexical/text',
'@lexical/utils',
const optimizedPackageImports = [
'@chakra-ui/react',
'@chakra-ui/system',
'@chakra-ui/icons',
'lodash',
'framer-motion',
'@emotion/react',
'@emotion/styled',
'use-context-selector'
];
return Object.fromEntries(dups.map((pkg) => [pkg, resolve(pkg)]).filter(([, v]) => v));
})())
});
'@emotion/styled'
];
config.module = {
...config.module,
rules: (config.module?.rules || []).concat([
const nextConfig: NextConfig = {
basePath,
i18n: {
defaultLocale: 'en',
locales: ['en', 'zh-CN', 'zh-Hant'],
localeDetection: false
},
output: 'standalone',
// Strict Mode is development-only; keep it disabled until double-render unsafe code is migrated.
reactStrictMode: false,
compress: true,
poweredByHeader: false,
productionBrowserSourceMaps: false,
async headers() {
return [
{
test: /\.svg$/i,
issuer: /\.[jt]sx?$/,
use: ['@svgr/webpack']
}
]),
exprContextCritical: false,
unknownContextCritical: false
};
if (!config.externals) {
config.externals = [];
source: '/((?!chat/share$).*)',
headers: securityHeaders
}
if (isServer) {
// 这些包只在服务端运行,且内部使用动态 import / 原生可选依赖(ws 的 bufferutil、
// utf-8-validate,pi-ai 的 node:os/provider dynamicImport 等),让 webpack 直接
// externalize,避免扫描源码产生 Critical dependency / Module not found 警告。
config.externals.push({
'@node-rs/jieba': '@node-rs/jieba',
'@mariozechner/pi-ai': 'commonjs @mariozechner/pi-ai',
'@mariozechner/pi-agent-core': 'commonjs @mariozechner/pi-agent-core',
'@google/genai': 'commonjs @google/genai',
ws: 'commonjs ws',
bufferutil: 'commonjs bufferutil',
'utf-8-validate': 'commonjs utf-8-validate'
});
];
},
turbopack: {
root: path.join(__dirname, '../../'),
rules: {
'*.svg': {
loaders: ['@svgr/webpack'],
as: '*.js'
}
config.experiments = {
...config.experiments,
asyncWebAssembly: true
};
if (isDev && !isServer) {
config.devtool = 'cheap-module-source-map';
config.watchOptions = {
...config.watchOptions,
ignored: [
'**/node_modules',
'**/.git',
'**/dist',
'**/coverage',
'../../packages/**/node_modules',
'../../packages/**/dist',
'**/.next',
'**/out'
],
// 减少轮询频率,降低 CPU 和内存占用
poll: 1000,
aggregateTimeout: 300
};
}
return config;
},
transpilePackages: ['@modelcontextprotocol/sdk', 'ahooks'],
serverExternalPackages: [
'mongoose',
'pg',
'@node-rs/jieba',
'bullmq',
'@zilliz/milvus2-sdk-node',
'tiktoken',
......@@ -191,20 +77,7 @@ const nextConfig: NextConfig = {
],
// 优化大库的 barrel exports tree-shaking
experimental: {
optimizePackageImports: [
'@chakra-ui/react',
'@chakra-ui/icons',
'lodash',
'ahooks',
'framer-motion',
'@emotion/react',
'@emotion/styled',
'react-syntax-highlighter',
'recharts',
'@tanstack/react-query',
'react-hook-form',
'react-markdown'
],
optimizePackageImports: optimizedPackageImports,
// 按页面拆分 CSS chunk,减少首屏 CSS 体积
cssChunking: 'strict',
// 减少内存占用
......@@ -231,5 +104,4 @@ const nextConfig: NextConfig = {
}
};
const config = withBundleAnalyzer(nextConfig);
export default isRspack ? withRspack(config) : config;
export default nextConfig;
{
"name": "app",
"name": "@fastgpt/app",
"version": "4.14.16",
"private": false,
"scripts": {
"dev": "NODE_OPTIONS='--max-old-space-size=8192' npm run build:workers && next dev",
"dev:skill": "NODE_OPTIONS='--max-old-space-size=8192' npm run build:workers && tsx server.ts",
"dev:webpack": "NODE_OPTIONS='--max-old-space-size=8192' npm run build:workers && WEBPACK=1 next dev --webpack",
"build": "npm run build:workers && next build --debug --webpack",
"start": "NODE_ENV=production tsx server.ts",
"build:workers": "npx tsx scripts/build-workers.ts",
"dev": "NODE_OPTIONS='--max-old-space-size=8192' pnpm run build:workers && next dev",
"build": "pnpm run build:workers && next build --debug",
"build:webpack": "pnpm run build:workers && next build --webpack --debug",
"analyze": "next experimental-analyze",
"start": "next start",
"test": "vitest run -c vitest.config.ts",
"test:watch": "vitest -c vitest.config.ts",
"build:workers": "tsx scripts/build-workers.ts",
"typecheck": "tsc --noEmit --pretty",
"build:workers:watch": "npx tsx scripts/build-workers.ts --watch"
"build:workers:watch": "tsx scripts/build-workers.ts --watch"
},
"engines": {
"node": ">=20",
"pnpm": "9.x"
"pnpm": "10.x"
},
"dependencies": {
"@chakra-ui/anatomy": "catalog:",
......@@ -26,17 +28,18 @@
"@dagrejs/dagre": "^1.1.4",
"@emotion/react": "catalog:",
"@emotion/styled": "catalog:",
"@fastgpt-sdk/storage": "catalog:",
"@fastgpt-sdk/otel": "workspace:*",
"@fastgpt-sdk/storage": "workspace:*",
"@fastgpt/global": "workspace:*",
"@fastgpt/service": "workspace:*",
"@fastgpt/web": "workspace:*",
"@fortaine/fetch-event-source": "^3.0.6",
"@modelcontextprotocol/sdk": "catalog:",
"@monaco-editor/react": "^4.7.0",
"@node-rs/jieba": "2.0.1",
"@node-rs/jieba": "catalog:",
"@scalar/api-reference-react": "^0.8.1",
"@tanstack/react-query": "^4.24.10",
"ahooks": "^3.9.5",
"@tanstack/react-query": "catalog:",
"ahooks": "catalog:",
"archiver": "^7.0.1",
"axios": "catalog:",
"date-fns": "catalog:",
......@@ -45,6 +48,7 @@
"echarts-gl": "2.0.9",
"esbuild": "^0.25.11",
"framer-motion": "9.1.7",
"http-proxy": "^1.18.1",
"hyperdown": "^2.4.29",
"i18next": "catalog:",
"immer": "^9.0.19",
......@@ -52,13 +56,15 @@
"js-yaml": "catalog:",
"json5": "catalog:",
"jsondiffpatch": "^0.7.2",
"jsonwebtoken": "^9.0.2",
"lodash": "catalog:",
"jsonwebtoken": "catalog:",
"jszip": "^3.10.1",
"katex": "0.16.22",
"lodash": "catalog:",
"mermaid": "^10.9.4",
"mime": "catalog:",
"minio": "catalog:",
"nanoid": "^5.1.3",
"mongoose": "catalog:",
"nanoid": "catalog:",
"next": "catalog:",
"next-i18next": "catalog:",
"nprogress": "^0.2.0",
......@@ -66,47 +72,46 @@
"qrcode": "^1.5.4",
"react": "catalog:",
"react-dom": "catalog:",
"react-hook-form": "7.43.1",
"react-hook-form": "catalog:",
"react-i18next": "catalog:",
"react-markdown": "^9.0.1",
"react-markdown": "catalog:",
"react-syntax-highlighter": "^15.5.0",
"react-textarea-autosize": "^8.5.4",
"reactflow": "^11.7.4",
"recharts": "^2.15.0",
"recharts": "catalog:",
"rehype-external-links": "^3.0.0",
"rehype-katex": "^7.0.0",
"remark-breaks": "^4.0.0",
"remark-gfm": "^4.0.0",
"remark-gfm": "catalog:",
"remark-math": "^6.0.0",
"request-ip": "^3.3.0",
"request-ip": "catalog:",
"sass": "^1.58.3",
"http-proxy": "^1.18.1",
"undici": "^7.18.2",
"use-context-selector": "^1.4.4",
"zod": "catalog:"
},
"devDependencies": {
"@types/http-proxy": "^1.17.15",
"@next/bundle-analyzer": "16.1.6",
"@svgr/webpack": "^6.5.1",
"@svgr/webpack": "catalog:",
"@types/archiver": "^6.0.2",
"@types/js-yaml": "^4.0.9",
"@types/jsonwebtoken": "^9.0.3",
"@types/http-proxy": "^1.17.15",
"@types/js-yaml": "catalog:",
"@types/jsonwebtoken": "catalog:",
"@types/lodash": "catalog:",
"@types/node": "^20.14.2",
"@types/node": "catalog:",
"@types/nprogress": "^0.2.0",
"@types/qrcode": "^1.5.5",
"@types/react": "catalog:",
"@types/react-dom": "catalog:",
"@types/react-syntax-highlighter": "^15.5.6",
"@types/request-ip": "^0.0.37",
"@typescript-eslint/eslint-plugin": "^6.21.0",
"@typescript-eslint/parser": "^6.21.0",
"@types/request-ip": "catalog:",
"@typescript-eslint/eslint-plugin": "catalog:",
"@typescript-eslint/parser": "catalog:",
"eslint": "catalog:",
"eslint-config-next": "catalog:",
"next-rspack": "catalog:",
"tsx": "^4.20.6",
"typescript": "^5.1.3",
"vitest": "^3.0.9"
"tailwindcss": "^3",
"tsx": "catalog:",
"typescript": "catalog:",
"vitest": "catalog:"
}
}
......@@ -6,6 +6,7 @@ import path from 'path';
const ROOT_DIR = path.resolve(__dirname, '../../..');
const WORKER_SOURCE_DIR = path.join(ROOT_DIR, 'packages/service/worker');
const WORKER_OUTPUT_DIR = path.join(__dirname, '../worker');
const OTEL_SDK_DIR = path.join(ROOT_DIR, 'sdk/otel/src');
/**
* Worker 预编译脚本
......@@ -47,6 +48,12 @@ async function buildWorkers(watch: boolean = false) {
minify: true,
treeShaking: true,
keepNames: false,
alias: {
'@fastgpt-sdk/otel': path.join(OTEL_SDK_DIR, 'index.ts'),
'@fastgpt-sdk/otel/logger': path.join(OTEL_SDK_DIR, 'logger-entry.ts'),
'@fastgpt-sdk/otel/metrics': path.join(OTEL_SDK_DIR, 'metrics-entry.ts'),
'@fastgpt-sdk/otel/tracing': path.join(OTEL_SDK_DIR, 'tracing-entry.ts')
},
// 移除调试代码
drop: process.env.NODE_ENV === 'production' ? ['console', 'debugger'] : []
};
......
import type { IncomingMessage } from 'http';
import { createServer, ServerResponse } from 'http';
import { parse } from 'url';
import next from 'next';
import httpProxy from 'http-proxy';
import { Readable } from 'stream';
......@@ -267,7 +266,7 @@ async function main() {
}
const server = createServer(async (req: IncomingMessage, res: ServerResponse) => {
const parsedUrl = parse(req.url || '');
const parsedUrl = new URL(req.url || '/', 'http://localhost');
// ① Check subdomain proxy first: {port}--{sandboxId}.{baseDomain}
const subdomain = parseSubdomainProxy(req.headers.host);
......@@ -287,7 +286,7 @@ async function main() {
}
// ③ Fall through to Next.js handler
handle(req, res, parsedUrl as any);
handle(req, res);
});
// WebSocket upgrade handler — supports all three proxy modes
......
import { exit } from 'process';
import {
runBackgroundInitializationStep,
getInitializationErrorLog,
runInitializationStep
} from '@fastgpt/service/common/system/initError';
export async function registerNodeInstrumentation() {
try {
await runInitializationStep({
step: 'load-proxy',
action: async () => import('@fastgpt/service/common/proxy')
});
const [
{ connectMongo },
{ connectionMongo, connectionLogMongo, MONGO_URL, MONGO_LOG_URL },
{ systemStartCb },
{ initGlobalVariables, getInitConfig, initSystemPluginTags, initAppTemplateTypes },
{ initVectorStore },
{ initRootUser },
{ startMongoWatch },
{ startCron },
{ startTrainingQueue },
{ preLoadWorker },
{ loadSystemModels },
{ getSystemTools },
{ trackTimerProcess },
{ initBullMQWorkers },
{ initS3Buckets },
{ initGeo },
{ instrumentationCheck },
{ getErrText },
{ configureLogger, getLogger, LogCategories },
{ InitialErrorEnum }
] = await Promise.all([
import('@fastgpt/service/common/mongo/init'),
import('@fastgpt/service/common/mongo/index'),
import('@fastgpt/service/common/system/tools'),
import('@/service/common/system'),
import('@fastgpt/service/common/vectorDB/controller'),
import('@/service/mongo'),
import('@/service/common/system/volumnMongoWatch'),
import('@/service/common/system/cron'),
import('@/service/core/dataset/training/utils'),
import('@fastgpt/service/worker/preload'),
import('@fastgpt/service/core/ai/config/utils'),
import('@fastgpt/service/core/app/tool/controller'),
import('@fastgpt/service/common/middle/tracks/processor'),
import('@/service/common/bullmq'),
import('@fastgpt/service/common/s3'),
import('@fastgpt/service/common/geo'),
import('@/service/common/system/health'),
import('@fastgpt/global/common/error/utils'),
import('@fastgpt/service/common/logger'),
import('@fastgpt/service/common/system/constants')
]);
await runInitializationStep({
step: 'configure-logger',
action: () => configureLogger()
});
const logger = getLogger(LogCategories.SYSTEM);
logger.info('Starting system initialization...');
await runInitializationStep({
step: 'system-start-callback',
action: () => systemStartCb(),
logger
});
await runInitializationStep({
step: 'init-global-variables',
action: () => initGlobalVariables(),
logger
});
await Promise.all([
runInitializationStep({
step: 'init-s3-buckets',
stage: InitialErrorEnum.S3_ERROR,
action: () => initS3Buckets(),
logger,
getErrText
}),
runInitializationStep({
step: 'connect-main-mongo',
stage: InitialErrorEnum.MONGO_ERROR,
action: () =>
connectMongo({
db: connectionMongo,
url: MONGO_URL,
connectedCb: () => startMongoWatch()
}),
logger,
getErrText,
meta: {
mongoUrl: MONGO_URL
}
}),
runInitializationStep({
step: 'connect-log-mongo',
stage: InitialErrorEnum.MONGO_ERROR,
action: () =>
connectMongo({
db: connectionLogMongo,
url: MONGO_LOG_URL
}),
logger,
getErrText,
meta: {
mongoLogUrl: MONGO_LOG_URL
}
}),
runInitializationStep({
step: 'init-bullmq-workers',
stage: InitialErrorEnum.REDIS_ERROR,
action: () => initBullMQWorkers(),
logger,
getErrText
}),
runInitializationStep({
step: 'init-vector-store',
stage: InitialErrorEnum.VECTORDB_ERROR,
action: () => initVectorStore(),
logger,
getErrText
})
]);
await runInitializationStep({
step: 'get-init-config',
action: () => getInitConfig(),
logger,
getErrText
});
await runInitializationStep({
step: 'instrumentation-check',
action: () => instrumentationCheck(),
logger,
getErrText
});
await Promise.all([
runInitializationStep({
step: 'init-root-user',
action: () => initRootUser(),
logger,
getErrText
}),
runInitializationStep({
step: 'load-system-models',
stage: InitialErrorEnum.PLUGIN_ERROR,
action: () => loadSystemModels(),
logger,
getErrText
}),
runInitializationStep({
step: 'load-system-tools',
stage: InitialErrorEnum.PLUGIN_ERROR,
action: () => getSystemTools(),
logger,
getErrText
}),
runInitializationStep({
step: 'init-system-plugin-tags',
stage: InitialErrorEnum.PLUGIN_ERROR,
action: () => initSystemPluginTags(),
logger,
getErrText
}),
runInitializationStep({
step: 'init-app-template-types',
action: () => initAppTemplateTypes(),
logger,
getErrText
}),
runInitializationStep({
step: 'preload-worker',
action: () => preLoadWorker(),
logger,
getErrText
}).catch(() => undefined)
]);
await runInitializationStep({
step: 'init-geo',
action: () => initGeo(),
logger,
getErrText
});
await runInitializationStep({
step: 'start-cron',
action: () => startCron(),
logger,
getErrText
});
await runInitializationStep({
step: 'start-training-queue',
action: () => startTrainingQueue(true),
logger,
getErrText
});
runBackgroundInitializationStep({
step: 'track-timer-process',
action: () => trackTimerProcess(),
logger,
getErrText
});
logger.info('System initialized successfully');
} catch (error) {
const logPayload = {
nextRuntime: process.env.NEXT_RUNTIME,
nodeEnv: process.env.NODE_ENV,
...getInitializationErrorLog(error)
};
console.error('System initialization failed', logPayload);
try {
const { getLogger, LogCategories } = await import('@fastgpt/service/common/logger');
getLogger(LogCategories.SYSTEM).error('System initialization failed', logPayload);
} catch (loggerError) {
console.error('Failed to record system initialization failure', {
...getInitializationErrorLog(loggerError)
});
}
exit(1);
}
}
import { exit } from 'process';
/*
Init system
*/
export async function register() {
try {
if (process.env.NEXT_RUNTIME === 'nodejs') {
await import('@fastgpt/service/common/proxy');
// 基础系统初始化
const [
{ connectMongo },
{ connectionMongo, connectionLogMongo, MONGO_URL, MONGO_LOG_URL },
{ systemStartCb },
{ initGlobalVariables, getInitConfig, initSystemPluginTags, initAppTemplateTypes },
{ initVectorStore },
{ initRootUser },
{ startMongoWatch },
{ startCron },
{ startTrainingQueue },
{ preLoadWorker },
{ loadSystemModels },
{ getSystemTools },
{ trackTimerProcess },
{ initBullMQWorkers },
{ initS3Buckets },
{ initGeo },
{ instrumentationCheck },
{ getErrText },
{ configureMetrics },
{ configureTracing },
{ configureLogger, getLogger, LogCategories },
{ InitialErrorEnum }
] = await Promise.all([
import('@fastgpt/service/common/mongo/init'),
import('@fastgpt/service/common/mongo/index'),
import('@fastgpt/service/common/system/tools'),
import('@/service/common/system'),
import('@fastgpt/service/common/vectorDB/controller'),
import('@/service/mongo'),
import('@/service/common/system/volumnMongoWatch'),
import('@/service/common/system/cron'),
import('@/service/core/dataset/training/utils'),
import('@fastgpt/service/worker/preload'),
import('@fastgpt/service/core/ai/config/utils'),
import('@fastgpt/service/core/app/tool/controller'),
import('@fastgpt/service/common/middle/tracks/processor'),
import('@/service/common/bullmq'),
import('@fastgpt/service/common/s3'),
import('@fastgpt/service/common/geo'),
import('@/service/common/system/health'),
import('@fastgpt/global/common/error/utils'),
import('@fastgpt/service/common/metrics'),
import('@fastgpt/service/common/tracing'),
import('@fastgpt/service/common/logger'),
import('@fastgpt/service/common/system/constants')
]);
await configureMetrics();
await configureTracing();
await configureLogger();
const logger = getLogger(LogCategories.SYSTEM);
logger.info('Starting system initialization...');
// 执行初始化流程
systemStartCb();
initGlobalVariables();
// Init infra
await Promise.all([
initS3Buckets(),
connectMongo({
db: connectionMongo,
url: MONGO_URL,
connectedCb: () => startMongoWatch()
}).catch((err) => {
return Promise.reject(`[${InitialErrorEnum.MONGO_ERROR}]: ${getErrText(err)}`);
}),
connectMongo({
db: connectionLogMongo,
url: MONGO_LOG_URL
}).catch((err) => {
return Promise.reject(`[${InitialErrorEnum.MONGO_ERROR}]: ${getErrText(err)}`);
}),
initBullMQWorkers().catch((err) => {
return Promise.reject(`[${InitialErrorEnum.REDIS_ERROR}]: ${getErrText(err)}`);
}),
initVectorStore().catch((err) => {
return Promise.reject(`[${InitialErrorEnum.VECTORDB_ERROR}]: ${getErrText(err)}`);
})
]);
// Init system config
await getInitConfig();
// Check infrastructure
await instrumentationCheck();
// Load init data
await Promise.all([
initRootUser(),
loadSystemModels(),
getSystemTools(),
initSystemPluginTags(),
initAppTemplateTypes(),
preLoadWorker().catch()
]);
initGeo(); // init geo
startCron();
startTrainingQueue(true);
trackTimerProcess();
logger.info('System initialized successfully');
}
} catch (error) {
console.error('System initialization failed', error);
exit(1);
const { registerNodeInstrumentation } = await import('./instrumentation-node');
await registerNodeInstrumentation();
}
}
......@@ -9,6 +9,7 @@ import { formatFileSize } from '@fastgpt/global/common/file/tools';
import { formatTime2YMDHM } from '@fastgpt/global/common/string/time';
import {
DatasetCollectionDataProcessModeMap,
DatasetCollectionDataProcessModeEnum,
DatasetCollectionTypeMap,
DatasetCollectionTypeEnum
} from '@fastgpt/global/core/dataset/constants';
......@@ -44,6 +45,8 @@ const MetaDataCard = ({ datasetId }: { datasetId: string }) => {
if (!collection) return [];
const webSelector = collection?.metadata?.webPageSelector;
const trainingType = collection.trainingType ?? DatasetCollectionDataProcessModeEnum.chunk;
const trainingTypeConfig = DatasetCollectionDataProcessModeMap[trainingType];
return [
{
......@@ -96,7 +99,7 @@ const MetaDataCard = ({ datasetId }: { datasetId: string }) => {
? [
{
label: t('dataset:collection.training_type'),
value: t(DatasetCollectionDataProcessModeMap[collection.trainingType]?.label as any)
value: t(trainingTypeConfig.label as any)
}
]
: []),
......
......@@ -17,6 +17,8 @@ import SystemStoreContextProvider from '@fastgpt/web/context/useSystem';
import { useRouter } from 'next/router';
import { errorLogger } from '@/web/common/utils/errorLogger';
import '@scalar/api-reference-react/style.css';
type NextPageWithLayout = NextPage & {
setLayout?: (page: ReactElement) => JSX.Element;
};
......
......@@ -41,6 +41,7 @@ async function handler(req: ApiRequestProps): Promise<ReTrainingCollectionRespon
createCollectionParams: {
...collection,
...data,
parentId: collection.parentId ?? undefined,
updateTime: new Date(),
tags: await collectionTagsToTagLabel({
datasetId: collection.datasetId,
......
......@@ -47,7 +47,7 @@ export async function getDatasetCollectionPaths({
if (!parent) return [];
const paths = await find(parent.parentId);
const paths = await find(parent.parentId ?? undefined);
paths.push({ parentId, parentName: parent.name });
return paths;
......
......@@ -17,6 +17,7 @@ import { getLLMModel } from '@fastgpt/service/core/ai/model';
import { getVlmModel } from '@fastgpt/service/core/ai/model';
import { createTrainingUsage } from '@fastgpt/service/support/wallet/usage/controller';
import { mongoSessionRun } from '@fastgpt/service/common/mongo/sessionRun';
import { DatasetCollectionDataProcessModeEnum } from '@fastgpt/global/core/dataset/constants';
async function handler(req: ApiRequestProps): Promise<PushDataResponseType> {
const body = PushDataBodySchema.parse(req.body);
......
'use client';
import { Box } from '@chakra-ui/react';
import dynamic from 'next/dynamic';
// 动态加载 @scalar/api-reference-react,避免其 CSS side-effect 在 Node 端
// (next build 的 collecting page data 阶段) 被解析导致 ERR_UNKNOWN_FILE_EXTENSION。
const ApiReferenceReact = dynamic(
() =>
Promise.all([
import('@scalar/api-reference-react'),
import('@scalar/api-reference-react/style.css')
]).then(([mod]) => mod.ApiReferenceReact),
() => Promise.all([import('@scalar/api-reference-react')]).then(([mod]) => mod.ApiReferenceReact),
{ ssr: false }
);
......
......@@ -195,7 +195,7 @@ export const datasetParseQueue = async (): Promise<any> => {
try {
const trainingMode = getTrainingModeByCollection({
trainingType: collection.trainingType,
trainingType: collection.trainingType ?? DatasetCollectionDataProcessModeEnum.chunk,
autoIndexes: collection.autoIndexes,
imageIndex: collection.imageIndex
});
......
......@@ -737,7 +737,7 @@ describe('stream resume helpers', () => {
});
afterEach(() => {
vi.restoreAllMocks();
vi.useRealTimers();
resetStreamResumeMirrorGuardForTest();
});
......@@ -745,7 +745,7 @@ describe('stream resume helpers', () => {
vi.useFakeTimers();
try {
const redis = getGlobalRedisConnection() as any;
const delSpy = vi.spyOn(redis, 'del').mockResolvedValue(1);
redis.del.mockClear?.();
const mirror = mirrorChatStream({
teamId,
......@@ -761,8 +761,8 @@ describe('stream resume helpers', () => {
const keys = getStreamResumeRedisKeys({ teamId, appId, chatId });
const rawStream = `${FASTGPT_REDIS_PREFIX}${keys.keyOfStream}`;
expect(delSpy).toHaveBeenCalledWith(keys.keyOfUnavailable);
expect(delSpy).toHaveBeenCalledWith(keys.keyOfStream);
expect(redis.del).toHaveBeenCalledWith(keys.keyOfUnavailable);
expect(redis.del).toHaveBeenCalledWith(keys.keyOfStream);
expect(redis.call).toHaveBeenNthCalledWith(
1,
'XADD',
......@@ -810,6 +810,7 @@ describe('stream resume helpers', () => {
it('should clear old redis mirror when mirror starts (before first chunk)', async () => {
const redis = getGlobalRedisConnection() as any;
redis.del.mockClear?.();
const { keyOfStream } = getStreamResumeRedisKeys({
teamId,
appId,
......@@ -826,12 +827,13 @@ describe('stream resume helpers', () => {
await mirror.flush();
expect(redis.del).toHaveBeenCalledWith(keyOfStream);
expect(await redis.get(keyOfStream)).toBeFalsy();
});
it('should continue mirroring chunks after the original response is already closed', async () => {
const redis = getGlobalRedisConnection() as any;
const delSpy = vi.spyOn(redis, 'del').mockResolvedValue(1);
redis.del.mockClear?.();
const mirror = mirrorChatStream({
teamId,
......@@ -847,8 +849,8 @@ describe('stream resume helpers', () => {
const keys = getStreamResumeRedisKeys({ teamId, appId, chatId });
const rawStream = `${FASTGPT_REDIS_PREFIX}${keys.keyOfStream}`;
expect(delSpy).toHaveBeenCalledWith(keys.keyOfUnavailable);
expect(delSpy).toHaveBeenCalledWith(keys.keyOfStream);
expect(redis.del).toHaveBeenCalledWith(keys.keyOfUnavailable);
expect(redis.del).toHaveBeenCalledWith(keys.keyOfStream);
expect(redis.call).toHaveBeenNthCalledWith(1, 'XADD', rawStream, '*', 'raw', 'event: answer\n');
expect(redis.call).toHaveBeenNthCalledWith(2, 'XADD', rawStream, '*', 'raw', 'data: hello\n\n');
});
......@@ -878,7 +880,7 @@ describe('stream resume helpers', () => {
it('should skip creating a mirror when redis memory usage crosses the watermark', async () => {
const redis = getGlobalRedisConnection() as any;
const usedMemory = Math.ceil(STREAM_RESUME_REDIS_MAXMEMORY_RATIO * 100) + 1;
const setSpy = vi.spyOn(redis, 'set');
redis.set.mockClear?.();
redis.info = vi.fn().mockResolvedValue(`used_memory:${usedMemory}\r\nmaxmemory:100\r\n`);
const mirror = await getStreamResumeMirror({
......@@ -890,7 +892,7 @@ describe('stream resume helpers', () => {
expect(mirror).toBeUndefined();
expect(redis.info).toHaveBeenCalledTimes(1);
expect(setSpy).toHaveBeenCalledWith(
expect(redis.set).toHaveBeenCalledWith(
getStreamResumeRedisKeys({ teamId, appId, chatId }).keyOfUnavailable,
JSON.stringify({
reason: StreamResumeUnavailableReasonEnum.memoryPressure
......
......@@ -9,8 +9,8 @@ import { setCookie } from '@fastgpt/service/support/permission/auth/common';
import { pushTrack } from '@fastgpt/service/common/middle/tracks/utils';
import { addAuditLog } from '@fastgpt/service/support/user/audit/util';
import { UserErrEnum } from '@fastgpt/global/common/error/code/user';
import { Call } from '@test/utils/request';
import type { LoginByPasswordBodyType } from '@fastgpt/global/openapi/support/user/account/login/api';
import { Call } from '@test/utils/request';
import { initTeamFreePlan } from '@fastgpt/service/support/wallet/sub/utils';
describe('loginByPassword API', () => {
......
......@@ -17,7 +17,14 @@
"baseUrl": ".",
"paths": {
"@/*": ["../src/*"],
"@fastgpt-sdk/logger": ["../../../sdk/logger/src/index.ts"],
"@fastgpt-sdk/storage": ["../../../sdk/storage/src/index.ts"],
"@fastgpt-sdk/otel": ["../../../sdk/otel/src/index.ts"],
"@fastgpt-sdk/otel/logger": ["../../../sdk/otel/src/logger-entry.ts"],
"@fastgpt-sdk/otel/metrics": ["../../../sdk/otel/src/metrics-entry.ts"],
"@fastgpt-sdk/otel/tracing": ["../../../sdk/otel/src/tracing-entry.ts"],
"@fastgpt/*": ["../../../packages/*"],
"#fastgpt/app/test/*": ["./*"],
"@test/*": ["../../../test/*"]
}
},
......
......@@ -4,8 +4,8 @@
"baseUrl": ".",
"paths": {
"@/*": ["./src/*"],
"@test/*": ["../../test/*"],
"@t3-oss/env-core": ["../../packages/service/node_modules/@t3-oss/env-core/dist/index.d.ts"]
"#fastgpt/app/test/*": ["./test/*"],
"@test/*": ["../../test/*"]
}
},
"include": [
......@@ -15,5 +15,14 @@
"../../packages/**/*.ts",
"../../packages/**/*.tsx"
],
"exclude": ["**/*.test.ts", "**/*.test.tsx", ".next", "dist", "coverage"]
"exclude": [
"**/*.test.ts",
"**/*.test.tsx",
"../../packages/**/vitest.config.ts",
"../../packages/**/vitest.integration.config.ts",
"../../packages/**/test/**",
".next",
"dist",
"coverage"
]
}
import { resolve } from 'node:path';
import { defineConfig } from 'vitest/config';
export default defineConfig({
resolve: {
alias: {
'@': resolve('src'),
'@fastgpt-sdk/logger': resolve('../../sdk/logger/src/index.ts'),
'@fastgpt-sdk/storage': resolve('../../sdk/storage/src/index.ts'),
'@fastgpt-sdk/otel/logger': resolve('../../sdk/otel/src/logger-entry.ts'),
'@fastgpt-sdk/otel/metrics': resolve('../../sdk/otel/src/metrics-entry.ts'),
'@fastgpt-sdk/otel/tracing': resolve('../../sdk/otel/src/tracing-entry.ts'),
'@fastgpt-sdk/otel': resolve('../../sdk/otel/src/index.ts'),
'@fastgpt': resolve('../../packages'),
'#fastgpt/app/test': resolve('test'),
'@test': resolve('../../test')
}
},
test: {
env: {
FILE_TOKEN_KEY:
process.env.FILE_TOKEN_KEY ??
'bfd697e7e798f75deaf2d31210bc93a2e41ad4eed9e7831071d77821b7b97cff'
},
coverage: {
enabled: true,
reporter: ['text', 'text-summary', 'html', 'json-summary', 'json'],
reportOnFailure: true,
include: ['src/**/*.ts', 'src/**/*.tsx'],
exclude: [
'**/node_modules/**',
'**/*.spec.ts',
'**/*/*.d.ts',
'**/test/**',
'**/*.test.ts',
'**/*/constants.ts',
'**/*/*.const.ts',
'**/*/type.ts',
'**/*/types.ts',
'**/*/type/*',
'**/*/schema.ts',
'**/*/*.schema.ts'
],
cleanOnRerun: false
},
outputFile: 'test-results.json',
setupFiles: '../../test/setup.ts',
globalSetup: '../../test/globalSetup.ts',
fileParallelism: false,
maxConcurrency: 10,
pool: 'threads',
testTimeout: 20000,
hookTimeout: 30000,
reporters: ['github-actions', 'default'],
include: ['test/**/*.test.ts']
}
});
......@@ -5,12 +5,13 @@ WORKDIR /app
ARG proxy
# 安装 pnpm
RUN apk add --no-cache nodejs npm && npm install -g pnpm@9
RUN apk add --no-cache nodejs npm && npm install -g pnpm@10.33.2
# 复制 workspace 配置和依赖包
COPY pnpm-lock.yaml pnpm-workspace.yaml package.json ./
COPY packages/global ./packages/global
COPY packages/service ./packages/service
COPY sdk ./sdk
COPY projects/code-sandbox/ ./projects/code-sandbox/
RUN [ -z "$proxy" ] || sed -i 's/dl-cdn.alpinelinux.org/mirrors.ustc.edu.cn/g' /etc/apk/repositories
......@@ -23,6 +24,9 @@ RUN if [ -z "$proxy" ]; then \
pnpm install --frozen-lockfile --ignore-scripts --registry=https://registry.npmmirror.com; \
fi
# 先构建 SDK workspace 包,确保 dist 入口可被 bun build 解析
RUN pnpm --filter @fastgpt-sdk/logger --filter @fastgpt-sdk/otel --filter @fastgpt-sdk/storage build
# 编译主入口文件
RUN cd /app/projects/code-sandbox && pnpm build
......
{
"name": "code-sandbox",
"name": "@fastgpt/code-sandbox",
"version": "5.0.0",
"description": "FastGPT Code Sandbox - Bun + Hono + 统一子进程模型",
"author": "",
......@@ -14,10 +14,10 @@
},
"engines": {
"node": ">=20",
"pnpm": "9.x"
"pnpm": "10.x"
},
"dependencies": {
"@fastgpt-sdk/logger": "catalog:",
"@fastgpt-sdk/logger": "workspace:*",
"@fastgpt/service": "workspace:*",
"axios": "catalog:",
"crypto-js": "^4.2.0",
......@@ -33,9 +33,9 @@
},
"devDependencies": {
"@types/bun": "^1.2.4",
"@types/node": "^20.14.2",
"vitest": "^3.0.9",
"@vitest/coverage-v8": "^3.0.9",
"typescript": "^5.7.3"
"@types/node": "catalog:",
"vitest": "catalog:",
"@vitest/coverage-v8": "catalog:",
"typescript": "catalog:"
}
}
......@@ -6,7 +6,6 @@ export default defineConfig({
enabled: true,
reporter: ['text', 'text-summary', 'html', 'json-summary', 'json'],
reportOnFailure: true,
all: false, // 只包含被测试实际覆盖的文件,不包含空目录
include: ['src/**/*.ts'],
cleanOnRerun: false
},
......
# --------- install dependence -----------
FROM node:20.14.0-alpine AS maindeps
FROM node:24-alpine AS maindeps
WORKDIR /app
ARG proxy
RUN [ -z "$proxy" ] || sed -i 's/dl-cdn.alpinelinux.org/mirrors.ustc.edu.cn/g' /etc/apk/repositories
RUN apk add --no-cache libc6-compat && npm install -g pnpm@9
RUN apk add --no-cache libc6-compat && npm install -g pnpm@10.33.2
# copy packages and one project
COPY pnpm-lock.yaml pnpm-workspace.yaml .npmrc ./
COPY ./packages ./packages
COPY ./sdk ./sdk
COPY ./projects/marketplace/package.json ./projects/marketplace/package.json
RUN [ -f pnpm-lock.yaml ] || (echo "Lockfile not found." && exit 1)
......@@ -22,7 +23,7 @@ RUN if [ -z "$proxy" ]; then \
fi
# --------- builder -----------
FROM node:20.14.0-alpine AS builder
FROM node:24-alpine AS builder
WORKDIR /app
ARG proxy
......@@ -32,19 +33,21 @@ ARG base_url
COPY package.json pnpm-workspace.yaml .npmrc tsconfig.json ./
COPY --from=maindeps /app/node_modules ./node_modules
COPY --from=maindeps /app/packages ./packages
COPY --from=maindeps /app/sdk ./sdk
COPY ./projects/marketplace ./projects/marketplace
COPY --from=maindeps /app/projects/marketplace/node_modules ./projects/marketplace/node_modules
RUN [ -z "$proxy" ] || sed -i 's/dl-cdn.alpinelinux.org/mirrors.ustc.edu.cn/g' /etc/apk/repositories
RUN apk add --no-cache libc6-compat && npm install -g pnpm@9
RUN apk add --no-cache libc6-compat && npm install -g pnpm@10.33.2
ENV NODE_OPTIONS="--max-old-space-size=4096"
ENV NEXT_PUBLIC_BASE_URL=$base_url
RUN pnpm --filter @fastgpt-sdk/logger --filter @fastgpt-sdk/otel --filter @fastgpt-sdk/storage build
RUN pnpm --filter=marketplace build
# --------- runner -----------
FROM node:20.14.0-alpine AS runner
FROM node:24-alpine AS runner
WORKDIR /app
ARG proxy
......
......@@ -2,6 +2,8 @@ const { i18n } = require('./next-i18next.config.js');
const path = require('path');
const isDev = process.env.NODE_ENV === 'development';
const monorepoRoot = path.join(__dirname, '../../');
const emptyModulePath = './packages/service/common/system/emptyModule.js';
/** @type {import('next').NextConfig} */
const nextConfig = {
......@@ -41,82 +43,33 @@ const nextConfig = {
}
];
},
webpack(config, { isServer, nextRuntime }) {
Object.assign(config.resolve.alias, {
'@mongodb-js/zstd': false,
'@aws-sdk/credential-providers': false,
snappy: false,
aws4: false,
'mongodb-client-encryption': false,
kerberos: false,
'supports-color': false,
'bson-ext': false,
'pg-native': false
});
config.module = {
...config.module,
rules: config.module.rules.concat([
{
test: /\.svg$/i,
issuer: /\.[jt]sx?$/,
use: ['@svgr/webpack']
}
]),
exprContextCritical: false,
unknownContextCritical: false
};
if (!config.externals) {
config.externals = [];
}
if (isServer) {
config.externals.push('@node-rs/jieba');
if (nextRuntime === 'nodejs') {
}
} else {
config.resolve = {
...config.resolve,
fallback: {
...config.resolve.fallback,
fs: false
turbopack: {
root: monorepoRoot,
resolveAlias: {
'@mongodb-js/zstd': emptyModulePath,
'@aws-sdk/credential-providers': emptyModulePath,
snappy: emptyModulePath,
aws4: emptyModulePath,
'mongodb-client-encryption': emptyModulePath,
kerberos: emptyModulePath,
'supports-color': emptyModulePath,
'bson-ext': emptyModulePath,
'pg-native': emptyModulePath,
fs: {
browser: emptyModulePath
}
};
}
config.experiments = {
asyncWebAssembly: true,
layers: true
};
if (isDev && !isServer) {
// 使用更快的 source map
config.devtool = 'eval-cheap-module-source-map';
// 减少文件监听范围
config.watchOptions = {
...config.watchOptions,
ignored: ['**/node_modules', '**/.git', '**/dist', '**/coverage']
};
// 启用持久化缓存
config.cache = {
type: 'filesystem',
name: 'client',
buildDependencies: {
config: [__filename]
},
cacheDirectory: path.resolve(__dirname, '.next/cache/webpack'),
maxMemoryGenerations: isDev ? 5 : Infinity,
maxAge: 7 * 24 * 60 * 60 * 1000 // 7 天
};
rules: {
'*.svg': {
loaders: ['@svgr/webpack'],
as: '*.js'
}
}
return config;
},
// 需要转译的包
transpilePackages: ['@modelcontextprotocol/sdk', 'ahooks'],
serverExternalPackages: [
'@node-rs/jieba',
'mongoose',
'pg',
'bullmq',
......@@ -125,9 +78,9 @@ const nextConfig = {
'@opentelemetry/api-logs'
],
experimental: {
outputFileTracingRoot: path.join(__dirname, '../../'),
workerThreads: true
}
},
outputFileTracingRoot: monorepoRoot
};
module.exports = nextConfig;
{
"name": "marketplace",
"name": "@fastgpt/marketplace",
"version": "0.1.0",
"private": true,
"scripts": {
......@@ -10,7 +10,7 @@
},
"engines": {
"node": ">=20",
"pnpm": "9.x"
"pnpm": "10.x"
},
"dependencies": {
"@chakra-ui/anatomy": "catalog:",
......@@ -20,12 +20,12 @@
"@chakra-ui/styled-system": "catalog:",
"@chakra-ui/system": "catalog:",
"@fastgpt/global": "workspace:*",
"@fastgpt-sdk/logger": "catalog:",
"@fastgpt-sdk/logger": "workspace:*",
"@fastgpt/service": "workspace:*",
"@fastgpt/web": "workspace:*",
"axios": "catalog:",
"i18next": "catalog:",
"mongoose": "^8.10.1",
"mongoose": "catalog:",
"next": "catalog:",
"next-i18next": "catalog:",
"react": "catalog:",
......@@ -34,14 +34,14 @@
"zod": "catalog:"
},
"devDependencies": {
"@svgr/webpack": "^6.5.1",
"@types/node": "^20",
"@svgr/webpack": "catalog:",
"@types/node": "catalog:",
"@types/react": "catalog:",
"@types/react-dom": "catalog:",
"eslint": "catalog:",
"eslint-config-next": "catalog:",
"postcss": "^8",
"tailwindcss": "^3.4.1",
"typescript": "^5"
"typescript": "catalog:"
}
}
import { exit } from 'process';
import {
getInitializationErrorLog,
runInitializationStep
} from '@fastgpt/service/common/system/initError';
export async function register() {
try {
if (process.env.NEXT_RUNTIME === 'nodejs') {
const { configureLogger, getLogger, LogCategories } = await import('@/service/logger');
await configureLogger();
await runInitializationStep({
step: 'configure-logger',
action: () => configureLogger()
});
const logger = getLogger(LogCategories.SYSTEM);
await import('@fastgpt/service/common/proxy');
await runInitializationStep({
step: 'load-proxy',
action: async () => import('@fastgpt/service/common/proxy'),
logger
});
const [{ getToolList }, { connectMongo, connectionMongo, MONGO_URL }] = await Promise.all([
import('@/service/tool/data'),
import('@/service/mongo')
]);
await connectMongo(connectionMongo, MONGO_URL);
await getToolList();
await runInitializationStep({
step: 'connect-main-mongo',
action: () => connectMongo(connectionMongo, MONGO_URL),
logger,
meta: {
mongoUrl: MONGO_URL
}
});
await runInitializationStep({
step: 'load-tool-list',
action: () => getToolList(),
logger
});
logger.info('Init system success');
}
} catch (error) {
const logPayload = {
nextRuntime: process.env.NEXT_RUNTIME,
nodeEnv: process.env.NODE_ENV,
...getInitializationErrorLog(error)
};
console.error('Init system error', logPayload);
try {
const { getLogger, LogCategories } = await import('@/service/logger');
getLogger(LogCategories.SYSTEM).error('Init system error', { error });
getLogger(LogCategories.SYSTEM).error('Init system error', logPayload);
} catch (loggerError) {
console.error('Failed to record init system error', {
...getInitializationErrorLog(loggerError)
});
}
exit(1);
}
}
......@@ -17,7 +17,14 @@
"baseUrl": ".",
"paths": {
"@/*": ["../src/*"],
"@fastgpt-sdk/logger": ["../../../sdk/logger/src/index.ts"],
"@fastgpt-sdk/storage": ["../../../sdk/storage/src/index.ts"],
"@fastgpt-sdk/otel": ["../../../sdk/otel/src/index.ts"],
"@fastgpt-sdk/otel/logger": ["../../../sdk/otel/src/logger-entry.ts"],
"@fastgpt-sdk/otel/metrics": ["../../../sdk/otel/src/metrics-entry.ts"],
"@fastgpt-sdk/otel/tracing": ["../../../sdk/otel/src/tracing-entry.ts"],
"@fastgpt/*": ["../../../packages/*"],
"#fastgpt/marketplace/test/*": ["./*"],
"@test/*": ["../../../test/*"]
}
},
......
......@@ -3,10 +3,14 @@
"compilerOptions": {
"baseUrl": ".",
"paths": {
"@/*": ["./src/*"],
"@t3-oss/env-core": ["../../packages/service/node_modules/@t3-oss/env-core/dist/index.d.ts"]
"@/*": ["./src/*"]
}
},
"include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", "../../packages/**/*.ts"],
"exclude": ["**/*.test.ts"]
"exclude": [
"**/*.test.ts",
"../../packages/**/vitest.config.ts",
"../../packages/**/vitest.integration.config.ts",
"../../packages/**/test/**"
]
}
# --------- Install -----------
FROM node:20.14.0-alpine AS install
FROM node:24-alpine AS install
WORKDIR /app
ARG proxy
RUN npm install -g pnpm@9
RUN npm install -g pnpm@10.33.2
# 复制package.json
COPY pnpm-lock.yaml pnpm-workspace.yaml ./
COPY packages/global ./packages/global
COPY packages/service ./packages/service
COPY packages/web ./packages/web
COPY sdk ./sdk
COPY projects/mcp_server/package.json ./projects/mcp_server/package.json
RUN [ -z "$proxy" ] || sed -i 's/dl-cdn.alpinelinux.org/mirrors.ustc.edu.cn/g' /etc/apk/repositories
......@@ -28,7 +29,7 @@ RUN if [ -z "$proxy" ]; then \
fi
# --------- builder -----------
FROM node:20.14.0-alpine AS builder
FROM node:24-alpine AS builder
WORKDIR /app
ARG proxy
......@@ -36,20 +37,22 @@ ARG proxy
COPY package.json pnpm-workspace.yaml .npmrc tsconfig.json ./
COPY ./projects/mcp_server /app/projects/mcp_server
COPY --from=install /app/packages /app/packages
COPY --from=install /app/sdk /app/sdk
COPY --from=install /app/node_modules /app/node_modules
COPY --from=install /app/projects/mcp_server/node_modules /app/projects/mcp_server/node_modules
RUN [ -z "$proxy" ] || sed -i 's/dl-cdn.alpinelinux.org/mirrors.ustc.edu.cn/g' /etc/apk/repositories
RUN apk add --no-cache libc6-compat curl bash && npm install -g pnpm@9
RUN apk add --no-cache libc6-compat curl bash && npm install -g pnpm@10.33.2
# Install curl and bash, then install bun
RUN curl -fsSL https://bun.sh/install | bash
ENV PATH="/root/.bun/bin:$PATH"
RUN pnpm --filter=mcp_server build
RUN pnpm --filter @fastgpt-sdk/logger build
RUN pnpm --filter=@fastgpt/mcp_server build
# runner
FROM node:20.14.0-alpine AS runner
FROM node:24-alpine AS runner
WORKDIR /app
ARG proxy
......
{
"name": "mcp_server",
"name": "@fastgpt/mcp_server",
"version": "0.1",
"keywords": [],
"author": "fastgpt",
......@@ -15,13 +15,13 @@
},
"engines": {
"node": ">=20",
"pnpm": "9.x"
"pnpm": "10.x"
},
"dependencies": {
"@fastgpt/global": "workspace:*",
"@fastgpt-sdk/logger": "catalog:",
"@fastgpt-sdk/logger": "workspace:*",
"@modelcontextprotocol/sdk": "catalog:",
"chalk": "^5.3.0",
"chalk": "catalog:",
"dayjs": "catalog:",
"dotenv": "^16.5.0",
"express": "catalog:"
......
......@@ -11,10 +11,10 @@
},
"dependencies": {
"hono": "^4.6.0",
"zod": "^4"
"zod": "catalog:"
},
"devDependencies": {
"@types/bun": "latest",
"vitest": "^2.0.0"
"vitest": "catalog:"
}
}
{
"name": "icon",
"name": "@fastgpt/icon",
"version": "1.0.0",
"description": "",
"main": "index.js",
......
......@@ -29,7 +29,7 @@
"repository": {
"type": "git",
"url": "https://github.com/labring/FastGPT.git",
"directory": "FastGPT/sdk/logger"
"directory": "sdk/logger"
},
"homepage": "https://github.com/labring/FastGPT",
"bugs": {
......@@ -40,7 +40,7 @@
},
"engines": {
"node": ">=20",
"pnpm": "9.x"
"pnpm": "10.x"
},
"license": "Apache-2.0",
"dependencies": {
......
......@@ -54,7 +54,7 @@
},
"engines": {
"node": ">=20",
"pnpm": ">=9"
"pnpm": "10.x"
},
"license": "Apache-2.0",
"dependencies": {
......
......@@ -10,7 +10,7 @@
"repository": {
"type": "git",
"url": "https://github.com/labring/FastGPT.git",
"directory": "FastGPT/packages/storage"
"directory": "sdk/storage"
},
"homepage": "https://github.com/labring/FastGPT",
"bugs": {
......@@ -21,7 +21,7 @@
},
"engines": {
"node": ">=20",
"pnpm": "9.x"
"pnpm": "10.x"
},
"exports": {
".": {
......@@ -53,8 +53,8 @@
"ali-oss": "^6.23.0",
"cos-nodejs-sdk-v5": "^2.15.4",
"es-toolkit": "^1.43.0",
"minio": "8.0.7",
"vitest": "^4.0.16"
"minio": "catalog:",
"vitest": "catalog:"
},
"devDependencies": {
"@types/ali-oss": "^6.16.13",
......
import { describe, expect, it } from 'vitest';
import { ChatRoleEnum } from '@fastgpt/global/core/chat/constants';
import { FlowNodeTypeEnum } from '@fastgpt/global/core/workflow/node/constant';
import type { ChatItemMiniType } from '@fastgpt/global/core/chat/type';
import {
transformPreviewHistories,
addStatisticalDataToHistoryItem
} from '@/global/core/chat/utils';
const mockResponseData = {
id: '1',
nodeId: '1',
moduleName: 'test',
moduleType: FlowNodeTypeEnum.chatNode
};
describe('transformPreviewHistories', () => {
it('should transform histories correctly with responseDetail=true', () => {
const histories: ChatItemMiniType[] = [
{
obj: ChatRoleEnum.AI,
value: [{ text: { content: 'test response' } }],
responseData: [
{
...mockResponseData,
runningTime: 1.5
}
]
}
];
const result = transformPreviewHistories(histories, true);
expect(result[0]).toEqual({
obj: ChatRoleEnum.AI,
value: [{ text: { content: 'test response' } }],
responseData: undefined,
useAgentSandbox: false,
llmModuleAccount: 1,
totalQuoteList: [],
historyPreviewLength: undefined
});
});
it('should transform histories correctly with responseDetail=false', () => {
const histories: ChatItemMiniType[] = [
{
obj: ChatRoleEnum.AI,
value: [{ text: { content: 'test response' } }],
responseData: [
{
...mockResponseData,
runningTime: 1.5
}
]
}
];
const result = transformPreviewHistories(histories, false);
expect(result[0]).toEqual({
obj: ChatRoleEnum.AI,
value: [{ text: { content: 'test response' } }],
responseData: undefined,
useAgentSandbox: false,
llmModuleAccount: 1,
totalQuoteList: undefined,
historyPreviewLength: undefined
});
});
});
describe('addStatisticalDataToHistoryItem', () => {
it('should return original item if obj is not AI', () => {
const item: ChatItemMiniType = {
obj: ChatRoleEnum.Human,
value: [{ text: { content: 'test response' } }]
};
expect(addStatisticalDataToHistoryItem(item)).toBe(item);
});
it('should return original item if totalQuoteList is already defined', () => {
const item: ChatItemMiniType = {
obj: ChatRoleEnum.AI,
value: [{ text: { content: 'test response' } }],
totalQuoteList: []
};
expect(addStatisticalDataToHistoryItem(item)).toBe(item);
});
it('should return original item if responseData is undefined', () => {
const item: ChatItemMiniType = {
obj: ChatRoleEnum.AI,
value: [{ text: { content: 'test response' } }]
};
expect(addStatisticalDataToHistoryItem(item)).toBe(item);
});
it('should calculate statistics correctly', () => {
const quoteId = '507f1f77bcf86cd799439011'; // Valid 24-bit hex ID
const item: ChatItemMiniType = {
obj: ChatRoleEnum.AI,
value: [{ text: { content: `test response with citation [${quoteId}](CITE)` } }],
responseData: [
{
...mockResponseData,
moduleType: FlowNodeTypeEnum.chatNode,
runningTime: 1.5,
historyPreview: [{ obj: ChatRoleEnum.AI, value: 'preview1' }]
},
{
...mockResponseData,
moduleType: FlowNodeTypeEnum.datasetSearchNode,
quoteList: [
{
id: quoteId,
q: 'test',
a: 'answer',
datasetId: 'ds1',
collectionId: 'col1',
sourceName: 'source1',
chunkIndex: 0,
updateTime: new Date(),
score: []
}
],
runningTime: 0.5
},
{
...mockResponseData,
moduleType: FlowNodeTypeEnum.toolCall,
runningTime: 1,
toolDetail: [
{
id: 'detail1',
nodeId: 'detailNode1',
moduleName: 'Detail Chat',
moduleType: FlowNodeTypeEnum.chatNode,
runningTime: 0.5
}
]
}
]
};
const result = addStatisticalDataToHistoryItem(item);
expect(result).toEqual({
...item,
llmModuleAccount: 3,
useAgentSandbox: false,
totalQuoteList: [
{
id: quoteId,
q: 'test',
a: 'answer',
datasetId: 'ds1',
collectionId: 'col1',
sourceName: 'source1',
chunkIndex: 0,
updateTime: expect.any(Date),
score: []
}
],
historyPreviewLength: 1
});
});
it('should handle empty arrays and undefined values', () => {
const item: ChatItemMiniType = {
obj: ChatRoleEnum.AI,
value: [{ text: { content: 'test response' } }],
responseData: [
{
...mockResponseData,
runningTime: 0
}
]
};
const result = addStatisticalDataToHistoryItem(item);
expect(result).toEqual({
...item,
useAgentSandbox: false,
llmModuleAccount: 1,
totalQuoteList: [],
historyPreviewLength: undefined
});
});
it('should handle nested plugin and loop details', () => {
const item: ChatItemMiniType = {
obj: ChatRoleEnum.AI,
value: [{ text: { content: 'test response' } }],
responseData: [
{
...mockResponseData,
runningTime: 1,
pluginDetail: [
{
id: 'plugin1',
nodeId: 'pluginNode1',
moduleName: 'Plugin Chat',
moduleType: FlowNodeTypeEnum.chatNode,
runningTime: 0.5
}
],
loopDetail: [
{
id: 'loop1',
nodeId: 'loopNode1',
moduleName: 'Loop Tool',
moduleType: FlowNodeTypeEnum.toolCall,
runningTime: 0.3
}
]
}
]
};
const result = addStatisticalDataToHistoryItem(item);
expect(result).toEqual({
...item,
useAgentSandbox: false,
llmModuleAccount: 3,
totalQuoteList: [],
historyPreviewLength: undefined
});
});
});
......@@ -12,7 +12,7 @@ import { MongoUser } from '@fastgpt/service/support/user/schema';
import { MongoTeamMember } from '@fastgpt/service/support/user/team/teamMemberSchema';
import { MongoTeam } from '@fastgpt/service/support/user/team/teamSchema';
import { initTeamFreePlan } from '@fastgpt/service/support/wallet/sub/utils';
import type { parseHeaderCertRet } from 'test/mocks/request';
import type { parseHeaderCertRet } from '@test/mocks/request';
export async function getRootUser(): Promise<parseHeaderCertRet> {
const rootUser = await MongoUser.create({
......
# FastGPT 集成测试
## 目录
- vectorDB: 向量数据库
\ No newline at end of file
import { beforeAll, describe, expect, test } from 'vitest';
import type { VectorControllerType } from '@fastgpt/service/common/vectorDB/type';
import { createTestIds, QUERY_VECTOR, TEST_COLLECTION_IDS, TEST_VECTORS } from './testData';
const insertTestVectors = async (
vectorCtrl: VectorControllerType,
teamId: string,
datasetId: string
) => {
const insertIds: string[] = [];
await Promise.all(
TEST_VECTORS.map(async (vector, index) => {
const { insertIds: ids } = await vectorCtrl.insert({
teamId,
datasetId,
collectionId: TEST_COLLECTION_IDS[index],
vectors: [vector]
});
insertIds.push(ids[0]);
})
);
await new Promise((resolve) => setTimeout(resolve, 500));
return insertIds;
};
const cleanupTestVectors = async (
vectorCtrl: VectorControllerType,
teamId: string,
datasetId: string
) => {
try {
await vectorCtrl.delete({
teamId,
datasetIds: [datasetId]
});
} catch (error) {
// Ignore cleanup errors
}
};
export const createVectorDBTestSuite = (vectorCtrl: VectorControllerType) => {
describe.sequential('vectorDB integration', () => {
beforeAll(async () => {
await vectorCtrl.init();
});
test('insert and count', async () => {
const { teamId, datasetId } = createTestIds();
const insertIds = await insertTestVectors(vectorCtrl, teamId, datasetId);
expect(insertIds).toHaveLength(TEST_VECTORS.length);
const count = await vectorCtrl.getVectorCount({ teamId, datasetId });
expect(count).toBe(TEST_VECTORS.length);
const collectionCount = await vectorCtrl.getVectorCount({
teamId,
datasetId,
collectionId: TEST_COLLECTION_IDS[0]
});
expect(collectionCount).toBe(1);
await cleanupTestVectors(vectorCtrl, teamId, datasetId);
});
test('embRecall returns results', async () => {
const { teamId, datasetId } = createTestIds();
await insertTestVectors(vectorCtrl, teamId, datasetId);
const { results } = await vectorCtrl.embRecall({
teamId,
datasetIds: [datasetId],
vector: QUERY_VECTOR,
limit: 3,
forbidCollectionIdList: []
});
expect(results.length).toBeGreaterThan(0);
expect(results.every((item) => TEST_COLLECTION_IDS.includes(item.collectionId))).toBe(true);
await cleanupTestVectors(vectorCtrl, teamId, datasetId);
});
test('embRecall respects forbidCollectionIdList', async () => {
const { teamId, datasetId } = createTestIds();
await insertTestVectors(vectorCtrl, teamId, datasetId);
const { results } = await vectorCtrl.embRecall({
teamId,
datasetIds: [datasetId],
vector: QUERY_VECTOR,
limit: 10,
forbidCollectionIdList: [TEST_COLLECTION_IDS[0]]
});
expect(results.length).toBeGreaterThan(0);
expect(results.every((item) => item.collectionId !== TEST_COLLECTION_IDS[0])).toBe(true);
await cleanupTestVectors(vectorCtrl, teamId, datasetId);
});
test('embRecall respects filterCollectionIdList', async () => {
const { teamId, datasetId } = createTestIds();
await insertTestVectors(vectorCtrl, teamId, datasetId);
const { results } = await vectorCtrl.embRecall({
teamId,
datasetIds: [datasetId],
vector: QUERY_VECTOR,
limit: 10,
forbidCollectionIdList: [],
filterCollectionIdList: [TEST_COLLECTION_IDS[1]]
});
expect(results.length).toBeGreaterThan(0);
expect(results.every((item) => item.collectionId === TEST_COLLECTION_IDS[1])).toBe(true);
await cleanupTestVectors(vectorCtrl, teamId, datasetId);
});
test('getVectorDataByTime returns data', async () => {
const { teamId, datasetId } = createTestIds();
const insertIds = await insertTestVectors(vectorCtrl, teamId, datasetId);
await new Promise((resolve) => setTimeout(resolve, 500));
const start = new Date(0);
const end = new Date(Date.now() + 600_000);
const data = await vectorCtrl.getVectorDataByTime(start, end);
const matchedIds = data
.filter((item) => item.teamId === teamId && item.datasetId === datasetId)
.map((item) => item.id);
expect(matchedIds.length).toBeGreaterThan(0);
expect(matchedIds).toEqual(expect.arrayContaining(insertIds));
await cleanupTestVectors(vectorCtrl, teamId, datasetId);
});
test('delete by idList removes vectors', async () => {
const { teamId, datasetId } = createTestIds();
const insertIds = await insertTestVectors(vectorCtrl, teamId, datasetId);
await vectorCtrl.delete({
teamId,
idList: insertIds.slice(0, 2)
});
const count = await vectorCtrl.getVectorCount({ teamId, datasetId });
expect(count).toBe(TEST_VECTORS.length - 2);
await cleanupTestVectors(vectorCtrl, teamId, datasetId);
});
});
};
import path from 'node:path';
import { vi } from 'vitest';
vi.mock('@fastgpt/service/common/geo/constants', async (importOriginal) => {
const actual = (await importOriginal()) as any;
return {
...actual,
dbPath: path.join(process.cwd(), 'projects/app/data/GeoLite2-City.mmdb')
};
});
......@@ -29,7 +29,8 @@ export const mockGetVectorCountByTeamId = vi.fn().mockResolvedValue(100);
export const mockGetVectorCount = vi.fn().mockResolvedValue(50);
const MockVectorCtrl = vi.fn().mockImplementation(() => ({
const MockVectorCtrl = vi.fn().mockImplementation(function () {
return {
init: mockVectorInit,
insert: mockVectorInsert,
delete: mockVectorDelete,
......@@ -37,7 +38,8 @@ const MockVectorCtrl = vi.fn().mockImplementation(() => ({
getVectorDataByTime: mockGetVectorDataByTime,
getVectorCountByTeamId: mockGetVectorCountByTeamId,
getVectorCount: mockGetVectorCount
}));
};
});
// Mock PgVectorCtrl
vi.mock('@fastgpt/service/common/vectorDB/pg', () => ({
......
import './request';
import './common/geo';
import './common/mongo';
import './common/redis';
import './common/bullmq';
......
import { ModelTypeEnum } from 'packages/global/core/ai/constants';
import { ModelTypeEnum } from '@fastgpt/global/core/ai/constants';
export default async function setupModels() {
global.llmModelMap = new Map<string, any>();
......
......@@ -3,6 +3,12 @@
"compilerOptions": {
"baseUrl": "..",
"paths": {
"@fastgpt-sdk/logger": ["sdk/logger/src/index.ts"],
"@fastgpt-sdk/storage": ["sdk/storage/src/index.ts"],
"@fastgpt-sdk/otel": ["sdk/otel/src/index.ts"],
"@fastgpt-sdk/otel/logger": ["sdk/otel/src/logger-entry.ts"],
"@fastgpt-sdk/otel/metrics": ["sdk/otel/src/metrics-entry.ts"],
"@fastgpt-sdk/otel/tracing": ["sdk/otel/src/tracing-entry.ts"],
"@fastgpt/*": ["packages/*"],
"@test/*": ["test/*"],
"@/*": ["projects/app/src/*"]
......
{
"$schema": "https://turborepo.com/schema.json",
"ui": "tui",
"tasks": {
"dev": {
"with": ["@fastgpt/admin#dev"],
"cache": false,
"persistent": true
},
"build": {
"dependsOn": ["^build"],
"outputs": [".next/**", "dist/**", "worker/**"]
},
"lint": {
"outputs": []
},
"test": {
"outputs": ["coverage/**", "test-results.json"]
},
"test:integration": {
"cache": false,
"outputs": ["test-results.integration.json"]
},
"typecheck": {
"outputs": []
},
"build:workers": {
"cache": false,
"outputs": ["worker/**"]
}
}
}
......@@ -5,9 +5,13 @@ export default defineConfig({
resolve: {
alias: {
'@': resolve('projects/app/src'),
'@fastgpt/global': resolve('packages/global'),
'@fastgpt/service': resolve('packages/service'),
'@fastgpt/web': resolve('packages/web'),
'@fastgpt-sdk/logger': resolve('sdk/logger/src/index.ts'),
'@fastgpt-sdk/storage': resolve('sdk/storage/src/index.ts'),
'@fastgpt-sdk/otel/logger': resolve('sdk/otel/src/logger-entry.ts'),
'@fastgpt-sdk/otel/metrics': resolve('sdk/otel/src/metrics-entry.ts'),
'@fastgpt-sdk/otel/tracing': resolve('sdk/otel/src/tracing-entry.ts'),
'@fastgpt-sdk/otel': resolve('sdk/otel/src/index.ts'),
'@fastgpt': resolve('packages'),
'@test': resolve('test')
}
},
......@@ -22,7 +26,6 @@ export default defineConfig({
reporter: ['html', 'json-summary', 'json'],
// reporter: ['text', 'text-summary', 'html', 'json-summary', 'json'],
reportOnFailure: true,
all: false, // 只包含被测试实际覆盖的文件,不包含空目录
include: ['projects/app/**/*.ts', 'packages/**/*.ts'],
exclude: [
'**/node_modules/**',
......@@ -52,11 +55,8 @@ export default defineConfig({
pool: 'threads',
testTimeout: 20000,
hookTimeout: 30000,
passWithNoTests: true,
reporters: ['github-actions', 'default'],
include: [
'test/**/*.test.ts',
'projects/app/test/**/*.test.ts',
'projects/marketplace/test/**/*.test.ts'
]
include: ['test/**/*.test.ts']
}
});
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment