Commit c8abd09d by Finley Ge Committed by GitHub

fix(security): harden archive parsing and CI actions (#7293)

parent 16331137
......@@ -69,7 +69,7 @@ jobs:
EOF
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
with:
driver-opts: |
network=host
......@@ -96,7 +96,7 @@ jobs:
echo "SOURCE_URL=${{ github.server_url }}/${{ github.repository }}" >> "$GITHUB_ENV"
- name: Login to Container Registry
uses: docker/login-action@v3
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.repository_owner }}
......@@ -104,7 +104,7 @@ jobs:
- name: Build for ${{ matrix.arch }}
id: build
uses: docker/build-push-action@v6
uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6
env:
GODEBUG: madvdontneed=0
with:
......@@ -181,7 +181,7 @@ jobs:
EOF
- name: Login to Container Registry
uses: docker/login-action@v3
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.repository_owner }}
......@@ -200,7 +200,7 @@ jobs:
path: ${{ runner.temp }}/digests
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
with:
driver-opts: |
network=host
......@@ -290,7 +290,7 @@ jobs:
EOF
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
with:
driver-opts: |
network=host
......@@ -317,7 +317,7 @@ jobs:
echo "SOURCE_URL=${{ github.server_url }}/${{ github.repository }}" >> "$GITHUB_ENV"
- name: Login to Container Registry
uses: docker/login-action@v3
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.repository_owner }}
......@@ -325,7 +325,7 @@ jobs:
- name: Build for ${{ matrix.arch }}
id: build
uses: docker/build-push-action@v6
uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6
env:
GODEBUG: madvdontneed=0
with:
......@@ -403,7 +403,7 @@ jobs:
EOF
- name: Login to Container Registry
uses: docker/login-action@v3
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.repository_owner }}
......@@ -422,7 +422,7 @@ jobs:
path: ${{ runner.temp }}/digests
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
with:
driver-opts: |
network=host
......
......@@ -89,7 +89,7 @@ jobs:
EOF
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
with:
driver-opts: |
network=host
......@@ -116,7 +116,7 @@ jobs:
echo "SOURCE_URL=${{ github.server_url }}/${{ github.repository }}" >> "$GITHUB_ENV"
- name: Login to Container Registry
uses: docker/login-action@v3
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.repository_owner }}
......@@ -124,7 +124,7 @@ jobs:
- name: Build for ${{ matrix.arch }}
id: build
uses: docker/build-push-action@v6
uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6
env:
GODEBUG: madvdontneed=0
with:
......@@ -201,7 +201,7 @@ jobs:
EOF
- name: Login to Container Registry
uses: docker/login-action@v3
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.repository_owner }}
......@@ -220,7 +220,7 @@ jobs:
path: ${{ runner.temp }}/digests
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
with:
driver-opts: |
network=host
......
......@@ -80,7 +80,7 @@ jobs:
EOF
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
with:
driver-opts: |
network=host
......@@ -107,7 +107,7 @@ jobs:
echo "SOURCE_URL=${{ github.server_url }}/${{ github.repository }}" >> "$GITHUB_ENV"
- name: Login to Container Registry
uses: docker/login-action@v3
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.repository_owner }}
......@@ -115,7 +115,7 @@ jobs:
- name: Build for ${{ matrix.arch }}
id: build
uses: docker/build-push-action@v6
uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6
env:
GODEBUG: madvdontneed=0
with:
......@@ -199,7 +199,7 @@ jobs:
EOF
- name: Login to Container Registry
uses: docker/login-action@v3
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.repository_owner }}
......@@ -218,7 +218,7 @@ jobs:
path: ${{ runner.temp }}/digests
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
with:
driver-opts: |
network=host
......
......@@ -95,7 +95,7 @@ jobs:
EOF
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
with:
driver-opts: |
network=host
......@@ -123,7 +123,7 @@ jobs:
# login docker
- name: Login to Container Registry
uses: docker/login-action@v3
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.repository_owner }}
......@@ -131,7 +131,7 @@ jobs:
- name: Build for ${{ matrix.archs.arch }}
id: build
uses: docker/build-push-action@v6
uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6
env:
GODEBUG: madvdontneed=0
with:
......@@ -215,7 +215,7 @@ jobs:
EOF
- name: Login to Container Registry
uses: docker/login-action@v3
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.repository_owner }}
......@@ -234,7 +234,7 @@ jobs:
path: ${{ runner.temp }}/digests
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
with:
driver-opts: |
network=host
......
......@@ -52,7 +52,7 @@ jobs:
EOF
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
with:
driver-opts: |
network=host
......@@ -80,7 +80,7 @@ jobs:
# login docker
- name: Login to Container Registry
uses: docker/login-action@v3
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.repository_owner }}
......@@ -88,7 +88,7 @@ jobs:
- name: Build for ${{ matrix.arch }}
id: build
uses: docker/build-push-action@v6
uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6
env:
GODEBUG: madvdontneed=0
with:
......@@ -165,7 +165,7 @@ jobs:
EOF
- name: Login to Container Registry
uses: docker/login-action@v3
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.repository_owner }}
......@@ -184,7 +184,7 @@ jobs:
path: ${{ runner.temp }}/digests
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
with:
driver-opts: |
network=host
......
......@@ -69,7 +69,7 @@ jobs:
EOF
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
with:
driver-opts: |
network=host
......@@ -97,7 +97,7 @@ jobs:
# login docker
- name: Login to Container Registry
uses: docker/login-action@v3
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.repository_owner }}
......@@ -105,7 +105,7 @@ jobs:
- name: Build for ${{ matrix.arch }}
id: build
uses: docker/build-push-action@v6
uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6
env:
GODEBUG: madvdontneed=0
with:
......@@ -181,7 +181,7 @@ jobs:
EOF
- name: Login to Container Registry
uses: docker/login-action@v3
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.repository_owner }}
......@@ -200,7 +200,7 @@ jobs:
path: ${{ runner.temp }}/digests
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
with:
driver-opts: |
network=host
......
......@@ -38,7 +38,7 @@ jobs:
repository: ${{ github.event.pull_request.head.repo.full_name ||
github.repository }}
- uses: pnpm/action-setup@v4
- uses: pnpm/action-setup@b906affcce14559ad1aafd4ab0e942779e9f58b1 # v4
- uses: actions/setup-node@v4
with:
......@@ -69,7 +69,7 @@ jobs:
repository: ${{ github.event.pull_request.head.repo.full_name ||
github.repository }}
- uses: pnpm/action-setup@v4
- uses: pnpm/action-setup@b906affcce14559ad1aafd4ab0e942779e9f58b1 # v4
- uses: actions/setup-node@v4
with:
......@@ -124,7 +124,7 @@ jobs:
repository: ${{ github.event.pull_request.head.repo.full_name ||
github.repository }}
- uses: pnpm/action-setup@v4
- uses: pnpm/action-setup@b906affcce14559ad1aafd4ab0e942779e9f58b1 # v4
- uses: actions/setup-node@v4
with:
......
......@@ -28,12 +28,12 @@ jobs:
- uses: actions/checkout@v6
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@stable
uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 # stable
with:
components: clippy, rustfmt
- name: Rust Cache
uses: Swatinem/rust-cache@v2
uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2
with:
shared-key: "fastgpt-rust-agent-cache"
workspaces: |
......
......@@ -18,7 +18,7 @@ jobs:
ref: ${{ github.event.pull_request.head.ref || github.ref }}
repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }}
- uses: pnpm/action-setup@v4
- uses: pnpm/action-setup@b906affcce14559ad1aafd4ab0e942779e9f58b1 # v4
- uses: actions/setup-node@v4
with:
......
......@@ -67,7 +67,7 @@ jobs:
fi
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
with:
driver-opts: network=host
......@@ -82,7 +82,7 @@ jobs:
# login docker (GHCR only; Ali tags are pushed in release job via imagetools)
- name: Login to GitHub Container Registry
uses: docker/login-action@v3
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
......@@ -90,7 +90,7 @@ jobs:
- name: Build for ${{ matrix.archs.arch }}
id: build
uses: docker/build-push-action@v6
uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6
with:
context: .
file: pro/admin/Dockerfile
......@@ -136,13 +136,13 @@ jobs:
runs-on: ubuntu-24.04
steps:
- name: Login to GitHub Container Registry
uses: docker/login-action@v3
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Login to Ali Hub
uses: docker/login-action@v3
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
with:
registry: registry.cn-hangzhou.aliyuncs.com
username: ${{ secrets.FASTGPT_ALI_IMAGE_USER }}
......@@ -156,7 +156,7 @@ jobs:
merge-multiple: true
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
- name: Set image name and tag
run: |
......
......@@ -45,7 +45,7 @@ jobs:
with:
fetch-depth: 0
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
with:
driver-opts: network=host
- name: Cache Docker layers
......@@ -57,7 +57,7 @@ jobs:
${{ runner.os }}-agent-sandbox-buildx-
- name: Login to GitHub Container Registry
uses: docker/login-action@v3
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
......@@ -65,7 +65,7 @@ jobs:
- name: Build for ${{ matrix.arch }}
id: build
uses: docker/build-push-action@v6
uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6
with:
context: .
file: projects/agent-sandbox/Dockerfile
......@@ -101,19 +101,19 @@ jobs:
runs-on: ubuntu-24.04
steps:
- name: Login to GitHub Container Registry
uses: docker/login-action@v3
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Login to Ali Hub
uses: docker/login-action@v3
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
with:
registry: registry.cn-hangzhou.aliyuncs.com
username: ${{ secrets.FASTGPT_ALI_IMAGE_USER }}
password: ${{ secrets.FASTGPT_ALI_IMAGE_PSW }}
- name: Login to Docker Hub
uses: docker/login-action@v3
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
with:
username: ${{ secrets.DOCKER_HUB_NAME }}
password: ${{ secrets.DOCKER_HUB_PASSWORD }}
......@@ -126,7 +126,7 @@ jobs:
merge-multiple: true
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
- name: Set image name and tag
run: |
......@@ -188,7 +188,7 @@ jobs:
with:
fetch-depth: 0
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
with:
driver-opts: network=host
- name: Cache Docker layers
......@@ -200,7 +200,7 @@ jobs:
${{ runner.os }}-volume-manager-buildx-
- name: Login to GitHub Container Registry
uses: docker/login-action@v3
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
......@@ -208,7 +208,7 @@ jobs:
- name: Build for ${{ matrix.arch }}
id: build
uses: docker/build-push-action@v6
uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6
with:
context: .
file: projects/volume-manager/Dockerfile
......@@ -245,19 +245,19 @@ jobs:
runs-on: ubuntu-24.04
steps:
- name: Login to GitHub Container Registry
uses: docker/login-action@v3
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Login to Ali Hub
uses: docker/login-action@v3
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
with:
registry: registry.cn-hangzhou.aliyuncs.com
username: ${{ secrets.FASTGPT_ALI_IMAGE_USER }}
password: ${{ secrets.FASTGPT_ALI_IMAGE_PSW }}
- name: Login to Docker Hub
uses: docker/login-action@v3
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
with:
username: ${{ secrets.DOCKER_HUB_NAME }}
password: ${{ secrets.DOCKER_HUB_PASSWORD }}
......@@ -270,7 +270,7 @@ jobs:
merge-multiple: true
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
- name: Set image name and tag
run: |
......
......@@ -64,7 +64,7 @@ jobs:
fi
- name: Install pnpm
uses: pnpm/action-setup@v4
uses: pnpm/action-setup@b906affcce14559ad1aafd4ab0e942779e9f58b1 # v4
with:
version: 10.33.4
......@@ -126,7 +126,7 @@ jobs:
fi
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
with:
driver-opts: network=host
......@@ -139,7 +139,7 @@ jobs:
${{ runner.os }}-browser-sandbox-buildx-
- name: Login to GitHub Container Registry
uses: docker/login-action@v3
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
......@@ -147,7 +147,7 @@ jobs:
- name: Build for ${{ matrix.arch }}
id: build
uses: docker/build-push-action@v6
uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6
with:
context: .
file: pro/browser-sandbox/Dockerfile
......@@ -205,21 +205,21 @@ jobs:
runs-on: ubuntu-24.04
steps:
- name: Login to GitHub Container Registry
uses: docker/login-action@v3
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Login to Ali Hub
uses: docker/login-action@v3
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
with:
registry: registry.cn-hangzhou.aliyuncs.com
username: ${{ secrets.FASTGPT_ALI_IMAGE_USER }}
password: ${{ secrets.FASTGPT_ALI_IMAGE_PSW }}
- name: Login to Docker Hub
uses: docker/login-action@v3
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
with:
username: ${{ secrets.DOCKER_HUB_NAME }}
password: ${{ secrets.DOCKER_HUB_PASSWORD }}
......@@ -232,7 +232,7 @@ jobs:
merge-multiple: true
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
- name: Set image name and tag
run: |
......
......@@ -64,7 +64,7 @@ jobs:
with:
fetch-depth: 0
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
with:
driver-opts: network=host
- name: Cache Docker layers
......@@ -77,7 +77,7 @@ jobs:
# ghcr + any extra names in outputs (Ali is pushed only in release via imagetools)
- name: Login to GitHub Container Registry
uses: docker/login-action@v3
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
......@@ -85,7 +85,7 @@ jobs:
- name: Build for ${{ matrix.arch }}
id: build
uses: docker/build-push-action@v6
uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6
with:
context: .
file: projects/code-sandbox/Dockerfile
......@@ -121,13 +121,13 @@ jobs:
runs-on: ubuntu-24.04
steps:
- name: Login to GitHub Container Registry
uses: docker/login-action@v3
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Login to Ali Hub
uses: docker/login-action@v3
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
with:
registry: registry.cn-hangzhou.aliyuncs.com
username: ${{ secrets.FASTGPT_ALI_IMAGE_USER }}
......@@ -140,7 +140,7 @@ jobs:
merge-multiple: true
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
- name: Set image name and tag
run: |
......
......@@ -93,7 +93,7 @@ jobs:
- name: Docker meta
id: meta
uses: docker/metadata-action@v5
uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # v5
with:
# list of Docker images to use as base name for tags
images: |
......@@ -103,7 +103,7 @@ jobs:
flavor: latest=false
- name: Login to Aliyun
uses: docker/login-action@v3
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
with:
registry: registry.cn-hangzhou.aliyuncs.com
username: ${{ secrets.FASTGPT_ALI_IMAGE_USER }}
......@@ -114,7 +114,7 @@ jobs:
- name: Build and push Docker images (CN)
if: matrix.domain_config.suffix == 'cn'
uses: docker/build-push-action@v5
uses: docker/build-push-action@ca052bb54ab0790a636c9b5f226502c73d547a25 # v5
with:
context: ./document
file: ./document/Dockerfile
......@@ -129,7 +129,7 @@ jobs:
- name: Build and push Docker images (IO)
if: matrix.domain_config.suffix == 'io'
uses: docker/build-push-action@v5
uses: docker/build-push-action@ca052bb54ab0790a636c9b5f226502c73d547a25 # v5
with:
context: ./document
file: ./document/Dockerfile
......
......@@ -52,7 +52,7 @@ jobs:
with:
fetch-depth: 0
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
with:
driver-opts: network=host
- name: Cache Docker layers
......@@ -65,7 +65,7 @@ jobs:
# Push per-arch images by digest first; the release job publishes the final manifest tag.
- name: Login to GitHub Container Registry
uses: docker/login-action@v3
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
......@@ -73,7 +73,7 @@ jobs:
- name: Build for ${{ matrix.arch }}
id: build
uses: docker/build-push-action@v6
uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6
with:
context: ${{ matrix.context }}
file: ${{ matrix.dockerfile }}
......@@ -114,14 +114,14 @@ jobs:
runs-on: ubuntu-24.04
steps:
- name: Login to GitHub Container Registry
uses: docker/login-action@v3
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Login to Ali Hub
if: matrix.image == 'fastgpt-agent-sandbox-proxy'
uses: docker/login-action@v3
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
with:
registry: registry.cn-hangzhou.aliyuncs.com
username: ${{ secrets.FASTGPT_ALI_IMAGE_USER }}
......@@ -134,7 +134,7 @@ jobs:
merge-multiple: true
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
- name: Set image name and tag
run: |
......
......@@ -57,7 +57,7 @@ jobs:
fetch-depth: 1
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
with:
driver-opts: network=host
......@@ -71,7 +71,7 @@ jobs:
# login docker
- name: Login to GitHub Container Registry
uses: docker/login-action@v3
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
......@@ -79,7 +79,7 @@ jobs:
- name: Build for ${{ matrix.archs.arch }}
id: build
uses: docker/build-push-action@v6
uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6
with:
context: .
file: projects/app/Dockerfile
......@@ -123,19 +123,19 @@ jobs:
runs-on: ubuntu-24.04
steps:
- name: Login to GitHub Container Registry
uses: docker/login-action@v3
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Login to Ali Hub
uses: docker/login-action@v3
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
with:
registry: registry.cn-hangzhou.aliyuncs.com
username: ${{ secrets.FASTGPT_ALI_IMAGE_USER }}
password: ${{ secrets.FASTGPT_ALI_IMAGE_PSW }}
- name: Login to Docker Hub
uses: docker/login-action@v3
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
with:
username: ${{ secrets.DOCKER_HUB_NAME }}
password: ${{ secrets.DOCKER_HUB_PASSWORD }}
......@@ -148,7 +148,7 @@ jobs:
merge-multiple: true
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
- name: Set image name and tag
run: |
......
......@@ -22,7 +22,7 @@ jobs:
with:
fetch-depth: 0
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
with:
driver-opts: network=host
- name: Cache Docker layers
......@@ -35,7 +35,7 @@ jobs:
# login docker
- name: Login to GitHub Container Registry
uses: docker/login-action@v3
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
......@@ -43,7 +43,7 @@ jobs:
- name: Build for ${{ matrix.arch }}
id: build
uses: docker/build-push-action@v6
uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6
with:
context: .
file: projects/marketplace/Dockerfile
......@@ -79,13 +79,13 @@ jobs:
runs-on: ubuntu-24.04
steps:
- name: Login to GitHub Container Registry
uses: docker/login-action@v3
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Login to Ali Hub
uses: docker/login-action@v3
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
with:
registry: registry.cn-hangzhou.aliyuncs.com
username: ${{ secrets.FASTGPT_ALI_IMAGE_USER }}
......@@ -99,7 +99,7 @@ jobs:
merge-multiple: true
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
- name: Generate random tag
id: tag
......
......@@ -44,7 +44,7 @@ jobs:
with:
fetch-depth: 0
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
with:
driver-opts: network=host
- name: Cache Docker layers
......@@ -57,7 +57,7 @@ jobs:
# login docker
- name: Login to GitHub Container Registry
uses: docker/login-action@v3
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
......@@ -65,7 +65,7 @@ jobs:
- name: Build for ${{ matrix.arch }}
id: build
uses: docker/build-push-action@v6
uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6
with:
context: .
file: projects/mcp_server/Dockerfile
......@@ -101,19 +101,19 @@ jobs:
runs-on: ubuntu-24.04
steps:
- name: Login to GitHub Container Registry
uses: docker/login-action@v3
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Login to Ali Hub
uses: docker/login-action@v3
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
with:
registry: registry.cn-hangzhou.aliyuncs.com
username: ${{ secrets.FASTGPT_ALI_IMAGE_USER }}
password: ${{ secrets.FASTGPT_ALI_IMAGE_PSW }}
- name: Login to Docker Hub
uses: docker/login-action@v3
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
with:
username: ${{ secrets.DOCKER_HUB_NAME }}
password: ${{ secrets.DOCKER_HUB_PASSWORD }}
......@@ -126,7 +126,7 @@ jobs:
merge-multiple: true
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
- name: Set image name and tag
run: |
......
......@@ -52,9 +52,9 @@ jobs:
run: |
sudo apt update && sudo apt install -y nodejs npm
- name: Set up QEMU
uses: docker/setup-qemu-action@v3
uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
with:
driver-opts: network=host
- name: Cache Docker layers
......@@ -65,13 +65,13 @@ jobs:
restore-keys: |
${{ runner.os }}-buildx-
- name: Login to GitHub Container Registry
uses: docker/login-action@v3
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
with:
registry: ghcr.io
username: labring
password: ${{ secrets.GITHUB_TOKEN }}
- name: Login to Ali Hub
uses: docker/login-action@v3
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
with:
registry: registry.cn-hangzhou.aliyuncs.com
username: ${{ secrets.FASTGPT_ALI_IMAGE_USER }}
......
......@@ -52,10 +52,10 @@ jobs:
fi
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
- name: Build Docker image (no push)
uses: docker/build-push-action@v6
uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6
with:
context: .
file: pro/admin/Dockerfile
......
......@@ -42,10 +42,10 @@ jobs:
run: docker load --input /tmp/fastgpt-pro-image.tar
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
- name: Login to GitHub Container Registry
uses: docker/login-action@v3
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
......
......@@ -26,13 +26,13 @@ jobs:
repository: ${{ github.event.pull_request.head.repo.full_name }}
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
- name: Prepare Docker workspace catalog
run: cp pnpm-workspace.yaml document/pnpm-workspace.yaml
- name: Build Docker image (no push)
uses: docker/build-push-action@v6
uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6
with:
context: ./document
file: ./document/Dockerfile
......
......@@ -103,10 +103,10 @@ jobs:
run: docker load --input /tmp/docs-image.tar
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
- name: Login to GitHub Container Registry
uses: docker/login-action@v3
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
......
......@@ -105,10 +105,10 @@ jobs:
repository: ${{ github.event.pull_request.head.repo.full_name }}
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
- name: Build Docker image (no push)
uses: docker/build-push-action@v6
uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6
with:
context: .
file: ${{ matrix.dockerfile }}
......
......@@ -135,10 +135,10 @@ jobs:
run: docker load --input /tmp/${{ matrix.image_name }}-image.tar
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
- name: Login to GitHub Container Registry
uses: docker/login-action@v3
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
......
......@@ -35,7 +35,6 @@ jobs:
permissions:
contents: read
pull-requests: write
steps:
- uses: actions/checkout@v4
......@@ -64,7 +63,7 @@ jobs:
sudo apt-get update
sudo apt-get install -y libcairo2-dev libpango1.0-dev libjpeg-dev libgif-dev librsvg2-dev
- uses: pnpm/action-setup@v4
- uses: pnpm/action-setup@b906affcce14559ad1aafd4ab0e942779e9f58b1 # v4
- uses: actions/setup-node@v4
with:
......@@ -76,9 +75,33 @@ jobs:
env:
INVOKE_TOKEN_SECRET: fastgpt_test_invoke_token_secret_32
run: pnpm test:admin
- name: Report Coverage
- name: Upload Coverage
if: always() && hashFiles('pro/admin/coverage/coverage-summary.json') != ''
uses: davelosert/vitest-coverage-report-action@v2
uses: actions/upload-artifact@v4
with:
name: coverage-fastgpt-pro
path: |
pro/admin/coverage/coverage-final.json
pro/admin/coverage/coverage-summary.json
retention-days: 1
report-coverage:
needs: test
if: always()
runs-on: ubuntu-latest
permissions:
contents: read
pull-requests: write
steps:
- name: Download Coverage
uses: actions/download-artifact@v4
with:
name: coverage-fastgpt-pro
path: pro/admin/coverage
continue-on-error: true
- name: Report Coverage
if: hashFiles('pro/admin/coverage/coverage-summary.json') != ''
uses: davelosert/vitest-coverage-report-action@3c054a2d2e2ca45446417ad5d6d5eb33092af8f1 # v2
with:
json-final-path: pro/admin/coverage/coverage-final.json
json-summary-path: pro/admin/coverage/coverage-summary.json
......@@ -107,7 +107,7 @@ jobs:
repository: ${{ github.event.pull_request.head.repo.full_name ||
github.repository }}
- uses: pnpm/action-setup@v4
- uses: pnpm/action-setup@b906affcce14559ad1aafd4ab0e942779e9f58b1 # v4
- uses: actions/setup-node@v4
with:
......@@ -141,7 +141,7 @@ jobs:
repository: ${{ github.event.pull_request.head.repo.full_name ||
github.repository }}
- uses: pnpm/action-setup@v4
- uses: pnpm/action-setup@b906affcce14559ad1aafd4ab0e942779e9f58b1 # v4
- uses: actions/setup-node@v4
with:
......@@ -175,7 +175,7 @@ jobs:
repository: ${{ github.event.pull_request.head.repo.full_name ||
github.repository }}
- uses: pnpm/action-setup@v4
- uses: pnpm/action-setup@b906affcce14559ad1aafd4ab0e942779e9f58b1 # v4
- uses: actions/setup-node@v4
with:
......
......@@ -28,12 +28,12 @@ jobs:
- uses: actions/checkout@v6
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@stable
uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 # stable
with:
components: clippy, rustfmt
- name: Rust Cache
uses: Swatinem/rust-cache@v2
uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2
with:
shared-key: "fastgpt-rust-agent-cache"
workspaces: |
......
......@@ -18,7 +18,7 @@ jobs:
ref: ${{ github.event.pull_request.head.ref || github.ref }}
repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }}
- uses: pnpm/action-setup@v4
- uses: pnpm/action-setup@b906affcce14559ad1aafd4ab0e942779e9f58b1 # v4
- uses: actions/setup-node@v4
with:
......
......@@ -36,7 +36,6 @@
"cookie": "^0.7.1",
"date-fns": "catalog:",
"dayjs": "catalog:",
"decompress": "^4.2.1",
"domino-ext": "^2.1.4",
"encoding": "^0.1.13",
"file-type": "catalog:",
......@@ -77,12 +76,12 @@
"undici": "catalog:",
"winston": "^3.17.0",
"xlsx": "0.18.5",
"yauzl": "^3.4.0",
"zod": "catalog:"
},
"devDependencies": {
"@types/async-retry": "^1.4.9",
"@types/cookie": "^0.5.2",
"@types/decompress": "^4.2.7",
"@types/jsonwebtoken": "catalog:",
"@types/lodash": "catalog:",
"@types/mime-types": "catalog:",
......@@ -94,6 +93,7 @@
"@types/proxy-addr": "catalog:",
"@types/proxy-from-env": "^1.0.4",
"@types/tunnel": "^0.0.4",
"@types/turndown": "^5.0.4"
"@types/turndown": "^5.0.4",
"@types/yauzl": "^2.10.3"
}
}
import { describe, expect, it } from 'vitest';
import JSZip from 'jszip';
import { detectFileEncoding } from '@fastgpt/global/common/file/tools';
import { parseOffice } from '@fastgpt/service/worker/readFile/parseOffice';
type PptxEntry = {
path: string;
content: string;
unixPermissions?: number;
};
const createPptxWithEntries = async (entries: PptxEntry[]) => {
const zip = new JSZip();
entries.forEach(({ path, content, unixPermissions }) => {
zip.file(path, content, { unixPermissions });
});
return zip.generateAsync({
type: 'nodebuffer',
platform: 'UNIX'
});
};
const createPptx = async ({ path, content, unixPermissions }: PptxEntry) =>
createPptxWithEntries([{ path, content, unixPermissions }]);
const parsePptx = (buffer: Buffer) =>
parseOffice({
buffer,
encoding: 'utf-8',
extension: 'pptx'
});
const findZipRecord = (buffer: Buffer, signature: number, fromEnd = false) => {
const signatureBuffer = Buffer.allocUnsafe(4);
signatureBuffer.writeUInt32LE(signature);
const offset = fromEnd ? buffer.lastIndexOf(signatureBuffer) : buffer.indexOf(signatureBuffer);
expect(offset).toBeGreaterThanOrEqual(0);
return offset;
};
describe('parseOffice', () => {
it('解析合法 PPTX slide XML', async () => {
const buffer = await createPptx({
path: 'ppt/slides/slide1.xml',
content: '<a:p><a:t>Hello</a:t><a:t> FastGPT</a:t></a:p>'
});
const encoding = detectFileEncoding(buffer) as BufferEncoding;
await expect(
parseOffice({
buffer,
encoding,
extension: 'pptx'
})
).resolves.toBe('Hello FastGPT');
});
it('按页码解析多个 slide 和 notes XML', async () => {
const buffer = await createPptxWithEntries([
{
path: 'ppt/slides/slide2.xml',
content: '<a:p><a:t>Slide 2</a:t></a:p>'
},
{
path: 'ppt/slides/slide1.xml',
content: '<a:p><a:t>Slide 1</a:t></a:p>'
},
{
path: 'ppt/notesSlides/notesSlide1.xml',
content: '<a:p><a:t>Note 1</a:t></a:p>'
}
]);
await expect(parsePptx(buffer)).resolves.toBe('Slide 1\nNote 1\nSlide 2');
});
it('合法 slide XML 没有文本节点时返回空字符串', async () => {
const buffer = await createPptx({
path: 'ppt/slides/slide1.xml',
content: '<a:p></a:p>'
});
await expect(parsePptx(buffer)).resolves.toBe('');
});
it('拒绝伪装成 slide XML 的 symlink entry', async () => {
const buffer = await createPptx({
path: 'ppt/slides/slide1.xml',
content: '/etc/passwd',
unixPermissions: 0o120777
});
await expect(parsePptx(buffer)).rejects.toBe('解析 PPT 失败');
});
it('拒绝仅包含嵌套伪装路径的归档', async () => {
const buffer = await createPptx({
path: 'outside/ppt/slides/slide1.xml',
content: '<a:p><a:t>hidden</a:t></a:p>'
});
await expect(parsePptx(buffer)).rejects.toBe('解析 PPT 失败');
});
it('拒绝经 JSZip 规范化后伪装成 slide XML 的 traversal entry', async () => {
const buffer = await createPptx({
path: '../ppt/slides/slide1.xml',
content: '<a:p><a:t>hidden</a:t></a:p>'
});
await expect(parsePptx(buffer)).rejects.toThrow();
});
it('拒绝伪造 uncompressedSize 的高膨胀归档', async () => {
const zip = new JSZip();
zip.file('ppt/slides/slide1.xml', `<a:p><a:t>${'x'.repeat(10 * 1024 * 1024 + 1)}</a:t></a:p>`);
const buffer = await zip.generateAsync({
type: 'nodebuffer',
platform: 'UNIX',
compression: 'DEFLATE',
compressionOptions: { level: 9 }
});
const forgedBuffer = Buffer.from(buffer);
const centralDirectoryOffset = findZipRecord(forgedBuffer, 0x02014b50, true);
forgedBuffer.writeUInt32LE(1, centralDirectoryOffset + 24);
await expect(parsePptx(forgedBuffer)).rejects.toThrow();
}, 15000);
it('在读取 entry 前拒绝超过数量限制的归档', async () => {
const buffer = await createPptx({
path: 'ppt/slides/slide1.xml',
content: '<a:p><a:t>Hello</a:t></a:p>'
});
const forgedBuffer = Buffer.from(buffer);
const endRecordOffset = findZipRecord(forgedBuffer, 0x06054b50, true);
forgedBuffer.writeUInt16LE(10001, endRecordOffset + 8);
forgedBuffer.writeUInt16LE(10001, endRecordOffset + 10);
await expect(parsePptx(forgedBuffer)).rejects.toBe('解析 PPT 失败');
});
it('拒绝损坏的 PPTX 归档', async () => {
await expect(parsePptx(Buffer.from('not-a-zip'))).rejects.toThrow();
});
it('拒绝不支持的 Office 扩展名', async () => {
await expect(
parseOffice({
buffer: Buffer.alloc(0),
encoding: 'utf-8',
extension: 'docx'
})
).rejects.toBe('只能读取 .pptx 文件');
});
});
import { getNanoid } from '@fastgpt/global/common/string/tools';
import fs from 'fs';
import decompress from 'decompress';
import { DOMParser } from '@xmldom/xmldom';
import { clearDirFiles } from '../../common/file/utils';
import { getLogger, LogCategories } from '../../common/logger';
import { type Entry, fromBuffer, type ZipFile } from 'yauzl';
const DEFAULTDECOMPRESSSUBLOCATION = '/tmp';
const logger = getLogger(LogCategories.INFRA.WORKER);
function getNewFileName(ext: string) {
return `${DEFAULTDECOMPRESSSUBLOCATION}/${getNanoid()}.${ext}`;
}
const parseString = (xml: string) => {
let parser = new DOMParser();
return parser.parseFromString(xml, 'text/xml');
};
const powerPointXmlPathRegex = /^ppt\/(?:notesSlides\/notesSlide|slides\/slide)\d+\.xml$/;
const powerPointSlidePathRegex = /^ppt\/slides\/slide\d+\.xml$/;
const maxPowerPointEntries = 10000;
const maxPowerPointXmlFileBytes = 10 * 1024 * 1024;
const maxPowerPointXmlBytes = 100 * 1024 * 1024;
const parsePowerPoint = async ({
filepath,
decompressPath,
buffer,
encoding
}: {
filepath: string;
decompressPath: string;
buffer: Buffer;
encoding: BufferEncoding;
}) => {
// Files regex that hold our content of interest
const allFilesRegex = /ppt\/(notesSlides|slides)\/(notesSlide|slide)\d+.xml/g;
const slidesRegex = /ppt\/slides\/slide\d+.xml/g;
const decodeXml = (data: Buffer) => {
try {
return data.toString(encoding);
} catch {
// 上游按整个压缩包探测编码时可能得到 Node.js 不支持的编码名。
return data.toString('utf-8');
}
};
const zip = await new Promise<ZipFile>((resolve, reject) => {
fromBuffer(
buffer,
{
lazyEntries: true,
decodeStrings: true,
validateEntrySizes: true,
strictFileNames: true
},
(error, zipFile) => {
if (error) {
reject(error);
return;
}
resolve(zipFile);
}
);
});
const files = await new Promise<{ path: string; content: string }[]>((resolve, reject) => {
const result: { path: string; content: string }[] = [];
let entriesRead = 0;
let totalXmlBytes = 0;
let settled = false;
const fail = (error: unknown) => {
if (settled) return;
settled = true;
zip.close();
reject(error);
};
/** The decompress location which contains the filename in it */
const readEntryContent = (entry: Entry) => {
zip.openReadStream(entry, (error, stream) => {
if (error) {
fail(error);
return;
}
const chunks: Buffer[] = [];
let entryBytes = 0;
stream.on('data', (chunk: Buffer) => {
entryBytes += chunk.length;
totalXmlBytes += chunk.length;
// 元数据可被伪造,必须按流中真实输出字节中止解压。
if (entryBytes > maxPowerPointXmlFileBytes || totalXmlBytes > maxPowerPointXmlBytes) {
stream.destroy(new Error('解析 PPT 失败'));
return;
}
chunks.push(chunk);
});
stream.once('error', fail);
stream.once('end', () => {
if (settled) return;
result.push({
path: entry.fileName,
content: decodeXml(Buffer.concat(chunks, entryBytes))
});
zip.readEntry();
});
});
};
const files = await decompress(filepath, decompressPath, {
filter: (x) => !!x.path.match(allFilesRegex)
});
zip.once('error', fail);
zip.on('entry', (entry: Entry) => {
entriesRead += 1;
if (entriesRead > maxPowerPointEntries) {
fail('解析 PPT 失败');
return;
}
// Verify if atleast the slides xml files exist in the extracted files list.
if (
files.length == 0 ||
!files.map((file) => file.path).some((filename) => filename.match(slidesRegex))
) {
return Promise.reject('解析 PPT 失败');
}
const createdOnUnix = entry.versionMadeBy >>> 8 === 3;
const unixMode = entry.externalFileAttributes >>> 16;
const unixFileType = unixMode & 0xf000;
const isRegularFile =
!entry.fileName.endsWith('/') &&
(!createdOnUnix || unixFileType === 0 || unixFileType === 0x8000);
// yauzl 在 entry 事件前校验原始路径;这里只读取锚定 OOXML 路径下的普通文件。
if (!isRegularFile || !powerPointXmlPathRegex.test(entry.fileName)) {
zip.readEntry();
return;
}
if (
entry.uncompressedSize > maxPowerPointXmlFileBytes ||
totalXmlBytes + entry.uncompressedSize > maxPowerPointXmlBytes
) {
fail('解析 PPT 失败');
return;
}
readEntryContent(entry);
});
zip.once('end', () => {
if (settled) return;
settled = true;
if (!result.some((file) => powerPointSlidePathRegex.test(file.path))) {
reject('解析 PPT 失败');
return;
}
resolve(result);
});
if (zip.entryCount > maxPowerPointEntries) {
fail('解析 PPT 失败');
return;
}
zip.readEntry();
});
// Sort files by slide number to ensure correct order
const sortedFiles = files.sort((a, b) => {
const getSlideNumber = (path: string) => {
const match = path.match(/\d+/);
......@@ -52,44 +141,32 @@ const parsePowerPoint = async ({
};
return getSlideNumber(a.path) - getSlideNumber(b.path);
});
const parser = new DOMParser();
// Returning an array of all the xml contents read using fs.readFileSync
const xmlContentArray = await Promise.all(
sortedFiles.map(async (file) => {
try {
return await fs.promises.readFile(`${decompressPath}/${file.path}`, encoding);
} catch (err) {
return await fs.promises.readFile(`${decompressPath}/${file.path}`, 'utf-8');
}
})
);
let responseArr: string[] = [];
return sortedFiles
.map(({ content }) => {
const xmlParagraphNodesList = parser
.parseFromString(content, 'text/xml')
.getElementsByTagName('a:p');
xmlContentArray.forEach((xmlContent) => {
/** Find text nodes with a:p tags */
const xmlParagraphNodesList = parseString(xmlContent).getElementsByTagName('a:p');
/** Store all the text content to respond */
responseArr.push(
Array.from(xmlParagraphNodesList)
// Filter paragraph nodes than do not have any text nodes which are identifiable by a:t tag
return Array.from(xmlParagraphNodesList)
.filter((paragraphNode) => paragraphNode.getElementsByTagName('a:t').length != 0)
.map((paragraphNode) => {
/** Find text nodes with a:t tags */
const xmlTextNodeList = paragraphNode.getElementsByTagName('a:t');
return Array.from(xmlTextNodeList)
.filter((textNode) => textNode.childNodes[0] && textNode.childNodes[0].nodeValue)
.map((textNode) => textNode.childNodes[0].nodeValue)
.join('');
})
.join('\n')
);
});
return responseArr.join('\n');
.join('\n');
})
.join('\n');
};
/**
* 解析受支持的 Office 文件文本。
* PPTX 归档按 entry 流式读取,只解压固定 OOXML 路径下且满足大小限制的普通 XML 文件。
*/
export const parseOffice = async ({
buffer,
encoding,
......@@ -99,43 +176,10 @@ export const parseOffice = async ({
encoding: BufferEncoding;
extension: string;
}) => {
// Prepare file for processing
// create temp file subdirectory if it does not exist
if (!fs.existsSync(DEFAULTDECOMPRESSSUBLOCATION)) {
fs.mkdirSync(DEFAULTDECOMPRESSSUBLOCATION, { recursive: true });
switch (extension) {
case 'pptx':
return parsePowerPoint({ buffer, encoding });
default:
return Promise.reject('只能读取 .pptx 文件');
}
// temp file name
const filepath = getNewFileName(extension);
const decompressPath = `${DEFAULTDECOMPRESSSUBLOCATION}/${getNanoid()}`;
// const decompressPath = `${DEFAULTDECOMPRESSSUBLOCATION}/test`;
// write new file
try {
fs.writeFileSync(filepath, buffer, {
encoding
});
} catch (err) {
fs.writeFileSync(filepath, buffer, {
encoding: 'utf-8'
});
}
const text = await (async () => {
try {
switch (extension) {
case 'pptx':
return parsePowerPoint({ filepath, decompressPath, encoding });
default:
return Promise.reject('只能读取 .pptx 文件');
}
} catch (error) {
logger.error('Failed to parse pptx file', { extension, error });
}
return '';
})();
fs.unlinkSync(filepath);
clearDirFiles(decompressPath);
return text;
};
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment