Skip to content

赵月辉 / fastgpt-migrated

Go to a project
Commit 8a9f1ed2 by archer
Options

feat: 训练数据管理

parent c0dc5a74
Showing with 852 additions and 97 deletions
package.json
......@@ -27,6 +27,7 @@
"hyperdown": "^2.4.29",
"immer": "^9.0.19",
"jsonwebtoken": "^9.0.0",
"mammoth": "^1.5.1",
"mongoose": "^6.10.0",
"nanoid": "^4.0.1",
"next": "13.1.6",
......
pnpm-lock.yaml
......@@ -31,6 +31,7 @@ specifiers:
immer: ^9.0.19
jsonwebtoken: ^9.0.0
lint-staged: ^13.1.2
mammoth: ^1.5.1
mongoose: ^6.10.0
nanoid: ^4.0.1
next: 13.1.6
......@@ -69,6 +70,7 @@ dependencies:
hyperdown: registry.npmmirror.com/hyperdown/2.4.29
immer: registry.npmmirror.com/immer/9.0.19
jsonwebtoken: registry.npmmirror.com/jsonwebtoken/9.0.0
mammoth: registry.npmmirror.com/mammoth/1.5.1
mongoose: registry.npmmirror.com/mongoose/6.10.0
nanoid: registry.npmmirror.com/nanoid/4.0.1
next: registry.npmmirror.com/next/13.1.6_wiv434v7erz4aedd5whhdwmpv4
......@@ -5085,6 +5087,14 @@ packages:
picomatch: registry.npmmirror.com/picomatch/2.3.1
dev: false
registry.npmmirror.com/argparse/1.0.10:
resolution: {integrity: sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/argparse/-/argparse-1.0.10.tgz}
name: argparse
version: 1.0.10
dependencies:
sprintf-js: registry.npmmirror.com/sprintf-js/1.0.3
dev: false
registry.npmmirror.com/argparse/2.0.1:
resolution: {integrity: sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/argparse/-/argparse-2.0.1.tgz}
name: argparse
......@@ -5333,6 +5343,12 @@ packages:
readable-stream: registry.npmmirror.com/readable-stream/3.6.1
dev: false
registry.npmmirror.com/bluebird/3.4.7:
resolution: {integrity: sha512-iD3898SR7sWVRHbiQv+sHUtHnMvC1o3nW5rAcqnq3uOn07DSAppZYUkIGslDz6gXC7HfunPe7YVBgoEJASPcHA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/bluebird/-/bluebird-3.4.7.tgz}
name: bluebird
version: 3.4.7
dev: false
registry.npmmirror.com/boolbase/1.0.0:
resolution: {integrity: sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/boolbase/-/boolbase-1.0.0.tgz}
name: boolbase
......@@ -5672,6 +5688,12 @@ packages:
browserslist: registry.npmmirror.com/browserslist/4.21.5
dev: true
registry.npmmirror.com/core-util-is/1.0.3:
resolution: {integrity: sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/core-util-is/-/core-util-is-1.0.3.tgz}
name: core-util-is
version: 1.0.3
dev: false
registry.npmmirror.com/cosmiconfig/7.1.0:
resolution: {integrity: sha512-AdmX6xUzdNASswsFtmwSt7Vj8po9IuqXm0UXz7QKPuEUmPB4XyjGfaAr2PSuELMwkRMVH1EpIkX5bTZGRB3eCA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/cosmiconfig/-/cosmiconfig-7.1.0.tgz}
name: cosmiconfig
......@@ -5912,6 +5934,12 @@ packages:
engines: {node: '>=0.3.1'}
dev: false
registry.npmmirror.com/dingbat-to-unicode/1.0.1:
resolution: {integrity: sha512-98l0sW87ZT58pU4i61wa2OHwxbiYSbuxsCBozaVnYX2iCnr3bLM3fIes1/ej7h1YdOKuKt/MLs706TVnALA65w==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/dingbat-to-unicode/-/dingbat-to-unicode-1.0.1.tgz}
name: dingbat-to-unicode
version: 1.0.1
dev: false
registry.npmmirror.com/dir-glob/3.0.1:
resolution: {integrity: sha512-WkrWp9GR4KXfKGYzOLmTuGVi1UWFfws377n9cc55/tb6DuqyF6pcQ5AbiHEshaDpY9v6oaSr2XCDidGmMwdzIA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/dir-glob/-/dir-glob-3.0.1.tgz}
name: dir-glob
......@@ -5974,6 +6002,14 @@ packages:
domhandler: registry.npmmirror.com/domhandler/4.3.1
dev: true
registry.npmmirror.com/duck/0.1.12:
resolution: {integrity: sha512-wkctla1O6VfP89gQ+J/yDesM0S7B7XLXjKGzXxMDVFg7uEn706niAtyYovKbyq1oT9YwDcly721/iUWoc8MVRg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/duck/-/duck-0.1.12.tgz}
name: duck
version: 0.1.12
dependencies:
underscore: registry.npmmirror.com/underscore/1.13.6
dev: false
registry.npmmirror.com/eastasianwidth/0.2.0:
resolution: {integrity: sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/eastasianwidth/-/eastasianwidth-0.2.0.tgz}
name: eastasianwidth
......@@ -7133,6 +7169,12 @@ packages:
engines: {node: '>= 4'}
dev: true
registry.npmmirror.com/immediate/3.0.6:
resolution: {integrity: sha512-XXOFtyqDjNDAQxVfYxuF7g9Il/IbWmmlQg2MYKOH8ExIT1qg6xc4zyS3HaEEATgs1btfzxq15ciUiY7gjSXRGQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/immediate/-/immediate-3.0.6.tgz}
name: immediate
version: 3.0.6
dev: false
registry.npmmirror.com/immer/9.0.19:
resolution: {integrity: sha512-eY+Y0qcsB4TZKwgQzLaE/lqYMlKhv5J9dyd2RhhtGhNo2njPXDqU9XPfcNfa3MIDsdtZt5KlkIsirlo4dHsWdQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/immer/-/immer-9.0.19.tgz}
name: immer
......@@ -7506,6 +7548,12 @@ packages:
is-docker: registry.npmmirror.com/is-docker/2.2.1
dev: true
registry.npmmirror.com/isarray/1.0.0:
resolution: {integrity: sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/isarray/-/isarray-1.0.0.tgz}
name: isarray
version: 1.0.0
dev: false
registry.npmmirror.com/isarray/2.0.5:
resolution: {integrity: sha512-xHjhDr3cNBK0BzdUJSPXZntQUx/mwMS5Rw4A7lPJ90XGAO6ISP/ePDNuo0vhqOZU+UD5JoodwCAAoZQd3FeAKw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/isarray/-/isarray-2.0.5.tgz}
name: isarray
......@@ -7609,6 +7657,17 @@ packages:
object.assign: registry.npmmirror.com/object.assign/4.1.4
dev: true
registry.npmmirror.com/jszip/3.10.1:
resolution: {integrity: sha512-xXDvecyTpGLrqFrvkrUSoxxfJI5AH7U8zxxtVclpsUtMCq4JQ290LY8AW5c7Ggnr/Y/oK+bQMbqK2qmtk3pN4g==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/jszip/-/jszip-3.10.1.tgz}
name: jszip
version: 3.10.1
dependencies:
lie: registry.npmmirror.com/lie/3.3.0
pako: registry.npmmirror.com/pako/1.0.11
readable-stream: registry.npmmirror.com/readable-stream/2.3.8
setimmediate: registry.npmmirror.com/setimmediate/1.0.5
dev: false
registry.npmmirror.com/jwa/1.4.1:
resolution: {integrity: sha512-qiLX/xhEEFKUAJ6FiBMbes3w9ATzyk5W7Hvzpa/SLYdxNtng+gcurvrI7TbACjIXlsJyr05/S1oUhZrc63evQA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/jwa/-/jwa-1.4.1.tgz}
name: jwa
......@@ -7684,6 +7743,14 @@ packages:
type-check: registry.npmmirror.com/type-check/0.4.0
dev: true
registry.npmmirror.com/lie/3.3.0:
resolution: {integrity: sha512-UaiMJzeWRlEujzAuw5LokY1L5ecNQYZKfmyZ9L7wDHb/p5etKaxXhohBcrw0EYby+G/NA52vRSN4N39dxHAIwQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/lie/-/lie-3.3.0.tgz}
name: lie
version: 3.3.0
dependencies:
immediate: registry.npmmirror.com/immediate/3.0.6
dev: false
registry.npmmirror.com/lilconfig/2.0.6:
resolution: {integrity: sha512-9JROoBW7pobfsx+Sq2JsASvCo6Pfo6WWoUW79HuB1BCoBXD4PLWJPqDF6fNj67pqBYTbAHkE57M1kS/+L1neOg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/lilconfig/-/lilconfig-2.0.6.tgz}
name: lilconfig
......@@ -7801,6 +7868,16 @@ packages:
dependencies:
js-tokens: registry.npmmirror.com/js-tokens/4.0.0
registry.npmmirror.com/lop/0.4.1:
resolution: {integrity: sha512-9xyho9why2A2tzm5aIcMWKvzqKsnxrf9B5I+8O30olh6lQU8PH978LqZoI4++37RBgS1Em5i54v1TFs/3wnmXQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/lop/-/lop-0.4.1.tgz}
name: lop
version: 0.4.1
dependencies:
duck: registry.npmmirror.com/duck/0.1.12
option: registry.npmmirror.com/option/0.2.4
underscore: registry.npmmirror.com/underscore/1.13.6
dev: false
registry.npmmirror.com/lowlight/1.20.0:
resolution: {integrity: sha512-8Ktj+prEb1RoCPkEOrPMYUN/nCggB7qAWe3a7OpMjWQkh3l2RD5wKRQ+o8Q8YuI9RG/xs95waaI/E6ym/7NsTw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/lowlight/-/lowlight-1.20.0.tgz}
name: lowlight
......@@ -7826,6 +7903,23 @@ packages:
dependencies:
yallist: registry.npmmirror.com/yallist/4.0.0
registry.npmmirror.com/mammoth/1.5.1:
resolution: {integrity: sha512-7ZioZBf/1HjYrm1qZJOO+DD+rYxLvwrHS+HVOwW89hwIp+r6ZqJ/Eq2rXSS+8ezZ3/DuW6FUUp2Dfz6e7B2pBQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/mammoth/-/mammoth-1.5.1.tgz}
name: mammoth
version: 1.5.1
hasBin: true
dependencies:
argparse: registry.npmmirror.com/argparse/1.0.10
bluebird: registry.npmmirror.com/bluebird/3.4.7
dingbat-to-unicode: registry.npmmirror.com/dingbat-to-unicode/1.0.1
jszip: registry.npmmirror.com/jszip/3.10.1
lop: registry.npmmirror.com/lop/0.4.1
path-is-absolute: registry.npmmirror.com/path-is-absolute/1.0.1
sax: registry.npmmirror.com/sax/1.1.6
underscore: registry.npmmirror.com/underscore/1.13.6
xmlbuilder: registry.npmmirror.com/xmlbuilder/10.1.1
dev: false
registry.npmmirror.com/markdown-table/3.0.3:
resolution: {integrity: sha512-Z1NL3Tb1M9wH4XESsCDEksWoKTdlUafKc4pt0GRwjUyXaCFZ+dc3g2erqB6zm3szA2IUSi7VnPI+o/9jnxh9hw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/markdown-table/-/markdown-table-3.0.3.tgz}
name: markdown-table
......@@ -8750,6 +8844,12 @@ packages:
- debug
dev: false
registry.npmmirror.com/option/0.2.4:
resolution: {integrity: sha512-pkEqbDyl8ou5cpq+VsnQbe/WlEy5qS7xPzMS1U55OCG9KPvwFD46zDbxQIj3egJSFc3D+XhYOPUzz49zQAVy7A==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/option/-/option-0.2.4.tgz}
name: option
version: 0.2.4
dev: false
registry.npmmirror.com/optionator/0.9.1:
resolution: {integrity: sha512-74RlY5FCnhq4jRxVUPKDaRwrVNXMqsGsiW6AJw4XK8hmtm10wC0ypZBLw5IIp85NZMr91+qd1RvvENwg7jjRFw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/optionator/-/optionator-0.9.1.tgz}
name: optionator
......@@ -8791,6 +8891,12 @@ packages:
aggregate-error: registry.npmmirror.com/aggregate-error/3.1.0
dev: true
registry.npmmirror.com/pako/1.0.11:
resolution: {integrity: sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/pako/-/pako-1.0.11.tgz}
name: pako
version: 1.0.11
dev: false
registry.npmmirror.com/parent-module/1.0.1:
resolution: {integrity: sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/parent-module/-/parent-module-1.0.1.tgz}
name: parent-module
......@@ -8841,7 +8947,6 @@ packages:
name: path-is-absolute
version: 1.0.1
engines: {node: '>=0.10.0'}
dev: true
registry.npmmirror.com/path-key/3.1.1:
resolution: {integrity: sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/path-key/-/path-key-3.1.1.tgz}
......@@ -8948,6 +9053,12 @@ packages:
engines: {node: '>=6'}
dev: false
registry.npmmirror.com/process-nextick-args/2.0.1:
resolution: {integrity: sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/process-nextick-args/-/process-nextick-args-2.0.1.tgz}
name: process-nextick-args
version: 2.0.1
dev: false
registry.npmmirror.com/prop-types/15.8.1:
resolution: {integrity: sha512-oj87CgZICdulUohogVAR7AjlC0327U4el4L6eAvOqCeudMDVU0NThNaV+b9Df4dXgSP1gXMTnPdhfe/2qDH5cg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/prop-types/-/prop-types-15.8.1.tgz}
name: prop-types
......@@ -9211,6 +9322,20 @@ packages:
loose-envify: registry.npmmirror.com/loose-envify/1.4.0
dev: false
registry.npmmirror.com/readable-stream/2.3.8:
resolution: {integrity: sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/readable-stream/-/readable-stream-2.3.8.tgz}
name: readable-stream
version: 2.3.8
dependencies:
core-util-is: registry.npmmirror.com/core-util-is/1.0.3
inherits: registry.npmmirror.com/inherits/2.0.4
isarray: registry.npmmirror.com/isarray/1.0.0
process-nextick-args: registry.npmmirror.com/process-nextick-args/2.0.1
safe-buffer: registry.npmmirror.com/safe-buffer/5.1.2
string_decoder: registry.npmmirror.com/string_decoder/1.1.1
util-deprecate: registry.npmmirror.com/util-deprecate/1.0.2
dev: false
registry.npmmirror.com/readable-stream/3.6.1:
resolution: {integrity: sha512-+rQmrWMYGA90yenhTYsLWAsLsqVC8osOw6PKE1HDYiO0gdPeKe/xDHNzIAIn4C91YQ6oenEhfYqqc1883qHbjQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/readable-stream/-/readable-stream-3.6.1.tgz}
name: readable-stream
......@@ -9467,6 +9592,12 @@ packages:
mri: registry.npmmirror.com/mri/1.2.0
dev: false
registry.npmmirror.com/safe-buffer/5.1.2:
resolution: {integrity: sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/safe-buffer/-/safe-buffer-5.1.2.tgz}
name: safe-buffer
version: 5.1.2
dev: false
registry.npmmirror.com/safe-buffer/5.2.1:
resolution: {integrity: sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/safe-buffer/-/safe-buffer-5.2.1.tgz}
name: safe-buffer
......@@ -9506,6 +9637,12 @@ packages:
source-map-js: registry.npmmirror.com/source-map-js/1.0.2
dev: false
registry.npmmirror.com/sax/1.1.6:
resolution: {integrity: sha512-8zci48uUQyfqynGDSkUMD7FCJB96hwLnlZOXlgs1l3TX+LW27t3psSWKUxC0fxVgA86i8tL4NwGcY1h/6t3ESg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/sax/-/sax-1.1.6.tgz}
name: sax
version: 1.1.6
dev: false
registry.npmmirror.com/scheduler/0.23.0:
resolution: {integrity: sha512-CtuThmgHNg7zIZWAXi3AsyIzA3n4xx7aNyjwC2VJldO2LMVDhFK+63xGqq6CsJH4rTAt6/M+N4GhZiDYPx9eUw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/scheduler/-/scheduler-0.23.0.tgz}
name: scheduler
......@@ -9530,6 +9667,12 @@ packages:
dependencies:
lru-cache: registry.npmmirror.com/lru-cache/6.0.0
registry.npmmirror.com/setimmediate/1.0.5:
resolution: {integrity: sha512-MATJdZp8sLqDl/68LfQmbP8zKPLQNV6BIZoIgrscFDQ+RsvK/BxeDQOgyxKKoh0y/8h3BqVFnCqQ/gd+reiIXA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/setimmediate/-/setimmediate-1.0.5.tgz}
name: setimmediate
version: 1.0.5
dev: false
registry.npmmirror.com/sharp/0.31.3:
resolution: {integrity: sha512-XcR4+FCLBFKw1bdB+GEhnUNXNXvnt0tDo4WsBsraKymuo/IAuPuCBVAL2wIkUw2r/dwFW5Q5+g66Kwl2dgDFVg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/sharp/-/sharp-0.31.3.tgz}
name: sharp
......@@ -9713,6 +9856,12 @@ packages:
dev: false
optional: true
registry.npmmirror.com/sprintf-js/1.0.3:
resolution: {integrity: sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/sprintf-js/-/sprintf-js-1.0.3.tgz}
name: sprintf-js
version: 1.0.3
dev: false
registry.npmmirror.com/stable/0.1.8:
resolution: {integrity: sha512-ji9qxRnOVfcuLDySj9qzhGSEFVobyt1kIOSkj1qZzYLzq7Tos/oUUWvotUPQLlrsidqsK6tBH89Bc9kL5zHA6w==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/stable/-/stable-0.1.8.tgz}
name: stable
......@@ -9793,6 +9942,14 @@ packages:
es-abstract: registry.npmmirror.com/es-abstract/1.21.1
dev: true
registry.npmmirror.com/string_decoder/1.1.1:
resolution: {integrity: sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/string_decoder/-/string_decoder-1.1.1.tgz}
name: string_decoder
version: 1.1.1
dependencies:
safe-buffer: registry.npmmirror.com/safe-buffer/5.1.2
dev: false
registry.npmmirror.com/string_decoder/1.3.0:
resolution: {integrity: sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/string_decoder/-/string_decoder-1.3.0.tgz}
name: string_decoder
......@@ -10149,6 +10306,12 @@ packages:
which-boxed-primitive: registry.npmmirror.com/which-boxed-primitive/1.0.2
dev: true
registry.npmmirror.com/underscore/1.13.6:
resolution: {integrity: sha512-+A5Sja4HP1M08MaXya7p5LvjuM7K6q/2EaC0+iovj/wOcMsTzMvDFbasi/oSapiwOlt252IqsKqPjCl7huKS0A==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/underscore/-/underscore-1.13.6.tgz}
name: underscore
version: 1.13.6
dev: false
registry.npmmirror.com/unicode-canonical-property-names-ecmascript/2.0.0:
resolution: {integrity: sha512-yY5PpDlfVIU5+y/BSCxAJRBIS1Zc2dDG3Ujq+sR0U+JjUevW2JhocOF+soROYDSaAezOzOKuyyixhD6mBknSmQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/unicode-canonical-property-names-ecmascript/-/unicode-canonical-property-names-ecmascript-2.0.0.tgz}
name: unicode-canonical-property-names-ecmascript
......@@ -10489,6 +10652,13 @@ packages:
name: wrappy
version: 1.0.2
registry.npmmirror.com/xmlbuilder/10.1.1:
resolution: {integrity: sha512-OyzrcFLL/nb6fMGHbiRDuPup9ljBycsdCypwuyg5AAHvyWzGfChJpCXMG88AGTIMFhGZ9RccFN1e6lhg3hkwKg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/xmlbuilder/-/xmlbuilder-10.1.1.tgz}
name: xmlbuilder
version: 10.1.1
engines: {node: '>=4.0'}
dev: false
registry.npmmirror.com/xtend/4.0.2:
resolution: {integrity: sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/xtend/-/xtend-4.0.2.tgz}
name: xtend
......
public/js/pdf.js 0 → 100644
This source diff could not be displayed because it is too large. You can view the blob instead.
public/js/pdf.worker.js 0 → 100644
This source diff could not be displayed because it is too large. You can view the blob instead.
src/api/data.ts 0 → 100644
import { GET, POST, DELETE, PUT } from './request';
import { RequestPaging } from '../types/index';
import { Obj2Query } from '@/utils/tools';
import type { DataListItem } from '@/types/data';
import type { PagingData } from '../types/index';
export const getDataList = (data: RequestPaging) =>
GET<PagingData<DataListItem>>(`/data/getDataList?${Obj2Query(data)}`);
export const postData = (name: string) => POST<string>(`/data/postData?name=${name}`);
export const postSplitData = (dataId: string, text: string) =>
POST(`/data/splitData`, { dataId, text });
src/components/Layout/index.tsx
......@@ -26,12 +26,12 @@ const navbarList = [
link: '/model/list',
activeLink: ['/model/list', '/model/detail']
},
// {
// label: '数据',
// icon: 'icon-datafull',
// link: '/training/dataList',
// activeLink: ['/training/dataList']
// },
{
label: '数据',
icon: 'icon-datafull',
link: '/data/list',
activeLink: ['/data/list']
},
{
label: '账号',
icon: 'icon-yonghu-yuan',
......@@ -62,8 +62,8 @@ const Layout = ({ children }: { children: JSX.Element }) => {
<Box h={'100%'} position={'fixed'} left={0} top={0} w={'80px'}>
<Navbar navbarList={navbarList} />
</Box>
<Box ml={'80px'} p={7}>
<Box maxW={'1100px'} m={'auto'}>
<Box ml={'80px'} p={7} h={'100%'}>
<Box maxW={'1100px'} m={'auto'} h={'100%'}>
<Auth>{children}</Auth>
</Box>
</Box>
......
src/components/ScrollData/index.tsx 0 → 100644
import React from 'react';
import type { BoxProps } from '@chakra-ui/react';
import { Box } from '@chakra-ui/react';
interface Props extends BoxProps {
nextPage: () => void;
children: React.ReactNode;
}
const ScrollData = ({ children, nextPage, ...props }: Props) => {
return (
<Box {...props} overflow={'auto'}>
{children}
</Box>
);
};
export default ScrollData;
src/hooks/useConfirm.tsx
......@@ -19,9 +19,10 @@ export const useConfirm = ({ title = '提示', content }: { title?: string; cont
return {
openConfirm: useCallback(
(confirm?: any, cancel?: any) => {
onOpen();
confirmCb.current = confirm;
cancelCb.current = cancel;
return onOpen;
},
[onOpen]
),
......
src/hooks/usePaging.ts
import { useState, useCallback } from 'react';
import type { PagingData } from '../types/index';
import { useQuery } from '@tanstack/react-query';
import { useToast } from './useToast';
export const usePaging = <T = any>({
api,
pageSize = 10,
params
params = {}
}: {
api: (data: any) => Promise<PagingData<T>>;
pageSize?: number;
params?: Record<string, any>;
}) => {
const { toast } = useToast();
const [data, setData] = useState<T[]>([]);
const [pageNum, setPageNum] = useState(1);
const [total, setTotal] = useState(0);
......@@ -18,36 +20,40 @@ export const usePaging = <T = any>({
const [requesting, setRequesting] = useState(false);
const getData = useCallback(
async (init = false) => {
async (num: number, init = false) => {
if (requesting) return;
if (!init && isLoadAll) return;
setRequesting(true);
try {
const res = await api({
pageNum,
pageNum: num,
pageSize,
...(params ? params : {})
...params
});
setData((state) => {
const data = init ? res.data : state.concat(res.data);
if (data.length >= res.total) {
setIsLoadAll(true);
}
setTotal(res.total);
return data;
});
setTotal(res.total);
} catch (error) {
} catch (error: any) {
toast({
title: error?.message || '获取数据异常',
status: 'error'
});
console.log(error);
}
setRequesting(false);
return null;
},
[api, isLoadAll, pageNum, pageSize, params, requesting]
[api, isLoadAll, pageSize, params, requesting, toast]
);
useQuery(['init', pageNum], () => getData(pageNum === 1));
useQuery(['init', pageNum], () => getData(pageNum, pageNum === 1));
return {
pageNum,
......@@ -55,6 +61,7 @@ export const usePaging = <T = any>({
setPageNum,
total,
data,
getData
getData,
requesting
};
};
src/hooks/useSelectFile.tsx 0 → 100644
import React, { useRef, useCallback } from 'react';
import { Box } from '@chakra-ui/react';
export const useSelectFile = (props?: { fileType?: string; multiple?: boolean }) => {
const { fileType = '*', multiple = false } = props || {};
const SelectFileDom = useRef<HTMLInputElement>(null);
const File = useCallback(
({ onSelect }: { onSelect: (e: File[]) => void }) => (
<Box position={'absolute'} w={0} h={0} overflow={'hidden'}>
<input
ref={SelectFileDom}
type="file"
accept={fileType}
multiple={multiple}
onChange={(e) => {
if (!e.target.files || e.target.files?.length === 0) return;
onSelect(Array.from(e.target.files));
}}
/>
</Box>
),
[fileType, multiple]
);
const onOpen = useCallback(() => {
SelectFileDom.current && SelectFileDom.current.click();
}, []);
return {
File,
onOpen
};
};
src/hooks/useTabs.tsx 0 → 100644
import React, { useState, useCallback, useRef } from 'react';
export const useTabs = ({
tabs = []
}: {
tabs: {
id: string;
label: string;
}[];
}) => {
const [activeTab, setActiveTab] = useState(tabs[0].id);
return {
tabs,
activeTab,
setActiveTab
};
};
src/pages/_app.tsx
......@@ -38,8 +38,9 @@ export default function App({ Component, pageProps }: AppProps) {
/>
<link rel="icon" href="/favicon.ico" />
</Head>
<Script src="/iconfont.js" strategy="afterInteractive"></Script>
<Script src="/qrcode.min.js" strategy="afterInteractive"></Script>
<Script src="/js/iconfont.js" strategy="afterInteractive"></Script>
<Script src="/js/qrcode.min.js" strategy="afterInteractive"></Script>
<Script src="/js/pdf.js" strategy="afterInteractive"></Script>
<QueryClientProvider client={queryClient}>
<ChakraProvider theme={theme}>
<ColorModeScript initialColorMode={theme.config.initialColorMode} />
......
src/pages/api/data/getDataList.ts 0 → 100644
// Next.js API route support: https://nextjs.org/docs/api-routes/introduction
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase, Data, DataItem } from '@/service/mongo';
import { authToken } from '@/service/utils/tools';
import type { DataSchema } from '@/types/mongoSchema';
import type { DataListItem } from '@/types/data';
import type { PagingData } from '@/types';
import mongoose from 'mongoose';
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
const { authorization } = req.headers;
let { pageNum = 1, pageSize = 10 } = req.query as { pageNum: string; pageSize: string };
pageNum = +pageNum;
pageSize = +pageSize;
if (!authorization) {
throw new Error('缺少登录凭证');
}
const userId = await authToken(authorization);
await connectToDatabase();
// 根据 id 获取用户账单
const datalist = await Data.aggregate<DataListItem>([
{
$match: {
userId: new mongoose.Types.ObjectId(userId)
}
},
{
$sort: { createTime: -1 } // 按照创建时间倒序排列
},
{
$skip: (pageNum - 1) * pageSize // 跳过前面的数据
},
{
$limit: pageSize // 取出指定数量的数据
},
{
$lookup: {
from: 'dataitems',
localField: '_id',
foreignField: 'dataId',
as: 'items'
}
},
{
$addFields: {
totalData: {
$size: '$items' // 统计dataItem的总数
},
trainingData: {
$size: {
$filter: {
input: '$items',
as: 'item',
cond: { $eq: ['$$item.status', 1] } // 统计status为1的数量
}
}
}
}
},
{
$project: {
items: 0 // 不返回 items 字段
}
}
]);
jsonRes<PagingData<DataListItem>>(res, {
data: {
pageNum,
pageSize,
data: datalist,
total: 1
}
});
} catch (err) {
jsonRes(res, {
code: 500,
error: err
});
}
}
src/pages/api/data/postData.ts 0 → 100644
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase, Data } from '@/service/mongo';
import { authToken } from '@/service/utils/tools';
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
let { name } = req.query as { name: string };
if (!name) {
throw new Error('参数错误');
}
await connectToDatabase();
const { authorization } = req.headers;
const userId = await authToken(authorization);
// 生成 data 集合
const data = await Data.create({
userId,
name
});
jsonRes(res, {
data: data._id
});
} catch (err) {
jsonRes(res, {
code: 500,
error: err
});
}
}
src/pages/api/data/splitData.ts
......@@ -4,11 +4,10 @@ import { connectToDatabase, Data, DataItem } from '@/service/mongo';
import { authToken } from '@/service/utils/tools';
import { generateQA } from '@/service/events/generateQA';
/* 定时删除那些不活跃的内容 */
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
let { text, name } = req.body as { text: string; name: string };
if (!text || !name) {
let { text, dataId } = req.body as { text: string; dataId: string };
if (!text || !dataId) {
throw new Error('参数错误');
}
text = text.replace(/\n+/g, '\n');
......@@ -18,28 +17,16 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
const userId = await authToken(authorization);
// 生成 data 父级
const data = await Data.create({
userId,
name
});
const dataItems: any[] = [];
// 格式化文本长度
for (let i = 0; i <= text.length / 1000; i++) {
const dataItem = {
dataItems.push({
temperature: 0,
userId,
dataId: data._id,
dataId,
text: text.slice(i * 1000, (i + 1) * 1000),
status: 1
};
[0, 0.2, 0.4, 0.6, 0.8, 1.0].forEach((temperature) => {
dataItems.push({
temperature,
...dataItem
});
});
}
......@@ -58,8 +45,3 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
});
}
}
/**
* 检查文本是否按格式返回
*/
function splitText(text: string) {}
src/pages/data/components/CreateDataModal.tsx 0 → 100644
import React, { useState } from 'react';
import {
Modal,
ModalOverlay,
ModalContent,
ModalHeader,
ModalFooter,
ModalBody,
ModalCloseButton,
Button,
Input
} from '@chakra-ui/react';
import { postData } from '@/api/data';
import { useMutation } from '@tanstack/react-query';
const CreateDataModal = ({
onClose,
onSuccess
}: {
onClose: () => void;
onSuccess: () => void;
}) => {
const [inputVal, setInputVal] = useState('');
const { isLoading, mutate } = useMutation({
mutationFn: (name: string) => postData(name),
onSuccess() {
onSuccess();
onClose();
}
});
return (
<Modal isOpen={true} onClose={onClose}>
<ModalOverlay />
<ModalContent>
<ModalHeader>创建数据集</ModalHeader>
<ModalCloseButton />
<ModalBody display={'flex'}>
<Input
value={inputVal}
onChange={(e) => setInputVal(e.target.value)}
placeholder={'数据集名称'}
></Input>
</ModalBody>
<ModalFooter>
<Button colorScheme={'gray'} onClick={onClose}>
取消
</Button>
<Button
ml={3}
isDisabled={inputVal === ''}
isLoading={isLoading}
onClick={() => mutate(inputVal)}
>
确认
</Button>
</ModalFooter>
</ModalContent>
</Modal>
);
};
export default CreateDataModal;
src/pages/data/components/ImportDataModal.tsx 0 → 100644
import React, { useState, useCallback } from 'react';
import {
Modal,
ModalOverlay,
ModalContent,
ModalHeader,
ModalFooter,
ModalBody,
ModalCloseButton,
Button,
Input,
Box,
Flex,
Textarea
} from '@chakra-ui/react';
import { useTabs } from '@/hooks/useTabs';
import { useConfirm } from '@/hooks/useConfirm';
import { useSelectFile } from '@/hooks/useSelectFile';
import { readTxtContent, readPdfContent, readDocContent } from '@/utils/tools';
import { postSplitData } from '@/api/data';
import { useMutation } from '@tanstack/react-query';
import { useToast } from '@/hooks/useToast';
import { useLoading } from '@/hooks/useLoading';
const ImportDataModal = ({ dataId, onClose }: { dataId: string; onClose: () => void }) => {
const { openConfirm, ConfirmChild } = useConfirm({
content: '确认提交生成任务？该任务无法终止！'
});
const { toast } = useToast();
const { setIsLoading, Loading } = useLoading();
const { File, onOpen } = useSelectFile({ fileType: '.txt,.doc,.docx,.pdf', multiple: true });
const { tabs, activeTab, setActiveTab } = useTabs({
tabs: [
{ id: 'text', label: '文本' },
{ id: 'doc', label: '文件' },
{ id: 'url', label: '链接' }
]
});
const [textInput, setTextInput] = useState('');
const [fileText, setFileText] = useState('');
const { mutate: handleClickSubmit, isLoading } = useMutation({
mutationFn: async () => {
let text = '';
if (activeTab === 'text') {
text = textInput;
} else if (activeTab === 'doc') {
text = fileText;
} else if (activeTab === 'url') {
}
if (!text) return;
return postSplitData(dataId, text);
},
onSuccess() {
toast({
title: '任务提交成功',
status: 'success'
});
onClose();
},
onError(err: any) {
toast({
title: err?.message || '提交任务异常',
status: 'error'
});
}
});
const onSelectFile = useCallback(
async (e: File[]) => {
setIsLoading(true);
try {
const fileTexts = (
await Promise.all(
e.map((file) => {
// @ts-ignore
const extension = file?.name?.split('.').pop().toLowerCase();
if (extension === 'txt') {
return readTxtContent(file);
} else if (extension === 'pdf') {
return readPdfContent(file);
} else if (extension === 'docx' || extension === 'doc') {
return readDocContent(file);
}
return '';
})
)
).join('\n');
setFileText(fileTexts);
} catch (error: any) {
console.log(error);
toast({
title: typeof error === 'string' ? error : '解析文件失败',
status: 'error'
});
}
setIsLoading(false);
},
[setIsLoading, toast]
);
return (
<Modal isOpen={true} onClose={onClose}>
<ModalOverlay />
<ModalContent position={'relative'} maxW={['90vw', '800px']}>
<ModalHeader>导入数据，生成QA</ModalHeader>
<ModalCloseButton />
<ModalBody display={'flex'}>
<Box>
{tabs.map((item) => (
<Button
key={item.id}
display={'block'}
variant={activeTab === item.id ? 'solid' : 'outline'}
_notLast={{
mb: 3
}}
onClick={() => setActiveTab(item.id)}
>
{item.label}
</Button>
))}
</Box>
<Box flex={'1 0 0'} w={0} ml={3} minH={'200px'}>
{activeTab === 'text' && (
<Textarea
h={'100%'}
maxLength={-1}
value={textInput}
placeholder={'请粘贴或输入需要处理的文本'}
onChange={(e) => setTextInput(e.target.value)}
/>
)}
{activeTab === 'doc' && (
<Flex
flexDirection={'column'}
h={'100%'}
alignItems={'center'}
justifyContent={'center'}
border={'1px solid '}
borderColor={'blackAlpha.200'}
borderRadius={'md'}
>
<Button onClick={onOpen}>选择文件</Button>
{fileText && <Box mt={2}>一共 {fileText.length} 个字</Box>}
</Flex>
)}
</Box>
</ModalBody>
<ModalFooter>
<Button colorScheme={'gray'} onClick={onClose}>
取消
</Button>
<Button
ml={3}
isLoading={isLoading}
isDisabled={!textInput && !fileText}
onClick={openConfirm(handleClickSubmit)}
>
确认
</Button>
</ModalFooter>
<Loading />
</ModalContent>
<ConfirmChild />
<File onSelect={onSelectFile} />
</Modal>
);
};
export default ImportDataModal;
src/pages/data/list.tsx 0 → 100644
import React, { useState } from 'react';
import {
Card,
Box,
Flex,
Button,
Table,
Thead,
Tbody,
Tr,
Th,
Td,
TableContainer,
useDisclosure
} from '@chakra-ui/react';
import { getDataList } from '@/api/data';
import { usePaging } from '@/hooks/usePaging';
import type { DataListItem } from '@/types/data';
import ScrollData from '@/components/ScrollData';
import dayjs from 'dayjs';
import dynamic from 'next/dynamic';
const CreateDataModal = dynamic(() => import('./components/CreateDataModal'));
const ImportDataModal = dynamic(() => import('./components/ImportDataModal'));
const DataList = () => {
const {
setPageNum,
pageNum,
data: dataList,
getData
} = usePaging<DataListItem>({
api: getDataList,
pageSize: 20
});
const [ImportDataId, setImportDataId] = useState<string>();
const {
isOpen: isOpenCreateDataModal,
onOpen: onOpenCreateDataModal,
onClose: onCloseCreateDataModal
} = useDisclosure();
return (
<Box display={['block', 'flex']} flexDirection={'column'} h={'100%'}>
<Card px={6} py={4}>
<Flex>
<Box flex={1} mr={1}>
<Box fontSize={'xl'} fontWeight={'bold'}>
对话数据管理
</Box>
<Box fontSize={'xs'} color={'blackAlpha.600'}>
允许你将任意文本数据拆分成 QA 的形式。你可以使用这些 QA 去微调你的对话模型。
</Box>
</Box>
<Button variant={'outline'} onClick={onOpenCreateDataModal}>
创建数据集
</Button>
</Flex>
</Card>
{/* 数据表 */}
<Card mt={3} flex={'1 0 0'} h={['auto', '0']} px={6} py={4}>
<ScrollData h={'100%'} nextPage={() => setPageNum(pageNum + 1)}>
<TableContainer>
<Table>
<Thead>
<Tr>
<Th>集合名</Th>
<Th>创建时间</Th>
<Th>训练中 / 总数据</Th>
<Th></Th>
</Tr>
</Thead>
<Tbody>
{dataList.map((item, i) => (
<Tr key={item._id}>
<Td>{item.name}</Td>
<Td>{dayjs(item.createTime).format('YYYY/MM/DD HH:mm')}</Td>
<Td>
{item.trainingData} / {item.totalData}
</Td>
<Td>
<Button
size={'sm'}
variant={'outline'}
mr={2}
onClick={() => setImportDataId(item._id)}
>
导入
</Button>
<Button size={'sm'}>导出</Button>
</Td>
</Tr>
))}
</Tbody>
</Table>
</TableContainer>
</ScrollData>
</Card>
{ImportDataId && (
<ImportDataModal dataId={ImportDataId} onClose={() => setImportDataId(undefined)} />
)}
{isOpenCreateDataModal && (
<CreateDataModal onClose={onCloseCreateDataModal} onSuccess={() => getData(1, true)} />
)}
</Box>
);
};
export default DataList;
src/pages/model/detail.tsx
......@@ -291,15 +291,7 @@ const ModelDetail = ({ modelId }: { modelId: string }) => {
</Box>
<Flex mt={5} alignItems={'center'}>
<Box flex={'0 0 80px'}>删除模型:</Box>
<Button
colorScheme={'red'}
size={'sm'}
onClick={() => {
openConfirm(() => {
handleDelModel();
});
}}
>
<Button colorScheme={'red'} size={'sm'} onClick={openConfirm(handleDelModel)}>
删除模型
</Button>
</Flex>
......
src/pages/training/dataList.tsx deleted 100644 → 0
import React from 'react';
import { Card, Box, Flex, Button } from '@chakra-ui/react';
const TrainDataList = () => {
return (
<>
<Card px={6} py={4}>
<Flex alignItems={'center'}>
<Box fontSize={'xl'} fontWeight={'bold'} flex={1}>
训练数据管理
</Box>
<Button variant={'outline'} mr={6}>
导入数据
</Button>
<Button>插入一条数据</Button>
</Flex>
</Card>
{/* 数据表 */}
</>
);
};
export default TrainDataList;
src/service/events/generateQA.ts
......@@ -23,7 +23,7 @@ export async function generateQA(next = false): Promise<any> {
try {
// 找出一个需要生成的 dataItem
dataItem = await DataItem.findOne({
status: 1,
status: { $ne: 0 },
times: { $gt: 0 }
});
......@@ -56,7 +56,8 @@ export async function generateQA(next = false): Promise<any> {
throw new Error('获取 openai key 失败');
}
console.log('正在生成一个QA', dataItem._id);
console.log('正在生成一个QA, ID:', dataItem._id, 'temperature: ', dataItem.temperature / 100);
const startTime = Date.now();
// 获取 openai 请求实例
......@@ -65,7 +66,7 @@ export async function generateQA(next = false): Promise<any> {
const response = await chatAPI.createChatCompletion(
{
model: ChatModelNameEnum.GPT35,
temperature: dataItem.temperature,
temperature: dataItem.temperature / 100,
n: 1,
messages: [
systemPrompt,
......@@ -76,28 +77,27 @@ export async function generateQA(next = false): Promise<any> {
]
},
{
timeout: 60000,
timeout: 120000,
httpsAgent
}
);
const content = response.data.choices[0].message?.content;
// 从 content 中提取 QA
const splitResponse = splitText(content || '');
if (splitResponse.length > 0) {
// 插入数据库，并修改状态
await DataItem.findByIdAndUpdate(dataItem._id, {
status: 0,
$push: {
result: {
$each: splitResponse
}
// 插入数据库，并修改状态
await DataItem.findByIdAndUpdate(dataItem._id, {
status: dataItem.temperature >= 100 ? 0 : 1,
temperature: dataItem.temperature >= 100 ? dataItem.temperature : dataItem.temperature + 25,
$push: {
result: {
$each: splitResponse
}
});
console.log('生成成功，time:', `${(Date.now() - startTime) / 1000}s`);
}
}
});
console.log('生成成功，time:', `${(Date.now() - startTime) / 1000}s`);
} catch (error: any) {
console.log('error: 生成QA错误', dataItem?._id);
console.log('statusText:', error?.response?.statusText);
console.log('response:', error?.response);
// 重置状态
if (dataItem?._id) {
await DataItem.findByIdAndUpdate(dataItem._id, {
......
src/service/models/bill.ts
import { Schema, model, models } from 'mongoose';
import { ModelList } from '@/constants/model';
import { modelList } from '@/constants/model';
const BillSchema = new Schema({
userId: {
......@@ -14,7 +14,7 @@ const BillSchema = new Schema({
},
modelName: {
type: String,
enum: ModelList.map((item) => item.model),
enum: modelList.map((item) => item.model),
required: true
},
chatId: {
......
src/types/data.d.ts 0 → 100644
import type { DataSchema } from './mongoSchema';
export interface DataListItem extends DataSchema {
trainingData: number;
totalData: number;
}
src/types/index.d.ts
......@@ -4,6 +4,9 @@ declare global {
var mongodb: Mongoose | string | null;
var generatingQA: boolean;
var QRCode: any;
interface Window {
['pdfjs-dist/build/pdf']: any;
}
}
export type PagingData<T> = {
......
src/utils/tools.ts
import crypto from 'crypto';
import { useToast } from '@/hooks/useToast';
import mammoth from 'mammoth';
/**
* copy text data
......@@ -50,17 +51,76 @@ export const Obj2Query = (obj: Record<string, string | number>) => {
};
/**
* 读取文件内容
* 读取 txt 文件内容
*/
export const loadLocalFileContent = (file: File) => {
export const readTxtContent = (file: File) => {
return new Promise((resolve: (_: string) => void, reject) => {
const reader = new FileReader();
reader.onload = () => {
resolve(reader.result as string);
};
reader.onerror = (err) => {
reject(err);
console.log('error txt read:', err);
reject('读取 txt 文件失败');
};
reader.readAsText(file);
});
};
/**
* 读取 pdf 内容
*/
export const readPdfContent = (file: File) =>
new Promise<string>((resolve, reject) => {
const pdfjsLib = window['pdfjs-dist/build/pdf'];
pdfjsLib.workerSrc = '/js/pdf.worker.js';
const readPDFPage = async (doc: any, pageNo: number) => {
const page = await doc.getPage(pageNo);
const tokenizedText = await page.getTextContent();
const pageText = tokenizedText.items.map((token: any) => token.str).join('');
return pageText.replaceAll(/\s+/g, '\n');
};
let reader = new FileReader();
reader.readAsArrayBuffer(file);
reader.onload = async (event) => {
if (!event?.target?.result) return reject('解析 PDF 失败');
try {
const doc = await pdfjsLib.getDocument(event.target.result).promise;
const pageTextPromises = [];
for (let pageNo = 1; pageNo <= doc.numPages; pageNo++) {
pageTextPromises.push(readPDFPage(doc, pageNo));
}
const pageTexts = await Promise.all(pageTextPromises);
resolve(pageTexts.join('\n'));
} catch (err) {
console.log(err, 'pdfjs error');
reject('解析 PDF 失败');
}
};
reader.onerror = (err) => {
console.log(err, 'reader error');
reject('解析 PDF 失败');
};
});
/**
* 读取doc
*/
export const readDocContent = (file: File) =>
new Promise<string>((resolve, reject) => {
const reader = new FileReader();
reader.readAsArrayBuffer(file);
reader.onload = ({ target }) => {
if (!target?.result) return reject('读取 doc 文件失败');
return mammoth.extractRawText({ arrayBuffer: target.result as ArrayBuffer }).then((res) => {
resolve(res.value);
});
};
reader.onerror = (err) => {
console.log('error doc read:', err);
reject('读取 doc 文件失败');
};
});
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment