Skip to content

赵月辉 / fastgpt-migrated

Go to a project
Commit 9e951fbc by archer
Options

feat: pg引入

parent a540ee94
Showing with 256 additions and 146 deletions
package.json
......@@ -40,6 +40,7 @@
"nprogress": "^0.2.0",
"openai": "^3.2.1",
"papaparse": "^5.4.1",
"pg": "^8.10.0",
"react": "18.2.0",
"react-dom": "18.2.0",
"react-hook-form": "^7.43.1",
......@@ -63,11 +64,11 @@
"@types/node": "18.14.0",
"@types/nodemailer": "^6.4.7",
"@types/papaparse": "^5.3.7",
"@types/pg": "^8.6.6",
"@types/react": "18.0.28",
"@types/react-dom": "18.0.11",
"@types/react-syntax-highlighter": "^15.5.6",
"@types/tunnel": "^0.0.3",
"@types/uuid": "^9.0.1",
"eslint": "8.34.0",
"eslint-config-next": "13.1.6",
"husky": "^8.0.3",
......
pnpm-lock.yaml
......@@ -18,11 +18,11 @@ specifiers:
'@types/nodemailer': ^6.4.7
'@types/nprogress': ^0.2.0
'@types/papaparse': ^5.3.7
'@types/pg': ^8.6.6
'@types/react': 18.0.28
'@types/react-dom': 18.0.11
'@types/react-syntax-highlighter': ^15.5.6
'@types/tunnel': ^0.0.3
'@types/uuid': ^9.0.1
axios: ^1.3.3
crypto: ^1.0.1
dayjs: ^1.11.7
......@@ -46,6 +46,7 @@ specifiers:
nprogress: ^0.2.0
openai: ^3.2.1
papaparse: ^5.4.1
pg: ^8.10.0
prettier: ^2.8.4
react: 18.2.0
react-dom: 18.2.0
......@@ -93,6 +94,7 @@ dependencies:
nprogress: registry.npmmirror.com/nprogress/0.2.0
openai: registry.npmmirror.com/openai/3.2.1
papaparse: registry.npmmirror.com/papaparse/5.4.1
pg: registry.npmmirror.com/pg/8.10.0
react: registry.npmmirror.com/react/18.2.0
react-dom: registry.npmmirror.com/react-dom/18.2.0_react@18.2.0
react-hook-form: registry.npmmirror.com/react-hook-form/7.43.1_react@18.2.0
......@@ -116,11 +118,11 @@ devDependencies:
'@types/node': registry.npmmirror.com/@types/node/18.14.0
'@types/nodemailer': registry.npmmirror.com/@types/nodemailer/6.4.7
'@types/papaparse': registry.npmmirror.com/@types/papaparse/5.3.7
'@types/pg': registry.npmmirror.com/@types/pg/8.6.6
'@types/react': registry.npmmirror.com/@types/react/18.0.28
'@types/react-dom': registry.npmmirror.com/@types/react-dom/18.0.11
'@types/react-syntax-highlighter': registry.npmmirror.com/@types/react-syntax-highlighter/15.5.6
'@types/tunnel': registry.npmmirror.com/@types/tunnel/0.0.3
'@types/uuid': registry.npmmirror.com/@types/uuid/9.0.1
eslint: registry.npmmirror.com/eslint/8.34.0
eslint-config-next: registry.npmmirror.com/eslint-config-next/13.1.6_7kw3g6rralp5ps6mg3uyzz6azm
husky: registry.npmmirror.com/husky/8.0.3
......@@ -5192,6 +5194,16 @@ packages:
name: '@types/parse-json'
version: 4.0.0
registry.npmmirror.com/@types/pg/8.6.6:
resolution: {integrity: sha512-O2xNmXebtwVekJDD+02udOncjVcMZQuTEQEMpKJ0ZRf5E7/9JJX3izhKUcUifBkyKpljyUM6BTgy2trmviKlpw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@types/pg/-/pg-8.6.6.tgz}
name: '@types/pg'
version: 8.6.6
dependencies:
'@types/node': registry.npmmirror.com/@types/node/18.14.0
pg-protocol: registry.npmmirror.com/pg-protocol/1.6.0
pg-types: registry.npmmirror.com/pg-types/2.2.0
dev: true
registry.npmmirror.com/@types/prop-types/15.7.5:
resolution: {integrity: sha512-JCB8C6SnDoQf0cNycqd/35A7MjcnK+ZTqE7judS6o7utxUCg6imJg3QK2qzHKszlTjcj2cn+NwMB2i96ubpj7w==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@types/prop-types/-/prop-types-15.7.5.tgz}
name: '@types/prop-types'
......@@ -5241,12 +5253,6 @@ packages:
version: 2.0.6
dev: false
registry.npmmirror.com/@types/uuid/9.0.1:
resolution: {integrity: sha512-rFT3ak0/2trgvp4yYZo5iKFEPsET7vKydKF+VRCxlQ9bpheehyAJH89dAkaLEq/j/RZXJIqcgsmPJKUP1Z28HA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@types/uuid/-/uuid-9.0.1.tgz}
name: '@types/uuid'
version: 9.0.1
dev: true
registry.npmmirror.com/@types/webidl-conversions/7.0.0:
resolution: {integrity: sha512-xTE1E+YF4aWPJJeUzaZI5DRntlkY3+BCVJi0axFptnjGmAoWxkyREIh/XMrfxVLejwQxMCfDXdICo0VLxThrog==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@types/webidl-conversions/-/webidl-conversions-7.0.0.tgz}
name: '@types/webidl-conversions'
......@@ -5816,6 +5822,13 @@ packages:
version: 1.0.1
dev: false
registry.npmmirror.com/buffer-writer/2.0.0:
resolution: {integrity: sha512-a7ZpuTZU1TRtnwyCNW3I5dc0wWNC3VR9S++Ewyk2HHZdrO3CQJqSpd+95Us590V6AL7JqUAH2IwZ/398PmNFgw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/buffer-writer/-/buffer-writer-2.0.0.tgz}
name: buffer-writer
version: 2.0.0
engines: {node: '>=4'}
dev: false
registry.npmmirror.com/buffer/5.7.1:
resolution: {integrity: sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/buffer/-/buffer-5.7.1.tgz}
name: buffer
......@@ -9733,6 +9746,12 @@ packages:
netmask: registry.npmmirror.com/netmask/2.0.2
dev: false
registry.npmmirror.com/packet-reader/1.0.0:
resolution: {integrity: sha512-HAKu/fG3HpHFO0AA8WE8q2g+gBJaZ9MG7fcKk+IJPLTGAD6Psw4443l+9DGRbOIh3/aXr7Phy0TjilYivJo5XQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/packet-reader/-/packet-reader-1.0.0.tgz}
name: packet-reader
version: 1.0.0
dev: false
registry.npmmirror.com/pako/1.0.11:
resolution: {integrity: sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/pako/-/pako-1.0.11.tgz}
name: pako
......@@ -9829,6 +9848,74 @@ packages:
through: registry.npmmirror.com/through/2.3.8
dev: false
registry.npmmirror.com/pg-connection-string/2.5.0:
resolution: {integrity: sha512-r5o/V/ORTA6TmUnyWZR9nCj1klXCO2CEKNRlVuJptZe85QuhFayC7WeMic7ndayT5IRIR0S0xFxFi2ousartlQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/pg-connection-string/-/pg-connection-string-2.5.0.tgz}
name: pg-connection-string
version: 2.5.0
dev: false
registry.npmmirror.com/pg-int8/1.0.1:
resolution: {integrity: sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/pg-int8/-/pg-int8-1.0.1.tgz}
name: pg-int8
version: 1.0.1
engines: {node: '>=4.0.0'}
registry.npmmirror.com/pg-pool/3.6.0_pg@8.10.0:
resolution: {integrity: sha512-clFRf2ksqd+F497kWFyM21tMjeikn60oGDmqMT8UBrynEwVEX/5R5xd2sdvdo1cZCFlguORNpVuqxIj+aK4cfQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/pg-pool/-/pg-pool-3.6.0.tgz}
id: registry.npmmirror.com/pg-pool/3.6.0
name: pg-pool
version: 3.6.0
peerDependencies:
pg: '>=8.0'
dependencies:
pg: registry.npmmirror.com/pg/8.10.0
dev: false
registry.npmmirror.com/pg-protocol/1.6.0:
resolution: {integrity: sha512-M+PDm637OY5WM307051+bsDia5Xej6d9IR4GwJse1qA1DIhiKlksvrneZOYQq42OM+spubpcNYEo2FcKQrDk+Q==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/pg-protocol/-/pg-protocol-1.6.0.tgz}
name: pg-protocol
version: 1.6.0
registry.npmmirror.com/pg-types/2.2.0:
resolution: {integrity: sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/pg-types/-/pg-types-2.2.0.tgz}
name: pg-types
version: 2.2.0
engines: {node: '>=4'}
dependencies:
pg-int8: registry.npmmirror.com/pg-int8/1.0.1
postgres-array: registry.npmmirror.com/postgres-array/2.0.0
postgres-bytea: registry.npmmirror.com/postgres-bytea/1.0.0
postgres-date: registry.npmmirror.com/postgres-date/1.0.7
postgres-interval: registry.npmmirror.com/postgres-interval/1.2.0
registry.npmmirror.com/pg/8.10.0:
resolution: {integrity: sha512-ke7o7qSTMb47iwzOSaZMfeR7xToFdkE71ifIipOAAaLIM0DYzfOAXlgFFmYUIE2BcJtvnVlGCID84ZzCegE8CQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/pg/-/pg-8.10.0.tgz}
name: pg
version: 8.10.0
engines: {node: '>= 8.0.0'}
peerDependencies:
pg-native: '>=3.0.1'
peerDependenciesMeta:
pg-native:
optional: true
dependencies:
buffer-writer: registry.npmmirror.com/buffer-writer/2.0.0
packet-reader: registry.npmmirror.com/packet-reader/1.0.0
pg-connection-string: registry.npmmirror.com/pg-connection-string/2.5.0
pg-pool: registry.npmmirror.com/pg-pool/3.6.0_pg@8.10.0
pg-protocol: registry.npmmirror.com/pg-protocol/1.6.0
pg-types: registry.npmmirror.com/pg-types/2.2.0
pgpass: registry.npmmirror.com/pgpass/1.0.5
dev: false
registry.npmmirror.com/pgpass/1.0.5:
resolution: {integrity: sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/pgpass/-/pgpass-1.0.5.tgz}
name: pgpass
version: 1.0.5
dependencies:
split2: registry.npmmirror.com/split2/4.2.0
dev: false
registry.npmmirror.com/picocolors/1.0.0:
resolution: {integrity: sha512-1fygroTLlHu66zi26VoTDv8yRgm0Fccecssto+MhsZ0D/DGW2sm8E8AjW7NU5VVTRt5GxbeZ5qBuJr+HyLYkjQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/picocolors/-/picocolors-1.0.0.tgz}
name: picocolors
......@@ -9859,6 +9946,32 @@ packages:
source-map-js: registry.npmmirror.com/source-map-js/1.0.2
dev: false
registry.npmmirror.com/postgres-array/2.0.0:
resolution: {integrity: sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/postgres-array/-/postgres-array-2.0.0.tgz}
name: postgres-array
version: 2.0.0
engines: {node: '>=4'}
registry.npmmirror.com/postgres-bytea/1.0.0:
resolution: {integrity: sha512-xy3pmLuQqRBZBXDULy7KbaitYqLcmxigw14Q5sj8QBVLqEwXfeybIKVWiqAXTlcvdvb0+xkOtDbfQMOf4lST1w==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/postgres-bytea/-/postgres-bytea-1.0.0.tgz}
name: postgres-bytea
version: 1.0.0
engines: {node: '>=0.10.0'}
registry.npmmirror.com/postgres-date/1.0.7:
resolution: {integrity: sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/postgres-date/-/postgres-date-1.0.7.tgz}
name: postgres-date
version: 1.0.7
engines: {node: '>=0.10.0'}
registry.npmmirror.com/postgres-interval/1.2.0:
resolution: {integrity: sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/postgres-interval/-/postgres-interval-1.2.0.tgz}
name: postgres-interval
version: 1.2.0
engines: {node: '>=0.10.0'}
dependencies:
xtend: registry.npmmirror.com/xtend/4.0.2
registry.npmmirror.com/prebuild-install/7.1.1:
resolution: {integrity: sha512-jAXscXWMcCK8GgCoHOfIr0ODh5ai8mj63L2nWrjuAgXE6tDyYGnx4/8o/rCgU+B4JSyZBKbeZqzhtwtC3ovxjw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/prebuild-install/-/prebuild-install-7.1.1.tgz}
name: prebuild-install
......@@ -10831,6 +10944,13 @@ packages:
dev: false
optional: true
registry.npmmirror.com/split2/4.2.0:
resolution: {integrity: sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/split2/-/split2-4.2.0.tgz}
name: split2
version: 4.2.0
engines: {node: '>= 10.x'}
dev: false
registry.npmmirror.com/sprintf-js/1.0.3:
resolution: {integrity: sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/sprintf-js/-/sprintf-js-1.0.3.tgz}
name: sprintf-js
......@@ -11815,7 +11935,6 @@ packages:
name: xtend
version: 4.0.2
engines: {node: '>=0.4'}
dev: false
registry.npmmirror.com/yallist/3.1.1:
resolution: {integrity: sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/yallist/-/yallist-3.1.1.tgz}
......
src/api/model.ts
......@@ -79,7 +79,7 @@ export const getWebContent = (url: string) => POST<string>(`/model/data/fetching
*/
export const postModelDataInput = (data: {
modelId: string;
data: { text: ModelDataSchema['text']; q: ModelDataSchema['q'] }[];
data: { a: ModelDataSchema['a']; q: ModelDataSchema['q'] }[];
}) => POST<number>(`/model/data/pushModelDataInput`, data);
/**
......@@ -97,7 +97,7 @@ export const postModelDataCsvData = (modelId: string, data: string[][]) =>
/**
* 更新模型数据
*/
export const putModelDataById = (data: { dataId: string; text: string; q?: string }) =>
export const putModelDataById = (data: { dataId: string; a: string; q?: string }) =>
PUT('/model/data/putModelData', data);
/**
* 删除一条模型数据
......
src/constants/model.ts
import type { ServiceName, ModelDataType, ModelSchema } from '@/types/mongoSchema';
import type { RedisModelDataItemType } from '@/types/redis';
export enum ModelDataStatusEnum {
ready = 'ready',
waiting = 'waiting'
}
export enum ChatModelNameEnum {
GPT35 = 'gpt-3.5-turbo',
......@@ -80,7 +84,7 @@ export const formatModelStatus = {
}
};
export const ModelDataStatusMap: Record<RedisModelDataItemType['status'], string> = {
export const ModelDataStatusMap: Record<`${ModelDataStatusEnum}`, string> = {
ready: '训练完成',
waiting: '训练中'
};
......
src/pages/api/model/data/delModelDataById.ts
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { authToken } from '@/service/utils/tools';
import { connectRedis } from '@/service/redis';
import { connectPg } from '@/service/pg';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
......@@ -21,15 +21,9 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
// 凭证校验
const userId = await authToken(authorization);
const redis = await connectRedis();
const pg = await connectPg();
await pg.query(`DELETE FROM modelData WHERE user_id = '${userId}' AND id = '${dataId}'`);
// 校验是否为该用户的数据
const dataItemUserId = await redis.hGet(dataId, 'userId');
if (dataItemUserId !== userId) {
throw new Error('无权操作');
}
// 删除
await redis.del(dataId);
jsonRes(res);
} catch (err) {
console.log(err);
......
src/pages/api/model/data/getModelData.ts
......@@ -2,8 +2,8 @@ import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase } from '@/service/mongo';
import { authToken } from '@/service/utils/tools';
import { connectRedis } from '@/service/redis';
import { VecModelDataIdx } from '@/constants/redis';
import { connectPg } from '@/service/pg';
import type { PgModelDataItemType } from '@/types/pg';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
......@@ -35,34 +35,20 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
const userId = await authToken(authorization);
await connectToDatabase();
const redis = await connectRedis();
const pg = await connectPg();
// 从 redis 中获取数据
const searchRes = await redis.ft.search(
VecModelDataIdx,
`@modelId:{${modelId}} @userId:{${userId}} ${searchText ? `*${searchText}*` : ''}`,
{
RETURN: ['q', 'text', 'status'],
LIMIT: {
from: (pageNum - 1) * pageSize,
size: pageSize
},
SORTBY: {
BY: 'modelId',
DIRECTION: 'DESC'
}
}
);
const searchRes = await pg.query<PgModelDataItemType>(`SELECT id, q, a, status
FROM modelData
WHERE user_id='${userId}' AND model_id='${modelId}'
LIMIT ${pageSize} OFFSET ${pageSize * (pageNum - 1)}
`);
jsonRes(res, {
data: {
pageNum,
pageSize,
data: searchRes.documents.map((item) => ({
id: item.id,
...item.value
})),
total: searchRes.total
data: searchRes.rows,
total: searchRes.rowCount
}
});
} catch (err) {
......
src/pages/api/model/data/pushModelDataInput.ts
......@@ -4,14 +4,13 @@ import { connectToDatabase, Model } from '@/service/mongo';
import { authToken } from '@/service/utils/tools';
import { ModelDataSchema } from '@/types/mongoSchema';
import { generateVector } from '@/service/events/generateVector';
import { connectRedis } from '@/service/redis';
import { VecModelDataPrefix, ModelDataStatusEnum } from '@/constants/redis';
import { connectPg } from '@/service/pg';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
const { modelId, data } = req.body as {
modelId: string;
data: { text: ModelDataSchema['text']; q: ModelDataSchema['q'] }[];
data: { a: ModelDataSchema['a']; q: ModelDataSchema['q'] }[];
};
const { authorization } = req.headers;
......@@ -27,7 +26,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
const userId = await authToken(authorization);
await connectToDatabase();
const redis = await connectRedis();
const pg = await connectPg();
// 验证是否是该用户的 model
const model = await Model.findOne({
......@@ -39,29 +38,23 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
throw new Error('无权操作该模型');
}
const insertRes = await Promise.allSettled(
data.map((item) => {
return redis.sendCommand([
'HMSET',
`${VecModelDataPrefix}:${item.q.id}`,
'userId',
userId,
'modelId',
modelId,
'q',
item.q.text,
'text',
item.text,
'status',
ModelDataStatusEnum.waiting
]);
})
// 插入记录
await pg.query(
`INSERT INTO modelData (user_id, model_id, q, a, status) VALUES ${data
.map(
(item) =>
`('${userId}', '${modelId}', '${item.q.replace(/\'/g, '"')}', '${item.a.replace(
/\'/g,
'"'
)}', 'waiting')`
)
.join(',')}`
);
generateVector();
jsonRes(res, {
data: insertRes.filter((item) => item.status === 'rejected').length
data: 0
});
} catch (err) {
jsonRes(res, {
......
src/pages/model/detail/components/InputDataModal.tsx
......@@ -16,14 +16,14 @@ import { useToast } from '@/hooks/useToast';
import { customAlphabet } from 'nanoid';
const nanoid = customAlphabet('abcdefghijklmnopqrstuvwxyz1234567890', 12);
export type FormData = { dataId?: string; text: string; q: string };
export type FormData = { dataId?: string; a: string; q: string };
const InputDataModal = ({
onClose,
onSuccess,
modelId,
defaultValues = {
text: '',
a: '',
q: ''
}
}: {
......@@ -51,11 +51,8 @@ const InputDataModal = ({
modelId: modelId,
data: [
{
text: e.text,
q: {
id: nanoid(),
text: e.q
}
a: e.a,
q: e.q
}
]
});
......@@ -65,7 +62,7 @@ const InputDataModal = ({
status: res === 0 ? 'success' : 'warning'
});
reset({
text: '',
a: '',
q: ''
});
onSuccess();
......@@ -81,10 +78,10 @@ const InputDataModal = ({
async (e: FormData) => {
if (!e.dataId) return;
if (e.text !== defaultValues.text || e.q !== defaultValues.q) {
if (e.a !== defaultValues.a || e.q !== defaultValues.q) {
await putModelDataById({
dataId: e.dataId,
text: e.text,
a: e.a,
q: e.q === defaultValues.q ? '' : e.q
});
onSuccess();
......@@ -144,7 +141,7 @@ const InputDataModal = ({
maxLength={1000}
resize={'none'}
h={'calc(100% - 30px)'}
{...register(`text`, {
{...register(`a`, {
required: '知识点'
})}
/>
......
src/pages/model/detail/components/ModelDataCard.tsx
......@@ -19,7 +19,7 @@ import {
Input
} from '@chakra-ui/react';
import type { ModelSchema } from '@/types/mongoSchema';
import type { RedisModelDataItemType } from '@/types/redis';
import type { ModelDataItemType } from '@/types/model';
import { ModelDataStatusMap } from '@/constants/model';
import { usePagination } from '@/hooks/usePagination';
import {
......@@ -53,9 +53,9 @@ const ModelDataCard = ({ model }: { model: ModelSchema }) => {
total,
getData,
pageNum
} = usePagination<RedisModelDataItemType>({
} = usePagination<ModelDataItemType>({
api: getModelDataList,
pageSize: 8,
pageSize: 10,
params: {
modelId: model._id,
searchText
......@@ -149,7 +149,7 @@ const ModelDataCard = ({ model }: { model: ModelSchema }) => {
<MenuItem
onClick={() =>
setEditInputData({
text: '',
a: '',
q: ''
})
}
......@@ -216,7 +216,7 @@ const ModelDataCard = ({ model }: { model: ModelSchema }) => {
maxH={'250px'}
overflowY={'auto'}
>
{item.text}
{item.a}
</Box>
</Td>
<Td>{ModelDataStatusMap[item.status]}</Td>
......@@ -231,7 +231,7 @@ const ModelDataCard = ({ model }: { model: ModelSchema }) => {
setEditInputData({
dataId: item.id,
q: item.q,
text: item.text
a: item.a
})
}
/>
......
src/service/events/generateQA.ts
......@@ -6,12 +6,9 @@ import type { ChatCompletionRequestMessage } from 'openai';
import { ChatModelNameEnum } from '@/constants/model';
import { pushSplitDataBill } from '@/service/events/pushBill';
import { generateVector } from './generateVector';
import { connectRedis } from '../redis';
import { VecModelDataPrefix } from '@/constants/redis';
import { customAlphabet } from 'nanoid';
import { openaiError2 } from '../errorCode';
import { connectPg } from '@/service/pg';
import { ModelSplitDataSchema } from '@/types/mongoSchema';
const nanoid = customAlphabet('abcdefghijklmnopqrstuvwxyz1234567890', 12);
export async function generateQA(next = false): Promise<any> {
if (process.env.queueTask !== '1') {
......@@ -25,7 +22,7 @@ export async function generateQA(next = false): Promise<any> {
let dataId = null;
try {
const redis = await connectRedis();
const pg = await connectPg();
// 找出一个需要生成的 dataItem
const data = await SplitData.aggregate([
{ $match: { textList: { $exists: true, $ne: [] } } },
......@@ -139,23 +136,17 @@ export async function generateQA(next = false): Promise<any> {
SplitData.findByIdAndUpdate(dataItem._id, {
textList: dataItem.textList.slice(0, -5)
}), // 删掉后5个数据
...resultList.map((item) => {
// 插入 redis
return redis.sendCommand([
'HMSET',
`${VecModelDataPrefix}:${nanoid()}`,
'userId',
String(dataItem.userId),
'modelId',
String(dataItem.modelId),
'q',
item.q,
'text',
item.a,
'status',
'waiting'
]);
})
// 生成的内容插入 pg
pg.query(`INSERT INTO modelData (user_id, model_id, q, a, status) VALUES ${resultList
.map(
(item) =>
`('${String(dataItem.userId)}', '${String(dataItem.modelId)}', '${item.q.replace(
/\'/g,
'"'
)}', '${item.a.replace(/\'/g, '"')}', 'waiting')`
)
.join(',')}
`)
]);
console.log('生成QA成功，time:', `${(Date.now() - startTime) / 1000}s`);
......
src/service/events/generateVector.ts
import { connectRedis } from '../redis';
import { VecModelDataIdx } from '@/constants/redis';
import { vectorToBuffer } from '@/utils/tools';
import { ModelDataStatusEnum } from '@/constants/redis';
import { openaiCreateEmbedding, getOpenApiKey } from '../utils/openai';
import { openaiError2 } from '../errorCode';
import { connectPg } from '@/service/pg';
import type { PgModelDataItemType } from '@/types/pg';
export async function generateVector(next = false): Promise<any> {
if (process.env.queueTask !== '1') {
......@@ -15,32 +14,27 @@ export async function generateVector(next = false): Promise<any> {
global.generatingVector = true;
let dataId = null;
try {
const redis = await connectRedis();
const pg = await connectPg();
// 从找出一个 status = waiting 的数据
const searchRes = await redis.ft.search(
VecModelDataIdx,
`@status:{${ModelDataStatusEnum.waiting}}`,
{
RETURN: ['q', 'userId'],
LIMIT: {
from: 0,
size: 1
}
}
);
const searchRes = await pg.query<PgModelDataItemType>(`SELECT id, q, user_id
FROM modelData
WHERE status='waiting'
LIMIT 1
`);
if (searchRes.total === 0) {
if (searchRes.rowCount === 0) {
console.log('没有需要生成 【向量】 的数据');
global.generatingVector = false;
return;
}
const dataItem: { id: string; q: string; userId: string } = {
id: searchRes.documents[0].id,
q: String(searchRes.documents[0]?.value?.q || ''),
userId: String(searchRes.documents[0]?.value?.userId || '')
id: searchRes.rows[0].id,
q: searchRes.rows[0].q,
userId: searchRes.rows[0].user_id
};
dataId = dataItem.id;
......@@ -53,7 +47,7 @@ export async function generateVector(next = false): Promise<any> {
systemKey = res.systemKey;
} catch (error: any) {
if (error?.code === 501) {
await redis.del(dataItem.id);
await pg.query(`DELETE FROM modelData WHERE id = '${dataId}'`);
generateVector(true);
return;
}
......@@ -69,15 +63,10 @@ export async function generateVector(next = false): Promise<any> {
apiKey: userApiKey || systemKey
});
// 更新 redis 向量和状态数据
await redis.sendCommand([
'HMSET',
dataItem.id,
'vector',
vectorToBuffer(vector),
'status',
ModelDataStatusEnum.ready
]);
// 更新 pg 向量和状态数据
await pg.query(
`UPDATE modelData SET vector = '[${vector}]', status = 'ready' WHERE id = ${dataId}`
);
console.log(`生成向量成功: ${dataItem.id}`);
......
src/service/pg.ts 0 → 100644
import { Pool } from 'pg';
export const connectPg = async () => {
if (global.pgClient) {
return global.pgClient;
}
global.pgClient = new Pool({
host: process.env.PG_HOST,
port: process.env.PG_PORT ? +process.env.PG_PORT : 5432,
user: process.env.PG_USER,
password: process.env.PG_PASSWORD,
database: process.env.PG_DB_NAME,
max: 20,
idleTimeoutMillis: 30000,
connectionTimeoutMillis: 2000
});
global.pgClient.on('connect', () => {
console.log('pg connected');
});
global.pgClient.on('error', (err) => {
console.log(err);
global.pgClient = null;
});
try {
await global.pgClient.connect();
return global.pgClient;
} catch (error) {
global.pgClient = null;
return Promise.reject(error);
}
};
src/types/index.d.ts
import type { Mongoose } from 'mongoose';
import type { RedisClientType } from 'redis';
import type { Agent } from 'http';
import type { Pool } from 'pg';
declare global {
var mongodb: Mongoose | string | null;
var redisClient: RedisClientType | null;
var pgClient: Pool | null;
var generatingQA: boolean;
var generatingAbstract: boolean;
var generatingVector: boolean;
......
src/types/model.d.ts
......@@ -12,7 +12,7 @@ export interface ModelUpdateParams {
export interface ModelDataItemType {
id: string;
status: 0 | 1; // 1代表向量生成完毕
status: 'waiting' | 'ready';
q: string; // 提问词
a: string; // 原文
modelId: string;
......
src/types/mongoSchema.d.ts
......@@ -66,11 +66,8 @@ export interface ModelDataSchema {
_id: string;
modelId: string;
userId: string;
text: string;
q: {
id: string;
text: string;
};
a: string;
q: string;
status: ModelDataType;
}
......
src/types/redis.d.ts → src/types/pg.d.ts
import { ModelDataStatusEnum } from '@/constants/redis';
export interface RedisModelDataItemType {
import { ModelDataStatusEnum } from '@/constants/model';
export interface PgModelDataItemType {
id: string;
q: string;
text: string;
a: string;
status: `${ModelDataStatusEnum}`;
model_id: string;
user_id: string;
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment