Skip to content
Toggle navigation
P
Projects
G
Groups
S
Snippets
Help
赵月辉
/
fastgpt-migrated
This project
Loading...
Sign in
Toggle navigation
Go to a project
Project
Repository
Issues
0
Merge Requests
0
Pipelines
Wiki
Snippets
Members
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Unverified
Commit
3e4487ad
authored
Mar 26, 2023
by
archer
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
feat: 摘要拆分
parent
888642f1
Hide whitespace changes
Inline
Side-by-side
Showing
20 changed files
with
397 additions
and
83 deletions
+397
-83
src/api/data.ts
+3
-2
src/constants/data.ts
+6
-0
src/constants/user.ts
+5
-1
src/pages/api/data/postData.ts
+5
-3
src/pages/api/data/splitData.ts
+18
-5
src/pages/api/timer/initDataItemTime.ts
+3
-2
src/pages/data/components/CreateDataModal.tsx
+46
-14
src/pages/data/components/ImportDataModal.tsx
+9
-2
src/pages/data/detail.tsx
+8
-3
src/pages/data/list.tsx
+27
-10
src/pages/model/components/ModelEditForm.tsx
+1
-1
src/service/events/generateAbstract.ts
+177
-0
src/service/events/generateQA.ts
+38
-27
src/service/events/pushBill.ts
+11
-2
src/service/models/data.ts
+7
-1
src/service/models/dataItem.ts
+21
-6
src/service/mongo.ts
+3
-1
src/types/data.d.ts
+2
-0
src/types/index.d.ts
+1
-0
src/types/mongoSchema.d.ts
+6
-3
No files found.
src/api/data.ts
View file @
3e4487ad
...
...
@@ -3,12 +3,13 @@ import { RequestPaging } from '../types/index';
import
{
Obj2Query
}
from
'@/utils/tools'
;
import
type
{
DataListItem
}
from
'@/types/data'
;
import
type
{
PagingData
}
from
'../types/index'
;
import
{
DataItemSchema
}
from
'@/types/mongoSchema'
;
import
type
{
DataItemSchema
}
from
'@/types/mongoSchema'
;
import
type
{
CreateDataProps
}
from
'@/pages/data/components/CreateDataModal'
;
export
const
getDataList
=
(
data
:
RequestPaging
)
=>
GET
<
PagingData
<
DataListItem
>>
(
`/data/getDataList?
${
Obj2Query
(
data
)}
`
);
export
const
postData
=
(
name
:
string
)
=>
POST
<
string
>
(
`/data/postData?name=
${
name
}
`
);
export
const
postData
=
(
data
:
CreateDataProps
)
=>
POST
<
string
>
(
`/data/postData`
,
data
);
export
const
postSplitData
=
(
dataId
:
string
,
text
:
string
)
=>
POST
(
`/data/splitData`
,
{
dataId
,
text
});
...
...
src/constants/data.ts
0 → 100644
View file @
3e4487ad
import
type
{
DataType
}
from
'@/types/data'
;
export
const
DataTypeTextMap
:
Record
<
DataType
,
string
>
=
{
QA
:
'问答拆分'
,
abstract
:
'摘要总结'
};
src/constants/user.ts
View file @
3e4487ad
export
enum
BillTypeEnum
{
chat
=
'chat'
,
splitData
=
'splitData'
,
QA
=
'QA'
,
abstract
=
'abstract'
,
return
=
'return'
}
export
enum
PageTypeEnum
{
...
...
@@ -11,6 +13,8 @@ export enum PageTypeEnum {
export
const
BillTypeMap
:
Record
<
`
${
BillTypeEnum
}
`
,
string
>
=
{
[
BillTypeEnum
.
chat
]:
'对话'
,
[
BillTypeEnum
.
splitData
]:
'文本拆分'
,
[
BillTypeEnum
.
splitData
]:
'QA拆分'
,
[
BillTypeEnum
.
QA
]:
'QA拆分'
,
[
BillTypeEnum
.
abstract
]:
'摘要总结'
,
[
BillTypeEnum
.
return
]:
'退款'
};
src/pages/api/data/postData.ts
View file @
3e4487ad
...
...
@@ -2,11 +2,12 @@ import type { NextApiRequest, NextApiResponse } from 'next';
import
{
jsonRes
}
from
'@/service/response'
;
import
{
connectToDatabase
,
Data
}
from
'@/service/mongo'
;
import
{
authToken
}
from
'@/service/utils/tools'
;
import
type
{
DataType
}
from
'@/types/data'
;
export
default
async
function
handler
(
req
:
NextApiRequest
,
res
:
NextApiResponse
)
{
try
{
let
{
name
}
=
req
.
query
as
{
name
:
string
};
if
(
!
name
)
{
let
{
name
,
type
}
=
req
.
body
as
{
name
:
string
;
type
:
DataType
};
if
(
!
name
||
!
type
)
{
throw
new
Error
(
'参数错误'
);
}
await
connectToDatabase
();
...
...
@@ -18,7 +19,8 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
// 生成 data 集合
const
data
=
await
Data
.
create
({
userId
,
name
name
,
type
});
jsonRes
(
res
,
{
...
...
src/pages/api/data/splitData.ts
View file @
3e4487ad
import
type
{
NextApiRequest
,
NextApiResponse
}
from
'next'
;
import
{
jsonRes
}
from
'@/service/response'
;
import
{
connectToDatabase
,
Data
,
DataItem
}
from
'@/service/mongo'
;
import
{
connectToDatabase
,
Data
Item
,
Data
}
from
'@/service/mongo'
;
import
{
authToken
}
from
'@/service/utils/tools'
;
import
{
generateQA
}
from
'@/service/events/generateQA'
;
import
{
generateAbstract
}
from
'@/service/events/generateAbstract'
;
/* 拆分数据成QA */
export
default
async
function
handler
(
req
:
NextApiRequest
,
res
:
NextApiResponse
)
{
try
{
let
{
text
,
dataId
}
=
req
.
body
as
{
text
:
string
;
dataId
:
string
};
...
...
@@ -17,14 +19,20 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
const
userId
=
await
authToken
(
authorization
);
const
DataRecord
=
await
Data
.
findById
(
dataId
);
if
(
!
DataRecord
)
{
throw
new
Error
(
'找不到数据集'
);
}
const
dataItems
:
any
[]
=
[];
//
格式化文本长度
//
每 1000 字符一组
for
(
let
i
=
0
;
i
<=
text
.
length
/
1000
;
i
++
)
{
dataItems
.
push
({
temperature
:
0
,
userId
,
dataId
,
type
:
DataRecord
.
type
,
text
:
text
.
slice
(
i
*
1000
,
(
i
+
1
)
*
1000
),
status
:
1
});
...
...
@@ -33,10 +41,15 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
// 批量插入数据
await
DataItem
.
insertMany
(
dataItems
);
generateQA
();
try
{
generateQA
();
generateAbstract
();
}
catch
(
error
)
{
error
;
}
jsonRes
(
res
,
{
data
:
dataItems
.
length
data
:
''
});
}
catch
(
err
)
{
jsonRes
(
res
,
{
...
...
src/pages/api/timer/initDataItemTime.ts
View file @
3e4487ad
...
...
@@ -13,14 +13,15 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
// await DataItem.updateMany(
// {},
// {
// times: 2
// type: 'QA'
// // times: 2
// }
// );
await
Data
.
updateMany
(
{},
{
isDeleted
:
false
type
:
'QA'
}
);
...
...
src/pages/data/components/CreateDataModal.tsx
View file @
3e4487ad
...
...
@@ -8,10 +8,21 @@ import {
ModalBody
,
ModalCloseButton
,
Button
,
Input
Input
,
Select
,
FormControl
,
FormErrorMessage
}
from
'@chakra-ui/react'
;
import
{
postData
}
from
'@/api/data'
;
import
{
useMutation
}
from
'@tanstack/react-query'
;
import
{
useForm
,
SubmitHandler
}
from
'react-hook-form'
;
import
{
DataType
}
from
'@/types/data'
;
import
{
DataTypeTextMap
}
from
'@/constants/data'
;
export
interface
CreateDataProps
{
name
:
string
;
type
:
DataType
;
}
const
CreateDataModal
=
({
onClose
,
...
...
@@ -21,9 +32,20 @@ const CreateDataModal = ({
onSuccess
:
()
=>
void
;
})
=>
{
const
[
inputVal
,
setInputVal
]
=
useState
(
''
);
const
{
getValues
,
register
,
handleSubmit
,
formState
:
{
errors
}
}
=
useForm
<
CreateDataProps
>
({
defaultValues
:
{
name
:
''
,
type
:
'abstract'
}
});
const
{
isLoading
,
mutate
}
=
useMutation
({
mutationFn
:
(
name
:
string
)
=>
postData
(
nam
e
),
mutationFn
:
(
e
:
CreateDataProps
)
=>
postData
(
e
),
onSuccess
()
{
onSuccess
();
onClose
();
...
...
@@ -37,23 +59,33 @@ const CreateDataModal = ({
<
ModalHeader
>
创建数据集
</
ModalHeader
>
<
ModalCloseButton
/>
<
ModalBody
display=
{
'flex'
}
>
<
Input
value=
{
inputVal
}
onChange=
{
(
e
)
=>
setInputVal
(
e
.
target
.
value
)
}
placeholder=
{
'数据集名称'
}
></
Input
>
<
ModalBody
>
<
FormControl
mb=
{
8
}
isInvalid=
{
!!
errors
.
name
}
>
<
Input
placeholder=
"数据集名称"
{
...
register
('
name
',
{
required
:
'数据集名称不能为空'
})}
/>
<
FormErrorMessage
position=
{
'absolute'
}
fontSize=
"xs"
>
{
!!
errors
.
name
&&
errors
.
name
.
message
}
</
FormErrorMessage
>
</
FormControl
>
<
FormControl
>
<
Select
placeholder=
"数据集类型"
{
...
register
('
type
',
{})}
>
{
Object
.
entries
(
DataTypeTextMap
).
map
(([
key
,
value
])
=>
(
<
option
key=
{
key
}
value=
{
key
}
>
{
value
}
</
option
>
))
}
</
Select
>
</
FormControl
>
</
ModalBody
>
<
ModalFooter
>
<
Button
colorScheme=
{
'gray'
}
onClick=
{
onClose
}
>
取消
</
Button
>
<
Button
ml=
{
3
}
isDisabled=
{
inputVal
===
''
}
isLoading=
{
isLoading
}
onClick=
{
()
=>
mutate
(
inputVal
)
}
>
<
Button
ml=
{
3
}
isLoading=
{
isLoading
}
onClick=
{
handleSubmit
(
mutate
as
any
)
}
>
确认
</
Button
>
</
ModalFooter
>
...
...
src/pages/data/components/ImportDataModal.tsx
View file @
3e4487ad
...
...
@@ -22,6 +22,7 @@ import { useToast } from '@/hooks/useToast';
import
{
useLoading
}
from
'@/hooks/useLoading'
;
import
{
formatPrice
}
from
'@/utils/user'
;
import
{
modelList
,
ChatModelNameEnum
}
from
'@/constants/model'
;
import
{
encode
,
decode
}
from
'gpt-token-utils'
;
const
fileExtension
=
'.txt,.doc,.docx,.pdf,.md'
;
...
...
@@ -106,6 +107,7 @@ const ImportDataModal = ({
.
join
(
'\n'
)
.
replace
(
/
\n
+/g
,
'\n'
);
setFileText
(
fileTexts
);
console
.
log
(
encode
(
fileTexts
));
}
catch
(
error
:
any
)
{
console
.
log
(
error
);
toast
({
...
...
@@ -161,7 +163,9 @@ const ImportDataModal = ({
placeholder=
{
'请粘贴或输入需要处理的文本'
}
onChange=
{
(
e
)
=>
setTextInput
(
e
.
target
.
value
)
}
/>
<
Box
mt=
{
2
}
>
一共
{
textInput
.
length
}
个字
</
Box
>
<
Box
mt=
{
2
}
>
一共
{
textInput
.
length
}
个字,
{
encode
(
textInput
).
length
}
个tokens
</
Box
>
</>
)
}
{
activeTab
===
'doc'
&&
(
...
...
@@ -174,12 +178,15 @@ const ImportDataModal = ({
border=
{
'1px solid '
}
borderColor=
{
'blackAlpha.200'
}
borderRadius=
{
'md'
}
fontSize=
{
'sm'
}
>
<
Button
onClick=
{
onOpen
}
>
选择文件
</
Button
>
<
Box
mt=
{
2
}
>
支持
{
fileExtension
}
文件
</
Box
>
{
fileText
&&
(
<>
<
Box
mt=
{
2
}
>
一共
{
fileText
.
length
}
个字
</
Box
>
<
Box
mt=
{
2
}
>
一共
{
fileText
.
length
}
个字,
{
encode
(
fileText
).
length
}
个tokens
</
Box
>
<
Box
maxH=
{
'300px'
}
w=
{
'100%'
}
...
...
src/pages/data/detail.tsx
View file @
3e4487ad
...
...
@@ -22,7 +22,7 @@ const DataDetail = ({ dataName, dataId }: { dataName: string; dataId: string })
return
(
<
Card
py=
{
4
}
h=
{
'100%'
}
display=
{
'flex'
}
flexDirection=
{
'column'
}
>
<
Box
px=
{
6
}
fontSize=
{
'xl'
}
fontWeight=
{
'bold'
}
>
{
dataName
}
拆分
结果
{
dataName
}
结果
</
Box
>
<
ScrollData
flex=
{
'1 0 0'
}
...
...
@@ -38,8 +38,13 @@ const DataDetail = ({ dataName, dataId }: { dataName: string; dataId: string })
<
Box
key=
{
item
.
_id
}
>
{
item
.
result
.
map
((
result
,
i
)
=>
(
<
Box
key=
{
i
}
mb=
{
3
}
>
<
Box
fontWeight=
{
'bold'
}
>
Q:
{
result
.
q
}
</
Box
>
<
Box
>
A:
{
result
.
a
}
</
Box
>
{
item
.
type
===
'QA'
&&
(
<>
<
Box
fontWeight=
{
'bold'
}
>
Q:
{
result
.
q
}
</
Box
>
<
Box
>
A:
{
result
.
a
}
</
Box
>
</>
)
}
{
item
.
type
===
'abstract'
&&
<
Box
fontSize=
{
'sm'
}
>
{
result
.
abstract
}
</
Box
>
}
</
Box
>
))
}
</
Box
>
...
...
src/pages/data/list.tsx
View file @
3e4487ad
...
...
@@ -28,13 +28,14 @@ import { useRouter } from 'next/router';
import
{
useConfirm
}
from
'@/hooks/useConfirm'
;
import
{
useRequest
}
from
'@/hooks/useRequest'
;
import
{
DataItemSchema
}
from
'@/types/mongoSchema'
;
import
{
DataTypeTextMap
}
from
'@/constants/data'
;
import
{
customAlphabet
}
from
'nanoid'
;
const
nanoid
=
customAlphabet
(
'.,'
,
1
);
const
CreateDataModal
=
dynamic
(()
=>
import
(
'./components/CreateDataModal'
));
const
ImportDataModal
=
dynamic
(()
=>
import
(
'./components/ImportDataModal'
));
export
type
ExportDataType
=
'jsonl'
;
export
type
ExportDataType
=
'jsonl'
|
'txt'
;
const
DataList
=
()
=>
{
const
router
=
useRouter
();
...
...
@@ -84,21 +85,26 @@ const DataList = () => {
let
text
=
''
;
// 生成 jsonl
data
.
forEach
((
item
)
=>
{
const
result
=
JSON
.
stringify
({
prompt
:
`
${
item
.
q
.
toLocaleLowerCase
()}${
nanoid
()}
</s>`
,
completion
:
`
${
item
.
a
}
###`
});
text
+=
`
${
result
}
\n`
;
if
(
res
.
type
===
'jsonl'
&&
item
.
q
&&
item
.
a
)
{
const
result
=
JSON
.
stringify
({
prompt
:
`
${
item
.
q
.
toLocaleLowerCase
()}${
nanoid
()}
</s>`
,
completion
:
`
${
item
.
a
}
###`
});
text
+=
`
${
result
}
\n`
;
}
else
if
(
res
.
type
===
'txt'
&&
item
.
abstract
)
{
text
+=
`
${
item
.
abstract
}
\n`
;
}
});
// 去掉最后一个 \n
text
=
text
.
substring
(
0
,
text
.
length
-
1
);
// 导出为文件
const
blob
=
new
Blob
([
text
],
{
type
:
'application/json;charset=utf-8'
});
// 创建下载链接
const
downloadLink
=
document
.
createElement
(
'a'
);
downloadLink
.
href
=
window
.
URL
.
createObjectURL
(
blob
);
downloadLink
.
download
=
'file.jsonl'
;
downloadLink
.
download
=
`data.
${
res
.
type
}
`
;
// 添加链接到页面并触发下载
document
.
body
.
appendChild
(
downloadLink
);
...
...
@@ -138,6 +144,7 @@ const DataList = () => {
<
Thead
>
<
Tr
>
<
Th
>
集合名
</
Th
>
<
Th
>
类型
</
Th
>
<
Th
>
创建时间
</
Th
>
<
Th
>
训练中 / 总数据
</
Th
>
<
Th
></
Th
>
...
...
@@ -158,6 +165,7 @@ const DataList = () => {
}
}
/>
</
Td
>
<
Td
>
{
DataTypeTextMap
[
item
.
type
||
'QA'
]
}
</
Td
>
<
Td
>
{
dayjs
(
item
.
createTime
).
format
(
'YYYY/MM/DD HH:mm'
)
}
</
Td
>
<
Td
>
{
item
.
trainingData
}
/
{
item
.
totalData
}
...
...
@@ -187,9 +195,18 @@ const DataList = () => {
导出
</
MenuButton
>
<
MenuList
>
<
MenuItem
onClick=
{
()
=>
handleExportData
({
data
:
item
,
type
:
'jsonl'
})
}
>
jsonl
</
MenuItem
>
{
item
.
type
===
'QA'
&&
(
<
MenuItem
onClick=
{
()
=>
handleExportData
({
data
:
item
,
type
:
'jsonl'
})
}
>
jsonl
</
MenuItem
>
)
}
{
item
.
type
===
'abstract'
&&
(
<
MenuItem
onClick=
{
()
=>
handleExportData
({
data
:
item
,
type
:
'txt'
})
}
>
txt
</
MenuItem
>
)
}
</
MenuList
>
</
Menu
>
...
...
src/pages/model/components/ModelEditForm.tsx
View file @
3e4487ad
...
...
@@ -97,7 +97,7 @@ const ModelEditForm = ({ formHooks }: { formHooks: UseFormReturn<ModelSchema> })
<
Box
mb=
{
1
}
>
系统提示词
</
Box
>
<
Textarea
rows=
{
6
}
maxLength=
{
500
}
maxLength=
{
-
1
}
{
...
register
('
systemPrompt
')}
placeholder=
{
'模型默认的 prompt 词,通过调整该内容,可以生成一个限定范围的模型。\n\n注意,改功能会影响对话的整体朝向!'
...
...
src/service/events/generateAbstract.ts
0 → 100644
View file @
3e4487ad
import
{
DataItem
}
from
'@/service/mongo'
;
import
{
getOpenAIApi
}
from
'@/service/utils/chat'
;
import
{
httpsAgent
,
getOpenApiKey
}
from
'@/service/utils/tools'
;
import
type
{
ChatCompletionRequestMessage
}
from
'openai'
;
import
{
DataItemSchema
}
from
'@/types/mongoSchema'
;
import
{
ChatModelNameEnum
}
from
'@/constants/model'
;
import
{
pushSplitDataBill
}
from
'@/service/events/pushBill'
;
export
async
function
generateAbstract
(
next
=
false
):
Promise
<
any
>
{
if
(
global
.
generatingAbstract
&&
!
next
)
return
;
global
.
generatingAbstract
=
true
;
const
systemPrompt
:
ChatCompletionRequestMessage
=
{
role
:
'system'
,
content
:
`我会向你发送一段长文本,请从中总结出3~10个摘要,尽量详细,请按以下格式返回: "(1):"\n"(2):"\n"(3):"\n`
};
let
dataItem
:
DataItemSchema
|
null
=
null
;
try
{
// 找出一个需要生成的 dataItem
dataItem
=
await
DataItem
.
findOne
({
status
:
{
$ne
:
0
},
times
:
{
$gt
:
0
},
type
:
'abstract'
});
if
(
!
dataItem
)
{
console
.
log
(
'没有需要生成 【摘要】 的数据'
);
global
.
generatingAbstract
=
false
;
return
;
}
// 更新状态为生成中
await
DataItem
.
findByIdAndUpdate
(
dataItem
.
_id
,
{
status
:
2
});
// 获取 openapi Key
let
userApiKey
,
systemKey
;
try
{
const
key
=
await
getOpenApiKey
(
dataItem
.
userId
);
userApiKey
=
key
.
userApiKey
;
systemKey
=
key
.
systemKey
;
}
catch
(
error
)
{
// 余额不够了, 把用户所有记录改成闲置
await
DataItem
.
updateMany
({
userId
:
dataItem
.
userId
,
status
:
0
});
throw
new
Error
(
'获取 openai key 失败'
);
}
console
.
log
(
'正在生成一组摘要, ID:'
,
dataItem
.
_id
);
const
startTime
=
Date
.
now
();
// 获取 openai 请求实例
const
chatAPI
=
getOpenAIApi
(
userApiKey
||
systemKey
);
// 请求 chatgpt 获取摘要
const
abstractResponse
=
await
Promise
.
allSettled
(
[
0.5
,
1
].
map
((
temperature
)
=>
chatAPI
.
createChatCompletion
(
{
model
:
ChatModelNameEnum
.
GPT35
,
temperature
:
temperature
,
n
:
1
,
messages
:
[
systemPrompt
,
{
role
:
'user'
,
content
:
dataItem
?.
text
||
''
}
]
},
{
timeout
:
120000
,
httpsAgent
}
)
)
);
// 过滤出成功的响应
const
successAbstracts
=
abstractResponse
.
filter
((
item
)
=>
item
.
status
===
'fulfilled'
);
// 提取摘要内容
const
rawContents
:
string
[]
=
successAbstracts
.
map
(
(
item
:
any
)
=>
item
?.
value
?.
data
.
choices
[
0
].
message
?.
content
||
''
);
// 从 content 中提取摘要内容
const
splitContents
=
rawContents
.
map
((
content
)
=>
splitText
(
content
)).
flat
();
// 生成词向量
const
vectorResponse
=
await
Promise
.
allSettled
(
splitContents
.
map
((
item
)
=>
chatAPI
.
createEmbedding
({
model
:
'text-embedding-ada-002'
,
input
:
item
.
abstract
})
)
);
// 筛选成功的向量请求
const
vectorSuccessResponse
=
vectorResponse
.
map
((
item
:
any
,
i
)
=>
{
if
(
item
.
status
!==
'fulfilled'
)
return
''
;
return
{
abstract
:
splitContents
[
i
].
abstract
,
abstractVector
:
item
?.
value
?.
data
?.
data
?.[
0
]?.
embedding
};
})
.
filter
((
item
)
=>
item
);
// 插入数据库,并修改状态
await
DataItem
.
findByIdAndUpdate
(
dataItem
.
_id
,
{
status
:
0
,
$push
:
{
rawResponse
:
{
$each
:
rawContents
},
result
:
{
$each
:
vectorSuccessResponse
}
}
});
// 计费
!
userApiKey
&&
splitContents
.
length
>
0
&&
pushSplitDataBill
({
userId
:
dataItem
.
userId
,
type
:
'abstract'
,
text
:
systemPrompt
.
content
+
dataItem
.
text
+
rawContents
.
join
(
''
)
+
rawContents
.
join
(
''
).
substring
(
0
,
Math
.
floor
(
dataItem
.
text
.
length
/
10
))
// 向量价格是gpt35的1/10
});
console
.
log
(
'生成摘要成功,time:'
,
`
${(
Date
.
now
()
-
startTime
)
/
1000
}
s`
,
'摘要数量:'
,
splitContents
.
length
);
}
catch
(
error
:
any
)
{
console
.
log
(
'error: 生成摘要错误'
,
dataItem
?.
_id
);
console
.
log
(
'response:'
,
error
);
if
(
dataItem
?.
_id
)
{
await
DataItem
.
findByIdAndUpdate
(
dataItem
.
_id
,
{
status
:
dataItem
.
times
>
0
?
1
:
0
,
// 还有重试次数则可以继续进行
$inc
:
{
// 剩余尝试次数-1
times
:
-
1
}
});
}
}
generateAbstract
(
true
);
}
/**
* 检查文本是否按格式返回
*/
function
splitText
(
text
:
string
)
{
const
regex
=
/
\(\d
+
\)
:
(\s
*
)(
.*
)(\s
*
)
/g
;
const
matches
=
text
.
matchAll
(
regex
);
// 获取所有匹配到的结果
const
result
=
[];
// 存储最终的结果
for
(
const
match
of
matches
)
{
if
(
match
[
2
])
{
result
.
push
({
abstract
:
match
[
2
]
as
string
});
}
}
return
result
;
}
src/service/events/generateQA.ts
View file @
3e4487ad
...
...
@@ -20,7 +20,8 @@ export async function generateQA(next = false): Promise<any> {
// 找出一个需要生成的 dataItem
dataItem
=
await
DataItem
.
findOne
({
status
:
{
$ne
:
0
},
times
:
{
$gt
:
0
}
times
:
{
$gt
:
0
},
type
:
'QA'
});
if
(
!
dataItem
)
{
...
...
@@ -49,62 +50,72 @@ export async function generateQA(next = false): Promise<any> {
throw
new
Error
(
'获取 openai key 失败'
);
}
console
.
log
(
'正在生成一
个QA, ID:'
,
dataItem
.
_id
,
'temperature: '
,
dataItem
.
temperature
/
100
);
console
.
log
(
'正在生成一
组QA, ID:'
,
dataItem
.
_id
);
const
startTime
=
Date
.
now
();
// 获取 openai 请求实例
const
chatAPI
=
getOpenAIApi
(
userApiKey
||
systemKey
);
// 请求 chatgpt 获取回答
const
response
=
await
chatAPI
.
createChatCompletion
(
{
model
:
ChatModelNameEnum
.
GPT35
,
temperature
:
dataItem
.
temperature
/
100
,
n
:
1
,
messages
:
[
systemPrompt
,
const
response
=
await
Promise
.
allSettled
(
[
0
,
0.5
,
0.8
].
map
((
temperature
)
=>
chatAPI
.
createChatCompletion
(
{
role
:
'user'
,
content
:
dataItem
.
text
model
:
ChatModelNameEnum
.
GPT35
,
temperature
:
temperature
,
n
:
1
,
messages
:
[
systemPrompt
,
{
role
:
'user'
,
content
:
dataItem
?.
text
||
''
}
]
},
{
timeout
:
120000
,
httpsAgent
}
]
},
{
timeout
:
120000
,
httpsAgent
}
)
)
);
// 过滤出成功的响应
const
successResponse
=
response
.
filter
((
item
)
=>
item
.
status
===
'fulfilled'
);
// 提取响应内容
const
rawContents
:
string
[]
=
successResponse
.
map
(
(
item
:
any
)
=>
item
?.
value
?.
data
.
choices
[
0
].
message
?.
content
||
''
);
const
content
=
response
.
data
.
choices
[
0
].
message
?.
content
;
// 从 content 中提取 QA
const
splitResponse
=
splitText
(
content
||
''
);
const
splitResponse
s
=
rawContents
.
map
((
content
)
=>
splitText
(
content
)).
flat
(
);
// 插入数据库,并修改状态
await
DataItem
.
findByIdAndUpdate
(
dataItem
.
_id
,
{
status
:
dataItem
.
temperature
>=
90
?
0
:
1
,
// 需要生成 4 组内容。0,0.3,0.6,0.9
temperature
:
dataItem
.
temperature
>=
90
?
dataItem
.
temperature
:
dataItem
.
temperature
+
30
,
status
:
0
,
$push
:
{
rawResponse
:
content
,
rawResponse
:
{
$each
:
rawContents
},
result
:
{
$each
:
splitResponse
$each
:
splitResponse
s
}
}
});
// 计费
!
userApiKey
&&
splitResponse
.
length
>
0
&&
splitResponse
s
.
length
>
0
&&
pushSplitDataBill
({
userId
:
dataItem
.
userId
,
text
:
systemPrompt
.
content
+
dataItem
.
text
+
content
type
:
'QA'
,
text
:
systemPrompt
.
content
+
dataItem
.
text
+
rawContents
.
join
(
''
)
});
console
.
log
(
'生成QA成功,time:'
,
`
${(
Date
.
now
()
-
startTime
)
/
1000
}
s`
,
'QA数量:'
,
splitResponse
.
length
splitResponse
s
.
length
);
}
catch
(
error
:
any
)
{
console
.
log
(
'error: 生成QA错误'
,
dataItem
?.
_id
);
console
.
log
(
'response:'
,
error
?.
response
);
// 重置状态
if
(
dataItem
?.
_id
)
{
await
DataItem
.
findByIdAndUpdate
(
dataItem
.
_id
,
{
status
:
dataItem
.
times
>
0
?
1
:
0
,
// 还有重试次数则可以继续进行
...
...
src/service/events/pushBill.ts
View file @
3e4487ad
...
...
@@ -2,6 +2,7 @@ import { connectToDatabase, Bill, User } from '../mongo';
import
{
modelList
,
ChatModelNameEnum
}
from
'@/constants/model'
;
import
{
encode
}
from
'gpt-token-utils'
;
import
{
formatPrice
}
from
'@/utils/user'
;
import
type
{
DataType
}
from
'@/types/data'
;
export
const
pushChatBill
=
async
({
modelName
,
...
...
@@ -59,7 +60,15 @@ export const pushChatBill = async ({
}
};
export
const
pushSplitDataBill
=
async
({
userId
,
text
}:
{
userId
:
string
;
text
:
string
})
=>
{
export
const
pushSplitDataBill
=
async
({
userId
,
text
,
type
}:
{
userId
:
string
;
text
:
string
;
type
:
DataType
;
})
=>
{
await
connectToDatabase
();
let
billId
;
...
...
@@ -83,7 +92,7 @@ export const pushSplitDataBill = async ({ userId, text }: { userId: string; text
// 插入 Bill 记录
const
res
=
await
Bill
.
create
({
userId
,
type
:
'splitData'
,
type
,
modelName
:
ChatModelNameEnum
.
GPT35
,
textLen
:
text
.
length
,
tokenLen
:
tokens
.
length
,
...
...
src/service/models/data.ts
View file @
3e4487ad
import
{
Schema
,
model
,
models
,
Model
}
from
'mongoose'
;
import
{
DataItemSchema
as
Datatype
}
from
'@/types/mongoSchema'
;
import
{
DataSchema
as
Datatype
}
from
'@/types/mongoSchema'
;
import
{
DataTypeTextMap
}
from
'@/constants/data'
;
const
DataSchema
=
new
Schema
({
userId
:
{
...
...
@@ -15,6 +16,11 @@ const DataSchema = new Schema({
type
:
Date
,
default
:
()
=>
new
Date
()
},
type
:
{
type
:
String
,
required
:
true
,
enum
:
Object
.
keys
(
DataTypeTextMap
)
},
isDeleted
:
{
type
:
Boolean
,
default
:
false
...
...
src/service/models/dataItem.ts
View file @
3e4487ad
import
type
{
DataItemSchema
as
DataItemType
}
from
'@/types/mongoSchema'
;
import
{
Schema
,
model
,
models
,
Model
}
from
'mongoose'
;
import
{
DataTypeTextMap
}
from
'@/constants/data'
;
const
DataItemSchema
=
new
Schema
({
userId
:
{
...
...
@@ -12,19 +13,23 @@ const DataItemSchema = new Schema({
ref
:
'data'
,
required
:
true
},
type
:
{
type
:
String
,
required
:
true
,
enum
:
Object
.
keys
(
DataTypeTextMap
)
},
times
:
{
// 剩余重试次数
type
:
Number
,
default
:
3
},
text
:
{
// 文本内容
type
:
String
,
required
:
true
},
temperature
:
{
type
:
Number
,
required
:
true
},
rawResponse
:
{
// 原始拆分结果
type
:
[
String
],
default
:
[]
},
...
...
@@ -33,11 +38,21 @@ const DataItemSchema = new Schema({
{
q
:
{
type
:
String
,
required
:
true
default
:
''
},
a
:
{
type
:
String
,
required
:
true
default
:
''
},
abstract
:
{
// 摘要
type
:
String
,
default
:
''
},
abstractVector
:
{
// 摘要对应的向量
type
:
[
Number
],
default
:
[]
}
}
],
...
...
src/service/mongo.ts
View file @
3e4487ad
import
mongoose
from
'mongoose'
;
import
{
generateQA
}
from
'./events/generateQA'
;
import
{
generateAbstract
}
from
'./events/generateAbstract'
;
/**
* 连接 MongoDB 数据库
*/
...
...
@@ -24,8 +26,8 @@ export async function connectToDatabase(): Promise<void> {
global
.
mongodb
=
null
;
}
// 递归 QA 生成
generateQA
();
generateAbstract
();
}
export
*
from
'./models/authCode'
;
...
...
src/types/data.d.ts
View file @
3e4487ad
import
type
{
DataSchema
}
from
'./mongoSchema'
;
export
type
DataType
=
'QA'
|
'abstract'
;
export
interface
DataListItem
extends
DataSchema
{
trainingData
:
number
;
totalData
:
number
;
...
...
src/types/index.d.ts
View file @
3e4487ad
...
...
@@ -3,6 +3,7 @@ import type { Mongoose } from 'mongoose';
declare
global
{
var
mongodb
:
Mongoose
|
string
|
null
;
var
generatingQA
:
boolean
;
var
generatingAbstract
:
boolean
;
var
QRCode
:
any
;
interface
Window
{
[
'pdfjs-dist/build/pdf'
]:
any
;
...
...
src/types/mongoSchema.d.ts
View file @
3e4487ad
import
type
{
ChatItemType
}
from
'./chat'
;
import
{
ModelStatusEnum
,
TrainingStatusEnum
,
ChatModelNameEnum
}
from
'@/constants/model'
;
import
type
{
DataType
}
from
'./data'
;
export
type
ServiceName
=
'openai'
;
...
...
@@ -102,19 +103,21 @@ export interface DataSchema {
userId
:
string
;
name
:
string
;
createTime
:
string
;
type
:
DataType
;
}
export
interface
DataItemSchema
{
_id
:
string
;
userId
:
string
;
dataId
:
string
;
type
:
DataType
;
times
:
number
;
temperature
:
number
;
text
:
string
;
rawResponse
:
string
[];
result
:
{
q
:
string
;
a
:
string
;
q
?:
string
;
a
?:
string
;
abstract
?:
string
;
}[];
status
:
0
|
1
|
2
;
}
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment