perf: search prompt, upload step and psw len

63c832d8 · archer · 9ea19b8e · 63c832d8 · 63c832d8 · 63c832d8
Commit 63c832d8 authored Aug 10, 2023 by archer
14 changed files
--- a/client/public/locales/en/common.json
+++ b/client/public/locales/en/common.json
@@ -22,6 +22,9 @@
    "New Chat": "New Chat",
    "You need to a chat app": "You need to a chat app"
  },
+  "commom": {
+    "Password inconsistency": "Password inconsistency"
+  },
  "common": {
    "Add": "Add",
    "Cancel": "Cancel",

--- a/client/public/locales/zh/common.json
+++ b/client/public/locales/zh/common.json
@@ -22,11 +22,15 @@
    "New Chat": "新对话",
    "You need to a chat app": "你需要创建一个应用"
  },
+  "commom": {
+    "Password inconsistency": "两次密码不一致"
+  },
  "common": {
    "Add": "添加",
    "Cancel": "取消",
    "Collect": "收藏",
    "Copy": "复制",
+    "Course": "",
    "Delete": "删除",
    "Filed is repeat": "",
    "Filed is repeated": "字段重复了",

--- a/client/src/components/ChatBox/QuoteModal.tsx
+++ b/client/src/components/ChatBox/QuoteModal.tsx
 import React, { useCallback, useState } from 'react';
-import { ModalBody, ModalCloseButton, ModalHeader, Box, useTheme } from '@chakra-ui/react';
+import { ModalBody, Box, useTheme } from '@chakra-ui/react';
 import { getKbDataItemById } from '@/api/plugins/kb';
 import { useLoading } from '@/hooks/useLoading';
 import { useToast } from '@/hooks/useToast';
@@ -9,13 +9,21 @@ import MyIcon from '@/components/Icon';
 import InputDataModal from '@/pages/kb/detail/components/InputDataModal';
 import MyModal from '../MyModal';

+type SearchType = {
+  kb_id?: string;
+  id?: string;
+  q: string;
+  a?: string;
+  source?: string | undefined;
+};
+
 const QuoteModal = ({
  onUpdateQuote,
  rawSearch = [],
  onClose
 }: {
  onUpdateQuote: (quoteId: string, sourceText: string) => Promise<void>;
-  rawSearch: QuoteItemType[];
+  rawSearch: SearchType[];
  onClose: () => void;
 }) => {
  const theme = useTheme();
@@ -32,7 +40,8 @@ const QuoteModal = ({
   * click edit, get new kbDataItem
   */
  const onclickEdit = useCallback(
-    async (item: QuoteItemType) => {
+    async (item: SearchType) => {
+      if (!item.id) return;
      try {
        setIsLoading(true);
        const data = (await getKbDataItemById(item.id)) as QuoteItemType;
@@ -77,9 +86,9 @@ const QuoteModal = ({
        }
      >
        <ModalBody pt={0} whiteSpace={'pre-wrap'} textAlign={'justify'} fontSize={'sm'}>
-          {rawSearch.map((item) => (
+          {rawSearch.map((item, i) => (
            <Box
-              key={item.id}
+              key={i}
              flex={'1 0 0'}
              p={2}
              borderRadius={'lg'}
@@ -91,31 +100,33 @@ const QuoteModal = ({
              {item.source && <Box color={'myGray.600'}>({item.source})</Box>}
              <Box>{item.q}</Box>
              <Box>{item.a}</Box>
-              <Box
-                className="edit"
-                display={'none'}
-                position={'absolute'}
-                right={0}
-                top={0}
-                bottom={0}
-                w={'40px'}
-                bg={'rgba(255,255,255,0.9)'}
-                alignItems={'center'}
-                justifyContent={'center'}
-                boxShadow={'-10px 0 10px rgba(255,255,255,1)'}
-              >
-                <MyIcon
-                  name={'edit'}
-                  w={'18px'}
-                  h={'18px'}
-                  cursor={'pointer'}
-                  color={'myGray.600'}
-                  _hover={{
-                    color: 'myBlue.700'
-                  }}
-                  onClick={() => onclickEdit(item)}
-                />
-              </Box>
+              {item.id && (
+                <Box
+                  className="edit"
+                  display={'none'}
+                  position={'absolute'}
+                  right={0}
+                  top={0}
+                  bottom={0}
+                  w={'40px'}
+                  bg={'rgba(255,255,255,0.9)'}
+                  alignItems={'center'}
+                  justifyContent={'center'}
+                  boxShadow={'-10px 0 10px rgba(255,255,255,1)'}
+                >
+                  <MyIcon
+                    name={'edit'}
+                    w={'18px'}
+                    h={'18px'}
+                    cursor={'pointer'}
+                    color={'myGray.600'}
+                    _hover={{
+                      color: 'myBlue.700'
+                    }}
+                    onClick={() => onclickEdit(item)}
+                  />
+                </Box>
+              )}
            </Box>
          ))}
        </ModalBody>

--- a/client/src/pages/account/components/UpdatePswModal.tsx
+++ b/client/src/pages/account/components/UpdatePswModal.tsx
@@ -6,9 +6,15 @@ import { useForm } from 'react-hook-form';
 import { useRequest } from '@/hooks/useRequest';
 import { updatePasswordByOld } from '@/api/user';

+type FormType = {
+  oldPsw: string;
+  newPsw: string;
+  confirmPsw: string;
+};
+
 const UpdatePswModal = ({ onClose }: { onClose: () => void }) => {
  const { t } = useTranslation();
-  const { register, handleSubmit } = useForm({
+  const { register, handleSubmit } = useForm<FormType>({
    defaultValues: {
      oldPsw: '',
      newPsw: '',
@@ -17,7 +23,10 @@ const UpdatePswModal = ({ onClose }: { onClose: () => void }) => {
  });

  const { mutate: onSubmit, isLoading } = useRequest({
-    mutationFn: (data) => {
+    mutationFn: (data: FormType) => {
+      if (data.newPsw !== data.confirmPsw) {
+        return Promise.reject(t('commom.Password inconsistency'));
+      }
      return updatePasswordByOld(data);
    },
    onSuccess() {
@@ -36,11 +45,31 @@ const UpdatePswModal = ({ onClose }: { onClose: () => void }) => {
        </Flex>
        <Flex alignItems={'center'} mt={5}>
          <Box flex={'0 0 70px'}>新密码:</Box>
-          <Input flex={1} type={'password'} {...register('newPsw', { required: true })}></Input>
+          <Input
+            flex={1}
+            type={'password'}
+            {...register('newPsw', {
+              required: true,
+              maxLength: {
+                value: 20,
+                message: '密码最少 4 位最多 20 位'
+              }
+            })}
+          ></Input>
        </Flex>
        <Flex alignItems={'center'} mt={5}>
          <Box flex={'0 0 70px'}>确认密码:</Box>
-          <Input flex={1} type={'password'} {...register('confirmPsw', { required: true })}></Input>
+          <Input
+            flex={1}
+            type={'password'}
+            {...register('confirmPsw', {
+              required: true,
+              maxLength: {
+                value: 20,
+                message: '密码最少 4 位最多 20 位'
+              }
+            })}
+          ></Input>
        </Flex>
      </ModalBody>
      <ModalFooter>

--- a/client/src/pages/api/openapi/kb/pushData.ts
+++ b/client/src/pages/api/openapi/kb/pushData.ts
@@ -90,15 +90,14 @@ export async function pushDataToKb({
  data.forEach((item) => {
    const text = item.q + item.a;

-    if (mode === TrainingModeEnum.qa) {
-      // count token
-      const token = modelToolMap.countTokens({
-        model: 'gpt-3.5-turbo-16k',
-        messages: [{ obj: 'System', value: item.q }]
-      });
-      if (token > modeMaxToken[TrainingModeEnum.qa]) {
-        return;
-      }
+    // count token
+    const token = modelToolMap.countTokens({
+      model: 'gpt-3.5-turbo',
+      messages: [{ obj: 'System', value: item.q }]
+    });
+
+    if (token > modeMaxToken[TrainingModeEnum.qa]) {
+      return;
    }

    if (!set.has(text)) {

--- a/client/src/pages/kb/detail/components/Import/Chunk.tsx
+++ b/client/src/pages/kb/detail/components/Import/Chunk.tsx
@@ -142,7 +142,7 @@ const ChunkImport = ({ kbId }: { kbId: string }) => {

      // subsection import
      let success = 0;
-      const step = 100;
+      const step = 500;
      for (let i = 0; i < chunks.length; i += step) {
        const { insertLen } = await postKbDataFromList({
          kbId,

--- a/client/src/pages/kb/detail/components/Import/Csv.tsx
+++ b/client/src/pages/kb/detail/components/Import/Csv.tsx
@@ -13,7 +13,7 @@ import { TrainingModeEnum } from '@/constants/plugin';
 import FileSelect from './FileSelect';
 import { useRouter } from 'next/router';
 const nanoid = customAlphabet('abcdefghijklmnopqrstuvwxyz1234567890', 12);
-import { fileDownload, readCsvContent } from '@/utils/file';
+import { readCsvContent } from '@/utils/file';

 const fileExtension = '.csv';

@@ -98,7 +98,7 @@ const CsvImport = ({ kbId }: { kbId: string }) => {

      // subsection import
      let success = 0;
-      const step = 100;
+      const step = 500;
      for (let i = 0; i < chunks.length; i += step) {
        const { insertLen } = await postKbDataFromList({
          kbId,

--- a/client/src/pages/kb/detail/components/Import/QA.tsx
+++ b/client/src/pages/kb/detail/components/Import/QA.tsx
@@ -132,7 +132,7 @@ const QAImport = ({ kbId }: { kbId: string }) => {

      // subsection import
      let success = 0;
-      const step = 100;
+      const step = 500;
      for (let i = 0; i < chunks.length; i += step) {
        const { insertLen } = await postKbDataFromList({
          kbId,

--- a/client/src/pages/login/components/ForgetPasswordForm.tsx
+++ b/client/src/pages/login/components/ForgetPasswordForm.tsx
@@ -129,11 +129,11 @@ const RegisterForm = ({ setPageType, loginSuccess }: Props) => {
              required: '密码不能为空',
              minLength: {
                value: 4,
-                message: '密码最少4位最多12位'
+                message: '密码最少 4 位最多 20 位'
              },
              maxLength: {
-                value: 12,
-                message: '密码最少4位最多12位'
+                value: 20,
+                message: '密码最少 4 位最多 20 位'
              }
            })}
          ></Input>

--- a/client/src/pages/login/components/LoginForm.tsx
+++ b/client/src/pages/login/components/LoginForm.tsx
@@ -97,8 +97,8 @@ const LoginForm = ({ setPageType, loginSuccess }: Props) => {
            {...register('password', {
              required: '密码不能为空',
              maxLength: {
-                value: 12,
-                message: '密码最多12位'
+                value: 20,
+                message: '密码最多 20 位'
              }
            })}
          ></Input>

--- a/client/src/pages/login/components/RegisterForm.tsx
+++ b/client/src/pages/login/components/RegisterForm.tsx
@@ -142,11 +142,11 @@ const RegisterForm = ({ setPageType, loginSuccess }: Props) => {
              required: '密码不能为空',
              minLength: {
                value: 4,
-                message: '密码最少4位最多12位'
+                message: '密码最少 4 位最多 20 位'
              },
              maxLength: {
-                value: 12,
-                message: '密码最少4位最多12位'
+                value: 20,
+                message: '密码最少 4 位最多 20 位'
              }
            })}
          ></Input>

--- a/client/src/service/moduleDispatch/agent/classifyQuestion.ts
+++ b/client/src/service/moduleDispatch/agent/classifyQuestion.ts
@@ -30,6 +30,10 @@ const maxTokens = 3000;
 export const dispatchClassifyQuestion = async (props: Record<string, any>): Promise<CQResponse> => {
  const { agents, systemPrompt, history = [], userChatInput, userOpenaiAccount } = props as CQProps;

+  if (!userChatInput) {
+    return Promise.reject('Input is empty');
+  }
+
  const messages: ChatItemType[] = [
    ...(systemPrompt
      ? [

--- a/client/src/service/moduleDispatch/chat/oneapi.ts
+++ b/client/src/service/moduleDispatch/chat/oneapi.ts
@@ -192,7 +192,7 @@ function filterQuote({
    maxToken: model.quoteMaxToken,
    messages: quoteQA.map((item, i) => ({
      obj: ChatRoleEnum.System,
-      value: `${i + 1}. [${item.q}\n${item.a}]`
+      value: item.a ? `{instruction:${item.q},output:${item.a}}` : `{instruction:${item.q}}`
    }))
  });

@@ -202,7 +202,9 @@ function filterQuote({
  const quotePrompt =
    filterQuoteQA.length > 0
      ? `下面是知识库内容:
-${filterQuoteQA.map((item) => `{Q:${item.q},A:${item.a}}`).join('\n')}
+${filterQuoteQA
+  .map((item) => (item.a ? `{instruction:${item.q},output:${item.a}}` : `{instruction:${item.q}}`))
+  .join('\n')}
 `
      : '';


--- a/client/src/service/utils/chat/index.ts
+++ b/client/src/service/utils/chat/index.ts
@@ -21,13 +21,16 @@ export type StreamResponseType = {
 /* slice chat context by tokens */
 export const ChatContextFilter = ({
  model,
-  prompts,
+  prompts = [],
  maxTokens
 }: {
  model: string;
  prompts: ChatItemType[];
  maxTokens: number;
 }) => {
+  if (!Array.isArray(prompts)) {
+    return [];
+  }
  const rawTextLen = prompts.reduce((sum, item) => sum + item.value.length, 0);

  // If the text length is less than half of the maximum token, no calculation is required