chat logs filter & export (#3737)

* chat logs filter & export

* export chat detail

packages/web/i18n/en/app.json

@@ -81,7 +81,10 @@
   "llm_use_vision_tip": "After clicking on the model selection, you can see whether the model supports image recognition and the ability to control whether to start image recognition. \nAfter starting image recognition, the model will read the image content in the file link, and if the user question is less than 500 words, it will automatically parse the image in the user question.",
   "logs_chat_user": "user",
   "logs_empty": "No logs yet~",
+  "logs_export_confirm_tip": "There are a total of {{total}} dialogue records, confirm the export?",
+  "logs_export_title": "Time, source, user, title, total number of messages, user feedback, custom feedback, number of labeled answers, conversation details",
   "logs_message_total": "Total Messages",
+  "logs_source": "source",
   "logs_title": "Title",
   "look_ai_point_price": "View all model billing standards",
   "mark_count": "Number of Marked Answers",

packages/web/i18n/zh-CN/app.json

@@ -81,7 +81,10 @@
   "llm_use_vision_tip": "点击模型选择后，可以看到模型是否支持图片识别以及控制是否启动图片识别的能力。启动图片识别后，模型会读取文件链接里图片内容，并且如果用户问题少于 500 字，会自动解析用户问题中的图片。",
   "logs_chat_user": "使用者",
   "logs_empty": "还没有日志噢~",
+  "logs_export_confirm_tip": "当前共 {{total}} 条对话记录，确认导出？",
+  "logs_export_title": "时间,来源,使用者,标题,消息总数,用户反馈,自定义反馈,标注答案数量,对话详情",
   "logs_message_total": "消息总数",
+  "logs_source": "来源",
   "logs_title": "标题",
   "look_ai_point_price": "查看所有模型计费标准",
   "mark_count": "标注答案数量",

packages/web/i18n/zh-Hant/app.json

@@ -81,7 +81,10 @@
   "llm_use_vision_tip": "點選模型選擇後，可以看到模型是否支援圖片辨識以及控制是否啟用圖片辨識的功能。啟用圖片辨識後，模型會讀取檔案連結中的圖片內容，並且如果使用者問題少於 500 字，會自動解析使用者問題中的圖片。",
   "logs_chat_user": "使用者",
   "logs_empty": "還沒有紀錄喔～",
+  "logs_export_confirm_tip": "當前共 {{total}} 條對話記錄，確認導出？",
+  "logs_export_title": "時間,來源,使用者,標題,消息總數,用戶反饋,自定義反饋,標註答案數量,對話詳情",
   "logs_message_total": "訊息總數",
+  "logs_source": "来源",
   "logs_title": "標題",
   "look_ai_point_price": "查看所有模型計費標準",
   "mark_count": "標記答案數量",

projects/app/src/global/core/api/appReq.d.ts

+import { ChatSourceEnum } from '@fastgpt/global/core/chat/constants';
+import { UsageSourceEnum } from '@fastgpt/global/support/wallet/usage/constants';
 import { PaginationProps } from '@fastgpt/web/common/fetch/type';
 
-export type GetAppChatLogsParams = PaginationProps<{
+export type GetAppChatLogsProps = {
   appId: string;
   dateStart: Date;
   dateEnd: Date;
-}>;
+  sources?: ChatSourceEnum[];
+  logTitle?: string;
+};
+
+export type GetAppChatLogsParams = PaginationProps<GetAppChatLogsProps>;

projects/app/src/pages/api/core/app/exportChatLogs.ts

+import type { NextApiResponse } from 'next';
+import { responseWriteController } from '@fastgpt/service/common/response';
+import { addDays } from 'date-fns';
+import { WritePermissionVal } from '@fastgpt/global/support/permission/constant';
+import { readFromSecondary } from '@fastgpt/service/common/mongo/utils';
+import { addLog } from '@fastgpt/service/common/system/log';
+import dayjs from 'dayjs';
+import { ApiRequestProps } from '@fastgpt/service/type/next';
+import { replaceRegChars } from '@fastgpt/global/common/string/tools';
+import { NextAPI } from '@/service/middleware/entry';
+import { useIPFrequencyLimit } from '@fastgpt/service/common/middle/reqFrequencyLimit';
+import { GetAppChatLogsProps } from '@/global/core/api/appReq';
+import { authApp } from '@fastgpt/service/support/permission/app/auth';
+import { Types } from 'mongoose';
+import { MongoChat } from '@fastgpt/service/core/chat/chatSchema';
+import { ChatItemCollectionName } from '@fastgpt/service/core/chat/chatItemSchema';
+import { MongoTeamMember } from '@fastgpt/service/support/user/team/teamMemberSchema';
+import { ChatItemValueTypeEnum, ChatSourceEnum } from '@fastgpt/global/core/chat/constants';
+import { AIChatItemValueItemType } from '@fastgpt/global/core/chat/type';
+
+export type ExportChatLogsBody = GetAppChatLogsProps & {
+  title: string;
+  sourcesMap: Record<string, { label: string }>;
+};
+
+async function handler(req: ApiRequestProps<ExportChatLogsBody, {}>, res: NextApiResponse) {
+  let {
+    appId,
+    dateStart = addDays(new Date(), -7),
+    dateEnd = new Date(),
+    sources,
+    logTitle,
+
+    title,
+    sourcesMap
+  } = req.body;
+
+  if (!appId) {
+    throw new Error('缺少参数');
+  }
+
+  const { teamId } = await authApp({ req, authToken: true, appId, per: WritePermissionVal });
+  const teamMembers = await MongoTeamMember.find({ teamId });
+
+  const where = {
+    teamId: new Types.ObjectId(teamId),
+    appId: new Types.ObjectId(appId),
+    updateTime: {
+      $gte: new Date(dateStart),
+      $lte: new Date(dateEnd)
+    },
+    ...(sources && { source: { $in: sources } }),
+    ...(logTitle && {
+      $or: [
+        { title: { $regex: new RegExp(`${replaceRegChars(logTitle)}`, 'i') } },
+        { customTitle: { $regex: new RegExp(`${replaceRegChars(logTitle)}`, 'i') } }
+      ]
+    })
+  };
+
+  res.setHeader('Content-Type', 'text/csv; charset=utf-8;');
+  res.setHeader('Content-Disposition', 'attachment; filename=usage.csv; ');
+
+  const cursor = MongoChat.aggregate(
+    [
+      { $match: where },
+      {
+        $sort: {
+          userBadFeedbackCount: -1,
+          userGoodFeedbackCount: -1,
+          customFeedbacksCount: -1,
+          updateTime: -1
+        }
+      },
+      { $limit: 50000 },
+      {
+        $lookup: {
+          from: ChatItemCollectionName,
+          let: { chatId: '$chatId' },
+          pipeline: [
+            {
+              $match: {
+                $expr: {
+                  $and: [
+                    { $eq: ['$appId', new Types.ObjectId(appId)] },
+                    { $eq: ['$chatId', '$$chatId'] }
+                  ]
+                }
+              }
+            },
+            {
+              $project: {
+                value: 1,
+                userGoodFeedback: 1,
+                userBadFeedback: 1,
+                customFeedbacks: 1,
+                adminFeedback: 1
+              }
+            }
+          ],
+          as: 'chatitems'
+        }
+      },
+      {
+        $addFields: {
+          userGoodFeedbackCount: {
+            $size: {
+              $filter: {
+                input: '$chatitems',
+                as: 'item',
+                cond: { $ifNull: ['$$item.userGoodFeedback', false] }
+              }
+            }
+          },
+          userBadFeedbackCount: {
+            $size: {
+              $filter: {
+                input: '$chatitems',
+                as: 'item',
+                cond: { $ifNull: ['$$item.userBadFeedback', false] }
+              }
+            }
+          },
+          customFeedbacksCount: {
+            $size: {
+              $filter: {
+                input: '$chatitems',
+                as: 'item',
+                cond: { $gt: [{ $size: { $ifNull: ['$$item.customFeedbacks', []] } }, 0] }
+              }
+            }
+          },
+          markCount: {
+            $size: {
+              $filter: {
+                input: '$chatitems',
+                as: 'item',
+                cond: { $ifNull: ['$$item.adminFeedback', false] }
+              }
+            }
+          },
+          chatDetails: {
+            $map: {
+              input: '$chatitems',
+              as: 'item',
+              in: {
+                id: '$$item._id',
+                value: '$$item.value'
+              }
+            }
+          }
+        }
+      },
+      {
+        $project: {
+          _id: 1,
+          id: '$chatId',
+          title: 1,
+          customTitle: 1,
+          source: 1,
+          time: '$updateTime',
+          messageCount: { $size: '$chatitems' },
+          userGoodFeedbackCount: 1,
+          userBadFeedbackCount: 1,
+          customFeedbacksCount: 1,
+          markCount: 1,
+          outLinkUid: 1,
+          tmbId: 1,
+          chatDetails: 1
+        }
+      }
+    ],
+    {
+      ...readFromSecondary
+    }
+  ).cursor({ batchSize: 1000 });
+
+  const write = responseWriteController({
+    res,
+    readStream: cursor
+  });
+
+  write(`\uFEFF${title}`);
+
+  cursor.on('data', (doc) => {
+    const time = dayjs(doc.time.toISOString()).format('YYYY-MM-DD HH:mm:ss');
+    const source = sourcesMap[doc.source as ChatSourceEnum]?.label || doc.source;
+    const title = doc.customTitle || doc.title;
+    const tmb = doc.outLinkUid
+      ? doc.outLinkUid
+      : teamMembers.find((member) => String(member._id) === String(doc.tmbId))?.name;
+
+    const messageCount = doc.messageCount;
+    const userFeedbackCount = doc.userGoodFeedbackCount || doc.userBadFeedbackCount || '-';
+    const customFeedbacksCount = doc.customFeedbacksCount || '-';
+    const markCount = doc.markCount;
+    const chatDetails = doc.chatDetails.map(
+      (chat: { id: string; value: AIChatItemValueItemType[] }) => {
+        return chat.value.map((item) => {
+          if ([ChatItemValueTypeEnum.text, ChatItemValueTypeEnum.reasoning].includes(item.type)) {
+            return item;
+          }
+          if (item.type === ChatItemValueTypeEnum.tool) {
+            const newTools = item.tools?.map((tool) => {
+              const { functionName, toolAvatar, ...rest } = tool;
+              return {
+                ...rest
+              };
+            });
+
+            return {
+              ...item,
+              tools: newTools
+            };
+          }
+          if (item.type === ChatItemValueTypeEnum.interactive) {
+            const newInteractive = {
+              type: item.interactive?.type,
+              params: item.interactive?.params
+            };
+
+            return {
+              ...item,
+              interactive: newInteractive
+            };
+          }
+        });
+      }
+    );
+    let chatDetailsStr = '';
+    try {
+      chatDetailsStr = JSON.stringify(chatDetails);
+    } catch (e) {
+      addLog.error(`export chat logs error`, e);
+    }
+
+    const res = `\n"${time}","${source}","${tmb}","${title}","${messageCount}","${userFeedbackCount}","${customFeedbacksCount}","${markCount}","${chatDetailsStr}"`;
+
+    write(res);
+  });
+
+  cursor.on('end', () => {
+    cursor.close();
+    res.end();
+  });
+
+  cursor.on('error', (err) => {
+    addLog.error(`export chat logs error`, err);
+    res.status(500);
+    res.end();
+  });
+}
+
+export default NextAPI(
+  useIPFrequencyLimit({ id: 'export-chat-logs', seconds: 2, limit: 1, force: true }),
+  handler
+);

projects/app/src/pages/api/core/app/getChatLogs.ts

@@ -12,6 +12,7 @@ import { readFromSecondary } from '@fastgpt/service/common/mongo/utils';
 import { parsePaginationRequest } from '@fastgpt/service/common/api/pagination';
 import { PaginationResponse } from '@fastgpt/web/common/fetch/type';
 import { addSourceMember } from '@fastgpt/service/support/user/utils';
+import { replaceRegChars } from '@fastgpt/global/common/string/tools';
 
 async function handler(
   req: NextApiRequest,
@@ -20,7 +21,9 @@ async function handler(
   const {
     appId,
     dateStart = addDays(new Date(), -7),
-    dateEnd = new Date()
+    dateEnd = new Date(),
+    sources,
+    logTitle
   } = req.body as GetAppChatLogsParams;
 
   const { pageSize = 20, offset } = parsePaginationRequest(req);
@@ -38,7 +41,14 @@ async function handler(
     updateTime: {
       $gte: new Date(dateStart),
       $lte: new Date(dateEnd)
-    }
+    },
+    ...(sources && { source: { $in: sources } }),
+    ...(logTitle && {
+      $or: [
+        { title: { $regex: new RegExp(`${replaceRegChars(logTitle)}`, 'i') } },
+        { customTitle: { $regex: new RegExp(`${replaceRegChars(logTitle)}`, 'i') } }
+      ]
+    })
   };
 
   const [list, total] = await Promise.all([
@@ -127,6 +137,7 @@ async function handler(
             _id: 1,
             id: '$chatId',
             title: 1,
+            customTitle: 1,
             source: 1,
             time: '$updateTime',
             messageCount: { $size: '$chatitems' },

projects/app/src/types/app.d.ts

@@ -40,6 +40,7 @@ export type AppLogsListItemType = {
   source: string;
   time: Date;
   title: string;
+  customTitle: string;
   messageCount: number;
   userGoodFeedbackCount: number;
   userBadFeedbackCount: number;