Skip to content

ccran / lufa-contract

Go to a project
Commit a76f1b8c by ccran
Options

feat: add luchuan config;

parent 34f37d76
Showing with 142 additions and 99 deletions
README.md
......@@ -138,7 +138,7 @@ result = requests.get(
## 🔐 安全说明
- API Key 配置在 `core/config.py` 中
- 支持内外网环境切换 (`use_lufa` 参数)
- 支持按平台切换运行环境（`platform` 参数，如 `"jp"`、`"lufa"`）
- 临时文件自动清理
## 📊 数据格式
......
core/config.py
import platform
from dataclasses import dataclass
# 可配置运行参数
use_docker = False
# @dataclass
# class LLMConfig:
# base_url: str = "https://api.deepseek.com/v1"
# api_key: str = "sk-3df81e63afe44ca39cbd7108d59bc91a"
# model: str = "deepseek-v4-pro"
@dataclass
class LLMConfig:
base_url: str = "http://172.21.107.80:9002/v1"
api_key: str = "none"
model: str = "Qwen2-72B-Instruct"
# @dataclass
# class LLMConfig:
# base_url: str = "http://192.168.252.71:9002/v1"
# api_key: str = "none"
# model: str = "Qwen2-72B-Instruct"
# 最大分片数量
min_single_chunk_size = 2000
max_single_chunk_size = 100000
......@@ -59,29 +35,29 @@ FILE_SUFFIX = "-审核批注"
## 关键参数**
use_non_fastgpt_llm = False
use_lufa = False
use_jp_machine = True
# 运行平台。新增平台时在下方补充对应配置，并同步各平台专属提示词。
platform = "luchuan"
debug_mode = False
## 关键参数**
max_model_len = 131072
ocr_url = 'http://192.168.252.71:8202/openapi/ocrUploadFile'
if use_lufa:
ocr_url = ''
if platform == "lufa":
base_url = "http://192.168.252.71:9002/v1"
model = "Qwen2-72B-Instruct"
outer_backend_url = "http://znkf.lgfzgroup.com:48081"
base_fastgpt_url = "http://192.168.252.71:18089"
base_backend_url = "http://192.168.252.71:48081"
ocr_url = 'http://192.168.252.71:8202/openapi/ocrUploadFile'
segment_review_api_key = (
"fastgpt-zMavJKKgqA9jRNHLXxzXCVZx1JXxfuNkH1p2qfLhtPfMp41UvdSQvt8"
)
reflect_retry_api_key = (
"fastgpt-ao3al2vgfnArt9qi2bTpPeRHouCO7qngUZiQsIM1E2x91u22z65J"
)
else:
if not use_jp_machine:
outer_backend_url = "http://218.77.58.8:48080"
base_fastgpt_url = "http://192.168.252.71:18088"
base_backend_url = "http://192.168.252.71:48080"
else:
elif platform == "jp":
base_url = "http://172.21.107.80:9002/v1"
model = "Qwen2-72B-Instruct"
outer_backend_url = "https://172.21.107.45:48080"
base_fastgpt_url = "http://172.21.107.45:3030"
base_backend_url = "http://172.21.107.45:1122"
......@@ -92,18 +68,28 @@ else:
reflect_retry_api_key = (
"fastgpt-abxzi4CC7SGuVdxDVFmhAFFQHqi6owK5YsIfXdvOMEAcpIhZWDPObTz2Xn"
)
elif platform == "luchuan":
base_url = "http://znkf.lgfzgroup.com:18089/qwen/v1"
model = "Qwen2-72B-Instruct"
outer_backend_url = "http://nas.hnluchuan.com:14124"
base_fastgpt_url = "http://nas.hnluchuan.com:14125"
base_backend_url = "http://192.168.0.70:1122"
ocr_url = "http://192.168.0.70:8202/openapi/ocrUploadFile"
segment_review_api_key = (
"fastgpt-oZs0FUIgkiCmJtNcsHqaQRPI93RHhevq0i3CNGSsJlzdw9hLzyeLeaPepiitb3Hy"
)
reflect_retry_api_key = (
"fastgpt-dZXQ5YF3OuVuVqWsgYB8UTXvC6NzjvfdWhvfxfxhIT1RcMv5OtlQSyq9BqhUC"
)
else:
raise ValueError(f"Unsupported platform: {platform!r}")
@dataclass
class LLMConfig:
base_url: str = base_url
api_key: str = "none"
model: str = model
# 项目根目录
root_path = r"E:\PycharmProject\contract_review_agent"
system = platform.system()
if system == "Linux":
# root_path = "/data/home/ccran/contract_review_agent"
root_path = "/home/ccran/contract_review_agent"
elif system == "Darwin":
root_path = "/Users/chenran/PycharmProjects/contract_review_agent"
# docker设置
if use_docker:
root_path = "/app"
LLM = {
"base_tool_llm": LLMConfig(),
"fastgpt_segment_review": (
......
core/memory.py
......@@ -11,7 +11,7 @@ from uuid import uuid4
from utils.http_util import upload_file
from utils.doc_util import DocBase
from core.config import META_KEY, FILE_SUFFIX, use_lufa
from core.config import META_KEY, FILE_SUFFIX, platform
from core.finding_preprocessor import FindingPreprocessor
from core.tools.memory_export_filter import MemoryExportFilter
......@@ -468,7 +468,7 @@ class MemoryStore:
for idx, f in enumerate(target_findings, start=1):
segment_id = int(f.segment_id or 0)
chunk_id = max(segment_id, 0)
if use_lufa:
if platform == "lufa":
suggest_parts = []
if f.risk_level:
suggest_parts.append(f"风险等级：{f.risk_level}")
......
core/tools/reflect_retry.py
......@@ -5,7 +5,7 @@ from typing import Dict, List, Optional, Any
from core.tool import tool, tool_func
from core.tools.segment_llm import LLMTool
from core.config import use_lufa
from core.config import platform
REFLECT_SYSTEM_PROMPT_LF = """
你是合同审查反思智能体（ReviewReflection）。
......@@ -478,7 +478,7 @@ OUTPUT_FORMAT_SCHEMA = """
@tool("reflect_retry", "反思重试质量闸")
class ReflectRetryTool(LLMTool):
def __init__(self) -> None:
if use_lufa:
if platform == "lufa":
super().__init__(REFLECT_SYSTEM_PROMPT_LF, "fastgpt_reflect_retry")
else:
super().__init__(REFLECT_SYSTEM_PROMPT, "fastgpt_reflect_retry")
......@@ -516,7 +516,7 @@ class ReflectRetryTool(LLMTool):
# return []
user_content = (
REFLECT_USER_PROMPT_LF if use_lufa else REFLECT_USER_PROMPT
REFLECT_USER_PROMPT_LF if platform == "lufa" else REFLECT_USER_PROMPT
).format(
rule=self._stringify_rule(rule),
findings_json=json.dumps(base_findings, ensure_ascii=False),
......@@ -525,7 +525,7 @@ class ReflectRetryTool(LLMTool):
# TODO 不同规则可能有不同的反思思路提示，目前先不区分，后续可根据需要增加规则专属反思思路提示，如针对付款条款审查的反思思路提示、针对担保条款审查的反思思路提示等 --- IGNORE ---
rule_reflection_prompt=None,
) + (
OUTPUT_FORMAT_SCHEMA_LF if use_lufa else OUTPUT_FORMAT_SCHEMA
OUTPUT_FORMAT_SCHEMA_LF if platform == "lufa" else OUTPUT_FORMAT_SCHEMA
)
messages = self.build_messages(user_content)
......
core/tools/rules_retrieve_reference.py
......@@ -38,20 +38,36 @@ class RulesRetrieveReferenceTool(ToolBase):
"type": "object",
"properties": {
"ruleset_id": {"type": "string"},
"rules": {"type": "array", "items": {"type": "object"}},
"routed_rule_titles": {"type": "array", "items": {"type": "string"}},
},
"required": [],
}
)
def run(
self, ruleset_id: str = "", routed_rule_titles: List[str] | None = None
self,
ruleset_id: str = "",
rules: List[Dict[str, Any]] | None = None,
routed_rule_titles: List[str] | None = None,
) -> Dict[str, Any]:
target_ruleset_id = ruleset_id or self.default_ruleset_id
"""Retrieve rules from a ruleset or from caller-supplied rules.
A non-empty ``ruleset_id`` keeps the historical ruleset lookup behavior.
When it is empty, a supplied ``rules`` list is used directly. If neither
is supplied, the default ruleset remains the backwards-compatible fallback.
"""
target_ruleset_id = (ruleset_id or "").strip()
if target_ruleset_id:
full_rules = (
self.rulesets.get(target_ruleset_id)
or self.rulesets.get(self.default_ruleset_id, [])
or []
)
elif rules is not None:
full_rules = [rule for rule in rules if isinstance(rule, dict)]
else:
target_ruleset_id = self.default_ruleset_id
full_rules = self.rulesets.get(target_ruleset_id, []) or []
if routed_rule_titles is None:
rules = full_rules
else:
......
core/tools/segment_review.py
......@@ -5,7 +5,7 @@ import json
from typing import Dict, List, Optional
from core.tool import tool, tool_func
from core.config import use_lufa
from core.config import platform
from core.tools.segment_llm import LLMTool
import re
from loguru import logger
......@@ -281,7 +281,7 @@ def _has_evidence(f: Dict) -> bool:
@tool("segment_review", "合同分段审查")
class SegmentReviewTool(LLMTool):
def __init__(self):
if use_lufa:
if platform == "lufa":
super().__init__(REVIEW_SYSTEM_PROMPT_LF)
else:
super().__init__(REVIEW_SYSTEM_PROMPT_JP)
......@@ -349,7 +349,7 @@ class SegmentReviewTool(LLMTool):
context_memories: Optional[List[Dict]],
) -> List[Dict[str, str]]:
user_content = (
REVIEW_USER_PROMPT_JP if not use_lufa else REVIEW_USER_PROMPT_LF
REVIEW_USER_PROMPT_LF if platform == "lufa" else REVIEW_USER_PROMPT_JP
).format(
segment_id=segment_id,
segment_text=segment_text,
......@@ -358,7 +358,7 @@ class SegmentReviewTool(LLMTool):
# context_memories_json=json.dumps(context_memories or [], ensure_ascii=False),
ruleset_text=self._stringify_rule(rule),
) + (
REVIEW_OUTPUT_SCHEMA_JP if not use_lufa else REVIEW_OUTPUT_SCHEMA_LF
REVIEW_OUTPUT_SCHEMA_LF if platform == "lufa" else REVIEW_OUTPUT_SCHEMA_JP
)
return self.build_messages(user_content)
......@@ -373,7 +373,7 @@ class SegmentReviewTool(LLMTool):
) -> List[Dict[str, str]]:
ruleset_text = "\n\n".join([self._stringify_rule(rule) for rule in rules])
user_content = (
REVIEW_USER_PROMPT_JP if not use_lufa else REVIEW_USER_PROMPT_LF
REVIEW_USER_PROMPT_LF if platform == "lufa" else REVIEW_USER_PROMPT_JP
).format(
segment_id=segment_id,
segment_text=segment_text,
......@@ -383,7 +383,7 @@ class SegmentReviewTool(LLMTool):
context_memories or [], ensure_ascii=False
),
ruleset_text=ruleset_text,
) + (REVIEW_OUTPUT_SCHEMA_JP if not use_lufa else REVIEW_OUTPUT_SCHEMA_LF)
) + (REVIEW_OUTPUT_SCHEMA_LF if platform == "lufa" else REVIEW_OUTPUT_SCHEMA_JP)
return self.build_messages(user_content)
async def _evaluate_rules_async(
......
data/batch/batch.py
......@@ -13,13 +13,13 @@ from loguru import logger
from utils.common_util import random_str
from utils.http_util import upload_file, fastgpt_openai_chat, download_file
use_lufa = False
platform = "jp"
batch_size = 5
def get_params():
output_suffix = f"{time.strftime('%Y%m%d-%H%M%S')}-{time.time_ns() % 1_000_000:06d}"
if not use_lufa:
if platform == "jp":
return {
"suffix": "_麓发迁移",
"batch_size": batch_size,
......@@ -38,6 +38,7 @@ def get_params():
# "token": "fastgpt-n74gGX5ZqLT6o1ysMBSGUTjIciswYOWDRfQ75krMkE5gDVDkpzsbz8u",
}
if platform == "lufa":
return {
"suffix": "_麓发",
"batch_size": batch_size,
......@@ -51,6 +52,8 @@ def get_params():
"token": "fastgpt-mg5tQUgreJeF7peoOr5zqP0NR4EIrfS2bEVXge6FUL94Suu1TvEMR1sGNRSiV",
}
raise ValueError(f"Unsupported platform: {platform!r}")
def extract_url(text):
# \s * ([ ^ "\s]+?\.(?:docx?|pdf|xlsx))
......
main.py
......@@ -19,7 +19,7 @@ from core.config import (
pdf_support_formats,
MERGE_RULE_PROMPT,
FULL_TEXT_SEGMENT_ID,
use_lufa,
platform,
max_model_len,
debug_mode,
)
......@@ -45,7 +45,7 @@ summary_tool = SegmentSummaryTool()
review_tool = SegmentReviewTool()
rule_router_tool = SegmentRuleRouterTool()
rule_filter_tool = RuleFilterTool()
# if use_lufa:
# if platform == "lufa":
# rule_filter_tool = LufaPartyRuleFilterTool()
# else:
# rule_filter_tool = JpRuleFilterTool()
......@@ -60,12 +60,12 @@ party_role_tool = PartyRoleTool()
nanobot_llm_tool = LLMTool(llm_key="nanobot_llm")
@app.post("/sleep")
def sleep(t: int):
import time
time.sleep(t)
return {"res": f"sleep over for {t} seconds."}
class TestModel(BaseModel):
role: str
rules: List[Dict]
@app.post("/test")
def test(t: TestModel):
return t
class NanobotLLMChatRequest(BaseModel):
......@@ -100,7 +100,11 @@ class DocumentParseRequest(BaseModel):
conversation_id: str
urls: List[str] = Field(..., description="File download url")
file_ext: Optional[str] = None
ruleset_id: Optional[str] = "通用"
ruleset_id: Optional[str] = None
rules: Optional[List[Dict[str, Any]]] = Field(
default=None,
description="规则集为空时直接使用的规则列表",
)
class DocumentParseResponse(BaseModel):
......@@ -135,6 +139,19 @@ class PartyRoleResponse(BaseModel):
reason: Optional[str] = ""
def retrieve_rules(
ruleset_id: Optional[str] = None,
rules: Optional[List[Dict[str, Any]]] = None,
routed_rule_titles: Optional[List[str]] = None,
) -> Dict[str, Any]:
"""Resolve rules from a ruleset or directly from the request payload."""
return rules_reference_tool.run(
ruleset_id=ruleset_id or "",
rules=rules,
routed_rule_titles=routed_rule_titles,
)
@app.post("/rulesets/route", response_model=RulesetRouteResponse)
def route_ruleset(payload: RulesetRouteRequest) -> RulesetRouteResponse:
question = (payload.question or "").strip()
......@@ -197,8 +214,10 @@ async def parse_document(payload: DocumentParseRequest) -> DocumentParseResponse
# TODO: FastGPT BUG segment_ids必须从1开始，0开始会缺少第一段文本，后续需要修复
segment_ids = [idx + 1 for idx in segment_ids]
# get ruleset items
ruleset_id = payload.ruleset_id or rules_reference_tool.default_ruleset_id
ruleset_items = rules_reference_tool.run(ruleset_id=ruleset_id).get("rules", [])
ruleset_items = retrieve_rules(
ruleset_id=payload.ruleset_id,
rules=payload.rules,
).get("rules", [])
ruleset_review_items = [
t
for t in (r.get("title") for r in ruleset_items)
......@@ -207,7 +226,7 @@ async def parse_document(payload: DocumentParseRequest) -> DocumentParseResponse
summary_names = list(
dict.fromkeys(
s.strip()
for s in (r.get("summa y") for r in ruleset_items)
for s in (r.get("summary") for r in ruleset_items)
if isinstance(s, str) and s.strip()
)
)
......@@ -230,7 +249,8 @@ class SegmentSummaryRequest(BaseModel):
conversation_id: str
segment_id: int
party_role: Optional[str] = ""
ruleset_id: Optional[str] = "通用"
ruleset_id: Optional[str] = None
rules: Optional[List[Dict[str, Any]]] = None
routed_rule_titles: Optional[List[str]] = None
routed_summary_names: Optional[List[str]] = None
file_ext: str
......@@ -262,22 +282,25 @@ def summarize_facts(payload: SegmentSummaryRequest) -> SegmentSummaryResponse:
detail=f"Segment text not found for id {payload.segment_id}: {exc}. Please parse document first.",
)
ruleset_id = payload.ruleset_id or rules_reference_tool.default_ruleset_id
if payload.routed_summary_names is not None:
summary_names = {
name.strip()
for name in payload.routed_summary_names
if isinstance(name, str) and name.strip()
}
all_rules = rules_reference_tool.run(ruleset_id=ruleset_id).get("rules", [])
all_rules = retrieve_rules(
ruleset_id=payload.ruleset_id,
rules=payload.rules,
).get("rules", [])
rules = [
rule
for rule in all_rules
if str(rule.get("summary", "")).strip() in summary_names
]
else:
rules = rules_reference_tool.run(
ruleset_id=ruleset_id,
rules = retrieve_rules(
ruleset_id=payload.ruleset_id,
rules=payload.rules,
routed_rule_titles=payload.routed_rule_titles,
).get("rules", [])
result = summary_tool.run(
......@@ -310,7 +333,8 @@ class SegmentReviewRequest(BaseModel):
description="segment_id 为 FULL_TEXT_SEGMENT_ID 时用于审查的全文文本",
)
party_role: Optional[str] = ""
ruleset_id: Optional[str] = "通用"
ruleset_id: Optional[str] = None
rules: Optional[List[Dict[str, Any]]] = None
routed_rule_titles: Optional[List[str]] = None
file_ext: Optional[str] = None
context_memories: Optional[List[Dict]] = None
......@@ -380,9 +404,9 @@ def review_segment(payload: SegmentReviewRequest) -> SegmentReviewResponse:
store = get_cached_memory(payload.conversation_id)
segment_idx, segment_text, _ = _resolve_review_segment(payload)
ruleset_id = payload.ruleset_id or rules_reference_tool.default_ruleset_id
rules = rules_reference_tool.run(
ruleset_id=ruleset_id,
rules = retrieve_rules(
ruleset_id=payload.ruleset_id,
rules=payload.rules,
routed_rule_titles=payload.routed_rule_titles,
).get("rules", [])
# 暂时不添加摘要看下结果
......@@ -435,8 +459,11 @@ def review_segment(payload: SegmentReviewRequest) -> SegmentReviewResponse:
def route_segment_rules(payload: SegmentReviewRequest) -> SegmentRuleRouterResponse:
segment_idx, segment_text, doc_obj = _resolve_review_segment(payload)
ruleset_id = payload.ruleset_id or rules_reference_tool.default_ruleset_id
rules = rules_reference_tool.run(ruleset_id=ruleset_id).get("rules", [])
rules_result = retrieve_rules(
ruleset_id=payload.ruleset_id,
rules=payload.rules,
)
rules = rules_result.get("rules", [])
try:
total_segments = len(doc_obj.get_chunk_id_list() or [])
......@@ -470,7 +497,7 @@ def route_segment_rules(payload: SegmentReviewRequest) -> SegmentRuleRouterRespo
return SegmentRuleRouterResponse(
conversation_id=payload.conversation_id,
segment_id=payload.segment_id,
ruleset_id=ruleset_id,
ruleset_id=rules_result.get("ruleset_id", ""),
route_by=result.get("route_by", payload.route_by),
routed_rule_titles=routed_rule_titles,
routed_summary_names=result.get("routed_summary_names", []),
......@@ -482,12 +509,14 @@ def route_segment_rules(payload: SegmentReviewRequest) -> SegmentRuleRouterRespo
class ReflectReviewRequest(BaseModel):
conversation_id: str
party_role: str
ruleset_id: Optional[str] = "通用"
ruleset_id: Optional[str] = None
rules: Optional[List[Dict[str, Any]]] = None
rule_title: str
class RuleStringifyRequest(BaseModel):
ruleset_id: Optional[str] = "通用"
ruleset_id: Optional[str] = None
rules: Optional[List[Dict[str, Any]]] = None
rule_title: str
......@@ -530,8 +559,11 @@ class FactsMergerResponse(BaseModel):
@app.post("/rules/stringify", response_model=RuleStringifyResponse)
def stringify_rule(payload: RuleStringifyRequest) -> RuleStringifyResponse:
ruleset_id = payload.ruleset_id or rules_reference_tool.default_ruleset_id
ruleset_items = rules_reference_tool.run(ruleset_id=ruleset_id).get("rules", [])
rules_result = retrieve_rules(
ruleset_id=payload.ruleset_id,
rules=payload.rules,
)
ruleset_items = rules_result.get("rules", [])
rule = next(
(r for r in ruleset_items if r.get("title") == payload.rule_title), None
)
......@@ -541,7 +573,7 @@ def stringify_rule(payload: RuleStringifyRequest) -> RuleStringifyResponse:
)
return RuleStringifyResponse(
ruleset_id=ruleset_id,
ruleset_id=rules_result.get("ruleset_id", ""),
rule_title=payload.rule_title,
rule_string=reflect_tool._stringify_rule(rule),
)
......@@ -550,8 +582,10 @@ def stringify_rule(payload: RuleStringifyRequest) -> RuleStringifyResponse:
@app.post("/segments/review/reflect", response_model=ReflectReviewResponse)
def reflect_review(payload: ReflectReviewRequest) -> ReflectReviewResponse:
store = get_cached_memory(payload.conversation_id)
ruleset_id = payload.ruleset_id or rules_reference_tool.default_ruleset_id
ruleset_items = rules_reference_tool.run(ruleset_id=ruleset_id).get("rules", [])
ruleset_items = retrieve_rules(
ruleset_id=payload.ruleset_id,
rules=payload.rules,
).get("rules", [])
rule = next(
(r for r in ruleset_items if r.get("title") == payload.rule_title), None
)
......@@ -799,12 +833,10 @@ def export_facts_memory(payload: FactsMemoryExportRequest) -> FactsMemoryExportR
if __name__ == "__main__":
from core.config import use_lufa
if use_lufa:
if platform == "lufa" or platform == "luchuan":
port = 18168
else:
port = 18169
if debug_mode:
port += 10
uvicorn.run("main:app", host="0.0.0.0", port=port, log_level="info", reload=False)
uvicorn.run("main:app", host="0.0.0.0", port=port, log_level="info", reload=True)
requirements.txt
......@@ -76,3 +76,5 @@ urllib3==2.6.3
uvicorn==0.40.0
wheel==0.45.1
yarl==1.22.0
spire-doc==14.6.1
spire-pdf==12.6.2
\ No newline at end of file
utils/http_util.py
......@@ -8,7 +8,7 @@ import requests
from loguru import logger
from requests_toolbelt import MultipartEncoder
from core.config import base_fastgpt_url, base_backend_url, outer_backend_url
from core.config import base_fastgpt_url, base_backend_url, outer_backend_url,platform
def fastgpt_openai_chat(url, token, model, chat_id, file_url, text, stream=True):
......@@ -58,6 +58,9 @@ def fastgpt_openai_chat(url, token, model, chat_id, file_url, text, stream=True)
def upload_file(path, input_url_to_inner=True, output_url_to_inner=False) -> str:
# 登录获取token
if platform == "luchuan":
login_data = {"username": "chenran", "password": "CR@lgfz2026"}
else:
login_data = {"username": "admin", "password": "admin@jpai.com"}
login_url = f"{base_backend_url}/admin-api/system/auth/login"
response = requests.post(
......
utils/spire_word_util.py
......@@ -4,7 +4,7 @@ import re
from thefuzz import fuzz
from utils.doc_util import DocBase
from utils.common_util import adjust_single_chunk_size
from core.config import FULL_TEXT_SEGMENT_ID, use_lufa
from core.config import FULL_TEXT_SEGMENT_ID
import os
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment