Skip to content
Toggle navigation
P
Projects
G
Groups
S
Snippets
Help
ccran
/
lufa-contract
This project
Loading...
Sign in
Toggle navigation
Go to a project
Project
Repository
Issues
0
Merge Requests
0
Pipelines
Wiki
Snippets
Members
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Commit
a76f1b8c
authored
Jul 25, 2026
by
ccran
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
feat: add luchuan config;
parent
34f37d76
Hide whitespace changes
Inline
Side-by-side
Showing
11 changed files
with
164 additions
and
121 deletions
+164
-121
README.md
+1
-1
core/config.py
+35
-49
core/memory.py
+2
-2
core/tools/reflect_retry.py
+4
-4
core/tools/rules_retrieve_reference.py
+23
-7
core/tools/segment_review.py
+6
-6
data/batch/batch.py
+17
-14
main.py
+67
-35
requirements.txt
+3
-0
utils/http_util.py
+5
-2
utils/spire_word_util.py
+1
-1
No files found.
README.md
View file @
a76f1b8c
...
@@ -138,7 +138,7 @@ result = requests.get(
...
@@ -138,7 +138,7 @@ result = requests.get(
## 🔐 安全说明
## 🔐 安全说明
-
API Key 配置在
`core/config.py`
中
-
API Key 配置在
`core/config.py`
中
-
支持
内外网环境切换 (
`use_lufa`
参数)
-
支持
按平台切换运行环境(
`platform`
参数,如
`"jp"`
、
`"lufa"`
)
-
临时文件自动清理
-
临时文件自动清理
## 📊 数据格式
## 📊 数据格式
...
...
core/config.py
View file @
a76f1b8c
import
platform
from
dataclasses
import
dataclass
from
dataclasses
import
dataclass
# 可配置运行参数
use_docker
=
False
# @dataclass
# class LLMConfig:
# base_url: str = "https://api.deepseek.com/v1"
# api_key: str = "sk-3df81e63afe44ca39cbd7108d59bc91a"
# model: str = "deepseek-v4-pro"
@dataclass
class
LLMConfig
:
base_url
:
str
=
"http://172.21.107.80:9002/v1"
api_key
:
str
=
"none"
model
:
str
=
"Qwen2-72B-Instruct"
# @dataclass
# class LLMConfig:
# base_url: str = "http://192.168.252.71:9002/v1"
# api_key: str = "none"
# model: str = "Qwen2-72B-Instruct"
# 最大分片数量
# 最大分片数量
min_single_chunk_size
=
2000
min_single_chunk_size
=
2000
max_single_chunk_size
=
100000
max_single_chunk_size
=
100000
...
@@ -59,51 +35,61 @@ FILE_SUFFIX = "-审核批注"
...
@@ -59,51 +35,61 @@ FILE_SUFFIX = "-审核批注"
## 关键参数**
## 关键参数**
use_non_fastgpt_llm
=
False
use_non_fastgpt_llm
=
False
use_lufa
=
False
# 运行平台。新增平台时在下方补充对应配置,并同步各平台专属提示词。
use_jp_machine
=
True
platform
=
"luchuan"
debug_mode
=
False
debug_mode
=
False
## 关键参数**
## 关键参数**
max_model_len
=
131072
max_model_len
=
131072
ocr_url
=
'http://192.168.252.71:8202/openapi/ocrUploadFile'
ocr_url
=
''
if
use_lufa
:
if
platform
==
"lufa"
:
base_url
=
"http://192.168.252.71:9002/v1"
model
=
"Qwen2-72B-Instruct"
outer_backend_url
=
"http://znkf.lgfzgroup.com:48081"
outer_backend_url
=
"http://znkf.lgfzgroup.com:48081"
base_fastgpt_url
=
"http://192.168.252.71:18089"
base_fastgpt_url
=
"http://192.168.252.71:18089"
base_backend_url
=
"http://192.168.252.71:48081"
base_backend_url
=
"http://192.168.252.71:48081"
ocr_url
=
'http://192.168.252.71:8202/openapi/ocrUploadFile'
segment_review_api_key
=
(
segment_review_api_key
=
(
"fastgpt-zMavJKKgqA9jRNHLXxzXCVZx1JXxfuNkH1p2qfLhtPfMp41UvdSQvt8"
"fastgpt-zMavJKKgqA9jRNHLXxzXCVZx1JXxfuNkH1p2qfLhtPfMp41UvdSQvt8"
)
)
reflect_retry_api_key
=
(
reflect_retry_api_key
=
(
"fastgpt-ao3al2vgfnArt9qi2bTpPeRHouCO7qngUZiQsIM1E2x91u22z65J"
"fastgpt-ao3al2vgfnArt9qi2bTpPeRHouCO7qngUZiQsIM1E2x91u22z65J"
)
)
else
:
elif
platform
==
"jp"
:
if
not
use_jp_machine
:
base_url
=
"http://172.21.107.80:9002/v1"
outer_backend_url
=
"http://218.77.58.8:48080"
model
=
"Qwen2-72B-Instruct"
base_fastgpt_url
=
"http://192.168.252.71:18088"
outer_backend_url
=
"https://172.21.107.45:48080"
base_backend_url
=
"http://192.168.252.71:48080"
base_fastgpt_url
=
"http://172.21.107.45:3030"
else
:
base_backend_url
=
"http://172.21.107.45:1122"
outer_backend_url
=
"https://172.21.107.45:48080"
ocr_url
=
"http://172.21.107.45:8202/openapi/ocrUploadFile"
base_fastgpt_url
=
"http://172.21.107.45:3030"
base_backend_url
=
"http://172.21.107.45:1122"
ocr_url
=
"http://172.21.107.45:8202/openapi/ocrUploadFile"
segment_review_api_key
=
(
segment_review_api_key
=
(
"fastgpt-vLu2JHAfqwEq5FUQhvATFDK0yDS6fs804v7KwWBMyU4sRrHzh4UGl89Zpa"
"fastgpt-vLu2JHAfqwEq5FUQhvATFDK0yDS6fs804v7KwWBMyU4sRrHzh4UGl89Zpa"
)
)
reflect_retry_api_key
=
(
reflect_retry_api_key
=
(
"fastgpt-abxzi4CC7SGuVdxDVFmhAFFQHqi6owK5YsIfXdvOMEAcpIhZWDPObTz2Xn"
"fastgpt-abxzi4CC7SGuVdxDVFmhAFFQHqi6owK5YsIfXdvOMEAcpIhZWDPObTz2Xn"
)
)
elif
platform
==
"luchuan"
:
base_url
=
"http://znkf.lgfzgroup.com:18089/qwen/v1"
model
=
"Qwen2-72B-Instruct"
outer_backend_url
=
"http://nas.hnluchuan.com:14124"
base_fastgpt_url
=
"http://nas.hnluchuan.com:14125"
base_backend_url
=
"http://192.168.0.70:1122"
ocr_url
=
"http://192.168.0.70:8202/openapi/ocrUploadFile"
segment_review_api_key
=
(
"fastgpt-oZs0FUIgkiCmJtNcsHqaQRPI93RHhevq0i3CNGSsJlzdw9hLzyeLeaPepiitb3Hy"
)
reflect_retry_api_key
=
(
"fastgpt-dZXQ5YF3OuVuVqWsgYB8UTXvC6NzjvfdWhvfxfxhIT1RcMv5OtlQSyq9BqhUC"
)
else
:
raise
ValueError
(
f
"Unsupported platform: {platform!r}"
)
@dataclass
class
LLMConfig
:
base_url
:
str
=
base_url
api_key
:
str
=
"none"
model
:
str
=
model
# 项目根目录
root_path
=
r"E:\PycharmProject\contract_review_agent"
system
=
platform
.
system
()
if
system
==
"Linux"
:
# root_path = "/data/home/ccran/contract_review_agent"
root_path
=
"/home/ccran/contract_review_agent"
elif
system
==
"Darwin"
:
root_path
=
"/Users/chenran/PycharmProjects/contract_review_agent"
# docker设置
if
use_docker
:
root_path
=
"/app"
LLM
=
{
LLM
=
{
"base_tool_llm"
:
LLMConfig
(),
"base_tool_llm"
:
LLMConfig
(),
"fastgpt_segment_review"
:
(
"fastgpt_segment_review"
:
(
...
...
core/memory.py
View file @
a76f1b8c
...
@@ -11,7 +11,7 @@ from uuid import uuid4
...
@@ -11,7 +11,7 @@ from uuid import uuid4
from
utils.http_util
import
upload_file
from
utils.http_util
import
upload_file
from
utils.doc_util
import
DocBase
from
utils.doc_util
import
DocBase
from
core.config
import
META_KEY
,
FILE_SUFFIX
,
use_lufa
from
core.config
import
META_KEY
,
FILE_SUFFIX
,
platform
from
core.finding_preprocessor
import
FindingPreprocessor
from
core.finding_preprocessor
import
FindingPreprocessor
from
core.tools.memory_export_filter
import
MemoryExportFilter
from
core.tools.memory_export_filter
import
MemoryExportFilter
...
@@ -468,7 +468,7 @@ class MemoryStore:
...
@@ -468,7 +468,7 @@ class MemoryStore:
for
idx
,
f
in
enumerate
(
target_findings
,
start
=
1
):
for
idx
,
f
in
enumerate
(
target_findings
,
start
=
1
):
segment_id
=
int
(
f
.
segment_id
or
0
)
segment_id
=
int
(
f
.
segment_id
or
0
)
chunk_id
=
max
(
segment_id
,
0
)
chunk_id
=
max
(
segment_id
,
0
)
if
use_lufa
:
if
platform
==
"lufa"
:
suggest_parts
=
[]
suggest_parts
=
[]
if
f
.
risk_level
:
if
f
.
risk_level
:
suggest_parts
.
append
(
f
"风险等级:{f.risk_level}"
)
suggest_parts
.
append
(
f
"风险等级:{f.risk_level}"
)
...
...
core/tools/reflect_retry.py
View file @
a76f1b8c
...
@@ -5,7 +5,7 @@ from typing import Dict, List, Optional, Any
...
@@ -5,7 +5,7 @@ from typing import Dict, List, Optional, Any
from
core.tool
import
tool
,
tool_func
from
core.tool
import
tool
,
tool_func
from
core.tools.segment_llm
import
LLMTool
from
core.tools.segment_llm
import
LLMTool
from
core.config
import
use_lufa
from
core.config
import
platform
REFLECT_SYSTEM_PROMPT_LF
=
"""
REFLECT_SYSTEM_PROMPT_LF
=
"""
你是合同审查反思智能体(ReviewReflection)。
你是合同审查反思智能体(ReviewReflection)。
...
@@ -478,7 +478,7 @@ OUTPUT_FORMAT_SCHEMA = """
...
@@ -478,7 +478,7 @@ OUTPUT_FORMAT_SCHEMA = """
@tool
(
"reflect_retry"
,
"反思重试质量闸"
)
@tool
(
"reflect_retry"
,
"反思重试质量闸"
)
class
ReflectRetryTool
(
LLMTool
):
class
ReflectRetryTool
(
LLMTool
):
def
__init__
(
self
)
->
None
:
def
__init__
(
self
)
->
None
:
if
use_lufa
:
if
platform
==
"lufa"
:
super
()
.
__init__
(
REFLECT_SYSTEM_PROMPT_LF
,
"fastgpt_reflect_retry"
)
super
()
.
__init__
(
REFLECT_SYSTEM_PROMPT_LF
,
"fastgpt_reflect_retry"
)
else
:
else
:
super
()
.
__init__
(
REFLECT_SYSTEM_PROMPT
,
"fastgpt_reflect_retry"
)
super
()
.
__init__
(
REFLECT_SYSTEM_PROMPT
,
"fastgpt_reflect_retry"
)
...
@@ -516,7 +516,7 @@ class ReflectRetryTool(LLMTool):
...
@@ -516,7 +516,7 @@ class ReflectRetryTool(LLMTool):
# return []
# return []
user_content
=
(
user_content
=
(
REFLECT_USER_PROMPT_LF
if
use_lufa
else
REFLECT_USER_PROMPT
REFLECT_USER_PROMPT_LF
if
platform
==
"lufa"
else
REFLECT_USER_PROMPT
)
.
format
(
)
.
format
(
rule
=
self
.
_stringify_rule
(
rule
),
rule
=
self
.
_stringify_rule
(
rule
),
findings_json
=
json
.
dumps
(
base_findings
,
ensure_ascii
=
False
),
findings_json
=
json
.
dumps
(
base_findings
,
ensure_ascii
=
False
),
...
@@ -525,7 +525,7 @@ class ReflectRetryTool(LLMTool):
...
@@ -525,7 +525,7 @@ class ReflectRetryTool(LLMTool):
# TODO 不同规则可能有不同的反思思路提示,目前先不区分,后续可根据需要增加规则专属反思思路提示,如针对付款条款审查的反思思路提示、针对担保条款审查的反思思路提示等 --- IGNORE ---
# TODO 不同规则可能有不同的反思思路提示,目前先不区分,后续可根据需要增加规则专属反思思路提示,如针对付款条款审查的反思思路提示、针对担保条款审查的反思思路提示等 --- IGNORE ---
rule_reflection_prompt
=
None
,
rule_reflection_prompt
=
None
,
)
+
(
)
+
(
OUTPUT_FORMAT_SCHEMA_LF
if
use_lufa
else
OUTPUT_FORMAT_SCHEMA
OUTPUT_FORMAT_SCHEMA_LF
if
platform
==
"lufa"
else
OUTPUT_FORMAT_SCHEMA
)
)
messages
=
self
.
build_messages
(
user_content
)
messages
=
self
.
build_messages
(
user_content
)
...
...
core/tools/rules_retrieve_reference.py
View file @
a76f1b8c
...
@@ -38,20 +38,36 @@ class RulesRetrieveReferenceTool(ToolBase):
...
@@ -38,20 +38,36 @@ class RulesRetrieveReferenceTool(ToolBase):
"type"
:
"object"
,
"type"
:
"object"
,
"properties"
:
{
"properties"
:
{
"ruleset_id"
:
{
"type"
:
"string"
},
"ruleset_id"
:
{
"type"
:
"string"
},
"rules"
:
{
"type"
:
"array"
,
"items"
:
{
"type"
:
"object"
}},
"routed_rule_titles"
:
{
"type"
:
"array"
,
"items"
:
{
"type"
:
"string"
}},
"routed_rule_titles"
:
{
"type"
:
"array"
,
"items"
:
{
"type"
:
"string"
}},
},
},
"required"
:
[],
"required"
:
[],
}
}
)
)
def
run
(
def
run
(
self
,
ruleset_id
:
str
=
""
,
routed_rule_titles
:
List
[
str
]
|
None
=
None
self
,
ruleset_id
:
str
=
""
,
rules
:
List
[
Dict
[
str
,
Any
]]
|
None
=
None
,
routed_rule_titles
:
List
[
str
]
|
None
=
None
,
)
->
Dict
[
str
,
Any
]:
)
->
Dict
[
str
,
Any
]:
target_ruleset_id
=
ruleset_id
or
self
.
default_ruleset_id
"""Retrieve rules from a ruleset or from caller-supplied rules.
full_rules
=
(
self
.
rulesets
.
get
(
target_ruleset_id
)
A non-empty ``ruleset_id`` keeps the historical ruleset lookup behavior.
or
self
.
rulesets
.
get
(
self
.
default_ruleset_id
,
[])
When it is empty, a supplied ``rules`` list is used directly. If neither
or
[]
is supplied, the default ruleset remains the backwards-compatible fallback.
)
"""
target_ruleset_id
=
(
ruleset_id
or
""
)
.
strip
()
if
target_ruleset_id
:
full_rules
=
(
self
.
rulesets
.
get
(
target_ruleset_id
)
or
self
.
rulesets
.
get
(
self
.
default_ruleset_id
,
[])
or
[]
)
elif
rules
is
not
None
:
full_rules
=
[
rule
for
rule
in
rules
if
isinstance
(
rule
,
dict
)]
else
:
target_ruleset_id
=
self
.
default_ruleset_id
full_rules
=
self
.
rulesets
.
get
(
target_ruleset_id
,
[])
or
[]
if
routed_rule_titles
is
None
:
if
routed_rule_titles
is
None
:
rules
=
full_rules
rules
=
full_rules
else
:
else
:
...
...
core/tools/segment_review.py
View file @
a76f1b8c
...
@@ -5,7 +5,7 @@ import json
...
@@ -5,7 +5,7 @@ import json
from
typing
import
Dict
,
List
,
Optional
from
typing
import
Dict
,
List
,
Optional
from
core.tool
import
tool
,
tool_func
from
core.tool
import
tool
,
tool_func
from
core.config
import
use_lufa
from
core.config
import
platform
from
core.tools.segment_llm
import
LLMTool
from
core.tools.segment_llm
import
LLMTool
import
re
import
re
from
loguru
import
logger
from
loguru
import
logger
...
@@ -281,7 +281,7 @@ def _has_evidence(f: Dict) -> bool:
...
@@ -281,7 +281,7 @@ def _has_evidence(f: Dict) -> bool:
@tool
(
"segment_review"
,
"合同分段审查"
)
@tool
(
"segment_review"
,
"合同分段审查"
)
class
SegmentReviewTool
(
LLMTool
):
class
SegmentReviewTool
(
LLMTool
):
def
__init__
(
self
):
def
__init__
(
self
):
if
use_lufa
:
if
platform
==
"lufa"
:
super
()
.
__init__
(
REVIEW_SYSTEM_PROMPT_LF
)
super
()
.
__init__
(
REVIEW_SYSTEM_PROMPT_LF
)
else
:
else
:
super
()
.
__init__
(
REVIEW_SYSTEM_PROMPT_JP
)
super
()
.
__init__
(
REVIEW_SYSTEM_PROMPT_JP
)
...
@@ -349,7 +349,7 @@ class SegmentReviewTool(LLMTool):
...
@@ -349,7 +349,7 @@ class SegmentReviewTool(LLMTool):
context_memories
:
Optional
[
List
[
Dict
]],
context_memories
:
Optional
[
List
[
Dict
]],
)
->
List
[
Dict
[
str
,
str
]]:
)
->
List
[
Dict
[
str
,
str
]]:
user_content
=
(
user_content
=
(
REVIEW_USER_PROMPT_
JP
if
not
use_lufa
else
REVIEW_USER_PROMPT_LF
REVIEW_USER_PROMPT_
LF
if
platform
==
"lufa"
else
REVIEW_USER_PROMPT_JP
)
.
format
(
)
.
format
(
segment_id
=
segment_id
,
segment_id
=
segment_id
,
segment_text
=
segment_text
,
segment_text
=
segment_text
,
...
@@ -358,7 +358,7 @@ class SegmentReviewTool(LLMTool):
...
@@ -358,7 +358,7 @@ class SegmentReviewTool(LLMTool):
# context_memories_json=json.dumps(context_memories or [], ensure_ascii=False),
# context_memories_json=json.dumps(context_memories or [], ensure_ascii=False),
ruleset_text
=
self
.
_stringify_rule
(
rule
),
ruleset_text
=
self
.
_stringify_rule
(
rule
),
)
+
(
)
+
(
REVIEW_OUTPUT_SCHEMA_
JP
if
not
use_lufa
else
REVIEW_OUTPUT_SCHEMA_LF
REVIEW_OUTPUT_SCHEMA_
LF
if
platform
==
"lufa"
else
REVIEW_OUTPUT_SCHEMA_JP
)
)
return
self
.
build_messages
(
user_content
)
return
self
.
build_messages
(
user_content
)
...
@@ -373,7 +373,7 @@ class SegmentReviewTool(LLMTool):
...
@@ -373,7 +373,7 @@ class SegmentReviewTool(LLMTool):
)
->
List
[
Dict
[
str
,
str
]]:
)
->
List
[
Dict
[
str
,
str
]]:
ruleset_text
=
"
\n\n
"
.
join
([
self
.
_stringify_rule
(
rule
)
for
rule
in
rules
])
ruleset_text
=
"
\n\n
"
.
join
([
self
.
_stringify_rule
(
rule
)
for
rule
in
rules
])
user_content
=
(
user_content
=
(
REVIEW_USER_PROMPT_
JP
if
not
use_lufa
else
REVIEW_USER_PROMPT_LF
REVIEW_USER_PROMPT_
LF
if
platform
==
"lufa"
else
REVIEW_USER_PROMPT_JP
)
.
format
(
)
.
format
(
segment_id
=
segment_id
,
segment_id
=
segment_id
,
segment_text
=
segment_text
,
segment_text
=
segment_text
,
...
@@ -383,7 +383,7 @@ class SegmentReviewTool(LLMTool):
...
@@ -383,7 +383,7 @@ class SegmentReviewTool(LLMTool):
context_memories
or
[],
ensure_ascii
=
False
context_memories
or
[],
ensure_ascii
=
False
),
),
ruleset_text
=
ruleset_text
,
ruleset_text
=
ruleset_text
,
)
+
(
REVIEW_OUTPUT_SCHEMA_
JP
if
not
use_lufa
else
REVIEW_OUTPUT_SCHEMA_LF
)
)
+
(
REVIEW_OUTPUT_SCHEMA_
LF
if
platform
==
"lufa"
else
REVIEW_OUTPUT_SCHEMA_JP
)
return
self
.
build_messages
(
user_content
)
return
self
.
build_messages
(
user_content
)
async
def
_evaluate_rules_async
(
async
def
_evaluate_rules_async
(
...
...
data/batch/batch.py
View file @
a76f1b8c
...
@@ -13,13 +13,13 @@ from loguru import logger
...
@@ -13,13 +13,13 @@ from loguru import logger
from
utils.common_util
import
random_str
from
utils.common_util
import
random_str
from
utils.http_util
import
upload_file
,
fastgpt_openai_chat
,
download_file
from
utils.http_util
import
upload_file
,
fastgpt_openai_chat
,
download_file
use_lufa
=
False
platform
=
"jp"
batch_size
=
5
batch_size
=
5
def
get_params
():
def
get_params
():
output_suffix
=
f
"{time.strftime('
%
Y
%
m
%
d-
%
H
%
M
%
S')}-{time.time_ns()
% 1
_000_000:06d}"
output_suffix
=
f
"{time.strftime('
%
Y
%
m
%
d-
%
H
%
M
%
S')}-{time.time_ns()
% 1
_000_000:06d}"
if
not
use_lufa
:
if
platform
==
"jp"
:
return
{
return
{
"suffix"
:
"_麓发迁移"
,
"suffix"
:
"_麓发迁移"
,
"batch_size"
:
batch_size
,
"batch_size"
:
batch_size
,
...
@@ -38,18 +38,21 @@ def get_params():
...
@@ -38,18 +38,21 @@ def get_params():
# "token": "fastgpt-n74gGX5ZqLT6o1ysMBSGUTjIciswYOWDRfQ75krMkE5gDVDkpzsbz8u",
# "token": "fastgpt-n74gGX5ZqLT6o1ysMBSGUTjIciswYOWDRfQ75krMkE5gDVDkpzsbz8u",
}
}
return
{
if
platform
==
"lufa"
:
"suffix"
:
"_麓发"
,
return
{
"batch_size"
:
batch_size
,
"suffix"
:
"_麓发"
,
"batch_input_dir_path"
:
"4.24测财务合同审核"
,
"batch_size"
:
batch_size
,
"batch_output_dir_path"
:
f
"4.24测财务合同审核-batch-{output_suffix}"
,
"batch_input_dir_path"
:
"4.24测财务合同审核"
,
# 麓发fastgpt接口
"batch_output_dir_path"
:
f
"4.24测财务合同审核-batch-{output_suffix}"
,
"url"
:
"http://192.168.252.71:18089/api/v1/chat/completions"
,
# 麓发fastgpt接口
# 麓发合同审查生产token
"url"
:
"http://192.168.252.71:18089/api/v1/chat/completions"
,
# "token": "fastgpt-ek3Z6PxI6sXgYc0jxzZ5bVGqrxwM6aVyfSmA6JVErJYBMr2KmYxrHwEUOIMSYz",
# 麓发合同审查生产token
# 麓发合同审查生产token-标准化
# "token": "fastgpt-ek3Z6PxI6sXgYc0jxzZ5bVGqrxwM6aVyfSmA6JVErJYBMr2KmYxrHwEUOIMSYz",
"token"
:
"fastgpt-mg5tQUgreJeF7peoOr5zqP0NR4EIrfS2bEVXge6FUL94Suu1TvEMR1sGNRSiV"
,
# 麓发合同审查生产token-标准化
}
"token"
:
"fastgpt-mg5tQUgreJeF7peoOr5zqP0NR4EIrfS2bEVXge6FUL94Suu1TvEMR1sGNRSiV"
,
}
raise
ValueError
(
f
"Unsupported platform: {platform!r}"
)
def
extract_url
(
text
):
def
extract_url
(
text
):
...
...
main.py
View file @
a76f1b8c
...
@@ -19,7 +19,7 @@ from core.config import (
...
@@ -19,7 +19,7 @@ from core.config import (
pdf_support_formats
,
pdf_support_formats
,
MERGE_RULE_PROMPT
,
MERGE_RULE_PROMPT
,
FULL_TEXT_SEGMENT_ID
,
FULL_TEXT_SEGMENT_ID
,
use_lufa
,
platform
,
max_model_len
,
max_model_len
,
debug_mode
,
debug_mode
,
)
)
...
@@ -45,7 +45,7 @@ summary_tool = SegmentSummaryTool()
...
@@ -45,7 +45,7 @@ summary_tool = SegmentSummaryTool()
review_tool
=
SegmentReviewTool
()
review_tool
=
SegmentReviewTool
()
rule_router_tool
=
SegmentRuleRouterTool
()
rule_router_tool
=
SegmentRuleRouterTool
()
rule_filter_tool
=
RuleFilterTool
()
rule_filter_tool
=
RuleFilterTool
()
# if
use_lufa
:
# if
platform == "lufa"
:
# rule_filter_tool = LufaPartyRuleFilterTool()
# rule_filter_tool = LufaPartyRuleFilterTool()
# else:
# else:
# rule_filter_tool = JpRuleFilterTool()
# rule_filter_tool = JpRuleFilterTool()
...
@@ -60,12 +60,12 @@ party_role_tool = PartyRoleTool()
...
@@ -60,12 +60,12 @@ party_role_tool = PartyRoleTool()
nanobot_llm_tool
=
LLMTool
(
llm_key
=
"nanobot_llm"
)
nanobot_llm_tool
=
LLMTool
(
llm_key
=
"nanobot_llm"
)
@app.post
(
"/sleep"
)
class
TestModel
(
BaseModel
):
def
sleep
(
t
:
int
):
role
:
str
import
time
rules
:
List
[
Dict
]
@app.post
(
"/test"
)
time
.
sleep
(
t
)
def
test
(
t
:
TestModel
):
return
{
"res"
:
f
"sleep over for {t} seconds."
}
return
t
class
NanobotLLMChatRequest
(
BaseModel
):
class
NanobotLLMChatRequest
(
BaseModel
):
...
@@ -100,7 +100,11 @@ class DocumentParseRequest(BaseModel):
...
@@ -100,7 +100,11 @@ class DocumentParseRequest(BaseModel):
conversation_id
:
str
conversation_id
:
str
urls
:
List
[
str
]
=
Field
(
...
,
description
=
"File download url"
)
urls
:
List
[
str
]
=
Field
(
...
,
description
=
"File download url"
)
file_ext
:
Optional
[
str
]
=
None
file_ext
:
Optional
[
str
]
=
None
ruleset_id
:
Optional
[
str
]
=
"通用"
ruleset_id
:
Optional
[
str
]
=
None
rules
:
Optional
[
List
[
Dict
[
str
,
Any
]]]
=
Field
(
default
=
None
,
description
=
"规则集为空时直接使用的规则列表"
,
)
class
DocumentParseResponse
(
BaseModel
):
class
DocumentParseResponse
(
BaseModel
):
...
@@ -135,6 +139,19 @@ class PartyRoleResponse(BaseModel):
...
@@ -135,6 +139,19 @@ class PartyRoleResponse(BaseModel):
reason
:
Optional
[
str
]
=
""
reason
:
Optional
[
str
]
=
""
def
retrieve_rules
(
ruleset_id
:
Optional
[
str
]
=
None
,
rules
:
Optional
[
List
[
Dict
[
str
,
Any
]]]
=
None
,
routed_rule_titles
:
Optional
[
List
[
str
]]
=
None
,
)
->
Dict
[
str
,
Any
]:
"""Resolve rules from a ruleset or directly from the request payload."""
return
rules_reference_tool
.
run
(
ruleset_id
=
ruleset_id
or
""
,
rules
=
rules
,
routed_rule_titles
=
routed_rule_titles
,
)
@app.post
(
"/rulesets/route"
,
response_model
=
RulesetRouteResponse
)
@app.post
(
"/rulesets/route"
,
response_model
=
RulesetRouteResponse
)
def
route_ruleset
(
payload
:
RulesetRouteRequest
)
->
RulesetRouteResponse
:
def
route_ruleset
(
payload
:
RulesetRouteRequest
)
->
RulesetRouteResponse
:
question
=
(
payload
.
question
or
""
)
.
strip
()
question
=
(
payload
.
question
or
""
)
.
strip
()
...
@@ -197,8 +214,10 @@ async def parse_document(payload: DocumentParseRequest) -> DocumentParseResponse
...
@@ -197,8 +214,10 @@ async def parse_document(payload: DocumentParseRequest) -> DocumentParseResponse
# TODO: FastGPT BUG segment_ids必须从1开始,0开始会缺少第一段文本,后续需要修复
# TODO: FastGPT BUG segment_ids必须从1开始,0开始会缺少第一段文本,后续需要修复
segment_ids
=
[
idx
+
1
for
idx
in
segment_ids
]
segment_ids
=
[
idx
+
1
for
idx
in
segment_ids
]
# get ruleset items
# get ruleset items
ruleset_id
=
payload
.
ruleset_id
or
rules_reference_tool
.
default_ruleset_id
ruleset_items
=
retrieve_rules
(
ruleset_items
=
rules_reference_tool
.
run
(
ruleset_id
=
ruleset_id
)
.
get
(
"rules"
,
[])
ruleset_id
=
payload
.
ruleset_id
,
rules
=
payload
.
rules
,
)
.
get
(
"rules"
,
[])
ruleset_review_items
=
[
ruleset_review_items
=
[
t
t
for
t
in
(
r
.
get
(
"title"
)
for
r
in
ruleset_items
)
for
t
in
(
r
.
get
(
"title"
)
for
r
in
ruleset_items
)
...
@@ -207,7 +226,7 @@ async def parse_document(payload: DocumentParseRequest) -> DocumentParseResponse
...
@@ -207,7 +226,7 @@ async def parse_document(payload: DocumentParseRequest) -> DocumentParseResponse
summary_names
=
list
(
summary_names
=
list
(
dict
.
fromkeys
(
dict
.
fromkeys
(
s
.
strip
()
s
.
strip
()
for
s
in
(
r
.
get
(
"summa
y"
)
for
r
in
ruleset_items
)
for
s
in
(
r
.
get
(
"summa
r
y"
)
for
r
in
ruleset_items
)
if
isinstance
(
s
,
str
)
and
s
.
strip
()
if
isinstance
(
s
,
str
)
and
s
.
strip
()
)
)
)
)
...
@@ -230,7 +249,8 @@ class SegmentSummaryRequest(BaseModel):
...
@@ -230,7 +249,8 @@ class SegmentSummaryRequest(BaseModel):
conversation_id
:
str
conversation_id
:
str
segment_id
:
int
segment_id
:
int
party_role
:
Optional
[
str
]
=
""
party_role
:
Optional
[
str
]
=
""
ruleset_id
:
Optional
[
str
]
=
"通用"
ruleset_id
:
Optional
[
str
]
=
None
rules
:
Optional
[
List
[
Dict
[
str
,
Any
]]]
=
None
routed_rule_titles
:
Optional
[
List
[
str
]]
=
None
routed_rule_titles
:
Optional
[
List
[
str
]]
=
None
routed_summary_names
:
Optional
[
List
[
str
]]
=
None
routed_summary_names
:
Optional
[
List
[
str
]]
=
None
file_ext
:
str
file_ext
:
str
...
@@ -262,22 +282,25 @@ def summarize_facts(payload: SegmentSummaryRequest) -> SegmentSummaryResponse:
...
@@ -262,22 +282,25 @@ def summarize_facts(payload: SegmentSummaryRequest) -> SegmentSummaryResponse:
detail
=
f
"Segment text not found for id {payload.segment_id}: {exc}. Please parse document first."
,
detail
=
f
"Segment text not found for id {payload.segment_id}: {exc}. Please parse document first."
,
)
)
ruleset_id
=
payload
.
ruleset_id
or
rules_reference_tool
.
default_ruleset_id
if
payload
.
routed_summary_names
is
not
None
:
if
payload
.
routed_summary_names
is
not
None
:
summary_names
=
{
summary_names
=
{
name
.
strip
()
name
.
strip
()
for
name
in
payload
.
routed_summary_names
for
name
in
payload
.
routed_summary_names
if
isinstance
(
name
,
str
)
and
name
.
strip
()
if
isinstance
(
name
,
str
)
and
name
.
strip
()
}
}
all_rules
=
rules_reference_tool
.
run
(
ruleset_id
=
ruleset_id
)
.
get
(
"rules"
,
[])
all_rules
=
retrieve_rules
(
ruleset_id
=
payload
.
ruleset_id
,
rules
=
payload
.
rules
,
)
.
get
(
"rules"
,
[])
rules
=
[
rules
=
[
rule
rule
for
rule
in
all_rules
for
rule
in
all_rules
if
str
(
rule
.
get
(
"summary"
,
""
))
.
strip
()
in
summary_names
if
str
(
rule
.
get
(
"summary"
,
""
))
.
strip
()
in
summary_names
]
]
else
:
else
:
rules
=
rules_reference_tool
.
run
(
rules
=
retrieve_rules
(
ruleset_id
=
ruleset_id
,
ruleset_id
=
payload
.
ruleset_id
,
rules
=
payload
.
rules
,
routed_rule_titles
=
payload
.
routed_rule_titles
,
routed_rule_titles
=
payload
.
routed_rule_titles
,
)
.
get
(
"rules"
,
[])
)
.
get
(
"rules"
,
[])
result
=
summary_tool
.
run
(
result
=
summary_tool
.
run
(
...
@@ -310,7 +333,8 @@ class SegmentReviewRequest(BaseModel):
...
@@ -310,7 +333,8 @@ class SegmentReviewRequest(BaseModel):
description
=
"segment_id 为 FULL_TEXT_SEGMENT_ID 时用于审查的全文文本"
,
description
=
"segment_id 为 FULL_TEXT_SEGMENT_ID 时用于审查的全文文本"
,
)
)
party_role
:
Optional
[
str
]
=
""
party_role
:
Optional
[
str
]
=
""
ruleset_id
:
Optional
[
str
]
=
"通用"
ruleset_id
:
Optional
[
str
]
=
None
rules
:
Optional
[
List
[
Dict
[
str
,
Any
]]]
=
None
routed_rule_titles
:
Optional
[
List
[
str
]]
=
None
routed_rule_titles
:
Optional
[
List
[
str
]]
=
None
file_ext
:
Optional
[
str
]
=
None
file_ext
:
Optional
[
str
]
=
None
context_memories
:
Optional
[
List
[
Dict
]]
=
None
context_memories
:
Optional
[
List
[
Dict
]]
=
None
...
@@ -380,9 +404,9 @@ def review_segment(payload: SegmentReviewRequest) -> SegmentReviewResponse:
...
@@ -380,9 +404,9 @@ def review_segment(payload: SegmentReviewRequest) -> SegmentReviewResponse:
store
=
get_cached_memory
(
payload
.
conversation_id
)
store
=
get_cached_memory
(
payload
.
conversation_id
)
segment_idx
,
segment_text
,
_
=
_resolve_review_segment
(
payload
)
segment_idx
,
segment_text
,
_
=
_resolve_review_segment
(
payload
)
rules
et_id
=
payload
.
ruleset_id
or
rules_reference_tool
.
default_ruleset_id
rules
=
retrieve_rules
(
rules
=
rules_reference_tool
.
run
(
ruleset_id
=
payload
.
ruleset_id
,
rules
et_id
=
ruleset_id
,
rules
=
payload
.
rules
,
routed_rule_titles
=
payload
.
routed_rule_titles
,
routed_rule_titles
=
payload
.
routed_rule_titles
,
)
.
get
(
"rules"
,
[])
)
.
get
(
"rules"
,
[])
# 暂时不添加摘要看下结果
# 暂时不添加摘要看下结果
...
@@ -435,8 +459,11 @@ def review_segment(payload: SegmentReviewRequest) -> SegmentReviewResponse:
...
@@ -435,8 +459,11 @@ def review_segment(payload: SegmentReviewRequest) -> SegmentReviewResponse:
def
route_segment_rules
(
payload
:
SegmentReviewRequest
)
->
SegmentRuleRouterResponse
:
def
route_segment_rules
(
payload
:
SegmentReviewRequest
)
->
SegmentRuleRouterResponse
:
segment_idx
,
segment_text
,
doc_obj
=
_resolve_review_segment
(
payload
)
segment_idx
,
segment_text
,
doc_obj
=
_resolve_review_segment
(
payload
)
ruleset_id
=
payload
.
ruleset_id
or
rules_reference_tool
.
default_ruleset_id
rules_result
=
retrieve_rules
(
rules
=
rules_reference_tool
.
run
(
ruleset_id
=
ruleset_id
)
.
get
(
"rules"
,
[])
ruleset_id
=
payload
.
ruleset_id
,
rules
=
payload
.
rules
,
)
rules
=
rules_result
.
get
(
"rules"
,
[])
try
:
try
:
total_segments
=
len
(
doc_obj
.
get_chunk_id_list
()
or
[])
total_segments
=
len
(
doc_obj
.
get_chunk_id_list
()
or
[])
...
@@ -470,7 +497,7 @@ def route_segment_rules(payload: SegmentReviewRequest) -> SegmentRuleRouterRespo
...
@@ -470,7 +497,7 @@ def route_segment_rules(payload: SegmentReviewRequest) -> SegmentRuleRouterRespo
return
SegmentRuleRouterResponse
(
return
SegmentRuleRouterResponse
(
conversation_id
=
payload
.
conversation_id
,
conversation_id
=
payload
.
conversation_id
,
segment_id
=
payload
.
segment_id
,
segment_id
=
payload
.
segment_id
,
ruleset_id
=
rules
et_id
,
ruleset_id
=
rules
_result
.
get
(
"ruleset_id"
,
""
)
,
route_by
=
result
.
get
(
"route_by"
,
payload
.
route_by
),
route_by
=
result
.
get
(
"route_by"
,
payload
.
route_by
),
routed_rule_titles
=
routed_rule_titles
,
routed_rule_titles
=
routed_rule_titles
,
routed_summary_names
=
result
.
get
(
"routed_summary_names"
,
[]),
routed_summary_names
=
result
.
get
(
"routed_summary_names"
,
[]),
...
@@ -482,12 +509,14 @@ def route_segment_rules(payload: SegmentReviewRequest) -> SegmentRuleRouterRespo
...
@@ -482,12 +509,14 @@ def route_segment_rules(payload: SegmentReviewRequest) -> SegmentRuleRouterRespo
class
ReflectReviewRequest
(
BaseModel
):
class
ReflectReviewRequest
(
BaseModel
):
conversation_id
:
str
conversation_id
:
str
party_role
:
str
party_role
:
str
ruleset_id
:
Optional
[
str
]
=
"通用"
ruleset_id
:
Optional
[
str
]
=
None
rules
:
Optional
[
List
[
Dict
[
str
,
Any
]]]
=
None
rule_title
:
str
rule_title
:
str
class
RuleStringifyRequest
(
BaseModel
):
class
RuleStringifyRequest
(
BaseModel
):
ruleset_id
:
Optional
[
str
]
=
"通用"
ruleset_id
:
Optional
[
str
]
=
None
rules
:
Optional
[
List
[
Dict
[
str
,
Any
]]]
=
None
rule_title
:
str
rule_title
:
str
...
@@ -530,8 +559,11 @@ class FactsMergerResponse(BaseModel):
...
@@ -530,8 +559,11 @@ class FactsMergerResponse(BaseModel):
@app.post
(
"/rules/stringify"
,
response_model
=
RuleStringifyResponse
)
@app.post
(
"/rules/stringify"
,
response_model
=
RuleStringifyResponse
)
def
stringify_rule
(
payload
:
RuleStringifyRequest
)
->
RuleStringifyResponse
:
def
stringify_rule
(
payload
:
RuleStringifyRequest
)
->
RuleStringifyResponse
:
ruleset_id
=
payload
.
ruleset_id
or
rules_reference_tool
.
default_ruleset_id
rules_result
=
retrieve_rules
(
ruleset_items
=
rules_reference_tool
.
run
(
ruleset_id
=
ruleset_id
)
.
get
(
"rules"
,
[])
ruleset_id
=
payload
.
ruleset_id
,
rules
=
payload
.
rules
,
)
ruleset_items
=
rules_result
.
get
(
"rules"
,
[])
rule
=
next
(
rule
=
next
(
(
r
for
r
in
ruleset_items
if
r
.
get
(
"title"
)
==
payload
.
rule_title
),
None
(
r
for
r
in
ruleset_items
if
r
.
get
(
"title"
)
==
payload
.
rule_title
),
None
)
)
...
@@ -541,7 +573,7 @@ def stringify_rule(payload: RuleStringifyRequest) -> RuleStringifyResponse:
...
@@ -541,7 +573,7 @@ def stringify_rule(payload: RuleStringifyRequest) -> RuleStringifyResponse:
)
)
return
RuleStringifyResponse
(
return
RuleStringifyResponse
(
ruleset_id
=
rules
et_id
,
ruleset_id
=
rules
_result
.
get
(
"ruleset_id"
,
""
)
,
rule_title
=
payload
.
rule_title
,
rule_title
=
payload
.
rule_title
,
rule_string
=
reflect_tool
.
_stringify_rule
(
rule
),
rule_string
=
reflect_tool
.
_stringify_rule
(
rule
),
)
)
...
@@ -550,8 +582,10 @@ def stringify_rule(payload: RuleStringifyRequest) -> RuleStringifyResponse:
...
@@ -550,8 +582,10 @@ def stringify_rule(payload: RuleStringifyRequest) -> RuleStringifyResponse:
@app.post
(
"/segments/review/reflect"
,
response_model
=
ReflectReviewResponse
)
@app.post
(
"/segments/review/reflect"
,
response_model
=
ReflectReviewResponse
)
def
reflect_review
(
payload
:
ReflectReviewRequest
)
->
ReflectReviewResponse
:
def
reflect_review
(
payload
:
ReflectReviewRequest
)
->
ReflectReviewResponse
:
store
=
get_cached_memory
(
payload
.
conversation_id
)
store
=
get_cached_memory
(
payload
.
conversation_id
)
ruleset_id
=
payload
.
ruleset_id
or
rules_reference_tool
.
default_ruleset_id
ruleset_items
=
retrieve_rules
(
ruleset_items
=
rules_reference_tool
.
run
(
ruleset_id
=
ruleset_id
)
.
get
(
"rules"
,
[])
ruleset_id
=
payload
.
ruleset_id
,
rules
=
payload
.
rules
,
)
.
get
(
"rules"
,
[])
rule
=
next
(
rule
=
next
(
(
r
for
r
in
ruleset_items
if
r
.
get
(
"title"
)
==
payload
.
rule_title
),
None
(
r
for
r
in
ruleset_items
if
r
.
get
(
"title"
)
==
payload
.
rule_title
),
None
)
)
...
@@ -799,12 +833,10 @@ def export_facts_memory(payload: FactsMemoryExportRequest) -> FactsMemoryExportR
...
@@ -799,12 +833,10 @@ def export_facts_memory(payload: FactsMemoryExportRequest) -> FactsMemoryExportR
if
__name__
==
"__main__"
:
if
__name__
==
"__main__"
:
from
core.config
import
use_lufa
if
platform
==
"lufa"
or
platform
==
"luchuan"
:
if
use_lufa
:
port
=
18168
port
=
18168
else
:
else
:
port
=
18169
port
=
18169
if
debug_mode
:
if
debug_mode
:
port
+=
10
port
+=
10
uvicorn
.
run
(
"main:app"
,
host
=
"0.0.0.0"
,
port
=
port
,
log_level
=
"info"
,
reload
=
Fals
e
)
uvicorn
.
run
(
"main:app"
,
host
=
"0.0.0.0"
,
port
=
port
,
log_level
=
"info"
,
reload
=
Tru
e
)
requirements.txt
View file @
a76f1b8c
...
@@ -76,3 +76,5 @@ urllib3==2.6.3
...
@@ -76,3 +76,5 @@ urllib3==2.6.3
uvicorn
==0.40.0
uvicorn
==0.40.0
wheel
==0.45.1
wheel
==0.45.1
yarl
==1.22.0
yarl
==1.22.0
spire-doc
==14.6.1
spire-pdf
==12.6.2
\ No newline at end of file
utils/http_util.py
View file @
a76f1b8c
...
@@ -8,7 +8,7 @@ import requests
...
@@ -8,7 +8,7 @@ import requests
from
loguru
import
logger
from
loguru
import
logger
from
requests_toolbelt
import
MultipartEncoder
from
requests_toolbelt
import
MultipartEncoder
from
core.config
import
base_fastgpt_url
,
base_backend_url
,
outer_backend_url
from
core.config
import
base_fastgpt_url
,
base_backend_url
,
outer_backend_url
,
platform
def
fastgpt_openai_chat
(
url
,
token
,
model
,
chat_id
,
file_url
,
text
,
stream
=
True
):
def
fastgpt_openai_chat
(
url
,
token
,
model
,
chat_id
,
file_url
,
text
,
stream
=
True
):
...
@@ -58,7 +58,10 @@ def fastgpt_openai_chat(url, token, model, chat_id, file_url, text, stream=True)
...
@@ -58,7 +58,10 @@ def fastgpt_openai_chat(url, token, model, chat_id, file_url, text, stream=True)
def
upload_file
(
path
,
input_url_to_inner
=
True
,
output_url_to_inner
=
False
)
->
str
:
def
upload_file
(
path
,
input_url_to_inner
=
True
,
output_url_to_inner
=
False
)
->
str
:
# 登录获取token
# 登录获取token
login_data
=
{
"username"
:
"admin"
,
"password"
:
"admin@jpai.com"
}
if
platform
==
"luchuan"
:
login_data
=
{
"username"
:
"chenran"
,
"password"
:
"CR@lgfz2026"
}
else
:
login_data
=
{
"username"
:
"admin"
,
"password"
:
"admin@jpai.com"
}
login_url
=
f
"{base_backend_url}/admin-api/system/auth/login"
login_url
=
f
"{base_backend_url}/admin-api/system/auth/login"
response
=
requests
.
post
(
response
=
requests
.
post
(
url
=
login_url
,
url
=
login_url
,
...
...
utils/spire_word_util.py
View file @
a76f1b8c
...
@@ -4,7 +4,7 @@ import re
...
@@ -4,7 +4,7 @@ import re
from
thefuzz
import
fuzz
from
thefuzz
import
fuzz
from
utils.doc_util
import
DocBase
from
utils.doc_util
import
DocBase
from
utils.common_util
import
adjust_single_chunk_size
from
utils.common_util
import
adjust_single_chunk_size
from
core.config
import
FULL_TEXT_SEGMENT_ID
,
use_lufa
from
core.config
import
FULL_TEXT_SEGMENT_ID
import
os
import
os
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment