Skip to content

ccran / auto-prompt

Go to a project
Commit 37faa701 by ccran
Options

Initial commit

parents
Showing with 579 additions and 0 deletions
.idea/.gitignore 0 → 100644
# Default ignored files
/shelf/
/workspace.xml
# Editor-based HTTP Client requests
/httpRequests/
# Datasource local storage ignored files
/dataSources/
/dataSources.local.xml
.idea/auto-prompt.iml 0 → 100644
<?xml version="1.0" encoding="UTF-8"?>
<module type="PYTHON_MODULE" version="4">
<component name="NewModuleRootManager">
<content url="file://$MODULE_DIR$" />
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
</component>
</module>
\ No newline at end of file
.idea/deployment.xml 0 → 100644
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="PublishConfigData" remoteFilesAllowedToDisappearOnAutoupload="false">
<serverData>
<paths name="ccran@117.157.192.95:33333 password">
<serverdata>
<mappings>
<mapping local="$PROJECT_DIR$" web="/" />
</mappings>
</serverdata>
</paths>
<paths name="天水aidemo">
<serverdata>
<mappings>
<mapping local="$PROJECT_DIR$" web="/" />
</mappings>
</serverdata>
</paths>
<paths name="天水coast">
<serverdata>
<mappings>
<mapping local="$PROJECT_DIR$" web="/" />
</mappings>
</serverdata>
</paths>
<paths name="天水练手verl">
<serverdata>
<mappings>
<mapping local="$PROJECT_DIR$" web="/" />
</mappings>
</serverdata>
</paths>
</serverData>
</component>
</project>
\ No newline at end of file
.idea/inspectionProfiles/Project_Default.xml 0 → 100644
<component name="InspectionProjectProfileManager">
<profile version="1.0">
<option name="myName" value="Project Default" />
<inspection_tool class="Eslint" enabled="true" level="WARNING" enabled_by_default="true" />
<inspection_tool class="PyPackageRequirementsInspection" enabled="true" level="WARNING" enabled_by_default="true">
<option name="ignoredPackages">
<value>
<list size="70">
<item index="0" class="java.lang.String" itemvalue="azure_storage" />
<item index="1" class="java.lang.String" itemvalue="onnxruntime" />
<item index="2" class="java.lang.String" itemvalue="torch" />
<item index="3" class="java.lang.String" itemvalue="openai-whisper" />
<item index="4" class="java.lang.String" itemvalue="torchaudio" />
<item index="5" class="java.lang.String" itemvalue="kaldialign" />
<item index="6" class="java.lang.String" itemvalue="tiktoken" />
<item index="7" class="java.lang.String" itemvalue="whisperspeech" />
<item index="8" class="java.lang.String" itemvalue="faster-whisper" />
<item index="9" class="java.lang.String" itemvalue="braceexpand" />
<item index="10" class="java.lang.String" itemvalue="chromadb" />
<item index="11" class="java.lang.String" itemvalue="httpx" />
<item index="12" class="java.lang.String" itemvalue="alembic" />
<item index="13" class="java.lang.String" itemvalue="rebyte-langchain" />
<item index="14" class="java.lang.String" itemvalue="emoji" />
<item index="15" class="java.lang.String" itemvalue="pgvector" />
<item index="16" class="java.lang.String" itemvalue="SQLAlchemy" />
<item index="17" class="java.lang.String" itemvalue="psycopg2-binary" />
<item index="18" class="java.lang.String" itemvalue="python-dotenv" />
<item index="19" class="java.lang.String" itemvalue="firebase_admin" />
<item index="20" class="java.lang.String" itemvalue="numpy" />
<item index="21" class="java.lang.String" itemvalue="edge-tts" />
<item index="22" class="java.lang.String" itemvalue="aioconsole" />
<item index="23" class="java.lang.String" itemvalue="llama_index" />
<item index="24" class="java.lang.String" itemvalue="langchain" />
<item index="25" class="java.lang.String" itemvalue="starlette" />
<item index="26" class="java.lang.String" itemvalue="anthropic" />
<item index="27" class="java.lang.String" itemvalue="google-cloud-speech" />
<item index="28" class="java.lang.String" itemvalue="beautifulsoup4" />
<item index="29" class="java.lang.String" itemvalue="SpeechRecognition" />
<item index="30" class="java.lang.String" itemvalue="pydantic" />
<item index="31" class="java.lang.String" itemvalue="faster_whisper" />
<item index="32" class="java.lang.String" itemvalue="pytest" />
<item index="33" class="java.lang.String" itemvalue="readerwriterlock" />
<item index="34" class="java.lang.String" itemvalue="pypdf" />
<item index="35" class="java.lang.String" itemvalue="pyaudio" />
<item index="36" class="java.lang.String" itemvalue="openai" />
<item index="37" class="java.lang.String" itemvalue="fastapi" />
<item index="38" class="java.lang.String" itemvalue="twilio" />
<item index="39" class="java.lang.String" itemvalue="transformers" />
<item index="40" class="java.lang.String" itemvalue="chonkie" />
<item index="41" class="java.lang.String" itemvalue="fitz" />
<item index="42" class="java.lang.String" itemvalue="tenacity" />
<item index="43" class="java.lang.String" itemvalue="pymupdf" />
<item index="44" class="java.lang.String" itemvalue="streamlit" />
<item index="45" class="java.lang.String" itemvalue="loguru" />
<item index="46" class="java.lang.String" itemvalue="Requests" />
<item index="47" class="java.lang.String" itemvalue="requests_toolbelt" />
<item index="48" class="java.lang.String" itemvalue="pandas" />
<item index="49" class="java.lang.String" itemvalue="pdf2docx" />
<item index="50" class="java.lang.String" itemvalue="python_docx" />
<item index="51" class="java.lang.String" itemvalue="cn2an" />
<item index="52" class="java.lang.String" itemvalue="pdfminer.six" />
<item index="53" class="java.lang.String" itemvalue="qwen_agent" />
<item index="54" class="java.lang.String" itemvalue="aiohttp" />
<item index="55" class="java.lang.String" itemvalue="uvicorn" />
<item index="56" class="java.lang.String" itemvalue="openpyxl" />
<item index="57" class="java.lang.String" itemvalue="torchdata" />
<item index="58" class="java.lang.String" itemvalue="pre-commit" />
<item index="59" class="java.lang.String" itemvalue="flash-attn" />
<item index="60" class="java.lang.String" itemvalue="ray" />
<item index="61" class="java.lang.String" itemvalue="pybind11" />
<item index="62" class="java.lang.String" itemvalue="hydra-core" />
<item index="63" class="java.lang.String" itemvalue="liger-kernel" />
<item index="64" class="java.lang.String" itemvalue="peft" />
<item index="65" class="java.lang.String" itemvalue="wandb" />
<item index="66" class="java.lang.String" itemvalue="tensordict" />
<item index="67" class="java.lang.String" itemvalue="codetiming" />
<item index="68" class="java.lang.String" itemvalue="pylatexenc" />
<item index="69" class="java.lang.String" itemvalue="thefuzz" />
</list>
</value>
</option>
</inspection_tool>
</profile>
</component>
\ No newline at end of file
.idea/inspectionProfiles/profiles_settings.xml 0 → 100644
<component name="InspectionProjectProfileManager">
<settings>
<option name="USE_PROJECT_PROFILE" value="false" />
<version value="1.0" />
</settings>
</component>
\ No newline at end of file
.idea/misc.xml 0 → 100644
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="Black">
<option name="sdkName" value="D:\Anaconda" />
</component>
<component name="ProjectRootManager" version="2" project-jdk-name="D:\Anaconda" project-jdk-type="Python SDK" />
</project>
\ No newline at end of file
.idea/modules.xml 0 → 100644
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectModuleManager">
<modules>
<module fileurl="file://$PROJECT_DIR$/.idea/auto-prompt.iml" filepath="$PROJECT_DIR$/.idea/auto-prompt.iml" />
</modules>
</component>
</project>
\ No newline at end of file
dataset/train.json 0 → 100644
[
{
"original_text": "8.1.2.6 向甲方支付违约基数30%的违约金。",
"details": "句子主语为乙方",
"result": "不涉及",
"suggest": ""
},
{
"original_text": "8.1.2.2 要求乙方全额退还甲方已支付的预付款项。",
"details": "句子主语为乙方",
"result": "不涉及",
"suggest": ""
},
{
"original_text": "8.1.2.2 2.13.3.5 甲方有权取消该笔订单/合同，乙方需支付违约基数30%的违约金。",
"details": "",
"result": "不合格",
"suggest": ""
}
]
\ No newline at end of file
dataset/val.json 0 → 100644
[
{
"original_text": "8.1.2.6 向甲方支付违约基数30%的违约金。",
"details": "句子主语为乙方",
"result": "不涉及",
"suggest": ""
},
{
"original_text": "8.1.2.2 要求乙方全额退还甲方已支付的预付款项。",
"details": "句子主语为乙方",
"result": "不涉及",
"suggest": ""
},
{
"original_text": "8.1.2.2 2.13.3.5 甲方有权取消该笔订单/合同，乙方需支付违约基数30%的违约金。",
"details": "",
"result": "不合格",
"suggest": ""
}
]
\ No newline at end of file
demo/json-demo.py 0 → 100644
import json
json_list = [
{'original_text': '8.1.2.6 向甲方支付违约基数30%的违约金。', 'details': '句子主语为乙方', 'result': '不涉及',
'suggest': ''},
{'original_text': '8.1.2.2 要求乙方全额退还甲方已支付的预付款项。', 'details': '句子主语为乙方',
'result': '不涉及', 'suggest': ''},
{'original_text': '8.1.2.2 2.13.3.5 甲方有权取消该笔订单/合同，乙方需支付违约基数30%的违约金。',
'details': '',
'result': '不合格', 'suggest': ''}
]
print(json.dumps(json_list, ensure_ascii=False, indent=4))
demo/logs/2025-09-09_09-57-22.jsonl 0 → 100644
This source diff could not be displayed because it is too large. You can view the blob instead.
demo/textgrad-demo.py 0 → 100644
import os
from util import *
os.environ['OLLAMA_BASE_URL'] = 'http://192.168.252.71:9002/v1'
import textgrad as tg
from textgrad.tasks import load_task, Dataset
from textgrad.autograd.string_based_ops import StringBasedFunction
class ContractReviewDataset(Dataset):
def __init__(self):
self.datas = [
{'original_text': '8.1.2.6 向甲方支付违约基数30%的违约金。', 'details': '句子主语为乙方', 'result': '不涉及',
'suggest': ''},
{'original_text': '8.1.2.2 要求乙方全额退还甲方已支付的预付款项。', 'details': '句子主语为乙方',
'result': '不涉及', 'suggest': ''},
{'original_text': '8.1.2.2 2.13.3.5 甲方有权取消该笔订单/合同，乙方需支付违约基数30%的违约金。',
'details': '',
'result': '不合格', 'suggest': ''}
]
def __getitem__(self, index):
row = self.datas[index]
return row["original_text"], row["result"]
def __len__(self):
return len(self.datas)
def load_contract_review_task():
ds = ContractReviewDataset()
return ds, ds, ds, StringBasedFunction(string_based_equality_fn,
function_purpose="The runtime of string-based function that checks if the prediction is correct.")
# init engine
llm_engine = tg.get_engine("ollama-Qwen2-72B-Instruct")
tg.set_backward_engine("ollama-Qwen2-72B-Instruct")
# init datasets
train_set, val_set, _, eval_fn = load_contract_review_task()
train_loader = tg.tasks.DataLoader(train_set, batch_size=3, shuffle=True)
# prediction
init_prompt = '''
你是乙方（供方、卖方）法律部门的合同审查助手
# 审查要点
1）提取涉及到句子的主体为甲方/买方/需方，句子内容为“合同变更/取消”、“退货”相关的句子，没有则返回不涉及
2）句子明确提及了“双方协商”，审查合格
3）句子没有明确提及“合同变更/取消”、“中途退货”所需要承担的责任，审查不合格
3）“合同变更/取消”相关的句子，没有提及违约金额，审查不合格
4）“退货”相关的句子，违约金的比例低于80%，审查不合格
# 不合格建议
1、提醒用户不合规的变更取消责任
# 审查约束
- 输出包括审查的原文、详情、结果、建议
- 审查结果为合格/不合格/不涉及，合格/不涉及的审查结果无需输出建议
- 审查原文严格提取关键、无省略、无篡改的原文内容
- 结果以JSON数组的格式返回,例如```json [{"original_text":"xx","details":"xx","result":"xx","suggest":"xx"}]```
依据审查要点，遵循约束，完成合同审查，提供审查建议，一步步仔细思考。
'''
system_prompt = tg.Variable(init_prompt,
requires_grad=True,
role_description="system prompt to guide the LLM's reasoning strategy for accurate responses")
model = tg.BlackboxLLM(llm_engine, system_prompt=system_prompt)
optimizer = tg.TGD(parameters=list(model.parameters()))
results = {"train_acc": [], "prompt": [], "validation_acc": []}
results["train_acc"].append(eval_dataset(train_set, eval_fn, model))
results["validation_acc"].append(eval_dataset(val_set, eval_fn, model))
results["prompt"].append(system_prompt.get_value())
# 反向传播
for epoch in range(3):
for steps, (batch_x, batch_y) in enumerate((pbar := tqdm(train_loader, position=0))):
pbar.set_description(f"Training step {steps}. Epoch {epoch}")
optimizer.zero_grad()
losses = []
for (x, y) in zip(batch_x, batch_y):
x = tg.Variable(x, requires_grad=False, role_description="query to the language model")
y = tg.Variable(y, requires_grad=False, role_description="correct answer for the query")
response = model(x)
eval_output_variable = eval_fn(inputs=dict(prediction=response, ground_truth_answer=y))
losses.append(eval_output_variable)
total_loss = tg.sum(losses)
total_loss.backward()
optimizer.step()
run_validation_revert(system_prompt, results, model, eval_fn, val_set)
print("sys prompt: ", system_prompt)
results["validation_acc"].append(eval_dataset(val_set, eval_fn, model))
results["prompt"].append(system_prompt.get_value())
logs/2025-09-09_11-31-39.jsonl 0 → 100644
{"name": "textgrad", "msg": "LLMCall function forward", "args": "()", "levelname": "INFO", "levelno": "20", "pathname": "C:\\Users\\chenran\\AppData\\Roaming\\Python\\Python312\\site-packages\\textgrad\\autograd\\llm_ops.py", "filename": "llm_ops.py", "module": "llm_ops", "exc_info": "None", "exc_text": "None", "stack_info": "None", "lineno": "69", "funcName": "forward", "created": "1757388710.4996352", "msecs": "499.0", "relativeCreated": "11522.260665893555", "thread": "82088", "threadName": "ThreadPoolExecutor-2_0", "processName": "MainProcess", "process": "83832", "taskName": "None", "text": "System:你是乙方（供方、卖方）法律部门的合同审查助手\n# 审查要点\n1）提取涉及到句子的主体为甲方/买方/需方，句子内容为“合同变更/取消”、“退货”相关的句子，没有则返回不涉及\n2）句子明确提及了“双方协商”，审查合格\n3）句子没有明确提及“合同变更/取消”、“中途退货”所需要承担的责任，审查不合格\n3）“合同变更/取消”相关的句子，没有提及违约金额，审查不合格\n4）“退货”相关的句子，违约金的比例低于80%，审查不合格\n\n# 不合格建议\n1、提醒用户不合规的变更取消责任\n\n# 审查约束\n- 输出包括审查的原文、详情、结果、建议\n- 审查结果为合格/不合格/不涉及，合格/不涉及的审查结果无需输出建议\n- 审查原文严格提取关键、无省略、无篡改的原文内容\n- 结果以JSON数组的格式返回,例如```json [{\"original_text\":\"xx\",\"details\":\"xx\",\"result\":\"xx\",\"suggest\":\"xx\"}]```\n依据审查要点，遵循约束，完成合同审查，提供审查建议，一步步仔细思考。\nQuery: 8.1.2.6 向甲方支付违约基数30%的违约金。\nResponse: \n\n```json\n[\n {\n \"original_text\": \"8.1.2.6 向甲方支付违约基数30%的违约金。\",\n \"details\": \"该条款未明确提及'合同变更/取消'或'退货'相关表述，且未说明违约金对应的违约行为类型（如中途退货、合同变更等）。违约金比例为30%，但缺乏具体适用场景的关联性描述。\",\n \"result\": \"不涉及\",\n \"suggest\": \"\"\n }\n]\n```", "message": "LLMCall function forward"}
{"name": "textgrad", "msg": "LLMCall function forward", "args": "()", "levelname": "INFO", "levelno": "20", "pathname": "C:\\Users\\chenran\\AppData\\Roaming\\Python\\Python312\\site-packages\\textgrad\\autograd\\llm_ops.py", "filename": "llm_ops.py", "module": "llm_ops", "exc_info": "None", "exc_text": "None", "stack_info": "None", "lineno": "69", "funcName": "forward", "created": "1757388710.4996352", "msecs": "499.0", "relativeCreated": "11522.260665893555", "thread": "90772", "threadName": "ThreadPoolExecutor-2_1", "processName": "MainProcess", "process": "83832", "taskName": "None", "text": "System:你是乙方（供方、卖方）法律部门的合同审查助手\n# 审查要点\n1）提取涉及到句子的主体为甲方/买方/需方，句子内容为“合同变更/取消”、“退货”相关的句子，没有则返回不涉及\n2）句子明确提及了“双方协商”，审查合格\n3）句子没有明确提及“合同变更/取消”、“中途退货”所需要承担的责任，审查不合格\n3）“合同变更/取消”相关的句子，没有提及违约金额，审查不合格\n4）“退货”相关的句子，违约金的比例低于80%，审查不合格\n\n# 不合格建议\n1、提醒用户不合规的变更取消责任\n\n# 审查约束\n- 输出包括审查的原文、详情、结果、建议\n- 审查结果为合格/不合格/不涉及，合格/不涉及的审查结果无需输出建议\n- 审查原文严格提取关键、无省略、无篡改的原文内容\n- 结果以JSON数组的格式返回,例如```json [{\"original_text\":\"xx\",\"details\":\"xx\",\"result\":\"xx\",\"suggest\":\"xx\"}]```\n依据审查要点，遵循约束，完成合同审查，提供审查建议，一步步仔细思考。\nQuery: 8.1.2.2 要求乙方全额退还甲方已支付的预付款项。\nResponse: \n\n```json\n[\n {\n \"original_text\": \"8.1.2.2 要求乙方全额退还甲方已支付的预付款项。\",\n \"details\": \"该条款涉及甲方要求退还预付款，但未明确提及'双方协商'，且未说明合同变更/取消或中途退货所需承担的责任，也未约定违约金额或违约金比例。\",\n \"result\": \"不合格\",\n \"suggest\": \"建议补充'双方协商一致'的表述，并明确合同变更/取消或中途退货时的责任承担方式及违约金比例（若涉及退货且违约金比例低于80%需调整）。\"\n }\n]\n```", "message": "LLMCall function forward"}
{"name": "textgrad", "msg": "LLMCall function forward", "args": "()", "levelname": "INFO", "levelno": "20", "pathname": "C:\\Users\\chenran\\AppData\\Roaming\\Python\\Python312\\site-packages\\textgrad\\autograd\\llm_ops.py", "filename": "llm_ops.py", "module": "llm_ops", "exc_info": "None", "exc_text": "None", "stack_info": "None", "lineno": "69", "funcName": "forward", "created": "1757388710.4996352", "msecs": "499.0", "relativeCreated": "11522.260665893555", "thread": "32384", "threadName": "ThreadPoolExecutor-2_2", "processName": "MainProcess", "process": "83832", "taskName": "None", "text": "System:你是乙方（供方、卖方）法律部门的合同审查助手\n# 审查要点\n1）提取涉及到句子的主体为甲方/买方/需方，句子内容为“合同变更/取消”、“退货”相关的句子，没有则返回不涉及\n2）句子明确提及了“双方协商”，审查合格\n3）句子没有明确提及“合同变更/取消”、“中途退货”所需要承担的责任，审查不合格\n3）“合同变更/取消”相关的句子，没有提及违约金额，审查不合格\n4）“退货”相关的句子，违约金的比例低于80%，审查不合格\n\n# 不合格建议\n1、提醒用户不合规的变更取消责任\n\n# 审查约束\n- 输出包括审查的原文、详情、结果、建议\n- 审查结果为合格/不合格/不涉及，合格/不涉及的审查结果无需输出建议\n- 审查原文严格提取关键、无省略、无篡改的原文内容\n- 结果以JSON数组的格式返回,例如```json [{\"original_text\":\"xx\",\"details\":\"xx\",\"result\":\"xx\",\"suggest\":\"xx\"}]```\n依据审查要点，遵循约束，完成合同审查，提供审查建议，一步步仔细思考。\nQuery: 8.1.2.2 2.13.3.5 甲方有权取消该笔订单/合同，乙方需支付违约基数30%的违约金。\nResponse: \n\n```json\n[\n {\n \"original_text\": \"甲方有权取消该笔订单/合同，乙方需支付违约基数30%的违约金。\",\n \"details\": \"条款涉及甲方单方取消合同的权利，且明确乙方需支付违约基数30%的违约金。但未提及'双方协商'，且违约金比例（30%）未达到退货条款要求的80%标准。\",\n \"result\": \"不合格\",\n \"suggest\": \"建议补充'双方协商'条款，并明确合同取消/变更需经协商一致。若涉及退货条款，需将违约金比例调整至80%以上。\"\n }\n]\n```", "message": "LLMCall function forward"}
{"name": "textgrad", "msg": "StringBasedFunction", "args": "()", "levelname": "INFO", "levelno": "20", "pathname": "C:\\Users\\chenran\\AppData\\Roaming\\Python\\Python312\\site-packages\\textgrad\\autograd\\string_based_ops.py", "filename": "string_based_ops.py", "module": "string_based_ops", "exc_info": "None", "exc_text": "None", "stack_info": "None", "lineno": "79", "funcName": "forward", "created": "1757388710.4996352", "msecs": "499.0", "relativeCreated": "11522.260665893555", "thread": "82088", "threadName": "ThreadPoolExecutor-2_0", "processName": "MainProcess", "process": "83832", "taskName": "None", "text": "In: {'prediction': Variable(value=\n\n```json\n[\n {\n \"original_text\": \"8.1.2.6 向甲方支付违约基数30%的违约金。\",\n \"details\": \"该条款未明确提及'合同变更/取消'或'退货'相关表述，且未说明违约金对应的违约行为类型（如中途退货、合同变更等）。违约金比例为30%，但缺乏具体适用场景的关联性描述。\",\n \"result\": \"不涉及\",\n \"suggest\": \"\"\n }\n]\n```, role=response from the language model, grads=set()), 'ground_truth_answer': Variable(value=不涉及, role=correct answer for the query, grads=set())}, Out: 正确", "message": "StringBasedFunction"}
{"name": "textgrad", "msg": "StringBasedFunction", "args": "()", "levelname": "INFO", "levelno": "20", "pathname": "C:\\Users\\chenran\\AppData\\Roaming\\Python\\Python312\\site-packages\\textgrad\\autograd\\string_based_ops.py", "filename": "string_based_ops.py", "module": "string_based_ops", "exc_info": "None", "exc_text": "None", "stack_info": "None", "lineno": "79", "funcName": "forward", "created": "1757388710.5006971", "msecs": "500.0", "relativeCreated": "11523.322582244873", "thread": "90772", "threadName": "ThreadPoolExecutor-2_1", "processName": "MainProcess", "process": "83832", "taskName": "None", "text": "In: {'prediction': Variable(value=\n\n```json\n[\n {\n \"original_text\": \"8.1.2.2 要求乙方全额退还甲方已支付的预付款项。\",\n \"details\": \"该条款涉及甲方要求退还预付款，但未明确提及'双方协商'，且未说明合同变更/取消或中途退货所需承担的责任，也未约定违约金额或违约金比例。\",\n \"result\": \"不合格\",\n \"suggest\": \"建议补充'双方协商一致'的表述，并明确合同变更/取消或中途退货时的责任承担方式及违约金比例（若涉及退货且违约金比例低于80%需调整）。\"\n }\n]\n```, role=response from the language model, grads=set()), 'ground_truth_answer': Variable(value=不涉及, role=correct answer for the query, grads=set())}, Out: 错误", "message": "StringBasedFunction"}
{"name": "textgrad", "msg": "StringBasedFunction", "args": "()", "levelname": "INFO", "levelno": "20", "pathname": "C:\\Users\\chenran\\AppData\\Roaming\\Python\\Python312\\site-packages\\textgrad\\autograd\\string_based_ops.py", "filename": "string_based_ops.py", "module": "string_based_ops", "exc_info": "None", "exc_text": "None", "stack_info": "None", "lineno": "79", "funcName": "forward", "created": "1757388710.5006971", "msecs": "500.0", "relativeCreated": "11523.322582244873", "thread": "32384", "threadName": "ThreadPoolExecutor-2_2", "processName": "MainProcess", "process": "83832", "taskName": "None", "text": "In: {'prediction': Variable(value=\n\n```json\n[\n {\n \"original_text\": \"甲方有权取消该笔订单/合同，乙方需支付违约基数30%的违约金。\",\n \"details\": \"条款涉及甲方单方取消合同的权利，且明确乙方需支付违约基数30%的违约金。但未提及'双方协商'，且违约金比例（30%）未达到退货条款要求的80%标准。\",\n \"result\": \"不合格\",\n \"suggest\": \"建议补充'双方协商'条款，并明确合同取消/变更需经协商一致。若涉及退货条款，需将违约金比例调整至80%以上。\"\n }\n]\n```, role=response from the language model, grads=set()), 'ground_truth_answer': Variable(value=不合格, role=correct answer for the query, grads=set())}, Out: 正确", "message": "StringBasedFunction"}
{"name": "textgrad", "msg": "LLMCall function forward", "args": "()", "levelname": "INFO", "levelno": "20", "pathname": "C:\\Users\\chenran\\AppData\\Roaming\\Python\\Python312\\site-packages\\textgrad\\autograd\\llm_ops.py", "filename": "llm_ops.py", "module": "llm_ops", "exc_info": "None", "exc_text": "None", "stack_info": "None", "lineno": "69", "funcName": "forward", "created": "1757388710.5116394", "msecs": "511.0", "relativeCreated": "11534.26480293274", "thread": "96884", "threadName": "ThreadPoolExecutor-3_0", "processName": "MainProcess", "process": "83832", "taskName": "None", "text": "System:你是乙方（供方、卖方）法律部门的合同审查助手\n# 审查要点\n1）提取涉及到句子的主体为甲方/买方/需方，句子内容为“合同变更/取消”、“退货”相关的句子，没有则返回不涉及\n2）句子明确提及了“双方协商”，审查合格\n3）句子没有明确提及“合同变更/取消”、“中途退货”所需要承担的责任，审查不合格\n3）“合同变更/取消”相关的句子，没有提及违约金额，审查不合格\n4）“退货”相关的句子，违约金的比例低于80%，审查不合格\n\n# 不合格建议\n1、提醒用户不合规的变更取消责任\n\n# 审查约束\n- 输出包括审查的原文、详情、结果、建议\n- 审查结果为合格/不合格/不涉及，合格/不涉及的审查结果无需输出建议\n- 审查原文严格提取关键、无省略、无篡改的原文内容\n- 结果以JSON数组的格式返回,例如```json [{\"original_text\":\"xx\",\"details\":\"xx\",\"result\":\"xx\",\"suggest\":\"xx\"}]```\n依据审查要点，遵循约束，完成合同审查，提供审查建议，一步步仔细思考。\nQuery: 8.1.2.6 向甲方支付违约基数30%的违约金。\nResponse: \n\n```json\n[\n {\n \"original_text\": \"8.1.2.6 向甲方支付违约基数30%的违约金。\",\n \"details\": \"该条款未明确提及'合同变更/取消'或'退货'相关表述，且未说明违约金对应的违约行为类型（如中途退货、合同变更等）。违约金比例为30%，但缺乏具体适用场景的关联性描述。\",\n \"result\": \"不涉及\",\n \"suggest\": \"\"\n }\n]\n```", "message": "LLMCall function forward"}
{"name": "textgrad", "msg": "StringBasedFunction", "args": "()", "levelname": "INFO", "levelno": "20", "pathname": "C:\\Users\\chenran\\AppData\\Roaming\\Python\\Python312\\site-packages\\textgrad\\autograd\\string_based_ops.py", "filename": "string_based_ops.py", "module": "string_based_ops", "exc_info": "None", "exc_text": "None", "stack_info": "None", "lineno": "79", "funcName": "forward", "created": "1757388710.5116394", "msecs": "511.0", "relativeCreated": "11534.26480293274", "thread": "96884", "threadName": "ThreadPoolExecutor-3_0", "processName": "MainProcess", "process": "83832", "taskName": "None", "text": "In: {'prediction': Variable(value=\n\n```json\n[\n {\n \"original_text\": \"8.1.2.6 向甲方支付违约基数30%的违约金。\",\n \"details\": \"该条款未明确提及'合同变更/取消'或'退货'相关表述，且未说明违约金对应的违约行为类型（如中途退货、合同变更等）。违约金比例为30%，但缺乏具体适用场景的关联性描述。\",\n \"result\": \"不涉及\",\n \"suggest\": \"\"\n }\n]\n```, role=response from the language model, grads=set()), 'ground_truth_answer': Variable(value=不涉及, role=correct answer for the query, grads=set())}, Out: 正确", "message": "StringBasedFunction"}
{"name": "textgrad", "msg": "LLMCall function forward", "args": "()", "levelname": "INFO", "levelno": "20", "pathname": "C:\\Users\\chenran\\AppData\\Roaming\\Python\\Python312\\site-packages\\textgrad\\autograd\\llm_ops.py", "filename": "llm_ops.py", "module": "llm_ops", "exc_info": "None", "exc_text": "None", "stack_info": "None", "lineno": "69", "funcName": "forward", "created": "1757388710.5127144", "msecs": "512.0", "relativeCreated": "11535.339832305908", "thread": "97824", "threadName": "ThreadPoolExecutor-3_1", "processName": "MainProcess", "process": "83832", "taskName": "None", "text": "System:你是乙方（供方、卖方）法律部门的合同审查助手\n# 审查要点\n1）提取涉及到句子的主体为甲方/买方/需方，句子内容为“合同变更/取消”、“退货”相关的句子，没有则返回不涉及\n2）句子明确提及了“双方协商”，审查合格\n3）句子没有明确提及“合同变更/取消”、“中途退货”所需要承担的责任，审查不合格\n3）“合同变更/取消”相关的句子，没有提及违约金额，审查不合格\n4）“退货”相关的句子，违约金的比例低于80%，审查不合格\n\n# 不合格建议\n1、提醒用户不合规的变更取消责任\n\n# 审查约束\n- 输出包括审查的原文、详情、结果、建议\n- 审查结果为合格/不合格/不涉及，合格/不涉及的审查结果无需输出建议\n- 审查原文严格提取关键、无省略、无篡改的原文内容\n- 结果以JSON数组的格式返回,例如```json [{\"original_text\":\"xx\",\"details\":\"xx\",\"result\":\"xx\",\"suggest\":\"xx\"}]```\n依据审查要点，遵循约束，完成合同审查，提供审查建议，一步步仔细思考。\nQuery: 8.1.2.2 要求乙方全额退还甲方已支付的预付款项。\nResponse: \n\n```json\n[\n {\n \"original_text\": \"8.1.2.2 要求乙方全额退还甲方已支付的预付款项。\",\n \"details\": \"该条款涉及甲方要求退还预付款，但未明确提及'双方协商'，且未说明合同变更/取消或中途退货所需承担的责任，也未约定违约金额或违约金比例。\",\n \"result\": \"不合格\",\n \"suggest\": \"建议补充'双方协商一致'的表述，并明确合同变更/取消或中途退货时的责任承担方式及违约金比例（若涉及退货且违约金比例低于80%需调整）。\"\n }\n]\n```", "message": "LLMCall function forward"}
{"name": "textgrad", "msg": "LLMCall function forward", "args": "()", "levelname": "INFO", "levelno": "20", "pathname": "C:\\Users\\chenran\\AppData\\Roaming\\Python\\Python312\\site-packages\\textgrad\\autograd\\llm_ops.py", "filename": "llm_ops.py", "module": "llm_ops", "exc_info": "None", "exc_text": "None", "stack_info": "None", "lineno": "69", "funcName": "forward", "created": "1757388710.5127144", "msecs": "512.0", "relativeCreated": "11535.339832305908", "thread": "93036", "threadName": "ThreadPoolExecutor-3_2", "processName": "MainProcess", "process": "83832", "taskName": "None", "text": "System:你是乙方（供方、卖方）法律部门的合同审查助手\n# 审查要点\n1）提取涉及到句子的主体为甲方/买方/需方，句子内容为“合同变更/取消”、“退货”相关的句子，没有则返回不涉及\n2）句子明确提及了“双方协商”，审查合格\n3）句子没有明确提及“合同变更/取消”、“中途退货”所需要承担的责任，审查不合格\n3）“合同变更/取消”相关的句子，没有提及违约金额，审查不合格\n4）“退货”相关的句子，违约金的比例低于80%，审查不合格\n\n# 不合格建议\n1、提醒用户不合规的变更取消责任\n\n# 审查约束\n- 输出包括审查的原文、详情、结果、建议\n- 审查结果为合格/不合格/不涉及，合格/不涉及的审查结果无需输出建议\n- 审查原文严格提取关键、无省略、无篡改的原文内容\n- 结果以JSON数组的格式返回,例如```json [{\"original_text\":\"xx\",\"details\":\"xx\",\"result\":\"xx\",\"suggest\":\"xx\"}]```\n依据审查要点，遵循约束，完成合同审查，提供审查建议，一步步仔细思考。\nQuery: 8.1.2.2 2.13.3.5 甲方有权取消该笔订单/合同，乙方需支付违约基数30%的违约金。\nResponse: \n\n```json\n[\n {\n \"original_text\": \"甲方有权取消该笔订单/合同，乙方需支付违约基数30%的违约金。\",\n \"details\": \"条款涉及甲方单方取消合同的权利，且明确乙方需支付违约基数30%的违约金。但未提及'双方协商'，且违约金比例（30%）未达到退货条款要求的80%标准。\",\n \"result\": \"不合格\",\n \"suggest\": \"建议补充'双方协商'条款，并明确合同取消/变更需经协商一致。若涉及退货条款，需将违约金比例调整至80%以上。\"\n }\n]\n```", "message": "LLMCall function forward"}
{"name": "textgrad", "msg": "StringBasedFunction", "args": "()", "levelname": "INFO", "levelno": "20", "pathname": "C:\\Users\\chenran\\AppData\\Roaming\\Python\\Python312\\site-packages\\textgrad\\autograd\\string_based_ops.py", "filename": "string_based_ops.py", "module": "string_based_ops", "exc_info": "None", "exc_text": "None", "stack_info": "None", "lineno": "79", "funcName": "forward", "created": "1757388710.5127144", "msecs": "512.0", "relativeCreated": "11535.339832305908", "thread": "97824", "threadName": "ThreadPoolExecutor-3_1", "processName": "MainProcess", "process": "83832", "taskName": "None", "text": "In: {'prediction': Variable(value=\n\n```json\n[\n {\n \"original_text\": \"8.1.2.2 要求乙方全额退还甲方已支付的预付款项。\",\n \"details\": \"该条款涉及甲方要求退还预付款，但未明确提及'双方协商'，且未说明合同变更/取消或中途退货所需承担的责任，也未约定违约金额或违约金比例。\",\n \"result\": \"不合格\",\n \"suggest\": \"建议补充'双方协商一致'的表述，并明确合同变更/取消或中途退货时的责任承担方式及违约金比例（若涉及退货且违约金比例低于80%需调整）。\"\n }\n]\n```, role=response from the language model, grads=set()), 'ground_truth_answer': Variable(value=不涉及, role=correct answer for the query, grads=set())}, Out: 错误", "message": "StringBasedFunction"}
{"name": "textgrad", "msg": "StringBasedFunction", "args": "()", "levelname": "INFO", "levelno": "20", "pathname": "C:\\Users\\chenran\\AppData\\Roaming\\Python\\Python312\\site-packages\\textgrad\\autograd\\string_based_ops.py", "filename": "string_based_ops.py", "module": "string_based_ops", "exc_info": "None", "exc_text": "None", "stack_info": "None", "lineno": "79", "funcName": "forward", "created": "1757388710.5127144", "msecs": "512.0", "relativeCreated": "11535.339832305908", "thread": "93036", "threadName": "ThreadPoolExecutor-3_2", "processName": "MainProcess", "process": "83832", "taskName": "None", "text": "In: {'prediction': Variable(value=\n\n```json\n[\n {\n \"original_text\": \"甲方有权取消该笔订单/合同，乙方需支付违约基数30%的违约金。\",\n \"details\": \"条款涉及甲方单方取消合同的权利，且明确乙方需支付违约基数30%的违约金。但未提及'双方协商'，且违约金比例（30%）未达到退货条款要求的80%标准。\",\n \"result\": \"不合格\",\n \"suggest\": \"建议补充'双方协商'条款，并明确合同取消/变更需经协商一致。若涉及退货条款，需将违约金比例调整至80%以上。\"\n }\n]\n```, role=response from the language model, grads=set()), 'ground_truth_answer': Variable(value=不合格, role=correct answer for the query, grads=set())}, Out: 正确", "message": "StringBasedFunction"}
{"name": "textgrad", "msg": "LLMCall function forward", "args": "()", "levelname": "INFO", "levelno": "20", "pathname": "C:\\Users\\chenran\\AppData\\Roaming\\Python\\Python312\\site-packages\\textgrad\\autograd\\llm_ops.py", "filename": "llm_ops.py", "module": "llm_ops", "exc_info": "None", "exc_text": "None", "stack_info": "None", "lineno": "69", "funcName": "forward", "created": "1757388710.5147462", "msecs": "514.0", "relativeCreated": "11537.371635437012", "thread": "88636", "threadName": "MainThread", "processName": "MainProcess", "process": "83832", "taskName": "None", "text": "System:你是乙方（供方、卖方）法律部门的合同审查助手\n# 审查要点\n1）提取涉及到句子的主体为甲方/买方/需方，句子内容为“合同变更/取消”、“退货”相关的句子，没有则返回不涉及\n2）句子明确提及了“双方协商”，审查合格\n3）句子没有明确提及“合同变更/取消”、“中途退货”所需要承担的责任，审查不合格\n3）“合同变更/取消”相关的句子，没有提及违约金额，审查不合格\n4）“退货”相关的句子，违约金的比例低于80%，审查不合格\n\n# 不合格建议\n1、提醒用户不合规的变更取消责任\n\n# 审查约束\n- 输出包括审查的原文、详情、结果、建议\n- 审查结果为合格/不合格/不涉及，合格/不涉及的审查结果无需输出建议\n- 审查原文严格提取关键、无省略、无篡改的原文内容\n- 结果以JSON数组的格式返回,例如```json [{\"original_text\":\"xx\",\"details\":\"xx\",\"result\":\"xx\",\"suggest\":\"xx\"}]```\n依据审查要点，遵循约束，完成合同审查，提供审查建议，一步步仔细思考。\nQuery: 8.1.2.2 2.13.3.5 甲方有权取消该笔订单/合同，乙方需支付违约基数30%的违约金。\nResponse: \n\n```json\n[\n {\n \"original_text\": \"甲方有权取消该笔订单/合同，乙方需支付违约基数30%的违约金。\",\n \"details\": \"条款涉及甲方单方取消合同的权利，且明确乙方需支付违约基数30%的违约金。但未提及'双方协商'，且违约金比例（30%）未达到退货条款要求的80%标准。\",\n \"result\": \"不合格\",\n \"suggest\": \"建议补充'双方协商'条款，并明确合同取消/变更需经协商一致。若涉及退货条款，需将违约金比例调整至80%以上。\"\n }\n]\n```", "message": "LLMCall function forward"}
{"name": "textgrad", "msg": "StringBasedFunction", "args": "()", "levelname": "INFO", "levelno": "20", "pathname": "C:\\Users\\chenran\\AppData\\Roaming\\Python\\Python312\\site-packages\\textgrad\\autograd\\string_based_ops.py", "filename": "string_based_ops.py", "module": "string_based_ops", "exc_info": "None", "exc_text": "None", "stack_info": "None", "lineno": "79", "funcName": "forward", "created": "1757388710.5158186", "msecs": "515.0", "relativeCreated": "11538.44404220581", "thread": "88636", "threadName": "MainThread", "processName": "MainProcess", "process": "83832", "taskName": "None", "text": "In: {'prediction': Variable(value=\n\n```json\n[\n {\n \"original_text\": \"甲方有权取消该笔订单/合同，乙方需支付违约基数30%的违约金。\",\n \"details\": \"条款涉及甲方单方取消合同的权利，且明确乙方需支付违约基数30%的违约金。但未提及'双方协商'，且违约金比例（30%）未达到退货条款要求的80%标准。\",\n \"result\": \"不合格\",\n \"suggest\": \"建议补充'双方协商'条款，并明确合同取消/变更需经协商一致。若涉及退货条款，需将违约金比例调整至80%以上。\"\n }\n]\n```, role=response from the language model, grads=set()), 'ground_truth_answer': Variable(value=不合格, role=correct answer for the query, grads=set())}, Out: 正确", "message": "StringBasedFunction"}
{"name": "textgrad", "msg": "LLMCall function forward", "args": "()", "levelname": "INFO", "levelno": "20", "pathname": "C:\\Users\\chenran\\AppData\\Roaming\\Python\\Python312\\site-packages\\textgrad\\autograd\\llm_ops.py", "filename": "llm_ops.py", "module": "llm_ops", "exc_info": "None", "exc_text": "None", "stack_info": "None", "lineno": "69", "funcName": "forward", "created": "1757388710.5158186", "msecs": "515.0", "relativeCreated": "11538.44404220581", "thread": "88636", "threadName": "MainThread", "processName": "MainProcess", "process": "83832", "taskName": "None", "text": "System:你是乙方（供方、卖方）法律部门的合同审查助手\n# 审查要点\n1）提取涉及到句子的主体为甲方/买方/需方，句子内容为“合同变更/取消”、“退货”相关的句子，没有则返回不涉及\n2）句子明确提及了“双方协商”，审查合格\n3）句子没有明确提及“合同变更/取消”、“中途退货”所需要承担的责任，审查不合格\n3）“合同变更/取消”相关的句子，没有提及违约金额，审查不合格\n4）“退货”相关的句子，违约金的比例低于80%，审查不合格\n\n# 不合格建议\n1、提醒用户不合规的变更取消责任\n\n# 审查约束\n- 输出包括审查的原文、详情、结果、建议\n- 审查结果为合格/不合格/不涉及，合格/不涉及的审查结果无需输出建议\n- 审查原文严格提取关键、无省略、无篡改的原文内容\n- 结果以JSON数组的格式返回,例如```json [{\"original_text\":\"xx\",\"details\":\"xx\",\"result\":\"xx\",\"suggest\":\"xx\"}]```\n依据审查要点，遵循约束，完成合同审查，提供审查建议，一步步仔细思考。\nQuery: 8.1.2.2 要求乙方全额退还甲方已支付的预付款项。\nResponse: \n\n```json\n[\n {\n \"original_text\": \"8.1.2.2 要求乙方全额退还甲方已支付的预付款项。\",\n \"details\": \"该条款涉及甲方要求退还预付款，但未明确提及'双方协商'，且未说明合同变更/取消或中途退货所需承担的责任，也未约定违约金额或违约金比例。\",\n \"result\": \"不合格\",\n \"suggest\": \"建议补充'双方协商一致'的表述，并明确合同变更/取消或中途退货时的责任承担方式及违约金比例（若涉及退货且违约金比例低于80%需调整）。\"\n }\n]\n```", "message": "LLMCall function forward"}
{"name": "textgrad", "msg": "StringBasedFunction", "args": "()", "levelname": "INFO", "levelno": "20", "pathname": "C:\\Users\\chenran\\AppData\\Roaming\\Python\\Python312\\site-packages\\textgrad\\autograd\\string_based_ops.py", "filename": "string_based_ops.py", "module": "string_based_ops", "exc_info": "None", "exc_text": "None", "stack_info": "None", "lineno": "79", "funcName": "forward", "created": "1757388710.5158186", "msecs": "515.0", "relativeCreated": "11538.44404220581", "thread": "88636", "threadName": "MainThread", "processName": "MainProcess", "process": "83832", "taskName": "None", "text": "In: {'prediction': Variable(value=\n\n```json\n[\n {\n \"original_text\": \"8.1.2.2 要求乙方全额退还甲方已支付的预付款项。\",\n \"details\": \"该条款涉及甲方要求退还预付款，但未明确提及'双方协商'，且未说明合同变更/取消或中途退货所需承担的责任，也未约定违约金额或违约金比例。\",\n \"result\": \"不合格\",\n \"suggest\": \"建议补充'双方协商一致'的表述，并明确合同变更/取消或中途退货时的责任承担方式及违约金比例（若涉及退货且违约金比例低于80%需调整）。\"\n }\n]\n```, role=response from the language model, grads=set()), 'ground_truth_answer': Variable(value=不涉及, role=correct answer for the query, grads=set())}, Out: 错误", "message": "StringBasedFunction"}
{"name": "textgrad", "msg": "LLMCall function forward", "args": "()", "levelname": "INFO", "levelno": "20", "pathname": "C:\\Users\\chenran\\AppData\\Roaming\\Python\\Python312\\site-packages\\textgrad\\autograd\\llm_ops.py", "filename": "llm_ops.py", "module": "llm_ops", "exc_info": "None", "exc_text": "None", "stack_info": "None", "lineno": "69", "funcName": "forward", "created": "1757388710.5158186", "msecs": "515.0", "relativeCreated": "11538.44404220581", "thread": "88636", "threadName": "MainThread", "processName": "MainProcess", "process": "83832", "taskName": "None", "text": "System:你是乙方（供方、卖方）法律部门的合同审查助手\n# 审查要点\n1）提取涉及到句子的主体为甲方/买方/需方，句子内容为“合同变更/取消”、“退货”相关的句子，没有则返回不涉及\n2）句子明确提及了“双方协商”，审查合格\n3）句子没有明确提及“合同变更/取消”、“中途退货”所需要承担的责任，审查不合格\n3）“合同变更/取消”相关的句子，没有提及违约金额，审查不合格\n4）“退货”相关的句子，违约金的比例低于80%，审查不合格\n\n# 不合格建议\n1、提醒用户不合规的变更取消责任\n\n# 审查约束\n- 输出包括审查的原文、详情、结果、建议\n- 审查结果为合格/不合格/不涉及，合格/不涉及的审查结果无需输出建议\n- 审查原文严格提取关键、无省略、无篡改的原文内容\n- 结果以JSON数组的格式返回,例如```json [{\"original_text\":\"xx\",\"details\":\"xx\",\"result\":\"xx\",\"suggest\":\"xx\"}]```\n依据审查要点，遵循约束，完成合同审查，提供审查建议，一步步仔细思考。\nQuery: 8.1.2.6 向甲方支付违约基数30%的违约金。\nResponse: \n\n```json\n[\n {\n \"original_text\": \"8.1.2.6 向甲方支付违约基数30%的违约金。\",\n \"details\": \"该条款未明确提及'合同变更/取消'或'退货'相关表述，且未说明违约金对应的违约行为类型（如中途退货、合同变更等）。违约金比例为30%，但缺乏具体适用场景的关联性描述。\",\n \"result\": \"不涉及\",\n \"suggest\": \"\"\n }\n]\n```", "message": "LLMCall function forward"}
{"name": "textgrad", "msg": "StringBasedFunction", "args": "()", "levelname": "INFO", "levelno": "20", "pathname": "C:\\Users\\chenran\\AppData\\Roaming\\Python\\Python312\\site-packages\\textgrad\\autograd\\string_based_ops.py", "filename": "string_based_ops.py", "module": "string_based_ops", "exc_info": "None", "exc_text": "None", "stack_info": "None", "lineno": "79", "funcName": "forward", "created": "1757388710.5168757", "msecs": "516.0", "relativeCreated": "11539.501190185547", "thread": "88636", "threadName": "MainThread", "processName": "MainProcess", "process": "83832", "taskName": "None", "text": "In: {'prediction': Variable(value=\n\n```json\n[\n {\n \"original_text\": \"8.1.2.6 向甲方支付违约基数30%的违约金。\",\n \"details\": \"该条款未明确提及'合同变更/取消'或'退货'相关表述，且未说明违约金对应的违约行为类型（如中途退货、合同变更等）。违约金比例为30%，但缺乏具体适用场景的关联性描述。\",\n \"result\": \"不涉及\",\n \"suggest\": \"\"\n }\n]\n```, role=response from the language model, grads=set()), 'ground_truth_answer': Variable(value=不涉及, role=correct answer for the query, grads=set())}, Out: 正确", "message": "StringBasedFunction"}
{"name": "textgrad", "msg": "Idempotent backward", "args": "()", "levelname": "INFO", "levelno": "20", "pathname": "C:\\Users\\chenran\\AppData\\Roaming\\Python\\Python312\\site-packages\\textgrad\\autograd\\algebra.py", "filename": "algebra.py", "module": "algebra", "exc_info": "None", "exc_text": "None", "stack_info": "None", "lineno": "89", "funcName": "backward", "created": "1757388710.5168757", "msecs": "516.0", "relativeCreated": "11539.501190185547", "thread": "88636", "threadName": "MainThread", "processName": "MainProcess", "process": "83832", "taskName": "None", "v_gradient_value": "", "summation_role": "a combination of the following: Output of the string-based function with purpose: The runtime of string-based function that checks if the prediction is correct.", "message": "Idempotent backward"}
{"name": "textgrad", "msg": "Idempotent backward", "args": "()", "levelname": "INFO", "levelno": "20", "pathname": "C:\\Users\\chenran\\AppData\\Roaming\\Python\\Python312\\site-packages\\textgrad\\autograd\\algebra.py", "filename": "algebra.py", "module": "algebra", "exc_info": "None", "exc_text": "None", "stack_info": "None", "lineno": "89", "funcName": "backward", "created": "1757388710.5168757", "msecs": "516.0", "relativeCreated": "11539.501190185547", "thread": "88636", "threadName": "MainThread", "processName": "MainProcess", "process": "83832", "taskName": "None", "v_gradient_value": "", "summation_role": "a combination of the following: Output of the string-based function with purpose: The runtime of string-based function that checks if the prediction is correct.", "message": "Idempotent backward"}
{"name": "textgrad", "msg": "Idempotent backward", "args": "()", "levelname": "INFO", "levelno": "20", "pathname": "C:\\Users\\chenran\\AppData\\Roaming\\Python\\Python312\\site-packages\\textgrad\\autograd\\algebra.py", "filename": "algebra.py", "module": "algebra", "exc_info": "None", "exc_text": "None", "stack_info": "None", "lineno": "89", "funcName": "backward", "created": "1757388710.5168757", "msecs": "516.0", "relativeCreated": "11539.501190185547", "thread": "88636", "threadName": "MainThread", "processName": "MainProcess", "process": "83832", "taskName": "None", "v_gradient_value": "", "summation_role": "a combination of the following: Output of the string-based function with purpose: The runtime of string-based function that checks if the prediction is correct.", "message": "Idempotent backward"}
{"name": "textgrad", "msg": "_backward_through_string_fn prompt", "args": "()", "levelname": "INFO", "levelno": "20", "pathname": "C:\\Users\\chenran\\AppData\\Roaming\\Python\\Python312\\site-packages\\textgrad\\autograd\\string_based_ops.py", "filename": "string_based_ops.py", "module": "string_based_ops", "exc_info": "None", "exc_text": "None", "stack_info": "None", "lineno": "179", "funcName": "_backward_through_string_fn_base", "created": "1757388710.5168757", "msecs": "516.0", "relativeCreated": "11539.501190185547", "thread": "88636", "threadName": "MainThread", "processName": "MainProcess", "process": "83832", "taskName": "None", "_backward_through_string_fn": "You will give feedback to a variable with the following role: <ROLE> response from the language model </ROLE>. Here is an evaluation of the variable using a string-based function:\n\nFunction purpose: The runtime of string-based function that checks if the prediction is correct.\n\n<INPUTS_TO_FUNCTION> **Prediction(role: response from the language model)**: \n\n```json\n[\n {\n (...) \"suggest\": \"建议补充'双方协商一致'的表述，并明确合同变更/取消或中途退货时的责任承担方式及违约金比例（若涉及退货且违约金比例低于80%需调整）。\"\n }\n]\n```\n\n**Ground truth answer(role: correct answer for the query)**: 不涉及 </INPUTS_TO_FUNCTION>\n\n<OUTPUT_OF_FUNCTION> 错误 </OUTPUT_OF_FUNCTION>\n\n<OBJECTIVE_FUNCTION>Your goal is to give feedback and criticism to the variable given the above evaluation output. Our only goal is to improve the above metric, and nothing else. </OBJECTIVE_FUNCTION>\n\nWe are interested in giving feedback to the response from the language model for this conversation. Specifically, give feedback to the following span of text:\n\n<VARIABLE> \n\n```json\n[\n {\n (...) \"suggest\": \"建议补充'双方协商一致'的表述，并明确合同变更/取消或中途退货时的责任承担方式及违约金比例（若涉及退货且违约金比例低于80%需调整）。\"\n }\n]\n``` </VARIABLE>\n\nGiven the above history, describe how the response from the language model could be improved to improve the <OBJECTIVE_FUNCTION>. Be very creative, critical, and intelligent.\n\n", "message": "_backward_through_string_fn prompt"}
{"name": "textgrad", "msg": "_backward_through_string_fn gradient", "args": "()", "levelname": "INFO", "levelno": "20", "pathname": "C:\\Users\\chenran\\AppData\\Roaming\\Python\\Python312\\site-packages\\textgrad\\autograd\\string_based_ops.py", "filename": "string_based_ops.py", "module": "string_based_ops", "exc_info": "None", "exc_text": "None", "stack_info": "None", "lineno": "181", "funcName": "_backward_through_string_fn_base", "created": "1757388742.6378605", "msecs": "637.0", "relativeCreated": "43660.4859828949", "thread": "88636", "threadName": "MainThread", "processName": "MainProcess", "process": "83832", "taskName": "None", "_backward_through_string_fn": "\n\nThe language model's response is incorrect because it introduces legally relevant clauses (\"双方协商一致\", contract change/cancellation terms, return liability, and breach penalties) in a scenario where the ground truth explicitly states these are **not applicable** (\"不涉及\"). The model's suggestion creates a false positive by assuming contractual obligations exist where none are required. To improve this:\n\n1. **Contextual Relevance Filtering** \n The model should first *validate whether the query context involves* contract modification/return scenarios. If the original question or scenario does not involve these elements (e.g., a fixed-term service agreement with no return policy), the model must avoid suggesting clauses that would introduce unnecessary legal complexity. The response should instead align with the ground truth by stating \"不涉及\" or explaining why the suggested clauses are irrelevant.\n\n2. **Binary Applicability Check** \n Add a pre-response validation step: \"Does the query scenario involve contract amendments, cancellations, or return conditions?\" If the answer is \"no,\" the model should refrain from suggesting any related clauses. This would prevent the current error of recommending terms for situations where they are explicitly not required.\n\n3. **Precision in Legal Terminology** \n The model conflates general contract best practices with scenario-specific requirements. While \"双方协商一致\" is a standard legal principle, its inclusion here is misleading because the ground truth indicates no such negotiation is needed. The model should distinguish between universal legal principles (which may still be valid) and scenario-specific obligations (which are explicitly absent here).\n\n4. **Error Pattern Recognition** \n Train the model to recognize that when the ground truth is \"不涉及,\" any suggestion of new clauses (even well-intentioned ones) constitutes an error. The model should learn to prioritize factual alignment over proactive legal advice in such cases.\n\nBy implementing these strategies, the model would avoid introducing false positives in scenarios where the correct answer is \"not applicable,\" thereby improving accuracy in legal scenario classification.", "message": "_backward_through_string_fn gradient"}
{"name": "textgrad", "msg": "_backward_through_llm prompt", "args": "()", "levelname": "INFO", "levelno": "20", "pathname": "C:\\Users\\chenran\\AppData\\Roaming\\Python\\Python312\\site-packages\\textgrad\\autograd\\llm_ops.py", "filename": "llm_ops.py", "module": "llm_ops", "exc_info": "None", "exc_text": "None", "stack_info": "None", "lineno": "148", "funcName": "_backward_through_llm_chain", "created": "1757388742.6378605", "msecs": "637.0", "relativeCreated": "43660.4859828949", "thread": "88636", "threadName": "MainThread", "processName": "MainProcess", "process": "83832", "taskName": "None", "_backward_through_llm": "You will give feedback to a variable with the following role: <ROLE> system prompt to guide the LLM's reasoning strategy for accurate responses </ROLE>. Here is a conversation with a language model (LM):\n\n<LM_SYSTEM_PROMPT> 你是乙方（供方、卖方）法律部门的合同审查助手\n# 审查要点\n1）提取涉及到句子的主体为甲方/买方/需方，句子内容为“合同变更/取消”、“退货”相关的句子，没有则返回不涉及\n2）句子明确提及了“双方协商”，审查合格\n3）句子没有明确提及“合同变更/取消”、“中途退货”所需要承担的责任，审查不合格\n3）“合同变更/取消”相关的句子，没有提及违约金额，审查不合格\n4）“退货”相关的句子，违约金的比例低于80%，审查不合格\n\n# 不合格建议\n1、提醒用户不合规的变更取消责任\n\n# 审查约束\n- 输出包括审查的原文、详情、结果、建议\n- 审查结果为合格/不合格/不涉及，合格/不涉及的审查结果无需输出建议\n- 审查原文严格提取关键、无省略、无篡改的原文内容\n- 结果以JSON数组的格式返回,例如```json [{\"original_text\":\"xx\",\"details\":\"xx\",\"result\":\"xx\",\"suggest\":\"xx\"}]```\n依据审查要点，遵循约束，完成合同审查，提供审查建议，一步步仔细思考。 </LM_SYSTEM_PROMPT>\n\n<LM_INPUT> 8.1.2.2 要求乙方全额退还甲方已支付的预付款项。 </LM_INPUT>\n\n<LM_OUTPUT> \n\n```json\n[\n {\n \"original_text\": \"8.1.2.2 要求乙方全额退还甲方已支付的预付款项。\",\n \"details\": \"该条款涉及甲方要求退还预付款，但未明确提及'双方协商'，且未说明合同变更/取消或中途退货所需承担的责任，也未约定违约金额或违约金比例。\",\n \"result\": \"不合格\",\n \"suggest\": \"建议补充'双方协商一致'的表述，并明确合同变更/取消或中途退货时的责任承担方式及违约金比例（若涉及退货且违约金比例低于80%需调整）。\"\n }\n]\n``` </LM_OUTPUT>\n\nThis conversation is part of a larger system. The <LM_OUTPUT> was later used as response from the language model.\n\n<OBJECTIVE_FUNCTION>Your goal is to give feedback to the variable to address the following feedback on the LM_OUTPUT: \n\nThe language model's response is incorrect because it introduces legally relevant clauses (\"双方协商一致\", contract change/cancellation terms, return liability, and breach penalties) in a scenario where the ground truth explicitly states these are **not applicable** (\"不涉及\"). The model's suggestion creates a false positive by assuming contractual obligations exist where none are required. To improve this:\n\n1. **Contextual Relevance Filtering** \n The model should first *validate whether the query context involves* contract modification/return scenarios. If the original question or scenario does not involve these elements (e.g., a fixed-term service agreement with no return policy), the model must avoid suggesting clauses that would introduce unnecessary legal complexity. The response should instead align with the ground truth by stating \"不涉及\" or explaining why the suggested clauses are irrelevant.\n\n2. **Binary Applicability Check** \n Add a pre-response validation step: \"Does the query scenario involve contract amendments, cancellations, or return conditions?\" If the answer is \"no,\" the model should refrain from suggesting any related clauses. This would prevent the current error of recommending terms for situations where they are explicitly not required.\n\n3. **Precision in Legal Terminology** \n The model conflates general contract best practices with scenario-specific requirements. While \"双方协商一致\" is a standard legal principle, its inclusion here is misleading because the ground truth indicates no such negotiation is needed. The model should distinguish between universal legal principles (which may still be valid) and scenario-specific obligations (which are explicitly absent here).\n\n4. **Error Pattern Recognition** \n Train the model to recognize that when the ground truth is \"不涉及,\" any suggestion of new clauses (even well-intentioned ones) constitutes an error. The model should learn to prioritize factual alignment over proactive legal advice in such cases.\n\nBy implementing these strategies, the model would avoid introducing false positives in scenarios where the correct answer is \"not applicable,\" thereby improving accuracy in legal scenario classification. </OBJECTIVE_FUNCTION>\n\nWe are interested in giving feedback to the system prompt to guide the LLM's reasoning strategy for accurate responses for this conversation. Specifically, give feedback to the following span of text:\n\n<VARIABLE> 你是乙方（供方、卖方）法律部门的合同审查助手\n# 审查要点\n1）提取涉及到句子的主体为甲方/买方/需方，句子内容为“合同变更/取消”、“退货”相关的句子，没有则返回不涉及\n2）句子明确提及了“双方协商”，审查合格\n3）句子没有明确提及“合同变更/取消”、“中途退货”所需要承担的责任，审查不合格\n3）“合同变更/取消”相关的句子，没有提及违约金额，审查不合格\n4）“退货”相关的句子，违约金的比例低于80%，审查不合格\n\n# 不合格建议\n1、提醒用户不合规的变更取消责任\n\n# 审查约束\n- 输出包括审查的原文、详情、结果、建议\n- 审查结果为合格/不合格/不涉及，合格/不涉及的审查结果无需输出建议\n- 审查原文严格提取关键、无省略、无篡改的原文内容\n- 结果以JSON数组的格式返回,例如```json [{\"original_text\":\"xx\",\"details\":\"xx\",\"result\":\"xx\",\"suggest\":\"xx\"}]```\n依据审查要点，遵循约束，完成合同审查，提供审查建议，一步步仔细思考。 </VARIABLE>\n\nGiven the above history, describe how the system prompt to guide the LLM's reasoning strategy for accurate responses could be improved to improve the <OBJECTIVE_FUNCTION>. Be very creative, critical, and intelligent.\n\n", "message": "_backward_through_llm prompt"}
{"name": "textgrad", "msg": "_backward_through_llm gradient", "args": "()", "levelname": "INFO", "levelno": "20", "pathname": "C:\\Users\\chenran\\AppData\\Roaming\\Python\\Python312\\site-packages\\textgrad\\autograd\\llm_ops.py", "filename": "llm_ops.py", "module": "llm_ops", "exc_info": "None", "exc_text": "None", "stack_info": "None", "lineno": "150", "funcName": "_backward_through_llm_chain", "created": "1757388826.2383811", "msecs": "238.0", "relativeCreated": "127261.00659370422", "thread": "88636", "threadName": "MainThread", "processName": "MainProcess", "process": "83832", "taskName": "None", "_backward_through_llm": "\n\nTo address the false positive error in the LLM's response, the system prompt must be restructured to enforce **strict contextual relevance filtering** and **binary applicability checks** before applying review rules. Here's how to improve the system prompt:\n\n---\n\n### **1. Add a Pre-Validation Step for Scenario Applicability**\n**Current flaw**: The prompt assumes all input sentences inherently involve contract amendments/returns, leading to overgeneralization. \n**Fix**: Insert a mandatory initial step to validate whether the scenario involves the required elements. \n**Proposed addition to the system prompt**: \n```markdown\n# **Pre-Validation Check** \n- **First**, determine if the input sentence explicitly involves **contract amendment/cancellation** or **return conditions**. \n - If **no**, output `{\"result\": \"不涉及\"}` **without any suggestions**. \n - If **yes**, proceed to the review steps below. \n- **Rationale**: This prevents the model from applying rules to irrelevant scenarios (e.g., refund clauses unrelated to returns). \n```\n\n---\n\n### **2. Restructure Review Points to Prioritize Applicability**\n**Current flaw**: The review points are applied sequentially without first confirming the scenario's relevance. \n**Fix**: Reorder the review logic to first confirm applicability, then apply rules. \n**Proposed revision to the review points**: \n```markdown\n# **Revised Review Points** \n1. **Applicability Check**: \n - If the sentence does **not** involve \"合同变更/取消\" or \"退货\", return `{\"result\": \"不涉及\"}`. \n2. **主体与内容匹配**: \n - Extract sentences where the subject is 甲方/买方/需方 and the content relates to \"合同变更/取消\" or \"退货\". \n3. **合格条件**: \n - If the sentence explicitly includes \"双方协商\", mark as **合格**. \n4. **不合格条件**: \n - If the sentence lacks mention of responsibilities for \"合同变更/取消\" or \"中途退货\", mark as **不合格**. \n - If \"合同变更/取消\" is mentioned but no breach amount is specified, mark as **不合格**. \n - If \"退货\" is mentioned but the breach penalty is <80%, mark as **不合格**. \n```\n\n---\n\n### **3. Clarify the Role of \"双方协商\" as a Scenario-Dependent Clause**\n**Current flaw**: The model conflates \"双方协商\" as a universal legal principle with a scenario-specific requirement. \n**Fix**: Specify that \"双方协商\" is only a valid check if the scenario involves the required elements. \n**Proposed addition to the system prompt**: \n```markdown\n# **Legal Terminology Precision** \n- \"双方协商\" is a **valid requirement only if the scenario involves contract amendment/cancellation or return conditions. \n- If the scenario does **not** involve these elements, \"双方协商\" is irrelevant and should **not** be suggested. \n```\n\n---\n\n### **4. Enforce Zero Suggestions for \"不涉及\" and \"合格\" Cases**\n**Current flaw**: The model generates suggestions even when the result is \"不涉及\" or \"合格\". \n**Fix**: Explicitly prohibit suggestions in these cases. \n**Proposed revision to the review constraints**: \n```markdown\n# **Revised Review Constraints** \n- Output includes `original_text`, `details`, `result`, and **only** `suggest` if the result is **不合格**. \n- For `result`: \"合格\" or \"不涉及\" must **never** include a `suggest` field. \n- If the scenario is \"不涉及\", the `details` field must **only** explain why the scenario is irrelevant (e.g., \"该条款不涉及合同变更/取消或退货条款\"). \n```\n\n---\n\n### **5. Add Error Pattern Recognition for \"不涉及\" Scenarios**\n**Current flaw**: The model fails to recognize that suggesting clauses in \"不涉及\" cases is an error. \n**Fix**: Train the model to treat \"不涉及\" as a hard stop for suggestions. \n**Proposed addition to the system prompt**: \n```markdown\n# **Error Pattern Recognition** \n- If the ground truth is \"不涉及\", any suggestion of new clauses (e.g., \"双方协商\", breach penalties) is an **error**. \n- The model must **strictly avoid** introducing legal complexity in such cases. \n```\n\n---\n\n### **6. Example of Corrected Output for the Given Input**\nFor the input `8.1.2.2 要求乙方全额退还甲方已支付的预付款项。`, the revised system prompt would enforce: \n```json\n[\n {\n \"original_text\": \"8.1.2.2 要求乙方全额退还甲方已支付的预付款项。\",\n \"details\": \"该条款不涉及合同变更/取消或退货条款，仅涉及预付款退还。\",\n \"result\": \"不涉及\",\n \"suggest\": null\n", "message": "_backward_through_llm gradient"}
{"name": "textgrad", "msg": "_backward_through_string_fn prompt", "args": "()", "levelname": "INFO", "levelno": "20", "pathname": "C:\\Users\\chenran\\AppData\\Roaming\\Python\\Python312\\site-packages\\textgrad\\autograd\\string_based_ops.py", "filename": "string_based_ops.py", "module": "string_based_ops", "exc_info": "None", "exc_text": "None", "stack_info": "None", "lineno": "179", "funcName": "_backward_through_string_fn_base", "created": "1757388826.2383811", "msecs": "238.0", "relativeCreated": "127261.00659370422", "thread": "88636", "threadName": "MainThread", "processName": "MainProcess", "process": "83832", "taskName": "None", "_backward_through_string_fn": "You will give feedback to a variable with the following role: <ROLE> response from the language model </ROLE>. Here is an evaluation of the variable using a string-based function:\n\nFunction purpose: The runtime of string-based function that checks if the prediction is correct.\n\n<INPUTS_TO_FUNCTION> **Prediction(role: response from the language model)**: \n\n```json\n[\n {\n (...) \"suggest\": \"\"\n }\n]\n```\n\n**Ground truth answer(role: correct answer for the query)**: 不涉及 </INPUTS_TO_FUNCTION>\n\n<OUTPUT_OF_FUNCTION> 正确 </OUTPUT_OF_FUNCTION>\n\n<OBJECTIVE_FUNCTION>Your goal is to give feedback and criticism to the variable given the above evaluation output. Our only goal is to improve the above metric, and nothing else. </OBJECTIVE_FUNCTION>\n\nWe are interested in giving feedback to the response from the language model for this conversation. Specifically, give feedback to the following span of text:\n\n<VARIABLE> \n\n```json\n[\n {\n (...) \"suggest\": \"\"\n }\n]\n``` </VARIABLE>\n\nGiven the above history, describe how the response from the language model could be improved to improve the <OBJECTIVE_FUNCTION>. Be very creative, critical, and intelligent.\n\n", "message": "_backward_through_string_fn prompt"}
{"name": "textgrad", "msg": "_backward_through_string_fn gradient", "args": "()", "levelname": "INFO", "levelno": "20", "pathname": "C:\\Users\\chenran\\AppData\\Roaming\\Python\\Python312\\site-packages\\textgrad\\autograd\\string_based_ops.py", "filename": "string_based_ops.py", "module": "string_based_ops", "exc_info": "None", "exc_text": "None", "stack_info": "None", "lineno": "181", "funcName": "_backward_through_string_fn_base", "created": "1757388854.9604893", "msecs": "960.0", "relativeCreated": "155983.11471939087", "thread": "88636", "threadName": "MainThread", "processName": "MainProcess", "process": "83832", "taskName": "None", "_backward_through_string_fn": "\n\nThe response from the language model could be improved by making the \"suggest\" field explicitly reflect the ground truth label \"不涉及\" (not involved/related) instead of leaving it as an empty string. While the empty string was technically deemed correct by the evaluation function, this approach risks ambiguity or misinterpretation in downstream systems expecting explicit semantic values. \n\n**Key improvements:**\n1. **Semantic alignment**: Replace `\"\"` with `\"不涉及\"` in the \"suggest\" field to directly mirror the ground truth answer. This eliminates reliance on implicit assumptions about empty strings and ensures consistency with human-annotated labels.\n2. **Robustness against parsing errors**: Empty strings may be misinterpreted by downstream systems as missing data or formatting errors. Using a semantically meaningful value like \"不涉及\" improves interoperability and reduces edge-case failures.\n3. **Signal clarity for training feedback**: If this response were used in a learning loop, explicitly encoding the correct label would provide stronger gradient signals for alignment with the objective function, accelerating convergence in future iterations.\n\nThis change would maintain correctness while enhancing the response's interpretability, robustness, and utility in production workflows.", "message": "_backward_through_string_fn gradient"}
{"name": "textgrad", "msg": "_backward_through_llm prompt", "args": "()", "levelname": "INFO", "levelno": "20", "pathname": "C:\\Users\\chenran\\AppData\\Roaming\\Python\\Python312\\site-packages\\textgrad\\autograd\\llm_ops.py", "filename": "llm_ops.py", "module": "llm_ops", "exc_info": "None", "exc_text": "None", "stack_info": "None", "lineno": "148", "funcName": "_backward_through_llm_chain", "created": "1757388854.9604893", "msecs": "960.0", "relativeCreated": "155983.11471939087", "thread": "88636", "threadName": "MainThread", "processName": "MainProcess", "process": "83832", "taskName": "None", "_backward_through_llm": "You will give feedback to a variable with the following role: <ROLE> system prompt to guide the LLM's reasoning strategy for accurate responses </ROLE>. Here is a conversation with a language model (LM):\n\n<LM_SYSTEM_PROMPT> 你是乙方（供方、卖方）法律部门的合同审查助手\n# 审查要点\n1）提取涉及到句子的主体为甲方/买方/需方，句子内容为“合同变更/取消”、“退货”相关的句子，没有则返回不涉及\n2）句子明确提及了“双方协商”，审查合格\n3）句子没有明确提及“合同变更/取消”、“中途退货”所需要承担的责任，审查不合格\n3）“合同变更/取消”相关的句子，没有提及违约金额，审查不合格\n4）“退货”相关的句子，违约金的比例低于80%，审查不合格\n\n# 不合格建议\n1、提醒用户不合规的变更取消责任\n\n# 审查约束\n- 输出包括审查的原文、详情、结果、建议\n- 审查结果为合格/不合格/不涉及，合格/不涉及的审查结果无需输出建议\n- 审查原文严格提取关键、无省略、无篡改的原文内容\n- 结果以JSON数组的格式返回,例如```json [{\"original_text\":\"xx\",\"details\":\"xx\",\"result\":\"xx\",\"suggest\":\"xx\"}]```\n依据审查要点，遵循约束，完成合同审查，提供审查建议，一步步仔细思考。 </LM_SYSTEM_PROMPT>\n\n<LM_INPUT> 8.1.2.6 向甲方支付违约基数30%的违约金。 </LM_INPUT>\n\n<LM_OUTPUT> \n\n```json\n[\n {\n \"original_text\": \"8.1.2.6 向甲方支付违约基数30%的违约金。\",\n \"details\": \"该条款未明确提及'合同变更/取消'或'退货'相关表述，且未说明违约金对应的违约行为类型（如中途退货、合同变更等）。违约金比例为30%，但缺乏具体适用场景的关联性描述。\",\n \"result\": \"不涉及\",\n \"suggest\": \"\"\n }\n]\n``` </LM_OUTPUT>\n\nThis conversation is part of a larger system. The <LM_OUTPUT> was later used as response from the language model.\n\n<OBJECTIVE_FUNCTION>Your goal is to give feedback to the variable to address the following feedback on the LM_OUTPUT: \n\nThe response from the language model could be improved by making the \"suggest\" field explicitly reflect the ground truth label \"不涉及\" (not involved/related) instead of leaving it as an empty string. While the empty string was technically deemed correct by the evaluation function, this approach risks ambiguity or misinterpretation in downstream systems expecting explicit semantic values. \n\n**Key improvements:**\n1. **Semantic alignment**: Replace `\"\"` with `\"不涉及\"` in the \"suggest\" field to directly mirror the ground truth answer. This eliminates reliance on implicit assumptions about empty strings and ensures consistency with human-annotated labels.\n2. **Robustness against parsing errors**: Empty strings may be misinterpreted by downstream systems as missing data or formatting errors. Using a semantically meaningful value like \"不涉及\" improves interoperability and reduces edge-case failures.\n3. **Signal clarity for training feedback**: If this response were used in a learning loop, explicitly encoding the correct label would provide stronger gradient signals for alignment with the objective function, accelerating convergence in future iterations.\n\nThis change would maintain correctness while enhancing the response's interpretability, robustness, and utility in production workflows. </OBJECTIVE_FUNCTION>\n\nWe are interested in giving feedback to the system prompt to guide the LLM's reasoning strategy for accurate responses for this conversation. Specifically, give feedback to the following span of text:\n\n<VARIABLE> 你是乙方（供方、卖方）法律部门的合同审查助手\n# 审查要点\n1）提取涉及到句子的主体为甲方/买方/需方，句子内容为“合同变更/取消”、“退货”相关的句子，没有则返回不涉及\n2）句子明确提及了“双方协商”，审查合格\n3）句子没有明确提及“合同变更/取消”、“中途退货”所需要承担的责任，审查不合格\n3）“合同变更/取消”相关的句子，没有提及违约金额，审查不合格\n4）“退货”相关的句子，违约金的比例低于80%，审查不合格\n\n# 不合格建议\n1、提醒用户不合规的变更取消责任\n\n# 审查约束\n- 输出包括审查的原文、详情、结果、建议\n- 审查结果为合格/不合格/不涉及，合格/不涉及的审查结果无需输出建议\n- 审查原文严格提取关键、无省略、无篡改的原文内容\n- 结果以JSON数组的格式返回,例如```json [{\"original_text\":\"xx\",\"details\":\"xx\",\"result\":\"xx\",\"suggest\":\"xx\"}]```\n依据审查要点，遵循约束，完成合同审查，提供审查建议，一步步仔细思考。 </VARIABLE>\n\nGiven the above history, describe how the system prompt to guide the LLM's reasoning strategy for accurate responses could be improved to improve the <OBJECTIVE_FUNCTION>. Be very creative, critical, and intelligent.\n\n", "message": "_backward_through_llm prompt"}
{"name": "textgrad", "msg": "_backward_through_llm gradient", "args": "()", "levelname": "INFO", "levelno": "20", "pathname": "C:\\Users\\chenran\\AppData\\Roaming\\Python\\Python312\\site-packages\\textgrad\\autograd\\llm_ops.py", "filename": "llm_ops.py", "module": "llm_ops", "exc_info": "None", "exc_text": "None", "stack_info": "None", "lineno": "150", "funcName": "_backward_through_llm_chain", "created": "1757388924.0729938", "msecs": "72.0", "relativeCreated": "225095.61920166016", "thread": "88636", "threadName": "MainThread", "processName": "MainProcess", "process": "83832", "taskName": "None", "_backward_through_llm": "\n\nTo address the objective function's feedback and ensure the \"suggest\" field explicitly reflects \"不涉及\" when applicable, the system prompt requires the following strategic revisions:\n\n---\n\n### **1. Clarify the \"审查约束\" (Review Constraints)**\n**Current issue**: The constraint states *\"合格/不涉及的审查结果无需输出建议\"*, which ambiguously implies both \"合格\" and \"不涉及\" should omit the \"suggest\" field. This leads to empty strings for \"不涉及\", violating the objective of semantic alignment. \n\n**Feedback**: \n- **Modify the constraint** to explicitly differentiate between \"合格\" and \"不涉及\": \n - *\"合格的审查结果无需输出建议，不涉及的审查结果需在'suggest'字段中明确标注'不涉及'\"* \n - This ensures \"不涉及\" is semantically encoded in the \"suggest\" field, while \"合格\" retains an empty string. \n- **Add a parsing robustness note**: \n - *\"避免使用空字符串表示'不涉及'，改用'不涉及'文本以减少下游系统歧义\"* \n\n---\n\n### **2. Update the Example in \"审查约束\"**\n**Current issue**: The example in the system prompt shows `\"suggest\":\"\"` for \"不涉及\", reinforcing the incorrect pattern. \n\n**Feedback**: \n- **Revise the example** to reflect the desired behavior: \n ```json\n {\"original_text\":\"xx\",\"details\":\"xx\",\"result\":\"不涉及\",\"suggest\":\"不涉及\"}\n ``` \n This directly demonstrates the expected output and trains the LLM to associate \"不涉及\" with the \"suggest\" field. \n\n---\n\n### **3. Add a \"不涉及\" Case to \"不合格建议\" (Non-Compliance Suggestions)**\n**Current issue**: The \"不合格建议\" section only includes a suggestion for non-compliant change/cancellation responsibilities, leaving no guidance for \"不涉及\" scenarios. \n\n**Feedback**: \n- **Introduce a new bullet point**: \n - *\"若审查结果为'不涉及'，需在'suggest'字段中明确标注'不涉及'以确保语义一致性\"* \n This explicitly ties the \"不涉及\" label to the \"suggest\" field, reinforcing the objective function's requirement. \n\n---\n\n### **4. Strengthen the \"审查要点\" (Review Criteria)**\n**Current issue**: The first review criterion states *\"没有则返回不涉及\"*, but does not specify how to handle the \"suggest\" field. \n\n**Feedback**: \n- **Add a sub-point under review criterion 1**: \n - *\"若返回'不涉及'，需在'suggest'字段中明确标注'不涉及'，以确保输出与审查结果完全对齐\"* \n This creates a direct link between the \"不涉及\" result and the \"suggest\" field, reducing ambiguity. \n\n---\n\n### **5. Add a \"语义一致性\" (Semantic Consistency) Section**\n**Current issue**: The system prompt lacks explicit guidance on aligning output fields with semantic labels. \n\n**Feedback**: \n- **Insert a new section**: \n ```markdown\n # 语义一致性要求\n - 所有输出字段（如'result'、'suggest'）必须与审查结果的语义完全一致。例如：\n - '不涉及'结果必须在'suggest'字段中显式标注'不涉及'\n - '合格'结果的'suggest'字段可为空字符串\n - 禁止使用空字符串表示非'合格'结果，除非明确允许\n ``` \n This codifies the objective function's requirement into the system prompt, ensuring the LLM prioritizes semantic clarity. \n\n---\n\n### **6. Test Case Inclusion**\n**Current issue**: The system prompt lacks examples for edge cases like \"不涉及\" with explicit \"suggest\" fields. \n\n**Feedback**: \n- **Add a test case to the system prompt**: \n ```markdown\n # 测试用例\n - 输入: \"合同条款与变更/取消或退货无关\"\n - 预期输出: {\"original_text\":\"xx\",\"details\":\"xx\",\"result\":\"不涉及\",\"suggest\":\"不涉及\"}\n ``` \n This trains the LLM to recognize and replicate the desired pattern. \n\n---\n\n### **Why This Works** \n1. **Semantic alignment**: By explicitly tying \"不涉及\" to the \"suggest\" field, the LLM learns to avoid relying on implicit assumptions (e.g., empty strings). \n2. **Parsing robustness**: Downstream systems can unambiguously interpret \"不涉及\" without error-handling logic for empty strings. \n3. **Training signal clarity**: Future iterations of the LLM will receive stronger gradient signals when \"suggest\" fields are semantically consistent with \"result\" fields. \n\nThese changes ensure the system prompt directly addresses the objective function's feedback while maintaining the original constraints and logic.", "message": "_backward_through_llm gradient"}
{"name": "textgrad", "msg": "_backward_through_string_fn prompt", "args": "()", "levelname": "INFO", "levelno": "20", "pathname": "C:\\Users\\chenran\\AppData\\Roaming\\Python\\Python312\\site-packages\\textgrad\\autograd\\string_based_ops.py", "filename": "string_based_ops.py", "module": "string_based_ops", "exc_info": "None", "exc_text": "None", "stack_info": "None", "lineno": "179", "funcName": "_backward_through_string_fn_base", "created": "1757388924.0729938", "msecs": "72.0", "relativeCreated": "225095.61920166016", "thread": "88636", "threadName": "MainThread", "processName": "MainProcess", "process": "83832", "taskName": "None", "_backward_through_string_fn": "You will give feedback to a variable with the following role: <ROLE> response from the language model </ROLE>. Here is an evaluation of the variable using a string-based function:\n\nFunction purpose: The runtime of string-based function that checks if the prediction is correct.\n\n<INPUTS_TO_FUNCTION> **Prediction(role: response from the language model)**: \n\n```json\n[\n {\n (...) \"suggest\": \"建议补充'双方协商'条款，并明确合同取消/变更需经协商一致。若涉及退货条款，需将违约金比例调整至80%以上。\"\n }\n]\n```\n\n**Ground truth answer(role: correct answer for the query)**: 不合格 </INPUTS_TO_FUNCTION>\n\n<OUTPUT_OF_FUNCTION> 正确 </OUTPUT_OF_FUNCTION>\n\n<OBJECTIVE_FUNCTION>Your goal is to give feedback and criticism to the variable given the above evaluation output. Our only goal is to improve the above metric, and nothing else. </OBJECTIVE_FUNCTION>\n\nWe are interested in giving feedback to the response from the language model for this conversation. Specifically, give feedback to the following span of text:\n\n<VARIABLE> \n\n```json\n[\n {\n (...) \"suggest\": \"建议补充'双方协商'条款，并明确合同取消/变更需经协商一致。若涉及退货条款，需将违约金比例调整至80%以上。\"\n }\n]\n``` </VARIABLE>\n\nGiven the above history, describe how the response from the language model could be improved to improve the <OBJECTIVE_FUNCTION>. Be very creative, critical, and intelligent.\n\n", "message": "_backward_through_string_fn prompt"}
{"name": "textgrad", "msg": "_backward_through_string_fn gradient", "args": "()", "levelname": "INFO", "levelno": "20", "pathname": "C:\\Users\\chenran\\AppData\\Roaming\\Python\\Python312\\site-packages\\textgrad\\autograd\\string_based_ops.py", "filename": "string_based_ops.py", "module": "string_based_ops", "exc_info": "None", "exc_text": "None", "stack_info": "None", "lineno": "181", "funcName": "_backward_through_string_fn_base", "created": "1757388993.028691", "msecs": "28.0", "relativeCreated": "294051.3164997101", "thread": "88636", "threadName": "MainThread", "processName": "MainProcess", "process": "83832", "taskName": "None", "_backward_through_string_fn": "\n\nThe response from the language model is technically correct in its suggestion but could be improved by addressing the **root cause of the contract's unqualification** and aligning more explicitly with the ground truth label \"不合格\". Here’s how to refine it:\n\n1. **Link Suggestions to the Ground Truth**: \n The model’s suggestion focuses on procedural terms (\"双方协商\") and breach penalties (80%+), but the ground truth \"不合格\" likely indicates a **fundamental flaw** in the contract’s structure or compliance. The response should explicitly connect the suggested changes to the reasons the contract is unqualified (e.g., missing mandatory clauses, non-compliance with legal standards, or ambiguous terms). For example: \n *\"建议补充'双方协商'条款，以解决合同变更流程不明确的问题（当前合同因缺乏协商机制导致不合格）。同时，将退货条款的违约金比例调整至80%以上，以符合行业合规标准。\"*\n\n2. **Prioritize Critical Fixes Over General Advice**: \n The current suggestion is generic. Instead, identify **specific clauses or legal requirements** the contract violates. For instance: \n *\"当前合同因未包含[具体法律条款名称]而不合格。建议补充[具体条款]，并明确[具体义务]，以确保合规性。\"*\n\n3. **Quantify or Contextualize the 80% Breach Penalty**: \n The 80% threshold may be arbitrary. Clarify why this number is appropriate (e.g., legal precedent, industry benchmarks) to strengthen the suggestion’s validity. \n *\"根据[相关法规/案例]，违约金比例需不低于80%以保障双方权益，建议调整至80%以上。\"*\n\n4. **Address Structural Deficiencies**: \n If the contract is \"不合格\" due to missing sections (e.g., dispute resolution, termination conditions), the response should explicitly name these gaps and propose targeted fixes. For example: \n *\"建议补充[争议解决条款]，并明确[具体流程]，以解决当前合同因缺乏争议处理机制而不合格的问题。\"*\n\n5. **Use Clearer Language for Legal Precision**: \n Replace vague terms like \"协商一致\" with legally binding phrasing (e.g., \"书面协议\") to avoid ambiguity. \n *\"合同取消/变更需经双方书面协议，以确保法律效力。\"*\n\nBy anchoring the suggestions to the **specific reasons for the \"不合格\" label** and providing **actionable, legally grounded fixes**, the response would better align with the objective of improving the evaluation metric. This approach ensures the model’s output is not only technically correct but also directly addresses the root cause of the contract’s unqualification.", "message": "_backward_through_string_fn gradient"}
{"name": "textgrad", "msg": "_backward_through_llm prompt", "args": "()", "levelname": "INFO", "levelno": "20", "pathname": "C:\\Users\\chenran\\AppData\\Roaming\\Python\\Python312\\site-packages\\textgrad\\autograd\\llm_ops.py", "filename": "llm_ops.py", "module": "llm_ops", "exc_info": "None", "exc_text": "None", "stack_info": "None", "lineno": "148", "funcName": "_backward_through_llm_chain", "created": "1757388993.028691", "msecs": "28.0", "relativeCreated": "294051.3164997101", "thread": "88636", "threadName": "MainThread", "processName": "MainProcess", "process": "83832", "taskName": "None", "_backward_through_llm": "You will give feedback to a variable with the following role: <ROLE> system prompt to guide the LLM's reasoning strategy for accurate responses </ROLE>. Here is a conversation with a language model (LM):\n\n<LM_SYSTEM_PROMPT> 你是乙方（供方、卖方）法律部门的合同审查助手\n# 审查要点\n1）提取涉及到句子的主体为甲方/买方/需方，句子内容为“合同变更/取消”、“退货”相关的句子，没有则返回不涉及\n2）句子明确提及了“双方协商”，审查合格\n3）句子没有明确提及“合同变更/取消”、“中途退货”所需要承担的责任，审查不合格\n3）“合同变更/取消”相关的句子，没有提及违约金额，审查不合格\n4）“退货”相关的句子，违约金的比例低于80%，审查不合格\n\n# 不合格建议\n1、提醒用户不合规的变更取消责任\n\n# 审查约束\n- 输出包括审查的原文、详情、结果、建议\n- 审查结果为合格/不合格/不涉及，合格/不涉及的审查结果无需输出建议\n- 审查原文严格提取关键、无省略、无篡改的原文内容\n- 结果以JSON数组的格式返回,例如```json [{\"original_text\":\"xx\",\"details\":\"xx\",\"result\":\"xx\",\"suggest\":\"xx\"}]```\n依据审查要点，遵循约束，完成合同审查，提供审查建议，一步步仔细思考。 </LM_SYSTEM_PROMPT>\n\n<LM_INPUT> 8.1.2.2 2.13.3.5 甲方有权取消该笔订单/合同，乙方需支付违约基数30%的违约金。 </LM_INPUT>\n\n<LM_OUTPUT> \n\n```json\n[\n {\n \"original_text\": \"甲方有权取消该笔订单/合同，乙方需支付违约基数30%的违约金。\",\n \"details\": \"条款涉及甲方单方取消合同的权利，且明确乙方需支付违约基数30%的违约金。但未提及'双方协商'，且违约金比例（30%）未达到退货条款要求的80%标准。\",\n \"result\": \"不合格\",\n \"suggest\": \"建议补充'双方协商'条款，并明确合同取消/变更需经协商一致。若涉及退货条款，需将违约金比例调整至80%以上。\"\n }\n]\n``` </LM_OUTPUT>\n\nThis conversation is part of a larger system. The <LM_OUTPUT> was later used as response from the language model.\n\n<OBJECTIVE_FUNCTION>Your goal is to give feedback to the variable to address the following feedback on the LM_OUTPUT: \n\nThe response from the language model is technically correct in its suggestion but could be improved by addressing the **root cause of the contract's unqualification** and aligning more explicitly with the ground truth label \"不合格\". Here’s how to refine it:\n\n1. **Link Suggestions to the Ground Truth**: \n The model’s suggestion focuses on procedural terms (\"双方协商\") and breach penalties (80%+), but the ground truth \"不合格\" likely indicates a **fundamental flaw** in the contract’s structure or compliance. The response should explicitly connect the suggested changes to the reasons the contract is unqualified (e.g., missing mandatory clauses, non-compliance with legal standards, or ambiguous terms). For example: \n *\"建议补充'双方协商'条款，以解决合同变更流程不明确的问题（当前合同因缺乏协商机制导致不合格）。同时，将退货条款的违约金比例调整至80%以上，以符合行业合规标准。\"*\n\n2. **Prioritize Critical Fixes Over General Advice**: \n The current suggestion is generic. Instead, identify **specific clauses or legal requirements** the contract violates. For instance: \n *\"当前合同因未包含[具体法律条款名称]而不合格。建议补充[具体条款]，并明确[具体义务]，以确保合规性。\"*\n\n3. **Quantify or Contextualize the 80% Breach Penalty**: \n The 80% threshold may be arbitrary. Clarify why this number is appropriate (e.g., legal precedent, industry benchmarks) to strengthen the suggestion’s validity. \n *\"根据[相关法规/案例]，违约金比例需不低于80%以保障双方权益，建议调整至80%以上。\"*\n\n4. **Address Structural Deficiencies**: \n If the contract is \"不合格\" due to missing sections (e.g., dispute resolution, termination conditions), the response should explicitly name these gaps and propose targeted fixes. For example: \n *\"建议补充[争议解决条款]，并明确[具体流程]，以解决当前合同因缺乏争议处理机制而不合格的问题。\"*\n\n5. **Use Clearer Language for Legal Precision**: \n Replace vague terms like \"协商一致\" with legally binding phrasing (e.g., \"书面协议\") to avoid ambiguity. \n *\"合同取消/变更需经双方书面协议，以确保法律效力。\"*\n\nBy anchoring the suggestions to the **specific reasons for the \"不合格\" label** and providing **actionable, legally grounded fixes**, the response would better align with the objective of improving the evaluation metric. This approach ensures the model’s output is not only technically correct but also directly addresses the root cause of the contract’s unqualification. </OBJECTIVE_FUNCTION>\n\nWe are interested in giving feedback to the system prompt to guide the LLM's reasoning strategy for accurate responses for this conversation. Specifically, give feedback to the following span of text:\n\n<VARIABLE> 你是乙方（供方、卖方）法律部门的合同审查助手\n# 审查要点\n1）提取涉及到句子的主体为甲方/买方/需方，句子内容为“合同变更/取消”、“退货”相关的句子，没有则返回不涉及\n2）句子明确提及了“双方协商”，审查合格\n3）句子没有明确提及“合同变更/取消”、“中途退货”所需要承担的责任，审查不合格\n3）“合同变更/取消”相关的句子，没有提及违约金额，审查不合格\n4）“退货”相关的句子，违约金的比例低于80%，审查不合格\n\n# 不合格建议\n1、提醒用户不合规的变更取消责任\n\n# 审查约束\n- 输出包括审查的原文、详情、结果、建议\n- 审查结果为合格/不合格/不涉及，合格/不涉及的审查结果无需输出建议\n- 审查原文严格提取关键、无省略、无篡改的原文内容\n- 结果以JSON数组的格式返回,例如```json [{\"original_text\":\"xx\",\"details\":\"xx\",\"result\":\"xx\",\"suggest\":\"xx\"}]```\n依据审查要点，遵循约束，完成合同审查，提供审查建议，一步步仔细思考。 </VARIABLE>\n\nGiven the above history, describe how the system prompt to guide the LLM's reasoning strategy for accurate responses could be improved to improve the <OBJECTIVE_FUNCTION>. Be very creative, critical, and intelligent.\n\n", "message": "_backward_through_llm prompt"}
main.py 0 → 100644
import argparse
import os
from util import *
import textgrad as tg
from textgrad.tasks import load_task, Dataset
from textgrad.autograd.string_based_ops import StringBasedFunction
class ContractReviewDataset(Dataset):
def __init__(self, path, x_col, y_col):
with open(path, 'r', encoding='utf-8') as f:
self.data_list = json.load(f)
self.x_col = x_col
self.y_col = y_col
def __getitem__(self, index):
row = self.data_list[index]
return row[self.x_col], row[self.y_col]
def __len__(self):
return len(self.data_list)
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument(
"--train_path",
type=str,
default="dataset/train.json",
help="train dataset path",
)
parser.add_argument(
"--val_path",
type=str,
default="dataset/val.json",
help="val dataset path",
)
parser.add_argument("--prompt_path", type=str, default="prompt/init_prompt.txt", help="prompts dir")
parser.add_argument(
"--output_dir", type=str, default="output_dir", help="Path to output dir"
)
parser.add_argument(
"--x_col", type=str, default="original_text", help="dataset x column name"
)
parser.add_argument(
"--y_col", type=str, default="result", help="dataset y column name"
)
parser.add_argument(
"--batch_size", type=int, default=10, help="batch size"
)
parser.add_argument(
"--epoch", type=int, default=3, help="epoch"
)
args = parser.parse_args()
# create output dir
output_dir = os.path.join(args.output_dir)
if not os.path.exists(output_dir):
os.makedirs(output_dir)
output_path = os.path.join(output_dir, f'{get_now()}.jsonl')
print(output_path)
def load_contract_review_task():
train_ds = ContractReviewDataset(args.train_path, args.x_col, args.y_col)
val_ds = ContractReviewDataset(args.val_path, args.x_col, args.y_col)
return train_ds, val_ds, None, StringBasedFunction(string_based_equality_fn,
function_purpose="The runtime of string-based function that checks if the prediction is correct.")
def save_results(results):
output_dict = {}
for k, v in results.items():
output_dict[k] = v[-1]
append_dict_to_jsonl(output_dict, output_path)
# init engine,model,optimizer
os.environ['OLLAMA_BASE_URL'] = 'http://192.168.252.71:9002/v1'
llm_engine = tg.get_engine("ollama-Qwen2-72B-Instruct")
tg.set_backward_engine("ollama-Qwen2-72B-Instruct")
# init datasets prompt; init eval
train_set, val_set, _, eval_fn = load_contract_review_task()
train_loader = tg.tasks.DataLoader(train_set, batch_size=args.batch_size, shuffle=True)
with open(args.prompt_path, 'r', encoding='utf-8') as f:
init_prompt = f.read()
system_prompt = tg.Variable(init_prompt,
requires_grad=True,
role_description="system prompt to guide the LLM's reasoning strategy for accurate responses")
model = tg.BlackboxLLM(llm_engine, system_prompt=system_prompt)
optimizer = tg.TGD(parameters=list(model.parameters()))
results = {"train_acc": [], "prompt": [], "validation_acc": []}
results["train_acc"].append(np.mean(eval_dataset(train_set, eval_fn, model)))
results["validation_acc"].append(np.mean(eval_dataset(val_set, eval_fn, model)))
results["prompt"].append(system_prompt.get_value())
save_results(results)
# backward
for epoch in range(args.epoch):
for steps, (batch_x, batch_y) in enumerate((pbar := tqdm(train_loader, position=0))):
pbar.set_description(f"Training step {steps}. Epoch {epoch}")
optimizer.zero_grad()
losses = []
for (x, y) in zip(batch_x, batch_y):
x = tg.Variable(x, requires_grad=False, role_description="query to the language model")
y = tg.Variable(y, requires_grad=False, role_description="correct answer for the query")
response = model(x)
eval_output_variable = eval_fn(inputs=dict(prediction=response, ground_truth_answer=y))
losses.append(eval_output_variable)
total_loss = tg.sum(losses)
total_loss.backward()
optimizer.step()
run_validation_revert(system_prompt, results, model, eval_fn, val_set)
# print("sys prompt: ", system_prompt)
results["train_acc"].append(np.mean(eval_dataset(train_set, eval_fn, model)))
results["validation_acc"].append(np.mean(eval_dataset(val_set, eval_fn, model)))
results["prompt"].append(system_prompt.get_value())
save_results(results)
output_dir/20250909-113148.jsonl 0 → 100644
{"train_acc": 0.6666666666666666, "prompt": "你是乙方（供方、卖方）法律部门的合同审查助手\n# 审查要点\n1）提取涉及到句子的主体为甲方/买方/需方，句子内容为“合同变更/取消”、“退货”相关的句子，没有则返回不涉及\n2）句子明确提及了“双方协商”，审查合格\n3）句子没有明确提及“合同变更/取消”、“中途退货”所需要承担的责任，审查不合格\n3）“合同变更/取消”相关的句子，没有提及违约金额，审查不合格\n4）“退货”相关的句子，违约金的比例低于80%，审查不合格\n\n# 不合格建议\n1、提醒用户不合规的变更取消责任\n\n# 审查约束\n- 输出包括审查的原文、详情、结果、建议\n- 审查结果为合格/不合格/不涉及，合格/不涉及的审查结果无需输出建议\n- 审查原文严格提取关键、无省略、无篡改的原文内容\n- 结果以JSON数组的格式返回,例如```json [{\"original_text\":\"xx\",\"details\":\"xx\",\"result\":\"xx\",\"suggest\":\"xx\"}]```\n依据审查要点，遵循约束，完成合同审查，提供审查建议，一步步仔细思考。", "validation_acc": 0.6666666666666666}
prompt/init_prompt.txt 0 → 100644
你是乙方（供方、卖方）法律部门的合同审查助手
# 审查要点
1）提取涉及到句子的主体为甲方/买方/需方，句子内容为“合同变更/取消”、“退货”相关的句子，没有则返回不涉及
2）句子明确提及了“双方协商”，审查合格
3）句子没有明确提及“合同变更/取消”、“中途退货”所需要承担的责任，审查不合格
3）“合同变更/取消”相关的句子，没有提及违约金额，审查不合格
4）“退货”相关的句子，违约金的比例低于80%，审查不合格
# 不合格建议
1、提醒用户不合规的变更取消责任
# 审查约束
- 输出包括审查的原文、详情、结果、建议
- 审查结果为合格/不合格/不涉及，合格/不涉及的审查结果无需输出建议
- 审查原文严格提取关键、无省略、无篡改的原文内容
- 结果以JSON数组的格式返回,例如```json [{"original_text":"xx","details":"xx","result":"xx","suggest":"xx"}]```
依据审查要点，遵循约束，完成合同审查，提供审查建议，一步步仔细思考。
\ No newline at end of file
util.py 0 → 100644
import textgrad as tg
import re
import json
import numpy as np
import concurrent
from tqdm import tqdm
from datetime import datetime
def eval_sample(item, eval_fn, model):
"""
This function allows us to evaluate if an answer to a question in the prompt is a good answer.
"""
x, y = item
x = tg.Variable(x, requires_grad=False, role_description="query to the language model")
y = tg.Variable(y, requires_grad=False, role_description="correct answer for the query")
response = model(x)
try:
eval_output_variable = eval_fn(inputs=dict(prediction=response, ground_truth_answer=y))
return 1 if eval_output_variable.value == '正确' else 0
except:
return 0
def run_validation_revert(system_prompt: tg.Variable, results, model, eval_fn, val_set):
val_performance = np.mean(eval_dataset(val_set, eval_fn, model))
previous_performance = np.mean(results["validation_acc"][-1])
print("val_performance: ", val_performance)
print("previous_performance: ", previous_performance)
previous_prompt = results["prompt"][-1]
if val_performance < previous_performance:
# print(f"rejected prompt: {system_prompt.value}")
system_prompt.set_value(previous_prompt)
val_performance = previous_performance
results["validation_acc"].append(val_performance)
def eval_dataset(test_set, eval_fn, model, max_samples: int = None):
if max_samples is None:
max_samples = len(test_set)
accuracy_list = []
with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor:
futures = []
for _, sample in enumerate(test_set):
future = executor.submit(eval_sample, sample, eval_fn, model)
futures.append(future)
if len(futures) >= max_samples:
break
tqdm_loader = tqdm(concurrent.futures.as_completed(futures), total=len(futures), position=0)
for future in tqdm_loader:
acc_item = future.result()
accuracy_list.append(acc_item)
tqdm_loader.set_description(f"Accuracy: {np.mean(accuracy_list)}")
return accuracy_list
def extract_json(json_str):
json_pattern = r'```json([\s\S]*?)```'
matches = re.findall(json_pattern, json_str, re.DOTALL)
json_list = []
for match in matches:
# 去除可能存在的前后空白字符
clean_json_str = match.strip()
try:
json_obj = json.loads(clean_json_str)
if isinstance(json_obj, list):
json_list += json_obj
else:
json_list.append(json_obj)
except json.JSONDecodeError as e:
print(f"发现了一个无法解析的JSON字符串: {clean_json_str} {e}")
return json_list
def string_based_equality_fn(prediction: tg.Variable, ground_truth_answer: tg.Variable):
json_res = extract_json(prediction.value)
check_res = json_res[0]['result'] == ground_truth_answer.value if json_res else False
return '正确' if check_res else '错误'
def append_dict_to_jsonl(dictionary, file_path):
with open(file_path, 'a', encoding='utf-8') as f:
json.dump(dictionary, f, ensure_ascii=False)
f.write('\n')
f.flush()
def get_now():
now = datetime.now()
formatted_time = now.strftime("%Y%m%d-%H%M%S")
return formatted_time
if __name__ == '__main__':
# append_dict_to_jsonl({'a': 'zz'}, 'test.jsonl')
# append_dict_to_jsonl({'a': 'ff'}, 'test.jsonl')
print(get_now())
pass
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment