latex auto translation milestone
This commit is contained in:
qingxu fu
parent
5a1831db98
commit
ef61221241
@ -319,5 +319,28 @@ def get_crazy_functions():
|
||||
except:
|
||||
print('Load function plugin failed')
|
||||
|
||||
try:
|
||||
from crazy_functions.Latex输出PDF结果 import Latex英文纠错加PDF对比
|
||||
function_plugins.update({
|
||||
"Latex英文纠错+LatexDiff高亮修正位置": {
|
||||
"Color": "stop",
|
||||
"AsButton": False,
|
||||
# "AdvancedArgs": True,
|
||||
# "ArgsReminder": "",
|
||||
"Function": HotReload(Latex英文纠错加PDF对比)
|
||||
}
|
||||
})
|
||||
from crazy_functions.Latex输出PDF结果 import Latex翻译中文并重新编译PDF
|
||||
function_plugins.update({
|
||||
"Latex翻译中文+生成PDF": {
|
||||
"Color": "stop",
|
||||
"AsButton": False,
|
||||
# "AdvancedArgs": True,
|
||||
# "ArgsReminder": "",
|
||||
"Function": HotReload(Latex翻译中文并重新编译PDF)
|
||||
}
|
||||
})
|
||||
except:
|
||||
print('Load function plugin failed')
|
||||
###################### 第n组插件 ###########################
|
||||
return function_plugins
|
||||
|
||||
@ -1,78 +1,20 @@
|
||||
from toolbox import update_ui, trimmed_format_exc, objdump, objload
|
||||
from toolbox import CatchException, report_execption, write_results_to_file, zip_folder
|
||||
import glob, copy, os
|
||||
from toolbox import update_ui, trimmed_format_exc, get_conf, objdump, objload
|
||||
from toolbox import CatchException, report_execption, update_ui_lastest_msg, zip_result, gen_time_str
|
||||
import glob, os, requests, time
|
||||
pj = os.path.join
|
||||
|
||||
|
||||
def confirm_answer_is_health(bufo, buf, llm_kwargs, default = True):
|
||||
return len(buf) >= len(bufo) // 3
|
||||
|
||||
def Latex精细分解与转化(file_manifest, project_folder, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, mode='proofread'):
|
||||
import time, os, re
|
||||
from .crazy_utils import request_gpt_model_multi_threads_with_very_awesome_ui_and_high_efficiency
|
||||
from .latex_utils import LatexPaperFileGroup, merge_tex_files, LatexPaperSplit, 寻找Latex主文件
|
||||
|
||||
maintex = 寻找Latex主文件(file_manifest, mode)
|
||||
|
||||
# <-------- 读取Latex文件，删除其中的所有注释 ---------->
|
||||
with open(maintex, 'r', encoding='utf-8', errors='replace') as f:
|
||||
content = f.read()
|
||||
merged_content = merge_tex_files(project_folder, content, mode)
|
||||
merged_content = re.sub(r'(?<!\\)%.*', '', merged_content) # 使用正则表达式查找注释，并替换为空字符串
|
||||
|
||||
with open(project_folder + '/merge.tex', 'w', encoding='utf-8', errors='replace') as f:
|
||||
f.write(merged_content)
|
||||
|
||||
lps = LatexPaperSplit()
|
||||
res = lps.split(merged_content)
|
||||
|
||||
# <-------- 拆分过长的latex片段 ---------->
|
||||
pfg = LatexPaperFileGroup()
|
||||
for index, r in enumerate(res):
|
||||
pfg.file_paths.append(index)
|
||||
pfg.file_contents.append(r)
|
||||
|
||||
pfg.run_file_split(max_token_limit=1024)
|
||||
n_split = len(pfg.sp_file_contents)
|
||||
|
||||
inputs_array, sys_prompt_array = switch_prompt(pfg, mode)
|
||||
inputs_show_user_array = [f"{mode} {f}" for f in pfg.sp_file_tag]
|
||||
|
||||
gpt_response_collection = yield from request_gpt_model_multi_threads_with_very_awesome_ui_and_high_efficiency(
|
||||
inputs_array=inputs_array,
|
||||
inputs_show_user_array=inputs_show_user_array,
|
||||
llm_kwargs=llm_kwargs,
|
||||
chatbot=chatbot,
|
||||
history_array=[[""] for _ in range(n_split)],
|
||||
sys_prompt_array=sys_prompt_array,
|
||||
# max_workers=5, # 并行任务数量限制，最多同时执行5个，其他的排队等待
|
||||
scroller_max_len = 80
|
||||
)
|
||||
|
||||
# <-------- 文本碎片重组为完整的tex片段 ---------->
|
||||
pfg.sp_file_result = []
|
||||
for i_say, gpt_say, orig_content in zip(gpt_response_collection[0::2], gpt_response_collection[1::2], pfg.sp_file_contents):
|
||||
pfg.sp_file_result.append(gpt_say)
|
||||
pfg.merge_result()
|
||||
|
||||
# <-------- 临时存储用于调试 ---------->
|
||||
pfg.get_token_num = None
|
||||
objdump(pfg)
|
||||
pfg = objload()
|
||||
|
||||
# <-------- 写出文件 ---------->
|
||||
final_tex = lps.merge_result(pfg.sp_file_result)
|
||||
with open(project_folder + f'/merge_{mode}.tex', 'w', encoding='utf-8', errors='replace') as f:
|
||||
f.write(final_tex)
|
||||
# <-------- 整理结果，退出 ---------->
|
||||
# create_report_file_name = time.strftime("%Y-%m-%d-%H-%M-%S", time.localtime()) + f"-chatgpt.polish.md"
|
||||
# res = write_results_to_file(gpt_response_collection, file_name=create_report_file_name)
|
||||
# history = gpt_response_collection
|
||||
chatbot.append((f"完成了吗？", 'GPT结果已输出，正在编译PDF'))
|
||||
yield from update_ui(chatbot=chatbot, history=history) # 刷新界面
|
||||
return project_folder + f'/merge_{mode}.tex'
|
||||
|
||||
# =================================== 工具函数 ===============================================
|
||||
def switch_prompt(pfg, mode):
|
||||
"""
|
||||
Generate prompts and system prompts based on the mode for proofreading or translating.
|
||||
Args:
|
||||
- pfg: Proofreader or Translator instance.
|
||||
- mode: A string specifying the mode, either 'proofread' or 'translate_zh'.
|
||||
|
||||
Returns:
|
||||
- inputs_array: A list of strings containing prompts for users to respond to.
|
||||
- sys_prompt_array: A list of strings containing prompts for system prompts.
|
||||
"""
|
||||
n_split = len(pfg.sp_file_contents)
|
||||
if mode == 'proofread':
|
||||
inputs_array = [r"Below is a section from an academic paper, proofread this section." +
|
||||
@ -90,162 +32,98 @@ def switch_prompt(pfg, mode):
|
||||
assert False, "未知指令"
|
||||
return inputs_array, sys_prompt_array
|
||||
|
||||
def desend_to_extracted_folder_if_exist(project_folder):
|
||||
"""
|
||||
Descend into the extracted folder if it exists, otherwise return the original folder.
|
||||
|
||||
Args:
|
||||
- project_folder: A string specifying the folder path.
|
||||
|
||||
def 编译Latex(main_tex, work_folder):
|
||||
import os
|
||||
current_dir = os.getcwd()
|
||||
os.chdir(work_folder)
|
||||
main_file = os.path.basename(main_tex)
|
||||
assert main_file.endswith('.tex')
|
||||
main_file = main_file[:-4]
|
||||
os.system(f'pdflatex {main_file}.tex')
|
||||
os.system(f'bibtex {main_file}.aux')
|
||||
os.system(f'pdflatex {main_file}.tex')
|
||||
os.system(f'pdflatex {main_file}.tex')
|
||||
os.chdir(current_dir)
|
||||
pdf_output = pj(work_folder, f'{main_file}.pdf')
|
||||
Returns:
|
||||
- A string specifying the path to the extracted folder, or the original folder if there is no extracted folder.
|
||||
"""
|
||||
maybe_dir = [f for f in glob.glob(f'{project_folder}/*') if os.path.isdir(f)]
|
||||
if len(maybe_dir) == 0: return project_folder
|
||||
if maybe_dir[0].endswith('.extract'): return maybe_dir[0]
|
||||
return project_folder
|
||||
|
||||
assert os.path.exists(pdf_output)
|
||||
return pdf_output
|
||||
def move_project(project_folder):
|
||||
"""
|
||||
Create a new work folder and copy the project folder to it.
|
||||
|
||||
def Latex预处理(tar_file):
|
||||
from toolbox import extract_archive
|
||||
import shutil
|
||||
work_folder = 'private_upload/latex_workshop_temp'
|
||||
try:
|
||||
shutil.rmtree(work_folder)
|
||||
except:
|
||||
pass
|
||||
res = extract_archive(tar_file, dest_dir=work_folder)
|
||||
for texf in glob.glob('private_upload/latex_workshop_temp/*.tex'):
|
||||
with open(texf, 'r', encoding='utf8') as f:
|
||||
file_content = f.read()
|
||||
if r'\documentclass' in file_content:
|
||||
return texf, work_folder
|
||||
else:
|
||||
continue
|
||||
raise RuntimeError('无法找到一个主Tex文件（包含documentclass关键字）')
|
||||
Args:
|
||||
- project_folder: A string specifying the folder path of the project.
|
||||
|
||||
Returns:
|
||||
- A string specifying the path to the new work folder.
|
||||
"""
|
||||
import shutil, time
|
||||
time.sleep(2) # avoid time string conflict
|
||||
new_workfolder = f'gpt_log/{gen_time_str()}'
|
||||
shutil.copytree(src=project_folder, dst=new_workfolder)
|
||||
return new_workfolder
|
||||
|
||||
|
||||
def 编译Latex差别(main_file_original, main_file_modified, work_folder_original, work_folder_modified, work_folder):
|
||||
import os
|
||||
current_dir = os.getcwd()
|
||||
n_fix = 0
|
||||
while True:
|
||||
# <--------------------->
|
||||
import os, shutil
|
||||
|
||||
# https://stackoverflow.com/questions/738755/dont-make-me-manually-abort-a-latex-compile-when-theres-an-error
|
||||
os.chdir(work_folder_original); os.system(f'pdflatex -interaction=batchmode -file-line-error {main_file_original}.tex'); os.chdir(current_dir)
|
||||
os.chdir(work_folder_modified); os.system(f'pdflatex -interaction=batchmode -file-line-error {main_file_modified}.tex'); os.chdir(current_dir)
|
||||
os.chdir(work_folder_original); os.system(f'bibtex {main_file_original}.aux'); os.chdir(current_dir)
|
||||
os.chdir(work_folder_modified); os.system(f'bibtex {main_file_modified}.aux'); os.chdir(current_dir)
|
||||
os.chdir(work_folder_original); os.system(f'pdflatex -interaction=batchmode -file-line-error {main_file_original}.tex'); os.chdir(current_dir)
|
||||
os.chdir(work_folder_modified); os.system(f'pdflatex -interaction=batchmode -file-line-error {main_file_modified}.tex'); os.chdir(current_dir)
|
||||
os.chdir(work_folder_original); os.system(f'pdflatex -interaction=batchmode -file-line-error {main_file_original}.tex'); os.chdir(current_dir)
|
||||
os.chdir(work_folder_modified); os.system(f'pdflatex -interaction=batchmode -file-line-error {main_file_modified}.tex'); os.chdir(current_dir)
|
||||
|
||||
print( f'latexdiff --encoding=utf8 --append-safecmd=subfile {work_folder_original}/{main_file_original}.tex {work_folder_modified}/{main_file_modified}.tex --flatten > {work_folder}/merge_diff.tex')
|
||||
os.system(f'latexdiff --encoding=utf8 --append-safecmd=subfile {work_folder_original}/{main_file_original}.tex {work_folder_modified}/{main_file_modified}.tex --flatten > {work_folder}/merge_diff.tex')
|
||||
|
||||
os.chdir(work_folder); os.system(f'pdflatex -interaction=batchmode -file-line-error merge_diff.tex'); os.chdir(current_dir)
|
||||
os.chdir(work_folder); os.system(f'bibtex merge_diff.aux'); os.chdir(current_dir)
|
||||
os.chdir(work_folder); os.system(f'pdflatex -interaction=batchmode -file-line-error merge_diff.tex'); os.chdir(current_dir)
|
||||
os.chdir(work_folder); os.system(f'pdflatex -interaction=batchmode -file-line-error merge_diff.tex'); os.chdir(current_dir)
|
||||
|
||||
# <--------------------->
|
||||
os.chdir(current_dir)
|
||||
|
||||
if os.path.exists(pj(work_folder_modified, f'{main_file_modified}.pdf')):
|
||||
return pj(work_folder_modified, f'{main_file_modified}.pdf')
|
||||
else:
|
||||
if n_fix>=10: break
|
||||
n_fix += 1
|
||||
can_retry, main_file_modified = remove_buggy_lines(file_path=pj(work_folder_modified, f'{main_file_modified}.tex'),
|
||||
log_path=pj(work_folder_modified, f'{main_file_modified}.log'),
|
||||
tex_name=f'{main_file_modified}.tex',
|
||||
tex_name_pure=f'{main_file_modified}',
|
||||
n_fix=n_fix,
|
||||
work_folder_modified=work_folder_modified,
|
||||
)
|
||||
if not can_retry: break
|
||||
|
||||
return "失败"
|
||||
|
||||
def remove_buggy_lines(file_path, log_path, tex_name, tex_name_pure, n_fix, work_folder_modified):
|
||||
try:
|
||||
with open(log_path, 'r', encoding='utf-8', errors='replace') as f:
|
||||
log = f.read()
|
||||
with open(file_path, 'r', encoding='utf-8', errors='replace') as f:
|
||||
file_lines = f.readlines()
|
||||
import re
|
||||
buggy_lines = re.findall(tex_name+':([0-9]{1,5}):', log)
|
||||
buggy_lines = [int(l) for l in buggy_lines]
|
||||
buggy_lines = sorted(buggy_lines)
|
||||
print("removing lines that has errors", buggy_lines)
|
||||
file_lines.pop(buggy_lines[0]-1)
|
||||
with open(pj(work_folder_modified, f"{tex_name_pure}_fix_{n_fix}.tex"), 'w', encoding='utf-8', errors='replace') as f:
|
||||
f.writelines(file_lines)
|
||||
return True, f"{tex_name_pure}_fix_{n_fix}"
|
||||
except:
|
||||
return False, 0
|
||||
def arxiv_download(chatbot, history, txt):
|
||||
if not txt.startswith('https://arxiv.org'):
|
||||
return txt
|
||||
|
||||
def Latex预处理(pfg, project_folder):
|
||||
import shutil, os
|
||||
work_folder = 'private_upload/latex_workshop_temp'
|
||||
|
||||
try:
|
||||
shutil.rmtree(work_folder)
|
||||
except:
|
||||
pass
|
||||
finally:
|
||||
work_folder_original = 'private_upload/latex_workshop_temp/original'
|
||||
work_folder_modified = 'private_upload/latex_workshop_temp/modified'
|
||||
shutil.copytree(project_folder, work_folder_original, ignore=lambda a,b: ['.git'])
|
||||
shutil.copytree(project_folder, work_folder_modified, ignore=lambda a,b: ['.git'])
|
||||
|
||||
for path, result in zip(pfg.file_paths, pfg.file_result):
|
||||
path_old = os.path.relpath(path, start=project_folder)
|
||||
path_new = pj(work_folder_modified, path_old)
|
||||
with open(path_new, 'w', encoding='utf-8') as f:
|
||||
f.write(result)
|
||||
|
||||
for main_file_original in glob.glob('private_upload/latex_workshop_temp/original/*.tex'):
|
||||
with open(main_file_original, 'r', encoding='utf8') as f:
|
||||
file_content = f.read()
|
||||
if r'\documentclass' in file_content:
|
||||
path_old = os.path.relpath(main_file_original, start=work_folder_original)
|
||||
main_file_modified = os.path.relpath(work_folder_modified, start=work_folder_original)
|
||||
return main_file_original, main_file_modified, work_folder_original, work_folder_modified, work_folder
|
||||
else:
|
||||
continue
|
||||
raise RuntimeError('无法找到一个主Tex文件, 本程序寻找主Tex文件的方法是查找文件中的documentclass关键字。')
|
||||
# <-------------- inspect format ------------->
|
||||
chatbot.append([f"检测到arxiv文档连接", '尝试下载 ...'])
|
||||
yield from update_ui(chatbot=chatbot, history=history)
|
||||
time.sleep(1) # 刷新界面
|
||||
|
||||
url_ = txt # https://arxiv.org/abs/1707.06690
|
||||
if not txt.startswith('https://arxiv.org/abs/'):
|
||||
msg = f"解析arxiv网址失败, 期望格式例如: https://arxiv.org/abs/1707.06690。实际得到格式: {url_}"
|
||||
yield from update_ui_lastest_msg(msg, chatbot=chatbot, history=history) # 刷新界面
|
||||
return msg
|
||||
|
||||
# <-------------- set format ------------->
|
||||
arxiv_id = url_.split('/abs/')[-1]
|
||||
url_tar = url_.replace('/abs/', '/e-print/')
|
||||
download_dir = './gpt_log/arxiv/'
|
||||
os.makedirs(download_dir, exist_ok=True)
|
||||
|
||||
# <-------------- download arxiv source file ------------->
|
||||
yield from update_ui_lastest_msg("开始下载", chatbot=chatbot, history=history) # 刷新界面
|
||||
proxies, = get_conf('proxies')
|
||||
r = requests.get(url_tar, proxies=proxies)
|
||||
dst = pj(download_dir, arxiv_id+'.tar')
|
||||
with open(dst, 'wb+') as f:
|
||||
f.write(r.content)
|
||||
|
||||
# <-------------- extract file ------------->
|
||||
yield from update_ui_lastest_msg("下载完成", chatbot=chatbot, history=history) # 刷新界面
|
||||
from toolbox import extract_archive
|
||||
extract_dst = f'gpt_log/{gen_time_str()}'
|
||||
extract_archive(file_path=dst, dest_dir=extract_dst)
|
||||
return extract_dst
|
||||
# ========================================= 插件主程序1 =====================================================
|
||||
|
||||
|
||||
@CatchException
|
||||
def Latex英文纠错加PDF对比(txt, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, web_port):
|
||||
# 基本信息：功能、贡献者
|
||||
chatbot.append([
|
||||
"函数插件功能？",
|
||||
"对整个Latex项目进行纠错，用latex编译为PDF对修正处做高亮。函数插件贡献者: Binary-Husky"])
|
||||
# <-------------- information about this plugin ------------->
|
||||
chatbot.append([ "函数插件功能？",
|
||||
"对整个Latex项目进行纠错, 用latex编译为PDF对修正处做高亮。函数插件贡献者: Binary-Husky"])
|
||||
yield from update_ui(chatbot=chatbot, history=history) # 刷新界面
|
||||
|
||||
# 尝试导入依赖，如果缺少依赖，则给出安装建议
|
||||
|
||||
# <-------------- check deps ------------->
|
||||
try:
|
||||
import glob, os
|
||||
import glob, os, time
|
||||
os.system(f'pdflatex -version')
|
||||
from .latex_utils import Latex精细分解与转化, 编译Latex差别
|
||||
except Exception as e:
|
||||
print(trimmed_format_exc())
|
||||
report_execption(chatbot, history, a=f"解析项目: {txt}",
|
||||
b=f"尝试执行Latex指令失败。Latex没有安装，或者不在环境变量PATH中。")
|
||||
chatbot.append([ f"解析项目: {txt}",
|
||||
f"尝试执行Latex指令失败。Latex没有安装, 或者不在环境变量PATH中。报错信息\n\n```\n\n{trimmed_format_exc()}\n\n```\n\n"])
|
||||
yield from update_ui(chatbot=chatbot, history=history) # 刷新界面
|
||||
return
|
||||
|
||||
history = [] # 清空历史，以免输入溢出
|
||||
|
||||
# <-------------- clear history and read input ------------->
|
||||
history = []
|
||||
txt = yield from arxiv_download(chatbot, history, txt)
|
||||
if os.path.exists(txt):
|
||||
project_folder = txt
|
||||
else:
|
||||
@ -259,34 +137,64 @@ def Latex英文纠错加PDF对比(txt, llm_kwargs, plugin_kwargs, chatbot, histo
|
||||
yield from update_ui(chatbot=chatbot, history=history) # 刷新界面
|
||||
return
|
||||
|
||||
|
||||
# <-------------- if is a zip/tar file ------------->
|
||||
project_folder = desend_to_extracted_folder_if_exist(project_folder)
|
||||
|
||||
|
||||
# <-------------- move latex project away from temp folder ------------->
|
||||
project_folder = move_project(project_folder)
|
||||
|
||||
|
||||
# <-------------- if merge_translate_zh is already generated, skip gpt req ------------->
|
||||
if not os.path.exists(project_folder + '/merge_proofread.tex'):
|
||||
yield from Latex精细分解与转化(file_manifest, project_folder, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, mode='proofread_latex')
|
||||
yield from Latex精细分解与转化(file_manifest, project_folder, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, mode='proofread_latex', switch_prompt=switch_prompt)
|
||||
|
||||
res_pdf_path = 编译Latex差别(main_file_original='merge', main_file_modified='merge_proofread',
|
||||
|
||||
# <-------------- compile PDF ------------->
|
||||
success = yield from 编译Latex差别(chatbot, history, main_file_original='merge', main_file_modified='merge_proofread',
|
||||
work_folder_original=project_folder, work_folder_modified=project_folder, work_folder=project_folder)
|
||||
return res_pdf_path
|
||||
|
||||
|
||||
# <-------------- zip PDF ------------->
|
||||
zip_result(project_folder)
|
||||
if success:
|
||||
chatbot.append((f"成功啦", '请查收结果（压缩包）...'))
|
||||
yield from update_ui(chatbot=chatbot, history=history); time.sleep(1) # 刷新界面
|
||||
else:
|
||||
chatbot.append((f"失败了", '虽然PDF生成失败了, 但请查收结果（压缩包）, 内含已经翻译的Tex文档, 也是可读的, 您可以到Github Issue区, 用该压缩包+对话历史存档进行反馈 ...'))
|
||||
yield from update_ui(chatbot=chatbot, history=history); time.sleep(1) # 刷新界面
|
||||
|
||||
# <-------------- we are done ------------->
|
||||
return success
|
||||
|
||||
|
||||
# ========================================= 插件主程序2 =====================================================
|
||||
|
||||
@CatchException
|
||||
def Latex翻译中文并重新编译PDF(txt, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, web_port):
|
||||
# 基本信息：功能、贡献者
|
||||
# <-------------- information about this plugin ------------->
|
||||
chatbot.append([
|
||||
"函数插件功能？",
|
||||
"对整个Latex项目进行翻译，生成中文PDF。函数插件贡献者: Binary-Husky"])
|
||||
"对整个Latex项目进行翻译, 生成中文PDF。函数插件贡献者: Binary-Husky"])
|
||||
yield from update_ui(chatbot=chatbot, history=history) # 刷新界面
|
||||
|
||||
# 尝试导入依赖，如果缺少依赖，则给出安装建议
|
||||
|
||||
# <-------------- check deps ------------->
|
||||
try:
|
||||
import glob, os
|
||||
import glob, os, time
|
||||
os.system(f'pdflatex -version')
|
||||
from .latex_utils import Latex精细分解与转化, 编译Latex差别
|
||||
except Exception as e:
|
||||
print(trimmed_format_exc())
|
||||
report_execption(chatbot, history, a=f"解析项目: {txt}",
|
||||
b=f"尝试执行Latex指令失败。Latex没有安装，或者不在环境变量PATH中。")
|
||||
chatbot.append([ f"解析项目: {txt}",
|
||||
f"尝试执行Latex指令失败。Latex没有安装, 或者不在环境变量PATH中。报错信息\n\n```\n\n{trimmed_format_exc()}\n\n```\n\n"])
|
||||
yield from update_ui(chatbot=chatbot, history=history) # 刷新界面
|
||||
return
|
||||
|
||||
history = [] # 清空历史，以免输入溢出
|
||||
|
||||
# <-------------- clear history and read input ------------->
|
||||
history = []
|
||||
txt = yield from arxiv_download(chatbot, history, txt)
|
||||
if os.path.exists(txt):
|
||||
project_folder = txt
|
||||
else:
|
||||
@ -300,9 +208,32 @@ def Latex翻译中文并重新编译PDF(txt, llm_kwargs, plugin_kwargs, chatbot,
|
||||
yield from update_ui(chatbot=chatbot, history=history) # 刷新界面
|
||||
return
|
||||
|
||||
if not os.path.exists(project_folder + '/merge_translate_zh.tex'):
|
||||
yield from Latex精细分解与转化(file_manifest, project_folder, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, mode='translate_zh')
|
||||
|
||||
res_pdf_path = 编译Latex差别(main_file_original='merge', main_file_modified='merge_translate_zh',
|
||||
# <-------------- if is a zip/tar file ------------->
|
||||
project_folder = desend_to_extracted_folder_if_exist(project_folder)
|
||||
|
||||
|
||||
# <-------------- move latex project away from temp folder ------------->
|
||||
project_folder = move_project(project_folder)
|
||||
|
||||
|
||||
# <-------------- if merge_translate_zh is already generated, skip gpt req ------------->
|
||||
if not os.path.exists(project_folder + '/merge_translate_zh.tex'):
|
||||
yield from Latex精细分解与转化(file_manifest, project_folder, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, mode='translate_zh', switch_prompt=switch_prompt)
|
||||
|
||||
|
||||
# <-------------- compile PDF ------------->
|
||||
success = yield from 编译Latex差别(chatbot, history, main_file_original='merge', main_file_modified='merge_translate_zh',
|
||||
work_folder_original=project_folder, work_folder_modified=project_folder, work_folder=project_folder)
|
||||
return res_pdf_path
|
||||
|
||||
# <-------------- zip PDF ------------->
|
||||
zip_result(project_folder)
|
||||
if success:
|
||||
chatbot.append((f"成功啦", '请查收结果（压缩包）...'))
|
||||
yield from update_ui(chatbot=chatbot, history=history); time.sleep(1) # 刷新界面
|
||||
else:
|
||||
chatbot.append((f"失败了", '虽然PDF生成失败了, 但请查收结果（压缩包）, 内含已经翻译的Tex文档, 也是可读的, 您可以到Github Issue区, 用该压缩包+对话历史存档进行反馈 ...'))
|
||||
yield from update_ui(chatbot=chatbot, history=history); time.sleep(1) # 刷新界面
|
||||
|
||||
# <-------------- we are done ------------->
|
||||
return success
|
||||
|
||||
@ -1,7 +1,8 @@
|
||||
from toolbox import update_ui, trimmed_format_exc
|
||||
from toolbox import CatchException, report_execption, write_results_to_file, zip_folder
|
||||
from toolbox import update_ui, update_ui_lastest_msg # 刷新Gradio前端界面
|
||||
from toolbox import zip_folder
|
||||
import os
|
||||
import re
|
||||
pj = os.path.join
|
||||
|
||||
def 寻找Latex主文件(file_manifest, mode):
|
||||
for texf in file_manifest:
|
||||
@ -49,7 +50,7 @@ def mod_inbraket(match):
|
||||
|
||||
# modify the matched string
|
||||
str_to_modify = str_to_modify.replace('：', ':')
|
||||
str_to_modify = str_to_modify.replace('，', ',')
|
||||
str_to_modify = str_to_modify.replace(', ', ',')
|
||||
# str_to_modify = 'BOOM'
|
||||
# return the modified string as the replacement
|
||||
return "\\" + cmd + "{" + str_to_modify + "}"
|
||||
@ -215,3 +216,226 @@ class LatexPaperFileGroup():
|
||||
folder = os.path.dirname(self.file_paths[0])
|
||||
t = time.strftime("%Y-%m-%d-%H-%M-%S", time.localtime())
|
||||
zip_folder(folder, './gpt_log/', f'{t}-polished.zip')
|
||||
|
||||
|
||||
|
||||
def Latex精细分解与转化(file_manifest, project_folder, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, mode='proofread', switch_prompt=None):
|
||||
import time, os, re
|
||||
from .crazy_utils import request_gpt_model_multi_threads_with_very_awesome_ui_and_high_efficiency
|
||||
from .latex_utils import LatexPaperFileGroup, merge_tex_files, LatexPaperSplit, 寻找Latex主文件
|
||||
|
||||
# <-------- 寻找主tex文件 ---------->
|
||||
maintex = 寻找Latex主文件(file_manifest, mode)
|
||||
chatbot.append((f"定位主Latex文件", f'[Local Message] 分析结果：该项目的Latex主文件是{maintex}, 如果分析错误, 请立即终止程序, 删除或修改歧义文件, 然后重试。主程序即将开始, 请稍候。'))
|
||||
yield from update_ui(chatbot=chatbot, history=history) # 刷新界面
|
||||
time.sleep(5)
|
||||
|
||||
# <-------- 读取Latex文件, 将多文件tex工程融合为一个巨型tex ---------->
|
||||
with open(maintex, 'r', encoding='utf-8', errors='replace') as f:
|
||||
content = f.read()
|
||||
merged_content = merge_tex_files(project_folder, content, mode)
|
||||
merged_content = re.sub(r'(?<!\\)%.*', '', merged_content) # 使用正则表达式查找注释, 并替换为空字符串
|
||||
|
||||
with open(project_folder + '/merge.tex', 'w', encoding='utf-8', errors='replace') as f:
|
||||
f.write(merged_content)
|
||||
|
||||
# <-------- 精细切分latex文件 ---------->
|
||||
lps = LatexPaperSplit()
|
||||
res = lps.split(merged_content)
|
||||
|
||||
# <-------- 拆分过长的latex片段 ---------->
|
||||
pfg = LatexPaperFileGroup()
|
||||
for index, r in enumerate(res):
|
||||
pfg.file_paths.append(index)
|
||||
pfg.file_contents.append(r)
|
||||
|
||||
pfg.run_file_split(max_token_limit=1024)
|
||||
n_split = len(pfg.sp_file_contents)
|
||||
|
||||
# <-------- 根据需要切换prompt ---------->
|
||||
inputs_array, sys_prompt_array = switch_prompt(pfg, mode)
|
||||
inputs_show_user_array = [f"{mode} {f}" for f in pfg.sp_file_tag]
|
||||
|
||||
# <-------- gpt 多线程请求 ---------->
|
||||
gpt_response_collection = yield from request_gpt_model_multi_threads_with_very_awesome_ui_and_high_efficiency(
|
||||
inputs_array=inputs_array,
|
||||
inputs_show_user_array=inputs_show_user_array,
|
||||
llm_kwargs=llm_kwargs,
|
||||
chatbot=chatbot,
|
||||
history_array=[[""] for _ in range(n_split)],
|
||||
sys_prompt_array=sys_prompt_array,
|
||||
# max_workers=5, # 并行任务数量限制, 最多同时执行5个, 其他的排队等待
|
||||
scroller_max_len = 80
|
||||
)
|
||||
|
||||
# <-------- 文本碎片重组为完整的tex片段 ---------->
|
||||
pfg.sp_file_result = []
|
||||
for i_say, gpt_say, orig_content in zip(gpt_response_collection[0::2], gpt_response_collection[1::2], pfg.sp_file_contents):
|
||||
pfg.sp_file_result.append(gpt_say)
|
||||
pfg.merge_result()
|
||||
|
||||
# # <-------- 临时存储用于调试 ---------->
|
||||
# pfg.get_token_num = None
|
||||
# objdump(pfg)
|
||||
# pfg = objload()
|
||||
|
||||
# <-------- 写出文件 ---------->
|
||||
final_tex = lps.merge_result(pfg.sp_file_result)
|
||||
with open(project_folder + f'/merge_{mode}.tex', 'w', encoding='utf-8', errors='replace') as f:
|
||||
f.write(final_tex)
|
||||
|
||||
# <-------- 整理结果, 退出 ---------->
|
||||
chatbot.append((f"完成了吗？", 'GPT结果已输出, 正在编译PDF'))
|
||||
yield from update_ui(chatbot=chatbot, history=history) # 刷新界面
|
||||
|
||||
# <-------- 返回 ---------->
|
||||
return project_folder + f'/merge_{mode}.tex'
|
||||
|
||||
|
||||
# def Latex预处理(pfg, project_folder):
|
||||
# import shutil, os
|
||||
# work_folder = 'private_upload/latex_workshop_temp'
|
||||
|
||||
# try:
|
||||
# shutil.rmtree(work_folder)
|
||||
# except:
|
||||
# pass
|
||||
# finally:
|
||||
# work_folder_original = 'private_upload/latex_workshop_temp/original'
|
||||
# work_folder_modified = 'private_upload/latex_workshop_temp/modified'
|
||||
# shutil.copytree(project_folder, work_folder_original, ignore=lambda a,b: ['.git'])
|
||||
# shutil.copytree(project_folder, work_folder_modified, ignore=lambda a,b: ['.git'])
|
||||
|
||||
# for path, result in zip(pfg.file_paths, pfg.file_result):
|
||||
# path_old = os.path.relpath(path, start=project_folder)
|
||||
# path_new = pj(work_folder_modified, path_old)
|
||||
# with open(path_new, 'w', encoding='utf-8') as f:
|
||||
# f.write(result)
|
||||
|
||||
# for main_file_original in glob.glob('private_upload/latex_workshop_temp/original/*.tex'):
|
||||
# with open(main_file_original, 'r', encoding='utf8') as f:
|
||||
# file_content = f.read()
|
||||
# if r'\documentclass' in file_content:
|
||||
# path_old = os.path.relpath(main_file_original, start=work_folder_original)
|
||||
# main_file_modified = os.path.relpath(work_folder_modified, start=work_folder_original)
|
||||
# return main_file_original, main_file_modified, work_folder_original, work_folder_modified, work_folder
|
||||
# else:
|
||||
# continue
|
||||
# raise RuntimeError('无法找到一个主Tex文件, 本程序寻找主Tex文件的方法是查找文件中的documentclass关键字。')
|
||||
|
||||
|
||||
# def Latex预处理(tar_file):
|
||||
# from toolbox import extract_archive
|
||||
# import shutil
|
||||
# work_folder = 'private_upload/latex_workshop_temp'
|
||||
# try:
|
||||
# shutil.rmtree(work_folder)
|
||||
# except:
|
||||
# pass
|
||||
# res = extract_archive(tar_file, dest_dir=work_folder)
|
||||
# for texf in glob.glob('private_upload/latex_workshop_temp/*.tex'):
|
||||
# with open(texf, 'r', encoding='utf8') as f:
|
||||
# file_content = f.read()
|
||||
# if r'\documentclass' in file_content:
|
||||
# return texf, work_folder
|
||||
# else:
|
||||
# continue
|
||||
# raise RuntimeError('无法找到一个主Tex文件（包含documentclass关键字）')
|
||||
|
||||
|
||||
|
||||
|
||||
def remove_buggy_lines(file_path, log_path, tex_name, tex_name_pure, n_fix, work_folder_modified):
|
||||
try:
|
||||
|
||||
with open(log_path, 'r', encoding='utf-8', errors='replace') as f:
|
||||
log = f.read()
|
||||
with open(file_path, 'r', encoding='utf-8', errors='replace') as f:
|
||||
file_lines = f.readlines()
|
||||
import re
|
||||
buggy_lines = re.findall(tex_name+':([0-9]{1,5}):', log)
|
||||
buggy_lines = [int(l) for l in buggy_lines]
|
||||
buggy_lines = sorted(buggy_lines)
|
||||
print("removing lines that has errors", buggy_lines)
|
||||
file_lines.pop(buggy_lines[0]-1)
|
||||
with open(pj(work_folder_modified, f"{tex_name_pure}_fix_{n_fix}.tex"), 'w', encoding='utf-8', errors='replace') as f:
|
||||
f.writelines(file_lines)
|
||||
return True, f"{tex_name_pure}_fix_{n_fix}", buggy_lines
|
||||
|
||||
except:
|
||||
return False, 0
|
||||
|
||||
|
||||
|
||||
def 编译Latex差别(chatbot, history, main_file_original, main_file_modified, work_folder_original, work_folder_modified, work_folder):
|
||||
import os, time
|
||||
current_dir = os.getcwd()
|
||||
n_fix = 0
|
||||
chatbot.append([f"正在编译PDF文档", '编译已经开始...']); yield from update_ui(chatbot=chatbot, history=history); time.sleep(1); chatbot[-1] = list(chatbot[-1]) # 刷新界面
|
||||
yield from update_ui_lastest_msg('编译已经开始...', chatbot, history) # 刷新Gradio前端界面
|
||||
|
||||
while True:
|
||||
import os
|
||||
# https://stackoverflow.com/questions/738755/dont-make-me-manually-abort-a-latex-compile-when-theres-an-error
|
||||
yield from update_ui_lastest_msg(f'尝试第{n_fix}编译, 编译原始PDF ...', chatbot, history) # 刷新Gradio前端界面
|
||||
os.chdir(work_folder_original); os.system(f'pdflatex -interaction=batchmode -file-line-error {main_file_original}.tex'); os.chdir(current_dir)
|
||||
|
||||
yield from update_ui_lastest_msg(f'尝试第{n_fix}编译, 编译转化后的PDF ...', chatbot, history) # 刷新Gradio前端界面
|
||||
os.chdir(work_folder_modified); os.system(f'pdflatex -interaction=batchmode -file-line-error {main_file_modified}.tex'); os.chdir(current_dir)
|
||||
|
||||
yield from update_ui_lastest_msg(f'尝试第{n_fix}编译, 编译BibTex ...', chatbot, history) # 刷新Gradio前端界面
|
||||
os.chdir(work_folder_original); os.system(f'bibtex {main_file_original}.aux'); os.chdir(current_dir)
|
||||
os.chdir(work_folder_modified); os.system(f'bibtex {main_file_modified}.aux'); os.chdir(current_dir)
|
||||
|
||||
yield from update_ui_lastest_msg(f'尝试第{n_fix}编译, 编译文献交叉引用 ...', chatbot, history) # 刷新Gradio前端界面
|
||||
os.chdir(work_folder_original); os.system(f'pdflatex -interaction=batchmode -file-line-error {main_file_original}.tex'); os.chdir(current_dir)
|
||||
os.chdir(work_folder_modified); os.system(f'pdflatex -interaction=batchmode -file-line-error {main_file_modified}.tex'); os.chdir(current_dir)
|
||||
os.chdir(work_folder_original); os.system(f'pdflatex -interaction=batchmode -file-line-error {main_file_original}.tex'); os.chdir(current_dir)
|
||||
os.chdir(work_folder_modified); os.system(f'pdflatex -interaction=batchmode -file-line-error {main_file_modified}.tex'); os.chdir(current_dir)
|
||||
|
||||
yield from update_ui_lastest_msg(f'尝试第{n_fix}编译, 使用latexdiff生成论文转化前后对比 ...', chatbot, history) # 刷新Gradio前端界面
|
||||
print( f'latexdiff --encoding=utf8 --append-safecmd=subfile {work_folder_original}/{main_file_original}.tex {work_folder_modified}/{main_file_modified}.tex --flatten > {work_folder}/merge_diff.tex')
|
||||
os.system(f'latexdiff --encoding=utf8 --append-safecmd=subfile {work_folder_original}/{main_file_original}.tex {work_folder_modified}/{main_file_modified}.tex --flatten > {work_folder}/merge_diff.tex')
|
||||
|
||||
yield from update_ui_lastest_msg(f'尝试第{n_fix}编译, 正在编译对比PDF ...', chatbot, history) # 刷新Gradio前端界面
|
||||
os.chdir(work_folder); os.system(f'pdflatex -interaction=batchmode -file-line-error merge_diff.tex'); os.chdir(current_dir)
|
||||
os.chdir(work_folder); os.system(f'bibtex merge_diff.aux'); os.chdir(current_dir)
|
||||
os.chdir(work_folder); os.system(f'pdflatex -interaction=batchmode -file-line-error merge_diff.tex'); os.chdir(current_dir)
|
||||
os.chdir(work_folder); os.system(f'pdflatex -interaction=batchmode -file-line-error merge_diff.tex'); os.chdir(current_dir)
|
||||
|
||||
# <--------------------->
|
||||
os.chdir(current_dir)
|
||||
|
||||
# <---------- 检查结果 ----------->
|
||||
results_ = ""
|
||||
original_pdf_success = os.path.exists(pj(work_folder_original, f'{main_file_original}.pdf'))
|
||||
modified_pdf_success = os.path.exists(pj(work_folder_modified, f'{main_file_modified}.pdf'))
|
||||
diff_pdf_success = os.path.exists(pj(work_folder, f'merge_diff.pdf'))
|
||||
results_ += f"原始PDF编译是否成功: {original_pdf_success};"
|
||||
results_ += f"转化PDF编译是否成功: {modified_pdf_success};"
|
||||
results_ += f"对比PDF编译是否成功: {diff_pdf_success};"
|
||||
yield from update_ui_lastest_msg(f'第{n_fix}编译结束:<br/>{results_}...', chatbot, history) # 刷新Gradio前端界面
|
||||
|
||||
if modified_pdf_success:
|
||||
yield from update_ui_lastest_msg(f'转化PDF编译已经成功, 即将退出 ...', chatbot, history) # 刷新Gradio前端界面
|
||||
os.chdir(current_dir)
|
||||
return True # 成功啦
|
||||
else:
|
||||
if n_fix>=10: break
|
||||
n_fix += 1
|
||||
can_retry, main_file_modified, buggy_lines = remove_buggy_lines(
|
||||
file_path=pj(work_folder_modified, f'{main_file_modified}.tex'),
|
||||
log_path=pj(work_folder_modified, f'{main_file_modified}.log'),
|
||||
tex_name=f'{main_file_modified}.tex',
|
||||
tex_name_pure=f'{main_file_modified}',
|
||||
n_fix=n_fix,
|
||||
work_folder_modified=work_folder_modified,
|
||||
)
|
||||
yield from update_ui_lastest_msg(f'由于最为关键的转化PDF编译失败, 将根据报错信息修正tex源文件并重试, 当前报错的latex代码处于第{buggy_lines}行 ...', chatbot, history) # 刷新Gradio前端界面
|
||||
if not can_retry: break
|
||||
|
||||
os.chdir(current_dir)
|
||||
return False # 失败啦
|
||||
|
||||
|
||||
|
||||
|
||||
21
toolbox.py
21
toolbox.py
@ -1,6 +1,6 @@
|
||||
import markdown
|
||||
import importlib
|
||||
import traceback
|
||||
import time
|
||||
import inspect
|
||||
import re
|
||||
import os
|
||||
@ -78,6 +78,17 @@ def update_ui(chatbot, history, msg='正常', **kwargs): # 刷新界面
|
||||
assert isinstance(chatbot, ChatBotWithCookies), "在传递chatbot的过程中不要将其丢弃。必要时，可用clear将其清空，然后用for+append循环重新赋值。"
|
||||
yield chatbot.get_cookies(), chatbot, history, msg
|
||||
|
||||
def update_ui_lastest_msg(lastmsg, chatbot, history, delay=1): # 刷新界面
|
||||
"""
|
||||
刷新用户界面
|
||||
"""
|
||||
if len(chatbot) == 0: chatbot.append(["update_ui_last_msg", lastmsg])
|
||||
chatbot[-1] = list(chatbot[-1])
|
||||
chatbot[-1][-1] = lastmsg
|
||||
yield from update_ui(chatbot=chatbot, history=history)
|
||||
time.sleep(delay)
|
||||
|
||||
|
||||
def trimmed_format_exc():
|
||||
import os, traceback
|
||||
str = traceback.format_exc()
|
||||
@ -772,6 +783,11 @@ def zip_folder(source_folder, dest_folder, zip_name):
|
||||
|
||||
print(f"Zip file created at {zip_file}")
|
||||
|
||||
def zip_result(folder):
|
||||
import time
|
||||
t = time.strftime("%Y-%m-%d-%H-%M-%S", time.localtime())
|
||||
zip_folder(folder, './gpt_log/', f'{t}-result.zip')
|
||||
|
||||
def gen_time_str():
|
||||
import time
|
||||
return time.strftime("%Y-%m-%d-%H-%M-%S", time.localtime())
|
||||
@ -806,4 +822,5 @@ def objload():
|
||||
if not os.path.exists('objdump.tmp'):
|
||||
return
|
||||
with open('objdump.tmp', 'rb') as f:
|
||||
return pickle.load(f)
|
||||
return pickle.load(f)
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user