From ef612212416277d3637f9692eca48bac421d078f Mon Sep 17 00:00:00 2001 From: qingxu fu <505030475@qq.com> Date: Sat, 3 Jun 2023 13:46:40 +0800 Subject: [PATCH] latex auto translation milestone --- crazy_functional.py | 23 ++ crazy_functions/Latex输出PDF结果.py | 375 ++++++++++++---------------- crazy_functions/latex_utils.py | 230 ++++++++++++++++- toolbox.py | 21 +- 4 files changed, 422 insertions(+), 227 deletions(-) diff --git a/crazy_functional.py b/crazy_functional.py index 9ce8ee1..10d9ea2 100644 --- a/crazy_functional.py +++ b/crazy_functional.py @@ -319,5 +319,28 @@ def get_crazy_functions(): except: print('Load function plugin failed') + try: + from crazy_functions.Latex输出PDF结果 import Latex英文纠错加PDF对比 + function_plugins.update({ + "Latex英文纠错+LatexDiff高亮修正位置": { + "Color": "stop", + "AsButton": False, + # "AdvancedArgs": True, + # "ArgsReminder": "", + "Function": HotReload(Latex英文纠错加PDF对比) + } + }) + from crazy_functions.Latex输出PDF结果 import Latex翻译中文并重新编译PDF + function_plugins.update({ + "Latex翻译中文+生成PDF": { + "Color": "stop", + "AsButton": False, + # "AdvancedArgs": True, + # "ArgsReminder": "", + "Function": HotReload(Latex翻译中文并重新编译PDF) + } + }) + except: + print('Load function plugin failed') ###################### 第n组插件 ########################### return function_plugins diff --git a/crazy_functions/Latex输出PDF结果.py b/crazy_functions/Latex输出PDF结果.py index b3b7ec6..60982fe 100644 --- a/crazy_functions/Latex输出PDF结果.py +++ b/crazy_functions/Latex输出PDF结果.py @@ -1,78 +1,20 @@ -from toolbox import update_ui, trimmed_format_exc, objdump, objload -from toolbox import CatchException, report_execption, write_results_to_file, zip_folder -import glob, copy, os +from toolbox import update_ui, trimmed_format_exc, get_conf, objdump, objload +from toolbox import CatchException, report_execption, update_ui_lastest_msg, zip_result, gen_time_str +import glob, os, requests, time pj = os.path.join - -def confirm_answer_is_health(bufo, buf, llm_kwargs, default = True): - return len(buf) >= len(bufo) // 3 - -def Latex精细分解与转化(file_manifest, project_folder, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, mode='proofread'): - import time, os, re - from .crazy_utils import request_gpt_model_multi_threads_with_very_awesome_ui_and_high_efficiency - from .latex_utils import LatexPaperFileGroup, merge_tex_files, LatexPaperSplit, 寻找Latex主文件 - - maintex = 寻找Latex主文件(file_manifest, mode) - - # <-------- 读取Latex文件，删除其中的所有注释 ----------> - with open(maintex, 'r', encoding='utf-8', errors='replace') as f: - content = f.read() - merged_content = merge_tex_files(project_folder, content, mode) - merged_content = re.sub(r'(? - pfg = LatexPaperFileGroup() - for index, r in enumerate(res): - pfg.file_paths.append(index) - pfg.file_contents.append(r) - - pfg.run_file_split(max_token_limit=1024) - n_split = len(pfg.sp_file_contents) - - inputs_array, sys_prompt_array = switch_prompt(pfg, mode) - inputs_show_user_array = [f"{mode} {f}" for f in pfg.sp_file_tag] - - gpt_response_collection = yield from request_gpt_model_multi_threads_with_very_awesome_ui_and_high_efficiency( - inputs_array=inputs_array, - inputs_show_user_array=inputs_show_user_array, - llm_kwargs=llm_kwargs, - chatbot=chatbot, - history_array=[[""] for _ in range(n_split)], - sys_prompt_array=sys_prompt_array, - # max_workers=5, # 并行任务数量限制，最多同时执行5个，其他的排队等待 - scroller_max_len = 80 - ) - - # <-------- 文本碎片重组为完整的tex片段 ----------> - pfg.sp_file_result = [] - for i_say, gpt_say, orig_content in zip(gpt_response_collection[0::2], gpt_response_collection[1::2], pfg.sp_file_contents): - pfg.sp_file_result.append(gpt_say) - pfg.merge_result() - - # <-------- 临时存储用于调试 ----------> - pfg.get_token_num = None - objdump(pfg) - pfg = objload() - - # <-------- 写出文件 ----------> - final_tex = lps.merge_result(pfg.sp_file_result) - with open(project_folder + f'/merge_{mode}.tex', 'w', encoding='utf-8', errors='replace') as f: - f.write(final_tex) - # <-------- 整理结果，退出 ----------> - # create_report_file_name = time.strftime("%Y-%m-%d-%H-%M-%S", time.localtime()) + f"-chatgpt.polish.md" - # res = write_results_to_file(gpt_response_collection, file_name=create_report_file_name) - # history = gpt_response_collection - chatbot.append((f"完成了吗？", 'GPT结果已输出，正在编译PDF')) - yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 - return project_folder + f'/merge_{mode}.tex' - +# =================================== 工具函数 =============================================== def switch_prompt(pfg, mode): + """ + Generate prompts and system prompts based on the mode for proofreading or translating. + Args: + - pfg: Proofreader or Translator instance. + - mode: A string specifying the mode, either 'proofread' or 'translate_zh'. + + Returns: + - inputs_array: A list of strings containing prompts for users to respond to. + - sys_prompt_array: A list of strings containing prompts for system prompts. + """ n_split = len(pfg.sp_file_contents) if mode == 'proofread': inputs_array = [r"Below is a section from an academic paper, proofread this section." + @@ -90,162 +32,98 @@ def switch_prompt(pfg, mode): assert False, "未知指令" return inputs_array, sys_prompt_array +def desend_to_extracted_folder_if_exist(project_folder): + """ + Descend into the extracted folder if it exists, otherwise return the original folder. + Args: + - project_folder: A string specifying the folder path. -def 编译Latex(main_tex, work_folder): - import os - current_dir = os.getcwd() - os.chdir(work_folder) - main_file = os.path.basename(main_tex) - assert main_file.endswith('.tex') - main_file = main_file[:-4] - os.system(f'pdflatex {main_file}.tex') - os.system(f'bibtex {main_file}.aux') - os.system(f'pdflatex {main_file}.tex') - os.system(f'pdflatex {main_file}.tex') - os.chdir(current_dir) - pdf_output = pj(work_folder, f'{main_file}.pdf') + Returns: + - A string specifying the path to the extracted folder, or the original folder if there is no extracted folder. + """ + maybe_dir = [f for f in glob.glob(f'{project_folder}/*') if os.path.isdir(f)] + if len(maybe_dir) == 0: return project_folder + if maybe_dir[0].endswith('.extract'): return maybe_dir[0] + return project_folder - assert os.path.exists(pdf_output) - return pdf_output +def move_project(project_folder): + """ + Create a new work folder and copy the project folder to it. -def Latex预处理(tar_file): - from toolbox import extract_archive - import shutil - work_folder = 'private_upload/latex_workshop_temp' - try: - shutil.rmtree(work_folder) - except: - pass - res = extract_archive(tar_file, dest_dir=work_folder) - for texf in glob.glob('private_upload/latex_workshop_temp/*.tex'): - with open(texf, 'r', encoding='utf8') as f: - file_content = f.read() - if r'\documentclass' in file_content: - return texf, work_folder - else: - continue - raise RuntimeError('无法找到一个主Tex文件（包含documentclass关键字）') + Args: + - project_folder: A string specifying the folder path of the project. + Returns: + - A string specifying the path to the new work folder. + """ + import shutil, time + time.sleep(2) # avoid time string conflict + new_workfolder = f'gpt_log/{gen_time_str()}' + shutil.copytree(src=project_folder, dst=new_workfolder) + return new_workfolder - -def 编译Latex差别(main_file_original, main_file_modified, work_folder_original, work_folder_modified, work_folder): - import os - current_dir = os.getcwd() - n_fix = 0 - while True: - # <---------------------> - import os, shutil - - # https://stackoverflow.com/questions/738755/dont-make-me-manually-abort-a-latex-compile-when-theres-an-error - os.chdir(work_folder_original); os.system(f'pdflatex -interaction=batchmode -file-line-error {main_file_original}.tex'); os.chdir(current_dir) - os.chdir(work_folder_modified); os.system(f'pdflatex -interaction=batchmode -file-line-error {main_file_modified}.tex'); os.chdir(current_dir) - os.chdir(work_folder_original); os.system(f'bibtex {main_file_original}.aux'); os.chdir(current_dir) - os.chdir(work_folder_modified); os.system(f'bibtex {main_file_modified}.aux'); os.chdir(current_dir) - os.chdir(work_folder_original); os.system(f'pdflatex -interaction=batchmode -file-line-error {main_file_original}.tex'); os.chdir(current_dir) - os.chdir(work_folder_modified); os.system(f'pdflatex -interaction=batchmode -file-line-error {main_file_modified}.tex'); os.chdir(current_dir) - os.chdir(work_folder_original); os.system(f'pdflatex -interaction=batchmode -file-line-error {main_file_original}.tex'); os.chdir(current_dir) - os.chdir(work_folder_modified); os.system(f'pdflatex -interaction=batchmode -file-line-error {main_file_modified}.tex'); os.chdir(current_dir) - - print( f'latexdiff --encoding=utf8 --append-safecmd=subfile {work_folder_original}/{main_file_original}.tex {work_folder_modified}/{main_file_modified}.tex --flatten > {work_folder}/merge_diff.tex') - os.system(f'latexdiff --encoding=utf8 --append-safecmd=subfile {work_folder_original}/{main_file_original}.tex {work_folder_modified}/{main_file_modified}.tex --flatten > {work_folder}/merge_diff.tex') - - os.chdir(work_folder); os.system(f'pdflatex -interaction=batchmode -file-line-error merge_diff.tex'); os.chdir(current_dir) - os.chdir(work_folder); os.system(f'bibtex merge_diff.aux'); os.chdir(current_dir) - os.chdir(work_folder); os.system(f'pdflatex -interaction=batchmode -file-line-error merge_diff.tex'); os.chdir(current_dir) - os.chdir(work_folder); os.system(f'pdflatex -interaction=batchmode -file-line-error merge_diff.tex'); os.chdir(current_dir) - - # <---------------------> - os.chdir(current_dir) - - if os.path.exists(pj(work_folder_modified, f'{main_file_modified}.pdf')): - return pj(work_folder_modified, f'{main_file_modified}.pdf') - else: - if n_fix>=10: break - n_fix += 1 - can_retry, main_file_modified = remove_buggy_lines(file_path=pj(work_folder_modified, f'{main_file_modified}.tex'), - log_path=pj(work_folder_modified, f'{main_file_modified}.log'), - tex_name=f'{main_file_modified}.tex', - tex_name_pure=f'{main_file_modified}', - n_fix=n_fix, - work_folder_modified=work_folder_modified, - ) - if not can_retry: break - - return "失败" - -def remove_buggy_lines(file_path, log_path, tex_name, tex_name_pure, n_fix, work_folder_modified): - try: - with open(log_path, 'r', encoding='utf-8', errors='replace') as f: - log = f.read() - with open(file_path, 'r', encoding='utf-8', errors='replace') as f: - file_lines = f.readlines() - import re - buggy_lines = re.findall(tex_name+':([0-9]{1,5}):', log) - buggy_lines = [int(l) for l in buggy_lines] - buggy_lines = sorted(buggy_lines) - print("removing lines that has errors", buggy_lines) - file_lines.pop(buggy_lines[0]-1) - with open(pj(work_folder_modified, f"{tex_name_pure}_fix_{n_fix}.tex"), 'w', encoding='utf-8', errors='replace') as f: - f.writelines(file_lines) - return True, f"{tex_name_pure}_fix_{n_fix}" - except: - return False, 0 +def arxiv_download(chatbot, history, txt): + if not txt.startswith('https://arxiv.org'): + return txt -def Latex预处理(pfg, project_folder): - import shutil, os - work_folder = 'private_upload/latex_workshop_temp' - - try: - shutil.rmtree(work_folder) - except: - pass - finally: - work_folder_original = 'private_upload/latex_workshop_temp/original' - work_folder_modified = 'private_upload/latex_workshop_temp/modified' - shutil.copytree(project_folder, work_folder_original, ignore=lambda a,b: ['.git']) - shutil.copytree(project_folder, work_folder_modified, ignore=lambda a,b: ['.git']) - - for path, result in zip(pfg.file_paths, pfg.file_result): - path_old = os.path.relpath(path, start=project_folder) - path_new = pj(work_folder_modified, path_old) - with open(path_new, 'w', encoding='utf-8') as f: - f.write(result) - - for main_file_original in glob.glob('private_upload/latex_workshop_temp/original/*.tex'): - with open(main_file_original, 'r', encoding='utf8') as f: - file_content = f.read() - if r'\documentclass' in file_content: - path_old = os.path.relpath(main_file_original, start=work_folder_original) - main_file_modified = os.path.relpath(work_folder_modified, start=work_folder_original) - return main_file_original, main_file_modified, work_folder_original, work_folder_modified, work_folder - else: - continue - raise RuntimeError('无法找到一个主Tex文件, 本程序寻找主Tex文件的方法是查找文件中的documentclass关键字。') + # <-------------- inspect format -------------> + chatbot.append([f"检测到arxiv文档连接", '尝试下载 ...']) + yield from update_ui(chatbot=chatbot, history=history) + time.sleep(1) # 刷新界面 + url_ = txt # https://arxiv.org/abs/1707.06690 + if not txt.startswith('https://arxiv.org/abs/'): + msg = f"解析arxiv网址失败, 期望格式例如: https://arxiv.org/abs/1707.06690。实际得到格式: {url_}" + yield from update_ui_lastest_msg(msg, chatbot=chatbot, history=history) # 刷新界面 + return msg + + # <-------------- set format -------------> + arxiv_id = url_.split('/abs/')[-1] + url_tar = url_.replace('/abs/', '/e-print/') + download_dir = './gpt_log/arxiv/' + os.makedirs(download_dir, exist_ok=True) + + # <-------------- download arxiv source file -------------> + yield from update_ui_lastest_msg("开始下载", chatbot=chatbot, history=history) # 刷新界面 + proxies, = get_conf('proxies') + r = requests.get(url_tar, proxies=proxies) + dst = pj(download_dir, arxiv_id+'.tar') + with open(dst, 'wb+') as f: + f.write(r.content) + # <-------------- extract file -------------> + yield from update_ui_lastest_msg("下载完成", chatbot=chatbot, history=history) # 刷新界面 + from toolbox import extract_archive + extract_dst = f'gpt_log/{gen_time_str()}' + extract_archive(file_path=dst, dest_dir=extract_dst) + return extract_dst +# ========================================= 插件主程序1 ===================================================== @CatchException def Latex英文纠错加PDF对比(txt, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, web_port): - # 基本信息：功能、贡献者 - chatbot.append([ - "函数插件功能？", - "对整个Latex项目进行纠错，用latex编译为PDF对修正处做高亮。函数插件贡献者: Binary-Husky"]) + # <-------------- information about this plugin -------------> + chatbot.append([ "函数插件功能？", + "对整个Latex项目进行纠错, 用latex编译为PDF对修正处做高亮。函数插件贡献者: Binary-Husky"]) yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 - # 尝试导入依赖，如果缺少依赖，则给出安装建议 + + # <-------------- check deps -------------> try: - import glob, os + import glob, os, time os.system(f'pdflatex -version') + from .latex_utils import Latex精细分解与转化, 编译Latex差别 except Exception as e: - print(trimmed_format_exc()) - report_execption(chatbot, history, a=f"解析项目: {txt}", - b=f"尝试执行Latex指令失败。Latex没有安装，或者不在环境变量PATH中。") + chatbot.append([ f"解析项目: {txt}", + f"尝试执行Latex指令失败。Latex没有安装, 或者不在环境变量PATH中。报错信息\n\n```\n\n{trimmed_format_exc()}\n\n```\n\n"]) yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 return - history = [] # 清空历史，以免输入溢出 + + # <-------------- clear history and read input -------------> + history = [] + txt = yield from arxiv_download(chatbot, history, txt) if os.path.exists(txt): project_folder = txt else: @@ -259,34 +137,64 @@ def Latex英文纠错加PDF对比(txt, llm_kwargs, plugin_kwargs, chatbot, histo yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 return + + # <-------------- if is a zip/tar file -------------> + project_folder = desend_to_extracted_folder_if_exist(project_folder) + + + # <-------------- move latex project away from temp folder -------------> + project_folder = move_project(project_folder) + + + # <-------------- if merge_translate_zh is already generated, skip gpt req -------------> if not os.path.exists(project_folder + '/merge_proofread.tex'): - yield from Latex精细分解与转化(file_manifest, project_folder, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, mode='proofread_latex') + yield from Latex精细分解与转化(file_manifest, project_folder, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, mode='proofread_latex', switch_prompt=switch_prompt) - res_pdf_path = 编译Latex差别(main_file_original='merge', main_file_modified='merge_proofread', + + # <-------------- compile PDF -------------> + success = yield from 编译Latex差别(chatbot, history, main_file_original='merge', main_file_modified='merge_proofread', work_folder_original=project_folder, work_folder_modified=project_folder, work_folder=project_folder) - return res_pdf_path + + # <-------------- zip PDF -------------> + zip_result(project_folder) + if success: + chatbot.append((f"成功啦", '请查收结果（压缩包）...')) + yield from update_ui(chatbot=chatbot, history=history); time.sleep(1) # 刷新界面 + else: + chatbot.append((f"失败了", '虽然PDF生成失败了, 但请查收结果（压缩包）, 内含已经翻译的Tex文档, 也是可读的, 您可以到Github Issue区, 用该压缩包+对话历史存档进行反馈 ...')) + yield from update_ui(chatbot=chatbot, history=history); time.sleep(1) # 刷新界面 + + # <-------------- we are done -------------> + return success + + +# ========================================= 插件主程序2 ===================================================== @CatchException def Latex翻译中文并重新编译PDF(txt, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, web_port): - # 基本信息：功能、贡献者 + # <-------------- information about this plugin -------------> chatbot.append([ "函数插件功能？", - "对整个Latex项目进行翻译，生成中文PDF。函数插件贡献者: Binary-Husky"]) + "对整个Latex项目进行翻译, 生成中文PDF。函数插件贡献者: Binary-Husky"]) yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 - # 尝试导入依赖，如果缺少依赖，则给出安装建议 + + # <-------------- check deps -------------> try: - import glob, os + import glob, os, time os.system(f'pdflatex -version') + from .latex_utils import Latex精细分解与转化, 编译Latex差别 except Exception as e: - print(trimmed_format_exc()) - report_execption(chatbot, history, a=f"解析项目: {txt}", - b=f"尝试执行Latex指令失败。Latex没有安装，或者不在环境变量PATH中。") + chatbot.append([ f"解析项目: {txt}", + f"尝试执行Latex指令失败。Latex没有安装, 或者不在环境变量PATH中。报错信息\n\n```\n\n{trimmed_format_exc()}\n\n```\n\n"]) yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 return - history = [] # 清空历史，以免输入溢出 + + # <-------------- clear history and read input -------------> + history = [] + txt = yield from arxiv_download(chatbot, history, txt) if os.path.exists(txt): project_folder = txt else: @@ -300,9 +208,32 @@ def Latex翻译中文并重新编译PDF(txt, llm_kwargs, plugin_kwargs, chatbot, yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 return - if not os.path.exists(project_folder + '/merge_translate_zh.tex'): - yield from Latex精细分解与转化(file_manifest, project_folder, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, mode='translate_zh') - res_pdf_path = 编译Latex差别(main_file_original='merge', main_file_modified='merge_translate_zh', + # <-------------- if is a zip/tar file -------------> + project_folder = desend_to_extracted_folder_if_exist(project_folder) + + + # <-------------- move latex project away from temp folder -------------> + project_folder = move_project(project_folder) + + + # <-------------- if merge_translate_zh is already generated, skip gpt req -------------> + if not os.path.exists(project_folder + '/merge_translate_zh.tex'): + yield from Latex精细分解与转化(file_manifest, project_folder, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, mode='translate_zh', switch_prompt=switch_prompt) + + + # <-------------- compile PDF -------------> + success = yield from 编译Latex差别(chatbot, history, main_file_original='merge', main_file_modified='merge_translate_zh', work_folder_original=project_folder, work_folder_modified=project_folder, work_folder=project_folder) - return res_pdf_path + + # <-------------- zip PDF -------------> + zip_result(project_folder) + if success: + chatbot.append((f"成功啦", '请查收结果（压缩包）...')) + yield from update_ui(chatbot=chatbot, history=history); time.sleep(1) # 刷新界面 + else: + chatbot.append((f"失败了", '虽然PDF生成失败了, 但请查收结果（压缩包）, 内含已经翻译的Tex文档, 也是可读的, 您可以到Github Issue区, 用该压缩包+对话历史存档进行反馈 ...')) + yield from update_ui(chatbot=chatbot, history=history); time.sleep(1) # 刷新界面 + + # <-------------- we are done -------------> + return success diff --git a/crazy_functions/latex_utils.py b/crazy_functions/latex_utils.py index 54888f6..37a5cac 100644 --- a/crazy_functions/latex_utils.py +++ b/crazy_functions/latex_utils.py @@ -1,7 +1,8 @@ -from toolbox import update_ui, trimmed_format_exc -from toolbox import CatchException, report_execption, write_results_to_file, zip_folder +from toolbox import update_ui, update_ui_lastest_msg # 刷新Gradio前端界面 +from toolbox import zip_folder import os import re +pj = os.path.join def 寻找Latex主文件(file_manifest, mode): for texf in file_manifest: @@ -49,7 +50,7 @@ def mod_inbraket(match): # modify the matched string str_to_modify = str_to_modify.replace('：', ':') - str_to_modify = str_to_modify.replace('，', ',') + str_to_modify = str_to_modify.replace(', ', ',') # str_to_modify = 'BOOM' # return the modified string as the replacement return "\\" + cmd + "{" + str_to_modify + "}" @@ -215,3 +216,226 @@ class LatexPaperFileGroup(): folder = os.path.dirname(self.file_paths[0]) t = time.strftime("%Y-%m-%d-%H-%M-%S", time.localtime()) zip_folder(folder, './gpt_log/', f'{t}-polished.zip') + + + +def Latex精细分解与转化(file_manifest, project_folder, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, mode='proofread', switch_prompt=None): + import time, os, re + from .crazy_utils import request_gpt_model_multi_threads_with_very_awesome_ui_and_high_efficiency + from .latex_utils import LatexPaperFileGroup, merge_tex_files, LatexPaperSplit, 寻找Latex主文件 + + # <-------- 寻找主tex文件 ----------> + maintex = 寻找Latex主文件(file_manifest, mode) + chatbot.append((f"定位主Latex文件", f'[Local Message] 分析结果：该项目的Latex主文件是{maintex}, 如果分析错误, 请立即终止程序, 删除或修改歧义文件, 然后重试。主程序即将开始, 请稍候。')) + yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 + time.sleep(5) + + # <-------- 读取Latex文件, 将多文件tex工程融合为一个巨型tex ----------> + with open(maintex, 'r', encoding='utf-8', errors='replace') as f: + content = f.read() + merged_content = merge_tex_files(project_folder, content, mode) + merged_content = re.sub(r'(? + lps = LatexPaperSplit() + res = lps.split(merged_content) + + # <-------- 拆分过长的latex片段 ----------> + pfg = LatexPaperFileGroup() + for index, r in enumerate(res): + pfg.file_paths.append(index) + pfg.file_contents.append(r) + + pfg.run_file_split(max_token_limit=1024) + n_split = len(pfg.sp_file_contents) + + # <-------- 根据需要切换prompt ----------> + inputs_array, sys_prompt_array = switch_prompt(pfg, mode) + inputs_show_user_array = [f"{mode} {f}" for f in pfg.sp_file_tag] + + # <-------- gpt 多线程请求 ----------> + gpt_response_collection = yield from request_gpt_model_multi_threads_with_very_awesome_ui_and_high_efficiency( + inputs_array=inputs_array, + inputs_show_user_array=inputs_show_user_array, + llm_kwargs=llm_kwargs, + chatbot=chatbot, + history_array=[[""] for _ in range(n_split)], + sys_prompt_array=sys_prompt_array, + # max_workers=5, # 并行任务数量限制, 最多同时执行5个, 其他的排队等待 + scroller_max_len = 80 + ) + + # <-------- 文本碎片重组为完整的tex片段 ----------> + pfg.sp_file_result = [] + for i_say, gpt_say, orig_content in zip(gpt_response_collection[0::2], gpt_response_collection[1::2], pfg.sp_file_contents): + pfg.sp_file_result.append(gpt_say) + pfg.merge_result() + + # # <-------- 临时存储用于调试 ----------> + # pfg.get_token_num = None + # objdump(pfg) + # pfg = objload() + + # <-------- 写出文件 ----------> + final_tex = lps.merge_result(pfg.sp_file_result) + with open(project_folder + f'/merge_{mode}.tex', 'w', encoding='utf-8', errors='replace') as f: + f.write(final_tex) + + # <-------- 整理结果, 退出 ----------> + chatbot.append((f"完成了吗？", 'GPT结果已输出, 正在编译PDF')) + yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 + + # <-------- 返回 ----------> + return project_folder + f'/merge_{mode}.tex' + + +# def Latex预处理(pfg, project_folder): +# import shutil, os +# work_folder = 'private_upload/latex_workshop_temp' + +# try: +# shutil.rmtree(work_folder) +# except: +# pass +# finally: +# work_folder_original = 'private_upload/latex_workshop_temp/original' +# work_folder_modified = 'private_upload/latex_workshop_temp/modified' +# shutil.copytree(project_folder, work_folder_original, ignore=lambda a,b: ['.git']) +# shutil.copytree(project_folder, work_folder_modified, ignore=lambda a,b: ['.git']) + +# for path, result in zip(pfg.file_paths, pfg.file_result): +# path_old = os.path.relpath(path, start=project_folder) +# path_new = pj(work_folder_modified, path_old) +# with open(path_new, 'w', encoding='utf-8') as f: +# f.write(result) + +# for main_file_original in glob.glob('private_upload/latex_workshop_temp/original/*.tex'): +# with open(main_file_original, 'r', encoding='utf8') as f: +# file_content = f.read() +# if r'\documentclass' in file_content: +# path_old = os.path.relpath(main_file_original, start=work_folder_original) +# main_file_modified = os.path.relpath(work_folder_modified, start=work_folder_original) +# return main_file_original, main_file_modified, work_folder_original, work_folder_modified, work_folder +# else: +# continue +# raise RuntimeError('无法找到一个主Tex文件, 本程序寻找主Tex文件的方法是查找文件中的documentclass关键字。') + + +# def Latex预处理(tar_file): +# from toolbox import extract_archive +# import shutil +# work_folder = 'private_upload/latex_workshop_temp' +# try: +# shutil.rmtree(work_folder) +# except: +# pass +# res = extract_archive(tar_file, dest_dir=work_folder) +# for texf in glob.glob('private_upload/latex_workshop_temp/*.tex'): +# with open(texf, 'r', encoding='utf8') as f: +# file_content = f.read() +# if r'\documentclass' in file_content: +# return texf, work_folder +# else: +# continue +# raise RuntimeError('无法找到一个主Tex文件（包含documentclass关键字）') + + + + +def remove_buggy_lines(file_path, log_path, tex_name, tex_name_pure, n_fix, work_folder_modified): + try: + + with open(log_path, 'r', encoding='utf-8', errors='replace') as f: + log = f.read() + with open(file_path, 'r', encoding='utf-8', errors='replace') as f: + file_lines = f.readlines() + import re + buggy_lines = re.findall(tex_name+':([0-9]{1,5}):', log) + buggy_lines = [int(l) for l in buggy_lines] + buggy_lines = sorted(buggy_lines) + print("removing lines that has errors", buggy_lines) + file_lines.pop(buggy_lines[0]-1) + with open(pj(work_folder_modified, f"{tex_name_pure}_fix_{n_fix}.tex"), 'w', encoding='utf-8', errors='replace') as f: + f.writelines(file_lines) + return True, f"{tex_name_pure}_fix_{n_fix}", buggy_lines + + except: + return False, 0 + + + +def 编译Latex差别(chatbot, history, main_file_original, main_file_modified, work_folder_original, work_folder_modified, work_folder): + import os, time + current_dir = os.getcwd() + n_fix = 0 + chatbot.append([f"正在编译PDF文档", '编译已经开始...']); yield from update_ui(chatbot=chatbot, history=history); time.sleep(1); chatbot[-1] = list(chatbot[-1]) # 刷新界面 + yield from update_ui_lastest_msg('编译已经开始...', chatbot, history) # 刷新Gradio前端界面 + + while True: + import os + # https://stackoverflow.com/questions/738755/dont-make-me-manually-abort-a-latex-compile-when-theres-an-error + yield from update_ui_lastest_msg(f'尝试第{n_fix}编译, 编译原始PDF ...', chatbot, history) # 刷新Gradio前端界面 + os.chdir(work_folder_original); os.system(f'pdflatex -interaction=batchmode -file-line-error {main_file_original}.tex'); os.chdir(current_dir) + + yield from update_ui_lastest_msg(f'尝试第{n_fix}编译, 编译转化后的PDF ...', chatbot, history) # 刷新Gradio前端界面 + os.chdir(work_folder_modified); os.system(f'pdflatex -interaction=batchmode -file-line-error {main_file_modified}.tex'); os.chdir(current_dir) + + yield from update_ui_lastest_msg(f'尝试第{n_fix}编译, 编译BibTex ...', chatbot, history) # 刷新Gradio前端界面 + os.chdir(work_folder_original); os.system(f'bibtex {main_file_original}.aux'); os.chdir(current_dir) + os.chdir(work_folder_modified); os.system(f'bibtex {main_file_modified}.aux'); os.chdir(current_dir) + + yield from update_ui_lastest_msg(f'尝试第{n_fix}编译, 编译文献交叉引用 ...', chatbot, history) # 刷新Gradio前端界面 + os.chdir(work_folder_original); os.system(f'pdflatex -interaction=batchmode -file-line-error {main_file_original}.tex'); os.chdir(current_dir) + os.chdir(work_folder_modified); os.system(f'pdflatex -interaction=batchmode -file-line-error {main_file_modified}.tex'); os.chdir(current_dir) + os.chdir(work_folder_original); os.system(f'pdflatex -interaction=batchmode -file-line-error {main_file_original}.tex'); os.chdir(current_dir) + os.chdir(work_folder_modified); os.system(f'pdflatex -interaction=batchmode -file-line-error {main_file_modified}.tex'); os.chdir(current_dir) + + yield from update_ui_lastest_msg(f'尝试第{n_fix}编译, 使用latexdiff生成论文转化前后对比 ...', chatbot, history) # 刷新Gradio前端界面 + print( f'latexdiff --encoding=utf8 --append-safecmd=subfile {work_folder_original}/{main_file_original}.tex {work_folder_modified}/{main_file_modified}.tex --flatten > {work_folder}/merge_diff.tex') + os.system(f'latexdiff --encoding=utf8 --append-safecmd=subfile {work_folder_original}/{main_file_original}.tex {work_folder_modified}/{main_file_modified}.tex --flatten > {work_folder}/merge_diff.tex') + + yield from update_ui_lastest_msg(f'尝试第{n_fix}编译, 正在编译对比PDF ...', chatbot, history) # 刷新Gradio前端界面 + os.chdir(work_folder); os.system(f'pdflatex -interaction=batchmode -file-line-error merge_diff.tex'); os.chdir(current_dir) + os.chdir(work_folder); os.system(f'bibtex merge_diff.aux'); os.chdir(current_dir) + os.chdir(work_folder); os.system(f'pdflatex -interaction=batchmode -file-line-error merge_diff.tex'); os.chdir(current_dir) + os.chdir(work_folder); os.system(f'pdflatex -interaction=batchmode -file-line-error merge_diff.tex'); os.chdir(current_dir) + + # <---------------------> + os.chdir(current_dir) + + # <---------- 检查结果 -----------> + results_ = "" + original_pdf_success = os.path.exists(pj(work_folder_original, f'{main_file_original}.pdf')) + modified_pdf_success = os.path.exists(pj(work_folder_modified, f'{main_file_modified}.pdf')) + diff_pdf_success = os.path.exists(pj(work_folder, f'merge_diff.pdf')) + results_ += f"原始PDF编译是否成功: {original_pdf_success};" + results_ += f"转化PDF编译是否成功: {modified_pdf_success};" + results_ += f"对比PDF编译是否成功: {diff_pdf_success};" + yield from update_ui_lastest_msg(f'第{n_fix}编译结束:
{results_}...', chatbot, history) # 刷新Gradio前端界面 + + if modified_pdf_success: + yield from update_ui_lastest_msg(f'转化PDF编译已经成功, 即将退出 ...', chatbot, history) # 刷新Gradio前端界面 + os.chdir(current_dir) + return True # 成功啦 + else: + if n_fix>=10: break + n_fix += 1 + can_retry, main_file_modified, buggy_lines = remove_buggy_lines( + file_path=pj(work_folder_modified, f'{main_file_modified}.tex'), + log_path=pj(work_folder_modified, f'{main_file_modified}.log'), + tex_name=f'{main_file_modified}.tex', + tex_name_pure=f'{main_file_modified}', + n_fix=n_fix, + work_folder_modified=work_folder_modified, + ) + yield from update_ui_lastest_msg(f'由于最为关键的转化PDF编译失败, 将根据报错信息修正tex源文件并重试, 当前报错的latex代码处于第{buggy_lines}行 ...', chatbot, history) # 刷新Gradio前端界面 + if not can_retry: break + + os.chdir(current_dir) + return False # 失败啦 + + + diff --git a/toolbox.py b/toolbox.py index 8bd37a2..9b9a40d 100644 --- a/toolbox.py +++ b/toolbox.py @@ -1,6 +1,6 @@ import markdown import importlib -import traceback +import time import inspect import re import os @@ -78,6 +78,17 @@ def update_ui(chatbot, history, msg='正常', **kwargs): # 刷新界面 assert isinstance(chatbot, ChatBotWithCookies), "在传递chatbot的过程中不要将其丢弃。必要时，可用clear将其清空，然后用for+append循环重新赋值。" yield chatbot.get_cookies(), chatbot, history, msg +def update_ui_lastest_msg(lastmsg, chatbot, history, delay=1): # 刷新界面 + """ + 刷新用户界面 + """ + if len(chatbot) == 0: chatbot.append(["update_ui_last_msg", lastmsg]) + chatbot[-1] = list(chatbot[-1]) + chatbot[-1][-1] = lastmsg + yield from update_ui(chatbot=chatbot, history=history) + time.sleep(delay) + + def trimmed_format_exc(): import os, traceback str = traceback.format_exc() @@ -772,6 +783,11 @@ def zip_folder(source_folder, dest_folder, zip_name): print(f"Zip file created at {zip_file}") +def zip_result(folder): + import time + t = time.strftime("%Y-%m-%d-%H-%M-%S", time.localtime()) + zip_folder(folder, './gpt_log/', f'{t}-result.zip') + def gen_time_str(): import time return time.strftime("%Y-%m-%d-%H-%M-%S", time.localtime()) @@ -806,4 +822,5 @@ def objload(): if not os.path.exists('objdump.tmp'): return with open('objdump.tmp', 'rb') as f: - return pickle.load(f) \ No newline at end of file + return pickle.load(f) + \ No newline at end of file