diff --git a/crazy_functional.py b/crazy_functional.py index 40b38c3..ca6ed0a 100644 --- a/crazy_functional.py +++ b/crazy_functional.py @@ -345,5 +345,28 @@ def get_crazy_functions(): except: print('Load function plugin failed') + try: + from crazy_functions.Latex输出PDF结果 import Latex英文纠错加PDF对比 + function_plugins.update({ + "Latex英文纠错+LatexDiff高亮修正位置": { + "Color": "stop", + "AsButton": False, + # "AdvancedArgs": True, + # "ArgsReminder": "", + "Function": HotReload(Latex英文纠错加PDF对比) + } + }) + from crazy_functions.Latex输出PDF结果 import Latex翻译中文并重新编译PDF + function_plugins.update({ + "Latex翻译中文+生成PDF": { + "Color": "stop", + "AsButton": False, + # "AdvancedArgs": True, + # "ArgsReminder": "", + "Function": HotReload(Latex翻译中文并重新编译PDF) + } + }) + except: + print('Load function plugin failed') ###################### 第n组插件 ########################### return function_plugins diff --git a/crazy_functions/Latex全文润色.py b/crazy_functions/Latex全文润色.py index 8d3f97b..9e1d4b6 100644 --- a/crazy_functions/Latex全文润色.py +++ b/crazy_functions/Latex全文润色.py @@ -238,3 +238,6 @@ def Latex英文纠错(txt, llm_kwargs, plugin_kwargs, chatbot, history, system_p yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 return yield from 多文件润色(file_manifest, project_folder, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, language='en', mode='proofread') + + + diff --git a/crazy_functions/Latex输出PDF结果.py b/crazy_functions/Latex输出PDF结果.py new file mode 100644 index 0000000..13ba3d9 --- /dev/null +++ b/crazy_functions/Latex输出PDF结果.py @@ -0,0 +1,239 @@ +from toolbox import update_ui, trimmed_format_exc, get_conf, objdump, objload +from toolbox import CatchException, report_execption, update_ui_lastest_msg, zip_result, gen_time_str +import glob, os, requests, time +pj = os.path.join + +# =================================== 工具函数 =============================================== +def switch_prompt(pfg, mode): + """ + Generate prompts and system prompts based on the mode for proofreading or translating. + Args: + - pfg: Proofreader or Translator instance. + - mode: A string specifying the mode, either 'proofread' or 'translate_zh'. + + Returns: + - inputs_array: A list of strings containing prompts for users to respond to. + - sys_prompt_array: A list of strings containing prompts for system prompts. + """ + n_split = len(pfg.sp_file_contents) + if mode == 'proofread': + inputs_array = [r"Below is a section from an academic paper, proofread this section." + + r"Do not modify any latex command such as \section, \cite, \begin, \item and equations. " + + r"Answer me only with the revised text:" + + f"\n\n{frag}" for frag in pfg.sp_file_contents] + sys_prompt_array = ["You are a professional academic paper writer." for _ in range(n_split)] + elif mode == 'translate_zh': + inputs_array = [r"Below is a section from an English academic paper, translate it into Chinese." + + r"Do not modify any latex command such as \section, \cite, \begin, \item and equations. " + + r"Answer me only with the translated text:" + + f"\n\n{frag}" for frag in pfg.sp_file_contents] + sys_prompt_array = ["You are a professional translator." for _ in range(n_split)] + else: + assert False, "未知指令" + return inputs_array, sys_prompt_array + +def desend_to_extracted_folder_if_exist(project_folder): + """ + Descend into the extracted folder if it exists, otherwise return the original folder. + + Args: + - project_folder: A string specifying the folder path. + + Returns: + - A string specifying the path to the extracted folder, or the original folder if there is no extracted folder. + """ + maybe_dir = [f for f in glob.glob(f'{project_folder}/*') if os.path.isdir(f)] + if len(maybe_dir) == 0: return project_folder + if maybe_dir[0].endswith('.extract'): return maybe_dir[0] + return project_folder + +def move_project(project_folder): + """ + Create a new work folder and copy the project folder to it. + + Args: + - project_folder: A string specifying the folder path of the project. + + Returns: + - A string specifying the path to the new work folder. + """ + import shutil, time + time.sleep(2) # avoid time string conflict + new_workfolder = f'gpt_log/{gen_time_str()}' + shutil.copytree(src=project_folder, dst=new_workfolder) + return new_workfolder + +def arxiv_download(chatbot, history, txt): + if not txt.startswith('https://arxiv.org'): + return txt + + # <-------------- inspect format -------------> + chatbot.append([f"检测到arxiv文档连接", '尝试下载 ...']) + yield from update_ui(chatbot=chatbot, history=history) + time.sleep(1) # 刷新界面 + + url_ = txt # https://arxiv.org/abs/1707.06690 + if not txt.startswith('https://arxiv.org/abs/'): + msg = f"解析arxiv网址失败, 期望格式例如: https://arxiv.org/abs/1707.06690。实际得到格式: {url_}" + yield from update_ui_lastest_msg(msg, chatbot=chatbot, history=history) # 刷新界面 + return msg + + # <-------------- set format -------------> + arxiv_id = url_.split('/abs/')[-1] + url_tar = url_.replace('/abs/', '/e-print/') + download_dir = './gpt_log/arxiv/' + os.makedirs(download_dir, exist_ok=True) + + # <-------------- download arxiv source file -------------> + yield from update_ui_lastest_msg("开始下载", chatbot=chatbot, history=history) # 刷新界面 + proxies, = get_conf('proxies') + r = requests.get(url_tar, proxies=proxies) + dst = pj(download_dir, arxiv_id+'.tar') + with open(dst, 'wb+') as f: + f.write(r.content) + + # <-------------- extract file -------------> + yield from update_ui_lastest_msg("下载完成", chatbot=chatbot, history=history) # 刷新界面 + from toolbox import extract_archive + extract_dst = f'gpt_log/{gen_time_str()}' + extract_archive(file_path=dst, dest_dir=extract_dst) + return extract_dst +# ========================================= 插件主程序1 ===================================================== + + +@CatchException +def Latex英文纠错加PDF对比(txt, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, web_port): + # <-------------- information about this plugin -------------> + chatbot.append([ "函数插件功能?", + "对整个Latex项目进行纠错, 用latex编译为PDF对修正处做高亮。函数插件贡献者: Binary-Husky。注意事项: 目前仅支持GPT3.5/GPT4,其他模型转化效果未知。目前对机器学习类文献转化效果最好,其他类型文献转化效果未知。"]) + yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 + + + # <-------------- check deps -------------> + try: + import glob, os, time + os.system(f'pdflatex -version') + from .latex_utils import Latex精细分解与转化, 编译Latex差别 + except Exception as e: + chatbot.append([ f"解析项目: {txt}", + f"尝试执行Latex指令失败。Latex没有安装, 或者不在环境变量PATH中。报错信息\n\n```\n\n{trimmed_format_exc()}\n\n```\n\n"]) + yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 + return + + + # <-------------- clear history and read input -------------> + history = [] + txt = yield from arxiv_download(chatbot, history, txt) + if os.path.exists(txt): + project_folder = txt + else: + if txt == "": txt = '空空如也的输入栏' + report_execption(chatbot, history, a = f"解析项目: {txt}", b = f"找不到本地项目或无权访问: {txt}") + yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 + return + file_manifest = [f for f in glob.glob(f'{project_folder}/**/*.tex', recursive=True)] + if len(file_manifest) == 0: + report_execption(chatbot, history, a = f"解析项目: {txt}", b = f"找不到任何.tex文件: {txt}") + yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 + return + + + # <-------------- if is a zip/tar file -------------> + project_folder = desend_to_extracted_folder_if_exist(project_folder) + + + # <-------------- move latex project away from temp folder -------------> + project_folder = move_project(project_folder) + + + # <-------------- if merge_translate_zh is already generated, skip gpt req -------------> + if not os.path.exists(project_folder + '/merge_proofread.tex'): + yield from Latex精细分解与转化(file_manifest, project_folder, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, mode='proofread_latex', switch_prompt=switch_prompt) + + + # <-------------- compile PDF -------------> + success = yield from 编译Latex差别(chatbot, history, main_file_original='merge', main_file_modified='merge_proofread', + work_folder_original=project_folder, work_folder_modified=project_folder, work_folder=project_folder) + + + # <-------------- zip PDF -------------> + zip_result(project_folder) + if success: + chatbot.append((f"成功啦", '请查收结果(压缩包)...')) + yield from update_ui(chatbot=chatbot, history=history); time.sleep(1) # 刷新界面 + else: + chatbot.append((f"失败了", '虽然PDF生成失败了, 但请查收结果(压缩包), 内含已经翻译的Tex文档, 也是可读的, 您可以到Github Issue区, 用该压缩包+对话历史存档进行反馈 ...')) + yield from update_ui(chatbot=chatbot, history=history); time.sleep(1) # 刷新界面 + + # <-------------- we are done -------------> + return success + + +# ========================================= 插件主程序2 ===================================================== + +@CatchException +def Latex翻译中文并重新编译PDF(txt, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, web_port): + # <-------------- information about this plugin -------------> + chatbot.append([ + "函数插件功能?", + "对整个Latex项目进行翻译, 生成中文PDF。函数插件贡献者: Binary-Husky。注意事项: 目前仅支持GPT3.5/GPT4,其他模型转化效果未知。目前对机器学习类文献转化效果最好,其他类型文献转化效果未知。"]) + yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 + + + # <-------------- check deps -------------> + try: + import glob, os, time + os.system(f'pdflatex -version') + from .latex_utils import Latex精细分解与转化, 编译Latex差别 + except Exception as e: + chatbot.append([ f"解析项目: {txt}", + f"尝试执行Latex指令失败。Latex没有安装, 或者不在环境变量PATH中。报错信息\n\n```\n\n{trimmed_format_exc()}\n\n```\n\n"]) + yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 + return + + + # <-------------- clear history and read input -------------> + history = [] + txt = yield from arxiv_download(chatbot, history, txt) + if os.path.exists(txt): + project_folder = txt + else: + if txt == "": txt = '空空如也的输入栏' + report_execption(chatbot, history, a = f"解析项目: {txt}", b = f"找不到本地项目或无权访问: {txt}") + yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 + return + file_manifest = [f for f in glob.glob(f'{project_folder}/**/*.tex', recursive=True)] + if len(file_manifest) == 0: + report_execption(chatbot, history, a = f"解析项目: {txt}", b = f"找不到任何.tex文件: {txt}") + yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 + return + + + # <-------------- if is a zip/tar file -------------> + project_folder = desend_to_extracted_folder_if_exist(project_folder) + + + # <-------------- move latex project away from temp folder -------------> + project_folder = move_project(project_folder) + + + # <-------------- if merge_translate_zh is already generated, skip gpt req -------------> + if not os.path.exists(project_folder + '/merge_translate_zh.tex'): + yield from Latex精细分解与转化(file_manifest, project_folder, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, mode='translate_zh', switch_prompt=switch_prompt) + + + # <-------------- compile PDF -------------> + success = yield from 编译Latex差别(chatbot, history, main_file_original='merge', main_file_modified='merge_translate_zh', + work_folder_original=project_folder, work_folder_modified=project_folder, work_folder=project_folder) + + # <-------------- zip PDF -------------> + zip_result(project_folder) + if success: + chatbot.append((f"成功啦", '请查收结果(压缩包)...')) + yield from update_ui(chatbot=chatbot, history=history); time.sleep(1) # 刷新界面 + else: + chatbot.append((f"失败了", '虽然PDF生成失败了, 但请查收结果(压缩包), 内含已经翻译的Tex文档, 也是可读的, 您可以到Github Issue区, 用该压缩包+对话历史存档进行反馈 ...')) + yield from update_ui(chatbot=chatbot, history=history); time.sleep(1) # 刷新界面 + + # <-------------- we are done -------------> + return success diff --git a/crazy_functions/crazy_functions_test.py b/crazy_functions/crazy_functions_test.py index fd7d91d..5b4d2b5 100644 --- a/crazy_functions/crazy_functions_test.py +++ b/crazy_functions/crazy_functions_test.py @@ -179,6 +179,25 @@ def test_Langchain知识库读取(): for cookies, cb, hist, msg in silence_stdout(读取知识库作答)(txt, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, web_port): cli_printer.print(cb) # print(cb) +def test_Latex(): + from crazy_functions.Latex输出PDF结果 import Latex预处理, 编译Latex, Latex英文纠错加PDF对比, Latex翻译中文并重新编译PDF + txt = "C:/Users/fuqingxu/Desktop/proofread" + txt = "C:/Users/fuqingxu/Desktop/旧文件/gpt/paperx" + txt = "C:/Users/fuqingxu/Desktop/旧文件/gpt/papery" + + for cookies, cb, hist, msg in (Latex翻译中文并重新编译PDF)(txt, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, web_port): + cli_printer.print(cb) # print(cb) + + + + # txt = "2302.02948.tar" + # print(txt) + # main_tex, work_folder = Latex预处理(txt) + # print('main tex:', main_tex) + # res = 编译Latex(main_tex, work_folder) + # # for cookies, cb, hist, msg in silence_stdout(编译Latex)(txt, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, web_port): + # cli_printer.print(cb) # print(cb) + # test_解析一个Python项目() @@ -192,8 +211,8 @@ def test_Langchain知识库读取(): # test_联网回答问题() # test_解析ipynb文件() # test_数学动画生成manim() -test_Langchain知识库() +# test_Langchain知识库() # test_Langchain知识库读取() - +test_Latex() input("程序完成,回车退出。") print("退出。") \ No newline at end of file diff --git a/crazy_functions/latex_utils.py b/crazy_functions/latex_utils.py new file mode 100644 index 0000000..37a5cac --- /dev/null +++ b/crazy_functions/latex_utils.py @@ -0,0 +1,441 @@ +from toolbox import update_ui, update_ui_lastest_msg # 刷新Gradio前端界面 +from toolbox import zip_folder +import os +import re +pj = os.path.join + +def 寻找Latex主文件(file_manifest, mode): + for texf in file_manifest: + if os.path.basename(texf).startswith('merge'): + continue + with open(texf, 'r', encoding='utf8') as f: + file_content = f.read() + if r'\documentclass' in file_content: + return texf + else: + continue + raise RuntimeError('无法找到一个主Tex文件(包含documentclass关键字)') + +def merge_tex_files_(project_foler, main_file, mode): + for s in reversed([q for q in re.finditer(r"\\input\{(.*?)\}", main_file, re.M)]): + f = s.group(1) + fp = os.path.join(project_foler, f) + with open(fp, 'r', encoding='utf-8', errors='replace') as fx: + c = fx.read() + c = merge_tex_files_(project_foler, c, mode) + main_file = main_file[:s.span()[0]] + c + main_file[s.span()[1]:] + return main_file + +def merge_tex_files(project_foler, main_file, mode): + main_file = merge_tex_files_(project_foler, main_file, mode) + + if mode == 'translate_zh': + pattern = re.compile(r'\\documentclass.*\n') + match = pattern.search(main_file) + position = match.end() + main_file = main_file[:position] + '\\usepackage{CTEX}\n' + main_file[position:] + + return main_file + +class LinkTable(): + def __init__(self, string, preserve=True) -> None: + self.string = string + self.preserve = preserve + self.next = None + +def mod_inbraket(match): + # get the matched string + cmd = match.group(1) + str_to_modify = match.group(2) + + # modify the matched string + str_to_modify = str_to_modify.replace(':', ':') + str_to_modify = str_to_modify.replace(', ', ',') + # str_to_modify = 'BOOM' + # return the modified string as the replacement + return "\\" + cmd + "{" + str_to_modify + "}" + +def fix_content(final_tex): + """ + fix common GPT errors to increase success rate + """ + final_tex = final_tex.replace('%', r'\%') + final_tex = final_tex.replace(r'\%', r'\\%') + final_tex = re.sub(r"\\([a-z]{2,10})\ \{", r"\\\1{", string=final_tex) + final_tex = re.sub(r"\\([a-z]{2,10})\{([^\}]*?)\}", mod_inbraket, string=final_tex) + return final_tex + +class LatexPaperSplit(): + def __init__(self) -> None: + self.root = None + + def merge_result(self, arr): + result_string = "" + node = self.root + p = 0 + while True: + if node.preserve: + result_string += node.string + else: + result_string += fix_content(arr[p]) + p += 1 + node = node.next + if node is None: break + + return result_string + + def split(self, txt): + # def replace_with_hash() + root = LinkTable(txt, False) + def split_worker(root, pattern, flags=0): + lt = root + cnt = 0 + while True: + if not lt.preserve: + while True: + res = re.search(pattern, lt.string, flags) + if not res: break + before = res.string[:res.span()[0]] + this = res.group(0) + # core = res.group(1) + after = res.string[res.span()[1]:] + # ====== + if before.endswith('\n'): + this = '\n' + this + before = before[:-1] + if after.startswith('\n'): + # move \n + this = this + '\n' + after = after[1:] + # ====== + lt.string = before + tmp = lt.next + # ====== + mid = LinkTable(this, True) + lt.next = mid + # ====== + aft = LinkTable(after, False) + mid.next = aft + aft.next = tmp + # ====== + lt = aft + lt = lt.next + cnt += 1 + print(cnt) + if lt is None: break + + # root 是链表的头 + print('正在分解Latex源文件') + split_worker(root, r"(.*?)\\maketitle", re.DOTALL) + split_worker(root, r"\\section\{(.*?)\}") + split_worker(root, r"\\subsection\{(.*?)\}") + split_worker(root, r"\\subsubsection\{(.*?)\}") + split_worker(root, r"\\begin\{figure\}(.*?)\\end\{figure\}", re.DOTALL) + split_worker(root, r"\\begin\{figure\*\}(.*?)\\end\{figure\*\}", re.DOTALL) + split_worker(root, r"\\begin\{table\}(.*?)\\end\{table\}", re.DOTALL) + split_worker(root, r"\\begin\{table\*\}(.*?)\\end\{table\*\}", re.DOTALL) + split_worker(root, r"\\begin\{align\*\}(.*?)\\end\{align\*\}", re.DOTALL) + split_worker(root, r"\\begin\{align\}(.*?)\\end\{align\}", re.DOTALL) + split_worker(root, r"\\begin\{equation\}(.*?)\\end\{equation\}", re.DOTALL) + split_worker(root, r"\\begin\{equation\*\}(.*?)\\end\{equation\*\}", re.DOTALL) + split_worker(root, r"\$\$(.*?)\$\$", re.DOTALL) + split_worker(root, r"\\item ") + split_worker(root, r"\\begin\{(.*?)\}") + split_worker(root, r"\\end\{(.*?)\}") + + res = [] + node = root + while True: + res.append((node.string, node.preserve)) + if len(node.string.strip('\n').strip(''))==0: node.preserve = True + if len(node.string.strip('\n').strip(''))<50: node.preserve = True + node = node.next + if node is None: break + + with open('debug_log', 'w', encoding='utf8') as f: + res_to_t = [] + node = root + while True: + if not node.preserve: + res_to_t.append(node.string) + f.write(node.string + '\n ========================= \n') + node = node.next + if node is None: break + + self.root = root + self.sp = res_to_t + return self.sp + +class LatexPaperFileGroup(): + def __init__(self): + self.file_paths = [] + self.file_contents = [] + self.sp_file_contents = [] + self.sp_file_index = [] + self.sp_file_tag = [] + + # count_token + from request_llm.bridge_all import model_info + enc = model_info["gpt-3.5-turbo"]['tokenizer'] + def get_token_num(txt): return len(enc.encode(txt, disallowed_special=())) + self.get_token_num = get_token_num + + def run_file_split(self, max_token_limit=1900): + """ + 将长文本分离开来 + """ + for index, file_content in enumerate(self.file_contents): + if self.get_token_num(file_content) < max_token_limit: + self.sp_file_contents.append(file_content) + self.sp_file_index.append(index) + self.sp_file_tag.append(self.file_paths[index]) + else: + from .crazy_utils import breakdown_txt_to_satisfy_token_limit_for_pdf + segments = breakdown_txt_to_satisfy_token_limit_for_pdf(file_content, self.get_token_num, max_token_limit) + for j, segment in enumerate(segments): + self.sp_file_contents.append(segment) + self.sp_file_index.append(index) + self.sp_file_tag.append(self.file_paths[index] + f".part-{j}.tex") + print('Segmentation: done') + + def merge_result(self): + self.file_result = ["" for _ in range(len(self.file_paths))] + for r, k in zip(self.sp_file_result, self.sp_file_index): + self.file_result[k] += r + + def write_result(self): + manifest = [] + for path, res in zip(self.file_paths, self.file_result): + with open(path + '.polish.tex', 'w', encoding='utf8') as f: + manifest.append(path + '.polish.tex') + f.write(res) + return manifest + + def zip_result(self): + import os, time + folder = os.path.dirname(self.file_paths[0]) + t = time.strftime("%Y-%m-%d-%H-%M-%S", time.localtime()) + zip_folder(folder, './gpt_log/', f'{t}-polished.zip') + + + +def Latex精细分解与转化(file_manifest, project_folder, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, mode='proofread', switch_prompt=None): + import time, os, re + from .crazy_utils import request_gpt_model_multi_threads_with_very_awesome_ui_and_high_efficiency + from .latex_utils import LatexPaperFileGroup, merge_tex_files, LatexPaperSplit, 寻找Latex主文件 + + # <-------- 寻找主tex文件 ----------> + maintex = 寻找Latex主文件(file_manifest, mode) + chatbot.append((f"定位主Latex文件", f'[Local Message] 分析结果:该项目的Latex主文件是{maintex}, 如果分析错误, 请立即终止程序, 删除或修改歧义文件, 然后重试。主程序即将开始, 请稍候。')) + yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 + time.sleep(5) + + # <-------- 读取Latex文件, 将多文件tex工程融合为一个巨型tex ----------> + with open(maintex, 'r', encoding='utf-8', errors='replace') as f: + content = f.read() + merged_content = merge_tex_files(project_folder, content, mode) + merged_content = re.sub(r'(? + lps = LatexPaperSplit() + res = lps.split(merged_content) + + # <-------- 拆分过长的latex片段 ----------> + pfg = LatexPaperFileGroup() + for index, r in enumerate(res): + pfg.file_paths.append(index) + pfg.file_contents.append(r) + + pfg.run_file_split(max_token_limit=1024) + n_split = len(pfg.sp_file_contents) + + # <-------- 根据需要切换prompt ----------> + inputs_array, sys_prompt_array = switch_prompt(pfg, mode) + inputs_show_user_array = [f"{mode} {f}" for f in pfg.sp_file_tag] + + # <-------- gpt 多线程请求 ----------> + gpt_response_collection = yield from request_gpt_model_multi_threads_with_very_awesome_ui_and_high_efficiency( + inputs_array=inputs_array, + inputs_show_user_array=inputs_show_user_array, + llm_kwargs=llm_kwargs, + chatbot=chatbot, + history_array=[[""] for _ in range(n_split)], + sys_prompt_array=sys_prompt_array, + # max_workers=5, # 并行任务数量限制, 最多同时执行5个, 其他的排队等待 + scroller_max_len = 80 + ) + + # <-------- 文本碎片重组为完整的tex片段 ----------> + pfg.sp_file_result = [] + for i_say, gpt_say, orig_content in zip(gpt_response_collection[0::2], gpt_response_collection[1::2], pfg.sp_file_contents): + pfg.sp_file_result.append(gpt_say) + pfg.merge_result() + + # # <-------- 临时存储用于调试 ----------> + # pfg.get_token_num = None + # objdump(pfg) + # pfg = objload() + + # <-------- 写出文件 ----------> + final_tex = lps.merge_result(pfg.sp_file_result) + with open(project_folder + f'/merge_{mode}.tex', 'w', encoding='utf-8', errors='replace') as f: + f.write(final_tex) + + # <-------- 整理结果, 退出 ----------> + chatbot.append((f"完成了吗?", 'GPT结果已输出, 正在编译PDF')) + yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 + + # <-------- 返回 ----------> + return project_folder + f'/merge_{mode}.tex' + + +# def Latex预处理(pfg, project_folder): +# import shutil, os +# work_folder = 'private_upload/latex_workshop_temp' + +# try: +# shutil.rmtree(work_folder) +# except: +# pass +# finally: +# work_folder_original = 'private_upload/latex_workshop_temp/original' +# work_folder_modified = 'private_upload/latex_workshop_temp/modified' +# shutil.copytree(project_folder, work_folder_original, ignore=lambda a,b: ['.git']) +# shutil.copytree(project_folder, work_folder_modified, ignore=lambda a,b: ['.git']) + +# for path, result in zip(pfg.file_paths, pfg.file_result): +# path_old = os.path.relpath(path, start=project_folder) +# path_new = pj(work_folder_modified, path_old) +# with open(path_new, 'w', encoding='utf-8') as f: +# f.write(result) + +# for main_file_original in glob.glob('private_upload/latex_workshop_temp/original/*.tex'): +# with open(main_file_original, 'r', encoding='utf8') as f: +# file_content = f.read() +# if r'\documentclass' in file_content: +# path_old = os.path.relpath(main_file_original, start=work_folder_original) +# main_file_modified = os.path.relpath(work_folder_modified, start=work_folder_original) +# return main_file_original, main_file_modified, work_folder_original, work_folder_modified, work_folder +# else: +# continue +# raise RuntimeError('无法找到一个主Tex文件, 本程序寻找主Tex文件的方法是查找文件中的documentclass关键字。') + + +# def Latex预处理(tar_file): +# from toolbox import extract_archive +# import shutil +# work_folder = 'private_upload/latex_workshop_temp' +# try: +# shutil.rmtree(work_folder) +# except: +# pass +# res = extract_archive(tar_file, dest_dir=work_folder) +# for texf in glob.glob('private_upload/latex_workshop_temp/*.tex'): +# with open(texf, 'r', encoding='utf8') as f: +# file_content = f.read() +# if r'\documentclass' in file_content: +# return texf, work_folder +# else: +# continue +# raise RuntimeError('无法找到一个主Tex文件(包含documentclass关键字)') + + + + +def remove_buggy_lines(file_path, log_path, tex_name, tex_name_pure, n_fix, work_folder_modified): + try: + + with open(log_path, 'r', encoding='utf-8', errors='replace') as f: + log = f.read() + with open(file_path, 'r', encoding='utf-8', errors='replace') as f: + file_lines = f.readlines() + import re + buggy_lines = re.findall(tex_name+':([0-9]{1,5}):', log) + buggy_lines = [int(l) for l in buggy_lines] + buggy_lines = sorted(buggy_lines) + print("removing lines that has errors", buggy_lines) + file_lines.pop(buggy_lines[0]-1) + with open(pj(work_folder_modified, f"{tex_name_pure}_fix_{n_fix}.tex"), 'w', encoding='utf-8', errors='replace') as f: + f.writelines(file_lines) + return True, f"{tex_name_pure}_fix_{n_fix}", buggy_lines + + except: + return False, 0 + + + +def 编译Latex差别(chatbot, history, main_file_original, main_file_modified, work_folder_original, work_folder_modified, work_folder): + import os, time + current_dir = os.getcwd() + n_fix = 0 + chatbot.append([f"正在编译PDF文档", '编译已经开始...']); yield from update_ui(chatbot=chatbot, history=history); time.sleep(1); chatbot[-1] = list(chatbot[-1]) # 刷新界面 + yield from update_ui_lastest_msg('编译已经开始...', chatbot, history) # 刷新Gradio前端界面 + + while True: + import os + # https://stackoverflow.com/questions/738755/dont-make-me-manually-abort-a-latex-compile-when-theres-an-error + yield from update_ui_lastest_msg(f'尝试第{n_fix}编译, 编译原始PDF ...', chatbot, history) # 刷新Gradio前端界面 + os.chdir(work_folder_original); os.system(f'pdflatex -interaction=batchmode -file-line-error {main_file_original}.tex'); os.chdir(current_dir) + + yield from update_ui_lastest_msg(f'尝试第{n_fix}编译, 编译转化后的PDF ...', chatbot, history) # 刷新Gradio前端界面 + os.chdir(work_folder_modified); os.system(f'pdflatex -interaction=batchmode -file-line-error {main_file_modified}.tex'); os.chdir(current_dir) + + yield from update_ui_lastest_msg(f'尝试第{n_fix}编译, 编译BibTex ...', chatbot, history) # 刷新Gradio前端界面 + os.chdir(work_folder_original); os.system(f'bibtex {main_file_original}.aux'); os.chdir(current_dir) + os.chdir(work_folder_modified); os.system(f'bibtex {main_file_modified}.aux'); os.chdir(current_dir) + + yield from update_ui_lastest_msg(f'尝试第{n_fix}编译, 编译文献交叉引用 ...', chatbot, history) # 刷新Gradio前端界面 + os.chdir(work_folder_original); os.system(f'pdflatex -interaction=batchmode -file-line-error {main_file_original}.tex'); os.chdir(current_dir) + os.chdir(work_folder_modified); os.system(f'pdflatex -interaction=batchmode -file-line-error {main_file_modified}.tex'); os.chdir(current_dir) + os.chdir(work_folder_original); os.system(f'pdflatex -interaction=batchmode -file-line-error {main_file_original}.tex'); os.chdir(current_dir) + os.chdir(work_folder_modified); os.system(f'pdflatex -interaction=batchmode -file-line-error {main_file_modified}.tex'); os.chdir(current_dir) + + yield from update_ui_lastest_msg(f'尝试第{n_fix}编译, 使用latexdiff生成论文转化前后对比 ...', chatbot, history) # 刷新Gradio前端界面 + print( f'latexdiff --encoding=utf8 --append-safecmd=subfile {work_folder_original}/{main_file_original}.tex {work_folder_modified}/{main_file_modified}.tex --flatten > {work_folder}/merge_diff.tex') + os.system(f'latexdiff --encoding=utf8 --append-safecmd=subfile {work_folder_original}/{main_file_original}.tex {work_folder_modified}/{main_file_modified}.tex --flatten > {work_folder}/merge_diff.tex') + + yield from update_ui_lastest_msg(f'尝试第{n_fix}编译, 正在编译对比PDF ...', chatbot, history) # 刷新Gradio前端界面 + os.chdir(work_folder); os.system(f'pdflatex -interaction=batchmode -file-line-error merge_diff.tex'); os.chdir(current_dir) + os.chdir(work_folder); os.system(f'bibtex merge_diff.aux'); os.chdir(current_dir) + os.chdir(work_folder); os.system(f'pdflatex -interaction=batchmode -file-line-error merge_diff.tex'); os.chdir(current_dir) + os.chdir(work_folder); os.system(f'pdflatex -interaction=batchmode -file-line-error merge_diff.tex'); os.chdir(current_dir) + + # <---------------------> + os.chdir(current_dir) + + # <---------- 检查结果 -----------> + results_ = "" + original_pdf_success = os.path.exists(pj(work_folder_original, f'{main_file_original}.pdf')) + modified_pdf_success = os.path.exists(pj(work_folder_modified, f'{main_file_modified}.pdf')) + diff_pdf_success = os.path.exists(pj(work_folder, f'merge_diff.pdf')) + results_ += f"原始PDF编译是否成功: {original_pdf_success};" + results_ += f"转化PDF编译是否成功: {modified_pdf_success};" + results_ += f"对比PDF编译是否成功: {diff_pdf_success};" + yield from update_ui_lastest_msg(f'第{n_fix}编译结束:
{results_}...', chatbot, history) # 刷新Gradio前端界面 + + if modified_pdf_success: + yield from update_ui_lastest_msg(f'转化PDF编译已经成功, 即将退出 ...', chatbot, history) # 刷新Gradio前端界面 + os.chdir(current_dir) + return True # 成功啦 + else: + if n_fix>=10: break + n_fix += 1 + can_retry, main_file_modified, buggy_lines = remove_buggy_lines( + file_path=pj(work_folder_modified, f'{main_file_modified}.tex'), + log_path=pj(work_folder_modified, f'{main_file_modified}.log'), + tex_name=f'{main_file_modified}.tex', + tex_name_pure=f'{main_file_modified}', + n_fix=n_fix, + work_folder_modified=work_folder_modified, + ) + yield from update_ui_lastest_msg(f'由于最为关键的转化PDF编译失败, 将根据报错信息修正tex源文件并重试, 当前报错的latex代码处于第{buggy_lines}行 ...', chatbot, history) # 刷新Gradio前端界面 + if not can_retry: break + + os.chdir(current_dir) + return False # 失败啦 + + + diff --git a/toolbox.py b/toolbox.py index 159ac1f..319d19a 100644 --- a/toolbox.py +++ b/toolbox.py @@ -1,6 +1,6 @@ import markdown import importlib -import traceback +import time import inspect import re import os @@ -70,6 +70,17 @@ def update_ui(chatbot, history, msg='正常', **kwargs): # 刷新界面 assert isinstance(chatbot, ChatBotWithCookies), "在传递chatbot的过程中不要将其丢弃。必要时,可用clear将其清空,然后用for+append循环重新赋值。" yield chatbot.get_cookies(), chatbot, history, msg +def update_ui_lastest_msg(lastmsg, chatbot, history, delay=1): # 刷新界面 + """ + 刷新用户界面 + """ + if len(chatbot) == 0: chatbot.append(["update_ui_last_msg", lastmsg]) + chatbot[-1] = list(chatbot[-1]) + chatbot[-1][-1] = lastmsg + yield from update_ui(chatbot=chatbot, history=history) + time.sleep(delay) + + def trimmed_format_exc(): import os, traceback str = traceback.format_exc() @@ -728,6 +739,8 @@ def clip_history(inputs, history, tokenizer, max_token_limit): 其他小工具: - zip_folder: 把某个路径下所有文件压缩,然后转移到指定的另一个路径中(gpt写的) - gen_time_str: 生成时间戳 + - ProxyNetworkActivate: 临时地启动代理网络(如果有) + - objdump/objload: 快捷的调试函数 ======================================================================== """ @@ -762,11 +775,15 @@ def zip_folder(source_folder, dest_folder, zip_name): print(f"Zip file created at {zip_file}") +def zip_result(folder): + import time + t = time.strftime("%Y-%m-%d-%H-%M-%S", time.localtime()) + zip_folder(folder, './gpt_log/', f'{t}-result.zip') + def gen_time_str(): import time return time.strftime("%Y-%m-%d-%H-%M-%S", time.localtime()) - class ProxyNetworkActivate(): """ 这段代码定义了一个名为TempProxy的空上下文管理器, 用于给一小段代码上代理 @@ -784,4 +801,18 @@ class ProxyNetworkActivate(): os.environ['no_proxy'] = '*' if 'HTTP_PROXY' in os.environ: os.environ.pop('HTTP_PROXY') if 'HTTPS_PROXY' in os.environ: os.environ.pop('HTTPS_PROXY') - return \ No newline at end of file + return + +def objdump(obj): + import pickle + with open('objdump.tmp', 'wb+') as f: + pickle.dump(obj, f) + return + +def objload(): + import pickle, os + if not os.path.exists('objdump.tmp'): + return + with open('objdump.tmp', 'rb') as f: + return pickle.load(f) + \ No newline at end of file