From db16e85d8c4172d12725d442374eca7c52d4256a Mon Sep 17 00:00:00 2001 From: 505030475 <505030475@qq.com> Date: Tue, 23 May 2023 00:05:00 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BF=AE=E5=A4=8Dpdf=E7=BF=BB=E8=AF=91?= =?UTF-8?q?=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- crazy_functions/批量翻译PDF文档_多线程.py | 103 ++++++++++++++++++++-- toolbox.py | 11 ++- 2 files changed, 101 insertions(+), 13 deletions(-) diff --git a/crazy_functions/批量翻译PDF文档_多线程.py b/crazy_functions/批量翻译PDF文档_多线程.py index 4adb9a4..06d8a5a 100644 --- a/crazy_functions/批量翻译PDF文档_多线程.py +++ b/crazy_functions/批量翻译PDF文档_多线程.py @@ -58,14 +58,17 @@ def 批量翻译PDF文档(txt, llm_kwargs, plugin_kwargs, chatbot, history, sys_ def 解析PDF(file_manifest, project_folder, llm_kwargs, plugin_kwargs, chatbot, history, sys_prompt): import os + import copy import tiktoken TOKEN_LIMIT_PER_FRAGMENT = 1280 generated_conclusion_files = [] + generated_html_files = [] for index, fp in enumerate(file_manifest): # 读取PDF文件 file_content, page_one = read_and_clean_pdf_text(fp) - + file_content = file_content.encode('utf-8', 'ignore').decode() # avoid reading non-utf8 chars + page_one = str(page_one).encode('utf-8', 'ignore').decode() # avoid reading non-utf8 chars # 递归地切割PDF文件 from .crazy_utils import breakdown_txt_to_satisfy_token_limit_for_pdf from request_llm.bridge_all import model_info @@ -74,7 +77,7 @@ def 解析PDF(file_manifest, project_folder, llm_kwargs, plugin_kwargs, chatbot, paper_fragments = breakdown_txt_to_satisfy_token_limit_for_pdf( txt=file_content, get_token_fn=get_token_num, limit=TOKEN_LIMIT_PER_FRAGMENT) page_one_fragments = breakdown_txt_to_satisfy_token_limit_for_pdf( - txt=str(page_one), get_token_fn=get_token_num, limit=TOKEN_LIMIT_PER_FRAGMENT//4) + txt=page_one, get_token_fn=get_token_num, limit=TOKEN_LIMIT_PER_FRAGMENT//4) # 为了更好的效果，我们剥离Introduction之后的部分（如果有） paper_meta = page_one_fragments[0].split('introduction')[0].split('Introduction')[0].split('INTRODUCTION')[0] @@ -100,15 +103,15 @@ def 解析PDF(file_manifest, project_folder, llm_kwargs, plugin_kwargs, chatbot, "请你作为一个学术翻译，负责把学术论文准确翻译成中文。注意文章中的每一句话都要翻译。" for _ in paper_fragments], # max_workers=5 # OpenAI所允许的最大并行过载 ) - + gpt_response_collection_md = copy.deepcopy(gpt_response_collection) # 整理报告的格式 - for i,k in enumerate(gpt_response_collection): + for i,k in enumerate(gpt_response_collection_md): if i%2==0: - gpt_response_collection[i] = f"\n\n---\n\n ## 原文[{i//2}/{len(gpt_response_collection)//2}]： \n\n {paper_fragments[i//2].replace('#', '')} \n\n---\n\n ## 翻译[{i//2}/{len(gpt_response_collection)//2}]：\n " + gpt_response_collection_md[i] = f"\n\n---\n\n ## 原文[{i//2}/{len(gpt_response_collection_md)//2}]： \n\n {paper_fragments[i//2].replace('#', '')} \n\n---\n\n ## 翻译[{i//2}/{len(gpt_response_collection_md)//2}]：\n " else: - gpt_response_collection[i] = gpt_response_collection[i] + gpt_response_collection_md[i] = gpt_response_collection_md[i] final = ["一、论文概况\n\n---\n\n", paper_meta_info.replace('# ', '### ') + '\n\n---\n\n', "二、论文翻译", ""] - final.extend(gpt_response_collection) + final.extend(gpt_response_collection_md) create_report_file_name = f"{os.path.basename(fp)}.trans.md" res = write_results_to_file(final, file_name=create_report_file_name) @@ -117,15 +120,97 @@ def 解析PDF(file_manifest, project_folder, llm_kwargs, plugin_kwargs, chatbot, chatbot.append((f"{fp}完成了吗？", res)) yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 + # write html + try: + ch = construct_html() + orig = "" + trans = "" + gpt_response_collection_html = copy.deepcopy(gpt_response_collection) + for i,k in enumerate(gpt_response_collection_html): + if i%2==0: + gpt_response_collection_html[i] = paper_fragments[i//2].replace('#', '') + else: + gpt_response_collection_html[i] = gpt_response_collection_html[i] + final = ["论文概况", paper_meta_info.replace('# ', '### '), "二、论文翻译", ""] + final.extend(gpt_response_collection_html) + for i, k in enumerate(final): + if i%2==0: + orig = k + if i%2==1: + trans = k + ch.add_row(a=orig, b=trans) + create_report_file_name = f"{os.path.basename(fp)}.trans.html" + ch.save_file(create_report_file_name) + generated_html_files.append(f'./gpt_log/{create_report_file_name}') + except: + from toolbox import trimmed_format_exc + print('writing html result failed:', trimmed_format_exc()) + # 准备文件的下载 import shutil for pdf_path in generated_conclusion_files: # 重命名文件 - rename_file = f'./gpt_log/总结论文-{os.path.basename(pdf_path)}' + rename_file = f'./gpt_log/翻译-{os.path.basename(pdf_path)}' if os.path.exists(rename_file): os.remove(rename_file) shutil.copyfile(pdf_path, rename_file) if os.path.exists(pdf_path): os.remove(pdf_path) - chatbot.append(("给出输出文件清单", str(generated_conclusion_files))) + for html_path in generated_html_files: + # 重命名文件 + rename_file = f'./gpt_log/翻译-{os.path.basename(html_path)}' + if os.path.exists(rename_file): + os.remove(rename_file) + shutil.copyfile(html_path, rename_file) + if os.path.exists(html_path): + os.remove(html_path) + chatbot.append(("给出输出文件清单", str(generated_conclusion_files + generated_html_files))) yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 + + +class construct_html(): + def __init__(self) -> None: + self.css = """ +.row { + display: flex; + flex-wrap: wrap; +} + +.column { + flex: 1; + padding: 10px; +} + +.table-header { + font-weight: bold; + border-bottom: 1px solid black; +} + +.table-row { + border-bottom: 1px solid lightgray; +} + +.table-cell { + padding: 5px; +} + """ + self.html_string = f'翻译结果' + + + def add_row(self, a, b): + tmp = """ +
+
REPLACE_A
+
REPLACE_B
+
+ """ + from toolbox import markdown_convertion + tmp = tmp.replace('REPLACE_A', markdown_convertion(a)) + tmp = tmp.replace('REPLACE_B', markdown_convertion(b)) + self.html_string += tmp + + + def save_file(self, file_name): + with open(f'./gpt_log/{file_name}', 'w', encoding='utf8') as f: + f.write(self.html_string.encode('utf-8', 'ignore').decode()) + diff --git a/toolbox.py b/toolbox.py index 6f5469e..033c02f 100644 --- a/toolbox.py +++ b/toolbox.py @@ -168,14 +168,17 @@ def write_results_to_file(history, file_name=None): with open(f'./gpt_log/{file_name}', 'w', encoding='utf8') as f: f.write('# chatGPT 分析报告\n') for i, content in enumerate(history): - try: # 这个bug没找到触发条件，暂时先这样顶一下 - if type(content) != str: - content = str(content) + try: + if type(content) != str: content = str(content) except: continue if i % 2 == 0: f.write('## ') - f.write(content) + try: + f.write(content) + except: + # remove everything that cannot be handled by utf8 + f.write(content.encode('utf-8', 'ignore').decode()) f.write('\n\n') res = '以上材料已经被写入' + os.path.abspath(f'./gpt_log/{file_name}') print(res)