diff --git a/crazy_functions/下载arxiv论文翻译摘要.py b/crazy_functions/下载arxiv论文翻译摘要.py index 6cef3da..566d4c9 100644 --- a/crazy_functions/下载arxiv论文翻译摘要.py +++ b/crazy_functions/下载arxiv论文翻译摘要.py @@ -1,5 +1,4 @@ from toolbox import update_ui -from request_llm.bridge_chatgpt import predict_no_ui from toolbox import CatchException, report_execption, write_results_to_file, predict_no_ui_but_counting_down, get_conf import re, requests, unicodedata, os diff --git a/crazy_functions/总结word文档.py b/crazy_functions/总结word文档.py index 82eec1c..abf78b8 100644 --- a/crazy_functions/总结word文档.py +++ b/crazy_functions/总结word文档.py @@ -1,4 +1,3 @@ -from request_llm.bridge_chatgpt import predict_no_ui from toolbox import update_ui from toolbox import CatchException, report_execption, write_results_to_file, predict_no_ui_but_counting_down fast_debug = False diff --git a/crazy_functions/批量总结PDF文档.py b/crazy_functions/批量总结PDF文档.py index 4fa06d9..735ca4a 100644 --- a/crazy_functions/批量总结PDF文档.py +++ b/crazy_functions/批量总结PDF文档.py @@ -1,4 +1,3 @@ -from request_llm.bridge_chatgpt import predict_no_ui from toolbox import update_ui from toolbox import CatchException, report_execption, write_results_to_file, predict_no_ui_but_counting_down import re diff --git a/crazy_functions/批量总结PDF文档pdfminer.py b/crazy_functions/批量总结PDF文档pdfminer.py index 396a1f0..08c9bed 100644 --- a/crazy_functions/批量总结PDF文档pdfminer.py +++ b/crazy_functions/批量总结PDF文档pdfminer.py @@ -1,4 +1,3 @@ -from request_llm.bridge_chatgpt import predict_no_ui from toolbox import update_ui from toolbox import CatchException, report_execption, write_results_to_file, predict_no_ui_but_counting_down diff --git a/crazy_functions/批量翻译PDF文档_多线程.py b/crazy_functions/批量翻译PDF文档_多线程.py index 5a60024..08d0bd6 100644 --- a/crazy_functions/批量翻译PDF文档_多线程.py +++ b/crazy_functions/批量翻译PDF文档_多线程.py @@ -2,10 +2,12 @@ from toolbox import CatchException, report_execption, write_results_to_file from toolbox import update_ui from .crazy_utils import request_gpt_model_in_new_thread_with_ui_alive from .crazy_utils import request_gpt_model_multi_threads_with_very_awesome_ui_and_high_efficiency - +from colorful import * def read_and_clean_pdf_text(fp): """ + 这个函数用于分割pdf，用了很多trick，逻辑较乱，效果奇好，不建议任何人去读这个函数 + **输入参数说明** - `fp`：需要读取和清理文本的pdf文件路径 @@ -22,17 +24,43 @@ def read_and_clean_pdf_text(fp): - 清除重复的换行 - 将每个换行符替换为两个换行符，使每个段落之间有两个换行符分隔 """ - import fitz + import fitz, copy import re import numpy as np + fc = 0 + fs = 1 + fb = 2 + REMOVE_FOOT_NOTE = True + REMOVE_FOOT_FFSIZE_PERCENT = 0.95 + def primary_ffsize(l): + fsize_statiscs = {} + for wtf in l['spans']: + if wtf['size'] not in fsize_statiscs: fsize_statiscs[wtf['size']] = 0 + fsize_statiscs[wtf['size']] += len(wtf['text']) + return max(fsize_statiscs, key=fsize_statiscs.get) + + def ffsize_same(a,b): + return abs((a-b)/max(a,b)) < 0.02 # file_content = "" with fitz.open(fp) as doc: meta_txt = [] meta_font = [] + + meta_line = [] + meta_span = [] for index, page in enumerate(doc): # file_content += page.get_text() text_areas = page.get_text("dict") # 获取页面上的文本信息 - + for t in text_areas['blocks']: + if 'lines' in t: + pf = 998 + for l in t['lines']: + txt_line = "".join([wtf['text'] for wtf in l['spans']]) + pf = primary_ffsize(l) + meta_line.append([txt_line, pf, l['bbox'], l]) + for wtf in l['spans']: # for l in t['lines']: + meta_span.append([wtf['text'], wtf['size'], len(wtf['text'])]) + # meta_line.append(["NEW_BLOCK", pf]) # 块元提取 for each word segment with in line for each line cross-line words for each block meta_txt.extend([" ".join(["".join([wtf['text'] for wtf in l['spans']]) for l in t['lines']]).replace( '- ', '') for t in text_areas['blocks'] if 'lines' in t]) @@ -41,6 +69,56 @@ def read_and_clean_pdf_text(fp): if index == 0: page_one_meta = [" ".join(["".join([wtf['text'] for wtf in l['spans']]) for l in t['lines']]).replace( '- ', '') for t in text_areas['blocks'] if 'lines' in t] + # 获取正文主字体 + fsize_statiscs = {} + for span in meta_span: + if span[1] not in fsize_statiscs: fsize_statiscs[span[1]] = 0 + fsize_statiscs[span[1]] += span[2] + main_fsize = max(fsize_statiscs, key=fsize_statiscs.get) + if REMOVE_FOOT_NOTE: + give_up_fize_threshold = main_fsize * REMOVE_FOOT_FFSIZE_PERCENT + + # 切分和重新整合 + mega_sec = [] + sec = [] + for index, line in enumerate(meta_line): + if index == 0: + sec.append(line[fc]) + continue + if REMOVE_FOOT_NOTE: + if meta_line[index][fs] <= give_up_fize_threshold: + continue + if ffsize_same(meta_line[index][fs], meta_line[index-1][fs]): + # 尝试识别段落 + if meta_line[index][fc].endswith('.') and\ + (meta_line[index-1][fc] != 'NEW_BLOCK') and \ + (meta_line[index][fb][2] - meta_line[index][fb][0]) < (meta_line[index-1][fb][2] - meta_line[index-1][fb][0]) * 0.7: + sec[-1] += line[fc] + sec[-1] += "\n\n" + else: + sec[-1] += " " + sec[-1] += line[fc] + else: + if (index+1 < len(meta_line)) and \ + meta_line[index][fs] > main_fsize: + # 单行 + 字体大 + mega_sec.append(copy.deepcopy(sec)) + sec = [] + sec.append("# " + line[fc]) + else: + # 尝试识别section + if meta_line[index-1][fs] > meta_line[index][fs]: + sec.append("\n" + line[fc]) + else: + sec.append(line[fc]) + mega_sec.append(copy.deepcopy(sec)) + + finals = [] + for ms in mega_sec: + final = " ".join(ms) + final = final.replace('- ', ' ') + finals.append(final) + meta_txt = finals def 把字符太少的块清除为回车(meta_txt): for index, block_txt in enumerate(meta_txt): @@ -85,6 +163,10 @@ def read_and_clean_pdf_text(fp): # 换行 -> 双换行 meta_txt = meta_txt.replace('\n', '\n\n') + for f in finals: + print亮黄(f) + print亮绿('***************************') + return meta_txt, page_one_meta @@ -145,21 +227,23 @@ def 解析PDF(file_manifest, project_folder, llm_kwargs, plugin_kwargs, chatbot, TOKEN_LIMIT_PER_FRAGMENT = 1600 generated_conclusion_files = [] for index, fp in enumerate(file_manifest): + # 读取PDF文件 file_content, page_one = read_and_clean_pdf_text(fp) + # 递归地切割PDF文件 from .crazy_utils import breakdown_txt_to_satisfy_token_limit_for_pdf from toolbox import get_conf enc = tiktoken.encoding_for_model(*get_conf('LLM_MODEL')) def get_token_num(txt): return len(enc.encode(txt)) - # 分解文本 paper_fragments = breakdown_txt_to_satisfy_token_limit_for_pdf( txt=file_content, get_token_fn=get_token_num, limit=TOKEN_LIMIT_PER_FRAGMENT) page_one_fragments = breakdown_txt_to_satisfy_token_limit_for_pdf( txt=str(page_one), get_token_fn=get_token_num, limit=TOKEN_LIMIT_PER_FRAGMENT//4) - # 为了更好的效果，我们剥离Introduction之后的部分 - paper_meta = page_one_fragments[0].split('introduction')[0].split( - 'Introduction')[0].split('INTRODUCTION')[0] + + # 为了更好的效果，我们剥离Introduction之后的部分（如果有） + paper_meta = page_one_fragments[0].split('introduction')[0].split('Introduction')[0].split('INTRODUCTION')[0] + # 单线，获取文章meta信息 paper_meta_info = yield from request_gpt_model_in_new_thread_with_ui_alive( inputs=f"以下是一篇学术论文的基础信息，请从中提取出“标题”、“收录会议或期刊”、“作者”、“摘要”、“编号”、“作者邮箱”这六个部分。请用markdown格式输出，最后用中文翻译摘要部分。请提取：{paper_meta}", @@ -168,23 +252,32 @@ def 解析PDF(file_manifest, project_folder, llm_kwargs, plugin_kwargs, chatbot, chatbot=chatbot, history=[], sys_prompt="Your job is to collect information from materials。", ) + # 多线，翻译 gpt_response_collection = yield from request_gpt_model_multi_threads_with_very_awesome_ui_and_high_efficiency( inputs_array=[ - f"以下是你需要翻译的文章段落：\n{frag}" for frag in paper_fragments], - inputs_show_user_array=[f"" for _ in paper_fragments], + f"以下是你需要翻译的论文片段：\n{frag}" for frag in paper_fragments], + inputs_show_user_array=[f"\n---\n 原文： \n\n {frag.replace('#', '')} \n---\n 翻译：\n " for frag in paper_fragments], llm_kwargs=llm_kwargs, chatbot=chatbot, history_array=[[paper_meta] for _ in paper_fragments], sys_prompt_array=[ - "请你作为一个学术翻译，把整个段落翻译成中文，要求语言简洁，禁止重复输出原文。" for _ in paper_fragments], + "请你作为一个学术翻译，负责把学术论文的片段准确翻译成中文。" for _ in paper_fragments], max_workers=16 # OpenAI所允许的最大并行过载 ) - final = ["", paper_meta_info + '\n\n---\n\n---\n\n---\n\n'] + # 整理报告的格式 + for i,k in enumerate(gpt_response_collection): + if i%2==0: + gpt_response_collection[i] = f"\n\n---\n\n ## 原文[{i//2}/{len(gpt_response_collection)//2}]： \n\n {paper_fragments[i//2].replace('#', '')} \n\n---\n\n ## 翻译[{i//2}/{len(gpt_response_collection)//2}]：\n " + else: + gpt_response_collection[i] = gpt_response_collection[i] + final = ["一、论文概况\n\n---\n\n", paper_meta_info.replace('# ', '### ') + '\n\n---\n\n', "二、论文翻译", ""] final.extend(gpt_response_collection) create_report_file_name = f"{os.path.basename(fp)}.trans.md" res = write_results_to_file(final, file_name=create_report_file_name) + + # 更新UI generated_conclusion_files.append(f'./gpt_log/{create_report_file_name}') chatbot.append((f"{fp}完成了吗？", res)) yield from update_ui(chatbot=chatbot, history=chatbot) # 刷新界面 @@ -200,4 +293,4 @@ def 解析PDF(file_manifest, project_folder, llm_kwargs, plugin_kwargs, chatbot, if os.path.exists(pdf_path): os.remove(pdf_path) chatbot.append(("给出输出文件清单", str(generated_conclusion_files))) - yield from update_ui(chatbot=chatbot, history=chatbot, msg=msg) # 刷新界面 + yield from update_ui(chatbot=chatbot, history=chatbot) # 刷新界面 diff --git a/crazy_functions/理解PDF文档内容.py b/crazy_functions/理解PDF文档内容.py index 4ad3dbf..2b938b8 100644 --- a/crazy_functions/理解PDF文档内容.py +++ b/crazy_functions/理解PDF文档内容.py @@ -1,4 +1,3 @@ -from request_llm.bridge_chatgpt import predict_no_ui from toolbox import update_ui from toolbox import CatchException, report_execption, write_results_to_file, predict_no_ui_but_counting_down import re diff --git a/crazy_functions/生成函数注释.py b/crazy_functions/生成函数注释.py index 5cf25f2..87506cd 100644 --- a/crazy_functions/生成函数注释.py +++ b/crazy_functions/生成函数注释.py @@ -1,4 +1,3 @@ -from request_llm.bridge_chatgpt import predict_no_ui from toolbox import update_ui from toolbox import CatchException, report_execption, write_results_to_file, predict_no_ui_but_counting_down fast_debug = False diff --git a/crazy_functions/读文章写摘要.py b/crazy_functions/读文章写摘要.py index af57c84..663a062 100644 --- a/crazy_functions/读文章写摘要.py +++ b/crazy_functions/读文章写摘要.py @@ -1,4 +1,3 @@ -from request_llm.bridge_chatgpt import predict_no_ui from toolbox import update_ui from toolbox import CatchException, report_execption, write_results_to_file, predict_no_ui_but_counting_down fast_debug = False diff --git a/request_llm/bridge_chatgpt.py b/request_llm/bridge_chatgpt.py index 4efdf5e..b54badd 100644 --- a/request_llm/bridge_chatgpt.py +++ b/request_llm/bridge_chatgpt.py @@ -39,38 +39,6 @@ def get_full_error(chunk, stream_response): break return chunk -def predict_no_ui(inputs, top_p, temperature, history=[], sys_prompt=""): - """ - 发送至chatGPT，等待回复，一次性完成，不显示中间过程。 - predict函数的简化版。 - 用于payload比较大的情况，或者用于实现多线、带嵌套的复杂功能。 - - inputs 是本次问询的输入 - top_p, temperature是chatGPT的内部调优参数 - history 是之前的对话列表 - （注意无论是inputs还是history，内容太长了都会触发token数量溢出的错误，然后raise ConnectionAbortedError） - """ - headers, payload = generate_payload(inputs, top_p, temperature, history, system_prompt=sys_prompt, stream=False) - - retry = 0 - while True: - try: - # make a POST request to the API endpoint, stream=False - response = requests.post(API_URL, headers=headers, proxies=proxies, - json=payload, stream=False, timeout=TIMEOUT_SECONDS*2); break - except requests.exceptions.ReadTimeout as e: - retry += 1 - traceback.print_exc() - if retry > MAX_RETRY: raise TimeoutError - if MAX_RETRY!=0: print(f'请求超时，正在重试 ({retry}/{MAX_RETRY}) ……') - - try: - result = json.loads(response.text)["choices"][0]["message"]["content"] - return result - except Exception as e: - if "choices" not in response.text: print(response.text) - raise ConnectionAbortedError("Json解析不合常规，可能是文本过长" + response.text) - def predict_no_ui_long_connection(inputs, llm_kwargs, history=[], sys_prompt="", observe_window=None, console_slience=False): """ @@ -276,7 +244,10 @@ def generate_payload(inputs, llm_kwargs, history, system_prompt, stream): "presence_penalty": 0, "frequency_penalty": 0, } - print(f" {llm_kwargs['llm_model']} : {conversation_cnt} : {inputs[:100]}") + try: + print(f" {llm_kwargs['llm_model']} : {conversation_cnt} : {inputs[:100]}") + except: + print('输入中可能存在乱码。') return headers,payload diff --git a/toolbox.py b/toolbox.py index 1654e69..f16fc53 100644 --- a/toolbox.py +++ b/toolbox.py @@ -87,10 +87,10 @@ def predict_no_ui_but_counting_down(i_say, i_say_show_user, chatbot, top_p, temp top_p, temperature: gpt参数 history: gpt参数 对话历史 sys_prompt: gpt参数 sys_prompt - long_connection: 是否采用更稳定的连接方式（推荐） + long_connection: 是否采用更稳定的连接方式（推荐）（已弃用） """ import time - from request_llm.bridge_chatgpt import predict_no_ui, predict_no_ui_long_connection + from request_llm.bridge_chatgpt import predict_no_ui_long_connection from toolbox import get_conf TIMEOUT_SECONDS, MAX_RETRY = get_conf('TIMEOUT_SECONDS', 'MAX_RETRY') # 多线程的时候，需要一个mutable结构在不同线程之间传递信息 @@ -101,13 +101,9 @@ def predict_no_ui_but_counting_down(i_say, i_say_show_user, chatbot, top_p, temp def mt(i_say, history): while True: try: - if long_connection: - mutable[0] = predict_no_ui_long_connection( - inputs=i_say, top_p=top_p, temperature=temperature, history=history, sys_prompt=sys_prompt) - else: - mutable[0] = predict_no_ui( - inputs=i_say, top_p=top_p, temperature=temperature, history=history, sys_prompt=sys_prompt) - break + mutable[0] = predict_no_ui_long_connection( + inputs=i_say, top_p=top_p, temperature=temperature, history=history, sys_prompt=sys_prompt) + except ConnectionAbortedError as token_exceeded_error: # 尝试计算比例，尽可能多地保留文本 p_ratio, n_exceed = get_reduce_token_percent(