Merge pull request #1111 from kaixindelele/only_chinese_pdf
提升PDF翻译插件的效果
This commit is contained in:
		
						commit
						0801e4d881
					
				@ -1,6 +1,14 @@
 | 
			
		||||
from functools import lru_cache
 | 
			
		||||
from toolbox import gen_time_str
 | 
			
		||||
from toolbox import promote_file_to_downloadzone
 | 
			
		||||
from toolbox import write_history_to_file, promote_file_to_downloadzone
 | 
			
		||||
from colorful import *
 | 
			
		||||
import requests
 | 
			
		||||
import random
 | 
			
		||||
from functools import lru_cache
 | 
			
		||||
import copy
 | 
			
		||||
import os
 | 
			
		||||
import math
 | 
			
		||||
 | 
			
		||||
class GROBID_OFFLINE_EXCEPTION(Exception): pass
 | 
			
		||||
 | 
			
		||||
def get_avail_grobid_url():
 | 
			
		||||
@ -28,3 +36,133 @@ def parse_pdf(pdf_path, grobid_url):
 | 
			
		||||
        raise RuntimeError("解析PDF失败,请检查PDF是否损坏。")
 | 
			
		||||
    return article_dict
 | 
			
		||||
 | 
			
		||||
 | 
			
		||||
def produce_report_markdown(gpt_response_collection, meta, paper_meta_info, chatbot, fp, generated_conclusion_files):
 | 
			
		||||
    # -=-=-=-=-=-=-=-= 写出第1个文件:翻译前后混合 -=-=-=-=-=-=-=-=
 | 
			
		||||
    res_path = write_history_to_file(meta +  ["# Meta Translation" , paper_meta_info] + gpt_response_collection, file_basename=f"{gen_time_str()}translated_and_original.md", file_fullname=None)
 | 
			
		||||
    promote_file_to_downloadzone(res_path, rename_file=os.path.basename(res_path)+'.md', chatbot=chatbot)
 | 
			
		||||
    generated_conclusion_files.append(res_path)
 | 
			
		||||
 | 
			
		||||
    # -=-=-=-=-=-=-=-= 写出第2个文件:仅翻译后的文本 -=-=-=-=-=-=-=-=
 | 
			
		||||
    translated_res_array = []
 | 
			
		||||
    # 记录当前的大章节标题:
 | 
			
		||||
    last_section_name = ""
 | 
			
		||||
    for index, value in enumerate(gpt_response_collection):
 | 
			
		||||
        # 先挑选偶数序列号:
 | 
			
		||||
        if index % 2 != 0:
 | 
			
		||||
            # 先提取当前英文标题:
 | 
			
		||||
            cur_section_name = gpt_response_collection[index-1].split('\n')[0].split(" Part")[0]
 | 
			
		||||
            # 如果index是1的话,则直接使用first section name:
 | 
			
		||||
            if cur_section_name != last_section_name:
 | 
			
		||||
                cur_value = cur_section_name + '\n'
 | 
			
		||||
                last_section_name = copy.deepcopy(cur_section_name)
 | 
			
		||||
            else:
 | 
			
		||||
                cur_value = ""
 | 
			
		||||
            # 再做一个小修改:重新修改当前part的标题,默认用英文的
 | 
			
		||||
            cur_value += value
 | 
			
		||||
            translated_res_array.append(cur_value)
 | 
			
		||||
    res_path = write_history_to_file(meta +  ["# Meta Translation" , paper_meta_info] + translated_res_array, 
 | 
			
		||||
                                     file_basename = f"{gen_time_str()}-translated_only.md", 
 | 
			
		||||
                                     file_fullname = None,
 | 
			
		||||
                                     auto_caption = False)
 | 
			
		||||
    promote_file_to_downloadzone(res_path, rename_file=os.path.basename(res_path)+'.md', chatbot=chatbot)
 | 
			
		||||
    generated_conclusion_files.append(res_path)
 | 
			
		||||
    return res_path
 | 
			
		||||
 | 
			
		||||
def translate_pdf(article_dict, llm_kwargs, chatbot, fp, generated_conclusion_files, TOKEN_LIMIT_PER_FRAGMENT, DST_LANG):
 | 
			
		||||
    from crazy_functions.crazy_utils import construct_html
 | 
			
		||||
    from crazy_functions.crazy_utils import breakdown_txt_to_satisfy_token_limit_for_pdf
 | 
			
		||||
    from crazy_functions.crazy_utils import request_gpt_model_in_new_thread_with_ui_alive
 | 
			
		||||
    from crazy_functions.crazy_utils import request_gpt_model_multi_threads_with_very_awesome_ui_and_high_efficiency
 | 
			
		||||
 | 
			
		||||
    prompt = "以下是一篇学术论文的基本信息:\n"
 | 
			
		||||
    # title
 | 
			
		||||
    title = article_dict.get('title', '无法获取 title'); prompt += f'title:{title}\n\n'
 | 
			
		||||
    # authors
 | 
			
		||||
    authors = article_dict.get('authors', '无法获取 authors'); prompt += f'authors:{authors}\n\n'
 | 
			
		||||
    # abstract
 | 
			
		||||
    abstract = article_dict.get('abstract', '无法获取 abstract'); prompt += f'abstract:{abstract}\n\n'
 | 
			
		||||
    # command
 | 
			
		||||
    prompt += f"请将题目和摘要翻译为{DST_LANG}。"
 | 
			
		||||
    meta = [f'# Title:\n\n', title, f'# Abstract:\n\n', abstract ]
 | 
			
		||||
 | 
			
		||||
    # 单线,获取文章meta信息
 | 
			
		||||
    paper_meta_info = yield from request_gpt_model_in_new_thread_with_ui_alive(
 | 
			
		||||
        inputs=prompt,
 | 
			
		||||
        inputs_show_user=prompt,
 | 
			
		||||
        llm_kwargs=llm_kwargs,
 | 
			
		||||
        chatbot=chatbot, history=[],
 | 
			
		||||
        sys_prompt="You are an academic paper reader。",
 | 
			
		||||
    )
 | 
			
		||||
 | 
			
		||||
    # 多线,翻译
 | 
			
		||||
    inputs_array = []
 | 
			
		||||
    inputs_show_user_array = []
 | 
			
		||||
 | 
			
		||||
    # get_token_num
 | 
			
		||||
    from request_llm.bridge_all import model_info
 | 
			
		||||
    enc = model_info[llm_kwargs['llm_model']]['tokenizer']
 | 
			
		||||
    def get_token_num(txt): return len(enc.encode(txt, disallowed_special=()))
 | 
			
		||||
 | 
			
		||||
    def break_down(txt):
 | 
			
		||||
        raw_token_num = get_token_num(txt)
 | 
			
		||||
        if raw_token_num <= TOKEN_LIMIT_PER_FRAGMENT:
 | 
			
		||||
            return [txt]
 | 
			
		||||
        else:
 | 
			
		||||
            # raw_token_num > TOKEN_LIMIT_PER_FRAGMENT
 | 
			
		||||
            # find a smooth token limit to achieve even seperation
 | 
			
		||||
            count = int(math.ceil(raw_token_num / TOKEN_LIMIT_PER_FRAGMENT))
 | 
			
		||||
            token_limit_smooth = raw_token_num // count + count
 | 
			
		||||
            return breakdown_txt_to_satisfy_token_limit_for_pdf(txt, get_token_fn=get_token_num, limit=token_limit_smooth)
 | 
			
		||||
 | 
			
		||||
    for section in article_dict.get('sections'):
 | 
			
		||||
        if len(section['text']) == 0: continue
 | 
			
		||||
        section_frags = break_down(section['text'])
 | 
			
		||||
        for i, fragment in enumerate(section_frags):
 | 
			
		||||
            heading = section['heading']
 | 
			
		||||
            if len(section_frags) > 1: heading += f' Part-{i+1}'
 | 
			
		||||
            inputs_array.append(
 | 
			
		||||
                f"你需要翻译{heading}章节,内容如下: \n\n{fragment}"
 | 
			
		||||
            )
 | 
			
		||||
            inputs_show_user_array.append(
 | 
			
		||||
                f"# {heading}\n\n{fragment}"
 | 
			
		||||
            )
 | 
			
		||||
 | 
			
		||||
    gpt_response_collection = yield from request_gpt_model_multi_threads_with_very_awesome_ui_and_high_efficiency(
 | 
			
		||||
        inputs_array=inputs_array,
 | 
			
		||||
        inputs_show_user_array=inputs_show_user_array,
 | 
			
		||||
        llm_kwargs=llm_kwargs,
 | 
			
		||||
        chatbot=chatbot,
 | 
			
		||||
        history_array=[meta for _ in inputs_array],
 | 
			
		||||
        sys_prompt_array=[
 | 
			
		||||
            "请你作为一个学术翻译,负责把学术论文准确翻译成中文。注意文章中的每一句话都要翻译。" for _ in inputs_array],
 | 
			
		||||
    )
 | 
			
		||||
    # -=-=-=-=-=-=-=-= 写出Markdown文件 -=-=-=-=-=-=-=-=
 | 
			
		||||
    produce_report_markdown(gpt_response_collection, meta, paper_meta_info, chatbot, fp, generated_conclusion_files)
 | 
			
		||||
 | 
			
		||||
    # -=-=-=-=-=-=-=-= 写出HTML文件 -=-=-=-=-=-=-=-=
 | 
			
		||||
    ch = construct_html() 
 | 
			
		||||
    orig = ""
 | 
			
		||||
    trans = ""
 | 
			
		||||
    gpt_response_collection_html = copy.deepcopy(gpt_response_collection)
 | 
			
		||||
    for i,k in enumerate(gpt_response_collection_html): 
 | 
			
		||||
        if i%2==0:
 | 
			
		||||
            gpt_response_collection_html[i] = inputs_show_user_array[i//2]
 | 
			
		||||
        else:
 | 
			
		||||
            # 先提取当前英文标题:
 | 
			
		||||
            cur_section_name = gpt_response_collection[i-1].split('\n')[0].split(" Part")[0]
 | 
			
		||||
            cur_value = cur_section_name + "\n" + gpt_response_collection_html[i]
 | 
			
		||||
            gpt_response_collection_html[i] = cur_value
 | 
			
		||||
 | 
			
		||||
    final = ["", "", "一、论文概况",  "", "Abstract", paper_meta_info,  "二、论文翻译",  ""]
 | 
			
		||||
    final.extend(gpt_response_collection_html)
 | 
			
		||||
    for i, k in enumerate(final): 
 | 
			
		||||
        if i%2==0:
 | 
			
		||||
            orig = k
 | 
			
		||||
        if i%2==1:
 | 
			
		||||
            trans = k
 | 
			
		||||
            ch.add_row(a=orig, b=trans)
 | 
			
		||||
    create_report_file_name = f"{os.path.basename(fp)}.trans.html"
 | 
			
		||||
    html_file = ch.save_file(create_report_file_name)
 | 
			
		||||
    generated_conclusion_files.append(html_file)
 | 
			
		||||
    promote_file_to_downloadzone(html_file, rename_file=os.path.basename(html_file), chatbot=chatbot)
 | 
			
		||||
 | 
			
		||||
@ -1,11 +1,12 @@
 | 
			
		||||
from toolbox import CatchException, report_execption, gen_time_str
 | 
			
		||||
from toolbox import CatchException, report_execption, get_log_folder, gen_time_str
 | 
			
		||||
from toolbox import update_ui, promote_file_to_downloadzone, update_ui_lastest_msg, disable_auto_promotion
 | 
			
		||||
from toolbox import write_history_to_file, get_log_folder
 | 
			
		||||
from toolbox import write_history_to_file, promote_file_to_downloadzone
 | 
			
		||||
from .crazy_utils import request_gpt_model_in_new_thread_with_ui_alive
 | 
			
		||||
from .crazy_utils import request_gpt_model_multi_threads_with_very_awesome_ui_and_high_efficiency
 | 
			
		||||
from .crazy_utils import read_and_clean_pdf_text
 | 
			
		||||
from .pdf_fns.parse_pdf import parse_pdf, get_avail_grobid_url
 | 
			
		||||
from .pdf_fns.parse_pdf import parse_pdf, get_avail_grobid_url, translate_pdf
 | 
			
		||||
from colorful import *
 | 
			
		||||
import copy
 | 
			
		||||
import os
 | 
			
		||||
import math
 | 
			
		||||
import logging
 | 
			
		||||
@ -92,7 +93,7 @@ def 批量翻译PDF文档(txt, llm_kwargs, plugin_kwargs, chatbot, history, syst
 | 
			
		||||
def 解析PDF_基于NOUGAT(file_manifest, project_folder, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt):
 | 
			
		||||
    import copy
 | 
			
		||||
    import tiktoken
 | 
			
		||||
    TOKEN_LIMIT_PER_FRAGMENT = 1280
 | 
			
		||||
    TOKEN_LIMIT_PER_FRAGMENT = 512
 | 
			
		||||
    generated_conclusion_files = []
 | 
			
		||||
    generated_html_files = []
 | 
			
		||||
    DST_LANG = "中文"
 | 
			
		||||
@ -106,96 +107,7 @@ def 解析PDF_基于NOUGAT(file_manifest, project_folder, llm_kwargs, plugin_kwa
 | 
			
		||||
            article_content = f.readlines()
 | 
			
		||||
        article_dict = markdown_to_dict(article_content)
 | 
			
		||||
        logging.info(article_dict)
 | 
			
		||||
 | 
			
		||||
        prompt = "以下是一篇学术论文的基本信息:\n"
 | 
			
		||||
        # title
 | 
			
		||||
        title = article_dict.get('title', '无法获取 title'); prompt += f'title:{title}\n\n'
 | 
			
		||||
        # authors
 | 
			
		||||
        authors = article_dict.get('authors', '无法获取 authors'); prompt += f'authors:{authors}\n\n'
 | 
			
		||||
        # abstract
 | 
			
		||||
        abstract = article_dict.get('abstract', '无法获取 abstract'); prompt += f'abstract:{abstract}\n\n'
 | 
			
		||||
        # command
 | 
			
		||||
        prompt += f"请将题目和摘要翻译为{DST_LANG}。"
 | 
			
		||||
        meta = [f'# Title:\n\n', title, f'# Abstract:\n\n', abstract ]
 | 
			
		||||
 | 
			
		||||
        # 单线,获取文章meta信息
 | 
			
		||||
        paper_meta_info = yield from request_gpt_model_in_new_thread_with_ui_alive(
 | 
			
		||||
            inputs=prompt,
 | 
			
		||||
            inputs_show_user=prompt,
 | 
			
		||||
            llm_kwargs=llm_kwargs,
 | 
			
		||||
            chatbot=chatbot, history=[],
 | 
			
		||||
            sys_prompt="You are an academic paper reader。",
 | 
			
		||||
        )
 | 
			
		||||
 | 
			
		||||
        # 多线,翻译
 | 
			
		||||
        inputs_array = []
 | 
			
		||||
        inputs_show_user_array = []
 | 
			
		||||
 | 
			
		||||
        # get_token_num
 | 
			
		||||
        from request_llm.bridge_all import model_info
 | 
			
		||||
        enc = model_info[llm_kwargs['llm_model']]['tokenizer']
 | 
			
		||||
        def get_token_num(txt): return len(enc.encode(txt, disallowed_special=()))
 | 
			
		||||
        from .crazy_utils import breakdown_txt_to_satisfy_token_limit_for_pdf
 | 
			
		||||
 | 
			
		||||
        def break_down(txt):
 | 
			
		||||
            raw_token_num = get_token_num(txt)
 | 
			
		||||
            if raw_token_num <= TOKEN_LIMIT_PER_FRAGMENT:
 | 
			
		||||
                return [txt]
 | 
			
		||||
            else:
 | 
			
		||||
                # raw_token_num > TOKEN_LIMIT_PER_FRAGMENT
 | 
			
		||||
                # find a smooth token limit to achieve even seperation
 | 
			
		||||
                count = int(math.ceil(raw_token_num / TOKEN_LIMIT_PER_FRAGMENT))
 | 
			
		||||
                token_limit_smooth = raw_token_num // count + count
 | 
			
		||||
                return breakdown_txt_to_satisfy_token_limit_for_pdf(txt, get_token_fn=get_token_num, limit=token_limit_smooth)
 | 
			
		||||
 | 
			
		||||
        for section in article_dict.get('sections'):
 | 
			
		||||
            if len(section['text']) == 0: continue
 | 
			
		||||
            section_frags = break_down(section['text'])
 | 
			
		||||
            for i, fragment in enumerate(section_frags):
 | 
			
		||||
                heading = section['heading']
 | 
			
		||||
                if len(section_frags) > 1: heading += f' Part-{i+1}'
 | 
			
		||||
                inputs_array.append(
 | 
			
		||||
                    f"你需要翻译{heading}章节,内容如下: \n\n{fragment}"
 | 
			
		||||
                )
 | 
			
		||||
                inputs_show_user_array.append(
 | 
			
		||||
                    f"# {heading}\n\n{fragment}"
 | 
			
		||||
                )
 | 
			
		||||
 | 
			
		||||
        gpt_response_collection = yield from request_gpt_model_multi_threads_with_very_awesome_ui_and_high_efficiency(
 | 
			
		||||
            inputs_array=inputs_array,
 | 
			
		||||
            inputs_show_user_array=inputs_show_user_array,
 | 
			
		||||
            llm_kwargs=llm_kwargs,
 | 
			
		||||
            chatbot=chatbot,
 | 
			
		||||
            history_array=[meta for _ in inputs_array],
 | 
			
		||||
            sys_prompt_array=[
 | 
			
		||||
                "请你作为一个学术翻译,负责把学术论文准确翻译成中文。注意文章中的每一句话都要翻译。" for _ in inputs_array],
 | 
			
		||||
        )
 | 
			
		||||
        res_path = write_history_to_file(meta +  ["# Meta Translation" , paper_meta_info] + gpt_response_collection, file_basename=None, file_fullname=None)
 | 
			
		||||
        promote_file_to_downloadzone(res_path, rename_file=os.path.basename(fp)+'.md', chatbot=chatbot)
 | 
			
		||||
        generated_conclusion_files.append(res_path)
 | 
			
		||||
 | 
			
		||||
        ch = construct_html() 
 | 
			
		||||
        orig = ""
 | 
			
		||||
        trans = ""
 | 
			
		||||
        gpt_response_collection_html = copy.deepcopy(gpt_response_collection)
 | 
			
		||||
        for i,k in enumerate(gpt_response_collection_html): 
 | 
			
		||||
            if i%2==0:
 | 
			
		||||
                gpt_response_collection_html[i] = inputs_show_user_array[i//2]
 | 
			
		||||
            else:
 | 
			
		||||
                gpt_response_collection_html[i] = gpt_response_collection_html[i]
 | 
			
		||||
 | 
			
		||||
        final = ["", "", "一、论文概况",  "", "Abstract", paper_meta_info,  "二、论文翻译",  ""]
 | 
			
		||||
        final.extend(gpt_response_collection_html)
 | 
			
		||||
        for i, k in enumerate(final): 
 | 
			
		||||
            if i%2==0:
 | 
			
		||||
                orig = k
 | 
			
		||||
            if i%2==1:
 | 
			
		||||
                trans = k
 | 
			
		||||
                ch.add_row(a=orig, b=trans)
 | 
			
		||||
        create_report_file_name = f"{os.path.basename(fp)}.trans.html"
 | 
			
		||||
        html_file = ch.save_file(create_report_file_name)
 | 
			
		||||
        generated_html_files.append(html_file)
 | 
			
		||||
        promote_file_to_downloadzone(html_file, rename_file=os.path.basename(html_file), chatbot=chatbot)
 | 
			
		||||
        yield from translate_pdf(article_dict, llm_kwargs, chatbot, fp, generated_conclusion_files, TOKEN_LIMIT_PER_FRAGMENT, DST_LANG)
 | 
			
		||||
 | 
			
		||||
    chatbot.append(("给出输出文件清单", str(generated_conclusion_files + generated_html_files)))
 | 
			
		||||
    yield from update_ui(chatbot=chatbot, history=history) # 刷新界面
 | 
			
		||||
 | 
			
		||||
@ -1,12 +1,12 @@
 | 
			
		||||
from toolbox import CatchException, report_execption, get_log_folder
 | 
			
		||||
from toolbox import CatchException, report_execption, get_log_folder, gen_time_str
 | 
			
		||||
from toolbox import update_ui, promote_file_to_downloadzone, update_ui_lastest_msg, disable_auto_promotion
 | 
			
		||||
from toolbox import write_history_to_file, promote_file_to_downloadzone
 | 
			
		||||
from .crazy_utils import request_gpt_model_in_new_thread_with_ui_alive
 | 
			
		||||
from .crazy_utils import request_gpt_model_multi_threads_with_very_awesome_ui_and_high_efficiency
 | 
			
		||||
from .crazy_utils import read_and_clean_pdf_text
 | 
			
		||||
from .pdf_fns.parse_pdf import parse_pdf, get_avail_grobid_url
 | 
			
		||||
from .pdf_fns.parse_pdf import parse_pdf, get_avail_grobid_url, translate_pdf
 | 
			
		||||
from colorful import *
 | 
			
		||||
import glob
 | 
			
		||||
import copy
 | 
			
		||||
import os
 | 
			
		||||
import math
 | 
			
		||||
 | 
			
		||||
@ -58,8 +58,8 @@ def 批量翻译PDF文档(txt, llm_kwargs, plugin_kwargs, chatbot, history, syst
 | 
			
		||||
 | 
			
		||||
 | 
			
		||||
def 解析PDF_基于GROBID(file_manifest, project_folder, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, grobid_url):
 | 
			
		||||
    import copy
 | 
			
		||||
    TOKEN_LIMIT_PER_FRAGMENT = 1280
 | 
			
		||||
    import copy, json
 | 
			
		||||
    TOKEN_LIMIT_PER_FRAGMENT = 512
 | 
			
		||||
    generated_conclusion_files = []
 | 
			
		||||
    generated_html_files = []
 | 
			
		||||
    DST_LANG = "中文"
 | 
			
		||||
@ -67,104 +67,23 @@ def 解析PDF_基于GROBID(file_manifest, project_folder, llm_kwargs, plugin_kwa
 | 
			
		||||
    for index, fp in enumerate(file_manifest):
 | 
			
		||||
        chatbot.append(["当前进度:", f"正在连接GROBID服务,请稍候: {grobid_url}\n如果等待时间过长,请修改config中的GROBID_URL,可修改成本地GROBID服务。"]); yield from update_ui(chatbot=chatbot, history=history) # 刷新界面
 | 
			
		||||
        article_dict = parse_pdf(fp, grobid_url)
 | 
			
		||||
        grobid_json_res = os.path.join(get_log_folder(), gen_time_str() + "grobid.json")
 | 
			
		||||
        with open(grobid_json_res, 'w+', encoding='utf8') as f:
 | 
			
		||||
            f.write(json.dumps(article_dict, indent=4, ensure_ascii=False))
 | 
			
		||||
        promote_file_to_downloadzone(grobid_json_res, chatbot=chatbot)
 | 
			
		||||
        
 | 
			
		||||
        if article_dict is None: raise RuntimeError("解析PDF失败,请检查PDF是否损坏。")
 | 
			
		||||
        prompt = "以下是一篇学术论文的基本信息:\n"
 | 
			
		||||
        # title
 | 
			
		||||
        title = article_dict.get('title', '无法获取 title'); prompt += f'title:{title}\n\n'
 | 
			
		||||
        # authors
 | 
			
		||||
        authors = article_dict.get('authors', '无法获取 authors'); prompt += f'authors:{authors}\n\n'
 | 
			
		||||
        # abstract
 | 
			
		||||
        abstract = article_dict.get('abstract', '无法获取 abstract'); prompt += f'abstract:{abstract}\n\n'
 | 
			
		||||
        # command
 | 
			
		||||
        prompt += f"请将题目和摘要翻译为{DST_LANG}。"
 | 
			
		||||
        meta = [f'# Title:\n\n', title, f'# Abstract:\n\n', abstract ]
 | 
			
		||||
 | 
			
		||||
        # 单线,获取文章meta信息
 | 
			
		||||
        paper_meta_info = yield from request_gpt_model_in_new_thread_with_ui_alive(
 | 
			
		||||
            inputs=prompt,
 | 
			
		||||
            inputs_show_user=prompt,
 | 
			
		||||
            llm_kwargs=llm_kwargs,
 | 
			
		||||
            chatbot=chatbot, history=[],
 | 
			
		||||
            sys_prompt="You are an academic paper reader。",
 | 
			
		||||
        )
 | 
			
		||||
 | 
			
		||||
        # 多线,翻译
 | 
			
		||||
        inputs_array = []
 | 
			
		||||
        inputs_show_user_array = []
 | 
			
		||||
 | 
			
		||||
        # get_token_num
 | 
			
		||||
        from request_llm.bridge_all import model_info
 | 
			
		||||
        enc = model_info[llm_kwargs['llm_model']]['tokenizer']
 | 
			
		||||
        def get_token_num(txt): return len(enc.encode(txt, disallowed_special=()))
 | 
			
		||||
        from .crazy_utils import breakdown_txt_to_satisfy_token_limit_for_pdf
 | 
			
		||||
 | 
			
		||||
        def break_down(txt):
 | 
			
		||||
            raw_token_num = get_token_num(txt)
 | 
			
		||||
            if raw_token_num <= TOKEN_LIMIT_PER_FRAGMENT:
 | 
			
		||||
                return [txt]
 | 
			
		||||
            else:
 | 
			
		||||
                # raw_token_num > TOKEN_LIMIT_PER_FRAGMENT
 | 
			
		||||
                # find a smooth token limit to achieve even seperation
 | 
			
		||||
                count = int(math.ceil(raw_token_num / TOKEN_LIMIT_PER_FRAGMENT))
 | 
			
		||||
                token_limit_smooth = raw_token_num // count + count
 | 
			
		||||
                return breakdown_txt_to_satisfy_token_limit_for_pdf(txt, get_token_fn=get_token_num, limit=token_limit_smooth)
 | 
			
		||||
 | 
			
		||||
        for section in article_dict.get('sections'):
 | 
			
		||||
            if len(section['text']) == 0: continue
 | 
			
		||||
            section_frags = break_down(section['text'])
 | 
			
		||||
            for i, fragment in enumerate(section_frags):
 | 
			
		||||
                heading = section['heading']
 | 
			
		||||
                if len(section_frags) > 1: heading += f' Part-{i+1}'
 | 
			
		||||
                inputs_array.append(
 | 
			
		||||
                    f"你需要翻译{heading}章节,内容如下: \n\n{fragment}"
 | 
			
		||||
                )
 | 
			
		||||
                inputs_show_user_array.append(
 | 
			
		||||
                    f"# {heading}\n\n{fragment}"
 | 
			
		||||
                )
 | 
			
		||||
 | 
			
		||||
        gpt_response_collection = yield from request_gpt_model_multi_threads_with_very_awesome_ui_and_high_efficiency(
 | 
			
		||||
            inputs_array=inputs_array,
 | 
			
		||||
            inputs_show_user_array=inputs_show_user_array,
 | 
			
		||||
            llm_kwargs=llm_kwargs,
 | 
			
		||||
            chatbot=chatbot,
 | 
			
		||||
            history_array=[meta for _ in inputs_array],
 | 
			
		||||
            sys_prompt_array=[
 | 
			
		||||
                "请你作为一个学术翻译,负责把学术论文准确翻译成中文。注意文章中的每一句话都要翻译。" for _ in inputs_array],
 | 
			
		||||
        )
 | 
			
		||||
        res_path = write_history_to_file(meta +  ["# Meta Translation" , paper_meta_info] + gpt_response_collection, file_basename=None, file_fullname=None)
 | 
			
		||||
        promote_file_to_downloadzone(res_path, rename_file=os.path.basename(fp)+'.md', chatbot=chatbot)
 | 
			
		||||
        generated_conclusion_files.append(res_path)
 | 
			
		||||
 | 
			
		||||
        ch = construct_html() 
 | 
			
		||||
        orig = ""
 | 
			
		||||
        trans = ""
 | 
			
		||||
        gpt_response_collection_html = copy.deepcopy(gpt_response_collection)
 | 
			
		||||
        for i,k in enumerate(gpt_response_collection_html): 
 | 
			
		||||
            if i%2==0:
 | 
			
		||||
                gpt_response_collection_html[i] = inputs_show_user_array[i//2]
 | 
			
		||||
            else:
 | 
			
		||||
                gpt_response_collection_html[i] = gpt_response_collection_html[i]
 | 
			
		||||
 | 
			
		||||
        final = ["", "", "一、论文概况",  "", "Abstract", paper_meta_info,  "二、论文翻译",  ""]
 | 
			
		||||
        final.extend(gpt_response_collection_html)
 | 
			
		||||
        for i, k in enumerate(final): 
 | 
			
		||||
            if i%2==0:
 | 
			
		||||
                orig = k
 | 
			
		||||
            if i%2==1:
 | 
			
		||||
                trans = k
 | 
			
		||||
                ch.add_row(a=orig, b=trans)
 | 
			
		||||
        create_report_file_name = f"{os.path.basename(fp)}.trans.html"
 | 
			
		||||
        html_file = ch.save_file(create_report_file_name)
 | 
			
		||||
        generated_html_files.append(html_file)
 | 
			
		||||
        promote_file_to_downloadzone(html_file, rename_file=os.path.basename(html_file), chatbot=chatbot)
 | 
			
		||||
 | 
			
		||||
        yield from translate_pdf(article_dict, llm_kwargs, chatbot, fp, generated_conclusion_files, TOKEN_LIMIT_PER_FRAGMENT, DST_LANG)
 | 
			
		||||
    chatbot.append(("给出输出文件清单", str(generated_conclusion_files + generated_html_files)))
 | 
			
		||||
    yield from update_ui(chatbot=chatbot, history=history) # 刷新界面
 | 
			
		||||
 | 
			
		||||
 | 
			
		||||
def 解析PDF(file_manifest, project_folder, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt):
 | 
			
		||||
    """
 | 
			
		||||
    此函数已经弃用
 | 
			
		||||
    """
 | 
			
		||||
    import copy
 | 
			
		||||
    TOKEN_LIMIT_PER_FRAGMENT = 1280
 | 
			
		||||
    TOKEN_LIMIT_PER_FRAGMENT = 512
 | 
			
		||||
    generated_conclusion_files = []
 | 
			
		||||
    generated_html_files = []
 | 
			
		||||
    from crazy_functions.crazy_utils import construct_html
 | 
			
		||||
 | 
			
		||||
@ -19,3 +19,8 @@
 | 
			
		||||
.wrap.svelte-xwlu1w {
 | 
			
		||||
    min-height: var(--size-32);
 | 
			
		||||
}
 | 
			
		||||
 | 
			
		||||
/* status bar height */
 | 
			
		||||
.min.svelte-1yrv54 {
 | 
			
		||||
    min-height: var(--size-12);
 | 
			
		||||
}
 | 
			
		||||
@ -216,7 +216,7 @@ def get_reduce_token_percent(text):
 | 
			
		||||
        return 0.5, '不详'
 | 
			
		||||
 | 
			
		||||
 | 
			
		||||
def write_history_to_file(history, file_basename=None, file_fullname=None):
 | 
			
		||||
def write_history_to_file(history, file_basename=None, file_fullname=None, auto_caption=True):
 | 
			
		||||
    """
 | 
			
		||||
    将对话记录history以Markdown格式写入文件中。如果没有指定文件名,则使用当前时间生成文件名。
 | 
			
		||||
    """
 | 
			
		||||
@ -235,7 +235,7 @@ def write_history_to_file(history, file_basename=None, file_fullname=None):
 | 
			
		||||
                if type(content) != str: content = str(content)
 | 
			
		||||
            except:
 | 
			
		||||
                continue
 | 
			
		||||
            if i % 2 == 0:
 | 
			
		||||
            if i % 2 == 0 and auto_caption:
 | 
			
		||||
                f.write('## ')
 | 
			
		||||
            try:
 | 
			
		||||
                f.write(content)
 | 
			
		||||
 | 
			
		||||
		Loading…
	
	
			
			x
			
			
		
	
		Reference in New Issue
	
	Block a user