fix the markdown translation functionality
This commit is contained in:
qingxu fu
parent
2a362cec84
commit
524684f8bd
@ -1,5 +1,7 @@
|
||||
from toolbox import update_ui, trimmed_format_exc, gen_time_str
|
||||
from toolbox import CatchException, report_execption, write_results_to_file
|
||||
import glob, time, os, re
|
||||
from toolbox import update_ui, trimmed_format_exc, gen_time_str, disable_auto_promotion
|
||||
from toolbox import CatchException, report_execption, write_history_to_file
|
||||
from toolbox import promote_file_to_downloadzone, get_log_folder
|
||||
fast_debug = False
|
||||
|
||||
class PaperFileGroup():
|
||||
@ -42,13 +44,13 @@ class PaperFileGroup():
|
||||
def write_result(self, language):
|
||||
manifest = []
|
||||
for path, res in zip(self.file_paths, self.file_result):
|
||||
with open(path + f'.{gen_time_str()}.{language}.md', 'w', encoding='utf8') as f:
|
||||
manifest.append(path + f'.{gen_time_str()}.{language}.md')
|
||||
dst_file = os.path.join(get_log_folder(), f'{gen_time_str()}.md')
|
||||
with open(dst_file, 'w', encoding='utf8') as f:
|
||||
manifest.append(dst_file)
|
||||
f.write(res)
|
||||
return manifest
|
||||
|
||||
def 多文件翻译(file_manifest, project_folder, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, language='en'):
|
||||
import time, os, re
|
||||
from .crazy_utils import request_gpt_model_multi_threads_with_very_awesome_ui_and_high_efficiency
|
||||
|
||||
# <-------- 读取Markdown文件，删除其中的所有注释 ---------->
|
||||
@ -102,15 +104,15 @@ def 多文件翻译(file_manifest, project_folder, llm_kwargs, plugin_kwargs, ch
|
||||
print(trimmed_format_exc())
|
||||
|
||||
# <-------- 整理结果，退出 ---------->
|
||||
create_report_file_name = time.strftime("%Y-%m-%d-%H-%M-%S", time.localtime()) + f"-chatgpt.polish.md"
|
||||
res = write_results_to_file(gpt_response_collection, file_name=create_report_file_name)
|
||||
create_report_file_name = gen_time_str() + f"-chatgpt.md"
|
||||
res = write_history_to_file(gpt_response_collection, file_basename=create_report_file_name)
|
||||
promote_file_to_downloadzone(res, chatbot=chatbot)
|
||||
history = gpt_response_collection
|
||||
chatbot.append((f"{fp}完成了吗？", res))
|
||||
yield from update_ui(chatbot=chatbot, history=history) # 刷新界面
|
||||
|
||||
|
||||
def get_files_from_everything(txt, preference=''):
|
||||
import glob, os
|
||||
success = True
|
||||
if txt.startswith('http'):
|
||||
import requests
|
||||
@ -129,9 +131,9 @@ def get_files_from_everything(txt, preference=''):
|
||||
txt = txt.replace("/blob/", "/")
|
||||
|
||||
r = requests.get(txt, proxies=proxies)
|
||||
download_local = f'./gpt_log/temp_{gen_time_str()}.md'
|
||||
download_local = f'{get_log_folder(plugin_name="批量Markdown翻译")}/raw-readme-{gen_time_str()}.md'
|
||||
project_folder = f'{get_log_folder(plugin_name="批量Markdown翻译")}'
|
||||
with open(download_local, 'wb+') as f: f.write(r.content)
|
||||
project_folder = './gpt_log/'
|
||||
file_manifest = [download_local]
|
||||
elif txt.endswith('.md'):
|
||||
# 直接给定文件
|
||||
@ -154,11 +156,11 @@ def Markdown英译中(txt, llm_kwargs, plugin_kwargs, chatbot, history, system_p
|
||||
"函数插件功能？",
|
||||
"对整个Markdown项目进行翻译。函数插件贡献者: Binary-Husky"])
|
||||
yield from update_ui(chatbot=chatbot, history=history) # 刷新界面
|
||||
disable_auto_promotion(chatbot)
|
||||
|
||||
# 尝试导入依赖，如果缺少依赖，则给出安装建议
|
||||
try:
|
||||
import tiktoken
|
||||
import glob, os
|
||||
except:
|
||||
report_execption(chatbot, history,
|
||||
a=f"解析项目: {txt}",
|
||||
@ -194,11 +196,11 @@ def Markdown中译英(txt, llm_kwargs, plugin_kwargs, chatbot, history, system_p
|
||||
"函数插件功能？",
|
||||
"对整个Markdown项目进行翻译。函数插件贡献者: Binary-Husky"])
|
||||
yield from update_ui(chatbot=chatbot, history=history) # 刷新界面
|
||||
disable_auto_promotion(chatbot)
|
||||
|
||||
# 尝试导入依赖，如果缺少依赖，则给出安装建议
|
||||
try:
|
||||
import tiktoken
|
||||
import glob, os
|
||||
except:
|
||||
report_execption(chatbot, history,
|
||||
a=f"解析项目: {txt}",
|
||||
@ -227,11 +229,11 @@ def Markdown翻译指定语言(txt, llm_kwargs, plugin_kwargs, chatbot, history,
|
||||
"函数插件功能？",
|
||||
"对整个Markdown项目进行翻译。函数插件贡献者: Binary-Husky"])
|
||||
yield from update_ui(chatbot=chatbot, history=history) # 刷新界面
|
||||
disable_auto_promotion(chatbot)
|
||||
|
||||
# 尝试导入依赖，如果缺少依赖，则给出安装建议
|
||||
try:
|
||||
import tiktoken
|
||||
import glob, os
|
||||
except:
|
||||
report_execption(chatbot, history,
|
||||
a=f"解析项目: {txt}",
|
||||
|
||||
50
toolbox.py
50
toolbox.py
@ -196,11 +196,10 @@ def write_results_to_file(history, file_name=None):
|
||||
import time
|
||||
if file_name is None:
|
||||
# file_name = time.strftime("chatGPT分析报告%Y-%m-%d-%H-%M-%S", time.localtime()) + '.md'
|
||||
file_name = 'chatGPT分析报告' + \
|
||||
time.strftime("%Y-%m-%d-%H-%M-%S", time.localtime()) + '.md'
|
||||
file_name = 'GPT-Report-' + gen_time_str() + '.md'
|
||||
os.makedirs('./gpt_log/', exist_ok=True)
|
||||
with open(f'./gpt_log/{file_name}', 'w', encoding='utf8') as f:
|
||||
f.write('# chatGPT 分析报告\n')
|
||||
f.write('# GPT-Academic Report\n')
|
||||
for i, content in enumerate(history):
|
||||
try:
|
||||
if type(content) != str: content = str(content)
|
||||
@ -219,6 +218,37 @@ def write_results_to_file(history, file_name=None):
|
||||
return res
|
||||
|
||||
|
||||
def write_history_to_file(history, file_basename=None, file_fullname=None):
|
||||
"""
|
||||
将对话记录history以Markdown格式写入文件中。如果没有指定文件名，则使用当前时间生成文件名。
|
||||
"""
|
||||
import os
|
||||
import time
|
||||
if file_fullname is None:
|
||||
if file_basename is not None:
|
||||
file_fullname = os.path.join(get_log_folder(), file_basename)
|
||||
else:
|
||||
file_fullname = os.path.join(get_log_folder(), f'GPT-Academic-{gen_time_str()}.md')
|
||||
os.makedirs(os.path.dirname(file_fullname), exist_ok=True)
|
||||
with open(file_fullname, 'w', encoding='utf8') as f:
|
||||
f.write('# GPT-Academic Report\n')
|
||||
for i, content in enumerate(history):
|
||||
try:
|
||||
if type(content) != str: content = str(content)
|
||||
except:
|
||||
continue
|
||||
if i % 2 == 0:
|
||||
f.write('## ')
|
||||
try:
|
||||
f.write(content)
|
||||
except:
|
||||
# remove everything that cannot be handled by utf8
|
||||
f.write(content.encode('utf-8', 'ignore').decode())
|
||||
f.write('\n\n')
|
||||
res = os.path.abspath(file_fullname)
|
||||
return res
|
||||
|
||||
|
||||
def regular_txt_to_markdown(text):
|
||||
"""
|
||||
将普通文本转换为Markdown格式的文本。
|
||||
@ -477,6 +507,10 @@ def promote_file_to_downloadzone(file, rename_file=None, chatbot=None):
|
||||
else: current = []
|
||||
chatbot._cookies.update({'file_to_promote': [new_path] + current})
|
||||
|
||||
def disable_auto_promotion(chatbot):
|
||||
chatbot._cookies.update({'file_to_promote': []})
|
||||
return
|
||||
|
||||
def on_file_uploaded(files, chatbot, txt, txt2, checkboxes):
|
||||
"""
|
||||
当文件被上传时的回调函数
|
||||
@ -492,7 +526,7 @@ def on_file_uploaded(files, chatbot, txt, txt2, checkboxes):
|
||||
shutil.rmtree('./private_upload/')
|
||||
except:
|
||||
pass
|
||||
time_tag = time.strftime("%Y-%m-%d-%H-%M-%S", time.localtime())
|
||||
time_tag = gen_time_str()
|
||||
os.makedirs(f'private_upload/{time_tag}', exist_ok=True)
|
||||
err_msg = ''
|
||||
for file in files:
|
||||
@ -849,8 +883,7 @@ def zip_folder(source_folder, dest_folder, zip_name):
|
||||
print(f"Zip file created at {zip_file}")
|
||||
|
||||
def zip_result(folder):
|
||||
import time
|
||||
t = time.strftime("%Y-%m-%d-%H-%M-%S", time.localtime())
|
||||
t = gen_time_str()
|
||||
zip_folder(folder, './gpt_log/', f'{t}-result.zip')
|
||||
return pj('./gpt_log/', f'{t}-result.zip')
|
||||
|
||||
@ -858,6 +891,11 @@ def gen_time_str():
|
||||
import time
|
||||
return time.strftime("%Y-%m-%d-%H-%M-%S", time.localtime())
|
||||
|
||||
def get_log_folder(user='default', plugin_name='shared'):
|
||||
_dir = os.path.join(os.path.dirname(__file__), 'gpt_log', user, plugin_name)
|
||||
if not os.path.exists(_dir): os.makedirs(_dir)
|
||||
return _dir
|
||||
|
||||
class ProxyNetworkActivate():
|
||||
"""
|
||||
这段代码定义了一个名为TempProxy的空上下文管理器, 用于给一小段代码上代理
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user