diff --git a/config.py b/config.py index 2d57f09..94a5974 100644 --- a/config.py +++ b/config.py @@ -179,7 +179,10 @@ GROBID_URLS = [ # 是否允许通过自然语言描述修改本页的配置,该功能具有一定的危险性,默认关闭 ALLOW_RESET_CONFIG = False - +# 临时的上传文件夹位置,请勿修改 +PATH_PRIVATE_UPLOAD = "private_upload" +# 日志文件夹的位置,请勿修改 +PATH_LOGGING = "gpt_log" """ 在线大模型配置关联关系示意图 diff --git a/crazy_functional.py b/crazy_functional.py index c5ccda7..4df53f5 100644 --- a/crazy_functional.py +++ b/crazy_functional.py @@ -13,7 +13,6 @@ def get_crazy_functions(): from crazy_functions.解析项目源代码 import 解析一个Java项目 from crazy_functions.解析项目源代码 import 解析一个前端项目 from crazy_functions.高级功能函数模板 import 高阶功能模板函数 - from crazy_functions.代码重写为全英文_多线程 import 全项目切换英文 from crazy_functions.Latex全文润色 import Latex英文润色 from crazy_functions.询问多个大语言模型 import 同时问询 from crazy_functions.解析项目源代码 import 解析一个Lua项目 diff --git a/crazy_functions/CodeInterpreter.py b/crazy_functions/CodeInterpreter.py index 3c970f3..283dd87 100644 --- a/crazy_functions/CodeInterpreter.py +++ b/crazy_functions/CodeInterpreter.py @@ -1,6 +1,7 @@ from collections.abc import Callable, Iterable, Mapping from typing import Any -from toolbox import CatchException, update_ui, gen_time_str, trimmed_format_exc, promote_file_to_downloadzone, clear_file_downloadzone +from toolbox import CatchException, update_ui, gen_time_str, trimmed_format_exc +from toolbox import promote_file_to_downloadzone, get_log_folder from .crazy_utils import request_gpt_model_in_new_thread_with_ui_alive from .crazy_utils import input_clipping, try_install_deps from multiprocessing import Process, Pipe @@ -92,7 +93,7 @@ def gpt_interact_multi_step(txt, file_type, llm_kwargs, chatbot, history): def make_module(code): module_file = 'gpt_fn_' + gen_time_str().replace('-','_') - with open(f'gpt_log/{module_file}.py', 'w', encoding='utf8') as f: + with open(f'{get_log_folder()}/{module_file}.py', 'w', encoding='utf8') as f: f.write(code) def get_class_name(class_string): @@ -102,7 +103,7 @@ def make_module(code): return class_name class_name = get_class_name(code) - return f"gpt_log.{module_file}->{class_name}" + return f"{get_log_folder().replace('/', '.')}.{module_file}->{class_name}" def init_module_instance(module): import importlib @@ -171,7 +172,7 @@ def 虚空终端CodeInterpreter(txt, llm_kwargs, plugin_kwargs, chatbot, history file_type = file_path.split('.')[-1] # 粗心检查 - if 'private_upload' in txt: + if is_the_upload_folder(txt): chatbot.append([ "...", f"请在输入框内填写需求,然后再次点击该插件(文件路径 {file_path} 已经被记忆)" diff --git a/crazy_functions/Latex全文润色.py b/crazy_functions/Latex全文润色.py index 9aaff6c..462f965 100644 --- a/crazy_functions/Latex全文润色.py +++ b/crazy_functions/Latex全文润色.py @@ -1,5 +1,5 @@ -from toolbox import update_ui, trimmed_format_exc -from toolbox import CatchException, report_execption, write_results_to_file, zip_folder +from toolbox import update_ui, trimmed_format_exc, promote_file_to_downloadzone, get_log_folder +from toolbox import CatchException, report_execption, write_history_to_file, zip_folder class PaperFileGroup(): @@ -51,7 +51,7 @@ class PaperFileGroup(): import os, time folder = os.path.dirname(self.file_paths[0]) t = time.strftime("%Y-%m-%d-%H-%M-%S", time.localtime()) - zip_folder(folder, './gpt_log/', f'{t}-polished.zip') + zip_folder(folder, get_log_folder(), f'{t}-polished.zip') def 多文件润色(file_manifest, project_folder, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, language='en', mode='polish'): @@ -126,7 +126,9 @@ def 多文件润色(file_manifest, project_folder, llm_kwargs, plugin_kwargs, ch # <-------- 整理结果,退出 ----------> create_report_file_name = time.strftime("%Y-%m-%d-%H-%M-%S", time.localtime()) + f"-chatgpt.polish.md" - res = write_results_to_file(gpt_response_collection, file_name=create_report_file_name) + res = write_history_to_file(gpt_response_collection, file_basename=create_report_file_name) + promote_file_to_downloadzone(res, chatbot=chatbot) + history = gpt_response_collection chatbot.append((f"{fp}完成了吗?", res)) yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 diff --git a/crazy_functions/Latex全文翻译.py b/crazy_functions/Latex全文翻译.py index 554c485..b5aad71 100644 --- a/crazy_functions/Latex全文翻译.py +++ b/crazy_functions/Latex全文翻译.py @@ -1,5 +1,5 @@ -from toolbox import update_ui -from toolbox import CatchException, report_execption, write_results_to_file +from toolbox import update_ui, promote_file_to_downloadzone +from toolbox import CatchException, report_execption, write_history_to_file fast_debug = False class PaperFileGroup(): @@ -95,7 +95,8 @@ def 多文件翻译(file_manifest, project_folder, llm_kwargs, plugin_kwargs, ch # <-------- 整理结果,退出 ----------> create_report_file_name = time.strftime("%Y-%m-%d-%H-%M-%S", time.localtime()) + f"-chatgpt.polish.md" - res = write_results_to_file(gpt_response_collection, file_name=create_report_file_name) + res = write_history_to_file(gpt_response_collection, create_report_file_name) + promote_file_to_downloadzone(res, chatbot=chatbot) history = gpt_response_collection chatbot.append((f"{fp}完成了吗?", res)) yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 diff --git a/crazy_functions/Latex输出PDF结果.py b/crazy_functions/Latex输出PDF结果.py index 8686f7e..56ca1c6 100644 --- a/crazy_functions/Latex输出PDF结果.py +++ b/crazy_functions/Latex输出PDF结果.py @@ -1,4 +1,4 @@ -from toolbox import update_ui, trimmed_format_exc, get_conf, objdump, objload, promote_file_to_downloadzone +from toolbox import update_ui, trimmed_format_exc, get_conf, get_log_folder, promote_file_to_downloadzone from toolbox import CatchException, report_execption, update_ui_lastest_msg, zip_result, gen_time_str from functools import partial import glob, os, requests, time @@ -65,7 +65,7 @@ def move_project(project_folder, arxiv_id=None): if arxiv_id is not None: new_workfolder = pj(ARXIV_CACHE_DIR, arxiv_id, 'workfolder') else: - new_workfolder = f'gpt_log/{gen_time_str()}' + new_workfolder = f'{get_log_folder()}/{gen_time_str()}' try: shutil.rmtree(new_workfolder) except: diff --git a/crazy_functions/latex_fns/latex_actions.py b/crazy_functions/latex_fns/latex_actions.py index 04efc94..dcde0e9 100644 --- a/crazy_functions/latex_fns/latex_actions.py +++ b/crazy_functions/latex_fns/latex_actions.py @@ -1,4 +1,4 @@ -from toolbox import update_ui, update_ui_lastest_msg # 刷新Gradio前端界面 +from toolbox import update_ui, update_ui_lastest_msg, get_log_folder from toolbox import zip_folder, objdump, objload, promote_file_to_downloadzone from .latex_toolbox import PRESERVE, TRANSFORM from .latex_toolbox import set_forbidden_text, set_forbidden_text_begin_end, set_forbidden_text_careful_brace @@ -439,9 +439,9 @@ def write_html(sp_file_contents, sp_file_result, chatbot, project_folder): trans = k ch.add_row(a=orig, b=trans) create_report_file_name = f"{gen_time_str()}.trans.html" - ch.save_file(create_report_file_name) - shutil.copyfile(pj('./gpt_log/', create_report_file_name), pj(project_folder, create_report_file_name)) - promote_file_to_downloadzone(file=f'./gpt_log/{create_report_file_name}', chatbot=chatbot) + res = ch.save_file(create_report_file_name) + shutil.copyfile(res, pj(project_folder, create_report_file_name)) + promote_file_to_downloadzone(file=res, chatbot=chatbot) except: from toolbox import trimmed_format_exc print('writing html result failed:', trimmed_format_exc()) diff --git a/crazy_functions/下载arxiv论文翻译摘要.py b/crazy_functions/下载arxiv论文翻译摘要.py index 46275bf..8b4a503 100644 --- a/crazy_functions/下载arxiv论文翻译摘要.py +++ b/crazy_functions/下载arxiv论文翻译摘要.py @@ -1,5 +1,6 @@ -from toolbox import update_ui -from toolbox import CatchException, report_execption, write_results_to_file, get_conf +from toolbox import update_ui, get_log_folder +from toolbox import write_history_to_file, promote_file_to_downloadzone +from toolbox import CatchException, report_execption, get_conf import re, requests, unicodedata, os from .crazy_utils import request_gpt_model_in_new_thread_with_ui_alive def download_arxiv_(url_pdf): @@ -28,7 +29,7 @@ def download_arxiv_(url_pdf): if k in other_info['comment']: title = k + ' ' + title - download_dir = './gpt_log/arxiv/' + download_dir = get_log_folder(plugin_name='arxiv') os.makedirs(download_dir, exist_ok=True) title_str = title.replace('?', '?')\ @@ -40,9 +41,6 @@ def download_arxiv_(url_pdf): requests_pdf_url = url_pdf file_path = download_dir+title_str - # if os.path.exists(file_path): - # print('返回缓存文件') - # return './gpt_log/arxiv/'+title_str print('下载中') proxies, = get_conf('proxies') @@ -61,7 +59,7 @@ def download_arxiv_(url_pdf): .replace('\n', '')\ .replace(' ', ' ')\ .replace(' ', ' ') - return './gpt_log/arxiv/'+title_str, other_info + return file_path, other_info def get_name(_url_): @@ -184,11 +182,10 @@ def 下载arxiv论文并翻译摘要(txt, llm_kwargs, plugin_kwargs, chatbot, hi chatbot[-1] = (i_say_show_user, gpt_say) history.append(i_say_show_user); history.append(gpt_say) yield from update_ui(chatbot=chatbot, history=history, msg=msg) # 刷新界面 - # 写入文件 - import shutil - # 重置文件的创建时间 - shutil.copyfile(pdf_path, f'./gpt_log/{os.path.basename(pdf_path)}'); os.remove(pdf_path) - res = write_results_to_file(history) + res = write_history_to_file(history) + promote_file_to_downloadzone(res, chatbot=chatbot) + promote_file_to_downloadzone(pdf_path, chatbot=chatbot) + chatbot.append(("完成了吗?", res + "\n\nPDF文件也已经下载")) yield from update_ui(chatbot=chatbot, history=history, msg=msg) # 刷新界面 diff --git a/crazy_functions/代码重写为全英文_多线程.py b/crazy_functions/代码重写为全英文_多线程.py deleted file mode 100644 index e57f80f..0000000 --- a/crazy_functions/代码重写为全英文_多线程.py +++ /dev/null @@ -1,138 +0,0 @@ -import threading -from request_llm.bridge_all import predict_no_ui_long_connection -from toolbox import update_ui -from toolbox import CatchException, write_results_to_file, report_execption -from .crazy_utils import breakdown_txt_to_satisfy_token_limit - -def extract_code_block_carefully(txt): - splitted = txt.split('```') - n_code_block_seg = len(splitted) - 1 - if n_code_block_seg <= 1: return txt - # 剩下的情况都开头除去 ``` 结尾除去一次 ``` - txt_out = '```'.join(splitted[1:-1]) - return txt_out - - - -def break_txt_into_half_at_some_linebreak(txt): - lines = txt.split('\n') - n_lines = len(lines) - pre = lines[:(n_lines//2)] - post = lines[(n_lines//2):] - return "\n".join(pre), "\n".join(post) - - -@CatchException -def 全项目切换英文(txt, llm_kwargs, plugin_kwargs, chatbot, history, sys_prompt, web_port): - # 第1步:清空历史,以免输入溢出 - history = [] - - # 第2步:尝试导入依赖,如果缺少依赖,则给出安装建议 - try: - import tiktoken - except: - report_execption(chatbot, history, - a = f"解析项目: {txt}", - b = f"导入软件依赖失败。使用该模块需要额外依赖,安装方法```pip install --upgrade tiktoken```。") - yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 - return - - # 第3步:集合文件 - import time, glob, os, shutil, re - os.makedirs('gpt_log/generated_english_version', exist_ok=True) - os.makedirs('gpt_log/generated_english_version/crazy_functions', exist_ok=True) - file_manifest = [f for f in glob.glob('./*.py') if ('test_project' not in f) and ('gpt_log' not in f)] + \ - [f for f in glob.glob('./crazy_functions/*.py') if ('test_project' not in f) and ('gpt_log' not in f)] - # file_manifest = ['./toolbox.py'] - i_say_show_user_buffer = [] - - # 第4步:随便显示点什么防止卡顿的感觉 - for index, fp in enumerate(file_manifest): - # if 'test_project' in fp: continue - with open(fp, 'r', encoding='utf-8', errors='replace') as f: - file_content = f.read() - i_say_show_user =f'[{index}/{len(file_manifest)}] 接下来请将以下代码中包含的所有中文转化为英文,只输出转化后的英文代码,请用代码块输出代码: {os.path.abspath(fp)}' - i_say_show_user_buffer.append(i_say_show_user) - chatbot.append((i_say_show_user, "[Local Message] 等待多线程操作,中间过程不予显示.")) - yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 - - - # 第5步:Token限制下的截断与处理 - MAX_TOKEN = 3000 - from request_llm.bridge_all import model_info - enc = model_info["gpt-3.5-turbo"]['tokenizer'] - def get_token_fn(txt): return len(enc.encode(txt, disallowed_special=())) - - - # 第6步:任务函数 - mutable_return = [None for _ in file_manifest] - observe_window = [[""] for _ in file_manifest] - def thread_worker(fp,index): - if index > 10: - time.sleep(60) - print('Openai 限制免费用户每分钟20次请求,降低请求频率中。') - with open(fp, 'r', encoding='utf-8', errors='replace') as f: - file_content = f.read() - i_say_template = lambda fp, file_content: f'接下来请将以下代码中包含的所有中文转化为英文,只输出代码,文件名是{fp},文件代码是 ```{file_content}```' - try: - gpt_say = "" - # 分解代码文件 - file_content_breakdown = breakdown_txt_to_satisfy_token_limit(file_content, get_token_fn, MAX_TOKEN) - for file_content_partial in file_content_breakdown: - i_say = i_say_template(fp, file_content_partial) - # # ** gpt request ** - gpt_say_partial = predict_no_ui_long_connection(inputs=i_say, llm_kwargs=llm_kwargs, history=[], sys_prompt=sys_prompt, observe_window=observe_window[index]) - gpt_say_partial = extract_code_block_carefully(gpt_say_partial) - gpt_say += gpt_say_partial - mutable_return[index] = gpt_say - except ConnectionAbortedError as token_exceed_err: - print('至少一个线程任务Token溢出而失败', e) - except Exception as e: - print('至少一个线程任务意外失败', e) - - # 第7步:所有线程同时开始执行任务函数 - handles = [threading.Thread(target=thread_worker, args=(fp,index)) for index, fp in enumerate(file_manifest)] - for h in handles: - h.daemon = True - h.start() - chatbot.append(('开始了吗?', f'多线程操作已经开始')) - yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 - - # 第8步:循环轮询各个线程是否执行完毕 - cnt = 0 - while True: - cnt += 1 - time.sleep(0.2) - th_alive = [h.is_alive() for h in handles] - if not any(th_alive): break - # 更好的UI视觉效果 - observe_win = [] - for thread_index, alive in enumerate(th_alive): - observe_win.append("[ ..."+observe_window[thread_index][0][-60:].replace('\n','').replace('```','...').replace(' ','.').replace('
','.....').replace('$','.')+"... ]") - stat = [f'执行中: {obs}\n\n' if alive else '已完成\n\n' for alive, obs in zip(th_alive, observe_win)] - stat_str = ''.join(stat) - chatbot[-1] = (chatbot[-1][0], f'多线程操作已经开始,完成情况: \n\n{stat_str}' + ''.join(['.']*(cnt%10+1))) - yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 - - # 第9步:把结果写入文件 - for index, h in enumerate(handles): - h.join() # 这里其实不需要join了,肯定已经都结束了 - fp = file_manifest[index] - gpt_say = mutable_return[index] - i_say_show_user = i_say_show_user_buffer[index] - - where_to_relocate = f'gpt_log/generated_english_version/{fp}' - if gpt_say is not None: - with open(where_to_relocate, 'w+', encoding='utf-8') as f: - f.write(gpt_say) - else: # 失败 - shutil.copyfile(file_manifest[index], where_to_relocate) - chatbot.append((i_say_show_user, f'[Local Message] 已完成{os.path.abspath(fp)}的转化,\n\n存入{os.path.abspath(where_to_relocate)}')) - history.append(i_say_show_user); history.append(gpt_say) - yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 - time.sleep(1) - - # 第10步:备份一个文件 - res = write_results_to_file(history) - chatbot.append(("生成一份任务执行报告", res)) - yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 diff --git a/crazy_functions/图片生成.py b/crazy_functions/图片生成.py index a1cc244..51a1baf 100644 --- a/crazy_functions/图片生成.py +++ b/crazy_functions/图片生成.py @@ -1,4 +1,4 @@ -from toolbox import CatchException, update_ui, get_conf, select_api_key +from toolbox import CatchException, update_ui, get_conf, select_api_key, get_log_folder from .crazy_utils import request_gpt_model_in_new_thread_with_ui_alive import datetime @@ -33,7 +33,7 @@ def gen_image(llm_kwargs, prompt, resolution="256x256"): raise RuntimeError(response.content.decode()) # 文件保存到本地 r = requests.get(image_url, proxies=proxies) - file_path = 'gpt_log/image_gen/' + file_path = f'{get_log_folder()}/image_gen/' os.makedirs(file_path, exist_ok=True) file_name = 'Image' + time.strftime("%Y-%m-%d-%H-%M-%S", time.localtime()) + '.png' with open(file_path+file_name, 'wb+') as f: f.write(r.content) diff --git a/crazy_functions/对话历史存档.py b/crazy_functions/对话历史存档.py index a0d0ac8..99bce3d 100644 --- a/crazy_functions/对话历史存档.py +++ b/crazy_functions/对话历史存档.py @@ -1,4 +1,4 @@ -from toolbox import CatchException, update_ui, promote_file_to_downloadzone +from toolbox import CatchException, update_ui, promote_file_to_downloadzone, get_log_folder from .crazy_utils import request_gpt_model_in_new_thread_with_ui_alive import re @@ -10,8 +10,8 @@ def write_chat_to_file(chatbot, history=None, file_name=None): import time if file_name is None: file_name = 'chatGPT对话历史' + time.strftime("%Y-%m-%d-%H-%M-%S", time.localtime()) + '.html' - os.makedirs('./gpt_log/', exist_ok=True) - with open(f'./gpt_log/{file_name}', 'w', encoding='utf8') as f: + fp = os.path.join(get_log_folder(), file_name) + with open(fp, 'w', encoding='utf8') as f: from themes.theme import advanced_css f.write(f'对话历史') for i, contents in enumerate(chatbot): @@ -29,8 +29,8 @@ def write_chat_to_file(chatbot, history=None, file_name=None): for h in history: f.write("\n>>>" + h) f.write('') - promote_file_to_downloadzone(f'./gpt_log/{file_name}', rename_file=file_name, chatbot=chatbot) - return '对话历史写入:' + os.path.abspath(f'./gpt_log/{file_name}') + promote_file_to_downloadzone(fp, rename_file=file_name, chatbot=chatbot) + return '对话历史写入:' + os.path.abspath(fp) def gen_file_preview(file_name): try: @@ -106,7 +106,7 @@ def 载入对话历史存档(txt, llm_kwargs, plugin_kwargs, chatbot, history, s if not success: if txt == "": txt = '空空如也的输入栏' import glob - local_history = "
".join(["`"+hide_cwd(f)+f" ({gen_file_preview(f)})"+"`" for f in glob.glob(f'gpt_log/**/chatGPT对话历史*.html', recursive=True)]) + local_history = "
".join(["`"+hide_cwd(f)+f" ({gen_file_preview(f)})"+"`" for f in glob.glob(f'{get_log_folder()}/**/chatGPT对话历史*.html', recursive=True)]) chatbot.append([f"正在查找对话历史文件(html格式): {txt}", f"找不到任何html文件: {txt}。但本地存储了以下历史文件,您可以将任意一个文件路径粘贴到输入区,然后重试:
{local_history}"]) yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 return @@ -132,8 +132,8 @@ def 删除所有本地对话历史记录(txt, llm_kwargs, plugin_kwargs, chatbot """ import glob, os - local_history = "
".join(["`"+hide_cwd(f)+"`" for f in glob.glob(f'gpt_log/**/chatGPT对话历史*.html', recursive=True)]) - for f in glob.glob(f'gpt_log/**/chatGPT对话历史*.html', recursive=True): + local_history = "
".join(["`"+hide_cwd(f)+"`" for f in glob.glob(f'{get_log_folder()}/**/chatGPT对话历史*.html', recursive=True)]) + for f in glob.glob(f'{get_log_folder()}/**/chatGPT对话历史*.html', recursive=True): os.remove(f) chatbot.append([f"删除所有历史对话文件", f"已删除
{local_history}"]) yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 diff --git a/crazy_functions/总结word文档.py b/crazy_functions/总结word文档.py index 5af6960..4ea753c 100644 --- a/crazy_functions/总结word文档.py +++ b/crazy_functions/总结word文档.py @@ -1,5 +1,6 @@ from toolbox import update_ui -from toolbox import CatchException, report_execption, write_results_to_file +from toolbox import CatchException, report_execption +from toolbox import write_history_to_file, promote_file_to_downloadzone from .crazy_utils import request_gpt_model_in_new_thread_with_ui_alive fast_debug = False @@ -71,11 +72,13 @@ def 解析docx(file_manifest, project_folder, llm_kwargs, plugin_kwargs, chatbot history.extend([i_say,gpt_say]) this_paper_history.extend([i_say,gpt_say]) - res = write_results_to_file(history) + res = write_history_to_file(history) + promote_file_to_downloadzone(res, chatbot=chatbot) chatbot.append(("完成了吗?", res)) yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 - res = write_results_to_file(history) + res = write_history_to_file(history) + promote_file_to_downloadzone(res, chatbot=chatbot) chatbot.append(("所有文件都总结完成了吗?", res)) yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 diff --git a/crazy_functions/总结音视频.py b/crazy_functions/总结音视频.py index 62f05d3..7c113f4 100644 --- a/crazy_functions/总结音视频.py +++ b/crazy_functions/总结音视频.py @@ -1,5 +1,6 @@ -from toolbox import CatchException, report_execption, select_api_key, update_ui, write_results_to_file, get_conf +from toolbox import CatchException, report_execption, select_api_key, update_ui, get_conf from .crazy_utils import request_gpt_model_in_new_thread_with_ui_alive +from toolbox import write_history_to_file, promote_file_to_downloadzone, get_log_folder def split_audio_file(filename, split_duration=1000): """ @@ -15,7 +16,7 @@ def split_audio_file(filename, split_duration=1000): """ from moviepy.editor import AudioFileClip import os - os.makedirs('gpt_log/mp3/cut/', exist_ok=True) # 创建存储切割音频的文件夹 + os.makedirs(f"{get_log_folder(plugin_name='audio')}/mp3/cut/", exist_ok=True) # 创建存储切割音频的文件夹 # 读取音频文件 audio = AudioFileClip(filename) @@ -31,8 +32,8 @@ def split_audio_file(filename, split_duration=1000): start_time = split_points[i] end_time = split_points[i + 1] split_audio = audio.subclip(start_time, end_time) - split_audio.write_audiofile(f"gpt_log/mp3/cut/{filename[0]}_{i}.mp3") - filelist.append(f"gpt_log/mp3/cut/{filename[0]}_{i}.mp3") + split_audio.write_audiofile(f"{get_log_folder(plugin_name='audio')}/mp3/cut/{filename[0]}_{i}.mp3") + filelist.append(f"{get_log_folder(plugin_name='audio')}/mp3/cut/{filename[0]}_{i}.mp3") audio.close() return filelist @@ -52,7 +53,7 @@ def AnalyAudio(parse_prompt, file_manifest, llm_kwargs, chatbot, history): 'Authorization': f"Bearer {api_key}" } - os.makedirs('gpt_log/mp3/', exist_ok=True) + os.makedirs(f"{get_log_folder(plugin_name='audio')}/mp3/", exist_ok=True) for index, fp in enumerate(file_manifest): audio_history = [] # 提取文件扩展名 @@ -60,8 +61,8 @@ def AnalyAudio(parse_prompt, file_manifest, llm_kwargs, chatbot, history): # 提取视频中的音频 if ext not in [".mp3", ".wav", ".m4a", ".mpga"]: audio_clip = AudioFileClip(fp) - audio_clip.write_audiofile(f'gpt_log/mp3/output{index}.mp3') - fp = f'gpt_log/mp3/output{index}.mp3' + audio_clip.write_audiofile(f"{get_log_folder(plugin_name='audio')}/mp3/output{index}.mp3") + fp = f"{get_log_folder(plugin_name='audio')}/mp3/output{index}.mp3" # 调用whisper模型音频转文字 voice = split_audio_file(fp) for j, i in enumerate(voice): @@ -113,18 +114,19 @@ def AnalyAudio(parse_prompt, file_manifest, llm_kwargs, chatbot, history): history=audio_history, sys_prompt="总结文章。" ) - history.extend([i_say, gpt_say]) audio_history.extend([i_say, gpt_say]) - res = write_results_to_file(history) + res = write_history_to_file(history) + promote_file_to_downloadzone(res, chatbot=chatbot) chatbot.append((f"第{index + 1}段音频完成了吗?", res)) yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 # 删除中间文件夹 import shutil - shutil.rmtree('gpt_log/mp3') - res = write_results_to_file(history) + shutil.rmtree(f"{get_log_folder(plugin_name='audio')}/mp3") + res = write_history_to_file(history) + promote_file_to_downloadzone(res, chatbot=chatbot) chatbot.append(("所有音频都总结完成了吗?", res)) yield from update_ui(chatbot=chatbot, history=history) diff --git a/crazy_functions/批量Markdown翻译.py b/crazy_functions/批量Markdown翻译.py index 8af9864..277d942 100644 --- a/crazy_functions/批量Markdown翻译.py +++ b/crazy_functions/批量Markdown翻译.py @@ -1,7 +1,7 @@ import glob, time, os, re from toolbox import update_ui, trimmed_format_exc, gen_time_str, disable_auto_promotion -from toolbox import CatchException, report_execption, write_history_to_file -from toolbox import promote_file_to_downloadzone, get_log_folder +from toolbox import CatchException, report_execption, get_log_folder +from toolbox import write_history_to_file, promote_file_to_downloadzone fast_debug = False class PaperFileGroup(): diff --git a/crazy_functions/批量总结PDF文档.py b/crazy_functions/批量总结PDF文档.py index fc65f5c..b87d482 100644 --- a/crazy_functions/批量总结PDF文档.py +++ b/crazy_functions/批量总结PDF文档.py @@ -1,5 +1,6 @@ from toolbox import update_ui, promote_file_to_downloadzone, gen_time_str -from toolbox import CatchException, report_execption, write_results_to_file +from toolbox import CatchException, report_execption +from toolbox import write_history_to_file, promote_file_to_downloadzone from .crazy_utils import request_gpt_model_in_new_thread_with_ui_alive from .crazy_utils import read_and_clean_pdf_text from .crazy_utils import input_clipping @@ -99,8 +100,8 @@ do not have too much repetitive information, numerical values using the original _, final_results = input_clipping("", final_results, max_token_limit=3200) yield from update_ui(chatbot=chatbot, history=final_results) # 注意这里的历史记录被替代了 - res = write_results_to_file(file_write_buffer, file_name=gen_time_str()) - promote_file_to_downloadzone(res.split('\t')[-1], chatbot=chatbot) + res = write_history_to_file(file_write_buffer) + promote_file_to_downloadzone(res, chatbot=chatbot) yield from update_ui(chatbot=chatbot, history=final_results) # 刷新界面 diff --git a/crazy_functions/批量总结PDF文档pdfminer.py b/crazy_functions/批量总结PDF文档pdfminer.py index ffbb055..213d8bb 100644 --- a/crazy_functions/批量总结PDF文档pdfminer.py +++ b/crazy_functions/批量总结PDF文档pdfminer.py @@ -1,6 +1,7 @@ from toolbox import update_ui -from toolbox import CatchException, report_execption, write_results_to_file +from toolbox import CatchException, report_execption from .crazy_utils import request_gpt_model_in_new_thread_with_ui_alive +from toolbox import write_history_to_file, promote_file_to_downloadzone fast_debug = False @@ -115,7 +116,8 @@ def 解析Paper(file_manifest, project_folder, llm_kwargs, plugin_kwargs, chatbo chatbot[-1] = (i_say, gpt_say) history.append(i_say); history.append(gpt_say) yield from update_ui(chatbot=chatbot, history=history, msg=msg) # 刷新界面 - res = write_results_to_file(history) + res = write_history_to_file(history) + promote_file_to_downloadzone(res, chatbot=chatbot) chatbot.append(("完成了吗?", res)) yield from update_ui(chatbot=chatbot, history=history, msg=msg) # 刷新界面 diff --git a/crazy_functions/批量翻译PDF文档_多线程.py b/crazy_functions/批量翻译PDF文档_多线程.py index 440004e..d620715 100644 --- a/crazy_functions/批量翻译PDF文档_多线程.py +++ b/crazy_functions/批量翻译PDF文档_多线程.py @@ -1,6 +1,6 @@ -from toolbox import CatchException, report_execption, write_results_to_file +from toolbox import CatchException, report_execption, get_log_folder from toolbox import update_ui, promote_file_to_downloadzone, update_ui_lastest_msg, disable_auto_promotion -from toolbox import write_history_to_file, get_log_folder +from toolbox import write_history_to_file, promote_file_to_downloadzone from .crazy_utils import request_gpt_model_in_new_thread_with_ui_alive from .crazy_utils import request_gpt_model_multi_threads_with_very_awesome_ui_and_high_efficiency from .crazy_utils import read_and_clean_pdf_text @@ -218,10 +218,11 @@ def 解析PDF(file_manifest, project_folder, llm_kwargs, plugin_kwargs, chatbot, final = ["一、论文概况\n\n---\n\n", paper_meta_info.replace('# ', '### ') + '\n\n---\n\n', "二、论文翻译", ""] final.extend(gpt_response_collection_md) create_report_file_name = f"{os.path.basename(fp)}.trans.md" - res = write_results_to_file(final, file_name=create_report_file_name) + res = write_history_to_file(final, create_report_file_name) + promote_file_to_downloadzone(res, chatbot=chatbot) # 更新UI - generated_conclusion_files.append(f'./gpt_log/{create_report_file_name}') + generated_conclusion_files.append(f'{get_log_folder()}/{create_report_file_name}') chatbot.append((f"{fp}完成了吗?", res)) yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 diff --git a/crazy_functions/生成函数注释.py b/crazy_functions/生成函数注释.py index a564f21..bf3da6a 100644 --- a/crazy_functions/生成函数注释.py +++ b/crazy_functions/生成函数注释.py @@ -1,5 +1,6 @@ from toolbox import update_ui -from toolbox import CatchException, report_execption, write_results_to_file +from toolbox import CatchException, report_execption +from toolbox import write_history_to_file, promote_file_to_downloadzone from .crazy_utils import request_gpt_model_in_new_thread_with_ui_alive fast_debug = False @@ -27,7 +28,8 @@ def 生成函数注释(file_manifest, project_folder, llm_kwargs, plugin_kwargs, if not fast_debug: time.sleep(2) if not fast_debug: - res = write_results_to_file(history) + res = write_history_to_file(history) + promote_file_to_downloadzone(res, chatbot=chatbot) chatbot.append(("完成了吗?", res)) yield from update_ui(chatbot=chatbot, history=history, msg=msg) # 刷新界面 diff --git a/crazy_functions/虚空终端.py b/crazy_functions/虚空终端.py index e01b274..5f33249 100644 --- a/crazy_functions/虚空终端.py +++ b/crazy_functions/虚空终端.py @@ -46,7 +46,7 @@ explain_msg = """ from pydantic import BaseModel, Field from typing import List -from toolbox import CatchException, update_ui, gen_time_str +from toolbox import CatchException, update_ui, is_the_upload_folder from toolbox import update_ui_lastest_msg, disable_auto_promotion from request_llm.bridge_all import predict_no_ui_long_connection from crazy_functions.crazy_utils import request_gpt_model_in_new_thread_with_ui_alive @@ -112,7 +112,7 @@ def 虚空终端(txt, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt # 用简单的关键词检测用户意图 is_certain, _ = analyze_intention_with_simple_rules(txt) - if txt.startswith('private_upload/') and len(txt) == 34: + if is_the_upload_folder(txt): state.set_state(chatbot=chatbot, key='has_provided_explaination', value=False) appendix_msg = "\n\n**很好,您已经上传了文件**,现在请您描述您的需求。" diff --git a/crazy_functions/解析JupyterNotebook.py b/crazy_functions/解析JupyterNotebook.py index b4bcd56..d4a3b49 100644 --- a/crazy_functions/解析JupyterNotebook.py +++ b/crazy_functions/解析JupyterNotebook.py @@ -1,5 +1,6 @@ from toolbox import update_ui -from toolbox import CatchException, report_execption, write_results_to_file +from toolbox import CatchException, report_execption +from toolbox import write_history_to_file, promote_file_to_downloadzone fast_debug = True @@ -110,7 +111,8 @@ def ipynb解释(file_manifest, project_folder, llm_kwargs, plugin_kwargs, chatbo yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 # <-------- 写入文件,退出 ----------> - res = write_results_to_file(history) + res = write_history_to_file(history) + promote_file_to_downloadzone(res, chatbot=chatbot) chatbot.append(("完成了吗?", res)) yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 diff --git a/crazy_functions/解析项目源代码.py b/crazy_functions/解析项目源代码.py index debcb51..ab63bae 100644 --- a/crazy_functions/解析项目源代码.py +++ b/crazy_functions/解析项目源代码.py @@ -109,9 +109,9 @@ def 解析源代码新(file_manifest, project_folder, llm_kwargs, plugin_kwargs, def 解析项目本身(txt, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, web_port): history = [] # 清空历史,以免输入溢出 import glob - file_manifest = [f for f in glob.glob('./*.py') if ('test_project' not in f) and ('gpt_log' not in f)] + \ - [f for f in glob.glob('./crazy_functions/*.py') if ('test_project' not in f) and ('gpt_log' not in f)]+ \ - [f for f in glob.glob('./request_llm/*.py') if ('test_project' not in f) and ('gpt_log' not in f)] + file_manifest = [f for f in glob.glob('./*.py')] + \ + [f for f in glob.glob('./crazy_functions/**/*.py')]+ \ + [f for f in glob.glob('./request_llm/**/*.py')] project_folder = './' if len(file_manifest) == 0: report_execption(chatbot, history, a = f"解析项目: {txt}", b = f"找不到任何python文件: {txt}") diff --git a/crazy_functions/读文章写摘要.py b/crazy_functions/读文章写摘要.py index 72ffe6b..acdf632 100644 --- a/crazy_functions/读文章写摘要.py +++ b/crazy_functions/读文章写摘要.py @@ -1,7 +1,7 @@ from toolbox import update_ui -from toolbox import CatchException, report_execption, write_results_to_file +from toolbox import CatchException, report_execption +from toolbox import write_history_to_file, promote_file_to_downloadzone from .crazy_utils import request_gpt_model_in_new_thread_with_ui_alive -fast_debug = False def 解析Paper(file_manifest, project_folder, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt): @@ -17,32 +17,29 @@ def 解析Paper(file_manifest, project_folder, llm_kwargs, plugin_kwargs, chatbo chatbot.append((i_say_show_user, "[Local Message] waiting gpt response.")) yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 - if not fast_debug: - msg = '正常' - # ** gpt request ** - gpt_say = yield from request_gpt_model_in_new_thread_with_ui_alive(i_say, i_say_show_user, llm_kwargs, chatbot, history=[], sys_prompt=system_prompt) # 带超时倒计时 - - chatbot[-1] = (i_say_show_user, gpt_say) - history.append(i_say_show_user); history.append(gpt_say) - yield from update_ui(chatbot=chatbot, history=history, msg=msg) # 刷新界面 - if not fast_debug: time.sleep(2) + msg = '正常' + gpt_say = yield from request_gpt_model_in_new_thread_with_ui_alive(i_say, i_say_show_user, llm_kwargs, chatbot, history=[], sys_prompt=system_prompt) # 带超时倒计时 + chatbot[-1] = (i_say_show_user, gpt_say) + history.append(i_say_show_user); history.append(gpt_say) + yield from update_ui(chatbot=chatbot, history=history, msg=msg) # 刷新界面 + time.sleep(2) all_file = ', '.join([os.path.relpath(fp, project_folder) for index, fp in enumerate(file_manifest)]) i_say = f'根据以上你自己的分析,对全文进行概括,用学术性语言写一段中文摘要,然后再写一段英文摘要(包括{all_file})。' chatbot.append((i_say, "[Local Message] waiting gpt response.")) yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 - if not fast_debug: - msg = '正常' - # ** gpt request ** - gpt_say = yield from request_gpt_model_in_new_thread_with_ui_alive(i_say, i_say, llm_kwargs, chatbot, history=history, sys_prompt=system_prompt) # 带超时倒计时 + msg = '正常' + # ** gpt request ** + gpt_say = yield from request_gpt_model_in_new_thread_with_ui_alive(i_say, i_say, llm_kwargs, chatbot, history=history, sys_prompt=system_prompt) # 带超时倒计时 - chatbot[-1] = (i_say, gpt_say) - history.append(i_say); history.append(gpt_say) - yield from update_ui(chatbot=chatbot, history=history, msg=msg) # 刷新界面 - res = write_results_to_file(history) - chatbot.append(("完成了吗?", res)) - yield from update_ui(chatbot=chatbot, history=history, msg=msg) # 刷新界面 + chatbot[-1] = (i_say, gpt_say) + history.append(i_say); history.append(gpt_say) + yield from update_ui(chatbot=chatbot, history=history, msg=msg) # 刷新界面 + res = write_history_to_file(history) + promote_file_to_downloadzone(res, chatbot=chatbot) + chatbot.append(("完成了吗?", res)) + yield from update_ui(chatbot=chatbot, history=history, msg=msg) # 刷新界面 diff --git a/crazy_functions/辅助功能.py b/crazy_functions/辅助功能.py index e56deaa..16854e0 100644 --- a/crazy_functions/辅助功能.py +++ b/crazy_functions/辅助功能.py @@ -2,8 +2,8 @@ # @Time : 2023/4/19 # @Author : Spike # @Descr : -from toolbox import update_ui -from toolbox import CatchException, report_execption, write_results_to_file, get_log_folder +from toolbox import update_ui, get_conf +from toolbox import CatchException from crazy_functions.crazy_utils import request_gpt_model_in_new_thread_with_ui_alive @@ -30,14 +30,13 @@ def 猜你想问(txt, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt @CatchException def 清除缓存(txt, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, web_port): - chatbot.append(['清除本地缓存数据', '执行中. 删除 gpt_log & private_upload']) + chatbot.append(['清除本地缓存数据', '执行中. 删除数据']) yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 import shutil, os - gpt_log_dir = os.path.join(os.path.dirname(__file__), '..', 'gpt_log') - private_upload_dir = os.path.join(os.path.dirname(__file__), '..', 'private_upload') - shutil.rmtree(gpt_log_dir, ignore_errors=True) - shutil.rmtree(private_upload_dir, ignore_errors=True) + PATH_PRIVATE_UPLOAD, PATH_LOGGING = get_conf('PATH_PRIVATE_UPLOAD', 'PATH_LOGGING') + shutil.rmtree(PATH_LOGGING, ignore_errors=True) + shutil.rmtree(PATH_PRIVATE_UPLOAD, ignore_errors=True) chatbot.append(['清除本地缓存数据', '执行完成']) yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 \ No newline at end of file diff --git a/docs/translate_english.json b/docs/translate_english.json index a29c366..91ffaa8 100644 --- a/docs/translate_english.json +++ b/docs/translate_english.json @@ -299,7 +299,6 @@ "地址🚀": "Address 🚀", "感谢热情的": "Thanks to the enthusiastic", "开发者们❤️": "Developers ❤️", - "所有问询记录将自动保存在本地目录./gpt_log/chat_secrets.log": "All inquiry records will be automatically saved in the local directory ./gpt_log/chat_secrets.log", "请注意自我隐私保护哦!": "Please pay attention to self-privacy protection!", "当前模型": "Current model", "输入区": "Input area", @@ -892,7 +891,6 @@ "保存当前对话": "Save current conversation", "您可以调用“LoadConversationHistoryArchive”还原当下的对话": "You can call 'LoadConversationHistoryArchive' to restore the current conversation", "警告!被保存的对话历史可以被使用该系统的任何人查阅": "Warning! The saved conversation history can be viewed by anyone using this system", - "gpt_log/**/chatGPT对话历史*.html": "gpt_log/**/chatGPT conversation history *.html", "正在查找对话历史文件": "Looking for conversation history file", "html格式": "HTML format", "找不到任何html文件": "No HTML files found", @@ -908,7 +906,6 @@ "pip install pywin32 用于doc格式": "pip install pywin32 for doc format", "仅支持Win平台": "Only supports Win platform", "打开文件": "Open file", - "private_upload里面的文件名在解压zip后容易出现乱码": "The file name in private_upload is prone to garbled characters after unzipping", "rar和7z格式正常": "RAR and 7z formats are normal", "故可以只分析文章内容": "So you can only analyze the content of the article", "不输入文件名": "Do not enter the file name", @@ -1364,7 +1361,6 @@ "注意文章中的每一句话都要翻译": "Please translate every sentence in the article", "一、论文概况": "I. Overview of the paper", "二、论文翻译": "II. Translation of the paper", - "/gpt_log/总结论文-": "/gpt_log/Summary of the paper-", "给出输出文件清单": "Provide a list of output files", "第 0 步": "Step 0", "切割PDF": "Split PDF", @@ -1564,7 +1560,6 @@ "广义速度": "Generalized velocity", "粒子的固有": "Intrinsic of particle", "一个包含所有切割音频片段文件路径的列表": "A list containing the file paths of all segmented audio clips", - "/gpt_log/翻译-": "Translation log-", "计算文件总时长和切割点": "Calculate total duration and cutting points of the file", "总结音频": "Summarize audio", "作者": "Author", @@ -2339,7 +2334,6 @@ "将文件拖动到文件上传区": "Drag and drop the file to the file upload area", "如果意图模糊": "If the intent is ambiguous", "星火认知大模型": "Spark Cognitive Big Model", - "执行中. 删除 gpt_log & private_upload": "Executing. Delete gpt_log & private_upload", "默认 Color = secondary": "Default Color = secondary", "此处也不需要修改": "No modification is needed here", "⭐ ⭐ ⭐ 分析用户意图": "⭐ ⭐ ⭐ Analyze user intent", diff --git a/docs/translate_japanese.json b/docs/translate_japanese.json index 15a4b31..ecd370e 100644 --- a/docs/translate_japanese.json +++ b/docs/translate_japanese.json @@ -301,7 +301,6 @@ "缺少的依赖": "不足している依存関係", "紫色": "紫色", "唤起高级参数输入区": "高度なパラメータ入力エリアを呼び出す", - "所有问询记录将自动保存在本地目录./gpt_log/chat_secrets.log": "すべての問い合わせ記録は自動的にローカルディレクトリ./gpt_log/chat_secrets.logに保存されます", "则换行符更有可能表示段落分隔": "したがって、改行記号は段落の区切りを表す可能性がより高いです", ";4、引用数量": ";4、引用数量", "中转网址预览": "中継ウェブサイトのプレビュー", @@ -448,7 +447,6 @@ "表示函数是否成功执行": "関数が正常に実行されたかどうかを示す", "一般原样传递下去就行": "通常はそのまま渡すだけでよい", "琥珀色": "琥珀色", - "gpt_log/**/chatGPT对话历史*.html": "gpt_log/**/chatGPT対話履歴*.html", "jittorllms 没有 sys_prompt 接口": "jittorllmsにはsys_promptインターフェースがありません", "清除": "クリア", "小于正文的": "本文より小さい", @@ -1234,7 +1232,6 @@ "找不到任何前端相关文件": "No frontend-related files can be found", "Not enough point. API2D账户点数不足": "Not enough points. API2D account points are insufficient", "当前版本": "Current version", - "/gpt_log/总结论文-": "/gpt_log/Summary paper-", "1. 临时解决方案": "1. Temporary solution", "第8步": "Step 8", "历史": "History", diff --git a/docs/translate_traditionalchinese.json b/docs/translate_traditionalchinese.json index cf12de1..efaa62f 100644 --- a/docs/translate_traditionalchinese.json +++ b/docs/translate_traditionalchinese.json @@ -314,7 +314,6 @@ "请用markdown格式输出": "請用 Markdown 格式輸出", "模仿ChatPDF": "模仿 ChatPDF", "等待多久判定为超时": "等待多久判定為超時", - "/gpt_log/总结论文-": "/gpt_log/總結論文-", "请结合互联网信息回答以下问题": "請結合互聯網信息回答以下問題", "IP查询频率受限": "IP查詢頻率受限", "高级参数输入区的显示提示": "高級參數輸入區的顯示提示", @@ -511,7 +510,6 @@ "將生成的報告自動投射到文件上傳區": "將生成的報告自動上傳到文件區", "函數插件作者": "函數插件作者", "將要匹配的模式": "將要匹配的模式", - "所有问询记录将自动保存在本地目录./gpt_log/chat_secrets.log": "所有詢問記錄將自動保存在本地目錄./gpt_log/chat_secrets.log", "正在分析一个项目的源代码": "正在分析一個專案的源代碼", "使每个段落之间有两个换行符分隔": "使每個段落之間有兩個換行符分隔", "并在被装饰的函数上执行": "並在被裝飾的函數上執行", @@ -1059,7 +1057,6 @@ "重试中": "重試中", "月": "月份", "localhost意思是代理软件安装在本机上": "localhost意思是代理軟體安裝在本機上", - "gpt_log/**/chatGPT对话历史*.html": "gpt_log/**/chatGPT對話歷史*.html", "的长度必须小于 2500 个 Token": "長度必須小於 2500 個 Token", "抽取可用的api-key": "提取可用的api-key", "增强报告的可读性": "增強報告的可讀性", diff --git a/main.py b/main.py index 628d30b..aedbcfc 100644 --- a/main.py +++ b/main.py @@ -8,7 +8,7 @@ def main(): # 建议您复制一个config_private.py放自己的秘密, 如API和代理网址, 避免不小心传github被别人看到 proxies, WEB_PORT, LLM_MODEL, CONCURRENT_COUNT, AUTHENTICATION = get_conf('proxies', 'WEB_PORT', 'LLM_MODEL', 'CONCURRENT_COUNT', 'AUTHENTICATION') CHATBOT_HEIGHT, LAYOUT, AVAIL_LLM_MODELS, AUTO_CLEAR_TXT = get_conf('CHATBOT_HEIGHT', 'LAYOUT', 'AVAIL_LLM_MODELS', 'AUTO_CLEAR_TXT') - ENABLE_AUDIO, AUTO_CLEAR_TXT = get_conf('ENABLE_AUDIO', 'AUTO_CLEAR_TXT') + ENABLE_AUDIO, AUTO_CLEAR_TXT, PATH_LOGGING = get_conf('ENABLE_AUDIO', 'AUTO_CLEAR_TXT', 'PATH_LOGGING') # 如果WEB_PORT是-1, 则随机选取WEB端口 PORT = find_free_port() if WEB_PORT <= 0 else WEB_PORT @@ -21,12 +21,12 @@ def main(): # 问询记录, python 版本建议3.9+(越新越好) import logging, uuid - os.makedirs("gpt_log", exist_ok=True) - try:logging.basicConfig(filename="gpt_log/chat_secrets.log", level=logging.INFO, encoding="utf-8", format="%(asctime)s %(levelname)-8s %(message)s", datefmt="%Y-%m-%d %H:%M:%S") - except:logging.basicConfig(filename="gpt_log/chat_secrets.log", level=logging.INFO, format="%(asctime)s %(levelname)-8s %(message)s", datefmt="%Y-%m-%d %H:%M:%S") + os.makedirs(PATH_LOGGING, exist_ok=True) + try:logging.basicConfig(filename=f"{PATH_LOGGING}/chat_secrets.log", level=logging.INFO, encoding="utf-8", format="%(asctime)s %(levelname)-8s %(message)s", datefmt="%Y-%m-%d %H:%M:%S") + except:logging.basicConfig(filename=f"{PATH_LOGGING}/chat_secrets.log", level=logging.INFO, format="%(asctime)s %(levelname)-8s %(message)s", datefmt="%Y-%m-%d %H:%M:%S") # Disable logging output from the 'httpx' logger logging.getLogger("httpx").setLevel(logging.WARNING) - print("所有问询记录将自动保存在本地目录./gpt_log/chat_secrets.log, 请注意自我隐私保护哦!") + print(f"所有问询记录将自动保存在本地目录./{PATH_LOGGING}/chat_secrets.log, 请注意自我隐私保护哦!") # 一些普通功能模块 from core_functional import get_core_functions diff --git a/multi_language.py b/multi_language.py index c4ed36e..8e3ac9d 100644 --- a/multi_language.py +++ b/multi_language.py @@ -33,9 +33,11 @@ import functools import re import pickle import time +from toolbox import get_conf -CACHE_FOLDER = "gpt_log" -blacklist = ['multi-language', 'gpt_log', '.git', 'private_upload', 'multi_language.py', 'build', '.github', '.vscode', '__pycache__', 'venv'] +CACHE_FOLDER, = get_conf('PATH_LOGGING') + +blacklist = ['multi-language', CACHE_FOLDER, '.git', 'private_upload', 'multi_language.py', 'build', '.github', '.vscode', '__pycache__', 'venv'] # LANG = "TraditionalChinese" # TransPrompt = f"Replace each json value `#` with translated results in Traditional Chinese, e.g., \"原始文本\":\"翻譯後文字\". Keep Json format. Do not answer #." diff --git a/request_llm/bridge_chatgpt.py b/request_llm/bridge_chatgpt.py index 19d1002..6a458f2 100644 --- a/request_llm/bridge_chatgpt.py +++ b/request_llm/bridge_chatgpt.py @@ -21,7 +21,7 @@ import importlib # config_private.py放自己的秘密如API和代理网址 # 读取时首先看是否存在私密的config_private配置文件(不受git管控),如果有,则覆盖原config文件 -from toolbox import get_conf, update_ui, is_any_api_key, select_api_key, what_keys, clip_history, trimmed_format_exc +from toolbox import get_conf, update_ui, is_any_api_key, select_api_key, what_keys, clip_history, trimmed_format_exc, is_the_upload_folder proxies, TIMEOUT_SECONDS, MAX_RETRY, API_ORG = \ get_conf('proxies', 'TIMEOUT_SECONDS', 'MAX_RETRY', 'API_ORG') @@ -138,7 +138,7 @@ def predict(inputs, llm_kwargs, plugin_kwargs, chatbot, history=[], system_promp yield from update_ui(chatbot=chatbot, history=history, msg="等待响应") # 刷新界面 # check mis-behavior - if raw_input.startswith('private_upload/') and len(raw_input) == 34: + if is_the_upload_folder(raw_input): chatbot[-1] = (inputs, f"[Local Message] 检测到操作错误!当您上传文档之后,需要点击“函数插件区”按钮进行处理,而不是点击“提交”按钮。") yield from update_ui(chatbot=chatbot, history=history, msg="正常") # 刷新界面 time.sleep(2) diff --git a/toolbox.py b/toolbox.py index ab8d046..947044f 100644 --- a/toolbox.py +++ b/toolbox.py @@ -5,6 +5,8 @@ import inspect import re import os import gradio +import shutil +import glob from latex2mathml.converter import convert as tex2mathml from functools import wraps, lru_cache pj = os.path.join @@ -171,7 +173,7 @@ def HotReload(f): ======================================================================== 第二部分 其他小工具: - - write_results_to_file: 将结果写入markdown文件中 + - write_history_to_file: 将结果写入markdown文件中 - regular_txt_to_markdown: 将普通文本转换为Markdown格式的文本。 - report_execption: 向chatbot中添加简单的意外错误信息 - text_divide_paragraph: 将文本按照段落分隔符分割开,生成带有段落标签的HTML代码。 @@ -203,36 +205,6 @@ def get_reduce_token_percent(text): return 0.5, '不详' -def write_results_to_file(history, file_name=None): - """ - 将对话记录history以Markdown格式写入文件中。如果没有指定文件名,则使用当前时间生成文件名。 - """ - import os - import time - if file_name is None: - # file_name = time.strftime("chatGPT分析报告%Y-%m-%d-%H-%M-%S", time.localtime()) + '.md' - file_name = 'GPT-Report-' + gen_time_str() + '.md' - os.makedirs('./gpt_log/', exist_ok=True) - with open(f'./gpt_log/{file_name}', 'w', encoding='utf8') as f: - f.write('# GPT-Academic Report\n') - for i, content in enumerate(history): - try: - if type(content) != str: content = str(content) - except: - continue - if i % 2 == 0: - f.write('## ') - try: - f.write(content) - except: - # remove everything that cannot be handled by utf8 - f.write(content.encode('utf-8', 'ignore').decode()) - f.write('\n\n') - res = '以上材料已经被写入:\t' + os.path.abspath(f'./gpt_log/{file_name}') - print(res) - return res - - def write_history_to_file(history, file_basename=None, file_fullname=None): """ 将对话记录history以Markdown格式写入文件中。如果没有指定文件名,则使用当前时间生成文件名。 @@ -241,9 +213,9 @@ def write_history_to_file(history, file_basename=None, file_fullname=None): import time if file_fullname is None: if file_basename is not None: - file_fullname = os.path.join(get_log_folder(), file_basename) + file_fullname = pj(get_log_folder(), file_basename) else: - file_fullname = os.path.join(get_log_folder(), f'GPT-Academic-{gen_time_str()}.md') + file_fullname = pj(get_log_folder(), f'GPT-Academic-{gen_time_str()}.md') os.makedirs(os.path.dirname(file_fullname), exist_ok=True) with open(file_fullname, 'w', encoding='utf8') as f: f.write('# GPT-Academic Report\n') @@ -519,7 +491,7 @@ def find_recent_files(directory): if not os.path.exists(directory): os.makedirs(directory, exist_ok=True) for filename in os.listdir(directory): - file_path = os.path.join(directory, filename) + file_path = pj(directory, filename) if file_path.endswith('.log'): continue created_time = os.path.getmtime(file_path) @@ -534,7 +506,7 @@ def promote_file_to_downloadzone(file, rename_file=None, chatbot=None): # 将文件复制一份到下载区 import shutil if rename_file is None: rename_file = f'{gen_time_str()}-{os.path.basename(file)}' - new_path = os.path.join(get_log_folder(), rename_file) + new_path = pj(get_log_folder(), rename_file) # 如果已经存在,先删除 if os.path.exists(new_path) and not os.path.samefile(new_path, file): os.remove(new_path) # 把文件复制过去 @@ -549,44 +521,70 @@ def disable_auto_promotion(chatbot): chatbot._cookies.update({'files_to_promote': []}) return -def on_file_uploaded(files, chatbot, txt, txt2, checkboxes, cookies): +def is_the_upload_folder(string): + PATH_PRIVATE_UPLOAD, = get_conf('PATH_PRIVATE_UPLOAD') + pattern = r'^PATH_PRIVATE_UPLOAD/[A-Za-z0-9_-]+/\d{4}-\d{2}-\d{2}-\d{2}-\d{2}-\d{2}$' + pattern = pattern.replace('PATH_PRIVATE_UPLOAD', PATH_PRIVATE_UPLOAD) + if re.match(pattern, string): return True + else: return False + +def del_outdated_uploads(outdate_time_seconds): + PATH_PRIVATE_UPLOAD, = get_conf('PATH_PRIVATE_UPLOAD') + current_time = time.time() + one_hour_ago = current_time - outdate_time_seconds + # Get a list of all subdirectories in the PATH_PRIVATE_UPLOAD folder + # Remove subdirectories that are older than one hour + for subdirectory in glob.glob(f'{PATH_PRIVATE_UPLOAD}/*/*'): + subdirectory_time = os.path.getmtime(subdirectory) + if subdirectory_time < one_hour_ago: + try: shutil.rmtree(subdirectory) + except: pass + return + +def on_file_uploaded(request: gradio.Request, files, chatbot, txt, txt2, checkboxes, cookies): """ 当文件被上传时的回调函数 """ if len(files) == 0: return chatbot, txt - import shutil - import os - import time - import glob - from toolbox import extract_archive - try: - shutil.rmtree('./private_upload/') - except: - pass + + # 移除过时的旧文件从而节省空间&保护隐私 + outdate_time_seconds = 60 + del_outdated_uploads(outdate_time_seconds) + + # 创建工作路径 + user_name = "default" if not request.username else request.username time_tag = gen_time_str() - os.makedirs(f'private_upload/{time_tag}', exist_ok=True) - err_msg = '' + PATH_PRIVATE_UPLOAD, = get_conf('PATH_PRIVATE_UPLOAD') + target_path_base = pj(PATH_PRIVATE_UPLOAD, user_name, time_tag) + os.makedirs(target_path_base, exist_ok=True) + + # 逐个文件转移到目标路径 + upload_msg = '' for file in files: file_origin_name = os.path.basename(file.orig_name) - shutil.copy(file.name, f'private_upload/{time_tag}/{file_origin_name}') - err_msg += extract_archive(f'private_upload/{time_tag}/{file_origin_name}', - dest_dir=f'private_upload/{time_tag}/{file_origin_name}.extract') - moved_files = [fp for fp in glob.glob('private_upload/**/*', recursive=True)] - if "底部输入区" in checkboxes: - txt = "" - txt2 = f'private_upload/{time_tag}' + this_file_path = pj(target_path_base, file_origin_name) + shutil.move(file.name, this_file_path) + upload_msg += extract_archive(file_path=this_file_path, dest_dir=this_file_path+'.extract') + + # 整理文件集合 + moved_files = [fp for fp in glob.glob(f'{target_path_base}/**/*', recursive=True)] + if "底部输入区" in checkboxes: + txt, txt2 = "", target_path_base else: - txt = f'private_upload/{time_tag}' - txt2 = "" + txt, txt2 = target_path_base, "" + + # 输出消息 moved_files_str = '\t\n\n'.join(moved_files) - chatbot.append(['我上传了文件,请查收', + chatbot.append(['我上传了文件,请查收', f'[Local Message] 收到以下文件: \n\n{moved_files_str}' + f'\n\n调用路径参数已自动修正到: \n\n{txt}' + - f'\n\n现在您点击任意函数插件时,以上文件将被作为输入参数'+err_msg]) + f'\n\n现在您点击任意函数插件时,以上文件将被作为输入参数'+upload_msg]) + + # 记录近期文件 cookies.update({ 'most_recent_uploaded': { - 'path': f'private_upload/{time_tag}', + 'path': target_path_base, 'time': time.time(), 'time_str': time_tag }}) @@ -595,11 +593,12 @@ def on_file_uploaded(files, chatbot, txt, txt2, checkboxes, cookies): def on_report_generated(cookies, files, chatbot): from toolbox import find_recent_files + PATH_LOGGING, = get_conf('PATH_LOGGING') if 'files_to_promote' in cookies: report_files = cookies['files_to_promote'] cookies.pop('files_to_promote') else: - report_files = find_recent_files('gpt_log') + report_files = find_recent_files(PATH_LOGGING) if len(report_files) == 0: return cookies, None, chatbot # files.extend(report_files) @@ -909,34 +908,35 @@ def zip_folder(source_folder, dest_folder, zip_name): return # Create the name for the zip file - zip_file = os.path.join(dest_folder, zip_name) + zip_file = pj(dest_folder, zip_name) # Create a ZipFile object with zipfile.ZipFile(zip_file, 'w', zipfile.ZIP_DEFLATED) as zipf: # Walk through the source folder and add files to the zip file for foldername, subfolders, filenames in os.walk(source_folder): for filename in filenames: - filepath = os.path.join(foldername, filename) + filepath = pj(foldername, filename) zipf.write(filepath, arcname=os.path.relpath(filepath, source_folder)) # Move the zip file to the destination folder (if it wasn't already there) if os.path.dirname(zip_file) != dest_folder: - os.rename(zip_file, os.path.join(dest_folder, os.path.basename(zip_file))) - zip_file = os.path.join(dest_folder, os.path.basename(zip_file)) + os.rename(zip_file, pj(dest_folder, os.path.basename(zip_file))) + zip_file = pj(dest_folder, os.path.basename(zip_file)) print(f"Zip file created at {zip_file}") def zip_result(folder): t = gen_time_str() - zip_folder(folder, './gpt_log/', f'{t}-result.zip') - return pj('./gpt_log/', f'{t}-result.zip') + zip_folder(folder, get_log_folder(), f'{t}-result.zip') + return pj(get_log_folder(), f'{t}-result.zip') def gen_time_str(): import time return time.strftime("%Y-%m-%d-%H-%M-%S", time.localtime()) def get_log_folder(user='default', plugin_name='shared'): - _dir = os.path.join(os.path.dirname(__file__), 'gpt_log', user, plugin_name) + PATH_LOGGING, = get_conf('PATH_LOGGING') + _dir = pj(PATH_LOGGING, user, plugin_name) if not os.path.exists(_dir): os.makedirs(_dir) return _dir