From e20070939c6c7eeca33a8438041c9e038836957b Mon Sep 17 00:00:00 2001 From: saltfish Date: Sat, 22 Apr 2023 00:36:28 +0800 Subject: [PATCH 1/4] Parse and generate ipynb (Issue #501) Implemented code to parse and generate the ipynb files. The solution addresses Issue #501. --- crazy_functional.py | 6 ++ crazy_functions/解析JupyterNotebook.py | 135 +++++++++++++++++++++++++ 2 files changed, 141 insertions(+) create mode 100644 crazy_functions/解析JupyterNotebook.py diff --git a/crazy_functional.py b/crazy_functional.py index 8e3ab6a..8f3b682 100644 --- a/crazy_functional.py +++ b/crazy_functional.py @@ -19,12 +19,18 @@ def get_crazy_functions(): from crazy_functions.解析项目源代码 import 解析一个Lua项目 from crazy_functions.解析项目源代码 import 解析一个CSharp项目 from crazy_functions.总结word文档 import 总结word文档 + from crazy_functions.解析JupyterNotebook import 解析ipynb文件 function_plugins = { "解析整个Python项目": { "Color": "stop", # 按钮颜色 "Function": HotReload(解析一个Python项目) }, + "解析Jupyter Notebook文件": { + "Color": "stop", + "AsButton":False, + "Function": HotReload(解析ipynb文件), + }, "批量总结Word文档": { "Color": "stop", "Function": HotReload(总结word文档) diff --git a/crazy_functions/解析JupyterNotebook.py b/crazy_functions/解析JupyterNotebook.py new file mode 100644 index 0000000..86f246c --- /dev/null +++ b/crazy_functions/解析JupyterNotebook.py @@ -0,0 +1,135 @@ +from toolbox import update_ui +from toolbox import CatchException, report_execption +fast_debug = True + + +class PaperFileGroup(): + def __init__(self): + self.file_paths = [] + self.file_contents = [] + self.sp_file_contents = [] + self.sp_file_index = [] + self.sp_file_tag = [] + + # count_token + from request_llm.bridge_all import model_info + enc = model_info["gpt-3.5-turbo"]['tokenizer'] + def get_token_num(txt): return len( + enc.encode(txt, disallowed_special=())) + self.get_token_num = get_token_num + + def run_file_split(self, max_token_limit=1900): + """ + 将长文本分离开来 + """ + for index, file_content in enumerate(self.file_contents): + if self.get_token_num(file_content) < max_token_limit: + self.sp_file_contents.append(file_content) + self.sp_file_index.append(index) + self.sp_file_tag.append(self.file_paths[index]) + else: + from .crazy_utils import breakdown_txt_to_satisfy_token_limit_for_pdf + segments = breakdown_txt_to_satisfy_token_limit_for_pdf( + file_content, self.get_token_num, max_token_limit) + for j, segment in enumerate(segments): + self.sp_file_contents.append(segment) + self.sp_file_index.append(index) + self.sp_file_tag.append( + self.file_paths[index] + f".part-{j}.txt") + + + +def parseNotebook(filename, enable_markdown=1): + import json + + CodeBlocks = [] + with open(filename, 'r', encoding='utf-8', errors='replace') as f: + notebook = json.load(f) + for cell in notebook['cells']: + if cell['cell_type'] == 'code' and cell['source']: + # remove blank lines + cell['source'] = [line for line in cell['source'] if line.strip() + != ''] + CodeBlocks.append("".join(cell['source'])) + elif enable_markdown and cell['cell_type'] == 'markdown' and cell['source']: + cell['source'] = [line for line in cell['source'] if line.strip() + != ''] + CodeBlocks.append("Markdown:"+"".join(cell['source'])) + + Code = "" + for idx, code in enumerate(CodeBlocks): + Code += f"This is {idx+1}th code block: \n" + Code += code+"\n" + + return Code + + +def ipynb解释(file_manifest, project_folder, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt): + from .crazy_utils import request_gpt_model_multi_threads_with_very_awesome_ui_and_high_efficiency + + pfg = PaperFileGroup() + + print(file_manifest) + for fp in file_manifest: + file_content = parseNotebook(fp, enable_markdown=1) + pfg.file_paths.append(fp) + pfg.file_contents.append(file_content) + + # <-------- 拆分过长的IPynb文件 ----------> + pfg.run_file_split(max_token_limit=1024) + n_split = len(pfg.sp_file_contents) + + inputs_array = [f"This is a Jupyter Notebook file, tell me about Each Block in Chinese. Focus Just On Code." + + f"If a block starts with `Markdown` which means it's a markdown block in ipynbipynb. " + + f"Start a new line for a block and block num use Chinese." + + f"\n\n{frag}" for frag in pfg.sp_file_contents] + inputs_show_user_array = [f"{f}的分析如下" for f in pfg.sp_file_tag] + sys_prompt_array = ["You are a professional programmer."] * n_split + + gpt_response_collection = yield from request_gpt_model_multi_threads_with_very_awesome_ui_and_high_efficiency( + inputs_array=inputs_array, + inputs_show_user_array=inputs_show_user_array, + llm_kwargs=llm_kwargs, + chatbot=chatbot, + history_array=[[""] for _ in range(n_split)], + sys_prompt_array=sys_prompt_array, + # max_workers=5, # OpenAI所允许的最大并行过载 + scroller_max_len=80 + ) + + # <-------- 整理结果，退出 ----------> + block_result = " \n".join(gpt_response_collection) + chatbot.append(("解析的结果如下", block_result)) + yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 + + +@CatchException +def 解析ipynb文件(txt, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, web_port): + chatbot.append([ + "函数插件功能？", + "对IPynb文件进行解析"]) + yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 + + history = [] # 清空历史 + import glob + import os + if os.path.exists(txt): + project_folder = txt + else: + if txt == "": + txt = '空空如也的输入栏' + report_execption(chatbot, history, + a=f"解析项目: {txt}", b=f"找不到本地项目或无权访问: {txt}") + yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 + return + if txt.endswith('.ipynb'): + file_manifest = [txt] + else: + file_manifest = [f for f in glob.glob( + f'{project_folder}/**/*.ipynb', recursive=True)] + if len(file_manifest) == 0: + report_execption(chatbot, history, + a=f"解析项目: {txt}", b=f"找不到任何.ipynb文件: {txt}") + yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 + return + yield from ipynb解释(file_manifest, project_folder, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, ) From 64548d33a95df51dbd0efce7460e19d2a2dc4b08 Mon Sep 17 00:00:00 2001 From: binary-husky <96192199+binary-husky@users.noreply.github.com> Date: Sat, 22 Apr 2023 15:58:43 +0800 Subject: [PATCH 2/4] Update crazy_functional.py --- crazy_functional.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crazy_functional.py b/crazy_functional.py index 8f3b682..cdc47cf 100644 --- a/crazy_functional.py +++ b/crazy_functional.py @@ -26,7 +26,7 @@ def get_crazy_functions(): "Color": "stop", # 按钮颜色 "Function": HotReload(解析一个Python项目) }, - "解析Jupyter Notebook文件": { + "[测试功能] 解析Jupyter Notebook文件": { "Color": "stop", "AsButton":False, "Function": HotReload(解析ipynb文件), From 9d9df8aa573a9fb509e6836425d3e19cd336f78e Mon Sep 17 00:00:00 2001 From: binary-husky <96192199+binary-husky@users.noreply.github.com> Date: Sat, 22 Apr 2023 16:01:32 +0800 Subject: [PATCH 3/4] =?UTF-8?q?Update=20=E8=A7=A3=E6=9E=90JupyterNotebook.?= =?UTF-8?q?py?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- crazy_functions/解析JupyterNotebook.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crazy_functions/解析JupyterNotebook.py b/crazy_functions/解析JupyterNotebook.py index 86f246c..1627f88 100644 --- a/crazy_functions/解析JupyterNotebook.py +++ b/crazy_functions/解析JupyterNotebook.py @@ -107,7 +107,7 @@ def ipynb解释(file_manifest, project_folder, llm_kwargs, plugin_kwargs, chatbo def 解析ipynb文件(txt, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, web_port): chatbot.append([ "函数插件功能？", - "对IPynb文件进行解析"]) + "对IPynb文件进行解析。Contributor: codycjy."]) yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 history = [] # 清空历史 From 880be21bf713bfff940e81f0afebdc8fff98c88e Mon Sep 17 00:00:00 2001 From: Your Name Date: Sat, 22 Apr 2023 16:19:36 +0800 Subject: [PATCH 4/4] Add test for juptyer notebook plugin --- .gitignore | 1 + crazy_functions/crazy_functions_test.py | 12 +++++++++--- crazy_functions/解析JupyterNotebook.py | 13 +++++++++---- 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/.gitignore b/.gitignore index 987f054..7a9c92b 100644 --- a/.gitignore +++ b/.gitignore @@ -145,3 +145,4 @@ cradle* debug* private* crazy_functions/test_project/pdf_and_word +crazy_functions/test_samples diff --git a/crazy_functions/crazy_functions_test.py b/crazy_functions/crazy_functions_test.py index 7b0f072..6020fa2 100644 --- a/crazy_functions/crazy_functions_test.py +++ b/crazy_functions/crazy_functions_test.py @@ -108,6 +108,13 @@ def test_联网回答问题(): print("当前问答：", cb[-1][-1].replace("\n"," ")) for i, it in enumerate(cb): print亮蓝(it[0]); print亮黄(it[1]) +def test_解析ipynb文件(): + from crazy_functions.解析JupyterNotebook import 解析ipynb文件 + txt = "crazy_functions/test_samples" + for cookies, cb, hist, msg in 解析ipynb文件(txt, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, web_port): + print(cb) + + # test_解析一个Python项目() # test_Latex英文润色() # test_Markdown中译英() @@ -116,9 +123,8 @@ def test_联网回答问题(): # test_总结word文档() # test_下载arxiv论文并翻译摘要() # test_解析一个Cpp项目() - -test_联网回答问题() - +# test_联网回答问题() +test_解析ipynb文件() input("程序完成，回车退出。") print("退出。") \ No newline at end of file diff --git a/crazy_functions/解析JupyterNotebook.py b/crazy_functions/解析JupyterNotebook.py index 1627f88..3c9948b 100644 --- a/crazy_functions/解析JupyterNotebook.py +++ b/crazy_functions/解析JupyterNotebook.py @@ -1,5 +1,5 @@ from toolbox import update_ui -from toolbox import CatchException, report_execption +from toolbox import CatchException, report_execption, write_results_to_file fast_debug = True @@ -79,9 +79,9 @@ def ipynb解释(file_manifest, project_folder, llm_kwargs, plugin_kwargs, chatbo pfg.run_file_split(max_token_limit=1024) n_split = len(pfg.sp_file_contents) - inputs_array = [f"This is a Jupyter Notebook file, tell me about Each Block in Chinese. Focus Just On Code." + - f"If a block starts with `Markdown` which means it's a markdown block in ipynbipynb. " + - f"Start a new line for a block and block num use Chinese." + + inputs_array = [r"This is a Jupyter Notebook file, tell me about Each Block in Chinese. Focus Just On Code." + + r"If a block starts with `Markdown` which means it's a markdown block in ipynbipynb. " + + r"Start a new line for a block and block num use Chinese." + f"\n\n{frag}" for frag in pfg.sp_file_contents] inputs_show_user_array = [f"{f}的分析如下" for f in pfg.sp_file_tag] sys_prompt_array = ["You are a professional programmer."] * n_split @@ -100,8 +100,13 @@ def ipynb解释(file_manifest, project_folder, llm_kwargs, plugin_kwargs, chatbo # <-------- 整理结果，退出 ----------> block_result = " \n".join(gpt_response_collection) chatbot.append(("解析的结果如下", block_result)) + history.extend(["解析的结果如下", block_result]) yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 + # <-------- 写入文件，退出 ----------> + res = write_results_to_file(history) + chatbot.append(("完成了吗？", res)) + yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 @CatchException def 解析ipynb文件(txt, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, web_port):