From 49f3fcf2c08ca8ccad52784bda5b5816d0327b8b Mon Sep 17 00:00:00 2001 From: binary-husky Date: Tue, 5 Dec 2023 21:22:15 +0800 Subject: [PATCH] vector store external to internal --- crazy_functional.py | 6 +++--- crazy_functions/vector_fns/__init__.py | 0 crazy_functions/知识库问答.py | 16 ++++++++-------- docs/translate_english.json | 2 +- docs/translate_japanese.json | 2 +- docs/translate_std.json | 2 +- docs/translate_traditionalchinese.json | 2 +- tests/test_plugins.py | 6 +++--- tests/test_utils.py | 16 ++++++++-------- tests/test_vector_plugins.py | 17 +++++++++++++++++ 10 files changed, 43 insertions(+), 26 deletions(-) create mode 100644 crazy_functions/vector_fns/__init__.py create mode 100644 tests/test_vector_plugins.py diff --git a/crazy_functional.py b/crazy_functional.py index 0d665f1..0c1560f 100644 --- a/crazy_functional.py +++ b/crazy_functional.py @@ -440,7 +440,7 @@ def get_crazy_functions(): print('Load function plugin failed') try: - from crazy_functions.知识库问答 import 知识库问答 + from crazy_functions.知识库问答 import 知识库文件注入 function_plugins.update({ "构建知识库（先上传文件素材,再运行此插件）": { "Group": "对话", @@ -448,7 +448,7 @@ def get_crazy_functions(): "AsButton": False, "AdvancedArgs": True, "ArgsReminder": "此处待注入的知识库名称id, 默认为default。文件进入知识库后可长期保存。可以通过再次调用本插件的方式，向知识库追加更多文档。", - "Function": HotReload(知识库问答) + "Function": HotReload(知识库文件注入) } }) except: @@ -458,7 +458,7 @@ def get_crazy_functions(): try: from crazy_functions.知识库问答 import 读取知识库作答 function_plugins.update({ - "知识库问答（构建知识库后,再运行此插件）": { + "知识库文件注入（构建知识库后,再运行此插件）": { "Group": "对话", "Color": "stop", "AsButton": False, diff --git a/crazy_functions/vector_fns/__init__.py b/crazy_functions/vector_fns/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/crazy_functions/知识库问答.py b/crazy_functions/知识库问答.py index 9d53848..8521ca1 100644 --- a/crazy_functions/知识库问答.py +++ b/crazy_functions/知识库问答.py @@ -4,7 +4,7 @@ from .crazy_utils import request_gpt_model_in_new_thread_with_ui_alive, get_file @CatchException -def 知识库问答(txt, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, web_port): +def 知识库文件注入(txt, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, web_port): """ txt 输入栏用户输入的文本，例如需要翻译的一段话，再例如一个包含了待处理文件的路径 llm_kwargs gpt模型参数, 如温度和top_p等, 一般原样传递下去就行 @@ -25,9 +25,9 @@ def 知识库问答(txt, llm_kwargs, plugin_kwargs, chatbot, history, system_pro # resolve deps try: - from zh_langchain import construct_vector_store - from langchain.embeddings.huggingface import HuggingFaceEmbeddings - from .crazy_utils import knowledge_archive_interface + # from zh_langchain import construct_vector_store + # from langchain.embeddings.huggingface import HuggingFaceEmbeddings + from crazy_functions.vector_fns.vector_database import knowledge_archive_interface except Exception as e: chatbot.append(["依赖不足", "导入依赖失败。正在尝试自动安装，请查看终端的输出或耐心等待..."]) yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 @@ -68,7 +68,7 @@ def 知识库问答(txt, llm_kwargs, plugin_kwargs, chatbot, history, system_pro # chatbot.append(['知识库构建成功', "正在将知识库存储至cookie中"]) # yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 # chatbot._cookies['langchain_plugin_embedding'] = kai.get_current_archive_id() - # chatbot._cookies['lock_plugin'] = 'crazy_functions.知识库问答->读取知识库作答' + # chatbot._cookies['lock_plugin'] = 'crazy_functions.知识库文件注入->读取知识库作答' # chatbot.append(['完成', "“根据知识库作答”函数插件已经接管问答系统, 提问吧! 但注意, 您接下来不能再使用其他插件了，刷新页面即可以退出知识库问答模式。"]) chatbot.append(['构建完成', f"当前知识库内的有效文件：\n\n---\n\n{kai_files}\n\n---\n\n请切换至“知识库问答”插件进行知识库访问, 或者使用此插件继续上传更多文件。"]) yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 # 由于请求gpt需要一段时间，我们先及时地做一次界面更新 @@ -77,9 +77,9 @@ def 知识库问答(txt, llm_kwargs, plugin_kwargs, chatbot, history, system_pro def 读取知识库作答(txt, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, web_port=-1): # resolve deps try: - from zh_langchain import construct_vector_store - from langchain.embeddings.huggingface import HuggingFaceEmbeddings - from .crazy_utils import knowledge_archive_interface + # from zh_langchain import construct_vector_store + # from langchain.embeddings.huggingface import HuggingFaceEmbeddings + from crazy_functions.vector_fns.vector_database import knowledge_archive_interface except Exception as e: chatbot.append(["依赖不足", "导入依赖失败。正在尝试自动安装，请查看终端的输出或耐心等待..."]) yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 diff --git a/docs/translate_english.json b/docs/translate_english.json index 622a1d6..8221a1f 100644 --- a/docs/translate_english.json +++ b/docs/translate_english.json @@ -1666,7 +1666,7 @@ "连接bing搜索回答问题": "ConnectBingSearchAnswerQuestion", "联网的ChatGPT_bing版": "OnlineChatGPT_BingEdition", "Markdown翻译指定语言": "TranslateMarkdownToSpecifiedLanguage", - "知识库问答": "LangchainKnowledgeBase", + "知识库文件注入": "LangchainKnowledgeBase", "Latex英文纠错加PDF对比": "CorrectEnglishInLatexWithPDFComparison", "Latex输出PDF结果": "OutputPDFFromLatex", "Latex翻译中文并重新编译PDF": "TranslateChineseToEnglishInLatexAndRecompilePDF", diff --git a/docs/translate_japanese.json b/docs/translate_japanese.json index 2184d4a..820751a 100644 --- a/docs/translate_japanese.json +++ b/docs/translate_japanese.json @@ -1487,7 +1487,7 @@ "数学动画生成manim": "GenerateMathematicalAnimationManim", "Markdown翻译指定语言": "TranslateMarkdownSpecifiedLanguage", "知识库问答": "KnowledgeBaseQuestionAnswer", - "知识库问答": "LangchainKnowledgeBase", + "知识库文件注入": "LangchainKnowledgeBase", "读取知识库作答": "ReadKnowledgeBaseAnswer", "交互功能模板函数": "InteractiveFunctionTemplateFunction", "交互功能函数模板": "InteractiveFunctionFunctionTemplate", diff --git a/docs/translate_std.json b/docs/translate_std.json index 0c2a7bd..d286bac 100644 --- a/docs/translate_std.json +++ b/docs/translate_std.json @@ -75,7 +75,7 @@ "解析docx": "ParseDocx", "解析源代码新": "ParsingSourceCodeNew", "总结音视频": "SummaryAudioVideo", - "知识库问答": "UpdateKnowledgeArchive", + "知识库文件注入": "UpdateKnowledgeArchive", "多文件润色": "ProofreadMultipleFiles", "多文件翻译": "TranslateMultipleFiles", "解析PDF": "ParsePDF", diff --git a/docs/translate_traditionalchinese.json b/docs/translate_traditionalchinese.json index f0338fc..586e029 100644 --- a/docs/translate_traditionalchinese.json +++ b/docs/translate_traditionalchinese.json @@ -1463,7 +1463,7 @@ "数学动画生成manim": "GenerateMathematicalAnimationsWithManim", "Markdown翻译指定语言": "TranslateMarkdownToSpecifiedLanguage", "知识库问答": "KnowledgeBaseQA", - "知识库问答": "LangchainKnowledgeBase", + "知识库文件注入": "LangchainKnowledgeBase", "读取知识库作答": "ReadKnowledgeBaseAndAnswerQuestions", "交互功能模板函数": "InteractiveFunctionTemplateFunctions", "交互功能函数模板": "InteractiveFunctionFunctionTemplates", diff --git a/tests/test_plugins.py b/tests/test_plugins.py index aeefc19..13ec259 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -48,11 +48,11 @@ if __name__ == "__main__": # for lang in ["English", "French", "Japanese", "Korean", "Russian", "Italian", "German", "Portuguese", "Arabic"]: # plugin_test(plugin='crazy_functions.批量Markdown翻译->Markdown翻译指定语言', main_input="README.md", advanced_arg={"advanced_arg": lang}) - # plugin_test(plugin='crazy_functions.知识库问答->知识库问答', main_input="./") + # plugin_test(plugin='crazy_functions.知识库文件注入->知识库文件注入', main_input="./") - # plugin_test(plugin='crazy_functions.知识库问答->读取知识库作答', main_input="What is the installation method？") + # plugin_test(plugin='crazy_functions.知识库文件注入->读取知识库作答', main_input="What is the installation method？") - # plugin_test(plugin='crazy_functions.知识库问答->读取知识库作答', main_input="远程云服务器部署？") + # plugin_test(plugin='crazy_functions.知识库文件注入->读取知识库作答', main_input="远程云服务器部署？") # plugin_test(plugin='crazy_functions.Latex输出PDF结果->Latex翻译中文并重新编译PDF', main_input="2210.03629") diff --git a/tests/test_utils.py b/tests/test_utils.py index 1fdca1e..346f58f 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -49,14 +49,14 @@ class VoidTerminal(): pass vt = VoidTerminal() -vt.get_conf = silence_stdout_fn(get_conf) -vt.set_conf = silence_stdout_fn(set_conf) -vt.set_multi_conf = silence_stdout_fn(set_multi_conf) -vt.get_plugin_handle = silence_stdout_fn(get_plugin_handle) -vt.get_plugin_default_kwargs = silence_stdout_fn(get_plugin_default_kwargs) -vt.get_chat_handle = silence_stdout_fn(get_chat_handle) -vt.get_chat_default_kwargs = silence_stdout_fn(get_chat_default_kwargs) -vt.chat_to_markdown_str = chat_to_markdown_str +vt.get_conf = (get_conf) +vt.set_conf = (set_conf) +vt.set_multi_conf = (set_multi_conf) +vt.get_plugin_handle = (get_plugin_handle) +vt.get_plugin_default_kwargs = (get_plugin_default_kwargs) +vt.get_chat_handle = (get_chat_handle) +vt.get_chat_default_kwargs = (get_chat_default_kwargs) +vt.chat_to_markdown_str = (chat_to_markdown_str) proxies, WEB_PORT, LLM_MODEL, CONCURRENT_COUNT, AUTHENTICATION, CHATBOT_HEIGHT, LAYOUT, API_KEY = \ vt.get_conf('proxies', 'WEB_PORT', 'LLM_MODEL', 'CONCURRENT_COUNT', 'AUTHENTICATION', 'CHATBOT_HEIGHT', 'LAYOUT', 'API_KEY') diff --git a/tests/test_vector_plugins.py b/tests/test_vector_plugins.py new file mode 100644 index 0000000..9b75463 --- /dev/null +++ b/tests/test_vector_plugins.py @@ -0,0 +1,17 @@ +""" +对项目中的各个插件进行测试。运行方法：直接运行 python tests/test_plugins.py +""" + + +import os, sys +def validate_path(): dir_name = os.path.dirname(__file__); root_dir_assume = os.path.abspath(dir_name + '/..'); os.chdir(root_dir_assume); sys.path.append(root_dir_assume) +validate_path() # 返回项目根路径 + +if __name__ == "__main__": + from tests.test_utils import plugin_test + + plugin_test(plugin='crazy_functions.知识库问答->知识库文件注入', main_input="./README.md") + + plugin_test(plugin='crazy_functions.知识库问答->读取知识库作答', main_input="What is the installation method？") + + plugin_test(plugin='crazy_functions.知识库问答->读取知识库作答', main_input="远程云服务器部署？") \ No newline at end of file