diff --git a/README.md b/README.md index fd31353..761e303 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ To translate this project to arbitary language with GPT, read and run [`multi_la > > 1.请注意只有 **高亮(如红色)** 标识的函数插件（按钮）才支持读取文件，部分插件位于插件区的**下拉菜单**中。另外我们以**最高优先级**欢迎和处理任何新插件的PR。 > -> 2.本项目中每个文件的功能都在自译解[`self_analysis.md`](https://github.com/binary-husky/gpt_academic/wiki/chatgpt-academic%E9%A1%B9%E7%9B%AE%E8%87%AA%E8%AF%91%E8%A7%A3%E6%8A%A5%E5%91%8A)详细说明。随着版本的迭代，您也可以随时自行点击相关函数插件，调用GPT重新生成项目的自我解析报告。常见问题汇总在[`wiki`](https://github.com/binary-husky/gpt_academic/wiki/%E5%B8%B8%E8%A7%81%E9%97%AE%E9%A2%98)当中。[安装方法](#installation)。 +> 2.本项目中每个文件的功能都在[自译解报告`self_analysis.md`](https://github.com/binary-husky/gpt_academic/wiki/GPT‐Academic项目自译解报告)详细说明。随着版本的迭代，您也可以随时自行点击相关函数插件，调用GPT重新生成项目的自我解析报告。常见问题汇总在[`wiki`](https://github.com/binary-husky/gpt_academic/wiki/%E5%B8%B8%E8%A7%81%E9%97%AE%E9%A2%98)当中。[安装方法](#installation)。 > > 3.本项目兼容并鼓励尝试国产大语言模型ChatGLM和Moss等等。支持多个api-key共存，可在配置文件中填写如`API_KEY="openai-key1,openai-key2,azure-key3,api2d-key4"`。需要临时更换`API_KEY`时，在输入区输入临时的`API_KEY`然后回车键提交后即可生效。 @@ -102,7 +102,7 @@ cd gpt_academic 在`config.py`中，配置API KEY等设置，[点击查看特殊网络环境设置方法](https://github.com/binary-husky/gpt_academic/issues/1) 。 -(P.S. 程序运行时会优先检查是否存在名为`config_private.py`的私密配置文件，并用其中的配置覆盖`config.py`的同名配置。因此，如果您能理解我们的配置读取逻辑，我们强烈建议您在`config.py`旁边创建一个名为`config_private.py`的新配置文件，并把`config.py`中的配置转移（复制）到`config_private.py`中。`config_private.py`不受git管控，可以让您的隐私信息更加安全。P.S.项目同样支持通过`环境变量`配置大多数选项，环境变量的书写格式参考`docker-compose`文件。读取优先级: `环境变量` > `config_private.py` > `config.py`) +(P.S. 程序运行时会优先检查是否存在名为`config_private.py`的私密配置文件，并用其中的配置覆盖`config.py`的同名配置。因此，如果您能理解我们的配置读取逻辑，我们强烈建议您在`config.py`旁边创建一个名为`config_private.py`的新配置文件，并把`config.py`中的配置转移（复制）到`config_private.py`中（仅复制您修改过的配置条目即可）。`config_private.py`不受git管控，可以让您的隐私信息更加安全。P.S.项目同样支持通过`环境变量`配置大多数选项，环境变量的书写格式参考`docker-compose`文件。读取优先级: `环境变量` > `config_private.py` > `config.py`) 3. 安装依赖 diff --git a/config.py b/config.py index dc9ce88..2a4eddd 100644 --- a/config.py +++ b/config.py @@ -168,10 +168,13 @@ HUGGINGFACE_ACCESS_TOKEN = "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" # 获取方法：复制以下空间https://huggingface.co/spaces/qingxu98/grobid，设为public，然后GROBID_URL = "https://(你的hf用户名如qingxu98)-(你的填写的空间名如grobid).hf.space" GROBID_URLS = [ "https://qingxu98-grobid.hf.space","https://qingxu98-grobid2.hf.space","https://qingxu98-grobid3.hf.space", - "https://shaocongma-grobid.hf.space","https://FBR123-grobid.hf.space", + "https://shaocongma-grobid.hf.space","https://FBR123-grobid.hf.space", "https://yeku-grobid.hf.space", ] +# 是否允许通过自然语言描述修改本页的配置，该功能具有一定的危险性，默认关闭 +ALLOW_RESET_CONFIG = False + """ 在线大模型配置关联关系示意图 @@ -224,4 +227,4 @@ GROBID_URLS = [ ├── PDF文档精准解析 │ └── GROBID_URLS -""" \ No newline at end of file +""" diff --git a/crazy_functional.py b/crazy_functional.py index df7c034..fcc4e52 100644 --- a/crazy_functional.py +++ b/crazy_functional.py @@ -2,7 +2,6 @@ from toolbox import HotReload # HotReload 的意思是热更新，修改函数 def get_crazy_functions(): - ###################### 第一组插件 ########################### from crazy_functions.读文章写摘要 import 读文章写摘要 from crazy_functions.生成函数注释 import 批量生成函数注释 from crazy_functions.解析项目源代码 import 解析项目本身 @@ -25,114 +24,8 @@ def get_crazy_functions(): from crazy_functions.对话历史存档 import 载入对话历史存档 from crazy_functions.对话历史存档 import 删除所有本地对话历史记录 from crazy_functions.辅助功能 import 清除缓存 - from crazy_functions.批量Markdown翻译 import Markdown英译中 - function_plugins = { - "解析整个Python项目": { - "Color": "stop", # 按钮颜色 - "Function": HotReload(解析一个Python项目) - }, - "载入对话历史存档（先上传存档或输入路径）": { - "Color": "stop", - "AsButton":False, - "Function": HotReload(载入对话历史存档) - }, - "删除所有本地对话历史记录（请谨慎操作）": { - "AsButton":False, - "Function": HotReload(删除所有本地对话历史记录) - }, - "清除所有缓存文件（请谨慎操作）": { - "Color": "stop", - "AsButton": False, # 加入下拉菜单中 - "Function": HotReload(清除缓存) - }, - "解析Jupyter Notebook文件": { - "Color": "stop", - "AsButton":False, - "Function": HotReload(解析ipynb文件), - "AdvancedArgs": True, # 调用时，唤起高级参数输入区（默认False） - "ArgsReminder": "若输入0，则不解析notebook中的Markdown块", # 高级参数输入区的显示提示 - }, - "批量总结Word文档": { - "Color": "stop", - "Function": HotReload(总结word文档) - }, - "解析整个C++项目头文件": { - "Color": "stop", # 按钮颜色 - "AsButton": False, # 加入下拉菜单中 - "Function": HotReload(解析一个C项目的头文件) - }, - "解析整个C++项目（.cpp/.hpp/.c/.h）": { - "Color": "stop", # 按钮颜色 - "AsButton": False, # 加入下拉菜单中 - "Function": HotReload(解析一个C项目) - }, - "解析整个Go项目": { - "Color": "stop", # 按钮颜色 - "AsButton": False, # 加入下拉菜单中 - "Function": HotReload(解析一个Golang项目) - }, - "解析整个Rust项目": { - "Color": "stop", # 按钮颜色 - "AsButton": False, # 加入下拉菜单中 - "Function": HotReload(解析一个Rust项目) - }, - "解析整个Java项目": { - "Color": "stop", # 按钮颜色 - "AsButton": False, # 加入下拉菜单中 - "Function": HotReload(解析一个Java项目) - }, - "解析整个前端项目（js,ts,css等）": { - "Color": "stop", # 按钮颜色 - "AsButton": False, # 加入下拉菜单中 - "Function": HotReload(解析一个前端项目) - }, - "解析整个Lua项目": { - "Color": "stop", # 按钮颜色 - "AsButton": False, # 加入下拉菜单中 - "Function": HotReload(解析一个Lua项目) - }, - "解析整个CSharp项目": { - "Color": "stop", # 按钮颜色 - "AsButton": False, # 加入下拉菜单中 - "Function": HotReload(解析一个CSharp项目) - }, - "读Tex论文写摘要": { - "Color": "stop", # 按钮颜色 - "Function": HotReload(读文章写摘要) - }, - "Markdown/Readme英译中": { - # HotReload 的意思是热更新，修改函数插件代码后，不需要重启程序，代码直接生效 - "Color": "stop", - "Function": HotReload(Markdown英译中) - }, - "批量生成函数注释": { - "Color": "stop", # 按钮颜色 - "AsButton": False, # 加入下拉菜单中 - "Function": HotReload(批量生成函数注释) - }, - "保存当前的对话": { - "Function": HotReload(对话历史存档) - }, - "[多线程Demo] 解析此项目本身（源码自译解）": { - "AsButton": False, # 加入下拉菜单中 - "Function": HotReload(解析项目本身) - }, - # "[老旧的Demo] 把本项目源代码切换成全英文": { - # # HotReload 的意思是热更新，修改函数插件代码后，不需要重启程序，代码直接生效 - # "AsButton": False, # 加入下拉菜单中 - # "Function": HotReload(全项目切换英文) - # }, - "[插件demo] 历史上的今天": { - # HotReload 的意思是热更新，修改函数插件代码后，不需要重启程序，代码直接生效 - "Function": HotReload(高阶功能模板函数) - }, - - } - ###################### 第二组插件 ########################### - # [第二组插件]: 经过充分测试 from crazy_functions.批量总结PDF文档 import 批量总结PDF文档 - # from crazy_functions.批量总结PDF文档pdfminer import 批量总结PDF文档pdfminer from crazy_functions.批量翻译PDF文档_多线程 import 批量翻译PDF文档 from crazy_functions.谷歌检索小助手 import 谷歌检索小助手 from crazy_functions.理解PDF文档内容 import 理解PDF文档内容标准文件输入 @@ -142,81 +35,185 @@ def get_crazy_functions(): from crazy_functions.Latex全文翻译 import Latex英译中 from crazy_functions.批量Markdown翻译 import Markdown中译英 - function_plugins.update({ - "批量翻译PDF文档（多线程）": { + + function_plugins = { + "解析整个Python项目": { + "Color": "stop", + "AsButton": True, + "Info": "解析一个Python项目的所有源文件(.py) | 输入参数为路径", + "Function": HotReload(解析一个Python项目) + }, + "载入对话历史存档（先上传存档或输入路径）": { + "Color": "stop", + "AsButton": False, + "Info": "载入对话历史存档 | 输入参数为路径", + "Function": HotReload(载入对话历史存档) + }, + "删除所有本地对话历史记录（谨慎操作）": { + "AsButton": False, + "Info": "删除所有本地对话历史记录，谨慎操作 | 不需要输入参数", + "Function": HotReload(删除所有本地对话历史记录) + }, + "清除所有缓存文件（谨慎操作）": { + "Color": "stop", + "AsButton": False, # 加入下拉菜单中 + "Info": "清除所有缓存文件，谨慎操作 | 不需要输入参数", + "Function": HotReload(清除缓存) + }, + "批量总结Word文档": { + "Color": "stop", + "AsButton": True, + "Info": "批量总结word文档 | 输入参数为路径", + "Function": HotReload(总结word文档) + }, + "解析整个C++项目头文件": { + "Color": "stop", + "AsButton": False, # 加入下拉菜单中 + "Info": "解析一个C++项目的所有头文件(.h/.hpp) | 输入参数为路径", + "Function": HotReload(解析一个C项目的头文件) + }, + "解析整个C++项目（.cpp/.hpp/.c/.h）": { + "Color": "stop", + "AsButton": False, # 加入下拉菜单中 + "Info": "解析一个C++项目的所有源文件（.cpp/.hpp/.c/.h）| 输入参数为路径", + "Function": HotReload(解析一个C项目) + }, + "解析整个Go项目": { + "Color": "stop", + "AsButton": False, # 加入下拉菜单中 + "Info": "解析一个Go项目的所有源文件 | 输入参数为路径", + "Function": HotReload(解析一个Golang项目) + }, + "解析整个Rust项目": { + "Color": "stop", + "AsButton": False, # 加入下拉菜单中 + "Info": "解析一个Rust项目的所有源文件 | 输入参数为路径", + "Function": HotReload(解析一个Rust项目) + }, + "解析整个Java项目": { + "Color": "stop", + "AsButton": False, # 加入下拉菜单中 + "Info": "解析一个Java项目的所有源文件 | 输入参数为路径", + "Function": HotReload(解析一个Java项目) + }, + "解析整个前端项目（js,ts,css等）": { + "Color": "stop", + "AsButton": False, # 加入下拉菜单中 + "Info": "解析一个前端项目的所有源文件（js,ts,css等） | 输入参数为路径", + "Function": HotReload(解析一个前端项目) + }, + "解析整个Lua项目": { + "Color": "stop", + "AsButton": False, # 加入下拉菜单中 + "Info": "解析一个Lua项目的所有源文件 | 输入参数为路径", + "Function": HotReload(解析一个Lua项目) + }, + "解析整个CSharp项目": { + "Color": "stop", + "AsButton": False, # 加入下拉菜单中 + "Info": "解析一个CSharp项目的所有源文件 | 输入参数为路径", + "Function": HotReload(解析一个CSharp项目) + }, + "解析Jupyter Notebook文件": { + "Color": "stop", + "AsButton": False, + "Info": "解析Jupyter Notebook文件 | 输入参数为路径", + "Function": HotReload(解析ipynb文件), + "AdvancedArgs": True, # 调用时，唤起高级参数输入区（默认False） + "ArgsReminder": "若输入0，则不解析notebook中的Markdown块", # 高级参数输入区的显示提示 + }, + "读Tex论文写摘要": { + "Color": "stop", + "AsButton": True, + "Function": HotReload(读文章写摘要) + }, + "翻译README或.MD": { + "Color": "stop", + "AsButton": True, + "Info": "将Markdown翻译为中文 | 输入参数为路径或URL", + "Function": HotReload(Markdown英译中) + }, + "翻译Markdown或README（支持Github链接）": { + "Color": "stop", + "AsButton": False, + "Function": HotReload(Markdown英译中) + }, + "批量生成函数注释": { + "Color": "stop", + "AsButton": False, # 加入下拉菜单中 + "Function": HotReload(批量生成函数注释) + }, + "保存当前的对话": { + "AsButton": True, + "Info": "保存当前的对话 | 不需要输入参数", + "Function": HotReload(对话历史存档) + }, + "[多线程Demo] 解析此项目本身（源码自译解）": { + "AsButton": False, # 加入下拉菜单中 + "Function": HotReload(解析项目本身) + }, + "[插件demo] 历史上的今天": { + "AsButton": True, + "Function": HotReload(高阶功能模板函数) + }, + "精准翻译PDF论文": { "Color": "stop", "AsButton": True, # 加入下拉菜单中 "Function": HotReload(批量翻译PDF文档) }, "询问多个GPT模型": { - "Color": "stop", # 按钮颜色 + "Color": "stop", + "AsButton": True, "Function": HotReload(同时问询) }, - "[测试功能] 批量总结PDF文档": { + "批量总结PDF文档": { "Color": "stop", "AsButton": False, # 加入下拉菜单中 - # HotReload 的意思是热更新，修改函数插件代码后，不需要重启程序，代码直接生效 "Function": HotReload(批量总结PDF文档) }, - # "[测试功能] 批量总结PDF文档pdfminer": { - # "Color": "stop", - # "AsButton": False, # 加入下拉菜单中 - # "Function": HotReload(批量总结PDF文档pdfminer) - # }, "谷歌学术检索助手（输入谷歌学术搜索页url）": { "Color": "stop", "AsButton": False, # 加入下拉菜单中 "Function": HotReload(谷歌检索小助手) }, "理解PDF文档内容 （模仿ChatPDF）": { - # HotReload 的意思是热更新，修改函数插件代码后，不需要重启程序，代码直接生效 "Color": "stop", "AsButton": False, # 加入下拉菜单中 "Function": HotReload(理解PDF文档内容标准文件输入) }, "英文Latex项目全文润色（输入路径或上传压缩包）": { - # HotReload 的意思是热更新，修改函数插件代码后，不需要重启程序，代码直接生效 "Color": "stop", "AsButton": False, # 加入下拉菜单中 "Function": HotReload(Latex英文润色) }, "英文Latex项目全文纠错（输入路径或上传压缩包）": { - # HotReload 的意思是热更新，修改函数插件代码后，不需要重启程序，代码直接生效 "Color": "stop", "AsButton": False, # 加入下拉菜单中 "Function": HotReload(Latex英文纠错) }, "中文Latex项目全文润色（输入路径或上传压缩包）": { - # HotReload 的意思是热更新，修改函数插件代码后，不需要重启程序，代码直接生效 "Color": "stop", "AsButton": False, # 加入下拉菜单中 "Function": HotReload(Latex中文润色) }, "Latex项目全文中译英（输入路径或上传压缩包）": { - # HotReload 的意思是热更新，修改函数插件代码后，不需要重启程序，代码直接生效 "Color": "stop", "AsButton": False, # 加入下拉菜单中 "Function": HotReload(Latex中译英) }, "Latex项目全文英译中（输入路径或上传压缩包）": { - # HotReload 的意思是热更新，修改函数插件代码后，不需要重启程序，代码直接生效 "Color": "stop", "AsButton": False, # 加入下拉菜单中 "Function": HotReload(Latex英译中) }, "批量Markdown中译英（输入路径或上传压缩包）": { - # HotReload 的意思是热更新，修改函数插件代码后，不需要重启程序，代码直接生效 "Color": "stop", "AsButton": False, # 加入下拉菜单中 "Function": HotReload(Markdown中译英) }, + } - - }) - - ###################### 第三组插件 ########################### - # [第三组插件]: 尚未充分测试的函数插件 - + # -=--=- 尚未充分测试的实验性插件 & 需要额外依赖的插件 -=--=- try: from crazy_functions.下载arxiv论文翻译摘要 import 下载arxiv论文并翻译摘要 function_plugins.update({ @@ -255,8 +252,8 @@ def get_crazy_functions(): "解析项目源代码（手动指定和筛选源代码文件类型）": { "Color": "stop", "AsButton": False, - "AdvancedArgs": True, # 调用时，唤起高级参数输入区（默认False） - "ArgsReminder": "输入时用逗号隔开, *代表通配符, 加了^代表不匹配; 不输入代表全部匹配。例如: \"*.c, ^*.cpp, config.toml, ^*.toml\"", # 高级参数输入区的显示提示 + "AdvancedArgs": True, # 调用时，唤起高级参数输入区（默认False） + "ArgsReminder": "输入时用逗号隔开, *代表通配符, 加了^代表不匹配; 不输入代表全部匹配。例如: \"*.c, ^*.cpp, config.toml, ^*.toml\"", # 高级参数输入区的显示提示 "Function": HotReload(解析任意code项目) }, }) @@ -269,8 +266,8 @@ def get_crazy_functions(): "询问多个GPT模型（手动指定询问哪些模型）": { "Color": "stop", "AsButton": False, - "AdvancedArgs": True, # 调用时，唤起高级参数输入区（默认False） - "ArgsReminder": "支持任意数量的llm接口，用&符号分隔。例如chatglm&gpt-3.5-turbo&api2d-gpt-4", # 高级参数输入区的显示提示 + "AdvancedArgs": True, # 调用时，唤起高级参数输入区（默认False） + "ArgsReminder": "支持任意数量的llm接口，用&符号分隔。例如chatglm&gpt-3.5-turbo&api2d-gpt-4", # 高级参数输入区的显示提示 "Function": HotReload(同时问询_指定模型) }, }) @@ -283,8 +280,9 @@ def get_crazy_functions(): "图片生成（先切换模型到openai或api2d）": { "Color": "stop", "AsButton": False, - "AdvancedArgs": True, # 调用时，唤起高级参数输入区（默认False） - "ArgsReminder": "在这里输入分辨率, 如256x256（默认）", # 高级参数输入区的显示提示 + "AdvancedArgs": True, # 调用时，唤起高级参数输入区（默认False） + "ArgsReminder": "在这里输入分辨率, 如256x256（默认）", # 高级参数输入区的显示提示 + "Info": "图片生成 | 输入参数字符串，提供图像的内容", "Function": HotReload(图片生成) }, }) @@ -299,6 +297,7 @@ def get_crazy_functions(): "AsButton": False, "AdvancedArgs": True, "ArgsReminder": "调用openai api 使用whisper-1模型, 目前支持的格式:mp4, m4a, wav, mpga, mpeg, mp3。此处可以输入解析提示，例如：解析为简体中文（默认）。", + "Info": "批量总结音频或视频 | 输入参数为路径", "Function": HotReload(总结音视频) } }) @@ -358,7 +357,7 @@ def get_crazy_functions(): }) except: print('Load function plugin failed') - + try: from crazy_functions.交互功能函数模板 import 交互功能模板函数 function_plugins.update({ @@ -371,20 +370,6 @@ def get_crazy_functions(): except: print('Load function plugin failed') - # try: - # from crazy_functions.chatglm微调工具 import 微调数据集生成 - # function_plugins.update({ - # "黑盒模型学习: 微调数据集生成 (先上传数据集)": { - # "Color": "stop", - # "AsButton": False, - # "AdvancedArgs": True, - # "ArgsReminder": "针对数据集输入（如 绿帽子*深蓝色衬衫*黑色运动裤）给出指令，例如您可以将以下命令复制到下方: --llm_to_learn=azure-gpt-3.5 --prompt_prefix='根据下面的服装类型提示，想象一个穿着者，对这个人外貌、身处的环境、内心世界、过去经历进行描写。要求：100字以内，用第二人称。' --system_prompt=''", - # "Function": HotReload(微调数据集生成) - # } - # }) - # except: - # print('Load function plugin failed') - try: from crazy_functions.Latex输出PDF结果 import Latex英文纠错加PDF对比 function_plugins.update({ @@ -402,9 +387,10 @@ def get_crazy_functions(): "Color": "stop", "AsButton": False, "AdvancedArgs": True, - "ArgsReminder": - "如果有必要, 请在此处给出自定义翻译命令, 解决部分词汇翻译不准确的问题。 "+ - "例如当单词'agent'翻译不准确时, 请尝试把以下指令复制到高级参数区: " + 'If the term "agent" is used in this section, it should be translated to "智能体". ', + "ArgsReminder": + "如果有必要, 请在此处给出自定义翻译命令, 解决部分词汇翻译不准确的问题。 " + + "例如当单词'agent'翻译不准确时, 请尝试把以下指令复制到高级参数区: " + + 'If the term "agent" is used in this section, it should be translated to "智能体". ', "Function": HotReload(Latex翻译中文并重新编译PDF) } }) @@ -413,16 +399,16 @@ def get_crazy_functions(): "Color": "stop", "AsButton": False, "AdvancedArgs": True, - "ArgsReminder": - "如果有必要, 请在此处给出自定义翻译命令, 解决部分词汇翻译不准确的问题。 "+ - "例如当单词'agent'翻译不准确时, 请尝试把以下指令复制到高级参数区: " + 'If the term "agent" is used in this section, it should be translated to "智能体". ', + "ArgsReminder": + "如果有必要, 请在此处给出自定义翻译命令, 解决部分词汇翻译不准确的问题。 " + + "例如当单词'agent'翻译不准确时, 请尝试把以下指令复制到高级参数区: " + + 'If the term "agent" is used in this section, it should be translated to "智能体". ', "Function": HotReload(Latex翻译中文并重新编译PDF) } }) except: print('Load function plugin failed') - try: from toolbox import get_conf ENABLE_AUDIO, = get_conf('ENABLE_AUDIO') @@ -437,19 +423,30 @@ def get_crazy_functions(): }) except: print('Load function plugin failed') - + + try: + from crazy_functions.虚空终端 import 自动终端 + function_plugins.update({ + "自动终端": { + "Color": "stop", + "AsButton": False, + "Function": HotReload(自动终端) + } + }) + except: + print('Load function plugin failed') + # try: - # from crazy_functions.虚空终端 import 终端 + # from crazy_functions.chatglm微调工具 import 微调数据集生成 # function_plugins.update({ - # "超级终端": { + # "黑盒模型学习: 微调数据集生成 (先上传数据集)": { # "Color": "stop", # "AsButton": False, - # # "AdvancedArgs": True, - # # "ArgsReminder": "", - # "Function": HotReload(终端) + # "AdvancedArgs": True, + # "ArgsReminder": "针对数据集输入（如 绿帽子*深蓝色衬衫*黑色运动裤）给出指令，例如您可以将以下命令复制到下方: --llm_to_learn=azure-gpt-3.5 --prompt_prefix='根据下面的服装类型提示，想象一个穿着者，对这个人外貌、身处的环境、内心世界、过去经历进行描写。要求：100字以内，用第二人称。' --system_prompt=''", + # "Function": HotReload(微调数据集生成) # } # }) # except: # print('Load function plugin failed') - return function_plugins diff --git a/crazy_functions/json_fns/pydantic_io.py b/crazy_functions/json_fns/pydantic_io.py new file mode 100644 index 0000000..db92412 --- /dev/null +++ b/crazy_functions/json_fns/pydantic_io.py @@ -0,0 +1,100 @@ +""" +https://github.com/langchain-ai/langchain/blob/master/docs/extras/modules/model_io/output_parsers/pydantic.ipynb + +Example 1. + +# Define your desired data structure. +class Joke(BaseModel): + setup: str = Field(description="question to set up a joke") + punchline: str = Field(description="answer to resolve the joke") + + # You can add custom validation logic easily with Pydantic. + @validator("setup") + def question_ends_with_question_mark(cls, field): + if field[-1] != "?": + raise ValueError("Badly formed question!") + return field + + +Example 2. + +# Here's another example, but with a compound typed field. +class Actor(BaseModel): + name: str = Field(description="name of an actor") + film_names: List[str] = Field(description="list of names of films they starred in") +""" + +import json, re, logging + + +PYDANTIC_FORMAT_INSTRUCTIONS = """The output should be formatted as a JSON instance that conforms to the JSON schema below. + +As an example, for the schema {{"properties": {{"foo": {{"title": "Foo", "description": "a list of strings", "type": "array", "items": {{"type": "string"}}}}}}, "required": ["foo"]}} +the object {{"foo": ["bar", "baz"]}} is a well-formatted instance of the schema. The object {{"properties": {{"foo": ["bar", "baz"]}}}} is not well-formatted. + +Here is the output schema: +``` +{schema} +```""" + +class GptJsonIO(): + + def __init__(self, schema): + self.pydantic_object = schema + self.format_instructions = self.generate_format_instructions() + + def generate_format_instructions(self): + schema = self.pydantic_object.schema() + + # Remove extraneous fields. + reduced_schema = schema + if "title" in reduced_schema: + del reduced_schema["title"] + if "type" in reduced_schema: + del reduced_schema["type"] + # Ensure json in context is well-formed with double quotes. + schema_str = json.dumps(reduced_schema) + + return PYDANTIC_FORMAT_INSTRUCTIONS.format(schema=schema_str) + + def generate_output(self, text): + # Greedy search for 1st json candidate. + match = re.search( + r"\{.*\}", text.strip(), re.MULTILINE | re.IGNORECASE | re.DOTALL + ) + json_str = "" + if match: json_str = match.group() + json_object = json.loads(json_str, strict=False) + final_object = self.pydantic_object.parse_obj(json_object) + return final_object + + def generate_repair_prompt(self, broken_json, error): + prompt = "Fix a broken json string.\n\n" + \ + "(1) The broken json string need to fix is: \n\n" + \ + "```" + "\n" + \ + broken_json + "\n" + \ + "```" + "\n\n" + \ + "(2) The error message is: \n\n" + \ + error + "\n\n" + \ + "Now, fix this json string. \n\n" + return prompt + + def generate_output_auto_repair(self, response, gpt_gen_fn): + """ + response: string containing canidate json + gpt_gen_fn: gpt_gen_fn(inputs, sys_prompt) + """ + try: + result = self.generate_output(response) + except Exception as e: + try: + logging.info(f'Repairing json：{response}') + repair_prompt = self.generate_repair_prompt(broken_json = response, error=repr(e)) + result = self.generate_output(gpt_gen_fn(repair_prompt, self.format_instructions)) + logging.info('Repaire json success.') + except Exception as e: + # 没辙了，放弃治疗 + logging.info('Repaire json fail.') + raise RuntimeError('Cannot repair json.', str(e)) + return result + diff --git a/crazy_functions/test_project/cpp/cppipc/buffer.cpp b/crazy_functions/test_project/cpp/cppipc/buffer.cpp deleted file mode 100644 index 084b815..0000000 --- a/crazy_functions/test_project/cpp/cppipc/buffer.cpp +++ /dev/null @@ -1,87 +0,0 @@ -#include "libipc/buffer.h" -#include "libipc/utility/pimpl.h" - -#include - -namespace ipc { - -bool operator==(buffer const & b1, buffer const & b2) { - return (b1.size() == b2.size()) && (std::memcmp(b1.data(), b2.data(), b1.size()) == 0); -} - -bool operator!=(buffer const & b1, buffer const & b2) { - return !(b1 == b2); -} - -class buffer::buffer_ : public pimpl { -public: - void* p_; - std::size_t s_; - void* a_; - buffer::destructor_t d_; - - buffer_(void* p, std::size_t s, buffer::destructor_t d, void* a) - : p_(p), s_(s), a_(a), d_(d) { - } - - ~buffer_() { - if (d_ == nullptr) return; - d_((a_ == nullptr) ? p_ : a_, s_); - } -}; - -buffer::buffer() - : buffer(nullptr, 0, nullptr, nullptr) { -} - -buffer::buffer(void* p, std::size_t s, destructor_t d) - : p_(p_->make(p, s, d, nullptr)) { -} - -buffer::buffer(void* p, std::size_t s, destructor_t d, void* additional) - : p_(p_->make(p, s, d, additional)) { -} - -buffer::buffer(void* p, std::size_t s) - : buffer(p, s, nullptr) { -} - -buffer::buffer(char const & c) - : buffer(const_cast(&c), 1) { -} - -buffer::buffer(buffer&& rhs) - : buffer() { - swap(rhs); -} - -buffer::~buffer() { - p_->clear(); -} - -void buffer::swap(buffer& rhs) { - std::swap(p_, rhs.p_); -} - -buffer& buffer::operator=(buffer rhs) { - swap(rhs); - return *this; -} - -bool buffer::empty() const noexcept { - return (impl(p_)->p_ == nullptr) || (impl(p_)->s_ == 0); -} - -void* buffer::data() noexcept { - return impl(p_)->p_; -} - -void const * buffer::data() const noexcept { - return impl(p_)->p_; -} - -std::size_t buffer::size() const noexcept { - return impl(p_)->s_; -} - -} // namespace ipc diff --git a/crazy_functions/test_project/cpp/cppipc/ipc.cpp b/crazy_functions/test_project/cpp/cppipc/ipc.cpp deleted file mode 100644 index 4dc71c0..0000000 --- a/crazy_functions/test_project/cpp/cppipc/ipc.cpp +++ /dev/null @@ -1,701 +0,0 @@ - -#include -#include -#include -#include // std::pair, std::move, std::forward -#include -#include // aligned_storage_t -#include -#include -#include -#include - -#include "libipc/ipc.h" -#include "libipc/def.h" -#include "libipc/shm.h" -#include "libipc/pool_alloc.h" -#include "libipc/queue.h" -#include "libipc/policy.h" -#include "libipc/rw_lock.h" -#include "libipc/waiter.h" - -#include "libipc/utility/log.h" -#include "libipc/utility/id_pool.h" -#include "libipc/utility/scope_guard.h" -#include "libipc/utility/utility.h" - -#include "libipc/memory/resource.h" -#include "libipc/platform/detail.h" -#include "libipc/circ/elem_array.h" - -namespace { - -using msg_id_t = std::uint32_t; -using acc_t = std::atomic; - -template -struct msg_t; - -template -struct msg_t<0, AlignSize> { - msg_id_t cc_id_; - msg_id_t id_; - std::int32_t remain_; - bool storage_; -}; - -template -struct msg_t : msg_t<0, AlignSize> { - std::aligned_storage_t data_ {}; - - msg_t() = default; - msg_t(msg_id_t cc_id, msg_id_t id, std::int32_t remain, void const * data, std::size_t size) - : msg_t<0, AlignSize> {cc_id, id, remain, (data == nullptr) || (size == 0)} { - if (this->storage_) { - if (data != nullptr) { - // copy storage-id - *reinterpret_cast(&data_) = - *static_cast(data); - } - } - else std::memcpy(&data_, data, size); - } -}; - -template -ipc::buff_t make_cache(T& data, std::size_t size) { - auto ptr = ipc::mem::alloc(size); - std::memcpy(ptr, &data, (ipc::detail::min)(sizeof(data), size)); - return { ptr, size, ipc::mem::free }; -} - -struct cache_t { - std::size_t fill_; - ipc::buff_t buff_; - - cache_t(std::size_t f, ipc::buff_t && b) - : fill_(f), buff_(std::move(b)) - {} - - void append(void const * data, std::size_t size) { - if (fill_ >= buff_.size() || data == nullptr || size == 0) return; - auto new_fill = (ipc::detail::min)(fill_ + size, buff_.size()); - std::memcpy(static_cast(buff_.data()) + fill_, data, new_fill - fill_); - fill_ = new_fill; - } -}; - -auto cc_acc() { - static ipc::shm::handle acc_h("__CA_CONN__", sizeof(acc_t)); - return static_cast(acc_h.get()); -} - -IPC_CONSTEXPR_ std::size_t align_chunk_size(std::size_t size) noexcept { - return (((size - 1) / ipc::large_msg_align) + 1) * ipc::large_msg_align; -} - -IPC_CONSTEXPR_ std::size_t calc_chunk_size(std::size_t size) noexcept { - return ipc::make_align(alignof(std::max_align_t), align_chunk_size( - ipc::make_align(alignof(std::max_align_t), sizeof(std::atomic)) + size)); -} - -struct chunk_t { - std::atomic &conns() noexcept { - return *reinterpret_cast *>(this); - } - - void *data() noexcept { - return reinterpret_cast(this) - + ipc::make_align(alignof(std::max_align_t), sizeof(std::atomic)); - } -}; - -struct chunk_info_t { - ipc::id_pool<> pool_; - ipc::spin_lock lock_; - - IPC_CONSTEXPR_ static std::size_t chunks_mem_size(std::size_t chunk_size) noexcept { - return ipc::id_pool<>::max_count * chunk_size; - } - - ipc::byte_t *chunks_mem() noexcept { - return reinterpret_cast(this + 1); - } - - chunk_t *at(std::size_t chunk_size, ipc::storage_id_t id) noexcept { - if (id < 0) return nullptr; - return reinterpret_cast(chunks_mem() + (chunk_size * id)); - } -}; - -auto& chunk_storages() { - class chunk_handle_t { - ipc::shm::handle handle_; - - public: - chunk_info_t *get_info(std::size_t chunk_size) { - if (!handle_.valid() && - !handle_.acquire( ("__CHUNK_INFO__" + ipc::to_string(chunk_size)).c_str(), - sizeof(chunk_info_t) + chunk_info_t::chunks_mem_size(chunk_size) )) { - ipc::error("[chunk_storages] chunk_shm.id_info_.acquire failed: chunk_size = %zd\n", chunk_size); - return nullptr; - } - auto info = static_cast(handle_.get()); - if (info == nullptr) { - ipc::error("[chunk_storages] chunk_shm.id_info_.get failed: chunk_size = %zd\n", chunk_size); - return nullptr; - } - return info; - } - }; - static ipc::map chunk_hs; - return chunk_hs; -} - -chunk_info_t *chunk_storage_info(std::size_t chunk_size) { - auto &storages = chunk_storages(); - std::decay_t::iterator it; - { - static ipc::rw_lock lock; - IPC_UNUSED_ std::shared_lock guard {lock}; - if ((it = storages.find(chunk_size)) == storages.end()) { - using chunk_handle_t = std::decay_t::value_type::second_type; - guard.unlock(); - IPC_UNUSED_ std::lock_guard guard {lock}; - it = storages.emplace(chunk_size, chunk_handle_t{}).first; - } - } - return it->second.get_info(chunk_size); -} - -std::pair acquire_storage(std::size_t size, ipc::circ::cc_t conns) { - std::size_t chunk_size = calc_chunk_size(size); - auto info = chunk_storage_info(chunk_size); - if (info == nullptr) return {}; - - info->lock_.lock(); - info->pool_.prepare(); - // got an unique id - auto id = info->pool_.acquire(); - info->lock_.unlock(); - - auto chunk = info->at(chunk_size, id); - if (chunk == nullptr) return {}; - chunk->conns().store(conns, std::memory_order_relaxed); - return { id, chunk->data() }; -} - -void *find_storage(ipc::storage_id_t id, std::size_t size) { - if (id < 0) { - ipc::error("[find_storage] id is invalid: id = %ld, size = %zd\n", (long)id, size); - return nullptr; - } - std::size_t chunk_size = calc_chunk_size(size); - auto info = chunk_storage_info(chunk_size); - if (info == nullptr) return nullptr; - return info->at(chunk_size, id)->data(); -} - -void release_storage(ipc::storage_id_t id, std::size_t size) { - if (id < 0) { - ipc::error("[release_storage] id is invalid: id = %ld, size = %zd\n", (long)id, size); - return; - } - std::size_t chunk_size = calc_chunk_size(size); - auto info = chunk_storage_info(chunk_size); - if (info == nullptr) return; - info->lock_.lock(); - info->pool_.release(id); - info->lock_.unlock(); -} - -template -bool sub_rc(ipc::wr, - std::atomic &/*conns*/, ipc::circ::cc_t /*curr_conns*/, ipc::circ::cc_t /*conn_id*/) noexcept { - return true; -} - -template -bool sub_rc(ipc::wr, - std::atomic &conns, ipc::circ::cc_t curr_conns, ipc::circ::cc_t conn_id) noexcept { - auto last_conns = curr_conns & ~conn_id; - for (unsigned k = 0;;) { - auto chunk_conns = conns.load(std::memory_order_acquire); - if (conns.compare_exchange_weak(chunk_conns, chunk_conns & last_conns, std::memory_order_release)) { - return (chunk_conns & last_conns) == 0; - } - ipc::yield(k); - } -} - -template -void recycle_storage(ipc::storage_id_t id, std::size_t size, ipc::circ::cc_t curr_conns, ipc::circ::cc_t conn_id) { - if (id < 0) { - ipc::error("[recycle_storage] id is invalid: id = %ld, size = %zd\n", (long)id, size); - return; - } - std::size_t chunk_size = calc_chunk_size(size); - auto info = chunk_storage_info(chunk_size); - if (info == nullptr) return; - - auto chunk = info->at(chunk_size, id); - if (chunk == nullptr) return; - - if (!sub_rc(Flag{}, chunk->conns(), curr_conns, conn_id)) { - return; - } - info->lock_.lock(); - info->pool_.release(id); - info->lock_.unlock(); -} - -template -bool clear_message(void* p) { - auto msg = static_cast(p); - if (msg->storage_) { - std::int32_t r_size = static_cast(ipc::data_length) + msg->remain_; - if (r_size <= 0) { - ipc::error("[clear_message] invalid msg size: %d\n", (int)r_size); - return true; - } - release_storage( - *reinterpret_cast(&msg->data_), - static_cast(r_size)); - } - return true; -} - -struct conn_info_head { - - ipc::string name_; - msg_id_t cc_id_; // connection-info id - ipc::detail::waiter cc_waiter_, wt_waiter_, rd_waiter_; - ipc::shm::handle acc_h_; - - conn_info_head(char const * name) - : name_ {name} - , cc_id_ {(cc_acc() == nullptr) ? 0 : cc_acc()->fetch_add(1, std::memory_order_relaxed)} - , cc_waiter_{("__CC_CONN__" + name_).c_str()} - , wt_waiter_{("__WT_CONN__" + name_).c_str()} - , rd_waiter_{("__RD_CONN__" + name_).c_str()} - , acc_h_ {("__AC_CONN__" + name_).c_str(), sizeof(acc_t)} { - } - - void quit_waiting() { - cc_waiter_.quit_waiting(); - wt_waiter_.quit_waiting(); - rd_waiter_.quit_waiting(); - } - - auto acc() { - return static_cast(acc_h_.get()); - } - - auto& recv_cache() { - thread_local ipc::unordered_map tls; - return tls; - } -}; - -template -bool wait_for(W& waiter, F&& pred, std::uint64_t tm) { - if (tm == 0) return !pred(); - for (unsigned k = 0; pred();) { - bool ret = true; - ipc::sleep(k, [&k, &ret, &waiter, &pred, tm] { - ret = waiter.wait_if(std::forward(pred), tm); - k = 0; - }); - if (!ret) return false; // timeout or fail - if (k == 0) break; // k has been reset - } - return true; -} - -template -struct queue_generator { - - using queue_t = ipc::queue, Policy>; - - struct conn_info_t : conn_info_head { - queue_t que_; - - conn_info_t(char const * name) - : conn_info_head{name} - , que_{("__QU_CONN__" + - ipc::to_string(DataSize) + "__" + - ipc::to_string(AlignSize) + "__" + name).c_str()} { - } - - void disconnect_receiver() { - bool dis = que_.disconnect(); - this->quit_waiting(); - if (dis) { - this->recv_cache().clear(); - } - } - }; -}; - -template -struct detail_impl { - -using policy_t = Policy; -using flag_t = typename policy_t::flag_t; -using queue_t = typename queue_generator::queue_t; -using conn_info_t = typename queue_generator::conn_info_t; - -constexpr static conn_info_t* info_of(ipc::handle_t h) noexcept { - return static_cast(h); -} - -constexpr static queue_t* queue_of(ipc::handle_t h) noexcept { - return (info_of(h) == nullptr) ? nullptr : &(info_of(h)->que_); -} - -/* API implementations */ - -static void disconnect(ipc::handle_t h) { - auto que = queue_of(h); - if (que == nullptr) { - return; - } - que->shut_sending(); - assert(info_of(h) != nullptr); - info_of(h)->disconnect_receiver(); -} - -static bool reconnect(ipc::handle_t * ph, bool start_to_recv) { - assert(ph != nullptr); - assert(*ph != nullptr); - auto que = queue_of(*ph); - if (que == nullptr) { - return false; - } - if (start_to_recv) { - que->shut_sending(); - if (que->connect()) { // wouldn't connect twice - info_of(*ph)->cc_waiter_.broadcast(); - return true; - } - return false; - } - // start_to_recv == false - if (que->connected()) { - info_of(*ph)->disconnect_receiver(); - } - return que->ready_sending(); -} - -static bool connect(ipc::handle_t * ph, char const * name, bool start_to_recv) { - assert(ph != nullptr); - if (*ph == nullptr) { - *ph = ipc::mem::alloc(name); - } - return reconnect(ph, start_to_recv); -} - -static void destroy(ipc::handle_t h) { - disconnect(h); - ipc::mem::free(info_of(h)); -} - -static std::size_t recv_count(ipc::handle_t h) noexcept { - auto que = queue_of(h); - if (que == nullptr) { - return ipc::invalid_value; - } - return que->conn_count(); -} - -static bool wait_for_recv(ipc::handle_t h, std::size_t r_count, std::uint64_t tm) { - auto que = queue_of(h); - if (que == nullptr) { - return false; - } - return wait_for(info_of(h)->cc_waiter_, [que, r_count] { - return que->conn_count() < r_count; - }, tm); -} - -template -static bool send(F&& gen_push, ipc::handle_t h, void const * data, std::size_t size) { - if (data == nullptr || size == 0) { - ipc::error("fail: send(%p, %zd)\n", data, size); - return false; - } - auto que = queue_of(h); - if (que == nullptr) { - ipc::error("fail: send, queue_of(h) == nullptr\n"); - return false; - } - if (que->elems() == nullptr) { - ipc::error("fail: send, queue_of(h)->elems() == nullptr\n"); - return false; - } - if (!que->ready_sending()) { - ipc::error("fail: send, que->ready_sending() == false\n"); - return false; - } - ipc::circ::cc_t conns = que->elems()->connections(std::memory_order_relaxed); - if (conns == 0) { - ipc::error("fail: send, there is no receiver on this connection.\n"); - return false; - } - // calc a new message id - auto acc = info_of(h)->acc(); - if (acc == nullptr) { - ipc::error("fail: send, info_of(h)->acc() == nullptr\n"); - return false; - } - auto msg_id = acc->fetch_add(1, std::memory_order_relaxed); - auto try_push = std::forward(gen_push)(info_of(h), que, msg_id); - if (size > ipc::large_msg_limit) { - auto dat = acquire_storage(size, conns); - void * buf = dat.second; - if (buf != nullptr) { - std::memcpy(buf, data, size); - return try_push(static_cast(size) - - static_cast(ipc::data_length), &(dat.first), 0); - } - // try using message fragment - //ipc::log("fail: shm::handle for big message. msg_id: %zd, size: %zd\n", msg_id, size); - } - // push message fragment - std::int32_t offset = 0; - for (std::int32_t i = 0; i < static_cast(size / ipc::data_length); ++i, offset += ipc::data_length) { - if (!try_push(static_cast(size) - offset - static_cast(ipc::data_length), - static_cast(data) + offset, ipc::data_length)) { - return false; - } - } - // if remain > 0, this is the last message fragment - std::int32_t remain = static_cast(size) - offset; - if (remain > 0) { - if (!try_push(remain - static_cast(ipc::data_length), - static_cast(data) + offset, - static_cast(remain))) { - return false; - } - } - return true; -} - -static bool send(ipc::handle_t h, void const * data, std::size_t size, std::uint64_t tm) { - return send([tm](auto info, auto que, auto msg_id) { - return [tm, info, que, msg_id](std::int32_t remain, void const * data, std::size_t size) { - if (!wait_for(info->wt_waiter_, [&] { - return !que->push( - [](void*) { return true; }, - info->cc_id_, msg_id, remain, data, size); - }, tm)) { - ipc::log("force_push: msg_id = %zd, remain = %d, size = %zd\n", msg_id, remain, size); - if (!que->force_push( - clear_message, - info->cc_id_, msg_id, remain, data, size)) { - return false; - } - } - info->rd_waiter_.broadcast(); - return true; - }; - }, h, data, size); -} - -static bool try_send(ipc::handle_t h, void const * data, std::size_t size, std::uint64_t tm) { - return send([tm](auto info, auto que, auto msg_id) { - return [tm, info, que, msg_id](std::int32_t remain, void const * data, std::size_t size) { - if (!wait_for(info->wt_waiter_, [&] { - return !que->push( - [](void*) { return true; }, - info->cc_id_, msg_id, remain, data, size); - }, tm)) { - return false; - } - info->rd_waiter_.broadcast(); - return true; - }; - }, h, data, size); -} - -static ipc::buff_t recv(ipc::handle_t h, std::uint64_t tm) { - auto que = queue_of(h); - if (que == nullptr) { - ipc::error("fail: recv, queue_of(h) == nullptr\n"); - return {}; - } - if (!que->connected()) { - // hasn't connected yet, just return. - return {}; - } - auto& rc = info_of(h)->recv_cache(); - for (;;) { - // pop a new message - typename queue_t::value_t msg; - if (!wait_for(info_of(h)->rd_waiter_, [que, &msg] { - return !que->pop(msg); - }, tm)) { - // pop failed, just return. - return {}; - } - info_of(h)->wt_waiter_.broadcast(); - if ((info_of(h)->acc() != nullptr) && (msg.cc_id_ == info_of(h)->cc_id_)) { - continue; // ignore message to self - } - // msg.remain_ may minus & abs(msg.remain_) < data_length - std::int32_t r_size = static_cast(ipc::data_length) + msg.remain_; - if (r_size <= 0) { - ipc::error("fail: recv, r_size = %d\n", (int)r_size); - return {}; - } - std::size_t msg_size = static_cast(r_size); - // large message - if (msg.storage_) { - ipc::storage_id_t buf_id = *reinterpret_cast(&msg.data_); - void* buf = find_storage(buf_id, msg_size); - if (buf != nullptr) { - struct recycle_t { - ipc::storage_id_t storage_id; - ipc::circ::cc_t curr_conns; - ipc::circ::cc_t conn_id; - } *r_info = ipc::mem::alloc(recycle_t{ - buf_id, que->elems()->connections(std::memory_order_relaxed), que->connected_id() - }); - if (r_info == nullptr) { - ipc::log("fail: ipc::mem::alloc.\n"); - return ipc::buff_t{buf, msg_size}; // no recycle - } else { - return ipc::buff_t{buf, msg_size, [](void* p_info, std::size_t size) { - auto r_info = static_cast(p_info); - IPC_UNUSED_ auto finally = ipc::guard([r_info] { - ipc::mem::free(r_info); - }); - recycle_storage(r_info->storage_id, size, r_info->curr_conns, r_info->conn_id); - }, r_info}; - } - } else { - ipc::log("fail: shm::handle for large message. msg_id: %zd, buf_id: %zd, size: %zd\n", msg.id_, buf_id, msg_size); - continue; - } - } - // find cache with msg.id_ - auto cac_it = rc.find(msg.id_); - if (cac_it == rc.end()) { - if (msg_size <= ipc::data_length) { - return make_cache(msg.data_, msg_size); - } - // gc - if (rc.size() > 1024) { - std::vector need_del; - for (auto const & pair : rc) { - auto cmp = std::minmax(msg.id_, pair.first); - if (cmp.second - cmp.first > 8192) { - need_del.push_back(pair.first); - } - } - for (auto id : need_del) rc.erase(id); - } - // cache the first message fragment - rc.emplace(msg.id_, cache_t { ipc::data_length, make_cache(msg.data_, msg_size) }); - } - // has cached before this message - else { - auto& cac = cac_it->second; - // this is the last message fragment - if (msg.remain_ <= 0) { - cac.append(&(msg.data_), msg_size); - // finish this message, erase it from cache - auto buff = std::move(cac.buff_); - rc.erase(cac_it); - return buff; - } - // there are remain datas after this message - cac.append(&(msg.data_), ipc::data_length); - } - } -} - -static ipc::buff_t try_recv(ipc::handle_t h) { - return recv(h, 0); -} - -}; // detail_impl - -template -using policy_t = ipc::policy::choose; - -} // internal-linkage - -namespace ipc { - -template -ipc::handle_t chan_impl::inited() { - ipc::detail::waiter::init(); - return nullptr; -} - -template -bool chan_impl::connect(ipc::handle_t * ph, char const * name, unsigned mode) { - return detail_impl>::connect(ph, name, mode & receiver); -} - -template -bool chan_impl::reconnect(ipc::handle_t * ph, unsigned mode) { - return detail_impl>::reconnect(ph, mode & receiver); -} - -template -void chan_impl::disconnect(ipc::handle_t h) { - detail_impl>::disconnect(h); -} - -template -void chan_impl::destroy(ipc::handle_t h) { - detail_impl>::destroy(h); -} - -template -char const * chan_impl::name(ipc::handle_t h) { - auto info = detail_impl>::info_of(h); - return (info == nullptr) ? nullptr : info->name_.c_str(); -} - -template -std::size_t chan_impl::recv_count(ipc::handle_t h) { - return detail_impl>::recv_count(h); -} - -template -bool chan_impl::wait_for_recv(ipc::handle_t h, std::size_t r_count, std::uint64_t tm) { - return detail_impl>::wait_for_recv(h, r_count, tm); -} - -template -bool chan_impl::send(ipc::handle_t h, void const * data, std::size_t size, std::uint64_t tm) { - return detail_impl>::send(h, data, size, tm); -} - -template -buff_t chan_impl::recv(ipc::handle_t h, std::uint64_t tm) { - return detail_impl>::recv(h, tm); -} - -template -bool chan_impl::try_send(ipc::handle_t h, void const * data, std::size_t size, std::uint64_t tm) { - return detail_impl>::try_send(h, data, size, tm); -} - -template -buff_t chan_impl::try_recv(ipc::handle_t h) { - return detail_impl>::try_recv(h); -} - -template struct chan_impl>; -// template struct chan_impl>; // TBD -// template struct chan_impl>; // TBD -template struct chan_impl>; -template struct chan_impl>; - -} // namespace ipc diff --git a/crazy_functions/test_project/cpp/cppipc/policy.h b/crazy_functions/test_project/cpp/cppipc/policy.h deleted file mode 100644 index 8959607..0000000 --- a/crazy_functions/test_project/cpp/cppipc/policy.h +++ /dev/null @@ -1,25 +0,0 @@ -#pragma once - -#include - -#include "libipc/def.h" -#include "libipc/prod_cons.h" - -#include "libipc/circ/elem_array.h" - -namespace ipc { -namespace policy { - -template