允许加入ChatGLM微调模型

config.py

@@ -74,6 +74,10 @@ AVAIL_LLM_MODELS = ["gpt-3.5-turbo-16k", "gpt-3.5-turbo", "azure-gpt-3.5", "api2
 # P.S. 其他可用的模型还包括 ["gpt-3.5-turbo-0613", "gpt-3.5-turbo-16k-0613", "newbing-free", "jittorllms_rwkv", "jittorllms_pangualpha", "jittorllms_llama"]
 
 
+# ChatGLM(2) Finetune Model Path （如果使用ChatGLM2微调模型，需要把"chatglmft"加入AVAIL_LLM_MODELS中）
+ChatGLM_PTUNING_CHECKPOINT = "" # 例如"/home/hmp/ChatGLM2-6B/ptuning/output/6b-pt-128-1e-2/checkpoint-100"
+
+
 # 本地LLM模型如ChatGLM的执行方式 CPU/GPU
 LOCAL_MODEL_DEVICE = "cpu" # 可选 "cuda"
 

crazy_functions/chatglm微调工具.py

@@ -69,3 +69,57 @@ def 微调数据集生成(txt, llm_kwargs, plugin_kwargs, chatbot, history, syst
 
     promote_file_to_downloadzone(txt+'.generated.json', rename_file='generated.json', chatbot=chatbot)
     return
+
+
+
+def 启动微调(arguments):
+    """
+    txt             输入栏用户输入的文本，例如需要翻译的一段话，再例如一个包含了待处理文件的路径
+    llm_kwargs      gpt模型参数，如温度和top_p等，一般原样传递下去就行
+    plugin_kwargs   插件模型的参数
+    chatbot         聊天显示框的句柄，用于显示给用户
+    history         聊天历史，前情提要
+    system_prompt   给gpt的静默提醒
+    web_port        当前软件运行的端口号
+    """
+    history = []    # 清空历史，以免输入溢出
+    import subprocess
+    PRE_SEQ_LEN = 128
+    LR = 2e-2
+    NUM_GPUS = 1
+    JSON_FILE = 't_code.json'
+    tune_work_path = '/home/hmp/ChatGLM2-6B/ptuning'
+
+    command = f"torchrun --standalone --nnodes=1 --nproc-per-node={NUM_GPUS} main.py \
+        --do_train \
+        --train_file AdvertiseGen/{JSON_FILE} \
+        --validation_file AdvertiseGen/{JSON_FILE} \
+        --preprocessing_num_workers 20 \
+        --prompt_column content \
+        --response_column summary \
+        --overwrite_cache \
+        --model_name_or_path THUDM/chatglm2-6b \
+        --output_dir output/clothgen-chatglm2-6b-pt-{PRE_SEQ_LEN}-{LR} \
+        --overwrite_output_dir \
+        --max_source_length 256 \
+        --max_target_length 256 \
+        --per_device_train_batch_size 1 \
+        --per_device_eval_batch_size 1 \
+        --gradient_accumulation_steps 16 \
+        --predict_with_generate \
+        --max_steps 100 \
+        --logging_steps 10 \
+        --save_steps 20 \
+        --learning_rate {LR} \
+        --pre_seq_len {PRE_SEQ_LEN} \
+        --quantization_bit 4"
+
+    process = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=tune_work_path)
+    try:
+        stdout, stderr = process.communicate(timeout=3600*5)
+    except subprocess.TimeoutExpired:
+        process.kill()
+        stdout, stderr = process.communicate()
+        print("Process timed out!")
+        return False
+    return

request_llm/bridge_all.py

@@ -269,6 +269,24 @@ if "newbing" in AVAIL_LLM_MODELS:   # same with newbing-free
         })
     except:
         print(trimmed_format_exc())
+if "chatglmft" in AVAIL_LLM_MODELS:   # same with newbing-free
+    try:
+        from .bridge_chatglmft import predict_no_ui_long_connection as chatglmft_noui
+        from .bridge_chatglmft import predict as chatglmft_ui
+        # claude
+        model_info.update({
+            "chatglmft": {
+                "fn_with_ui": chatglmft_ui,
+                "fn_without_ui": chatglmft_noui,
+                "endpoint": None,
+                "max_token": 4096,
+                "tokenizer": tokenizer_gpt35,
+                "token_cnt": get_token_num_gpt35,
+            }
+        })
+    except:
+        print(trimmed_format_exc())
+
 
 def LLM_CATCH_EXCEPTION(f):
     """
@@ -372,6 +390,6 @@ def predict(inputs, llm_kwargs, *args, **kwargs):
     additional_fn代表点击的哪个按钮，按钮见functional.py
     """
 
-    method = model_info[llm_kwargs['llm_model']]["fn_with_ui"]
+    method = model_info[llm_kwargs['llm_model']]["fn_with_ui"]  # 如果这里报错，检查config中的AVAIL_LLM_MODELS选项
     yield from method(inputs, llm_kwargs, *args, **kwargs)
 

request_llm/bridge_chatglmft.py

@@ -0,0 +1,210 @@
+
+from transformers import AutoModel, AutoTokenizer
+import time
+import os
+import json
+import threading
+import importlib
+from toolbox import update_ui, get_conf
+from multiprocessing import Process, Pipe
+
+load_message = "ChatGLMFT尚未加载，加载需要一段时间。注意，取决于`config.py`的配置，ChatGLMFT消耗大量的内存（CPU）或显存（GPU），也许会导致低配计算机卡死 ……"
+
+def string_to_options(arguments):
+    import argparse
+    import shlex
+    # Create an argparse.ArgumentParser instance
+    parser = argparse.ArgumentParser()
+    # Add command-line arguments
+    parser.add_argument("--llm_to_learn", type=str, help="LLM model to learn", default="gpt-3.5-turbo")
+    parser.add_argument("--prompt_prefix", type=str, help="Prompt prefix", default='')
+    parser.add_argument("--system_prompt", type=str, help="System prompt", default='')
+    parser.add_argument("--batch", type=int, help="System prompt", default=50)
+    # Parse the arguments
+    args = parser.parse_args(shlex.split(arguments))
+    return args
+
+
+#################################################################################
+class GetGLMFTHandle(Process):
+    def __init__(self):
+        super().__init__(daemon=True)
+        self.parent, self.child = Pipe()
+        self.chatglmft_model = None
+        self.chatglmft_tokenizer = None
+        self.info = ""
+        self.success = True
+        self.check_dependency()
+        self.start()
+        self.threadLock = threading.Lock()
+        
+    def check_dependency(self):
+        try:
+            import sentencepiece
+            self.info = "依赖检测通过"
+            self.success = True
+        except:
+            self.info = "缺少ChatGLMFT的依赖，如果要使用ChatGLMFT，除了基础的pip依赖以外，您还需要运行`pip install -r request_llm/requirements_chatglm.txt`安装ChatGLM的依赖。"
+            self.success = False
+
+    def ready(self):
+        return self.chatglmft_model is not None
+
+    def run(self):
+        # 子进程执行
+        # 第一次运行，加载参数
+        retry = 0
+        while True:
+            try:
+                if self.chatglmft_model is None:
+                    from transformers import AutoConfig
+                    import torch
+                    # conf = 'request_llm/current_ptune_model.json'
+                    # if not os.path.exists(conf): raise RuntimeError('找不到微调模型信息')
+                    # with open(conf, 'r', encoding='utf8') as f:
+                    #     model_args = json.loads(f.read())
+                    ChatGLM_PTUNING_CHECKPOINT, = get_conf('ChatGLM_PTUNING_CHECKPOINT')
+                    assert os.path.exists(ChatGLM_PTUNING_CHECKPOINT), "找不到微调模型检查点"
+                    conf = os.path.join(ChatGLM_PTUNING_CHECKPOINT, "config.json")
+                    with open(conf, 'r', encoding='utf8') as f:
+                        model_args = json.loads(f.read())
+                    if 'model_name_or_path' not in model_args:
+                        model_args['model_name_or_path'] = model_args['_name_or_path']
+                    self.chatglmft_tokenizer = AutoTokenizer.from_pretrained(
+                        model_args['model_name_or_path'], trust_remote_code=True)
+                    config = AutoConfig.from_pretrained(
+                        model_args['model_name_or_path'], trust_remote_code=True)
+
+                    config.pre_seq_len = model_args['pre_seq_len']
+                    config.prefix_projection = model_args['prefix_projection']
+
+                    print(f"Loading prefix_encoder weight from {ChatGLM_PTUNING_CHECKPOINT}")
+                    model = AutoModel.from_pretrained(model_args['model_name_or_path'], config=config, trust_remote_code=True)
+                    prefix_state_dict = torch.load(os.path.join(ChatGLM_PTUNING_CHECKPOINT, "pytorch_model.bin"))
+                    new_prefix_state_dict = {}
+                    for k, v in prefix_state_dict.items():
+                        if k.startswith("transformer.prefix_encoder."):
+                            new_prefix_state_dict[k[len("transformer.prefix_encoder."):]] = v
+                    model.transformer.prefix_encoder.load_state_dict(new_prefix_state_dict)
+
+                    if model_args['quantization_bit'] is not None:
+                        print(f"Quantized to {model_args['quantization_bit']} bit")
+                        model = model.quantize(model_args['quantization_bit'])
+                    model = model.cuda()
+                    if model_args['pre_seq_len'] is not None:
+                        # P-tuning v2
+                        model.transformer.prefix_encoder.float()
+                    self.chatglmft_model = model.eval()
+
+                    break
+                else:
+                    break
+            except Exception as e:
+                retry += 1
+                if retry > 3: 
+                    self.child.send('[Local Message] Call ChatGLMFT fail 不能正常加载ChatGLMFT的参数。')
+                    raise RuntimeError("不能正常加载ChatGLMFT的参数！")
+
+        while True:
+            # 进入任务等待状态
+            kwargs = self.child.recv()
+            # 收到消息，开始请求
+            try:
+                for response, history in self.chatglmft_model.stream_chat(self.chatglmft_tokenizer, **kwargs):
+                    self.child.send(response)
+                    # # 中途接收可能的终止指令（如果有的话）
+                    # if self.child.poll(): 
+                    #     command = self.child.recv()
+                    #     if command == '[Terminate]': break
+            except:
+                from toolbox import trimmed_format_exc
+                self.child.send('[Local Message] Call ChatGLMFT fail.' + '\n```\n' + trimmed_format_exc() + '\n```\n')
+            # 请求处理结束，开始下一个循环
+            self.child.send('[Finish]')
+
+    def stream_chat(self, **kwargs):
+        # 主进程执行
+        self.threadLock.acquire()
+        self.parent.send(kwargs)
+        while True:
+            res = self.parent.recv()
+            if res != '[Finish]':
+                yield res
+            else:
+                break
+        self.threadLock.release()
+    
+global glmft_handle
+glmft_handle = None
+#################################################################################
+def predict_no_ui_long_connection(inputs, llm_kwargs, history=[], sys_prompt="", observe_window=[], console_slience=False):
+    """
+        多线程方法
+        函数的说明请见 request_llm/bridge_all.py
+    """
+    global glmft_handle
+    if glmft_handle is None:
+        glmft_handle = GetGLMFTHandle()
+        if len(observe_window) >= 1: observe_window[0] = load_message + "\n\n" + glmft_handle.info
+        if not glmft_handle.success: 
+            error = glmft_handle.info
+            glmft_handle = None
+            raise RuntimeError(error)
+
+    # chatglmft 没有 sys_prompt 接口，因此把prompt加入 history
+    history_feedin = []
+    history_feedin.append(["What can I do?", sys_prompt])
+    for i in range(len(history)//2):
+        history_feedin.append([history[2*i], history[2*i+1]] )
+
+    watch_dog_patience = 5 # 看门狗 (watchdog) 的耐心, 设置5秒即可
+    response = ""
+    for response in glmft_handle.stream_chat(query=inputs, history=history_feedin, max_length=llm_kwargs['max_length'], top_p=llm_kwargs['top_p'], temperature=llm_kwargs['temperature']):
+        if len(observe_window) >= 1:  observe_window[0] = response
+        if len(observe_window) >= 2:  
+            if (time.time()-observe_window[1]) > watch_dog_patience:
+                raise RuntimeError("程序终止。")
+    return response
+
+
+
+def predict(inputs, llm_kwargs, plugin_kwargs, chatbot, history=[], system_prompt='', stream = True, additional_fn=None):
+    """
+        单线程方法
+        函数的说明请见 request_llm/bridge_all.py
+    """
+    chatbot.append((inputs, ""))
+
+    global glmft_handle
+    if glmft_handle is None:
+        glmft_handle = GetGLMFTHandle()
+        chatbot[-1] = (inputs, load_message + "\n\n" + glmft_handle.info)
+        yield from update_ui(chatbot=chatbot, history=[])
+        if not glmft_handle.success: 
+            glmft_handle = None
+            return
+
+    if additional_fn is not None:
+        import core_functional
+        importlib.reload(core_functional)    # 热更新prompt
+        core_functional = core_functional.get_core_functions()
+        if "PreProcess" in core_functional[additional_fn]: inputs = core_functional[additional_fn]["PreProcess"](inputs)  # 获取预处理函数（如果有的话）
+        inputs = core_functional[additional_fn]["Prefix"] + inputs + core_functional[additional_fn]["Suffix"]
+
+    # 处理历史信息
+    history_feedin = []
+    history_feedin.append(["What can I do?", system_prompt] )
+    for i in range(len(history)//2):
+        history_feedin.append([history[2*i], history[2*i+1]] )
+
+    # 开始接收chatglmft的回复
+    response = "[Local Message]: 等待ChatGLMFT响应中 ..."
+    for response in glmft_handle.stream_chat(query=inputs, history=history_feedin, max_length=llm_kwargs['max_length'], top_p=llm_kwargs['top_p'], temperature=llm_kwargs['temperature']):
+        chatbot[-1] = (inputs, response)
+        yield from update_ui(chatbot=chatbot, history=history)
+
+    # 总结输出
+    if response == "[Local Message]: 等待ChatGLMFT响应中 ...":
+        response = "[Local Message]: ChatGLMFT响应异常 ..."
+    history.extend([inputs, response])
+    yield from update_ui(chatbot=chatbot, history=history)