From 4d70b3786f8be564afa799acef371ea6e5b0c33d Mon Sep 17 00:00:00 2001
From: binary-husky <qingxu.fu@outlook.com>
Date: Mon, 7 Aug 2023 01:24:41 +0800
Subject: [PATCH] interface with qwen

---
 request_llm/bridge_chatglmonnx.py |  2 +-
 request_llm/bridge_internlm.py    |  2 +-
 request_llm/bridge_qwen.py        | 71 +++++++++++++++++++++++++++++++
 request_llm/requirements_qwen.txt |  2 +
 request_llm/test_llms.py          |  3 +-
 5 files changed, 77 insertions(+), 3 deletions(-)
 create mode 100644 request_llm/bridge_qwen.py
 create mode 100644 request_llm/requirements_qwen.txt

diff --git a/request_llm/bridge_chatglmonnx.py b/request_llm/bridge_chatglmonnx.py
index fbe64b4..594bcca 100644
--- a/request_llm/bridge_chatglmonnx.py
+++ b/request_llm/bridge_chatglmonnx.py
@@ -1,5 +1,5 @@
 model_name = "ChatGLM-ONNX"
-cmd_to_install = "`pip install request_llm/requirements_chatglm_onnx.txt`"
+cmd_to_install = "`pip install -r request_llm/requirements_chatglm_onnx.txt`"
 
 
 from transformers import AutoModel, AutoTokenizer
diff --git a/request_llm/bridge_internlm.py b/request_llm/bridge_internlm.py
index 804edc8..0ec65b6 100644
--- a/request_llm/bridge_internlm.py
+++ b/request_llm/bridge_internlm.py
@@ -1,5 +1,5 @@
 model_name = "InternLM"
-cmd_to_install = "`pip install request_llm/requirements_chatglm.txt`"
+cmd_to_install = "`pip install -r request_llm/requirements_chatglm.txt`"
 
 from transformers import AutoModel, AutoTokenizer
 import time
diff --git a/request_llm/bridge_qwen.py b/request_llm/bridge_qwen.py
new file mode 100644
index 0000000..3ca36ab
--- /dev/null
+++ b/request_llm/bridge_qwen.py
@@ -0,0 +1,71 @@
+model_name = "Qwen"
+cmd_to_install = "`pip install -r request_llm/requirements_qwen.txt`"
+
+
+from transformers import AutoModel, AutoTokenizer
+import time
+import threading
+import importlib
+from toolbox import update_ui, get_conf
+from multiprocessing import Process, Pipe
+from .local_llm_class import LocalLLMHandle, get_local_llm_predict_fns, SingletonLocalLLM
+
+
+
+# ------------------------------------------------------------------------------------------------------------------------
+# 🔌💻 Local Model
+# ------------------------------------------------------------------------------------------------------------------------
+@SingletonLocalLLM
+class GetONNXGLMHandle(LocalLLMHandle):
+
+    def load_model_info(self):
+        # 🏃‍♂️🏃‍♂️🏃‍♂️ 子进程执行
+        self.model_name = model_name
+        self.cmd_to_install = cmd_to_install
+
+    def load_model_and_tokenizer(self):
+        # 🏃‍♂️🏃‍♂️🏃‍♂️ 子进程执行
+        import os, glob
+        import os
+        import platform
+        from modelscope import AutoModelForCausalLM, AutoTokenizer, GenerationConfig
+
+        model_id = 'qwen/Qwen-7B-Chat'
+        revision = 'v1.0.1'
+        tokenizer = AutoTokenizer.from_pretrained(model_id, revision=revision, trust_remote_code=True)
+        # use fp16
+        model = AutoModelForCausalLM.from_pretrained(model_id, device_map="auto", revision=revision, 
+                                                    trust_remote_code=True, fp16=True).eval()
+        model.generation_config = GenerationConfig.from_pretrained(model_id,
+                                                                trust_remote_code=True)  # 可指定不同的生成长度、top_p等相关超参
+        self._model = model
+
+        return self._model, None
+
+    def llm_stream_generator(self, **kwargs):
+        # 🏃‍♂️🏃‍♂️🏃‍♂️ 子进程执行
+        def adaptor(kwargs):
+            query = kwargs['query']
+            max_length = kwargs['max_length']
+            top_p = kwargs['top_p']
+            temperature = kwargs['temperature']
+            history = kwargs['history']
+            return query, max_length, top_p, temperature, history
+
+        query, max_length, top_p, temperature, history = adaptor(kwargs)
+
+        prompt = chat_template(history, query)
+        for response in model.chat(tokenizer, query, history=history, stream=True):
+            yield response
+        
+    def try_to_import_special_deps(self, **kwargs):
+        # import something that will raise error if the user does not install requirement_*.txt
+        # 🏃‍♂️🏃‍♂️🏃‍♂️ 主进程执行
+        # from modelscope import AutoModelForCausalLM, AutoTokenizer, GenerationConfig
+        pass
+
+
+# ------------------------------------------------------------------------------------------------------------------------
+# 🔌💻 GPT-Academic Interface
+# ------------------------------------------------------------------------------------------------------------------------
+predict_no_ui_long_connection, predict = get_local_llm_predict_fns(GetONNXGLMHandle, model_name)
\ No newline at end of file
diff --git a/request_llm/requirements_qwen.txt b/request_llm/requirements_qwen.txt
new file mode 100644
index 0000000..3d7d62a
--- /dev/null
+++ b/request_llm/requirements_qwen.txt
@@ -0,0 +1,2 @@
+modelscope
+transformers_stream_generator
\ No newline at end of file
diff --git a/request_llm/test_llms.py b/request_llm/test_llms.py
index 2127046..ce52589 100644
--- a/request_llm/test_llms.py
+++ b/request_llm/test_llms.py
@@ -15,7 +15,8 @@ if __name__ == "__main__":
     # from request_llm.bridge_jittorllms_pangualpha import predict_no_ui_long_connection
     # from request_llm.bridge_jittorllms_llama import predict_no_ui_long_connection
     # from request_llm.bridge_claude import predict_no_ui_long_connection
-    from request_llm.bridge_internlm import predict_no_ui_long_connection
+    # from request_llm.bridge_internlm import predict_no_ui_long_connection
+    from request_llm.bridge_qwen import predict_no_ui_long_connection
 
     llm_kwargs = {
         'max_length': 512,