fix missing get_token_num method

2024-02-12 15:58:20 +08:00 · 2024-02-12 15:58:20 +08:00 · b9b1e12dc9
commit b9b1e12dc9
parent 8814026ec3
1 changed files with 7 additions and 1 deletions
--- a/crazy_functions/解析JupyterNotebook.py
+++ b/crazy_functions/解析JupyterNotebook.py
@ -12,6 +12,12 @@ class PaperFileGroup():
        self.sp_file_index = []
        self.sp_file_tag = []

+        # count_token
+        from request_llms.bridge_all import model_info
+        enc = model_info["gpt-3.5-turbo"]['tokenizer']
+        def get_token_num(txt): return len(enc.encode(txt, disallowed_special=()))
+        self.get_token_num = get_token_num
+
    def run_file_split(self, max_token_limit=1900):
        """
        将长文本分离开来