From c37c49dd5141fdc37fe5222d4d0252a706daf9dd Mon Sep 17 00:00:00 2001 From: Your Name Date: Wed, 19 Apr 2023 14:58:39 +0800 Subject: [PATCH] =?UTF-8?q?lazy=20load=20+decode=20=E6=96=B9=E6=B3=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- request_llm/bridge_all.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/request_llm/bridge_all.py b/request_llm/bridge_all.py index f416ad6..f1f4ee1 100644 --- a/request_llm/bridge_all.py +++ b/request_llm/bridge_all.py @@ -38,7 +38,11 @@ class LazyloadTiktoken(object): def encode(self, *args, **kwargs): encoder = self.get_encoder(self.model) return encoder.encode(*args, **kwargs) - + + def decode(self, *args, **kwargs): + encoder = self.get_encoder(self.model) + return encoder.decode(*args, **kwargs) + tokenizer_gpt35 = LazyloadTiktoken("gpt-3.5-turbo") tokenizer_gpt4 = LazyloadTiktoken("gpt-4") get_token_num_gpt35 = lambda txt: len(tokenizer_gpt35.encode(txt, disallowed_special=()))