chunzhi/chatgpt_academic
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

GROBID服务代理访问支持

Browse Source
This commit is contained in:
binary-husky 2023-09-27 15:40:55 +08:00
parent 9cb05e5724
commit 0844b6e9cf
2 changed files with 7 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
config.py
View File

@ -184,7 +184,7 @@ PATH_PRIVATE_UPLOAD = "private_upload"
# 日志文件夹的位置，请勿修改 # 日志文件夹的位置，请勿修改
PATH_LOGGING = "gpt_log" PATH_LOGGING = "gpt_log"
# 除了连接OpenAI之外，还有哪些场合允许使用代理，请勿修改 # 除了连接OpenAI之外，还有哪些场合允许使用代理，请勿修改
WHEN_TO_USE_PROXY = ["Download_LLM", "Download_Gradio_Theme"] WHEN_TO_USE_PROXY = ["Download_LLM", "Download_Gradio_Theme", "Connect_Grobid"]
""" """

9
crazy_functions/pdf_fns/parse_pdf.py
View File

@ -2,6 +2,8 @@ from functools import lru_cache
from toolbox import gen_time_str from toolbox import gen_time_str
from toolbox import promote_file_to_downloadzone from toolbox import promote_file_to_downloadzone
from toolbox import write_history_to_file, promote_file_to_downloadzone from toolbox import write_history_to_file, promote_file_to_downloadzone
from toolbox import get_conf
from toolbox import ProxyNetworkActivate
from colorful import * from colorful import *
import requests import requests
import random import random
@ -12,13 +14,13 @@ import math
class GROBID_OFFLINE_EXCEPTION(Exception): pass class GROBID_OFFLINE_EXCEPTION(Exception): pass
def get_avail_grobid_url(): def get_avail_grobid_url():
from toolbox import get_conf
GROBID_URLS, = get_conf('GROBID_URLS') GROBID_URLS, = get_conf('GROBID_URLS')
if len(GROBID_URLS) == 0: return None if len(GROBID_URLS) == 0: return None
try: try:
_grobid_url = random.choice(GROBID_URLS) # 随机负载均衡 _grobid_url = random.choice(GROBID_URLS) # 随机负载均衡
if _grobid_url.endswith('/'): _grobid_url = _grobid_url.rstrip('/') if _grobid_url.endswith('/'): _grobid_url = _grobid_url.rstrip('/')
res = requests.get(_grobid_url+'/api/isalive') with ProxyNetworkActivate('Connect_Grobid'):
res = requests.get(_grobid_url+'/api/isalive')
if res.text=='true': return _grobid_url if res.text=='true': return _grobid_url
else: return None else: return None
except: except:
@ -29,7 +31,8 @@ def parse_pdf(pdf_path, grobid_url):
import scipdf # pip install scipdf_parser import scipdf # pip install scipdf_parser
if grobid_url.endswith('/'): grobid_url = grobid_url.rstrip('/') if grobid_url.endswith('/'): grobid_url = grobid_url.rstrip('/')
try: try:
article_dict = scipdf.parse_pdf_to_dict(pdf_path, grobid_url=grobid_url) with ProxyNetworkActivate('Connect_Grobid'):
article_dict = scipdf.parse_pdf_to_dict(pdf_path, grobid_url=grobid_url)
except GROBID_OFFLINE_EXCEPTION: except GROBID_OFFLINE_EXCEPTION:
raise GROBID_OFFLINE_EXCEPTION("GROBID服务不可用，请修改config中的GROBID_URL，可修改成本地GROBID服务。") raise GROBID_OFFLINE_EXCEPTION("GROBID服务不可用，请修改config中的GROBID_URL，可修改成本地GROBID服务。")
except: except: