176 lines
6.4 KiB
Python
176 lines
6.4 KiB
Python
from toolbox import update_ui, trimmed_format_exc
|
|
from toolbox import CatchException, report_execption, write_results_to_file, zip_folder
|
|
import os
|
|
import re
|
|
|
|
|
|
|
|
def merge_tex_files(project_foler, main_file):
|
|
# Get the directory of the main tex file
|
|
|
|
# re.findall(r"\\input\{(.*?)\}", main_file, re.M)
|
|
for s in reversed([q for q in re.finditer(r"\\input\{(.*?)\}", main_file, re.M)]):
|
|
f = s.group(1)
|
|
fp = os.path.join(project_foler, f)
|
|
with open(fp, 'r', encoding='utf-8', errors='replace') as fx:
|
|
c = fx.read()
|
|
c = merge_tex_files(project_foler, c)
|
|
main_file = main_file[:s.span()[0]] + c + main_file[s.span()[1]:]
|
|
|
|
return main_file
|
|
|
|
class LinkTable():
|
|
def __init__(self, string, preserve=True) -> None:
|
|
self.string = string
|
|
self.preserve = preserve
|
|
self.next = None
|
|
|
|
class LatexPaperSplit():
|
|
def __init__(self) -> None:
|
|
self.root = None
|
|
|
|
def merge_result(self, arr):
|
|
def remove_special_chars(s):
|
|
s.replace('%', 'Percent')
|
|
return s
|
|
|
|
result_string = ""
|
|
|
|
node = self.root
|
|
p = 0
|
|
while True:
|
|
if node.preserve:
|
|
result_string += node.string
|
|
else:
|
|
result_string += remove_special_chars(arr[p])
|
|
p += 1
|
|
node = node.next
|
|
if node is None: break
|
|
|
|
return result_string
|
|
|
|
def split(self, txt):
|
|
# def replace_with_hash()
|
|
root = LinkTable(txt, False)
|
|
def split_worker(root, pattern, flags=0):
|
|
lt = root
|
|
cnt = 0
|
|
while True:
|
|
if not lt.preserve:
|
|
while True:
|
|
res = re.search(pattern, lt.string, flags)
|
|
if not res: break
|
|
before = res.string[:res.span()[0]]
|
|
this = res.group(0)
|
|
# core = res.group(1)
|
|
after = res.string[res.span()[1]:]
|
|
|
|
lt.string = before
|
|
tmp = lt.next
|
|
# ======
|
|
if after.startswith('\n'):
|
|
# move \n
|
|
this = this + '\n'
|
|
after = after[1:]
|
|
mid = LinkTable(this, True)
|
|
lt.next = mid
|
|
# ======
|
|
aft = LinkTable(after, False)
|
|
mid.next = aft
|
|
aft.next = tmp
|
|
# ======
|
|
lt = aft
|
|
lt = lt.next
|
|
cnt += 1
|
|
print(cnt)
|
|
if lt is None: break
|
|
|
|
# root 是链表的头
|
|
print('正在分解Latex源文件')
|
|
split_worker(root, r"(.*?)\\maketitle", re.DOTALL)
|
|
split_worker(root, r"\\section\{(.*?)\}")
|
|
split_worker(root, r"\\subsection\{(.*?)\}")
|
|
split_worker(root, r"\\subsubsection\{(.*?)\}")
|
|
split_worker(root, r"\\begin\{figure\}(.*?)\\end\{figure\}", re.DOTALL)
|
|
split_worker(root, r"\\begin\{figure\*\}(.*?)\\end\{figure\*\}", re.DOTALL)
|
|
split_worker(root, r"\\begin\{table\}(.*?)\\end\{table\}", re.DOTALL)
|
|
split_worker(root, r"\\begin\{table\*\}(.*?)\\end\{table\*\}", re.DOTALL)
|
|
split_worker(root, r"\\item ")
|
|
split_worker(root, r"\\begin\{(.*?)\}")
|
|
split_worker(root, r"\\end\{(.*?)\}")
|
|
|
|
res = []
|
|
node = root
|
|
while True:
|
|
res.append((node.string, node.preserve))
|
|
if len(node.string.strip('\n').strip(''))==0: node.preserve = True
|
|
if len(node.string.strip('\n').strip(''))<50: node.preserve = True
|
|
node = node.next
|
|
if node is None: break
|
|
|
|
print('======================================')
|
|
res_to_t = []
|
|
node = root
|
|
while True:
|
|
if not node.preserve:
|
|
print(node.string)
|
|
res_to_t.append(node.string)
|
|
print('======================================')
|
|
node = node.next
|
|
if node is None: break
|
|
print('======================================')
|
|
self.root = root
|
|
self.sp = res_to_t
|
|
return self.sp
|
|
|
|
class LatexPaperFileGroup():
|
|
def __init__(self):
|
|
self.file_paths = []
|
|
self.file_contents = []
|
|
self.sp_file_contents = []
|
|
self.sp_file_index = []
|
|
self.sp_file_tag = []
|
|
|
|
# count_token
|
|
from request_llm.bridge_all import model_info
|
|
enc = model_info["gpt-3.5-turbo"]['tokenizer']
|
|
def get_token_num(txt): return len(enc.encode(txt, disallowed_special=()))
|
|
self.get_token_num = get_token_num
|
|
|
|
def run_file_split(self, max_token_limit=1900):
|
|
"""
|
|
将长文本分离开来
|
|
"""
|
|
for index, file_content in enumerate(self.file_contents):
|
|
if self.get_token_num(file_content) < max_token_limit:
|
|
self.sp_file_contents.append(file_content)
|
|
self.sp_file_index.append(index)
|
|
self.sp_file_tag.append(self.file_paths[index])
|
|
else:
|
|
from .crazy_utils import breakdown_txt_to_satisfy_token_limit_for_pdf
|
|
segments = breakdown_txt_to_satisfy_token_limit_for_pdf(file_content, self.get_token_num, max_token_limit)
|
|
for j, segment in enumerate(segments):
|
|
self.sp_file_contents.append(segment)
|
|
self.sp_file_index.append(index)
|
|
self.sp_file_tag.append(self.file_paths[index] + f".part-{j}.tex")
|
|
print('Segmentation: done')
|
|
|
|
def merge_result(self):
|
|
self.file_result = ["" for _ in range(len(self.file_paths))]
|
|
for r, k in zip(self.sp_file_result, self.sp_file_index):
|
|
self.file_result[k] += r
|
|
|
|
def write_result(self):
|
|
manifest = []
|
|
for path, res in zip(self.file_paths, self.file_result):
|
|
with open(path + '.polish.tex', 'w', encoding='utf8') as f:
|
|
manifest.append(path + '.polish.tex')
|
|
f.write(res)
|
|
return manifest
|
|
|
|
def zip_result(self):
|
|
import os, time
|
|
folder = os.path.dirname(self.file_paths[0])
|
|
t = time.strftime("%Y-%m-%d-%H-%M-%S", time.localtime())
|
|
zip_folder(folder, './gpt_log/', f'{t}-polished.zip')
|