From aba871342f984fa0f2f50ccf9178f0cf65b6117a Mon Sep 17 00:00:00 2001 From: Menghuan1918 Date: Wed, 3 Jan 2024 19:49:17 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E5=88=86=E5=89=B2=E5=87=BD?= =?UTF-8?q?=E6=95=B0=E4=B8=AD=E4=BD=BF=E7=94=A8=E7=9A=84=E5=8F=98=E9=87=8F?= =?UTF-8?q?=E9=94=99=E8=AF=AF=20(#1443)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Fix force_breakdown function parameter name * Add handling for PDFs with lowercase starting paragraphs * Change first lowercase word in meta_txt to uppercase --- crazy_functions/crazy_utils.py | 3 +++ crazy_functions/pdf_fns/breakdown_txt.py | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/crazy_functions/crazy_utils.py b/crazy_functions/crazy_utils.py index 4d3b195..e7e625b 100644 --- a/crazy_functions/crazy_utils.py +++ b/crazy_functions/crazy_utils.py @@ -466,6 +466,9 @@ def read_and_clean_pdf_text(fp): return True else: return False + # 对于某些PDF会有第一个段落就以小写字母开头,为了避免索引错误将其更改为大写 + if starts_with_lowercase_word(meta_txt[0]): + meta_txt[0] = meta_txt[0].capitalize() for _ in range(100): for index, block_txt in enumerate(meta_txt): if starts_with_lowercase_word(block_txt): diff --git a/crazy_functions/pdf_fns/breakdown_txt.py b/crazy_functions/pdf_fns/breakdown_txt.py index a961481..e7c7673 100644 --- a/crazy_functions/pdf_fns/breakdown_txt.py +++ b/crazy_functions/pdf_fns/breakdown_txt.py @@ -65,10 +65,10 @@ def cut(limit, get_token_fn, txt_tocut, must_break_at_empty_line, break_anyway=F # 如果没有找到合适的切分点 if break_anyway: # 是否允许暴力切分 - prev, post = force_breakdown(txt_tocut, limit, get_token_fn) + prev, post = force_breakdown(remain_txt_to_cut, limit, get_token_fn) else: # 不允许直接报错 - raise RuntimeError(f"存在一行极长的文本！{txt_tocut}") + raise RuntimeError(f"存在一行极长的文本！{remain_txt_to_cut}") # 追加列表 res.append(prev); fin_len+=len(prev)