diff --git a/rag/app/manual.py b/rag/app/manual.py index 8c663e7879..b9546083b5 100644 --- a/rag/app/manual.py +++ b/rag/app/manual.py @@ -67,9 +67,11 @@ def __call__(self, filename, binary=None, from_page=0, return [(b["text"], b.get("layout_no", ""), self.get_position(b, zoomin)) for i, b in enumerate(self.boxes)], tbls + class Docx(DocxParser): def __init__(self): pass + def get_picture(self, document, paragraph): img = paragraph._element.xpath('.//pic:pic') if not img: @@ -80,6 +82,7 @@ def get_picture(self, document, paragraph): image = related_part.image image = Image.open(BytesIO(image.blob)) return image + def concat_img(self, img1, img2): if img1 and not img2: return img1 @@ -160,6 +163,7 @@ def __call__(self, filename, binary=None, from_page=0, to_page=100000, callback= tbls.append(((None, html), "")) return ti_list, tbls + def chunk(filename, binary=None, from_page=0, to_page=100000, lang="Chinese", callback=None, **kwargs): """ @@ -244,6 +248,7 @@ def tag(pn, left, right, top, bottom): res = tokenize_table(tbls, doc, eng) res.extend(tokenize_chunks(chunks, doc, eng, pdf_parser)) return res + if re.search(r"\.docx$", filename, re.IGNORECASE): docx_parser = Docx() ti_list, tbls = docx_parser(filename, binary, @@ -259,8 +264,6 @@ def tag(pn, left, right, top, bottom): raise NotImplementedError("file type not supported yet(pdf and docx supported)") - - if __name__ == "__main__": import sys diff --git a/rag/app/naive.py b/rag/app/naive.py index bd507ffe10..f86d9ed271 100644 --- a/rag/app/naive.py +++ b/rag/app/naive.py @@ -168,8 +168,12 @@ def __call__(self, filename, binary=None): sections.append((sec[:int(len(sec) / 2)], "")) sections.append((sec[int(len(sec) / 2):], "")) else: - sections.append((sec, "")) - print(tables) + if sections and sections[-1][0].strip().find("#") == 0: + sec_, _ = sections.pop(-1) + sections.append((sec_+"\n"+sec, "")) + else: + sections.append((sec, "")) + for table in tables: tbls.append(((None, markdown(table, extensions=['markdown.extensions.tables'])), "")) return sections, tbls diff --git a/rag/app/qa.py b/rag/app/qa.py index fec6e62364..fba0300a0e 100644 --- a/rag/app/qa.py +++ b/rag/app/qa.py @@ -393,6 +393,7 @@ def chunk(filename, binary=None, lang="Chinese", callback=None, **kwargs): if sum_question: res.append(beAdoc(deepcopy(doc), sum_question, markdown(last_answer, extensions=['markdown.extensions.tables']), eng)) return res + elif re.search(r"\.docx$", filename, re.IGNORECASE): docx_parser = Docx() qai_list, tbls = docx_parser(filename, binary,