diff --git a/CHANGELOG.md b/CHANGELOG.md index e99fe8b6..8e08028f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,10 +1,13 @@ # Changelogs ### 2022-12-30 -[v1.3.18](https://github.com/dmMaze/BallonsTranslator/releases/tag/v1.3.18)发布 +[v1.3.20](https://github.com/dmMaze/BallonsTranslator/releases/tag/v1.3.20)发布 1. 适应具有极端宽高比的图片比如条漫 2. 支持粘贴到多个选中的文本编辑框 3. 修bug +4. OCR/翻译/修复选中文字区域, 填字样式会继承选中的文字框自己的 + 单行文本建议选用ctc_48px, 多行日文选mangocr, 目前对多行其它语言不太行, 需要重新训练检测模型 + ### 2022-11-29 [v1.3.15](https://github.com/dmMaze/BallonsTranslator/releases/tag/v1.3.15)发布 diff --git a/CHANGELOG_EN.md b/CHANGELOG_EN.md index ea1d5fcb..1bbf14c5 100644 --- a/CHANGELOG_EN.md +++ b/CHANGELOG_EN.md @@ -1,10 +1,14 @@ # Changelogs -### 2022-12-30 -[v1.3.18](https://github.com/dmMaze/BallonsTranslator/releases/tag/v1.3.18) released +### 2022-12-31 +[v1.3.20](https://github.com/dmMaze/BallonsTranslator/releases/tag/v1.3.20) released 1. Adapted to images with extreme aspect ratio such as webtoons 2. Support paste text to multiple selected Text blocks. 3. Bugfixes +4. OCR/Translate/Inpaint selected text blocks + lettering style will inherit from corresponding selected block. + ctc_48px is more recommended for single line text, mangocr for multi-line Japanese, need to retrain detection model to perform better + ### 2022-11-29 [v1.3.15](https://github.com/dmMaze/BallonsTranslator/releases/tag/v1.3.15) released diff --git a/ballontranslator/__init__.py b/ballontranslator/__init__.py index 6b7d1663..f7ceb75a 100644 --- a/ballontranslator/__init__.py +++ b/ballontranslator/__init__.py @@ -4,4 +4,4 @@ # 1. MAJOR version when you make incompatible API changes; # 2. MINOR version when you add functionality in a backwards-compatible manner; # 3. PATCH version when you make backwards-compatible bug fixes. -__version__ = "1.3.19" \ No newline at end of file +__version__ = "1.3.20" \ No newline at end of file diff --git a/ballontranslator/data/translate/zh_CN.qm b/ballontranslator/data/translate/zh_CN.qm index 1411d3b0..d846f485 100644 Binary files a/ballontranslator/data/translate/zh_CN.qm and b/ballontranslator/data/translate/zh_CN.qm differ diff --git a/ballontranslator/data/translate/zh_CN.ts b/ballontranslator/data/translate/zh_CN.ts index 2a006fdc..3a05ca08 100644 --- a/ballontranslator/data/translate/zh_CN.ts +++ b/ballontranslator/data/translate/zh_CN.ts @@ -52,20 +52,50 @@ Canvas - + Delete 删除 - + Apply font formatting 应用字体格式 - + Auto layout 自动排版 + + + Copy + 复制 + + + + Paste + 粘贴 + + + + translate + 翻译 + + + + OCR + OCR + + + + OCR and translate + OCR并翻译 + + + + OCR, translate and inpaint + OCR，翻译并抹字 + ConfigPanel @@ -178,7 +208,7 @@ DLManager - + Invalid 不可用 @@ -186,7 +216,7 @@ DrawingPanel - + Mask Transparency 掩膜透明度 @@ -366,22 +396,22 @@ ImgtransProgressMessageBox - + Detecting: 检测: - + OCR: OCR: - + Inpainting: 修复: - + Translating: 翻译: @@ -389,7 +419,7 @@ ImgtransThread - + Translation Failed. 翻译失败. @@ -427,11 +457,26 @@ pen thickness 画笔大小 + + + Shape + 形状 + + + + Circle + 圆形 + + + + Rectangle + 方形 + InpaintThread - + Inpainting Failed. 修复失败. @@ -495,27 +540,27 @@ MainWindow - + Failed to load project 项目加载失败 - + unsaved 未保存 - + saved 已保存 - + Saving image... 保存中... - + Export to 导出至 @@ -523,7 +568,7 @@ ModuleThread - + Failed to set 无法设置 @@ -604,30 +649,45 @@ PenConfigPanel - + alpha value alpha值 - + Color 颜色 - + Alpha Alpha - + Thickness 大小 - + pen thickness 画笔大小 + + + Shape + 形状 + + + + Circle + 圆形 + + + + Rectangle + 方形 + PresetListWidget @@ -716,52 +776,52 @@ RectPanel - + method 1 方法1 - + method 2 方法2 - + Auto 自动 - + run inpainting automatically. 自动运行修复函数. - + Inpaint 图像修复 - + Delete 删除 - + Dilate 膨胀 - + kernel size: 核大小: - + Space 空格 - + Ctrl+D @@ -769,57 +829,57 @@ TextEffectPanel - + Effect 特效 - + Opacity 不透明度 - + Shadow 阴影 - + Change shadow color 修改阴影颜色 - + Apply 应用 - + Cancel 取消 - + Opacity: 不透明度: - + radius: 半径: - + strength: 强度: - + x offset: x偏移: - + y offset: y偏移: @@ -923,17 +983,17 @@ 支持语言列表: - + Failed to set translator 翻译器设置失败 - + is required for 是翻译器必填项 - + Translation Failed. 翻译失败. diff --git a/ballontranslator/dl/inpaint/__init__.py b/ballontranslator/dl/inpaint/__init__.py index 14870dcd..071c4626 100644 --- a/ballontranslator/dl/inpaint/__init__.py +++ b/ballontranslator/dl/inpaint/__init__.py @@ -29,8 +29,21 @@ def __init__(self, **setup_params) -> None: def setup_inpainter(self): raise NotImplementedError - def inpaint(self, img: np.ndarray, mask: np.ndarray, textblock_list: List[TextBlock] = None) -> np.ndarray: + def inpaint(self, img: np.ndarray, mask: np.ndarray, textblock_list: List[TextBlock] = None, check_need_inpaint: bool = False) -> np.ndarray: if not self.inpaint_by_block or textblock_list is None: + if check_need_inpaint: + ballon_msk, non_text_msk = extract_ballon_mask(img, mask) + if ballon_msk is not None: + non_text_region = np.where(non_text_msk > 0) + non_text_px = img[non_text_region] + average_bg_color = np.mean(non_text_px, axis=0) + std_bgr = np.std(non_text_px - average_bg_color, axis=0) + std_max = np.max(std_bgr) + inpaint_thresh = 7 if np.std(std_bgr) > 1 else 10 + if std_max < inpaint_thresh: + img = img.copy() + img[np.where(ballon_msk > 0)] = average_bg_color + return img return self._inpaint(img, mask) else: im_h, im_w = img.shape[:2] @@ -41,7 +54,7 @@ def inpaint(self, img: np.ndarray, mask: np.ndarray, textblock_list: List[TextBl im = inpainted[xyxy_e[1]:xyxy_e[3], xyxy_e[0]:xyxy_e[2]] msk = mask[xyxy_e[1]:xyxy_e[3], xyxy_e[0]:xyxy_e[2]] need_inpaint = True - if self.check_need_inpaint: + if self.check_need_inpaint or check_need_inpaint: ballon_msk, non_text_msk = extract_ballon_mask(im, msk) if ballon_msk is not None: non_text_region = np.where(non_text_msk > 0) diff --git a/ballontranslator/dl/ocr/__init__.py b/ballontranslator/dl/ocr/__init__.py index 2a37b91c..9fa1b6c7 100644 --- a/ballontranslator/dl/ocr/__init__.py +++ b/ballontranslator/dl/ocr/__init__.py @@ -1,6 +1,7 @@ from typing import Tuple, List, Dict, Union import numpy as np import cv2 +import logging from ..textdetector.textblock import TextBlock @@ -139,9 +140,15 @@ def ocr_img(self, img: np.ndarray) -> str: return self.model(img) def ocr_blk_list(self, img: np.ndarray, blk_list: List[TextBlock]): + im_h, im_w = img.shape[:2] for blk in blk_list: x1, y1, x2, y2 = blk.xyxy - blk.text = self.model(img[y1:y2, x1:x2]) + if y2 < im_h and x2 < im_w and \ + x1 > 0 and y1 > 0 and x1 < x2 and y1 < y2: + blk.text = self.model(img[y1:y2, x1:x2]) + else: + logging.warning('invalid textbbox to target img') + blk.text = [''] def updateParam(self, param_key: str, param_content): super().updateParam(param_key, param_content) diff --git a/ballontranslator/dl/textdetector/textblock.py b/ballontranslator/dl/textdetector/textblock.py index 91f95d96..02e44c55 100644 --- a/ballontranslator/dl/textdetector/textblock.py +++ b/ballontranslator/dl/textdetector/textblock.py @@ -97,7 +97,10 @@ def __init__(self, xyxy: List, self.shadow_color = shadow_color self.shadow_offset = shadow_offset - def adjust_bbox(self, with_bbox=False): + self.region_mask: np.ndarray = None + self.region_inpaint_dict: dict = None + + def adjust_bbox(self, with_bbox=False, x_range=None, y_range=None): lines = self.lines_array().astype(np.int32) if with_bbox: self.xyxy[0] = min(lines[..., 0].min(), self.xyxy[0]) @@ -110,6 +113,13 @@ def adjust_bbox(self, with_bbox=False): self.xyxy[2] = lines[..., 0].max() self.xyxy[3] = lines[..., 1].max() + if x_range is not None: + self.xyxy[0] = np.clip(self.xyxy[0], x_range[0], x_range[1]) + self.xyxy[2] = np.clip(self.xyxy[2], x_range[0], x_range[1]) + if y_range is not None: + self.xyxy[1] = np.clip(self.xyxy[1], y_range[0], y_range[1]) + self.xyxy[3] = np.clip(self.xyxy[3], y_range[0], y_range[1]) + def sort_lines(self): if self.distance is not None: idx = np.argsort(self.distance) @@ -120,6 +130,26 @@ def sort_lines(self): def lines_array(self, dtype=np.float64): return np.array(self.lines, dtype=dtype) + def set_lines_by_xywh(self, xywh: np.ndarray, angle=0, x_range=None, y_range=None, adjust_bbox=False): + if isinstance(xywh, List): + xywh = np.array(xywh) + lines = xywh2xyxypoly(np.array([xywh])) + if angle != 0: + cx, cy = xywh[0], xywh[1] + cx += xywh[2] / 2. + cy += xywh[3] / 2. + lines = rotate_polygons([cx, cy], lines, angle) + + lines = lines.reshape(-1, 4, 2) + if x_range is not None: + lines[..., 0] = np.clip(lines[..., 0], x_range[0], x_range[1]) + if y_range is not None: + lines[..., 1] = np.clip(lines[..., 1], y_range[0], y_range[1]) + self.lines = lines.tolist() + + if adjust_bbox: + self.adjust_bbox() + def aspect_ratio(self) -> float: min_rect = self.min_rect() middle_pnts = (min_rect[:, [1, 2, 3, 0]] + min_rect) / 2 @@ -188,25 +218,38 @@ def to_dict(self): def get_transformed_region(self, img: np.ndarray, idx: int, textheight: int, maxwidth: int = None) -> np.ndarray : direction = 'v' if self.vertical else 'h' src_pts = np.array(self.lines[idx], dtype=np.float64) + im_h, im_w = img.shape[:2] middle_pnt = (src_pts[[1, 2, 3, 0]] + src_pts) / 2 vec_v = middle_pnt[2] - middle_pnt[0] # vertical vectors of textlines vec_h = middle_pnt[1] - middle_pnt[3] # horizontal vectors of textlines - ratio = np.linalg.norm(vec_v) / np.linalg.norm(vec_h) + norm_v = np.linalg.norm(vec_v) + norm_h = np.linalg.norm(vec_h) + if norm_v <= 0 or norm_h <= 0: + print('invalid textpolygon to target img') + return np.zeros((textheight, textheight, 3), dtype=np.uint8) + ratio = norm_v / norm_h if direction == 'h' : h = int(textheight) w = int(round(textheight / ratio)) dst_pts = np.array([[0, 0], [w - 1, 0], [w - 1, h - 1], [0, h - 1]]).astype(np.float32) M, _ = cv2.findHomography(src_pts, dst_pts, cv2.RANSAC, 5.0) + if M is None: + print('invalid textpolygon to target img') + return np.zeros((textheight, textheight, 3), dtype=np.uint8) region = cv2.warpPerspective(img, M, (w, h)) elif direction == 'v' : w = int(textheight) h = int(round(textheight * ratio)) dst_pts = np.array([[0, 0], [w - 1, 0], [w - 1, h - 1], [0, h - 1]]).astype(np.float32) M, _ = cv2.findHomography(src_pts, dst_pts, cv2.RANSAC, 5.0) + if M is None: + print('invalid textpolygon to target img') + return np.zeros((textheight, textheight, 3), dtype=np.uint8) region = cv2.warpPerspective(img, M, (w, h)) region = cv2.rotate(region, cv2.ROTATE_90_COUNTERCLOCKWISE) + if maxwidth is not None: h, w = region.shape[: 2] if w > maxwidth: diff --git a/ballontranslator/scripts/build_win.bat b/ballontranslator/scripts/build_win.bat index c4aa3346..8ebcf91d 100644 --- a/ballontranslator/scripts/build_win.bat +++ b/ballontranslator/scripts/build_win.bat @@ -2,6 +2,6 @@ nuitka --standalone --mingw64 --show-memory --show-progress ^ --enable-plugin=pyqt5 --include-qt-plugins=sensible,styles ^ --nofollow-import-to=fw_qt6,numpy,urllib3,jaconv,torch,torchvision,transformers,fugashi,unidic_lite,tqdm,shapely,pyclipper,einops,termcolor,bs4,deepl,qtpy,pkuseg,pandas,spacy_pkuseg,sentencepiece,ctranslate2,python-docx,docx2txt,piexif,docx,argparse,colorama,http,email,chardet,requests,pkg_resources,yaml,PIL,multiprocessing,dbm ^ --follow-import-to=dl,utils,ui --include-plugin-directory=ballontranslator/dl,ballontranslator/ui,ballontranslator/utils ^ - --windows-product-version=1.3.19 --windows-company-name=DUMMY_WINDOWS_COMPANY_NAME --windows-product-name=BallonTranslator ^ + --windows-product-version=1.3.20 --windows-company-name=DUMMY_WINDOWS_COMPANY_NAME --windows-product-name=BallonTranslator ^ --output-dir=release BallonTranslator \ No newline at end of file diff --git a/ballontranslator/ui/canvas.py b/ballontranslator/ui/canvas.py index 0a6baff0..5f3bd283 100644 --- a/ballontranslator/ui/canvas.py +++ b/ballontranslator/ui/canvas.py @@ -93,6 +93,8 @@ class Canvas(QGraphicsScene): format_textblks = Signal() layout_textblks = Signal() + run_blktrans = Signal(int) + begin_scale_tool = Signal(QPointF) scale_tool = Signal(QPointF) end_scale_tool = Signal() @@ -542,7 +544,14 @@ def contextMenuEvent(self, event: QGraphicsSceneContextMenuEvent): menu.addSeparator() format_act = menu.addAction(self.tr("Apply font formatting")) layout_act = menu.addAction(self.tr("Auto layout")) + menu.addSeparator() + translate_act = menu.addAction(self.tr("translate")) + ocr_act = menu.addAction(self.tr("OCR")) + ocr_translate_act = menu.addAction(self.tr("OCR and translate")) + ocr_translate_inpaint_act = menu.addAction(self.tr("OCR, translate and inpaint")) + rst = menu.exec_(event.screenPos()) + if rst == delete_act: self.delete_textblks.emit() elif rst == copy_act: @@ -553,6 +562,14 @@ def contextMenuEvent(self, event: QGraphicsSceneContextMenuEvent): self.format_textblks.emit() elif rst == layout_act: self.layout_textblks.emit() + elif rst == translate_act: + self.run_blktrans.emit(-1) + elif rst == ocr_act: + self.run_blktrans.emit(0) + elif rst == ocr_translate_act: + self.run_blktrans.emit(1) + elif rst == ocr_translate_inpaint_act: + self.run_blktrans.emit(2) def on_hide_canvas(self): self.clear_states() diff --git a/ballontranslator/ui/dl_manager.py b/ballontranslator/ui/dl_manager.py index fd5588da..296a31da 100644 --- a/ballontranslator/ui/dl_manager.py +++ b/ballontranslator/ui/dl_manager.py @@ -8,6 +8,7 @@ from utils.logger import logger as LOGGER from utils.registry import Registry +from utils.imgproc_utils import enlarge_window from dl.translators import MissingTranslatorParams from dl import INPAINTERS, TRANSLATORS, TEXTDETECTORS, OCR, \ VALID_TRANSLATORS, VALID_TEXTDETECTORS, VALID_INPAINTERS, VALID_OCR, \ @@ -19,6 +20,8 @@ from .configpanel import ConfigPanel from .misc import DLModuleConfig, ProgramConfig from .imgtrans_proj import ProjImgTrans +from dl.textdetector import TextBlock + class ModuleThread(QThread): @@ -246,6 +249,8 @@ class ImgtransThread(QThread): update_inpaint_progress = Signal(int) exception_occurred = Signal(str, str) + finish_blktrans_stage = Signal(str, int) + def __init__(self, dl_config: DLModuleConfig, textdetect_thread: TextDetectThread, @@ -284,6 +289,36 @@ def runImgtransPipeline(self, imgtrans_proj: ProjImgTrans): self.job = self._imgtrans_pipeline self.start() + def runBlktransPipeline(self, blk_list: List[TextBlock], tgt_img: np.ndarray, mode: int): + self.job = lambda : self._blktrans_pipeline(blk_list, tgt_img, mode) + self.start() + + def _blktrans_pipeline(self, blk_list: List[TextBlock], tgt_img: np.ndarray, mode: int): + if mode >= 0: + self.ocr_thread.module.ocr_blk_list(tgt_img, blk_list) + self.finish_blktrans_stage.emit('ocr', 100) + if mode != 0: + self.translate_thread.module.translate_textblk_lst(blk_list) + self.finish_blktrans_stage.emit('translate', 100) + if mode > 1: + im_h, im_w = tgt_img.shape[:2] + progress_prod = 100. / len(blk_list) if len(blk_list) > 0 else 0 + for ii, blk in enumerate(blk_list): + xyxy = enlarge_window(blk.xyxy, im_w, im_h) + xyxy = np.array(xyxy) + x1, y1, x2, y2 = xyxy.astype(np.int64) + blk.region_inpaint_dict = None + if y2 - y1 > 2 and x2 - x1 > 2: + im = np.copy(tgt_img[y1: y2, x1: x2]) + maskseg_method = self.get_maskseg_method() + inpaint_mask_array, ballon_mask, bub_dict = maskseg_method(im) + mask = self.post_process_mask(inpaint_mask_array) + if mask.sum() > 0: + inpainted = self.inpaint_thread.inpainter.inpaint(im, mask) + blk.region_inpaint_dict = {'img': im, 'mask': mask, 'inpaint_rect': [x1, y1, x2, y2], 'inpainted': inpainted} + self.finish_blktrans_stage.emit('inpaint', int((ii+1) * progress_prod)) + self.finish_blktrans_stage.emit(str(mode), 0) + def _imgtrans_pipeline(self): self.detect_counter = 0 self.ocr_counter = 0 @@ -409,6 +444,7 @@ class DLManager(QObject): canvas_inpaint_finished = Signal(dict) imgtrans_pipeline_finished = Signal() + blktrans_pipeline_finished = Signal(int) page_trans_finished = Signal(int) run_canvas_inpaint = False @@ -450,6 +486,7 @@ def setupThread(self, config_panel: ConfigPanel, imgtrans_progress_msgbox: Imgtr self.imgtrans_thread.update_translate_progress.connect(self.on_update_translate_progress) self.imgtrans_thread.update_inpaint_progress.connect(self.on_update_inpaint_progress) self.imgtrans_thread.exception_occurred.connect(self.handleRunTimeException) + self.imgtrans_thread.finish_blktrans_stage.connect(self.on_finish_blktrans_stage) self.translator_panel = translator_panel = config_panel.trans_config_panel translator_setup_params = merge_config_module_params(dl_config.translator_setup_params, VALID_TRANSLATORS, TRANSLATORS.get) @@ -518,12 +555,7 @@ def inpaint(self, img: np.ndarray, mask: np.ndarray, img_key: str = None, inpain return self.inpaint_thread.inpaint(img, mask, img_key, inpaint_rect) - def runImgtransPipeline(self): - if self.imgtrans_proj.is_empty: - LOGGER.info('proj file is empty, nothing to do') - self.progress_msgbox.hide() - return - self.last_finished_index = -1 + def terminateRunningThread(self): if self.textdetect_thread.isRunning(): self.textdetect_thread.terminate() if self.ocr_thread.isRunning(): @@ -532,7 +564,16 @@ def runImgtransPipeline(self): self.inpaint_thread.terminate() if self.translate_thread.isRunning(): self.translate_thread.terminate() + + def runImgtransPipeline(self): + if self.imgtrans_proj.is_empty: + LOGGER.info('proj file is empty, nothing to do') + self.progress_msgbox.hide() + return + self.last_finished_index = -1 + self.terminateRunningThread() + self.progress_msgbox.show_all_bars() if not self.dl_config.enable_ocr: self.progress_msgbox.ocr_bar.hide() self.progress_msgbox.translate_bar.hide() @@ -546,6 +587,32 @@ def runImgtransPipeline(self): self.progress_msgbox.show() self.imgtrans_thread.runImgtransPipeline(self.imgtrans_proj) + def runBlktransPipeline(self, blk_list: List[TextBlock], tgt_img: np.ndarray, mode: int): + self.terminateRunningThread() + self.progress_msgbox.hide_all_bars() + if mode >= 0: + self.progress_msgbox.ocr_bar.show() + if mode == 2: + self.progress_msgbox.inpaint_bar.show() + if mode != 0: + self.progress_msgbox.translate_bar.show() + self.progress_msgbox.zero_progress() + self.progress_msgbox.show() + self.imgtrans_thread.runBlktransPipeline(blk_list, tgt_img, mode) + + def on_finish_blktrans_stage(self, stage: str, progress: int): + if stage == 'ocr': + self.progress_msgbox.updateOCRProgress(progress) + elif stage == 'translate': + self.progress_msgbox.updateTranslateProgress(progress) + elif stage == 'inpaint': + self.progress_msgbox.updateInpaintProgress(progress) + elif stage in {'-1', '0', '1', '2'}: + self.blktrans_pipeline_finished.emit(int(stage)) + self.progress_msgbox.hide() + else: + raise NotImplementedError(f'Unknown stage: {stage}') + def on_update_detect_progress(self, progress: int): ri = self.imgtrans_thread.recent_finished_index(progress) progress = int(progress / self.imgtrans_thread.num_pages * 100) diff --git a/ballontranslator/ui/drawing_commands.py b/ballontranslator/ui/drawing_commands.py index dc4a0ecf..fdef7d0f 100644 --- a/ballontranslator/ui/drawing_commands.py +++ b/ballontranslator/ui/drawing_commands.py @@ -10,7 +10,8 @@ from utils.logger import logger from .image_edit import ImageEditMode, PixmapItem, DrawingLayer, StrokeImgItem -from .canvas import Canvas +from .canvas import Canvas, TextBlkItem +from .textedit_area import TransPairWidget class StrokeItemUndoCommand(QUndoCommand): @@ -71,4 +72,100 @@ def undo(self) -> None: img_view[:] = self.undo_img mask_view[:] = self.undo_mask self.canvas.setInpaintLayer() - self.canvas.setMaskLayer() \ No newline at end of file + self.canvas.setMaskLayer() + + + +class RunBlkTransCommand(QUndoCommand): + def __init__(self, canvas: Canvas, blkitems: List[TextBlkItem], transpairw_list: List[TransPairWidget], mode: int): + super().__init__() + + self.op_counter = -1 + self.blkitems = blkitems + self.transpairw_list = transpairw_list + + for blkitem, transpairw in zip(self.blkitems, self.transpairw_list): + if mode != 0: + trs = blkitem.blk.translation + transpairw.e_trans.setPlainTextAndKeepUndoStack(trs) + blkitem.setPlainTextAndKeepUndoStack(trs) + blkitem.blk.rich_text = '' + if mode >= 0: + transpairw.e_source.setPlainTextAndKeepUndoStack(blkitem.blk.get_text()) + + self.canvas = canvas + self.mode = mode + if mode > 1: + self.undo_img_list = [] + self.undo_mask_list = [] + self.redo_img_list = [] + self.redo_mask_list = [] + self.inpaint_rect_lst = [] + img_array = self.canvas.imgtrans_proj.inpainted_array + mask_array = self.canvas.imgtrans_proj.mask_array + self.num_inpainted = 0 + for item in self.blkitems: + inpainted_dict = item.blk.region_inpaint_dict + item.blk.region_inpaint_dict = None + if inpainted_dict is None: + self.undo_img_list.append(None) + self.undo_mask_list.append(None) + self.redo_mask_list.append(None) + self.redo_img_list.append(None) + self.inpaint_rect_lst.append(None) + else: + inpaint_rect = inpainted_dict['inpaint_rect'] + img_view = img_array[inpaint_rect[1]: inpaint_rect[3], inpaint_rect[0]: inpaint_rect[2]] + mask_view = mask_array[inpaint_rect[1]: inpaint_rect[3], inpaint_rect[0]: inpaint_rect[2]] + self.undo_img_list.append(np.copy(img_view)) + self.undo_mask_list.append(np.copy(mask_view)) + self.redo_img_list.append(inpainted_dict['inpainted']) + self.redo_mask_list.append(inpainted_dict['mask']) + self.inpaint_rect_lst.append(inpaint_rect) + self.num_inpainted += 1 + + def redo(self) -> None: + if self.mode > 1 and self.num_inpainted > 0: + img_array = self.canvas.imgtrans_proj.inpainted_array + mask_array = self.canvas.imgtrans_proj.mask_array + for inpaint_rect, redo_img, redo_mask in zip(self.inpaint_rect_lst, self.redo_img_list, self.redo_mask_list): + if inpaint_rect is None: + continue + img_view = img_array[inpaint_rect[1]: inpaint_rect[3], inpaint_rect[0]: inpaint_rect[2]] + mask_view = mask_array[inpaint_rect[1]: inpaint_rect[3], inpaint_rect[0]: inpaint_rect[2]] + img_view[:] = redo_img + mask_view[:] = redo_mask + self.canvas.setInpaintLayer() + self.canvas.setMaskLayer() + + if self.op_counter < 0: + self.op_counter += 1 + return + + for blkitem, transpairw in zip(self.blkitems, self.transpairw_list): + if self.mode != 0: + transpairw.e_trans.redo() + blkitem.redo() + if self.mode >= 0: + transpairw.e_source.redo() + + def undo(self) -> None: + if self.mode > 1 and self.num_inpainted > 0: + img_array = self.canvas.imgtrans_proj.inpainted_array + mask_array = self.canvas.imgtrans_proj.mask_array + for inpaint_rect, undo_img, undo_mask in zip(self.inpaint_rect_lst, self.undo_img_list, self.undo_mask_list): + if inpaint_rect is None: + continue + img_view = img_array[inpaint_rect[1]: inpaint_rect[3], inpaint_rect[0]: inpaint_rect[2]] + mask_view = mask_array[inpaint_rect[1]: inpaint_rect[3], inpaint_rect[0]: inpaint_rect[2]] + img_view[:] = undo_img + mask_view[:] = undo_mask + self.canvas.setInpaintLayer() + self.canvas.setMaskLayer() + + for blkitem, transpairw in zip(self.blkitems, self.transpairw_list): + if self.mode != 0: + transpairw.e_trans.undo() + blkitem.undo() + if self.mode >= 0: + transpairw.e_source.undo() \ No newline at end of file diff --git a/ballontranslator/ui/mainwindow.py b/ballontranslator/ui/mainwindow.py index 4851d5c3..476adf98 100644 --- a/ballontranslator/ui/mainwindow.py +++ b/ballontranslator/ui/mainwindow.py @@ -1,5 +1,5 @@ import os.path as osp -import os, re +import os, re, copy from typing import List from qtpy.QtWidgets import QHBoxLayout, QVBoxLayout, QApplication, QStackedWidget, QSplitter, QListWidget, QShortcut, QListWidgetItem, QMessageBox, QTextEdit, QPlainTextEdit @@ -9,7 +9,8 @@ from utils.logger import logger as LOGGER from utils.io_utils import json_dump_nested_obj from utils.text_processing import is_cjk, full_len, half_len -from dl.textdetector import TextBlock +from dl.textdetector.textblock import TextBlock, visualize_textblocks +from utils.imgproc_utils import xywh2xyxypoly from .misc import pt2px, parse_stylesheet from .imgtrans_proj import ProjImgTrans @@ -28,6 +29,7 @@ from . import constants as C from .textedit_commands import GlobalRepalceAllCommand from .framelesswindow import FramelessWindow +from .drawing_commands import RunBlkTransCommand class PageListView(QListWidget): def __init__(self, *args, **kwargs) -> None: @@ -130,6 +132,7 @@ def setupUi(self): self.canvas.gv.hide_canvas.connect(self.onHideCanvas) self.canvas.proj_savestate_changed.connect(self.on_savestate_changed) self.canvas.textstack_changed.connect(self.on_textstack_changed) + self.canvas.run_blktrans.connect(self.on_run_blktrans) self.bottomBar.originalSlider.valueChanged.connect(self.canvas.setOriginalTransparencyBySlider) @@ -203,9 +206,12 @@ def setupConfig(self): dl_manager.finish_translate_page.connect(self.finishTranslatePage) dl_manager.imgtrans_pipeline_finished.connect(self.on_imgtrans_pipeline_finished) dl_manager.page_trans_finished.connect(self.on_pagtrans_finished) - self.dl_manager.setupThread(self.configPanel, self.imgtrans_progress_msgbox) + dl_manager.setupThread(self.configPanel, self.imgtrans_progress_msgbox) dl_manager.progress_msgbox.showed.connect(self.on_imgtrans_progressbox_showed) dl_manager.imgtrans_thread.mask_postprocess = self.drawingPanel.rectPanel.post_process_mask + dl_manager.blktrans_pipeline_finished.connect(self.on_blktrans_finished) + dl_manager.imgtrans_thread.get_maskseg_method = self.drawingPanel.rectPanel.get_maskseg_method + dl_manager.imgtrans_thread.post_process_mask = self.drawingPanel.rectPanel.post_process_mask self.leftBar.run_imgtrans.connect(self.on_run_imgtrans) self.bottomBar.ocrcheck_statechanged.connect(dl_manager.setOCRMode) @@ -789,6 +795,39 @@ def on_textstack_changed(self): if not self.page_changing: self.global_search_widget.set_document_edited() + def on_run_blktrans(self, mode: int): + tgt_img = self.imgtrans_proj.img_array + if tgt_img is None: + return + blkitem_list = self.canvas.selected_text_items() + if len(blkitem_list) < 1: + return + + im_h, im_w = tgt_img.shape[:2] + blk_list = [] + for blkitem in blkitem_list: + blk = blkitem.blk + blk._bounding_rect = blkitem.absBoundingRect() + blk.vertical = blkitem.is_vertical + blk.text = [self.st_manager.pairwidget_list[blkitem.idx].e_source.toPlainText()] + blk.set_lines_by_xywh(blk._bounding_rect, angle=-blk.angle, x_range=[0, im_w-1], y_range=[0, im_h-1], adjust_bbox=True) + blk_list.append(blk) + + # c = visualize_textblocks(tgt_img.copy(), blk_list) + # import cv2 + # cv2.imshow('xx', c) + # cv2.waitKey(0) + self.dl_manager.runBlktransPipeline(blk_list, tgt_img, mode) + + def on_blktrans_finished(self, mode: int): + blkitem_list = self.canvas.selected_text_items() + if len(blkitem_list) < 1: + return + pairw_list = [] + for blk in blkitem_list: + pairw_list.append(self.st_manager.pairwidget_list[blk.idx]) + self.canvas.push_undo_command(RunBlkTransCommand(self.canvas, blkitem_list, pairw_list, mode)) + def on_imgtrans_progressbox_showed(self): msg_size = self.dl_manager.progress_msgbox.size() size = self.size() diff --git a/ballontranslator/ui/scenetext_manager.py b/ballontranslator/ui/scenetext_manager.py index 42fe35ed..f47e4eb4 100644 --- a/ballontranslator/ui/scenetext_manager.py +++ b/ballontranslator/ui/scenetext_manager.py @@ -27,21 +27,21 @@ def __init__(self, blk_item: TextBlkItem, ctrl, parent=None): super().__init__(parent) self.blk_item = blk_item self.ctrl: SceneTextManager = ctrl - - def redo(self): + self.op_count = -1 self.ctrl.addTextBlock(self.blk_item) + self.pairw = self.ctrl.pairwidget_list[self.blk_item.idx] self.ctrl.txtblkShapeControl.setBlkItem(self.blk_item) + def redo(self): + if self.op_count < 0: + self.op_count += 1 + self.blk_item.setSelected(True) + return + self.ctrl.recoverTextblkItem(self.blk_item, self.pairw) + def undo(self): self.ctrl.deleteTextblkItem(self.blk_item) - def mergeWith(self, command: QUndoCommand): - blk_item = command.blk_item - if self.blk_item != blk_item: - return False - self.blk_item = blk_item - return True - class DeleteBlkItemsCommand(QUndoCommand): def __init__(self, blk_list: List[TextBlkItem], ctrl, parent=None): @@ -662,7 +662,7 @@ def onEndCreateTextBlock(self, rect: QRectF): block = TextBlock(xyxy) xywh = np.copy(xyxy) xywh[[2, 3]] -= xywh[[0, 1]] - block.lines = xywh2xyxypoly(np.array([xywh])).reshape(-1, 4, 2).tolist() + block.set_lines_by_xywh(xywh) blk_item = TextBlkItem(block, len(self.textblk_item_list), set_format=False, show_rect=True) blk_item.set_fontformat(self.formatpanel.global_format) self.canvas.push_undo_command(CreateItemCommand(blk_item, self)) diff --git a/ballontranslator/ui/stylewidgets.py b/ballontranslator/ui/stylewidgets.py index f4260805..84460684 100644 --- a/ballontranslator/ui/stylewidgets.py +++ b/ballontranslator/ui/stylewidgets.py @@ -146,6 +146,17 @@ def zero_progress(self): self.updateInpaintProgress(0) self.updateTranslateProgress(0) + def show_all_bars(self): + self.detect_bar.show() + self.ocr_bar.show() + self.translate_bar.show() + self.inpaint_bar.show() + + def hide_all_bars(self): + self.detect_bar.hide() + self.ocr_bar.hide() + self.translate_bar.hide() + self.inpaint_bar.hide() class ColorPicker(QLabel): colorChanged = Signal(bool) diff --git a/ballontranslator/utils/imgproc_utils.py b/ballontranslator/utils/imgproc_utils.py index 32f7c4a3..631718e2 100644 --- a/ballontranslator/utils/imgproc_utils.py +++ b/ballontranslator/utils/imgproc_utils.py @@ -168,6 +168,9 @@ def enlarge_window(rect, im_w, im_h, ratio=2.5, aspect_ratio=1.0) -> List: w = x2 - x1 h = y2 - y1 + if w <= 0 or h <= 0: + return [0, 0, 0, 0] + # https://numpy.org/doc/stable/reference/generated/numpy.roots.html coeff = [aspect_ratio, w+h*aspect_ratio, (1-ratio)*w*h] roots = np.roots(coeff) @@ -177,6 +180,8 @@ def enlarge_window(rect, im_w, im_h, ratio=2.5, aspect_ratio=1.0) -> List: delta_w = min(x1, im_w - x2, delta_w) delta = min(y1, im_h - y2, delta) rect = np.array([x1-delta_w, y1-delta, x2+delta_w, y2+delta], dtype=np.int64) + rect[::2] = np.clip(rect[::2], 0, im_w - 1) + rect[1::2] = np.clip(rect[1::2], 0, im_h - 1) return rect.tolist() def draw_connected_labels(num_labels, labels, stats, centroids, names="draw_connected_labels", skip_background=True): diff --git a/doc/src/ocrselected.gif b/doc/src/ocrselected.gif new file mode 100644 index 00000000..8586ab83 Binary files /dev/null and b/doc/src/ocrselected.gif differ