diff --git a/src/core.py b/src/core.py index 06ff886..f6ea1ac 100644 --- a/src/core.py +++ b/src/core.py @@ -38,7 +38,6 @@ if windows: from subprocess import CREATE_NO_WINDOW - from signal import SIGBREAK else: from os import getpgid, killpg @@ -202,10 +201,11 @@ def get_command(executable,parameters,full_file_path,shell): #'ignore','replace','backslashreplace' def popen_win(command,shell,stdin=DEVNULL): - return Popen(command, stdout=PIPE, stderr=STDOUT,stdin=stdin,shell=shell,text=True,universal_newlines=True,creationflags=CREATE_NO_WINDOW,close_fds=False,errors='ignore') + return Popen(command, stdout=PIPE, stderr=STDOUT,stdin=stdin,shell=shell,text=True,universal_newlines=True,bufsize=-1,errors='ignore',creationflags=CREATE_NO_WINDOW) + #,close_fds=False def popen_lin(command,shell,stdin=DEVNULL): - return Popen(command, stdout=PIPE, stderr=STDOUT,stdin=stdin,shell=shell,text=True,universal_newlines=True,start_new_session=True,errors='ignore') + return Popen(command, stdout=PIPE, stderr=STDOUT,stdin=stdin,shell=shell,text=True,universal_newlines=True,bufsize=-1,errors='ignore',start_new_session=True) uni_popen = (lambda command,shell=False,stdin=DEVNULL : popen_win(command,shell,stdin)) if windows else (lambda command,shell=False,stdin=DEVNULL : popen_lin(command,shell,stdin)) @@ -682,9 +682,9 @@ def threaded_cde(timeout_semi_list): output_list_append = output_list.append while True: - line = subprocess_stdout_readline().rstrip() + line = subprocess_stdout_readline() - output_list_append(line) + output_list_append(line.rstrip()) if not line and subprocess_poll() is not None: returncode=subprocess.returncode @@ -759,8 +759,11 @@ def threaded_cde(timeout_semi_list): kill_subprocess(subprocess,print_func) self.killed=True abort_list[1]=False + sleep(0.2) else: - sleep(0.5) + sleep(0.4) + + print_func( ('info','custom data extraction finished.'),True) cde_thread.join() @@ -975,7 +978,7 @@ def find_items(self, #katalog moze spelniac kryteria naazwy pliku ale nie ma rozmiaru i custom data if name_func_to_call: if name_func_to_call(name): - print_func([search_progress,size,mtime,*next_level]) + print_func( (search_progress,size,mtime,*next_level) ) #search_progress_update_quant=0 #progress_update_time = perf_counter() @@ -1043,11 +1046,12 @@ def find_items(self, else: continue - print_func([search_progress,size,mtime,*next_level]) + print_func( (search_progress,size,mtime,*next_level) ) #search_progress_update_quant=0 #progress_update_time = perf_counter() - print_func([search_progress]) + #print_func((search_progress)) + #t_now = perf_counter() #if t_now>progress_update_time+1.0: # progress_update_time = t_now @@ -1057,7 +1061,7 @@ def find_items(self, #else: # search_progress_update_quant+=1 - print_func([search_progress]) + print_func( [search_progress] ) #print_func(True) def find_items_sort(self,what,reverse): @@ -1739,10 +1743,7 @@ def find_items_in_records(self, find_cd_search_kind,cd_expr,cd_case_sens, filename_fuzzy_threshold,cd_fuzzy_threshold): - self.log.info(f'find_items_in_records {size_min},{size_max},\ - {find_filename_search_kind},{name_expr},{name_case_sens},\ - {find_cd_search_kind},{cd_expr},{cd_case_sens},\ - {filename_fuzzy_threshold},{cd_fuzzy_threshold}') + self.log.info(f'find_items_in_records:{size_min},{size_max},{find_filename_search_kind},{name_expr},{name_case_sens},{find_cd_search_kind},{cd_expr},{cd_case_sens},{filename_fuzzy_threshold},{cd_fuzzy_threshold}') self.find_results_clean() @@ -1764,53 +1765,10 @@ def find_items_in_records(self, print(e) record_command_list={} - is_frozen = bool(getattr(sys, 'frozen', False)) for record_nr,record in enumerate(records_to_process): curr_command_list = record_command_list[record_nr] = self.record_exe() - curr_command_list.extend(['search',record.file_path,temp_dir]) - - if t_min: - curr_command_list.extend( ['--timestamp_min',str(t_min) ] ) - - if t_max: - curr_command_list.extend( ['--timestamp_max',str(t_max)] ) - - if size_min: - curr_command_list.extend( ['--size_min',str(size_min).replace(' ','') ] ) - - if size_max: - curr_command_list.extend( ['--size_max',str(size_max).replace(' ','')] ) - - if name_expr: - if find_filename_search_kind == 'regexp': - curr_command_list.extend(['--file_regexp',name_expr]) - elif find_filename_search_kind == 'glob': - curr_command_list.extend(['--file_glob',name_expr]) - if name_case_sens: - curr_command_list.append('--file_case_sensitive') - elif find_filename_search_kind == 'fuzzy': - curr_command_list.extend(['--file_fuzzy',name_expr,'--file_fuzzy_threshold',filename_fuzzy_threshold]) - elif find_filename_search_kind == 'error': - curr_command_list.append('--file_error') - - if cd_expr: - if find_cd_search_kind == 'regexp': - curr_command_list.extend( ['--cd_regexp',cd_expr] ) - elif find_cd_search_kind == 'glob': - curr_command_list.extend( ['--cd_glob',cd_expr] ) - if cd_case_sens: - curr_command_list.append('--cd_case_sensitive') - elif find_cd_search_kind == 'fuzzy': - curr_command_list.extend( ['--cd_fuzzy',cd_expr,'--cd_fuzzy_threshold',cd_fuzzy_threshold] ) - elif find_cd_search_kind == 'without': - curr_command_list.append('--cd_without') - elif find_cd_search_kind == 'any': - curr_command_list.append('--cd_ok') - elif find_cd_search_kind == 'error': - curr_command_list.append('--cd_error') - self.log.info(f'curr_command_list: {curr_command_list}') self.find_res_quant = 0 diff --git a/src/dialogs.py b/src/dialogs.py index 396881c..7cb56e0 100644 --- a/src/dialogs.py +++ b/src/dialogs.py @@ -367,7 +367,7 @@ def __init__(self,parent,icon,bg_color,pre_show=None,post_close=None,min_width=1 self.find_info_var=StringVar() self.find_info_var.set('-/-') self.find_info_lab=Label(self.area_mark, textvariable=self.find_info_var, width=8,relief='groove',bd=2,bg=self.bg_color) - self.find_info_lab.pack(side='right', anchor='w',padx=2,pady=5,expand=1) + self.find_info_lab.pack(side='right', anchor='w',padx=2,pady=5,expand=1,fill='both') self.find_prev_butt=Button(self.area_mark, command=lambda : self.find_next_prev(-1), width=1) self.find_prev_butt.pack(side='right', anchor='w',padx=2,pady=5,fill='both') diff --git a/src/librer.py b/src/librer.py index 34e2138..584ea2e 100644 --- a/src/librer.py +++ b/src/librer.py @@ -56,7 +56,6 @@ if windows: from os import startfile from win32api import GetVolumeInformation - from signal import SIGBREAK #l_debug = logging.debug l_info = logging.info diff --git a/src/record.py b/src/record.py index be55c98..e50c60e 100644 --- a/src/record.py +++ b/src/record.py @@ -51,9 +51,6 @@ from core import * -#if windows: -# from signal import SIGBREAK - import logging l_info = logging.info @@ -83,56 +80,20 @@ def parse_args(ver): parser.add_argument('comm_dir',type=str,help='internal communication dir') - file_group = parser.add_argument_group() - - file_group.add_argument('-fre' ,'--file_regexp',type=str,help='serch files by regular expression') - - #file_glob_group = file_group.add_argument_group('file name glob matching') - file_group.add_argument('-fg' ,'--file_glob',type=str,help='serch files by glob expression') - file_group.add_argument('-fgcs' ,'--file_case_sensitive',action='store_true',help='serch files by case sensitive glob expression') - - #file_fuzzy_group = file_group.add_argument_group('file name fuzzy matching') - file_group.add_argument('-ff' ,'--file_fuzzy',type=str,help='serch files by fuzzy match with threshold') - file_group.add_argument('-fft' ,'--file_fuzzy_threshold', type=float,help='threshold value') - - file_group.add_argument('-fe' ,'--file_error',action='store_true',help='serch files with error on access') - - cd_group = parser.add_argument_group() - cd_group.add_argument('-cdw','--cd_without',action='store_true',help='serch for riles without custom data') - cd_group.add_argument('-cdok','--cd_ok',action='store_true',help='serch for riles with correct custom data') - cd_group.add_argument('-cderror','--cd_error',action='store_true',help='serch for riles with error status on custom data extraction') - - cd_group.add_argument('-cdre' ,'--cd_regexp',type=str,help='serch by regular expression on custom data') - - #cd_glob_group = file_group.add_argument_group('Custom data glob matching') - cd_group.add_argument('-cdg' ,'--cd_glob',type=str,help='serch by glob expression on custom data') - cd_group.add_argument('-cdgcs' ,'--cd_case_sensitive',action='store_true',help='serch by case sensitive glob expression on custom data') - - #cd_fuzzy_group = cd_group.add_argument_group('Custom data fuzzy matching') - cd_group.add_argument('-cdf' ,'--cd_fuzzy',type=str,help='serch by fuzzy match with threshold on custom data') - cd_group.add_argument('-cdft' ,'--cd_fuzzy_threshold',type=float,help='threshold value on custom data') - - parser.add_argument('-min','--size_min',type=str,help='minimum size') - parser.add_argument('-max','--size_max',type=str,help='maximum size') - - parser.add_argument('-tmin','--timestamp_min',type=str,help='minimum modification timestamp') - parser.add_argument('-tmax','--timestamp_max',type=str,help='maximum modification timestamp') - return parser.parse_args() def find_params_check(self, size_min,size_max, find_filename_search_kind,name_expr,name_case_sens, find_cd_search_kind,cd_expr,cd_case_sens, - file_fuzzy_threshold,cd_fuzzy_threshold): - + filename_fuzzy_threshold,cd_fuzzy_threshold): if find_filename_search_kind == 'fuzzy': if name_expr: try: - float(file_fuzzy_threshold) + float(filename_fuzzy_threshold) except ValueError: - return f"wrong threshold value:{file_fuzzy_threshold}" + return f"wrong threshold value:{filename_fuzzy_threshold}" else: return "empty file expression" @@ -148,70 +109,67 @@ def find_params_check(self, return None stdout_data_queue=deque() -stdout_data_last_not_printed=None -stdout_data_queue_print_time=0 +stdout_data_queue_append = stdout_data_queue.append def print_func(data,always=False): - now=perf_counter() - global stdout_data_queue_print_time - global stdout_data_last_not_printed - if now>stdout_data_queue_print_time or always: - stdout_data_queue.append(data) - stdout_data_queue_print_time=now+0.1 - stdout_data_last_not_printed=None - else: - stdout_data_last_not_printed=data + stdout_data_queue_append( (data,always) ) def print_func_always(data): - stdout_data_queue.append(data) + print_func(data,True) -def printer_stop(): - stdout_data_queue.append(True) +def caretaker_stop(): + stdout_data_queue_append( (True,True) ) -def printer(): +abort_list=[False,False] + +def caretaker(signal_file): stdout_data_queue_get = stdout_data_queue.popleft - last_print_time=0 - global stdout_data_last_not_printed - try: - while True: - if stdout_data_queue: - result=stdout_data_queue_get() - if result==True: - break - print(json_dumps(result),flush=True) - last_print_time=time() - else: - sleep(0.01) - if stdout_data_last_not_printed: - if time()>last_print_time+0.5: - print(json_dumps(stdout_data_last_not_printed),flush=True) - stdout_data_last_not_printed=None - except Exception as pe: - print_info(f'printer error:{pe}') + next_signal_file_check=0 + next_time_print=0 - sys.exit(0) #thread + global abort_list -abort_list=[False,False] + print_min_time_period=0.1 + signal_file_check_period=0.5 + last_data_not_printed=None -def check_abort(signal_file): + sys_stdout_flush = sys.stdout.flush while True: - if path_exists(signal_file): - try: - with open(signal_file,'r') as sf: - got_int = int(sf.read().strip()) + now=perf_counter() - print_info(f'got abort int:{got_int}') + if stdout_data_queue: + data,always=stdout_data_queue_get() - global abort_list - abort_list[got_int]=True + if data==True: + break - remove(signal_file) + if always or now>next_time_print: + print(json_dumps(data),flush=True) + next_time_print=now+print_min_time_period + else: + last_data_not_printed=data - except Exception as pe: - print_info(f'check_abort error:{pe}') - else: - sleep(0.1) + continue + + if now>next_signal_file_check: + next_signal_file_check=now+signal_file_check_period + if path_exists(signal_file): + try: + with open(signal_file,'r') as sf: + got_int = int(sf.read().strip()) + print_info(f'got abort int:{got_int}') + abort_list[got_int]=True + remove(signal_file) + + except Exception as pe: + print_info(f'check_abort error:{pe}') + + if last_data_not_printed and now>next_time_print: + print(json_dumps(last_data_not_printed),flush=True) + last_data_not_printed=None + + sleep(0.01) sys.exit(0) #thread @@ -228,13 +186,10 @@ def print_info(*args): gc_disable() - printer_thread = Thread(target=printer,daemon=True) - printer_thread.start() - signal_file = sep.join([comm_dir,SIGINT_FILE]) - abort_thread = Thread(target=lambda : check_abort(signal_file),daemon=True) - abort_thread.start() + caretaker_thread = Thread(target=lambda : caretaker(signal_file),daemon=False) + caretaker_thread.start() if args.command == 'search': ##################################################################### @@ -250,37 +205,41 @@ def print_info(*args): except Exception as e: print_info(e) exit(2) - else: - (size_min,size_max,t_min,t_max,find_filename_search_kind,name_expr,name_case_sens,find_cd_search_kind,cd_expr,cd_case_sens,filename_fuzzy_threshold,cd_fuzzy_threshold) = params - name_case_sens=args.file_case_sensitive - cd_case_sens=args.cd_case_sensitive + (size_min,size_max,t_min,t_max,find_filename_search_kind,name_expr,name_case_sens,find_cd_search_kind,cd_expr,cd_case_sens,filename_fuzzy_threshold_str,cd_fuzzy_threshold_str) = params - name_regexp=args.file_regexp - name_glob=args.file_glob - name_fuzzy=args.file_fuzzy - file_error=args.file_error + try: + filename_fuzzy_threshold = float(filename_fuzzy_threshold_str) + except Exception as ce: + filename_fuzzy_threshold = 0.0 + + try: + cd_fuzzy_threshold = float(cd_fuzzy_threshold_str) + except Exception as ce: + cd_fuzzy_threshold = 0.0 - file_fuzzy_threshold = args.file_fuzzy_threshold - cd_fuzzy_threshold = args.cd_fuzzy_threshold + name_regexp=bool(find_filename_search_kind == 'regexp') + name_glob=bool(find_filename_search_kind == 'glob') + name_fuzzy=bool(find_filename_search_kind == 'fuzzy') + file_error=bool(find_filename_search_kind == 'error') if name_regexp: - if res := test_regexp(name_regexp): + if res := test_regexp(name_expr): exit(res) - re_obj_name=re_compile(name_regexp) + re_obj_name=re_compile(name_expr) name_func_to_call = lambda x : re_obj_name.match(x) name_search_kind='regexp' elif name_glob: if name_case_sens: - re_obj_name=re_compile(translate(name_glob)) + re_obj_name=re_compile(translate(name_expr)) name_func_to_call = lambda x : re_obj_name.match(x) else: - re_obj_name=re_compile(translate(name_glob), IGNORECASE) + re_obj_name=re_compile(translate(name_expr), IGNORECASE) name_func_to_call = lambda x : re_obj_name.match(x) name_search_kind='glob' elif name_fuzzy: - name_func_to_call = lambda x : bool(SequenceMatcher(None, name_fuzzy, x).ratio()>file_fuzzy_threshold) + name_func_to_call = lambda x : bool(SequenceMatcher(None, name_expr, x).ratio()>filename_fuzzy_threshold) name_search_kind='fuzzy' elif file_error: name_func_to_call = None @@ -292,35 +251,34 @@ def print_info(*args): custom_data_needed=False - cd_without=args.cd_without - cd_error=args.cd_error - cd_ok=args.cd_ok - - cd_regexp=args.cd_regexp - cd_glob=args.cd_glob - cd_fuzzy=args.cd_fuzzy + cd_regexp=bool(find_cd_search_kind=='regexp') + cd_glob=bool(find_cd_search_kind=='glob') + cd_fuzzy=bool(find_cd_search_kind=='fuzzy') + cd_error=bool(find_cd_search_kind=='error') + cd_without=bool(find_cd_search_kind=='without') + cd_ok=bool(find_cd_search_kind=='any') if cd_regexp: custom_data_needed=True cd_search_kind='regexp' - if res := test_regexp(cd_regexp): + if res := test_regexp(cd_expr): exit(res) - re_obj_cd=re_compile(cd_regexp, MULTILINE | DOTALL) + re_obj_cd=re_compile(cd_expr, MULTILINE | DOTALL) cd_func_to_call = lambda x : re_obj_cd.match(x) elif cd_glob: custom_data_needed=True cd_search_kind='glob' if cd_case_sens: - re_obj_cd=re_compile(translate(cd_glob), MULTILINE | DOTALL) + re_obj_cd=re_compile(translate(cd_expr), MULTILINE | DOTALL) cd_func_to_call = lambda x : re_obj_cd.match(x) else: - re_obj_cd=re_compile(translate(cd_glob), MULTILINE | DOTALL | IGNORECASE) + re_obj_cd=re_compile(translate(cd_expr), MULTILINE | DOTALL | IGNORECASE) cd_func_to_call = lambda x : re_obj_cd.match(x) elif cd_fuzzy: custom_data_needed=True cd_search_kind='fuzzy' - cd_func_to_call = lambda x : bool(SequenceMatcher(None,cd_fuzzy, x).ratio()>cd_fuzzy_threshold) + cd_func_to_call = lambda x : bool(SequenceMatcher(None,cd_expr, x).ratio()>cd_fuzzy_threshold) elif cd_without: cd_search_kind='without' cd_func_to_call = None @@ -344,18 +302,12 @@ def print_info(*args): print_info('search start') t1 = perf_counter() - size_min=str_to_bytes(args.size_min) if args.size_min else None - size_max=str_to_bytes(args.size_max) if args.size_max else None - - timestamp_min=int(args.timestamp_min) if args.timestamp_min else None - timestamp_max=int(args.timestamp_max) if args.timestamp_max else None - print_info(f'args:{args}') try: record.find_items(print_func_always,abort_list, size_min,size_max, - timestamp_min,timestamp_max, + t_min,t_max, name_search_kind,name_func_to_call, cd_search_kind,cd_func_to_call, print_info) @@ -403,7 +355,8 @@ def print_info(*args): print_info(f'parse problem. command:{args.command}') exit(1) - printer_stop() - printer_thread.join() + caretaker_stop() + caretaker_thread.join() + sys.exit(0)