diff --git a/py/common/results.py b/py/common/results.py index f4b45d5b..1f324cc0 100644 --- a/py/common/results.py +++ b/py/common/results.py @@ -272,6 +272,20 @@ def append(self, key, value): self.write("%s = %s\n" % (key, val_str)) +def re_from_checker_set(checker_set): + """return operand for the --checker option of csgrep based on checker_set""" + chk_re = "^(" + first = True + for chk in sorted(checker_set): + if first: + first = False + else: + chk_re += "|" + chk_re += chk + chk_re += ")$" + return chk_re + + def transform_results(js_file, results): err_file = re.sub("\\.js", ".err", js_file) html_file = re.sub("\\.js", ".html", js_file) @@ -283,3 +297,98 @@ def transform_results(js_file, results): results.exec_cmd("csgrep --mode=evtstat %s '%s' | tee '%s'" % \ (CSGREP_FINAL_FILTER_ARGS, js_file, stat_file), shell=True) return err_file, html_file + + +def finalize_results(js_file, results, props): + """transform scan-results.js to scan-results.{err,html} and write stats""" + if props.imp_checker_set: + # filter out "important" defects, first based on checkers only + cmd = "csgrep '%s' --mode=json --checker '%s'" % \ + (js_file, re_from_checker_set(props.imp_checker_set)) + + # then apply custom per-checker filters + for (chk, csgrep_args) in props.imp_csgrep_filters: + chk_re = re_from_checker_set(props.imp_checker_set - set([chk])) + cmd += " | csdiff <(csgrep '%s' --mode=json --drop-scan-props --invert-regex --checker '%s' %s) -" \ + % (js_file, chk_re, csgrep_args) + + # write the result into *-imp.js + imp_js_file = re.sub("\\.js", "-imp.js", js_file) + cmd += " > '%s'" % imp_js_file + + # bash is needed to process <(...) + cmd = strlist_to_shell_cmd(["bash", "-c", cmd], escape_special=True) + results.exec_cmd(cmd, shell=True) + + # generate *-imp.{err,html} + transform_results(imp_js_file, results) + + # initialize the "imp" flag in the resulting full .js output file + tmp_js_file = re.sub("\\.js", "-tmp.js", js_file) + cmd = "cslinker --implist '%s' '%s' > '%s' && mv -v '%s' '%s'" \ + % (imp_js_file, js_file, tmp_js_file, tmp_js_file, js_file) + results.exec_cmd(cmd, shell=True) + + (err_file, _) = transform_results(js_file, results) + + if props.print_defects: + os.system("csgrep '%s'" % err_file) + + +def apply_result_filters(props, results, supp_filters=[]): + """apply filters, sort the list and record suppressed results""" + js_file = os.path.join(results.resdir, "scan-results.js") + all_file = os.path.join(results.dbgdir, "scan-results-all.js") + + # apply filters, sort the list and store the result as scan-results.js + cmd = f"cat '{all_file}'" + for filt in props.result_filters: + cmd += f" | {filt}" + cmd += f" | cssort --key=path > '{js_file}'" + results.exec_cmd(cmd, shell=True) + + # record suppressed results + js_supp = os.path.join(results.dbgdir, "suppressed-results.js") + cmd = f"cat '{all_file}'" + for filt in supp_filters: + cmd += f" | {filt}" + cmd += f" | csdiff --show-internal '{js_file}' -" + cmd += f" | cssort > '{js_supp}'" + results.exec_cmd(cmd, shell=True) + finalize_results(js_supp, results, props) + finalize_results(js_file, results, props) + + +def handle_known_fp_list(props, results): + """Update props.result_filters based on props.known_false_positives""" + if not props.known_false_positives: + return + + # update scan metadata + results.ini_writer.append("known-false-positives", props.known_false_positives) + + # install global filter of known false positives + filter_cmd = f'csdiff --json-output --show-internal "{props.known_false_positives}" -' + props.result_filters += [filter_cmd] + + if props.pkg is None: + # no package name available + return + + kfp_dir = re.sub("\\.js", ".d", props.known_false_positives) + if not os.path.isdir(kfp_dir): + # no per-pkg known false positives available + return + + ep_file = os.path.join(kfp_dir, props.pkg, "exclude-paths.txt") + if not os.path.exists(ep_file): + # no list of path regexes to exclude for this pkg + return + + # install path exclusion filters for this pkg + with open(ep_file) as file_handle: + lines = file_handle.readlines() + for line in lines: + path_re = line.strip() + filter_cmd = f'csgrep --mode=json --invert-match --path="{shell_quote(path_re)}"' + props.result_filters += [filter_cmd] diff --git a/py/common/util.py b/py/common/util.py index cfab8979..e0058ade 100644 --- a/py/common/util.py +++ b/py/common/util.py @@ -15,6 +15,7 @@ # You should have received a copy of the GNU General Public License # along with csmock. If not, see . +import os import re @@ -121,3 +122,9 @@ def dirs_to_scan_by_args(parser, args, props, tool): props.need_rpm_bi = True return dirs_to_scan + + +def require_file(parser, name): + """Print an error and exit unsuccessfully if 'name' is not a file""" + if not os.path.isfile(name): + parser.error(f"'{name}' is not a file") diff --git a/py/csmock b/py/csmock index 5402dd4d..e1f27b91 100755 --- a/py/csmock +++ b/py/csmock @@ -32,10 +32,14 @@ import time # local imports import csmock.common.util +from csmock.common.util import require_file from csmock.common.util import shell_quote from csmock.common.util import strlist_to_shell_cmd from csmock.common.results import FatalError from csmock.common.results import ScanResults +from csmock.common.results import apply_result_filters +from csmock.common.results import finalize_results +from csmock.common.results import handle_known_fp_list from csmock.common.results import transform_results @@ -657,84 +661,6 @@ class PluginManager: return len(self.enabled_plugins()) -def re_from_checker_set(checker_set): - """return operand for the --checker option of csgrep based on checker_set""" - chk_re = "^(" - first = True - for chk in sorted(checker_set): - if first: - first = False - else: - chk_re += "|" - chk_re += chk - chk_re += ")$" - return chk_re - - -def handle_known_fp_list(props): - # install global filter of known false positives - filter_cmd = 'csdiff --json-output --show-internal "%s" -' % props.known_false_positives - props.result_filters += [ filter_cmd ] - - if props.pkg is None: - # no package name available - return - - kfp_dir = re.sub("\\.js", ".d", props.known_false_positives) - if not os.path.isdir(kfp_dir): - # no per-pkg known false positives available - return - - ep_file = os.path.join(kfp_dir, props.pkg, "exclude-paths.txt") - if not os.path.exists(ep_file): - # no list of path regexes to exclude for this pkg - return - - # install path exclusion filters for this pkg - with open(ep_file) as f: - lines = f.readlines() - for l in lines: - path_re = l.strip() - filter_cmd = 'csgrep --mode=json --invert-match --path=%s' % shell_quote(path_re) - props.result_filters += [ filter_cmd ] - - -# transform scan-results.js to scan-results.{err,html} and write stats -def finalize_results(js_file, results, props): - if props.imp_checker_set: - # filter out "important" defects, first based on checkers only - cmd = "csgrep '%s' --mode=json --checker '%s'" % \ - (js_file, re_from_checker_set(props.imp_checker_set)) - - # then apply custom per-checker filters - for (chk, csgrep_args) in props.imp_csgrep_filters: - chk_re = re_from_checker_set(props.imp_checker_set - set([chk])) - cmd += " | csdiff <(csgrep '%s' --mode=json --drop-scan-props --invert-regex --checker '%s' %s) -" \ - % (js_file, chk_re, csgrep_args) - - # write the result into *-imp.js - imp_js_file = re.sub("\\.js", "-imp.js", js_file) - cmd += " > '%s'" % imp_js_file - - # bash is needed to process <(...) - cmd = strlist_to_shell_cmd(["bash", "-c", cmd], escape_special=True) - results.exec_cmd(cmd, shell=True) - - # generate *-imp.{err,html} - transform_results(imp_js_file, results) - - # initialize the "imp" flag in the resulting full .js output file - tmp_js_file = re.sub("\\.js", "-tmp.js", js_file) - cmd = "cslinker --implist '%s' '%s' > '%s' && mv -v '%s' '%s'" \ - % (imp_js_file, js_file, tmp_js_file, tmp_js_file, js_file) - results.exec_cmd(cmd, shell=True) - - (err_file, _) = transform_results(js_file, results) - - if props.print_defects: - os.system("csgrep '%s'" % err_file) - - # argparse._VersionAction would write to stderr, which breaks help2man class VersionPrinter(argparse.Action): def __init__(self, option_strings, dest=None, default=None, help=None): @@ -758,12 +684,6 @@ is a file name, please use the './' prefix." % val) setattr(namespace, self.dest, val) -def require_file(parser, name): - """Print an error and exit unsuccessfully if 'name' is not a file""" - if not os.path.isfile(name): - parser.error("'%s' is not a file" % name) - - def main(): # load plug-ins plugins = PluginManager() @@ -1031,11 +951,10 @@ key event (defaults to 3).") if os.path.exists(output) and not args.force: parser.error("'%s' already exists, use --force to proceed" % output) - # handle --known-false-positives + # check the path given to --known-false-positives props.known_false_positives = args.known_false_positives if props.known_false_positives: require_file(parser, props.known_false_positives) - handle_known_fp_list(props) # poll plug-ins to reflect themselves in ScanProps plugins.handle_args(parser, args, props) @@ -1070,8 +989,7 @@ def do_scan(props, output): results.ini_writer.append("enabled-plugins", ", ".join(enabled_plugins)) results.ini_writer.append("mock-config", props.mock_profile) results.ini_writer.append("project-name", props.nvr) - if props.known_false_positives: - results.ini_writer.append("known-false-positives", props.known_false_positives) + handle_known_fp_list(props, results) if not props.any_tool: # no tool enabled @@ -1319,23 +1237,10 @@ cd %%s*/ || cd *\n\ # we are done with mock - # apply filters, sort the list and store the result as scan-results.js - cmd = "cat '%s'" % all_file - for filt in props.result_filters: - cmd += " | %s" % filt - cmd += " | cssort --key=path > '%s'" % js_file - results.exec_cmd(cmd, shell=True) - - # record suppressed results - js_supp = "%s/suppressed-results.js" % results.dbgdir - cmd = "%s '%s' " % (RPM_BI_FILTER, all_file) - cmd += "| csgrep --mode=json --strip-path-prefix /builddir/build/BUILD/ " - cmd += "| csdiff --show-internal '%s' - " % js_file - cmd += "| cssort > '%s'" % js_supp - results.exec_cmd(cmd, shell=True) - finalize_results(js_supp, results, props) + # apply filters, sort the list and record suppressed results + supp_filters = [RPM_BI_FILTER, "csgrep --mode=json --strip-path-prefix /builddir/build/BUILD/"] + apply_result_filters(props, results, supp_filters=supp_filters) - finalize_results(js_file, results, props) return results.ec except FatalError as error: