Merge pull request #459 from bensonhome/main

🎨(client v20220718.1)QuickScan支持指定分析方案模板进行扫描;支持从环境变量读取文件服务器url和token

client/node/common/cmdarg.py

@@ -83,12 +83,24 @@ def parse_args():
         quickscan_parser.add_argument("-l", "--label", dest="label", type=str,
                                       help="规则标签，可以指定多个标签,用英文逗号(,)分隔")
         quickscan_parser.add_argument("--language", dest="language", type=str, help="代码语言类型,可以指定多门语言,用英文逗号(,)分隔")
+        quickscan_parser.add_argument("-t", "--token", dest='token', type=str,
+                                      help="个人Token,在代码分析网站获取, 与--scheme-template-id,--org-sid配合使用")
+        quickscan_parser.add_argument("--scheme-template-id", dest="scheme_template_id", type=str,
+                                      help="分析方案模板id, 与--token,--org-sid配合使用")
+        quickscan_parser.add_argument("--org-sid", dest="org_sid", help="团队编号,在代码分析网站获取,"
+                                                                        "与--scheme-template-id,--token配合使用")
 
         # quickinit命令
         quickinit_parser = subparsers.add_parser('quickinit', help="初始化快速分析")
         quickinit_parser.add_argument("-l", "--label", dest="label", type=str,
                                       help="规则标签，可以指定多个标签,用英文逗号(,)分隔")
         quickinit_parser.add_argument("--language", dest="language", type=str, help="代码语言类型,可以指定多门语言,用英文逗号(,)分隔")
+        quickinit_parser.add_argument("-t", "--token", dest='token', type=str,
+                                      help="个人Token,在代码分析网站获取, 与--scheme-template-id,--org-sid配合使用")
+        quickinit_parser.add_argument("--scheme-template-id", dest="scheme_template_id", type=str,
+                                      help="分析方案模板id, 与--token,--org-sid配合使用")
+        quickinit_parser.add_argument("--org-sid", dest="org_sid", help="团队编号,在代码分析网站获取,"
+                                                                        "与--scheme-template-id,--token配合使用")
 
         # task命令
         task_parser = subparsers.add_parser('task', help="执行单个任务，本地调试用")

client/node/localtask/projectmgr.py

@@ -118,6 +118,7 @@ def _check_scheme(self, repo_schemes):
         is_scheme_existed, scheme_params = SchemeManager.check_and_create_scheme(self._repo_id,
                                                                                  repo_schemes,
                                                                                  self._scan_plan,
+                                                                                 self._ref_scheme_id,
                                                                                  self._scheme_template_ids,
                                                                                  self._scheme_templates,
                                                                                  self._languages,

client/node/localtask/schememgr.py

@@ -43,13 +43,20 @@ def check_scheme_by_template_ids(repo_schemes, scheme_template_ids):
         return False, {}
 
     @staticmethod
-    def check_and_create_scheme(repo_id, repo_schemes, scan_plan, scheme_template_ids, scheme_templates, languages,
+    def check_and_create_scheme(repo_id, repo_schemes, scan_plan, ref_scheme_id, scheme_template_ids, scheme_templates, languages,
                                 url_mgr, dog_server, org_sid, team_name, source_dir, include_paths, exclude_paths,
                                 proj_id):
         """
         检查并创建分析方案
         :return: <bool>方案是否已存在或被创建，<dict>方案参数（如果返回False, 方案参数只包含方案名name）
         """
+        # 指定了分析方案模板id，没有指定方案名称，会使用模板名称作为方案名称
+        if not scan_plan and ref_scheme_id:
+            ref_scheme = dog_server.get_scheme_by_id(ref_scheme_id, org_sid)
+            if ref_scheme:
+                scan_plan = ref_scheme.get("name")
+                LogPrinter.info(f"use the name of refer scheme(id:{ref_scheme_id}) as scheme name: {scan_plan}")
+
         # 有传分析方案名称，先通过名称判断，如果存在，直接复用
         if scan_plan:
             is_existed, scheme_params = SchemeManager.check_scheme_by_name(repo_schemes, scan_plan)

client/node/quicktask/quickrunner.py

@@ -15,11 +15,13 @@
 import sys
 
 from node.localtask.runtask import ConcurrentTasksRuner
+from node.localtask.localconfig import LocalConfig
 from node.common.taskdirmgr import TaskDirCtl
 from node.common.taskrunner import TaskRunner
 from node.common.userinput import UserInput
 from node.quicktask.quickscan import QuickScan
 from util import errcode
+from util.api.dogserver import RetryDogServer
 from util.exceptions import NodeError
 from util.pathlib import PathMgr
 from util.scmurlmgr import BaseScmUrlMgr
@@ -67,6 +69,18 @@ def __init__(self, args):
             self._labels = StringMgr.str_to_list(args.label)
         else:
             self._labels = []
+        if args.token:
+            self._token = args.token
+        else:
+            self._token = None
+        if args.scheme_template_id:
+            self._scheme_template_id = args.scheme_template_id
+        else:
+            self._scheme_template_id = None
+        if args.org_sid:
+            self._org_sid = args.org_sid
+        else:
+            self._org_sid = None
 
         # 其他变量
         self._scm_type = None
@@ -195,6 +209,14 @@ def check_language(self, input_params):
             self._languages = list(language_types)
             LogPrinter.info(f"languages: {self._languages}")
 
+    def _merge_path_filters(self, default_filtered_paths, path_filters):
+        return {
+            "inclusion": path_filters["inclusion"] + default_filtered_paths["inclusion"],
+            "exclusion": path_filters["exclusion"] + default_filtered_paths["exclusion"],
+            "re_inclusion": path_filters["re_inclusion"] + default_filtered_paths["re_inclusion"],
+            "re_exclusion": path_filters["re_exclusion"] + default_filtered_paths["re_exclusion"]
+        }
+
     def _generate_request(self, proj_conf, path_filters):
         """
         根据项目配置信息生成任务参数列表
@@ -204,20 +226,19 @@ def _generate_request(self, proj_conf, path_filters):
         job_context = proj_conf["job_context"]
         # 全量分析
         job_context["incr_scan"] = False
-
-        # # 检查代码库的账号密码权限
-        # self._check_scm_authority()
+        # 合并过滤路径
+        new_path_filters = self._merge_path_filters(job_context["path_filters"], path_filters)
 
         task_list = proj_conf["tasks"]
 
         for task_request in task_list:
             task_params = task_request.get("task_params")
             task_params["incr_scan"] = False
             task_params["scm_last_revision"] = ""
-            task_params["path_filters"] = path_filters
+            task_params["path_filters"] = new_path_filters
             task_params["scm_url"] = self._scm_url
             task_params["scm_type"] = self._scm_type
-            task_params["project_id"] = job_context["project_id"]
+            task_params["project_id"] = job_context.get("project_id")
 
             task_request["execute_processes"] = task_request["processes"]
             task_request["private_processes"] = []
@@ -272,7 +293,18 @@ def run(self):
             self._scan_rel_paths, self._path_info = QuickScan.get_scan_paths(input_params, self._source_dir)
             self.check_language(input_params)
             # LogPrinter.info(f">>> input_params: {json.dumps(input_params, indent=2)}")
-            proj_conf = QuickScan.get_proj_config(self._scm_url, self._languages, self._labels, input_params)
+            if self._scheme_template_id:
+                if self._token and self._org_sid:  # 通过server获取指定的分析方案模板的任务参数
+                    LogPrinter.info(f"token and scheme_template_id are set, get task config from server ...")
+                    server_url = LocalConfig.get_server_url()
+                    dog_server = RetryDogServer(server_url, self._token).get_api_server(retry_times=2)
+                    proj_conf = dog_server.get_jobconfs_by_scheme_template(self._scheme_template_id, self._org_sid)
+                else:
+                    raise NodeError(code=errcode.E_NODE_TASK_CONFIG, msg="已输入scheme_template_id, "
+                                                                         "但缺少--token和--org-sid参数, "
+                                                                         "无法获取到分析方案模板, 请补充参数!")
+            else:
+                proj_conf = QuickScan.get_proj_config(self._scm_url, self._languages, self._labels, input_params)
 
             # debug打印调试
             # with open("quickscan_proj_config.json", "w") as wf:

client/node/quicktask/toolloader.py

@@ -8,12 +8,16 @@
 """拉取quickscan工具
 """
 
+from node.localtask.localconfig import LocalConfig
 from node.toolloader.loadtool import ToolLoader, ToolConfigLoader
 from node.quicktask.configloader import QuickScanConfigLoader
 from node.quicktask.quickscan import QuickScan
 from node.common.userinput import UserInput
+from util import errcode
+from util.api.dogserver import RetryDogServer
 from util.textutil import StringMgr
 from util.logutil import LogPrinter
+from util.exceptions import NodeError
 
 
 class QuickScanToolLoader(object):
@@ -35,7 +39,19 @@ def load_tools(args):
         # 从git拉取工具配置库
         ToolConfigLoader().load_tool_config()
         # LogPrinter.info(f"---> labels: {labels}")
-        tool_tasks = QuickScan.get_scan_tasks(languages, labels, {})
+        if args.scheme_template_id:
+            if args.token and args.org_sid:  # 通过server获取指定的分析方案模板的任务参数
+                LogPrinter.info(f"token and scheme_template_id are set, get task config from server ...")
+                server_url = LocalConfig.get_server_url()
+                dog_server = RetryDogServer(server_url, args.token).get_api_server(retry_times=2)
+                proj_conf = dog_server.get_jobconfs_by_scheme_template(args.scheme_template_id, args.org_sid)
+                tool_tasks = proj_conf["tasks"]
+            else:
+                raise NodeError(code=errcode.E_NODE_TASK_CONFIG, msg="已输入scheme_template_id, "
+                                                                     "但缺少--token和--org-sid参数, "
+                                                                     "无法获取到分析方案模板, 请补充参数!")
+        else:
+            tool_tasks = QuickScan.get_scan_tasks(languages, labels, {})
         tool_names = [task['task_name'] for task in tool_tasks]
         LogPrinter.info("Initing other tools ...")
         ToolLoader(tool_names=tool_names, task_list=tool_tasks, include_common=False).git_load_tools(print_enable=False)

client/settings/edition.py

@@ -28,4 +28,4 @@ class Edition(Enum):
 # 版本号
 # ========================
 # puppy版本号,格式：浮点数，整数部分为8位日期，小数部分为编号(从1开始)
-VERSION = 20220620.1
+VERSION = 20220718.1

client/util/api/dogapi.py

@@ -453,3 +453,22 @@ def get_scan_conf(self, org_sid, team_name, repo_id, proj_id):
         rel_url = f"api/orgs/{org_sid}/teams/{team_name}/repos/{repo_id}/projects/{proj_id}/confs/"
         rsp = CodeDogHttpClient(self._server_url, rel_url, headers=self._headers).get()
         return self.get_data_from_result(rsp)
+
+    def get_scheme_by_id(self, scheme_id, org_sid):
+        """根据分析方案模板id，获取方案模板的名称"""
+        rel_url = f"api/orgs/{org_sid}/schemes/{scheme_id}/"
+        rsp = CodeDogHttpClient(self._server_url, rel_url, headers=self._headers).get()
+        data = self.get_data_from_result(rsp)
+        return data
+
+    def get_jobconfs_by_scheme_template(self, scheme_id, org_sid):
+        """
+        获取分析方案模板的任务执行参数
+        :param scheme_id:分析方案模板id
+        :param org_sid: 团队编号
+        :return:
+        """
+        rel_url = f"api/orgs/{org_sid}/schemes/{scheme_id}/jobconfs/"
+        rsp = CodeDogHttpClient(self._server_url, rel_url, headers=self._headers).get()
+        data = self.get_data_from_result(rsp)
+        return data

client/util/api/fileserver.py

@@ -11,6 +11,7 @@
 
 import logging
 import hashlib
+import os
 
 from urllib.parse import urljoin
 from util import wrapper
@@ -50,8 +51,10 @@ def __init__(self):
         构造函数
         :return:
         """
-        self._server_url = settings.FILE_SERVER['URL']
-        self._headers = {'Authorization': 'Token %s' % settings.FILE_SERVER['TOKEN']}
+        # 优先从环境变量读取文件服务器url和token，没有再使用默认值
+        self._server_url = os.getenv("FILE_SERVER_URL", settings.FILE_SERVER['URL'])
+        file_server_token = os.getenv("FILE_SERVER_TOKEN", settings.FILE_SERVER['TOKEN'])
+        self._headers = {'Authorization': 'Token %s' % file_server_token}
         self._proxies = None
 
     def modify_save_time(self, rel_dir, days):
@@ -75,7 +78,6 @@ def __upload_data(self, data, rel_url, headers):
         :return: 上传到服务器后的完整url, 如果上传失败, 返回 None
         """
         file_url = urljoin(self._server_url, rel_url)
-        # logger.info(f">> file_url: {file_url}")
         HttpClient(self._server_url, rel_url, headers=headers, data=data, proxies=self._proxies).put()
         return file_url