From 33b2015674b340c6178cec1141c6525b673f22c6 Mon Sep 17 00:00:00 2001 From: bensonhome <410554565@qq.com> Date: Mon, 18 Jul 2022 12:24:47 +0800 Subject: [PATCH] =?UTF-8?q?:art:(client=20v20220718.1)QuickScan=E6=94=AF?= =?UTF-8?q?=E6=8C=81=E6=8C=87=E5=AE=9A=E5=88=86=E6=9E=90=E6=96=B9=E6=A1=88?= =?UTF-8?q?=E6=A8=A1=E6=9D=BF=E8=BF=9B=E8=A1=8C=E6=89=AB=E6=8F=8F;?= =?UTF-8?q?=E6=94=AF=E6=8C=81=E4=BB=8E=E7=8E=AF=E5=A2=83=E5=8F=98=E9=87=8F?= =?UTF-8?q?=E8=AF=BB=E5=8F=96=E6=96=87=E4=BB=B6=E6=9C=8D=E5=8A=A1=E5=99=A8?= =?UTF-8?q?url=E5=92=8Ctoken?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- client/node/common/cmdarg.py | 12 ++++++++ client/node/localtask/projectmgr.py | 1 + client/node/localtask/schememgr.py | 9 +++++- client/node/quicktask/quickrunner.py | 44 ++++++++++++++++++++++++---- client/node/quicktask/toolloader.py | 18 +++++++++++- client/settings/edition.py | 2 +- client/util/api/dogapi.py | 19 ++++++++++++ client/util/api/fileserver.py | 8 +++-- 8 files changed, 101 insertions(+), 12 deletions(-) diff --git a/client/node/common/cmdarg.py b/client/node/common/cmdarg.py index 2e3f32512..4f0fe8a4b 100644 --- a/client/node/common/cmdarg.py +++ b/client/node/common/cmdarg.py @@ -83,12 +83,24 @@ def parse_args(): quickscan_parser.add_argument("-l", "--label", dest="label", type=str, help="规则标签，可以指定多个标签,用英文逗号(,)分隔") quickscan_parser.add_argument("--language", dest="language", type=str, help="代码语言类型,可以指定多门语言,用英文逗号(,)分隔") + quickscan_parser.add_argument("-t", "--token", dest='token', type=str, + help="个人Token,在代码分析网站获取, 与--scheme-template-id,--org-sid配合使用") + quickscan_parser.add_argument("--scheme-template-id", dest="scheme_template_id", type=str, + help="分析方案模板id, 与--token,--org-sid配合使用") + quickscan_parser.add_argument("--org-sid", dest="org_sid", help="团队编号,在代码分析网站获取," + "与--scheme-template-id,--token配合使用") # quickinit命令 quickinit_parser = subparsers.add_parser('quickinit', help="初始化快速分析") quickinit_parser.add_argument("-l", "--label", dest="label", type=str, help="规则标签，可以指定多个标签,用英文逗号(,)分隔") quickinit_parser.add_argument("--language", dest="language", type=str, help="代码语言类型,可以指定多门语言,用英文逗号(,)分隔") + quickinit_parser.add_argument("-t", "--token", dest='token', type=str, + help="个人Token,在代码分析网站获取, 与--scheme-template-id,--org-sid配合使用") + quickinit_parser.add_argument("--scheme-template-id", dest="scheme_template_id", type=str, + help="分析方案模板id, 与--token,--org-sid配合使用") + quickinit_parser.add_argument("--org-sid", dest="org_sid", help="团队编号,在代码分析网站获取," + "与--scheme-template-id,--token配合使用") # task命令 task_parser = subparsers.add_parser('task', help="执行单个任务，本地调试用") diff --git a/client/node/localtask/projectmgr.py b/client/node/localtask/projectmgr.py index df226c7b1..983555d8a 100644 --- a/client/node/localtask/projectmgr.py +++ b/client/node/localtask/projectmgr.py @@ -118,6 +118,7 @@ def _check_scheme(self, repo_schemes): is_scheme_existed, scheme_params = SchemeManager.check_and_create_scheme(self._repo_id, repo_schemes, self._scan_plan, + self._ref_scheme_id, self._scheme_template_ids, self._scheme_templates, self._languages, diff --git a/client/node/localtask/schememgr.py b/client/node/localtask/schememgr.py index 50be3a090..18325fd59 100644 --- a/client/node/localtask/schememgr.py +++ b/client/node/localtask/schememgr.py @@ -43,13 +43,20 @@ def check_scheme_by_template_ids(repo_schemes, scheme_template_ids): return False, {} @staticmethod - def check_and_create_scheme(repo_id, repo_schemes, scan_plan, scheme_template_ids, scheme_templates, languages, + def check_and_create_scheme(repo_id, repo_schemes, scan_plan, ref_scheme_id, scheme_template_ids, scheme_templates, languages, url_mgr, dog_server, org_sid, team_name, source_dir, include_paths, exclude_paths, proj_id): """ 检查并创建分析方案 :return: 方案是否已存在或被创建，方案参数（如果返回False, 方案参数只包含方案名name） """ + # 指定了分析方案模板id，没有指定方案名称，会使用模板名称作为方案名称 + if not scan_plan and ref_scheme_id: + ref_scheme = dog_server.get_scheme_by_id(ref_scheme_id, org_sid) + if ref_scheme: + scan_plan = ref_scheme.get("name") + LogPrinter.info(f"use the name of refer scheme(id:{ref_scheme_id}) as scheme name: {scan_plan}") + # 有传分析方案名称，先通过名称判断，如果存在，直接复用 if scan_plan: is_existed, scheme_params = SchemeManager.check_scheme_by_name(repo_schemes, scan_plan) diff --git a/client/node/quicktask/quickrunner.py b/client/node/quicktask/quickrunner.py index a004f1b4c..ce0653ff1 100644 --- a/client/node/quicktask/quickrunner.py +++ b/client/node/quicktask/quickrunner.py @@ -15,11 +15,13 @@ import sys from node.localtask.runtask import ConcurrentTasksRuner +from node.localtask.localconfig import LocalConfig from node.common.taskdirmgr import TaskDirCtl from node.common.taskrunner import TaskRunner from node.common.userinput import UserInput from node.quicktask.quickscan import QuickScan from util import errcode +from util.api.dogserver import RetryDogServer from util.exceptions import NodeError from util.pathlib import PathMgr from util.scmurlmgr import BaseScmUrlMgr @@ -67,6 +69,18 @@ def __init__(self, args): self._labels = StringMgr.str_to_list(args.label) else: self._labels = [] + if args.token: + self._token = args.token + else: + self._token = None + if args.scheme_template_id: + self._scheme_template_id = args.scheme_template_id + else: + self._scheme_template_id = None + if args.org_sid: + self._org_sid = args.org_sid + else: + self._org_sid = None # 其他变量 self._scm_type = None @@ -195,6 +209,14 @@ def check_language(self, input_params): self._languages = list(language_types) LogPrinter.info(f"languages: {self._languages}") + def _merge_path_filters(self, default_filtered_paths, path_filters): + return { + "inclusion": path_filters["inclusion"] + default_filtered_paths["inclusion"], + "exclusion": path_filters["exclusion"] + default_filtered_paths["exclusion"], + "re_inclusion": path_filters["re_inclusion"] + default_filtered_paths["re_inclusion"], + "re_exclusion": path_filters["re_exclusion"] + default_filtered_paths["re_exclusion"] + } + def _generate_request(self, proj_conf, path_filters): """ 根据项目配置信息生成任务参数列表 @@ -204,9 +226,8 @@ def _generate_request(self, proj_conf, path_filters): job_context = proj_conf["job_context"] # 全量分析 job_context["incr_scan"] = False - - # # 检查代码库的账号密码权限 - # self._check_scm_authority() + # 合并过滤路径 + new_path_filters = self._merge_path_filters(job_context["path_filters"], path_filters) task_list = proj_conf["tasks"] @@ -214,10 +235,10 @@ def _generate_request(self, proj_conf, path_filters): task_params = task_request.get("task_params") task_params["incr_scan"] = False task_params["scm_last_revision"] = "" - task_params["path_filters"] = path_filters + task_params["path_filters"] = new_path_filters task_params["scm_url"] = self._scm_url task_params["scm_type"] = self._scm_type - task_params["project_id"] = job_context["project_id"] + task_params["project_id"] = job_context.get("project_id") task_request["execute_processes"] = task_request["processes"] task_request["private_processes"] = [] @@ -272,7 +293,18 @@ def run(self): self._scan_rel_paths, self._path_info = QuickScan.get_scan_paths(input_params, self._source_dir) self.check_language(input_params) # LogPrinter.info(f">>> input_params: {json.dumps(input_params, indent=2)}") - proj_conf = QuickScan.get_proj_config(self._scm_url, self._languages, self._labels, input_params) + if self._scheme_template_id: + if self._token and self._org_sid: # 通过server获取指定的分析方案模板的任务参数 + LogPrinter.info(f"token and scheme_template_id are set, get task config from server ...") + server_url = LocalConfig.get_server_url() + dog_server = RetryDogServer(server_url, self._token).get_api_server(retry_times=2) + proj_conf = dog_server.get_jobconfs_by_scheme_template(self._scheme_template_id, self._org_sid) + else: + raise NodeError(code=errcode.E_NODE_TASK_CONFIG, msg="已输入scheme_template_id, " + "但缺少--token和--org-sid参数, " + "无法获取到分析方案模板, 请补充参数!") + else: + proj_conf = QuickScan.get_proj_config(self._scm_url, self._languages, self._labels, input_params) # debug打印调试 # with open("quickscan_proj_config.json", "w") as wf: diff --git a/client/node/quicktask/toolloader.py b/client/node/quicktask/toolloader.py index d4231773f..b2d2f1200 100644 --- a/client/node/quicktask/toolloader.py +++ b/client/node/quicktask/toolloader.py @@ -8,12 +8,16 @@ """拉取quickscan工具 """ +from node.localtask.localconfig import LocalConfig from node.toolloader.loadtool import ToolLoader, ToolConfigLoader from node.quicktask.configloader import QuickScanConfigLoader from node.quicktask.quickscan import QuickScan from node.common.userinput import UserInput +from util import errcode +from util.api.dogserver import RetryDogServer from util.textutil import StringMgr from util.logutil import LogPrinter +from util.exceptions import NodeError class QuickScanToolLoader(object): @@ -35,7 +39,19 @@ def load_tools(args): # 从git拉取工具配置库 ToolConfigLoader().load_tool_config() # LogPrinter.info(f"---> labels: {labels}") - tool_tasks = QuickScan.get_scan_tasks(languages, labels, {}) + if args.scheme_template_id: + if args.token and args.org_sid: # 通过server获取指定的分析方案模板的任务参数 + LogPrinter.info(f"token and scheme_template_id are set, get task config from server ...") + server_url = LocalConfig.get_server_url() + dog_server = RetryDogServer(server_url, args.token).get_api_server(retry_times=2) + proj_conf = dog_server.get_jobconfs_by_scheme_template(args.scheme_template_id, args.org_sid) + tool_tasks = proj_conf["tasks"] + else: + raise NodeError(code=errcode.E_NODE_TASK_CONFIG, msg="已输入scheme_template_id, " + "但缺少--token和--org-sid参数, " + "无法获取到分析方案模板, 请补充参数!") + else: + tool_tasks = QuickScan.get_scan_tasks(languages, labels, {}) tool_names = [task['task_name'] for task in tool_tasks] LogPrinter.info("Initing other tools ...") ToolLoader(tool_names=tool_names, task_list=tool_tasks, include_common=False).git_load_tools(print_enable=False) diff --git a/client/settings/edition.py b/client/settings/edition.py index c9a6dfb74..9b233325e 100644 --- a/client/settings/edition.py +++ b/client/settings/edition.py @@ -28,4 +28,4 @@ class Edition(Enum): # 版本号 # ======================== # puppy版本号,格式：浮点数，整数部分为8位日期，小数部分为编号(从1开始) -VERSION = 20220620.1 +VERSION = 20220718.1 diff --git a/client/util/api/dogapi.py b/client/util/api/dogapi.py index 9ff1577d4..a1e69176b 100644 --- a/client/util/api/dogapi.py +++ b/client/util/api/dogapi.py @@ -453,3 +453,22 @@ def get_scan_conf(self, org_sid, team_name, repo_id, proj_id): rel_url = f"api/orgs/{org_sid}/teams/{team_name}/repos/{repo_id}/projects/{proj_id}/confs/" rsp = CodeDogHttpClient(self._server_url, rel_url, headers=self._headers).get() return self.get_data_from_result(rsp) + + def get_scheme_by_id(self, scheme_id, org_sid): + """根据分析方案模板id，获取方案模板的名称""" + rel_url = f"api/orgs/{org_sid}/schemes/{scheme_id}/" + rsp = CodeDogHttpClient(self._server_url, rel_url, headers=self._headers).get() + data = self.get_data_from_result(rsp) + return data + + def get_jobconfs_by_scheme_template(self, scheme_id, org_sid): + """ + 获取分析方案模板的任务执行参数 + :param scheme_id:分析方案模板id + :param org_sid: 团队编号 + :return: + """ + rel_url = f"api/orgs/{org_sid}/schemes/{scheme_id}/jobconfs/" + rsp = CodeDogHttpClient(self._server_url, rel_url, headers=self._headers).get() + data = self.get_data_from_result(rsp) + return data diff --git a/client/util/api/fileserver.py b/client/util/api/fileserver.py index 91029fda4..4ca2c380d 100644 --- a/client/util/api/fileserver.py +++ b/client/util/api/fileserver.py @@ -11,6 +11,7 @@ import logging import hashlib +import os from urllib.parse import urljoin from util import wrapper @@ -50,8 +51,10 @@ def __init__(self): 构造函数 :return: """ - self._server_url = settings.FILE_SERVER['URL'] - self._headers = {'Authorization': 'Token %s' % settings.FILE_SERVER['TOKEN']} + # 优先从环境变量读取文件服务器url和token，没有再使用默认值 + self._server_url = os.getenv("FILE_SERVER_URL", settings.FILE_SERVER['URL']) + file_server_token = os.getenv("FILE_SERVER_TOKEN", settings.FILE_SERVER['TOKEN']) + self._headers = {'Authorization': 'Token %s' % file_server_token} self._proxies = None def modify_save_time(self, rel_dir, days): @@ -75,7 +78,6 @@ def __upload_data(self, data, rel_url, headers): :return: 上传到服务器后的完整url, 如果上传失败, 返回 None """ file_url = urljoin(self._server_url, rel_url) - # logger.info(f">> file_url: {file_url}") HttpClient(self._server_url, rel_url, headers=headers, data=data, proxies=self._proxies).put() return file_url