Merge pull request #458 from Lingghh/dev/add_scanscheme_apis

✨增加扫描方案模板API&优化文件服务器鉴权交互

server/configs/django/local_analysis.py

@@ -80,6 +80,6 @@
 # 文件服务器
 FILE_SERVER = {
     "URL": os.environ.get("FILE_SERVER_URL", "http://127.0.0.1:8000/files/"),
-    "TOKEN": os.environ.get("FILE_SERVER_TOKEN"),
+    # "TOKEN": os.environ.get("FILE_SERVER_TOKEN"),
     "TYPE_PREFIX": os.environ.get("FILE_SERVER_TYPE", "public")
 }

server/configs/django/local_main.py

@@ -96,7 +96,7 @@
 # 文件服务器
 FILE_SERVER = {
     "URL": os.environ.get("FILE_SERVER_URL", "http://127.0.0.1:8000/files/"),
-    "TOKEN": os.environ.get("FILE_SERVER_TOKEN"),
+    # "TOKEN": os.environ.get("FILE_SERVER_TOKEN"),
     "TYPE_PREFIX": os.environ.get("FILE_SERVER_TYPE", "public")
 }
 

server/projects/analysis/util/fileserver.py

@@ -22,6 +22,7 @@
 from util.retrylib import RetryDecor
 from util.httpclient import HttpClient
 from util.exceptions import ServerError
+from util.authticket import MainServerTicket
 
 logger = logging.getLogger(__name__)
 
@@ -33,6 +34,8 @@ class FileServer(object):
     class TypeEnum(object):
         TEMPORARY = "server_temp"
         BUSINESS = "server_business"
+        CORE = "server_core"
+        ARCHIVE = "server_archive"
 
         @classmethod
         def values(cls):
@@ -47,27 +50,36 @@ def values(cls):
     def __init__(self, server_conf):
         self._http_client = HttpClient()
         self._server_url = server_conf["URL"]
-        self._server_token = server_conf["TOKEN"]
         self._type_prefix = server_conf["TYPE_PREFIX"]
-        self._headers = {'Authorization': 'Token %s' % self._server_token}
+        self._server_token = server_conf.get("TOKEN")
 
     @classmethod
     def get_data_md5(cls, data):
         """获取指定数据的md5
         """
         md5_val = hashlib.md5()
-        md5_val.update(data.encode("utf-8"))
+        tempdata = data if type(data) == bytes else data.encode("utf-8")
+        md5_val.update(tempdata)
         return md5_val.hexdigest()
 
     @classmethod
     def get_data_sha256(cls, data):
         """获取指定数据的sha256
         """
         sha256_val = hashlib.sha256()
-        sha256_val.update(data.encode("utf-8"))
+        tempdata = data if type(data) == bytes else data.encode("utf-8")
+        sha256_val.update(tempdata)
         return sha256_val.hexdigest()
 
-    @RetryDecor()
+    def get_auth_headers(self):
+        """获取头部信息
+        """
+        if self._server_token:
+            return {"Authorization": "Token %s" % self._server_token}
+        else:
+            return {"SERVER-TICKET": MainServerTicket.generate_ticket()}
+
+    @RetryDecor(interval=3)
     def put_file(self, data, to_file_url_or_path=None, type=TypeEnum.TEMPORARY):
         """将指定数据存储到文件服务器的指定路径下
         :params data: 需要上传的数据
@@ -87,20 +99,22 @@ def put_file(self, data, to_file_url_or_path=None, type=TypeEnum.TEMPORARY):
             # 无命名文件默认上传到临时文件夹，数据保留7天
             file_url = urllib.parse.urljoin(self._server_url, "%s_%s/unnamed/%s.file" % (
                 self._type_prefix, self.TypeEnum.TEMPORARY, uuid.uuid1().hex))
-        headers = copy.copy(self._headers)
+        headers = copy.copy(self.get_auth_headers())
         headers.update({"Content-SHA256": self.get_data_sha256(data), "Content-MD5": self.get_data_md5(data)})
         rsp = self._http_client.put(file_url, data=data, headers=headers)
         if rsp.status == self.OK_STATUS:
             return file_url
         else:
+            logger.error('return code %d when put file to %s, content: %s' % (
+                rsp.status, file_url, rsp.data.decode("utf-8")))
             raise ServerError(errcode.E_SERVER_FILE_SERVICE_ERROR,
                               'return code %d when put file to %s' % (rsp.status, file_url))
 
     @RetryDecor()
     def get_file(self, file_url):
         """获取指定路径文件数据，返回该文件的路径信息"""
         try:
-            rsp = self._http_client.get(file_url, headers=self._headers)
+            rsp = self._http_client.get(file_url, headers=self.get_auth_headers())
         except Exception as err:
             raise ServerError(errcode.E_SERVER_FILE_SERVICE_ERROR, ' get file exception: %s' % err)
         if rsp.status == self.OK_STATUS:
@@ -111,25 +125,26 @@ def get_file(self, file_url):
 
     def delete_file(self, file_url):
         """删除指定路径文件"""
-        rsp = self._http_client.delete(file_url, headers=self._headers)
+        rsp = self._http_client.delete(file_url, headers=self.get_auth_headers())
         if rsp.status == self.OK_STATUS:
             return True
         else:
             raise ServerError(errcode.E_SERVER_FILE_SERVICE_ERROR,
                               'return code %d when delete file %s' % (rsp.status, file_url))
 
     @RetryDecor()
-    def download_file(self, file_url):
+    def download_file(self, file_url, dir_path=None):
         """下载指定路径文件"""
         file_name = os.path.basename(file_url)
-        dir_path = os.path.join(settings.BASE_DIR, 'tempdata', "%s_%s" % (gen_path_key(file_url), uuid.uuid1().hex))
+        if not dir_path:
+            dir_path = os.path.join(settings.BASE_DIR, 'tempdata', "%s_%s" % (gen_path_key(file_url), uuid.uuid1().hex))
         file_path = os.path.join(dir_path, file_name)
         logger.info("Downloading file from %s to %s", file_url, file_path)
         if os.path.exists(file_path):
             logger.info("File already exists, remove...: %s" % file_path)
             os.remove(file_path)
         try:
-            rsp = self._http_client.get(file_url, headers=self._headers, stream=True)
+            rsp = self._http_client.get(file_url, headers=self.get_auth_headers(), stream=True)
         except Exception as err:
             raise ServerError(errcode.E_SERVER_FILE_SERVICE_ERROR, ' download file exception: %s' % err)
         if not os.path.isdir(dir_path):

server/projects/main/apps/codeproj/api_urls/v1_org.py

@@ -17,9 +17,10 @@
 # 项目内 import
 from apps.codeproj.apis import v1
 from apps.codeproj.apis import v1_pt
+from apps.codeproj.apis import v1_scheme
 
 # 前缀 /api/orgs/<org_sid>/teams/<team_name>/repos/
-repos_url_patterns = [
+repo_url_patterns = [
     # 节点使用
     path("defaultpaths/", v1.DefaultScanPathListApiView.as_view(),
          name="apiv1_pt_repo_defaultpath_list"),
@@ -43,11 +44,9 @@
          name="apiv1_pt_project_clocscans_overview"),
     path("<int:repo_id>/projects/<int:project_id>/codelint/issues/", v1_pt.PTAnalyseServerProxyAPIView.as_view(),
          name="apiv1_pt_project_codelint_issue"),
-    path("<int:repo_id>/projects/<int:project_id>/codelint/issues/epreport/",
-         v1_pt.PTAnalyseServerProxyAPIView.as_view(),
+    path("<int:repo_id>/projects/<int:project_id>/codelint/issues/epreport/", v1_pt.PTAnalyseServerProxyAPIView.as_view(),
          name="apiv1_pt_project_codelint_issue_epreport"),
-    path("<int:repo_id>/projects/<int:project_id>/codelint/issues/summary/",
-         v1_pt.PTAnalyseServerProxyAPIView.as_view(),
+    path("<int:repo_id>/projects/<int:project_id>/codelint/issues/summary/", v1_pt.PTAnalyseServerProxyAPIView.as_view(),
          name="apiv1_pt_project_codelint_summary"),
     path("<int:repo_id>/projects/<int:project_id>/scheme/defaultpaths/",
          v1_pt.PTProjectScanSchemeDefaultScanPathListAPIView.as_view(),
@@ -92,13 +91,29 @@
 ]
 
 # 前缀 /api/orgs/<org_sid>/teams/<team_name>/projects/
-projects_url_patterns = [
+project_url_patterns = [
     path("", v1_pt.PTProjectListCreateView.as_view(),
          name="apiv1_pt_project_list"),
 ]
 
-# 前缀 /api/orgs/<org_sid>/teams/
+# 前缀 /api/orgs/<org_sid>/schemes/
+# 分析方案模板路由
+global_scheme_urlpatterns = [
+    path("",
+         v1_scheme.OrgScanSchemeListAPIView.as_view(),
+         name="apiv1_org_scheme_list"),
+    path("<int:scheme_id>/",
+         v1_scheme.OrgScanSchemeDetailAPIView.as_view(),
+         name="apiv1_org_scheme_detail"),
+    path("<int:scheme_id>/jobconfs/",
+         v1_scheme.OrgScanSchemeScanJobConfAPIView.as_view(),
+         name="apiv1_org_scheme_scan_jobconfs"),
+
+]
+
+# 前缀 /api/orgs/<org_sid>/
 urlpatterns = [
-    path("<str:team_name>/repos/", include(repos_url_patterns)),
-    path("<str:team_name>/projects/", include(projects_url_patterns))
+    path("schemes/", include(global_scheme_urlpatterns)),
+    path("teams/<str:team_name>/repos/", include(repo_url_patterns)),
+    path("teams/<str:team_name>/projects/", include(project_url_patterns)),
 ]

server/projects/main/apps/codeproj/apis/v1.py

@@ -241,6 +241,25 @@ def post(self, request, *args, **kwargs):
                 return Response(e.data, status=status.HTTP_400_BAD_REQUEST)
 
 
+class ScanSchemeScanJobConfApiView(generics.GenericAPIView):
+    """获取指定扫描方案模板扫描配置
+    使用对象：节点
+
+    ### GET
+    应用场景：获取指定扫描方案模板扫描配置信息，供节点端离线扫描使用
+    """
+    schema = None
+    authentication_classes = [TCANodeTokenBackend]
+
+    def get(self, request, *args, **kwargs):
+        scan_scheme = get_object_or_404(models.ScanScheme, id=kwargs["scheme_id"], repo=None)
+        job_manager = core.JobManager(project=None)
+        try:
+            return Response(job_manager.get_job_confs(scan_scheme=scan_scheme))
+        except CDErrorBase as e:
+            return ParseError({"errcode": e.code, "errmsg": e.msg})
+
+
 class ProjectScanDetailApiView(generics.GenericAPIView, ProjectBaseAPIView):
     """项目指定扫描详情接口
     使用对象：节点

server/projects/main/apps/codeproj/apis/v1_scheme.py

@@ -0,0 +1,70 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2021-2022 THL A29 Limited
+#
+# This source code file is made available under MIT License
+# See LICENSE for details
+# ==============================================================================
+
+"""
+codeproj - api v1 for scheme templates
+"""
+# 原生
+import logging
+
+# 第三方
+from django_filters.rest_framework import DjangoFilterBackend
+from rest_framework import generics
+
+# 项目内
+from apps.authen.permissions import OrganizationOperationPermission
+from apps.base.apimixins import V3GetModelMixinAPIView
+from apps.codeproj.api_filters import base as base_filters
+from apps.codeproj.apis import v1
+from apps.codeproj.core import ScanSchemeManager
+from apps.codeproj.permissions import GlobalSchemeDefaultPermission
+from apps.codeproj.serializers import v3_scheme as v3_scheme_serializers
+
+logger = logging.getLogger(__name__)
+
+
+class OrgScanSchemeListAPIView(generics.ListAPIView, V3GetModelMixinAPIView):
+    """分析方案模板列表接口
+
+    ### GET
+    应用场景：获取分析方案列表
+    筛选项：
+    ```python
+    name: str, 方案模板名称，包含
+    scope: str, 过滤范围，all全部，system系统模板，not_system非系统模板，editable有权限编辑模板
+    ```
+    """
+    permission_classes = [OrganizationOperationPermission]
+    serializer_class = v3_scheme_serializers.GlobalScanSchemeTemplateSerializer
+    filter_backends = (DjangoFilterBackend,)
+    filterset_class = base_filters.GlobalScanSchemeTemplateFilter
+
+    def get_queryset(self):
+        org = self.get_org()
+        return ScanSchemeManager.filter_usable_global_scheme_templates(org.org_sid, self.request.user)
+
+
+class OrgScanSchemeDetailAPIView(generics.RetrieveAPIView, V3GetModelMixinAPIView):
+    """分析方案模板详情接口
+
+    ### GET
+    应用场景：获取分析方案模板详情
+    """
+    permission_classes = [GlobalSchemeDefaultPermission]
+    serializer_class = v3_scheme_serializers.GlobalScanSchemeTemplateSerializer
+
+    def get_object(self):
+        return self.get_scheme()
+
+
+class OrgScanSchemeScanJobConfAPIView(v1.ScanSchemeScanJobConfApiView):
+    """获取指定扫描方案模板的扫描配置
+
+    ### GET
+    应用场景：获取指定扫描方案模板配置的api，供节点端离线扫描使用
+    """
+    permission_classes = [GlobalSchemeDefaultPermission]