From 79d1f8e7a0198c5c01dbb3d1708f1add1dedbfd6 Mon Sep 17 00:00:00 2001 From: devopstales <42894256+devopstales@users.noreply.github.com> Date: Sat, 9 Oct 2021 09:54:33 +0200 Subject: [PATCH 1/8] Admission Controller base commit --- trivy-operator.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/trivy-operator.py b/trivy-operator.py index a65d81f..1c9ab6e 100644 --- a/trivy-operator.py +++ b/trivy-operator.py @@ -24,6 +24,10 @@ password: "password" """ +############################################################################# +# Pretasks +############################################################################# + """Deploy CRDs""" @kopf.on.startup() async def startup_fn_crd(logger, **kwargs): @@ -116,6 +120,10 @@ async def startup_fn_trivy_cache(logger, **kwargs): ) logger.info("trivy cache created...") +############################################################################# +# Operator +############################################################################# + """Scanner Creation""" @kopf.on.create('trivy-operator.devopstales.io', 'v1', 'namespace-scanners') async def create_fn(logger, spec, **kwargs): @@ -230,6 +238,21 @@ async def create_fn(logger, spec, **kwargs): else: await asyncio.sleep(15) +############################################################################# +# Admission Controller +############################################################################# + +@kopf.on.startup() +def configure(settings: kopf.OperatorSettings, **_): + # Auto-detect the best server (K3d/Minikube/simple) with external tunneling as a fallback: + settings.admission.server = kopf.WebhookAutoTunnel() + settings.admission.managed = 'trivy-image-validator.devopstales.io' + +@kopf.on.validate('pod') +def validate1(logger, spec, dryrun, **_): + logger.info("Admission Controller is working") + +############################################################################# ## print to operator log # print(f"And here we are! Creating: %s" % (ns_name), file=sys.stderr) # debug ## message to CR From fd376e6c86cc795b17b1853ac2d5536119e91f47 Mon Sep 17 00:00:00 2001 From: devopstales <42894256+devopstales@users.noreply.github.com> Date: Sun, 10 Oct 2021 14:12:10 +0200 Subject: [PATCH 2/8] add main AC logic --- docker/Dockerfile | 2 +- trivy-operator.py | 133 ++++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 125 insertions(+), 10 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 7446e81..d36c8fe 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -6,7 +6,7 @@ ENV TRIVY_CACHE_DIR=/home/trivy-operator/trivy-cache \ RUN apk add --no-cache gcc musl-dev -RUN pip3 install --no-cache-dir kopf kubernetes asyncio pycron prometheus_client +RUN pip3 install --no-cache-dir kopf kubernetes asyncio pycron prometheus_client certvalidator certbuilder COPY trivy-operator.py /trivy-operator.py COPY trivy /usr/local/bin diff --git a/trivy-operator.py b/trivy-operator.py index 1c9ab6e..9c4a172 100644 --- a/trivy-operator.py +++ b/trivy-operator.py @@ -24,6 +24,25 @@ password: "password" """ +############################################################################# +# ToDo +############################################################################# +# initContainers ??? +# AC: config from CR? +### gen CRD +### get registry user pass from CR if exists +# AC: prometheus] +## container vulns +## block and accept +# AC: cache scanned images + +############################################################################# +# Global Variables +############################################################################# +CONTAINER_VULN = prometheus_client.Gauge('so_vulnerabilities', 'Container vulnerabilities', ['exported_namespace', 'image', 'severity']) +AC_VULN = prometheus_client.Gauge('ac_vulnerabilities', 'Admission Controller vulnerabilities', ['exported_namespace', 'image', 'severity']) +IN_CLUSTER = os.getenv("IN_CLUSTER", False) + ############################################################################# # Pretasks ############################################################################# @@ -120,6 +139,12 @@ async def startup_fn_trivy_cache(logger, **kwargs): ) logger.info("trivy cache created...") +#"""Start Prometheus Exporter""" +@kopf.on.startup() +async def startup_fn_prometheus_client(logger, **kwargs): + prometheus_client.start_http_server(9115) + logger.info("Prometheus Exporter started...") + ############################################################################# # Operator ############################################################################# @@ -127,11 +152,7 @@ async def startup_fn_trivy_cache(logger, **kwargs): """Scanner Creation""" @kopf.on.create('trivy-operator.devopstales.io', 'v1', 'namespace-scanners') async def create_fn(logger, spec, **kwargs): - CONTAINER_VULN = prometheus_client.Gauge('so_vulnerabilities', 'Container vulnerabilities', ['exported_namespace', 'image', 'severity']) - - """Start Prometheus Exporter""" - prometheus_client.start_http_server(9115) - logger.info("Prometheus Exporter started...") + logger.info("NamespaceScanner Created") try: crontab = spec['crontab'] @@ -148,7 +169,6 @@ async def create_fn(logger, spec, **kwargs): while True: if pycron.is_now(crontab): """Find Namespaces""" - IN_CLUSTER = os.getenv("IN_CLUSTER", False) image_list = {} vul_list = {} tagged_ns_list = [] @@ -245,12 +265,107 @@ async def create_fn(logger, spec, **kwargs): @kopf.on.startup() def configure(settings: kopf.OperatorSettings, **_): # Auto-detect the best server (K3d/Minikube/simple) with external tunneling as a fallback: - settings.admission.server = kopf.WebhookAutoTunnel() + settings.admission.server = kopf.WebhookAutoTunnel(port=443) + # settings.admission.server = kopf.WebhookServer(port=443, host="k3s") settings.admission.managed = 'trivy-image-validator.devopstales.io' -@kopf.on.validate('pod') -def validate1(logger, spec, dryrun, **_): +@kopf.on.validate('pod', operation='CREATE') +def validate1(logger, namespace, name, **_): logger.info("Admission Controller is working") + vul_list = {} + image_list = {} + + if IN_CLUSTER: + k8s_config.load_incluster_config() + else: + k8s_config.load_kube_config() + + """Find pods in namespace""" + pod_list = k8s_client.CoreV1Api().list_namespaced_pod(namespace) + + """Find images in pods""" + for pod in pod_list.items: + pod_name = pod.metadata.name + if pod_name == name: + images = pod.status.container_statuses + annotations = pod.metadata.annotations + for image in images: + image_name = image.image + image_id = image.image_id + image_list[pod_name] = list() + image_list[pod_name].append(image_name) + image_list[pod_name].append(image_id) + image_list[pod_name].append(namespace) + + """Scan images""" + for image in image_list: + image_name = image_list[image][0] + image_id = image_list[image][1] + ns_name = image_list[image][2] + registry = image_name.split('/')[0] + logger.info("Scanning Image: %s" % (image_name)) + + TRIVY = ["trivy", "-q", "i", "-f", "json", image_name] + # --ignore-policy trivy.rego + + res = subprocess.Popen(TRIVY,stdout=subprocess.PIPE,stderr=subprocess.PIPE); + output,error = res.communicate() + if output: + trivy_result = json.loads(output.decode("UTF-8")) + item_list = trivy_result['Results'][0]["Vulnerabilities"] + vuls = { + "UNKNOWN": 0,"LOW": 0, + "MEDIUM": 0,"HIGH": 0, + "CRITICAL": 0 + } + for item in item_list: + vuls[item["Severity"]] += 1 + vul_list[image_name] = vuls + + """Generate Metricfile""" + for image_name in vul_list.keys(): + for severity in vul_list[image_name][0].keys(): + AC_VULN.labels(vul_list[image_name][1], image_name, severity).set(int(vul_list[image_name][0][severity])) + + if error: + logger.error("TRIVY ERROR: return %s" % (res.returncode)) + if b"401" in error.strip(): + logger.error("Repository: Unauthorized authentication required") + if b"UNAUTHORIZED" in error.strip(): + logger.error("Repository: Unauthorized authentication required") + if b"You have reached your pull rate limit." in error.strip(): + logger.error("You have reached your pull rate limit.") + + """Get vulnerabilities from annotations""" + vul_annotations= { + "UNKNOWN": 0,"LOW": 0, + "MEDIUM": 0,"HIGH": 0, + "CRITICAL": 0 + } + for sev in vul_annotations: + try: + print(sev + ": " + annotations['trivy.security.devopstales.io/' + sev.lower()], file=sys.stderr) # Debug + vul_annotations[sev["Severity"]] = int(annotations['trivy.security.devopstales.io/' + sev.lower()]) + except: + continue + + """Check vulnerabilities""" + print("Check vulnerabilities:", file=sys.stderr) # Debug + for sev in vul_annotations: + try: + an_vul_num = vul_annotations[sev] + vul_num = vul_list[image_name][sev] + if vul_num > an_vul_num: + print(sev + " is bigger", file=sys.stderr) # Debug + raise kopf.AdmissionError(f"Too much vulnerability in the image: %s" % (image_name)) + else: + print(sev + " is ok", file=sys.stderr) # Debug + continue + except: + continue + + # print(f"%s" % (image_name), file=sys.stderr) # debug + ############################################################################# ## print to operator log From 221870be684929a2d1046efcb37b9133baa2c06f Mon Sep 17 00:00:00 2001 From: devopstales <42894256+devopstales@users.noreply.github.com> Date: Sun, 10 Oct 2021 14:26:11 +0200 Subject: [PATCH 3/8] add main AC logic --- README.md | 31 +++++++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 0a1bd4b..427b2f3 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,21 @@ # Trivy Operator -Trivy Operator is an operator that default every 5 minutes execute a scan script. It will get image list from all namespaces with the label `trivy-scan=true`, and then scan this images with trivy, finally we will get metrics on `http://[pod-ip]:9115/metrics` - Built with [kopf](https://github.com/nolar/kopf) +Main functions: + +* Scheduled Image scans on running pods +* Trivy Image Validator Admission controller + +Inspirated by [knqyf263](https://github.com/knqyf263)'s [trivy-enforcer](https://github.com/aquasecurity/trivy-enforcer) and [fleeto](https://github.com/fleeto)'s [trivy-scanner](https://github.com/fleeto/trivy-scanner). + +### Schefuled Image scans +Default every 5 minutes execute a scan script. It will get image list from all namespaces with the label `trivy-scan=true`, and then scan this images with trivy, finally we will get metrics on `http://[pod-ip]:9115/metrics` + +### Trivy Image Validator +The admission controller function can be configured as a ValidatingWebhook in a k8s cluster. Kubernetes will send requests to the admission server when a Pod creation is initiated. The admission controller checks the image using trivy. + + ## Usage ```bash @@ -54,3 +66,18 @@ kubectl logs [2021-10-02 09:45:52,227] kopf.objects [INFO ] [trivytest/main-config] Scanning Image: docker.io/library/nginx:1.18 [2021-10-02 09:45:55,556] kopf.objects [INFO ] [trivytest/main-config] Scanning Image: docker.io/library/nginx:latest ~~~ + +### Development + +To run kopf development you need to install the fallowing packages to the k3s host: + +```bash +yum install python3-8 +pip3 install kopf kubernetes asyncio pycron prometheus_client certvalidator certbuilder +``` + +The admission webhook try to call the host with the domain name `host.k3d.internal` so I added to the host's `/etc/host` file. + +```yaml +echo "172.17.12.10 host.k3d.internal" >> /etc/host +``` \ No newline at end of file From 57c600fda25440fff1fdfce15a4b4e5f1f9d356f Mon Sep 17 00:00:00 2001 From: devopstales <42894256+devopstales@users.noreply.github.com> Date: Sun, 10 Oct 2021 14:28:31 +0200 Subject: [PATCH 4/8] add main AC logic --- README.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/README.md b/README.md index 427b2f3..51ef487 100644 --- a/README.md +++ b/README.md @@ -67,6 +67,21 @@ kubectl logs [2021-10-02 09:45:55,556] kopf.objects [INFO ] [trivytest/main-config] Scanning Image: docker.io/library/nginx:latest ~~~ +### Example Deploy: +You can define policy to the Admission Controller, by adding annotation to the pod trough the deployment: + +```yaml +spec: + ... + template: + metadata: + annotations: + trivy.security.devopstales.io/medium: "5" + trivy.security.devopstales.io/low: "10" + trivy.security.devopstales.io/critical: "2" +... +``` + ### Development To run kopf development you need to install the fallowing packages to the k3s host: From 60f55b4a4c14fbd5f9b3d5fbaeee1a5e9ec49f75 Mon Sep 17 00:00:00 2001 From: devopstales <42894256+devopstales@users.noreply.github.com> Date: Sat, 13 Nov 2021 09:33:15 +0100 Subject: [PATCH 5/8] add CRD #1 --- deploy/02_crds.yaml | 51 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 deploy/02_crds.yaml diff --git a/deploy/02_crds.yaml b/deploy/02_crds.yaml new file mode 100644 index 0000000..7b2815a --- /dev/null +++ b/deploy/02_crds.yaml @@ -0,0 +1,51 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: namespace-scanners.trivy-operator.devopstales.io +spec: + conversion: + strategy: None + group: trivy-operator.devopstales.io + names: + kind: NamespaceScanner + listKind: NamespaceScannerList + plural: namespace-scanners + shortNames: + - ns-scan + singular: namespace-scanner + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Namespace Selector for pod scanning + jsonPath: .spec.namespace_selector + name: NamespaceSelector + type: string + - description: crontab value + jsonPath: .spec.crontab + name: Crontab + type: string + - description: As returned from the handler (sometimes). + jsonPath: .status.create_fn.message + name: Message + type: string + name: v1 + schema: + openAPIV3Schema: + properties: + crontab: + pattern: ^(\d+|\*)(/\d+)?(\s+(\d+|\*)(/\d+)?){4}$ + type: string + namespace_selector: + type: string + spec: + type: object + x-kubernetes-preserve-unknown-fields: true + status: + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: true + subresources: + status: {} + From 72845b1ff61b64b0acf3e808f8cbb5117156ecca Mon Sep 17 00:00:00 2001 From: devopstales <42894256+devopstales@users.noreply.github.com> Date: Sat, 13 Nov 2021 18:29:37 +0100 Subject: [PATCH 6/8] release 2.0.0-devel-1 --- README.md | 26 ++++++-- deploy/02_crds.yaml | 51 --------------- docker/Dockerfile | 3 +- trivy-operator.py | 147 ++++++++++++++++++++++++++++---------------- 4 files changed, 117 insertions(+), 110 deletions(-) delete mode 100644 deploy/02_crds.yaml diff --git a/README.md b/README.md index 51ef487..2a01e2f 100644 --- a/README.md +++ b/README.md @@ -84,15 +84,33 @@ spec: ### Development +Install trivy: + +```bash +nano /etc/yum.repos.d/trivy.repo +[trivy] +name=Trivy repository +baseurl=https://aquasecurity.github.io/trivy-repo/rpm/releases/$releasever/$basearch/ +gpgcheck=0 +enabled=1 + +sudo yum -y install trivy +``` + To run kopf development you need to install the fallowing packages to the k3s host: ```bash -yum install python3-8 -pip3 install kopf kubernetes asyncio pycron prometheus_client certvalidator certbuilder +yum install -y python3.8 +pip3 install --no-cache-dir kopf kubernetes asyncio pycron prometheus_client certvalidator certbuilder +pip3 install --no-cache-dir kopf[devel] ``` The admission webhook try to call the host with the domain name `host.k3d.internal` so I added to the host's `/etc/host` file. -```yaml +```bash echo "172.17.12.10 host.k3d.internal" >> /etc/host -``` \ No newline at end of file +``` + +```bash +kopf run -A ./trivy-operator.py +``` diff --git a/deploy/02_crds.yaml b/deploy/02_crds.yaml deleted file mode 100644 index 7b2815a..0000000 --- a/deploy/02_crds.yaml +++ /dev/null @@ -1,51 +0,0 @@ -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: namespace-scanners.trivy-operator.devopstales.io -spec: - conversion: - strategy: None - group: trivy-operator.devopstales.io - names: - kind: NamespaceScanner - listKind: NamespaceScannerList - plural: namespace-scanners - shortNames: - - ns-scan - singular: namespace-scanner - scope: Namespaced - versions: - - additionalPrinterColumns: - - description: Namespace Selector for pod scanning - jsonPath: .spec.namespace_selector - name: NamespaceSelector - type: string - - description: crontab value - jsonPath: .spec.crontab - name: Crontab - type: string - - description: As returned from the handler (sometimes). - jsonPath: .status.create_fn.message - name: Message - type: string - name: v1 - schema: - openAPIV3Schema: - properties: - crontab: - pattern: ^(\d+|\*)(/\d+)?(\s+(\d+|\*)(/\d+)?){4}$ - type: string - namespace_selector: - type: string - spec: - type: object - x-kubernetes-preserve-unknown-fields: true - status: - type: object - x-kubernetes-preserve-unknown-fields: true - type: object - served: true - storage: true - subresources: - status: {} - diff --git a/docker/Dockerfile b/docker/Dockerfile index d36c8fe..5cbb25e 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -6,7 +6,8 @@ ENV TRIVY_CACHE_DIR=/home/trivy-operator/trivy-cache \ RUN apk add --no-cache gcc musl-dev -RUN pip3 install --no-cache-dir kopf kubernetes asyncio pycron prometheus_client certvalidator certbuilder +RUN pip3 install --no-cache-dir kopf kubernetes asyncio pycron prometheus_client certvalidator certbuilder \ + pip3 install --no-cache-dir kopf[dev] COPY trivy-operator.py /trivy-operator.py COPY trivy /usr/local/bin diff --git a/trivy-operator.py b/trivy-operator.py index 9c4a172..461c279 100644 --- a/trivy-operator.py +++ b/trivy-operator.py @@ -8,6 +8,7 @@ import sys import subprocess import json +import validators """ apiVersion: trivy-operator.devopstales.io/v1 @@ -113,7 +114,6 @@ async def startup_fn_crd(logger, **kwargs): ) ) - IN_CLUSTER = os.getenv("IN_CLUSTER", False) if IN_CLUSTER: k8s_config.load_incluster_config() else: @@ -228,7 +228,16 @@ async def create_fn(logger, spec, **kwargs): res = subprocess.Popen(TRIVY,stdout=subprocess.PIPE,stderr=subprocess.PIPE); output,error = res.communicate() - if output: + + if error: + logger.error("TRIVY ERROR: return %s" % (res.returncode)) + if b"401" in error.strip(): + logger.error("Repository: Unauthorized authentication required") + if b"UNAUTHORIZED" in error.strip(): + logger.error("Repository: Unauthorized authentication required") + if b"You have reached your pull rate limit." in error.strip(): + logger.error("You have reached your pull rate limit.") + elif output: trivy_result = json.loads(output.decode("UTF-8")) item_list = trivy_result['Results'][0]["Vulnerabilities"] vuls = { @@ -245,15 +254,6 @@ async def create_fn(logger, spec, **kwargs): for severity in vul_list[image_name][0].keys(): CONTAINER_VULN.labels(vul_list[image_name][1], image_name, severity).set(int(vul_list[image_name][0][severity])) - if error: - logger.error("TRIVY ERROR: return %s" % (res.returncode)) - if b"401" in error.strip(): - logger.error("Repository: Unauthorized authentication required") - if b"UNAUTHORIZED" in error.strip(): - logger.error("Repository: Unauthorized authentication required") - if b"You have reached your pull rate limit." in error.strip(): - logger.error("You have reached your pull rate limit.") - await asyncio.sleep(15) else: await asyncio.sleep(15) @@ -261,56 +261,100 @@ async def create_fn(logger, spec, **kwargs): ############################################################################# # Admission Controller ############################################################################# +# namespace selector for admission controller [ ] @kopf.on.startup() def configure(settings: kopf.OperatorSettings, **_): - # Auto-detect the best server (K3d/Minikube/simple) with external tunneling as a fallback: - settings.admission.server = kopf.WebhookAutoTunnel(port=443) - # settings.admission.server = kopf.WebhookServer(port=443, host="k3s") + # Auto-detect the best server (K3d/Minikube/simple): + settings.admission.server = kopf.WebhookAutoServer(port=443) settings.admission.managed = 'trivy-image-validator.devopstales.io' @kopf.on.validate('pod', operation='CREATE') -def validate1(logger, namespace, name, **_): +def validate1(logger, namespace, name, annotations, spec, **_): logger.info("Admission Controller is working") + image_list = [] vul_list = {} - image_list = {} + registry_list = {} + """Try to get Registry auth values""" if IN_CLUSTER: k8s_config.load_incluster_config() else: k8s_config.load_kube_config() - - """Find pods in namespace""" - pod_list = k8s_client.CoreV1Api().list_namespaced_pod(namespace) - - """Find images in pods""" - for pod in pod_list.items: - pod_name = pod.metadata.name - if pod_name == name: - images = pod.status.container_statuses - annotations = pod.metadata.annotations - for image in images: - image_name = image.image - image_id = image.image_id - image_list[pod_name] = list() - image_list[pod_name].append(image_name) - image_list[pod_name].append(image_id) - image_list[pod_name].append(namespace) - - """Scan images""" + try: # if no namespace-scanners created + nsScans = k8s_client.CustomObjectsApi().list_cluster_custom_object( + group="trivy-operator.devopstales.io", + version="v1", + plural="namespace-scanners", + ) + for nss in nsScans["items"]: + registry_list = nss["spec"]["registry"] + except: + logger.info("No ns-scan object created yet.") + + """Get conainers""" + containers = spec.get('containers') + initContainers = spec.get('initContainers') + + for icn in initContainers: + try: + initContainers_array = json.dumps(icn) + initContainer = json.loads(initContainers_array) + image_name = initContainer["image"] + image_list.append(image_name) + except: + continue + + for cn in containers: + container_array = json.dumps(cn) + container = json.loads(container_array) + image_name = container["image"] + image_list.append(image_name) + + """Get Images""" for image in image_list: - image_name = image_list[image][0] - image_id = image_list[image][1] - ns_name = image_list[image][2] + image_name = image registry = image_name.split('/')[0] logger.info("Scanning Image: %s" % (image_name)) + """Login to refistry""" + try: + for reg in registry_list: + if reg['name'] == registry: + os.environ['DOCKER_REGISTRY']=reg['name'] + os.environ['TRIVY_USERNAME']=reg['user'] + os.environ['TRIVY_PASSWORD']=reg['password'] + elif not validators.domain(registry): + """If registry is not an url""" + if reg['name'] == "docker.io": + os.environ['DOCKER_REGISTRY']=reg['name'] + os.environ['TRIVY_USERNAME']=reg['user'] + os.environ['TRIVY_PASSWORD']=reg['password'] + except: + logger.info("No registry auth config is defined.") + ACTIVE_REGISTRY = os.getenv("DOCKER_REGISTRY") + logger.info("Active Registry: %s" % (ACTIVE_REGISTRY)) + + """Scan Images""" TRIVY = ["trivy", "-q", "i", "-f", "json", image_name] # --ignore-policy trivy.rego res = subprocess.Popen(TRIVY,stdout=subprocess.PIPE,stderr=subprocess.PIPE); output,error = res.communicate() - if output: + if error: + logger.error("TRIVY ERROR: return %s" % (res.returncode)) + if b"401" in error.strip(): + logger.error("Repository: Unauthorized authentication required") + elif b"UNAUTHORIZED" in error.strip(): + logger.error("Repository: Unauthorized authentication required") + elif b"You have reached your pull rate limit." in error.strip(): + logger.error("You have reached your pull rate limit.") + elif b"unsupported MediaType" in error.strip(): + logger.error("Unsupported MediaType: see https://github.com/google/go-containerregistry/issues/377") + else: + logger.error("%s" % (error.strip())) + + elif output: trivy_result = json.loads(output.decode("UTF-8")) item_list = trivy_result['Results'][0]["Vulnerabilities"] vuls = { @@ -320,23 +364,18 @@ def validate1(logger, namespace, name, **_): } for item in item_list: vuls[item["Severity"]] += 1 - vul_list[image_name] = vuls + vul_list[image_name] = [vuls, namespace] """Generate Metricfile""" for image_name in vul_list.keys(): for severity in vul_list[image_name][0].keys(): + """Generate log""" +# logger.info("%s - %s: %s" % (vul_list[image_name][1], image_name, severity)) AC_VULN.labels(vul_list[image_name][1], image_name, severity).set(int(vul_list[image_name][0][severity])) - if error: - logger.error("TRIVY ERROR: return %s" % (res.returncode)) - if b"401" in error.strip(): - logger.error("Repository: Unauthorized authentication required") - if b"UNAUTHORIZED" in error.strip(): - logger.error("Repository: Unauthorized authentication required") - if b"You have reached your pull rate limit." in error.strip(): - logger.error("You have reached your pull rate limit.") - - """Get vulnerabilities from annotations""" +############################################################################# +""" + # Get vulnerabilities from annotations vul_annotations= { "UNKNOWN": 0,"LOW": 0, "MEDIUM": 0,"HIGH": 0, @@ -349,7 +388,7 @@ def validate1(logger, namespace, name, **_): except: continue - """Check vulnerabilities""" + # Check vulnerabilities print("Check vulnerabilities:", file=sys.stderr) # Debug for sev in vul_annotations: try: @@ -365,12 +404,12 @@ def validate1(logger, namespace, name, **_): continue # print(f"%s" % (image_name), file=sys.stderr) # debug - - +""" ############################################################################# ## print to operator log # print(f"And here we are! Creating: %s" % (ns_name), file=sys.stderr) # debug ## message to CR # return {'message': 'hello world'} # will be the new status ## events to CR describe -# kopf.event(body, type="SomeType", reason="SomeReason", message="Some message") \ No newline at end of file +# kopf.event(body, type="SomeType", reason="SomeReason", message="Some message") + From 1f6e982484668159228e0cbc4f8471f3f6e274ac Mon Sep 17 00:00:00 2001 From: devopstales <42894256+devopstales@users.noreply.github.com> Date: Sun, 14 Nov 2021 14:36:38 +0100 Subject: [PATCH 7/8] release 2.0.0-devel-2 --- deploy/10_demo.yaml | 12 +++- docker/Dockerfile | 4 +- trivy-operator.py | 151 +++++++++++++++++++++++++------------------- 3 files changed, 99 insertions(+), 68 deletions(-) diff --git a/deploy/10_demo.yaml b/deploy/10_demo.yaml index 4532543..3623c22 100644 --- a/deploy/10_demo.yaml +++ b/deploy/10_demo.yaml @@ -12,6 +12,13 @@ metadata: name: nginx namespace: trivytest spec: + initContainers: + - name: init + image: nginxinc/nginx-unprivileged:latest + command: ['sh', '-c', 'echo The app is running! && sleep 10'] + - name: init2 + image: nginxinc/nginx-unprivileged:latest + command: ['sh', '-c', 'echo The app is running! && sleep 10'] containers: - image: nginx:1.18 imagePullPolicy: IfNotPresent @@ -26,4 +33,7 @@ spec: containers: - image: nginx imagePullPolicy: IfNotPresent - name: nginx \ No newline at end of file + name: nginx + - image: nginx + imagePullPolicy: IfNotPresent + name: nginx2 diff --git a/docker/Dockerfile b/docker/Dockerfile index 5cbb25e..bee5b2c 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -6,8 +6,8 @@ ENV TRIVY_CACHE_DIR=/home/trivy-operator/trivy-cache \ RUN apk add --no-cache gcc musl-dev -RUN pip3 install --no-cache-dir kopf kubernetes asyncio pycron prometheus_client certvalidator certbuilder \ - pip3 install --no-cache-dir kopf[dev] +RUN pip3 install --no-cache-dir kopf kubernetes asyncio pycron prometheus_client certvalidator certbuilder +RUN pip3 install --no-cache-dir kopf[dev] COPY trivy-operator.py /trivy-operator.py COPY trivy /usr/local/bin diff --git a/trivy-operator.py b/trivy-operator.py index 461c279..dcfad41 100644 --- a/trivy-operator.py +++ b/trivy-operator.py @@ -28,14 +28,11 @@ ############################################################################# # ToDo ############################################################################# -# initContainers ??? -# AC: config from CR? -### gen CRD -### get registry user pass from CR if exists -# AC: prometheus] -## container vulns -## block and accept -# AC: cache scanned images +# OP +# AC +## namespace selector for admission controller webhook +# cache scanned images ??? + ############################################################################# # Global Variables @@ -195,17 +192,34 @@ async def create_fn(logger, spec, **kwargs): pod_list = k8s_client.CoreV1Api().list_namespaced_pod(tagged_ns) """Find images in pods""" for pod in pod_list.items: - pod_name = pod.metadata.name - images = pod.status.container_statuses - for image in images: + Containers = pod.status.container_statuses + for image in Containers: + pod_name = pod.metadata.name + pod_name += '_' + pod_name += image.name + image_list[pod_name] = list() image_name = image.image image_id = image.image_id - image_list[pod_name] = list() image_list[pod_name].append(image_name) image_list[pod_name].append(image_id) image_list[pod_name].append(tagged_ns) + try: + initContainers = pod.status.init_container_statuses + for image in initContainers: + pod_name = pod.metadata.name + pod_name += '_' + pod_name += image.name + image_list[pod_name] = list() + image_name = image.image + image_id = image.image_id + image_list[pod_name].append(image_name) + image_list[pod_name].append(image_id) + image_list[pod_name].append(tagged_ns) + except: + continue """Scan images""" + logger.info("%s" % (image_list)) # debug for image in image_list: logger.info("Scanning Image: %s" % (image_list[image][0])) image_name = image_list[image][0] @@ -218,10 +232,19 @@ async def create_fn(logger, spec, **kwargs): for reg in registry_list: if reg['name'] == registry: + os.environ['DOCKER_REGISTRY']=reg['name'] os.environ['TRIVY_USERNAME']=reg['user'] os.environ['TRIVY_PASSWORD']=reg['password'] + elif not validators.domain(registry): + """If registry is not an url""" + if reg['name'] == "docker.io": + os.environ['DOCKER_REGISTRY']=reg['name'] + os.environ['TRIVY_USERNAME']=reg['user'] + os.environ['TRIVY_PASSWORD']=reg['password'] except: - logger.info("no registry auth config is defined") + logger.info("No registry auth config is defined.") + ACTIVE_REGISTRY = os.getenv("DOCKER_REGISTRY") + logger.info("Active Registry: %s" % (ACTIVE_REGISTRY)) # Debug TRIVY = ["trivy", "-q", "i", "-f", "json", image_name] # --ignore-policy trivy.rego @@ -230,29 +253,33 @@ async def create_fn(logger, spec, **kwargs): output,error = res.communicate() if error: + """Error Logging""" logger.error("TRIVY ERROR: return %s" % (res.returncode)) if b"401" in error.strip(): logger.error("Repository: Unauthorized authentication required") - if b"UNAUTHORIZED" in error.strip(): + elif b"UNAUTHORIZED" in error.strip(): logger.error("Repository: Unauthorized authentication required") - if b"You have reached your pull rate limit." in error.strip(): + elif b"You have reached your pull rate limit." in error.strip(): logger.error("You have reached your pull rate limit.") + elif b"unsupported MediaType" in error.strip(): + logger.error("Unsupported MediaType: see https://github.com/google/go-containerregistry/issues/377") + else: + logger.error("%s" % (error.strip())) + """Error action""" + vuls = { "scanning_error": 1 } + vul_list[image_name] = [vuls, ns_name] elif output: trivy_result = json.loads(output.decode("UTF-8")) item_list = trivy_result['Results'][0]["Vulnerabilities"] - vuls = { - "UNKNOWN": 0,"LOW": 0, - "MEDIUM": 0,"HIGH": 0, - "CRITICAL": 0 - } + vuls = { "UNKNOWN": 0,"LOW": 0,"MEDIUM": 0,"HIGH": 0,"CRITICAL": 0 } for item in item_list: vuls[item["Severity"]] += 1 vul_list[image_name] = [vuls, ns_name] - """Generate Metricfile""" - for image_name in vul_list.keys(): - for severity in vul_list[image_name][0].keys(): - CONTAINER_VULN.labels(vul_list[image_name][1], image_name, severity).set(int(vul_list[image_name][0][severity])) + """Generate Metricfile""" + for image_name in vul_list.keys(): + for severity in vul_list[image_name][0].keys(): + CONTAINER_VULN.labels(vul_list[image_name][1], image_name, severity).set(int(vul_list[image_name][0][severity])) await asyncio.sleep(15) else: @@ -261,7 +288,6 @@ async def create_fn(logger, spec, **kwargs): ############################################################################# # Admission Controller ############################################################################# -# namespace selector for admission controller [ ] @kopf.on.startup() def configure(settings: kopf.OperatorSettings, **_): @@ -281,7 +307,8 @@ def validate1(logger, namespace, name, annotations, spec, **_): k8s_config.load_incluster_config() else: k8s_config.load_kube_config() - try: # if no namespace-scanners created + try: + # if no namespace-scanners created nsScans = k8s_client.CustomObjectsApi().list_cluster_custom_object( group="trivy-operator.devopstales.io", version="v1", @@ -296,14 +323,14 @@ def validate1(logger, namespace, name, annotations, spec, **_): containers = spec.get('containers') initContainers = spec.get('initContainers') - for icn in initContainers: - try: + try: + for icn in initContainers: initContainers_array = json.dumps(icn) initContainer = json.loads(initContainers_array) image_name = initContainer["image"] image_list.append(image_name) - except: - continue + except: + print("") for cn in containers: container_array = json.dumps(cn) @@ -312,12 +339,11 @@ def validate1(logger, namespace, name, annotations, spec, **_): image_list.append(image_name) """Get Images""" - for image in image_list: - image_name = image + for image_name in image_list: registry = image_name.split('/')[0] logger.info("Scanning Image: %s" % (image_name)) - """Login to refistry""" + """Login to registry""" try: for reg in registry_list: if reg['name'] == registry: @@ -333,7 +359,7 @@ def validate1(logger, namespace, name, annotations, spec, **_): except: logger.info("No registry auth config is defined.") ACTIVE_REGISTRY = os.getenv("DOCKER_REGISTRY") - logger.info("Active Registry: %s" % (ACTIVE_REGISTRY)) +# logger.info("Active Registry: %s" % (ACTIVE_REGISTRY)) # Debug """Scan Images""" TRIVY = ["trivy", "-q", "i", "-f", "json", image_name] @@ -342,6 +368,7 @@ def validate1(logger, namespace, name, annotations, spec, **_): res = subprocess.Popen(TRIVY,stdout=subprocess.PIPE,stderr=subprocess.PIPE); output,error = res.communicate() if error: + """Error Logging""" logger.error("TRIVY ERROR: return %s" % (res.returncode)) if b"401" in error.strip(): logger.error("Repository: Unauthorized authentication required") @@ -353,58 +380,52 @@ def validate1(logger, namespace, name, annotations, spec, **_): logger.error("Unsupported MediaType: see https://github.com/google/go-containerregistry/issues/377") else: logger.error("%s" % (error.strip())) + """Error action""" + se = { "scanning_error": 1 } + vul_list[image_name] = [se, namespace] elif output: trivy_result = json.loads(output.decode("UTF-8")) item_list = trivy_result['Results'][0]["Vulnerabilities"] - vuls = { - "UNKNOWN": 0,"LOW": 0, - "MEDIUM": 0,"HIGH": 0, - "CRITICAL": 0 - } + vuls = { "UNKNOWN": 0,"LOW": 0,"MEDIUM": 0,"HIGH": 0,"CRITICAL": 0 } for item in item_list: vuls[item["Severity"]] += 1 vul_list[image_name] = [vuls, namespace] - """Generate Metricfile""" - for image_name in vul_list.keys(): - for severity in vul_list[image_name][0].keys(): - """Generate log""" -# logger.info("%s - %s: %s" % (vul_list[image_name][1], image_name, severity)) - AC_VULN.labels(vul_list[image_name][1], image_name, severity).set(int(vul_list[image_name][0][severity])) + """Generate log""" + logger.info("severity: %s" % (vul_list[image_name][0])) # Logging + + """Generate Metricfile""" + for image_name in vul_list.keys(): + for severity in vul_list[image_name][0].keys(): + AC_VULN.labels(vul_list[image_name][1], image_name, severity).set(int(vul_list[image_name][0][severity])) + # logger.info("Prometheus Done") # Debug -############################################################################# -""" # Get vulnerabilities from annotations - vul_annotations= { - "UNKNOWN": 0,"LOW": 0, - "MEDIUM": 0,"HIGH": 0, - "CRITICAL": 0 - } + vul_annotations= { "UNKNOWN": 0,"LOW": 0,"MEDIUM": 0,"HIGH": 0,"CRITICAL": 0 } for sev in vul_annotations: try: - print(sev + ": " + annotations['trivy.security.devopstales.io/' + sev.lower()], file=sys.stderr) # Debug - vul_annotations[sev["Severity"]] = int(annotations['trivy.security.devopstales.io/' + sev.lower()]) +# logger.info("%s: %s" % (sev, annotations['trivy.security.devopstales.io/' + sev.lower()])) # Debug + vul_annotations[sev] = annotations['trivy.security.devopstales.io/' + sev.lower()] except: continue # Check vulnerabilities - print("Check vulnerabilities:", file=sys.stderr) # Debug - for sev in vul_annotations: - try: + # logger.info("Check vulnerabilities:") # Debug + if "scanning_error" in vul_list[image_name][0]: + logger.error("Trivy can't scann the image") + raise kopf.AdmissionError(f"Trivy can't scann the image: %s" % (image_name)) + else: + for sev in vul_annotations: an_vul_num = vul_annotations[sev] - vul_num = vul_list[image_name][sev] - if vul_num > an_vul_num: - print(sev + " is bigger", file=sys.stderr) # Debug + vul_num = vul_list[image_name][0][sev] + if int(vul_num) > int(an_vul_num): +# logger.error("%s is bigger" % (sev)) # Debug raise kopf.AdmissionError(f"Too much vulnerability in the image: %s" % (image_name)) else: - print(sev + " is ok", file=sys.stderr) # Debug +# logger.info("%s is ok" % (sev)) # Debug continue - except: - continue - # print(f"%s" % (image_name), file=sys.stderr) # debug -""" ############################################################################# ## print to operator log # print(f"And here we are! Creating: %s" % (ns_name), file=sys.stderr) # debug From 0ccb96eb654482131d5b78402f36d9c9e1233b3f Mon Sep 17 00:00:00 2001 From: devopstales <42894256+devopstales@users.noreply.github.com> Date: Thu, 18 Nov 2021 14:40:03 +0100 Subject: [PATCH 8/8] release 2.0.0-devel-3 --- docker/Dockerfile | 11 ++++------- trivy-operator.py | 26 +++++++++++++++++++++++++- 2 files changed, 29 insertions(+), 8 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index bee5b2c..abe214c 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,19 +1,16 @@ -FROM python:3.8-alpine +FROM python:3.8.12-slim-buster ENV TRIVY_CACHE_DIR=/home/trivy-operator/trivy-cache \ TRIVY_QUIET=true \ IN_CLUSTER=true -RUN apk add --no-cache gcc musl-dev - -RUN pip3 install --no-cache-dir kopf kubernetes asyncio pycron prometheus_client certvalidator certbuilder -RUN pip3 install --no-cache-dir kopf[dev] +RUN pip3 install --no-cache-dir kopf[dev] kubernetes asyncio pycron prometheus_client oscrypto certvalidator certbuilder validators COPY trivy-operator.py /trivy-operator.py COPY trivy /usr/local/bin -RUN addgroup -S -g 10001 trivy-operator && \ - adduser -S -u 10001 trivy-operator -G trivy-operator && \ +RUN addgroup --gid 10001 trivy-operator && \ + adduser --uid 10001 trivy-operator --ingroup trivy-operator && \ mkdir /home/trivy-operator/trivy-cache && \ chown -R trivy-operator:trivy-operator /home/trivy-operator/trivy-cache diff --git a/trivy-operator.py b/trivy-operator.py index dcfad41..9c2df13 100644 --- a/trivy-operator.py +++ b/trivy-operator.py @@ -9,6 +9,7 @@ import subprocess import json import validators +from typing import AsyncIterator """ apiVersion: trivy-operator.devopstales.io/v1 @@ -288,11 +289,34 @@ async def create_fn(logger, spec, **kwargs): ############################################################################# # Admission Controller ############################################################################# +# https://github.com/nolar/kopf/issues/785#issuecomment-859931945 +if IN_CLUSTER: + class ServiceTunnel: + async def __call__( + self, fn: kopf.WebhookFn + ) -> AsyncIterator[kopf.WebhookClientConfig]: + # https://github.com/kubernetes-client/python/issues/363 + # Use field reference to environment variable instad + namespace = os.environ.get("POD_NAMESPACE", "trivy-operator") + name = "trivy-image-validator" + service_port = int(443) + container_port = int(8443) + server = kopf.WebhookServer(port=container_port, host=f"{name}.{namespace}.svc") + async for client_config in server(fn): + client_config["url"] = None + client_config["service"] = kopf.WebhookClientConfigService( + name=name, namespace=namespace, port=service_port + ) + yield client_config @kopf.on.startup() def configure(settings: kopf.OperatorSettings, **_): # Auto-detect the best server (K3d/Minikube/simple): - settings.admission.server = kopf.WebhookAutoServer(port=443) + if IN_CLUSTER: +# settings.admission.server = kopf.WebhookServer(addr='0.0.0.0', port=8443, host="trivy-image-validator.trivy-operator.svc") + settings.admission.server = ServiceTunnel() + else: + settings.admission.server = kopf.WebhookAutoServer(port=443) settings.admission.managed = 'trivy-image-validator.devopstales.io' @kopf.on.validate('pod', operation='CREATE')