Generate VDR Using Github Actions (#15)

* rename ci, add vdr creation yml

* try to run the downloader and upload partial

* add trigger on PR

* make data dir

* run the conversion script

* run vdr creation for the whole vdr period

* fix vdr artifact name

* try to make nist calls use api key

* fix the build

* add flush

* Apply suggestions from gadams

Co-authored-by: George Adams <[email protected]>

* add the api key secret, remove branches from workflows

* check api key is actually there

* remove pip and flake8 from the installs

* update comments

* rm newlines

* fix flaky inability to find risk matrix

* Update .github/workflows/vdr-creation.yml

---------

Co-authored-by: George Adams <[email protected]>
Co-authored-by: Martijn Verburg <[email protected]>

.github/workflows/vdr-creation.yml

@@ -0,0 +1,43 @@
+name: VDR Creation
+
+on:
+  workflow_dispatch:
+  push:
+    branches: [ main ]
+  pull_request: # prs don't get secrets, but the API works (albeit 10x slower) without the api key
+    branches: [ main ]
+
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@44c2b7a8a4ea60a981eaca3cf939b5f4305c123b # v4.1.5
+
+    - name: Set up Python 3.10
+      uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0
+      with:
+        python-version: "3.10"
+
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+
+    - name: Download ojvg
+      env:
+        NIST_NVD_TOKEN: ${{ secrets.NIST_NVD_TOKEN }}
+      run: |
+        mkdir -p data
+        python3 ojvg_download.py
+        python3 ojvg_convert.py
+    - name: Upload data directory (for debugging/introspection)
+      uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
+      with:
+        name: data directory
+        path: data
+    - name: Upload final vdr
+      uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
+      with:
+        name: final vdr
+        path: data/vdr.json

cvereporter/fetch_vulnerabilities.py

@@ -102,7 +102,9 @@ def parse_to_dict(resp_text: str, date: str) -> list[dict]:
 
     # find the table with the CVEs
     table = soup.find("table", attrs={"class": "risk-matrix"})
-
+    if table is None:
+        print("unable to find risk matrix for "+date)
+        return None
     # find all the rows in the table
     rows = table.find_all("tr")
     dicts = []
@@ -180,8 +182,8 @@ def dict_to_vulns(dicts: list[dict]) -> list[Vulnerability]:
 """
 We assume the text for the affected versions is in a block like:
 
-"The following vulnerabilities in OpenJDK source code were fixed in this release. 
-The affected versions are 12, 11.0.2, 8u202, 7u211, and earlier. 
+"The following vulnerabilities in OpenJDK source code were fixed in this release.
+The affected versions are 12, 11.0.2, 8u202, 7u211, and earlier.
 We recommend that you upgrade as soon as possible."
 
 """

cvereporter/nist_enhance.py

@@ -6,6 +6,8 @@
 )
 import requests
 import json
+import time
+import os
 
 """
 this file has the utilities for downloading data about cves from NIST and updating Vulnerability objects with the data
@@ -14,7 +16,15 @@
 
 def fetch_nist(url: str, id: str) -> dict:
     data = None
-    nist_resp = requests.get(url)
+    nist_resp = None
+    if "NIST_NVD_TOKEN" in os.environ and os.environ["NIST_NVD_TOKEN"]: # check not empty
+        print("making call to NIST using api key! "+url, flush=True)
+        time.sleep(1) # stay well within 50 requests/30 seconds
+        nist_resp = requests.get(url, headers= {"apiKey": os.environ["NIST_NVD_TOKEN"]})
+    else:
+        print("making call to NIST without using api key! "+url, flush=True)
+        time.sleep(10) # stay well within 5 requests/30 seconds
+        nist_resp = requests.get(url)
     if nist_resp.status_code != 200:
         print(
             "error fetching {}; status code: {}; text: {}".format(