CKAN Coding Standards

.github/workflows/flake8.yml

@@ -0,0 +1,22 @@
+name: Lint
+on: [pull_request]
+
+permissions:
+  contents: read
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+        with:
+          python-version: '3.9'
+      - name: Install requirements
+        run: pip install flake8 pycodestyle
+      - name: Check syntax
+        # Stop the build if there are Python syntax errors or undefined names
+        run: flake8 --show-source
+
+      - name: Warnings
+        run: flake8

.github/workflows/pyright.yml

@@ -0,0 +1,54 @@
+name: Check types
+on: [pull_request]
+env:
+  NODE_VERSION: '18'
+  PYTHON_VERSION: '3.9'
+
+permissions:
+  contents: read
+
+jobs:
+  typecheck:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+        id: gitcheckout
+      - uses: actions/setup-python@v2
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+      - uses: actions/setup-node@v3
+        with:
+          node-version: ${{ env.NODE_VERSION }}
+      - name: Install python deps
+        id: pydepends
+        if: steps.gitcheckout.outcome == 'success'
+        run: |
+            python3 -m venv /home/runner/work/app
+            mkdir -p /home/runner/work/app/src/ckanext-canada
+            cp -R . /home/runner/work/app/src/ckanext-canada
+            source /home/runner/work/app/bin/activate
+            pip install --upgrade setuptools==70.0.0
+            pip install --upgrade pip==23.2.1
+            pip install -e 'git+https://github.com/open-data/[email protected]#egg=ckan' -r 'https://raw.githubusercontent.com/open-data/ckan/canada-v2.10/requirement-setuptools.txt' -r 'https://raw.githubusercontent.com/open-data/ckan/canada-v2.10/requirements.txt' -r 'https://raw.githubusercontent.com/open-data/ckan/canada-v2.10/dev-requirements.txt'
+            pip install -e 'git+https://github.com/ckan/ckanapi.git#egg=ckanapi' -r 'https://raw.githubusercontent.com/ckan/ckanapi/master/requirements.txt'
+            pip install -e /home/runner/work/app/src/ckanext-canada/. -r /home/runner/work/app/src/ckanext-canada/requirements.txt -r /home/runner/work/app/src/ckanext-canada/test-requirements.txt
+            pip install -e 'git+https://github.com/open-data/[email protected]#egg=ckanext-cloudstorage' -r 'https://raw.githubusercontent.com/open-data/ckanext-cloudstorage/canada-v2.10/requirements.txt'
+            pip install -e 'git+https://github.com/ckan/ckanext-fluent.git#egg=ckanext-fluent' -r 'https://raw.githubusercontent.com/ckan/ckanext-fluent/master/requirements.txt'
+            pip install -e 'git+https://github.com/open-data/[email protected]#egg=ckanext-recombinant' -r 'https://raw.githubusercontent.com/open-data/ckanext-recombinant/canada-v2.10/requirements.txt'
+            pip install -e 'git+https://github.com/ckan/ckanext-scheming.git#egg=ckanext-scheming'
+            pip install -e 'git+https://github.com/open-data/[email protected]#egg=ckanext-validation' -r 'https://raw.githubusercontent.com/open-data/ckanext-validation/canada-v2.10/requirements.txt' -r 'https://raw.githubusercontent.com/open-data/ckanext-validation/canada-v2.10/dev-requirements.txt'
+            pip install -e 'git+https://github.com/open-data/[email protected]#egg=ckanext-xloader' -r 'https://raw.githubusercontent.com/open-data/ckanext-xloader/canada-v2.10/requirements.txt' -r 'https://raw.githubusercontent.com/open-data/ckanext-xloader/canada-v2.10/dev-requirements.txt'
+            pip install -e 'git+https://github.com/ckan/ckantoolkit.git#egg=ckantoolkit' -r 'https://raw.githubusercontent.com/ckan/ckantoolkit/master/requirements.txt'
+            pip install -e 'git+https://github.com/open-data/[email protected]#egg=frictionless' -r 'https://raw.githubusercontent.com/open-data/frictionless-py/canada-v2.10/requirements.txt'
+            pip install -e 'git+https://github.com/open-data/[email protected]#egg=ckanext-security' -r 'https://raw.githubusercontent.com/open-data/ckanext-security/canada-v2.10/requirements.txt'
+            find /home/runner/work/app -name '*.pyc' -delete
+      - name: Install node deps
+        if: steps.pydepends.outcome == 'success'
+        run: |
+          cd /home/runner/work/app/src/ckanext-canada
+          npm ci
+      - name: Check types
+        if: steps.pydepends.outcome == 'success'
+        run: |
+          cd /home/runner/work/app/src/ckanext-canada
+          npx pyright

.gitignore

@@ -20,3 +20,4 @@ ckanext/canada/templates/public/snippets/site_message_*
 ckanext/canada/download/*.jsonl
 **.mo
 *.xcf
+node_modules/**

bin/datasets_by_orgs_report.py

@@ -21,9 +21,11 @@ def main():
     opts = docopt(__doc__)
 
     num_months = int(opts['MONTHS'])
-    portal = ckanapi.RemoteCKAN(opts['PORTAL_URL'],
+    portal = ckanapi.RemoteCKAN(
+        opts['PORTAL_URL'],
         user_agent='datasets_by_orgs_report.py (ckanext-canada)')
-    registry = ckanapi.RemoteCKAN(opts['REGISTRY_URL'],
+    registry = ckanapi.RemoteCKAN(
+        opts['REGISTRY_URL'],
         user_agent='datasets_by_orgs_report.py (ckanext-canada)')
 
     sys.stderr.write('getting org list...\n')
@@ -34,7 +36,6 @@ def main():
     sys.stderr.write('getting published datasets...\n')
     published_datasets = set(portal.action.package_list())
 
-    orgs = {o['id']: o for o in org_list}
     now = datetime.utcnow()
     months = [(now.year, now.month)]
     counts = {o['id']: [0] for o in org_list}
@@ -51,7 +52,7 @@ def main():
                 for c in counts:
                     counts[c].append(0)
                 sys.stderr.write(str(processed) + u'\n'
-                    + ym_head(months[-1]) + u':')
+                                 + ym_head(months[-1]) + u':')
                 processed = 0
                 if months[-1] == act_ym:
                     break
@@ -88,10 +89,9 @@ def main():
             o['id'],
             o['title'].split(' | ')[0],
             o['title'].split(' | ')[-1],
-            opts['PORTAL_URL'].rstrip('/')
-                + u'/organization/' + o['name'],
+            opts['PORTAL_URL'].rstrip('/') + u'/organization/' + o['name'],
             o['package_count']
-            ] + counts[o['id']])
+        ] + counts[o['id']])
 
 
 def prior_month(ym):
@@ -118,4 +118,5 @@ def activities(registry):
         offset += len(batch)
     sys.stderr.write(u'activity list ended at %d\n' % offset)
 
+
 main()

bin/download_country.py

@@ -6,13 +6,16 @@
 import sys
 import os.path
 
-OUTPUT_FILE = os.path.join(os.path.split(__file__)[0],
-    '../ckanext/canada/tables/choices/country.json')
+OUTPUT_FILE = os.path.join(
+                os.path.split(__file__)[0],
+                '../ckanext/canada/tables/choices/country.json')
 
-DATA_URL = 'https://raw.githubusercontent.com/datasets/country-codes/master/data/country-codes.csv'
+DATA_URL = 'https://raw.githubusercontent.com/datasets/'\
+           'country-codes/master/data/country-codes.csv'
 
 choices = {}
 
+
 def download_csv_filter_output(url):
     """
     Download CSV and print the columns we need