adds invoke task for checking notebooks for errors

Pipfile

@@ -1,78 +0,0 @@
-[[source]]
-url = "https://pypi.org/simple"
-verify_ssl = true
-name = "pypi"
-
-[packages]
-anyio = "==3.6.2"
-argon2-cffi = "==21.3.0"
-argon2-cffi-bindings = "==21.2.0"
-asttokens = "==2.1.0"
-attrs = "==22.1.0"
-backcall = "==0.2.0"
-beautifulsoup4 = "==4.11.1"
-bleach = "==5.0.1"
-certifi = "==2022.12.7"
-cffi = "==1.15.1"
-decorator = "==5.1.1"
-defusedxml = "==0.7.1"
-elastic-transport = "==8.4.0"
-elasticsearch = "==8.5.0"
-entrypoints = "==0.4"
-executing = "==1.2.0"
-fastjsonschema = "==2.16.2"
-idna = "==3.4"
-ipython = "==8.6.0"
-jedi = "==0.18.1"
-jinja2 = "==3.1.2"
-jsonschema = "==4.17.0"
-jupyter-client = "==7.4.4"
-jupyter-core = "==4.11.2"
-jupyter-server = "==1.21.0"
-jupyterlab-pygments = "==0.2.2"
-markdown = "==3.1.1"
-markupsafe = "==2.1.1"
-matplotlib-inline = "==0.1.6"
-mistune = "==2.0.4"
-nbclient = "==0.7.0"
-nbconvert = "==7.2.3"
-nbformat = "==5.7.0"
-nest-asyncio = "==1.5.6"
-newrelic = "==8.4.0"
-packaging = "==21.3"
-pandocfilters = "==1.5.0"
-parso = "==0.8.3"
-pexpect = "==4.8.0"
-pickleshare = "==0.7.5"
-prometheus-client = "==0.15.0"
-prompt-toolkit = "==3.0.32"
-ptyprocess = "==0.7.0"
-pure-eval = "==0.2.2"
-pycparser = "==2.21"
-pycurl = "==7.45.1"
-pygments = "==2.13.0"
-pylibmc = "==1.6.3"
-pyparsing = "==3.0.9"
-pyrsistent = "==0.19.2"
-python-dateutil = "==2.8.2"
-pyzmq = "==24.0.1"
-send2trash = "==1.8.0"
-six = "==1.16.0"
-sniffio = "==1.3.0"
-soupsieve = "==2.3.2.post1"
-stack-data = "==0.6.0"
-statsd = "==4.0.1"
-terminado = "==0.17.0"
-tinycss2 = "==1.2.1"
-tornado = "==6.2"
-traitlets = "==5.5.0"
-urllib3 = "==1.26.12"
-wcwidth = "==0.2.5"
-webencodings = "==0.5.1"
-websocket-client = "==1.4.2"
-
-[dev-packages]
-
-[requires]
-python_version = "3.10"
-python_full_version = "3.10.6"

nbviewer/app.py

@@ -558,6 +558,9 @@ def frontpage_setup(self):
         # check if the JSON has a 'sections' field, otherwise assume it is just a list of sessions,
         # and provide the defaults of the other fields
         try:
+            with io.open(self.cases_modules, "r") as r:
+                modules = json.load(r)
+                frontpage_setup["modules"] = modules['modules']
             with io.open(self.aca_domains, "r") as r:
                 domains = json.load(r)
                 frontpage_setup["domains"] = domains['member']
@@ -567,9 +570,9 @@ def frontpage_setup(self):
             with io.open(self.cases_themes, "r") as r:
                 themes = json.load(r)
                 frontpage_setup["themes"] = themes['member']
-            with io.open(self.cases_modules, "r") as r:
-                modules = json.load(r)
-                frontpage_setup["modules"] = modules['modules']
+            # TODO: add xrefs, perhaps by parsing jsonld with rdflib?
+            #xref_modules_by_attribute(frontpage_setup, "themes", "about")
+            #xref_modules_by_attribute(frontpage_setup, "practices", "teaches")
         except Exception as ex:
             self.log.warning("Failed to load: "+ str(ex))
         self.log.info(json.dumps(frontpage_setup))

tasks.py

@@ -10,8 +10,11 @@
 import sys
 import tempfile
 from tarfile import TarFile
-
+from urllib.parse import urlparse
+import re
+import glob
 import invoke
+from invoke.exceptions import UnexpectedExit
 
 NOTEBOOK_VERSION = "5.7.8"  # the notebook version whose LESS we will use
 NOTEBOOK_CHECKSUM = "573e0ae650c5d76b18b6e564ba6d21bf321d00847de1d215b418acb64f056eb8"  # sha256 checksum of notebook tarball
@@ -21,7 +24,8 @@
 NOTEBOOK_STATIC_PATH = os.path.join(
     APP_ROOT, "notebook-%s" % NOTEBOOK_VERSION, "notebook", "static"
 )
-
+CASES_MODULES_INVENTORY_PATH = os.path.join(APP_ROOT, "nbviewer", "cases_data", "CASES_modules.json")
+CASES_MODULES_TEST_PATH = "/home/jansen/module_tests"
 
 @invoke.task
 def test(ctx):
@@ -76,6 +80,63 @@ def notebook_static(ctx):
         print("Extract {0} in {1}".format(nb_archive, nb_archive_file.extractall()))
 
 
+@invoke.task
+def update_case_module_dependencies(c, clean=False, update=False, debug=True):
+    repos = {}
+    inv = {}
+    with open(CASES_MODULES_INVENTORY_PATH) as f:
+        inv = json.load(f)
+    for m in inv['modules']:
+        code_repo = m['codeRepository']
+        # https://github.com/cases-umd/Baltimore-Redlining-1/blob/master/index.ipynb
+        url = urlparse(code_repo)
+        path = url.path[0:url.path.index('/blob/')].removesuffix("/blob/")
+        git_repo = f'{url.scheme}://{url.netloc}{path}'
+        repos[git_repo] = {
+            'path': path,
+            'dirname': path[path.index("/", 1)+1:],
+            'repo': git_repo
+        }
+    c.run(f'mkdir -p {CASES_MODULES_TEST_PATH}', echo=True)
+    with c.cd(CASES_MODULES_TEST_PATH):
+        for m in repos.values():
+            if os.path.exists(f'{CASES_MODULES_TEST_PATH}/{m["dirname"]}'):
+                if update:
+                    with c.cd(m["dirname"]):
+                        c.run(f'git pull', echo=True)
+            else:
+                c.run(f'git clone {m["repo"]}', echo=True)
+    modules_with_errors = {}
+    for m in repos.values():
+        notebooks_with_errors = []
+        with c.cd(f'{CASES_MODULES_TEST_PATH}/{m["dirname"]}'):
+            c.run('pipenv --rm -q', echo=True, warn=True)
+            c.run('pipenv --python 3.10', echo=True, warn=True)
+            text2 = ""
+            with open(f"{c.cwd}/Pipfile") as fh:
+                text = fh.read()
+                text2 = re.sub(r'"==\d*\.\d*(\.\d*)?"', '"*"', text)
+                text2 = re.sub(r'^python_full_version.*', '', text2, flags=re.MULTILINE)
+            with open(f"{c.cwd}/Pipfile", mode='w') as fh:
+                fh.write(text2)
+            c.run('pipenv install --dev', echo=True)   
+            c.run('pipenv install --dev notebook', echo=True)
+            with open(f'{c.cwd}/notebook_executions.log', mode="w") as out:
+                for nb in glob.glob("*.ipynb", root_dir=c.cwd, recursive=True):
+                    try:
+                        c.run(f'TERM=xterm-mono; pipenv run jupyter execute {nb}', pty=True, warn=False, echo=True, err_stream=out, out_stream=out)
+                    except UnexpectedExit:
+                        notebooks_with_errors.append(f'{c.cwd}/{nb}')
+            if len(notebooks_with_errors) > 0:
+                modules_with_errors[c.cwd] = notebooks_with_errors
+    print('Notebooks with errors:')
+    for moduledir in modules_with_errors:
+        print(moduledir)
+        for notebookf in modules_with_errors[moduledir]:
+            print(f'\t{notebookf}')
+
+
+
 @invoke.task
 def less(ctx, debug=False):
     notebook_static(ctx)