From bc044fa808d5cabb9e6b3d6c90ef280f10aa9477 Mon Sep 17 00:00:00 2001 From: "Gregory N. Jansen" Date: Mon, 26 Feb 2024 13:20:28 -0500 Subject: [PATCH] adds invoke task for checking notebooks for errors --- Pipfile | 78 ------------------------------------------------- nbviewer/app.py | 9 ++++-- tasks.py | 65 +++++++++++++++++++++++++++++++++++++++-- 3 files changed, 69 insertions(+), 83 deletions(-) diff --git a/Pipfile b/Pipfile index dfc55106..e69de29b 100644 --- a/Pipfile +++ b/Pipfile @@ -1,78 +0,0 @@ -[[source]] -url = "https://pypi.org/simple" -verify_ssl = true -name = "pypi" - -[packages] -anyio = "==3.6.2" -argon2-cffi = "==21.3.0" -argon2-cffi-bindings = "==21.2.0" -asttokens = "==2.1.0" -attrs = "==22.1.0" -backcall = "==0.2.0" -beautifulsoup4 = "==4.11.1" -bleach = "==5.0.1" -certifi = "==2022.12.7" -cffi = "==1.15.1" -decorator = "==5.1.1" -defusedxml = "==0.7.1" -elastic-transport = "==8.4.0" -elasticsearch = "==8.5.0" -entrypoints = "==0.4" -executing = "==1.2.0" -fastjsonschema = "==2.16.2" -idna = "==3.4" -ipython = "==8.6.0" -jedi = "==0.18.1" -jinja2 = "==3.1.2" -jsonschema = "==4.17.0" -jupyter-client = "==7.4.4" -jupyter-core = "==4.11.2" -jupyter-server = "==1.21.0" -jupyterlab-pygments = "==0.2.2" -markdown = "==3.1.1" -markupsafe = "==2.1.1" -matplotlib-inline = "==0.1.6" -mistune = "==2.0.4" -nbclient = "==0.7.0" -nbconvert = "==7.2.3" -nbformat = "==5.7.0" -nest-asyncio = "==1.5.6" -newrelic = "==8.4.0" -packaging = "==21.3" -pandocfilters = "==1.5.0" -parso = "==0.8.3" -pexpect = "==4.8.0" -pickleshare = "==0.7.5" -prometheus-client = "==0.15.0" -prompt-toolkit = "==3.0.32" -ptyprocess = "==0.7.0" -pure-eval = "==0.2.2" -pycparser = "==2.21" -pycurl = "==7.45.1" -pygments = "==2.13.0" -pylibmc = "==1.6.3" -pyparsing = "==3.0.9" -pyrsistent = "==0.19.2" -python-dateutil = "==2.8.2" -pyzmq = "==24.0.1" -send2trash = "==1.8.0" -six = "==1.16.0" -sniffio = "==1.3.0" -soupsieve = "==2.3.2.post1" -stack-data = "==0.6.0" -statsd = "==4.0.1" -terminado = "==0.17.0" -tinycss2 = "==1.2.1" -tornado = "==6.2" -traitlets = "==5.5.0" -urllib3 = "==1.26.12" -wcwidth = "==0.2.5" -webencodings = "==0.5.1" -websocket-client = "==1.4.2" - -[dev-packages] - -[requires] -python_version = "3.10" -python_full_version = "3.10.6" diff --git a/nbviewer/app.py b/nbviewer/app.py index bc9c46b4..d4496796 100644 --- a/nbviewer/app.py +++ b/nbviewer/app.py @@ -558,6 +558,9 @@ def frontpage_setup(self): # check if the JSON has a 'sections' field, otherwise assume it is just a list of sessions, # and provide the defaults of the other fields try: + with io.open(self.cases_modules, "r") as r: + modules = json.load(r) + frontpage_setup["modules"] = modules['modules'] with io.open(self.aca_domains, "r") as r: domains = json.load(r) frontpage_setup["domains"] = domains['member'] @@ -567,9 +570,9 @@ def frontpage_setup(self): with io.open(self.cases_themes, "r") as r: themes = json.load(r) frontpage_setup["themes"] = themes['member'] - with io.open(self.cases_modules, "r") as r: - modules = json.load(r) - frontpage_setup["modules"] = modules['modules'] + # TODO: add xrefs, perhaps by parsing jsonld with rdflib? + #xref_modules_by_attribute(frontpage_setup, "themes", "about") + #xref_modules_by_attribute(frontpage_setup, "practices", "teaches") except Exception as ex: self.log.warning("Failed to load: "+ str(ex)) self.log.info(json.dumps(frontpage_setup)) diff --git a/tasks.py b/tasks.py index 2ca8babd..627480cb 100644 --- a/tasks.py +++ b/tasks.py @@ -10,8 +10,11 @@ import sys import tempfile from tarfile import TarFile - +from urllib.parse import urlparse +import re +import glob import invoke +from invoke.exceptions import UnexpectedExit NOTEBOOK_VERSION = "5.7.8" # the notebook version whose LESS we will use NOTEBOOK_CHECKSUM = "573e0ae650c5d76b18b6e564ba6d21bf321d00847de1d215b418acb64f056eb8" # sha256 checksum of notebook tarball @@ -21,7 +24,8 @@ NOTEBOOK_STATIC_PATH = os.path.join( APP_ROOT, "notebook-%s" % NOTEBOOK_VERSION, "notebook", "static" ) - +CASES_MODULES_INVENTORY_PATH = os.path.join(APP_ROOT, "nbviewer", "cases_data", "CASES_modules.json") +CASES_MODULES_TEST_PATH = "/home/jansen/module_tests" @invoke.task def test(ctx): @@ -76,6 +80,63 @@ def notebook_static(ctx): print("Extract {0} in {1}".format(nb_archive, nb_archive_file.extractall())) +@invoke.task +def update_case_module_dependencies(c, clean=False, update=False, debug=True): + repos = {} + inv = {} + with open(CASES_MODULES_INVENTORY_PATH) as f: + inv = json.load(f) + for m in inv['modules']: + code_repo = m['codeRepository'] + # https://github.com/cases-umd/Baltimore-Redlining-1/blob/master/index.ipynb + url = urlparse(code_repo) + path = url.path[0:url.path.index('/blob/')].removesuffix("/blob/") + git_repo = f'{url.scheme}://{url.netloc}{path}' + repos[git_repo] = { + 'path': path, + 'dirname': path[path.index("/", 1)+1:], + 'repo': git_repo + } + c.run(f'mkdir -p {CASES_MODULES_TEST_PATH}', echo=True) + with c.cd(CASES_MODULES_TEST_PATH): + for m in repos.values(): + if os.path.exists(f'{CASES_MODULES_TEST_PATH}/{m["dirname"]}'): + if update: + with c.cd(m["dirname"]): + c.run(f'git pull', echo=True) + else: + c.run(f'git clone {m["repo"]}', echo=True) + modules_with_errors = {} + for m in repos.values(): + notebooks_with_errors = [] + with c.cd(f'{CASES_MODULES_TEST_PATH}/{m["dirname"]}'): + c.run('pipenv --rm -q', echo=True, warn=True) + c.run('pipenv --python 3.10', echo=True, warn=True) + text2 = "" + with open(f"{c.cwd}/Pipfile") as fh: + text = fh.read() + text2 = re.sub(r'"==\d*\.\d*(\.\d*)?"', '"*"', text) + text2 = re.sub(r'^python_full_version.*', '', text2, flags=re.MULTILINE) + with open(f"{c.cwd}/Pipfile", mode='w') as fh: + fh.write(text2) + c.run('pipenv install --dev', echo=True) + c.run('pipenv install --dev notebook', echo=True) + with open(f'{c.cwd}/notebook_executions.log', mode="w") as out: + for nb in glob.glob("*.ipynb", root_dir=c.cwd, recursive=True): + try: + c.run(f'TERM=xterm-mono; pipenv run jupyter execute {nb}', pty=True, warn=False, echo=True, err_stream=out, out_stream=out) + except UnexpectedExit: + notebooks_with_errors.append(f'{c.cwd}/{nb}') + if len(notebooks_with_errors) > 0: + modules_with_errors[c.cwd] = notebooks_with_errors + print('Notebooks with errors:') + for moduledir in modules_with_errors: + print(moduledir) + for notebookf in modules_with_errors[moduledir]: + print(f'\t{notebookf}') + + + @invoke.task def less(ctx, debug=False): notebook_static(ctx)