diff --git a/data/program_analysis/ABM-COVID-ABS.zip b/data/program_analysis/ABM-COVID-ABS.zip deleted file mode 100644 index f906eddc34a..00000000000 Binary files a/data/program_analysis/ABM-COVID-ABS.zip and /dev/null differ diff --git a/data/program_analysis/ABM-COmplexVID-19.zip b/data/program_analysis/ABM-COmplexVID-19.zip deleted file mode 100644 index 160fdb0382e..00000000000 Binary files a/data/program_analysis/ABM-COmplexVID-19.zip and /dev/null differ diff --git a/data/program_analysis/ABM-Covasim.zip b/data/program_analysis/ABM-Covasim.zip deleted file mode 100644 index c1a244bdec0..00000000000 Binary files a/data/program_analysis/ABM-Covasim.zip and /dev/null differ diff --git a/data/program_analysis/ABM-REINA.zip b/data/program_analysis/ABM-REINA.zip deleted file mode 100644 index 4377c6f118a..00000000000 Binary files a/data/program_analysis/ABM-REINA.zip and /dev/null differ diff --git a/data/program_analysis/Bucky.zip b/data/program_analysis/Bucky.zip deleted file mode 100644 index 0d143a03989..00000000000 Binary files a/data/program_analysis/Bucky.zip and /dev/null differ diff --git a/data/program_analysis/CHIME-SIR-model.zip b/data/program_analysis/CHIME-SIR-model.zip deleted file mode 100644 index 63a27b171fe..00000000000 Binary files a/data/program_analysis/CHIME-SIR-model.zip and /dev/null differ diff --git a/data/program_analysis/CHIME-SVIIvR-model.zip b/data/program_analysis/CHIME-SVIIvR-model.zip deleted file mode 100644 index 1f8f7913478..00000000000 Binary files a/data/program_analysis/CHIME-SVIIvR-model.zip and /dev/null differ diff --git a/data/program_analysis/CHIME-penn-full-model.zip b/data/program_analysis/CHIME-penn-full-model.zip deleted file mode 100644 index db3376fdb30..00000000000 Binary files a/data/program_analysis/CHIME-penn-full-model.zip and /dev/null differ diff --git a/data/program_analysis/Climlab.zip b/data/program_analysis/Climlab.zip deleted file mode 100644 index c8b94b28838..00000000000 Binary files a/data/program_analysis/Climlab.zip and /dev/null differ diff --git a/data/program_analysis/Cornell-COVID19-sim-Frazier.zip b/data/program_analysis/Cornell-COVID19-sim-Frazier.zip deleted file mode 100644 index 007e1ef2b7e..00000000000 Binary files a/data/program_analysis/Cornell-COVID19-sim-Frazier.zip and /dev/null differ diff --git a/data/program_analysis/Generated-Halfar.zip b/data/program_analysis/Generated-Halfar.zip deleted file mode 100644 index d37e3256f39..00000000000 Binary files a/data/program_analysis/Generated-Halfar.zip and /dev/null differ diff --git a/data/program_analysis/MechBayes.zip b/data/program_analysis/MechBayes.zip deleted file mode 100644 index 15cb0ecb3e2..00000000000 Binary files a/data/program_analysis/MechBayes.zip and /dev/null differ diff --git a/data/program_analysis/SIDARTHE.zip b/data/program_analysis/SIDARTHE.zip deleted file mode 100644 index 833a6f51c4f..00000000000 Binary files a/data/program_analysis/SIDARTHE.zip and /dev/null differ diff --git a/data/program_analysis/SV2AIR3-Waterloo-MATLAB.zip b/data/program_analysis/SV2AIR3-Waterloo-MATLAB.zip deleted file mode 100644 index 96ee3f54aaf..00000000000 Binary files a/data/program_analysis/SV2AIR3-Waterloo-MATLAB.zip and /dev/null differ diff --git a/data/program_analysis/Simple-SIR.zip b/data/program_analysis/Simple-SIR.zip deleted file mode 100644 index 856019c0e74..00000000000 Binary files a/data/program_analysis/Simple-SIR.zip and /dev/null differ diff --git a/data/program_analysis/TIE-GCM.zip b/data/program_analysis/TIE-GCM.zip deleted file mode 100644 index 5c176cafe9d..00000000000 Binary files a/data/program_analysis/TIE-GCM.zip and /dev/null differ diff --git a/data/program_analysis/cism_v3.zip b/data/program_analysis/cism_v3.zip deleted file mode 100644 index f91b294b18f..00000000000 Binary files a/data/program_analysis/cism_v3.zip and /dev/null differ diff --git a/data/program_analysis/climlab-v2.zip b/data/program_analysis/climlab-v2.zip deleted file mode 100644 index e17db0abee6..00000000000 Binary files a/data/program_analysis/climlab-v2.zip and /dev/null differ diff --git a/data/program_analysis/code_sir.zip b/data/program_analysis/code_sir.zip deleted file mode 100644 index a64fefd55d5..00000000000 Binary files a/data/program_analysis/code_sir.zip and /dev/null differ diff --git a/data/program_analysis/examples_python.zip b/data/program_analysis/examples_python.zip deleted file mode 100644 index 274a8914039..00000000000 Binary files a/data/program_analysis/examples_python.zip and /dev/null differ diff --git a/pyproject.toml b/pyproject.toml index d55b46bae25..437a653bde0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -100,6 +100,7 @@ all = ["skema[core]", "skema[dev]", "skema[doc]", "skema[demo]", "skema[annotati "skema.rest" = "skema/rest" "skema.skema_py" = "skema/skema_py" "skema.utils" = "skema/utils" +"skema.data" = "skema/data" # re-map skema/text_reading/python to skema.text_reading #"skema.text_reading" = "skema/text_reading/python" diff --git a/skema/program_analysis/model_coverage_report/model_coverage_report.py b/skema/program_analysis/model_coverage_report/model_coverage_report.py index e76059f755c..d86812ecdb6 100644 --- a/skema/program_analysis/model_coverage_report/model_coverage_report.py +++ b/skema/program_analysis/model_coverage_report/model_coverage_report.py @@ -30,6 +30,8 @@ from skema.program_analysis.tree_sitter_parsers.util import extension_to_language from skema.rest.utils import fn_preprocessor from skema.utils.fold import del_nulls, dictionary_to_gromet_json +from skema.data.program_analysis import MODEL_ZIP_ROOT_PATH + THIS_PATH = Path(__file__).parent.resolve() MODEL_YAML_PATH = Path(__file__).parent / "models.yaml" @@ -141,11 +143,14 @@ def process_single_model(html: HTML_Instance, output_dir: str, model_name: str): """Generate an HTML report for a single model""" html.add_model(model_name) - if model_name in MODEL_YAML: - model_url = MODEL_YAML[model_name]["zip_archive"] - response = requests.get(model_url) - - zip = ZipFile(BytesIO(response.content)) + + + if not model_name in MODEL_YAML: + return + + model_path = MODEL_ZIP_ROOT_PATH.resolve() / MODEL_YAML[model_name]["zip_archive"] + + zip = ZipFile(BytesIO(model_path.read_bytes())) with TemporaryDirectory() as temp: # We need to write all the files to the temporary directory before processing # This is because some steps may require additional files, such as include directories in Fortran diff --git a/skema/program_analysis/model_coverage_report/models.yaml b/skema/program_analysis/model_coverage_report/models.yaml index 3d326f2db58..b77505c793d 100644 --- a/skema/program_analysis/model_coverage_report/models.yaml +++ b/skema/program_analysis/model_coverage_report/models.yaml @@ -1,57 +1,56 @@ ---- CHIME-penn-full: - zip_archive: "https://artifacts.askem.lum.ai/askem/data/models/zip-archives/CHIME-penn-full-model.zip" + zip_archive: "CHIME-penn-full-model.zip" CHIME-SIR: - zip_archive: "https://artifacts.askem.lum.ai/askem/data/models/zip-archives/CHIME-SIR-model.zip" + zip_archive: "CHIME-SIR-model.zip" CHIME-SVIIvR: - zip_archive: "https://artifacts.askem.lum.ai/askem/data/models/zip-archives/CHIME-SVIIvR-model.zip" + zip_archive: "CHIME-SVIIvR-model.zip" ABM-COmplexVID-19: - zip_archive: "https://artifacts.askem.lum.ai/askem/data/models/zip-archives/ABM-COmplexVID-19.zip" + zip_archive: "ABM-COmplexVID-19.zip" ABM-COVID-ABS: - zip_archive: "https://artifacts.askem.lum.ai/askem/data/models/zip-archives/ABM-COVID-ABS.zip" + zip_archive: "ABM-COVID-ABS.zip" CHIME-penn-full: - zip_archive: "https://artifacts.askem.lum.ai/askem/data/models/zip-archives/CHIME-penn-full-model.zip" + zip_archive: "CHIME-penn-full-model.zip" MechBayes: - zip_archive: "https://artifacts.askem.lum.ai/askem/data/models/zip-archives/MechBayes.zip" + zip_archive: "MechBayes.zip" SIDARTHE: - zip_archive: "https://artifacts.askem.lum.ai/askem/data/models/zip-archives/SIDARTHE.zip" + zip_archive: "SIDARTHE.zip" Simple-SIR: - zip_archive: "https://artifacts.askem.lum.ai/askem/data/models/zip-archives/Simple-SIR.zip" + zip_archive: "Simple-SIR.zip" Climlab-v1: - zip_archive: "https://artifacts.askem.lum.ai/askem/data/models/zip-archives/Climlab.zip" + zip_archive: "Climlab.zip" Climlab-v2: - zip_archive: "https://artifacts.askem.lum.ai/askem/data/models/zip-archives/climlab-v2.zip" + zip_archive: "climlab-v2.zip" Examples-Python: - zip_archive: "https://artifacts.askem.lum.ai/askem/data/models/zip-archives/examples_python.zip" + zip_archive: "examples_python.zip" Generated-Halfar: - zip_archive: "https://artifacts.askem.lum.ai/askem/data/models/zip-archives/Generated-Halfar.zip" + zip_archive: "Generated-Halfar.zip" SV2AIR3-Waterloo-MATLAB: - zip_archive: "https://artifacts.askem.lum.ai/askem/data/models/zip-archives/SV2AIR3-Waterloo-MATLAB.zip" + zip_archive: "SV2AIR3-Waterloo-MATLAB.zip" Bucky: - zip_archive: "https://artifacts.askem.lum.ai/askem/data/models/zip-archives/Bucky.zip" + zip_archive: "Bucky.zip" ABM-REINA: - zip_archive: "https://artifacts.askem.lum.ai/askem/data/models/zip-archives/ABM-REINA.zip" + zip_archive: "ABM-REINA.zip" Cornell-COVID19-sim-Frazier: - zip_archive: "https://artifacts.askem.lum.ai/askem/data/models/zip-archives/Cornell-COVID19-sim-Frazier.zip" + zip_archive: "Cornell-COVID19-sim-Frazier.zip" ABM-Covasim: - zip_archive: "https://artifacts.askem.lum.ai/askem/data/models/zip-archives/ABM-Covasim.zip" + zip_archive: "ABM-Covasim.zip" TIE-GCM: - zip_archive: "https://artifacts.askem.lum.ai/askem/data/models/zip-archives/TIE-GCM.zip" + zip_archive: "TIE-GCM.zip" diff --git a/skema/program_analysis/tests/test_code2fn.py b/skema/program_analysis/tests/test_code2fn.py index 3b17ec03965..800eee8f900 100644 --- a/skema/program_analysis/tests/test_code2fn.py +++ b/skema/program_analysis/tests/test_code2fn.py @@ -7,8 +7,9 @@ from skema.program_analysis.multi_file_ingester import process_file_system from skema.gromet.fn import GrometFNModuleCollection +from skema.data.program_analysis import MODEL_ZIP_ROOT_PATH -BUCKY_ZIP_URL = "https://artifacts.askem.lum.ai/askem/data/models/zip-archives/Bucky.zip" +BUCKY_ZIP_URL = MODEL_ZIP_ROOT_PATH.resolve() / "Bucky.zip" def test_code2fn(): """This is simply a smokescreen test to see if the PA pipeline runs to @@ -19,8 +20,7 @@ def test_code2fn(): flag or postprocessing function to be able to compare two GroMEts modulo their random components.""" - response = requests.get(BUCKY_ZIP_URL) - zip = zipfile.ZipFile(io.BytesIO(response.content)) + zip = zipfile.ZipFile(io.BytesIO(BUCKY_ZIP_URL.read_bytes())) with TemporaryDirectory() as temp: system_filepaths_path = Path(temp) / "system_filepaths.txt" diff --git a/skema/rest/tests/test_model_to_amr.py b/skema/rest/tests/test_model_to_amr.py index 5d168c37a57..6d3a533ad26 100644 --- a/skema/rest/tests/test_model_to_amr.py +++ b/skema/rest/tests/test_model_to_amr.py @@ -1,3 +1,6 @@ +import json +import httpx +import pytest import asyncio import requests from pathlib import Path @@ -16,17 +19,12 @@ from skema.rest.llm_proxy import Dynamics from skema.rest.proxies import SKEMA_RS_ADDESS from skema.skema_py import server as code2fn -import json -import httpx -import pytest +from skema.data.program_analysis import MODEL_ZIP_ROOT_PATH -CHIME_SIR_URL = ( - "https://artifacts.askem.lum.ai/askem/data/models/zip-archives/CHIME-SIR-model.zip" -) -SIDARTHE_URL = ( - "https://artifacts.askem.lum.ai/askem/data/models/zip-archives/SIDARTHE.zip" -) +CHIME_SIR_PATH = MODEL_ZIP_ROOT_PATH.resolve() / "CHIME-SIR-model.zip" +SIDARTHE_PATH = MODEL_ZIP_ROOT_PATH.resolve() / "SIDARTHE.zip" + @pytest.mark.asyncio async def test_any_amr_chime_sir(): @@ -34,8 +32,8 @@ async def test_any_amr_chime_sir(): Unit test for checking that Chime-SIR model produces any AMR. This test zip contains 4 versions of CHIME SIR. This will test if just the core dynamics works, the whole script, and also rewritten scripts work. """ - response = requests.get(CHIME_SIR_URL) - zip_bytes = BytesIO(response.content) + + zip_bytes = BytesIO(CHIME_SIR_PATH.read_bytes()) # NOTE: For CI we are unable to use the LLM assisted functions due to API keys # So, we will instead mock the output for those functions instead @@ -136,8 +134,7 @@ async def test_any_amr_sidarthe(): Unit test for checking that Chime-SIR model produces any AMR. This test zip contains 4 versions of CHIME SIR. This will test if just the core dynamics works, the whole script, and also rewritten scripts work. """ - response = requests.get(SIDARTHE_URL) - zip_bytes = BytesIO(response.content) + zip_bytes = BytesIO(SIDARTHE_PATH.read_bytes()) # NOTE: For CI we are unable to use the LLM assisted functions due to API keys # So, we will instead mock the output for those functions instead