-
Notifications
You must be signed in to change notification settings - Fork 40
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Check Models Action and Front End showing Action findings (#687)
* Check Models Action * test for nod20 actions * toml updates * entry point * entry point * entry point * allow install true * debuging check model outputs when unable to install package via pip * debuging check model outputs when unable to install package via pip * debuging check model outputs when unable to install package via pip * subprocess.check_call shell=True * model catalog checks * model catalog checks * model catalog checks * model catalog checks requests * model catalog action * model catalog action * check_models * url updates * url update VEdge * print config * Update path to use os.path.join * model load successfully check * flake8 updates * flak8 updates * flake8 updates * workflow schedule updated * updates to function description and return report * check models formatting * save action report as csv * temporary change of run action on push so to check results * check model report * model report * json fix * stardist fix * stardist fix * json serializable * falke8 * flake8 * update website with check models * test * json changes * json changes * json changes * bug * bug * checkreport * switch back to at * removal of artefact saving in yml and updates to checkreport in Model Table * turing to fork * table not loading debugging * removal of store artifact component of action * model table now populates test outcomes, paths sets back to alan turing institute, store artifact is removed and tested to make sure still runs successfully --------- Co-authored-by: Isabel Fenton <[email protected]>
- Loading branch information
Showing
8 changed files
with
241 additions
and
29 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,36 @@ | ||
| name: Check models | ||
| on: | ||
| schedule: | ||
| - cron: '18 3 * * *' | ||
| jobs: | ||
| check-models: | ||
| runs-on: ubuntu-latest | ||
| permissions: | ||
| contents: write | ||
| steps: | ||
| - uses: actions/checkout@v4 | ||
| - uses: actions/setup-python@v4 | ||
| with: | ||
| python-version: '3.10' | ||
| - name: Upgrade pip | ||
| run: python -m pip install --upgrade pip | ||
| - name: Install Scivision | ||
| run: pip install -e . | ||
| - name: Install chardet | ||
| run: pip install chardet | ||
| - name: Check model catalog | ||
| run: scivision-check-models | ||
| # - name: Store artifact | ||
| # uses: actions/upload-artifact@v3 | ||
| # with: | ||
| # name: check-models-report | ||
| # path: check_models.csv | ||
| - name: Create Release | ||
| uses: ncipollo/release-action@v1 | ||
| with: | ||
| tag: model-checks-report-latest-release | ||
| name: Models checks report | ||
| commit: ${{ github.ref }} | ||
| allowUpdates: true | ||
| removeArtifacts: true | ||
| artifacts: check_models.js |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,99 @@ | ||
| """ | ||
| Automated Models Checks | ||
| Iterate through model catalog via scivision.load_pretrained_model | ||
| to check if the model can be loaded and if the model scivision_usable = True. | ||
| If not scivision_usable, check if the model url is accessible, if return 200 reponse log as passed. | ||
| Otherwise, load the model using load_pretrained_model and log as passed if successful. | ||
| """ | ||
|
|
||
| import logging | ||
| import json | ||
| import requests | ||
| from datetime import datetime | ||
|
|
||
| from scivision import default_catalog, load_pretrained_model | ||
| from tqdm import tqdm | ||
|
|
||
| # Create Logger | ||
| logger = logging.getLogger(__name__) | ||
| # Set log level | ||
| logger.setLevel(logging.INFO) | ||
| file_handler = logging.FileHandler('check_models.log') | ||
| formatter = logging.Formatter('%(asctime)s : %(levelname)s : %(name)s : %(message)s') | ||
| file_handler.setFormatter(formatter) | ||
| logger.addHandler(file_handler) | ||
|
|
||
|
|
||
| def check_models(): | ||
| """ | ||
| For each model in the catalog, check that the URL can be loaded | ||
| with `load_pretrained_model`. | ||
| Returns a json report | ||
| Model information includes | ||
| - name | ||
| - tasks | ||
| - pkg_url | ||
| - url | ||
| - scivision_usable | ||
| """ | ||
| # Load model catalog | ||
| model_catalog = default_catalog.models.to_dataframe() | ||
| # Load model using model and record response | ||
| rows = {} | ||
| for model in tqdm(model_catalog.itertuples()): | ||
| name = model.name | ||
| yml_path = model.url | ||
| print(f'\nValidating: {name}') | ||
| if not model.scivision_usable: | ||
| response = requests.get(model.url) | ||
| row_data = { | ||
| 'url': model.url, | ||
| 'check_result': 'Pass' if response.status_code == 200 else 'Fail', | ||
| 'response': f'Scivision_usable = False but model url response: {response.status_code}', | ||
| } | ||
| print(f'Model is not scivision usable but model url response: {response.status_code}') | ||
| else: | ||
| try: | ||
| if not yml_path.endswith((".yml", ".yaml",)): | ||
| load_pretrained_model(yml_path, allow_install=True) | ||
| print('Model Loaded Successfully') | ||
| check_result = "Pass" | ||
| response = None | ||
| except Exception as e: | ||
| print(e) | ||
| logger.exception("Automated Model Check has failed!") | ||
| check_result = "Fail" | ||
| response = logger.error(e, exc_info=True) | ||
| # Convert response to JSON serializable format | ||
| if response is not None: | ||
| response = str(response) | ||
| row_data = { | ||
| 'url': yml_path, | ||
| 'check_result': check_result, | ||
| 'response': response, | ||
| } | ||
|
|
||
| rows.update({model.name: row_data}) | ||
|
|
||
| automated_checks_report = { | ||
| "time": datetime.now().isoformat(), | ||
| "report": rows | ||
| } | ||
| automated_checks_report_json = json.dumps(automated_checks_report) | ||
|
|
||
| return automated_checks_report_json | ||
|
|
||
|
|
||
| def entry_point(): | ||
| """This is the entry point for the 'scivision-check-models' | ||
| command. | ||
| """ | ||
| automated_checks_report_json = check_models() | ||
|
|
||
| with open('check_models.js', 'w') as f: | ||
| print('// This file was generated automatically by check_models.py', file=f) | ||
| print(f'var global_CheckModelReport = {automated_checks_report_json};', file=f) | ||
| # ^^^ requires changes to ModelTable.jsx similar to DataTable.jsx |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.