From d5be5216af48643874c227ea4961502ed18abcfd Mon Sep 17 00:00:00 2001 From: willdunklin Date: Tue, 14 Nov 2023 10:50:34 -0500 Subject: [PATCH 1/4] Make dicomweb plugin an explicit dependency for import_tracker --- import_tracker/web_client/package.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/import_tracker/web_client/package.json b/import_tracker/web_client/package.json index 5141dc2..bd19cf8 100644 --- a/import_tracker/web_client/package.json +++ b/import_tracker/web_client/package.json @@ -13,7 +13,8 @@ "name": "import_tracker", "main": "./main.js", "dependencies": [ - "jobs" + "jobs", + "dicomweb" ] }, "scripts": { From 4fa7c52f7de2191bfc2c390dae9d386f14efd368 Mon Sep 17 00:00:00 2001 From: willdunklin Date: Tue, 14 Nov 2023 10:51:03 -0500 Subject: [PATCH 2/4] Add JS wrappers/support for dicomweb elements --- import_tracker/web_client/main.js | 40 +++++++++++++++++++++++++++++-- 1 file changed, 38 insertions(+), 2 deletions(-) diff --git a/import_tracker/web_client/main.js b/import_tracker/web_client/main.js index 94b072a..d39793c 100644 --- a/import_tracker/web_client/main.js +++ b/import_tracker/web_client/main.js @@ -3,6 +3,7 @@ import $ from 'jquery'; import AssetstoreView from '@girder/core/views/body/AssetstoresView'; import FilesystemImportView from '@girder/core/views/body/FilesystemImportView'; import S3ImportView from '@girder/core/views/body/S3ImportView'; +import DICOMwebImportView from '@girder/dicomweb/views/DICOMwebImportView'; import CollectionModel from '@girder/core/models/CollectionModel'; import FolderModel from '@girder/core/models/FolderModel'; @@ -54,6 +55,11 @@ wrap(S3ImportView, 'render', function (render) { this.$('.form-group').last().after(excludeExistingInput({ type: 's3' })); }); +wrap(DICOMwebImportView, 'render', function (render) { + render.call(this); + + this.$('.form-group').last().after(excludeExistingInput({ type: 'dwas' })); +}); const setBrowserRoot = (view, type) => { const browserWidget = view._browserWidgetView; @@ -97,13 +103,17 @@ wrap(S3ImportView, '_openBrowser', function (_openBrowser) { setBrowserRoot(this, 's3'); _openBrowser.call(this); }); +wrap(DICOMwebImportView, '_openBrowser', function (_openBrowser) { + setBrowserRoot(this, 'dwas'); + _openBrowser.call(this); +}); // We can't just wrap the submit events, as we need to modify what is passed to // the assetstore import method FilesystemImportView.prototype.events['submit .g-filesystem-import-form'] = function (e) { e.preventDefault(); - var destId = this.$('#g-filesystem-import-dest-id').val().trim().split(/\s/)[0], + let destId = this.$('#g-filesystem-import-dest-id').val().trim().split(/\s/)[0], destType = this.$('#g-filesystem-import-dest-type').val(), foldersAsItems = this.$('#g-filesystem-import-leaf-items').val(), excludeExisting = this.$('#g-filesystem-import-exclude-existing').val(); @@ -126,7 +136,7 @@ FilesystemImportView.prototype.events['submit .g-filesystem-import-form'] = func S3ImportView.prototype.events['submit .g-s3-import-form'] = function (e) { e.preventDefault(); - var destId = this.$('#g-s3-import-dest-id').val().trim().split(/\s/)[0], + let destId = this.$('#g-s3-import-dest-id').val().trim().split(/\s/)[0], destType = this.$('#g-s3-import-dest-type').val(), excludeExisting = this.$('#g-s3-import-exclude-existing').val(); @@ -144,6 +154,32 @@ S3ImportView.prototype.events['submit .g-s3-import-form'] = function (e) { progress: true }); }; +DICOMwebImportView.prototype.events['submit .g-dwas-import-form'] = function (e) { + e.preventDefault(); + + let destId = this.$('#g-dwas-import-dest-id').val().trim().split(/\s/)[0], + destType = this.$('#g-dwas-import-dest-type').val(), + excludeExisting = this.$('#g-dwas-import-exclude-existing').val(), + filters = this.$('#g-dwas-import-filters').val().trim(), + limit = this.$('#g-dwas-import-limit').val().trim(); + + this.$('.g-validation-failed-message').empty(); + + this.$('.g-submit-dwas-import').addClass('disabled'); + this.model.off().on('g:imported', function () { + router.navigate(destType + '/' + destId, { trigger: true }); + }, this).on('g:error', function (err) { + this.$('.g-submit-dwas-import').removeClass('disabled'); + this.$('.g-validation-failed-message').html(err.responseJSON.message); + }, this).dicomwebImport({ + destinationId: destId, + destinationType: destType, + limit, + filters, + excludeExisting, + progress: true + }); +}; // Setup router to assetstore imports view router.route('assetstore/:id/imports', 'importsPage', function (id) { From fc66fd0124989fd45972d169aedceb3e5088fdb3 Mon Sep 17 00:00:00 2001 From: willdunklin Date: Wed, 15 Nov 2023 10:13:03 -0500 Subject: [PATCH 3/4] Refactor assetstore import submit wrapping --- import_tracker/web_client/main.js | 111 ++++++++++++------------------ 1 file changed, 44 insertions(+), 67 deletions(-) diff --git a/import_tracker/web_client/main.js b/import_tracker/web_client/main.js index d39793c..640e155 100644 --- a/import_tracker/web_client/main.js +++ b/import_tracker/web_client/main.js @@ -63,9 +63,9 @@ wrap(DICOMwebImportView, 'render', function (render) { const setBrowserRoot = (view, type) => { const browserWidget = view._browserWidgetView; - const destType = view.$(`#g-${type}-import-dest-type`).val(); - const destId = view.$(`#g-${type}-import-dest-id`).val(); - const resourceId = destId.trim().split(/\s/)[0]; + const destinationType = view.$(`#g-${type}-import-dest-type`).val(); + const destinationId = view.$(`#g-${type}-import-dest-id`).val(); + const resourceId = destinationId.trim().split(/\s/)[0]; const models = { collection: CollectionModel, @@ -73,8 +73,8 @@ const setBrowserRoot = (view, type) => { user: UserModel }; - if (resourceId && destType in models) { - const model = new models[destType]({ _id: resourceId }); + if (resourceId && destinationType in models) { + const model = new models[destinationType]({ _id: resourceId }); model.once('g:fetched', () => { if (!browserWidget.root || browserWidget.root.id !== model.id) { @@ -85,7 +85,7 @@ const setBrowserRoot = (view, type) => { if (err.status === 400) { events.trigger('g:alert', { icon: 'cancel', - text: `No ${destType.toUpperCase()} with ID ${resourceId} found.`, + text: `No ${destinationType.toUpperCase()} with ID ${resourceId} found.`, type: 'danger', timeout: 4000 }); @@ -110,75 +110,52 @@ wrap(DICOMwebImportView, '_openBrowser', function (_openBrowser) { // We can't just wrap the submit events, as we need to modify what is passed to // the assetstore import method +const importSubmit = (view, type) => { + const destinationId = view.$(`#g-${type}-import-dest-id`).val().trim().split(/\s/)[0]; + const destinationType = view.$(`#g-${type}-import-dest-type`).val(); + const excludeExisting = view.$(`#g-${type}-import-exclude-existing`).val(); + + const importParams = { destinationId, destinationType, excludeExisting, progress: true }; + + if (type === 'filesystem') { + importParams.leafFoldersAsItems = view.$(`#g-${type}-import-leaf-items`).val(); + } + if (type === 'dwas') { + importParams.filters = view.$(`#g-${type}-import-filters`).val().trim(); + importParams.limit = view.$(`#g-${type}-import-limit`).val().trim(); + } else { + importParams.importPath = view.$(`#g-${type}-import-path`).val().trim(); + } + + view.$('.g-validation-failed-message').empty(); + view.$(`.g-submit-${type}-import`).addClass('disabled'); + + let model = type === 'dwas' ? view.model : view.assetstore; + model = model.off().on('g:imported', function () { + router.navigate(destinationType + '/' + destinationId, { trigger: true }); + }, view).on('g:error', function (err) { + view.$(`.g-submit-${type}-import`).removeClass('disabled'); + view.$('.g-validation-failed-message').html(err.responseJSON.message); + }, view); + + if (type === 'dwas') { + model.dicomwebImport(importParams); + } else { + model.import(importParams); + } +}; + FilesystemImportView.prototype.events['submit .g-filesystem-import-form'] = function (e) { e.preventDefault(); - - let destId = this.$('#g-filesystem-import-dest-id').val().trim().split(/\s/)[0], - destType = this.$('#g-filesystem-import-dest-type').val(), - foldersAsItems = this.$('#g-filesystem-import-leaf-items').val(), - excludeExisting = this.$('#g-filesystem-import-exclude-existing').val(); - - this.$('.g-validation-failed-message').empty(); - - this.assetstore.off('g:imported').on('g:imported', function () { - router.navigate(destType + '/' + destId, { trigger: true }); - }, this).on('g:error', function (resp) { - this.$('.g-validation-failed-message').text(resp.responseJSON.message); - }, this).import({ - importPath: this.$('#g-filesystem-import-path').val().trim(), - leafFoldersAsItems: foldersAsItems, - destinationId: destId, - destinationType: destType, - excludeExisting: excludeExisting, - progress: true - }); + importSubmit(this, 'filesystem'); }; S3ImportView.prototype.events['submit .g-s3-import-form'] = function (e) { e.preventDefault(); - - let destId = this.$('#g-s3-import-dest-id').val().trim().split(/\s/)[0], - destType = this.$('#g-s3-import-dest-type').val(), - excludeExisting = this.$('#g-s3-import-exclude-existing').val(); - - this.$('.g-validation-failed-message').empty(); - - this.assetstore.off('g:imported').on('g:imported', function () { - router.navigate(destType + '/' + destId, { trigger: true }); - }, this).on('g:error', function (resp) { - this.$('.g-validation-failed-message').text(resp.responseJSON.message); - }, this).import({ - importPath: this.$('#g-s3-import-path').val().trim(), - destinationId: destId, - destinationType: destType, - excludeExisting: excludeExisting, - progress: true - }); + importSubmit(this, 's3'); }; DICOMwebImportView.prototype.events['submit .g-dwas-import-form'] = function (e) { e.preventDefault(); - - let destId = this.$('#g-dwas-import-dest-id').val().trim().split(/\s/)[0], - destType = this.$('#g-dwas-import-dest-type').val(), - excludeExisting = this.$('#g-dwas-import-exclude-existing').val(), - filters = this.$('#g-dwas-import-filters').val().trim(), - limit = this.$('#g-dwas-import-limit').val().trim(); - - this.$('.g-validation-failed-message').empty(); - - this.$('.g-submit-dwas-import').addClass('disabled'); - this.model.off().on('g:imported', function () { - router.navigate(destType + '/' + destId, { trigger: true }); - }, this).on('g:error', function (err) { - this.$('.g-submit-dwas-import').removeClass('disabled'); - this.$('.g-validation-failed-message').html(err.responseJSON.message); - }, this).dicomwebImport({ - destinationId: destId, - destinationType: destType, - limit, - filters, - excludeExisting, - progress: true - }); + importSubmit(this, 'dwas'); }; // Setup router to assetstore imports view From 8ee131443a538ff8d109cfe852d9013e608d7b4c Mon Sep 17 00:00:00 2001 From: willdunklin Date: Thu, 2 May 2024 13:26:06 -0400 Subject: [PATCH 4/4] Update importList/reImport for generic model.import changes --- import_tracker/__init__.py | 114 ++---------------- import_tracker/web_client/main.js | 8 +- import_tracker/web_client/package.json | 3 +- import_tracker/web_client/views/importList.js | 16 +-- import_tracker/web_client/views/reImport.js | 33 +++-- 5 files changed, 42 insertions(+), 132 deletions(-) diff --git a/import_tracker/__init__.py b/import_tracker/__init__.py index 862d146..05bab59 100644 --- a/import_tracker/__init__.py +++ b/import_tracker/__init__.py @@ -1,6 +1,6 @@ import time -from girder import logger, plugin +from girder import plugin from girder.api.describe import autoDescribeRoute from girder.api.rest import boundHandler from girder.constants import AccessType @@ -28,30 +28,34 @@ def wrapImportData(assetstoreResource): def importDataWrapper( self, assetstore, importPath, destinationId, destinationType, progress, leafFoldersAsItems, fileIncludeRegex, fileExcludeRegex, - excludeExisting): + excludeExisting, **kwargs): # We don't actually wrap importData, as it would be excessive # monkey-patching to make the import trackable and cancelable. user = self.getCurrentUser() parent = ModelImporter.model(destinationType).load( destinationId, user=user, level=AccessType.ADMIN, exc=True) - # create a job, set it to running - # alter ProgressContext so update updates the job or raises if - # canceling + # Capture any additional parameters passed to route + extraParams = kwargs.get('params', {}) + params = { 'destinationId': destinationId, 'destinationType': destinationType, 'importPath': importPath, 'leafFoldersAsItems': str(leafFoldersAsItems).lower(), 'progress': str(progress).lower(), + **extraParams } + if fileIncludeRegex: params['fileIncludeRegex'] = fileIncludeRegex if fileExcludeRegex: params['fileExcludeRegex'] = fileExcludeRegex if excludeExisting: params['excludeExisting'] = str(excludeExisting).lower() + importRecord = AssetstoreImport().createAssetstoreImport(assetstore, params) + job = Job().createJob( title='Import from %s : %s' % (assetstore['name'], importPath), type='assetstore_import', @@ -63,6 +67,7 @@ def importDataWrapper( time.strftime('%Y-%m-%d %H:%M:%S'), assetstore['name'], importPath, ), status=JobStatus.RUNNING) + try: with ProgressContext(progress, user=user, title='Importing data') as ctx: try: @@ -76,10 +81,7 @@ def importDataWrapper( self._model.importData( assetstore, parent=parent, parentType=destinationType, params={ - 'fileIncludeRegex': fileIncludeRegex, - 'fileExcludeRegex': fileExcludeRegex, - 'importPath': importPath, - 'excludeExisting': excludeExisting, + **params, '_job': jobRec, }, progress=ctx, user=user, @@ -103,6 +105,7 @@ def importDataWrapper( exc, ), status=JobStatus.ERROR) success = False + importRecord = AssetstoreImport().markEnded(importRecord, success) return importRecord @@ -158,100 +161,12 @@ def shouldImportFileWrapper(self, path, params): AbstractAssetstoreAdapter.shouldImportFile = shouldImportFileWrapper -def wrapDICOMImport(assetstoreResource): - baseImportData = assetstoreResource.importData - baseImportData.description.param( - 'excludeExisting', - 'If true, then a file with an import path that is already in the ' - 'system is not imported, even if it is not in the destination ' - 'hierarchy.', dataType='boolean', required=False, default=False) - - @boundHandler(ctx=assetstoreResource) - @autoDescribeRoute(baseImportData.description) - def dwaImportDataWrapper(self, assetstore, destinationId, destinationType, filters, - progress, excludeExisting): - - user = self.getCurrentUser() - params = { - 'destinationId': destinationId, - 'destinationType': destinationType, - 'filters': filters, - 'progress': str(progress).lower(), - } - if excludeExisting: - params['excludeExisting'] = str(excludeExisting).lower() - - importRecord = AssetstoreImport().createAssetstoreImport(assetstore, params) - job = Job().createJob( - title=f'Import from {assetstore["name"]}', - type='assetstore_import', - public=False, - user=user, - kwargs=params, - ) - job = Job().updateJob(job, '%s - Starting import from %s\n' % ( - time.strftime('%Y-%m-%d %H:%M:%S'), - assetstore['name'] - ), status=JobStatus.RUNNING) - - try: - with ProgressContext(progress, user=user, title='Importing data') as ctx: - try: - jobRec = { - 'id': str(job['_id']), - 'count': 0, - 'skip': 0, - 'lastlog': time.time(), - 'logcount': 0, - } - self._importData( - assetstore, - params={ - **params, - '_job': jobRec}, - progress=ctx) - - success = True - Job().updateJob(job, '%s - Finished. Checked %d, skipped %d\n' % ( - time.strftime('%Y-%m-%d %H:%M:%S'), - jobRec['count'], jobRec['skip'], - ), status=JobStatus.SUCCESS) - - except ImportTrackerCancelError: - Job().updateJob(job, '%s - Canceled' % ( - time.strftime('%Y-%m-%d %H:%M:%S'), - )) - success = 'canceled' - - except Exception as exc: - Job().updateJob(job, '%s - Failed with %s\n' % ( - time.strftime('%Y-%m-%d %H:%M:%S'), - exc, - ), status=JobStatus.ERROR) - success = False - - importRecord = AssetstoreImport().markEnded(importRecord, success) - return importRecord - - for key in {'accessLevel', 'description', 'requiredScopes'}: - setattr(dwaImportDataWrapper, key, getattr(baseImportData, key)) - - assetstoreResource.importData = dwaImportDataWrapper - assetstoreResource.removeRoute('POST', (':id', 'import')) - assetstoreResource.route('POST', (':id', 'import'), assetstoreResource.importData) - - class GirderPlugin(plugin.GirderPlugin): DISPLAY_NAME = 'import_tracker' CLIENT_SOURCE_PATH = 'web_client' def load(self, info): plugin.getPlugin('jobs').load(info) - try: - import large_image_source_dicom # noqa - plugin.getPlugin('dicomweb').load(info) - except (ImportError, AttributeError): - pass ModelImporter.registerModel( 'assetstoreImport', AssetstoreImport, 'import_tracker' ) @@ -263,8 +178,3 @@ def load(self, info): wrapImportData(info['apiRoot'].assetstore) info['apiRoot'].folder.route('PUT', (':id', 'move'), moveFolder) - - if hasattr(info['apiRoot'], 'dicomweb_assetstore'): - wrapDICOMImport(info['apiRoot'].dicomweb_assetstore) - else: - logger.info('dicomweb_assetstore not found') diff --git a/import_tracker/web_client/main.js b/import_tracker/web_client/main.js index 7b0ddab..6cf26b7 100644 --- a/import_tracker/web_client/main.js +++ b/import_tracker/web_client/main.js @@ -130,7 +130,7 @@ const importSubmit = (view, type) => { view.$('.g-validation-failed-message').empty(); view.$(`.g-submit-${type}-import`).addClass('disabled'); - let model = type === 'dwas' ? view.model : view.assetstore; + let model = view.assetstore; model = model.off().on('g:imported', function () { router.navigate(destinationType + '/' + destinationId, { trigger: true }); }, view).on('g:error', function (err) { @@ -138,11 +138,7 @@ const importSubmit = (view, type) => { view.$('.g-validation-failed-message').html(err.responseJSON.message); }, view); - if (type === 'dwas') { - model.dicomwebImport(importParams); - } else { - model.import(importParams); - } + model.import(importParams); }; FilesystemImportView.prototype.events['submit .g-filesystem-import-form'] = function (e) { diff --git a/import_tracker/web_client/package.json b/import_tracker/web_client/package.json index bd19cf8..5141dc2 100644 --- a/import_tracker/web_client/package.json +++ b/import_tracker/web_client/package.json @@ -13,8 +13,7 @@ "name": "import_tracker", "main": "./main.js", "dependencies": [ - "jobs", - "dicomweb" + "jobs" ] }, "scripts": { diff --git a/import_tracker/web_client/views/importList.js b/import_tracker/web_client/views/importList.js index e4db3df..b1dc3a8 100644 --- a/import_tracker/web_client/views/importList.js +++ b/import_tracker/web_client/views/importList.js @@ -5,7 +5,7 @@ import PaginateWidget from '@girder/core/views/widgets/PaginateWidget'; import Collection from '@girder/core/collections/Collection'; import AssetstoreModel from '@girder/core/models/AssetstoreModel'; -import { AssetstoreType, SORT_DESC } from '@girder/core/constants'; +import { SORT_DESC } from '@girder/core/constants'; import View from '@girder/core/views/View'; import router from '@girder/core/router'; import { restRequest } from '@girder/core/rest'; @@ -34,11 +34,7 @@ var importList = View.extend({ }, this); assetstore.once('g:fetched', () => { - if (assetstore.get('type') === AssetstoreType.DICOMWEB) { - assetstore.dicomwebImport(importEvent.get('params')); - } else { - assetstore.import(importEvent.get('params')); - } + assetstore.import(importEvent.get('params')); }).fetch(); }, 'click .re-import-edit-btn': function (e) { @@ -52,13 +48,7 @@ var importList = View.extend({ const navigate = (assetstoreId, importId) => { const assetstore = new AssetstoreModel({ _id: assetstoreId }); assetstore.once('g:fetched', () => { - if (assetstore.get('type') === AssetstoreType.DICOMWEB) { - // Avoid adding previous import data for DICOMweb imports by navigating to blank import - // TODO: Add DICOMweb-specific re-import view - router.navigate(`dicomweb_assetstore/${assetstoreId}/import`, { trigger: true }); - } else { - router.navigate(`assetstore/${assetstoreId}/re-import/${importId}`, { trigger: true }); - } + router.navigate(`assetstore/${assetstoreId}/re-import/${importId}`, { trigger: true }); }).fetch(); }; diff --git a/import_tracker/web_client/views/reImport.js b/import_tracker/web_client/views/reImport.js index ab78041..05fcdb2 100644 --- a/import_tracker/web_client/views/reImport.js +++ b/import_tracker/web_client/views/reImport.js @@ -1,11 +1,17 @@ import AssetstoreModel from '@girder/core/models/AssetstoreModel'; import View from '@girder/core/views/View'; +import { AssetstoreType } from '@girder/core/constants'; import router from '@girder/core/router'; import events from '@girder/core/events'; import { restRequest } from '@girder/core/rest'; -const goBack = (assetstoreId) => { +const goBack = (assetstoreId, message) => { + events.trigger('g:alert', { + icon: 'cancel', + text: `Could not re-import: ${message}`, + type: 'danger' + }); router.navigate( `assetstore/${assetstoreId}/import`, { trigger: true, replace: true } @@ -23,7 +29,7 @@ var reImportView = View.extend({ error: null }).done((assetstoreImport) => { if (!assetstoreImport) { - goBack(this.assetstoreId); + goBack(this.assetstoreId, `Unable to find import ${importId}`); return; } @@ -32,16 +38,20 @@ var reImportView = View.extend({ // collect assetstore type info and render const assetstore = new AssetstoreModel({ _id: assetstoreId }); assetstore.once('g:fetched', () => { - this.type = assetstore.get('type') === 0 ? 'filesystem' : 's3'; + const assetstoreType = assetstore.get('type'); + if (assetstoreType === AssetstoreType.FILESYSTEM) { + this.type = 'filesystem'; + } else if (assetstoreType === AssetstoreType.S3) { + this.type = 's3'; + } else if (assetstoreType === AssetstoreType.DICOMWEB) { + this.type = 'dwas'; + } else { + goBack(this.assetstoreId, `Unsupported assetstore type '${assetstoreType}'`); + } this.render(); }).fetch(); }).fail(() => { - events.trigger('g:alert', { - icon: 'cancel', - text: 'Unable to fetch base import information. Redirected to empty import page', - type: 'danger' - }); - goBack(this.assetstoreId); + goBack(this.assetstoreId, 'Unable to fetch base import information'); }); }, @@ -57,6 +67,11 @@ var reImportView = View.extend({ this.$(`#g-${this.type}-import-leaf-items`).val(params.leafFoldersAsItems); this.$(`#g-${this.type}-import-exclude-existing`).val(excludeExisting); + if (this.type === 'dwas') { + this.$(`#g-${this.type}-import-filters`).val(params.filters); + this.$(`#g-${this.type}-import-limit`).val(params.limit); + } + restRequest({ url: `resource/${destId}/path`, method: 'GET',