diff --git a/import_tracker/__init__.py b/import_tracker/__init__.py index 862d146..05bab59 100644 --- a/import_tracker/__init__.py +++ b/import_tracker/__init__.py @@ -1,6 +1,6 @@ import time -from girder import logger, plugin +from girder import plugin from girder.api.describe import autoDescribeRoute from girder.api.rest import boundHandler from girder.constants import AccessType @@ -28,30 +28,34 @@ def wrapImportData(assetstoreResource): def importDataWrapper( self, assetstore, importPath, destinationId, destinationType, progress, leafFoldersAsItems, fileIncludeRegex, fileExcludeRegex, - excludeExisting): + excludeExisting, **kwargs): # We don't actually wrap importData, as it would be excessive # monkey-patching to make the import trackable and cancelable. user = self.getCurrentUser() parent = ModelImporter.model(destinationType).load( destinationId, user=user, level=AccessType.ADMIN, exc=True) - # create a job, set it to running - # alter ProgressContext so update updates the job or raises if - # canceling + # Capture any additional parameters passed to route + extraParams = kwargs.get('params', {}) + params = { 'destinationId': destinationId, 'destinationType': destinationType, 'importPath': importPath, 'leafFoldersAsItems': str(leafFoldersAsItems).lower(), 'progress': str(progress).lower(), + **extraParams } + if fileIncludeRegex: params['fileIncludeRegex'] = fileIncludeRegex if fileExcludeRegex: params['fileExcludeRegex'] = fileExcludeRegex if excludeExisting: params['excludeExisting'] = str(excludeExisting).lower() + importRecord = AssetstoreImport().createAssetstoreImport(assetstore, params) + job = Job().createJob( title='Import from %s : %s' % (assetstore['name'], importPath), type='assetstore_import', @@ -63,6 +67,7 @@ def importDataWrapper( time.strftime('%Y-%m-%d %H:%M:%S'), assetstore['name'], importPath, ), status=JobStatus.RUNNING) + try: with ProgressContext(progress, user=user, title='Importing data') as ctx: try: @@ -76,10 +81,7 @@ def importDataWrapper( self._model.importData( assetstore, parent=parent, parentType=destinationType, params={ - 'fileIncludeRegex': fileIncludeRegex, - 'fileExcludeRegex': fileExcludeRegex, - 'importPath': importPath, - 'excludeExisting': excludeExisting, + **params, '_job': jobRec, }, progress=ctx, user=user, @@ -103,6 +105,7 @@ def importDataWrapper( exc, ), status=JobStatus.ERROR) success = False + importRecord = AssetstoreImport().markEnded(importRecord, success) return importRecord @@ -158,100 +161,12 @@ def shouldImportFileWrapper(self, path, params): AbstractAssetstoreAdapter.shouldImportFile = shouldImportFileWrapper -def wrapDICOMImport(assetstoreResource): - baseImportData = assetstoreResource.importData - baseImportData.description.param( - 'excludeExisting', - 'If true, then a file with an import path that is already in the ' - 'system is not imported, even if it is not in the destination ' - 'hierarchy.', dataType='boolean', required=False, default=False) - - @boundHandler(ctx=assetstoreResource) - @autoDescribeRoute(baseImportData.description) - def dwaImportDataWrapper(self, assetstore, destinationId, destinationType, filters, - progress, excludeExisting): - - user = self.getCurrentUser() - params = { - 'destinationId': destinationId, - 'destinationType': destinationType, - 'filters': filters, - 'progress': str(progress).lower(), - } - if excludeExisting: - params['excludeExisting'] = str(excludeExisting).lower() - - importRecord = AssetstoreImport().createAssetstoreImport(assetstore, params) - job = Job().createJob( - title=f'Import from {assetstore["name"]}', - type='assetstore_import', - public=False, - user=user, - kwargs=params, - ) - job = Job().updateJob(job, '%s - Starting import from %s\n' % ( - time.strftime('%Y-%m-%d %H:%M:%S'), - assetstore['name'] - ), status=JobStatus.RUNNING) - - try: - with ProgressContext(progress, user=user, title='Importing data') as ctx: - try: - jobRec = { - 'id': str(job['_id']), - 'count': 0, - 'skip': 0, - 'lastlog': time.time(), - 'logcount': 0, - } - self._importData( - assetstore, - params={ - **params, - '_job': jobRec}, - progress=ctx) - - success = True - Job().updateJob(job, '%s - Finished. Checked %d, skipped %d\n' % ( - time.strftime('%Y-%m-%d %H:%M:%S'), - jobRec['count'], jobRec['skip'], - ), status=JobStatus.SUCCESS) - - except ImportTrackerCancelError: - Job().updateJob(job, '%s - Canceled' % ( - time.strftime('%Y-%m-%d %H:%M:%S'), - )) - success = 'canceled' - - except Exception as exc: - Job().updateJob(job, '%s - Failed with %s\n' % ( - time.strftime('%Y-%m-%d %H:%M:%S'), - exc, - ), status=JobStatus.ERROR) - success = False - - importRecord = AssetstoreImport().markEnded(importRecord, success) - return importRecord - - for key in {'accessLevel', 'description', 'requiredScopes'}: - setattr(dwaImportDataWrapper, key, getattr(baseImportData, key)) - - assetstoreResource.importData = dwaImportDataWrapper - assetstoreResource.removeRoute('POST', (':id', 'import')) - assetstoreResource.route('POST', (':id', 'import'), assetstoreResource.importData) - - class GirderPlugin(plugin.GirderPlugin): DISPLAY_NAME = 'import_tracker' CLIENT_SOURCE_PATH = 'web_client' def load(self, info): plugin.getPlugin('jobs').load(info) - try: - import large_image_source_dicom # noqa - plugin.getPlugin('dicomweb').load(info) - except (ImportError, AttributeError): - pass ModelImporter.registerModel( 'assetstoreImport', AssetstoreImport, 'import_tracker' ) @@ -263,8 +178,3 @@ def load(self, info): wrapImportData(info['apiRoot'].assetstore) info['apiRoot'].folder.route('PUT', (':id', 'move'), moveFolder) - - if hasattr(info['apiRoot'], 'dicomweb_assetstore'): - wrapDICOMImport(info['apiRoot'].dicomweb_assetstore) - else: - logger.info('dicomweb_assetstore not found') diff --git a/import_tracker/web_client/main.js b/import_tracker/web_client/main.js index 80ad0ad..6cf26b7 100644 --- a/import_tracker/web_client/main.js +++ b/import_tracker/web_client/main.js @@ -3,6 +3,7 @@ import $ from 'jquery'; import AssetstoreView from '@girder/core/views/body/AssetstoresView'; import FilesystemImportView from '@girder/core/views/body/FilesystemImportView'; import S3ImportView from '@girder/core/views/body/S3ImportView'; +import DICOMwebImportView from '@girder/dicomweb/views/DICOMwebImportView'; import CollectionModel from '@girder/core/models/CollectionModel'; import FolderModel from '@girder/core/models/FolderModel'; @@ -54,12 +55,17 @@ wrap(S3ImportView, 'render', function (render) { this.$('.form-group').last().after(excludeExistingInput({ type: 's3' })); }); +wrap(DICOMwebImportView, 'render', function (render) { + render.call(this); + + this.$('.form-group').last().after(excludeExistingInput({ type: 'dwas' })); +}); const setBrowserRoot = (view, type) => { const browserWidget = view._browserWidgetView; - const destType = view.$(`#g-${type}-import-dest-type`).val(); - const destId = view.$(`#g-${type}-import-dest-id`).val(); - const resourceId = destId.trim().split(/\s/)[0]; + const destinationType = view.$(`#g-${type}-import-dest-type`).val(); + const destinationId = view.$(`#g-${type}-import-dest-id`).val(); + const resourceId = destinationId.trim().split(/\s/)[0]; const models = { collection: CollectionModel, @@ -67,8 +73,8 @@ const setBrowserRoot = (view, type) => { user: UserModel }; - if (resourceId && destType in models) { - const model = new models[destType]({ _id: resourceId }); + if (resourceId && destinationType in models) { + const model = new models[destinationType]({ _id: resourceId }); model.once('g:fetched', () => { if (!browserWidget.root || browserWidget.root.id !== model.id) { @@ -79,7 +85,7 @@ const setBrowserRoot = (view, type) => { if (err.status === 400) { events.trigger('g:alert', { icon: 'cancel', - text: `No ${destType.toUpperCase()} with ID ${resourceId} found.`, + text: `No ${destinationType.toUpperCase()} with ID ${resourceId} found.`, type: 'danger', timeout: 4000 }); @@ -97,52 +103,55 @@ wrap(S3ImportView, '_openBrowser', function (_openBrowser) { setBrowserRoot(this, 's3'); _openBrowser.call(this); }); +wrap(DICOMwebImportView, '_openBrowser', function (_openBrowser) { + setBrowserRoot(this, 'dwas'); + _openBrowser.call(this); +}); // We can't just wrap the submit events, as we need to modify what is passed to // the assetstore import method +const importSubmit = (view, type) => { + const destinationId = view.$(`#g-${type}-import-dest-id`).val().trim().split(/\s/)[0]; + const destinationType = view.$(`#g-${type}-import-dest-type`).val(); + const excludeExisting = view.$(`#g-${type}-import-exclude-existing`).val(); + + const importParams = { destinationId, destinationType, excludeExisting, progress: true }; + + if (type === 'filesystem') { + importParams.leafFoldersAsItems = view.$(`#g-${type}-import-leaf-items`).val(); + } + if (type === 'dwas') { + importParams.filters = view.$(`#g-${type}-import-filters`).val().trim(); + importParams.limit = view.$(`#g-${type}-import-limit`).val().trim(); + } else { + importParams.importPath = view.$(`#g-${type}-import-path`).val().trim(); + } + + view.$('.g-validation-failed-message').empty(); + view.$(`.g-submit-${type}-import`).addClass('disabled'); + + let model = view.assetstore; + model = model.off().on('g:imported', function () { + router.navigate(destinationType + '/' + destinationId, { trigger: true }); + }, view).on('g:error', function (err) { + view.$(`.g-submit-${type}-import`).removeClass('disabled'); + view.$('.g-validation-failed-message').html(err.responseJSON.message); + }, view); + + model.import(importParams); +}; + FilesystemImportView.prototype.events['submit .g-filesystem-import-form'] = function (e) { e.preventDefault(); - - var destId = this.$('#g-filesystem-import-dest-id').val().trim().split(/\s/)[0], - destType = this.$('#g-filesystem-import-dest-type').val(), - foldersAsItems = this.$('#g-filesystem-import-leaf-items').val(), - excludeExisting = this.$('#g-filesystem-import-exclude-existing').val(); - - this.$('.g-validation-failed-message').empty(); - - this.assetstore.off('g:imported').on('g:imported', function () { - router.navigate(destType + '/' + destId, { trigger: true }); - }, this).on('g:error', function (resp) { - this.$('.g-validation-failed-message').text(resp.responseJSON.message); - }, this).import({ - importPath: this.$('#g-filesystem-import-path').val().trim(), - leafFoldersAsItems: foldersAsItems, - destinationId: destId, - destinationType: destType, - excludeExisting: excludeExisting, - progress: true - }); + importSubmit(this, 'filesystem'); }; S3ImportView.prototype.events['submit .g-s3-import-form'] = function (e) { e.preventDefault(); - - var destId = this.$('#g-s3-import-dest-id').val().trim().split(/\s/)[0], - destType = this.$('#g-s3-import-dest-type').val(), - excludeExisting = this.$('#g-s3-import-exclude-existing').val(); - - this.$('.g-validation-failed-message').empty(); - - this.assetstore.off('g:imported').on('g:imported', function () { - router.navigate(destType + '/' + destId, { trigger: true }); - }, this).on('g:error', function (resp) { - this.$('.g-validation-failed-message').text(resp.responseJSON.message); - }, this).import({ - importPath: this.$('#g-s3-import-path').val().trim(), - destinationId: destId, - destinationType: destType, - excludeExisting: excludeExisting, - progress: true - }); + importSubmit(this, 's3'); +}; +DICOMwebImportView.prototype.events['submit .g-dwas-import-form'] = function (e) { + e.preventDefault(); + importSubmit(this, 'dwas'); }; // Setup router to assetstore imports view diff --git a/import_tracker/web_client/views/importList.js b/import_tracker/web_client/views/importList.js index e4db3df..b1dc3a8 100644 --- a/import_tracker/web_client/views/importList.js +++ b/import_tracker/web_client/views/importList.js @@ -5,7 +5,7 @@ import PaginateWidget from '@girder/core/views/widgets/PaginateWidget'; import Collection from '@girder/core/collections/Collection'; import AssetstoreModel from '@girder/core/models/AssetstoreModel'; -import { AssetstoreType, SORT_DESC } from '@girder/core/constants'; +import { SORT_DESC } from '@girder/core/constants'; import View from '@girder/core/views/View'; import router from '@girder/core/router'; import { restRequest } from '@girder/core/rest'; @@ -34,11 +34,7 @@ var importList = View.extend({ }, this); assetstore.once('g:fetched', () => { - if (assetstore.get('type') === AssetstoreType.DICOMWEB) { - assetstore.dicomwebImport(importEvent.get('params')); - } else { - assetstore.import(importEvent.get('params')); - } + assetstore.import(importEvent.get('params')); }).fetch(); }, 'click .re-import-edit-btn': function (e) { @@ -52,13 +48,7 @@ var importList = View.extend({ const navigate = (assetstoreId, importId) => { const assetstore = new AssetstoreModel({ _id: assetstoreId }); assetstore.once('g:fetched', () => { - if (assetstore.get('type') === AssetstoreType.DICOMWEB) { - // Avoid adding previous import data for DICOMweb imports by navigating to blank import - // TODO: Add DICOMweb-specific re-import view - router.navigate(`dicomweb_assetstore/${assetstoreId}/import`, { trigger: true }); - } else { - router.navigate(`assetstore/${assetstoreId}/re-import/${importId}`, { trigger: true }); - } + router.navigate(`assetstore/${assetstoreId}/re-import/${importId}`, { trigger: true }); }).fetch(); }; diff --git a/import_tracker/web_client/views/reImport.js b/import_tracker/web_client/views/reImport.js index ab78041..05fcdb2 100644 --- a/import_tracker/web_client/views/reImport.js +++ b/import_tracker/web_client/views/reImport.js @@ -1,11 +1,17 @@ import AssetstoreModel from '@girder/core/models/AssetstoreModel'; import View from '@girder/core/views/View'; +import { AssetstoreType } from '@girder/core/constants'; import router from '@girder/core/router'; import events from '@girder/core/events'; import { restRequest } from '@girder/core/rest'; -const goBack = (assetstoreId) => { +const goBack = (assetstoreId, message) => { + events.trigger('g:alert', { + icon: 'cancel', + text: `Could not re-import: ${message}`, + type: 'danger' + }); router.navigate( `assetstore/${assetstoreId}/import`, { trigger: true, replace: true } @@ -23,7 +29,7 @@ var reImportView = View.extend({ error: null }).done((assetstoreImport) => { if (!assetstoreImport) { - goBack(this.assetstoreId); + goBack(this.assetstoreId, `Unable to find import ${importId}`); return; } @@ -32,16 +38,20 @@ var reImportView = View.extend({ // collect assetstore type info and render const assetstore = new AssetstoreModel({ _id: assetstoreId }); assetstore.once('g:fetched', () => { - this.type = assetstore.get('type') === 0 ? 'filesystem' : 's3'; + const assetstoreType = assetstore.get('type'); + if (assetstoreType === AssetstoreType.FILESYSTEM) { + this.type = 'filesystem'; + } else if (assetstoreType === AssetstoreType.S3) { + this.type = 's3'; + } else if (assetstoreType === AssetstoreType.DICOMWEB) { + this.type = 'dwas'; + } else { + goBack(this.assetstoreId, `Unsupported assetstore type '${assetstoreType}'`); + } this.render(); }).fetch(); }).fail(() => { - events.trigger('g:alert', { - icon: 'cancel', - text: 'Unable to fetch base import information. Redirected to empty import page', - type: 'danger' - }); - goBack(this.assetstoreId); + goBack(this.assetstoreId, 'Unable to fetch base import information'); }); }, @@ -57,6 +67,11 @@ var reImportView = View.extend({ this.$(`#g-${this.type}-import-leaf-items`).val(params.leafFoldersAsItems); this.$(`#g-${this.type}-import-exclude-existing`).val(excludeExisting); + if (this.type === 'dwas') { + this.$(`#g-${this.type}-import-filters`).val(params.filters); + this.$(`#g-${this.type}-import-limit`).val(params.limit); + } + restRequest({ url: `resource/${destId}/path`, method: 'GET',