Merge pull request #101 from scottwittenburg/display-import-errors-on-client

Provide more details to client when encountering errors on import

Author: scottwittenburg

client/src/components/DataImportExport.vue

@@ -9,7 +9,9 @@ export default {
     importEnabled: false,
     exportEnabled: false,
     importing: false,
-    importDialog: false
+    importDialog: false,
+    importErrorText: "",
+    importErrors: false
   }),
   async created() {
     var { data: result } = await this.girderRest.get(
@@ -22,19 +24,26 @@ export default {
     ...mapActions(["loadSessions"]),
     async importData() {
       this.importing = true;
+      this.importErrorText = "";
+      this.importErrors = false;
       try {
         var { data: result } = await this.girderRest.post("miqa/data/import");
         this.importing = false;
-        this.$snackbar({
-          text: `Import finished.
-          With ${result.success} scans succeeded and ${result.failed} failed.`,
-          timeout: 6000
-        });
+        if (result.errorMsg) {
+          this.importErrorText = `${result.success} scans succeeded, ${result.failed} failed.\n\n${result.errorMsg}`;
+          this.importErrors = true;
+        } else {
+          this.$snackbar({
+            text: `Import finished.
+            With ${result.success} scans succeeded and ${result.failed} failed.`,
+            timeout: 6000
+          });
+        }
         this.loadSessions();
       } catch (ex) {
         this.importing = false;
         this.$snackbar({
-          text: "Import failed. Refer console for detail."
+          text: "Import failed. Refer to server logs for details."
         });
         console.error(ex.response);
       }
@@ -84,7 +93,31 @@ export default {
         </v-card-actions>
       </v-card>
     </v-dialog>
+    <v-dialog v-model="importErrors" content-class="import-error-dialog">
+      <v-card>
+        <v-card-title class="title">Import Errors Encountered</v-card-title>
+        <v-card-text class="console-format">
+          {{ importErrorText }}
+        </v-card-text>
+        <v-card-actions>
+          <v-spacer></v-spacer>
+          <v-btn color="primary" text @click="importErrors = false">
+            Ok
+          </v-btn>
+        </v-card-actions>
+      </v-card>
+    </v-dialog>
   </div>
 </template>
 
-<style lang="scss" scoped></style>
+<style lang="scss">
+.import-error-dialog {
+  position: relative;
+  width: 100%;
+  margin: 48px;
+}
+.console-format {
+  white-space: pre-wrap;
+  font-family: monospace;
+}
+</style>

server/miqa_server/session.py

@@ -4,6 +4,8 @@
 from jsonschema import validate
 from jsonschema.exceptions import ValidationError as JSONValidationError
 import os
+import sys
+import traceback
 
 from girder.api.rest import Resource, setResponseHeader, setContentDisposition
 from girder.api import access, rest
@@ -135,125 +137,126 @@ def _getSessions(self):
         Description('')
         .errorResponse())
     def dataImport(self, params):
-        user = self.getCurrentUser()
-        importpath = os.path.expanduser(Setting().get(importpathKey))
-        if not os.path.isfile(importpath):
-            raise RestException('import path does not exist ({0}'.format(importpath), code=404)
+        successCount = 0
+        failedCount = 0
+        errorMsg = ''
+        try:
+            user = self.getCurrentUser()
+            importpath = os.path.expanduser(Setting().get(importpathKey))
 
-        json_content = None
+            json_content = None
 
-        if importpath.endswith('.csv'):
-            with open(importpath) as fd:
-                csv_content = fd.read()
-                try:
+            if importpath.endswith('.csv'):
+                with open(importpath) as fd:
+                    csv_content = fd.read()
                     json_content = csvContentToJsonObject(csv_content)
                     validate(json_content, schema)
-                except (JSONValidationError, Exception) as inst:
-                    return {
-                        "error": 'Invalid CSV file: {0}'.format(inst.message),
-                        "success": successCount,
-                        "failed": failedCount
-                    }
-        else:
-            with open(importpath) as json_file:
-                json_content = json.load(json_file)
-                try:
+            else:
+                with open(importpath) as json_file:
+                    json_content = json.load(json_file)
                     validate(json_content, schema)
-                except JSONValidationError as inst:
-                    return {
-                        "error": 'Invalid JSON file: {0}'.format(inst.message),
-                        "success": successCount,
-                        "failed": failedCount
-                    }
 
-        existingSessionsFolder = self.findSessionsFolder(user)
-        if existingSessionsFolder:
-            existingSessionsFolder['name'] = 'sessions_' + \
-                datetime.datetime.now().strftime("%Y-%m-%d %I:%M:%S %p")
-            Folder().save(existingSessionsFolder)
-        sessionsFolder = self.findSessionsFolder(user, True)
-        Item().createItem('json', user, sessionsFolder, description=json.dumps(json_content))
+            existingSessionsFolder = self.findSessionsFolder(user)
+            if existingSessionsFolder:
+                existingSessionsFolder['name'] = 'sessions_' + \
+                    datetime.datetime.now().strftime("%Y-%m-%d %I:%M:%S %p")
+                Folder().save(existingSessionsFolder)
+            sessionsFolder = self.findSessionsFolder(user, True)
+            Item().createItem('json', user, sessionsFolder, description=json.dumps(json_content))
 
-        datasetRoot = json_content['data_root']
-        experiments = json_content['experiments']
-        sites = json_content['sites']
+            datasetRoot = json_content['data_root']
+            experiments = json_content['experiments']
+            sites = json_content['sites']
 
-        successCount = 0
-        failedCount = 0
-        sites = set()
-        for scan in json_content['scans']:
-            experimentId = scan['experiment_id']
-            experimentNote = ''
-            for experiment in experiments:
-                if experiment['id'] == experimentId:
-                    experimentNote = experiment['note']
-            scanPath = scan['path']
-            site = scan['site_id']
-            sites.add(site)
-            scanId = scan['id']
-            scanType = scan['type']
-            scanName = scanId+'_'+scanType
-            niftiFolder = os.path.expanduser(os.path.join(datasetRoot, scanPath))
-            if not os.path.isdir(niftiFolder):
-                failedCount += 1
-                continue
-            experimentFolder = Folder().createFolder(
-                sessionsFolder, experimentId, parentType='folder', reuseExisting=True)
-            scanFolder = Folder().createFolder(
-                experimentFolder, scanName, parentType='folder', reuseExisting=True)
-            meta = {
-                'experimentId': experimentId,
-                'experimentNote': experimentNote,
-                'site': site,
-                'scanId': scanId,
-                'scanType': scanType
-            }
-            if 'decision' in scan:
-                meta['rating'] = convertDecisionToRating(scan['decision'])
-            if 'note' in scan:
-                meta['note'] = scan['note']
-            Folder().setMetadata(scanFolder, meta)
-            currentAssetstore = Assetstore().getCurrent()
-            if 'images' in scan:
-                scanImages = scan['images']
-                # Import images one at a time because the user provided a list
-                for scanImage in scanImages:
-                    absImagePath = os.path.join(niftiFolder, scanImage)
+
+            sites = set()
+            for scan in json_content['scans']:
+                experimentId = scan['experiment_id']
+                experimentNote = ''
+                for experiment in experiments:
+                    if experiment['id'] == experimentId:
+                        experimentNote = experiment['note']
+                scanPath = scan['path']
+                site = scan['site_id']
+                sites.add(site)
+                scanId = scan['id']
+                scanType = scan['type']
+                scanName = scanId+'_'+scanType
+                niftiFolder = os.path.expanduser(os.path.join(datasetRoot, scanPath))
+                if not os.path.isdir(niftiFolder):
+                    errorMsg += '{0}: path {1} does not exist\n'.format(experimentId, niftiFolder)
+                    failedCount += 1
+                    continue
+                experimentFolder = Folder().createFolder(
+                    sessionsFolder, experimentId, parentType='folder', reuseExisting=True)
+                scanFolder = Folder().createFolder(
+                    experimentFolder, scanName, parentType='folder', reuseExisting=True)
+                meta = {
+                    'experimentId': experimentId,
+                    'experimentNote': experimentNote,
+                    'site': site,
+                    'scanId': scanId,
+                    'scanType': scanType
+                }
+                if 'decision' in scan:
+                    meta['rating'] = convertDecisionToRating(scan['decision'])
+                if 'note' in scan:
+                    meta['note'] = scan['note']
+                Folder().setMetadata(scanFolder, meta)
+                currentAssetstore = Assetstore().getCurrent()
+                if 'images' in scan:
+                    scanImages = scan['images']
+                    # Import images one at a time because the user provided a list
+                    for scanImage in scanImages:
+                        absImagePath = os.path.join(niftiFolder, scanImage)
+                        Assetstore().importData(
+                            currentAssetstore, parent=scanFolder, parentType='folder', params={
+                                'fileIncludeRegex': '^{0}$'.format(scanImage),
+                                'importPath': niftiFolder,
+                            }, progress=noProgress, user=user, leafFoldersAsItems=False)
+                    imageOrderDescription = {
+                        'orderDescription': {
+                            'images': scanImages
+                        }
+                    }
+                else:
+                    scanImagePattern = scan['imagePattern']
+                    # Import all images in directory at once because user provide a file pattern
                     Assetstore().importData(
                         currentAssetstore, parent=scanFolder, parentType='folder', params={
-                            'fileIncludeRegex': '^{0}$'.format(scanImage),
+                            'fileIncludeRegex': scanImagePattern,
                             'importPath': niftiFolder,
                         }, progress=noProgress, user=user, leafFoldersAsItems=False)
-                imageOrderDescription = {
-                    'orderDescription': {
-                        'images': scanImages
-                    }
-                }
-            else:
-                scanImagePattern = scan['imagePattern']
-                # Import all images in directory at once because user provide a file pattern
-                Assetstore().importData(
-                    currentAssetstore, parent=scanFolder, parentType='folder', params={
-                        'fileIncludeRegex': scanImagePattern,
-                        'importPath': niftiFolder,
-                    }, progress=noProgress, user=user, leafFoldersAsItems=False)
-                imageOrderDescription = {
-                    'orderDescription': {
-                        'imagePattern': scanImagePattern
+                    imageOrderDescription = {
+                        'orderDescription': {
+                            'imagePattern': scanImagePattern
+                        }
                     }
-                }
-            Item().createItem(name='imageOrderDescription',
-                              creator=user,
-                              folder=scanFolder,
-                              reuseExisting=True,
-                              description=json.dumps(imageOrderDescription))
-            successCount += 1
-        tryAddSites(sites, self.getCurrentUser())
-        return {
-            "success": successCount,
-            "failed": failedCount
+                Item().createItem(name='imageOrderDescription',
+                                  creator=user,
+                                  folder=scanFolder,
+                                  reuseExisting=True,
+                                  description=json.dumps(imageOrderDescription))
+                successCount += 1
+            tryAddSites(sites, self.getCurrentUser())
+        except Exception as inst:
+            errorMsg += '{0}\n\n'.format(inst)
+
+            stringBuffer = io.StringIO()
+            excType, excVal, excTb = sys.exc_info()
+            traceback.print_exception(excType, excVal, excTb, file=stringBuffer)
+            errorMsg += stringBuffer.getvalue()
+
+        result = {
+            'success': successCount,
+            'failed': failedCount
         }
 
+        if errorMsg != '':
+            result['errorMsg'] = errorMsg
+
+        return result
+
     @access.user
     @autoDescribeRoute(
         Description('')