Multi genbank/fasta files import

api/src/org/labkey/api/pipeline/AppPipelineJobNotificationProvider.java

@@ -182,7 +182,7 @@ private String getJobSuccessMsg(PipelineJob job, @NotNull ImportType importType,
                     .append(!crossTypeImport ? type + " " : "")
                     .append(importType.name())
                     .append(" from ")
-                    .append(queryImportPipelineJob.getImportContextBuilder().getPrimaryFile().getName());
+                    .append(queryImportPipelineJob.getImportSourceDescription());
 
             if (job.getContainer().hasProductFolders())
             {
@@ -214,7 +214,7 @@ else if (job instanceof AssayUploadPipelineJob<?> assayJob)
         return null;
     }
 
-    private String getJobErrorMsg(PipelineJob job, String rawErrorMsg)
+    private String getJobErrorMsg(PipelineJob job, @NotNull String errorMsg)
     {
         if (job instanceof QueryImportPipelineJob queryImportPipelineJob)
         {
@@ -224,16 +224,14 @@ private String getJobErrorMsg(PipelineJob job, String rawErrorMsg)
             return "Failed to import " +
                     type +
                     " from " +
-                    queryImportPipelineJob.getImportContextBuilder().getPrimaryFile().getName() +
-                    "\n" +
-                    rawErrorMsg;// resolveErrorMessage on client
+                    queryImportPipelineJob.getImportSourceDescription() +
+                    errorMsg;// resolveErrorMessage on client
         }
         else if (job instanceof AssayUploadPipelineJob<?> assayJob)
         {
             return "Failed to import assay run from " +
                     assayJob.getPrimaryFile().getName() +
-                    "\n" +
-                    rawErrorMsg;
+                    errorMsg;
         }
 
         return null;
@@ -332,7 +330,7 @@ public void sendJobNotification(PipelineJob job, @NotNull ImportType importType,
         }
         else
         {
-            n.setContent(getJobErrorMsg(job, msgContent), "text/plain");
+            n.setContent(getJobErrorMsg(job, msgContent == null ? "" : "\n" + msgContent), "text/plain");
             n.setActionLinkURL(getJobErrorUrl(job));
             n.setActionLinkText("view error details");
         }

api/src/org/labkey/api/query/AbstractQueryImportAction.java

@@ -45,6 +45,7 @@
 import org.labkey.api.module.Module;
 import org.labkey.api.module.ModuleLoader;
 import org.labkey.api.pipeline.PipeRoot;
+import org.labkey.api.pipeline.PipelineJob;
 import org.labkey.api.pipeline.PipelineService;
 import org.labkey.api.reader.ColumnDescriptor;
 import org.labkey.api.reader.DataLoader;
@@ -71,6 +72,7 @@
 import org.labkey.api.webdav.WebdavService;
 import org.labkey.api.workflow.WorkflowService;
 import org.labkey.vfs.FileLike;
+import org.springframework.util.MultiValueMap;
 import org.springframework.validation.BindException;
 import org.springframework.web.multipart.MultipartFile;
 import org.springframework.web.multipart.MultipartHttpServletRequest;
@@ -435,7 +437,7 @@ public final ApiResponse _execute(FORM form, BindException errors) throws Except
         if (errors.hasErrors())
             throw errors;
 
-        FileLike dataFile = null;
+        List<FileLike> dataFiles = new ArrayList<>();
         boolean hasPostData = false;
         FileStream file = null;
         String originalName = null;
@@ -547,17 +549,19 @@ else if (getViewContext().getRequest() instanceof MultipartHttpServletRequest)
             {
                 Map<String, MultipartFile> files = getFileMap();
                 MultipartFile multipartfile = null==files ? null : files.get("file");
-                if (null != multipartfile && multipartfile.getSize() > 0)
+                List<MultipartFile> uploadedFiles = getUploadedFiles();
+                if (!uploadedFiles.isEmpty())
                 {
-                    transactionDetails.put(TransactionAuditProvider.TransactionDetail.ImportFileName, multipartfile.getOriginalFilename());
                     hasPostData = true;
-                    originalName = multipartfile.getOriginalFilename();
-                    // can't read the multipart file twice so create temp file (12800)
-                    dataFile = FileUtil.createTempFileLike("~upload", multipartfile.getOriginalFilename());
                     PipeRoot root = PipelineService.get().findPipelineRoot(getContainer());
+                    originalName = uploadedFiles.getFirst().getOriginalFilename();
+                    transactionDetails.put(TransactionAuditProvider.TransactionDetail.ImportFileName, originalName);
+
+                    FileLike dataFileDir = null;
+
                     if (null != root && (Boolean.parseBoolean(saveToPipeline) || _useAsync))
                     {
-                        FileLike dataFileDir = root.getRootFileLike().resolveChild("QueryImportFiles");
+                        dataFileDir = root.getRootFileLike().resolveChild("QueryImportFiles");
                         if (dataFileDir.isFile())
                         {
                             dataFileDir = FileUtil.findUniqueFileName("QueryImportFiles", root.getRootFileLike());
@@ -571,49 +575,35 @@ else if (!dataFileDir.exists())
                                 throw new RuntimeException("Error attempting to create directory " + dataFileDir
                                         + " for uploaded query import file " + multipartfile.getOriginalFilename());
                         }
+                    }
+
+                    for (MultipartFile uploadedFile : uploadedFiles)
+                    {
+                        // can't read the multipart file twice so create temp file (12800)
+                        FileLike uploaded = FileUtil.createTempFileLike("~upload", uploadedFile.getOriginalFilename());
 
-                        dataFile = FileUtil.findUniqueFileName(multipartfile.getOriginalFilename(), dataFileDir);
+                        if (dataFileDir != null)
+                            uploaded = FileUtil.findUniqueFileName(uploadedFile.getOriginalFilename(), dataFileDir);
 
+                        uploadedFile.transferTo(uploaded.toNioPathForWrite());
+
+                        dataFiles.add(uploaded);
                     }
-                    multipartfile.transferTo(dataFile.toNioPathForWrite());
+
                     if (_useAsync)
                     {
-                        if (!isBackgroundImportSupported(dataFile.getName()))
+                        if (!isBackgroundImportSupported(dataFiles.getFirst().getName()))
                             throw new RuntimeException("Importing in background currently is not supported for this table");
 
                         ViewBackgroundInfo info = new ViewBackgroundInfo(getContainer(), getUser(), new ActionURL());
 
                         UserSchema schema = getTargetSchema();
                         if (schema != null)
                         {
-                            String schemaName = schema.getSchemaName();
-                            String queryName = getPipelineTargetQueryName();
-
-                            QueryImportPipelineJob.QueryImportAsyncContextBuilder importContextBuilder = new QueryImportPipelineJob.QueryImportAsyncContextBuilder();
-                            importContextBuilder
-                                .setPrimaryFile(dataFile)
-                                .setHasColumnHeaders(_hasColumnHeaders)
-                                .setFileContentType(multipartfile.getContentType())
-                                .setSchemaName(schemaName)
-                                .setQueryName(queryName)
-                                .setRenamedColumns(getRenamedColumns())
-                                .setLineageImportAliases(getLineageImportAliases())
-                                .setInsertOption(_insertOption)
-                                .setAuditBehaviorType(behaviorType)
-                                .setAuditUserComment(_auditUserComment)
-                                .setOptionParamsMap(getOptionParamsMap())
-                                .setLookupResolutionType(getLookupResolutionType())
-                                .setAllowLineageColumns(allowLineageColumns())
-                                .setJobDescription(getQueryImportDescription())
-                                .setJobNotificationProvider(getQueryImportJobNotificationProviderName());
-                            if (WorkflowService.get() != null)
-                            {
-                                Map<String, Object> workflowParams = WorkflowService.get().getConfigParameters(getViewContext().getRequest());
-                                importContextBuilder.setWorkflowParams(workflowParams);
-                            }
-
-                            importContextBuilder.setTransactionDetails(transactionDetails);
-                            QueryImportPipelineJob job = new QueryImportPipelineJob(getQueryImportProviderName(), info, root, importContextBuilder);
+                            QueryImportPipelineJob.QueryImportAsyncContextBuilder importContextBuilder = getBackgroundImportContext(dataFiles.getFirst(), multipartfile, behaviorType, transactionDetails);
+
+                            PipelineJob job = getImportPipelineJob(dataFiles, info, root, importContextBuilder);
+
                             PipelineService.get().queueJob(job, getQueryImportJobNotificationProviderName());
 
                             if (_target != null)
@@ -626,8 +616,9 @@ else if (!dataFileDir.exists())
 
                     }
 
-                    loader = DataLoader.get().createLoader(dataFile.toNioPathForRead().toFile(), multipartfile.getContentType(), _hasColumnHeaders, null, null);
-                    file = new FileAttachmentFile(dataFile, multipartfile.getOriginalFilename());
+                    Pair<DataLoader, FileStream> loadedFile = prepareDataFileLoader(dataFiles.getFirst(), uploadedFiles.getFirst());
+                    loader = loadedFile.first;
+                    file = loadedFile.second;
                 }
             }
 
@@ -636,25 +627,11 @@ else if (!dataFileDir.exists())
             if (errors.hasErrors())
                 throw errors;
 
-            BatchValidationException ve = new BatchValidationException();
-            //di = wrap(di, ve);
-            //importData(di, ve);
-
-            configureLoader(loader, _target, getRenamedColumns(), allowLineageColumns(), getLineageImportAliases(), getOptionParamsMap());
-
             TransactionAuditProvider.TransactionAuditEvent auditEvent = null;
             if (isCrossTypeImport || (behaviorType != null && behaviorType != AuditBehaviorType.NONE))
                 auditEvent = createTransactionAuditEvent(getContainer(), _insertOption.auditAction, transactionDetails);
 
-            int rowCount = importData(loader, file, originalName, ve, behaviorType, auditEvent, _auditUserComment);
-
-            if (ve.hasErrors())
-            {
-                addImportValidationErrorMetric(_insertOption, _target, (form instanceof QueryForm qf) ? qf.getQueryName() : null);
-                throw ve;
-            }
-
-            JSONObject response = createSuccessResponse(rowCount);
+            JSONObject response = handleImportData(loader, dataFiles, file, originalName, behaviorType, auditEvent, (form instanceof QueryForm qf ? qf.getQueryName() : null));
             if (auditEvent != null)
             {
                 response.put("transactionAuditId", auditEvent.getRowId());
@@ -675,10 +652,93 @@ else if (!dataFileDir.exists())
                 loader.close();
             if (null != file)
                 file.closeInputStream();
-            if (null != dataFile && !Boolean.parseBoolean(saveToPipeline) && !_useAsync)
-                dataFile.delete();
+            if (!dataFiles.isEmpty() && !Boolean.parseBoolean(saveToPipeline) && !_useAsync)
+            {
+                for (FileLike tmpFile : dataFiles)
+                    tmpFile.delete();
+            }
+
+        }
+
+    }
+
+    protected Pair<DataLoader, FileStream> prepareDataFileLoader(FileLike dataFile, MultipartFile multipartFile) throws IOException
+    {
+        DataLoader loader = DataLoader.get().createLoader(dataFile.toNioPathForRead().toFile(), multipartFile.getContentType(), _hasColumnHeaders, null, null);
+        FileStream file = new FileAttachmentFile(dataFile, multipartFile.getOriginalFilename());
+        return new Pair<>(loader, file);
+    }
+
+    protected JSONObject handleImportData(DataLoader loader, List<FileLike> dataFiles, FileStream file, String originalName, AuditBehaviorType behaviorType,
+                                          TransactionAuditProvider.TransactionAuditEvent auditEvent, String queryName) throws BatchValidationException, IOException
+    {
+        BatchValidationException ve = new BatchValidationException();
+
+        configureLoader(loader, _target, getRenamedColumns(), allowLineageColumns(), getLineageImportAliases(), getOptionParamsMap());
+
+        int rowCount = importData(loader, file, originalName, ve, behaviorType, auditEvent, _auditUserComment);
+
+        if (ve.hasErrors())
+        {
+            addImportValidationErrorMetric(_insertOption, _target, queryName);
+            throw ve;
+        }
+
+        return createSuccessResponse(rowCount);
+    }
+
+    private @NotNull List<MultipartFile> getUploadedFiles()
+    {
+        MultiValueMap<String, MultipartFile> fileMap = PageFlowUtil.getMultiFileMap(getViewContext().getRequest());
+        List<MultipartFile> files = new ArrayList<>();
+        if (fileMap != null && fileMap.get("file") != null)
+        {
+            for (MultipartFile mf : fileMap.get("file"))
+            {
+                if (mf != null && mf.getSize() > 0)
+                    files.add(mf);
+            }
+        }
+        return files;
+    }
+
+    protected PipelineJob getImportPipelineJob(List<FileLike> dataFiles, ViewBackgroundInfo info, PipeRoot root, QueryImportPipelineJob.QueryImportAsyncContextBuilder importContextBuilder)
+    {
+        return new QueryImportPipelineJob(getQueryImportProviderName(), info, root, importContextBuilder);
+    }
+
+    protected QueryImportPipelineJob.QueryImportAsyncContextBuilder getBackgroundImportContext(FileLike dataFile, @Nullable MultipartFile multipartfile, AuditBehaviorType behaviorType, Map<TransactionAuditProvider.TransactionDetail, Object> transactionDetails) throws IOException, ValidationException
+    {
+        UserSchema schema = getTargetSchema();
+        String schemaName = schema.getSchemaName();
+        String queryName = getPipelineTargetQueryName();
+
+        QueryImportPipelineJob.QueryImportAsyncContextBuilder importContextBuilder = new QueryImportPipelineJob.QueryImportAsyncContextBuilder();
+        importContextBuilder
+                .setPrimaryFile(dataFile)
+                .setHasColumnHeaders(_hasColumnHeaders)
+                .setFileContentType(multipartfile != null ? multipartfile.getContentType() : null)
+                .setSchemaName(schemaName)
+                .setQueryName(queryName)
+                .setRenamedColumns(getRenamedColumns())
+                .setLineageImportAliases(getLineageImportAliases())
+                .setInsertOption(_insertOption)
+                .setAuditBehaviorType(behaviorType)
+                .setAuditUserComment(_auditUserComment)
+                .setOptionParamsMap(getOptionParamsMap())
+                .setLookupResolutionType(getLookupResolutionType())
+                .setAllowLineageColumns(allowLineageColumns())
+                .setJobDescription(getQueryImportDescription())
+                .setJobNotificationProvider(getQueryImportJobNotificationProviderName());
+        if (WorkflowService.get() != null)
+        {
+            Map<String, Object> workflowParams = WorkflowService.get().getConfigParameters(getViewContext().getRequest());
+            importContextBuilder.setWorkflowParams(workflowParams);
         }
 
+        importContextBuilder.setTransactionDetails(transactionDetails);
+
+        return importContextBuilder;
     }
 
     public static void configureLoader(DataLoader loader, @Nullable TableInfo target, @Nullable Map<String, String> renamedColumns, boolean allowLineageColumns, @Nullable Set<String> lineageAliasNames, @Nullable Map<Params, Boolean> optionParamsMap) throws IOException

api/src/org/labkey/api/query/QueryImportPipelineJob.java

@@ -441,4 +441,10 @@ public Map<String, Object> getAdditionalJobResponseInfo()
         return _additionalJobResponseInfo;
     }
 
+    public String getImportSourceDescription()
+    {
+        FileLike primaryFile = _importContextBuilder == null ? null : _importContextBuilder.getPrimaryFile();
+        return primaryFile != null ? primaryFile.getName() : "";
+    }
+
 }

api/src/org/labkey/api/reader/FastaDataLoader.java

@@ -42,7 +42,7 @@
  */
 public class FastaDataLoader extends DataLoader
 {
-    public static FileType FILE_TYPE = new FileType(Arrays.asList(".fna", ".fasta"), ".fna");
+    public static FileType FILE_TYPE = new FileType(Arrays.asList(".fna", ".fasta", ".fa"), ".fna");
     static {
         FILE_TYPE.setExtensionsMutuallyExclusive(false);
     }