From ed0f1d689f3de36cec5d80533a059cd698482591 Mon Sep 17 00:00:00 2001 From: bbimber Date: Thu, 18 Feb 2021 13:28:53 -0800 Subject: [PATCH 1/4] Use pipeline job's logger --- .../userRequests/Pending Requests.qview.xml | 8 +++ .../pipeline/MhcMigrationPipelineJob.java | 70 +++++++++---------- 2 files changed, 42 insertions(+), 36 deletions(-) create mode 100644 mcc/resources/queries/mcc/userRequests/Pending Requests.qview.xml diff --git a/mcc/resources/queries/mcc/userRequests/Pending Requests.qview.xml b/mcc/resources/queries/mcc/userRequests/Pending Requests.qview.xml new file mode 100644 index 000000000..b849a22a8 --- /dev/null +++ b/mcc/resources/queries/mcc/userRequests/Pending Requests.qview.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/primeseq/src/org/labkey/primeseq/pipeline/MhcMigrationPipelineJob.java b/primeseq/src/org/labkey/primeseq/pipeline/MhcMigrationPipelineJob.java index dfd668352..0539a3761 100644 --- a/primeseq/src/org/labkey/primeseq/pipeline/MhcMigrationPipelineJob.java +++ b/primeseq/src/org/labkey/primeseq/pipeline/MhcMigrationPipelineJob.java @@ -68,8 +68,6 @@ public class MhcMigrationPipelineJob extends PipelineJob { - private static final Logger _log = LogManager.getLogger(MhcMigrationPipelineJob.class); - private String remoteServerFolder; private String remoteConnectionName; @@ -123,7 +121,7 @@ public ActionURL getStatusHref() @Override public String getDescription() { - return "Find Orphan Sequence Files"; + return "Migrate MHC Data"; } @Override @@ -184,7 +182,7 @@ private MhcMigrationPipelineJob getPipelineJob() private Connection getConnection() { - DataIntegrationService.RemoteConnection rc = DataIntegrationService.get().getRemoteConnection(getPipelineJob().remoteConnectionName, getPipelineJob().targetContainer, _log); + DataIntegrationService.RemoteConnection rc = DataIntegrationService.get().getRemoteConnection(getPipelineJob().remoteConnectionName, getPipelineJob().targetContainer, getJob().getLogger()); return(rc.connection); } @@ -308,7 +306,7 @@ else if ("analysis_id".equals(colName)) private void createLibraryMembers() { - _log.info("Creating library members"); + getJob().getLogger().info("Creating library members"); final UserSchema us = QueryService.get().getUserSchema(getJob().getUser(), getPipelineJob().targetContainer, "sequenceanalysis"); final TableInfo ti = us.getTable("reference_library_members"); @@ -360,14 +358,14 @@ private void createLibraryMembers() } catch (Exception e) { - _log.error(e.getMessage(), e); + getJob().getLogger().error(e.getMessage(), e); throw new RuntimeException(e); } }); } catch (Exception e) { - _log.error(e.getMessage(), e); + getJob().getLogger().error(e.getMessage(), e); throw new RuntimeException(e); } } @@ -387,14 +385,14 @@ private int getOrCreateSequence(int remoteSeqId, String name, int seqLength, Tab { if (ts.getRowCount() > 1) { - _log.info("Duplicate ref name: " + name); + getJob().getLogger().info("Duplicate ref name: " + name); } AtomicInteger localId = new AtomicInteger(-1); ts.forEachResults(rs -> { if (rs.getInt(FieldKey.fromString("seqLength")) < seqLength) { - _log.warn("length doesnt match for " + name + ", expected: " + seqLength); + getJob().getLogger().warn("length doesnt match for " + name + ", expected: " + seqLength); return; } @@ -410,7 +408,7 @@ private int getOrCreateSequence(int remoteSeqId, String name, int seqLength, Tab //TODO: Create sequence? //throw new IllegalStateException("Expected sequence to exist: " + name); - _log.error("Sequence missing: " + name); + getJob().getLogger().error("Sequence missing: " + name); return -1; } } @@ -426,7 +424,7 @@ public String getParent(String path) private void createLibraries() { - _log.info("Creating libraries"); + getJob().getLogger().info("Creating libraries"); try { final TableInfo libraryTable = QueryService.get().getUserSchema(getJob().getUser(), getPipelineJob().targetContainer, "sequenceanalysis").getTable("reference_libraries"); @@ -469,8 +467,8 @@ private void createLibraries() localJobRootFile.getParentFile().mkdirs(); } - _log.info(remoteJobRoot); - _log.info(localJobRoot.getPath()); + getJob().getLogger().info(remoteJobRoot); + getJob().getLogger().info(localJobRoot.getPath()); File remoteJobRootFile = new File(remoteJobRoot); if (remoteJobRootFile.exists()) { @@ -495,14 +493,14 @@ private void createLibraries() } catch (Exception e) { - _log.error(e.getMessage(), e); + getJob().getLogger().error(e.getMessage(), e); throw new RuntimeException(e); } } private void createOutputFiles() { - _log.info("Creating outputfiles"); + getJob().getLogger().info("Creating outputfiles"); try { final TableInfo outputTable = QueryService.get().getUserSchema(getJob().getUser(), getPipelineJob().targetContainer, "sequenceanalysis").getTable("outputfiles"); @@ -580,7 +578,7 @@ private void createOutputFiles() } else { - _log.error("output missing runid: " + remoteId); + getJob().getLogger().error("output missing runid: " + remoteId); } BatchValidationException bve = new BatchValidationException(); @@ -601,14 +599,14 @@ private void createOutputFiles() } catch (Exception e) { - _log.error(e.getMessage(), e); + getJob().getLogger().error(e.getMessage(), e); throw new RuntimeException(e); } } private void createAnalyses() { - _log.info("Creating analyses"); + getJob().getLogger().info("Creating analyses"); try { final TableInfo analysisTable = QueryService.get().getUserSchema(getJob().getUser(), getPipelineJob().targetContainer, "sequenceanalysis").getTable("sequence_analyses"); @@ -622,7 +620,7 @@ private void createAnalyses() int remoteId = Integer.parseInt(String.valueOf(rd.getValue("rowid"))); if (rd.getValue("readset") == null) { - _log.warn("analysis lacks readset, skipping: " + remoteId); + getJob().getLogger().warn("analysis lacks readset, skipping: " + remoteId); return; } @@ -675,7 +673,7 @@ private void createAnalyses() { if (rd.getValue("runid/JobId") == null) { - _log.info("skipping analysis without runid: " + remoteId); + getJob().getLogger().info("skipping analysis without runid: " + remoteId); return; } @@ -703,7 +701,7 @@ private void createAnalyses() } else { - _log.error("analysis missing runid: " + remoteId); + getJob().getLogger().error("analysis missing runid: " + remoteId); } BatchValidationException bve = new BatchValidationException(); @@ -724,14 +722,14 @@ private void createAnalyses() } catch (Exception e) { - _log.error(e.getMessage(), e); + getJob().getLogger().error(e.getMessage(), e); throw new RuntimeException(e); } } private void createReaddata() { - _log.info("Creating read data"); + getJob().getLogger().info("Creating read data"); try { final TableInfo readdataTable = QueryService.get().getUserSchema(getJob().getUser(), getPipelineJob().targetContainer, "sequenceanalysis").getTable("readdata"); @@ -802,7 +800,7 @@ private void createReaddata() } else { - _log.error("readddata missing jobid: " + remoteId); + getJob().getLogger().error("readddata missing jobid: " + remoteId); } //Create run: @@ -815,7 +813,7 @@ private void createReaddata() } else { - _log.error("readddata missing runid: " + remoteId); + getJob().getLogger().error("readddata missing runid: " + remoteId); } BatchValidationException bve = new BatchValidationException(); @@ -836,7 +834,7 @@ private void createReaddata() } catch (Exception e) { - _log.error(e.getMessage(), e); + getJob().getLogger().error(e.getMessage(), e); throw new RuntimeException(e); } } @@ -856,7 +854,7 @@ private int getOrCreateExpData(URI file, Container workbook) private void createReadsets() { - _log.info("Creating readsets"); + getJob().getLogger().info("Creating readsets"); try { final UserSchema us = QueryService.get().getUserSchema(getJob().getUser(), getPipelineJob().targetContainer, "sequenceanalysis"); @@ -920,7 +918,7 @@ private void createReadsets() } else { - _log.error("readset missing run id: " + remoteId); + getJob().getLogger().error("readset missing run id: " + remoteId); } BatchValidationException bve = new BatchValidationException(); @@ -941,7 +939,7 @@ private void createReadsets() } catch (Exception e) { - _log.error(e.getMessage(), e); + getJob().getLogger().error(e.getMessage(), e); throw new RuntimeException(e); } } @@ -981,7 +979,7 @@ else if (filepath.contains("sequenceAnalysis")) } else { - _log.error("Unexpected filepath: " + pj.getValue("FilePath")); + getJob().getLogger().error("Unexpected filepath: " + pj.getValue("FilePath")); } } @@ -1015,14 +1013,14 @@ else if (filepath.contains("sequenceAnalysis")) if (localDir.exists()) { - _log.info("Directory exists, will not re-copy: " + localDir.getPath()); + getJob().getLogger().info("Directory exists, will not re-copy: " + localDir.getPath()); return; } try { - _log.info(remoteDir.getPath()); - _log.info(localDir.getPath()); + getJob().getLogger().info(remoteDir.getPath()); + getJob().getLogger().info(localDir.getPath()); if (!localDir.getParentFile().exists()) { @@ -1035,7 +1033,7 @@ else if (filepath.contains("sequenceAnalysis")) } else { - _log.error("source folder not found: " + remoteDir.getPath()); + getJob().getLogger().error("source folder not found: " + remoteDir.getPath()); } } catch (Exception e) @@ -1050,7 +1048,7 @@ else if (filepath.contains("sequenceAnalysis")) } catch (Exception e) { - _log.error(e.getMessage(), e); + getJob().getLogger().error(e.getMessage(), e); throw new RuntimeException(e); } } @@ -1072,7 +1070,7 @@ private int createExpRun(int remoteId, Container c, String name, int localJobId) private void createWorkbooks() { - _log.info("Creating workbooks"); + getJob().getLogger().info("Creating workbooks"); try { TableInfo containers = QueryService.get().getUserSchema(getJob().getUser(), getPipelineJob().targetContainer, "core").getTable("containers"); From 654076f74d8caf8968a3a5cc6bdcda65aa976c3d Mon Sep 17 00:00:00 2001 From: bbimber Date: Thu, 18 Feb 2021 15:10:20 -0800 Subject: [PATCH 2/4] allow greater decimal precision for filtering --- .../pipeline/MhcMigrationPipelineJob.java | 109 ++++++++++++++++-- 1 file changed, 99 insertions(+), 10 deletions(-) diff --git a/primeseq/src/org/labkey/primeseq/pipeline/MhcMigrationPipelineJob.java b/primeseq/src/org/labkey/primeseq/pipeline/MhcMigrationPipelineJob.java index 0539a3761..fe6007851 100644 --- a/primeseq/src/org/labkey/primeseq/pipeline/MhcMigrationPipelineJob.java +++ b/primeseq/src/org/labkey/primeseq/pipeline/MhcMigrationPipelineJob.java @@ -1,8 +1,6 @@ package org.labkey.primeseq.pipeline; import org.apache.commons.io.FileUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; import org.labkey.api.collections.CaseInsensitiveHashMap; import org.labkey.api.data.CompareType; import org.labkey.api.data.Container; @@ -10,6 +8,7 @@ import org.labkey.api.data.DbSchema; import org.labkey.api.data.DbSchemaType; import org.labkey.api.data.DbScope; +import org.labkey.api.data.Results; import org.labkey.api.data.SimpleFilter; import org.labkey.api.data.Sort; import org.labkey.api.data.Table; @@ -58,6 +57,7 @@ import java.io.File; import java.io.IOException; import java.net.URI; +import java.sql.SQLException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -205,9 +205,11 @@ public RecordedActionSet run() throws PipelineJobException createAnalyses(); createOutputFiles(); + createAlignmentSummary(); + //TODO: //samples - //alignment_summary + //alignment_summary_junction //quality_metrics //subjects @@ -225,6 +227,75 @@ public RecordedActionSet run() throws PipelineJobException return new RecordedActionSet(); } + private void createAlignmentSummary() throws PipelineJobException + { + try + { + TableInfo alignmentSummary = DbSchema.get("sequenceanalysis", DbSchemaType.Module).getTable("alignment_summary"); + TableInfo alignmentSummaryJunction = DbSchema.get("sequenceanalysis", DbSchemaType.Module).getTable("alignment_summary_junction"); + + SelectRowsCommand sr = new SelectRowsCommand("sequenceanalysis", "alignment_summary"); + sr.setColumns(Arrays.asList("rowid", "analysis_id", "file_id", "total", "total_forward", "total_reverse", "valid_pairs", "workbook/workbookId")); + + SelectRowsResponse srr = sr.execute(getConnection(), getPipelineJob().remoteServerFolder); + + Map alignmentSummaryMap = new HashMap<>(); + srr.getRowset().forEach(rs -> { + CaseInsensitiveHashMap map = new CaseInsensitiveHashMap<>(); + Integer localId = analysisMap.get(rs.getValue("analysis_id")); + if (localId == null) + { + throw new RuntimeException("Unable to find analysis: " + rs.getValue("analysis_id")); + } + map.put("analysis_id", localId); + map.put("file_id", analysisToFileMap.get(localId)); + + map.put("total", rs.getValue("total")); + map.put("total_forward", rs.getValue("total_forward")); + map.put("total_reverse", rs.getValue("total_reverse")); + map.put("valid_pairs", rs.getValue("valid_pairs")); + + Container c = workbookMap.get((int)rs.getValue("workbook/workbookId")); + map.put("container", c.getId()); + + map = Table.insert(getJob().getUser(), alignmentSummary, map); + alignmentSummaryMap.put((int)rs.getValue("rowid"), (int)map.get("rowid")); + }); + + SelectRowsCommand sr2 = new SelectRowsCommand("sequenceanalysis", "alignment_summary_junction"); + sr2.setColumns(Arrays.asList("analysis_id", "alignment_id", "ref_nt_id", "analysis_id/workbook/workbookId")); + + SelectRowsResponse srr2 = sr2.execute(getConnection(), getPipelineJob().remoteServerFolder); + srr2.getRowset().forEach(rs -> { + CaseInsensitiveHashMap map = new CaseInsensitiveHashMap<>(); + Integer localId = analysisMap.get(rs.getValue("analysis_id")); + if (localId == null) + { + throw new RuntimeException("Unable to find analysis: " + rs.getValue("analysis_id")); + } + map.put("analysis_id", localId); + + Integer localNT = sequenceMap.get(rs.getValue("ref_nt_id")); + if (localNT == null) + { + throw new RuntimeException("Unable to find ref_nt_id: " + rs.getValue("ref_nt_id")); + } + map.put("ref_nt_id", localNT); + + map.put("alignment_id", alignmentSummaryMap.get("alignment_id")); + + Container c = workbookMap.get((int)rs.getValue("analysis_id/workbook/workbookId")); + map.put("container", c.getId()); + + map = Table.insert(getJob().getUser(), alignmentSummaryJunction, map); + }); + } + catch (CommandException | IOException e) + { + throw new PipelineJobException(e); + } + } + private void replaceEntireTable(String schema, String query, List columns, String workbookColName, boolean truncateExisting) throws Exception { SelectRowsCommand sr = new SelectRowsCommand(schema, query); @@ -298,6 +369,7 @@ else if ("analysis_id".equals(colName)) private final Map readsetMap = new HashMap<>(); private final Map readdataMap = new HashMap<>(); private final Map analysisMap = new HashMap<>(); + private final Map analysisToFileMap = new HashMap<>(); //local analysis_id -> alignment file private final Map libraryMap = new HashMap<>(); private final Map outputFileMap = new HashMap<>(); private final Map sequenceMap = new HashMap<>(); @@ -392,7 +464,7 @@ private int getOrCreateSequence(int remoteSeqId, String name, int seqLength, Tab ts.forEachResults(rs -> { if (rs.getInt(FieldKey.fromString("seqLength")) < seqLength) { - getJob().getLogger().warn("length doesnt match for " + name + ", expected: " + seqLength); + getJob().getLogger().warn("length doesnt match for " + name + ", expected: " + seqLength + ", was: " + rs.getInt(FieldKey.fromString("seqLength"))); return; } @@ -526,11 +598,19 @@ private void createOutputFiles() throw new IllegalArgumentException("Unable to find genome for remote id: " + remoteLibrary); } - int remoteAnalysis = Integer.parseInt(String.valueOf(rd.getValue("analysis_id"))); - Integer localAnalysis = analysisMap.get(remoteAnalysis); - if (localAnalysis == null) + Integer localAnalysis; + if (rd.getValue("analysis_id") != null) { - throw new IllegalArgumentException("Unable to find analysis for remote id: " + remoteAnalysis); + int remoteAnalysis = Integer.parseInt(String.valueOf(rd.getValue("analysis_id"))); + localAnalysis = analysisMap.get(remoteAnalysis); + if (localAnalysis == null) + { + throw new IllegalArgumentException("Unable to find analysis for remote id: " + remoteAnalysis); + } + } + else + { + localAnalysis = null; } Readset rs = SequenceAnalysisService.get().getReadset(localReadset, getJob().getUser()); @@ -649,10 +729,19 @@ private void createAnalyses() filter.addCondition(FieldKey.fromString("runid/JobId/Description"), rd.getValue("runid/JobId/Description")); filter.addCondition(FieldKey.fromString("container"), targetWorkbook.getId(), CompareType.EQUAL); - TableSelector tsAnalyses = new TableSelector(analysisTable, PageFlowUtil.set("rowid"), filter, null); + TableSelector tsAnalyses = new TableSelector(analysisTable, PageFlowUtil.set("rowid", "alignmentfile"), filter, null); if (tsAnalyses.exists()) { - analysisMap.put(remoteId, tsAnalyses.getObject(Integer.class)); + Results results = tsAnalyses.getResults(); + try + { + analysisMap.put(remoteId, results.getInt("rowid")); + analysisToFileMap.put(results.getInt("rowid"), results.getInt("alignmentfile")); + } + catch (SQLException e) + { + throw new RuntimeException(e); + } } else { From b83db1cfa9d1bddd0af7261f8571b5ec407164e5 Mon Sep 17 00:00:00 2001 From: bbimber Date: Fri, 19 Feb 2021 12:20:55 -0800 Subject: [PATCH 3/4] Improve reporting for GenomicsDBImport --- .../labkey/primeseq/pipeline/MhcMigrationPipelineJob.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/primeseq/src/org/labkey/primeseq/pipeline/MhcMigrationPipelineJob.java b/primeseq/src/org/labkey/primeseq/pipeline/MhcMigrationPipelineJob.java index fe6007851..c979ea32e 100644 --- a/primeseq/src/org/labkey/primeseq/pipeline/MhcMigrationPipelineJob.java +++ b/primeseq/src/org/labkey/primeseq/pipeline/MhcMigrationPipelineJob.java @@ -640,6 +640,11 @@ private void createOutputFiles() try { + if (rd.getValue("runid/JobId") == null) + { + throw new PipelineJobException("Output missing runId"); + } + int remoteJobId = Integer.parseInt(String.valueOf(rd.getValue("runid/JobId"))); int jobId = getOrCreateJob(remoteJobId, targetWorkbook); PipelineStatusFile sf = PipelineService.get().getStatusFile(jobId); From cc57cc656a561ea6df61db063eb53b2714093b51 Mon Sep 17 00:00:00 2001 From: bbimber Date: Fri, 19 Feb 2021 12:55:33 -0800 Subject: [PATCH 4/4] Update Seurat/subset step --- .../labkey/primeseq/pipeline/MhcMigrationPipelineJob.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/primeseq/src/org/labkey/primeseq/pipeline/MhcMigrationPipelineJob.java b/primeseq/src/org/labkey/primeseq/pipeline/MhcMigrationPipelineJob.java index c979ea32e..b535c27d5 100644 --- a/primeseq/src/org/labkey/primeseq/pipeline/MhcMigrationPipelineJob.java +++ b/primeseq/src/org/labkey/primeseq/pipeline/MhcMigrationPipelineJob.java @@ -397,6 +397,13 @@ private void createLibraryMembers() int remoteSeqId = Integer.parseInt(String.valueOf(rd.getValue("ref_nt_id"))); String name = String.valueOf(rd.getValue("ref_nt_id/name")); + + //Skip all pigtail MHC. + if (name.startsWith("Mane")) + { + return; + } + int localSeqId = getOrCreateSequence(remoteSeqId, name, seqLength, refNtTable); int remoteLibraryId = Integer.parseInt(String.valueOf(rd.getValue("library_id")));