From 39725fbc9c6eccb16c660442f0d2484d32cc83f1 Mon Sep 17 00:00:00 2001 From: bbimber Date: Mon, 25 Jan 2021 12:22:29 -0800 Subject: [PATCH 01/29] Fix comment --- mcc/resources/etls/wnprc.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mcc/resources/etls/wnprc.xml b/mcc/resources/etls/wnprc.xml index e5f1ee6d1..de5850d40 100644 --- a/mcc/resources/etls/wnprc.xml +++ b/mcc/resources/etls/wnprc.xml @@ -13,7 +13,7 @@ - + From ed4935d3e9a8d21ed75a913ce5d2b4e3481b9d1e Mon Sep 17 00:00:00 2001 From: bbimber Date: Mon, 25 Jan 2021 12:30:55 -0800 Subject: [PATCH 02/29] Fix markdown syntax --- mcc/src/org/labkey/mcc/MccModule.java | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/mcc/src/org/labkey/mcc/MccModule.java b/mcc/src/org/labkey/mcc/MccModule.java index 03182cb68..96e462c0f 100644 --- a/mcc/src/org/labkey/mcc/MccModule.java +++ b/mcc/src/org/labkey/mcc/MccModule.java @@ -86,19 +86,5 @@ public Set getSchemaNames() private void registerEHRResources() { EHRService.get().registerModule(this); - //EHRService.get().registerTableCustomizer(this, ONPRC_EHRCustomizer.class); - - //Resource r = getModuleResource("/scripts/mcc/mcc_triggers.js"); - //assert r != null; - //EHRService.get().registerTriggerScript(this, r); - - //EHRService.get().registerClientDependency(ClientDependency.supplierFromPath("Ext4"), this); - //EHRService.get().registerClientDependency(ClientDependency.supplierFromPath("onprc_ehr/panel/BloodSummaryPanel.js"), this); - - //EHRService.get().registerReportLink(EHRService.REPORT_LINK_TYPE.housing, "List Single Housed Animals", this, DetailsURL.fromString("/query/executeQuery.view?schemaName=study&query.queryName=demographicsPaired&query.viewName=Single Housed"), "Commonly Used Queries"); - //EHRService.get().registerReportLink(EHRService.REPORT_LINK_TYPE.moreReports, "Clinical Snapshot Printable Report", this, DetailsURL.fromString("/onprc_ehr/SnapshotPrintableReport.view"), "Clinical"); - - //EHRService.get().registerDemographicsProvider(new ActiveCasesDemographicsProvider(this)); - //EHRService.get().registerHistoryDataSource(new DefaultSustainedReleaseDatasource(this)); } } \ No newline at end of file From 5cc59ff833eb4c357b07eb8733ffea6f2bce1734 Mon Sep 17 00:00:00 2001 From: bbimber Date: Tue, 26 Jan 2021 12:42:10 -0800 Subject: [PATCH 03/29] Bugfix TCR import --- .../CellRangerVDJCellHashingHandler.java | 2 +- .../tcrdb/pipeline/CellRangerVDJUtils.java | 25 +++++++++---------- 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java index 71d1c1820..d2443b30f 100644 --- a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java +++ b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java @@ -160,7 +160,7 @@ public void complete(PipelineJob job, List inputFiles, List< for (SequenceOutputFile so : inputFiles) { AnalysisModel model = support.getCachedAnalysis(so.getAnalysis_id()); - new CellRangerVDJUtils(job.getLogger()).importAssayData(job, model, job.getLogFile().getParentFile(), assayId, null, deleteExistingData); + new CellRangerVDJUtils(job.getLogger()).importAssayData(job, model, so.getFile(), job.getLogFile().getParentFile(), assayId, null, deleteExistingData); } } } diff --git a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJUtils.java b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJUtils.java index b6b9640c3..e09c0e871 100644 --- a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJUtils.java +++ b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJUtils.java @@ -63,8 +63,10 @@ public CellRangerVDJUtils(Logger log) _log = log; } - public void importAssayData(PipelineJob job, AnalysisModel model, File outDir, Integer assayId, @Nullable Integer runId, boolean deleteExisting) throws PipelineJobException + public void importAssayData(PipelineJob job, AnalysisModel model, File vLoupeFile, File outDir, Integer assayId, @Nullable Integer runId, boolean deleteExisting) throws PipelineJobException { + File cellRangerOutDir = vLoupeFile.getParentFile(); + if (assayId == null) { _log.info("No assay selected, will not import"); @@ -77,32 +79,29 @@ public void importAssayData(PipelineJob job, AnalysisModel model, File outDir, I throw new PipelineJobException("Unable to find protocol: " + assayId); } - File allCsv = getPerCellCsv(outDir); + File allCsv = getPerCellCsv(cellRangerOutDir); if (!allCsv.exists()) { _log.warn("unable to find consensus contigs: " + allCsv .getPath()); return; } - File consensusCsv = new File(outDir, "consensus_annotations.csv"); + File consensusCsv = new File(cellRangerOutDir, "consensus_annotations.csv"); if (!consensusCsv .exists()) { - _log.warn("unable to find consensus contigs: " + consensusCsv .getPath()); - return; + throw new PipelineJobException("unable to find consensus contigs: " + consensusCsv .getPath()); } - File consensusFasta = new File(outDir, "consensus.fasta"); + File consensusFasta = new File(cellRangerOutDir, "consensus.fasta"); if (!consensusFasta.exists()) { - _log.warn("unable to find FASTA: " + consensusFasta.getPath()); - return; + throw new PipelineJobException("unable to find FASTA: " + consensusFasta.getPath()); } - File allFasta = new File(outDir, "all_contig.fasta"); + File allFasta = new File(cellRangerOutDir, "all_contig.fasta"); if (!allFasta.exists()) { - _log.warn("unable to find FASTA: " + allFasta.getPath()); - return; + throw new PipelineJobException("unable to find FASTA: " + allFasta.getPath()); } _log.info("loading results into assay: " + assayId); @@ -647,8 +646,8 @@ public static void deleteExistingData(AssayProvider ap, ExpProtocol protocol, Co } } - public static File getPerCellCsv(File outDir) + public static File getPerCellCsv(File cellRangerOutDir) { - return new File(outDir, "all_contig_annotations.csv"); + return new File(cellRangerOutDir, "all_contig_annotations.csv"); } } From 60f7ce1a2fd91686fd8839a2410305008628fdbf Mon Sep 17 00:00:00 2001 From: bbimber Date: Tue, 26 Jan 2021 14:05:06 -0800 Subject: [PATCH 04/29] Bugfix TCR hashing --- .../tcrdb/pipeline/CellRangerVDJCellHashingHandler.java | 1 + tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJUtils.java | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java index d2443b30f..a54aea590 100644 --- a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java +++ b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java @@ -221,6 +221,7 @@ private void processVloupeFile(JobContext ctx, File perCellTsv, Readset rs, Reco } + action.addOutput(cellToHto, CellRangerVDJUtils.TCR_HASHING_CALLS, false); ctx.getFileManager().addStepOutputs(action, output); } else if (htosPerReadset.size() == 1) diff --git a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJUtils.java b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJUtils.java index e09c0e871..425d940b1 100644 --- a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJUtils.java +++ b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJUtils.java @@ -56,6 +56,8 @@ public class CellRangerVDJUtils { + public static final String TCR_HASHING_CALLS = "Cell Hashing TCR Calls"; + private Logger _log; public CellRangerVDJUtils(Logger log) @@ -459,7 +461,7 @@ private AssayModel createForRow(String[] line, String sequenceContigName, Intege private File getCellToHtoFile(ExpRun run) throws PipelineJobException { - List datas = run.getInputDatas(CellHashingService.HASHING_CALLS, ExpProtocol.ApplicationType.ExperimentRunOutput); + List datas = run.getInputDatas(TCR_HASHING_CALLS, ExpProtocol.ApplicationType.ExperimentRunOutput); if (datas.isEmpty()) { throw new PipelineJobException("Unable to find hashing calls output"); From 4f0fefbfa7b7e77323dc7ff6e855d609b7934c34 Mon Sep 17 00:00:00 2001 From: bbimber Date: Tue, 26 Jan 2021 18:51:56 -0800 Subject: [PATCH 05/29] Allow CDR3s from rows lacking C-Gene --- .../tcrdb/pipeline/CellRangerVDJUtils.java | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJUtils.java b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJUtils.java index 425d940b1..b52974def 100644 --- a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJUtils.java +++ b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJUtils.java @@ -292,10 +292,15 @@ else if ("Negative".equals(hto)) continue; } - if ("None".equals(line[9])) + String cGene = removeNone(line[9]); + if (cGene == null) { - noCGene++; - continue; + // Only discard these if chain type doesnt match between JGene and VGene. + if (!line[8].substring(0, 3).equals(line[6].substring(0,3))) + { + noCGene++; + continue; + } } if ("False".equals(line[10])) @@ -343,9 +348,9 @@ else if (discordantBarcodes.contains(barcode)) //NOTE: chimeras with a TRDV / TRAJ / TRAC are relatively common. categorize as TRA for reporting ease String locus = line[5]; - if (locus.equals("Multi") && removeNone(line[9]) != null && removeNone(line[8]) != null && removeNone(line[6]) != null) + if (locus.equals("Multi") && cGene != null && removeNone(line[8]) != null && removeNone(line[6]) != null) { - if (removeNone(line[9]).contains("TRAC") && removeNone(line[8]).contains("TRAJ") && removeNone(line[6]).contains("TRDV")) + if (cGene.contains("TRAC") && removeNone(line[8]).contains("TRAJ") && removeNone(line[6]).contains("TRDV")) { locus = "TRA"; multiChainConverted++; @@ -353,7 +358,7 @@ else if (discordantBarcodes.contains(barcode)) } // Aggregate by: cDNA_ID, cdr3, chain, raw_clonotype_id, sequenceContigName, vHit, dHit, jHit, cHit, cdr3_nt - String key = StringUtils.join(new String[]{cDNA.toString(), line[12], locus, clonotypeId, sequenceContigName, removeNone(line[6]), removeNone(line[7]), removeNone(line[8]), removeNone(line[9]), removeNone(line[13])}, "<>"); + String key = StringUtils.join(new String[]{cDNA.toString(), line[12], locus, clonotypeId, sequenceContigName, removeNone(line[6]), removeNone(line[7]), removeNone(line[8]), cGene, removeNone(line[13])}, "<>"); AssayModel am; if (!rows.containsKey(key)) { From 79ac3e623dc70fe630abb62582cf37e69e1e6539 Mon Sep 17 00:00:00 2001 From: bbimber Date: Wed, 27 Jan 2021 17:08:19 -0800 Subject: [PATCH 06/29] Update field name --- .../assay/TCRdb/queries/Data/cDNA Info.qview.xml | 6 +++--- .../singlecell/cdna_libraries/Assay Info.qview.xml | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/tcrdb/resources/assay/TCRdb/queries/Data/cDNA Info.qview.xml b/tcrdb/resources/assay/TCRdb/queries/Data/cDNA Info.qview.xml index d8518ac12..2b3104ee9 100644 --- a/tcrdb/resources/assay/TCRdb/queries/Data/cDNA Info.qview.xml +++ b/tcrdb/resources/assay/TCRdb/queries/Data/cDNA Info.qview.xml @@ -2,9 +2,9 @@ - - - + + + diff --git a/tcrdb/resources/queries/singlecell/cdna_libraries/Assay Info.qview.xml b/tcrdb/resources/queries/singlecell/cdna_libraries/Assay Info.qview.xml index 8a69ed8a5..109d35ad5 100644 --- a/tcrdb/resources/queries/singlecell/cdna_libraries/Assay Info.qview.xml +++ b/tcrdb/resources/queries/singlecell/cdna_libraries/Assay Info.qview.xml @@ -2,10 +2,10 @@ - - - - + + + + From 539a1965aa892d9dbc3fdbc7956bb9796954ac4c Mon Sep 17 00:00:00 2001 From: bbimber Date: Thu, 28 Jan 2021 13:56:55 -0800 Subject: [PATCH 07/29] Checkpoint for MHC migration code --- .../src/org/labkey/primeseq/MhcMigration.java | 1009 +++++++++++++++++ .../labkey/primeseq/PrimeseqController.java | 43 + 2 files changed, 1052 insertions(+) create mode 100644 primeseq/src/org/labkey/primeseq/MhcMigration.java diff --git a/primeseq/src/org/labkey/primeseq/MhcMigration.java b/primeseq/src/org/labkey/primeseq/MhcMigration.java new file mode 100644 index 000000000..76246381b --- /dev/null +++ b/primeseq/src/org/labkey/primeseq/MhcMigration.java @@ -0,0 +1,1009 @@ +package org.labkey.primeseq; + +import org.apache.commons.io.FileUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.labkey.api.collections.CaseInsensitiveHashMap; +import org.labkey.api.data.CompareType; +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerManager; +import org.labkey.api.data.DbSchema; +import org.labkey.api.data.DbSchemaType; +import org.labkey.api.data.DbScope; +import org.labkey.api.data.SimpleFilter; +import org.labkey.api.data.Sort; +import org.labkey.api.data.Table; +import org.labkey.api.data.TableInfo; +import org.labkey.api.data.TableSelector; +import org.labkey.api.data.WorkbookContainerType; +import org.labkey.api.di.DataIntegrationService; +import org.labkey.api.exp.api.DataType; +import org.labkey.api.exp.api.ExpData; +import org.labkey.api.exp.api.ExpRun; +import org.labkey.api.exp.api.ExperimentService; +import org.labkey.api.pipeline.PipelineService; +import org.labkey.api.pipeline.PipelineStatusFile; +import org.labkey.api.pipeline.RecordedActionSet; +import org.labkey.api.query.BatchValidationException; +import org.labkey.api.query.FieldKey; +import org.labkey.api.query.QueryService; +import org.labkey.api.query.UserSchema; +import org.labkey.api.security.User; +import org.labkey.api.sequenceanalysis.SequenceAnalysisService; +import org.labkey.api.sequenceanalysis.model.Readset; +import org.labkey.api.util.PageFlowUtil; +import org.labkey.remoteapi.CommandException; +import org.labkey.remoteapi.Connection; +import org.labkey.remoteapi.query.Filter; +import org.labkey.remoteapi.query.SelectRowsCommand; +import org.labkey.remoteapi.query.SelectRowsResponse; + +import java.io.File; +import java.io.IOException; +import java.net.URI; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; + +public class MhcMigration +{ + private static final Logger _log = LogManager.getLogger(MhcMigration.class); + + private final String remoteServerFolder; + private final String remoteConnectionName; + + private final User user; + private final Container target; + + public MhcMigration(Container c, User u, String remoteConnectionName, String remoteServerFolder) + { + this.target = c; + this.user = u; + this.remoteConnectionName = remoteConnectionName; + this.remoteServerFolder = remoteServerFolder; + } + + private Connection getConnection() + { + DataIntegrationService.RemoteConnection rc = DataIntegrationService.get().getRemoteConnection(remoteConnectionName, target, _log); + + return(rc.connection); + } + + public void doWork() + { + try (DbScope.Transaction transaction = DbScope.getLabKeyScope().ensureTransaction()) + { + createWorkbooks(); + + createLibraries(); + createLibraryMembers(); + + createReadsets(); + transaction.commitAndKeepConnection(); + + createReaddata(); + + createAnalyses(); + createOutputFiles(); + + //TODO: + //samples + //alignment_summary + //alignment_summary_junction + //quality_metrics + //subjects + //WaNPRC + + //sequenceanalysis.haplotypes + //sequenceanalysis.haplotype_types + //sequenceanalysis.haplotype_sequences + + //Create assay runs, including data and haplotypes + + transaction.commit(); + } + } + + private void replaceEntireTable(String schema, String query, List columns, String workbookColName, boolean truncateExisting) throws Exception + { + SelectRowsCommand sr = new SelectRowsCommand(schema, query); + sr.setColumns(columns); + SelectRowsResponse srr = sr.execute(getConnection(), remoteServerFolder); + + List> toInsert = new ArrayList<>(); + srr.getRowset().forEach(r -> { + Map row = new CaseInsensitiveHashMap<>(); + srr.getColumnModel().forEach(col -> { + String colName = (String)col.get("Name"); + Object val = r.getValue(colName); + if ("readset".equals(colName) || "readsetid".equals(colName)) + { + if (!readsetMap.containsKey((int)val)) + { + throw new IllegalStateException("Unable to find readset: " + val); + } + + val = readsetMap.get((int)val); + } + else if ("library_id".equals(colName)) + { + if (!libraryMap.containsKey((int)val)) + { + throw new IllegalStateException("Unable to find library: " + val); + } + + val = libraryMap.get((int)val); + + } + else if ("ref_nt_id".equals(colName)) + { + if (!sequenceMap.containsKey((int)val)) + { + throw new IllegalStateException("Unable to find sequence: " + val); + } + + val = sequenceMap.get((int)val); + } + else if ("analysis_id".equals(colName)) + { + if (!analysisMap.containsKey((int)val)) + { + throw new IllegalStateException("Unable to find analysis: " + val); + } + + val = analysisMap.get((int)val); + } + + row.put(colName, val); + }); + + if (workbookColName != null) + { + Object workbookId = r.getValue(workbookColName); + if (workbookId != null) + { + row.put("container", workbookMap.get(Integer.parseInt(String.valueOf(workbookId))).getId()); + } + } + + toInsert.add(row); + }); + + + + } + + //All of these map remote Id to local Id + private final Map workbookMap = new HashMap<>(); + private final Map readsetMap = new HashMap<>(); + private final Map readdataMap = new HashMap<>(); + private final Map analysisMap = new HashMap<>(); + private final Map libraryMap = new HashMap<>(); + private final Map outputFileMap = new HashMap<>(); + private final Map sequenceMap = new HashMap<>(); + private final Map runIdMap = new HashMap<>(); + private final Map jobIdMap = new HashMap<>(); + + private void createLibraryMembers() + { + _log.info("Creating library members"); + + final UserSchema us = QueryService.get().getUserSchema(user, target, "sequenceanalysis"); + final TableInfo ti = us.getTable("reference_library_members"); + final TableInfo refNtTable = us.getTable("ref_nt_sequences"); + + try + { + SelectRowsCommand sr = new SelectRowsCommand("sequenceanalysis", "reference_library_members"); + sr.setColumns(Arrays.asList("rowid", "library_id", "ref_nt_id", "ref_nt_id/name", "ref_nt_id/seqLength", "workbook/workbookId")); + + SelectRowsResponse srr = sr.execute(getConnection(), remoteServerFolder); + + srr.getRowset().forEach(rd -> { + int remoteId = Integer.parseInt(String.valueOf(rd.getValue("rowid"))); + int seqLength = Integer.parseInt(String.valueOf(rd.getValue("ref_nt_id/seqLength"))); + + int remoteSeqId = Integer.parseInt(String.valueOf(rd.getValue("ref_nt_id"))); + String name = String.valueOf(rd.getValue("ref_nt_id/name")); + int localSeqId = getOrCreateSequence(remoteSeqId, name, seqLength, refNtTable); + + int remoteLibraryId = Integer.parseInt(String.valueOf(rd.getValue("library_id"))); + Integer localLibraryId = libraryMap.get(remoteLibraryId); + if (localLibraryId == null) + { + throw new IllegalStateException("Unable to find library id: " + remoteLibraryId); + } + + SimpleFilter filter = new SimpleFilter(FieldKey.fromString("library_id"), localLibraryId); + filter.addCondition(FieldKey.fromString("ref_nt_id"), localSeqId); + + if (new TableSelector(ti, PageFlowUtil.set("rowid"), filter, null).exists()) + { + //Already exists: + return; + } + + Map toCreate = new CaseInsensitiveHashMap<>(); + toCreate.put("library_id", localLibraryId); + toCreate.put("ref_nt_id", localSeqId); + + try + { + BatchValidationException bve = new BatchValidationException(); + List> created = ti.getUpdateService().insertRows(user, target, Arrays.asList(toCreate), bve, null, null); + if (bve.hasErrors()) + { + throw new RuntimeException(bve); + } + } + catch (Exception e) + { + _log.error(e.getMessage(), e); + throw new RuntimeException(e); + } + }); + } + catch (Exception e) + { + _log.error(e.getMessage(), e); + throw new RuntimeException(e); + } + } + + private int getOrCreateSequence(int remoteSeqId, String name, int seqLength, TableInfo refNtTable) + { + if (sequenceMap.containsKey(remoteSeqId)) + { + return sequenceMap.get(remoteSeqId); + } + else + { + SimpleFilter filter = new SimpleFilter(FieldKey.fromString("name"), name); + filter.addCondition(FieldKey.fromString("datedisabled"), null, CompareType.ISBLANK); + TableSelector ts = new TableSelector(refNtTable, PageFlowUtil.set("rowid", "seqLength"), filter, new Sort("rowid")); + if (ts.exists()) + { + if (ts.getRowCount() > 1) + { + _log.info("Duplicate ref name: " + name); + } + + AtomicInteger localId = new AtomicInteger(-1); + ts.forEachResults(rs -> { + if (rs.getInt(FieldKey.fromString("seqLength")) < seqLength) + { + _log.warn("length doesnt match for " + name + ", expected: " + seqLength); + return; + } + + localId.set(rs.getInt(FieldKey.fromString("rowid"))); + }); + + if (localId.get() != -1) + { + sequenceMap.put(remoteSeqId, localId.get()); + return localId.get(); + } + } + + //TODO: Create sequence? + //throw new IllegalStateException("Expected sequence to exist: " + name); + _log.error("Sequence missing: " + name); + return -1; + } + } + + public String getParent(String path) { + final char separatorChar = '/'; + + int index = path.lastIndexOf(separatorChar); + + return path.substring(0, index); + } + + private void createLibraries() + { + _log.info("Creating libraries"); + try + { + final TableInfo libraryTable = QueryService.get().getUserSchema(user, target, "sequenceanalysis").getTable("reference_libraries"); + + SelectRowsCommand sr = new SelectRowsCommand("sequenceanalysis", "reference_libraries"); + sr.setColumns(Arrays.asList("rowid", "name", "description", "fasta_file", "datedisabled", "assemblyId", "fasta_file/DataFileUrl", "workbook/workbookId")); + + SelectRowsResponse srr = sr.execute(getConnection(), remoteServerFolder); + + srr.getRowset().forEach(rd -> { + int remoteId = Integer.parseInt(String.valueOf(rd.getValue("rowid"))); + + Integer remoteWorkbook = rd.getValue("workbook/workbookId") == null ? null : Integer.parseInt(String.valueOf(rd.getValue("workbook/workbookId"))); + Container targetContainer = remoteWorkbook == null ? target : workbookMap.get(remoteWorkbook); + + SimpleFilter filter = new SimpleFilter(FieldKey.fromString("name"), rd.getValue("name")); + TableSelector ts = new TableSelector(libraryTable, PageFlowUtil.set("rowid"), filter, null); + if (ts.exists()) + { + libraryMap.put(remoteId, ts.getObject(Integer.class)); + } + else + { + Map toCreate = new CaseInsensitiveHashMap<>(); + toCreate.put("name", rd.getValue("name")); + toCreate.put("description", rd.getValue("description")); + toCreate.put("datedisabled", rd.getValue("datedisabled")); + toCreate.put("assemblyId", rd.getValue("assemblyId")); + try + { + String remoteJobRoot = getParent(URI.create(String.valueOf(rd.getValue("fasta_file/DatafileUrl"))).getPath()); + URI localJobRoot = PipelineService.get().getPipelineRootSetting(targetContainer).getRootPath().toURI(); + URI localFasta = translateURI(String.valueOf(rd.getValue("fasta_file/DatafileUrl")), remoteJobRoot, localJobRoot.getPath()); + toCreate.put("fasta_file", getOrCreateExpData(localFasta, targetContainer)); + + //Ensure parent folder exists: + File localJobRootFile = new File(localFasta).getParentFile(); + if (!localJobRootFile.getParentFile().exists()) + { + localJobRootFile.getParentFile().mkdirs(); + } + + _log.info(remoteJobRoot); + _log.info(localJobRoot.getPath()); + File remoteJobRootFile = new File(remoteJobRoot); + if (remoteJobRootFile.exists()) + { + FileUtils.copyDirectory(remoteJobRootFile, localJobRootFile); + } + + BatchValidationException bve = new BatchValidationException(); + List> created = libraryTable.getUpdateService().insertRows(user, target, Arrays.asList(toCreate), bve, null, null); + if (bve.hasErrors()) + { + throw new RuntimeException(bve); + } + + libraryMap.put(remoteId, Integer.parseInt(String.valueOf(created.get(0).get("rowid")))); + } + catch (Exception e) + { + throw new RuntimeException(e); + } + } + }); + } + catch (Exception e) + { + _log.error(e.getMessage(), e); + throw new RuntimeException(e); + } + } + + private void createOutputFiles() + { + _log.info("Creating outputfiles"); + try + { + final TableInfo outputTable = QueryService.get().getUserSchema(user, target, "sequenceanalysis").getTable("outputfiles"); + + SelectRowsCommand sr = new SelectRowsCommand("sequenceanalysis", "outputfiles"); + sr.setColumns(Arrays.asList("rowid", "name", "description", "dataid", "library_id", "readset", "analysis_id", "category", "sra_accession", "dataid/DataFileUrl", "runid/jobid", "runid/Name", "workbook/workbookId", "runid/JobId", "runid/Name", "runid/JobId/FilePath")); + + SelectRowsResponse srr = sr.execute(getConnection(), remoteServerFolder); + + srr.getRowset().forEach(rd -> { + int remoteId = Integer.parseInt(String.valueOf(rd.getValue("rowid"))); + int remoteReadset = Integer.parseInt(String.valueOf(rd.getValue("readset"))); + Integer localReadset = readsetMap.get(remoteReadset); + if (localReadset == null) + { + throw new IllegalArgumentException("Unable to find readset for remote id: " + remoteReadset); + } + + int remoteLibrary = Integer.parseInt(String.valueOf(rd.getValue("library_id"))); + Integer localLibrary = libraryMap.get(remoteLibrary); + if (localLibrary == null) + { + throw new IllegalArgumentException("Unable to find genome for remote id: " + remoteLibrary); + } + + int remoteAnalysis = Integer.parseInt(String.valueOf(rd.getValue("analysis_id"))); + Integer localAnalysis = analysisMap.get(remoteAnalysis); + if (localAnalysis == null) + { + throw new IllegalArgumentException("Unable to find analysis for remote id: " + remoteAnalysis); + } + + Readset rs = SequenceAnalysisService.get().getReadset(localReadset, user); + Container targetWorkbook = ContainerManager.getForId(rs.getContainer()); + + SimpleFilter filter = new SimpleFilter(FieldKey.fromString("readset"), rs.getRowId()); + filter.addCondition(FieldKey.fromString("name"), rd.getValue("name")); + filter.addCondition(FieldKey.fromString("category"), rd.getValue("category")); + filter.addCondition(FieldKey.fromString("analysis_id"), localAnalysis); + filter.addCondition(FieldKey.fromString("container"), targetWorkbook.getId(), CompareType.EQUAL); + + TableSelector tsOutputFiles = new TableSelector(outputTable, PageFlowUtil.set("rowid"), filter, null); + if (tsOutputFiles.exists()) + { + outputFileMap.put(remoteId, tsOutputFiles.getObject(Integer.class)); + } + else + { + Map toCreate = new CaseInsensitiveHashMap<>(); + toCreate.put("readset", rs.getRowId()); + toCreate.put("analysis_id", localAnalysis); + toCreate.put("description", rd.getValue("description")); + toCreate.put("sra_accession", rd.getValue("sra_accession")); + toCreate.put("library_id", localLibrary); + toCreate.put("name", rd.getValue("name")); + toCreate.put("category", rd.getValue("category")); + + try + { + int remoteJobId = Integer.parseInt(String.valueOf(rd.getValue("runid/JobId"))); + int jobId = getOrCreateJob(remoteJobId, targetWorkbook); + PipelineStatusFile sf = PipelineService.get().getStatusFile(jobId); + + String localJobRoot = getParent(sf.getFilePath()); + String remoteJobRoot = getParent(URI.create(String.valueOf(rd.getValue("runid/JobId/FilePath")).replaceAll(" ", "_")).getPath()); + + URI newFileAlignment = translateURI(String.valueOf(rd.getValue("dataid/DatafileUrl")), remoteJobRoot, localJobRoot); + toCreate.put("dataid", getOrCreateExpData(newFileAlignment, targetWorkbook)); + + //Create run: + if (rd.getValue("runid") != null && rd.getValue("runid/JobId") != null) + { + int runId = createExpRun(Integer.parseInt(String.valueOf(rd.getValue("runid"))), targetWorkbook, String.valueOf(rd.getValue("runid/Name")), jobId); + toCreate.put("runid", runId); + } + else + { + _log.error("output missing runid: " + remoteId); + } + + BatchValidationException bve = new BatchValidationException(); + List> created = outputTable.getUpdateService().insertRows(user, target, Arrays.asList(toCreate), bve, null, null); + if (bve.hasErrors()) + { + throw new RuntimeException(bve); + } + + outputFileMap.put(remoteId, Integer.parseInt(String.valueOf(created.get(0).get("rowid")))); + } + catch (Exception e) + { + throw new RuntimeException(e); + } + } + }); + } + catch (Exception e) + { + _log.error(e.getMessage(), e); + throw new RuntimeException(e); + } + } + + private void createAnalyses() + { + _log.info("Creating analyses"); + try + { + final TableInfo analysisTable = QueryService.get().getUserSchema(user, target, "sequenceanalysis").getTable("sequence_analyses"); + + SelectRowsCommand sr = new SelectRowsCommand("sequenceanalysis", "sequence_analyses"); + sr.setColumns(Arrays.asList("rowid", "type", "description", "synopsis", "runid", "readset", "alignmentfile", "reference_library", "library_id", "sra_accession", "alignmentfile/DataFileUrl", "alignmentfile/Name", "reference_library", "reference_library/DataFileUrl", "runid/jobid", "runid/Name", "workbook/workbookId", "runid/JobId", "runid/Name", "runid/JobId/FilePath", "runid/JobId/Description")); + + SelectRowsResponse srr = sr.execute(getConnection(), remoteServerFolder); + + srr.getRowset().forEach(rd -> { + int remoteId = Integer.parseInt(String.valueOf(rd.getValue("rowid"))); + if (rd.getValue("readset") == null) + { + _log.warn("analysis lacks readset, skipping: " + remoteId); + return; + } + + int remoteReadset = Integer.parseInt(String.valueOf(rd.getValue("readset"))); + Integer localReadset = readsetMap.get(remoteReadset); + if (localReadset == null) + { + throw new IllegalArgumentException("Unable to find readset for remote id: " + remoteReadset); + } + + Integer localLibrary = null; + if (rd.getValue("library_id") != null) + { + int remoteLibrary = Integer.parseInt(String.valueOf(rd.getValue("library_id"))); + localLibrary = libraryMap.get(remoteLibrary); + if (localLibrary == null) + { + throw new IllegalArgumentException("Unable to find genome for remote id: " + remoteLibrary); + } + } + + Readset rs = SequenceAnalysisService.get().getReadset(localReadset, user); + Container targetWorkbook = ContainerManager.getForId(rs.getContainer()); + + SimpleFilter filter = new SimpleFilter(FieldKey.fromString("readset"), rs.getRowId()); + filter.addCondition(FieldKey.fromString("runid/JobId/Description"), rd.getValue("runid/JobId/Description")); + filter.addCondition(FieldKey.fromString("container"), targetWorkbook.getId(), CompareType.EQUAL); + + TableSelector tsAnalyses = new TableSelector(analysisTable, PageFlowUtil.set("rowid"), filter, null); + if (tsAnalyses.exists()) + { + analysisMap.put(remoteId, tsAnalyses.getObject(Integer.class)); + } + else + { + Map toCreate = new CaseInsensitiveHashMap<>(); + + toCreate.put("readset", rs.getRowId()); + toCreate.put("synopsis", rd.getValue("synopsis")); + toCreate.put("centerName", rd.getValue("centerName")); + toCreate.put("type", rd.getValue("type")); + toCreate.put("description", rd.getValue("description")); + toCreate.put("sra_accession", rd.getValue("sra_accession")); + if (localLibrary != null) + { + toCreate.put("library_id", localLibrary); + } + + try + { + if (rd.getValue("runid/JobId") == null) + { + _log.info("skipping analysis without runid: " + remoteId); + return; + } + + int remoteJobId = Integer.parseInt(String.valueOf(rd.getValue("runid/JobId"))); + int jobId = getOrCreateJob(remoteJobId, targetWorkbook); + PipelineStatusFile sf = PipelineService.get().getStatusFile(jobId); + + String localJobRoot = getParent(sf.getFilePath()); + String remoteJobRoot = getParent(URI.create(String.valueOf(rd.getValue("runid/JobId/FilePath")).replaceAll(" ", "_")).getPath()); + + URI newFileAlignment = translateURI(String.valueOf(rd.getValue("alignmentfile/DatafileUrl")), remoteJobRoot, localJobRoot); + toCreate.put("alignmentfile", getOrCreateExpData(newFileAlignment, targetWorkbook)); + + if (rd.getValue("reference_library") != null) + { + URI newFile2 = translateURI(String.valueOf(rd.getValue("reference_library/DatafileUrl")), remoteJobRoot, localJobRoot); + toCreate.put("reference_library", getOrCreateExpData(newFile2, targetWorkbook)); + } + + //Create run: + if (rd.getValue("runid") != null && rd.getValue("runid/JobId") != null) + { + int runId = createExpRun(Integer.parseInt(String.valueOf(rd.getValue("runid"))), targetWorkbook, String.valueOf(rd.getValue("runid/Name")), jobId); + toCreate.put("runid", runId); + } + else + { + _log.error("analysis missing runid: " + remoteId); + } + + BatchValidationException bve = new BatchValidationException(); + List> created = analysisTable.getUpdateService().insertRows(user, target, Arrays.asList(toCreate), bve, null, null); + if (bve.hasErrors()) + { + throw new RuntimeException(bve); + } + + analysisMap.put(remoteId, Integer.parseInt(String.valueOf(created.get(0).get("rowid")))); + } + catch (Exception e) + { + throw new RuntimeException(e); + } + } + }); + } + catch (Exception e) + { + _log.error(e.getMessage(), e); + throw new RuntimeException(e); + } + } + + private void createReaddata() + { + _log.info("Creating read data"); + try + { + final TableInfo readdataTable = QueryService.get().getUserSchema(user, target, "sequenceanalysis").getTable("readdata"); + + SelectRowsCommand sr = new SelectRowsCommand("sequenceanalysis", "readdata"); + sr.setColumns(Arrays.asList("rowid", "readset", "platformUnit", "centerName", "date", "fileid1", "fileid1/DataFileUrl", "fileid2", "fileid2/DataFileUrl", "fileid1/Name", "description", "sra_accession", "runid", "runid/jobid", "runid/Name", "readset/workbook/workbookId", "runid/JobId", "runid/Name", "runid/JobId/FilePath", "runid/JobId/Description")); + + SelectRowsResponse srr = sr.execute(getConnection(), remoteServerFolder); + + srr.getRowset().forEach(rd -> { + int remoteId = Integer.parseInt(String.valueOf(rd.getValue("rowid"))); + int remoteReadset = Integer.parseInt(String.valueOf(rd.getValue("readset"))); + Integer localReadset = readsetMap.get(remoteReadset); + if (localReadset == null) + { + throw new IllegalArgumentException("Unable to find readset for remote id: " + remoteReadset); + } + + Readset rs = SequenceAnalysisService.get().getReadset(localReadset, user); + Container targetWorkbook = ContainerManager.getForId(rs.getContainer()); + + SimpleFilter rdFilter = new SimpleFilter(FieldKey.fromString("readset"), rs.getRowId()); + rdFilter.addCondition(FieldKey.fromString("runid/JobId/Description"), rd.getValue("runid/JobId/Description")); + rdFilter.addCondition(FieldKey.fromString("fileid1/Name"), rd.getValue("fileid1/Name")); + rdFilter.addCondition(FieldKey.fromString("container"), targetWorkbook.getId(), CompareType.EQUAL); + + if (rd.getValue("platformUnit") != null) + { + rdFilter.addCondition(FieldKey.fromString("platformUnit"), rd.getValue("platformUnit")); + } + + TableSelector tsReaddata = new TableSelector(readdataTable, PageFlowUtil.set("rowid"), rdFilter, null); + if (tsReaddata.exists()) + { + readdataMap.put(remoteId, tsReaddata.getObject(Integer.class)); + } + else + { + Map toCreate = new CaseInsensitiveHashMap<>(); + toCreate.put("readset", rs.getRowId()); + toCreate.put("platformUnit", rd.getValue("platformUnit")); + toCreate.put("centerName", rd.getValue("centerName")); + toCreate.put("date", rd.getValue("date")); + toCreate.put("description", rd.getValue("description")); + toCreate.put("sra_accession", rd.getValue("sra_accession")); + try + { + if (rd.getValue("runid/JobId") != null) + { + int remoteJobId = Integer.parseInt(String.valueOf(rd.getValue("runid/JobId"))); + int jobId = getOrCreateJob(remoteJobId, targetWorkbook); + PipelineStatusFile sf = PipelineService.get().getStatusFile(jobId); + + String localJobRoot = getParent(sf.getFilePath()); + String remoteJobRoot = getParent(URI.create(String.valueOf(rd.getValue("runid/JobId/FilePath")).replaceAll(" ", "_")).getPath()); + + if (rd.getValue("fileid1/DataFileUrl") != null) + { + URI newFile1 = translateURI(String.valueOf(rd.getValue("fileid1/DataFileUrl")), remoteJobRoot, localJobRoot); + toCreate.put("fileid1", getOrCreateExpData(newFile1, targetWorkbook)); + } + + if (rd.getValue("fileid2/DataFileUrl") != null) + { + URI newFile2 = translateURI(String.valueOf(rd.getValue("fileid2/DatafileUrl")), remoteJobRoot, localJobRoot); + toCreate.put("fileid2", getOrCreateExpData(newFile2, targetWorkbook)); + } + } + else + { + _log.error("readddata missing jobid: " + remoteId); + } + + //Create run: + if (rd.getValue("runid") != null && rd.getValue("runid/JobId") != null) + { + int remoteJobId = Integer.parseInt(String.valueOf(rd.getValue("runid/JobId"))); + int jobId = getOrCreateJob(remoteJobId, targetWorkbook); + int runId = createExpRun(Integer.parseInt(String.valueOf(rd.getValue("runid"))), targetWorkbook, String.valueOf(rd.getValue("runid/Name")), jobId); + toCreate.put("runid", runId); + } + else + { + _log.error("readddata missing runid: " + remoteId); + } + + BatchValidationException bve = new BatchValidationException(); + List> created = readdataTable.getUpdateService().insertRows(user, target, Arrays.asList(toCreate), bve, null, null); + if (bve.hasErrors()) + { + throw new RuntimeException(bve); + } + + readdataMap.put(remoteId, Integer.parseInt(String.valueOf(created.get(0).get("rowid")))); + } + catch (Exception e) + { + throw new RuntimeException(e); + } + } + }); + } + catch (Exception e) + { + _log.error(e.getMessage(), e); + throw new RuntimeException(e); + } + } + + private int getOrCreateExpData(URI file, Container workbook) + { + ExpData ret = ExperimentService.get().getExpDataByURL(new File(file), workbook); + if (ret == null) + { + ret = ExperimentService.get().createData(workbook, new DataType("Data")); + ret.setDataFileURI(file); + ret.save(user); + } + + return ret.getRowId(); + } + + private void createReadsets() + { + _log.info("Creating readsets"); + try + { + final UserSchema us = QueryService.get().getUserSchema(user, target, "sequenceanalysis"); + final TableInfo readsetTable = us.getTable("sequence_readsets"); + + SelectRowsCommand sr = new SelectRowsCommand("sequenceanalysis", "sequence_readsets"); + sr.setColumns(Arrays.asList("rowid", "name", "platform", "application", "librarytype", "chemistry", "comments", "status", "subjectid", "subjectdate", "sampletype", "sampleid", "barcode5", "barcode3", "runid", "runid/jobid", "runid/Name", "workbook/workbookId", "runid/JobId", "runid/Name", "runid/JobId/FilePath")); + + SelectRowsResponse srr = sr.execute(getConnection(), remoteServerFolder); + + srr.getRowset().forEach(rs -> { + int remoteId = Integer.parseInt(String.valueOf(rs.getValue("rowid"))); + int sourceWorkbook = Integer.parseInt(String.valueOf(rs.getValue("workbook/workbookId"))); + Container targetWorkbook = workbookMap.get(sourceWorkbook); + if (targetWorkbook == null) + { + throw new IllegalArgumentException("Unable to find local workbook for source: " + sourceWorkbook); + } + + SimpleFilter rsFilter = new SimpleFilter(FieldKey.fromString("name"), rs.getValue("name")); + rsFilter.addCondition(FieldKey.fromString("container"), targetWorkbook.getId(), CompareType.EQUAL); + if (rs.getValue("subjectid") != null) + { + rsFilter.addCondition(FieldKey.fromString("subjectid"), rs.getValue("subjectid"), CompareType.EQUAL); + } + + TableSelector tsReadset = new TableSelector(readsetTable, PageFlowUtil.set("rowid"), rsFilter, null); + if (tsReadset.exists()) + { + readsetMap.put(remoteId, tsReadset.getObject(Integer.class)); + } + else + { + Map toCreate = new CaseInsensitiveHashMap<>(); + toCreate.put("name", rs.getValue("name")); + toCreate.put("platform", rs.getValue("platform")); + toCreate.put("application", rs.getValue("application")); + toCreate.put("barcode5", rs.getValue("barcode5")); + toCreate.put("barcode3", rs.getValue("barcode3")); + toCreate.put("subjectid", rs.getValue("subjectid")); + + toCreate.put("sampleid", rs.getValue("sampleid")); + toCreate.put("sampledate", rs.getValue("sampledate")); + toCreate.put("librarytype", rs.getValue("librarytype")); + toCreate.put("sampletype", rs.getValue("sampletype")); + toCreate.put("chemistry", rs.getValue("chemistry")); + toCreate.put("comments", rs.getValue("comments")); + toCreate.put("status", rs.getValue("status")); + + toCreate.put("container", targetWorkbook.getId()); + + try + { + //Create run: + if (rs.getValue("runid") != null && rs.getValue("runid/JobId") != null) + { + int remoteJobId = Integer.parseInt(String.valueOf(rs.getValue("runid/JobId"))); + int jobId = getOrCreateJob(remoteJobId, targetWorkbook); + int runid = createExpRun(Integer.parseInt(String.valueOf(rs.getValue("runid"))), targetWorkbook, String.valueOf(rs.getValue("runid/Name")), jobId); + toCreate.put("runid", runid); + } + else + { + _log.error("readset missing run id: " + remoteId); + } + + BatchValidationException bve = new BatchValidationException(); + List> created = readsetTable.getUpdateService().insertRows(user, target, Arrays.asList(toCreate), bve, null, null); + if (bve.hasErrors()) + { + throw new RuntimeException(bve); + } + + readsetMap.put(remoteId, Integer.parseInt(String.valueOf(created.get(0).get("rowid")))); + } + catch (Exception e) + { + throw new RuntimeException(e); + } + } + }); + } + catch (Exception e) + { + _log.error(e.getMessage(), e); + throw new RuntimeException(e); + } + } + + private int getOrCreateJob(int remoteJobId, Container targetWorkbook) + { + if (jobIdMap.containsKey(remoteJobId)) + { + return jobIdMap.get(remoteJobId); + } + + TableInfo ti = DbSchema.get("pipeline", DbSchemaType.Module).getTable("StatusFiles"); + + try + { + SelectRowsCommand sr = new SelectRowsCommand("pipeline", "job"); + sr.addFilter(new Filter("rowid", remoteJobId, Filter.Operator.EQUAL)); + sr.setColumns(Arrays.asList("RowId", "Info", "FilePath", "Email", "Description", "DataUrl", "Job", "Provider", "HadError", "ActiveTaskId")); + + SelectRowsResponse srr = sr.execute(getConnection(), remoteServerFolder); + + File fr = PipelineService.get().getPipelineRootSetting(targetWorkbook).getRootPath(); + + AtomicInteger ret = new AtomicInteger(); + srr.getRowset().forEach(pj -> { + String filepath = String.valueOf(pj.getValue("FilePath")); + if (!filepath.contains("@files")) + { + //This appears to be an error in PRIMe's data: + if (filepath.contains("illuminaImport")) + { + filepath = filepath.replace("illuminaImport", "@files/illuminaImport"); + } + else + { + _log.error("Unexpected filepath: " + pj.getValue("FilePath")); + } + } + + File remoteDir = new File(URI.create(filepath.replaceAll(" ", "_")).getPath()); + File localDir = new File(fr, filepath.split("@files")[1]); + + //Check for existing row: + TableSelector ts = new TableSelector(ti, PageFlowUtil.set("RowId"), new SimpleFilter(FieldKey.fromString("Job"), pj.getValue("Job")), null); + if (ts.exists()) + { + ret.set(ts.getObject(Integer.class)); + } + else + { + Map toCreate = new CaseInsensitiveHashMap<>(); + toCreate.put("Info", pj.getValue("Info")); + toCreate.put("FilePath", localDir.getPath()); + toCreate.put("Email", pj.getValue("Email")); + toCreate.put("Description", pj.getValue("Description")); + toCreate.put("DataUrl", pj.getValue("DataUrl")); + toCreate.put("Job", pj.getValue("Job")); + toCreate.put("Provider", pj.getValue("Provider")); + toCreate.put("HadError", pj.getValue("HadError")); + toCreate.put("ActiveTaskId", pj.getValue("ActiveTaskId")); + toCreate.put("Container", targetWorkbook.getId()); + + toCreate = Table.insert(user, ti, toCreate); + + ret.set((int) toCreate.get("RowId")); + } + + if (localDir.exists()) + { + _log.info("Directory exists, will not re-copy: " + localDir.getPath()); + return; + } + + try + { + _log.info(remoteDir.getPath()); + _log.info(localDir.getPath()); + + if (!localDir.getParentFile().exists()) + { + localDir.getParentFile().mkdirs(); + } + + if (remoteDir.exists()) + { + FileUtils.copyDirectory(remoteDir, localDir); + } + else + { + _log.error("source folder not found: " + remoteDir.getPath()); + } + } + catch (Exception e) + { + throw new RuntimeException(e); + } + }); + + jobIdMap.put(remoteJobId, ret.get()); + + return ret.get(); + } + catch (Exception e) + { + _log.error(e.getMessage(), e); + throw new RuntimeException(e); + } + } + + private int createExpRun(int remoteId, Container c, String name, int localJobId) throws Exception + { + if (runIdMap.containsKey(remoteId)) + { + return runIdMap.get(remoteId); + } + else + { + ExpRun ret = ExperimentService.get().createRunForProvenanceRecording(c, user, new RecordedActionSet(), name, localJobId); + runIdMap.put(remoteId, ret.getRowId()); + + return ret.getRowId(); + } + } + + private void createWorkbooks() + { + _log.info("Creating workbooks"); + try + { + TableInfo containers = QueryService.get().getUserSchema(user, target, "core").getTable("containers"); + + SelectRowsCommand sr = new SelectRowsCommand("core", "workbooks"); + sr.setColumns(Arrays.asList("Name", "Title", "Description")); + SelectRowsResponse srr = sr.execute(getConnection(), remoteServerFolder); + + srr.getRowset().forEach(wb -> { + String localTitle = (String)wb.getValue("Title"); + + TableSelector ts = new TableSelector(containers, PageFlowUtil.set("RowId"), new SimpleFilter(FieldKey.fromString("Title"), localTitle), null); + if (ts.exists()) + { + Container workbook = ContainerManager.getForRowId(ts.getObject(Integer.class)); + workbookMap.put(Integer.parseInt(String.valueOf(wb.getValue("Name"))), workbook); + } + else + { + String description = String.valueOf(wb.getValue("Description")); + if (description != null) + { + description = description + ". "; + } + else + { + description = ""; + } + + description = description + "Originally PRIMe workbook: " + wb.getValue("Name"); + + Container workbook = ContainerManager.createContainer(target, null, localTitle, description, WorkbookContainerType.NAME, user); + workbookMap.put(Integer.parseInt(String.valueOf(wb.getValue("Name"))), workbook); + } + }); + } + catch (CommandException | IOException e) + { + throw new RuntimeException(e); + } + } + + private URI translateURI(String databaseURI, String remoteFolderRoot, String localFolderRoot) + { + databaseURI = databaseURI.replace("\\", "/"); + remoteFolderRoot = remoteFolderRoot.replace("\\", "/").split("@files")[0]; + localFolderRoot = localFolderRoot.replace("\\", "/").split("@files")[0]; + if (localFolderRoot.startsWith("C:")) + { + localFolderRoot = localFolderRoot.replaceAll("^C:", ""); + } + + databaseURI = databaseURI.replace(remoteFolderRoot, localFolderRoot); + + return URI.create(databaseURI); + } +} diff --git a/primeseq/src/org/labkey/primeseq/PrimeseqController.java b/primeseq/src/org/labkey/primeseq/PrimeseqController.java index bc680d82c..f077e2267 100644 --- a/primeseq/src/org/labkey/primeseq/PrimeseqController.java +++ b/primeseq/src/org/labkey/primeseq/PrimeseqController.java @@ -203,4 +203,47 @@ public URLHelper getSuccessURL(Object o) } } + @RequiresSiteAdmin + public class SyncMhcAction extends ConfirmAction + { + @Override + public ModelAndView getConfirmView(Object o, BindException errors) throws Exception + { + setTitle("Sync MHC Data from PRIMe"); + + return new HtmlView(HtmlString.of("This will attempt to sync MHC typing data from PRIMe to the current folder, creating all sequence records and workbooks. Do you want to continue?")); + } + + @Override + public boolean handlePost(Object o, BindException errors) throws Exception + { + try + { + MhcMigration mhc = new MhcMigration(getContainer(), getUser(), "PRIMe", "ONPRC/Core Facilities/Genetics Core/MHC_Typing/"); + mhc.doWork(); + } + catch (Exception e) + { + _log.error(e); + errors.reject(ERROR_MSG, e.getMessage()); + return false; + + } + + return true; + } + + @Override + public void validateCommand(Object o, Errors errors) + { + + } + + @NotNull + @Override + public URLHelper getSuccessURL(Object o) + { + return PageFlowUtil.urlProvider(PipelineUrls.class).urlBegin(getContainer()); + } + } } \ No newline at end of file From 14215142f6bc391966ca8c0119ed9bd850878e53 Mon Sep 17 00:00:00 2001 From: bbimber Date: Fri, 29 Jan 2021 07:07:27 -0800 Subject: [PATCH 08/29] Add validation --- primeseq/src/org/labkey/primeseq/MhcMigration.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/primeseq/src/org/labkey/primeseq/MhcMigration.java b/primeseq/src/org/labkey/primeseq/MhcMigration.java index 76246381b..b17a87fe2 100644 --- a/primeseq/src/org/labkey/primeseq/MhcMigration.java +++ b/primeseq/src/org/labkey/primeseq/MhcMigration.java @@ -856,6 +856,10 @@ private int getOrCreateJob(int remoteJobId, Container targetWorkbook) { filepath = filepath.replace("illuminaImport", "@files/illuminaImport"); } + else if (filepath.contains("sequenceAnalysis")) + { + filepath = filepath.replace("sequenceAnalysis", "@files/sequenceAnalysis"); + } else { _log.error("Unexpected filepath: " + pj.getValue("FilePath")); From 29cdf6dbbcbd0ca366743ad8579135d15ab973b6 Mon Sep 17 00:00:00 2001 From: bbimber Date: Fri, 29 Jan 2021 09:30:54 -0800 Subject: [PATCH 09/29] Convert MHC migration code to a pipeline job --- .../src/org/labkey/primeseq/MhcMigration.java | 1013 --------------- .../labkey/primeseq/PrimeseqController.java | 8 +- .../org/labkey/primeseq/PrimeseqModule.java | 4 + .../pipeline/MhcMigrationPipelineJob.java | 1133 +++++++++++++++++ primeseq/webapp/WEB-INF/primeseqContext.xml | 22 + 5 files changed, 1165 insertions(+), 1015 deletions(-) delete mode 100644 primeseq/src/org/labkey/primeseq/MhcMigration.java create mode 100644 primeseq/src/org/labkey/primeseq/pipeline/MhcMigrationPipelineJob.java diff --git a/primeseq/src/org/labkey/primeseq/MhcMigration.java b/primeseq/src/org/labkey/primeseq/MhcMigration.java deleted file mode 100644 index b17a87fe2..000000000 --- a/primeseq/src/org/labkey/primeseq/MhcMigration.java +++ /dev/null @@ -1,1013 +0,0 @@ -package org.labkey.primeseq; - -import org.apache.commons.io.FileUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.labkey.api.collections.CaseInsensitiveHashMap; -import org.labkey.api.data.CompareType; -import org.labkey.api.data.Container; -import org.labkey.api.data.ContainerManager; -import org.labkey.api.data.DbSchema; -import org.labkey.api.data.DbSchemaType; -import org.labkey.api.data.DbScope; -import org.labkey.api.data.SimpleFilter; -import org.labkey.api.data.Sort; -import org.labkey.api.data.Table; -import org.labkey.api.data.TableInfo; -import org.labkey.api.data.TableSelector; -import org.labkey.api.data.WorkbookContainerType; -import org.labkey.api.di.DataIntegrationService; -import org.labkey.api.exp.api.DataType; -import org.labkey.api.exp.api.ExpData; -import org.labkey.api.exp.api.ExpRun; -import org.labkey.api.exp.api.ExperimentService; -import org.labkey.api.pipeline.PipelineService; -import org.labkey.api.pipeline.PipelineStatusFile; -import org.labkey.api.pipeline.RecordedActionSet; -import org.labkey.api.query.BatchValidationException; -import org.labkey.api.query.FieldKey; -import org.labkey.api.query.QueryService; -import org.labkey.api.query.UserSchema; -import org.labkey.api.security.User; -import org.labkey.api.sequenceanalysis.SequenceAnalysisService; -import org.labkey.api.sequenceanalysis.model.Readset; -import org.labkey.api.util.PageFlowUtil; -import org.labkey.remoteapi.CommandException; -import org.labkey.remoteapi.Connection; -import org.labkey.remoteapi.query.Filter; -import org.labkey.remoteapi.query.SelectRowsCommand; -import org.labkey.remoteapi.query.SelectRowsResponse; - -import java.io.File; -import java.io.IOException; -import java.net.URI; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.atomic.AtomicInteger; - -public class MhcMigration -{ - private static final Logger _log = LogManager.getLogger(MhcMigration.class); - - private final String remoteServerFolder; - private final String remoteConnectionName; - - private final User user; - private final Container target; - - public MhcMigration(Container c, User u, String remoteConnectionName, String remoteServerFolder) - { - this.target = c; - this.user = u; - this.remoteConnectionName = remoteConnectionName; - this.remoteServerFolder = remoteServerFolder; - } - - private Connection getConnection() - { - DataIntegrationService.RemoteConnection rc = DataIntegrationService.get().getRemoteConnection(remoteConnectionName, target, _log); - - return(rc.connection); - } - - public void doWork() - { - try (DbScope.Transaction transaction = DbScope.getLabKeyScope().ensureTransaction()) - { - createWorkbooks(); - - createLibraries(); - createLibraryMembers(); - - createReadsets(); - transaction.commitAndKeepConnection(); - - createReaddata(); - - createAnalyses(); - createOutputFiles(); - - //TODO: - //samples - //alignment_summary - //alignment_summary_junction - //quality_metrics - //subjects - //WaNPRC - - //sequenceanalysis.haplotypes - //sequenceanalysis.haplotype_types - //sequenceanalysis.haplotype_sequences - - //Create assay runs, including data and haplotypes - - transaction.commit(); - } - } - - private void replaceEntireTable(String schema, String query, List columns, String workbookColName, boolean truncateExisting) throws Exception - { - SelectRowsCommand sr = new SelectRowsCommand(schema, query); - sr.setColumns(columns); - SelectRowsResponse srr = sr.execute(getConnection(), remoteServerFolder); - - List> toInsert = new ArrayList<>(); - srr.getRowset().forEach(r -> { - Map row = new CaseInsensitiveHashMap<>(); - srr.getColumnModel().forEach(col -> { - String colName = (String)col.get("Name"); - Object val = r.getValue(colName); - if ("readset".equals(colName) || "readsetid".equals(colName)) - { - if (!readsetMap.containsKey((int)val)) - { - throw new IllegalStateException("Unable to find readset: " + val); - } - - val = readsetMap.get((int)val); - } - else if ("library_id".equals(colName)) - { - if (!libraryMap.containsKey((int)val)) - { - throw new IllegalStateException("Unable to find library: " + val); - } - - val = libraryMap.get((int)val); - - } - else if ("ref_nt_id".equals(colName)) - { - if (!sequenceMap.containsKey((int)val)) - { - throw new IllegalStateException("Unable to find sequence: " + val); - } - - val = sequenceMap.get((int)val); - } - else if ("analysis_id".equals(colName)) - { - if (!analysisMap.containsKey((int)val)) - { - throw new IllegalStateException("Unable to find analysis: " + val); - } - - val = analysisMap.get((int)val); - } - - row.put(colName, val); - }); - - if (workbookColName != null) - { - Object workbookId = r.getValue(workbookColName); - if (workbookId != null) - { - row.put("container", workbookMap.get(Integer.parseInt(String.valueOf(workbookId))).getId()); - } - } - - toInsert.add(row); - }); - - - - } - - //All of these map remote Id to local Id - private final Map workbookMap = new HashMap<>(); - private final Map readsetMap = new HashMap<>(); - private final Map readdataMap = new HashMap<>(); - private final Map analysisMap = new HashMap<>(); - private final Map libraryMap = new HashMap<>(); - private final Map outputFileMap = new HashMap<>(); - private final Map sequenceMap = new HashMap<>(); - private final Map runIdMap = new HashMap<>(); - private final Map jobIdMap = new HashMap<>(); - - private void createLibraryMembers() - { - _log.info("Creating library members"); - - final UserSchema us = QueryService.get().getUserSchema(user, target, "sequenceanalysis"); - final TableInfo ti = us.getTable("reference_library_members"); - final TableInfo refNtTable = us.getTable("ref_nt_sequences"); - - try - { - SelectRowsCommand sr = new SelectRowsCommand("sequenceanalysis", "reference_library_members"); - sr.setColumns(Arrays.asList("rowid", "library_id", "ref_nt_id", "ref_nt_id/name", "ref_nt_id/seqLength", "workbook/workbookId")); - - SelectRowsResponse srr = sr.execute(getConnection(), remoteServerFolder); - - srr.getRowset().forEach(rd -> { - int remoteId = Integer.parseInt(String.valueOf(rd.getValue("rowid"))); - int seqLength = Integer.parseInt(String.valueOf(rd.getValue("ref_nt_id/seqLength"))); - - int remoteSeqId = Integer.parseInt(String.valueOf(rd.getValue("ref_nt_id"))); - String name = String.valueOf(rd.getValue("ref_nt_id/name")); - int localSeqId = getOrCreateSequence(remoteSeqId, name, seqLength, refNtTable); - - int remoteLibraryId = Integer.parseInt(String.valueOf(rd.getValue("library_id"))); - Integer localLibraryId = libraryMap.get(remoteLibraryId); - if (localLibraryId == null) - { - throw new IllegalStateException("Unable to find library id: " + remoteLibraryId); - } - - SimpleFilter filter = new SimpleFilter(FieldKey.fromString("library_id"), localLibraryId); - filter.addCondition(FieldKey.fromString("ref_nt_id"), localSeqId); - - if (new TableSelector(ti, PageFlowUtil.set("rowid"), filter, null).exists()) - { - //Already exists: - return; - } - - Map toCreate = new CaseInsensitiveHashMap<>(); - toCreate.put("library_id", localLibraryId); - toCreate.put("ref_nt_id", localSeqId); - - try - { - BatchValidationException bve = new BatchValidationException(); - List> created = ti.getUpdateService().insertRows(user, target, Arrays.asList(toCreate), bve, null, null); - if (bve.hasErrors()) - { - throw new RuntimeException(bve); - } - } - catch (Exception e) - { - _log.error(e.getMessage(), e); - throw new RuntimeException(e); - } - }); - } - catch (Exception e) - { - _log.error(e.getMessage(), e); - throw new RuntimeException(e); - } - } - - private int getOrCreateSequence(int remoteSeqId, String name, int seqLength, TableInfo refNtTable) - { - if (sequenceMap.containsKey(remoteSeqId)) - { - return sequenceMap.get(remoteSeqId); - } - else - { - SimpleFilter filter = new SimpleFilter(FieldKey.fromString("name"), name); - filter.addCondition(FieldKey.fromString("datedisabled"), null, CompareType.ISBLANK); - TableSelector ts = new TableSelector(refNtTable, PageFlowUtil.set("rowid", "seqLength"), filter, new Sort("rowid")); - if (ts.exists()) - { - if (ts.getRowCount() > 1) - { - _log.info("Duplicate ref name: " + name); - } - - AtomicInteger localId = new AtomicInteger(-1); - ts.forEachResults(rs -> { - if (rs.getInt(FieldKey.fromString("seqLength")) < seqLength) - { - _log.warn("length doesnt match for " + name + ", expected: " + seqLength); - return; - } - - localId.set(rs.getInt(FieldKey.fromString("rowid"))); - }); - - if (localId.get() != -1) - { - sequenceMap.put(remoteSeqId, localId.get()); - return localId.get(); - } - } - - //TODO: Create sequence? - //throw new IllegalStateException("Expected sequence to exist: " + name); - _log.error("Sequence missing: " + name); - return -1; - } - } - - public String getParent(String path) { - final char separatorChar = '/'; - - int index = path.lastIndexOf(separatorChar); - - return path.substring(0, index); - } - - private void createLibraries() - { - _log.info("Creating libraries"); - try - { - final TableInfo libraryTable = QueryService.get().getUserSchema(user, target, "sequenceanalysis").getTable("reference_libraries"); - - SelectRowsCommand sr = new SelectRowsCommand("sequenceanalysis", "reference_libraries"); - sr.setColumns(Arrays.asList("rowid", "name", "description", "fasta_file", "datedisabled", "assemblyId", "fasta_file/DataFileUrl", "workbook/workbookId")); - - SelectRowsResponse srr = sr.execute(getConnection(), remoteServerFolder); - - srr.getRowset().forEach(rd -> { - int remoteId = Integer.parseInt(String.valueOf(rd.getValue("rowid"))); - - Integer remoteWorkbook = rd.getValue("workbook/workbookId") == null ? null : Integer.parseInt(String.valueOf(rd.getValue("workbook/workbookId"))); - Container targetContainer = remoteWorkbook == null ? target : workbookMap.get(remoteWorkbook); - - SimpleFilter filter = new SimpleFilter(FieldKey.fromString("name"), rd.getValue("name")); - TableSelector ts = new TableSelector(libraryTable, PageFlowUtil.set("rowid"), filter, null); - if (ts.exists()) - { - libraryMap.put(remoteId, ts.getObject(Integer.class)); - } - else - { - Map toCreate = new CaseInsensitiveHashMap<>(); - toCreate.put("name", rd.getValue("name")); - toCreate.put("description", rd.getValue("description")); - toCreate.put("datedisabled", rd.getValue("datedisabled")); - toCreate.put("assemblyId", rd.getValue("assemblyId")); - try - { - String remoteJobRoot = getParent(URI.create(String.valueOf(rd.getValue("fasta_file/DatafileUrl"))).getPath()); - URI localJobRoot = PipelineService.get().getPipelineRootSetting(targetContainer).getRootPath().toURI(); - URI localFasta = translateURI(String.valueOf(rd.getValue("fasta_file/DatafileUrl")), remoteJobRoot, localJobRoot.getPath()); - toCreate.put("fasta_file", getOrCreateExpData(localFasta, targetContainer)); - - //Ensure parent folder exists: - File localJobRootFile = new File(localFasta).getParentFile(); - if (!localJobRootFile.getParentFile().exists()) - { - localJobRootFile.getParentFile().mkdirs(); - } - - _log.info(remoteJobRoot); - _log.info(localJobRoot.getPath()); - File remoteJobRootFile = new File(remoteJobRoot); - if (remoteJobRootFile.exists()) - { - FileUtils.copyDirectory(remoteJobRootFile, localJobRootFile); - } - - BatchValidationException bve = new BatchValidationException(); - List> created = libraryTable.getUpdateService().insertRows(user, target, Arrays.asList(toCreate), bve, null, null); - if (bve.hasErrors()) - { - throw new RuntimeException(bve); - } - - libraryMap.put(remoteId, Integer.parseInt(String.valueOf(created.get(0).get("rowid")))); - } - catch (Exception e) - { - throw new RuntimeException(e); - } - } - }); - } - catch (Exception e) - { - _log.error(e.getMessage(), e); - throw new RuntimeException(e); - } - } - - private void createOutputFiles() - { - _log.info("Creating outputfiles"); - try - { - final TableInfo outputTable = QueryService.get().getUserSchema(user, target, "sequenceanalysis").getTable("outputfiles"); - - SelectRowsCommand sr = new SelectRowsCommand("sequenceanalysis", "outputfiles"); - sr.setColumns(Arrays.asList("rowid", "name", "description", "dataid", "library_id", "readset", "analysis_id", "category", "sra_accession", "dataid/DataFileUrl", "runid/jobid", "runid/Name", "workbook/workbookId", "runid/JobId", "runid/Name", "runid/JobId/FilePath")); - - SelectRowsResponse srr = sr.execute(getConnection(), remoteServerFolder); - - srr.getRowset().forEach(rd -> { - int remoteId = Integer.parseInt(String.valueOf(rd.getValue("rowid"))); - int remoteReadset = Integer.parseInt(String.valueOf(rd.getValue("readset"))); - Integer localReadset = readsetMap.get(remoteReadset); - if (localReadset == null) - { - throw new IllegalArgumentException("Unable to find readset for remote id: " + remoteReadset); - } - - int remoteLibrary = Integer.parseInt(String.valueOf(rd.getValue("library_id"))); - Integer localLibrary = libraryMap.get(remoteLibrary); - if (localLibrary == null) - { - throw new IllegalArgumentException("Unable to find genome for remote id: " + remoteLibrary); - } - - int remoteAnalysis = Integer.parseInt(String.valueOf(rd.getValue("analysis_id"))); - Integer localAnalysis = analysisMap.get(remoteAnalysis); - if (localAnalysis == null) - { - throw new IllegalArgumentException("Unable to find analysis for remote id: " + remoteAnalysis); - } - - Readset rs = SequenceAnalysisService.get().getReadset(localReadset, user); - Container targetWorkbook = ContainerManager.getForId(rs.getContainer()); - - SimpleFilter filter = new SimpleFilter(FieldKey.fromString("readset"), rs.getRowId()); - filter.addCondition(FieldKey.fromString("name"), rd.getValue("name")); - filter.addCondition(FieldKey.fromString("category"), rd.getValue("category")); - filter.addCondition(FieldKey.fromString("analysis_id"), localAnalysis); - filter.addCondition(FieldKey.fromString("container"), targetWorkbook.getId(), CompareType.EQUAL); - - TableSelector tsOutputFiles = new TableSelector(outputTable, PageFlowUtil.set("rowid"), filter, null); - if (tsOutputFiles.exists()) - { - outputFileMap.put(remoteId, tsOutputFiles.getObject(Integer.class)); - } - else - { - Map toCreate = new CaseInsensitiveHashMap<>(); - toCreate.put("readset", rs.getRowId()); - toCreate.put("analysis_id", localAnalysis); - toCreate.put("description", rd.getValue("description")); - toCreate.put("sra_accession", rd.getValue("sra_accession")); - toCreate.put("library_id", localLibrary); - toCreate.put("name", rd.getValue("name")); - toCreate.put("category", rd.getValue("category")); - - try - { - int remoteJobId = Integer.parseInt(String.valueOf(rd.getValue("runid/JobId"))); - int jobId = getOrCreateJob(remoteJobId, targetWorkbook); - PipelineStatusFile sf = PipelineService.get().getStatusFile(jobId); - - String localJobRoot = getParent(sf.getFilePath()); - String remoteJobRoot = getParent(URI.create(String.valueOf(rd.getValue("runid/JobId/FilePath")).replaceAll(" ", "_")).getPath()); - - URI newFileAlignment = translateURI(String.valueOf(rd.getValue("dataid/DatafileUrl")), remoteJobRoot, localJobRoot); - toCreate.put("dataid", getOrCreateExpData(newFileAlignment, targetWorkbook)); - - //Create run: - if (rd.getValue("runid") != null && rd.getValue("runid/JobId") != null) - { - int runId = createExpRun(Integer.parseInt(String.valueOf(rd.getValue("runid"))), targetWorkbook, String.valueOf(rd.getValue("runid/Name")), jobId); - toCreate.put("runid", runId); - } - else - { - _log.error("output missing runid: " + remoteId); - } - - BatchValidationException bve = new BatchValidationException(); - List> created = outputTable.getUpdateService().insertRows(user, target, Arrays.asList(toCreate), bve, null, null); - if (bve.hasErrors()) - { - throw new RuntimeException(bve); - } - - outputFileMap.put(remoteId, Integer.parseInt(String.valueOf(created.get(0).get("rowid")))); - } - catch (Exception e) - { - throw new RuntimeException(e); - } - } - }); - } - catch (Exception e) - { - _log.error(e.getMessage(), e); - throw new RuntimeException(e); - } - } - - private void createAnalyses() - { - _log.info("Creating analyses"); - try - { - final TableInfo analysisTable = QueryService.get().getUserSchema(user, target, "sequenceanalysis").getTable("sequence_analyses"); - - SelectRowsCommand sr = new SelectRowsCommand("sequenceanalysis", "sequence_analyses"); - sr.setColumns(Arrays.asList("rowid", "type", "description", "synopsis", "runid", "readset", "alignmentfile", "reference_library", "library_id", "sra_accession", "alignmentfile/DataFileUrl", "alignmentfile/Name", "reference_library", "reference_library/DataFileUrl", "runid/jobid", "runid/Name", "workbook/workbookId", "runid/JobId", "runid/Name", "runid/JobId/FilePath", "runid/JobId/Description")); - - SelectRowsResponse srr = sr.execute(getConnection(), remoteServerFolder); - - srr.getRowset().forEach(rd -> { - int remoteId = Integer.parseInt(String.valueOf(rd.getValue("rowid"))); - if (rd.getValue("readset") == null) - { - _log.warn("analysis lacks readset, skipping: " + remoteId); - return; - } - - int remoteReadset = Integer.parseInt(String.valueOf(rd.getValue("readset"))); - Integer localReadset = readsetMap.get(remoteReadset); - if (localReadset == null) - { - throw new IllegalArgumentException("Unable to find readset for remote id: " + remoteReadset); - } - - Integer localLibrary = null; - if (rd.getValue("library_id") != null) - { - int remoteLibrary = Integer.parseInt(String.valueOf(rd.getValue("library_id"))); - localLibrary = libraryMap.get(remoteLibrary); - if (localLibrary == null) - { - throw new IllegalArgumentException("Unable to find genome for remote id: " + remoteLibrary); - } - } - - Readset rs = SequenceAnalysisService.get().getReadset(localReadset, user); - Container targetWorkbook = ContainerManager.getForId(rs.getContainer()); - - SimpleFilter filter = new SimpleFilter(FieldKey.fromString("readset"), rs.getRowId()); - filter.addCondition(FieldKey.fromString("runid/JobId/Description"), rd.getValue("runid/JobId/Description")); - filter.addCondition(FieldKey.fromString("container"), targetWorkbook.getId(), CompareType.EQUAL); - - TableSelector tsAnalyses = new TableSelector(analysisTable, PageFlowUtil.set("rowid"), filter, null); - if (tsAnalyses.exists()) - { - analysisMap.put(remoteId, tsAnalyses.getObject(Integer.class)); - } - else - { - Map toCreate = new CaseInsensitiveHashMap<>(); - - toCreate.put("readset", rs.getRowId()); - toCreate.put("synopsis", rd.getValue("synopsis")); - toCreate.put("centerName", rd.getValue("centerName")); - toCreate.put("type", rd.getValue("type")); - toCreate.put("description", rd.getValue("description")); - toCreate.put("sra_accession", rd.getValue("sra_accession")); - if (localLibrary != null) - { - toCreate.put("library_id", localLibrary); - } - - try - { - if (rd.getValue("runid/JobId") == null) - { - _log.info("skipping analysis without runid: " + remoteId); - return; - } - - int remoteJobId = Integer.parseInt(String.valueOf(rd.getValue("runid/JobId"))); - int jobId = getOrCreateJob(remoteJobId, targetWorkbook); - PipelineStatusFile sf = PipelineService.get().getStatusFile(jobId); - - String localJobRoot = getParent(sf.getFilePath()); - String remoteJobRoot = getParent(URI.create(String.valueOf(rd.getValue("runid/JobId/FilePath")).replaceAll(" ", "_")).getPath()); - - URI newFileAlignment = translateURI(String.valueOf(rd.getValue("alignmentfile/DatafileUrl")), remoteJobRoot, localJobRoot); - toCreate.put("alignmentfile", getOrCreateExpData(newFileAlignment, targetWorkbook)); - - if (rd.getValue("reference_library") != null) - { - URI newFile2 = translateURI(String.valueOf(rd.getValue("reference_library/DatafileUrl")), remoteJobRoot, localJobRoot); - toCreate.put("reference_library", getOrCreateExpData(newFile2, targetWorkbook)); - } - - //Create run: - if (rd.getValue("runid") != null && rd.getValue("runid/JobId") != null) - { - int runId = createExpRun(Integer.parseInt(String.valueOf(rd.getValue("runid"))), targetWorkbook, String.valueOf(rd.getValue("runid/Name")), jobId); - toCreate.put("runid", runId); - } - else - { - _log.error("analysis missing runid: " + remoteId); - } - - BatchValidationException bve = new BatchValidationException(); - List> created = analysisTable.getUpdateService().insertRows(user, target, Arrays.asList(toCreate), bve, null, null); - if (bve.hasErrors()) - { - throw new RuntimeException(bve); - } - - analysisMap.put(remoteId, Integer.parseInt(String.valueOf(created.get(0).get("rowid")))); - } - catch (Exception e) - { - throw new RuntimeException(e); - } - } - }); - } - catch (Exception e) - { - _log.error(e.getMessage(), e); - throw new RuntimeException(e); - } - } - - private void createReaddata() - { - _log.info("Creating read data"); - try - { - final TableInfo readdataTable = QueryService.get().getUserSchema(user, target, "sequenceanalysis").getTable("readdata"); - - SelectRowsCommand sr = new SelectRowsCommand("sequenceanalysis", "readdata"); - sr.setColumns(Arrays.asList("rowid", "readset", "platformUnit", "centerName", "date", "fileid1", "fileid1/DataFileUrl", "fileid2", "fileid2/DataFileUrl", "fileid1/Name", "description", "sra_accession", "runid", "runid/jobid", "runid/Name", "readset/workbook/workbookId", "runid/JobId", "runid/Name", "runid/JobId/FilePath", "runid/JobId/Description")); - - SelectRowsResponse srr = sr.execute(getConnection(), remoteServerFolder); - - srr.getRowset().forEach(rd -> { - int remoteId = Integer.parseInt(String.valueOf(rd.getValue("rowid"))); - int remoteReadset = Integer.parseInt(String.valueOf(rd.getValue("readset"))); - Integer localReadset = readsetMap.get(remoteReadset); - if (localReadset == null) - { - throw new IllegalArgumentException("Unable to find readset for remote id: " + remoteReadset); - } - - Readset rs = SequenceAnalysisService.get().getReadset(localReadset, user); - Container targetWorkbook = ContainerManager.getForId(rs.getContainer()); - - SimpleFilter rdFilter = new SimpleFilter(FieldKey.fromString("readset"), rs.getRowId()); - rdFilter.addCondition(FieldKey.fromString("runid/JobId/Description"), rd.getValue("runid/JobId/Description")); - rdFilter.addCondition(FieldKey.fromString("fileid1/Name"), rd.getValue("fileid1/Name")); - rdFilter.addCondition(FieldKey.fromString("container"), targetWorkbook.getId(), CompareType.EQUAL); - - if (rd.getValue("platformUnit") != null) - { - rdFilter.addCondition(FieldKey.fromString("platformUnit"), rd.getValue("platformUnit")); - } - - TableSelector tsReaddata = new TableSelector(readdataTable, PageFlowUtil.set("rowid"), rdFilter, null); - if (tsReaddata.exists()) - { - readdataMap.put(remoteId, tsReaddata.getObject(Integer.class)); - } - else - { - Map toCreate = new CaseInsensitiveHashMap<>(); - toCreate.put("readset", rs.getRowId()); - toCreate.put("platformUnit", rd.getValue("platformUnit")); - toCreate.put("centerName", rd.getValue("centerName")); - toCreate.put("date", rd.getValue("date")); - toCreate.put("description", rd.getValue("description")); - toCreate.put("sra_accession", rd.getValue("sra_accession")); - try - { - if (rd.getValue("runid/JobId") != null) - { - int remoteJobId = Integer.parseInt(String.valueOf(rd.getValue("runid/JobId"))); - int jobId = getOrCreateJob(remoteJobId, targetWorkbook); - PipelineStatusFile sf = PipelineService.get().getStatusFile(jobId); - - String localJobRoot = getParent(sf.getFilePath()); - String remoteJobRoot = getParent(URI.create(String.valueOf(rd.getValue("runid/JobId/FilePath")).replaceAll(" ", "_")).getPath()); - - if (rd.getValue("fileid1/DataFileUrl") != null) - { - URI newFile1 = translateURI(String.valueOf(rd.getValue("fileid1/DataFileUrl")), remoteJobRoot, localJobRoot); - toCreate.put("fileid1", getOrCreateExpData(newFile1, targetWorkbook)); - } - - if (rd.getValue("fileid2/DataFileUrl") != null) - { - URI newFile2 = translateURI(String.valueOf(rd.getValue("fileid2/DatafileUrl")), remoteJobRoot, localJobRoot); - toCreate.put("fileid2", getOrCreateExpData(newFile2, targetWorkbook)); - } - } - else - { - _log.error("readddata missing jobid: " + remoteId); - } - - //Create run: - if (rd.getValue("runid") != null && rd.getValue("runid/JobId") != null) - { - int remoteJobId = Integer.parseInt(String.valueOf(rd.getValue("runid/JobId"))); - int jobId = getOrCreateJob(remoteJobId, targetWorkbook); - int runId = createExpRun(Integer.parseInt(String.valueOf(rd.getValue("runid"))), targetWorkbook, String.valueOf(rd.getValue("runid/Name")), jobId); - toCreate.put("runid", runId); - } - else - { - _log.error("readddata missing runid: " + remoteId); - } - - BatchValidationException bve = new BatchValidationException(); - List> created = readdataTable.getUpdateService().insertRows(user, target, Arrays.asList(toCreate), bve, null, null); - if (bve.hasErrors()) - { - throw new RuntimeException(bve); - } - - readdataMap.put(remoteId, Integer.parseInt(String.valueOf(created.get(0).get("rowid")))); - } - catch (Exception e) - { - throw new RuntimeException(e); - } - } - }); - } - catch (Exception e) - { - _log.error(e.getMessage(), e); - throw new RuntimeException(e); - } - } - - private int getOrCreateExpData(URI file, Container workbook) - { - ExpData ret = ExperimentService.get().getExpDataByURL(new File(file), workbook); - if (ret == null) - { - ret = ExperimentService.get().createData(workbook, new DataType("Data")); - ret.setDataFileURI(file); - ret.save(user); - } - - return ret.getRowId(); - } - - private void createReadsets() - { - _log.info("Creating readsets"); - try - { - final UserSchema us = QueryService.get().getUserSchema(user, target, "sequenceanalysis"); - final TableInfo readsetTable = us.getTable("sequence_readsets"); - - SelectRowsCommand sr = new SelectRowsCommand("sequenceanalysis", "sequence_readsets"); - sr.setColumns(Arrays.asList("rowid", "name", "platform", "application", "librarytype", "chemistry", "comments", "status", "subjectid", "subjectdate", "sampletype", "sampleid", "barcode5", "barcode3", "runid", "runid/jobid", "runid/Name", "workbook/workbookId", "runid/JobId", "runid/Name", "runid/JobId/FilePath")); - - SelectRowsResponse srr = sr.execute(getConnection(), remoteServerFolder); - - srr.getRowset().forEach(rs -> { - int remoteId = Integer.parseInt(String.valueOf(rs.getValue("rowid"))); - int sourceWorkbook = Integer.parseInt(String.valueOf(rs.getValue("workbook/workbookId"))); - Container targetWorkbook = workbookMap.get(sourceWorkbook); - if (targetWorkbook == null) - { - throw new IllegalArgumentException("Unable to find local workbook for source: " + sourceWorkbook); - } - - SimpleFilter rsFilter = new SimpleFilter(FieldKey.fromString("name"), rs.getValue("name")); - rsFilter.addCondition(FieldKey.fromString("container"), targetWorkbook.getId(), CompareType.EQUAL); - if (rs.getValue("subjectid") != null) - { - rsFilter.addCondition(FieldKey.fromString("subjectid"), rs.getValue("subjectid"), CompareType.EQUAL); - } - - TableSelector tsReadset = new TableSelector(readsetTable, PageFlowUtil.set("rowid"), rsFilter, null); - if (tsReadset.exists()) - { - readsetMap.put(remoteId, tsReadset.getObject(Integer.class)); - } - else - { - Map toCreate = new CaseInsensitiveHashMap<>(); - toCreate.put("name", rs.getValue("name")); - toCreate.put("platform", rs.getValue("platform")); - toCreate.put("application", rs.getValue("application")); - toCreate.put("barcode5", rs.getValue("barcode5")); - toCreate.put("barcode3", rs.getValue("barcode3")); - toCreate.put("subjectid", rs.getValue("subjectid")); - - toCreate.put("sampleid", rs.getValue("sampleid")); - toCreate.put("sampledate", rs.getValue("sampledate")); - toCreate.put("librarytype", rs.getValue("librarytype")); - toCreate.put("sampletype", rs.getValue("sampletype")); - toCreate.put("chemistry", rs.getValue("chemistry")); - toCreate.put("comments", rs.getValue("comments")); - toCreate.put("status", rs.getValue("status")); - - toCreate.put("container", targetWorkbook.getId()); - - try - { - //Create run: - if (rs.getValue("runid") != null && rs.getValue("runid/JobId") != null) - { - int remoteJobId = Integer.parseInt(String.valueOf(rs.getValue("runid/JobId"))); - int jobId = getOrCreateJob(remoteJobId, targetWorkbook); - int runid = createExpRun(Integer.parseInt(String.valueOf(rs.getValue("runid"))), targetWorkbook, String.valueOf(rs.getValue("runid/Name")), jobId); - toCreate.put("runid", runid); - } - else - { - _log.error("readset missing run id: " + remoteId); - } - - BatchValidationException bve = new BatchValidationException(); - List> created = readsetTable.getUpdateService().insertRows(user, target, Arrays.asList(toCreate), bve, null, null); - if (bve.hasErrors()) - { - throw new RuntimeException(bve); - } - - readsetMap.put(remoteId, Integer.parseInt(String.valueOf(created.get(0).get("rowid")))); - } - catch (Exception e) - { - throw new RuntimeException(e); - } - } - }); - } - catch (Exception e) - { - _log.error(e.getMessage(), e); - throw new RuntimeException(e); - } - } - - private int getOrCreateJob(int remoteJobId, Container targetWorkbook) - { - if (jobIdMap.containsKey(remoteJobId)) - { - return jobIdMap.get(remoteJobId); - } - - TableInfo ti = DbSchema.get("pipeline", DbSchemaType.Module).getTable("StatusFiles"); - - try - { - SelectRowsCommand sr = new SelectRowsCommand("pipeline", "job"); - sr.addFilter(new Filter("rowid", remoteJobId, Filter.Operator.EQUAL)); - sr.setColumns(Arrays.asList("RowId", "Info", "FilePath", "Email", "Description", "DataUrl", "Job", "Provider", "HadError", "ActiveTaskId")); - - SelectRowsResponse srr = sr.execute(getConnection(), remoteServerFolder); - - File fr = PipelineService.get().getPipelineRootSetting(targetWorkbook).getRootPath(); - - AtomicInteger ret = new AtomicInteger(); - srr.getRowset().forEach(pj -> { - String filepath = String.valueOf(pj.getValue("FilePath")); - if (!filepath.contains("@files")) - { - //This appears to be an error in PRIMe's data: - if (filepath.contains("illuminaImport")) - { - filepath = filepath.replace("illuminaImport", "@files/illuminaImport"); - } - else if (filepath.contains("sequenceAnalysis")) - { - filepath = filepath.replace("sequenceAnalysis", "@files/sequenceAnalysis"); - } - else - { - _log.error("Unexpected filepath: " + pj.getValue("FilePath")); - } - } - - File remoteDir = new File(URI.create(filepath.replaceAll(" ", "_")).getPath()); - File localDir = new File(fr, filepath.split("@files")[1]); - - //Check for existing row: - TableSelector ts = new TableSelector(ti, PageFlowUtil.set("RowId"), new SimpleFilter(FieldKey.fromString("Job"), pj.getValue("Job")), null); - if (ts.exists()) - { - ret.set(ts.getObject(Integer.class)); - } - else - { - Map toCreate = new CaseInsensitiveHashMap<>(); - toCreate.put("Info", pj.getValue("Info")); - toCreate.put("FilePath", localDir.getPath()); - toCreate.put("Email", pj.getValue("Email")); - toCreate.put("Description", pj.getValue("Description")); - toCreate.put("DataUrl", pj.getValue("DataUrl")); - toCreate.put("Job", pj.getValue("Job")); - toCreate.put("Provider", pj.getValue("Provider")); - toCreate.put("HadError", pj.getValue("HadError")); - toCreate.put("ActiveTaskId", pj.getValue("ActiveTaskId")); - toCreate.put("Container", targetWorkbook.getId()); - - toCreate = Table.insert(user, ti, toCreate); - - ret.set((int) toCreate.get("RowId")); - } - - if (localDir.exists()) - { - _log.info("Directory exists, will not re-copy: " + localDir.getPath()); - return; - } - - try - { - _log.info(remoteDir.getPath()); - _log.info(localDir.getPath()); - - if (!localDir.getParentFile().exists()) - { - localDir.getParentFile().mkdirs(); - } - - if (remoteDir.exists()) - { - FileUtils.copyDirectory(remoteDir, localDir); - } - else - { - _log.error("source folder not found: " + remoteDir.getPath()); - } - } - catch (Exception e) - { - throw new RuntimeException(e); - } - }); - - jobIdMap.put(remoteJobId, ret.get()); - - return ret.get(); - } - catch (Exception e) - { - _log.error(e.getMessage(), e); - throw new RuntimeException(e); - } - } - - private int createExpRun(int remoteId, Container c, String name, int localJobId) throws Exception - { - if (runIdMap.containsKey(remoteId)) - { - return runIdMap.get(remoteId); - } - else - { - ExpRun ret = ExperimentService.get().createRunForProvenanceRecording(c, user, new RecordedActionSet(), name, localJobId); - runIdMap.put(remoteId, ret.getRowId()); - - return ret.getRowId(); - } - } - - private void createWorkbooks() - { - _log.info("Creating workbooks"); - try - { - TableInfo containers = QueryService.get().getUserSchema(user, target, "core").getTable("containers"); - - SelectRowsCommand sr = new SelectRowsCommand("core", "workbooks"); - sr.setColumns(Arrays.asList("Name", "Title", "Description")); - SelectRowsResponse srr = sr.execute(getConnection(), remoteServerFolder); - - srr.getRowset().forEach(wb -> { - String localTitle = (String)wb.getValue("Title"); - - TableSelector ts = new TableSelector(containers, PageFlowUtil.set("RowId"), new SimpleFilter(FieldKey.fromString("Title"), localTitle), null); - if (ts.exists()) - { - Container workbook = ContainerManager.getForRowId(ts.getObject(Integer.class)); - workbookMap.put(Integer.parseInt(String.valueOf(wb.getValue("Name"))), workbook); - } - else - { - String description = String.valueOf(wb.getValue("Description")); - if (description != null) - { - description = description + ". "; - } - else - { - description = ""; - } - - description = description + "Originally PRIMe workbook: " + wb.getValue("Name"); - - Container workbook = ContainerManager.createContainer(target, null, localTitle, description, WorkbookContainerType.NAME, user); - workbookMap.put(Integer.parseInt(String.valueOf(wb.getValue("Name"))), workbook); - } - }); - } - catch (CommandException | IOException e) - { - throw new RuntimeException(e); - } - } - - private URI translateURI(String databaseURI, String remoteFolderRoot, String localFolderRoot) - { - databaseURI = databaseURI.replace("\\", "/"); - remoteFolderRoot = remoteFolderRoot.replace("\\", "/").split("@files")[0]; - localFolderRoot = localFolderRoot.replace("\\", "/").split("@files")[0]; - if (localFolderRoot.startsWith("C:")) - { - localFolderRoot = localFolderRoot.replaceAll("^C:", ""); - } - - databaseURI = databaseURI.replace(remoteFolderRoot, localFolderRoot); - - return URI.create(databaseURI); - } -} diff --git a/primeseq/src/org/labkey/primeseq/PrimeseqController.java b/primeseq/src/org/labkey/primeseq/PrimeseqController.java index f077e2267..66d1de408 100644 --- a/primeseq/src/org/labkey/primeseq/PrimeseqController.java +++ b/primeseq/src/org/labkey/primeseq/PrimeseqController.java @@ -30,6 +30,8 @@ import org.labkey.api.data.ContainerType; import org.labkey.api.module.Module; import org.labkey.api.module.ModuleLoader; +import org.labkey.api.pipeline.PipeRoot; +import org.labkey.api.pipeline.PipelineService; import org.labkey.api.pipeline.PipelineUrls; import org.labkey.api.security.RequiresPermission; import org.labkey.api.security.RequiresSiteAdmin; @@ -39,6 +41,7 @@ import org.labkey.api.util.URLHelper; import org.labkey.api.view.ActionURL; import org.labkey.api.view.HtmlView; +import org.labkey.primeseq.pipeline.MhcMigrationPipelineJob; import org.springframework.validation.BindException; import org.springframework.validation.Errors; import org.springframework.web.servlet.ModelAndView; @@ -219,8 +222,9 @@ public boolean handlePost(Object o, BindException errors) throws Exception { try { - MhcMigration mhc = new MhcMigration(getContainer(), getUser(), "PRIMe", "ONPRC/Core Facilities/Genetics Core/MHC_Typing/"); - mhc.doWork(); + PipeRoot pipelineRoot = PipelineService.get().findPipelineRoot(getContainer()); + MhcMigrationPipelineJob job = new MhcMigrationPipelineJob(getContainer(), getUser(), getViewContext().getActionURL(), pipelineRoot, "PRIMe", "ONPRC/Core Facilities/Genetics Core/MHC_Typing/"); + PipelineService.get().queueJob(job); } catch (Exception e) { diff --git a/primeseq/src/org/labkey/primeseq/PrimeseqModule.java b/primeseq/src/org/labkey/primeseq/PrimeseqModule.java index bc65a9262..bd36014d1 100644 --- a/primeseq/src/org/labkey/primeseq/PrimeseqModule.java +++ b/primeseq/src/org/labkey/primeseq/PrimeseqModule.java @@ -23,6 +23,7 @@ import org.labkey.api.data.Container; import org.labkey.api.ldk.ExtendedSimpleModule; import org.labkey.api.module.ModuleContext; +import org.labkey.api.pipeline.PipelineService; import org.labkey.api.sequenceanalysis.SequenceAnalysisService; import org.labkey.api.sequenceanalysis.pipeline.SequencePipelineService; import org.labkey.api.util.PageFlowUtil; @@ -36,6 +37,7 @@ import org.labkey.primeseq.pipeline.BlastPipelineJobResourceAllocator; import org.labkey.primeseq.pipeline.ClusterMaintenanceTask; import org.labkey.primeseq.pipeline.ExacloudResourceSettings; +import org.labkey.primeseq.pipeline.MhcMigrationPipelineJob; import org.labkey.primeseq.pipeline.SequenceJobResourceAllocator; import java.util.Collection; @@ -75,6 +77,8 @@ protected void doStartupAfterSpringConfig(ModuleContext moduleContext) ClusterService.get().registerResourceAllocator(new BlastPipelineJobResourceAllocator.Factory()); ClusterService.get().registerResourceAllocator(new SequenceJobResourceAllocator.Factory()); + PipelineService.get().registerPipelineProvider(new MhcMigrationPipelineJob.Provider(this)); + //register resources new PipelineStartup(); diff --git a/primeseq/src/org/labkey/primeseq/pipeline/MhcMigrationPipelineJob.java b/primeseq/src/org/labkey/primeseq/pipeline/MhcMigrationPipelineJob.java new file mode 100644 index 000000000..dfd668352 --- /dev/null +++ b/primeseq/src/org/labkey/primeseq/pipeline/MhcMigrationPipelineJob.java @@ -0,0 +1,1133 @@ +package org.labkey.primeseq.pipeline; + +import org.apache.commons.io.FileUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.labkey.api.collections.CaseInsensitiveHashMap; +import org.labkey.api.data.CompareType; +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerManager; +import org.labkey.api.data.DbSchema; +import org.labkey.api.data.DbSchemaType; +import org.labkey.api.data.DbScope; +import org.labkey.api.data.SimpleFilter; +import org.labkey.api.data.Sort; +import org.labkey.api.data.Table; +import org.labkey.api.data.TableInfo; +import org.labkey.api.data.TableSelector; +import org.labkey.api.data.WorkbookContainerType; +import org.labkey.api.di.DataIntegrationService; +import org.labkey.api.exp.api.DataType; +import org.labkey.api.exp.api.ExpData; +import org.labkey.api.exp.api.ExpRun; +import org.labkey.api.exp.api.ExperimentService; +import org.labkey.api.files.FileUrls; +import org.labkey.api.module.Module; +import org.labkey.api.pipeline.AbstractTaskFactory; +import org.labkey.api.pipeline.AbstractTaskFactorySettings; +import org.labkey.api.pipeline.PipeRoot; +import org.labkey.api.pipeline.PipelineDirectory; +import org.labkey.api.pipeline.PipelineJob; +import org.labkey.api.pipeline.PipelineJobException; +import org.labkey.api.pipeline.PipelineJobService; +import org.labkey.api.pipeline.PipelineProvider; +import org.labkey.api.pipeline.PipelineService; +import org.labkey.api.pipeline.PipelineStatusFile; +import org.labkey.api.pipeline.RecordedActionSet; +import org.labkey.api.pipeline.TaskId; +import org.labkey.api.pipeline.TaskPipeline; +import org.labkey.api.query.BatchValidationException; +import org.labkey.api.query.FieldKey; +import org.labkey.api.query.QueryService; +import org.labkey.api.query.UserSchema; +import org.labkey.api.security.User; +import org.labkey.api.sequenceanalysis.SequenceAnalysisService; +import org.labkey.api.sequenceanalysis.model.Readset; +import org.labkey.api.util.FileType; +import org.labkey.api.util.FileUtil; +import org.labkey.api.util.PageFlowUtil; +import org.labkey.api.view.ActionURL; +import org.labkey.api.view.ViewBackgroundInfo; +import org.labkey.api.view.ViewContext; +import org.labkey.remoteapi.CommandException; +import org.labkey.remoteapi.Connection; +import org.labkey.remoteapi.query.Filter; +import org.labkey.remoteapi.query.SelectRowsCommand; +import org.labkey.remoteapi.query.SelectRowsResponse; + +import java.io.File; +import java.io.IOException; +import java.net.URI; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; + +public class MhcMigrationPipelineJob extends PipelineJob +{ + private static final Logger _log = LogManager.getLogger(MhcMigrationPipelineJob.class); + + private String remoteServerFolder; + private String remoteConnectionName; + + private Container targetContainer; + + public static class Provider extends PipelineProvider + { + public static final String NAME = "mhcMigrationPipeline"; + + public Provider(Module owningModule) + { + super(NAME, owningModule); + } + + @Override + public void updateFileProperties(ViewContext context, PipeRoot pr, PipelineDirectory directory, boolean includeAll) + { + + } + } + + // Default constructor for serialization + protected MhcMigrationPipelineJob() + { + } + + public MhcMigrationPipelineJob(Container c, User u, ActionURL url, PipeRoot pipeRoot, String remoteConnectionName, String remoteServerFolder) + { + super(Provider.NAME, new ViewBackgroundInfo(c, u, url), pipeRoot); + + this.targetContainer = c; + this.remoteConnectionName = remoteConnectionName; + this.remoteServerFolder = remoteServerFolder; + + File subdir = new File(pipeRoot.getRootPath(), Provider.NAME); + if (!subdir.exists()) + { + subdir.mkdirs(); + } + + setLogFile(new File(subdir, FileUtil.makeFileNameWithTimestamp("mhcMigration", "log"))); + + } + + @Override + public ActionURL getStatusHref() + { + return PageFlowUtil.urlProvider(FileUrls.class).urlBegin(getContainer()); + } + + @Override + public String getDescription() + { + return "Find Orphan Sequence Files"; + } + + @Override + public TaskPipeline getTaskPipeline() + { + return PipelineJobService.get().getTaskPipeline(new TaskId(MhcMigrationPipelineJob.class)); + } + + public static class Task extends PipelineJob.Task + { + protected Task(Factory factory, PipelineJob job) + { + super(factory, job); + } + + public static class Factory extends AbstractTaskFactory + { + public Factory() + { + super(Task.class); + } + + @Override + public List getInputTypes() + { + return Collections.emptyList(); + } + + @Override + public String getStatusName() + { + return PipelineJob.TaskStatus.running.toString(); + } + + @Override + public List getProtocolActionNames() + { + return Arrays.asList("Migrate MHC Data"); + } + + @Override + public PipelineJob.Task createTask(PipelineJob job) + { + return new Task(this, job); + } + + @Override + public boolean isJobComplete(PipelineJob job) + { + return false; + } + } + + private MhcMigrationPipelineJob getPipelineJob() + { + return (MhcMigrationPipelineJob)getJob(); + } + + private Connection getConnection() + { + DataIntegrationService.RemoteConnection rc = DataIntegrationService.get().getRemoteConnection(getPipelineJob().remoteConnectionName, getPipelineJob().targetContainer, _log); + + return(rc.connection); + } + + @Override + public RecordedActionSet run() throws PipelineJobException + { + try (DbScope.Transaction transaction = DbScope.getLabKeyScope().ensureTransaction()) + { + createWorkbooks(); + + createLibraries(); + createLibraryMembers(); + + createReadsets(); + transaction.commitAndKeepConnection(); + + createReaddata(); + + createAnalyses(); + createOutputFiles(); + + //TODO: + //samples + //alignment_summary + //alignment_summary_junction + //quality_metrics + //subjects + //WaNPRC + + //sequenceanalysis.haplotypes + //sequenceanalysis.haplotype_types + //sequenceanalysis.haplotype_sequences + + //Create assay runs, including data and haplotypes + + transaction.commit(); + } + + return new RecordedActionSet(); + } + + private void replaceEntireTable(String schema, String query, List columns, String workbookColName, boolean truncateExisting) throws Exception + { + SelectRowsCommand sr = new SelectRowsCommand(schema, query); + sr.setColumns(columns); + SelectRowsResponse srr = sr.execute(getConnection(), getPipelineJob().remoteServerFolder); + + List> toInsert = new ArrayList<>(); + srr.getRowset().forEach(r -> { + Map row = new CaseInsensitiveHashMap<>(); + srr.getColumnModel().forEach(col -> { + String colName = (String) col.get("Name"); + Object val = r.getValue(colName); + if ("readset".equals(colName) || "readsetid".equals(colName)) + { + if (!readsetMap.containsKey((int) val)) + { + throw new IllegalStateException("Unable to find readset: " + val); + } + + val = readsetMap.get((int) val); + } + else if ("library_id".equals(colName)) + { + if (!libraryMap.containsKey((int) val)) + { + throw new IllegalStateException("Unable to find library: " + val); + } + + val = libraryMap.get((int) val); + + } + else if ("ref_nt_id".equals(colName)) + { + if (!sequenceMap.containsKey((int) val)) + { + throw new IllegalStateException("Unable to find sequence: " + val); + } + + val = sequenceMap.get((int) val); + } + else if ("analysis_id".equals(colName)) + { + if (!analysisMap.containsKey((int) val)) + { + throw new IllegalStateException("Unable to find analysis: " + val); + } + + val = analysisMap.get((int) val); + } + + row.put(colName, val); + }); + + if (workbookColName != null) + { + Object workbookId = r.getValue(workbookColName); + if (workbookId != null) + { + row.put("container", workbookMap.get(Integer.parseInt(String.valueOf(workbookId))).getId()); + } + } + + toInsert.add(row); + }); + + + } + + //All of these map remote Id to local Id + private final Map workbookMap = new HashMap<>(); + private final Map readsetMap = new HashMap<>(); + private final Map readdataMap = new HashMap<>(); + private final Map analysisMap = new HashMap<>(); + private final Map libraryMap = new HashMap<>(); + private final Map outputFileMap = new HashMap<>(); + private final Map sequenceMap = new HashMap<>(); + private final Map runIdMap = new HashMap<>(); + private final Map jobIdMap = new HashMap<>(); + + private void createLibraryMembers() + { + _log.info("Creating library members"); + + final UserSchema us = QueryService.get().getUserSchema(getJob().getUser(), getPipelineJob().targetContainer, "sequenceanalysis"); + final TableInfo ti = us.getTable("reference_library_members"); + final TableInfo refNtTable = us.getTable("ref_nt_sequences"); + + try + { + SelectRowsCommand sr = new SelectRowsCommand("sequenceanalysis", "reference_library_members"); + sr.setColumns(Arrays.asList("rowid", "library_id", "ref_nt_id", "ref_nt_id/name", "ref_nt_id/seqLength", "workbook/workbookId")); + + SelectRowsResponse srr = sr.execute(getConnection(), getPipelineJob().remoteServerFolder); + + srr.getRowset().forEach(rd -> { + int remoteId = Integer.parseInt(String.valueOf(rd.getValue("rowid"))); + int seqLength = Integer.parseInt(String.valueOf(rd.getValue("ref_nt_id/seqLength"))); + + int remoteSeqId = Integer.parseInt(String.valueOf(rd.getValue("ref_nt_id"))); + String name = String.valueOf(rd.getValue("ref_nt_id/name")); + int localSeqId = getOrCreateSequence(remoteSeqId, name, seqLength, refNtTable); + + int remoteLibraryId = Integer.parseInt(String.valueOf(rd.getValue("library_id"))); + Integer localLibraryId = libraryMap.get(remoteLibraryId); + if (localLibraryId == null) + { + throw new IllegalStateException("Unable to find library id: " + remoteLibraryId); + } + + SimpleFilter filter = new SimpleFilter(FieldKey.fromString("library_id"), localLibraryId); + filter.addCondition(FieldKey.fromString("ref_nt_id"), localSeqId); + + if (new TableSelector(ti, PageFlowUtil.set("rowid"), filter, null).exists()) + { + //Already exists: + return; + } + + Map toCreate = new CaseInsensitiveHashMap<>(); + toCreate.put("library_id", localLibraryId); + toCreate.put("ref_nt_id", localSeqId); + + try + { + BatchValidationException bve = new BatchValidationException(); + List> created = ti.getUpdateService().insertRows(getJob().getUser(), getPipelineJob().targetContainer, Arrays.asList(toCreate), bve, null, null); + if (bve.hasErrors()) + { + throw new RuntimeException(bve); + } + } + catch (Exception e) + { + _log.error(e.getMessage(), e); + throw new RuntimeException(e); + } + }); + } + catch (Exception e) + { + _log.error(e.getMessage(), e); + throw new RuntimeException(e); + } + } + + private int getOrCreateSequence(int remoteSeqId, String name, int seqLength, TableInfo refNtTable) + { + if (sequenceMap.containsKey(remoteSeqId)) + { + return sequenceMap.get(remoteSeqId); + } + else + { + SimpleFilter filter = new SimpleFilter(FieldKey.fromString("name"), name); + filter.addCondition(FieldKey.fromString("datedisabled"), null, CompareType.ISBLANK); + TableSelector ts = new TableSelector(refNtTable, PageFlowUtil.set("rowid", "seqLength"), filter, new Sort("rowid")); + if (ts.exists()) + { + if (ts.getRowCount() > 1) + { + _log.info("Duplicate ref name: " + name); + } + + AtomicInteger localId = new AtomicInteger(-1); + ts.forEachResults(rs -> { + if (rs.getInt(FieldKey.fromString("seqLength")) < seqLength) + { + _log.warn("length doesnt match for " + name + ", expected: " + seqLength); + return; + } + + localId.set(rs.getInt(FieldKey.fromString("rowid"))); + }); + + if (localId.get() != -1) + { + sequenceMap.put(remoteSeqId, localId.get()); + return localId.get(); + } + } + + //TODO: Create sequence? + //throw new IllegalStateException("Expected sequence to exist: " + name); + _log.error("Sequence missing: " + name); + return -1; + } + } + + public String getParent(String path) + { + final char separatorChar = '/'; + + int index = path.lastIndexOf(separatorChar); + + return path.substring(0, index); + } + + private void createLibraries() + { + _log.info("Creating libraries"); + try + { + final TableInfo libraryTable = QueryService.get().getUserSchema(getJob().getUser(), getPipelineJob().targetContainer, "sequenceanalysis").getTable("reference_libraries"); + + SelectRowsCommand sr = new SelectRowsCommand("sequenceanalysis", "reference_libraries"); + sr.setColumns(Arrays.asList("rowid", "name", "description", "fasta_file", "datedisabled", "assemblyId", "fasta_file/DataFileUrl", "workbook/workbookId")); + + SelectRowsResponse srr = sr.execute(getConnection(), getPipelineJob().remoteServerFolder); + + srr.getRowset().forEach(rd -> { + int remoteId = Integer.parseInt(String.valueOf(rd.getValue("rowid"))); + + Integer remoteWorkbook = rd.getValue("workbook/workbookId") == null ? null : Integer.parseInt(String.valueOf(rd.getValue("workbook/workbookId"))); + Container targetContainer = remoteWorkbook == null ? getPipelineJob().targetContainer : workbookMap.get(remoteWorkbook); + + SimpleFilter filter = new SimpleFilter(FieldKey.fromString("name"), rd.getValue("name")); + TableSelector ts = new TableSelector(libraryTable, PageFlowUtil.set("rowid"), filter, null); + if (ts.exists()) + { + libraryMap.put(remoteId, ts.getObject(Integer.class)); + } + else + { + Map toCreate = new CaseInsensitiveHashMap<>(); + toCreate.put("name", rd.getValue("name")); + toCreate.put("description", rd.getValue("description")); + toCreate.put("datedisabled", rd.getValue("datedisabled")); + toCreate.put("assemblyId", rd.getValue("assemblyId")); + try + { + String remoteJobRoot = getParent(URI.create(String.valueOf(rd.getValue("fasta_file/DatafileUrl"))).getPath()); + URI localJobRoot = PipelineService.get().getPipelineRootSetting(targetContainer).getRootPath().toURI(); + URI localFasta = translateURI(String.valueOf(rd.getValue("fasta_file/DatafileUrl")), remoteJobRoot, localJobRoot.getPath()); + toCreate.put("fasta_file", getOrCreateExpData(localFasta, targetContainer)); + + //Ensure parent folder exists: + File localJobRootFile = new File(localFasta).getParentFile(); + if (!localJobRootFile.getParentFile().exists()) + { + localJobRootFile.getParentFile().mkdirs(); + } + + _log.info(remoteJobRoot); + _log.info(localJobRoot.getPath()); + File remoteJobRootFile = new File(remoteJobRoot); + if (remoteJobRootFile.exists()) + { + FileUtils.copyDirectory(remoteJobRootFile, localJobRootFile); + } + + BatchValidationException bve = new BatchValidationException(); + List> created = libraryTable.getUpdateService().insertRows(getJob().getUser(), getPipelineJob().targetContainer, Arrays.asList(toCreate), bve, null, null); + if (bve.hasErrors()) + { + throw new RuntimeException(bve); + } + + libraryMap.put(remoteId, Integer.parseInt(String.valueOf(created.get(0).get("rowid")))); + } + catch (Exception e) + { + throw new RuntimeException(e); + } + } + }); + } + catch (Exception e) + { + _log.error(e.getMessage(), e); + throw new RuntimeException(e); + } + } + + private void createOutputFiles() + { + _log.info("Creating outputfiles"); + try + { + final TableInfo outputTable = QueryService.get().getUserSchema(getJob().getUser(), getPipelineJob().targetContainer, "sequenceanalysis").getTable("outputfiles"); + + SelectRowsCommand sr = new SelectRowsCommand("sequenceanalysis", "outputfiles"); + sr.setColumns(Arrays.asList("rowid", "name", "description", "dataid", "library_id", "readset", "analysis_id", "category", "sra_accession", "dataid/DataFileUrl", "runid/jobid", "runid/Name", "workbook/workbookId", "runid/JobId", "runid/Name", "runid/JobId/FilePath")); + + SelectRowsResponse srr = sr.execute(getConnection(), getPipelineJob().remoteServerFolder); + + srr.getRowset().forEach(rd -> { + int remoteId = Integer.parseInt(String.valueOf(rd.getValue("rowid"))); + int remoteReadset = Integer.parseInt(String.valueOf(rd.getValue("readset"))); + Integer localReadset = readsetMap.get(remoteReadset); + if (localReadset == null) + { + throw new IllegalArgumentException("Unable to find readset for remote id: " + remoteReadset); + } + + int remoteLibrary = Integer.parseInt(String.valueOf(rd.getValue("library_id"))); + Integer localLibrary = libraryMap.get(remoteLibrary); + if (localLibrary == null) + { + throw new IllegalArgumentException("Unable to find genome for remote id: " + remoteLibrary); + } + + int remoteAnalysis = Integer.parseInt(String.valueOf(rd.getValue("analysis_id"))); + Integer localAnalysis = analysisMap.get(remoteAnalysis); + if (localAnalysis == null) + { + throw new IllegalArgumentException("Unable to find analysis for remote id: " + remoteAnalysis); + } + + Readset rs = SequenceAnalysisService.get().getReadset(localReadset, getJob().getUser()); + Container targetWorkbook = ContainerManager.getForId(rs.getContainer()); + + SimpleFilter filter = new SimpleFilter(FieldKey.fromString("readset"), rs.getRowId()); + filter.addCondition(FieldKey.fromString("name"), rd.getValue("name")); + filter.addCondition(FieldKey.fromString("category"), rd.getValue("category")); + filter.addCondition(FieldKey.fromString("analysis_id"), localAnalysis); + filter.addCondition(FieldKey.fromString("container"), targetWorkbook.getId(), CompareType.EQUAL); + + TableSelector tsOutputFiles = new TableSelector(outputTable, PageFlowUtil.set("rowid"), filter, null); + if (tsOutputFiles.exists()) + { + outputFileMap.put(remoteId, tsOutputFiles.getObject(Integer.class)); + } + else + { + Map toCreate = new CaseInsensitiveHashMap<>(); + toCreate.put("readset", rs.getRowId()); + toCreate.put("analysis_id", localAnalysis); + toCreate.put("description", rd.getValue("description")); + toCreate.put("sra_accession", rd.getValue("sra_accession")); + toCreate.put("library_id", localLibrary); + toCreate.put("name", rd.getValue("name")); + toCreate.put("category", rd.getValue("category")); + + try + { + int remoteJobId = Integer.parseInt(String.valueOf(rd.getValue("runid/JobId"))); + int jobId = getOrCreateJob(remoteJobId, targetWorkbook); + PipelineStatusFile sf = PipelineService.get().getStatusFile(jobId); + + String localJobRoot = getParent(sf.getFilePath()); + String remoteJobRoot = getParent(URI.create(String.valueOf(rd.getValue("runid/JobId/FilePath")).replaceAll(" ", "_")).getPath()); + + URI newFileAlignment = translateURI(String.valueOf(rd.getValue("dataid/DatafileUrl")), remoteJobRoot, localJobRoot); + toCreate.put("dataid", getOrCreateExpData(newFileAlignment, targetWorkbook)); + + //Create run: + if (rd.getValue("runid") != null && rd.getValue("runid/JobId") != null) + { + int runId = createExpRun(Integer.parseInt(String.valueOf(rd.getValue("runid"))), targetWorkbook, String.valueOf(rd.getValue("runid/Name")), jobId); + toCreate.put("runid", runId); + } + else + { + _log.error("output missing runid: " + remoteId); + } + + BatchValidationException bve = new BatchValidationException(); + List> created = outputTable.getUpdateService().insertRows(getJob().getUser(), getPipelineJob().targetContainer, Arrays.asList(toCreate), bve, null, null); + if (bve.hasErrors()) + { + throw new RuntimeException(bve); + } + + outputFileMap.put(remoteId, Integer.parseInt(String.valueOf(created.get(0).get("rowid")))); + } + catch (Exception e) + { + throw new RuntimeException(e); + } + } + }); + } + catch (Exception e) + { + _log.error(e.getMessage(), e); + throw new RuntimeException(e); + } + } + + private void createAnalyses() + { + _log.info("Creating analyses"); + try + { + final TableInfo analysisTable = QueryService.get().getUserSchema(getJob().getUser(), getPipelineJob().targetContainer, "sequenceanalysis").getTable("sequence_analyses"); + + SelectRowsCommand sr = new SelectRowsCommand("sequenceanalysis", "sequence_analyses"); + sr.setColumns(Arrays.asList("rowid", "type", "description", "synopsis", "runid", "readset", "alignmentfile", "reference_library", "library_id", "sra_accession", "alignmentfile/DataFileUrl", "alignmentfile/Name", "reference_library", "reference_library/DataFileUrl", "runid/jobid", "runid/Name", "workbook/workbookId", "runid/JobId", "runid/Name", "runid/JobId/FilePath", "runid/JobId/Description")); + + SelectRowsResponse srr = sr.execute(getConnection(), getPipelineJob().remoteServerFolder); + + srr.getRowset().forEach(rd -> { + int remoteId = Integer.parseInt(String.valueOf(rd.getValue("rowid"))); + if (rd.getValue("readset") == null) + { + _log.warn("analysis lacks readset, skipping: " + remoteId); + return; + } + + int remoteReadset = Integer.parseInt(String.valueOf(rd.getValue("readset"))); + Integer localReadset = readsetMap.get(remoteReadset); + if (localReadset == null) + { + throw new IllegalArgumentException("Unable to find readset for remote id: " + remoteReadset); + } + + Integer localLibrary = null; + if (rd.getValue("library_id") != null) + { + int remoteLibrary = Integer.parseInt(String.valueOf(rd.getValue("library_id"))); + localLibrary = libraryMap.get(remoteLibrary); + if (localLibrary == null) + { + throw new IllegalArgumentException("Unable to find genome for remote id: " + remoteLibrary); + } + } + + Readset rs = SequenceAnalysisService.get().getReadset(localReadset, getJob().getUser()); + Container targetWorkbook = ContainerManager.getForId(rs.getContainer()); + + SimpleFilter filter = new SimpleFilter(FieldKey.fromString("readset"), rs.getRowId()); + filter.addCondition(FieldKey.fromString("runid/JobId/Description"), rd.getValue("runid/JobId/Description")); + filter.addCondition(FieldKey.fromString("container"), targetWorkbook.getId(), CompareType.EQUAL); + + TableSelector tsAnalyses = new TableSelector(analysisTable, PageFlowUtil.set("rowid"), filter, null); + if (tsAnalyses.exists()) + { + analysisMap.put(remoteId, tsAnalyses.getObject(Integer.class)); + } + else + { + Map toCreate = new CaseInsensitiveHashMap<>(); + + toCreate.put("readset", rs.getRowId()); + toCreate.put("synopsis", rd.getValue("synopsis")); + toCreate.put("centerName", rd.getValue("centerName")); + toCreate.put("type", rd.getValue("type")); + toCreate.put("description", rd.getValue("description")); + toCreate.put("sra_accession", rd.getValue("sra_accession")); + if (localLibrary != null) + { + toCreate.put("library_id", localLibrary); + } + + try + { + if (rd.getValue("runid/JobId") == null) + { + _log.info("skipping analysis without runid: " + remoteId); + return; + } + + int remoteJobId = Integer.parseInt(String.valueOf(rd.getValue("runid/JobId"))); + int jobId = getOrCreateJob(remoteJobId, targetWorkbook); + PipelineStatusFile sf = PipelineService.get().getStatusFile(jobId); + + String localJobRoot = getParent(sf.getFilePath()); + String remoteJobRoot = getParent(URI.create(String.valueOf(rd.getValue("runid/JobId/FilePath")).replaceAll(" ", "_")).getPath()); + + URI newFileAlignment = translateURI(String.valueOf(rd.getValue("alignmentfile/DatafileUrl")), remoteJobRoot, localJobRoot); + toCreate.put("alignmentfile", getOrCreateExpData(newFileAlignment, targetWorkbook)); + + if (rd.getValue("reference_library") != null) + { + URI newFile2 = translateURI(String.valueOf(rd.getValue("reference_library/DatafileUrl")), remoteJobRoot, localJobRoot); + toCreate.put("reference_library", getOrCreateExpData(newFile2, targetWorkbook)); + } + + //Create run: + if (rd.getValue("runid") != null && rd.getValue("runid/JobId") != null) + { + int runId = createExpRun(Integer.parseInt(String.valueOf(rd.getValue("runid"))), targetWorkbook, String.valueOf(rd.getValue("runid/Name")), jobId); + toCreate.put("runid", runId); + } + else + { + _log.error("analysis missing runid: " + remoteId); + } + + BatchValidationException bve = new BatchValidationException(); + List> created = analysisTable.getUpdateService().insertRows(getJob().getUser(), getPipelineJob().targetContainer, Arrays.asList(toCreate), bve, null, null); + if (bve.hasErrors()) + { + throw new RuntimeException(bve); + } + + analysisMap.put(remoteId, Integer.parseInt(String.valueOf(created.get(0).get("rowid")))); + } + catch (Exception e) + { + throw new RuntimeException(e); + } + } + }); + } + catch (Exception e) + { + _log.error(e.getMessage(), e); + throw new RuntimeException(e); + } + } + + private void createReaddata() + { + _log.info("Creating read data"); + try + { + final TableInfo readdataTable = QueryService.get().getUserSchema(getJob().getUser(), getPipelineJob().targetContainer, "sequenceanalysis").getTable("readdata"); + + SelectRowsCommand sr = new SelectRowsCommand("sequenceanalysis", "readdata"); + sr.setColumns(Arrays.asList("rowid", "readset", "platformUnit", "centerName", "date", "fileid1", "fileid1/DataFileUrl", "fileid2", "fileid2/DataFileUrl", "fileid1/Name", "description", "sra_accession", "runid", "runid/jobid", "runid/Name", "readset/workbook/workbookId", "runid/JobId", "runid/Name", "runid/JobId/FilePath", "runid/JobId/Description")); + + SelectRowsResponse srr = sr.execute(getConnection(), getPipelineJob().remoteServerFolder); + + srr.getRowset().forEach(rd -> { + int remoteId = Integer.parseInt(String.valueOf(rd.getValue("rowid"))); + int remoteReadset = Integer.parseInt(String.valueOf(rd.getValue("readset"))); + Integer localReadset = readsetMap.get(remoteReadset); + if (localReadset == null) + { + throw new IllegalArgumentException("Unable to find readset for remote id: " + remoteReadset); + } + + Readset rs = SequenceAnalysisService.get().getReadset(localReadset, getJob().getUser()); + Container targetWorkbook = ContainerManager.getForId(rs.getContainer()); + + SimpleFilter rdFilter = new SimpleFilter(FieldKey.fromString("readset"), rs.getRowId()); + rdFilter.addCondition(FieldKey.fromString("runid/JobId/Description"), rd.getValue("runid/JobId/Description")); + rdFilter.addCondition(FieldKey.fromString("fileid1/Name"), rd.getValue("fileid1/Name")); + rdFilter.addCondition(FieldKey.fromString("container"), targetWorkbook.getId(), CompareType.EQUAL); + + if (rd.getValue("platformUnit") != null) + { + rdFilter.addCondition(FieldKey.fromString("platformUnit"), rd.getValue("platformUnit")); + } + + TableSelector tsReaddata = new TableSelector(readdataTable, PageFlowUtil.set("rowid"), rdFilter, null); + if (tsReaddata.exists()) + { + readdataMap.put(remoteId, tsReaddata.getObject(Integer.class)); + } + else + { + Map toCreate = new CaseInsensitiveHashMap<>(); + toCreate.put("readset", rs.getRowId()); + toCreate.put("platformUnit", rd.getValue("platformUnit")); + toCreate.put("centerName", rd.getValue("centerName")); + toCreate.put("date", rd.getValue("date")); + toCreate.put("description", rd.getValue("description")); + toCreate.put("sra_accession", rd.getValue("sra_accession")); + try + { + if (rd.getValue("runid/JobId") != null) + { + int remoteJobId = Integer.parseInt(String.valueOf(rd.getValue("runid/JobId"))); + int jobId = getOrCreateJob(remoteJobId, targetWorkbook); + PipelineStatusFile sf = PipelineService.get().getStatusFile(jobId); + + String localJobRoot = getParent(sf.getFilePath()); + String remoteJobRoot = getParent(URI.create(String.valueOf(rd.getValue("runid/JobId/FilePath")).replaceAll(" ", "_")).getPath()); + + if (rd.getValue("fileid1/DataFileUrl") != null) + { + URI newFile1 = translateURI(String.valueOf(rd.getValue("fileid1/DataFileUrl")), remoteJobRoot, localJobRoot); + toCreate.put("fileid1", getOrCreateExpData(newFile1, targetWorkbook)); + } + + if (rd.getValue("fileid2/DataFileUrl") != null) + { + URI newFile2 = translateURI(String.valueOf(rd.getValue("fileid2/DatafileUrl")), remoteJobRoot, localJobRoot); + toCreate.put("fileid2", getOrCreateExpData(newFile2, targetWorkbook)); + } + } + else + { + _log.error("readddata missing jobid: " + remoteId); + } + + //Create run: + if (rd.getValue("runid") != null && rd.getValue("runid/JobId") != null) + { + int remoteJobId = Integer.parseInt(String.valueOf(rd.getValue("runid/JobId"))); + int jobId = getOrCreateJob(remoteJobId, targetWorkbook); + int runId = createExpRun(Integer.parseInt(String.valueOf(rd.getValue("runid"))), targetWorkbook, String.valueOf(rd.getValue("runid/Name")), jobId); + toCreate.put("runid", runId); + } + else + { + _log.error("readddata missing runid: " + remoteId); + } + + BatchValidationException bve = new BatchValidationException(); + List> created = readdataTable.getUpdateService().insertRows(getJob().getUser(), getPipelineJob().targetContainer, Arrays.asList(toCreate), bve, null, null); + if (bve.hasErrors()) + { + throw new RuntimeException(bve); + } + + readdataMap.put(remoteId, Integer.parseInt(String.valueOf(created.get(0).get("rowid")))); + } + catch (Exception e) + { + throw new RuntimeException(e); + } + } + }); + } + catch (Exception e) + { + _log.error(e.getMessage(), e); + throw new RuntimeException(e); + } + } + + private int getOrCreateExpData(URI file, Container workbook) + { + ExpData ret = ExperimentService.get().getExpDataByURL(new File(file), workbook); + if (ret == null) + { + ret = ExperimentService.get().createData(workbook, new DataType("Data")); + ret.setDataFileURI(file); + ret.save(getJob().getUser()); + } + + return ret.getRowId(); + } + + private void createReadsets() + { + _log.info("Creating readsets"); + try + { + final UserSchema us = QueryService.get().getUserSchema(getJob().getUser(), getPipelineJob().targetContainer, "sequenceanalysis"); + final TableInfo readsetTable = us.getTable("sequence_readsets"); + + SelectRowsCommand sr = new SelectRowsCommand("sequenceanalysis", "sequence_readsets"); + sr.setColumns(Arrays.asList("rowid", "name", "platform", "application", "librarytype", "chemistry", "comments", "status", "subjectid", "subjectdate", "sampletype", "sampleid", "barcode5", "barcode3", "runid", "runid/jobid", "runid/Name", "workbook/workbookId", "runid/JobId", "runid/Name", "runid/JobId/FilePath")); + + SelectRowsResponse srr = sr.execute(getConnection(), getPipelineJob().remoteServerFolder); + + srr.getRowset().forEach(rs -> { + int remoteId = Integer.parseInt(String.valueOf(rs.getValue("rowid"))); + int sourceWorkbook = Integer.parseInt(String.valueOf(rs.getValue("workbook/workbookId"))); + Container targetWorkbook = workbookMap.get(sourceWorkbook); + if (targetWorkbook == null) + { + throw new IllegalArgumentException("Unable to find local workbook for source: " + sourceWorkbook); + } + + SimpleFilter rsFilter = new SimpleFilter(FieldKey.fromString("name"), rs.getValue("name")); + rsFilter.addCondition(FieldKey.fromString("container"), targetWorkbook.getId(), CompareType.EQUAL); + if (rs.getValue("subjectid") != null) + { + rsFilter.addCondition(FieldKey.fromString("subjectid"), rs.getValue("subjectid"), CompareType.EQUAL); + } + + TableSelector tsReadset = new TableSelector(readsetTable, PageFlowUtil.set("rowid"), rsFilter, null); + if (tsReadset.exists()) + { + readsetMap.put(remoteId, tsReadset.getObject(Integer.class)); + } + else + { + Map toCreate = new CaseInsensitiveHashMap<>(); + toCreate.put("name", rs.getValue("name")); + toCreate.put("platform", rs.getValue("platform")); + toCreate.put("application", rs.getValue("application")); + toCreate.put("barcode5", rs.getValue("barcode5")); + toCreate.put("barcode3", rs.getValue("barcode3")); + toCreate.put("subjectid", rs.getValue("subjectid")); + + toCreate.put("sampleid", rs.getValue("sampleid")); + toCreate.put("sampledate", rs.getValue("sampledate")); + toCreate.put("librarytype", rs.getValue("librarytype")); + toCreate.put("sampletype", rs.getValue("sampletype")); + toCreate.put("chemistry", rs.getValue("chemistry")); + toCreate.put("comments", rs.getValue("comments")); + toCreate.put("status", rs.getValue("status")); + + toCreate.put("container", targetWorkbook.getId()); + + try + { + //Create run: + if (rs.getValue("runid") != null && rs.getValue("runid/JobId") != null) + { + int remoteJobId = Integer.parseInt(String.valueOf(rs.getValue("runid/JobId"))); + int jobId = getOrCreateJob(remoteJobId, targetWorkbook); + int runid = createExpRun(Integer.parseInt(String.valueOf(rs.getValue("runid"))), targetWorkbook, String.valueOf(rs.getValue("runid/Name")), jobId); + toCreate.put("runid", runid); + } + else + { + _log.error("readset missing run id: " + remoteId); + } + + BatchValidationException bve = new BatchValidationException(); + List> created = readsetTable.getUpdateService().insertRows(getJob().getUser(), getPipelineJob().targetContainer, Arrays.asList(toCreate), bve, null, null); + if (bve.hasErrors()) + { + throw new RuntimeException(bve); + } + + readsetMap.put(remoteId, Integer.parseInt(String.valueOf(created.get(0).get("rowid")))); + } + catch (Exception e) + { + throw new RuntimeException(e); + } + } + }); + } + catch (Exception e) + { + _log.error(e.getMessage(), e); + throw new RuntimeException(e); + } + } + + private int getOrCreateJob(int remoteJobId, Container targetWorkbook) + { + if (jobIdMap.containsKey(remoteJobId)) + { + return jobIdMap.get(remoteJobId); + } + + TableInfo ti = DbSchema.get("pipeline", DbSchemaType.Module).getTable("StatusFiles"); + + try + { + SelectRowsCommand sr = new SelectRowsCommand("pipeline", "job"); + sr.addFilter(new Filter("rowid", remoteJobId, Filter.Operator.EQUAL)); + sr.setColumns(Arrays.asList("RowId", "Info", "FilePath", "Email", "Description", "DataUrl", "Job", "Provider", "HadError", "ActiveTaskId")); + + SelectRowsResponse srr = sr.execute(getConnection(), getPipelineJob().remoteServerFolder); + + File fr = PipelineService.get().getPipelineRootSetting(targetWorkbook).getRootPath(); + + AtomicInteger ret = new AtomicInteger(); + srr.getRowset().forEach(pj -> { + String filepath = String.valueOf(pj.getValue("FilePath")); + if (!filepath.contains("@files")) + { + //This appears to be an error in PRIMe's data: + if (filepath.contains("illuminaImport")) + { + filepath = filepath.replace("illuminaImport", "@files/illuminaImport"); + } + else if (filepath.contains("sequenceAnalysis")) + { + filepath = filepath.replace("sequenceAnalysis", "@files/sequenceAnalysis"); + } + else + { + _log.error("Unexpected filepath: " + pj.getValue("FilePath")); + } + } + + File remoteDir = new File(URI.create(filepath.replaceAll(" ", "_")).getPath()); + File localDir = new File(fr, filepath.split("@files")[1]); + + //Check for existing row: + TableSelector ts = new TableSelector(ti, PageFlowUtil.set("RowId"), new SimpleFilter(FieldKey.fromString("Job"), pj.getValue("Job")), null); + if (ts.exists()) + { + ret.set(ts.getObject(Integer.class)); + } + else + { + Map toCreate = new CaseInsensitiveHashMap<>(); + toCreate.put("Info", pj.getValue("Info")); + toCreate.put("FilePath", localDir.getPath()); + toCreate.put("Email", pj.getValue("Email")); + toCreate.put("Description", pj.getValue("Description")); + toCreate.put("DataUrl", pj.getValue("DataUrl")); + toCreate.put("Job", pj.getValue("Job")); + toCreate.put("Provider", pj.getValue("Provider")); + toCreate.put("HadError", pj.getValue("HadError")); + toCreate.put("ActiveTaskId", pj.getValue("ActiveTaskId")); + toCreate.put("Container", targetWorkbook.getId()); + + toCreate = Table.insert(getJob().getUser(), ti, toCreate); + + ret.set((int) toCreate.get("RowId")); + } + + if (localDir.exists()) + { + _log.info("Directory exists, will not re-copy: " + localDir.getPath()); + return; + } + + try + { + _log.info(remoteDir.getPath()); + _log.info(localDir.getPath()); + + if (!localDir.getParentFile().exists()) + { + localDir.getParentFile().mkdirs(); + } + + if (remoteDir.exists()) + { + FileUtils.copyDirectory(remoteDir, localDir); + } + else + { + _log.error("source folder not found: " + remoteDir.getPath()); + } + } + catch (Exception e) + { + throw new RuntimeException(e); + } + }); + + jobIdMap.put(remoteJobId, ret.get()); + + return ret.get(); + } + catch (Exception e) + { + _log.error(e.getMessage(), e); + throw new RuntimeException(e); + } + } + + private int createExpRun(int remoteId, Container c, String name, int localJobId) throws Exception + { + if (runIdMap.containsKey(remoteId)) + { + return runIdMap.get(remoteId); + } + else + { + ExpRun ret = ExperimentService.get().createRunForProvenanceRecording(c, getJob().getUser(), new RecordedActionSet(), name, localJobId); + runIdMap.put(remoteId, ret.getRowId()); + + return ret.getRowId(); + } + } + + private void createWorkbooks() + { + _log.info("Creating workbooks"); + try + { + TableInfo containers = QueryService.get().getUserSchema(getJob().getUser(), getPipelineJob().targetContainer, "core").getTable("containers"); + + SelectRowsCommand sr = new SelectRowsCommand("core", "workbooks"); + sr.setColumns(Arrays.asList("Name", "Title", "Description")); + SelectRowsResponse srr = sr.execute(getConnection(), getPipelineJob().remoteServerFolder); + + srr.getRowset().forEach(wb -> { + String localTitle = (String) wb.getValue("Title"); + + TableSelector ts = new TableSelector(containers, PageFlowUtil.set("RowId"), new SimpleFilter(FieldKey.fromString("Title"), localTitle), null); + if (ts.exists()) + { + Container workbook = ContainerManager.getForRowId(ts.getObject(Integer.class)); + workbookMap.put(Integer.parseInt(String.valueOf(wb.getValue("Name"))), workbook); + } + else + { + String description = String.valueOf(wb.getValue("Description")); + if (description != null) + { + description = description + ". "; + } + else + { + description = ""; + } + + description = description + "Originally PRIMe workbook: " + wb.getValue("Name"); + + Container workbook = ContainerManager.createContainer(getPipelineJob().targetContainer, null, localTitle, description, WorkbookContainerType.NAME, getJob().getUser()); + workbookMap.put(Integer.parseInt(String.valueOf(wb.getValue("Name"))), workbook); + } + }); + } + catch (CommandException | IOException e) + { + throw new RuntimeException(e); + } + } + + private URI translateURI(String databaseURI, String remoteFolderRoot, String localFolderRoot) + { + databaseURI = databaseURI.replace("\\", "/"); + remoteFolderRoot = remoteFolderRoot.replace("\\", "/").split("@files")[0]; + localFolderRoot = localFolderRoot.replace("\\", "/").split("@files")[0]; + if (localFolderRoot.startsWith("C:")) + { + localFolderRoot = localFolderRoot.replaceAll("^C:", ""); + } + + databaseURI = databaseURI.replace(remoteFolderRoot, localFolderRoot); + + return URI.create(databaseURI); + } + } +} diff --git a/primeseq/webapp/WEB-INF/primeseqContext.xml b/primeseq/webapp/WEB-INF/primeseqContext.xml index 411bb99a8..b91d3dfb3 100644 --- a/primeseq/webapp/WEB-INF/primeseqContext.xml +++ b/primeseq/webapp/WEB-INF/primeseqContext.xml @@ -3,6 +3,28 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-2.0.xsd"> + + + + + + + + + + + + + + + org.labkey.primeseq.pipeline.MhcMigrationPipelineJob.Task + + + + + + + From 15819417e8ebdceaac6202c1199e686dcda7dbe0 Mon Sep 17 00:00:00 2001 From: bbimber Date: Fri, 29 Jan 2021 15:26:18 -0800 Subject: [PATCH 10/29] Store barcodes as list to enforce uniqueness --- .../labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java index a54aea590..d2fc05b73 100644 --- a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java +++ b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java @@ -200,7 +200,7 @@ private void processVloupeFile(JobContext ctx, File perCellTsv, Readset rs, Reco { AlignmentOutputImpl output = new AlignmentOutputImpl(); - List htosPerReadset = CellHashingService.get().getHtosForParentReadset(rs.getReadsetId(), ctx.getSourceDirectory(), ctx.getSequenceSupport()); + Set htosPerReadset = CellHashingService.get().getHtosForParentReadset(rs.getReadsetId(), ctx.getSourceDirectory(), ctx.getSequenceSupport()); if (htosPerReadset.size() > 1) { ctx.getLogger().info("Total HTOs for readset: " + htosPerReadset.size()); From ae386b5542a5fbdacd2600d53902263867fde35e Mon Sep 17 00:00:00 2001 From: bbimber Date: Sat, 30 Jan 2021 16:29:50 -0800 Subject: [PATCH 11/29] Include default cite-seq-count params for CiteSeq handler --- .../labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java index d2fc05b73..4e121453e 100644 --- a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java +++ b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java @@ -112,7 +112,7 @@ public class Processor implements SequenceOutputHandler.SequenceOutputProcessor public void init(JobContext ctx, List inputFiles, List actions, List outputsToCreate) throws UnsupportedOperationException, PipelineJobException { //NOTE: this is the pathway to import assay data, whether hashing is used or not - CellHashingService.get().prepareHashingAndCiteSeqFilesIfNeeded(ctx.getOutputDir(), ctx.getJob(), ctx.getSequenceSupport(), "tcrReadsetId", ctx.getParams().optBoolean("excludeFailedcDNA", true), false, false); + CellHashingService.get().prepareHashingAndCiteSeqFilesIfNeeded(ctx.getOutputDir(), ctx.getJob(), ctx.getSequenceSupport(), "tcrReadsetId", ctx.getParams().optBoolean("excludeFailedcDNA", false), false, false); if (ctx.getParams().optBoolean(USE_GEX_BARCODES, false)) { From 67f89f6877c153de4084a0bdee11b30edc9a9cfc Mon Sep 17 00:00:00 2001 From: bbimber Date: Mon, 1 Feb 2021 13:12:32 -0800 Subject: [PATCH 12/29] Add default trigger scripts --- .../queries/study/animalGroupMembership.js | 14 +++ mcc/resources/queries/study/deaths.js | 97 +++++++++++++++++++ mcc/resources/queries/study/demographics.js | 23 +++++ mcc/resources/queries/study/encounters.js | 13 +++ mcc/resources/queries/study/flags.js | 50 ++++++++++ mcc/resources/queries/study/labworkResults.js | 14 +++ mcc/resources/queries/study/parentage.js | 13 +++ mcc/resources/queries/study/samples.js | 7 ++ mcc/resources/queries/study/weight.js | 89 +++++++++++++++++ 9 files changed, 320 insertions(+) create mode 100644 mcc/resources/queries/study/animalGroupMembership.js create mode 100644 mcc/resources/queries/study/deaths.js create mode 100644 mcc/resources/queries/study/demographics.js create mode 100644 mcc/resources/queries/study/encounters.js create mode 100644 mcc/resources/queries/study/flags.js create mode 100644 mcc/resources/queries/study/labworkResults.js create mode 100644 mcc/resources/queries/study/parentage.js create mode 100644 mcc/resources/queries/study/samples.js create mode 100644 mcc/resources/queries/study/weight.js diff --git a/mcc/resources/queries/study/animalGroupMembership.js b/mcc/resources/queries/study/animalGroupMembership.js new file mode 100644 index 000000000..c8510ccb2 --- /dev/null +++ b/mcc/resources/queries/study/animalGroupMembership.js @@ -0,0 +1,14 @@ +/* + * Copyright (c) 2011-2014 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0 + */ + +require("ehr/triggers").initScript(this); + +function onInit(event, helper){ + helper.setScriptOptions({ + allowFutureDates: true, + removeTimeFromDate: true + }); +} \ No newline at end of file diff --git a/mcc/resources/queries/study/deaths.js b/mcc/resources/queries/study/deaths.js new file mode 100644 index 000000000..9dda9861c --- /dev/null +++ b/mcc/resources/queries/study/deaths.js @@ -0,0 +1,97 @@ +/* + * Copyright (c) 2018-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0 + */ + +require("ehr/triggers").initScript(this); +EHR.Server.Utils = require("ehr/utils").EHR.Server.Utils; + +var demographicsUpdates = []; +var validIds = []; + +function onInit(event, helper){ + helper.setScriptOptions({ + requiresStatusRecalc: true + }); + + helper.decodeExtraContextProperty('deathsInTransaction'); + + // Cache valid Ids for check on each row + LABKEY.Query.selectRows({ + requiredVersion: 9.1, + schemaName: 'study', + queryName: 'demographics', + columns: ['Id'], + scope: this, + success: function (results) { + if (!results || !results.rows || results.rows.length < 1) + return; + + for(var i=0; i 0) { + console.log('updating demographics death date for ' + demographicsUpdates.length + " animals"); + helper.getJavaHelper().updateDemographicsRecord(demographicsUpdates); + } + + var deaths = helper.getDeaths(); + if (deaths){ + var ids = []; + for (var id in deaths){ + ids.push(id); + } + + if (!helper.isETL()) { + console.log('sending death notification'); + helper.getJavaHelper().sendDeathNotification(ids); + } + } +} \ No newline at end of file diff --git a/mcc/resources/queries/study/demographics.js b/mcc/resources/queries/study/demographics.js new file mode 100644 index 000000000..992138d43 --- /dev/null +++ b/mcc/resources/queries/study/demographics.js @@ -0,0 +1,23 @@ +/* + * Copyright (c) 2010-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0 + */ + +require("ehr/triggers").initScript(this); + +function onInit(event, helper){ + helper.setScriptOptions({ + allowAnyId: true, + requiresStatusRecalc: false, + allowDatesInDistantPast: true + }); +} + +function onUpsert(helper, scriptErrors, row, oldRow){ + //NOTE: this should be getting set by the birth, death, arrival & departure tables + //ALSO: it should be rare to insert directly into this table. usually this record will be created by inserting into either birth or arrival + if (!row.calculated_status && !helper.isETL()){ + row.calculated_status = helper.getJavaHelper().getCalculatedStatusValue(row.Id); + } +} \ No newline at end of file diff --git a/mcc/resources/queries/study/encounters.js b/mcc/resources/queries/study/encounters.js new file mode 100644 index 000000000..fbb17ad72 --- /dev/null +++ b/mcc/resources/queries/study/encounters.js @@ -0,0 +1,13 @@ +/* + * Copyright (c) 2018-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0 + */ + +require("ehr/triggers").initScript(this); + +function onUpsert(helper, scriptErrors, row, oldRow){ + if (!helper.isETL() && row.date && !row.requestdate){ + row.requestdate = row.date; + } +} \ No newline at end of file diff --git a/mcc/resources/queries/study/flags.js b/mcc/resources/queries/study/flags.js new file mode 100644 index 000000000..454371e4c --- /dev/null +++ b/mcc/resources/queries/study/flags.js @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2012-2018 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0 + */ + +require("ehr/triggers").initScript(this); + +function onInit(event, helper){ + helper.setScriptOptions({ + allowFutureDates: true, + removeTimeFromDate: true, + removeTimeFromEndDate: true + }); +} + +function onUpsert(helper, scriptErrors, row, oldRow){ + //if the animal is not at the center, automatically set the enddate + if (!helper.isETL() && row.Id && !row.enddate){ + EHR.Server.Utils.findDemographics({ + participant: row.Id, + helper: helper, + scope: this, + callback: function(data){ + if (!data) + return; + + if (data && data.calculated_status && data.calculated_status != 'Alive'){ + row.enddate = data.death || data.departure; + } + } + }); + + } + + if (!helper.isETL() && row.Id && row.date && row.flag){ + var active = helper.getJavaHelper().getOverlappingFlags(row.Id, row.flag, row.objectid || null, row.date); + if (active > 0){ + EHR.Server.Utils.addError(scriptErrors, 'flag', 'There are already ' + active + ' active flag(s) of the same type spanning this date.', 'INFO'); + } + } +} + +function onAfterInsert(helper, errors, row){ + //if this category enforces only a single active flag at once, enforce it + //note: if this flag has a future date, preemptively set enddate on flags, since isActive should handle this + if (!helper.isETL() && row.Id && row.flag && !row.enddate && row.date){ + helper.getJavaHelper().ensureSingleFlagCategoryActive(row.Id, row.flag, row.objectId, row.date); + } +} \ No newline at end of file diff --git a/mcc/resources/queries/study/labworkResults.js b/mcc/resources/queries/study/labworkResults.js new file mode 100644 index 000000000..2004fb6cf --- /dev/null +++ b/mcc/resources/queries/study/labworkResults.js @@ -0,0 +1,14 @@ +/* + * Copyright (c) 2012-2018 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0 + */ + +require("ehr/triggers").initScript(this); + +function onInit(event, helper){ + helper.setScriptOptions({ + removeTimeFromDate: false, + allowDatesInDistantPast: true + }); +} \ No newline at end of file diff --git a/mcc/resources/queries/study/parentage.js b/mcc/resources/queries/study/parentage.js new file mode 100644 index 000000000..0cea7a1c5 --- /dev/null +++ b/mcc/resources/queries/study/parentage.js @@ -0,0 +1,13 @@ +/* + * Copyright (c) 2013 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0 + */ + +require("ehr/triggers").initScript(this); + +function onInit(event, helper){ + helper.setScriptOptions({ + lookupValidationFields: ['relationship', 'method'] + }); +} diff --git a/mcc/resources/queries/study/samples.js b/mcc/resources/queries/study/samples.js new file mode 100644 index 000000000..64a117a3e --- /dev/null +++ b/mcc/resources/queries/study/samples.js @@ -0,0 +1,7 @@ +/* + * Copyright (c) 2011-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0 + */ + +require("ehr/triggers").initScript(this); \ No newline at end of file diff --git a/mcc/resources/queries/study/weight.js b/mcc/resources/queries/study/weight.js new file mode 100644 index 000000000..a515311e1 --- /dev/null +++ b/mcc/resources/queries/study/weight.js @@ -0,0 +1,89 @@ +/* + * Copyright (c) 2010-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0 + */ + +require("ehr/triggers").initScript(this); + +function onInit(event, helper){ + helper.setScriptOptions({ + allowAnyId: true, + allowDeadIds: true, + skipIdFormatCheck: true + }); + + helper.registerRowProcessor(function(helper, row){ + if (!row) + return; + + if (!row.Id || !row.weight){ + return; + } + + var weightInTransaction = helper.getProperty('weightInTransaction'); + weightInTransaction = weightInTransaction || {}; + weightInTransaction[row.Id] = weightInTransaction[row.Id] || []; + + var shouldAdd = true; + if (row.objectid){ + LABKEY.ExtAdapter.each(weightInTransaction[row.Id], function(r){ + if (r.objectid === row.objectid){ + if (r.weight !== row.weight){ + r.weight = row.weight; + } + else { + shouldAdd = false; + return false; + } + } + }, this); + } + + if (shouldAdd){ + weightInTransaction[row.Id].push({ + objectid: row.objectid, + date: row.date, + qcstate: row.QCState, + weight: row.weight + }); + } + + helper.setProperty('weightInTransaction', weightInTransaction); + }); +} + +function onUpsert(helper, scriptErrors, row, oldRow){ + if (!row.weight){ + EHR.Server.Utils.addError(scriptErrors, 'weight', 'This field is required', 'WARN'); + } + + // warn if more than 10% different from last weight + // the highest error this can produce is WARN. therefore skip this check if we would ignore it anyway in order to save the overhead. + // this would normally occur when finalizing a form + if (!helper.isETL() && row.Id && row.weight && EHR.Server.Utils.shouldIncludeError('WARN', helper.getErrorThreshold(), helper)){ + EHR.Server.Utils.findDemographics({ + participant: row.Id, + helper: helper, + scope: this, + callback: function(data){ + if (!data) + return; + + if (data.mostRecentWeight && (row.weight <= data.mostRecentWeight * 0.9)){ + EHR.Server.Utils.addError(scriptErrors, 'weight', 'Weight drop of >10%. Last weight ' + data.mostRecentWeight + ' kg', 'INFO'); + } + else if (data.mostRecentWeight && (row.weight >= data.mostRecentWeight / 0.9)){ + EHR.Server.Utils.addError(scriptErrors, 'weight', 'Weight gain of >10%. Last weight ' + data.mostRecentWeight + ' kg', 'INFO'); + } + + if (data && data.species){ + var msg = helper.getJavaHelper().verifyWeightRange(row.id, row.weight, data.species); + if (msg != null){ + EHR.Server.Utils.addError(scriptErrors, 'weight', msg, 'WARN'); + } + } + } + }); + } +} \ No newline at end of file From 5a559830c7dc11cad60cd7b6c138221936e413fb Mon Sep 17 00:00:00 2001 From: bbimber Date: Thu, 4 Feb 2021 08:54:38 -0800 Subject: [PATCH 13/29] Dont enforce calling methods for cite-seq only --- .../labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java index 4e121453e..85ac41dea 100644 --- a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java +++ b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java @@ -65,7 +65,7 @@ private static List getDefaultParams() }}, false) )); - ret.addAll(CellHashingService.get().getDefaultHashingParams(true)); + ret.addAll(CellHashingService.get().getDefaultHashingParams(true, CellHashingService.BARCODE_TYPE.hashing)); return ret; } From c17cbfc6acf4341668b34b60de7a09b7676013aa Mon Sep 17 00:00:00 2001 From: bbimber Date: Thu, 4 Feb 2021 12:40:43 -0800 Subject: [PATCH 14/29] Update reference study --- .../datasets/datasets_manifest.xml | 10 +- .../datasets/datasets_metadata.xml | 482 +----------------- 2 files changed, 17 insertions(+), 475 deletions(-) diff --git a/mcc/resources/referenceStudy/datasets/datasets_manifest.xml b/mcc/resources/referenceStudy/datasets/datasets_manifest.xml index 3600f8261..1d43fa147 100644 --- a/mcc/resources/referenceStudy/datasets/datasets_manifest.xml +++ b/mcc/resources/referenceStudy/datasets/datasets_manifest.xml @@ -18,7 +18,10 @@ - + + + + @@ -30,7 +33,7 @@ - + @@ -45,9 +48,6 @@ - - - diff --git a/mcc/resources/referenceStudy/datasets/datasets_metadata.xml b/mcc/resources/referenceStudy/datasets/datasets_metadata.xml index 8f19b36f9..6a154bcc1 100644 --- a/mcc/resources/referenceStudy/datasets/datasets_metadata.xml +++ b/mcc/resources/referenceStudy/datasets/datasets_metadata.xml @@ -135,33 +135,12 @@ varchar - - varchar - - - varchar - varchar - - varchar - - - varchar - - - timestamp - - - integer - varchar - - timestamp - varchar urn:ehr.labkey.org/#ObjectId @@ -207,36 +186,6 @@ urn:ehr.labkey.org/#ObjectId true - - varchar - urn:ehr.labkey.org/#VetReview - - - timestamp - urn:ehr.labkey.org/#VetReviewDate - - - varchar - - - varchar - - - varchar - - - varchar - - - timestamp - urn:ehr.labkey.org/#DateRequested - - - varchar - - - varchar - integer @@ -264,6 +213,9 @@ integer urn:ehr.labkey.org/#Project + + varchar + varchar @@ -272,35 +224,6 @@ urn:ehr.labkey.org/#ObjectId true - - varchar - urn:ehr.labkey.org/#VetReview - - - timestamp - urn:ehr.labkey.org/#VetReviewDate - - - varchar - - - varchar - - - varchar - - - varchar - - - varchar - - - varchar - - - varchar - timestamp urn:ehr.labkey.org/#EndDate @@ -308,16 +231,10 @@ timestamp - - varchar - - - varchar - Clinical Remarks - +
varchar @@ -389,12 +306,6 @@ varchar - - timestamp - - - varchar - Medication Administration
@@ -503,7 +414,7 @@ Weight - +
varchar @@ -524,33 +435,15 @@ varchar - - varchar - integer urn:ehr.labkey.org/#Project - - varchar - entityid urn:ehr.labkey.org/#ObjectId true - - varchar - - - varchar - - - varchar - - - varchar - varchar @@ -566,16 +459,13 @@ varchar - - varchar - timestamp - Clinpath Runs + Labwork
- +
varchar @@ -630,132 +520,9 @@ varchar - Hematology Results -
- - - - varchar - http://cpas.labkey.com/Study#ParticipantId - - ptid - - - - timestamp - http://cpas.labkey.com/Study#VisitDate - http://cpas.labkey.com/Study#VisitDate - - - varchar - - - entityid - urn:ehr.labkey.org/#ObjectId - true - - - varchar - - - integer - urn:ehr.labkey.org/#Project - - - timestamp - urn:ehr.labkey.org/#EndDate - - - double - - - varchar - - - varchar - - - varchar - - - varchar - - - Parasitology Results -
- - - - varchar - http://cpas.labkey.com/Study#ParticipantId - - ptid - - - - timestamp - http://cpas.labkey.com/Study#VisitDate - http://cpas.labkey.com/Study#VisitDate - - - varchar - - - double - - - varchar - - - double - - - double - - - double - - - varchar - - - varchar - - - entityid - urn:ehr.labkey.org/#ObjectId - true - - - integer - urn:ehr.labkey.org/#Project - - - timestamp - urn:ehr.labkey.org/#EndDate - - - varchar - - - double - - - varchar - - - double - - - double - - - varchar - - - Urinalysis Results + Lab Results
- +
varchar @@ -791,45 +558,6 @@ boolean - - varchar - - - varchar - - - varchar - - - varchar - - - varchar - - - varchar - - - varchar - - - integer - - - integer - - - varchar - - - varchar - - - varchar - - - timestamp - Arrival
@@ -860,34 +588,10 @@ urn:ehr.labkey.org/#ObjectId true - - timestamp - - - integer - - - integer - - - integer - - - varchar - - - timestamp - - - timestamp - - - varchar - Assignment - +
varchar @@ -913,9 +617,6 @@ urn:ehr.labkey.org/#ObjectId true - - varchar - Animal Group Members
@@ -936,9 +637,6 @@ varchar - - varchar - varchar urn:ehr.labkey.org/#ObjectId @@ -951,21 +649,6 @@ timestamp urn:ehr.labkey.org/#EndDate - - varchar - - - varchar - - - varchar - - - varchar - - - boolean - Deaths @@ -1013,151 +696,10 @@ varchar - - Demographics - - - - - varchar - http://cpas.labkey.com/Study#ParticipantId - - ptid - - - - timestamp - http://cpas.labkey.com/Study#VisitDate - http://cpas.labkey.com/Study#VisitDate - - - varchar - - - varchar - - - entityid - urn:ehr.labkey.org/#ObjectId - true - - - integer - urn:ehr.labkey.org/#Project - - - timestamp - urn:ehr.labkey.org/#EndDate - - - Departure -
- - - - varchar - http://cpas.labkey.com/Study#ParticipantId - - ptid - - - - timestamp - http://cpas.labkey.com/Study#VisitDate - http://cpas.labkey.com/Study#VisitDate - - - timestamp - urn:ehr.labkey.org/#EndDate - - - varchar - - + varchar - - integer - - - integer - - - integer - - - entityid - urn:ehr.labkey.org/#ObjectId - true - - - varchar - - - integer - urn:ehr.labkey.org/#Project - - - Housing -
- - - - varchar - http://cpas.labkey.com/Study#ParticipantId - - ptid - - - - timestamp - http://cpas.labkey.com/Study#VisitDate - http://cpas.labkey.com/Study#VisitDate - - - varchar - - - varchar - - - entityid - urn:ehr.labkey.org/#ObjectId - true - - - integer - urn:ehr.labkey.org/#Project - - - timestamp - urn:ehr.labkey.org/#EndDate - - - - - - - - - varchar - - - varchar - - - varchar - - - - - - double - - - boolean - - Tissue Samples + Demographics
From b18d2a1e9614a5518ea5719f33a5709218dba8e2 Mon Sep 17 00:00:00 2001 From: bbimber Date: Thu, 4 Feb 2021 14:28:22 -0800 Subject: [PATCH 15/29] Add stubs of various files for MCC --- mGAP/resources/views/contact.html | 2 +- mcc/resources/module.xml | 21 + .../queries/wnprcSource/demographics.sql | 6 + .../postgresql/mcc-20.000-20.001.sql | 18 + .../dbscripts/sqlserver/mcc-20.000-20.001.sql | 18 + mcc/resources/schemas/mcc.xml | 90 +++- mcc/resources/views/about.html | 3 + mcc/resources/views/about.view.xml | 8 + mcc/resources/views/about.webpart.xml | 6 + mcc/resources/views/contact.html | 75 +++ mcc/resources/views/contact.view.xml | 8 + mcc/resources/views/helpMenu.html | 1 + mcc/resources/views/helpMenu.view.xml | 8 + mcc/resources/views/helpMenu.webpart.xml | 6 + mcc/resources/views/login.html | 40 ++ mcc/resources/views/login.view.xml | 9 + mcc/resources/views/overview.html | 1 + mcc/resources/views/overview.view.xml | 10 + mcc/resources/views/overview.webpart.xml | 6 + mcc/resources/views/requestLogin.html | 117 +++++ mcc/resources/views/requestLogin.view.xml | 8 + mcc/resources/web/mcc/Security.js | 49 ++ mcc/src/org/labkey/mcc/MccController.java | 494 +++++++++++++++++- mcc/src/org/labkey/mcc/MccManager.java | 70 +++ mcc/src/org/labkey/mcc/MccModule.java | 2 +- mcc/src/org/labkey/mcc/MccSchema.java | 2 + .../mcc/query/UserRequestCustomizer.java | 44 ++ 27 files changed, 1115 insertions(+), 7 deletions(-) create mode 100644 mcc/resources/module.xml create mode 100644 mcc/resources/queries/wnprcSource/demographics.sql create mode 100644 mcc/resources/schemas/dbscripts/postgresql/mcc-20.000-20.001.sql create mode 100644 mcc/resources/schemas/dbscripts/sqlserver/mcc-20.000-20.001.sql create mode 100644 mcc/resources/views/about.html create mode 100644 mcc/resources/views/about.view.xml create mode 100644 mcc/resources/views/about.webpart.xml create mode 100644 mcc/resources/views/contact.html create mode 100644 mcc/resources/views/contact.view.xml create mode 100644 mcc/resources/views/helpMenu.html create mode 100644 mcc/resources/views/helpMenu.view.xml create mode 100644 mcc/resources/views/helpMenu.webpart.xml create mode 100644 mcc/resources/views/login.html create mode 100644 mcc/resources/views/login.view.xml create mode 100644 mcc/resources/views/overview.html create mode 100644 mcc/resources/views/overview.view.xml create mode 100644 mcc/resources/views/overview.webpart.xml create mode 100644 mcc/resources/views/requestLogin.html create mode 100644 mcc/resources/views/requestLogin.view.xml create mode 100644 mcc/resources/web/mcc/Security.js create mode 100644 mcc/src/org/labkey/mcc/query/UserRequestCustomizer.java diff --git a/mGAP/resources/views/contact.html b/mGAP/resources/views/contact.html index 1b110d8cc..e1f2f6b53 100644 --- a/mGAP/resources/views/contact.html +++ b/mGAP/resources/views/contact.html @@ -49,7 +49,7 @@ success: function(response){ console.log(response); - Ext4.Msg.alert('Success', 'An account has been requested. You should receive a reply shortly.', function(){ + Ext4.Msg.alert('Success', 'Your request has been sent. You should receive a reply shortly.', function(){ window.location = LABKEY.ActionURL.getContextPath() + '/'; }); }, diff --git a/mcc/resources/module.xml b/mcc/resources/module.xml new file mode 100644 index 000000000..b07899cb8 --- /dev/null +++ b/mcc/resources/module.xml @@ -0,0 +1,21 @@ + + + + false + This is the path to the container holding the primary MCC Study. Use of slashes is very important - it should be in the format '/myProject/mcc' + + ADMIN + + + + false + This is a comma separated list of LabKey user names of users that should be notified by email when requests are submitted through MCC. + + ADMIN + + + + + + + diff --git a/mcc/resources/queries/wnprcSource/demographics.sql b/mcc/resources/queries/wnprcSource/demographics.sql new file mode 100644 index 000000000..c63412b14 --- /dev/null +++ b/mcc/resources/queries/wnprcSource/demographics.sql @@ -0,0 +1,6 @@ +SELECT + +Id, date, gender, geographic_origin, birth, death, species, objectid + +FROM "/WNPRC/EHR/".study.demographics +WHERE species = 'Marmoset'; \ No newline at end of file diff --git a/mcc/resources/schemas/dbscripts/postgresql/mcc-20.000-20.001.sql b/mcc/resources/schemas/dbscripts/postgresql/mcc-20.000-20.001.sql new file mode 100644 index 000000000..567bfef27 --- /dev/null +++ b/mcc/resources/schemas/dbscripts/postgresql/mcc-20.000-20.001.sql @@ -0,0 +1,18 @@ +CREATE TABLE mcc.userRequests ( + rowid serial, + email varchar(1000), + firstName varchar(1000), + lastName varchar(1000), + title varchar(1000), + institution varchar(1000), + reason varchar(4000), + userid userid, + + container entityid, + created timestamp, + createdby userid, + modified timestamp, + modifiedby userid, + + CONSTRAINT PK_userRequests PRIMARY KEY (rowid) +); \ No newline at end of file diff --git a/mcc/resources/schemas/dbscripts/sqlserver/mcc-20.000-20.001.sql b/mcc/resources/schemas/dbscripts/sqlserver/mcc-20.000-20.001.sql new file mode 100644 index 000000000..de27af5ef --- /dev/null +++ b/mcc/resources/schemas/dbscripts/sqlserver/mcc-20.000-20.001.sql @@ -0,0 +1,18 @@ +CREATE TABLE mcc.userRequests ( + rowid int identity(1,1), + email varchar(1000), + firstName varchar(1000), + lastName varchar(1000), + title varchar(1000), + institution varchar(1000), + reason varchar(4000), + userid userid, + + container entityid, + created datetime, + createdby userid, + modified datetime, + modifiedby userid, + + CONSTRAINT PK_userRequests PRIMARY KEY (rowid) +); \ No newline at end of file diff --git a/mcc/resources/schemas/mcc.xml b/mcc/resources/schemas/mcc.xml index 2bba6c71d..e69524513 100644 --- a/mcc/resources/schemas/mcc.xml +++ b/mcc/resources/schemas/mcc.xml @@ -17,4 +17,92 @@ --> \ No newline at end of file + xmlns="http://labkey.org/data/xml" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"> + + + + + + + + rowid + Requests For Logins + DETAILED + + + true + false + false + false + false + Request Id + + + Email + false + + + First Name + false + + + Last Name + false + + + Title + false + + + Institution + false + + + Reason For Request + false + + + false + + core + Users + UserId + + + + true + + + true + + + false + false + false + true + true + + + true + + + false + false + false + true + true + + + + ldk.context + /mcc/Security.js + + MCC.Security.approveUserRequests(dataRegionName); + + + +
+ + \ No newline at end of file diff --git a/mcc/resources/views/about.html b/mcc/resources/views/about.html new file mode 100644 index 000000000..436f532b2 --- /dev/null +++ b/mcc/resources/views/about.html @@ -0,0 +1,3 @@ +MCC is supported by NIH U24 xxxxxxx. +

+Please remember to cite this funding source in all publications that make use of MCC data. \ No newline at end of file diff --git a/mcc/resources/views/about.view.xml b/mcc/resources/views/about.view.xml new file mode 100644 index 000000000..48ca9ced8 --- /dev/null +++ b/mcc/resources/views/about.view.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/mcc/resources/views/about.webpart.xml b/mcc/resources/views/about.webpart.xml new file mode 100644 index 000000000..f40c189c7 --- /dev/null +++ b/mcc/resources/views/about.webpart.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/mcc/resources/views/contact.html b/mcc/resources/views/contact.html new file mode 100644 index 000000000..714f95170 --- /dev/null +++ b/mcc/resources/views/contact.html @@ -0,0 +1,75 @@ + \ No newline at end of file diff --git a/mcc/resources/views/contact.view.xml b/mcc/resources/views/contact.view.xml new file mode 100644 index 000000000..0fc3bfa31 --- /dev/null +++ b/mcc/resources/views/contact.view.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/mcc/resources/views/helpMenu.html b/mcc/resources/views/helpMenu.html new file mode 100644 index 000000000..932f848cc --- /dev/null +++ b/mcc/resources/views/helpMenu.html @@ -0,0 +1 @@ +We'd love to hear your feedback! Click here to send a help or feature request, or email mcc@ohsu.edu for any questions. \ No newline at end of file diff --git a/mcc/resources/views/helpMenu.view.xml b/mcc/resources/views/helpMenu.view.xml new file mode 100644 index 000000000..3960a69c1 --- /dev/null +++ b/mcc/resources/views/helpMenu.view.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/mcc/resources/views/helpMenu.webpart.xml b/mcc/resources/views/helpMenu.webpart.xml new file mode 100644 index 000000000..3f4624d6a --- /dev/null +++ b/mcc/resources/views/helpMenu.webpart.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/mcc/resources/views/login.html b/mcc/resources/views/login.html new file mode 100644 index 000000000..71bcbf7d3 --- /dev/null +++ b/mcc/resources/views/login.html @@ -0,0 +1,40 @@ + +
+
Sign In
+
+
+ + + + + Remember my email address + + +
+ + + + or + Request an account + +
+ + + +
+
+ \ No newline at end of file diff --git a/mcc/resources/views/login.view.xml b/mcc/resources/views/login.view.xml new file mode 100644 index 000000000..7ce47abe2 --- /dev/null +++ b/mcc/resources/views/login.view.xml @@ -0,0 +1,9 @@ + + + + + + + + + \ No newline at end of file diff --git a/mcc/resources/views/overview.html b/mcc/resources/views/overview.html new file mode 100644 index 000000000..f03b886bc --- /dev/null +++ b/mcc/resources/views/overview.html @@ -0,0 +1 @@ +This will hold the overview information for MCC. \ No newline at end of file diff --git a/mcc/resources/views/overview.view.xml b/mcc/resources/views/overview.view.xml new file mode 100644 index 000000000..f9e80a68a --- /dev/null +++ b/mcc/resources/views/overview.view.xml @@ -0,0 +1,10 @@ + + + + + + + + + + \ No newline at end of file diff --git a/mcc/resources/views/overview.webpart.xml b/mcc/resources/views/overview.webpart.xml new file mode 100644 index 000000000..c81f56bb6 --- /dev/null +++ b/mcc/resources/views/overview.webpart.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/mcc/resources/views/requestLogin.html b/mcc/resources/views/requestLogin.html new file mode 100644 index 000000000..6c3b1fb7f --- /dev/null +++ b/mcc/resources/views/requestLogin.html @@ -0,0 +1,117 @@ + \ No newline at end of file diff --git a/mcc/resources/views/requestLogin.view.xml b/mcc/resources/views/requestLogin.view.xml new file mode 100644 index 000000000..13c485f9f --- /dev/null +++ b/mcc/resources/views/requestLogin.view.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/mcc/resources/web/mcc/Security.js b/mcc/resources/web/mcc/Security.js new file mode 100644 index 000000000..5ab385833 --- /dev/null +++ b/mcc/resources/web/mcc/Security.js @@ -0,0 +1,49 @@ +Ext4.namespace('MCC.Security'); + +MCC.Security = new function(){ + return { + approveUserRequests: function(dataRegionName){ + var dr = LABKEY.DataRegions[dataRegionName]; + if (!dr){ + alert('Unable to find DataRegion with name: ' + dataRegionName); + return; + } + + var rowIds = dr.getChecked(); + if (!rowIds.length){ + alert('Must select one or more rows'); + return; + } + + Ext4.Msg.confirm('Approve Requests', 'You are able to approve ' + rowIds.length + ' user requests. Continue?', function(val){ + if (val === 'yes'){ + Ext4.Msg.wait('Loading...'); + LABKEY.Ajax.request({ + method: 'POST', + url: LABKEY.ActionURL.buildURL('mcc', 'approveUserRequests'), + params: { + requestIds: rowIds + }, + success: function(){ + Ext4.Msg.hide(); + Ext4.Msg.alert('Success', 'Requests approved!', function(){ + //note: drop view, so we see the newly added user(s) + LABKEY.DataRegions[dataRegionName].changeView(null); + }); + }, + failure: LDK.Utils.getErrorCallback({ + showAlertOnError: false, + scope: this, + callback: function(responseObj){ + if (responseObj.errorMsg){ + Ext4.Msg.alert('Error', responseObj.errorMsg); + } + } + }) + }); + + } + }, this); + } + } +}; \ No newline at end of file diff --git a/mcc/src/org/labkey/mcc/MccController.java b/mcc/src/org/labkey/mcc/MccController.java index 0967188da..ea84cef04 100644 --- a/mcc/src/org/labkey/mcc/MccController.java +++ b/mcc/src/org/labkey/mcc/MccController.java @@ -16,22 +16,508 @@ package org.labkey.mcc; -import org.labkey.api.action.SimpleViewAction; +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.labkey.api.action.ApiSimpleResponse; +import org.labkey.api.action.MutatingApiAction; import org.labkey.api.action.SpringActionController; +import org.labkey.api.data.Container; +import org.labkey.api.data.CoreSchema; +import org.labkey.api.data.DbScope; +import org.labkey.api.data.SimpleFilter; +import org.labkey.api.data.Table; +import org.labkey.api.data.TableInfo; +import org.labkey.api.data.TableSelector; +import org.labkey.api.module.AllowedDuringUpgrade; +import org.labkey.api.query.DetailsURL; +import org.labkey.api.query.FieldKey; +import org.labkey.api.security.IgnoresTermsOfUse; +import org.labkey.api.security.MutableSecurityPolicy; +import org.labkey.api.security.RequiresNoPermission; import org.labkey.api.security.RequiresPermission; +import org.labkey.api.security.SecurityManager; +import org.labkey.api.security.SecurityPolicyManager; +import org.labkey.api.security.User; +import org.labkey.api.security.UserManager; +import org.labkey.api.security.ValidEmail; +import org.labkey.api.security.permissions.AdminPermission; import org.labkey.api.security.permissions.ReadPermission; -import org.labkey.api.view.JspView; -import org.labkey.api.view.NavTree; +import org.labkey.api.security.roles.ReaderRole; +import org.labkey.api.settings.AppProps; +import org.labkey.api.settings.LookAndFeelProperties; +import org.labkey.api.util.ConfigurationException; +import org.labkey.api.util.ExceptionUtil; +import org.labkey.api.util.MailHelper; +import org.labkey.api.util.PageFlowUtil; import org.springframework.validation.BindException; -import org.springframework.web.servlet.ModelAndView; +import org.springframework.validation.Errors; + +import javax.mail.Address; +import javax.mail.Message; +import javax.mail.internet.InternetAddress; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; public class MccController extends SpringActionController { private static final DefaultActionResolver _actionResolver = new DefaultActionResolver(MccController.class); public static final String NAME = "mcc"; + private static final Logger _log = LogManager.getLogger(MccController.class); + public MccController() { setActionResolver(_actionResolver); } + + @RequiresNoPermission + @IgnoresTermsOfUse + @AllowedDuringUpgrade + public class RequestUserAction extends MutatingApiAction + { + @Override + public void validateForm(RequestUserForm form, Errors errors) + { + Container mccContainer = MccManager.get().getMCCContainer(); + if (mccContainer == null) + { + errors.reject(ERROR_MSG, "The MCC project has not been set on this server. This is an administrator error."); + return; + } + + if (StringUtils.isEmpty(form.getEmail()) || StringUtils.isEmpty(form.getEmailConfirmation())) + { + errors.reject(ERROR_REQUIRED, "No email address provided"); + } + else if (StringUtils.isEmpty(form.getFirstName()) || StringUtils.isEmpty(form.getLastName()) || StringUtils.isEmpty(form.getTitle()) || StringUtils.isEmpty(form.getInstitution()) || StringUtils.isEmpty(form.getReason())) + { + errors.reject(ERROR_REQUIRED, "You must provide your first and last name, title, institution, and reason for requesting access"); + } + else + { + try + { + ValidEmail email = new ValidEmail(form.getEmail()); + if (!form.getEmail().equals(form.getEmailConfirmation())) + { + errors.reject(ERROR_MSG, "The email addresses you have entered do not match. Please verify your email addresses below."); + } + + TableInfo ti = MccSchema.getInstance().getSchema().getTable(MccSchema.TABLE_USER_REQUESTS); + + //first check if this email exists: + SimpleFilter filter = new SimpleFilter(FieldKey.fromString("email"), form.getEmail()); + filter.addCondition(FieldKey.fromString("container"), mccContainer.getId()); + if (new TableSelector(ti, filter, null).exists()) + { + errors.reject(ERROR_MSG, "A login has already been requested for this email. You should receive a reply shortly from the site administrator."); + } + } + catch (ValidEmail.InvalidEmailException e) + { + errors.reject(ERROR_MSG, "Your email address is not valid. Please verify your email address below."); + } + } + } + + @Override + public Object execute(RequestUserForm form, BindException errors) throws Exception + { + ApiSimpleResponse response = new ApiSimpleResponse(); + + try + { + TableInfo ti = MccSchema.getInstance().getSchema().getTable(MccSchema.TABLE_USER_REQUESTS); + Map row = new HashMap<>(); + row.put("email", form.getEmail()); + row.put("firstName", form.getFirstName()); + row.put("lastName", form.getLastName()); + row.put("title", form.getTitle()); + row.put("institution", form.getInstitution()); + row.put("reason", form.getReason()); + row.put("container", MccManager.get().getMCCContainer().getId()); + + Table.insert(UserManager.getGuestUser(), ti, row); + + Set users = MccManager.get().getNotificationUsers(); + if (users != null && !users.isEmpty()) + { + try + { + Set
emails = new HashSet<>(); + for (User u : users) + { + emails.add(new InternetAddress(u.getEmail())); + } + + MailHelper.MultipartMessage mail = MailHelper.createMultipartMessage(); + Container c = MccManager.get().getMCCContainer(); + if (c == null) + { + c = getContainer(); + _log.warn("MCC container was not set, using: " + c.getPath()); + } + + DetailsURL url = DetailsURL.fromString("/query/executeQuery.view?schemaName=mcc&query.queryName=userRequests&query.viewName=Pending Requests", c); + mail.setEncodedHtmlContent("A user requested an account on MCC. Click here to view/approve this request"); + mail.setFrom(getReplyEmail(getContainer())); + mail.setSubject("MCC Account Request"); + mail.addRecipients(Message.RecipientType.TO, emails.toArray(new Address[0])); + + MailHelper.send(mail, getUser(), c); + } + catch (Exception e) + { + ExceptionUtil.logExceptionToMothership(null, e); + } + } + + + } + catch (ConfigurationException e) + { + errors.reject(ERROR_MSG, "There was a problem sending the registration email. Please contact your administrator."); + _log.error("Error adding self registered user", e); + } + + response.put("success", !errors.hasErrors()); + if (!errors.hasErrors()) + response.put("email", form.getEmail()); + + return response; + } + } + + public static class RequestUserForm extends Object + { + private String email; + private String emailConfirmation; + private String firstName; + private String lastName; + private String title; + private String institution; + private String reason; + + public void setEmail(String email) + { + this.email = email; + } + + public String getEmail() + { + return this.email; + } + + public void setEmailConfirmation(String email) + { + this.emailConfirmation = email; + } + + public String getEmailConfirmation() + { + return this.emailConfirmation; + } + + public String getFirstName() + { + return firstName; + } + + public void setFirstName(String firstName) + { + this.firstName = firstName; + } + + public String getLastName() + { + return lastName; + } + + public void setLastName(String lastName) + { + this.lastName = lastName; + } + + public String getTitle() + { + return title; + } + + public void setTitle(String title) + { + this.title = title; + } + + public String getInstitution() + { + return institution; + } + + public void setInstitution(String institution) + { + this.institution = institution; + } + + public String getReason() + { + return reason; + } + + public void setReason(String reason) + { + this.reason = reason; + } + } + + @RequiresPermission(AdminPermission.class) + public class ApproveUserRequestsAction extends MutatingApiAction + { + @Override + public void validateForm(ApproveUserRequestsForm form, Errors errors) + { + Container mccContainer = MccManager.get().getMCCContainer(); + if (mccContainer == null) + { + errors.reject(ERROR_MSG, "The MCC project has not been set on this server. This is an administrator error."); + return; + } + + if (form.getRequestIds() == null || form.getRequestIds().length == 0) + { + errors.reject(ERROR_MSG, "No request IDs provided"); + } + + TableInfo ti = MccSchema.getInstance().getSchema().getTable(MccSchema.TABLE_USER_REQUESTS); + for (int requestId : form.getRequestIds()) + { + TableSelector ts = new TableSelector(ti, PageFlowUtil.set("userId"), new SimpleFilter(FieldKey.fromString("rowId"), requestId), null); + if (!ts.exists()) + { + errors.reject(ERROR_MSG, "No request found for request ID: " + requestId); + break; + } + } + } + + @Override + public Object execute(ApproveUserRequestsForm form, BindException errors) throws Exception + { + ApiSimpleResponse response = new ApiSimpleResponse(); + MutableSecurityPolicy policy = new MutableSecurityPolicy(MccManager.get().getMCCContainer().getPolicy()); + List newUserStatusList = new ArrayList<>(); + List existingUsersGivenAccess = new ArrayList<>(); + try (DbScope.Transaction transaction = CoreSchema.getInstance().getScope().ensureTransaction()) + { + TableInfo ti = MccSchema.getInstance().getSchema().getTable(MccSchema.TABLE_USER_REQUESTS); + for (int requestId : form.getRequestIds()) + { + TableSelector ts = new TableSelector(ti, new SimpleFilter(FieldKey.fromString("rowId"), requestId), null); + Map map = ts.getMap(requestId); + + User u; + if (map.get("userId") != null) + { + Integer userId = (Integer)map.get("userId"); + u = UserManager.getUser(userId); + existingUsersGivenAccess.add(u); + } + else + { + ValidEmail ve = new ValidEmail((String)map.get("email")); + u = UserManager.getUser(ve); + if (u != null) + { + existingUsersGivenAccess.add(u); + } + else + { + SecurityManager.NewUserStatus st = SecurityManager.addUser(ve, getUser()); + u = st.getUser(); + u.setFirstName((String)map.get("firstName")); + u.setLastName((String)map.get("lastName")); + UserManager.updateUser(getUser(), u); + + if (st.isLdapEmail()) + { + existingUsersGivenAccess.add(st.getUser()); + } + else + { + newUserStatusList.add(st); + } + } + } + + Map row = new HashMap<>(); + row.put("rowId", requestId); + row.put("userId", u.getUserId()); + Table.update(getUser(), ti, row, requestId); + + if (!policy.hasPermission(u, ReadPermission.class)) + { + policy.addRoleAssignment(u, ReaderRole.class); + } + else + { + _log.info("user already has read permission on MCC container: " + u.getDisplayName(getUser())); + } + } + + SecurityPolicyManager.savePolicy(policy); + + transaction.commit(); + } + + //send emails: + for (SecurityManager.NewUserStatus st : newUserStatusList) + { + SecurityManager.sendRegistrationEmail(getViewContext(), st.getEmail(), null, st, null); + } + + for (User u : existingUsersGivenAccess) + { + Container mccContainer = MccManager.get().getMCCContainer(); + boolean isLDAP = SecurityManager.isLdapEmail(new ValidEmail(u.getEmail())); + + MailHelper.MultipartMessage mail = MailHelper.createMultipartMessage(); + mail.setEncodedHtmlContent("Your account request has been approved for MCC! " + "Click here to access the site." + (isLDAP ? " Use your normal OHSU email/password to login." : "")); + mail.setFrom(getReplyEmail(getContainer())); + mail.setSubject("MCC Account Request"); + mail.addRecipients(Message.RecipientType.TO, u.getEmail()); + + MailHelper.send(mail, getUser(), getContainer()); + } + + response.put("success", !errors.hasErrors()); + + return response; + } + } + + private String getReplyEmail(Container c) + { + LookAndFeelProperties lfp = LookAndFeelProperties.getInstance(getContainer()); + String email = lfp.getSystemEmailAddress(); + if (email == null) + { + return AppProps.getInstance().getAdministratorContactEmail(true); + } + + return email; + } + + public static class ApproveUserRequestsForm + { + private int[] requestIds; + + public int[] getRequestIds() + { + return requestIds; + } + + public void setRequestIds(int[] requestIds) + { + this.requestIds = requestIds; + } + } + + @RequiresNoPermission + @IgnoresTermsOfUse + @AllowedDuringUpgrade + public class RequestHelpAction extends MutatingApiAction + { + @Override + public void validateForm(RequestHelpForm form, Errors errors) + { + Container mccContainer = MccManager.get().getMCCContainer(); + if (mccContainer == null) + { + errors.reject(ERROR_MSG, "The MCC project has not been set on this server. This is an administrator error."); + return; + } + + if (StringUtils.isEmpty(form.getEmail()) || StringUtils.isEmpty(form.getComment())) + { + errors.reject(ERROR_REQUIRED, "Must provide both an email address and question/comment"); + } + else + { + try + { + new ValidEmail(form.getEmail()); + } + catch (ValidEmail.InvalidEmailException e) + { + errors.reject(ERROR_MSG, "Your email address is not valid. Please verify your email address below."); + } + } + } + + @Override + public Object execute(RequestHelpForm form, BindException errors) throws Exception + { + Set users = MccManager.get().getNotificationUsers(); + if (users != null && !users.isEmpty()) + { + try + { + Set
emails = new HashSet<>(); + for (User u : users) + { + emails.add(new InternetAddress(u.getEmail())); + } + + MailHelper.MultipartMessage mail = MailHelper.createMultipartMessage(); + mail.setEncodedHtmlContent("A support request was submitted from MCC by: " + form.getEmail() + "

Message:
" + form.getComment()); + mail.setFrom(form.getEmail()); + mail.setSubject("MCC Help Request"); + mail.addRecipients(Message.RecipientType.TO, emails.toArray(new Address[0])); + + MailHelper.send(mail, getUser(), getContainer()); + } + catch (Exception e) + { + ExceptionUtil.logExceptionToMothership(null, e); + } + } + else + { + _log.error("A help request was received by MCC, but the admin emails have not been configured. The request from: " + form.getEmail()); + _log.error(form.getComment()); + } + + return new ApiSimpleResponse("success", true); + } + } + + public static class RequestHelpForm + { + private String _email; + private String _comment; + + public String getEmail() + { + return _email; + } + + public void setEmail(String email) + { + _email = email; + } + + public String getComment() + { + return _comment; + } + + public void setComment(String comment) + { + _comment = comment; + } + } } diff --git a/mcc/src/org/labkey/mcc/MccManager.java b/mcc/src/org/labkey/mcc/MccManager.java index d6417b79d..d0f657de9 100644 --- a/mcc/src/org/labkey/mcc/MccManager.java +++ b/mcc/src/org/labkey/mcc/MccManager.java @@ -16,8 +16,28 @@ package org.labkey.mcc; +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerManager; +import org.labkey.api.module.Module; +import org.labkey.api.module.ModuleLoader; +import org.labkey.api.module.ModuleProperty; +import org.labkey.api.security.User; +import org.labkey.api.security.UserManager; +import org.labkey.api.security.ValidEmail; + +import java.util.HashSet; +import java.util.Set; + public class MccManager { + private static final Logger _log = LogManager.getLogger(MccManager.class); + + public static final String ContainerPropName = "MCCContainer"; + public static final String NotifyPropName = "MCCContactUsers"; + private static final MccManager _instance = new MccManager(); private MccManager() @@ -29,4 +49,54 @@ public static MccManager get() { return _instance; } + + public Container getMCCContainer() + { + Module m = ModuleLoader.getInstance().getModule(MccModule.NAME); + ModuleProperty mp = m.getModuleProperties().get(MccManager.ContainerPropName); + String path = mp.getEffectiveValue(ContainerManager.getRoot()); + if (path == null) + return null; + + return ContainerManager.getForPath(path); + } + + public Set getNotificationUsers() + { + Module m = ModuleLoader.getInstance().getModule(MccModule.NAME); + ModuleProperty mp = m.getModuleProperties().get(MccManager.NotifyPropName); + String userNames = mp.getEffectiveValue(ContainerManager.getRoot()); + userNames = StringUtils.trimToNull(userNames); + if (userNames == null) + return null; + + Set ret = new HashSet<>(); + for (String username : userNames.split(",")) + { + User u = UserManager.getUserByDisplayName(username); + if (u == null) + { + try + { + u = UserManager.getUser(new ValidEmail(username)); + } + catch (ValidEmail.InvalidEmailException e) + { + //ignore + } + } + + if (u == null) + { + _log.error("Unknown user registered for MCC notifcations: " + username); + } + + if (u != null) + { + ret.add(u); + } + } + + return ret; + } } \ No newline at end of file diff --git a/mcc/src/org/labkey/mcc/MccModule.java b/mcc/src/org/labkey/mcc/MccModule.java index 96e462c0f..ea28010a7 100644 --- a/mcc/src/org/labkey/mcc/MccModule.java +++ b/mcc/src/org/labkey/mcc/MccModule.java @@ -41,7 +41,7 @@ public String getName() @Override public @Nullable Double getSchemaVersion() { - return 20.000; + return 20.001; } @Override diff --git a/mcc/src/org/labkey/mcc/MccSchema.java b/mcc/src/org/labkey/mcc/MccSchema.java index d072b94a9..be31ef6f1 100644 --- a/mcc/src/org/labkey/mcc/MccSchema.java +++ b/mcc/src/org/labkey/mcc/MccSchema.java @@ -25,6 +25,8 @@ public class MccSchema private static final MccSchema _instance = new MccSchema(); public static final String NAME = "mcc"; + public static final String TABLE_USER_REQUESTS = "userRequests"; + public static MccSchema getInstance() { return _instance; diff --git a/mcc/src/org/labkey/mcc/query/UserRequestCustomizer.java b/mcc/src/org/labkey/mcc/query/UserRequestCustomizer.java new file mode 100644 index 000000000..7f789d464 --- /dev/null +++ b/mcc/src/org/labkey/mcc/query/UserRequestCustomizer.java @@ -0,0 +1,44 @@ +package org.labkey.mcc.query; + +import org.labkey.api.data.AbstractTableInfo; +import org.labkey.api.data.JdbcType; +import org.labkey.api.data.SQLFragment; +import org.labkey.api.data.TableCustomizer; +import org.labkey.api.data.TableInfo; +import org.labkey.api.ldk.LDKService; +import org.labkey.api.query.ExprColumn; + +public class UserRequestCustomizer implements TableCustomizer +{ + @Override + public void customize(TableInfo tableInfo) + { + LDKService.get().getDefaultTableCustomizer().customize(tableInfo); + + if (tableInfo instanceof AbstractTableInfo) + { + addUserCol((AbstractTableInfo)tableInfo); + } + } + + public void addUserCol(AbstractTableInfo ti) + { + String colName = "hasAccess"; + if (ti.getColumn(colName) != null) + { + return; + } + + ExprColumn col = new ExprColumn(ti, colName, new SQLFragment("(CASE WHEN (exists (" + + "select u.rowid from mcc.userrequests u " + + "left join core.RoleAssignments ra " + + "on (u.userid = ra.UserId AND u.container = ra.ResourceId) " + + "WHERE ra.Role = 'org.labkey.api.security.roles.ReaderRole' AND u.rowid = " + ExprColumn.STR_TABLE_ALIAS + ".rowid " + + ")) THEN " + ti.getSqlDialect().getBooleanTRUE() + " ELSE " + ti.getSqlDialect().getBooleanFALSE() + " END)"), JdbcType.BOOLEAN, ti.getColumn("userId")); + col.setLabel("Has MCC Access?"); + col.setReadOnly(true); + col.setIsUnselectable(true); + col.setUserEditable(false); + ti.addColumn(col); + } +} From c3cf85e99b4f189afbe20b0b8aadac0a65e93d94 Mon Sep 17 00:00:00 2001 From: bbimber Date: Thu, 4 Feb 2021 15:16:49 -0800 Subject: [PATCH 16/29] Add release notes page --- mGAP/resources/folderTypes/mGAP.folderType.xml | 4 ++++ mGAP/resources/views/releaseNotes.html | 12 ++++++++++++ mGAP/resources/views/releaseNotes.view.xml | 8 ++++++++ mGAP/resources/views/releaseNotes.webpart.xml | 6 ++++++ mGAP/resources/views/variants.html | 2 +- .../labkey/mgap/pipeline/mGapReleaseGenerator.java | 4 ++-- 6 files changed, 33 insertions(+), 3 deletions(-) create mode 100644 mGAP/resources/views/releaseNotes.html create mode 100644 mGAP/resources/views/releaseNotes.view.xml create mode 100644 mGAP/resources/views/releaseNotes.webpart.xml diff --git a/mGAP/resources/folderTypes/mGAP.folderType.xml b/mGAP/resources/folderTypes/mGAP.folderType.xml index bee390d74..bf7576130 100644 --- a/mGAP/resources/folderTypes/mGAP.folderType.xml +++ b/mGAP/resources/folderTypes/mGAP.folderType.xml @@ -84,6 +84,10 @@ mGAP Variant Releases body + + mGAP Release Notes + body + mGAP Gene Search right diff --git a/mGAP/resources/views/releaseNotes.html b/mGAP/resources/views/releaseNotes.html new file mode 100644 index 000000000..110244e37 --- /dev/null +++ b/mGAP/resources/views/releaseNotes.html @@ -0,0 +1,12 @@ +

Release 2.0:

+
    +
  • Substantial revamp of all data. All samples have been realigned to the MMul_10 reference genome, followed by our standard GenotypeGVCFs pipeline. The MMul_10 is the most complete rhesus macaque assembly to date, and we expect this should improve accuracy of variant calls. Further, because our data are now aligned to the same assembly as NCBI/Ensembl, it should be easier to translate between mGAP and other databases.
    • +
  • Our internal variant calling process has switched to use GATK's GenomicsDB to pre-aggregate data prior to calling with GenotypeGVCFs, as opposed to CombineGVCFs, which was used in prior releases. This should be a purely technical difference with no change in the resulting data
    • +
+ +

Future Plans:

+
    +
  • We expect to upgrade the genome browser to use the redesigned JBrowse 2 browser. This should provide general performance improvements and will make future mGAP-specific customization easier.
    • +
  • We will support other modes of viewing and downloading variant data, in particular tabular views by gene.
    • +
  • We recognize that the mGAP release VCF can be enormous, particularly because of all the site-specific functional annotation. To support different types of users, upcoming releases will include 'slim' versions of the data, which will be downloadable files with certain information removed to save file size.
    • +
\ No newline at end of file diff --git a/mGAP/resources/views/releaseNotes.view.xml b/mGAP/resources/views/releaseNotes.view.xml new file mode 100644 index 000000000..c9a2b3e33 --- /dev/null +++ b/mGAP/resources/views/releaseNotes.view.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/mGAP/resources/views/releaseNotes.webpart.xml b/mGAP/resources/views/releaseNotes.webpart.xml new file mode 100644 index 000000000..f7f58d800 --- /dev/null +++ b/mGAP/resources/views/releaseNotes.webpart.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/mGAP/resources/views/variants.html b/mGAP/resources/views/variants.html index 3d66a6667..f59586d79 100644 --- a/mGAP/resources/views/variants.html +++ b/mGAP/resources/views/variants.html @@ -9,7 +9,7 @@ title: 'Variant Catalog Releases', schemaName: 'mgap', queryName: 'variantCatalogReleases', - maxRows: 50, + maxRows: 3, showRecordSelectors: false, showDetailsColumn: false, buttonBar: {position: 'none', includeStandardButtons: false, items: []} diff --git a/mGAP/src/org/labkey/mgap/pipeline/mGapReleaseGenerator.java b/mGAP/src/org/labkey/mgap/pipeline/mGapReleaseGenerator.java index 848a0a09d..a3fd61c17 100644 --- a/mGAP/src/org/labkey/mgap/pipeline/mGapReleaseGenerator.java +++ b/mGAP/src/org/labkey/mgap/pipeline/mGapReleaseGenerator.java @@ -953,7 +953,7 @@ private void inspectAndSummarizeVcf(JobContext ctx, File vcfInput, GeneToNameTra File interestingVariantTable = getVariantTableName(ctx, vcfInput); try (VCFFileReader reader = new VCFFileReader(vcfInput); CloseableIterator it = reader.iterator(); CSVWriter writer = new CSVWriter(PrintWriters.getPrintWriter(interestingVariantTable), '\t', CSVWriter.NO_QUOTE_CHARACTER)) { - writer.writeNext(new String[]{"Chromosome", "Position", "Reference", "Allele", "Source", "Reason", "Description", "Overlapping Gene(s)", "OMIM Entries", "OMIM Phenotypes", "AF", "CADD_PH"}); + writer.writeNext(new String[]{"Chromosome", "Position", "Reference", "Allele", "Source", "Reason", "Description", "Overlapping Gene(s)", "OMIM Entries", "OMIM Phenotypes", "AF", "Identifier", "CADD_PH"}); while (it.hasNext()) { Set> queuedLines = new LinkedHashSet<>(); @@ -1113,7 +1113,7 @@ private void inspectAndSummarizeVcf(JobContext ctx, File vcfInput, GeneToNameTra try { String allele = clnAlleles.get(i); - maybeWriteVariantLine(queuedLines, vc, allele, "ClinVar", diseaseSplit.get(j), description, overlappingGenes, omims, omimds, ctx.getLogger(), "ClinVar:" + clnAlleleIds.get(j)); + maybeWriteVariantLine(queuedLines, vc, allele, "ClinVar", diseaseSplit.get(j), description, overlappingGenes, omims, omimds, ctx.getLogger(), "ClinVar:" + clnAlleleIds.get(i)); } catch (IndexOutOfBoundsException e) From 8ca15eef8fb6cc0e7c68fb6f67a975f421a2e078 Mon Sep 17 00:00:00 2001 From: bbimber Date: Thu, 4 Feb 2021 16:02:07 -0800 Subject: [PATCH 17/29] Add more WNPRC ETL code --- mcc/resources/etls/wnprc.xml | 92 ++++++++++++++++++- mcc/resources/queries/wnprcSource/birth.sql | 6 ++ mcc/resources/queries/wnprcSource/deaths.sql | 8 ++ .../queries/wnprcSource/demographics.sql | 2 +- .../queries/wnprcSource/parentage.sql | 27 ++++++ mcc/resources/queries/wnprcSource/weight.sql | 8 ++ .../datasets/datasets_manifest.xml | 6 -- .../datasets/datasets_metadata.xml | 59 ------------ 8 files changed, 138 insertions(+), 70 deletions(-) create mode 100644 mcc/resources/queries/wnprcSource/birth.sql create mode 100644 mcc/resources/queries/wnprcSource/deaths.sql create mode 100644 mcc/resources/queries/wnprcSource/parentage.sql create mode 100644 mcc/resources/queries/wnprcSource/weight.sql diff --git a/mcc/resources/etls/wnprc.xml b/mcc/resources/etls/wnprc.xml index de5850d40..d9d4bf593 100644 --- a/mcc/resources/etls/wnprc.xml +++ b/mcc/resources/etls/wnprc.xml @@ -3,14 +3,98 @@ WNPRC_Data WNPRC Clinical/Demographics Data - + Copy to target - - + + + Id + date + gender + geographic_origin + birth + death + species + objectid + + + + + + + + + + Copy to target + + + Id + date + parent + relationship + method + objectid + + + + + + + + + + Copy to target + + + Id + date + gender + species + geographic_origin + dam + sire + objectid + + + + + + + + + + Copy to target + + + Id + date + weight + objectid + + + + + + + + + + Copy to target + + + Id + date + cause + objectid + + + + + - + + diff --git a/mcc/resources/queries/wnprcSource/birth.sql b/mcc/resources/queries/wnprcSource/birth.sql new file mode 100644 index 000000000..06808b9af --- /dev/null +++ b/mcc/resources/queries/wnprcSource/birth.sql @@ -0,0 +1,6 @@ +SELECT + +Id, date, gender, species, geographic_origin, dam, sire, objectid, modified + +FROM "/WNPRC/EHR/".study.birth +WHERE species = 'Marmoset'; \ No newline at end of file diff --git a/mcc/resources/queries/wnprcSource/deaths.sql b/mcc/resources/queries/wnprcSource/deaths.sql new file mode 100644 index 000000000..94c301bb7 --- /dev/null +++ b/mcc/resources/queries/wnprcSource/deaths.sql @@ -0,0 +1,8 @@ +SELECT + + Id, date, + cause, + objectid, modified + +FROM "/WNPRC/EHR/".study.weight +WHERE Id.demographics.species = 'Marmoset'; \ No newline at end of file diff --git a/mcc/resources/queries/wnprcSource/demographics.sql b/mcc/resources/queries/wnprcSource/demographics.sql index c63412b14..7a860fef3 100644 --- a/mcc/resources/queries/wnprcSource/demographics.sql +++ b/mcc/resources/queries/wnprcSource/demographics.sql @@ -1,6 +1,6 @@ SELECT -Id, date, gender, geographic_origin, birth, death, species, objectid +Id, date, gender, geographic_origin, birth, death, species, objectid, modified FROM "/WNPRC/EHR/".study.demographics WHERE species = 'Marmoset'; \ No newline at end of file diff --git a/mcc/resources/queries/wnprcSource/parentage.sql b/mcc/resources/queries/wnprcSource/parentage.sql new file mode 100644 index 000000000..a8fe2e80c --- /dev/null +++ b/mcc/resources/queries/wnprcSource/parentage.sql @@ -0,0 +1,27 @@ +SELECT + + Id, + date, + sire as parent, + 'Sire' as relationship, + 'Observed' as method, + cast(objectid as varchar) || '-Sire' as objectid, + modified + +FROM "/WNPRC/EHR/".study.demographics +WHERE species = 'Marmoset' and sire is not null + +UNION ALL + +SELECT + + Id, + date, + sire as parent, + 'Dam' as relationship, + 'Observed' as method, + cast(objectid as varchar) || '-Dam' as objectid, + modified + +FROM "/WNPRC/EHR/".study.demographics +WHERE species = 'Marmoset' and dam is not null \ No newline at end of file diff --git a/mcc/resources/queries/wnprcSource/weight.sql b/mcc/resources/queries/wnprcSource/weight.sql new file mode 100644 index 000000000..db7bce107 --- /dev/null +++ b/mcc/resources/queries/wnprcSource/weight.sql @@ -0,0 +1,8 @@ +SELECT + + Id, date, + weight, + objectid, modified + +FROM "/WNPRC/EHR/".study.weight +WHERE Id.demographics.species = 'Marmoset'; \ No newline at end of file diff --git a/mcc/resources/referenceStudy/datasets/datasets_manifest.xml b/mcc/resources/referenceStudy/datasets/datasets_manifest.xml index 1d43fa147..47d75b91c 100644 --- a/mcc/resources/referenceStudy/datasets/datasets_manifest.xml +++ b/mcc/resources/referenceStudy/datasets/datasets_manifest.xml @@ -39,17 +39,11 @@ - - - - - - diff --git a/mcc/resources/referenceStudy/datasets/datasets_metadata.xml b/mcc/resources/referenceStudy/datasets/datasets_metadata.xml index 6a154bcc1..480c6a89d 100644 --- a/mcc/resources/referenceStudy/datasets/datasets_metadata.xml +++ b/mcc/resources/referenceStudy/datasets/datasets_metadata.xml @@ -561,65 +561,6 @@ Arrival - - - - varchar - http://cpas.labkey.com/Study#ParticipantId - - ptid - - - - timestamp - http://cpas.labkey.com/Study#VisitDate - http://cpas.labkey.com/Study#VisitDate - - - timestamp - urn:ehr.labkey.org/#EndDate - - - integer - urn:ehr.labkey.org/#Project - - - entityid - urn:ehr.labkey.org/#ObjectId - true - - - Assignment -
- - - - varchar - http://cpas.labkey.com/Study#ParticipantId - - ptid - - - - timestamp - http://cpas.labkey.com/Study#VisitDate - http://cpas.labkey.com/Study#VisitDate - - - timestamp - urn:ehr.labkey.org/#EndDate - - - integer - - - entityid - urn:ehr.labkey.org/#ObjectId - true - - - Animal Group Members -
From a578266fb27f75c453f445a4365fe1ea0bb129c3 Mon Sep 17 00:00:00 2001 From: bbimber Date: Fri, 5 Feb 2021 14:44:24 -0800 Subject: [PATCH 18/29] Update ETLs --- mcc/resources/etls/snprc.xml | 103 ++++++++++++++++++++++++++++++++++- mcc/resources/etls/wnprc.xml | 15 +++-- 2 files changed, 107 insertions(+), 11 deletions(-) diff --git a/mcc/resources/etls/snprc.xml b/mcc/resources/etls/snprc.xml index 127947d46..adadf3468 100644 --- a/mcc/resources/etls/snprc.xml +++ b/mcc/resources/etls/snprc.xml @@ -1,14 +1,111 @@ SNPRC_Data + + SNPRC Clinical/Demographics Data - + Copy to target - - + + + AnimalId + date + gender + geographic_origin + birth + death + species + objectid + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Copy to target + + + AnimalId + birth + gender + species + dam + sire + objectid + + + + + + + + + + + + + Copy to target + + + AnimalId + date + weight + objectid + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/mcc/resources/etls/wnprc.xml b/mcc/resources/etls/wnprc.xml index d9d4bf593..9b3aac007 100644 --- a/mcc/resources/etls/wnprc.xml +++ b/mcc/resources/etls/wnprc.xml @@ -17,7 +17,7 @@ objectid - + @@ -35,7 +35,7 @@ objectid - + @@ -55,7 +55,7 @@ objectid - + @@ -71,7 +71,7 @@ objectid - + @@ -87,17 +87,16 @@ objectid - + - - - + + From 871f4c4d3d5ed41daaff334f055b8055930aeec3 Mon Sep 17 00:00:00 2001 From: bbimber Date: Sun, 7 Feb 2021 11:09:23 -0800 Subject: [PATCH 19/29] Allow cellranger VDJ to finish when there are no a/b hits --- .../tcrdb/pipeline/CellRangerVDJCellHashingHandler.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java index 85ac41dea..8e52ec939 100644 --- a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java +++ b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java @@ -38,7 +38,9 @@ public class CellRangerVDJCellHashingHandler extends AbstractParameterizedOutputHandler { - private FileType _fileType = new FileType("vloupe", false); + private FileType _vloupeFileType = new FileType("vloupe", false); + private FileType _htmlFileType = new FileType("html", false); + public static final String CATEGORY = "Cell Hashing Calls (VDJ)"; public static final String TARGET_ASSAY = "targetAssay"; @@ -73,7 +75,7 @@ private static List getDefaultParams() @Override public boolean canProcess(SequenceOutputFile o) { - return o.getFile() != null && _fileType.isType(o.getFile()); + return o.getFile() != null && (_vloupeFileType.isType(o.getFile()) || (_htmlFileType.isType(o.getFile()) && "10x Run Summary".equals(o.getCategory()) && o.getName().contains("VDJ Summary"))); } @Override From daaf73d87b2be6a770f67a28c5dc606633b9ee87 Mon Sep 17 00:00:00 2001 From: bbimber Date: Mon, 8 Feb 2021 08:42:24 -0800 Subject: [PATCH 20/29] Fail more clearly if no cell barcodes found --- .../tcrdb/pipeline/CellRangerVDJCellHashingHandler.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java index 8e52ec939..e7ae5b2c1 100644 --- a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java +++ b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java @@ -244,6 +244,8 @@ private File createCellbarcodeWhitelist(JobContext ctx, File perCellTsv, boolean Set uniqueBarcodesIncludingNoCDR3 = new HashSet<>(); ctx.getLogger().debug("writing cell barcodes, using file: " + perCellTsv.getPath()); ctx.getLogger().debug("allow cells lacking CDR3: " + allowCellsLackingCDR3); + + int totalBarcodeWritten = 0; try (CSVWriter writer = new CSVWriter(PrintWriters.getPrintWriter(cellBarcodeWhitelist), ',', CSVWriter.NO_QUOTE_CHARACTER); CSVReader reader = new CSVReader(Readers.getReader(perCellTsv), ',')) { int rowIdx = 0; @@ -275,6 +277,7 @@ private File createCellbarcodeWhitelist(JobContext ctx, File perCellTsv, boolean { writer.writeNext(new String[]{barcode}); uniqueBarcodes.add(barcode); + totalBarcodeWritten++; } uniqueBarcodesIncludingNoCDR3.add(barcode); @@ -301,6 +304,7 @@ private File createCellbarcodeWhitelist(JobContext ctx, File perCellTsv, boolean for (String barcode : uniqueBarcodesIncludingNoCDR3) { writer.writeNext(new String[]{barcode}); + totalBarcodeWritten++; } } catch (IOException e) @@ -309,6 +313,11 @@ private File createCellbarcodeWhitelist(JobContext ctx, File perCellTsv, boolean } } + if (totalBarcodeWritten == 0) + { + throw new PipelineJobException("No valid cell barcodes found!"); + } + //TODO: consider looking up GEX data? return cellBarcodeWhitelist; From 3bf50badf6213000c919d921dc7fd990543e2dd1 Mon Sep 17 00:00:00 2001 From: bbimber Date: Tue, 9 Feb 2021 14:38:37 -0800 Subject: [PATCH 21/29] Refactor citeseq/hashing to use pre-computed count matrix --- mcc/resources/etls/snprc.xml | 2 +- mcc/resources/etls/wnprc.xml | 2 +- .../pipeline/CellRangerVDJCellHashingHandler.java | 10 +++++----- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/mcc/resources/etls/snprc.xml b/mcc/resources/etls/snprc.xml index adadf3468..103417512 100644 --- a/mcc/resources/etls/snprc.xml +++ b/mcc/resources/etls/snprc.xml @@ -7,7 +7,7 @@ Copy to target - + AnimalId date diff --git a/mcc/resources/etls/wnprc.xml b/mcc/resources/etls/wnprc.xml index 9b3aac007..c4433e54c 100644 --- a/mcc/resources/etls/wnprc.xml +++ b/mcc/resources/etls/wnprc.xml @@ -5,7 +5,7 @@ Copy to target - + Id date diff --git a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java index e7ae5b2c1..79204c155 100644 --- a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java +++ b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java @@ -67,7 +67,7 @@ private static List getDefaultParams() }}, false) )); - ret.addAll(CellHashingService.get().getDefaultHashingParams(true, CellHashingService.BARCODE_TYPE.hashing)); + ret.addAll(CellHashingService.get().getHashingCallingParams()); return ret; } @@ -114,7 +114,7 @@ public class Processor implements SequenceOutputHandler.SequenceOutputProcessor public void init(JobContext ctx, List inputFiles, List actions, List outputsToCreate) throws UnsupportedOperationException, PipelineJobException { //NOTE: this is the pathway to import assay data, whether hashing is used or not - CellHashingService.get().prepareHashingAndCiteSeqFilesIfNeeded(ctx.getOutputDir(), ctx.getJob(), ctx.getSequenceSupport(), "tcrReadsetId", ctx.getParams().optBoolean("excludeFailedcDNA", false), false, false); + CellHashingService.get().prepareHashingAndCiteSeqFilesIfNeeded(ctx.getOutputDir(), ctx.getJob(), ctx.getSequenceSupport(), "tcrReadsetId", false, false); if (ctx.getParams().optBoolean(USE_GEX_BARCODES, false)) { @@ -210,17 +210,17 @@ private void processVloupeFile(JobContext ctx, File perCellTsv, Readset rs, Reco //TODO: allow union of GEX and TCR cell barcodes for whitelist! CellHashingService.CellHashingParameters parameters = CellHashingService.CellHashingParameters.createFromJson(CellHashingService.BARCODE_TYPE.hashing, ctx.getSourceDirectory(), ctx.getParams(), null, rs, null); - parameters.cellBarcodeWhitelistFile = createCellbarcodeWhitelist(ctx, perCellTsv, true); parameters.genomeId = genomeId; parameters.outputCategory = CATEGORY; parameters.basename = FileUtil.makeLegalName(rs.getName()); parameters.allowableHtoOrCiteseqBarcodes = htosPerReadset; + parameters.cellBarcodeWhitelistFile = createCellbarcodeWhitelist(ctx, perCellTsv, true); + File existingCountMatrixUmiDir = CellHashingService.get().getExistingFeatureBarcodeCountDir(rs, CellHashingService.BARCODE_TYPE.hashing, ctx.getSequenceSupport()); - File cellToHto = CellHashingService.get().processCellHashingOrCiteSeqForParent(rs, output, ctx, parameters); + File cellToHto = CellHashingService.get().generateHashingCallsForRawMatrix(rs, output, ctx, parameters, existingCountMatrixUmiDir); if (CellHashingService.get().usesCellHashing(ctx.getSequenceSupport(), ctx.getSourceDirectory()) && cellToHto == null) { throw new PipelineJobException("Missing cell to HTO file"); - } action.addOutput(cellToHto, CellRangerVDJUtils.TCR_HASHING_CALLS, false); From 8c41f3f72a5881901a152c11e319e7e7663c3aa3 Mon Sep 17 00:00:00 2001 From: bbimber Date: Wed, 10 Feb 2021 17:17:47 -0800 Subject: [PATCH 22/29] Prepare MCC examples --- mcc/resources/queries/study/labwork.js | 8 ++++++++ mcc/resources/queries/study/medicationAdministration.js | 8 ++++++++ mcc/resources/queries/study/medicationOrders.js | 8 ++++++++ mcc/resources/views/dashboard.html | 9 +++++++++ mcc/resources/views/dashboard.view.xml | 9 +++++++++ mcc/resources/views/dashboard.webpart.xml | 6 ++++++ mcc/resources/web/mcc/dashboard.js | 0 7 files changed, 48 insertions(+) create mode 100644 mcc/resources/queries/study/labwork.js create mode 100644 mcc/resources/queries/study/medicationAdministration.js create mode 100644 mcc/resources/queries/study/medicationOrders.js create mode 100644 mcc/resources/views/dashboard.html create mode 100644 mcc/resources/views/dashboard.view.xml create mode 100644 mcc/resources/views/dashboard.webpart.xml create mode 100644 mcc/resources/web/mcc/dashboard.js diff --git a/mcc/resources/queries/study/labwork.js b/mcc/resources/queries/study/labwork.js new file mode 100644 index 000000000..d4075c56c --- /dev/null +++ b/mcc/resources/queries/study/labwork.js @@ -0,0 +1,8 @@ +/* + * Copyright (c) 2018-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0 + */ + +require("ehr/triggers").initScript(this); + diff --git a/mcc/resources/queries/study/medicationAdministration.js b/mcc/resources/queries/study/medicationAdministration.js new file mode 100644 index 000000000..d4075c56c --- /dev/null +++ b/mcc/resources/queries/study/medicationAdministration.js @@ -0,0 +1,8 @@ +/* + * Copyright (c) 2018-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0 + */ + +require("ehr/triggers").initScript(this); + diff --git a/mcc/resources/queries/study/medicationOrders.js b/mcc/resources/queries/study/medicationOrders.js new file mode 100644 index 000000000..d4075c56c --- /dev/null +++ b/mcc/resources/queries/study/medicationOrders.js @@ -0,0 +1,8 @@ +/* + * Copyright (c) 2018-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0 + */ + +require("ehr/triggers").initScript(this); + diff --git a/mcc/resources/views/dashboard.html b/mcc/resources/views/dashboard.html new file mode 100644 index 000000000..014a425a7 --- /dev/null +++ b/mcc/resources/views/dashboard.html @@ -0,0 +1,9 @@ + \ No newline at end of file diff --git a/mcc/resources/views/dashboard.view.xml b/mcc/resources/views/dashboard.view.xml new file mode 100644 index 000000000..252242cd6 --- /dev/null +++ b/mcc/resources/views/dashboard.view.xml @@ -0,0 +1,9 @@ + + + + + + + + + \ No newline at end of file diff --git a/mcc/resources/views/dashboard.webpart.xml b/mcc/resources/views/dashboard.webpart.xml new file mode 100644 index 000000000..4e56bf8c1 --- /dev/null +++ b/mcc/resources/views/dashboard.webpart.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/mcc/resources/web/mcc/dashboard.js b/mcc/resources/web/mcc/dashboard.js new file mode 100644 index 000000000..e69de29bb From bfb90055898f3df53c58da1a6f48bea3b40b80a5 Mon Sep 17 00:00:00 2001 From: bbimber Date: Thu, 11 Feb 2021 09:46:58 -0800 Subject: [PATCH 23/29] Fix case in ETL --- mcc/resources/etls/snprc.xml | 8 ++++---- mcc/resources/etls/wnprc.xml | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/mcc/resources/etls/snprc.xml b/mcc/resources/etls/snprc.xml index 103417512..9eccdb1aa 100644 --- a/mcc/resources/etls/snprc.xml +++ b/mcc/resources/etls/snprc.xml @@ -29,7 +29,7 @@ - + @@ -49,7 +49,7 @@ Copy to target - + AnimalId birth @@ -71,7 +71,7 @@ Copy to target - + AnimalId date @@ -89,7 +89,7 @@ - + diff --git a/mcc/resources/etls/wnprc.xml b/mcc/resources/etls/wnprc.xml index c4433e54c..3147ab99d 100644 --- a/mcc/resources/etls/wnprc.xml +++ b/mcc/resources/etls/wnprc.xml @@ -25,7 +25,7 @@ Copy to target - + Id date @@ -43,7 +43,7 @@ Copy to target - + Id date @@ -63,7 +63,7 @@ Copy to target - + Id date @@ -79,7 +79,7 @@ Copy to target - + Id date From 4bfda5679138fc0fe64771d0eb43f80b22c0ec05 Mon Sep 17 00:00:00 2001 From: bbimber Date: Thu, 11 Feb 2021 11:03:21 -0800 Subject: [PATCH 24/29] Remove ETL step --- mcc/resources/etls/snprc.xml | 41 ++++++++++++++++++------------------ 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/mcc/resources/etls/snprc.xml b/mcc/resources/etls/snprc.xml index 9eccdb1aa..9f856c935 100644 --- a/mcc/resources/etls/snprc.xml +++ b/mcc/resources/etls/snprc.xml @@ -47,27 +47,28 @@ - - Copy to target - - - AnimalId - birth - gender - species - dam - sire - objectid - - - - - - - - + + + + + + + + + + + + + + + + + + + + - + Copy to target From e68be3401bed407f8b8d3ce860c7ab3d32de60bf Mon Sep 17 00:00:00 2001 From: bbimber Date: Thu, 11 Feb 2021 13:22:16 -0800 Subject: [PATCH 25/29] Further simplify hashing params code --- .../tcrdb/pipeline/CellRangerVDJCellHashingHandler.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java index 79204c155..26b1e01fa 100644 --- a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java +++ b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java @@ -49,7 +49,7 @@ public class CellRangerVDJCellHashingHandler extends AbstractParameterizedOutput public CellRangerVDJCellHashingHandler() { - super(ModuleLoader.getInstance().getModule(TCRdbModule.class), "CellRanger VDJ Import", "This will either directly import data (if cell hashing is not used), or run CiteSeqCount/MultiSeqClassifier to generate a sample-to-cellbarcode TSV based on the filtered barcodes from CellRanger VDJ and then import.", new LinkedHashSet<>(PageFlowUtil.set("tcrdb/field/AssaySelectorField.js")), getDefaultParams()); + super(ModuleLoader.getInstance().getModule(TCRdbModule.class), "CellRanger VDJ Import", "This will either directly import data (if cell hashing is not used), or run cellhashR on the hashing count matrix to generate a sample-to-cellbarcode TSV based on the filtered barcodes from CellRanger VDJ and then import.", new LinkedHashSet<>(PageFlowUtil.set("tcrdb/field/AssaySelectorField.js")), getDefaultParams()); } private static List getDefaultParams() @@ -209,11 +209,11 @@ private void processVloupeFile(JobContext ctx, File perCellTsv, Readset rs, Reco //TODO: allow union of GEX and TCR cell barcodes for whitelist! - CellHashingService.CellHashingParameters parameters = CellHashingService.CellHashingParameters.createFromJson(CellHashingService.BARCODE_TYPE.hashing, ctx.getSourceDirectory(), ctx.getParams(), null, rs, null); + CellHashingService.CellHashingParameters parameters = CellHashingService.CellHashingParameters.createFromJson(CellHashingService.BARCODE_TYPE.hashing, ctx.getSourceDirectory(), ctx.getParams(), null, rs); parameters.genomeId = genomeId; parameters.outputCategory = CATEGORY; parameters.basename = FileUtil.makeLegalName(rs.getName()); - parameters.allowableHtoOrCiteseqBarcodes = htosPerReadset; + parameters.allowableHtoBarcodes = htosPerReadset; parameters.cellBarcodeWhitelistFile = createCellbarcodeWhitelist(ctx, perCellTsv, true); File existingCountMatrixUmiDir = CellHashingService.get().getExistingFeatureBarcodeCountDir(rs, CellHashingService.BARCODE_TYPE.hashing, ctx.getSequenceSupport()); From b4ef0c8bdbbd966c9e27370ab909d2d7a47059d5 Mon Sep 17 00:00:00 2001 From: bbimber Date: Thu, 11 Feb 2021 14:07:07 -0800 Subject: [PATCH 26/29] Add alternate keys --- mcc/resources/etls/snprc.xml | 9 +++++++++ mcc/resources/etls/wnprc.xml | 22 +++++++++++++++------- 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/mcc/resources/etls/snprc.xml b/mcc/resources/etls/snprc.xml index 9f856c935..4a959b289 100644 --- a/mcc/resources/etls/snprc.xml +++ b/mcc/resources/etls/snprc.xml @@ -23,6 +23,9 @@ + + + @@ -84,6 +87,9 @@ + + + @@ -102,6 +108,9 @@ + + + diff --git a/mcc/resources/etls/wnprc.xml b/mcc/resources/etls/wnprc.xml index 3147ab99d..f8efab248 100644 --- a/mcc/resources/etls/wnprc.xml +++ b/mcc/resources/etls/wnprc.xml @@ -18,7 +18,9 @@ - + + + @@ -36,7 +38,9 @@ - + + + @@ -56,7 +60,9 @@ - + + + @@ -72,9 +78,10 @@ - + + + - @@ -88,9 +95,10 @@ - + + + - From ed9495b51110637d18122d6c5ad6fb65cfd7ce47 Mon Sep 17 00:00:00 2001 From: bbimber Date: Thu, 11 Feb 2021 14:28:44 -0800 Subject: [PATCH 27/29] Add dummy data loading --- mcc/resources/web/mcc/dashboard.js | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/mcc/resources/web/mcc/dashboard.js b/mcc/resources/web/mcc/dashboard.js index e69de29bb..4c94d7e87 100644 --- a/mcc/resources/web/mcc/dashboard.js +++ b/mcc/resources/web/mcc/dashboard.js @@ -0,0 +1,18 @@ +var MCC = {}; + +MCC.Dashboard = new function() { + return { + loadData: function () { + LABKEY.Query.selectRows({ + schemaName: 'study', + queryName: 'demographics', + columns: 'Id,birth,death,gender,species,Id/age/AgeFriendly', + success: function(results) { + console.log(results.rows); + }, + error: LDK.Utils.getErrorCallback(), + scope: this + }); + } + } +}; From 098d19f1749a77c5aa7e0115a5708121490d9636 Mon Sep 17 00:00:00 2001 From: bbimber Date: Fri, 12 Feb 2021 06:18:34 -0800 Subject: [PATCH 28/29] Save metadata table from Seurat objects --- primeseq/resources/views/geneticsMenu.html | 6 +++--- primeseq/src/org/labkey/primeseq/PrimeseqController.java | 8 +------- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/primeseq/resources/views/geneticsMenu.html b/primeseq/resources/views/geneticsMenu.html index 3ff3df451..4b6308609 100644 --- a/primeseq/resources/views/geneticsMenu.html +++ b/primeseq/resources/views/geneticsMenu.html @@ -88,15 +88,15 @@ items: [{ title: 'Public Resources', itemId: 'public' - },{ - itemId: 'collaborations', - title: 'Collaborations' },{ title: 'Labs', itemId: 'labs' },{ itemId: 'internal', title: 'Internal Projects' + },{ + itemId: 'collaborations', + title: 'Bimber Lab Collaborations' }] }); diff --git a/primeseq/src/org/labkey/primeseq/PrimeseqController.java b/primeseq/src/org/labkey/primeseq/PrimeseqController.java index 66d1de408..de1835eab 100644 --- a/primeseq/src/org/labkey/primeseq/PrimeseqController.java +++ b/primeseq/src/org/labkey/primeseq/PrimeseqController.java @@ -71,7 +71,7 @@ public ApiResponse execute(Object form, BindException errors) { Map resultProperties = new HashMap<>(); - resultProperties.put("collaborations", getSection("/Public/Collaborations")); + resultProperties.put("collaborations", getSection("/Labs/Bimber/Collaborations")); resultProperties.put("internal", getSection("/Internal")); resultProperties.put("labs", getSection("/Labs")); @@ -128,12 +128,6 @@ private List getSection(String path) { for (Container c : mainContainer.getChildren()) { - //NOTE: unlike EHR, omit children if the current user cannot read them - if (!c.hasPermission(getUser(), ReadPermission.class)) - { - continue; - } - JSONObject json = new JSONObject(); json.put("name", c.getName()); json.put("title", c.getTitle()); From 8a6033bb6f57054bad1187042de4ffaead334e3f Mon Sep 17 00:00:00 2001 From: bbimber Date: Fri, 12 Feb 2021 06:50:34 -0800 Subject: [PATCH 29/29] Left align text --- primeseq/resources/views/geneticsMenu.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/primeseq/resources/views/geneticsMenu.html b/primeseq/resources/views/geneticsMenu.html index 4b6308609..5177cd413 100644 --- a/primeseq/resources/views/geneticsMenu.html +++ b/primeseq/resources/views/geneticsMenu.html @@ -44,7 +44,7 @@ 'data-qtip="You do not have permission to view this page"', 'style="width: 300px;height: auto;" class="thumb-wrap thumb-wrap-side">', '', - '{title:htmlEncode}', + '{title:htmlEncode}', '', '', '',