From f65aef11c92360337bad6645a8fbc8828c75ab36 Mon Sep 17 00:00:00 2001 From: bbimber Date: Wed, 4 Nov 2020 11:30:34 -0800 Subject: [PATCH 01/98] Add support for cellranger vdj --inner-enrichment-primers --- .../CellRangerCellHashingHandler.java | 2 +- .../tcrdb/pipeline/CellRangerVDJUtils.java | 2 +- .../tcrdb/pipeline/CellRangerVDJWrapper.java | 27 +++++++++++++++++++ 3 files changed, 29 insertions(+), 2 deletions(-) diff --git a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerCellHashingHandler.java b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerCellHashingHandler.java index 00be83f8c..79c2bcc53 100644 --- a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerCellHashingHandler.java +++ b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerCellHashingHandler.java @@ -248,7 +248,7 @@ public static File processBarcodeFile(SequenceOutputHandler.JobContext ctx, File //prepare whitelist of cell indexes File cellBarcodeWhitelist = utils.getValidCellIndexFile(); Set uniqueBarcodes = new HashSet<>(); - ctx.getLogger().debug("writing cell barcodes"); + ctx.getLogger().debug("writing cell barcodes, using file: " + perCellTsv.getPath()); try (CSVWriter writer = new CSVWriter(PrintWriters.getPrintWriter(cellBarcodeWhitelist), ',', CSVWriter.NO_QUOTE_CHARACTER);CSVReader reader = new CSVReader(IOUtil.openFileForBufferedUtf8Reading(perCellTsv), '\t')) { int rowIdx = 0; diff --git a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJUtils.java b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJUtils.java index f9d8ce89d..41748d31a 100644 --- a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJUtils.java +++ b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJUtils.java @@ -376,7 +376,7 @@ public File runRemoteVdjCellHashingTasks(PipelineStepOutput output, String outpu File cellBarcodeWhitelist = getValidCellIndexFile(); Set uniqueBarcodes = new HashSet<>(); Set uniqueBarcodesIncludingNoCDR3 = new HashSet<>(); - _log.debug("writing cell barcodes"); + _log.debug("writing cell barcodes, using file: " + perCellTsv.getPath()); try (CSVWriter writer = new CSVWriter(PrintWriters.getPrintWriter(cellBarcodeWhitelist), ',', CSVWriter.NO_QUOTE_CHARACTER); CSVReader reader = new CSVReader(Readers.getReader(perCellTsv), ',')) { int rowIdx = 0; diff --git a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJWrapper.java b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJWrapper.java index 613f72914..64c53ba96 100644 --- a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJWrapper.java +++ b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJWrapper.java @@ -68,6 +68,7 @@ public CellRangerVDJWrapper(@Nullable Logger logger) public static final String TARGET_ASSAY = "targetAssay"; public static final String DELETE_EXISTING_ASSAY_DATA = "deleteExistingAssayData"; + public static final String INNER_ENRICHMENT_PRIMERS = "innerEnrichmentPrimers"; public static class VDJProvider extends AbstractAlignmentStepProvider { @@ -82,6 +83,9 @@ public VDJProvider() ToolParameterDescriptor.createCommandLineParam(CommandLineParam.create("--force-cells"), "force-cells", "Force Cells", "Force pipeline to use this number of cells, bypassing the cell detection algorithm. Use this if the number of cells estimated by Cell Ranger is not consistent with the barcode rank plot.", "ldk-integerfield", new JSONObject(){{ put("minValue", 0); }}, null), + ToolParameterDescriptor.create(INNER_ENRICHMENT_PRIMERS, "Inner Enrichment Primers", "An option comma-separated list of the inner primers used for TCR enrichment. These will be used for trimming.", "textfield", new JSONObject(){{ + + }}, true), ToolParameterDescriptor.create(TARGET_ASSAY, "Target Assay", "Results will be loaded into this assay. If no assay is selected, a table will be created with nothing in the DB.", "tcr-assayselectorfield", new JSONObject(){{ put("autoSelectAssay", false); }}, null), @@ -296,6 +300,29 @@ public AlignmentStep.AlignmentOutput performAlignment(Readset rs, File inputFast File indexDir = AlignerIndexUtil.getIndexDir(referenceGenome, getIndexCachedDirName(getPipelineCtx().getJob())); args.add("--reference=" + indexDir.getPath()); + String primers = StringUtils.trimToNull(getProvider().getParameterByName(INNER_ENRICHMENT_PRIMERS).extractValue(getPipelineCtx().getJob(), getProvider(), getStepIdx(), String.class, null)); + if (primers != null) + { + File primerFile = new File(outputDirectory, "primers.txt"); + try (PrintWriter writer = PrintWriters.getPrintWriter(primerFile)) + { + Arrays.stream(primers.split(",")).forEach(x -> { + x = StringUtils.trimToNull(x); + if (x != null) + { + writer.println(x); + } + }); + } + catch (IOException e) + { + throw new PipelineJobException(e); + } + + output.addIntermediateFile(primerFile); + args.add("--inner-enrichment-primers=" + primerFile.getPath()); + } + args.addAll(getClientCommandArgs("=")); Integer maxThreads = SequencePipelineService.get().getMaxThreads(getPipelineCtx().getLogger()); From cbf62febf88f5107cd9fbfdf2018175b53d266b0 Mon Sep 17 00:00:00 2001 From: bbimber Date: Wed, 4 Nov 2020 11:38:41 -0800 Subject: [PATCH 02/98] Add support for cellranger vdj --disable-ui --- tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJWrapper.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJWrapper.java b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJWrapper.java index 64c53ba96..a50cc0f23 100644 --- a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJWrapper.java +++ b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJWrapper.java @@ -83,6 +83,9 @@ public VDJProvider() ToolParameterDescriptor.createCommandLineParam(CommandLineParam.create("--force-cells"), "force-cells", "Force Cells", "Force pipeline to use this number of cells, bypassing the cell detection algorithm. Use this if the number of cells estimated by Cell Ranger is not consistent with the barcode rank plot.", "ldk-integerfield", new JSONObject(){{ put("minValue", 0); }}, null), + ToolParameterDescriptor.createCommandLineParam(CommandLineParam.create("--disable--ui"), "disable--ui", "Disable UI", "If checked, this will run cellranger with the optional web-based UI disabled.", "checkbox", new JSONObject(){{ + put("checked", true); + }}, true), ToolParameterDescriptor.create(INNER_ENRICHMENT_PRIMERS, "Inner Enrichment Primers", "An option comma-separated list of the inner primers used for TCR enrichment. These will be used for trimming.", "textfield", new JSONObject(){{ }}, true), From 633d7f4344d56503f6d849ad93e68a33e57b58da Mon Sep 17 00:00:00 2001 From: bbimber Date: Wed, 4 Nov 2020 12:11:06 -0800 Subject: [PATCH 03/98] Larger input and validation for cellranger primer input --- .../labkey/tcrdb/pipeline/CellRangerVDJWrapper.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJWrapper.java b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJWrapper.java index a50cc0f23..7b9c53fbf 100644 --- a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJWrapper.java +++ b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJWrapper.java @@ -86,9 +86,10 @@ public VDJProvider() ToolParameterDescriptor.createCommandLineParam(CommandLineParam.create("--disable--ui"), "disable--ui", "Disable UI", "If checked, this will run cellranger with the optional web-based UI disabled.", "checkbox", new JSONObject(){{ put("checked", true); }}, true), - ToolParameterDescriptor.create(INNER_ENRICHMENT_PRIMERS, "Inner Enrichment Primers", "An option comma-separated list of the inner primers used for TCR enrichment. These will be used for trimming.", "textfield", new JSONObject(){{ - - }}, true), + ToolParameterDescriptor.create(INNER_ENRICHMENT_PRIMERS, "Inner Enrichment Primers", "An option comma-separated list of the inner primers used for TCR enrichment. These will be used for trimming.", "textarea", new JSONObject(){{ + put("height", 100); + put("width", 400); + }}, null), ToolParameterDescriptor.create(TARGET_ASSAY, "Target Assay", "Results will be loaded into this assay. If no assay is selected, a table will be created with nothing in the DB.", "tcr-assayselectorfield", new JSONObject(){{ put("autoSelectAssay", false); }}, null), @@ -306,6 +307,9 @@ public AlignmentStep.AlignmentOutput performAlignment(Readset rs, File inputFast String primers = StringUtils.trimToNull(getProvider().getParameterByName(INNER_ENRICHMENT_PRIMERS).extractValue(getPipelineCtx().getJob(), getProvider(), getStepIdx(), String.class, null)); if (primers != null) { + primers = primers.replaceAll("\\s+", ","); + primers = primers.replaceAll(",+", ","); + File primerFile = new File(outputDirectory, "primers.txt"); try (PrintWriter writer = PrintWriters.getPrintWriter(primerFile)) { From 039f82a8c61cb89cfa98115dd909096e7b8969e4 Mon Sep 17 00:00:00 2001 From: bbimber Date: Wed, 4 Nov 2020 12:12:51 -0800 Subject: [PATCH 04/98] cellranger param is a switch --- tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJWrapper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJWrapper.java b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJWrapper.java index 7b9c53fbf..8ebd7ea51 100644 --- a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJWrapper.java +++ b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJWrapper.java @@ -83,7 +83,7 @@ public VDJProvider() ToolParameterDescriptor.createCommandLineParam(CommandLineParam.create("--force-cells"), "force-cells", "Force Cells", "Force pipeline to use this number of cells, bypassing the cell detection algorithm. Use this if the number of cells estimated by Cell Ranger is not consistent with the barcode rank plot.", "ldk-integerfield", new JSONObject(){{ put("minValue", 0); }}, null), - ToolParameterDescriptor.createCommandLineParam(CommandLineParam.create("--disable--ui"), "disable--ui", "Disable UI", "If checked, this will run cellranger with the optional web-based UI disabled.", "checkbox", new JSONObject(){{ + ToolParameterDescriptor.createCommandLineParam(CommandLineParam.createSwitch("--disable--ui"), "disable--ui", "Disable UI", "If checked, this will run cellranger with the optional web-based UI disabled.", "checkbox", new JSONObject(){{ put("checked", true); }}, true), ToolParameterDescriptor.create(INNER_ENRICHMENT_PRIMERS, "Inner Enrichment Primers", "An option comma-separated list of the inner primers used for TCR enrichment. These will be used for trimming.", "textarea", new JSONObject(){{ From e0cb5a706759c210038b5dca9555da68903ebdc2 Mon Sep 17 00:00:00 2001 From: bbimber Date: Wed, 4 Nov 2020 13:21:10 -0800 Subject: [PATCH 05/98] Fix param name --- tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJWrapper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJWrapper.java b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJWrapper.java index 8ebd7ea51..3a7cdd341 100644 --- a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJWrapper.java +++ b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJWrapper.java @@ -83,7 +83,7 @@ public VDJProvider() ToolParameterDescriptor.createCommandLineParam(CommandLineParam.create("--force-cells"), "force-cells", "Force Cells", "Force pipeline to use this number of cells, bypassing the cell detection algorithm. Use this if the number of cells estimated by Cell Ranger is not consistent with the barcode rank plot.", "ldk-integerfield", new JSONObject(){{ put("minValue", 0); }}, null), - ToolParameterDescriptor.createCommandLineParam(CommandLineParam.createSwitch("--disable--ui"), "disable--ui", "Disable UI", "If checked, this will run cellranger with the optional web-based UI disabled.", "checkbox", new JSONObject(){{ + ToolParameterDescriptor.createCommandLineParam(CommandLineParam.createSwitch("--disable-ui"), "disable-ui", "Disable UI", "If checked, this will run cellranger with the optional web-based UI disabled.", "checkbox", new JSONObject(){{ put("checked", true); }}, true), ToolParameterDescriptor.create(INNER_ENRICHMENT_PRIMERS, "Inner Enrichment Primers", "An option comma-separated list of the inner primers used for TCR enrichment. These will be used for trimming.", "textarea", new JSONObject(){{ From 54663869eacd141ea45ce7b0bbe9b12bd73f1eb3 Mon Sep 17 00:00:00 2001 From: Adam Rauch Date: Wed, 4 Nov 2020 14:06:28 -0800 Subject: [PATCH 06/98] Use new signature (#53) --- GenotypeAssays/module.properties | 1 - elispot_assay/module.properties | 1 - flowassays/module.properties | 1 - mGAP/module.properties | 1 - primeseq/module.properties | 1 - tcrdb/module.properties | 1 - .../src/org/labkey/tcrdb/TCRdbController.java | 52 ++++----- .../tcrdb/pipeline/CellRangerVDJWrapper.java | 4 +- variantdb/module.properties | 1 - .../labkey/variantdb/VariantDBManager.java | 101 +++++++++--------- 10 files changed, 73 insertions(+), 91 deletions(-) diff --git a/GenotypeAssays/module.properties b/GenotypeAssays/module.properties index 9a8f3c517..26b7527df 100644 --- a/GenotypeAssays/module.properties +++ b/GenotypeAssays/module.properties @@ -1,3 +1,2 @@ ModuleClass: org.labkey.genotypeassays.GenotypeAssaysModule -ConsolidateScripts: false ManageVersion: false diff --git a/elispot_assay/module.properties b/elispot_assay/module.properties index 77a02d564..4f2d4f3f5 100644 --- a/elispot_assay/module.properties +++ b/elispot_assay/module.properties @@ -1,3 +1,2 @@ ModuleClass: org.labkey.elispot_assay.ELISPOT_AssayModule -ConsolidateScripts: false ManageVersion: false diff --git a/flowassays/module.properties b/flowassays/module.properties index 384050e17..df44cb72e 100644 --- a/flowassays/module.properties +++ b/flowassays/module.properties @@ -1,3 +1,2 @@ ModuleClass: org.labkey.flowassays.FlowAssaysModule -ConsolidateScripts: false ManageVersion: false diff --git a/mGAP/module.properties b/mGAP/module.properties index 5378792a1..18b50ca5b 100644 --- a/mGAP/module.properties +++ b/mGAP/module.properties @@ -1,5 +1,4 @@ ModuleClass: org.labkey.mgap.mGAPModule License: Apache 2.0 LicenseURL: http://www.apache.org/licenses/LICENSE-2.0 -ConsolidateScripts: false ManageVersion: false diff --git a/primeseq/module.properties b/primeseq/module.properties index 6e8b00db6..47985036b 100644 --- a/primeseq/module.properties +++ b/primeseq/module.properties @@ -3,5 +3,4 @@ Label: PRIMe-Seq Description: This module contains code related to our internal server, PRIMe-Seq License: Apache 2.0 LicenseURL: http://www.apache.org/licenses/LICENSE-2.0 -ConsolidateScripts: false ManageVersion: false \ No newline at end of file diff --git a/tcrdb/module.properties b/tcrdb/module.properties index 7ee0ae800..b282909c1 100644 --- a/tcrdb/module.properties +++ b/tcrdb/module.properties @@ -3,5 +3,4 @@ Label: TCRdb Description: The TCRdb module is designed to manage TCR sequence from either clones or populations. License: Apache 2.0 LicenseURL: http://www.apache.org/licenses/LICENSE-2.0 -ConsolidateScripts: false ManageVersion: false diff --git a/tcrdb/src/org/labkey/tcrdb/TCRdbController.java b/tcrdb/src/org/labkey/tcrdb/TCRdbController.java index 59b0acc6b..250069efd 100644 --- a/tcrdb/src/org/labkey/tcrdb/TCRdbController.java +++ b/tcrdb/src/org/labkey/tcrdb/TCRdbController.java @@ -136,44 +136,38 @@ public ModelAndView getView(ExportAlignmentsForm form, BindException errors) thr TableInfo ti = us.getTable("data"); final Map> VDJMap = new HashMap<>(); - List rowIds = new ArrayList<>(); - rowIds.addAll(Arrays.asList(form.getAssayRowIds())); + List rowIds = new ArrayList<>(Arrays.asList(form.getAssayRowIds())); TableSelector ts = new TableSelector(ti, new SimpleFilter(FieldKey.fromString("rowid"), rowIds, CompareType.IN), null); final StringWriter writer = new StringWriter(); - ts.forEach(new Selector.ForEachBlock() - { - @Override - public void exec(AssayRecord r) throws SQLException, StopIteratingException + ts.forEach(AssayRecord.class, r -> { + if (r.getVdjFile() == null) { - if (r.getVdjFile() == null) - { - writer.write("ERROR: Row lacks VDJCA file: " + r.getRowId() + "\n"); - return; - } - - ExpData d = ExperimentService.get().getExpData(r.getVdjFile()); - if (d == null) - { - writer.write("ERROR: Unable to find VDJCA file for row: " + r.getRowId() + ", ExpData: " + r.getVdjFile() + "\n"); - return; - } + writer.write("ERROR: Row lacks VDJCA file: " + r.getRowId() + "\n"); + return; + } - if (!d.getFile().exists()) - { - writer.write("ERROR: Unable to find VDJCA file for row: " + r.getRowId() + ", file does not exist: " + d.getFile().getPath() + "\n"); - return; - } + ExpData d = ExperimentService.get().getExpData(r.getVdjFile()); + if (d == null) + { + writer.write("ERROR: Unable to find VDJCA file for row: " + r.getRowId() + ", ExpData: " + r.getVdjFile() + "\n"); + return; + } - if (!VDJMap.containsKey(d.getFile())) - { - VDJMap.put(d.getFile(), new ArrayList<>()); - } + if (!d.getFile().exists()) + { + writer.write("ERROR: Unable to find VDJCA file for row: " + r.getRowId() + ", file does not exist: " + d.getFile().getPath() + "\n"); + return; + } - VDJMap.get(d.getFile()).add(r); + if (!VDJMap.containsKey(d.getFile())) + { + VDJMap.put(d.getFile(), new ArrayList<>()); } - }, AssayRecord.class); + + VDJMap.get(d.getFile()).add(r); + }); if (VDJMap.isEmpty()) { diff --git a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJWrapper.java b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJWrapper.java index 1fc0aaf0d..b63bdedf5 100644 --- a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJWrapper.java +++ b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJWrapper.java @@ -154,7 +154,7 @@ public void init(SequenceAnalysisJobSupport support) throws PipelineJobException final AtomicInteger i = new AtomicInteger(0); UserSchema us = QueryService.get().getUserSchema(getPipelineCtx().getJob().getUser(), getPipelineCtx().getJob().getContainer(), "sequenceanalysis"); List seqIds = new TableSelector(us.getTable("reference_library_members", null), PageFlowUtil.set("ref_nt_id"), new SimpleFilter(FieldKey.fromString("library_id"), referenceGenome.getGenomeId()), null).getArrayList(Integer.class); - new TableSelector(us.getTable("ref_nt_sequences", null), new SimpleFilter(FieldKey.fromString("rowid"), seqIds, CompareType.IN), null).forEach(nt -> { + new TableSelector(us.getTable("ref_nt_sequences", null), new SimpleFilter(FieldKey.fromString("rowid"), seqIds, CompareType.IN), null).forEach(RefNtSequenceModel.class, nt -> { if (nt.getLocus() == null) { @@ -209,7 +209,7 @@ else if (nt.getLineage().contains("D")) writer.write(seq + "\n"); } nt.clearCachedSequence(); - }, RefNtSequenceModel.class); + }); } catch (IllegalArgumentException | IOException e) { diff --git a/variantdb/module.properties b/variantdb/module.properties index 251fad2e6..b3faf5d00 100644 --- a/variantdb/module.properties +++ b/variantdb/module.properties @@ -1,3 +1,2 @@ ModuleClass: org.labkey.variantdb.VariantDBModule -ConsolidateScripts: false ManageVersion: false diff --git a/variantdb/src/org/labkey/variantdb/VariantDBManager.java b/variantdb/src/org/labkey/variantdb/VariantDBManager.java index 984551cb2..8a5445e20 100644 --- a/variantdb/src/org/labkey/variantdb/VariantDBManager.java +++ b/variantdb/src/org/labkey/variantdb/VariantDBManager.java @@ -149,71 +149,66 @@ public void exec(ResultSet rs) throws SQLException final Pair matches = Pair.of(0, 0); TableSelector variantTs = new TableSelector(VariantDBSchema.getInstance().getSchema().getTable(VariantDBSchema.TABLE_VARIANTS), variantFilter, null); - variantTs.forEach(new Selector.ForEachBlock() - { - @Override - public void exec(Variant v) throws SQLException + variantTs.forEach(Variant.class, v -> { + String name = resolveSequenceName(v.getSequenceId()); + if (name != null) { - String name = resolveSequenceName(v.getSequenceId()); - if (name != null) - { - matches.second++; + matches.second++; - //only delete once - deletePs.setString(1, v.getObjectid()); - deletePs.addBatch(); + //only delete once + deletePs.setString(1, v.getObjectid()); + deletePs.addBatch(); - v.setSequenceName(name); - for (Integer targetId : liftOverMap.keySet()) + v.setSequenceName(name); + for (Integer targetId : liftOverMap.keySet()) + { + LiftedVariant lv = VariantDBManager.get().liftOverVariant(liftOverMap.get(targetId), v, chainFileMap.get(targetId)); + if (lv.successfulLiftover()) { - LiftedVariant lv = VariantDBManager.get().liftOverVariant(liftOverMap.get(targetId), v, chainFileMap.get(targetId)); - if (lv.successfulLiftover()) - { - matches.first++; - } - - //variantid, sequenceid, startPosition, endPosition, reference, allele, referenceVariantId, referenceAlleleId, batchId, chainFile, created, createdBy, modified, modifiedBy - insertPs.setString(1, v.getObjectid()); - if (lv.successfulLiftover()) - { - insertPs.setInt(2, lv.getSequenceId()); - insertPs.setInt(3, lv.getStartPosition()); - insertPs.setInt(4, lv.getEndPosition()); - } - else - { - insertPs.setInt(2, -1); - insertPs.setInt(3, 0); - insertPs.setInt(4, 0); - } - insertPs.setString(5, null); - insertPs.setString(6, null); - - insertPs.setString(7, v.getReferenceVariantId()); - insertPs.setString(8, v.getReferenceAlleleId()); - insertPs.setString(9, batchId); - insertPs.setInt(10, lv.getChainFile()); - insertPs.setDate(11, new Date(System.currentTimeMillis())); - insertPs.setInt(12, u.getUserId()); - insertPs.setDate(13, new Date(System.currentTimeMillis())); - insertPs.setInt(14, u.getUserId()); - - insertPs.addBatch(); + matches.first++; } - if (matches.second % batchSize == 0) + //variantid, sequenceid, startPosition, endPosition, reference, allele, referenceVariantId, referenceAlleleId, batchId, chainFile, created, createdBy, modified, modifiedBy + insertPs.setString(1, v.getObjectid()); + if (lv.successfulLiftover()) + { + insertPs.setInt(2, lv.getSequenceId()); + insertPs.setInt(3, lv.getStartPosition()); + insertPs.setInt(4, lv.getEndPosition()); + } + else { - log.info("processed: " + matches.second + " variants"); - deletePs.executeBatch(); - insertPs.executeBatch(); + insertPs.setInt(2, -1); + insertPs.setInt(3, 0); + insertPs.setInt(4, 0); } + insertPs.setString(5, null); + insertPs.setString(6, null); + + insertPs.setString(7, v.getReferenceVariantId()); + insertPs.setString(8, v.getReferenceAlleleId()); + insertPs.setString(9, batchId); + insertPs.setInt(10, lv.getChainFile()); + insertPs.setDate(11, new Date(System.currentTimeMillis())); + insertPs.setInt(12, u.getUserId()); + insertPs.setDate(13, new Date(System.currentTimeMillis())); + insertPs.setInt(14, u.getUserId()); + + insertPs.addBatch(); } - else + + if (matches.second % batchSize == 0) { - log.error("unable to resolve sequenceId: " + v.getSequenceId()); + log.info("processed: " + matches.second + " variants"); + deletePs.executeBatch(); + insertPs.executeBatch(); } } - }, Variant.class); + else + { + log.error("unable to resolve sequenceId: " + v.getSequenceId()); + } + }); //execute any remaining commands log.info("processed: " + matches.second + " variants"); From 74e310e2ac5d5358bc2d81852c545919cc7eeefe Mon Sep 17 00:00:00 2001 From: bbimber Date: Fri, 6 Nov 2020 15:11:32 -0800 Subject: [PATCH 07/98] set markdown output format --- .../src/org/labkey/tcrdb/pipeline/CellRangerSeuratHandler.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerSeuratHandler.java b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerSeuratHandler.java index 7d49bd571..06c8236e1 100644 --- a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerSeuratHandler.java +++ b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerSeuratHandler.java @@ -447,7 +447,7 @@ public void processFilesRemote(List inputFiles, JobContext c writer.println(); writer.println("setwd('/work')"); - writer.println("rmarkdown::render('" + rmdScript.getName() + "', clean=TRUE, output_file='" + outHtml.getName() + "')"); + writer.println("rmarkdown::render('" + rmdScript.getName() + "', clean=TRUE, output_format = 'html_document', output_file='" + outHtml.getName() + "')"); } catch (IOException e) { From 579191438a12956ffc9c3445d9c2cd27d24566df Mon Sep 17 00:00:00 2001 From: bbimber Date: Sat, 7 Nov 2020 15:35:57 -0800 Subject: [PATCH 08/98] If user supplies GTF file to seurat, use this instead of inferring --- .../pipeline/CellRangerSeuratHandler.java | 56 ++++++++++--------- 1 file changed, 29 insertions(+), 27 deletions(-) diff --git a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerSeuratHandler.java b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerSeuratHandler.java index 06c8236e1..8f988337a 100644 --- a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerSeuratHandler.java +++ b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerSeuratHandler.java @@ -60,6 +60,7 @@ public class CellRangerSeuratHandler extends AbstractParameterizedOutputHandler< { private FileType _fileType = new FileType("cloupe", false); public static final String SEURAT_MAX_THREADS = "seuratMaxThreads"; + private static final String GTF_FILE_ID = "gtfFileId"; public CellRangerSeuratHandler() { @@ -103,11 +104,11 @@ private static List getDefaultParams() put("storeValues", "simple;cca"); }}, "simple"), ToolParameterDescriptor.create(SEURAT_MAX_THREADS, "Seurat Max Threads", "Because seurat can behave badly with multiple threads, this allows a separate cap to be used from the main job. This will allow CITE-Seq-Count and other tools to run with more threads.", "ldk-integerfield", null, 1), - ToolParameterDescriptor.createExpDataParam("gtfFile", "Gene File", "This is the ID of a GTF file containing genes from this genome.", "sequenceanalysis-genomefileselectorfield", new JSONObject() + ToolParameterDescriptor.createExpDataParam(GTF_FILE_ID, "Gene File", "This is the ID of a GTF file containing genes from this genome.", "sequenceanalysis-genomefileselectorfield", new JSONObject() {{ put("extensions", Arrays.asList("gtf")); put("width", 400); - put("allowBlank", false); + put("allowBlank", true); }}, null) )); @@ -165,8 +166,6 @@ public boolean doSplitJobs() public class Processor implements SequenceOutputProcessor { - private static final String GTF_FILE_ID = "gtfFileIf"; - @Override public void init(PipelineJob job, SequenceAnalysisJobSupport support, List inputFiles, JSONObject params, File outputDir, List actions, List outputsToCreate) throws UnsupportedOperationException, PipelineJobException { @@ -184,19 +183,16 @@ public void init(PipelineJob job, SequenceAnalysisJobSupport support, List gtfIds = new HashSet<>(); - for (SequenceOutputFile so : inputFiles) + if (params.get(GTF_FILE_ID) == null) { - ExpData gtf = null; - ExpRun run = ExperimentService.get().getExpRun(so.getRunId()); - if (run != null) + job.getLogger().info("attempting to infer GTF:"); + + Set gtfIds = new HashSet<>(); + for (SequenceOutputFile so : inputFiles) { - List gtfDatas = run.getInputDatas("GTF File", null); - if (!gtfDatas.isEmpty()) - { - gtf = gtfDatas.get(0); - } - else + ExpData gtf = null; + ExpRun run = ExperimentService.get().getExpRun(so.getRunId()); + if (run != null) { //Because existing runs didnt explicitly track GTF as an input, try to infer: PipelineStatusFile sf = PipelineService.get().getStatusFile(run.getJobId()); @@ -226,23 +222,23 @@ public void init(PipelineJob job, SequenceAnalysisJobSupport support, List inputFiles, JobContext c RecordedAction action = new RecordedAction(getName()); ctx.addActions(action); - int gtfId = ctx.getSequenceSupport().getCachedObject(GTF_FILE_ID, Integer.class); + int gtfId = ctx.getParams().optInt(GTF_FILE_ID, -1); + if (gtfId == -1) + { + ctx.getLogger().debug("GTF file was not specified, defaulting to inferred file"); + gtfId = ctx.getSequenceSupport().getCachedObject(GTF_FILE_ID, Integer.class); + } + File gtfFile = ctx.getSequenceSupport().getCachedData(gtfId); if (!gtfFile.exists()) { From 035bc9d14bb116d15aad633a2bc4c205403f0cd9 Mon Sep 17 00:00:00 2001 From: bbimber Date: Tue, 10 Nov 2020 08:44:30 -0800 Subject: [PATCH 09/98] Add more options to LoFreq analysis --- tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerSeuratHandler.java | 1 + 1 file changed, 1 insertion(+) diff --git a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerSeuratHandler.java b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerSeuratHandler.java index 8f988337a..e9b98234d 100644 --- a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerSeuratHandler.java +++ b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerSeuratHandler.java @@ -187,6 +187,7 @@ public void init(PipelineJob job, SequenceAnalysisJobSupport support, List gtfIds = new HashSet<>(); for (SequenceOutputFile so : inputFiles) { From b653dac595e315eafb38fd56e54aa08fe7045feb Mon Sep 17 00:00:00 2001 From: bbimber Date: Wed, 11 Nov 2020 14:23:28 -0800 Subject: [PATCH 10/98] Cell hashing calling should not be performed when only one HTO is actually used --- .../org/labkey/tcrdb/pipeline/CellRangerSeuratHandler.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerSeuratHandler.java b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerSeuratHandler.java index e9b98234d..3330051a9 100644 --- a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerSeuratHandler.java +++ b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerSeuratHandler.java @@ -692,11 +692,15 @@ else if (rs.getReadsetId() == null) throw new PipelineJobException(e); } - if (htosForReadset > 0) + if (htosForReadset > 1) { ctx.getLogger().info("Total HTOs for readset: " + htosForReadset); finalCalls.put(barcodePrefix, CellRangerCellHashingHandler.processBarcodeFile(ctx, barcodes, rs, htoReadset, so.getLibrary_id(), action, getClientCommandArgs(ctx.getParams()), false, SeuratCellHashingHandler.CATEGORY, true, perReadsetHtos, true)); } + else if (htosForReadset == 1) + { + ctx.getLogger().info("Only single HTO used for lane, skipping cell hashing calling"); + } else { ctx.getLogger().info("No HTOs found for readset"); From 18746c72b7966f289068d6e1348ab00a31ff540f Mon Sep 17 00:00:00 2001 From: bbimber Date: Fri, 13 Nov 2020 10:39:38 -0800 Subject: [PATCH 11/98] Bugfix TagPcr metrics file parsing --- tcrdb/resources/external/scRNAseq/Seurat3.rmd | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tcrdb/resources/external/scRNAseq/Seurat3.rmd b/tcrdb/resources/external/scRNAseq/Seurat3.rmd index a3911a6c7..e06fddaf7 100644 --- a/tcrdb/resources/external/scRNAseq/Seurat3.rmd +++ b/tcrdb/resources/external/scRNAseq/Seurat3.rmd @@ -1,13 +1,16 @@ --- title: 'Seurat scRNA-seq Analysis' +output: html_document + --- ```{r Setup} -knitr::opts_chunk$set(message=FALSE, warning=FALSE,echo=TRUE,error = FALSE) library(knitr) library(OOSAP) +knitr::opts_chunk$set(message=FALSE, warning=FALSE, echo=TRUE, error = FALSE) + cores <- Sys.getenv('SEQUENCEANALYSIS_MAX_THREADS') if (cores != ''){ print(paste0('Setting future::plan to ', cores, ' cores')) From 5ffa101e7ee3a1b49a66ddcc79a9b885638f00f6 Mon Sep 17 00:00:00 2001 From: bbimber Date: Mon, 16 Nov 2020 09:58:01 -0800 Subject: [PATCH 12/98] Show error in rmarkdown/html for better debugging --- tcrdb/resources/external/scRNAseq/Seurat3.rmd | 2 +- .../org/labkey/tcrdb/pipeline/CellRangerCellHashingHandler.java | 2 +- tcrdb/src/org/labkey/tcrdb/pipeline/SeuratCiteSeqHandler.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tcrdb/resources/external/scRNAseq/Seurat3.rmd b/tcrdb/resources/external/scRNAseq/Seurat3.rmd index e06fddaf7..9fb58d5aa 100644 --- a/tcrdb/resources/external/scRNAseq/Seurat3.rmd +++ b/tcrdb/resources/external/scRNAseq/Seurat3.rmd @@ -9,7 +9,7 @@ output: html_document library(knitr) library(OOSAP) -knitr::opts_chunk$set(message=FALSE, warning=FALSE, echo=TRUE, error = FALSE) +knitr::opts_chunk$set(message=FALSE, warning=FALSE, echo=TRUE, error = TRUE) cores <- Sys.getenv('SEQUENCEANALYSIS_MAX_THREADS') if (cores != ''){ diff --git a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerCellHashingHandler.java b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerCellHashingHandler.java index 79c2bcc53..3cd2ecf90 100644 --- a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerCellHashingHandler.java +++ b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerCellHashingHandler.java @@ -63,7 +63,7 @@ public static List getDefaultHashingParams(boolean incl ToolParameterDescriptor.create("scanEditDistances", "Scan Edit Distances", "If checked, CITE-seq-count will be run using edit distances from 0-3 and the iteration with the highest singlets will be used.", "checkbox", new JSONObject(){{ put("checked", false); }}, false), - ToolParameterDescriptor.create("editDistance", "Edit Distance", null, "ldk-integerfield", null, 3), + ToolParameterDescriptor.create("editDistance", "Edit Distance", null, "ldk-integerfield", null, 2), ToolParameterDescriptor.create("minCountPerCell", "Min Reads/Cell", null, "ldk-integerfield", null, 5), ToolParameterDescriptor.create("useSeurat", "Use Seurat Calling", "If checked, the seurat HTO calling algorithm will be used.", "checkbox", null, true), ToolParameterDescriptor.create("useMultiSeq", "Use MultiSeq Calling", "If checked, the MultiSeq HTO calling algorithm will be used.", "checkbox", null, true) diff --git a/tcrdb/src/org/labkey/tcrdb/pipeline/SeuratCiteSeqHandler.java b/tcrdb/src/org/labkey/tcrdb/pipeline/SeuratCiteSeqHandler.java index 853a68a0b..582b5dd92 100644 --- a/tcrdb/src/org/labkey/tcrdb/pipeline/SeuratCiteSeqHandler.java +++ b/tcrdb/src/org/labkey/tcrdb/pipeline/SeuratCiteSeqHandler.java @@ -27,7 +27,7 @@ public class SeuratCiteSeqHandler extends AbstractParameterizedOutputHandler Date: Tue, 17 Nov 2020 14:23:28 -0800 Subject: [PATCH 13/98] Allow wildcards when specifying plate lists --- .../web/tcrdb/panel/LibraryExportPanel.js | 63 ++++++++++++++++++- 1 file changed, 61 insertions(+), 2 deletions(-) diff --git a/tcrdb/resources/web/tcrdb/panel/LibraryExportPanel.js b/tcrdb/resources/web/tcrdb/panel/LibraryExportPanel.js index 4988c82e5..b17e8cf4a 100644 --- a/tcrdb/resources/web/tcrdb/panel/LibraryExportPanel.js +++ b/tcrdb/resources/web/tcrdb/panel/LibraryExportPanel.js @@ -88,7 +88,7 @@ Ext4.define('TCRdb.panel.LibraryExportPanel', { border: false }, items: [{ - html: 'Add an ordered list of plates, using tab-delimited columns. The first column(s) are plate ID and library type (GEX, VDJ, CITE, or HTO). These can either be one column (i.e. G234-1, C234-1, H234-1, or T234-1), or as two columns (234-1 GEX or 234-1 HTO). An optional next column is the lane assignment (i.e. Novaseq1, HiSeq1, HiSeq2). Finally, an optional final column can be used to provide the alias for this pool. This is mostly used for CITE-Seq/HTOs, where multiple libraries are pre-pooled. See these examples:
' + + html: 'Add an ordered list of plates, using tab-delimited columns. The first column(s) are plate ID and library type (GEX, VDJ, CITE, or HTO). These can either be one column (i.e. G234-1, C234-1, H234-1, or T234-1), or as two columns (234-1 GEX or 234-1 HTO). An optional next column is the lane assignment (i.e. Novaseq1, HiSeq1, HiSeq2). Finally, an optional final column can be used to provide the alias for this pool. This is mostly used for CITE-Seq/HTOs, where multiple libraries are pre-pooled. Note, a wildcard can be used to specify all plates beginning with that prefix. See these examples:
' + '
' +
                                                 '234-2\tGEX
' +
                                                 '234-2\tVDJ
' +
@@ -101,6 +101,7 @@ Ext4.define('TCRdb.panel.LibraryExportPanel', {
                                                 '235-2\tHTO\tHiSeq2\tBNB-HTO-1
' +
                                                 'H235-2\tHiSeq1\tBNB-HTO-1
' +
                                                 'C235-2\tHiSeq1\tBNB-HTO-1' +
+                                                'C235-*\tHiSeq2\tBNB-HTO-2' +
                                             '
', border: false },{ @@ -171,6 +172,7 @@ Ext4.define('TCRdb.panel.LibraryExportPanel', { }, this); var hadError = false; + var wildcards = {}; Ext4.Array.forEach(text, function(r){ if (r.length < 2){ hadError = true; @@ -186,6 +188,12 @@ Ext4.define('TCRdb.panel.LibraryExportPanel', { Ext4.Array.forEach(r, function(val, idx){ r[idx] = Ext4.String.trim(val); }, this); + + if (r[0].match('\\*$')) { + var m = r[0].match('\\*$'); + var val = r[0].substr(0, m.index); + wildcards[val] = r; + } }, this); if (hadError) { @@ -193,7 +201,58 @@ Ext4.define('TCRdb.panel.LibraryExportPanel', { return; } - this.onSubmit(btn, text); + if (!Ext4.Object.isEmpty(wildcards)) { + LABKEY.Query.selectRows({ + method: 'POST', + containerPath: Laboratory.Utils.getQueryContainerPath(), + schemaName: 'tcrdb', + queryName: 'cdnas', + columns: 'rowid,plateId', + filterArray: [LABKEY.Filter.create('plateId', Ext4.Object.getKeys(wildcards).join(';'), LABKEY.Filter.Types.CONTAINS_ONE_OF)], + scope: this, + failure: LDK.Utils.getErrorCallback(), + success: function (results) { + if (results.rows.length) { + var prefixToPlate = {}; + Ext4.Array.forEach(results.rows, function (row) { + Ext4.Array.forEach(Ext4.Object.getKeys(wildcards), function (prefix) { + if (row.plateId && row.plateId.includes(prefix)) { + prefix = prefix + '*'; + prefixToPlate[prefix] = prefixToPlate[prefix] || []; + prefixToPlate[prefix].push(row.plateId); + } + }, this); + }, this); + + Ext4.Array.forEach(Ext4.Object.getKeys(prefixToPlate), function (prefix) { + prefixToPlate[prefix] = Ext4.unique(prefixToPlate[prefix]); + }, this); + + var updatedText = []; + var prefixes = Ext4.Object.getKeys(prefixToPlate); + Ext4.Array.forEach(text, function (r, idx) { + var plateId = r[0]; + if (prefixes.indexOf(plateId) == -1) { + updatedText.push(r); + } + else { + Ext4.Array.forEach(prefixToPlate[plateId], function(newPlate){ + var r2 = [].concat(r); + r2[0] = newPlate; + updatedText.push(r2); + }, this); + } + }, this); + + text = updatedText; + } + + this.onSubmit(btn, text); + } + }); + } else { + + } } }] }); From eb4fe290a344b06ba73e38ae99c5fc3066664b4c Mon Sep 17 00:00:00 2001 From: bbimber Date: Tue, 17 Nov 2020 15:47:28 -0800 Subject: [PATCH 14/98] bugfix library export panel --- tcrdb/resources/web/tcrdb/panel/LibraryExportPanel.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tcrdb/resources/web/tcrdb/panel/LibraryExportPanel.js b/tcrdb/resources/web/tcrdb/panel/LibraryExportPanel.js index b17e8cf4a..2626e445d 100644 --- a/tcrdb/resources/web/tcrdb/panel/LibraryExportPanel.js +++ b/tcrdb/resources/web/tcrdb/panel/LibraryExportPanel.js @@ -251,7 +251,7 @@ Ext4.define('TCRdb.panel.LibraryExportPanel', { } }); } else { - + this.onSubmit(btn, text); } } }] From 6fdf04d0603ea99275b4e2a3f3e11389e3602854 Mon Sep 17 00:00:00 2001 From: bbimber Date: Wed, 18 Nov 2020 20:58:55 -0800 Subject: [PATCH 15/98] Reformat phiX --- tcrdb/resources/web/tcrdb/panel/LibraryExportPanel.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tcrdb/resources/web/tcrdb/panel/LibraryExportPanel.js b/tcrdb/resources/web/tcrdb/panel/LibraryExportPanel.js index 2626e445d..7e3354dc8 100644 --- a/tcrdb/resources/web/tcrdb/panel/LibraryExportPanel.js +++ b/tcrdb/resources/web/tcrdb/panel/LibraryExportPanel.js @@ -816,10 +816,10 @@ Ext4.define('TCRdb.panel.LibraryExportPanel', { var delim = instrument === 'Novogene' ? '\t' : ','; Ext4.Array.forEach(sortedRows, function (r) { - processType(readsetIds, rows, r, 'readsetId', 'GEX', 500, 1, 'G', null, false); - processType(readsetIds, rows, r, 'enrichedReadsetId', 'TCR', 700, 1, 'T', null, false); - processType(readsetIds, rows, r, 'hashingReadsetId', 'HTO', 182, 5, 'H', 'Cell hashing, 190bp amplicon. Please QC individually and pool in equal amounts per lane', true); - processType(readsetIds, rows, r, 'citeseqReadsetId', 'CITE', 182, 5, 'C', 'CITE-Seq, 190bp amplicon. Please QC individually and pool in equal amounts per lane', false); + processType(readsetIds, rows, r, 'readsetId', 'GEX', 500, 0.01, 'G', null, false); + processType(readsetIds, rows, r, 'enrichedReadsetId', 'TCR', 700, 0.01, 'T', null, false); + processType(readsetIds, rows, r, 'hashingReadsetId', 'HTO', 182, 0.05, 'H', 'Cell hashing, 190bp amplicon. Please QC individually and pool in equal amounts per lane', true); + processType(readsetIds, rows, r, 'citeseqReadsetId', 'CITE', 182, 0.05, 'C', 'CITE-Seq, 190bp amplicon. Please QC individually and pool in equal amounts per lane', false); }, this); //add missing barcodes: From 0f301adbdd3ca6c362142b3e28f8be250f336cf7 Mon Sep 17 00:00:00 2001 From: bbimber Date: Fri, 20 Nov 2020 13:17:11 -0800 Subject: [PATCH 16/98] Support custom cropping of F/R reads --- tcrdb/resources/web/tcrdb/panel/LibraryExportPanel.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tcrdb/resources/web/tcrdb/panel/LibraryExportPanel.js b/tcrdb/resources/web/tcrdb/panel/LibraryExportPanel.js index 7e3354dc8..e3d3b9934 100644 --- a/tcrdb/resources/web/tcrdb/panel/LibraryExportPanel.js +++ b/tcrdb/resources/web/tcrdb/panel/LibraryExportPanel.js @@ -324,11 +324,11 @@ Ext4.define('TCRdb.panel.LibraryExportPanel', { var instrument = btn.up('tcrdb-libraryexportpanel').down('#instrument').getValue(); var plateId = btn.up('tcrdb-libraryexportpanel').down('#sourcePlates').getValue(); var delim = 'TAB'; - var extention = 'txt'; + var extension = 'txt'; var split = '\t'; if (instrument !== 'NextSeq (MPSSR)') { delim = 'COMMA'; - extention = 'csv'; + extension = 'csv'; split = ','; } @@ -336,7 +336,7 @@ Ext4.define('TCRdb.panel.LibraryExportPanel', { var rows = LDK.Utils.CSVToArray(Ext4.String.trim(val), split); LABKEY.Utils.convertToTable({ - fileName: plateId + '.' + extention, + fileName: plateId + '.' + extension, rows: rows, delim: delim }); From 743ad06970856964dac47858291ada9a8288967f Mon Sep 17 00:00:00 2001 From: bbimber Date: Sun, 29 Nov 2020 22:43:40 -0800 Subject: [PATCH 17/98] Add action to build discvr modules using github actions --- .github/workflows/build.yml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 .github/workflows/build.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 000000000..b08a42c19 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,21 @@ +name: Build DISCVR +on: + push: + branches: [ * ] + pull_request: + branches: [ * ] +jobs: + sync-release-branches: + # See: https://help.github.com/en/actions/reference/contexts-and-expression-syntax-for-github-actions#github-context + # https://help.github.com/en/actions/configuring-and-managing-workflows/using-environment-variables#default-environment-variables + if: github.repository == 'BimberLabInternal/BimberLabKeyModules' + runs-on: ubuntu-latest + steps: + - name: "Build DISCVR" + uses: bimberlabinternal/DevOps/githubActions/discvr-build@master + with: + artifactory_user: ${{secrets.artifactory_user}} + artifactory_password: ${{secrets.artifactory_password}} + # NOTE: permissions are limited on the default secrets.GITHUB_TOKEN, including updating workflows, so use a personal access token + github_token: ${{ secrets.PAT }} + From d12ff73b85bc3bbd874b89449b241b16804f150d Mon Sep 17 00:00:00 2001 From: bbimber Date: Sun, 29 Nov 2020 22:45:53 -0800 Subject: [PATCH 18/98] Fix yml syntax --- .github/workflows/build.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b08a42c19..f4bfb67cd 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,9 +1,6 @@ name: Build DISCVR on: - push: - branches: [ * ] - pull_request: - branches: [ * ] + [ push, pull_request ] jobs: sync-release-branches: # See: https://help.github.com/en/actions/reference/contexts-and-expression-syntax-for-github-actions#github-context From 295eb252ee32f94729855f49a34585ee4c8d8e81 Mon Sep 17 00:00:00 2001 From: bbimber Date: Sun, 29 Nov 2020 22:51:06 -0800 Subject: [PATCH 19/98] Update yml syntax --- .github/workflows/build.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f4bfb67cd..49d0114fb 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,8 +1,7 @@ name: Build DISCVR -on: - [ push, pull_request ] +on: [ push, pull_request ] jobs: - sync-release-branches: + build-modules: # See: https://help.github.com/en/actions/reference/contexts-and-expression-syntax-for-github-actions#github-context # https://help.github.com/en/actions/configuring-and-managing-workflows/using-environment-variables#default-environment-variables if: github.repository == 'BimberLabInternal/BimberLabKeyModules' From b4e5e08e64fac48e499b0fdd4469d3cea9167a61 Mon Sep 17 00:00:00 2001 From: bbimber Date: Sun, 29 Nov 2020 22:53:04 -0800 Subject: [PATCH 20/98] Update yml syntax --- .github/workflows/build.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 49d0114fb..aca42609f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,5 +1,7 @@ name: Build DISCVR -on: [ push, pull_request ] +on: + push: + pull_request: jobs: build-modules: # See: https://help.github.com/en/actions/reference/contexts-and-expression-syntax-for-github-actions#github-context From 2d7ffc6ec751f506123986f04ad9663e308d5e43 Mon Sep 17 00:00:00 2001 From: bbimber Date: Mon, 30 Nov 2020 00:40:40 -0800 Subject: [PATCH 21/98] travisci -> github actions --- .travis.yml | 22 ---------------------- 1 file changed, 22 deletions(-) delete mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 4f4ef89f4..000000000 --- a/.travis.yml +++ /dev/null @@ -1,22 +0,0 @@ -language: java -dist: trusty -git: - depth: 9999999 -jdk: - - openjdk13 - -before_cache: - - rm -f $HOME/.gradle/caches/modules-2/modules-2.lock - - rm -fr $HOME/.gradle/caches/*/plugin-resolution/ - -cache: - directories: - - $HOME/.gradle/caches/ - - $HOME/.gradle/wrapper/ - - $HOME/.m2 - - $HOME/site-library - -install: skip -script: - - wget -O ./travis.sh https://github.com/bimberlabinternal/DevOps/raw/master/travisci/travis.sh - - bash ./travis.sh \ No newline at end of file From e832f7fdac890c8a1368a14b0a86a1654d9ed0d5 Mon Sep 17 00:00:00 2001 From: bbimber Date: Mon, 30 Nov 2020 07:38:56 -0800 Subject: [PATCH 22/98] Add dependabot config --- .github/dependabot.yml | 7 +++++++ elispot_assay/resources/credits/dependencies.txt | 2 ++ variantdb/resources/credits/dependencies.txt | 3 +++ 3 files changed, 12 insertions(+) create mode 100644 .github/dependabot.yml create mode 100644 elispot_assay/resources/credits/dependencies.txt create mode 100644 variantdb/resources/credits/dependencies.txt diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 000000000..583decfd1 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,7 @@ +version: 2 +updates: + # Maintain dependencies for GitHub Actions + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "daily" \ No newline at end of file diff --git a/elispot_assay/resources/credits/dependencies.txt b/elispot_assay/resources/credits/dependencies.txt new file mode 100644 index 000000000..2b459f3fc --- /dev/null +++ b/elispot_assay/resources/credits/dependencies.txt @@ -0,0 +1,2 @@ +# direct external dependencies for project :server:modules:BimberLabKeyModules:elispot_assay +commons-math3-3.6.1.jar diff --git a/variantdb/resources/credits/dependencies.txt b/variantdb/resources/credits/dependencies.txt new file mode 100644 index 000000000..4e8ead70b --- /dev/null +++ b/variantdb/resources/credits/dependencies.txt @@ -0,0 +1,3 @@ +# direct external dependencies for project :server:modules:BimberLabKeyModules:variantdb +commons-net-3.5.jar +commons-math3-3.6.1.jar From 6c71079d1137f4f64cae60bf062aa37e56418d4a Mon Sep 17 00:00:00 2001 From: bbimber Date: Mon, 7 Dec 2020 22:14:33 -0800 Subject: [PATCH 23/98] Update status badges --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index c498f9a50..b5f6334a8 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -[![Build Status](https://api.travis-ci.com/BimberLab/DiscvrLabKeyModules.svg)](https://travis-ci.com/BimberLab/DiscvrLabKeyModules) +![Build DISCVR](https://github.com/bimberlabinternal/BimberLabKeyModules/workflows/Build%20DISCVR/badge.svg) ## Overview From a3c4855619c50616d2ff9d4c144bb3c9224d48ef Mon Sep 17 00:00:00 2001 From: bbimber Date: Tue, 15 Dec 2020 06:30:22 -0800 Subject: [PATCH 24/98] Support newer novogene export format --- .../web/tcrdb/panel/LibraryExportPanel.js | 54 ++++++++++++++++--- 1 file changed, 46 insertions(+), 8 deletions(-) diff --git a/tcrdb/resources/web/tcrdb/panel/LibraryExportPanel.js b/tcrdb/resources/web/tcrdb/panel/LibraryExportPanel.js index e3d3b9934..47e347716 100644 --- a/tcrdb/resources/web/tcrdb/panel/LibraryExportPanel.js +++ b/tcrdb/resources/web/tcrdb/panel/LibraryExportPanel.js @@ -48,7 +48,7 @@ Ext4.define('TCRdb.panel.LibraryExportPanel', { forceSelection: true, editable: false, labelWidth: 160, - storeValues: ['NextSeq (MPSSR)', 'MiSeq (ONPRC)', 'Basic List (MedGenome)', '10x Sample Sheet', 'Novogene'] + storeValues: ['NextSeq (MPSSR)', 'MiSeq (ONPRC)', 'Basic List (MedGenome)', '10x Sample Sheet', 'Novogene', 'Novogene-New'] },{ xtype: 'ldk-simplecombo', itemId: 'application', @@ -105,9 +105,14 @@ Ext4.define('TCRdb.panel.LibraryExportPanel', { '', border: false },{ - xtype: 'hidden', + xtype: 'ldk-simplecombo', itemId: 'instrument', - value: 'Novogene' + value: 'Novogene-New', + fieldLabel: 'Format', + forceSelection: true, + editable: true, + allowBlank: true, + storeValues: ['Novogene', 'Novogene-New'] },{ xtype: 'textarea', itemId: 'plateList', @@ -759,7 +764,7 @@ Ext4.define('TCRdb.panel.LibraryExportPanel', { }, this); } } - else if (instrument === '10x Sample Sheet' || instrument === 'Novogene') { + else if (instrument === '10x Sample Sheet' || instrument === 'Novogene' || instrument === 'Novogene-New') { //we make the default assumption that we're using 10x primers, which are listed in the sample-sheet orientation var doRC = false; var rows = []; @@ -777,7 +782,7 @@ Ext4.define('TCRdb.panel.LibraryExportPanel', { var cleanedName = r[fieldName] + '_' + r[fieldName + '/name'].replace(/ /g, '_'); cleanedName = cleanedName.replace(/\//g, '-'); - var sampleName = getSampleName(simpleSampleNames, r[fieldName], r[fieldName + '/name']) + (suffix && instrument === 'Novogene' ? '' : '-' + suffix); + var sampleName = getSampleName(simpleSampleNames, r[fieldName], r[fieldName + '/name']) + (suffix && instrument.startsWith('Novogene') ? '' : '-' + suffix); var barcode5s = r[fieldName + '/barcode5/sequence'] ? r[fieldName + '/barcode5/sequence'].split(',') : []; if (!barcode5s) { @@ -785,10 +790,22 @@ Ext4.define('TCRdb.panel.LibraryExportPanel', { } barcodeCombosUsed.push([r[fieldName + '/barcode5'], '', r.laneAssignment || ''].join('/')); + + //The new format requires one/line + if (instrument === 'Novogene-New') { + if (doRC && barcode5s.length > 1) { + var msg = 'Did not expect Novogene-New, reverse complement and multiple barcodes'; + LDK.Utils.logError(msg); + Ext4.Msg.alert('Error', msg); + return; + } + barcode5s = [barcode5s.join(',')]; + } + Ext4.Array.forEach(barcode5s, function (bc, idx) { bc = doRC ? doReverseComplement(bc) : bc; - var data = [sampleName, (instrument === 'Novogene' ? '' : cleanedName), bc, '']; + var data = [sampleName, (instrument.startsWith('Novogene') ? '' : cleanedName), bc, '']; if (instrument === 'Novogene') { data = [sampleName]; if (r.plateAlias) { @@ -809,12 +826,33 @@ Ext4.define('TCRdb.panel.LibraryExportPanel', { data.push(r.laneAssignment || ''); data.push(comment || 'Please QC individually and pool in equal amounts per lane'); } + else if (instrument === 'Novogene-New') { + data = ['Premade-10X transcriptome library']; + data.push(r.plateAlias ? r.plateAlias : samplePrefix + r.plateId.replace(/-/g, '_')); + data.push(sampleName); + data.push('Partial lane sequencing-With Demultiplexing'); //TODO: HiSeq? + data.push(bc); + data.push(''); //P5 + data.push(size); + data.push('Others'); //Library Status + data.push('ddH2O'); + data.push('Partial Lane sequencing-lib QC'); + data.push(200); //Total data + data.push('M raw reads'); + data.push(r[fieldName + '/concentration'] || ''); + data.push(defaultVolume); + data.push(comment || 'Please QC individually and pool in equal amounts per lane'); + + //data.push(phiX); //PhiX + //data.push(r.laneAssignment || ''); + + } rows.push(data.join(delim)); }, this); } }; - var delim = instrument === 'Novogene' ? '\t' : ','; + var delim = instrument.startsWith('Novogene') ? '\t' : ','; Ext4.Array.forEach(sortedRows, function (r) { processType(readsetIds, rows, r, 'readsetId', 'GEX', 500, 0.01, 'G', null, false); processType(readsetIds, rows, r, 'enrichedReadsetId', 'TCR', 700, 0.01, 'T', null, false); @@ -823,7 +861,7 @@ Ext4.define('TCRdb.panel.LibraryExportPanel', { }, this); //add missing barcodes: - if (includeBlanks && instrument !== 'Novogene') { + if (includeBlanks && !instrument.startsWith('Novogene')) { var blankIdx = 0; Ext4.Array.forEach(TCRdb.panel.LibraryExportPanel.TENX_BARCODES, function (barcode5) { if (barcodeCombosUsed.indexOf(barcode5) === -1) { From f126755c71755e30c619f8dd79dea6a9401b2831 Mon Sep 17 00:00:00 2001 From: bbimber Date: Tue, 15 Dec 2020 12:18:59 -0800 Subject: [PATCH 25/98] Retain lane assignment --- tcrdb/resources/web/tcrdb/panel/LibraryExportPanel.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tcrdb/resources/web/tcrdb/panel/LibraryExportPanel.js b/tcrdb/resources/web/tcrdb/panel/LibraryExportPanel.js index 47e347716..f9a7a19fe 100644 --- a/tcrdb/resources/web/tcrdb/panel/LibraryExportPanel.js +++ b/tcrdb/resources/web/tcrdb/panel/LibraryExportPanel.js @@ -844,7 +844,7 @@ Ext4.define('TCRdb.panel.LibraryExportPanel', { data.push(comment || 'Please QC individually and pool in equal amounts per lane'); //data.push(phiX); //PhiX - //data.push(r.laneAssignment || ''); + data.push(r.laneAssignment || ''); } rows.push(data.join(delim)); From 76268a3771f00be1f4a9da6d4f8a6f6485e76870 Mon Sep 17 00:00:00 2001 From: bbimber Date: Tue, 15 Dec 2020 14:56:41 -0800 Subject: [PATCH 26/98] When adding plates by wildcard, only include those of matching type --- .../web/tcrdb/panel/LibraryExportPanel.js | 31 +++++++++++-------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/tcrdb/resources/web/tcrdb/panel/LibraryExportPanel.js b/tcrdb/resources/web/tcrdb/panel/LibraryExportPanel.js index f9a7a19fe..46542e463 100644 --- a/tcrdb/resources/web/tcrdb/panel/LibraryExportPanel.js +++ b/tcrdb/resources/web/tcrdb/panel/LibraryExportPanel.js @@ -100,7 +100,7 @@ Ext4.define('TCRdb.panel.LibraryExportPanel', { 'H235-2\tHiSeq1\tBNB-HTO-1
' + '235-2\tHTO\tHiSeq2\tBNB-HTO-1
' + 'H235-2\tHiSeq1\tBNB-HTO-1
' + - 'C235-2\tHiSeq1\tBNB-HTO-1' + + 'C235-2\tHiSeq1\tBNB-HTO-1
' + 'C235-*\tHiSeq2\tBNB-HTO-2' + '', border: false @@ -212,7 +212,7 @@ Ext4.define('TCRdb.panel.LibraryExportPanel', { containerPath: Laboratory.Utils.getQueryContainerPath(), schemaName: 'tcrdb', queryName: 'cdnas', - columns: 'rowid,plateId', + columns: 'rowid,plateId,hashingReadsetId,citeseqReadsetId', filterArray: [LABKEY.Filter.create('plateId', Ext4.Object.getKeys(wildcards).join(';'), LABKEY.Filter.Types.CONTAINS_ONE_OF)], scope: this, failure: LDK.Utils.getErrorCallback(), @@ -223,28 +223,33 @@ Ext4.define('TCRdb.panel.LibraryExportPanel', { Ext4.Array.forEach(Ext4.Object.getKeys(wildcards), function (prefix) { if (row.plateId && row.plateId.includes(prefix)) { prefix = prefix + '*'; - prefixToPlate[prefix] = prefixToPlate[prefix] || []; - prefixToPlate[prefix].push(row.plateId); + prefixToPlate[prefix] = prefixToPlate[prefix] || {}; + prefixToPlate[prefix][row.plateId] = prefixToPlate[prefix][row.plateId] || {} + if (row.hashingReadsetId) { + prefixToPlate[prefix][row.plateId].HTO = true; + } + + if (row.citeseqReadsetId) { + prefixToPlate[prefix][row.plateId].CITE = true; + } } }, this); }, this); - Ext4.Array.forEach(Ext4.Object.getKeys(prefixToPlate), function (prefix) { - prefixToPlate[prefix] = Ext4.unique(prefixToPlate[prefix]); - }, this); - var updatedText = []; var prefixes = Ext4.Object.getKeys(prefixToPlate); Ext4.Array.forEach(text, function (r, idx) { var plateId = r[0]; - if (prefixes.indexOf(plateId) == -1) { + if (prefixes.indexOf(plateId) === -1) { updatedText.push(r); } else { - Ext4.Array.forEach(prefixToPlate[plateId], function(newPlate){ - var r2 = [].concat(r); - r2[0] = newPlate; - updatedText.push(r2); + Ext4.Array.forEach(Ext4.Object.getKeys(prefixToPlate[plateId]), function(newPlateId){ + if (Ext4.Object.getKeys(prefixToPlate[plateId][newPlateId]).indexOf(r[1]) > -1) { + var r2 = [].concat(r); + r2[0] = newPlateId; + updatedText.push(r2); + } }, this); } }, this); From 51f97baf6a5b63a336dcd94b70caebcbd8218599 Mon Sep 17 00:00:00 2001 From: bbimber Date: Tue, 15 Dec 2020 18:02:20 -0800 Subject: [PATCH 27/98] Switch to PAT instead of GITHUB_TOKEN --- .github/workflows/sync-repos.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sync-repos.yml b/.github/workflows/sync-repos.yml index 199e5f75a..e5201d176 100644 --- a/.github/workflows/sync-repos.yml +++ b/.github/workflows/sync-repos.yml @@ -31,4 +31,4 @@ jobs: source_repo: "labkey/BimberLabKeyModules" source_branch: "develop" destination_branch: "develop" - github_token: ${{ secrets.GITHUB_TOKEN }} + github_token: ${{ secrets.PAT }} From 475a11b84ac917f9fb8f4a75c7a7becf8cc212ca Mon Sep 17 00:00:00 2001 From: bbimber Date: Wed, 23 Dec 2020 12:31:43 -0800 Subject: [PATCH 28/98] Add stubs for LabPurchasing and MCC modules --- LabPurchasing/module.properties | 5 ++ .../postgresql/labpurchasing-0.00-20.000.sql | 19 ++++ .../sqlserver/labpurchasing-0.00-20.000.sql | 20 +++++ .../resources/schemas/labpurchasing.xml | 20 +++++ .../LabPurchasingController.java | 37 ++++++++ .../labpurchasing/LabPurchasingManager.java | 32 +++++++ .../labpurchasing/LabPurchasingModule.java | 85 ++++++++++++++++++ .../labpurchasing/LabPurchasingSchema.java | 49 +++++++++++ .../labpurchasing/LabPurchasingWebPart.java | 69 +++++++++++++++ .../test/pages/labpurchasing/BeginPage.java | 59 +++++++++++++ .../labpurchasing/LabPurchasingTest.java | 88 +++++++++++++++++++ mcc/module.properties | 5 ++ .../dbscripts/postgresql/mcc-0.00-20.000.sql | 19 ++++ .../dbscripts/sqlserver/mcc-0.00-20.000.sql | 20 +++++ mcc/resources/schemas/mcc.xml | 20 +++++ mcc/src/org/labkey/mcc/MccController.java | 37 ++++++++ mcc/src/org/labkey/mcc/MccManager.java | 32 +++++++ mcc/src/org/labkey/mcc/MccModule.java | 84 ++++++++++++++++++ mcc/src/org/labkey/mcc/MccSchema.java | 49 +++++++++++ .../test/components/mcc/MccWebPart.java | 69 +++++++++++++++ .../org/labkey/test/pages/mcc/BeginPage.java | 59 +++++++++++++ .../org/labkey/test/tests/mcc/MccTest.java | 88 +++++++++++++++++++ 22 files changed, 965 insertions(+) create mode 100644 LabPurchasing/module.properties create mode 100644 LabPurchasing/resources/schemas/dbscripts/postgresql/labpurchasing-0.00-20.000.sql create mode 100644 LabPurchasing/resources/schemas/dbscripts/sqlserver/labpurchasing-0.00-20.000.sql create mode 100644 LabPurchasing/resources/schemas/labpurchasing.xml create mode 100644 LabPurchasing/src/org/labkey/labpurchasing/LabPurchasingController.java create mode 100644 LabPurchasing/src/org/labkey/labpurchasing/LabPurchasingManager.java create mode 100644 LabPurchasing/src/org/labkey/labpurchasing/LabPurchasingModule.java create mode 100644 LabPurchasing/src/org/labkey/labpurchasing/LabPurchasingSchema.java create mode 100644 LabPurchasing/test/src/org/labkey/test/components/labpurchasing/LabPurchasingWebPart.java create mode 100644 LabPurchasing/test/src/org/labkey/test/pages/labpurchasing/BeginPage.java create mode 100644 LabPurchasing/test/src/org/labkey/test/tests/labpurchasing/LabPurchasingTest.java create mode 100644 mcc/module.properties create mode 100644 mcc/resources/schemas/dbscripts/postgresql/mcc-0.00-20.000.sql create mode 100644 mcc/resources/schemas/dbscripts/sqlserver/mcc-0.00-20.000.sql create mode 100644 mcc/resources/schemas/mcc.xml create mode 100644 mcc/src/org/labkey/mcc/MccController.java create mode 100644 mcc/src/org/labkey/mcc/MccManager.java create mode 100644 mcc/src/org/labkey/mcc/MccModule.java create mode 100644 mcc/src/org/labkey/mcc/MccSchema.java create mode 100644 mcc/test/src/org/labkey/test/components/mcc/MccWebPart.java create mode 100644 mcc/test/src/org/labkey/test/pages/mcc/BeginPage.java create mode 100644 mcc/test/src/org/labkey/test/tests/mcc/MccTest.java diff --git a/LabPurchasing/module.properties b/LabPurchasing/module.properties new file mode 100644 index 000000000..577c924b4 --- /dev/null +++ b/LabPurchasing/module.properties @@ -0,0 +1,5 @@ +ModuleClass: org.labkey.labpurchasing.LabPurchasingModule +Label: Lab Purchacing +Description: A module designed to assist with purchasing supplies for academic labs +License: Apache 2.0 +LicenseURL: http://www.apache.org/licenses/LICENSE-2.0 diff --git a/LabPurchasing/resources/schemas/dbscripts/postgresql/labpurchasing-0.00-20.000.sql b/LabPurchasing/resources/schemas/dbscripts/postgresql/labpurchasing-0.00-20.000.sql new file mode 100644 index 000000000..fe2e39c0f --- /dev/null +++ b/LabPurchasing/resources/schemas/dbscripts/postgresql/labpurchasing-0.00-20.000.sql @@ -0,0 +1,19 @@ +/* + * Copyright (c) 2020 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +-- Create schema, tables, indexes, and constraints used for LabPurchasing module here +-- All SQL VIEW definitions should be created in labpurchasing-create.sql and dropped in labpurchasing-drop.sql +CREATE SCHEMA labpurchasing; diff --git a/LabPurchasing/resources/schemas/dbscripts/sqlserver/labpurchasing-0.00-20.000.sql b/LabPurchasing/resources/schemas/dbscripts/sqlserver/labpurchasing-0.00-20.000.sql new file mode 100644 index 000000000..35a4a8574 --- /dev/null +++ b/LabPurchasing/resources/schemas/dbscripts/sqlserver/labpurchasing-0.00-20.000.sql @@ -0,0 +1,20 @@ +/* + * Copyright (c) 2020 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +-- Create schema, tables, indexes, and constraints used for LabPurchasing module here +-- All SQL VIEW definitions should be created in labpurchasing-create.sql and dropped in labpurchasing-drop.sql +CREATE SCHEMA labpurchasing; +GO \ No newline at end of file diff --git a/LabPurchasing/resources/schemas/labpurchasing.xml b/LabPurchasing/resources/schemas/labpurchasing.xml new file mode 100644 index 000000000..2bba6c71d --- /dev/null +++ b/LabPurchasing/resources/schemas/labpurchasing.xml @@ -0,0 +1,20 @@ + + + \ No newline at end of file diff --git a/LabPurchasing/src/org/labkey/labpurchasing/LabPurchasingController.java b/LabPurchasing/src/org/labkey/labpurchasing/LabPurchasingController.java new file mode 100644 index 000000000..e53cbc6c5 --- /dev/null +++ b/LabPurchasing/src/org/labkey/labpurchasing/LabPurchasingController.java @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2020 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.labpurchasing; + +import org.labkey.api.action.SimpleViewAction; +import org.labkey.api.action.SpringActionController; +import org.labkey.api.security.RequiresPermission; +import org.labkey.api.security.permissions.ReadPermission; +import org.labkey.api.view.JspView; +import org.labkey.api.view.NavTree; +import org.springframework.validation.BindException; +import org.springframework.web.servlet.ModelAndView; + +public class LabPurchasingController extends SpringActionController +{ + private static final DefaultActionResolver _actionResolver = new DefaultActionResolver(LabPurchasingController.class); + public static final String NAME = "labpurchasing"; + + public LabPurchasingController() + { + setActionResolver(_actionResolver); + } +} diff --git a/LabPurchasing/src/org/labkey/labpurchasing/LabPurchasingManager.java b/LabPurchasing/src/org/labkey/labpurchasing/LabPurchasingManager.java new file mode 100644 index 000000000..43b0d9543 --- /dev/null +++ b/LabPurchasing/src/org/labkey/labpurchasing/LabPurchasingManager.java @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2020 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.labpurchasing; + +public class LabPurchasingManager +{ + private static final LabPurchasingManager _instance = new LabPurchasingManager(); + + private LabPurchasingManager() + { + // prevent external construction with a private default constructor + } + + public static LabPurchasingManager get() + { + return _instance; + } +} \ No newline at end of file diff --git a/LabPurchasing/src/org/labkey/labpurchasing/LabPurchasingModule.java b/LabPurchasing/src/org/labkey/labpurchasing/LabPurchasingModule.java new file mode 100644 index 000000000..230a72130 --- /dev/null +++ b/LabPurchasing/src/org/labkey/labpurchasing/LabPurchasingModule.java @@ -0,0 +1,85 @@ +/* + * Copyright (c) 2020 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.labpurchasing; + +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerManager; +import org.labkey.api.module.DefaultModule; +import org.labkey.api.module.ModuleContext; +import org.labkey.api.view.WebPartFactory; + +import java.util.Collection; +import java.util.Collections; +import java.util.Set; + +public class LabPurchasingModule extends DefaultModule +{ + public static final String NAME = "LabPurchasing"; + + @Override + public String getName() + { + return NAME; + } + + @Override + public @Nullable Double getSchemaVersion() + { + return 20.000; + } + + @Override + public boolean hasScripts() + { + return true; + } + + @Override + @NotNull + protected Collection createWebPartFactories() + { + return Collections.emptyList(); + } + + @Override + protected void init() + { + addController(LabPurchasingController.NAME, LabPurchasingController.class); + } + + @Override + public void doStartup(ModuleContext moduleContext) + { + + } + + @Override + @NotNull + public Collection getSummary(Container c) + { + return Collections.emptyList(); + } + + @Override + @NotNull + public Set getSchemaNames() + { + return Collections.singleton(LabPurchasingSchema.NAME); + } +} \ No newline at end of file diff --git a/LabPurchasing/src/org/labkey/labpurchasing/LabPurchasingSchema.java b/LabPurchasing/src/org/labkey/labpurchasing/LabPurchasingSchema.java new file mode 100644 index 000000000..d4425241d --- /dev/null +++ b/LabPurchasing/src/org/labkey/labpurchasing/LabPurchasingSchema.java @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2020 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.labpurchasing; + +import org.labkey.api.data.DbSchema; +import org.labkey.api.data.DbSchemaType; +import org.labkey.api.data.dialect.SqlDialect; + +public class LabPurchasingSchema +{ + private static final LabPurchasingSchema _instance = new LabPurchasingSchema(); + public static final String NAME = "labpurchasing"; + + public static LabPurchasingSchema getInstance() + { + return _instance; + } + + private LabPurchasingSchema() + { + // private constructor to prevent instantiation from + // outside this class: this singleton should only be + // accessed via org.labkey.labpurchasing.LabPurchasingSchema.getInstance() + } + + public DbSchema getSchema() + { + return DbSchema.get(NAME, DbSchemaType.Module); + } + + public SqlDialect getSqlDialect() + { + return getSchema().getSqlDialect(); + } +} diff --git a/LabPurchasing/test/src/org/labkey/test/components/labpurchasing/LabPurchasingWebPart.java b/LabPurchasing/test/src/org/labkey/test/components/labpurchasing/LabPurchasingWebPart.java new file mode 100644 index 000000000..eaa9965df --- /dev/null +++ b/LabPurchasing/test/src/org/labkey/test/components/labpurchasing/LabPurchasingWebPart.java @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2020 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.test.components.labpurchasing; + +import org.labkey.test.Locator; +import org.labkey.test.components.BodyWebPart; +import org.labkey.test.components.html.Input; +import org.labkey.test.pages.LabKeyPage; +import org.openqa.selenium.WebDriver; +import org.openqa.selenium.WebElement; + +import static org.labkey.test.components.html.Input.Input; + +/** + * TODO: Component for a hypothetical webpart containing an input and a save button + * Component classes should handle all timing and functionality for a component + */ +public class LabPurchasingWebPart extends BodyWebPart +{ + public LabPurchasingWebPart(WebDriver driver) + { + this(driver, 0); + } + + public LabPurchasingWebPart(WebDriver driver, int index) + { + super(driver, "LabPurchasing", index); + } + + public LabPurchasingWebPart setInput(String value) + { + elementCache().input.set(value); + // TODO: Methods that don't navigate should return this object + return this; + } + + public LabKeyPage clickSave() + { + getWrapper().clickAndWait(elementCache().button); + // TODO: Methods that navigate should return an appropriate page object + return new LabKeyPage(getDriver()); + } + + @Override + protected ElementCache newElementCache() + { + return new ElementCache(); + } + + protected class ElementCache extends BodyWebPart.ElementCache + { + protected final WebElement button = Locator.tag("button").withText("Save").findWhenNeeded(this); + protected final Input input = Input(Locator.tag("input"), getDriver()).findWhenNeeded(this); + } +} \ No newline at end of file diff --git a/LabPurchasing/test/src/org/labkey/test/pages/labpurchasing/BeginPage.java b/LabPurchasing/test/src/org/labkey/test/pages/labpurchasing/BeginPage.java new file mode 100644 index 000000000..7e36f3e45 --- /dev/null +++ b/LabPurchasing/test/src/org/labkey/test/pages/labpurchasing/BeginPage.java @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2020 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.test.pages.labpurchasing; + +import org.labkey.test.BaseWebDriverTest; +import org.labkey.test.WebDriverWrapper; +import org.labkey.test.Locator; +import org.labkey.test.WebTestHelper; +import org.labkey.test.pages.LabKeyPage; +import org.openqa.selenium.WebElement; + +public class BeginPage extends LabKeyPage +{ + public BeginPage(WebDriverWrapper driver) + { + super(driver); + } + + public static BeginPage beginAt(WebDriverWrapper driver) + { + return beginAt(driver, driver.getCurrentContainerPath()); + } + + public static BeginPage beginAt(WebDriverWrapper driver, String containerPath) + { + driver.beginAt(WebTestHelper.buildURL("labpurchasing", containerPath, "begin")); + return new BeginPage(driver); + } + + public String getHelloMessage() + { + return elementCache().helloMessage.getText(); + } + + @Override + protected ElementCache newElementCache() + { + return new ElementCache(); + } + + protected class ElementCache extends LabKeyPage.ElementCache + { + protected final WebElement helloMessage = Locator.tagWithName("div", "helloMessage").findWhenNeeded(this); + } +} diff --git a/LabPurchasing/test/src/org/labkey/test/tests/labpurchasing/LabPurchasingTest.java b/LabPurchasing/test/src/org/labkey/test/tests/labpurchasing/LabPurchasingTest.java new file mode 100644 index 000000000..1fa717924 --- /dev/null +++ b/LabPurchasing/test/src/org/labkey/test/tests/labpurchasing/LabPurchasingTest.java @@ -0,0 +1,88 @@ +/* + * Copyright (c) 2020 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.test.tests.labpurchasing; + +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.labkey.test.BaseWebDriverTest; +import org.labkey.test.TestTimeoutException; +import org.labkey.test.categories.InDevelopment; +import org.labkey.test.pages.labpurchasing.BeginPage; + +import java.util.Collections; +import java.util.List; + +import static org.junit.Assert.*; + +@Category({InDevelopment.class}) +public class LabPurchasingTest extends BaseWebDriverTest +{ + @Override + protected void doCleanup(boolean afterTest) throws TestTimeoutException + { + _containerHelper.deleteProject(getProjectName(), afterTest); + } + + @BeforeClass + public static void setupProject() + { + LabPurchasingTest init = (LabPurchasingTest)getCurrentTest(); + + init.doSetup(); + } + + private void doSetup() + { + _containerHelper.createProject(getProjectName(), null); + } + + @Before + public void preTest() + { + goToProjectHome(); + } + + @Test + public void testLabPurchasingModule() + { + _containerHelper.enableModule("LabPurchasing"); + BeginPage beginPage = BeginPage.beginAt(this, getProjectName()); + assertEquals(200, getResponseCode()); + final String expectedHello = "Hello, and welcome to the LabPurchasing module."; + assertEquals("Wrong hello message", expectedHello, beginPage.getHelloMessage()); + } + + @Override + protected BrowserType bestBrowser() + { + return BrowserType.CHROME; + } + + @Override + protected String getProjectName() + { + return "LabPurchasingTest Project"; + } + + @Override + public List getAssociatedModules() + { + return Collections.singletonList("LabPurchasing"); + } +} \ No newline at end of file diff --git a/mcc/module.properties b/mcc/module.properties new file mode 100644 index 000000000..0ddf5e2dd --- /dev/null +++ b/mcc/module.properties @@ -0,0 +1,5 @@ +ModuleClass: org.labkey.mcc.MccModule +Label: Maromoset Coordinating Center +Description: This module is used by the BRAIN Initiative Maromoset Coordinating Center +License: Apache 2.0 +LicenseURL: http://www.apache.org/licenses/LICENSE-2.0 diff --git a/mcc/resources/schemas/dbscripts/postgresql/mcc-0.00-20.000.sql b/mcc/resources/schemas/dbscripts/postgresql/mcc-0.00-20.000.sql new file mode 100644 index 000000000..53628e9d3 --- /dev/null +++ b/mcc/resources/schemas/dbscripts/postgresql/mcc-0.00-20.000.sql @@ -0,0 +1,19 @@ +/* + * Copyright (c) 2020 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +-- Create schema, tables, indexes, and constraints used for Mcc module here +-- All SQL VIEW definitions should be created in mcc-create.sql and dropped in mcc-drop.sql +CREATE SCHEMA mcc; diff --git a/mcc/resources/schemas/dbscripts/sqlserver/mcc-0.00-20.000.sql b/mcc/resources/schemas/dbscripts/sqlserver/mcc-0.00-20.000.sql new file mode 100644 index 000000000..5609630ff --- /dev/null +++ b/mcc/resources/schemas/dbscripts/sqlserver/mcc-0.00-20.000.sql @@ -0,0 +1,20 @@ +/* + * Copyright (c) 2020 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +-- Create schema, tables, indexes, and constraints used for Mcc module here +-- All SQL VIEW definitions should be created in mcc-create.sql and dropped in mcc-drop.sql +CREATE SCHEMA mcc; +GO \ No newline at end of file diff --git a/mcc/resources/schemas/mcc.xml b/mcc/resources/schemas/mcc.xml new file mode 100644 index 000000000..2bba6c71d --- /dev/null +++ b/mcc/resources/schemas/mcc.xml @@ -0,0 +1,20 @@ + + + \ No newline at end of file diff --git a/mcc/src/org/labkey/mcc/MccController.java b/mcc/src/org/labkey/mcc/MccController.java new file mode 100644 index 000000000..0967188da --- /dev/null +++ b/mcc/src/org/labkey/mcc/MccController.java @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2020 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.mcc; + +import org.labkey.api.action.SimpleViewAction; +import org.labkey.api.action.SpringActionController; +import org.labkey.api.security.RequiresPermission; +import org.labkey.api.security.permissions.ReadPermission; +import org.labkey.api.view.JspView; +import org.labkey.api.view.NavTree; +import org.springframework.validation.BindException; +import org.springframework.web.servlet.ModelAndView; + +public class MccController extends SpringActionController +{ + private static final DefaultActionResolver _actionResolver = new DefaultActionResolver(MccController.class); + public static final String NAME = "mcc"; + + public MccController() + { + setActionResolver(_actionResolver); + } +} diff --git a/mcc/src/org/labkey/mcc/MccManager.java b/mcc/src/org/labkey/mcc/MccManager.java new file mode 100644 index 000000000..d6417b79d --- /dev/null +++ b/mcc/src/org/labkey/mcc/MccManager.java @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2020 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.mcc; + +public class MccManager +{ + private static final MccManager _instance = new MccManager(); + + private MccManager() + { + // prevent external construction with a private default constructor + } + + public static MccManager get() + { + return _instance; + } +} \ No newline at end of file diff --git a/mcc/src/org/labkey/mcc/MccModule.java b/mcc/src/org/labkey/mcc/MccModule.java new file mode 100644 index 000000000..29361b628 --- /dev/null +++ b/mcc/src/org/labkey/mcc/MccModule.java @@ -0,0 +1,84 @@ +/* + * Copyright (c) 2020 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.mcc; + +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.labkey.api.data.Container; +import org.labkey.api.module.DefaultModule; +import org.labkey.api.module.ModuleContext; +import org.labkey.api.view.WebPartFactory; + +import java.util.Collection; +import java.util.Collections; +import java.util.Set; + +public class MccModule extends DefaultModule +{ + public static final String NAME = "MCC"; + + @Override + public String getName() + { + return NAME; + } + + @Override + public @Nullable Double getSchemaVersion() + { + return 20.000; + } + + @Override + public boolean hasScripts() + { + return true; + } + + @Override + @NotNull + protected Collection createWebPartFactories() + { + return Collections.emptyList(); + } + + @Override + protected void init() + { + addController(MccController.NAME, MccController.class); + } + + @Override + public void doStartup(ModuleContext moduleContext) + { + + } + + @Override + @NotNull + public Collection getSummary(Container c) + { + return Collections.emptyList(); + } + + @Override + @NotNull + public Set getSchemaNames() + { + return Collections.singleton(MccSchema.NAME); + } +} \ No newline at end of file diff --git a/mcc/src/org/labkey/mcc/MccSchema.java b/mcc/src/org/labkey/mcc/MccSchema.java new file mode 100644 index 000000000..d072b94a9 --- /dev/null +++ b/mcc/src/org/labkey/mcc/MccSchema.java @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2020 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.mcc; + +import org.labkey.api.data.DbSchema; +import org.labkey.api.data.DbSchemaType; +import org.labkey.api.data.dialect.SqlDialect; + +public class MccSchema +{ + private static final MccSchema _instance = new MccSchema(); + public static final String NAME = "mcc"; + + public static MccSchema getInstance() + { + return _instance; + } + + private MccSchema() + { + // private constructor to prevent instantiation from + // outside this class: this singleton should only be + // accessed via org.labkey.mcc.MccSchema.getInstance() + } + + public DbSchema getSchema() + { + return DbSchema.get(NAME, DbSchemaType.Module); + } + + public SqlDialect getSqlDialect() + { + return getSchema().getSqlDialect(); + } +} diff --git a/mcc/test/src/org/labkey/test/components/mcc/MccWebPart.java b/mcc/test/src/org/labkey/test/components/mcc/MccWebPart.java new file mode 100644 index 000000000..a48615354 --- /dev/null +++ b/mcc/test/src/org/labkey/test/components/mcc/MccWebPart.java @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2020 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.test.components.mcc; + +import org.labkey.test.Locator; +import org.labkey.test.components.BodyWebPart; +import org.labkey.test.components.html.Input; +import org.labkey.test.pages.LabKeyPage; +import org.openqa.selenium.WebDriver; +import org.openqa.selenium.WebElement; + +import static org.labkey.test.components.html.Input.Input; + +/** + * TODO: Component for a hypothetical webpart containing an input and a save button + * Component classes should handle all timing and functionality for a component + */ +public class MccWebPart extends BodyWebPart +{ + public MccWebPart(WebDriver driver) + { + this(driver, 0); + } + + public MccWebPart(WebDriver driver, int index) + { + super(driver, "Mcc", index); + } + + public MccWebPart setInput(String value) + { + elementCache().input.set(value); + // TODO: Methods that don't navigate should return this object + return this; + } + + public LabKeyPage clickSave() + { + getWrapper().clickAndWait(elementCache().button); + // TODO: Methods that navigate should return an appropriate page object + return new LabKeyPage(getDriver()); + } + + @Override + protected ElementCache newElementCache() + { + return new ElementCache(); + } + + protected class ElementCache extends BodyWebPart.ElementCache + { + protected final WebElement button = Locator.tag("button").withText("Save").findWhenNeeded(this); + protected final Input input = Input(Locator.tag("input"), getDriver()).findWhenNeeded(this); + } +} \ No newline at end of file diff --git a/mcc/test/src/org/labkey/test/pages/mcc/BeginPage.java b/mcc/test/src/org/labkey/test/pages/mcc/BeginPage.java new file mode 100644 index 000000000..3baa4087e --- /dev/null +++ b/mcc/test/src/org/labkey/test/pages/mcc/BeginPage.java @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2020 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.test.pages.mcc; + +import org.labkey.test.BaseWebDriverTest; +import org.labkey.test.WebDriverWrapper; +import org.labkey.test.Locator; +import org.labkey.test.WebTestHelper; +import org.labkey.test.pages.LabKeyPage; +import org.openqa.selenium.WebElement; + +public class BeginPage extends LabKeyPage +{ + public BeginPage(WebDriverWrapper driver) + { + super(driver); + } + + public static BeginPage beginAt(WebDriverWrapper driver) + { + return beginAt(driver, driver.getCurrentContainerPath()); + } + + public static BeginPage beginAt(WebDriverWrapper driver, String containerPath) + { + driver.beginAt(WebTestHelper.buildURL("mcc", containerPath, "begin")); + return new BeginPage(driver); + } + + public String getHelloMessage() + { + return elementCache().helloMessage.getText(); + } + + @Override + protected ElementCache newElementCache() + { + return new ElementCache(); + } + + protected class ElementCache extends LabKeyPage.ElementCache + { + protected final WebElement helloMessage = Locator.tagWithName("div", "helloMessage").findWhenNeeded(this); + } +} diff --git a/mcc/test/src/org/labkey/test/tests/mcc/MccTest.java b/mcc/test/src/org/labkey/test/tests/mcc/MccTest.java new file mode 100644 index 000000000..1c53ac721 --- /dev/null +++ b/mcc/test/src/org/labkey/test/tests/mcc/MccTest.java @@ -0,0 +1,88 @@ +/* + * Copyright (c) 2020 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.test.tests.mcc; + +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.labkey.test.BaseWebDriverTest; +import org.labkey.test.TestTimeoutException; +import org.labkey.test.categories.InDevelopment; +import org.labkey.test.pages.mcc.BeginPage; + +import java.util.Collections; +import java.util.List; + +import static org.junit.Assert.*; + +@Category({InDevelopment.class}) +public class MccTest extends BaseWebDriverTest +{ + @Override + protected void doCleanup(boolean afterTest) throws TestTimeoutException + { + _containerHelper.deleteProject(getProjectName(), afterTest); + } + + @BeforeClass + public static void setupProject() + { + MccTest init = (MccTest)getCurrentTest(); + + init.doSetup(); + } + + private void doSetup() + { + _containerHelper.createProject(getProjectName(), null); + } + + @Before + public void preTest() + { + goToProjectHome(); + } + + @Test + public void testMccModule() + { + _containerHelper.enableModule("Mcc"); + BeginPage beginPage = BeginPage.beginAt(this, getProjectName()); + assertEquals(200, getResponseCode()); + final String expectedHello = "Hello, and welcome to the Mcc module."; + assertEquals("Wrong hello message", expectedHello, beginPage.getHelloMessage()); + } + + @Override + protected BrowserType bestBrowser() + { + return BrowserType.CHROME; + } + + @Override + protected String getProjectName() + { + return "MccTest Project"; + } + + @Override + public List getAssociatedModules() + { + return Collections.singletonList("Mcc"); + } +} \ No newline at end of file From 7587b685b4c8720c0288d18919aa3bc2ac9c6af0 Mon Sep 17 00:00:00 2001 From: bbimber Date: Tue, 29 Dec 2020 15:19:49 -0800 Subject: [PATCH 29/98] Major refactor to split out single-cell code into new module --- mGAP/resources/credits/jars.txt | 2 +- mcc/resources/etls/snprc.xml | 17 + mcc/resources/etls/wnprc.xml | 19 + tcrdb/build.gradle | 2 + .../assay/TCRdb/queries/Data.query.xml | 4 +- tcrdb/resources/external/install.R | 1 - .../resources/external/installCiteSeqCount.sh | 9 - tcrdb/resources/external/scRNAseq/Seurat3.rmd | 186 -- .../external/scRNAseq/seuratWrapper.sh | 27 - .../cdna_libraries}/Assay Info.qview.xml | 8 +- tcrdb/resources/queries/tcrdb/cdnas.js | 39 - .../resources/queries/tcrdb/cdnas/.qview.xml | 29 - .../tcrdb/citeseq_panel_names.query.xml | 13 - .../queries/tcrdb/citeseq_panel_names.sql | 6 - .../queries/tcrdb/citeseq_panels/.qview.xml | 12 - .../queries/tcrdb/hashtag_oligos.sql | 7 - .../queries/tcrdb/sortStatusByPlate.query.xml | 21 - .../queries/tcrdb/sortStatusByPlate.sql | 88 - .../tcrdb/sortStatusByPlate/.qview.xml | 6 - .../sortStatusByPlateAndSample.query.xml | 21 - .../tcrdb/sortStatusByPlateAndSample.sql | 72 - .../sortStatusByPlateAndSample/.qview.xml | 7 - tcrdb/resources/queries/tcrdb/sorts.js | 82 - tcrdb/resources/queries/tcrdb/sorts.query.xml | 11 - .../resources/queries/tcrdb/sorts/.qview.xml | 26 - tcrdb/resources/queries/tcrdb/stims.js | 36 - .../postgresql/tcrdb-15.51-15.52.sql | 2 + .../dbscripts/sqlserver/tcrdb-15.51-15.52.sql | 48 + tcrdb/resources/views/cDNAImport.html | 8 - tcrdb/resources/views/cDNAImport.view.xml | 31 - tcrdb/resources/views/libraryExport.html | 8 - tcrdb/resources/views/libraryExport.view.xml | 7 - tcrdb/resources/views/poolImport.html | 8 - tcrdb/resources/views/poolImport.view.xml | 30 - tcrdb/resources/views/stimDashboard.html | 8 - tcrdb/resources/views/stimDashboard.view.xml | 7 - tcrdb/resources/web/tcrdb/buttons.js | 3 - .../web/tcrdb/exampleData/ImportExample.xlsx | Bin 11860 -> 0 bytes .../exampleData/ImportReadsetTemplate.xlsx | Bin 9435 -> 0 bytes .../web/tcrdb/exampleData/ImportTemplate.xlsx | Bin 9653 -> 0 bytes .../web/tcrdb/panel/LibraryExportPanel.js | 924 ---------- .../web/tcrdb/panel/PoolImportPanel.js | 1017 ----------- tcrdb/resources/web/tcrdb/panel/StimPanel.js | 1515 ----------------- .../web/tcrdb/panel/cDNAImportPanel.js | 378 ---- tcrdb/src/org/labkey/tcrdb/ImportHelper.java | 110 -- .../labkey/tcrdb/TCRdbBulkImportNavItem.java | 53 - .../src/org/labkey/tcrdb/TCRdbController.java | 250 +-- .../org/labkey/tcrdb/TCRdbImportNavItem.java | 50 - tcrdb/src/org/labkey/tcrdb/TCRdbManager.java | 4 +- tcrdb/src/org/labkey/tcrdb/TCRdbModule.java | 25 +- tcrdb/src/org/labkey/tcrdb/TCRdbProvider.java | 95 +- tcrdb/src/org/labkey/tcrdb/TCRdbSchema.java | 13 +- .../labkey/tcrdb/TCRdbTableCustomizer.java | 136 +- .../src/org/labkey/tcrdb/TCRdbUserSchema.java | 7 +- .../CellRangerCellHashingHandler.java | 361 ---- .../pipeline/CellRangerSeuratHandler.java | 1061 ------------ .../CellRangerVDJCellHashingHandler.java | 171 +- .../tcrdb/pipeline/CellRangerVDJUtils.java | 736 +------- .../tcrdb/pipeline/CellRangerVDJWrapper.java | 727 -------- .../labkey/tcrdb/pipeline/MiXCRAnalysis.java | 25 +- .../pipeline/SeuratCellHashingHandler.java | 153 -- .../tcrdb/pipeline/SeuratCiteSeqHandler.java | 131 -- 62 files changed, 160 insertions(+), 8693 deletions(-) create mode 100644 mcc/resources/etls/snprc.xml create mode 100644 mcc/resources/etls/wnprc.xml delete mode 100644 tcrdb/resources/external/install.R delete mode 100644 tcrdb/resources/external/installCiteSeqCount.sh delete mode 100644 tcrdb/resources/external/scRNAseq/Seurat3.rmd delete mode 100644 tcrdb/resources/external/scRNAseq/seuratWrapper.sh rename tcrdb/resources/queries/{tcrdb/cdnas => singlecell/cdna_libraries}/Assay Info.qview.xml (84%) delete mode 100644 tcrdb/resources/queries/tcrdb/cdnas.js delete mode 100644 tcrdb/resources/queries/tcrdb/cdnas/.qview.xml delete mode 100644 tcrdb/resources/queries/tcrdb/citeseq_panel_names.query.xml delete mode 100644 tcrdb/resources/queries/tcrdb/citeseq_panel_names.sql delete mode 100644 tcrdb/resources/queries/tcrdb/citeseq_panels/.qview.xml delete mode 100644 tcrdb/resources/queries/tcrdb/hashtag_oligos.sql delete mode 100644 tcrdb/resources/queries/tcrdb/sortStatusByPlate.query.xml delete mode 100644 tcrdb/resources/queries/tcrdb/sortStatusByPlate.sql delete mode 100644 tcrdb/resources/queries/tcrdb/sortStatusByPlate/.qview.xml delete mode 100644 tcrdb/resources/queries/tcrdb/sortStatusByPlateAndSample.query.xml delete mode 100644 tcrdb/resources/queries/tcrdb/sortStatusByPlateAndSample.sql delete mode 100644 tcrdb/resources/queries/tcrdb/sortStatusByPlateAndSample/.qview.xml delete mode 100644 tcrdb/resources/queries/tcrdb/sorts.js delete mode 100644 tcrdb/resources/queries/tcrdb/sorts.query.xml delete mode 100644 tcrdb/resources/queries/tcrdb/sorts/.qview.xml delete mode 100644 tcrdb/resources/queries/tcrdb/stims.js create mode 100644 tcrdb/resources/schemas/dbscripts/postgresql/tcrdb-15.51-15.52.sql create mode 100644 tcrdb/resources/schemas/dbscripts/sqlserver/tcrdb-15.51-15.52.sql delete mode 100644 tcrdb/resources/views/cDNAImport.html delete mode 100644 tcrdb/resources/views/cDNAImport.view.xml delete mode 100644 tcrdb/resources/views/libraryExport.html delete mode 100644 tcrdb/resources/views/libraryExport.view.xml delete mode 100644 tcrdb/resources/views/poolImport.html delete mode 100644 tcrdb/resources/views/poolImport.view.xml delete mode 100644 tcrdb/resources/views/stimDashboard.html delete mode 100644 tcrdb/resources/views/stimDashboard.view.xml delete mode 100644 tcrdb/resources/web/tcrdb/exampleData/ImportExample.xlsx delete mode 100644 tcrdb/resources/web/tcrdb/exampleData/ImportReadsetTemplate.xlsx delete mode 100644 tcrdb/resources/web/tcrdb/exampleData/ImportTemplate.xlsx delete mode 100644 tcrdb/resources/web/tcrdb/panel/LibraryExportPanel.js delete mode 100644 tcrdb/resources/web/tcrdb/panel/PoolImportPanel.js delete mode 100644 tcrdb/resources/web/tcrdb/panel/StimPanel.js delete mode 100644 tcrdb/resources/web/tcrdb/panel/cDNAImportPanel.js delete mode 100644 tcrdb/src/org/labkey/tcrdb/ImportHelper.java delete mode 100644 tcrdb/src/org/labkey/tcrdb/TCRdbBulkImportNavItem.java delete mode 100644 tcrdb/src/org/labkey/tcrdb/TCRdbImportNavItem.java delete mode 100644 tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerCellHashingHandler.java delete mode 100644 tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerSeuratHandler.java delete mode 100644 tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJWrapper.java delete mode 100644 tcrdb/src/org/labkey/tcrdb/pipeline/SeuratCellHashingHandler.java delete mode 100644 tcrdb/src/org/labkey/tcrdb/pipeline/SeuratCiteSeqHandler.java diff --git a/mGAP/resources/credits/jars.txt b/mGAP/resources/credits/jars.txt index b3fbb0b53..0a2c7d2f9 100644 --- a/mGAP/resources/credits/jars.txt +++ b/mGAP/resources/credits/jars.txt @@ -1,4 +1,4 @@ {table} Filename|Component|Version|Source|License|LabKey Dev|Purpose -htsjdk-2.21.3.jar|htsjdk|2.21.3|{link:htsjdk|http://samtools.github.io/htsjdk/}|{link:MIT License|http://opensource.org/licenses/MIT}|bbimber|Description A Java API for high-throughput sequencing data (HTS) formats +htsjdk-2.21.3.jar|htsjdk|2.21.3|{link:htsjdk|http://samtools.github.io/htsjdk/}|{link:MIT License|http://opensource.org/licenses/MIT}|bbimber|A Java API for high-throughput sequencing data (HTS) formats {table} \ No newline at end of file diff --git a/mcc/resources/etls/snprc.xml b/mcc/resources/etls/snprc.xml new file mode 100644 index 000000000..127947d46 --- /dev/null +++ b/mcc/resources/etls/snprc.xml @@ -0,0 +1,17 @@ + + + SNPRC_Data + SNPRC Clinical/Demographics Data + + + Copy to target + + + + + + + + + + diff --git a/mcc/resources/etls/wnprc.xml b/mcc/resources/etls/wnprc.xml new file mode 100644 index 000000000..f0d8a115f --- /dev/null +++ b/mcc/resources/etls/wnprc.xml @@ -0,0 +1,19 @@ + + + SNPRC_Data + SNPRC Clinical/Demographics Data + + + Copy to target + + + + + + + + + + + + diff --git a/tcrdb/build.gradle b/tcrdb/build.gradle index c1117ff51..03b7b0bac 100644 --- a/tcrdb/build.gradle +++ b/tcrdb/build.gradle @@ -1,6 +1,7 @@ import org.labkey.gradle.util.BuildUtils; dependencies { + BuildUtils.addLabKeyDependency(project: project, config: "implementation", depProjectPath: ":server:modules:DiscvrLabKeyModules:singlecell", depProjectConfig: "apiJarFile") BuildUtils.addLabKeyDependency(project: project, config: "implementation", depProjectPath: ":server:modules:DiscvrLabKeyModules:SequenceAnalysis", depProjectConfig: "apiJarFile") BuildUtils.addLabKeyDependency(project: project, config: "implementation", depProjectPath: ":server:modules:DiscvrLabKeyModules:SequenceAnalysis", depProjectConfig: "apiElements") BuildUtils.addLabKeyDependency(project: project, config: "implementation", depProjectPath: ":server:modules:LabDevKitModules:laboratory", depProjectConfig: "apiJarFile") @@ -14,5 +15,6 @@ dependencies { BuildUtils.addLabKeyDependency(project: project, config: "modules", depProjectPath: ":server:modules:LabDevKitModules:laboratory", depProjectConfig: "published", depExtension: "module") BuildUtils.addLabKeyDependency(project: project, config: "modules", depProjectPath: ":server:modules:LabDevKitModules:LDK", depProjectConfig: "published", depExtension: "module") BuildUtils.addLabKeyDependency(project: project, config: "modules", depProjectPath: ":server:modules:DiscvrLabKeyModules:SequenceAnalysis", depProjectConfig: "published", depExtension: "module") + BuildUtils.addLabKeyDependency(project: project, config: "modules", depProjectPath: ":server:modules:DiscvrLabKeyModules:singlecell", depProjectConfig: "published", depExtension: "module") } diff --git a/tcrdb/resources/assay/TCRdb/queries/Data.query.xml b/tcrdb/resources/assay/TCRdb/queries/Data.query.xml index f46107a76..20a2fb77e 100644 --- a/tcrdb/resources/assay/TCRdb/queries/Data.query.xml +++ b/tcrdb/resources/assay/TCRdb/queries/Data.query.xml @@ -69,8 +69,8 @@ - tcrdb - cdnas + singlecell + cdna_libraries rowid rowid diff --git a/tcrdb/resources/external/install.R b/tcrdb/resources/external/install.R deleted file mode 100644 index 8000b3313..000000000 --- a/tcrdb/resources/external/install.R +++ /dev/null @@ -1 +0,0 @@ -install.packages(c("reshape2", "FField", "reshape", "gplots", "gridExtra", "circlize", "ggplot2", "grid", "VennDiagram", "ape", "MASS", "plotrix", "RColorBrewer", "scales"), dependencies=TRUE, repos='http://cran.rstudio.com') \ No newline at end of file diff --git a/tcrdb/resources/external/installCiteSeqCount.sh b/tcrdb/resources/external/installCiteSeqCount.sh deleted file mode 100644 index 286748329..000000000 --- a/tcrdb/resources/external/installCiteSeqCount.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -scl enable rh-python36 bash -virtualenv /home/groups/prime-seq/pipeline_tools/bin/primeseq-python -source /home/groups/prime-seq/pipeline_tools/bin/primeseq-python/bin/activate -pip install --upgrade pip -pip install CITE-seq-Count - - diff --git a/tcrdb/resources/external/scRNAseq/Seurat3.rmd b/tcrdb/resources/external/scRNAseq/Seurat3.rmd deleted file mode 100644 index 9fb58d5aa..000000000 --- a/tcrdb/resources/external/scRNAseq/Seurat3.rmd +++ /dev/null @@ -1,186 +0,0 @@ ---- -title: 'Seurat scRNA-seq Analysis' -output: html_document - ---- - -```{r Setup} - -library(knitr) -library(OOSAP) - -knitr::opts_chunk$set(message=FALSE, warning=FALSE, echo=TRUE, error = TRUE) - -cores <- Sys.getenv('SEQUENCEANALYSIS_MAX_THREADS') -if (cores != ''){ - print(paste0('Setting future::plan to ', cores, ' cores')) - future::plan("multiprocess", workers = as.integer(cores)) - Sys.setenv('OMP_NUM_THREADS' = cores) -} else { - print('SEQUENCEANALYSIS_MAX_THREADS not set, will not set cores') -} - -print('Updating future.globals.maxSize') -options(future.globals.maxSize = Inf) - -print('Global variables: ') -for (v in c('outPrefix', 'resolutionToUse', 'dimsToUse', 'minDimsToUse', 'doCellFilter', 'doCellCycle', 'useSCTransform', 'runSingleR', 'mergeMethod', 'skipProcessing', 'gtfFile')){ - if (exists(v)){ - print(paste0(v, ': ', get(v))) - } else { - print(paste0(v, ': not defined')) - } -} - -``` - -## Prepare data - -```{r PreparingData, fig.width=12} - -rawDataSaveFile <- paste0(outPrefix, '.rawData.rds') -seuratObjs <- list() -if (file.exists(rawDataSaveFile)) { - print('resuming from file') - seuratObjs <- readRDS(rawDataSaveFile) -} else { - for (datasetName in names(data)) { - print(paste0('Loading dataset: ', datasetName)) - seuratObjs[[datasetName]] <- ReadAndFilter10xData(dataDir = data[[datasetName]], datasetName = datasetName, gtfFile = gtfFile) - - print(seuratObjs[[datasetName]]) - } - - saveRDS(seuratObjs, file = rawDataSaveFile) -} - -``` - -## Merge data - -```{r MergeDatasets} - -seuratObj <- NULL -saveFile <- paste0(outPrefix, '.seurat.rds') -if (file.exists(saveFile)) { - print('resuming from file') - seuratObj <- readRDS(saveFile) -} else { - seuratObj <- MergeSeuratObjs(seuratObjs, metadata = data, method = mergeMethod) - saveRDS(seuratObj, file = saveFile) - rm(seuratObjs) -} - -print(seuratObj) - -``` - -## Initial Processing - -```{r InitialProcessing, fig.width=12} - -if (!skipProcessing) { - seuratObj <- ProcessSeurat1(seuratObj, variableGeneTable = paste0(outPrefix, '.variableGenes.txt'), doCellFilter = doCellFilter, doCellCycle = doCellCycle, useSCTransform = useSCTransform, saveFile = saveFile) - - print(seuratObj) -} else { - print('Downstream processing will be skipped') -} - -``` - -## DimRedux - -```{r DimRedux, fig.width=12} - -if (!skipProcessing) { - seuratObj <- FindClustersAndDimRedux(seuratObj, dimsToUse = dimsToUse, minDimsToUse = minDimsToUse, saveFile = saveFile) - - Find_Markers(seuratObj, resolutionToUse = resolutionToUse, outFile = paste0(outPrefix, '.markers.txt'), saveFileMarkers = paste0(outPrefix, '.markers.rds')) - - print(seuratObj) -} - -``` - -## SingleR - -```{r SingleR, fig.width=12} - -if (!skipProcessing && runSingleR) { - tryCatch({ - seuratObj <- RunSingleR(seuratObj = seuratObj, resultTableFile = paste0(outPrefix, '.singleR.txt')) - saveRDS(seuratObj, file = saveFile) - - DimPlot_SingleRClassLabs(seuratObj, plotIndividually = T) - - Tabulate_SingleRClassLabs(seuratObj, plotIndividually = T) - }, error = function(e){ - print('There was an error in SingleR') - - saveRDS(e, file = 'error.rds') - }) - - print(seuratObj) -} else { - print('SingleR will not be run') -} - -``` - -## Phenotypes - -```{r Phenotypes, fig.width=12} - -if ( !skipProcessing ) { - PlotImmuneMarkers(seuratObj, reductions = c('tsne', 'umap')) - - if (length(unique(seuratObj$BarcodePrefix)) > 1) { - print(Seurat::DimPlot(seuratObj, reduction = 'pca', group.by = 'BarcodePrefix', label = T)) - print(Seurat::DimPlot(seuratObj, reduction = 'tsne', group.by = 'BarcodePrefix', label = T)) - print(Seurat::DimPlot(seuratObj, reduction = 'umap', group.by = 'BarcodePrefix', label = T)) - - t <- table(Cluster = Seurat::Idents(seuratObj), Dataset = seuratObj$BarcodePrefix) - t <- round(t / colSums(t), 2) - knitr::kable(t) - } -} - -``` - -## Activation - -```{r ActivationScore} - -if ( !skipProcessing ) { - seuratObj <- ClassifySGSAndApply(seuratObj = seuratObj, geneSetName = 'HighlyActivated', geneList = OOSAP::Phenotyping_GeneList()$HighlyActivated, positivityThreshold = 0.5, saveFilePath = paste0(outPrefix, '.ha.txt')) - saveRDS(seuratObj, file = saveFile) -} - -``` - -## Write Summary - -```{r Summary} - -saveRDS(seuratObj, file = saveFile) -unlink(rawDataSaveFile) - -WriteSummaryMetrics(seuratObj, file = paste0(outPrefix, '.summary.txt')) - -if ( !skipProcessing ) { - SaveDimRedux(seuratObj, file = paste0(outPrefix, '.DimReduxComps.csv')) -} - -WriteCellBarcodes(seuratObj, file = paste0(outPrefix, '.cellBarcodes.csv')) - -``` - -## Print Session Info - -```{r SessionInfo} - -sessionInfo() - -``` - diff --git a/tcrdb/resources/external/scRNAseq/seuratWrapper.sh b/tcrdb/resources/external/scRNAseq/seuratWrapper.sh deleted file mode 100644 index 5a6f2d81b..000000000 --- a/tcrdb/resources/external/scRNAseq/seuratWrapper.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash - -set -e -set -u -set -x - -WD=`pwd` -HOME=`echo ~/` - -DOCKER=/opt/acc/sbin/exadocker -LK_ROOT=$1 - -RAM_OPTS="" -ENV_OPTS="" -if [ ! -z $SEQUENCEANALYSIS_MAX_RAM ];then - RAM_OPTS=" --memory=${SEQUENCEANALYSIS_MAX_RAM}g" - - ENV_OPTS=" -e SEQUENCEANALYSIS_MAX_RAM" -fi - -if [ ! -z SEQUENCEANALYSIS_MAX_THREADS ];then - ENV_OPTS=${ENV_OPTS}" -e SEQUENCEANALYSIS_MAX_THREADS="${SEQUENCEANALYSIS_MAX_THREADS} -fi - -sudo $DOCKER pull bimberlab/oosap - -sudo $DOCKER run --rm=true $RAM_OPTS $ENV_OPTS -v "${WD}:/work" -v "${HOME}:/homeDir" -u $UID -e USERID=$UID -w /work -e HOME=/homeDir bimberlab/oosap Rscript --vanilla script.R \ No newline at end of file diff --git a/tcrdb/resources/queries/tcrdb/cdnas/Assay Info.qview.xml b/tcrdb/resources/queries/singlecell/cdna_libraries/Assay Info.qview.xml similarity index 84% rename from tcrdb/resources/queries/tcrdb/cdnas/Assay Info.qview.xml rename to tcrdb/resources/queries/singlecell/cdna_libraries/Assay Info.qview.xml index 482b26d5d..8a69ed8a5 100644 --- a/tcrdb/resources/queries/tcrdb/cdnas/Assay Info.qview.xml +++ b/tcrdb/resources/queries/singlecell/cdna_libraries/Assay Info.qview.xml @@ -2,10 +2,10 @@ - - + + - + @@ -13,7 +13,7 @@ - + diff --git a/tcrdb/resources/queries/tcrdb/cdnas.js b/tcrdb/resources/queries/tcrdb/cdnas.js deleted file mode 100644 index e465f71e1..000000000 --- a/tcrdb/resources/queries/tcrdb/cdnas.js +++ /dev/null @@ -1,39 +0,0 @@ -var console = require("console"); -var LABKEY = require("labkey"); -var importHelper = org.labkey.tcrdb.ImportHelper.create(LABKEY.Security.currentContainer.id, LABKEY.Security.currentUser.id, 'cdnas'); - -var wellMap = importHelper.getInitialWells(); - -function beforeInsert(row, errors){ - beforeUpsert(row, null, errors); -} - -function beforeUpdate(row, oldRow, errors){ - beforeUpsert(row, oldRow, errors); -} - -var rowIdx = -1; - -function beforeUpsert(row, oldRow, errors){ - //check for duplicate plate/well - oldRow = oldRow || {}; - var rowId = row.rowId || oldRow.rowId || rowIdx; - rowIdx--; - - var well = row.well || oldRow.well || ''; - if ('pool' !== well.toLowerCase()) { - var wellArr = [(row.plateId || oldRow.plateId), well]; - var wellKey = wellArr.join('<>').toUpperCase(); - if (wellMap[wellKey] && wellMap[wellKey] !== rowId) { - errors.well = 'Duplicate entry for plate/well: ' + wellArr.join('/'); - } - else { - wellMap[wellKey] = rowId; - } - } - - //Note: this will only work if the incoming row has a container property - //if (row.sortId && !row.container){ - // row.container = importHelper.getContainerForSort(row.sortId); - //} -} \ No newline at end of file diff --git a/tcrdb/resources/queries/tcrdb/cdnas/.qview.xml b/tcrdb/resources/queries/tcrdb/cdnas/.qview.xml deleted file mode 100644 index fafeb6227..000000000 --- a/tcrdb/resources/queries/tcrdb/cdnas/.qview.xml +++ /dev/null @@ -1,29 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/tcrdb/resources/queries/tcrdb/citeseq_panel_names.query.xml b/tcrdb/resources/queries/tcrdb/citeseq_panel_names.query.xml deleted file mode 100644 index bf5d41a17..000000000 --- a/tcrdb/resources/queries/tcrdb/citeseq_panel_names.query.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - true - - -
- - - \ No newline at end of file diff --git a/tcrdb/resources/queries/tcrdb/citeseq_panel_names.sql b/tcrdb/resources/queries/tcrdb/citeseq_panel_names.sql deleted file mode 100644 index c4140f37f..000000000 --- a/tcrdb/resources/queries/tcrdb/citeseq_panel_names.sql +++ /dev/null @@ -1,6 +0,0 @@ -SELECT - distinct name, - count(*) as totalMarkers - -FROM tcrdb.citeseq_panels -GROUP BY name \ No newline at end of file diff --git a/tcrdb/resources/queries/tcrdb/citeseq_panels/.qview.xml b/tcrdb/resources/queries/tcrdb/citeseq_panels/.qview.xml deleted file mode 100644 index 570773573..000000000 --- a/tcrdb/resources/queries/tcrdb/citeseq_panels/.qview.xml +++ /dev/null @@ -1,12 +0,0 @@ - - - - - - - - - - - - \ No newline at end of file diff --git a/tcrdb/resources/queries/tcrdb/hashtag_oligos.sql b/tcrdb/resources/queries/tcrdb/hashtag_oligos.sql deleted file mode 100644 index 16e695b4c..000000000 --- a/tcrdb/resources/queries/tcrdb/hashtag_oligos.sql +++ /dev/null @@ -1,7 +0,0 @@ -SELECT - b.tag_name, - b.sequence, - b.group_name - -FROM sequenceanalysis.barcodes b -WHERE group_name IN ('5p-HTOs', 'MultiSeq Barcodes') \ No newline at end of file diff --git a/tcrdb/resources/queries/tcrdb/sortStatusByPlate.query.xml b/tcrdb/resources/queries/tcrdb/sortStatusByPlate.query.xml deleted file mode 100644 index 6ea09e41f..000000000 --- a/tcrdb/resources/queries/tcrdb/sortStatusByPlate.query.xml +++ /dev/null @@ -1,21 +0,0 @@ - - - - - TCR Status By Plate - - - true - /query/executeQuery.view?schemaName=tcrdb&query.queryName=sort&query.plateId~eq=${plateId} - - - true - - - Plate Complete - - -
- - - \ No newline at end of file diff --git a/tcrdb/resources/queries/tcrdb/sortStatusByPlate.sql b/tcrdb/resources/queries/tcrdb/sortStatusByPlate.sql deleted file mode 100644 index ae28bf06a..000000000 --- a/tcrdb/resources/queries/tcrdb/sortStatusByPlate.sql +++ /dev/null @@ -1,88 +0,0 @@ -SELECT - t3.sortPlateId, - t3.container, - t3.workbook, - t3.sortType, - t3.processingRequested, - t3.animals, - t3.stims, - t3.sampleDates, - t3.totalSorts, - t3.totalBulkSorts, - t3.totalSingleCells, - t3.totalLibraries, - t3.totalLibrariesWithData, - t3.totalLibrariesWithBulkData, - t3.totalLibrariesWithEnrichedData, - t3.librariesComplete, - t3.sequencingComplete, - CASE - WHEN (t3.librariesComplete = TRUE AND t3.sequencingComplete = TRUE) THEN TRUE - ELSE FALSE - END as isComplete - -FROM ( -SELECT - t2.sortPlateId, - t2.container, - t2.workbook, - t2.animals, - t2.stims, - t2.sampleDates, - t2.totalSorts, - t2.totalBulkSorts, - t2.totalSingleCells, - t2.totalLibraries, - t2.totalLibrariesWithData, - t2.totalLibrariesWithBulkData, - t2.totalLibrariesWithEnrichedData, - CASE WHEN t2.totalSorts - t2.totalLibraries <= 0 THEN true ELSE false END as librariesComplete, - CASE - WHEN (t2.processingRequested LIKE '%Whole Transcriptome%' AND t2.totalSorts - t2.totalLibrariesWithBulkData > 0) THEN FALSE - WHEN (t2.processingRequested LIKE '%Enriched%' AND t2.totalSorts - t2.totalLibrariesWithEnrichedData > 0) THEN FALSE - WHEN (t2.totalSorts - t2.totalLibrariesWithData > 0) THEN FALSE - ELSE TRUE - END as sequencingComplete, - CASE - WHEN (t2.totalBulkSorts > 0 AND t2.totalSingleCells > 0) THEN 'MIXED' - WHEN (t2.totalBulkSorts > 0) THEN 'BULK' - WHEN (t2.totalSingleCells > 0) THEN 'SINGLE' - END as sortType, - t2.processingRequested -FROM ( -SELECT - t.plateId as sortPlateId, - t.totalSorts, - t.totalBulkSorts, - t.totalSingleCells, - t.animals, - t.stims, - t.sampleDates, - t.container, - t.workbook, - t.processingRequested, - (SELECT count(*) as expr from tcrdb.cdnas c1 WHERE c1.sortId.plateId = t.plateId) as totalLibraries, - (SELECT count(*) as expr from tcrdb.cdnas c2 WHERE c2.sortId.plateId = t.plateId AND c2.hasReadsetWithData = true) as totalLibrariesWithData, - (SELECT count(*) as expr from tcrdb.cdnas c3 WHERE c3.sortId.plateId = t.plateId AND c3.readsetId.totalFiles > 0) as totalLibrariesWithBulkData, - (SELECT count(*) as expr from tcrdb.cdnas c4 WHERE c4.sortId.plateId = t.plateId AND c4.enrichedReadsetId.totalFiles > 0) as totalLibrariesWithEnrichedData, - (SELECT group_concat(distinct c3.plateId, chr(10)) as expr from tcrdb.cdnas c3 WHERE c3.sortId.plateId = t.plateId) as libraryPlates - -FROM ( - SELECT - s.plateId, - s.container, - s.workbook, - count(*) AS totalSorts, - SUM(CASE WHEN s.cells > 1 THEN 1 ELSE 0 END) AS totalBulkSorts, - SUM(CASE WHEN s.cells = 1 THEN 1 ELSE 0 END) AS totalSingleCells, - group_concat(distinct s.stimId.animalId) as animals, - group_concat(distinct s.stimId.stim) as stims, - group_concat(distinct ((year(s.stimId.date) || '-' || month(s.stimId.date) || '-' || dayofmonth(s.stimId.date)))) as sampleDates, - group_concat(distinct s.processingRequested) as processingRequested - - FROM tcrdb.sorts s - GROUP BY s.plateId, s.container, s.workbook - -) t -) t2 -) t3 \ No newline at end of file diff --git a/tcrdb/resources/queries/tcrdb/sortStatusByPlate/.qview.xml b/tcrdb/resources/queries/tcrdb/sortStatusByPlate/.qview.xml deleted file mode 100644 index 366339a29..000000000 --- a/tcrdb/resources/queries/tcrdb/sortStatusByPlate/.qview.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/tcrdb/resources/queries/tcrdb/sortStatusByPlateAndSample.query.xml b/tcrdb/resources/queries/tcrdb/sortStatusByPlateAndSample.query.xml deleted file mode 100644 index 6ea09e41f..000000000 --- a/tcrdb/resources/queries/tcrdb/sortStatusByPlateAndSample.query.xml +++ /dev/null @@ -1,21 +0,0 @@ - - - - - TCR Status By Plate - - - true - /query/executeQuery.view?schemaName=tcrdb&query.queryName=sort&query.plateId~eq=${plateId} - - - true - - - Plate Complete - - -
- - - \ No newline at end of file diff --git a/tcrdb/resources/queries/tcrdb/sortStatusByPlateAndSample.sql b/tcrdb/resources/queries/tcrdb/sortStatusByPlateAndSample.sql deleted file mode 100644 index e8aa0bb97..000000000 --- a/tcrdb/resources/queries/tcrdb/sortStatusByPlateAndSample.sql +++ /dev/null @@ -1,72 +0,0 @@ -SELECT - t3.sortPlateId, - t3.container, - t3.workbook, - t3.sortType, - t3.animalId, - t3.stim, - t3.date, - t3.totalSorts, - t3.totalBulkSorts, - t3.totalSingleCells, - t3.totalLibraries, - t3.totalLibrariesWithData, - t3.totalLibrariesWithBulkData, - t3.totalLibrariesWithEnrichedData, - t3.librariesComplete - -FROM ( -SELECT - t2.sortPlateId, - t2.container, - t2.workbook, - t2.animalId, - t2.stim, - t2.date, - t2.totalSorts, - t2.totalBulkSorts, - t2.totalSingleCells, - t2.totalLibraries, - t2.totalLibrariesWithData, - t2.totalLibrariesWithBulkData, - t2.totalLibrariesWithEnrichedData, - CASE WHEN t2.totalSorts - t2.totalLibraries <= 0 THEN true ELSE false END as librariesComplete, - CASE - WHEN (t2.totalBulkSorts > 0 AND t2.totalSingleCells > 0) THEN 'MIXED' - WHEN (t2.totalBulkSorts > 0) THEN 'BULK' - WHEN (t2.totalSingleCells > 0) THEN 'SINGLE' - END as sortType -FROM ( -SELECT - t.plateId as sortPlateId, - t.totalSorts, - t.totalBulkSorts, - t.totalSingleCells, - t.animalId, - t.stim, - t.date, - t.container, - t.workbook, - (SELECT count(*) as expr from tcrdb.cdnas c1 WHERE c1.sortId.plateId = t.plateId) as totalLibraries, - (SELECT count(*) as expr from tcrdb.cdnas c2 WHERE c2.sortId.plateId = t.plateId AND c2.hasReadsetWithData = true) as totalLibrariesWithData, - (SELECT count(*) as expr from tcrdb.cdnas c3 WHERE c3.sortId.plateId = t.plateId AND c3.readsetId.totalFiles > 0) as totalLibrariesWithBulkData, - (SELECT count(*) as expr from tcrdb.cdnas c4 WHERE c4.sortId.plateId = t.plateId AND c4.enrichedReadsetId.totalFiles > 0) as totalLibrariesWithEnrichedData, - (SELECT group_concat(distinct c3.plateId, chr(10)) as expr from tcrdb.cdnas c3 WHERE c3.sortId.plateId = t.plateId) as libraryPlates - -FROM ( - SELECT - s.plateId, - s.container, - s.workbook, - s.stimId.animalId, - s.stimId.stim, - s.stimId.date, - count(*) AS totalSorts, - SUM(CASE WHEN s.cells > 1 THEN 1 ELSE 0 END) AS totalBulkSorts, - SUM(CASE WHEN s.cells = 1 THEN 1 ELSE 0 END) AS totalSingleCells - FROM tcrdb.sorts s - GROUP BY s.plateId, s.container, s.workbook, s.stimId.animalId, s.stimId.stim, s.stimId.date - -) t -) t2 -) t3 \ No newline at end of file diff --git a/tcrdb/resources/queries/tcrdb/sortStatusByPlateAndSample/.qview.xml b/tcrdb/resources/queries/tcrdb/sortStatusByPlateAndSample/.qview.xml deleted file mode 100644 index f6e7f6a7f..000000000 --- a/tcrdb/resources/queries/tcrdb/sortStatusByPlateAndSample/.qview.xml +++ /dev/null @@ -1,7 +0,0 @@ - - - - - - - \ No newline at end of file diff --git a/tcrdb/resources/queries/tcrdb/sorts.js b/tcrdb/resources/queries/tcrdb/sorts.js deleted file mode 100644 index 7c62e725f..000000000 --- a/tcrdb/resources/queries/tcrdb/sorts.js +++ /dev/null @@ -1,82 +0,0 @@ -var console = require("console"); -var LABKEY = require("labkey"); -var helper = org.labkey.ldk.query.LookupValidationHelper.create(LABKEY.Security.currentContainer.id, LABKEY.Security.currentUser.id, 'tcrdb', 'sorts'); -var wellHelper = org.labkey.tcrdb.ImportHelper.create(LABKEY.Security.currentContainer.id, LABKEY.Security.currentUser.id, 'sorts'); - -var wellMap = wellHelper.getInitialWells(); - -function beforeInsert(row, errors){ - beforeUpsert(row, null, errors); -} - -function beforeUpdate(row, oldRow, errors){ - beforeUpsert(row, oldRow, errors); -} - -var rowIdx = -1; - -function beforeUpsert(row, oldRow, errors){ - if (row.well){ - row.well = row.well.toUpperCase(); - } - - if (['TNF+', 'TNF Pos', 'CD69/TNF', 'CD69+/TNF+', 'CD69-Pos/TNF-Pos', 'CD69/TNFa', 'TNF+/CD69+'].indexOf(row.population) !== -1){ - row.population = 'TNF-Pos'; - } - else if (['TNF-', 'CD69-/TNF-', 'TNF Neg', 'CD69-Neg/TNF-Neg'].indexOf(row.population) !== -1){ - row.population = 'TNF-Neg'; - } - else if (['Bulk CD8', 'Bulk CD8 T-cells', 'Bulk-CD8', 'CD8+', 'CD8', 'CD8s'].indexOf(row.population) !== -1){ - row.population = 'Bulk CD8s'; - } - else if (['CD8-CD69-Pos', 'CD69-Pos/TNF-Neg', 'TNF-/CD69+', 'CD69+', 'CD69+/TNF-'].indexOf(row.population) !== -1){ - row.population = 'CD69-Pos'; - } - - //Naive cells - if (row.population && row.population.match(/ï/)){ - row.population = row.population.replace(/ï/g, 'i'); - } - - //Tetramer/spaces: - if (row.population && row.population.match(/ Tet$/)){ - row.population = row.population.replace(/ /g, '-'); - } - - //check for duplicate plate/well - oldRow = oldRow || {}; - var rowId = row.rowId || oldRow.rowId || rowIdx; - rowIdx--; - - var lookupFields = ['stimId']; - - //for 10x-style pooled expts, support 'pool' as a special-case for well name - var well = row.well || oldRow.well || ''; - if ('pool' !== well.toLowerCase()) { - var wellArr = [(row.plateId || oldRow.plateId), well]; - var wellKey = wellArr.join('<>').toUpperCase(); - if (wellMap[wellKey] && wellMap[wellKey] !== rowId) { - errors.well = 'Duplicate entry for plate/well: ' + wellArr.join('/'); - } - else { - wellMap[wellKey] = rowId; - } - - lookupFields.push('well'); - } - - for (var i=0;i - - - - - - -
- - - \ No newline at end of file diff --git a/tcrdb/resources/queries/tcrdb/sorts/.qview.xml b/tcrdb/resources/queries/tcrdb/sorts/.qview.xml deleted file mode 100644 index b561d0bb4..000000000 --- a/tcrdb/resources/queries/tcrdb/sorts/.qview.xml +++ /dev/null @@ -1,26 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/tcrdb/resources/queries/tcrdb/stims.js b/tcrdb/resources/queries/tcrdb/stims.js deleted file mode 100644 index bca243881..000000000 --- a/tcrdb/resources/queries/tcrdb/stims.js +++ /dev/null @@ -1,36 +0,0 @@ -var console = require("console"); -var LABKEY = require("labkey"); -var helper = org.labkey.ldk.query.LookupValidationHelper.create(LABKEY.Security.currentContainer.id, LABKEY.Security.currentUser.id, 'tcrdb', 'stims'); - -function beforeInsert(row, errors){ - beforeUpsert(row, null, errors); -} - -function beforeUpdate(row, oldRow, errors){ - beforeUpsert(row, oldRow, errors); -} - -function beforeUpsert(row, oldRow, errors){ - if (['IE1', 'IE-1', 'IE1 Pool', 'IE-1 Pool', 'CMV IE-1 Pool', 'CMV IE1 Pool'].indexOf(row.stim) !== -1){ - row.stim = 'CMV IE-1'; - } - else if (['IE2', 'IE-2', 'IE2 Pool', 'IE-2 Pool', 'CMV IE-2 Pool', 'CMV IE2 Pool'].indexOf(row.stim) !== -1){ - row.stim = 'CMV IE-2'; - } - else if (['IE-1|IE-2', 'IE1/IE2', 'IE-1/IE-2', 'IE1IE2', 'CMV IE-2 Pool', 'CMV IE2 Pool'].indexOf(row.stim) !== -1){ - row.stim = 'IE-1/IE-2'; - } - - var lookupFields = ['stim']; - for (var i=0;i - - Ext4.onReady(function(){ - var webpart = <%=webpartContext%>; - Ext4.create('TCRdb.panel.cDNAImportPanel').render(webpart.wrapperDivId); - }); - - \ No newline at end of file diff --git a/tcrdb/resources/views/cDNAImport.view.xml b/tcrdb/resources/views/cDNAImport.view.xml deleted file mode 100644 index 7164ffd11..000000000 --- a/tcrdb/resources/views/cDNAImport.view.xml +++ /dev/null @@ -1,31 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/tcrdb/resources/views/libraryExport.html b/tcrdb/resources/views/libraryExport.html deleted file mode 100644 index 77fe3e9c7..000000000 --- a/tcrdb/resources/views/libraryExport.html +++ /dev/null @@ -1,8 +0,0 @@ - \ No newline at end of file diff --git a/tcrdb/resources/views/libraryExport.view.xml b/tcrdb/resources/views/libraryExport.view.xml deleted file mode 100644 index 0a2d0b33e..000000000 --- a/tcrdb/resources/views/libraryExport.view.xml +++ /dev/null @@ -1,7 +0,0 @@ - - - - - - - \ No newline at end of file diff --git a/tcrdb/resources/views/poolImport.html b/tcrdb/resources/views/poolImport.html deleted file mode 100644 index aca7a6e09..000000000 --- a/tcrdb/resources/views/poolImport.html +++ /dev/null @@ -1,8 +0,0 @@ - \ No newline at end of file diff --git a/tcrdb/resources/views/poolImport.view.xml b/tcrdb/resources/views/poolImport.view.xml deleted file mode 100644 index ea2101326..000000000 --- a/tcrdb/resources/views/poolImport.view.xml +++ /dev/null @@ -1,30 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/tcrdb/resources/views/stimDashboard.html b/tcrdb/resources/views/stimDashboard.html deleted file mode 100644 index 111c5d08e..000000000 --- a/tcrdb/resources/views/stimDashboard.html +++ /dev/null @@ -1,8 +0,0 @@ - \ No newline at end of file diff --git a/tcrdb/resources/views/stimDashboard.view.xml b/tcrdb/resources/views/stimDashboard.view.xml deleted file mode 100644 index f452ee3c7..000000000 --- a/tcrdb/resources/views/stimDashboard.view.xml +++ /dev/null @@ -1,7 +0,0 @@ - - - - - - - \ No newline at end of file diff --git a/tcrdb/resources/web/tcrdb/buttons.js b/tcrdb/resources/web/tcrdb/buttons.js index 2db248537..a59d12a5a 100644 --- a/tcrdb/resources/web/tcrdb/buttons.js +++ b/tcrdb/resources/web/tcrdb/buttons.js @@ -1,9 +1,6 @@ Ext4.ns('TCRdb.buttons'); - TCRdb.buttons = new function(){ - - return { createMixcrGenome: function(dataRegionName) { var dataRegion = LABKEY.DataRegions[dataRegionName]; diff --git a/tcrdb/resources/web/tcrdb/exampleData/ImportExample.xlsx b/tcrdb/resources/web/tcrdb/exampleData/ImportExample.xlsx deleted file mode 100644 index 4f5d287cf3fdc86eff630fd0f84ab2fb39f543f7..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 11860 zcmeHNg;10pv-6gm?LvVKj1c%@O0>Rxi1ef3hhv4q+Z?b#u+uiKG z_Yd6L^L^9ZUw8fboH}Kvs+44*pn(8b06YKyAO#3>1&-Q7007W1001Tc9#U7-!QREp z-o-%8!_f?+&**MvOOgu>N%Ii^34Z^-$A2&fO4SDJzcQn>Y3>P%wQEG=gcTRV6E+j- zQtZHeaF5rbsNck+dGMh&S5~ir@1#_=9o3~e=ff?Vr_vS?TDN7q(^jM~MEo3G=6uTk zD3vW9_^DJQx+XS~<0CHe@iFXV9h_X_yfLFS*)tg>Hb?CXw(ZU{g!kZ-66?h zi9Pjb3x)@UqFF5gmYWaJF9<V&srUL}cBBrCLbUeEp>tQR|}fC3@PnWv{n~sV1{aq2qWAhquh!U0BQb!l$?M(9LR0RX-onvQIVVdvm9BQ9yG~zD zU!_XQc+j}E#n6>B7UswftdL7go{CkWjWTNDBVgtehGO%l2I%z4YcCt$ltWC5s2-Mu zRyJ^b+)Efu_nS#B*hLl$=aD;@NX6<08C%Sid-vOtUf$xXt6K6{RT^hG^OAWOnA(0h z7fbKJeDr2e$mmxgW5>8*9hK~({CMf7Q_Frjl*ck^q6k zlyfjgn!!qf4}gbow`KZWPuv`wZHyfpY<_yPf9nhc*rkC{{2Wpdw}Sy;vcjYu4BLF3SYv5bAOKt&c8vy8y}Vt1xmtTVjS%4mmDIZq=7J-1mgSYvw+-NH#C%ZVR;2)Vi!#l zq3qTp%x_t(L1Qlm!7?*cyWxKqta?xuSliG*l16L3`P?MBa0Qzk=s`Swj)x%4hwK3u z-#HzLt%CM3K-!Z?{7TUy=P|_C)I~>ED$CjcrF|szUiPAO zOBLmSOF7Qk(1aj0NCNoj@$u-~#wo0P3CBLSrCIb3)0SdV(_YLQt6$(5D6C-0zQO0x z^ug}zzvJ)}!Q9|?ge+;PuZjR73mxEDY#9N>lSy{3v}8lB3FnVmEk!~P2`oFq<8_FJ zkv?X`ZKV71&z!baZF)NyW-Lcl>o!`o8;@(Q_OEHBza2ngU;~QMC|;EyS>TRB&>nri6z~?1Po}POJ(mou}HqA$5%kK zJvj=wxGAgfBNd=(dxK)he#LV1JoU=pBFLv}M_Mp)Axy(QV^@Q3Fg)_42h-wov5}p{$o8cdi>VnmH?-o0Sx5YbG1BF(N>h_G7%kO2^JLL~% z?2h@X)dmgo>hrlS*nK3JsZ$2`QdY$anZEy?=_R$XJWWm`dx^IMOzqwiChR7Vdg;y zLId>Z2g8?G0f&<5U*1UA%v`N~X;^n$ms}mZ?&|j=qieoC1%BYJC3m^OqQn#(&zBW1 zwllKh7WnFYAJhy*>h6;`mg?E>2lY=PEYWfS9R{{!G0*@2@OKdgSsFQ;nX0-tTiIKH zej2^tL<7aI%)p)%`3DqvgDJR!&_K(2yg)K4=m>?pn71wOsUCyuEXEX%4O!CkLt`G7 zK6?^vuCSjZ0)iv2RUy)~B z*;aqVFreiMFv1?d?^96PeJ2YqG`dDKtT`@Wj^*X zWt6oe{se_HS%f#M^fB31LU>sW$KK1>^nDBG5|9>$&F)T?)TwpT<07NGCy&Dwl7-Qj zDOc+bxe-hkW4warw=HSnI5siZ@ZfM4bROXiU|C+s+8L+A861;Z!nbeLiHn-MT(6Q8 zDPSC$AQw^z!lJUG8&>O5Uf-LWFA%B^nl`_xFR1~{u`SZi2gbdv!@t#B$Hr8yuEX!q zHA!@eRvsgd#kjql35Xll9?DzM(CP3P(=bJZnNQ#xi0OEq?n}l=k0k2Gsg0^qW#f{l zuJsoGHaBhCh!;b+7nn4ex3ZvpnM3i7|NB)QH0a==vxq$xVxqiE0 z`Cq>k9Mky@ypH16P*0-nUJ2hrkVI9v$QsqRp^We5VHZ+juYr$VHH2ngHa8tdH#yVq z`HBe4QR3*Qnju3rOmHkKPrT+CI)tmZ_ZEQ&41BRhlZ@?YPznk%5q+^*Scp|2$`p^d zUB2CORqqjsVgq+2?hP zNS%!aOY_7U3gk-h^ot*!`5O!GyIp13_@F=+*bNVzUm^UH=$0Kxb_H=n;* zUl&U=J2R%={(qbHf%cO90w=JYxY7;Ht=Sa17TH=*8g(Z21I*}So6M5q)s z$2&-`KnSR2BQ7Y{hR7x0Y*ta%cm8e^+Bq_l=3T`XE?veXv4t^V#G^b9H*V+OClh(M zxwnJO_#2$MKrb_TY$dA2M}4^QW_ZiHU@tC6l3Ikt|O^{NON{vfyb3w9uAu@iUcv4mT}N$N>&7r;kcUWk#^? z64^(^(6pKd4h4SL`aHqTX-#u*P)dsK{f#{B^>+A`o~1=EBh^ij1hz2pdb)^H2wyK%H=dP>VP|2PfZ zGe|1$&r)Yj?imUBJ52_G>Y71(Z+Il-C@2_p)g_|v8^hx4Ggpnu0ib6tC;b|M~gi;BT&gA z+w}ZjCLflpNTk^L9P&wewB| z_QmUuj8~P}V{fg=K*A%ooH6Joy|d;P%7GL4No)fcFYjRlgMi{AK2)|RnC9=%Zc&^7 zOS3pflF$UJ?K<#8ft7oS#n5pE8cmeRk{?pno#eOSUnM~-ac%HX&Lz$JN>2yHl@tr3$n35hKe(#3j>Kf~_%o%3k@; zbJA5>IvW;EgP=NQpSv_(DDx#a(Y=S{98c0P@jUX@GL zHH6m+i^y^%1Sf|gq^+{PzJyBHk9yniqDWY=*_aKV4*zhZFT4&ICp$01rcZ0RrFXGd ziS;E_D=L}a4dvd)T`nc|wH~7@y8f3OZTd6gFC8(}^dQF2aO#k*q#dAjEJgq*EWUPo z@W`Piz+6TY*dT!ky_!Hd1EYOA)h|5ujb#0nuulZFMT3lAzcuhHj*ih|adC&B)_2oa z@YhzXEsg~)NrFPL#= zjr4%fBx}ykTdo%vTOJe$5_Qw(*6&BS3bI%#`E0$*BhOyKGpU5dV>91#u`YjxUt*2H zp@lV3R*&2|(R?JaPoi8_(~VVgq$>xHUsuc$lxEucs!OG2NF`yOgAtxxG1%*&K9jsO z%Y7x4cu#*shez1R7S9>DR&9n9rsSgk zrm65O(0*Gj^EaR&( z>NO13tHth#C8>Q?$f+fPP4-;Z$S)`(Cb7Gc zV}1BQhdxTI-n}DECV>O4I9iYt&-$SV#S^(~Cjg_9wTw>v118|Dx^iWdzy`0|Hsb&+ zg})D8JaO}zF}y6XkoW?T)Z+Yzs!c0tCi_p#2}MhRAATIEVW_W+Gpv#zRHloiuE~%? zjlk;qyowVWJL-m6wJl4%b+Ik*jW;-&-uu2r=bgo zSO(Au8E3>C>_JOIld`D|0bR$R?maaFAuy^-f&$6}vz9HZ&x0IYI_*wYT@?>sWG zzVGcpQos=F;BAuly$<2c!VfN(_&TW&57eWuyV{&R!A|Y_(f6Q+O3tWp-UAlf6EeLi zARNr7#oNYcDebc*9t#56!f#$Sl%}=~V!HWrZk6K&p0M|7CifzE$a~s=|O5wcP#nl+9Uktkh#qTK#%5RAVyzDyBEkk-N zwdza@==rQuTBNWr!e#6``rdEkp)h5YYlcXEt!|n)xZZE5x%MvIkk_EtbyO@l^^-{JZ^jSn&bRjM z&2mr{TMQN>8;PMKe0t>>j?PtjWfM&}?l^*yU3&&m#fOH-kK@_&bSO||%LFadejNSA zd^+Bhlgh8CdN=TCI`+~<+{+XPeQt?w(ooe z3Aub5ndOv5Oo>YbL`-O*5t0>gjpVJ!S5cnpC_4U8Hyy&QVHFSofN5R;0QYy@{JE9? zS2+DgAYGcStGdpM(YMlI-62YMoX|}ruoZri11RAO+!H|;$bbGZ>4U{bF z7&CphX$0$QU8lQOzi6*L)^K$*}^0wf)!f&KQ?yk+J z#@k2Fes5;4;JIf0`nmD(ex9T%&m{1zJ z6=JVd#pHWxt+;ySj5fz{GTMWsc^Nu2+0k~WBZe=3a@HrGZF@hWew?0d{FQ%N`dv1y z)7=WAfccsUrNT3DH+7ju=GpR|&&t$7SEZH%zJyJ>Bv%)0uSZyCxWU0ossl)(Xq`90 z1no>z?~XV)JW~Rejd=O&{0G^-@18f4jx$kfyE|2$tl6%$_bYkUBNyJA)wH%PPp^2n zuuYmCtRx#-oj}H1j5C=wWVqoOl|{j4otnLN_c~FUf`98KvLgRnypo6F_3p}{!9sJU z%K`Z16(y@U`f*0&Fg+QL-o`SS^ME*!l4&>P7PgHKW3^CRRSspXCC|DJ8Spe3u?L&M zq@1!UT&_{dIu)BSPCF5IGFw+HYL|(gtoI@%nZ>ODiT0rx7bu%e6jTqNrxXuC6--Of z##zrtUfbMja&y~DG4cs%$yh)5E;P+6Ujp+q%EOo25*^(h`bN(o53UfM@la+Z+K~OE zH%0RxEQZz;iU^sJvG3E_jR_Ky%33WQQJ;mF$*|g8LI6u+a9s?)382U<8F|G*u8Su+ z4wY{E4dIQsdlLf<1Gc*)b@h+j>4S?O7+=e5x@(k5EmXxN4)cU#Qz=Gnlrvp$a6vbY zV`OW6AD)_)w~QYArAcGqrI08f;von8H-ui0LZz0D92>0KkZGJ8Pj&0pj(1<0_j* zmKI4du5Zf6a+DuDDXt>c+~0_>?U~*qZm0}T0f_-6?f*%Lm4qi2QmP1ZD|hHQC^8tu z7l$^tNeW|0yqEm)jSROwh4`n)s3TC@w>7d#8fXlNhyV&96%xEyLMez(QwX3qn)ewQ zZ(bOk!Y&CQMFD+^@oB60w=}j&A`^3T@76C=-Z3kt?rsW4N}?~bfIAVOY<9=Jh^mAdt`AH)X z6d0$rag~}_)$yXxre7j`^veVqLo`}XIEEN)E|T*5LLoKQZXc>QWrBw_0F- z&JDD?lOxk4D)wyhDM;ZSK1)?OsT!>W1SU)TGDHE)<4tOdd}GAb56Sp$0~ce1XDTMQ z>9nL4X*s4DsaBu+lGCbtWedH}jP#Jb-`uo%-WZ!Mk{+|*M^WRH8(X;~x|x>f6-izy zri=tfs$NdhH@S`QyqOH&9p3%2NVsX*4RR%ar13RUwn~RN#^vBc6Te)&5DMPwQM1)e zEeRGo(n??*$=pkW~ZBQYRMe~z+6o)4?S+pC&4*6u)_>5)( z{SKEBy5-Qxp_W#`G!lsLcY*^ZCw^pH*sVIesObegs6fAxDGqD*B!7oa8LlFRmrJM|P-q(NHB?H| z)E1IYZa}uhKt5<8)0;F27X2EHf!TBN@0R!Fa5mD#i09Z+Jv2#w=08JC^2QZ5cC>fI zhlZ#g!va&ixl#-#FWCyRR09|q7-Ee$wU(_kH059US+-J4cB`M0E31n&(e~Dy@4Gqp zi+y1EZ`hbVWIC4#u)UyvMf!ng&64aF4RCnWD9V2v`{1BK+udqLfyG9-K4_Q}zMvb1 z@xE;MGgxsLXb?iU)8Lh(XoM5V2GKUdUx-nPF6w?xeA5jqJXSFi?>gS2ziyW%@*fVM z_YW2_CZUZ*l0pI7DdHCGIY+;X!=XWc0oMdR|6#&@NGW&<*xxO;Oy)^dI#!@TlzzL> zz6sYrkV8CAEKDpK*tWyOLkt8Y@~)b2t*{0*15FvfFT$rGqM_cXFG2u0O+dJ2$L9wGe3 zf*&iIV?X-}890-UKKQq9SMQ`k8w-x)T4IDVa7PwMlW?w8=A1y?5e|Q)4wy~o3hhSS zdCiWP4Q(unMu5$}iMXH0^q^c>sla2W50h&jG%vD8o)tFhybb5Z*>yzKnYmQirOZNJ zjEz}6!x=w@y3@rUDUZqe{kzc~oT_fvE(O|sS7C%Jc~KAN(QE6bnuf8c#W|>yyKp+p zH(_+JF4&`-kt~SMUfR!Ay($CsZ}D5i`O!azY7;h zV|XVSAf-faB<&+cI*Q$&CqyAN#mZN%*D&KSv`wdblr>Xvzg*yHje0W<pvl7le`r(WcKo{T6f+#7K7Wq|I^c(B|1t5EOJ6Fp~bH z+h*fB^mY=*LM{E$x~a)ecao{?FZ=l#k@_RuB`S_)o->^h8W2(6# zW7@3vI>Fb&P;5)U;niXOp+OQ|aj!EN+^MgQ+7jBT%J+r&txbX-pgT&Bv*(MeroNSPf27CCa7G*^NH3#o z=uQ19i7HOYBSrwn*<_5ZZwz=9j7-6PaQ7AjXM0|Sc;D+r5bXli8M z(&ZS|b?Za^njDI}>s7h;dmzcTQMAr(=udBPg+HOrE@D(77}^2Rqm-89gz$s410bIR z082_ZiXBRnuuug=$pGvcFIVpydMub*n2M5#Er&Gz}ExI;;Q@(c*DU!qn95cl|;PAhrc+A#0MCW z#+JiOEDN!q)eCo#tP7i;m55rR5%j`EbTV_Yn0i>(NG1$e!Xs^>!7IuupA0dnGFK}x zv)8iL*0QoTuyS##F|#t)ehlpxbUUT~a>iaN2f!x&F^m`w#R6vTKk(g18UZZ={0JSK zE<^jXDKK(${I4Ydv+Q3-M!c}Y0y9#`G1MI~(h>pBk`hutsj-yiMWqo$e7%Kyfk{p^ zMR;l4!Awm3_|%-|Ui5~S`<bGB{hUJS|xPa?uUA^=A@c@ z-95x}+RuEdz`YRQzzZ7TdXuVnNA2~p#q?`ji0Wlmh=sRQi@Oc{n;8!QwT17dg|{#Y z`vS;Q{0^-5q^3)0m0@?&=N-&^PwN+5muH^JzKpl}7HpI&Pt{N0oYC)Q0|Jr(TtWZm z&e(r1*T2vI&?l=T`&WR!_U!$e@aOq6SSS9}!S^fSU;CK;OgIcSasPK$)2}$cHn;qN z^bBtP{iW6ASK?pW3jQGe3T7CX_-{=Hzasov-~I!^3!FRu*XRGgYWJ^zzm|*t0Q5om z4e;0E@vi{Crtkj%V8r@;PXC+C|CRLD?BXA!gWzQ2UnlcNuJKotzs5&@U;qFL6ac{A zqNQKSfAwX50IX8|Hi%z5**|)=UqSzBsQ-Wj0JLfTG^T%B*GxvP%`aGkefPe@BAOlbV001RGoHt<54h{f7L;?Wt0I2Y~ zlJ<7argqK-&paGVA^I%twl)-5i13dy0PwKu|2O`NEl~XIi(LmBcJuQsQK=Tq(9Gb% zLR8WQQeB!&l=tp2S~N9lM31k%>C99$%2C_Fsy2hVw8uh(B{Q_z5@IVhELYmfG=}K; z5hYHC!Z)(%PXcm^H6tpc-*9ITV(jlDk5;27*3B5RSW(?kfgf^LJ>cAEKT`TGQ4}I7 z32tUsFND9G+7|xOp$)G8!>VsI@OFdQAOk)b?Z;Z|?88ijtG2sz&2>d7q$W$x86-BCJ6yLBft^*Xg@$~h+R zAKau!V~;-cXbwb$K(Q_70pA*SaZX4fCGeYQhbNl61P25^-eE?L@$p5a%m5PHj0oPA z3Y-EF{ee4@P8L&V;iQ*EvzYU#S>oi2@rRfsw=h4uyF&n|{LM|PHQ1<6VAxZHokfSa zsezNJ4TP2D=k|Z@`Csgle|hxcn5W8}Y(VI~>~(O@`S^S^wuFM4xNIZkGao;>Ijri4 zY#NgJRytDbXT+}&WPF=_u6t+ag(9|kD9@I;ir?UZ1gUG>N}vf>jxHz+v`+ETj>Su# za9qbv$IlXFRW|0P%lMn+?;oNOlfAe<4ilVQtfDLW1)VjqMxOutzSF_69s4I zQAwTDk3hMs2H0FUtRO>Vy)ex5RjvA61*-3q1XR`5QY6uvt=-j2&YmHs2Y8SV9}}U; z31N5uhBwD3886zOoS_~yU7J1i7_e^7z5=e17AZU``r8+^oJJfylc6X{O!dd^@nLw z=AdJQs-;5N5K%&0lm~enFG%r@cDEIs^K}?j?T;pgj_?$IZ z7+DU+qOuHSn=W6{mvR&Lcv3QL2Iy#g=*UvUg1`Gn!+gf@Tnn&)-11xxVWL^pOr(?5 zR6Qxl+6t0CLPc5hU1SA%GjwZ$y@z43Q_=iP8{D>&pqH2N(c%hcApMb7{CB2#>ViV9 zPC+`^^HeM=1_d~4tM9kkM4Q{hcTXrETUQ;cd}PVDyUTYy)*fB0B~@CN+x+ajCHTT* z>s@pkxeOoA#gGh6pZuyAxOA&zR`oq-NVM5)lph@z=`w~wj{rqMFrA*%Src}iORd74xr)W z8_kMuwS94 zKh7{45>Z%5%Ph`E??0NKnSSu<(+eye7hMVW=GdWa$oo6we}i&#VS4>O48oJJs0R8S z$`A`9CsPx3XD3TLbI8x&7yefHr@wdY2P{yoE?e?ys7Q_MWEMA2JjNZio{5|-+g0R^ z?cJ;8edF7v^J%5&eEkMln$=aG@f$@M-Pg76Lb32=Su-mF3Cd@$T)E1@EU(F!T zCPbfuZagbVP21Mi90u2Tl79$&ATh&y%QV&iAGG?Cz@qHHbB4K9yqs@q4ur-m6m2xh z(vpOrq$D5O9j%3rULK@Mb3@qa^J%&~6755@$ojNgs0Z7Z%QqxL#f4V3M+PWWtV-4U zOsn3{UkeDUY2WTddXo#;Ww#E<9t{S{@kbkqWXbaPJlQ?+GZx==J4>_n#zf4s{n~eY zhW2k3I93s;_k)2akoae__gl1cwlK9dW&LgY+hgx&&)d!NfLh4Q+_0VZb!x!Vm|LJ4 zn|$5a8jCr*<{Z-*W64mcEFE_kyk`I$LW2=6f@|%YdC*kaht3J%Pnh)6R4*I8D?f1V zG^U6yhzcej(~h)OX;$au6Q!&%||pTP}+&qYZg)gJfGuU z-z(`H6m1N}lnPMa5xWT|&qzx~Ef*@7^Z<5+EEp=JwAb?YsnaZS8;qIrHvk&xOL&D@ zi}OQFN@FuYU%w>}%7v$fa(#bf7asMv$t<8R;Qf032p5mlLkI9kTX3C z^Uo}_7attR!s`66geTQ~?QRat$jIKs6-=o=FXGkoHEpiw*eU)n^Oa!KHoJxV!pPU! z0Ip!FD`Fk!6)%f5wc7g`vg9;R*Gmv zWrw@Dd@|tfXrqX34aeEH|T(s$Wo^14FHbKRoQ*sotnET=f~5DdIr24a^tm`LPFQ)$H6 z#e$YAYH$KoNLu(kR)K`or>{Z=pV9z?UJC8i@_eu`jE29ac`MF;g!K-t#2Ld3TQn)Tp#nfHk+j!!pG z8ROvQc~^zN({VFCa^tU~O150RTOJD6Cf{O%_cuNXq}dsa?0eVwW?EqlGc-mSW$X;- z+`T!w%i6u&c;dOB)#CG^bc`slQ75|y-?PE3U4-skS{Qd%w_eFK|5v$6c2Hq|;Ebvr zuR|Z=qrP2#f8ABtaYrZG7Ah+Kr$e|8&oHWQ^bST&0t6L48#R9n8v8C^`x3 z5A(LlraaFHa(AF9bYh)~?|dW)+AP!GZLV1#w{y)r>mh-!*lUa)BIcBShKHfd!pZQ%I0T_}>aJ7sfvQlfBSR!S&v0CO{er%nJ}Z^2t_`UYmr&r14UC7PB`t9Xo+89;nGN53soo9W27tu?BG z3Bm#mp$qDa+XPufjc)&fUYRJw0RwFX>tyko=q?#L zPq~WtM^*6ot!KJUR-PUa7g^zQYvGTS)L=A?H0()lQ>d0ye!{QZ(^W(zt}bMMon+F) z*r`_8rxrWI&4S9M9O!vnlS*Bj=Dv_fzNJ5)!!K?#5yx;i9u(|D*Pf>=CIof3YRn2K zQt610j5Hj6o$nE0(o%im{+me_Ot@i> zhdkZ$nr_$=4JEgA>sTnMI=l~1s*MP%gIA0|&J8OYrI6A#pH}?Kr2=(y64htNlFB3d z&epcA(>p(Qeau(S(-Zm#H$5oHb6MM(!zPzRVd?0f<$2FHb`C5A0Dv2o&MsdBzU-BJa>{o6#!+Z(>yCYoOpayme6i*Esew^{(w{a8Sr|w!}ZRj2` z>rU>OdWPnX&hNME$D~vTWP28p&tp$@l{DxwSbTCF_;fZ8n~+N%%C@;Zs5B{YMyuxg z)M>RRtKDH~+NRRaqx*xq5V}y!fvgq}22+!O(P~S9xUA8EBTxpV@@WZtOiI&=9A;*9 zp;SIRLGa-AW}JA>gWk)CCqHx|juOGPL90ru9eQw7&IuSN4uOWJ-2)SQ#@>Y4E>qM~ z^zW}W$_h#_FYKeGb4o97Qz|1sRg5!C{!ZJa7DZg1hy=E&4-b^MYgW3-jHSvS)r=>n zTts~p!%7;ft#UzFB*%(KHS|t&A<}7xbj{vn1r1bYI`zZvvTwDg3mgn7^kDXYd@D#o zqk#JRguIF(quzenHQhd#J3?%(jde<*7XCFXmQP&g<{mv#_Y(5W0ndhB`fqbKKL0G9 zbK$;MC!$5B0-VJ7`tl+x)r|huwf>sTK!5!$FM2$uy_5V)`$w$LUkKG}m17K%YM?WY7vTF>=puUGC; zcToP-bJp?XTDxG4Z5`}1;qRXFv$6f7*!@-JzKhefPi6ykFVO7@YWc82xoWeb3$1%& z)v2@jLLD2_EE z-d!cW=w}~+P7pDh_Tzc25-+fb`s$G8yqZATK=zj?!P6rg{S;`Sc@VZG#?u6)tAB>i z?NH<@#Tz#IwfR|`2?(mLrmDK4NvXL%9PPnEXdqYf(cM-Dr%S<#!bd+u+RX~zGfMBt z_;E|UafOwbF{8Xuq!K7QtSecc8Gk-L`rMi;uXxmz7vWN#4P5>8DZnWQ!}XwMIL!S zuk?BWzt>&qJhwh!jr?sQJXeRINb=%dJC=tQq?{2bl!$odufQ=7$yiGCj#v+ont&n} z9EdFQ3{8+zg#p$dtSMMSs&YwYhXyV-13NDdXdii8oT3FE&(?hu5m|EZi4w^9ZX65= z9@Ia|AEM-~nUvPlA8wk;(_bImzPuxgsWB@RX0+OL%3OtOZ+nd}ukzzk$`Ntlk)Bvy z$zY9JWVtQTpv}$Q@WaPtagCL)(2#*IhS*G4`=$e$+Dyvu4ozhhy_bDt!@Z2TX-d79 zjby{UsmO3MmA%Mt@zS-hLVM_VB#rm*q9m8s6|px*>NVKPUKD-16=~qm6=H$<(pFk1 zcS$A)pc9Ub{IklHnS!?wg{cMH3?6$t3qa~|yKXi=Yig24tlgwJd4wOCjf{&Z-I?=7 zfG7RAT=N*kVnshb^|y$tp-mO+rN=EwGh`RAP2EBx46F3G;U~mUx5c{PT^nai?C4{T zMcpn5{w--l$?ng-fc1DTF!X5P=`V_eVs(5`I$TgrlsoDl{4}F5L;*ZO_zp^muy=b$YopoP=DQu$Uz_5_x16 z#`BFA^s<{b<2?ydFs89-v9Sp0E-`PoZyEO@mt1K%{2<~nu(qv^A7>bj{sL{_SbN)>9HBv+}Yo=69#B^ ziPcj<70cw2kTH-&k+BU;jLaH49fCVAy{QGM5vV&EOT8jN6#awP?Vk{HTnNQ;u%_m4 z%g_vMK{(+m^NM1`f!hA?cm9BR6$0f}6)-YF-lKTH!%9zAuL~x8q)ViaMF)Lqdqt{v z$O4G6p+*RX9q4Bmhw$~T`Hhs$D*)2w-wc} zH^7K2171bKllp6$gG7wK5oL5K(#SV4cI+DQPKp(Av!fzO3v80lD533aJnSYOR+Tcb zUo238YuKpDPgM{4Sk>7oRM@zxII5~RIBGd~d7iOxuvKM1TYKFO>DrFCiWLD5$rrz( z#~`r7jQcNkHkRqJMn26E zE%WS~+7O#B@ysM(!hb2MXo`2BK;<4w_zrrU_en#U9HS(g?Bc-DO%Yf;Ui+@85_%DT zm0!tkY*TRIQ??kF@+RzijiEtBb~Z~~O0AW3L@+nX_GfoQ@B;GIqpxLE5spOFTbARm zbKj#o`?}jc&$_V1Moy7Gp%ugnDMy|i3$Ip|)fVl#4rc~r#@M`vzT`h2us(mnDuY0< z@tM*P-e`%`aia&}){2LARPa+D|6vc?p%UM$zr6rtfcHhB@)eyYS))nE+aUGR&LDFy z*+zG3)EAQ;BF|DLrQ<&Zf9~^PKJk}@q$vR?`;V8lD@Chldj`}5y2+kK^f57fU@0sxDT{&G)$ Ygl!cCBp3z(03hr~3xjbw<Cr?hvF~N>Wl9q@+QR20^+T0ZHkSlK9pe zpZ9X__x%I!JNrBn&&>Jl%$ak}@5Cs}0g)g86aX3k0H6Yh@%RtfA^-qL$N&Hy01Z(` z!p_#&#MW8=sfWFZlOBt^jWuO965_*503!VQ|1ST*8Yp@C*|r@9ZqnEik!)5E%L*wf zLL+M+)1lo&ed8XhNn5){{P61K15*{X3bZz;iuI5V-LW81=`5X=xaf*C%cYhQtpP?s zWU1q!(6!9lNB+4b>XB73uedUaF!%RS#%fUI>t~HvEU9m)p#)sj_c%7%jugL%7l(>S zK${rWix4lSw}n2pYaz58(mfk9e%8abDi&v{^usn<3%r zGXC!%BtOuOgroWNX$09t@f_AddbSwFQsN;N=?&b^Zf}7A<-fUUjT(&R1a5os@Us|j zH`RAEv36o*`MLhDd;SOeEMFOK5_=d&m`%5dbT z_Ru%BZatPvZNa;sUvvagEva_@L$!3503<8|e;3)suOUV-jxlVXU z$G8cBxuv?|<@Iio^fG)1>${g zV=N5EMS6i}y)S@lH*h=vL^Ikgz3(rLjEB3kwOzc6ftBKf_dz(ehbaTXyMF!m9W2mZ_UtFpZqqg3+WbztMmt;8kx z#k7!G;0R^mqf5Gtyl*44o)~Glm(2`%`Nm;jdcye(9p$ew+zaFwc@_+?-PveQ*9S3C(Qr) zV_{@D7>~|2kokD=oW6{Uq{owrY13a@^IdzkJT~I(ds^mG1`SQX21>JrE^x9*#Z#^3@Y8{#4y6omB=PiL} z##`|*trXI{+#6q_q^21QP7k8YxVs-mzP%6zh^!N&$BhvARB<+0%<@njmY3$AF&PLj z5!LIZXWPWG4gfBA69yWrE`51l^g86YB`>az-+n4In3}anJf)V}G!nIWACW#}-(_?d zc=hp(^@_P8o8$F&YS*0CBCbi^&dXB8-|4g!zuMzO`3PB1T+g9_9(>7t>pL5re@)|0 z%fUPRcse~-S~E$89wfjQvx1TB0Ji<=vWarY?M!%Rx_OxcvzH#>4$V;mOFA6LJe{;# zp{GC2G#wU}Tgk{ODZm&wT9}==_oDL|wzi9oxO-FF@V3*NTai=Cr`W=Yl)ZV9lf<|PHO1irq< z$7u6P`23mUWVUuq%q>f0;M2tOoo09Wz#-fU@)nH^=^fjvPjs4X{GYL+YZDK#PHBD8Qqp-!Jp~6qytNJWvHCxcjoG1|PA~yHTS23Xm;l!G7?SU$x zaFzP2>~slhoiBnB&CiUx>%qGxv~;U+M)Ne9*Q$J`E~d{zb&A8!()@}LPUa>i&Q5>u%Wns4|I03c(QV=FFvz1b;Eja4XWV2ENJ5o|x?XJq zXmmM?GM5x{2D$dEA~R`STeBZp<4*k{_>R;R>owDO17gtXb3*g-1J7CJ7O@K6t$7GK zvtW$j7)x^sP*L%5Sa*yjK1M~53hgyfhj-^pMHKqG7~%C9*)R{-=ZjaQ!zD$QHb?rX z)vSs&`%J4ZH3ImBo@m|dM7^XCw9RQ5lsOs-l;w*t5YCq2>v^<$Ln*ZN**$B;;47?z~J>W=7j1Vv^>Dq4kL;gkocD|FF7F0HMOuTPbBiA#UnjIROE zNMFh$#9C4iYFrkV1?m5iIwTvB9>)3YzHLPG!;hx^eg1FO3r0D)Eg$aelu+S#4bY?r zY=oZbnwx!Mp*w$PPaaSKF-U`kFNpHMihs!`0N?qkwa*}hZqZnmFr%qFLq z;@r^3N*|$cx+`)v#~xz4OAiQtzG>0+sE!!a5MZcm#HD;_ULj=t{vBl@|)5Uh(Tm6>vX7dyFjO}x(K>~fe_@;LVn>K5~8X_OBR?d>% zdtaRn>FaCmoH=n$EaSx*?ag-)4gwR!H<*N4$F3GE<#8n=NFE~05scB;r(=C=lo}5v z^~H0Z&mn_+LJ`y!Vyn-fyLb$i>vM}82V}scP?_rZm!hoYjt`6zn0F#K7|G87Sx!20 zIK2_zXmz<5bt4X9=yayX+idwv58ja23W_uOJxTqH{`lSA!E;(% z($8Zq_qXeD2*KZw^ZSVhU>mjkXO5fi2~GquSx(E}4!c-VJBbZib4TMm|1@oCrs6-E zm%#BEm;48^NC4!~;7dB|13c3x@CB9wU}2hYPa27Ixmg?SzJK{vd=b)Xef1B}MCmul zD-H@9Xp9L63p}fW(3yl;Z`p}}=+Z6Mm(2u1b*VRC=>A40e}=98=>E%kpDau45r)QS z!_1vQ?b}zUx7oWl8;?8}HJiQPm5mbzHfrY-<9jx^wFy6n&j{!0>eelt;p>;JVuKV7 z1kS41^4Rww-S6A=^V3%M(phwavi-DGSU8Rh^v)l~j=8gvMCY*pR)3+V!_f!Pq92lYyxknP( zK1|B&JJC&W0Mx;$phD_}1l==NrPDH*n!3yJbxWL3<^ps_Gf$zY$?Fg5;d+)@Q z8jLtd7)W*pyF+UrujOV%IrQkw*L6?k%kf*2H6s#*+^~MUbeB(x5zu9E#nEfc(qcL? zYHf+GWO8B&4t)^Rk+2D|jKTGD3W=@W=-soc@-uxb0jZP1Lt2i5P9bYujQ5BQ4^XaL zkade=H>y($>3#HXC)75)E-Gpf(VR49L_4!!Z?w;MP7o1|F%oN}BIl(ci+V{C#AAju z$W|w=zP6Yb_AXxUno?y=StV+dU2H8P`;`ZR26aPT&X-!@;V-?y0TOO{ym~$8rvYXQ z#RZn%_!Kyc`Nov-`K)HUPF56-NQy0Sxis-dOKUMdjyCK`ZBwe0R(0Z6?diy)k<=8i z1*8~%WbAlS)%PTBmWu_AQz_8%sy3abB*T3%nPN+CP@7N8dNP6Ga3Uzg`$1d2lBi&? z{bggef3b3Vq(qd#h(;O(VI_jZ>br+ip$Draj-yU*J;WBHQ^cMg^S2eQ*Ll64M87Q6 z7D3aFR2e=94p?`3qbV87?f@MJ^jP5nAHYPvShJ=7XQqMDEuLx^+3%0`(}S*v#o{>5^kDh8>_ z(_;yx(S2tto0gfKAG_XWt7mVM`iM3?s3`JSTbshCmPO!n^v|Te=L;Jdg3}r1 z?}X*#>|t&4TM9ILB%aGZ!*u00`_ z{>vE3Ta*1oXylC3i>kbFO5YfT@VZaq7>jn{%=o!{DqIN%Pw1IL{ffVwE0ogHlQPoQ zQy{mE69&r8Q!z;%;)2Ca7RpZVK`o7@QyffBu`p;cI3Z>>7cr<07t{`3e^3gi!EJ5EqK~+y+#(x_eJnecDZZQQcK~eQdg1V3 zBA|aa4(h6_H7c0x+3tsHC?a>SD?tjyAG0|)kvc4syc>Dhx-=Q8E(vSi?@=6FE7-sT zWt)8PvXZ{F5Dj(kE@!Aj0Z_?NE(KdAcOZY?n0tx21s#_{T{-veXc{r5kUErUb-Pz( zTfSIc7Y$aQ}H{(V(1L7!acT-b%* zHbyO>$+>6lVYP^Xzm}`Wx2|S1^6O_18QYi5zy0kDuG!XVrKn_(8Kw z{X#I*|J8mD*I6)hL!F(|j8}`3Og(EaeOK=z-{=#;2Hm;OJEt+&rqMQUrN@>8v4E^1 znBsQqP!2n8p~xmdtegqw%8wDp?#7;1y)`7~hP)*c+!2Jq=)K8gj$Fp(@efitAoHL`; zm_XPYMcjLVR--3JOx6e~(}|2>x;M>?eEH_Prm=+OQPI(-J7qh&J}(EK?7x=ndUXGF zIX*agtwP4VIkIPnB8OG)E4{*TU_P}Oj)!5m7rEEXxDW1;*e=o7{fDaz)B>fiGWLtz zyV;Ahvkmy>XV8*WFy%gcSEC|U*pblMstZ`^A9^?>`))XHe*~P089k%i`6FVj0rHwD zkL0U{z_Y@~%&!({l2k6rS(`hl8AXi-RLk^B9v)WKyHyPY{eC{)SUdy~V2dt`w*; z-Ph!k_d)KkPEub)<)?`T5Q~xvnnqI#lGls|#UifQ^?{gI9a!B{6Sg!u))Hs$ znhCzyWo(k~kW@Ep6y!D>>Io_kE$(s7^VuYmax*urn)Eycb@^p#+gQ;P(D!t_PjVPD ziP62(owd(P@MinG_XdW2a>{s(q{}l}S`2JDejIXZ+E9$;FlVsTlI>QSI;BZD?f$;H zr-+HSB3(^gUiJ%)u4XbD;RMhoj*i!4^2gkdpqM~*H?s!|xBN_fx~2;Ro-$nICwcBi zOEJmb4-5G!4Os<49!ZMUaB8V9I?5XgY;aevw~L2G%;$}~LgV#Cg;X!EPzrAUh#uBG zWbCS?%&yk!bwQA?6K2fdaD%jxZLaQsUF;HnCst) zz(``VL+dnQA|dM8uk^!hU?}EC#^0|MLz-AG^E-OaD_p&vC9-VHdwpSgrbp~5IeYpv z#q}!K4+rHF(HjGUa6e*{bs|jzgD}4doTc-ijx&Uo;NA1CRy5#qkUR|5)d~VSftXB`ye+_m*|ls+f|2K&^C1XKm}!(iR5h(I||NtExX z9iL|WVv4`*jDIilP(o5=Bst2G4o#mQm3dnYK@c%c|7tk_NptyxC8{cQKn_`TW)Qdd zrX_-^-$;|$3w?KSMc~5;B}}Ayu`^-kamtoeAPylNgX&uA`EW)M=+Dt-yk;AKXi-@x z$g=|SQb@P%LYiwgWSjARsIY#?sKU_{?Ukpnk?2xuiwC7>xgZY=^$&zS4MQBX4du0#pGE26uIw{ zvpzcQq;#+6__`xO^!ARUpLoHU;h)*o*6tfh4!BX)!?UdizjKB?yy4_*;;3rk z?EJIy#CqqLjm5~q_Q(LuK-s|FP{!n;X^&|S;wV(KnJoYzfF8^Ub7yTQ23E# z!VG~1?HH$+hln4ve8mvrLDFsJXiFeJ009chTVDXSG6}|NeLA3h$GoT_uLaGZ*WZvl z6H!^*lcvAbUOd*%kSeAOdGw1Y8@N`igK|a8^r%?E98CHNHLMNB&1URjStT9!*&GeD z21Zj-P&w#hRfSb5!#JzitE<`B>)3g?pTgK-)tSL9y>5pOT8}tOLyrcjvkn$}kgZxU2WHe67 z4H05%%@p#VXI0XMmb~7Ximn|QpLwwrx$5bDDJi=^#;<=8Gt#9Tww4V+Mk2K$JWX() zUeNQ#M~+3WIb;)sM)`h~()W6jN``0b=!-0kR3a%Cul^+4CX1uqfUkVCbDsDs_r8fW ziPX1~!j03uOQ+b2& zr}-ivGQ(Tb|NiO8pReoB?Z148qAd4!fWHsL{}lYWEr9#PUq2U2|uabXWLpzy6nSFI-`8;eT}R?;_l7P5(lmf|tF2{QdvesJ;t$ zx6}IzkOu2Fz`K3ly8w49?_U7p@R7vNXZqhd{I2NT(&LvX7Q7Pq`@{SyMDC*eJwy71 z0RWVe0s#NWm+p$+<+5J@;`e{Mi91a8ugrEA^zVWC7bJX8P4Sm|`Xg*B%OS&U5C8zd Ne{^s&_MrZG^gq10b5Q^Q diff --git a/tcrdb/resources/web/tcrdb/panel/LibraryExportPanel.js b/tcrdb/resources/web/tcrdb/panel/LibraryExportPanel.js deleted file mode 100644 index 46542e463..000000000 --- a/tcrdb/resources/web/tcrdb/panel/LibraryExportPanel.js +++ /dev/null @@ -1,924 +0,0 @@ -Ext4.define('TCRdb.panel.LibraryExportPanel', { - extend: 'Ext.panel.Panel', - alias: 'widget.tcrdb-libraryexportpanel', - - statics: { - BARCODES5: ['N701', 'N702', 'N703', 'N704', 'N705', 'N706', 'N707', 'N708', 'N709', 'N710', 'N711', 'N712'], - - BARCODES3: ['S517', 'S502', 'S503', 'S504', 'S505', 'S506', 'S507', 'S508'], - - TENX_BARCODES: ['SI-GA-A1','SI-GA-A2','SI-GA-A3','SI-GA-A4','SI-GA-A5','SI-GA-A6','SI-GA-A7','SI-GA-A8','SI-GA-A9','SI-GA-A10','SI-GA-A11','SI-GA-A12','SI-GA-B1','SI-GA-B2','SI-GA-B3','SI-GA-B4','SI-GA-B5','SI-GA-B6','SI-GA-B7','SI-GA-B8','SI-GA-B9','SI-GA-B10','SI-GA-B11','SI-GA-B12','SI-GA-C1','SI-GA-C2','SI-GA-C3','SI-GA-C4','SI-GA-C5','SI-GA-C6','SI-GA-C7','SI-GA-C8','SI-GA-C9','SI-GA-C10','SI-GA-C11','SI-GA-C12','SI-GA-D1','SI-GA-D2','SI-GA-D3','SI-GA-D4','SI-GA-D5','SI-GA-D6','SI-GA-D7','SI-GA-D8','SI-GA-D9','SI-GA-D10','SI-GA-D11','SI-GA-D12','SI-GA-E1','SI-GA-E2','SI-GA-E3','SI-GA-E4','SI-GA-E5','SI-GA-E6','SI-GA-E7','SI-GA-E8','SI-GA-E9','SI-GA-E10','SI-GA-E11','SI-GA-E12','SI-GA-F1','SI-GA-F2','SI-GA-F3','SI-GA-F4','SI-GA-F5','SI-GA-F6','SI-GA-F7','SI-GA-F8','SI-GA-F9','SI-GA-F10','SI-GA-F11','SI-GA-F12','SI-GA-G1','SI-GA-G2','SI-GA-G3','SI-GA-G4','SI-GA-G5','SI-GA-G6','SI-GA-G7','SI-GA-G8','SI-GA-G9','SI-GA-G10','SI-GA-G11','SI-GA-G12','SI-GA-H1','SI-GA-H2','SI-GA-H3','SI-GA-H4','SI-GA-H5','SI-GA-H6','SI-GA-H7','SI-GA-H8','SI-GA-H9','SI-GA-H10','SI-GA-H11','SI-GA-H12'] - }, - - initComponent: function () { - Ext4.apply(this, { - title: null, - border: false, - defaults: { - border: false - }, - items: [{ - xtype: 'radiogroup', - name: 'importType', - columns: 1, - items: [{ - boxLabel: 'Novogene/Plate List', - inputValue: 'plateList', - name: 'importType', - checked: true - },{ - boxLabel: 'Other', - inputValue: 'other', - name: 'importType' - }], - listeners: { - scope: this, - afterrender: function(field) { - field.fireEvent('change', field, field.getValue()); - }, - change: function(field, val) { - val = val.importType; - var target = field.up('panel').down('#importArea'); - target.removeAll(); - if (val === 'other') { - target.add([{ - xtype: 'ldk-simplecombo', - itemId: 'instrument', - fieldLabel: 'Instrument/Core', - forceSelection: true, - editable: false, - labelWidth: 160, - storeValues: ['NextSeq (MPSSR)', 'MiSeq (ONPRC)', 'Basic List (MedGenome)', '10x Sample Sheet', 'Novogene', 'Novogene-New'] - },{ - xtype: 'ldk-simplecombo', - itemId: 'application', - fieldLabel: 'Application/Type', - forceSelection: true, - editable: true, - labelWidth: 160, - allowBlank: true, - storeValues: ['Whole Transcriptome RNA-Seq', 'TCR Enriched', '10x GEX', '10x VDJ'] - },{ - xtype: 'labkey-combo', - forceSelection: true, - multiSelect: true, - displayField: 'plateId', - valueField: 'plateId', - itemId: 'sourcePlates', - fieldLabel: 'Source Plate Id', - store: { - type: 'labkey-store', - schemaName: 'tcrdb', - sql: 'SELECT distinct plateId as plateId from tcrdb.cdnas c WHERE c.allReadsetsHaveData = false', - autoLoad: true - }, - labelWidth: 160 - },{ - xtype: 'textfield', - itemId: 'adapter', - fieldLabel: 'Adapter', - labelWidth: 160, - value: 'CTGTCTCTTATACACATCT' - }]); - } - else { - target.add({ - border: false, - defaults: { - border: false - }, - items: [{ - html: 'Add an ordered list of plates, using tab-delimited columns. The first column(s) are plate ID and library type (GEX, VDJ, CITE, or HTO). These can either be one column (i.e. G234-1, C234-1, H234-1, or T234-1), or as two columns (234-1 GEX or 234-1 HTO). An optional next column is the lane assignment (i.e. Novaseq1, HiSeq1, HiSeq2). Finally, an optional final column can be used to provide the alias for this pool. This is mostly used for CITE-Seq/HTOs, where multiple libraries are pre-pooled. Note, a wildcard can be used to specify all plates beginning with that prefix. See these examples:
' + - '
' +
-                                                '234-2\tGEX
' +
-                                                '234-2\tVDJ
' +
-                                                'G233-2
' +
-                                                'T235-2
' +
-                                                '234-2\tVDJ\tNovaSeq1
' +
-                                                'G233-2\tNovaSeq1
' +
-                                                '235-2\tHTO\tHiSeq1\tBNB-HTO-1
' +
-                                                'H235-2\tHiSeq1\tBNB-HTO-1
' +
-                                                '235-2\tHTO\tHiSeq2\tBNB-HTO-1
' +
-                                                'H235-2\tHiSeq1\tBNB-HTO-1
' +
-                                                'C235-2\tHiSeq1\tBNB-HTO-1
' +
-                                                'C235-*\tHiSeq2\tBNB-HTO-2' +
-                                            '
', - border: false - },{ - xtype: 'ldk-simplecombo', - itemId: 'instrument', - value: 'Novogene-New', - fieldLabel: 'Format', - forceSelection: true, - editable: true, - allowBlank: true, - storeValues: ['Novogene', 'Novogene-New'] - },{ - xtype: 'textarea', - itemId: 'plateList', - fieldLabel: 'Plate List', - labelAlign: 'top', - width: 270, - height: 200, - enableKeyEvents: true, - listeners: { - specialkey: function (field, e) { - if (e.getKey() === e.TAB) { - field.setValue(field.getValue() + '\t'); - e.preventDefault(); - } - } - }, - },{ - xtype: 'ldk-numberfield', - itemId: 'defaultVolume', - fieldLabel: 'Default Volume (uL)', - value: 10 - }], - buttonAlign: 'left', - buttons: [{ - text: 'Add', - scope: this, - handler: function (btn) { - var text = btn.up('panel').down('#plateList').getValue(); - if (!text) { - Ext4.Msg.alert('Error', 'Must enter a list of plates'); - return; - } - - text = LDK.Utils.CSVToArray(Ext4.String.trim(text), '\t'); - Ext4.Array.forEach(text, function(r, idx){ - var val = r[0]; - if (val.startsWith('G')){ - val = val.substr(1); - val = val.replace('_', '-'); - r[0] = 'GEX'; - r.unshift(val); - - } - else if (val.startsWith('T')){ - val = val.substr(1); - val = val.replace('_', '-'); - r[0] = 'VDJ'; - r.unshift(val); - } - else if (val.startsWith('H')){ - val = val.substr(1); - val = val.replace('_', '-'); - r[0] = 'HTO'; - r.unshift(val); - } - else if (val.startsWith('C')){ - val = val.substr(1); - val = val.replace('_', '-'); - r[0] = 'CITE'; - r.unshift(val); - } - }, this); - - var hadError = false; - var wildcards = {}; - Ext4.Array.forEach(text, function(r){ - if (r.length < 2){ - hadError = true; - } - - //ensure all rows are of length 4 - if (r.length !== 4) { - for (i=0;i<(4-r.length);i++) { - r.push(''); - } - } - - Ext4.Array.forEach(r, function(val, idx){ - r[idx] = Ext4.String.trim(val); - }, this); - - if (r[0].match('\\*$')) { - var m = r[0].match('\\*$'); - var val = r[0].substr(0, m.index); - wildcards[val] = r; - } - }, this); - - if (hadError) { - Ext4.Msg.alert('Error', 'All rows must have at least 2 values'); - return; - } - - if (!Ext4.Object.isEmpty(wildcards)) { - LABKEY.Query.selectRows({ - method: 'POST', - containerPath: Laboratory.Utils.getQueryContainerPath(), - schemaName: 'tcrdb', - queryName: 'cdnas', - columns: 'rowid,plateId,hashingReadsetId,citeseqReadsetId', - filterArray: [LABKEY.Filter.create('plateId', Ext4.Object.getKeys(wildcards).join(';'), LABKEY.Filter.Types.CONTAINS_ONE_OF)], - scope: this, - failure: LDK.Utils.getErrorCallback(), - success: function (results) { - if (results.rows.length) { - var prefixToPlate = {}; - Ext4.Array.forEach(results.rows, function (row) { - Ext4.Array.forEach(Ext4.Object.getKeys(wildcards), function (prefix) { - if (row.plateId && row.plateId.includes(prefix)) { - prefix = prefix + '*'; - prefixToPlate[prefix] = prefixToPlate[prefix] || {}; - prefixToPlate[prefix][row.plateId] = prefixToPlate[prefix][row.plateId] || {} - if (row.hashingReadsetId) { - prefixToPlate[prefix][row.plateId].HTO = true; - } - - if (row.citeseqReadsetId) { - prefixToPlate[prefix][row.plateId].CITE = true; - } - } - }, this); - }, this); - - var updatedText = []; - var prefixes = Ext4.Object.getKeys(prefixToPlate); - Ext4.Array.forEach(text, function (r, idx) { - var plateId = r[0]; - if (prefixes.indexOf(plateId) === -1) { - updatedText.push(r); - } - else { - Ext4.Array.forEach(Ext4.Object.getKeys(prefixToPlate[plateId]), function(newPlateId){ - if (Ext4.Object.getKeys(prefixToPlate[plateId][newPlateId]).indexOf(r[1]) > -1) { - var r2 = [].concat(r); - r2[0] = newPlateId; - updatedText.push(r2); - } - }, this); - } - }, this); - - text = updatedText; - } - - this.onSubmit(btn, text); - } - }); - } else { - this.onSubmit(btn, text); - } - } - }] - }); - } - } - } - }, { - bodyStyle: 'padding: 5px;', - itemId: 'importArea', - border: false, - defaults: { - border: false - } - },{ - xtype: 'checkbox', - boxLabel: 'Allow Duplicate Barcodes', - checked: false, - itemId: 'allowDuplicates' - },{ - xtype: 'checkbox', - boxLabel: 'Use Simple Sample Names', - checked: true, - itemId: 'simpleSampleNames' - },{ - xtype: 'checkbox', - boxLabel: 'Include Blanks', - checked: true, - itemId: 'includeBlanks' - },{ - xtype: 'checkbox', - boxLabel: 'Include Libraries With Data', - checked: false, - itemId: 'includeWithData', - listeners: { - change: function (field, val) { - var target = field.up('tcrdb-libraryexportpanel').down('#sourcePlates'); - if (target) { - var sql = 'SELECT distinct plateId as plateId from tcrdb.cdnas ' + (val ? '' : 'c WHERE c.allReadsetsHaveData = false'); - target.store.sql = sql; - target.store.removeAll(); - target.store.load(function () { - if (target.getPicker()) { - target.getPicker().refresh(); - } - }, this); - } - } - } - },{ - xtype: 'textarea', - itemId: 'outputArea', - fieldLabel: 'Output', - labelAlign: 'top', - width: 1000, - height: 400 - }], - buttonAlign: 'left', - buttons: [{ - text: 'Submit', - scope: this, - handler: function(btn){ - this.onSubmit(btn); - } - }, { - text: 'Download Data', - itemId: 'downloadData', - disabled: true, - handler: function (btn) { - var instrument = btn.up('tcrdb-libraryexportpanel').down('#instrument').getValue(); - var plateId = btn.up('tcrdb-libraryexportpanel').down('#sourcePlates').getValue(); - var delim = 'TAB'; - var extension = 'txt'; - var split = '\t'; - if (instrument !== 'NextSeq (MPSSR)') { - delim = 'COMMA'; - extension = 'csv'; - split = ','; - } - - var val = btn.up('tcrdb-libraryexportpanel').down('#outputArea').getValue(); - var rows = LDK.Utils.CSVToArray(Ext4.String.trim(val), split); - - LABKEY.Utils.convertToTable({ - fileName: plateId + '.' + extension, - rows: rows, - delim: delim - }); - } - },{ - text: 'Assign Readsets To Batch', - itemId: 'readsetBatch', - disabled: true, - handler: function (btn) { - var panel = btn.up('tcrdb-libraryexportpanel'); - var readsetIds = btn.readsetIds; - if (!readsetIds) { - Ext4.Msg.alert('Error', 'No Readset IDs Found'); - return; - } - - Ext4.Msg.wait('Loading...'); - LABKEY.Query.selectRows({ - containerPath: Laboratory.Utils.getQueryContainerPath(), - schemaName: 'sequenceanalysis', - queryName: 'sequence_readsets', - columns: 'rowid,container', - filterArray: [LABKEY.Filter.create('rowid', readsetIds.join(';'), LABKEY.Filter.Types.IN)], - scope: this, - failure: LDK.Utils.getErrorCallback(), - success: function (results) { - Ext4.Msg.hide(); - - if (!results || !results.rows || !results.rows.length) { - Ext4.Msg.hide(); - Ext4.Msg.alert('Error', 'Readsets not found: ' + readsetIds.join(';')); - return; - } - - var readsetRows = results.rows; - - Ext4.create('Ext.window.Window', { - title: 'Assign Readsets To Batch', - width: 800, - bodyStyle: 'padding: 10px;', - readsetIds: readsetIds, - items: [{ - html: 'The following readsets will be assigned to an instrument run/batch: ' + readsetIds.join(', '), - style: 'padding-bottom: 10px;', - border: false - }, { - xtype: 'textfield', - fieldLabel: 'Run/Batch Name', - labelWidth: 150, - itemId: 'batchName' - }, { - xtype: 'ldk-integerfield', - fieldLabel: 'Target Workbook', - labelWidth: 150, - itemId: 'targetWorkbook' - }], - buttons: [{ - text: 'Submit', - scope: this, - handler: function (btn) { - var win = btn.up('window'); - var batchId = win.down('#batchName').getValue(); - if (!batchId) { - Ext4.Msg.alert('Error', 'Must enter a batch name'); - return; - } - - var workbook = win.down('#targetWorkbook').getValue(); - if (workbook) { - Ext4.Msg.wait('Loading...'); - LABKEY.Query.selectRows({ - containerPath: Laboratory.Utils.getQueryContainerPath(), - schemaName: 'core', - queryName: 'workbooks', - columns: 'EntityId', - filterArray: [LABKEY.Filter.create('workbookId/workbookId', workbook, LABKEY.Filter.Types.EQUAL)], - scope: this, - failure: LDK.Utils.getErrorCallback(), - success: function (results) { - if (!results || !results.rows || !results.rows.length) { - Ext4.Msg.hide(); - Ext4.Msg.alert('Error', 'Workbook not found: ' + workbook); - return; - } - - LDK.Assert.assertEquality('Expected single workbook to be returned', results.rows.length, 1); - - win.close(); - panel.createInstrumentRun(readsetRows, batchId, results.rows[0].EntityId); - } - }); - } - else { - panel.createInstrumentRun(readsetRows, batchId); - } - } - }, { - text: 'Cancel', - handler: function (btn) { - btn.up('window').close(); - } - }] - }).show(); - } - }); - } - }] - }); - - this.callParent(arguments); - - Ext4.Msg.wait('Loading...'); - LABKEY.Query.selectRows({ - schemaName: 'sequenceanalysis', - queryName: 'barcodes', - sort: 'group_name,tag_name', - scope: this, - failure: LDK.Utils.getErrorCallback(), - success: function(results){ - this.barcodeMap = {}; - - Ext4.Array.forEach(results.rows, function(r){ - this.barcodeMap[r.group_name] = this.barcodeMap[r.group_name] || {}; - this.barcodeMap[r.group_name][r.tag_name] = r.sequence; - }, this); - - Ext4.Msg.hide(); - } - }); - }, - - createInstrumentRun: function (readsetRows, batchId, containerId) { - containerId = containerId || Laboratory.Utils.getQueryContainerPath(); - LABKEY.Query.insertRows({ - containerPath: containerId, - schemaName: 'sequenceanalysis', - queryName: 'instrument_runs', - scope: this, - rows: [{ - name: batchId - }], - failure: LDK.Utils.getErrorCallback(), - success: function (results) { - var runId = results.rows[0].rowId; - LDK.Assert.assertNotEmpty('Error creating instrument run', runId); - - Ext4.Array.forEach(readsetRows, function (rs) { - rs.instrument_run_id = runId - }, this); - - LABKEY.Query.updateRows({ - containerPath: Laboratory.Utils.getQueryContainerPath(), - schemaName: 'sequenceanalysis', - queryName: 'sequence_readsets', - rows: readsetRows, - failure: LDK.Utils.getErrorCallback(), - success: function (results) { - Ext4.Msg.hide(); - Ext4.Msg.alert('Success', 'Readsets updated'); - } - }); - } - }); - }, - - onSubmit: function(btn, expectedPairs){ - var plateIds = []; - - if (expectedPairs) { - var hadError = false; - Ext4.Array.forEach(expectedPairs, function(p){ - plateIds.push(Ext4.String.trim(p[0])); - }, this); - } - else { - plateIds = btn.up('tcrdb-libraryexportpanel').down('#sourcePlates').getValue(); - } - - if (!plateIds || !plateIds.length){ - Ext4.Msg.alert('Error', 'Must provide the plate Id(s)'); - return; - } - - plateIds = Ext4.unique(plateIds); - - var instrument = btn.up('tcrdb-libraryexportpanel').down('#instrument').getValue(); - var application = btn.up('tcrdb-libraryexportpanel').down('#application') ? btn.up('tcrdb-libraryexportpanel').down('#application').getValue() : null; - var defaultVolume = btn.up('tcrdb-libraryexportpanel').down('#defaultVolume') ? btn.up('tcrdb-libraryexportpanel').down('#defaultVolume').getValue() : ''; - var adapter = btn.up('tcrdb-libraryexportpanel').down('#adapter') ? btn.up('tcrdb-libraryexportpanel').down('#adapter').getValue() : null; - var includeWithData = btn.up('tcrdb-libraryexportpanel').down('#includeWithData').getValue(); - var allowDuplicates = btn.up('tcrdb-libraryexportpanel').down('#allowDuplicates').getValue(); - var simpleSampleNames = btn.up('tcrdb-libraryexportpanel').down('#simpleSampleNames').getValue(); - var includeBlanks = btn.up('tcrdb-libraryexportpanel').down('#includeBlanks').getValue(); - var doReverseComplement = btn.up('tcrdb-libraryexportpanel').doReverseComplement; - - var isMatchingApplication = function(application, libraryType, readsetApplication, rowLevelApplication){ - if (!application && !rowLevelApplication){ - return true; - } - - if (application === 'Whole Transcriptome RNA-Seq'){ - return readsetApplication === 'RNA-seq' || readsetApplication === 'RNA-seq, Single Cell'; - } - else if (application === 'TCR Enriched'){ - return readsetApplication === 'RNA-seq + Enrichment'; - } - else if (readsetApplication === 'RNA-seq, Single Cell'){ - application = rowLevelApplication || application; - return (libraryType.match(/^10x [35]\' GEX/) && application === '10x GEX') || (libraryType.match(/^10x 5' VDJ/) && application === '10x VDJ'); - } - else if (readsetApplication === 'Cell Hashing'){ - application = rowLevelApplication || application; - return (application === '10x HTO'); - } - else if (readsetApplication === 'CITE-Seq'){ - application = rowLevelApplication || application; - return (application === '10x CITE-Seq'); - } - }; - - var getSampleName = function(simpleSampleNames, readsetId, readsetName, suffix){ - return (simpleSampleNames ? 's_' + readsetId : readsetId + '_' + readsetName) + (suffix ? '_' + suffix : ''); - }; - - Ext4.Msg.wait('Loading cDNA data'); - LABKEY.Query.selectRows({ - method: 'POST', - containerPath: Laboratory.Utils.getQueryContainerPath(), - schemaName: 'tcrdb', - queryName: 'cdnas', - sort: 'plateId,well/addressByColumn', - columns: 'rowid,plateid' + - ',readsetId,readsetId/name,readsetId/application,readsetId/librarytype,readsetId/barcode5,readsetId/barcode5/sequence,readsetId/barcode3,readsetId/barcode3/sequence,readsetId/totalFiles,readsetId/concentration' + - ',enrichedReadsetId,enrichedReadsetId/name,enrichedReadsetId/application,enrichedReadsetId/librarytype,enrichedReadsetId/barcode5,enrichedReadsetId/barcode5/sequence,enrichedReadsetId/barcode3,enrichedReadsetId/barcode3/sequence,enrichedReadsetId/totalFiles,enrichedReadsetId/concentration' + - ',hashingReadsetId,hashingReadsetId/name,hashingReadsetId/application,hashingReadsetId/librarytype,hashingReadsetId/barcode5,hashingReadsetId/barcode5/sequence,hashingReadsetId/barcode3,hashingReadsetId/barcode3/sequence,hashingReadsetId/totalFiles,hashingReadsetId/concentration' + - ',citeseqReadsetId,citeseqReadsetId/name,citeseqReadsetId/application,citeseqReadsetId/librarytype,citeseqReadsetId/barcode5,citeseqReadsetId/barcode5/sequence,citeseqReadsetId/barcode3,citeseqReadsetId/barcode3/sequence,citeseqReadsetId/totalFiles,citeseqReadsetId/concentration', - scope: this, - filterArray: [LABKEY.Filter.create('plateId', plateIds.join(';'), LABKEY.Filter.Types.IN)], - failure: LDK.Utils.getErrorCallback(), - success: function (results) { - Ext4.Msg.hide(); - - if (!results || !results.rows || !results.rows.length) { - Ext4.Msg.alert('Error', 'No libraries found for the selected plates'); - return; - } - - var sortedRows = results.rows; - if (expectedPairs) { - sortedRows = []; - var missingRows = []; - Ext4.Array.forEach(expectedPairs, function(p){ - var found = false; - Ext4.Array.forEach(results.rows, function(row){ - if (row.plateId === p[0]) { - if (p[1] === 'GEX') { - if (includeWithData || row['readsetId/totalFiles'] === 0) { - if (row['readsetId'] && row['readsetId/librarytype'] && row['readsetId/librarytype'].match('GEX')) { - sortedRows.push(Ext4.apply({targetApplication: '10x GEX', laneAssignment: (p.length > 2 ? p[2] : null), plateAlias: (p.length > 3 ? p[3] : null)}, row)); - found = true; - return false; - } - } - } - else if (p[1] === 'HTO') { - if (includeWithData || row['hashingReadsetId/totalFiles'] === 0) { - if (row['hashingReadsetId'] && row['hashingReadsetId/application'] && row['hashingReadsetId/application'].match('Cell Hashing')) { - sortedRows.push(Ext4.apply({targetApplication: '10x HTO', laneAssignment: (p.length > 2 ? p[2] : null), plateAlias: (p.length > 3 ? p[3] : null)}, row)); - found = true; - return false; - } - } - } - else if (p[1] === 'CITE') { - if (includeWithData || row['citeseqReadsetId/totalFiles'] === 0) { - if (row['citeseqReadsetId'] && row['citeseqReadsetId/application'] && row['citeseqReadsetId/application'].match('CITE-Seq')) { - sortedRows.push(Ext4.apply({targetApplication: '10x CITE-Seq', laneAssignment: (p.length > 2 ? p[2] : null), plateAlias: (p.length > 3 ? p[3] : null)}, row)); - found = true; - return false; - } - } - } - else if (p[1] === 'VDJ') { - if (includeWithData || row['enrichedReadsetId/totalFiles'] === 0) { - if (row['enrichedReadsetId'] && row['enrichedReadsetId/librarytype'].match('VDJ')) { - sortedRows.push(Ext4.apply({targetApplication: '10x VDJ', laneAssignment: (p.length > 2 ? p[2] : null), plateAlias: (p.length > 3 ? p[3] : null)}, row)); - found = true; - return false; - } - } - } - } - }, this); - - if (!found) { - missingRows.push(p[0] + '/' + p[1]); - } - }, this); - - if (missingRows.length){ - Ext4.Msg.alert('Error', 'The following plates were not found:
' + missingRows.join('
')); - return; - } - } - - var barcodes = 'Illumina'; - var readsetIds = {}; - var barcodeCombosUsed = []; - if (instrument === 'NextSeq (MPSSR)' || instrument === 'Basic List (MedGenome)') { - var rc5 = (instrument === 'NextSeq (MPSSR)'); - var rc3 = (instrument === 'NextSeq (MPSSR)'); - - var rows = [['Name', 'Adapter', 'I7_Index_ID', 'I7_Seq', 'I5_Index_ID', 'I5_Seq'].join('\t')]; - Ext4.Array.forEach(sortedRows, function (r) { - //only include readsets without existing data - var processSample = function(rows, r, fieldName) { - if (!readsetIds[r[fieldName]] && r[fieldName] && (includeWithData || r[fieldName + '/totalFiles'] === 0) && isMatchingApplication(application, r[fieldName + '/librarytype'], r[fieldName + '/application'], r.targetApplication)) { - //allow for cell hashing / shared readsets - readsetIds[r[fieldName]] = true; - - //reverse complement both barcodes: - var barcode5 = rc5 ? doReverseComplement(r[fieldName + '/barcode5/sequence']) : r[fieldName + '/barcode5/sequence']; - var barcode3 = rc3 ? doReverseComplement(r[fieldName + '/barcode3/sequence']) : r[fieldName + '/barcode3/sequence']; - barcodeCombosUsed.push(r[fieldName + '/barcode5'] + '/' + r[fieldName + '/barcode3']); - rows.push([getSampleName(simpleSampleNames, r[fieldName], r[fieldName + '/name']), adapter, r[fieldName + '/barcode5'], barcode5, r[fieldName + '/barcode3'], barcode3].join('\t')); - } - }; - - processSample(rows, r, 'readsetId'); - processSample(rows, r, 'enrichedReadsetId'); - processSample(rows, r, 'hashingReadsetId'); - processSample(rows, r, 'citeseqReadsetId'); - }, this); - - //add missing barcodes: - if (includeBlanks) { - var blankIdx = 0; - Ext4.Array.forEach(TCRdb.panel.LibraryExportPanel.BARCODES5, function (barcode5) { - Ext4.Array.forEach(TCRdb.panel.LibraryExportPanel.BARCODES3, function (barcode3) { - var combo = barcode5 + '/' + barcode3; - if (barcodeCombosUsed.indexOf(combo) === -1) { - blankIdx++; - var barcode5Seq = rc5 ? doReverseComplement(this.barcodeMap[barcodes][barcode5]) : this.barcodeMap[barcodes][barcode5]; - var barcode3Seq = rc3 ? doReverseComplement(this.barcodeMap[barcodes][barcode3]) : this.barcodeMap[barcodes][barcode3]; - - var name = simpleSampleNames ? 's_Blank' + blankIdx : plateIds.join(';').replace(/\//g, '-') + '_Blank' + blankIdx; - rows.push([name, adapter, barcode5, barcode5Seq, barcode3, barcode3Seq].join('\t')); - } - }, this); - }, this); - } - } - else if (instrument === 'MiSeq (ONPRC)') { - var rows = []; - rows.push('[Header]'); - rows.push('IEMFileVersion,4'); - rows.push('Investigator Name,Bimber'); - rows.push('Experiment Name,' + plateIds.join(';')); - rows.push('Date,11/16/2017'); - rows.push('Workflow,GenerateFASTQ'); - rows.push('Application,FASTQ Only'); - rows.push('Assay,Nextera XT'); - rows.push('Description,'); - rows.push('Chemistry,Amplicon'); - rows.push(''); - rows.push('[Reads]'); - rows.push('251'); - rows.push('251'); - rows.push(''); - rows.push('[Settings]'); - rows.push('ReverseComplement,0'); - rows.push('Adapter,' + adapter); - rows.push(''); - rows.push('[Data]'); - rows.push('Sample_ID,Sample_Name,Sample_Plate,Sample_Well,I7_Index_ID,index,I5_Index_ID,index2,Sample_Project,Description'); - - Ext4.Array.forEach(sortedRows, function (r) { - //only include readsets without existing data - if (!readsetIds[r.readsetId] && r.readsetId && (includeWithData || r['readsetId/totalFiles'] === 0) && isMatchingApplication(application, r['readsetId/librarytype'], r['readsetId/application'], r.targetApplication)) { - //allow for cell hashing / shared readsets - readsetIds[r.readsetId] = true; - - //reverse complement both barcodes: - var barcode5 = doReverseComplement(r['readsetId/barcode5/sequence']); - var barcode3 = r['readsetId/barcode3/sequence']; - var cleanedName = r.readsetId + '_' + r['readsetId/name'].replace(/ /g, '_'); - cleanedName = cleanedName.replace(/\//g, '-'); - - barcodeCombosUsed.push(r['readsetId/barcode5'] + '/' + r['readsetId/barcode3']); - rows.push([r.readsetId, cleanedName, '', '', r['readsetId/barcode5'], barcode5, r['readsetId/barcode3'], barcode3].join(',')); - } - - if (!readsetIds[r.enrichedReadsetId] && r.enrichedReadsetId && (includeWithData || r['enrichedReadsetId/totalFiles'] == 0) && isMatchingApplication(application, r['enrichedReadsetId/librarytype'], r['enrichedReadsetId/application'], r.targetApplication)) { - //allow for cell hashing / shared readsets - readsetIds[r.enrichedReadsetId] = true; - - var barcode5 = doReverseComplement(r['enrichedReadsetId/barcode5/sequence']); - var barcode3 = r['enrichedReadsetId/barcode3/sequence']; - var cleanedName = r.enrichedReadsetId + '_' + r['enrichedReadsetId/name'].replace(/ /g, '_'); - cleanedName = cleanedName.replace(/\//g, '-'); - - barcodeCombosUsed.push(r['enrichedReadsetId/barcode5'] + '/' + r['enrichedReadsetId/barcode3']); - rows.push([r.enrichedReadsetId, cleanedName, '', '', r['enrichedReadsetId/barcode5'], barcode5, r['enrichedReadsetId/barcode3'], barcode3].join(',')) - } - }, this); - - //add missing barcodes: - if (includeBlanks) { - var blankIdx = 0; - Ext4.Array.forEach(TCRdb.panel.LibraryExportPanel.BARCODES5, function (barcode5) { - Ext4.Array.forEach(TCRdb.panel.LibraryExportPanel.BARCODES3, function (barcode3) { - var combo = barcode5 + '/' + barcode3; - if (barcodeCombosUsed.indexOf(combo) === -1) { - blankIdx++; - var barcode5Seq = doReverseComplement(this.barcodeMap[barcodes][barcode5]); - var barcode3Seq = this.barcodeMap[barcodes][barcode3]; - rows.push([plateIds.join(';').replace(/\//g, '-') + '_Blank' + blankIdx, null, null, null, barcode5, barcode5Seq, barcode3, barcode3Seq].join(',')); - } - }, this); - }, this); - } - } - else if (instrument === '10x Sample Sheet' || instrument === 'Novogene' || instrument === 'Novogene-New') { - //we make the default assumption that we're using 10x primers, which are listed in the sample-sheet orientation - var doRC = false; - var rows = []; - var barcodes = '10x Chromium Single Cell v2'; - - if (instrument === '10x Sample Sheet') { - rows.push('Sample_ID,Sample_Name,index,Sample_Project'); - } - - //only include readsets without existing data - var processType = function(readsetIds, rows, r, fieldName, suffix, size, phiX, samplePrefix, comment, doRC) { - if (!readsetIds[r[fieldName]] && r[fieldName] && (includeWithData || r[fieldName + '/totalFiles'] === 0) && isMatchingApplication(application, r[fieldName + '/librarytype'], r[fieldName + '/application'], r.targetApplication)) { - //allow for shared readsets across cDNAs (hashing, etc.) - readsetIds[r[fieldName]] = true; - - var cleanedName = r[fieldName] + '_' + r[fieldName + '/name'].replace(/ /g, '_'); - cleanedName = cleanedName.replace(/\//g, '-'); - var sampleName = getSampleName(simpleSampleNames, r[fieldName], r[fieldName + '/name']) + (suffix && instrument.startsWith('Novogene') ? '' : '-' + suffix); - - var barcode5s = r[fieldName + '/barcode5/sequence'] ? r[fieldName + '/barcode5/sequence'].split(',') : []; - if (!barcode5s) { - LDK.Utils.logError('Sample missing barcode: ' + sampleName); - } - - barcodeCombosUsed.push([r[fieldName + '/barcode5'], '', r.laneAssignment || ''].join('/')); - - //The new format requires one/line - if (instrument === 'Novogene-New') { - if (doRC && barcode5s.length > 1) { - var msg = 'Did not expect Novogene-New, reverse complement and multiple barcodes'; - LDK.Utils.logError(msg); - Ext4.Msg.alert('Error', msg); - return; - } - barcode5s = [barcode5s.join(',')]; - } - - Ext4.Array.forEach(barcode5s, function (bc, idx) { - bc = doRC ? doReverseComplement(bc) : bc; - - var data = [sampleName, (instrument.startsWith('Novogene') ? '' : cleanedName), bc, '']; - if (instrument === 'Novogene') { - data = [sampleName]; - if (r.plateAlias) { - data.unshift(r.plateAlias); - } - else { - data.unshift(samplePrefix + r.plateId.replace(/-/g, '_')); - } - - data.push('Macaca mulatta'); - data.push(bc); - data.push(''); - data.push(r[fieldName + '/concentration'] || ''); - data.push(defaultVolume); - data.push(''); - data.push(size); - data.push(phiX); //PhiX - data.push(r.laneAssignment || ''); - data.push(comment || 'Please QC individually and pool in equal amounts per lane'); - } - else if (instrument === 'Novogene-New') { - data = ['Premade-10X transcriptome library']; - data.push(r.plateAlias ? r.plateAlias : samplePrefix + r.plateId.replace(/-/g, '_')); - data.push(sampleName); - data.push('Partial lane sequencing-With Demultiplexing'); //TODO: HiSeq? - data.push(bc); - data.push(''); //P5 - data.push(size); - data.push('Others'); //Library Status - data.push('ddH2O'); - data.push('Partial Lane sequencing-lib QC'); - data.push(200); //Total data - data.push('M raw reads'); - data.push(r[fieldName + '/concentration'] || ''); - data.push(defaultVolume); - data.push(comment || 'Please QC individually and pool in equal amounts per lane'); - - //data.push(phiX); //PhiX - data.push(r.laneAssignment || ''); - - } - rows.push(data.join(delim)); - }, this); - } - }; - - var delim = instrument.startsWith('Novogene') ? '\t' : ','; - Ext4.Array.forEach(sortedRows, function (r) { - processType(readsetIds, rows, r, 'readsetId', 'GEX', 500, 0.01, 'G', null, false); - processType(readsetIds, rows, r, 'enrichedReadsetId', 'TCR', 700, 0.01, 'T', null, false); - processType(readsetIds, rows, r, 'hashingReadsetId', 'HTO', 182, 0.05, 'H', 'Cell hashing, 190bp amplicon. Please QC individually and pool in equal amounts per lane', true); - processType(readsetIds, rows, r, 'citeseqReadsetId', 'CITE', 182, 0.05, 'C', 'CITE-Seq, 190bp amplicon. Please QC individually and pool in equal amounts per lane', false); - }, this); - - //add missing barcodes: - if (includeBlanks && !instrument.startsWith('Novogene')) { - var blankIdx = 0; - Ext4.Array.forEach(TCRdb.panel.LibraryExportPanel.TENX_BARCODES, function (barcode5) { - if (barcodeCombosUsed.indexOf(barcode5) === -1) { - blankIdx++; - var barcode5Seq = this.barcodeMap[barcodes][barcode5].split(','); - Ext4.Array.forEach(barcode5Seq, function (seq, idx) { - seq = doRC ? doReverseComplement(seq) : seq; - rows.push([barcode5 + '_' + (idx + 1), plateIds.join(';').replace(/\//g, '-') + '_Blank' + blankIdx, seq, ''].join(delim)); - }, this); - } - }, this); - } - } - - //check for unique barcodes - var sorted = barcodeCombosUsed.slice().sort(); - var duplicates = []; - for (var i = 0; i < sorted.length - 1; i++) { - if (sorted[i + 1] === sorted[i]) { - duplicates.push(sorted[i]); - } - } - - duplicates = Ext4.unique(duplicates); - if (!allowDuplicates && duplicates.length){ - Ext4.Msg.alert('Error', 'Duplicate barcodes: ' + duplicates.join(', ')); - btn.up('tcrdb-libraryexportpanel').down('#outputArea').setValue(null); - btn.up('tcrdb-libraryexportpanel').down('#downloadData').setDisabled(true); - } - else { - btn.up('tcrdb-libraryexportpanel').down('#outputArea').setValue(rows.join('\n')); - btn.up('tcrdb-libraryexportpanel').down('#downloadData').setDisabled(false); - - var rsBtn = btn.up('tcrdb-libraryexportpanel').down('#readsetBatch'); - rsBtn.readsetIds = Ext4.Object.getKeys(readsetIds); - rsBtn.setDisabled(false); - } - } - }); - }, - - doReverseComplement: function(seq){ - if (!seq){ - return seq; - } - var match={'a': 'T', 'A': 'T', 't': 'A', 'T': 'A', 'g': 'C', 'G': 'C', 'c': 'G', 'C': 'G'}; - var o = ''; - for (var i = seq.length - 1; i >= 0; i--) { - if (match[seq[i]] === undefined) break; - o += match[seq[i]]; - } - - return o; - } -}); diff --git a/tcrdb/resources/web/tcrdb/panel/PoolImportPanel.js b/tcrdb/resources/web/tcrdb/panel/PoolImportPanel.js deleted file mode 100644 index fc4a4dbf8..000000000 --- a/tcrdb/resources/web/tcrdb/panel/PoolImportPanel.js +++ /dev/null @@ -1,1017 +0,0 @@ -Ext4.define('TCRdb.panel.PoolImportPanel', { - extend: 'Ext.panel.Panel', - - COLUMNS: [{ - name: 'workbook', - labels: ['Experiment/Workbook', 'Expt', 'Expt #', 'Experiment', 'Exp#', 'Exp #', 'Workbook', 'Workbook #'], - allowRowSpan: true, - alwaysShow: true, - transform: 'expt', - allowBlank: false - },{ - name: 'plateId', - labels: ['Pool/Tube', 'Pool', 'Pool Num', 'Pool #', 'Tube #', 'Tube#'], - allowRowSpan: true, - alwaysShow: true, - transform: 'pool', - allowBlank: false - },{ - name: 'stimId', - labels: ['Stim Id'], - allowRowSpan: false, - allowBlank: true, - alwaysShow: true - },{ - name: 'animalId', - labels: ['Animal', 'Animal Id', 'SubjectId', 'Subject Id'], - allowRowSpan: true, - allowBlank: false, - transform: 'animal' - },{ - name: 'sampleDate', - labels: ['Sample Date', 'Date'], - alwaysShow: true, - allowRowSpan: true, - transform: 'sampleDate', - allowBlank: false - },{ - name: 'effector', - labels: ['Effectors', 'Effector'], - alwaysShow: true, - allowRowSpan: true, - transform: 'effector', - allowBlank: false - },{ - name: 'tissue', - labels: ['Tissue', 'Tissue Sample'], - alwaysShow: false, - allowRowSpan: true, - allowBlank: true - },{ - name: 'stim', - labels: ['Stim', 'Peptide Only Conditions'], - allowRowSpan: false, - allowBlank: false, - transform: 'stim' - },{ - name: 'stim_num', - labels: ['Stim #'], - allowRowSpan: false, - alwaysShow: true - },{ - name: 'population', - labels: ['Population', 'Target Population', 'Target Pop'], - allowRowSpan: true, - allowBlank: false, - transform: 'population' - },{ - name: 'tetramer', - labels: ['Tetramer'], - allowRowSpan: false, - allowBlank: true, - transform: 'tetramer' - },{ - name: 'sortId', - labels: ['Sort Id'], - allowRowSpan: false, - allowBlank: true, - alwaysShow: true - },{ - name: 'hto', - labels: ['HTO', 'HTO Oligo', 'HTO-Oligo', 'HTO barcode', 'Barcode'], - allowRowSpan: false, - transform: 'hto' - },{ - name: 'cells', - labels: ['Cells', 'Cell #', 'Sort', 'Sort Cell Count'], - allowRowSpan: false, - allowBlank: false, - transform: 'cells' - },{ - name: 'hto_library_index', - labels: ['HTO Library Index', 'HTO Index', 'MultiSeq Index', 'MultiSeq Library Index'], - allowRowSpan: true, - transform: 'htoIndex' - },{ - name: 'hto_library_conc', - labels: ['HTO Library Conc', 'HTO Library Conc (ng/uL)', 'HTO (qubit) ng/uL', 'HTO (quibit) ng/uL', 'MultiSeq Library Conc', 'MultiSeq Library (qubit) ng/uL', 'MultiSeq Library Conc (qubit) ng/uL'], - allowRowSpan: true - },{ - name: 'citeseqpanel', - labels: ['Cite-Seq Panel', 'Cite-Seq Panel Name', 'CiteSeq Panel'], - allowRowSpan: true - },{ - name: 'citeseq_library_index', - labels: ['Cite-Seq Library Index', 'Cite-Seq Index', 'CiteSeq Library Index', 'CiteSeq Index', 'Cite-Seq Library Index', 'Cite-Seq Index', 'CiteSeq Library (qubit) ng/uL'], - allowRowSpan: true, - transform: 'citeSeqTenXBarcode' - },{ - name: 'citeseq_library_conc', - labels: ['Cite-Seq Library Conc', 'Cite-Seq Library Conc (ng/uL)', 'Cite-Seq (qubit) ng/uL', 'Cite-Seq (quibit) ng/uL'], - allowRowSpan: true - },{ - name: 'gex_library_index', - labels: ['5\' GEX Library Index', '5\' GEX Index', 'GEX Index', 'GEX Library Index', '5-GEX Index', '5\'GEX Library Index'], - allowRowSpan: true, - transform: 'tenXBarcode' - },{ - name: 'gex_library_conc', - labels: ['5\' GEX Library Conc', 'GEX Library Conc', 'GEX Library Conc (ng/uL)', '5\' GEX Conc', 'GEX Conc', 'GEX Conc (ng/uL)', '5\' GEX (qubit) ng/uL', '5\' GEX Library (qubit) ng/uL'], - allowRowSpan: true - },{ - name: 'gex_library_fragment', - labels: ['5\' GEX Library Fragment Size', 'GEX Library Fragment Size', '5\' GEX Fragment Size', 'GEX Fragment Size', 'GEX Library Fragment Size (bp)'], - allowRowSpan: true - },{ - name: 'tcr_library_index', - labels: ['TCR Library Index', 'TCR Index', 'TCR Libray Index'], - allowRowSpan: true, - transform: 'tenXBarcode' - },{ - name: 'tcr_library_conc', - labels: ['TCR Library Conc', 'TCR Library Conc (ng/uL)', 'TCR (qubit) ng/uL', 'TCR library (qubit) ng/uL'], - allowRowSpan: true - },{ - name: 'tcr_library_fragment', - labels: ['TCR Library Fragment Size', 'TCR Library Fragment Size (bp)'], - allowRowSpan: true - }], - - IGNORED_COLUMNS: [], - - transforms: { - stim: function(val, panel) { - if (val && (val === '--' || val === '-')) { - val = 'NoStim'; - } - - return val; - }, - - animal: function(val, panel) { - if (val) { - val = val.replace(/ PBMC/, ''); - } - - return val; - }, - - htoIndex: function(val, panel) { - if (Ext4.isNumeric(val)) { - //indexes are named D7XX. accept rows named '1', '12', etc. - var type = panel.down('#hashingType').getValue(); - if (type === 'CD298') { - val = parseInt(val); - if (val < 100) { - val = val + 700; - } - return 'D' + val; - } - else if (type === 'MultiSeq') { - val = parseInt(val); - - return 'MultiSeq-Idx-RP' + val; - } - else { - LDK.Utils.logError('Unknown or missing hashingType: ' + type); - } - } - else if (val) { - var type = panel.down('#hashingType').getValue(); - if (type === 'MultiSeq') { - val = String(val); - if (val.match(/^MS-[0-9]+$/i)) { - val = val.replace(/^MS(-)*/ig, 'MultiSeq-Idx-RP'); - } - - val = val.replace(/^MS[- ]Idx/ig, 'MultiSeq-Idx'); - val = val.replace(/^MultiSeq[- ]Idx[- ]RP/ig, 'MultiSeq-Idx-RP'); - - return val; - } - } - - return val; - }, - - citeSeqTenXBarcode: function(val, panel){ - if (!val){ - return; - } - - var barcodeSeries = panel.down('#citeseqBarcodeSeries').getValue(); - val = val.toUpperCase(); - var re = new RegExp('^' + barcodeSeries + '-', 'i'); - if (!val.match(re)) { - if (val.length > 3) { - //errorMsgs.push('Every row must have name, application and proper barcodes'); - } - else { - val = barcodeSeries + '-' + val; - } - } - - return val; - }, - - tenXBarcode: function(val, panel){ - if (!val){ - return; - } - - var barcodeSeries = panel.down('#barcodeSeries').getValue(); - val = val.toUpperCase(); - var re = new RegExp('^' + barcodeSeries + '-', 'i'); - if (!val.match(re)) { - if (val.length > 3) { - //errorMsgs.push('Every row must have name, application and proper barcodes'); - } - else { - val = barcodeSeries + '-' + val; - } - } - - return val; - }, - - hto: function(val, panel){ - if (Ext4.isNumeric(val)){ - var type = panel.down('#hashingType').getValue(); - if (type === 'CD298') { - return 'HTO-' + val; - } - else if (type === 'MultiSeq') { - return 'MS-' + val; - } - } - else if (val) { - //Normalize hyphen use - val = String(val); - val = val.replace(/^MS(-)*/, 'MS-'); - val = val.replace(/^HTO(-)*/, 'HTO-'); - } - - return val; - }, - - expt: function(val, panel){ - return val || panel.EXPERIMENT; - }, - - cells: function(val, panel){ - return val ? Ext4.data.Types.INTEGER.convert(val) : val; - }, - - pool: function(val, panel, row){ - var workbook = row.workbook || panel.EXPERIMENT; - //Note: convert values like 2B -> 2 - if (val && !Ext4.isNumeric(val)) { - val = val.replace(/[^0-9]+/, ''); - } - if (workbook && Ext4.isNumeric(val) && workbook !== val){ - return workbook + '-' + val; - } - - return val; - }, - - tetramer: function(val, panel, row){ - if (val) { - if (['Tet+', 'Tetramer+', 'Tetramer'].indexOf(val) > -1) { - row.population = null; - } - - row.population = row.population || val; - } - - return val; - }, - - population: function(val, panel, row){ - if (val && ['Tet+', 'Tetramer+', 'Tetramer'].indexOf(val) > -1) { - val = row.tetramer; - } - - return val; - }, - - sampleDate: function(val, panel){ - return val || panel.SAMPLE_DATE; - }, - - effector: function(val, panel, row){ - if (val && val.endsWith('PBMC')) { - var tmp = val.replace(/( )+PBMC$/,''); - row.animalId = tmp; - val = 'PBMC'; - } - return val || panel.EFFECTOR; - } - }, - - COLUMN_MAP: null, - - initComponent: function () { - this.COLUMN_MAP = {}; - Ext4.Array.forEach(this.COLUMNS, function(col){ - this.COLUMN_MAP[col.name.toLowerCase()] = col; - Ext4.Array.forEach(col.labels, function(alias){ - this.COLUMN_MAP[alias.toLowerCase()] = col; - }, this); - }, this); - - Ext4.apply(this, { - title: null, - border: false, - defaults: { - border: false - }, - items: this.getPanelItems() - }); - - this.callParent(arguments); - }, - - getPanelItems: function(){ - return [{ - layout: { - type: 'hbox' - }, - items: [{ - xtype: 'ldk-integerfield', - style: 'margin-right: 5px;', - fieldLabel: 'Current Folder/Workbook', - labelWidth: 200, - minValue: 1, - value: LABKEY.Security.currentContainer.type === 'workbook' ? LABKEY.Security.currentContainer.name : null, - emptyText: LABKEY.Security.currentContainer.type === 'workbook' ? null : 'Showing All', - listeners: { - afterRender: function(field){ - new Ext4.util.KeyNav(field.getEl(), { - enter : function(e){ - var btn = field.up('panel').down('#goButton'); - btn.handler(btn); - }, - scope : this - }); - } - } - },{ - xtype: 'button', - itemId: 'goButton', - scope: this, - text: 'Go', - handler: function(btn){ - var wb = btn.up('panel').down('ldk-integerfield').getValue(); - if (!wb){ - wb = ''; - } - - var container = LABKEY.Security.currentContainer.type === 'workbook' ? LABKEY.Security.currentContainer.parentPath + '/' + wb : LABKEY.Security.currentContainer.path + '/' + wb; - window.location = LABKEY.ActionURL.buildURL('tcrdb', 'poolImport', container); - } - },{ - xtype: 'button', - scope: this, - hidden: !LABKEY.Security.currentUser.canInsert, - text: 'Create Workbook', - handler: function(btn){ - Ext4.create('Laboratory.window.WorkbookCreationWindow', { - abortIfContainerIsWorkbook: false, - canAddToExistingExperiment: false, - controller: 'tcrdb', - action: 'poolImport', - title: 'Create Workbook' - }).show(); - } - }] - }, { - style: 'padding-top: 10px;', - html: 'This page is designed to help import TCR/10x data, including pooled samples. Each sample tends to create many libraries with many indexes/barcodes to track. Use the fields below to download the excel template and paste data to import.

' - },{ - layout: 'hbox', - items: [{ - xtype: 'button', - text: 'Download Template', - border: true, - scope: this, - href: LABKEY.ActionURL.getContextPath() + '/tcrdb/exampleData/ImportTemplate.xlsx' - },{ - xtype: 'button', - text: 'Download Example Import', - border: true, - scope: this, - href: LABKEY.ActionURL.getContextPath() + '/tcrdb/exampleData/ImportExample.xlsx' - }] - }, { - xtype: 'ldk-linkbutton', - text: 'Manage Allowable Values for Stims', - linkCls: 'labkey-text-link', - href: LABKEY.ActionURL.buildURL('query', 'executeQuery', Laboratory.Utils.getQueryContainerPath(), {schemaName: 'tcrdb', 'query.queryName': 'peptides'}), - style: 'margin-top: 10px;' - },{ - xtype: 'textfield', - style: 'margin-top: 20px;', - fieldLabel: 'Expt Number', - itemId: 'exptNum', - value: LABKEY.Security.currentContainer.type === 'workbook' ? LABKEY.Security.currentContainer.name : null - },{ - xtype: 'datefield', - fieldLabel: 'Sample Date', - itemId: 'sampleDate' - },{ - xtype: 'textfield', - fieldLabel: 'Effectors', - itemId: 'effector', - value: 'PBMC' - },{ - xtype: 'checkbox', - fieldLabel: 'Require HTO', - itemId: 'requireHashTag', - checked: true - },{ - xtype: 'checkbox', - fieldLabel: 'Require GEX Library', - itemId: 'requireGEX', - checked: false - },{ - xtype: 'checkbox', - fieldLabel: 'Require TCR Library', - itemId: 'requireTCR', - checked: false - },{ - xtype: 'checkbox', - fieldLabel: 'Require HTO Library', - itemId: 'requireHTO', - checked: false - },{ - xtype: 'checkbox', - fieldLabel: 'Require Cite-Seq Library', - itemId: 'requireCITE', - checked: false - },{ - xtype: 'checkbox', - fieldLabel: 'Require Library Concentrations', - itemId: 'requireConc', - checked: false - }, { - xtype: 'checkbox', - fieldLabel: 'Skip Readsets', - itemId: 'skipReadsets', - checked: true, - listeners: { - scope: this, - change: function(field, val) { - field.up('panel').down('#requireGEX').setValue(!val); - field.up('panel').down('#requireTCR').setValue(!val); - field.up('panel').down('#requireHTO').setValue(!val); - field.up('panel').down('#requireCITE').setValue(!val); - } - } - },{ - xtype: 'ldk-simplecombo', - fieldLabel: '10x GEX/TCR Barcode Series', - itemId: 'barcodeSeries', - forceSelection: true, - storeValues: ['SI-GA'], - value: 'SI-GA' - },{ - xtype: 'ldk-simplecombo', - fieldLabel: '10x Cite-Seq Barcode Series', - itemId: 'citeseqBarcodeSeries', - forceSelection: true, - storeValues: ['SI-NA'], - value: 'SI-NA' - },{ - xtype: 'ldk-simplecombo', - fieldLabel: 'Hashing Type', - itemId: 'hashingType', - forceSelection: true, - storeValues: ['CD298', 'MultiSeq'], - value: 'MultiSeq' - },{ - xtype: 'textarea', - fieldLabel: 'Paste Data Below', - labelAlign: 'top', - itemId: 'data', - width: 1000, - height: 300 - },{ - xtype: 'button', - text: 'Preview', - border: true, - scope: this, - handler: this.onPreview - },{ - itemId: 'previewArea', - style: 'margin-top: 20px;margin-bottom: 10px;', - autoEl: 'table', - cls: 'stripe hover' - }]; - }, - - onPreview: function(btn) { - var text = this.down('#data').getValue(); - if (!text) { - Ext4.Msg.alert('Error', 'Must provide the table of data'); - return; - } - - this.EXPERIMENT = this.down('#exptNum').getValue(); - this.SAMPLE_DATE = this.down('#sampleDate').getValue(); - if (this.SAMPLE_DATE) { - this.SAMPLE_DATE = Ext4.Date.format(this.SAMPLE_DATE, 'Y-m-d'); - } - this.EFFECTOR = this.down('#effector').getValue(); - - //this is a special case. if the first character is Tab, this indicates a blank field. Add a placeholder so it's not trimmed: - if (text .startsWith("\t")) { - text = 'Column1' + text; - } - text = Ext4.String.trim(text); - - var rows = LDK.Utils.CSVToArray(text, '\t'); - var colArray = this.parseHeader(rows.shift()); - var parsedRows = this.parseRows(colArray, rows); - var stimRows = []; - Ext4.Array.forEach(parsedRows, function(r){ - LDK.Assert.assertNotEmpty('Expected non-null workbook', r.workbook); - stimRows.push({ - animalId: r.animalId, - date: r.sampleDate, - stim: r.stim, - treatment: r.treatment || 'None', - tissue: r.tissue, - objectId: r.objectId, - population: r.population, - workbook: r.workbook - }); - }, this); - - Ext4.Msg.wait('Looking for matching stims'); - LABKEY.Ajax.request({ - url: LABKEY.ActionURL.buildURL('tcrdb', 'getMatchingStims', Laboratory.Utils.getQueryContainerPath(), null), - timeout: 99999, - method: 'POST', - jsonData: { - stimRows: stimRows - }, - scope: this, - success: LABKEY.Utils.getCallbackWrapper(function(results){ - Ext4.Msg.hide(); - - Ext4.Array.forEach(parsedRows, function(r){ - if (results.stimMap[r.objectId]){ - r.stimId = results.stimMap[r.objectId]; - } - - if (results.sortMap[r.objectId]){ - r.sortId = results.sortMap[r.objectId]; - } - }, this); - - var groupedRows = this.groupForImport(colArray, parsedRows); - if (!groupedRows){ - console.log('No rows after grouping'); - return; - } - - this.renderPreview(colArray, parsedRows, groupedRows); - - if (results.recordErrors) { - Ext4.Array.forEach(results.recordErrors, function(e){ - console.error(e); - }, this); - } - }, this), - failure: LDK.Utils.getErrorCallback() - }); - }, - - parseHeader: function(headerRow){ - var colArray = []; - var colNames = {}; - Ext4.Array.forEach(headerRow, function(headerText, idx){ - //replace common terms: - if (headerText.match(/ng\/ul/i) || headerText.match(/qubit/i)) { - headerText = headerText.replace(/( )+(\()*ng\/ul(\))*/i, ''); - headerText = headerText.replace(/( )+(\()*qubit(\))*/i, ''); - headerText = Ext4.String.trim(headerText); - if (!headerText.match(/Conc/i)) { - headerText = headerText + ' Conc'; - } - } - headerText = headerText.replace(/CiteSeq/i, 'Cite-Seq'); - headerText = headerText.replace(/Cite Seq/i, 'Cite-Seq'); - headerText = headerText.replace(/^MS /i, 'MultiSeq '); - headerText = headerText.replace(/Multi Seq/i, 'MultiSeq'); - headerText = headerText.replace(/Multi-Seq/i, 'MultiSeq'); - headerText = headerText.replace(/Library Index/i, 'Index'); - headerText = headerText.replace(/ RP#/i, ''); - headerText = headerText.replace(/:( )+10X Plate N Set A/i, ''); - headerText = headerText.replace(/:( )+10X Plate T Kit A/i, ''); - - headerText = headerText.replace(/5'[- ]*GEX/i, 'GEX'); - headerText = headerText.replace(/5[- ]GEX/i, 'GEX'); - headerText = Ext4.String.trim(headerText); - - var colData = this.COLUMN_MAP[headerText.toLowerCase()]; - if (colData){ - colNames[colData.name] = idx; - } - }, this); - - Ext4.Array.forEach(this.COLUMNS, function(colData, idx){ - if (this.IGNORED_COLUMNS.indexOf(colData.name) > -1) { - return; - } - - if (colData.alwaysShow || colData.allowBlank === false || colNames[colData.name]){ - colData = Ext4.apply({}, colData); - colData.dataIdx = colNames[colData.name]; - - colArray.push(colData); - } - },this); - - return colArray; - }, - - parseRows: function(colArray, rows){ - var lastValueByCol = new Array(colArray.length); - var ret = []; - - var doSplitCellsByPool = false; - Ext4.Array.forEach(rows, function(row, rowIdx){ - var data = { - objectId: LABKEY.Utils.generateUUID() - }; - - Ext4.Array.forEach(colArray, function(col, colIdx){ - var cell = Ext4.isDefined(col.dataIdx) ? row[col.dataIdx] : ''; - if (cell){ - if (col.transform && this.transforms[col.transform]){ - cell = this.transforms[col.transform](cell, this, data); - } - - data[col.name] = cell; - lastValueByCol[colIdx] = cell; - } - else if (col.allowRowSpan && lastValueByCol[colIdx]){ - data[col.name] = lastValueByCol[colIdx]; - } - else { - //allow transform even if value is null - if (col.transform && this.transforms[col.transform]){ - cell = this.transforms[col.transform](cell, this, data); - } - - data[col.name] = cell; - - if (!cell && col.name === 'cells' && lastValueByCol[colIdx]) { - doSplitCellsByPool = true; - } - } - }, this); - - ret.push(data); - }, this); - - //split cells across rows - if (doSplitCellsByPool) { - var cellCountMap = {}; - Ext4.Array.forEach(ret, function(data) { - if (data.plateId) { - cellCountMap[data.plateId] = cellCountMap[data.plateId] || []; - cellCountMap[data.plateId].push(data.cells); - } - }, this); - - Ext4.Array.forEach(Ext4.Object.getKeys(cellCountMap), function(plateId) { - var arr = cellCountMap[plateId]; - var size = arr.length; - arr = Ext4.Array.remove(arr, null); - arr = Ext4.Array.remove(arr, ''); - if (arr.length === 1) { - cellCountMap[plateId] = arr[0] / size; - } - else { - delete cellCountMap[plateId]; - } - }, this); - - Ext4.Array.forEach(ret, function(data) { - if (data.plateId && cellCountMap[data.plateId]) { - data.cells = cellCountMap[data.plateId]; - } - }, this); - } - - return ret; - }, - - groupForImport: function(colArray, parsedRows){ - var ret = { - stimRows: [], - sortRows: [], - cDNARows: [], - readsetRows: [] - }; - - var errorsMsgs = []; - - //stims: - var stimMap = {}; - var stimIdxs = {}; - var stimIdx = 0; - Ext4.Array.forEach(parsedRows, function(row){ - var key = this.getStimKey(row); - if (!stimMap[key]){ - var guid = LABKEY.Utils.generateUUID(); - stimIdx++; - - stimMap[key] = guid; - stimIdxs[key] = stimIdx; - LDK.Assert.assertNotEmpty('Expected non-null workbook', row.workbook); - ret.stimRows.push({ - rowId: row.stimId || null, - animalId: row.animalId, - date: row.sampleDate, - stim: row.stim, - effector: row.effector, - tissue: row.tissue, - treatment: row.treatment || 'None', - objectId: guid, - workbook: row.workbook - }); - } - - row.stim_num = row.stim_num || stimIdxs[key]; - }, this); - - //sorts: - var sortMap = {}; - Ext4.Array.forEach(parsedRows, function(row){ - LDK.Assert.assertNotEmpty('Expected non-null workbook', row.workbook); - var stimGUID = stimMap[this.getStimKey(row)]; - var key = this.getSortKey(row); - if (!sortMap[key]){ - var guid = LABKEY.Utils.generateUUID(); - sortMap[key] = guid; - ret.sortRows.push({ - rowId: row.sortId || null, - stimGUID: stimGUID, - population: row.population, - replicate: row.replicate, - cells: row.cells, - well: row.well || 'Pool', - hto: row.hto, - buffer: row.buffer, - objectId: guid, - workbook: row.workbook - }); - } - }, this); - - //cDNA/readsets: group by pool - var poolMap = {}; - Ext4.Array.forEach(parsedRows, function(row) { - poolMap[row.plateId] = poolMap[row.plateId] || []; - poolMap[row.plateId].push(row); - }, this); - - Ext4.Object.each(poolMap, function(poolName, rowArr){ - var readsetGUIDs = {}; - - var requireHTO = this.down('#requireHTO').getValue(); - var hashingType = this.down('#hashingType').getValue(); - var libraryType = null; - if (hashingType === 'CD298'){ - libraryType = 'CD298 Hashing'; - } - else if (hashingType === 'MultiSeq'){ - libraryType = 'MultiSeq'; - } - - var rs = this.processReadsetForGroup(poolName, rowArr, ret.readsetRows, 'hto', 'HTO', 'Cell Hashing', libraryType); - if (Ext4.isString(rs)) { - readsetGUIDs.hashingReadsetGUID = rs; - } - else if (requireHTO){ - errorsMsgs.push('Missing HTO library'); - errorsMsgs = errorsMsgs.concat(rs); - return false; - } - - var requireCITE = this.down('#requireCITE').getValue(); - var rs = this.processReadsetForGroup(poolName, rowArr, ret.readsetRows, 'citeseq', 'CITE', 'CITE-Seq', null); - if (Ext4.isString(rs)) { - readsetGUIDs.citeseqReadsetGUID = rs; - } - else if (requireCITE){ - errorsMsgs.push('Missing CITE-Seq library'); - errorsMsgs = errorsMsgs.concat(rs); - return false; - } - - var requireGEX = this.down('#requireGEX').getValue(); - rs = this.processReadsetForGroup(poolName, rowArr, ret.readsetRows, 'gex', 'GEX', 'RNA-seq, Single Cell', '10x 5\' GEX'); - if (Ext4.isString(rs)) { - readsetGUIDs.readsetGUID = rs; - } - else if (requireGEX){ - errorsMsgs.push('Missing GEX library'); - errorsMsgs = errorsMsgs.concat(rs); - return false; - } - - var requireTCR = this.down('#requireTCR').getValue(); - rs = this.processReadsetForGroup(poolName, rowArr, ret.readsetRows, 'tcr', 'TCR', 'RNA-seq, Single Cell', '10x 5\' VDJ (Rhesus A/B/G)'); - if (Ext4.isString(rs)) { - readsetGUIDs.enrichedReadsetGUID = rs; - } - else if (requireTCR){ - errorsMsgs.push('Missing TCR library'); - errorsMsgs = errorsMsgs.concat(rs); - return false; - } - - Ext4.Array.forEach(rowArr, function(row) { - var sortKey = this.getSortKey(row); - - LDK.Assert.assertNotEmpty('Expected non-null workbook', row.workbook); - var cDNA = Ext4.apply({ - sortGUID: sortMap[sortKey], - chemistry: null, - plateId: row.plateId, - well: row.well || 'Pool', - citeseqpanel: row.citeseqpanel, - workbook: row.workbook - }, readsetGUIDs); - - ret.cDNARows.push(cDNA); - }, this); - }, this); - - if (errorsMsgs.length) { - errorsMsgs = Ext4.unique(errorsMsgs); - Ext4.Msg.alert('Error', errorsMsgs.join('
')); - return null; - } - - return ret; - }, - - processReadsetForGroup: function(poolName, rowArr, readsetRows, prefix, type, application, librarytype){ - var idxValues = this.getUniqueValues(rowArr, prefix + '_library_index'); - var conc = this.getUniqueValues(rowArr, prefix + '_library_conc'); - var fragment = this.getUniqueValues(rowArr, prefix + '_library_fragment'); - var workbook = this.getUniqueValues(rowArr, 'workbook'); - if (workbook.length > 1) { - return ['Error', 'Pool ' + poolName + ' uses more workbook ' + workbook.join(';')]; - } - workbook = workbook.length === 1 ? workbook[0] : null; - - var subjectid = this.getUniqueValues(rowArr, 'animalId'); - subjectid = subjectid.length === 1 ? subjectid[0] : null; - - var requireConc = this.down('#requireConc').getValue(); - - if (idxValues.length === 1){ - if (requireConc && !conc[0]) { - return ['Pool ' + poolName + ': did not provide concentration for library: ' + type]; - } - - var guid = LABKEY.Utils.generateUUID(); - LDK.Assert.assertNotEmpty('Expected non-null workbook', workbook); - readsetRows.push({ - name: poolName + '-' + type, - barcode5: idxValues[0], - concentration: conc[0], - fragmentSize: fragment[0], - platform: 'ILLUMINA', - application: application, - librarytype: librarytype, - subjectid: subjectid, - sampleType: 'mRNA', - objectId: guid, - workbook: workbook - }); - - return guid; - } - else if (idxValues.length > 1) { - return ['Error', 'Pool ' + poolName + ' uses more than one ' + type + ' index']; - } - else if (idxValues.length === 0) { - var required = this.down('#require' + type).getValue(); - if (required) { - return ['Error', 'No index found for pool: ' + poolName + ', for library type: ' + type]; - } - } - }, - - getUniqueValues: function(rowArr, colName){ - var ret = []; - Ext4.Array.forEach(rowArr, function(row){ - if (row[colName]) - ret.push(row[colName]); - }, this); - - return Ext4.unique(ret); - }, - - renderPreview: function(colArray, parsedRows, groupedRows){ - var previewArea = this.down('#previewArea'); - previewArea.removeAll(); - - var columns = [{title: 'Row #'}]; - var colIdxs = []; - Ext4.Array.forEach(colArray, function(col, idx){ - if (col){ - columns.push({title: col.labels[0], className: 'dt-center'}); - colIdxs.push(idx); - } - }, this); - - var data = []; - var missingValues = false; - var requireHTO = this.down('#requireHTO').getValue() || (this.down('#requireHashTag') && this.down('#requireHashTag').getValue()); - Ext4.Array.forEach(parsedRows, function(row, rowIdx){ - var toAdd = [rowIdx + 1]; - Ext4.Array.forEach(colIdxs, function(colIdx){ - var colDef = colArray[colIdx]; - var propName = colDef.name; - - var allowBlank = colDef.allowBlank; - if (requireHTO && colDef.name === 'hto') { - allowBlank = false; - } - - if (allowBlank === false && Ext4.isEmpty(row[propName])){ - missingValues = true; - toAdd.push('MISSING'); - } - else { - toAdd.push(row[propName] || 'ND'); - } - - }, this); - - data.push(toAdd); - }, this); - - var id = '#' + previewArea.getId(); - if ( jQuery.fn.dataTable.isDataTable(id) ) { - jQuery(id).DataTable().destroy(); - } - - jQuery(id).DataTable({ - data: data, - pageLength: 500, - dom: 'rt<"bottom"BS><"clear">', - buttons: missingValues ? [] : [{ - text: 'Submit', - action: this.onSubmit, - rowData: { - colArray: colArray, - parsedRows: parsedRows, - groupedRows: groupedRows, - panel: this - } - }], - columns: columns - }); - - previewArea.doLayout(); - - if (missingValues){ - Ext4.Msg.alert('Error', 'One or more rows is missing data. Any required cells without values are marked MISSING'); - } - }, - - onSubmit: function(e, dt, node, config){ - Ext4.Msg.wait('Saving...'); - LABKEY.Ajax.request({ - url: LABKEY.ActionURL.buildURL('tcrdb', 'importTenx', Laboratory.Utils.getQueryContainerPath()), - method: 'POST', - jsonData: config.rowData.groupedRows, - scope: this, - success: function(){ - Ext4.Msg.hide(); - Ext4.Msg.alert('Success', 'Data Saved', function(){ - window.location = LABKEY.ActionURL.buildURL('query', 'executeQuery.view', Laboratory.Utils.getQueryContainerPath(), {'query.queryName': 'cdnas', schemaName: 'tcrdb', 'query.sort': '-created'}) - }, this); - }, - failure: LDK.Utils.getErrorCallback() - }); - }, - - getStimKey: function(data){ - return [data.stimId, data.animalId, data.stim, data.treatment, data.tissue, (Ext4.isDate(data.sampleDate) ? Ext4.Date.format(data.sampleDate, 'Y-m-d') : data.sampleDate)].join('|'); - }, - - getSortKey: function(data){ - return [this.getStimKey(data), data.sortId, data.population, data.hto].join('|'); - } -}); \ No newline at end of file diff --git a/tcrdb/resources/web/tcrdb/panel/StimPanel.js b/tcrdb/resources/web/tcrdb/panel/StimPanel.js deleted file mode 100644 index 62e1f2af9..000000000 --- a/tcrdb/resources/web/tcrdb/panel/StimPanel.js +++ /dev/null @@ -1,1515 +0,0 @@ -Ext4.define('TCRdb.panel.StimPanel', { - extend: 'Ext.panel.Panel', - alias: 'widget.tcrdb-stimpanel', - - initComponent: function(){ - Ext4.apply(this, { - title: null, - border: false, - defaults: { - border: false - }, - items: [{ - layout: { - type: 'hbox' - }, - items: [{ - xtype: 'ldk-integerfield', - style: 'margin-right: 5px;', - fieldLabel: 'Current Folder/Workbook', - labelWidth: 200, - minValue: 1, - value: LABKEY.Security.currentContainer.type === 'workbook' ? LABKEY.Security.currentContainer.name : null, - emptyText: LABKEY.Security.currentContainer.type === 'workbook' ? null : 'Showing All', - listeners: { - afterRender: function(field){ - new Ext4.util.KeyNav(field.getEl(), { - enter : function(e){ - var btn = field.up('panel').down('#goButton'); - btn.handler(btn); - }, - scope : this - }); - } - } - },{ - xtype: 'button', - itemId: 'goButton', - scope: this, - text: 'Go', - handler: function(btn){ - var wb = btn.up('panel').down('ldk-integerfield').getValue(); - if (!wb){ - wb = ''; - } - - var container = LABKEY.Security.currentContainer.type === 'workbook' ? LABKEY.Security.currentContainer.parentPath + '/' + wb : LABKEY.Security.currentContainer.path + '/' + wb; - window.location = LABKEY.ActionURL.buildURL('tcrdb', 'stimDashboard', container); - } - },{ - xtype: 'button', - scope: this, - hidden: !LABKEY.Security.currentUser.canInsert, - text: 'Create Workbook', - handler: function(btn){ - Ext4.create('Laboratory.window.WorkbookCreationWindow', { - abortIfContainerIsWorkbook: false, - canAddToExistingExperiment: false, - controller: 'tcrdb', - action: 'stimDashboard', - title: 'Create Workbook' - }).show(); - } - }] - },{ - style: 'padding-top: 10px;', - html: 'This page is designed to help manage samples for the TCR sequencing project. Where possible we try to carry sample information from to step to step; however, each step often generates new info we need to track, and sometimes samples and plates generated at different times are combined for later steps. The basic steps are:

' - }] - }); - - this.callParent(arguments); - - Ext4.Msg.wait('Loading...'); - this.loadData(); - }, - - getFolderSummaryConfig: function(){ - - }, - - loadData: function(){ - var multi = new LABKEY.MultiRequest(); - multi.add(LABKEY.Query.selectRows, { - schemaName: 'laboratory', - queryName: 'well_layout', - columns: 'well_96,addressbycolumn_96', - filterArray: [LABKEY.Filter.create('plate', 1)], - sort: 'addressbycolumn_96', - scope: this, - failure: LDK.Utils.getErrorCallback(), - success: function(results){ - this.wellNames96 = []; - - Ext4.Array.forEach(results.rows, function(r){ - this.wellNames96.push(r.well_96); - }, this); - } - }); - - multi.add(LABKEY.Query.selectRows, { - schemaName: 'sequenceanalysis', - queryName: 'barcodes', - sort: 'group_name,tag_name', - scope: this, - failure: LDK.Utils.getErrorCallback(), - success: function(results){ - this.barcodeMap = {}; - - Ext4.Array.forEach(results.rows, function(r){ - this.barcodeMap[r.group_name] = this.barcodeMap[r.group_name] || {}; - this.barcodeMap[r.group_name][r.tag_name] = r.sequence; - }, this); - } - }); - - multi.add(LABKEY.Query.selectRows, { - schemaName: 'tcrdb', - queryName: 'stims', - columns: 'rowid,tubeNum,animalId,effector,effectors,date,stim,treatment,costim,background,activated,comment,numSorts,status', - scope: this, - failure: LDK.Utils.getErrorCallback(), - success: function(results){ - this.stimRows = results.rows; - this.stimStats = { - totalStims: 0, - lackingSort: 0, - hasStatus: 0 - }; - - Ext4.Array.forEach(results.rows, function(r){ - this.stimStats.totalStims++; - if (!r.numSorts && !r.status){ - this.stimStats.lackingSort++; - } - - if (r.status){ - this.stimStats.hasStatus++; - } - }, this); - } - }); - - multi.add(LABKEY.Query.selectRows, { - schemaName: 'tcrdb', - queryName: 'sorts', - columns: 'rowid,stimId,stimId/animalId,stimId/effector,stimId/date,stimId/treatment,population,replicate,cells,plateId,well,well/addressByColumn,numLibraries,maxCellsForPlate,container', - scope: this, - failure: LDK.Utils.getErrorCallback(), - success: function(results){ - this.sortRows = results.rows; - this.sortStats = { - totalSorts: 0, - totalPlates: [], - lackingLibraries: 0, - bulkLackingLibraries: 0, - totalPlatesLackingLibraries: [], - totalBulkPlatesLackingLibraries: [] - }; - - Ext4.Array.forEach(results.rows, function(r){ - this.sortStats.totalSorts++; - if (!r.numLibraries){ - this.sortStats.lackingLibraries++; - - if (r.cells > 1) { - this.sortStats.bulkLackingLibraries++; - } - - if (r.plateId){ - this.sortStats.totalPlatesLackingLibraries.push(r.plateId); - - if (r.maxCellsForPlate > 1){ - this.sortStats.totalBulkPlatesLackingLibraries.push(r.plateId); - } - } - } - - if (r.plateId){ - this.sortStats.totalPlates.push(r.plateId); - } - }, this); - - this.sortStats.totalPlates = Ext4.unique(this.sortStats.totalPlates); - this.sortStats.totalPlatesLackingLibraries = Ext4.unique(this.sortStats.totalPlatesLackingLibraries); - this.sortStats.totalBulkPlatesLackingLibraries = Ext4.unique(this.sortStats.totalBulkPlatesLackingLibraries); - } - }); - - multi.add(LABKEY.Query.selectRows, { - schemaName: 'tcrdb', - queryName: 'cdnas', - columns: 'rowid,sortId,cells,plateId,well,well/addressByColumn,readsetId,readsetId/totalFiles,enrichedReadsetId,enrichedReadsetId/totalFiles,sortId/stimId,sortId/stimId/animalId,sortId/stimId/effector,sortId/stimId/date,sortId/stimId/treatment,sortId/population,sortId/replicate,sortId/cells,sortId/plateId,sortId/sortId/well,sortId/well/addressByColumn,sortId/stimId/stim', - sort: 'plateId,well/addressByColumn', - scope: this, - failure: LDK.Utils.getErrorCallback(), - success: function(results){ - this.libraryRows = results.rows; - this.libraryStats = { - totalLibraries: 0, - totalPlates: [], - lackingAnyReadset: 0, - totalPlatesLackingAnyReadset: [], - withReadset: 0, - totalPlatesWithReadset: [], - withTCRReadset: 0, - totalPlatesWithTCRReadset: [], - lackingBarcodes: 0 - - }; - - Ext4.Array.forEach(results.rows, function(r){ - this.libraryStats.totalLibraries++; - if (r.readsetId){ - this.libraryStats.withReadset++; - if (r.plateId){ - this.libraryStats.totalPlatesWithReadset.push(r.plateId); - } - } - - if (r.enrichedReadsetId) { - this.libraryStats.withTCRReadset++; - if (r.plateId){ - this.libraryStats.totalPlatesWithTCRReadset.push(r.plateId); - } - } - - if (!r.enrichedReadsetId && !r.readsetId) { - this.libraryStats.lackingAnyReadset++; - if (r.plateId){ - this.libraryStats.totalPlatesLackingAnyReadset.push(r.plateId); - } - } - - if (r.plateId){ - this.libraryStats.totalPlates.push(r.plateId); - } - }, this); - - this.libraryStats.totalPlates = Ext4.unique(this.libraryStats.totalPlates); - this.libraryStats.totalPlatesWithReadset = Ext4.unique(this.libraryStats.totalPlatesWithReadset); - this.libraryStats.totalPlatesWithTCRReadset = Ext4.unique(this.libraryStats.totalPlatesWithTCRReadset); - this.libraryStats.totalPlatesLackingAnyReadset = Ext4.unique(this.libraryStats.totalPlatesLackingAnyReadset); - } - }); - - multi.add(LABKEY.Query.selectRows, { - schemaName: 'sequenceanalysis', - queryName: 'sequence_readsets', - columns: 'rowid,name,application,totalFiles', - scope: this, - failure: LDK.Utils.getErrorCallback(), - success: function(results){ - this.readsetRows = results.rows; - this.readsetStats = { - totalReadsets: 0, - lackingData: 0, - dataImported: 0 - }; - - Ext4.Array.forEach(results.rows, function(r){ - this.readsetStats.totalReadsets++; - if (!r.totalFiles){ - this.readsetStats.lackingData++; - } - else { - this.readsetStats.dataImported++; - } - }, this); - } - }); - - multi.send(this.onDataLoad, this); - }, - - onDataLoad: function(){ - this.add(this.getItemConfig()); - - Ext4.Msg.hide(); - }, - - getItemConfig: function(){ - return { - defaults: { - border: true, - style: 'padding-bottom: 10px;', - bodyStyle: 'padding: 5px;' - }, - items: [{ - defaults: { - border: false - }, - title: 'Step 1: Stims/Blood Draws', - layout: { - type: 'table', - columns: 2, - tdAttrs: { style: 'padding-right: 10px;' } - }, - items: [{ - html: 'Total Stims:' - },{ - html: '' + this.stimStats.totalStims + '' - },{ - html: 'Lacking Sorts:' - },{ - html: '' + this.stimStats.lackingSort + '' - },{ - html: 'Non-passing Status:' - },{ - html: '' + this.stimStats.hasStatus + '' - },{ - xtype: 'ldk-linkbutton', - text: 'Import Stims', - href: 'javascript:void(0);', - linkCls: 'labkey-text-link', - handler: function(btn){ - if (LABKEY.Security.currentContainer.type === 'workbook'){ - Ext4.define('TCRdb.window.StimUploadWindow', { - extend: 'Ext.window.Window', - initComponent: function(){ - Ext4.apply(this, { - title: 'Import Stims', - items: [{ - xtype: 'labkey-exceluploadpanel', - bubbleEvents: ['uploadexception', 'uploadcomplete'], - itemId: 'theForm', - title: null, - buttons: null, - containerPath: Laboratory.Utils.getQueryContainerPath(), - schemaName: 'tcrdb', - queryName: 'stims', - populateTemplates: function(meta){ - Ext4.Msg.hide(); - var toAdd = []; - - toAdd.push({ - html: 'Use the button below to download an excel template for uploading stims.', - border: false, - style: 'padding-bottom: 10px;', - width: 700 - }); - - toAdd.push({ - xtype: 'ldk-integerfield', - itemId: 'templateRows', - fieldLabel: 'Total Stims', - labelWidth: 120, - value: 10 - }); - - toAdd.push({ - xtype: 'textfield', - itemId: 'treatment', - fieldLabel: 'Treatment', - labelWidth: 120, - value: 'TAPI-0' - }); - - toAdd.push({ - xtype: 'textfield', - itemId: 'coStim', - fieldLabel: 'Co-Stim', - labelWidth: 120, - value: 'CD28/CD49d' - }); - - toAdd.push({ - xtype: 'textfield', - itemId: 'effectors', - fieldLabel: 'Effector', - labelWidth: 120, - value: 'PBMC' - }); - - toAdd.push({ - xtype: 'ldk-numberfield', - itemId: 'numEffectors', - fieldLabel: '# Effectors', - labelWidth: 120, - value: 1000000 - }); - - toAdd.push({ - xtype: 'textfield', - itemId: 'apc', - fieldLabel: 'APCs', - labelWidth: 120, - value: 'PBMC' - }); - - toAdd.push({ - xtype: 'ldk-numberfield', - itemId: 'numAPC', - fieldLabel: '# APCs', - labelWidth: 120, - value: null - }); - - toAdd.push({ - xtype: 'button', - style: 'margin-bottom: 10px;', - text: 'Download Template', - border: true, - handler: this.generateExcelTemplate - }); - - this.down('#templateArea').add(toAdd); - }, - generateExcelTemplate: function(){ - var win = this.up('window'); - var numRows = win.down('#templateRows').getValue() || 1; - var effectors = win.down('#effectors').getValue(); - var numEffectors = win.down('#numEffectors').getValue(); - var apc = win.down('#apc').getValue(); - var numAPC = win.down('#numAPC').getValue(); - var treatment = win.down('#treatment').getValue(); - var coStim = win.down('#coStim').getValue(); - - var data = []; - data.push(['Tube #', 'Animal/Cell', 'Sample Date', 'Effectors', '# Effectors', 'APCs', '# APCs', 'Treatment', 'Co-stim', 'Peptide/Stim', 'Comment']); - for (var i=0;i' + this.sortStats.totalSorts + '' - }, { - xtype: 'ldk-linkbutton', - text: '(' + this.sortStats.totalPlates.length + ' plates)', - href: 'javascript:void(0);', - handler: this.getPlateCallback(this.sortStats.totalPlates) - }, { - html: 'Lacking cDNA Libraries (All):' - }, { - html: '' + this.sortStats.lackingLibraries + '' - }, { - xtype: 'ldk-linkbutton', - text: '(' + this.sortStats.totalPlatesLackingLibraries.length + ' plates)', - href: 'javascript:void(0);', - handler: this.getPlateCallback(this.sortStats.totalPlatesLackingLibraries) - }, { - html: 'Lacking cDNA Libraries (Bulk):' - }, { - html: '' + this.sortStats.bulkLackingLibraries + '' - }, { - xtype: 'ldk-linkbutton', - text: '(' + this.sortStats.totalBulkPlatesLackingLibraries.length + ' plates)', - href: 'javascript:void(0);', - handler: this.getPlateCallback(this.sortStats.totalBulkPlatesLackingLibraries) - }] - },{ - xtype: 'ldk-linkbutton', - text: 'Import Sort Data For Stims', - href: 'javascript:void(0);', - linkCls: 'labkey-text-link', - handler: function (btn) { - if (LABKEY.Security.currentContainer.type === 'workbook') { - Ext4.define('TCRdb.window.SortUploadWindow', { - extend: 'Ext.window.Window', - initComponent: function () { - Ext4.apply(this, { - title: 'Import Sorts for Stims', - items: [{ - xtype: 'labkey-exceluploadpanel', - bubbleEvents: ['uploadexception', 'uploadcomplete'], - itemId: 'theForm', - title: null, - buttons: null, - containerPath: Laboratory.Utils.getQueryContainerPath(), - schemaName: 'tcrdb', - queryName: 'sorts', - populateTemplates: function (meta) { - Ext4.Msg.hide(); - var toAdd = []; - - toAdd.push({ - html: 'Use the button below to download an excel template pre-populated with data from the sorts imported into this workbook.', - border: false, - style: 'padding-bottom: 10px;', - width: 700 - }); - - toAdd.push({ - xtype: 'textfield', - itemId: 'buffer', - fieldLabel: 'Sort Buffer', - labelWidth: 120, - value: 'Takara Buffer' - }); - - toAdd.push({ - xtype: 'checkbox', - itemId: 'skipWithData', - fieldLabel: 'Skip Stims With Sorts Imported', - labelWidth: 120, - helpPopup: 'If checked, stims with sort records already importd will be skipped', - checked: true - }); - - toAdd.push({ - xtype: 'ldk-integerfield', - itemId: 'templateRows', - fieldLabel: 'Rows Per Stim', - labelWidth: 120, - helpPopup: 'For each stim, the template will include this many rows', - value: 2 - }); - - toAdd.push({ - xtype: 'button', - style: 'margin-bottom: 10px;', - text: 'Download Template', - border: true, - handler: this.generateExcelTemplate - }); - - this.down('#templateArea').add(toAdd); - }, - generateExcelTemplate: function () { - var win = this.up('window'); - var rowsPer = win.down('#templateRows').getValue() || 1; - var skipWithData = win.down('#skipWithData').getValue(); - var buffer = win.down('#buffer').getValue(); - - var data = []; - data.push(['TubeNum', 'StimId', 'AnimalId', 'SampleDate', 'Peptide/Stim', 'Treatment', 'Buffer', 'Population', 'Replicate', 'Cells', 'PlateId', 'Well', 'Comment']); - Ext4.Array.forEach(win.stimRows, function (r) { - if (skipWithData && r.numSorts) { - return; - } - - for (var i = 0; i < rowsPer; i++) { - data.push([r.tubeNum, r.rowid, r.animalId, r.date, r.stim, r.treatment, buffer, null, null, null, null, null, null, null]); - } - }, this); - - LABKEY.Utils.convertToExcel({ - fileName: 'SortImport_' + Ext4.Date.format(new Date(), 'Y-m-d H_i_s') + '.xls', - sheets: [{ - name: 'Sorts', - data: data - }] - }); - }, - listeners: { - uploadcomplete: function (panel, response) { - Ext4.Msg.alert('Success', 'Upload Complete!', function (btn) { - this.up('window').close(); - location.reload(); - }, this); - } - } - }] - }); - - this.callParent(); - }, - buttons: [{ - text: 'Upload', - width: 50, - handler: function (btn) { - var form = btn.up('window').down('#theForm'); - form.formSubmit.call(form, btn); - }, - scope: this, - formBind: true - }, { - text: 'Close', - width: 50, - handler: function (btn) { - btn.up('window').close(); - } - }] - }); - - Ext4.create('TCRdb.window.SortUploadWindow', { - stimRows: this.up('tcrdb-stimpanel').stimRows - }).show(); - } - else { - Ext4.Msg.alert('Error', 'This is only allowed when in a specific workbook. Please enter the workbook into the box at the top of the page and hit \'Go\''); - } - } - }] - },{ - title: 'Step 3: cDNA Synthesis / Library Prep', - defaults: { - border: false - }, - items: [{ - layout: { - type: 'table', - columns: 3, - tdAttrs: { style: 'padding-right: 10px;' } - }, - defaults: { - border: false - }, - items: [{ - html: 'Total cDNA Libraries:' - },{ - html: '' + this.libraryStats.totalLibraries + '' - },{ - xtype: 'ldk-linkbutton', - text: '(' + this.libraryStats.totalPlates.length + ' plates)', - href: 'javascript:void(0);', - handler: this.getPlateCallback(this.libraryStats.totalPlates) - },{ - html: 'Lacking Any Readset:' - },{ - html: '' + this.libraryStats.lackingAnyReadset + '' - },{ - xtype: 'ldk-linkbutton', - text: '(' + this.libraryStats.totalPlatesLackingAnyReadset.length + ' plates)', - href: 'javascript:void(0);', - handler: this.getPlateCallback(this.libraryStats.totalPlatesLackingAnyReadset) - },{ - html: 'With Whole Transcriptome Readset:' - },{ - html: '' + this.libraryStats.withReadset + '' - },{ - xtype: 'ldk-linkbutton', - text: '(' + this.libraryStats.totalPlatesWithReadset.length + ' plates)', - href: 'javascript:void(0);', - handler: this.getPlateCallback(this.libraryStats.totalPlatesWithReadset) - },{ - html: 'With TCR Enriched Readset:' - },{ - html: '' + this.libraryStats.withTCRReadset + '' - },{ - xtype: 'ldk-linkbutton', - text: '(' + this.libraryStats.totalPlatesWithTCRReadset.length + ' plates)', - href: 'javascript:void(0);', - handler: this.getPlateCallback(this.libraryStats.totalPlatesWithTCRReadset) - }] - },{ - xtype: 'ldk-linkbutton', - text: 'Create cDNA Libraries From Sorts', - href: 'javascript:void(0);', - scope: this, - linkCls: 'labkey-text-link', - handler: function(){ - if (LABKEY.Security.currentContainer.type === 'workbook'){ - Ext4.define('TCRdb.window.cDNAUploadWindow', { - extend: 'Ext.window.Window', - initComponent: function(){ - Ext4.apply(this, { - title: 'Create cDNA Libraries From Sorts', - items: [{ - xtype: 'labkey-exceluploadpanel', - bubbleEvents: ['uploadexception', 'uploadcomplete'], - itemId: 'theForm', - title: null, - buttons: null, - containerPath: Laboratory.Utils.getQueryContainerPath(), - schemaName: 'tcrdb', - queryName: 'cdnas', - populateTemplates: function(meta){ - Ext4.Msg.hide(); - var toAdd = []; - - toAdd.push({ - html: 'Use the button below to download an excel template pre-populated with data from the selected plate IDs.', - border: false, - style: 'padding-bottom: 10px;', - width: 700 - }); - - toAdd.push({ - xtype: 'textfield', - itemId: 'destPlate', - fieldLabel: 'Destination Plate ID', - labelWidth: 160 - }); - - toAdd.push({ - xtype: 'ldk-simplecombo', - itemId: 'chemistry', - fieldLabel: 'Chemistry', - labelWidth: 160, - storeValues: ['SMART-Seq2', 'Takara SMART-Seq HT', '10x GEX/VDJ'], - value: 'SMART-Seq2' - }); - - toAdd.push({ - xtype: 'textarea', - itemId: 'plates', - fieldLabel: 'Source Plates', - labelWidth: 160, - //width: 200, - height: 100 - }); - - var win = this.up('window'); - toAdd.push({ - xtype: 'ldk-linkbutton', - itemId: 'showIds', - style: 'margin-left: 165px;', - text: 'Show Plate IDs', - scope: this, - handler: win.getPlateCallback(win.sortStats.totalPlatesLackingLibraries, 'plates'), - linkCls: 'labkey-text-link' - }); - - toAdd.push({ - xtype: 'button', - style: 'margin-bottom: 10px;', - text: 'Download Template', - border: true, - handler: this.generateExcelTemplate - }); - - toAdd.push({ - xtype: 'checkbox', - itemId: 'keepWell', - checked: true, - fieldLabel: 'Keep Original Well', - labelWidth: 160 - }); - - this.down('#templateArea').add(toAdd); - }, - - generateExcelTemplate: function(btn) { - var win = btn.up('window'); - var chemistry = win.down('#chemistry').getValue(); - var destPlate = win.down('#destPlate').getValue(); - var keepWell = win.down('#keepWell').getValue(); - - if (!destPlate) { - Ext4.Msg.alert('Error', 'Must provide destination plate IDs'); - return; - } - - var plates = Ext4.String.trim(btn.up('window').down('textarea').getValue()); - if (!plates) { - Ext4.Msg.alert('Error', 'Must provide source plate IDs'); - return; - } - - plates = plates.replace(/[\r\n]+/g, '\n'); - plates = plates.replace(/[\n]+/g, '\n'); - plates = Ext4.String.trim(plates); - if (plates){ - plates = plates.split('\n'); - } - - Ext4.Msg.wait('Loading...'); - LABKEY.Query.selectRows({ - containerPath: Laboratory.Utils.getQueryContainerPath(), - schemaName: 'tcrdb', - queryName: 'sorts', - sort: 'well/addressByColumn', - columns: 'rowid,stimId,stimId/animalId,stimId/effector,stimId/stim,stimId/date,stimId/treatment,population,replicate,cells,plateId,well,well/addressByColumn,numLibraries', - scope: win, - filterArray: [LABKEY.Filter.create('plateId', plates.join(';'), LABKEY.Filter.Types.IN)], - failure: LDK.Utils.getErrorCallback(), - success: function (results) { - Ext4.Msg.hide(); - - if (!results || !results.rows || !results.rows.length) { - Ext4.Msg.alert('Error', 'No sorts found for the selected plates'); - return; - } - - var data = []; - data.push(['Source Plate', 'Source Well', 'SortId', 'Plate Id', 'Well', 'Name', 'Chemistry', 'Comments']); - var wellIdx = 0; - var wellsUsed = {}; - var errors = []; - Ext4.Array.forEach(plates, function (sourcePlateId) { - Ext4.Array.forEach(results.rows, function (r) { - if (r.plateId !== sourcePlateId){ - return; - } - - var name = TCRdb.panel.StimPanel.getNameFromSort(r); - var targetWell = keepWell ? r.well : this.wellNames96[wellIdx]; - if (wellsUsed[targetWell]){ - errors.push('Duplicate well: ' + targetWell); - } - wellsUsed[targetWell] = true; - data.push([r.plateId, r.well, r.rowid, destPlate, targetWell, name, chemistry, null]); - wellIdx++; - }, this); - }, this); - - for (var i=0;i'), function(){ - LABKEY.Utils.convertToExcel({ - fileName: 'cDNAImport_' + Ext4.Date.format(new Date(), 'Y-m-d H_i_s') + '.xls', - sheets: [{ - name: 'cDNA Libraries', - data: data - }] - }); - }, this); - } - else { - LABKEY.Utils.convertToExcel({ - fileName: 'cDNAImport_' + Ext4.Date.format(new Date(), 'Y-m-d H_i_s') + '.xls', - sheets: [{ - name: 'cDNA Libraries', - data: data - }] - }); - } - } - }); - }, - listeners: { - uploadcomplete: function(panel, response){ - Ext4.Msg.alert('Success', 'Upload Complete!', function(btn){ - this.up('window').close(); - location.reload(); - }, this); - } - } - }] - }); - - this.callParent(); - }, - getWellSort: function(wellNames96){ - return function(a, b){ - var idx1 = wellNames96.indexOf(a[4]); - var idx2 = wellNames96.indexOf(b[4]); - - return idx1 - idx2; - } - }, - buttons: [{ - text: 'Upload', - width: 50, - handler: function(btn){ - var form = btn.up('window').down('#theForm'); - form.formSubmit.call(form, btn); - }, - scope: this, - formBind: true - },{ - text: 'Close', - width: 50, - handler: function(btn){ - btn.up('window').close(); - } - }] - }); - - Ext4.create('TCRdb.window.cDNAUploadWindow', { - wellNames96: this.wellNames96, - sortStats: this.sortStats, - getPlateCallback: this.getPlateCallback - }).show(); - } - else { - Ext4.Msg.alert('Error', 'This is only allowed when in a specific workbook. Please enter the workbook into the box at the top of the page and hit \'Go\''); - } - } - },{ - xtype: 'ldk-linkbutton', - text: 'Download Library Prep Template (box)', - linkCls: 'labkey-text-link', - href: 'https://ohsu.box.com/s/6kncrzm4ba9mxjput12v8u500tjlsip7', - linkTarget: '_blank' - },{ - xtype: 'ldk-linkbutton', - text: 'Download TCR Enrichment Library Prep Template (box)', - linkCls: 'labkey-text-link', - href: 'https://ohsu.box.com/s/js55a347q5mioqxwowe1dk3prkvn4d29', - linkTarget: '_blank' - },{ - xtype: 'ldk-linkbutton', - text: 'Download Names To Use In Protocols', - href: 'javascript:void(0);', - linkCls: 'labkey-text-link', - handler: function(){ - if (LABKEY.Security.currentContainer.type === 'workbook'){ - Ext4.Msg.wait('Loading...'); - LABKEY.Query.selectRows({ - containerPath: Laboratory.Utils.getQueryContainerPath(), - schemaName: 'tcrdb', - queryName: 'cdnas', - sort: 'well/addressByColumn', - scope: this, - failure: LDK.Utils.getErrorCallback(), - success: function(results){ - Ext4.Msg.hide(); - if (!results || !results.rows || !results.rows.length){ - Ext4.Msg.alert('Error', 'No cDNA libraries found'); - return; - } - - var rows = []; - rows.push(['Well', 'Name'].join('\t')); - Ext4.Array.forEach(results.rows, function(r){ - var name = TCRdb.panel.StimPanel.getNameFromCDNAs(r); - rows.push([r.well, name].join('\t')); - }, this); - - Ext4.create('Ext.window.Window', { - bodyStyle: 'padding: 5px;', - items: [{ - html: 'Please use the following as names for the sorts in the folder', - border: false, - style: 'padding-bottom: 10px;' - },{ - xtype: 'textarea', - width: 500, - height: 200, - value: rows.join('\n') - }], - buttons: [{ - text: 'Close', - handler: function(btn){ - btn.up('window').close(); - } - }] - }).show(); - } - }); - } - else { - Ext4.Msg.alert('Error', 'This is only allowed when in a specific workbook. Please enter the workbook into the box at the top of the page and hit \'Go\''); - } - } - }] - },{ - title: 'Step 4: Create Readsets / Template for Sequencing', - defaults: { - border: false - }, - items: [{ - layout: { - type: 'table', - columns: 2, - tdAttrs: { style: 'padding-right: 10px;' } - }, - defaults: { - border: false - }, - items: [{ - html: 'Total Readsets:' - },{ - html: '' + this.readsetStats.totalReadsets + '' - },{ - html: 'Data Not Imported:' - },{ - html: '' + this.readsetStats.lackingData + '' - }] - },{ - xtype: 'ldk-linkbutton', - text: 'Create Readsets From cDNA Libraries', - href: 'javascript:void(0);', - linkCls: 'labkey-text-link', - handler: function(){ - if (LABKEY.Security.currentContainer.type === 'workbook'){ - Ext4.define('TCRdb.window.ReadsetUploadWindow', { - extend: 'Ext.window.Window', - initComponent: function(){ - Ext4.apply(this, { - title: 'Create Readsets From cDNAs', - bodyStyle: 'padding: 5px;', - items: [{ - html: 'Use the button below to download an excel template pre-populated with data from the sorts imported into this workbook.', - border: false, - style: 'padding-bottom: 10px;', - width: 700 - },{ - xtype: 'textarea', - itemId: 'plateIds', - fieldLabel: 'Plate Id(s)', - labelWidth: 120, - height: 100 - },{ - xtype: 'ldk-linkbutton', - itemId: 'showIds', - text: 'Show Plate IDs', - style: 'margin-left: 125px;', - scope: this, - handler: this.getPlateCallback(this.libraryStats.totalPlatesLackingAnyReadset, 'plateIds'), - linkCls: 'labkey-text-link' - },{ - xtype: 'ldk-simplecombo', - itemId: 'application', - fieldLabel: 'Application', - labelWidth: 120, - storeValues: ['Whole Transcriptome RNA-Seq', 'TCR Enrichment', '10x GEX Only', '10x GEX/TCR'], - forceSelection: true, - multiSelect: true - },{ - xtype: 'labkey-combo', - itemId: 'chemistry', - fieldLabel: 'Chemistry', - labelWidth: 120, - store: { - type: 'labkey-store', - schemaName: 'sequenceanalysis', - queryName: 'sequence_chemistries', - autoLoad: true - }, - displayField: 'chemistry', - valueField: 'chemistry', - value: 'Illumina HiSeq3000', - forceSelection: true - },{ - xtype: 'checkbox', - itemId: 'includeImported', - fieldLabel: 'Include Those With Existing Readsets' - },{ - xtype: 'button', - style: 'margin-bottom: 10px;', - text: 'Download Template', - border: true, - handler: this.generateExcelTemplate - },{ - xtype: 'textarea', - height: 350, - width: 700, - itemId: 'template' - }] - }); - - this.callParent(); - }, - generateExcelTemplate: function(){ - var win = this.up('window'); - - //'Whole Transcriptome RNA-Seq', 'TCR Enrichment', 10x GEX Only, 10x GEX/TCR - var types = win.down('#application').getValue(); - var chemistry = win.down('#chemistry').getValue(); - var includeImported = win.down('#includeImported').getValue(); - var plates = Ext4.String.trim(win.down('textarea').getValue()); - if (!plates) { - Ext4.Msg.alert('Error', 'Must provide source plate IDs'); - return; - } - - plates = plates.replace(/[\r\n]+/g, '\n'); - plates = plates.replace(/[\n]+/g, '\n'); - plates = Ext4.String.trim(plates); - if (plates){ - plates = plates.split('\n'); - } - - if (!types || !types.length){ - Ext4.Msg.alert('Error', 'Must choose the application(s)'); - return; - } - - var applications = []; - Ext4.Array.forEach(types, function(type) { - switch (type) { - case 'Whole Transcriptome RNA-Seq': - applications.push('RNA-seq'); - break; - case 'TCR Enrichment': - applications.push('RNA-seq + Enrichment'); - break; - case '10x GEX Only': - applications.push('10x GEX'); - break; - case '10x GEX/TCR': - applications.push('10x GEX'); - applications.push('10x VDJ'); - } - }, this); - applications = Ext4.unique(applications); - - var data = []; - data.push(['LibraryId', 'PlateId', 'Source Well', 'Name', 'Subject Id', 'Sample Date', '5-Barcode', '3-Barcode', 'Sample Type', 'Sequencing Platform', 'Application', 'Chemistry', 'Library Type', 'Comments']); - Ext4.Array.forEach(win.libraryRows, function(r){ - Ext4.Array.forEach(applications, function(application){ - if (plates.indexOf(r.plateId) > -1) { - if (includeImported || (['RNA-seq + Enrichment', '10x VDJ'].indexOf(application) > -1 && !r.enrichedReadsetId) || (['RNA-seq', '10x GEX'].indexOf(application) > -1 && !r.readsetId)) { - var applicationValue = application; - if (application === 'RNA-seq' && r.cells === 1) { - applicationValue = 'RNA-seq, Single Cell'; - } - else if (['10x GEX', '10x VDJ'].indexOf(application) > -1){ - applicationValue = 'RNA-seq, Single Cell'; - } - - var libraryType = null; - switch (application){ - case 'RNA-seq': - libraryType = 'SMART-Seq2'; - break; - case '10x VDJ': - libraryType = '10x 5\' VDJ (Rhesus A/B/G)'; - break; - case '10x GEX': - libraryType = '10x 5\' GEX'; - } - - var name = TCRdb.panel.StimPanel.getNameFromCDNAs(r); - data.push([r.rowid, r.plateId, r.well, name, r['sortId/stimId/animalId'], r['sortId/stimId/date'], null, null, 'mRNA', 'ILLUMINA', applicationValue, chemistry, libraryType, null]); - } - } - }, this); - }, this); - - if (data.length === 1){ - Ext4.Msg.alert('Error', 'No matching rows found'); - return; - } - - LABKEY.Utils.convertToExcel({ - fileName: 'ReadsetImport_' + Ext4.Date.format(new Date(), 'Y-m-d H_i_s') + '.xls', - sheets: [{ - name: 'Readsets', - data: data - }] - }); - }, - buttons: [{ - text: 'Upload', - width: 50, - handler: function(btn){ - btn.up('window').processUpload(); - }, - scope: this, - formBind: true - },{ - text: 'Close', - width: 50, - handler: function(btn){ - btn.up('window').close(); - } - }], - processUpload: function(){ - var text = this.down('#template').getValue(); - if (!text){ - Ext4.Msg.alert('Error', 'No rows provided'); - return; - } - text = LDK.Utils.CSVToArray(Ext4.String.trim(text), '\t'); - - var header = text.shift(); - var headerToField = { - Name: 'name', - 'Subject Id': 'subjectid', - 'Sample Date': 'sampledate', - '5-Barcode': 'barcode5', - '3-Barcode': 'barcode3', - 'Sample Type': 'sampletype', - 'Sequencing Platform': 'platform', - 'Application': 'application', - 'Chemistry': 'chemsitry', - 'Library Type': 'libraryType', - 'Comments': 'comments', - 'LibraryId': 'libraryId' - - }; - - var readsetToInsert = []; - var cDNAsToUpdate = {}; - var errorMsgs = []; - - Ext4.Array.forEach(text, function (row, rowIdx) { - var r = {}; - for (var headerName in headerToField){ - var idx = header.indexOf(headerName); - if (idx !== -1 && row.length > idx){ - r[headerToField[headerName]] = row[idx]; - } - } - - var hasBarcodes; - switch (r.libraryType){ - case '10x 5\' GEX': - case '10x 5\' VDJ (Rhesus A/B/G)': - hasBarcodes = !!r.barcode5 && !r.barcode3; - if (!hasBarcodes) {errorMsgs.push('10x data must have the 5\' barcode but not 3\'')}; - break; - default: - hasBarcodes = !!r.barcode5 && !!r.barcode3; - } - - if (!r.name || !r.application || !hasBarcodes || !r.libraryId){ - errorMsgs.push('Every row must have name, application and proper barcodes'); - return; - } - - //TODO: set container to match sort - // if (row.libraryId && containerMap[row.libraryId]){ - // row.container = containerMap[row.libraryId]; - // } - // - // if (!row.container){ - // //TODO - // } - - if (['10x 5\' GEX', '10x 5\' VDJ (Rhesus A/B/G)'].indexOf(r.libraryType) > -1 ) { - r.barcode5 = r.barcode5.toUpperCase(); - if (!r.barcode5.match(/^SI-GA-/)) { - if (r.barcode5.length > 3) { - errorMsgs.push('Every row must have name, application and proper barcodes'); - } - else { - r.barcode5 = 'SI-GA-' + r.barcode5; - } - } - } - - readsetToInsert.push(r); - - cDNAsToUpdate[r.libraryId] = cDNAsToUpdate[r.libraryId] || {}; - cDNAsToUpdate[r.libraryId].container = row.container; - if ('rna-seq' === r.application.toLowerCase()){ - cDNAsToUpdate[r.libraryId].readsetIdx = rowIdx; - } - else if ('rna-seq, single cell' === r.application.toLowerCase() && r.libraryType === '10x 5\' GEX'){ - cDNAsToUpdate[r.libraryId].readsetIdx = rowIdx; - } - else if ('rna-seq + enrichment' === r.application.toLowerCase()){ - cDNAsToUpdate[r.libraryId].enrichedReadsetIdx = rowIdx; - } - else if ('rna-seq, single cell' === r.application.toLowerCase() && r.libraryType === '10x 5\' VDJ (Rhesus A/B/G)'){ - cDNAsToUpdate[r.libraryId].enrichedReadsetIdx = rowIdx; - } - else { - errorMsgs.push('Unknown application/libraryType: ' + r.application + ' / ' + r.libraryType); - } - }, this); - - if (errorMsgs.length){ - errorMsgs = Ext4.unique(errorMsgs); - Ext4.Msg.alert('Error', errorMsgs.join('
')); - return; - } - - Ext4.Msg.wait('Saving...'); - LABKEY.Query.insertRows({ - //containerPath: Laboratory.Utils.getQueryContainerPath(), - schemaName: 'sequenceanalysis', - queryName: 'sequence_readsets', - rows: readsetToInsert, - scope: this, - failure: LDK.Utils.getErrorCallback(), - success: function (results) { - var toUpdate = []; - for (var libraryId in cDNAsToUpdate){ - //TODO: add container - var r = {rowid: libraryId}; - if (Ext4.isDefined(cDNAsToUpdate[libraryId].readsetIdx)){ - r.readsetId = results.rows[cDNAsToUpdate[libraryId].readsetIdx].rowId - } - - if (Ext4.isDefined(cDNAsToUpdate[libraryId].enrichedReadsetIdx)){ - r.enrichedReadsetId = results.rows[cDNAsToUpdate[libraryId].enrichedReadsetIdx].rowId - } - - if (Ext4.isDefined(cDNAsToUpdate[libraryId].container)){ - r.container = cDNAsToUpdate[libraryId].container; - } - - if (r.readsetId || r.enrichedReadsetId){ - toUpdate.push(r); - } - } - - if (toUpdate.length){ - LABKEY.Query.updateRows({ - //containerPath: Laboratory.Utils.getQueryContainerPath(), - schemaName: 'tcrdb', - queryName: 'cdnas', - rows: toUpdate, - scope: this, - failure: LDK.Utils.getErrorCallback(), - success: function (results) { - Ext4.Msg.hide(); - - Ext4.Msg.alert('Success', 'Rows saved', function(){ - window.location.reload(); - }); - } - }); - } - else { - Ext4.Msg.hide(); - Ext4.Msg.alert('Error', 'There were no readsets to update'); - } - } - }); - } - }); - - Ext4.create('TCRdb.window.ReadsetUploadWindow', { - libraryRows: this.up('tcrdb-stimpanel').libraryRows, - libraryStats: this.up('tcrdb-stimpanel').libraryStats, - getPlateCallback: this.up('tcrdb-stimpanel').getPlateCallback - }).show(); - } - else { - Ext4.Msg.alert('Error', 'This is only allowed when in a specific workbook. Please enter the workbook into the box at the top of the page and hit \'Go\''); - } - } - },{ - xtype: 'ldk-linkbutton', - text: 'Download Blank MPSSR Template (box)', - linkCls: 'labkey-text-link', - href: 'https://ohsu.box.com/s/awhkmncp3gphs60inlu0mnts1yd22z25' - },{ - xtype: 'ldk-linkbutton', - text: 'Request Runs From MPSSR (iLABS)', - linkCls: 'labkey-text-link', - href: 'https://ohsu.corefacilities.org/account/pending/ohsu' - },{ - xtype: 'ldk-linkbutton', - text: 'MedGenome Information', - linkCls: 'labkey-text-link', - href: 'https://prime-seq.ohsu.edu/wiki/Internal/Bimber/page.view?name=tcrSequenceShipping' - },{ - xtype: 'ldk-linkbutton', - text: 'Shipment List', - linkCls: 'labkey-text-link', - href: LABKEY.ActionURL.buildURL('query', 'executeQuery', Laboratory.Utils.getQueryContainerPath(), {schemaName: 'lists', queryName: 'MedGenomeShipments'}) - }] - },{ - title: 'Plate Summary', - defaults: { - border: false - }, - items: [{ - xtype: 'ldk-linkbutton', - text: 'View Summary of Sorts By Plate (this workbook)', - linkCls: 'labkey-text-link', - hidden: LABKEY.Security.currentContainer.type != 'workbook', - href: LABKEY.ActionURL.buildURL('query', 'executeQuery', null, {schemaName: 'tcrdb', queryName: 'sortStatusByPlate', 'query.isComplete~eq': false}) - },{ - xtype: 'ldk-linkbutton', - text: 'View Summary of Sorts By Plate (entire folder)', - linkCls: 'labkey-text-link', - href: LABKEY.ActionURL.buildURL('query', 'executeQuery', Laboratory.Utils.getQueryContainerPath(), {schemaName: 'tcrdb', queryName: 'sortStatusByPlate', 'query.isComplete~eq': false}) - },{ - xtype: 'ldk-linkbutton', - text: 'View Summary of Sorts By Animal/Plate (this workbook)', - linkCls: 'labkey-text-link', - hidden: LABKEY.Security.currentContainer.type !== 'workbook', - href: LABKEY.ActionURL.buildURL('query', 'executeQuery', null, {schemaName: 'tcrdb', queryName: 'sortStatusByPlateAndSample'}) - },{ - xtype: 'ldk-linkbutton', - text: 'View Summary of Sorts By Animal/Plate (entire folder)', - linkCls: 'labkey-text-link', - href: LABKEY.ActionURL.buildURL('query', 'executeQuery', Laboratory.Utils.getQueryContainerPath(), {schemaName: 'tcrdb', queryName: 'sortStatusByPlateAndSample'}) - }] - }] - } - }, - - getPlateCallback: function(plateIds, fieldId){ - return function(f){ - var target; - if (fieldId){ - target = f.up('window').down('#' + fieldId); - } - - var items = []; - Ext4.Array.forEach(plateIds, function(id){ - var listener = target ? { - scope: this, - afterrender: function(panel){ - panel.mon(panel.getEl(), 'click', function(){ - target.setValue(target.getValue() + (target.getValue() ? '\n' : '') + id); - }, this); - } - } : null; - items.push({ - html: id, - bodyStyle: target ? 'text-decoration: underline;cursor: pointer;' : null, - border: false, - listeners: listener - }); - }, this); - - if (plateIds.length === 0){ - items.push({html: 'There are no plates in this folder', border: false}); - } - - Ext4.create('Ext.window.Window', { - modal: true, - title: 'Plate IDs', - maxHeight: '400', - autoScroll: true, - width: 300, - bodyStyle: 'padding: 5px;', - items: [{ - xtype: 'container', - items: items - }], - buttons: [{ - text: 'Close', - handler: function(btn){ - btn.up('window').close(); - } - }] - }).show(); - } - }, - - statics: { - getNameFromSort: function(r){ - return [ - r['plateId'], - r['well'], - r['stimId/animalId'], - r['stimId/stim'], - r['stimId/treatment'], - r.population + (r.replicate ? '_' + r.replicate : '') - ].join('_').replace(/ /g, '-'); - }, - - getNameFromCDNAs: function(r){ - return [ - //NOTE: preferentially retain the original sort plate name, in case of combined cDNA plates - r['sortId/plateId'] || r['plateId'], - r['well'], - r['sortId/stimId/animalId'], - r['sortId/stimId/stim'], - r['sortId/stimId/treatment'], - r['sortId/population'] + (r['sortId/cells'] === 1 ? '_Clone' : '') + (r['sortId/replicate'] ? '_' + r['sortId/replicate'] : '') - ].join('_').replace(/ /g, '-').replace(/\(/g, '').replace(/\)/g, '').replace(/\+/g, 'Pos'); - } - } -}); \ No newline at end of file diff --git a/tcrdb/resources/web/tcrdb/panel/cDNAImportPanel.js b/tcrdb/resources/web/tcrdb/panel/cDNAImportPanel.js deleted file mode 100644 index 28c8ef451..000000000 --- a/tcrdb/resources/web/tcrdb/panel/cDNAImportPanel.js +++ /dev/null @@ -1,378 +0,0 @@ -Ext4.define('TCRdb.panel.cDNAImportPanel', { - extend: 'TCRdb.panel.PoolImportPanel', - - IGNORED_COLUMNS: ['animalId', 'sampleDate', 'stimId', 'population', 'sortId', 'hto', 'cells', 'stim', 'effector', 'tissue', 'stim_num'], - - initComponent: function () { - this.COLUMN_MAP = {}; - Ext4.Array.forEach(this.COLUMNS, function (col) { - if (this.IGNORED_COLUMNS.indexOf(col.name) > -1) { - return; - } - - //Do not allow rowspan for this type of import - col.allowRowSpan = false; - - this.COLUMN_MAP[col.name.toLowerCase()] = col; - Ext4.Array.forEach(col.labels, function (alias) { - this.COLUMN_MAP[alias.toLowerCase()] = col; - }, this); - }, this); - - Ext4.apply(this, { - title: null, - border: false, - defaults: { - border: false - }, - items: this.getPanelItems() - }); - - this.callParent(arguments); - }, - - getPanelItems: function(){ - return [{ - layout: { - type: 'hbox' - }, - items: [{ - xtype: 'ldk-integerfield', - style: 'margin-right: 5px;', - fieldLabel: 'Current Folder/Workbook', - labelWidth: 200, - minValue: 1, - value: LABKEY.Security.currentContainer.type === 'workbook' ? LABKEY.Security.currentContainer.name : null, - emptyText: LABKEY.Security.currentContainer.type === 'workbook' ? null : 'Showing All', - listeners: { - afterRender: function (field) { - new Ext4.util.KeyNav(field.getEl(), { - enter: function (e) { - var btn = field.up('panel').down('#goButton'); - btn.handler(btn); - }, - scope: this - }); - } - } - }, { - xtype: 'button', - itemId: 'goButton', - scope: this, - text: 'Go', - handler: function (btn) { - var wb = btn.up('panel').down('ldk-integerfield').getValue(); - if (!wb) { - wb = ''; - } - - var container = LABKEY.Security.currentContainer.type === 'workbook' ? LABKEY.Security.currentContainer.parentPath + '/' + wb : LABKEY.Security.currentContainer.path + '/' + wb; - window.location = LABKEY.ActionURL.buildURL('tcrdb', 'poolImport', container); - } - }, { - xtype: 'button', - scope: this, - hidden: !LABKEY.Security.currentUser.canInsert, - text: 'Create Workbook', - handler: function (btn) { - Ext4.create('Laboratory.window.WorkbookCreationWindow', { - abortIfContainerIsWorkbook: false, - canAddToExistingExperiment: false, - controller: 'tcrdb', - action: 'poolImport', - title: 'Create Workbook' - }).show(); - } - }] - }, { - style: 'padding-top: 10px;', - html: 'This page is designed to help import the readset/index information for 10x libraries, after the stim/sort data has already been imported.

' - }, { - layout: 'hbox', - items: [{ - xtype: 'button', - text: 'Download Template', - border: true, - scope: this, - href: LABKEY.ActionURL.getContextPath() + '/tcrdb/exampleData/ImportReadsetTemplate.xlsx' - }] - }, { - xtype: 'textfield', - style: 'margin-top: 20px;', - fieldLabel: 'Expt Number', - itemId: 'exptNum', - value: LABKEY.Security.currentContainer.type === 'workbook' ? LABKEY.Security.currentContainer.name : null - }, { - xtype: 'checkbox', - fieldLabel: 'Require GEX Library', - itemId: 'requireGEX', - checked: true - }, { - xtype: 'checkbox', - fieldLabel: 'Require TCR Library', - itemId: 'requireTCR', - checked: true - }, { - xtype: 'checkbox', - fieldLabel: 'Require HTO Library', - itemId: 'requireHTO', - checked: true - },{ - xtype: 'checkbox', - fieldLabel: 'Require Cite-Seq Library', - itemId: 'requireCITE', - checked: false - }, { - xtype: 'checkbox', - fieldLabel: 'Require Library Concentrations', - itemId: 'requireConc', - checked: true - },{ - xtype: 'ldk-simplecombo', - fieldLabel: '10x GEX/TCR Barcode Series', - itemId: 'barcodeSeries', - forceSelection: true, - storeValues: ['SI-GA'], - value: 'SI-GA' - },{ - xtype: 'ldk-simplecombo', - fieldLabel: '10x Cite-Seq Barcode Series', - itemId: 'citeseqBarcodeSeries', - forceSelection: true, - storeValues: ['SI-NA'], - value: 'SI-NA' - },{ - xtype: 'ldk-simplecombo', - fieldLabel: 'Hashing Type', - itemId: 'hashingType', - forceSelection: true, - storeValues: ['CD298', 'MultiSeq'], - value: 'MultiSeq' - }, { - xtype: 'textarea', - fieldLabel: 'Paste Data Below', - labelAlign: 'top', - itemId: 'data', - width: 1000, - height: 300 - }, { - xtype: 'button', - text: 'Preview', - border: true, - scope: this, - handler: this.onPreview - }, { - style: 'margin-top: 20px;margin-bottom: 10px;', - itemId: 'previewArea', - autoEl: 'table', - cls: 'stripe hover' - }]; - }, - - onPreview: function (btn) { - var text = this.down('#data').getValue(); - if (!text) { - Ext4.Msg.alert('Error', 'Must provide the table of data'); - return; - } - - this.EXPERIMENT = this.down('#exptNum').getValue(); - - text = Ext4.String.trim(text); - - var rows = LDK.Utils.CSVToArray(text, '\t'); - var colArray = this.parseHeader(rows.shift()); - var parsedRows = this.parseRows(colArray, rows); - - var groupedRows = this.groupForImport(colArray, parsedRows); - if (!groupedRows) { - console.log('No rows after grouping'); - return; - } - - var workbooks = []; - var hadError = false; - Ext4.Array.forEach(parsedRows, function(row){ - if (!row.workbook) { - hadError = true; - } - else { - workbooks.push(row.workbook); - } - }, this); - - if (hadError) { - Ext4.Msg.alert('Error', 'One or more rows missing a workbook ID'); - return; - } - - Ext4.Msg.wait('Loading workbooks'); - LABKEY.Query.selectRows({ - containerPath: Laboratory.Utils.getQueryContainerPath(), - schemaName: 'core', - queryName: 'workbooks', - columns: 'Name,EntityId', - filterArray: [LABKEY.Filter.create('Name', Ext4.unique(workbooks).join(';'), LABKEY.Filter.Types.IN)], - scope: this, - success: function(results) { - Ext4.Msg.hide(); - - var workbookMap = {}; - Ext4.Array.forEach(results.rows, function(r){ - workbookMap[r.Name] = r.EntityId; - }, this); - - Ext4.Array.forEach(groupedRows.cDNARows, function(r){ - LDK.Assert.assertNotEmpty('Unable to find workbook in map: ' + r.workbook, workbookMap[r.workbook]); - r.container = workbookMap[r.workbook]; - }, this); - - Ext4.Array.forEach(groupedRows.readsetRows, function(r){ - LDK.Assert.assertNotEmpty('Unable to find workbook in map: ' + r.workbook, workbookMap[r.workbook]); - r.container = workbookMap[r.workbook]; - }, this); - - this.onWorkbookQueryLoad(colArray, parsedRows, groupedRows); - }, - failure: LDK.Utils.getErrorCallback() - }); - }, - - onWorkbookQueryLoad: function(colArray, parsedRows, groupedRows) { - var plateIDs = []; - var hadError = false; - Ext4.Array.forEach(parsedRows, function(row){ - if (!row.plateId) { - hadError = true; - } - else { - plateIDs.push(row.plateId); - } - }, this); - - if (hadError) { - Ext4.Msg.alert('Error', 'One or more rows missing plate ID'); - return; - } - - plateIDs = Ext4.unique(plateIDs); - - Ext4.Msg.wait('Looking for matching cDNA'); - LABKEY.Query.selectRows({ - containerPath: Laboratory.Utils.getQueryContainerPath(), - schemaName: 'tcrdb', - queryName: 'cdnas', - columns: 'rowid,plateid,readsetid,enrichedreadsetid,hashingreadsetid,citeseqreadsetid,sortid/population,sortid,sortid/stimid,citeseqpanel', - filterArray: [LABKEY.Filter.create('plateId', plateIDs.join(';'), LABKEY.Filter.Types.IN)], - scope: this, - success: function(results) { - Ext4.Msg.hide(); - - if (!results.rows || !results.rows.length) { - Ext4.Msg.alert('Error', 'No matching rows found'); - return; - } - - var plateToCDNAMap = {}; - Ext4.Array.forEach(results.rows, function(row) { - plateToCDNAMap[row.plateId] = plateToCDNAMap[row.plateId] || []; - plateToCDNAMap[row.plateId].push(row.rowid); - }, this); - - var missing = []; - Ext4.Array.forEach(plateIDs, function (r) { - if (!plateToCDNAMap[r]) { - missing.push(r); - } - }, this); - - if (missing.length) { - Ext4.Msg.alert('Error', 'No cDNA records found for plates: ' + missing.join(', ')); - return; - } - - Ext4.Array.forEach(groupedRows.cDNARows, function(r){ - r.rowIds = plateToCDNAMap[r.plateId]; - }, this); - - this.renderPreview(colArray, parsedRows, groupedRows); - }, - failure: LDK.Utils.getErrorCallback() - }); - }, - - onSubmit: function (e, dt, node, config) { - Ext4.Msg.wait('Saving...'); - - var data = config.rowData.groupedRows; - - LABKEY.Query.insertRows({ - containerPath: Laboratory.Utils.getQueryContainerPath(), - schemaName: 'sequenceanalysis', - queryName: 'sequence_readsets', - rows: data.readsetRows, - success: function(results){ - var readsetMap = {}; - Ext4.Array.forEach(results.rows, function(row){ - readsetMap[row.name] = row.rowId; - }, this); - - var toUpdate = []; - Ext4.Array.forEach(data.cDNARows, function(row){ - var baseRow = {}; - - var gexReadsetId = readsetMap[row.plateId + '-GEX']; - if (gexReadsetId) { - baseRow.readsetId = gexReadsetId; - } - - var tcrReadsetId = readsetMap[row.plateId + '-TCR']; - if (tcrReadsetId) { - baseRow.enrichedReadsetId = tcrReadsetId; - } - - var htoReadsetId = readsetMap[row.plateId + '-HTO']; - if (htoReadsetId) { - baseRow.hashingReadsetId = htoReadsetId; - } - - var citeseqReadsetId = readsetMap[row.plateId + '-CITE']; - if (citeseqReadsetId) { - baseRow.citeseqReadsetId = citeseqReadsetId; - } - - baseRow.container = row.container; - - if (row.rowIds) { - Ext4.Array.forEach(row.rowIds, function(r){ - var toAdd = Ext4.apply({ - rowId: r - }, baseRow); - - toUpdate.push(toAdd); - }, this); - } - }, this); - - if (toUpdate.length) { - LABKEY.Query.updateRows({ - containerPath: Laboratory.Utils.getQueryContainerPath(), - schemaName: 'tcrdb', - queryName: 'cdnas', - rows: toUpdate, - success: function (results) { - Ext4.Msg.hide(); - Ext4.Msg.alert('Success', 'Data Saved', function(){ - window.location = LABKEY.ActionURL.buildURL('query', 'executeQuery.view', Laboratory.Utils.getQueryContainerPath(), {'query.queryName': 'cdnas', schemaName: 'tcrdb', 'query.sort': '-created'}); - }, this); - }, - failure: LDK.Utils.getErrorCallback(), - scope: this - }); - } - }, - failure: LDK.Utils.getErrorCallback(), - scope: this - }); - } -}); \ No newline at end of file diff --git a/tcrdb/src/org/labkey/tcrdb/ImportHelper.java b/tcrdb/src/org/labkey/tcrdb/ImportHelper.java deleted file mode 100644 index 4ac075926..000000000 --- a/tcrdb/src/org/labkey/tcrdb/ImportHelper.java +++ /dev/null @@ -1,110 +0,0 @@ -package org.labkey.tcrdb; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.labkey.api.data.Container; -import org.labkey.api.data.ContainerManager; -import org.labkey.api.data.TableInfo; -import org.labkey.api.data.TableSelector; -import org.labkey.api.query.FieldKey; -import org.labkey.api.query.QueryService; -import org.labkey.api.query.UserSchema; -import org.labkey.api.security.User; -import org.labkey.api.security.UserManager; -import org.labkey.api.util.MemTracker; -import org.labkey.api.util.PageFlowUtil; - -import java.util.HashMap; -import java.util.Map; - -public class ImportHelper -{ - private Container _container; - private User _user; - private TableInfo _table; - private TableInfo _sortTable; - - private static final Logger _log = LogManager.getLogger(ImportHelper.class); - - private Map _userSchemaMap = new HashMap<>(); - - private ImportHelper(String containerId, int userId, String queryName) - { - String schemaName = "tcrdb"; - - _container = ContainerManager.getForId(containerId); - if (_container == null) - throw new IllegalArgumentException("Unknown container: " + containerId); - - _container = _container.isWorkbook() ? _container.getParent() : _container; - - _user = UserManager.getUser(userId); - if (_user == null) - throw new IllegalArgumentException("Unknown user: " + userId); - - UserSchema us = getUserSchema(schemaName); - if (us == null) - throw new IllegalArgumentException("Unknown schema: " + schemaName); - - _table = us.getTable(queryName); - if (_table == null) - throw new IllegalArgumentException("Unknown table: " + schemaName + "." + queryName); - - _sortTable = us.getTable(TCRdbSchema.TABLE_SORTS); - if (_sortTable == null) - throw new IllegalArgumentException("Unknown table: " + schemaName + "." + queryName); - - MemTracker.getInstance().put(this); - } - - public static ImportHelper create(String containerId, int userId, String queryName) - { - return new ImportHelper(containerId, userId, queryName); - } - - private UserSchema getUserSchema(String name) - { - if (_userSchemaMap.containsKey(name)) - return _userSchemaMap.get(name); - - UserSchema us = QueryService.get().getUserSchema(_user, _container, name); - _userSchemaMap.put(name, us); - - return us; - } - - public Map getInitialWells() - { - TableSelector ts = new TableSelector(_table, PageFlowUtil.set("plateId", "well", "rowid")); - final Map ret = new HashMap<>(); - ts.forEachResults(rs -> { - String key = (rs.getString(FieldKey.fromString("plateId")) + "<>" + rs.getString(FieldKey.fromString("well"))).toUpperCase(); - ret.put(key, rs.getInt(FieldKey.fromString("rowId"))); - }); - - return ret; - } - - private Map sortToContainer = null; - - public String getContainerForSort(int sortId) - { - if (sortToContainer == null) - { - sortToContainer = new HashMap<>(); - new TableSelector(_sortTable, PageFlowUtil.set("rowId", "container")).forEachResults(rs -> { - sortToContainer.put(rs.getInt(FieldKey.fromString("rowId")), rs.getString(FieldKey.fromString("container"))); - }); - } - - if (sortToContainer != null && sortToContainer.containsKey(sortId)) - { - return sortToContainer.get(sortId); - } - - String containerId = new TableSelector(_sortTable, PageFlowUtil.set("container")).getObject(sortId, String.class); - sortToContainer.put(sortId, containerId); - - return containerId; - } -} diff --git a/tcrdb/src/org/labkey/tcrdb/TCRdbBulkImportNavItem.java b/tcrdb/src/org/labkey/tcrdb/TCRdbBulkImportNavItem.java deleted file mode 100644 index 4203f5097..000000000 --- a/tcrdb/src/org/labkey/tcrdb/TCRdbBulkImportNavItem.java +++ /dev/null @@ -1,53 +0,0 @@ -package org.labkey.tcrdb; - -import org.labkey.api.data.Container; -import org.labkey.api.laboratory.AbstractImportingNavItem; -import org.labkey.api.laboratory.DataProvider; -import org.labkey.api.laboratory.LaboratoryService; -import org.labkey.api.module.ModuleLoader; -import org.labkey.api.query.DetailsURL; -import org.labkey.api.security.User; -import org.labkey.api.view.ActionURL; - -public class TCRdbBulkImportNavItem extends AbstractImportingNavItem -{ - public static final String NAME = "TCR/10x Import"; - - private String _url; - - public TCRdbBulkImportNavItem(DataProvider provider, String label, LaboratoryService.NavItemCategory itemType, String reportCategory, String url) - { - super(provider, NAME, label, itemType, (reportCategory == null ? "TCRdb" : reportCategory)); - _url = url; - } - - @Override - public ActionURL getImportUrl(Container c, User u) - { - return DetailsURL.fromString(_url).getActionURL(); - } - - @Override - public ActionURL getSearchUrl(Container c, User u) - { - return null; - } - - @Override - public ActionURL getBrowseUrl(Container c, User u) - { - return null; - } - - @Override - public boolean isImportIntoWorkbooks(Container c, User u) - { - return true; - } - - @Override - public boolean getDefaultVisibility(Container c, User u) - { - return getTargetContainer(c).getActiveModules().contains(ModuleLoader.getInstance().getModule(TCRdbModule.NAME)); - } -} diff --git a/tcrdb/src/org/labkey/tcrdb/TCRdbController.java b/tcrdb/src/org/labkey/tcrdb/TCRdbController.java index 250069efd..7647b1d63 100644 --- a/tcrdb/src/org/labkey/tcrdb/TCRdbController.java +++ b/tcrdb/src/org/labkey/tcrdb/TCRdbController.java @@ -22,32 +22,19 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.jetbrains.annotations.NotNull; -import org.json.JSONArray; -import org.labkey.api.action.ApiSimpleResponse; -import org.labkey.api.action.ApiUsageException; import org.labkey.api.action.ConfirmAction; import org.labkey.api.action.ExportAction; -import org.labkey.api.action.MutatingApiAction; -import org.labkey.api.action.ReadOnlyApiAction; -import org.labkey.api.action.SimpleApiJsonForm; import org.labkey.api.action.SimpleViewAction; import org.labkey.api.action.SpringActionController; -import org.labkey.api.collections.CaseInsensitiveHashMap; import org.labkey.api.data.ColumnInfo; import org.labkey.api.data.CompareType; import org.labkey.api.data.Container; -import org.labkey.api.data.ContainerManager; -import org.labkey.api.data.ContainerType; -import org.labkey.api.data.DbScope; -import org.labkey.api.data.Selector; import org.labkey.api.data.SimpleFilter; -import org.labkey.api.data.StopIteratingException; import org.labkey.api.data.TableInfo; import org.labkey.api.data.TableSelector; import org.labkey.api.exp.api.ExpData; import org.labkey.api.exp.api.ExperimentService; import org.labkey.api.pipeline.PipelineJobException; -import org.labkey.api.query.BatchValidationException; import org.labkey.api.query.FieldKey; import org.labkey.api.query.QueryAction; import org.labkey.api.query.QueryService; @@ -56,7 +43,6 @@ import org.labkey.api.security.IgnoresTermsOfUse; import org.labkey.api.security.RequiresPermission; import org.labkey.api.security.permissions.AdminPermission; -import org.labkey.api.security.permissions.InsertPermission; import org.labkey.api.security.permissions.ReadPermission; import org.labkey.api.sequenceanalysis.RefNtSequenceModel; import org.labkey.api.sequenceanalysis.SequenceAnalysisService; @@ -78,7 +64,6 @@ import java.io.File; import java.io.FileInputStream; import java.io.StringWriter; -import java.sql.SQLException; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Arrays; @@ -967,239 +952,6 @@ public void setQueryName(String queryName) } } - @RequiresPermission(InsertPermission.class) - public static class ImportTenXAction extends MutatingApiAction - { - @Override - public Object execute(SimpleApiJsonForm form, BindException errors) throws Exception - { - List> stimRows = parseRows(form, "stimRows", getContainer()); - List> sortRows = parseRows(form, "sortRows", getContainer()); - List> readsetRows = parseRows(form, "readsetRows", getContainer()); - List> cDNARows = parseRows(form, "cDNARows", getContainer()); - - UserSchema tcrdb = QueryService.get().getUserSchema(getUser(), getContainer(), TCRdbSchema.NAME); - UserSchema sequenceAnalysis = QueryService.get().getUserSchema(getUser(), getContainer(), "sequenceanalysis"); - - try (DbScope.Transaction transaction = DbScope.getLabKeyScope().ensureTransaction()) - { - BatchValidationException bve = new BatchValidationException(); - - Map stimMap = new HashMap<>(); - final List> stimRowsToInsert = new ArrayList<>(); - stimRows.forEach(r -> { - if (r.get("objectId") == null) - { - throw new ApiUsageException("Missing objectId for stim row"); - } - - if (r.get("rowId") != null && StringUtils.trimToNull(r.get("rowId").toString()) != null) - { - stimMap.put((String) r.get("objectId"), (Integer) r.get("rowId")); - } - else - { - stimRowsToInsert.add(r); - } - }); - - - List> insertedStimRows = tcrdb.getTable(TCRdbSchema.TABLE_STIMS, null).getUpdateService().insertRows(getUser(), getContainer(), stimRowsToInsert, bve, null, new HashMap<>()); - if (bve.hasErrors()) - { - throw bve; - } - - insertedStimRows.forEach(r -> { - if (r.get("rowId") == null) - { - throw new ApiUsageException("Missing rowId for inserted stim row"); - } - - stimMap.put((String) r.get("objectId"), (Integer) r.get("rowId")); - }); - - Map sortMap = new HashMap<>(); - final List> sortRowsToInsert = new ArrayList<>(); - sortRows.forEach(r -> { - if (stimMap.get(r.get("stimGUID")) == null) - { - throw new ApiUsageException("Unable to find stimId for row"); - } - - if (r.get("rowId") != null && StringUtils.trimToNull(r.get("rowId").toString()) != null) - { - sortMap.put((String) r.get("objectId"), (Integer) r.get("rowId")); - } - else - { - r.put("stimId", stimMap.get(r.get("stimGUID"))); - sortRowsToInsert.add(r); - } - }); - - sortRows = tcrdb.getTable(TCRdbSchema.TABLE_SORTS, null).getUpdateService().insertRows(getUser(), getContainer(), sortRowsToInsert, bve, null, new HashMap<>()); - if (bve.hasErrors()) - { - throw bve; - } - - sortRows.forEach(r -> { - if (r.get("objectId") == null) - { - throw new ApiUsageException("Missing objectId for sort row"); - } - - sortMap.put((String) r.get("objectId"), (Integer) r.get("rowId")); - }); - - readsetRows = sequenceAnalysis.getTable("sequence_readsets", null).getUpdateService().insertRows(getUser(), getContainer(), readsetRows, bve, null, new HashMap<>()); - if (bve.hasErrors()) - { - throw bve; - } - - Map readsetMap = new HashMap<>(); - readsetRows.forEach(r -> { - if (r.get("objectId") == null) - { - throw new ApiUsageException("Missing objectId for readset row"); - } - - readsetMap.put((String)r.get("objectId"), (Integer)r.get("rowId")); - }); - - cDNARows.forEach(r -> { - if (sortMap.get(r.get("sortGUID")) == null) - { - throw new ApiUsageException("Unable to find sortId for row"); - } - r.put("sortId", sortMap.get((String)r.get("sortGUID"))); - }); - cDNARows.forEach(r -> r.put("readsetId", readsetMap.get((String)r.get("readsetGUID")))); - cDNARows.forEach(r -> r.put("hashingReadsetId", readsetMap.get((String)r.get("hashingReadsetGUID")))); - cDNARows.forEach(r -> r.put("enrichedReadsetId", readsetMap.get((String)r.get("enrichedReadsetGUID")))); - cDNARows.forEach(r -> r.put("citeseqReadsetId", readsetMap.get((String)r.get("citeseqReadsetGUID")))); - tcrdb.getTable(TCRdbSchema.TABLE_CDNAS, null).getUpdateService().insertRows(getUser(), getContainer(), cDNARows, bve, null, new HashMap<>()); - if (bve.hasErrors()) - { - throw bve; - } - - transaction.commit(); - } - - return new ApiSimpleResponse("success", true); - } - } - - private static List> parseRows(SimpleApiJsonForm form, String propName, Container container) throws ApiUsageException - { - if (!form.getJsonObject().containsKey(propName)) - { - throw new ApiUsageException("Missing property: " + propName); - } - - JSONArray arr = form.getJsonObject().getJSONArray(propName); - - List> ret = new ArrayList<>(); - Arrays.stream(arr.toJSONObjectArray()).forEach(m -> { - Map map = new CaseInsensitiveHashMap<>(); - map.putAll(m); - - if (map.containsKey("workbook")) - { - Container parent = container.getContainerFor(ContainerType.DataType.folderManagement); - Container workbook = ContainerManager.getForPath(parent.getPath() + "/" + map.get("workbook").toString()); - if (workbook == null) - { - throw new IllegalArgumentException("Unable to identify matching workbook for: " + map.get("workbook")); - } - - map.put("container", workbook == null ? null : workbook.getId()); - } - ret.add(map); - }); - - return ret; - } - - @RequiresPermission(ReadPermission.class) - public static class GetMatchingStimsAction extends ReadOnlyApiAction - { - final List FIELDS = Arrays.asList("animalId", "date", "stim", "treatment", "tissue"); - - @Override - public Object execute(SimpleApiJsonForm form, BindException errors) throws Exception - { - ApiSimpleResponse resp = new ApiSimpleResponse(); - - List> stimRows = parseRows(form, "stimRows", getContainer()); - - UserSchema us = QueryService.get().getUserSchema(getUser(), getContainer(), TCRdbSchema.NAME); - if (us == null) - { - throw new ApiUsageException("Unable to find schema: " + TCRdbSchema.NAME); - } - - TableInfo ti = us.getTable(TCRdbSchema.TABLE_STIMS, null); - TableInfo tiSort = us.getTable(TCRdbSchema.TABLE_SORTS, null); - - List retErrors = new ArrayList<>(); - Map stimRowMap = new HashMap<>(); - Map sortRowMap = new HashMap<>(); - stimRows.forEach(r -> { - List keys = new ArrayList<>(); - SimpleFilter filter = new SimpleFilter(); - FIELDS.forEach(f -> { - if (r.get(f) != null) - { - filter.addCondition(FieldKey.fromString(f), r.get(f), ("date".equals(f) ? CompareType.DATE_EQUAL : CompareType.EQUAL)); - keys.add(r.get(f)); - } - }); - - if (!filter.isEmpty()) - { - TableSelector ts = new TableSelector(ti, PageFlowUtil.set("rowId"), filter, null); - long count = ts.getRowCount(); - if (count == 1) - { - int rowId = ts.getObject(Integer.class); - stimRowMap.put(r.get("objectId"), rowId); - - if (r.get("population") != null) - { - SimpleFilter sortFilter = new SimpleFilter(FieldKey.fromString("stimId"), rowId); - sortFilter.addCondition(FieldKey.fromString("population"), r.get("population")); - TableSelector tsSort = new TableSelector(tiSort, PageFlowUtil.set("rowId"), sortFilter, null); - long countSort = tsSort.getRowCount(); - if (countSort == 1) - { - int sortRowId = tsSort.getObject(Integer.class); - sortRowMap.put(r.get("objectId"), sortRowId); - } - else if (countSort > 1) - { - retErrors.add("More than one matching sort found: " + StringUtils.join(keys, "|") + "|" + r.get("population")); - } - } - } - else if (count > 1 && filter.getClauses().size() == FIELDS.size()) - { - retErrors.add("More than one matching stim found: " + StringUtils.join(keys, "|")); - } - } - }); - - resp.put("stimMap", stimRowMap); - resp.put("sortMap", sortRowMap); - resp.put("recordErrors", retErrors); - - return resp; - } - } - @RequiresPermission(AdminPermission.class) public class CreateGenomeFromMixcrAction extends ConfirmAction { @@ -1241,7 +993,7 @@ public void validateCommand(CreateGenomeFromMixcrForm form, Errors errors) @Override public URLHelper getSuccessURL(CreateGenomeFromMixcrForm form) { - return QueryService.get().urlFor(getUser(), getContainer(), QueryAction.executeQuery, TCRdbSchema.NAME, TCRdbSchema.TABLE_LIBRARIES); + return QueryService.get().urlFor(getUser(), getContainer(), QueryAction.executeQuery, TCRdbSchema.NAME, TCRdbSchema.TABLE_MIXCR_LIBRARIES); } } diff --git a/tcrdb/src/org/labkey/tcrdb/TCRdbImportNavItem.java b/tcrdb/src/org/labkey/tcrdb/TCRdbImportNavItem.java deleted file mode 100644 index dc74a4fbe..000000000 --- a/tcrdb/src/org/labkey/tcrdb/TCRdbImportNavItem.java +++ /dev/null @@ -1,50 +0,0 @@ -package org.labkey.tcrdb; - -import org.labkey.api.data.Container; -import org.labkey.api.laboratory.AbstractImportingNavItem; -import org.labkey.api.laboratory.DataProvider; -import org.labkey.api.laboratory.LaboratoryService; -import org.labkey.api.module.ModuleLoader; -import org.labkey.api.query.DetailsURL; -import org.labkey.api.security.User; -import org.labkey.api.view.ActionURL; - -public class TCRdbImportNavItem extends AbstractImportingNavItem -{ - public static final String NAME = "TCR Sorts/Stims"; - - public TCRdbImportNavItem(DataProvider provider, String label, LaboratoryService.NavItemCategory itemType, String reportCategory) - { - super(provider, NAME, label, itemType, (reportCategory == null ? "TCRdb" : reportCategory)); - } - - @Override - public ActionURL getImportUrl(Container c, User u) - { - return DetailsURL.fromString("tcrdb/stimDashboard.view").getActionURL(); - } - - @Override - public ActionURL getSearchUrl(Container c, User u) - { - return null; - } - - @Override - public ActionURL getBrowseUrl(Container c, User u) - { - return DetailsURL.fromString("tcrdb/stimDashboard.view").getActionURL(); - } - - @Override - public boolean isImportIntoWorkbooks(Container c, User u) - { - return true; - } - - @Override - public boolean getDefaultVisibility(Container c, User u) - { - return getTargetContainer(c).getActiveModules().contains(ModuleLoader.getInstance().getModule(TCRdbModule.NAME)); - } -} diff --git a/tcrdb/src/org/labkey/tcrdb/TCRdbManager.java b/tcrdb/src/org/labkey/tcrdb/TCRdbManager.java index 79b3e06ac..efc318e7e 100644 --- a/tcrdb/src/org/labkey/tcrdb/TCRdbManager.java +++ b/tcrdb/src/org/labkey/tcrdb/TCRdbManager.java @@ -69,7 +69,7 @@ public static TCRdbManager get() public void createGenomeFromMixcrDb(int mixcrRowId, User u, Container c) throws Exception { - MixcrLibrary lib = new TableSelector(TCRdbSchema.getInstance().getSchema().getTable(TCRdbSchema.TABLE_LIBRARIES)).getObject(mixcrRowId, MixcrLibrary.class); + MixcrLibrary lib = new TableSelector(TCRdbSchema.getInstance().getSchema().getTable(TCRdbSchema.TABLE_MIXCR_LIBRARIES)).getObject(mixcrRowId, MixcrLibrary.class); if (lib == null) { throw new IllegalArgumentException("Unable to find MiXCR library: " + mixcrRowId); @@ -308,7 +308,7 @@ public void onCreate(Container c, User u, Logger log, int genomeId) { if (_mixcrId != null) { - TableInfo ti = QueryService.get().getUserSchema(u, c, TCRdbSchema.NAME).getTable(TCRdbSchema.TABLE_LIBRARIES); + TableInfo ti = QueryService.get().getUserSchema(u, c, TCRdbSchema.NAME).getTable(TCRdbSchema.TABLE_MIXCR_LIBRARIES); List> rows = new ArrayList<>(); List> oldKeys = new ArrayList<>(); Map row = new CaseInsensitiveHashMap<>(); diff --git a/tcrdb/src/org/labkey/tcrdb/TCRdbModule.java b/tcrdb/src/org/labkey/tcrdb/TCRdbModule.java index 0b1796b56..6d5f552f6 100644 --- a/tcrdb/src/org/labkey/tcrdb/TCRdbModule.java +++ b/tcrdb/src/org/labkey/tcrdb/TCRdbModule.java @@ -27,13 +27,8 @@ import org.labkey.api.module.ModuleContext; import org.labkey.api.sequenceanalysis.SequenceAnalysisService; import org.labkey.api.sequenceanalysis.pipeline.SequencePipelineService; -import org.labkey.tcrdb.pipeline.CellRangerCellHashingHandler; -import org.labkey.tcrdb.pipeline.CellRangerSeuratHandler; import org.labkey.tcrdb.pipeline.CellRangerVDJCellHashingHandler; -import org.labkey.tcrdb.pipeline.CellRangerVDJWrapper; import org.labkey.tcrdb.pipeline.MiXCRAnalysis; -import org.labkey.tcrdb.pipeline.SeuratCellHashingHandler; -import org.labkey.tcrdb.pipeline.SeuratCiteSeqHandler; import java.util.Collection; import java.util.Collections; @@ -51,7 +46,7 @@ public String getName() @Override public Double getSchemaVersion() { - return 15.51; + return 15.52; } @Override @@ -74,17 +69,13 @@ protected void doStartupAfterSpringConfig(ModuleContext moduleContext) LaboratoryService.get().registerDataProvider(new TCRdbProvider(this)); SequenceAnalysisService.get().registerDataProvider(new TCRdbProvider(this)); - LaboratoryService.get().registerTableCustomizer(this, TCRdbTableCustomizer.class, "sequenceanalysis", "sequence_readsets"); - LaboratoryService.get().registerTableCustomizer(this, TCRdbTableCustomizer.class, "sequenceanalysis", "sequence_analyses"); - LaboratoryService.get().registerTableCustomizer(this, TCRdbTableCustomizer.class, TCRdbSchema.NAME, TCRdbSchema.TABLE_STIMS); - LaboratoryService.get().registerTableCustomizer(this, TCRdbTableCustomizer.class, TCRdbSchema.NAME, TCRdbSchema.TABLE_SORTS); - LaboratoryService.get().registerTableCustomizer(this, TCRdbTableCustomizer.class, TCRdbSchema.NAME, TCRdbSchema.TABLE_CDNAS); + LaboratoryService.get().registerTableCustomizer(this, TCRdbTableCustomizer.class, TCRdbSchema.SEQUENCE_ANALYSIS, "sequence_readsets"); + LaboratoryService.get().registerTableCustomizer(this, TCRdbTableCustomizer.class, TCRdbSchema.SEQUENCE_ANALYSIS, "sequence_analyses"); LaboratoryService.get().registerTableCustomizer(this, TCRdbTableCustomizer.class, TCRdbSchema.NAME, TCRdbSchema.TABLE_CLONES); + LaboratoryService.get().registerTableCustomizer(this, TCRdbTableCustomizer.class, TCRdbSchema.SINGLE_CELL, TCRdbSchema.TABLE_CDNAS); - LDKService.get().registerQueryButton(new ChangeStatusButton(), "tcrdb", "stims"); + LDKService.get().registerQueryButton(new ChangeStatusButton(), TCRdbSchema.SINGLE_CELL, "samples"); - LDKService.get().registerQueryButton(new ShowBulkEditButton(this, TCRdbSchema.NAME, TCRdbSchema.TABLE_CDNAS), TCRdbSchema.NAME, TCRdbSchema.TABLE_CDNAS); - LDKService.get().registerQueryButton(new ShowBulkEditButton(this, TCRdbSchema.NAME, TCRdbSchema.TABLE_SORTS), TCRdbSchema.NAME, TCRdbSchema.TABLE_SORTS); LDKService.get().registerQueryButton(new ShowBulkEditButton(this, TCRdbSchema.NAME, TCRdbSchema.TABLE_CLONES), TCRdbSchema.NAME, TCRdbSchema.TABLE_CLONES); //register resources @@ -118,13 +109,7 @@ public PipelineStartup() else { SequencePipelineService.get().registerPipelineStep(new MiXCRAnalysis.Provider()); - SequencePipelineService.get().registerPipelineStep(new CellRangerVDJWrapper.VDJProvider()); - - SequenceAnalysisService.get().registerFileHandler(new CellRangerCellHashingHandler()); SequenceAnalysisService.get().registerFileHandler(new CellRangerVDJCellHashingHandler()); - SequenceAnalysisService.get().registerFileHandler(new SeuratCellHashingHandler()); - SequenceAnalysisService.get().registerFileHandler(new SeuratCiteSeqHandler()); - SequenceAnalysisService.get().registerFileHandler(new CellRangerSeuratHandler()); _hasRegistered = true; } diff --git a/tcrdb/src/org/labkey/tcrdb/TCRdbProvider.java b/tcrdb/src/org/labkey/tcrdb/TCRdbProvider.java index 1e3f0d31c..0359e45a7 100644 --- a/tcrdb/src/org/labkey/tcrdb/TCRdbProvider.java +++ b/tcrdb/src/org/labkey/tcrdb/TCRdbProvider.java @@ -3,20 +3,13 @@ import org.json.JSONObject; import org.labkey.api.data.Container; import org.labkey.api.data.ContainerManager; -import org.labkey.api.laboratory.DetailsUrlWithoutLabelNavItem; import org.labkey.api.laboratory.LaboratoryService; import org.labkey.api.laboratory.NavItem; import org.labkey.api.laboratory.QueryCountNavItem; import org.labkey.api.laboratory.QueryImportNavItem; -import org.labkey.api.laboratory.QueryTabbedReportItem; -import org.labkey.api.laboratory.SimpleSettingsItem; import org.labkey.api.laboratory.SummaryNavItem; -import org.labkey.api.laboratory.TabbedReportItem; import org.labkey.api.ldk.table.QueryCache; import org.labkey.api.module.Module; -import org.labkey.api.module.ModuleLoader; -import org.labkey.api.query.DetailsURL; -import org.labkey.api.query.FieldKey; import org.labkey.api.security.User; import org.labkey.api.sequenceanalysis.AbstractSequenceDataProvider; import org.labkey.api.view.ActionURL; @@ -60,15 +53,9 @@ public ActionURL getInstructionsUrl(Container c, User u) } @Override - public List getMiscItems(Container c, User u) + public List getSubjectIdSummary(Container c, User u, String subjectId) { - List items = new ArrayList<>(); - if (c.getActiveModules().contains(ModuleLoader.getInstance().getModule(TCRdbModule.class))) - { - items.add(new DetailsUrlWithoutLabelNavItem(this, "Export 10x Library Information", DetailsURL.fromString("tcrdb/libraryExport.view"), LaboratoryService.NavItemCategory.misc, NAME)); - } - - return items; + return Collections.emptyList(); } @Override @@ -81,27 +68,7 @@ public List getDataNavItems(Container c, User u) return Collections.emptyList(); } - TCRdbImportNavItem item = new TCRdbImportNavItem(this, "TCR Stims/Sorts (SMART-seq)", LaboratoryService.NavItemCategory.data, NAME); - item.setQueryCache(cache); - items.add(item); - - TCRdbBulkImportNavItem item2 = new TCRdbBulkImportNavItem(this, "TCR/10x Import 1: Stims/cDNA", LaboratoryService.NavItemCategory.data, NAME, "tcrdb/poolImport.view"); - item2.setQueryCache(cache); - items.add(item2); - - TCRdbBulkImportNavItem item3 = new TCRdbBulkImportNavItem(this, "TCR/10x Import 2: Libraries/Readsets", LaboratoryService.NavItemCategory.data, NAME, "tcrdb/cDNAImport.view"); - item3.setQueryCache(cache); - items.add(item3); - items.add(new QueryImportNavItem(this, TCRdbSchema.NAME, TCRdbSchema.TABLE_CLONES, "TCR Clones", LaboratoryService.NavItemCategory.data, NAME, cache)); - items.add(new QueryImportNavItem(this, TCRdbSchema.NAME, TCRdbSchema.TABLE_CDNAS, "TCR cDNA Libraries", LaboratoryService.NavItemCategory.data, NAME, cache){ - @Override - public ActionURL getImportUrl(Container c, User u) - { - return null; - } - }); - return Collections.unmodifiableList(items); } @@ -117,11 +84,7 @@ public List getSettingsItems(Container c, User u) List items = new ArrayList<>(); if (ContainerManager.getSharedContainer().equals(c)) { - items.add(new QueryImportNavItem(this, ContainerManager.getSharedContainer(), TCRdbSchema.NAME, TCRdbSchema.TABLE_LIBRARIES, LaboratoryService.NavItemCategory.settings, "MiXCR Libraries", NAME)); - } - else - { - items.add(new SimpleSettingsItem(this, TCRdbSchema.NAME, "peptides", NAME, "Peptides/Stims")); + items.add(new QueryImportNavItem(this, ContainerManager.getSharedContainer(), TCRdbSchema.NAME, TCRdbSchema.TABLE_MIXCR_LIBRARIES, LaboratoryService.NavItemCategory.settings, "MiXCR Libraries", NAME)); } return items; @@ -148,56 +111,6 @@ public Module getOwningModule() @Override public List getSummary(Container c, User u) { - List items = new ArrayList<>(); - - items.add(new QueryCountNavItem(this, TCRdbSchema.NAME, "stims", LaboratoryService.NavItemCategory.data, LaboratoryService.NavItemCategory.data.name(), "TCR Stims")); - items.add(new QueryCountNavItem(this, TCRdbSchema.NAME, "sorts", LaboratoryService.NavItemCategory.data, LaboratoryService.NavItemCategory.data.name(), "TCR Sorts")); - items.add(new QueryCountNavItem(this, TCRdbSchema.NAME, "cdnas", LaboratoryService.NavItemCategory.data, LaboratoryService.NavItemCategory.data.name(), "TCR cDNA Libraries")); - items.add(new QueryCountNavItem(this, TCRdbSchema.NAME, "clones", LaboratoryService.NavItemCategory.data, LaboratoryService.NavItemCategory.data.name(), "TCR Clones")); - - return Collections.unmodifiableList(items); - } - - @Override - public List getSubjectIdSummary(Container c, User u, String subjectId) - { - return Collections.emptyList(); - } - - @Override - public List getTabbedReportItems(Container c, User u) - { - if (!c.getActiveModules().contains(getOwningModule())) - { - return Collections.emptyList(); - } - - List items = new ArrayList<>(); - - NavItem owner = getDataNavItems(c, u).get(0); - String category = NAME; - QueryCache cache = new QueryCache(); - - TabbedReportItem stims = new QueryTabbedReportItem(cache, this, TCRdbSchema.NAME, TCRdbSchema.TABLE_STIMS, "TCR Stims/Blood Draws", category); - stims.setOwnerKey(owner.getPropertyManagerKey()); - items.add(stims); - - TabbedReportItem sorts = new QueryTabbedReportItem(cache, this, TCRdbSchema.NAME, TCRdbSchema.TABLE_SORTS, "TCR Sorts", category); - sorts.setSubjectIdFieldKey(FieldKey.fromString("stimId/animalId")); - sorts.setSampleDateFieldKey(FieldKey.fromString("stimId/date")); - sorts.setAllProjectsFieldKey(FieldKey.fromString("stimId/allProjectsPivot")); - sorts.setOverlappingProjectsFieldKey(FieldKey.fromString("stimId/overlappingProjectsPivot")); - sorts.setOwnerKey(owner.getPropertyManagerKey()); - items.add(sorts); - - TabbedReportItem cdnas = new QueryTabbedReportItem(cache, this, TCRdbSchema.NAME, TCRdbSchema.TABLE_CDNAS, "TCR cDNA Libraries", category); - cdnas.setSubjectIdFieldKey(FieldKey.fromString("sortId/stimId/animalId")); - cdnas.setSampleDateFieldKey(FieldKey.fromString("sortId/stimId/date")); - cdnas.setAllProjectsFieldKey(FieldKey.fromString("sortId/stimId/allProjectsPivot")); - cdnas.setOverlappingProjectsFieldKey(FieldKey.fromString("sortId/stimId/overlappingProjectsPivot")); - cdnas.setOwnerKey(owner.getPropertyManagerKey()); - items.add(cdnas); - - return items; + return Collections.singletonList(new QueryCountNavItem(this, TCRdbSchema.NAME, TCRdbSchema.TABLE_CLONES, LaboratoryService.NavItemCategory.data, LaboratoryService.NavItemCategory.data.name(), "TCR Clones")); } } diff --git a/tcrdb/src/org/labkey/tcrdb/TCRdbSchema.java b/tcrdb/src/org/labkey/tcrdb/TCRdbSchema.java index 0336e338c..32a0ca98f 100644 --- a/tcrdb/src/org/labkey/tcrdb/TCRdbSchema.java +++ b/tcrdb/src/org/labkey/tcrdb/TCRdbSchema.java @@ -24,16 +24,13 @@ public class TCRdbSchema { private static final TCRdbSchema _instance = new TCRdbSchema(); public static final String NAME = "tcrdb"; - public static final String SEQUENCE_ANALYSIS = "sequenceanalysis"; - public static final String TABLE_LIBRARIES = "mixcr_libraries"; - public static final String TABLE_SORTS = "sorts"; - public static final String TABLE_STIMS = "stims"; - public static final String TABLE_CDNAS = "cdnas"; + public static final String TABLE_MIXCR_LIBRARIES = "mixcr_libraries"; public static final String TABLE_CLONES = "clones"; - public static final String TABLE_CITE_SEQ_ANTIBODIES = "citeseq_antibodies"; - public static final String TABLE_CITE_SEQ_PANELS = "citeseq_panels"; - public static final String TABLE_PROCESSING = "plate_processing"; + + public static final String SEQUENCE_ANALYSIS = "sequenceanalysis"; + public static final String SINGLE_CELL = "singlecell"; + public static final String TABLE_CDNAS = "cdna_libraries"; public static TCRdbSchema getInstance() { diff --git a/tcrdb/src/org/labkey/tcrdb/TCRdbTableCustomizer.java b/tcrdb/src/org/labkey/tcrdb/TCRdbTableCustomizer.java index 9c7059a83..cf1a8e14e 100644 --- a/tcrdb/src/org/labkey/tcrdb/TCRdbTableCustomizer.java +++ b/tcrdb/src/org/labkey/tcrdb/TCRdbTableCustomizer.java @@ -19,9 +19,7 @@ import org.labkey.api.query.DetailsURL; import org.labkey.api.query.ExprColumn; import org.labkey.api.query.FieldKey; -import org.labkey.api.query.QueryForeignKey; import org.labkey.api.query.QueryService; -import org.labkey.api.query.UserSchema; import java.util.Arrays; import java.util.List; @@ -34,27 +32,19 @@ public void customize(TableInfo table) if (table instanceof AbstractTableInfo) { AbstractTableInfo ti = (AbstractTableInfo) table; - if (matches(ti, "sequenceanalysis", "sequence_analyses")) + if (matches(ti, TCRdbSchema.SEQUENCE_ANALYSIS, "sequence_analyses")) { addAssayFieldsToAnalyses(ti); } - else if (matches(ti, "sequenceanalysis", "sequence_readsets")) + else if (matches(ti, TCRdbSchema.SEQUENCE_ANALYSIS, "sequence_readsets")) { customizeReadsets(ti); } - else if (matches(ti, "tcrdb", "stims")) - { - customizeStims(ti); - } - else if (matches(ti, "tcrdb", "sorts")) - { - customizeSorts(ti); - } - else if (matches(ti, "tcrdb", "cdnas")) + else if (matches(ti, TCRdbSchema.SINGLE_CELL, TCRdbSchema.TABLE_CDNAS)) { customizeCdnas(ti); } - else if (matches(ti, "tcrdb", "clones")) + else if (matches(ti, TCRdbSchema.NAME, TCRdbSchema.TABLE_CLONES)) { customizeClones(ti); } @@ -67,130 +57,12 @@ else if (ti instanceof AssayResultTable) private void customizeCdnas(AbstractTableInfo ti) { - String name = "hasReadsetWithData"; - if (ti.getColumn(name) == null) - { - SQLFragment sql = new SQLFragment("CASE " + - " WHEN (select count(*) as expr FROM sequenceanalysis.sequence_readsets r JOIN sequenceanalysis.readdata d ON (r.rowid = d.readset) WHERE r.rowid = " + ExprColumn.STR_TABLE_ALIAS + ".readsetId) > 0 THEN " + ti.getSqlDialect().getBooleanTRUE() + - " WHEN (select count(*) as expr FROM sequenceanalysis.sequence_readsets r JOIN sequenceanalysis.readdata d ON (r.rowid = d.readset) WHERE r.rowid = " + ExprColumn.STR_TABLE_ALIAS + ".enrichedReadsetId) > 0 THEN " + ti.getSqlDialect().getBooleanTRUE() + - " ELSE " + ti.getSqlDialect().getBooleanFALSE() + " END"); - - ExprColumn newCol = new ExprColumn(ti, name, sql, JdbcType.BOOLEAN, ti.getColumn("readsetId"), ti.getColumn("enrichedReadsetId")); - newCol.setLabel("Has Any Readset With Data?"); - ti.addColumn(newCol); - } - - String name2 = "allReadsetsHaveData"; - if (ti.getColumn(name2) == null) - { - SQLFragment sql = new SQLFragment("CASE " + - " WHEN (" + ExprColumn.STR_TABLE_ALIAS + ".readsetId IS NOT NULL AND (select count(*) as expr FROM sequenceanalysis.sequence_readsets r JOIN sequenceanalysis.readdata d ON (r.rowid = d.readset) WHERE r.rowid = " + ExprColumn.STR_TABLE_ALIAS + ".readsetId) = 0) THEN " + ti.getSqlDialect().getBooleanFALSE() + - " WHEN (" + ExprColumn.STR_TABLE_ALIAS + ".enrichedReadsetId IS NOT NULL AND (select count(*) as expr FROM sequenceanalysis.sequence_readsets r JOIN sequenceanalysis.readdata d ON (r.rowid = d.readset) WHERE r.rowid = " + ExprColumn.STR_TABLE_ALIAS + ".enrichedReadsetId) = 0) THEN " + ti.getSqlDialect().getBooleanFALSE() + - " ELSE " + ti.getSqlDialect().getBooleanTRUE() + " END"); - - ExprColumn newCol = new ExprColumn(ti, name2, sql, JdbcType.BOOLEAN, ti.getColumn("readsetId"), ti.getColumn("enrichedReadsetId")); - newCol.setLabel("All Readsets Have Data?"); - ti.addColumn(newCol); - } - addAssayFieldsToCDnas(ti); - - LDKService.get().applyNaturalSort(ti, "plateId"); - } - - private void customizeSorts(AbstractTableInfo ti) - { - LDKService.get().applyNaturalSort(ti, "plateId"); - LDKService.get().applyNaturalSort(ti, "hto"); - - String name = "numLibraries"; - if (ti.getColumn(name) == null) - { - DetailsURL details = DetailsURL.fromString("/query/executeQuery.view?schemaName=tcrdb&query.queryName=cdnas&query.sortId~eq=${rowid}", (ti.getUserSchema().getContainer().isWorkbook() ? ti.getUserSchema().getContainer().getParent() : ti.getUserSchema().getContainer())); - - SQLFragment sql = new SQLFragment("(select count(*) as expr FROM " + TCRdbSchema.NAME + "." + TCRdbSchema.TABLE_CDNAS + " s WHERE s.sortId = " + ExprColumn.STR_TABLE_ALIAS + ".rowid)"); - ExprColumn newCol = new ExprColumn(ti, name, sql, JdbcType.INTEGER, ti.getColumn("rowid")); - newCol.setLabel("# cDNA Libraries"); - newCol.setURL(details); - ti.addColumn(newCol); - } - - name = "maxCellsForPlate"; - if (ti.getColumn(name) == null) - { - SQLFragment sql = new SQLFragment("(select count(*) as expr FROM " + TCRdbSchema.NAME + "." + TCRdbSchema.TABLE_SORTS + " s WHERE s.plateId = " + ExprColumn.STR_TABLE_ALIAS + ".plateId AND s.container = " + ExprColumn.STR_TABLE_ALIAS + ".container)"); - ExprColumn newCol = new ExprColumn(ti, name, sql, JdbcType.INTEGER, ti.getColumn("plateId"), ti.getColumn("container")); - newCol.setLabel("Max Cells/Well In Plate"); - ti.addColumn(newCol); - } - - name = "processingRequested"; - if (ti.getColumn(name) == null) - { - SQLFragment sql = new SQLFragment("(select ").append(ti.getSqlDialect().getGroupConcat(new SQLFragment("p.type"), true, true)).append(new SQLFragment(" as expr FROM " + TCRdbSchema.NAME + "." + TCRdbSchema.TABLE_PROCESSING + " p WHERE p.plateId = " + ExprColumn.STR_TABLE_ALIAS + ".plateId AND p.container = " + ExprColumn.STR_TABLE_ALIAS + ".container)")); - ExprColumn newCol = new ExprColumn(ti, name, sql, JdbcType.VARCHAR, ti.getColumn("plateId"), ti.getColumn("container")); - newCol.setLabel("Processing Requested"); - ti.addColumn(newCol); - } - } - - private void customizeStims(AbstractTableInfo ti) - { - String name = "numSorts"; - if (ti.getColumn(name) == null) - { - DetailsURL details = DetailsURL.fromString("/query/executeQuery.view?schemaName=tcrdb&query.queryName=sorts&query.stimId~eq=${rowid}", (ti.getUserSchema().getContainer().isWorkbook() ? ti.getUserSchema().getContainer().getParent() : ti.getUserSchema().getContainer())); - - SQLFragment sql = new SQLFragment("(select count(*) as expr FROM " + TCRdbSchema.NAME + "." + TCRdbSchema.TABLE_SORTS + " s WHERE s.stimId = " + ExprColumn.STR_TABLE_ALIAS + ".rowid)"); - ExprColumn newCol = new ExprColumn(ti, "numSorts", sql, JdbcType.INTEGER, ti.getColumn("rowid")); - newCol.setLabel("# Sorts"); - newCol.setURL(details); - ti.addColumn(newCol); - } - - name = "numLibraries"; - if (ti.getColumn(name) == null) - { - DetailsURL details = DetailsURL.fromString("/query/executeQuery.view?schemaName=tcrdb&query.queryName=cdnas&query.sortId/stimId~eq=${rowid}", (ti.getUserSchema().getContainer().isWorkbook() ? ti.getUserSchema().getContainer().getParent() : ti.getUserSchema().getContainer())); - - SQLFragment sql = new SQLFragment("(select count(c.rowid) as expr FROM " + TCRdbSchema.NAME + "." + TCRdbSchema.TABLE_SORTS + " so JOIN " + TCRdbSchema.NAME + "." + TCRdbSchema.TABLE_CDNAS + " c ON (so.rowid = c.sortId) WHERE so.stimId = " + ExprColumn.STR_TABLE_ALIAS + ".rowid)"); - ExprColumn newCol = new ExprColumn(ti, "numLibraries", sql, JdbcType.INTEGER, ti.getColumn("rowid")); - newCol.setLabel("# cDNA Libraries"); - newCol.setURL(details); - ti.addColumn(newCol); - } } private void customizeReadsets(AbstractTableInfo ti) { addAssayFieldsToTable(ti, "analysisId/readset", "LEFT JOIN sequenceanalysis.sequence_analyses a2 ON (a.analysisId = a2.rowId) WHERE a2.readset = " + ExprColumn.STR_TABLE_ALIAS + ".rowid", "rowid"); - - String name = "numTCRLibraries"; - if (ti.getColumn(name) == null) - { - SQLFragment sql = new SQLFragment("(select count(*) as expr FROM " + TCRdbSchema.NAME + "." + TCRdbSchema.TABLE_CDNAS + " c WHERE c.readsetId = " + ExprColumn.STR_TABLE_ALIAS + ".rowid OR c.enrichedReadsetId = " + ExprColumn.STR_TABLE_ALIAS + ".rowid)"); - ExprColumn newCol = new ExprColumn(ti, name, sql, JdbcType.INTEGER, ti.getColumn("rowid")); - newCol.setLabel("# TCR Libraries"); - ti.addColumn(newCol); - } - - String cDNA = "cDNA"; - if (ti.getColumn(cDNA) == null) - { - SQLFragment sql = new SQLFragment("(CASE" + - " WHEN ((select count(*) as expr FROM " + TCRdbSchema.NAME + "." + TCRdbSchema.TABLE_CDNAS + " c WHERE c.readsetId = " + ExprColumn.STR_TABLE_ALIAS + ".rowid OR c.enrichedReadsetId = " + ExprColumn.STR_TABLE_ALIAS + ".rowid) > 0) " + - " THEN (select max(c.rowid) FROM " + TCRdbSchema.NAME + "." + TCRdbSchema.TABLE_CDNAS + " c WHERE c.readsetId = " + ExprColumn.STR_TABLE_ALIAS + ".rowid OR c.enrichedReadsetId = " + ExprColumn.STR_TABLE_ALIAS + ".rowid) " + - " ELSE null " + - "END)"); - ExprColumn newCol = new ExprColumn(ti, cDNA, sql, JdbcType.INTEGER, ti.getColumn("rowid")); - newCol.setLabel("cDNA Library"); - UserSchema us = QueryService.get().getUserSchema(ti.getUserSchema().getUser(), (ti.getUserSchema().getContainer().isWorkbook() ? ti.getUserSchema().getContainer().getParent() : ti.getUserSchema().getContainer()), TCRdbSchema.NAME); - newCol.setFk(QueryForeignKey.from(us, ti.getContainerFilter()) - .table(TCRdbSchema.TABLE_CDNAS) - .key("rowid") - .display("rowid")); - ti.addColumn(newCol); - } } private void addAssayFieldsToAnalyses(AbstractTableInfo ti) diff --git a/tcrdb/src/org/labkey/tcrdb/TCRdbUserSchema.java b/tcrdb/src/org/labkey/tcrdb/TCRdbUserSchema.java index 402e11a85..188fcc112 100644 --- a/tcrdb/src/org/labkey/tcrdb/TCRdbUserSchema.java +++ b/tcrdb/src/org/labkey/tcrdb/TCRdbUserSchema.java @@ -6,7 +6,6 @@ import org.labkey.api.data.ContainerFilter; import org.labkey.api.data.DbSchema; import org.labkey.api.data.TableInfo; -import org.labkey.api.ldk.table.ContainerScopedTable; import org.labkey.api.ldk.table.SharedDataTable; import org.labkey.api.module.Module; import org.labkey.api.query.DefaultSchema; @@ -42,15 +41,11 @@ public QuerySchema createSchema(final DefaultSchema schema, Module module) @Nullable protected TableInfo createWrappedTable(String name, @NotNull TableInfo sourceTable, ContainerFilter cf) { - if (TCRdbSchema.TABLE_LIBRARIES.equalsIgnoreCase(name)) + if (TCRdbSchema.TABLE_MIXCR_LIBRARIES.equalsIgnoreCase(name)) { // TODO: assert cf is null or not default? return new SharedDataTable<>(this, sourceTable).init(); } - else if (TCRdbSchema.TABLE_CITE_SEQ_ANTIBODIES.equalsIgnoreCase(name)) - { - return new ContainerScopedTable<>(this, sourceTable, cf, "antibodyName").init(); - } return super.createWrappedTable(name, sourceTable, cf); } diff --git a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerCellHashingHandler.java b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerCellHashingHandler.java deleted file mode 100644 index 3cd2ecf90..000000000 --- a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerCellHashingHandler.java +++ /dev/null @@ -1,361 +0,0 @@ -package org.labkey.tcrdb.pipeline; - -import au.com.bytecode.opencsv.CSVReader; -import au.com.bytecode.opencsv.CSVWriter; -import htsjdk.samtools.util.IOUtil; -import org.json.JSONObject; -import org.labkey.api.module.ModuleLoader; -import org.labkey.api.pipeline.PipelineJob; -import org.labkey.api.pipeline.PipelineJobException; -import org.labkey.api.pipeline.RecordedAction; -import org.labkey.api.reader.Readers; -import org.labkey.api.sequenceanalysis.SequenceOutputFile; -import org.labkey.api.sequenceanalysis.model.Readset; -import org.labkey.api.sequenceanalysis.pipeline.AbstractParameterizedOutputHandler; -import org.labkey.api.sequenceanalysis.pipeline.DefaultPipelineStepOutput; -import org.labkey.api.sequenceanalysis.pipeline.PipelineStepOutput; -import org.labkey.api.sequenceanalysis.pipeline.SequenceAnalysisJobSupport; -import org.labkey.api.sequenceanalysis.pipeline.SequenceOutputHandler; -import org.labkey.api.sequenceanalysis.pipeline.SequencePipelineService; -import org.labkey.api.sequenceanalysis.pipeline.ToolParameterDescriptor; -import org.labkey.api.util.FileType; -import org.labkey.api.util.FileUtil; -import org.labkey.api.util.PageFlowUtil; -import org.labkey.api.writer.PrintWriters; -import org.labkey.tcrdb.TCRdbModule; - -import java.io.File; -import java.io.FileFilter; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashSet; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; - -public class CellRangerCellHashingHandler extends AbstractParameterizedOutputHandler -{ - private FileType _fileType = new FileType("cloupe", false); - public static String CATEGORY = "10x GEX Cell Hashing Calls"; - - public CellRangerCellHashingHandler() - { - super(ModuleLoader.getInstance().getModule(TCRdbModule.class), "CellRanger GEX/Cell Hashing", "This will run CiteSeqCount/MultiSeqClassifier to generate a sample-to-cellbarcode TSV based on the filtered barcodes from CellRanger.", new LinkedHashSet<>(PageFlowUtil.set("sequenceanalysis/field/CellRangerAggrTextarea.js")), getDefaultParams()); - } - - private static List getDefaultParams() - { - List ret = new ArrayList<>(getDefaultHashingParams(true)); - ret.add( - ToolParameterDescriptor.create("useOutputFileContainer", "Submit to Source File Workbook", "If checked, each job will be submitted to the same workbook as the input file, as opposed to submitting all jobs to the same workbook. This is primarily useful if submitting a large batch of files to process separately. This only applies if 'Run Separately' is selected.", "checkbox", new JSONObject(){{ - put("checked", true); - }}, true) - ); - - return ret; - } - - public static List getDefaultHashingParams(boolean includeExcludeFailedcDNA) - { - List ret = new ArrayList<>(Arrays.asList( - ToolParameterDescriptor.create("scanEditDistances", "Scan Edit Distances", "If checked, CITE-seq-count will be run using edit distances from 0-3 and the iteration with the highest singlets will be used.", "checkbox", new JSONObject(){{ - put("checked", false); - }}, false), - ToolParameterDescriptor.create("editDistance", "Edit Distance", null, "ldk-integerfield", null, 2), - ToolParameterDescriptor.create("minCountPerCell", "Min Reads/Cell", null, "ldk-integerfield", null, 5), - ToolParameterDescriptor.create("useSeurat", "Use Seurat Calling", "If checked, the seurat HTO calling algorithm will be used.", "checkbox", null, true), - ToolParameterDescriptor.create("useMultiSeq", "Use MultiSeq Calling", "If checked, the MultiSeq HTO calling algorithm will be used.", "checkbox", null, true) - )); - - if (includeExcludeFailedcDNA) - { - ret.add(ToolParameterDescriptor.create("excludeFailedcDNA", "Exclude Failed cDNA", "If selected, cDNAs with non-blank status fields will be omitted", "checkbox", null, true)); - } - - return ret; - } - - @Override - public boolean canProcess(SequenceOutputFile o) - { - return o.getFile() != null && _fileType.isType(o.getFile()); - } - - @Override - public boolean doRunRemote() - { - return true; - } - - @Override - public boolean doRunLocal() - { - return false; - } - - @Override - public SequenceOutputProcessor getProcessor() - { - return new CellRangerCellHashingHandler.Processor(); - } - - @Override - public boolean doSplitJobs() - { - return true; - } - - @Override - public boolean requiresSingleGenome() - { - return false; - } - - public class Processor implements SequenceOutputHandler.SequenceOutputProcessor - { - @Override - public void init(PipelineJob job, SequenceAnalysisJobSupport support, List inputFiles, JSONObject params, File outputDir, List actions, List outputsToCreate) throws UnsupportedOperationException, PipelineJobException - { - new CellRangerVDJUtils(job.getLogger(), outputDir).prepareHashingAndCiteSeqFilesIfNeeded(job, support, "readsetId", params.optBoolean("excludeFailedcDNA", true), true, false); - } - - @Override - public void processFilesOnWebserver(PipelineJob job, SequenceAnalysisJobSupport support, List inputFiles, JSONObject params, File outputDir, List actions, List outputsToCreate) throws UnsupportedOperationException, PipelineJobException - { - - } - - @Override - public void processFilesRemote(List inputFiles, SequenceOutputHandler.JobContext ctx) throws UnsupportedOperationException, PipelineJobException - { - RecordedAction action = new RecordedAction(getName()); - Map readsetToHashing = CellRangerVDJUtils.getCachedHashingReadsetMap(ctx.getSequenceSupport()); - ctx.getLogger().debug("total cached readset to hashing pairs: " + readsetToHashing.size()); - - for (SequenceOutputFile so : inputFiles) - { - ctx.getLogger().info("processing file: " + so.getName()); - - //find TSV: - File perCellTsv; - File barcodeDir = null; - for (String dirName : Arrays.asList("filtered_gene_bc_matrices", "filtered_feature_bc_matrix")) - { - File f = new File(so.getFile().getParentFile(), dirName); - if (f.exists()) - { - barcodeDir = f; - break; - } - } - - if (barcodeDir == null) - { - //this might be a re-analysis loupe directory. in this case, use the tsne projection.csv as the whitelist: - File dir = new File(so.getFile().getParentFile(), "analysis"); - dir = new File(dir, "tsne"); - dir = new File(dir, "2_components"); - if (!dir.exists()) - { - throw new PipelineJobException("Unable to find barcode or analysis directory: " + dir.getPath()); - } - - perCellTsv = new File(dir, "projection.csv"); - } - //cellranger 2 format - else if ("filtered_gene_bc_matrices".equals(barcodeDir.getName())) - { - File[] children = barcodeDir.listFiles(new FileFilter() - { - @Override - public boolean accept(File pathname) - { - return pathname.isDirectory(); - } - }); - - if (children == null || children.length != 1) - { - throw new PipelineJobException("Expected to find a single subfolder under: " + barcodeDir.getPath()); - } - - perCellTsv = new File(children[0], "barcodes.tsv"); - } - else - { - perCellTsv = new File(barcodeDir, "barcodes.tsv.gz"); - } - - if (!perCellTsv.exists()) - { - throw new PipelineJobException("Unable to find file: " + perCellTsv.getPath()); - } - - Readset rs = ctx.getSequenceSupport().getCachedReadset(so.getReadset()); - if (rs == null) - { - throw new PipelineJobException("Unable to find readset for outputfile: " + so.getRowid()); - } - else if (rs.getReadsetId() == null) - { - throw new PipelineJobException("Readset lacks a rowId for outputfile: " + so.getRowid()); - } - - Readset htoReadset = ctx.getSequenceSupport().getCachedReadset(readsetToHashing.get(rs.getReadsetId())); - if (htoReadset == null) - { - throw new PipelineJobException("Unable to find Hashing/Cite-seq readset for GEX readset: " + rs.getReadsetId()); - } - - processBarcodeFile(ctx, perCellTsv, rs, htoReadset, so.getLibrary_id(), action, getClientCommandArgs(ctx.getParams()), true, CATEGORY); - } - - ctx.addActions(action); - } - - @Override - public void complete(PipelineJob job, List inputs, List outputsCreated, SequenceAnalysisJobSupport support) throws PipelineJobException - { - for (SequenceOutputFile so : outputsCreated) - { - if (so.getCategory().equals(CATEGORY)) - { - CellRangerVDJCellHashingHandler.processMetrics(so, job, true); - } - } - } - } - - public static File processBarcodeFile(SequenceOutputHandler.JobContext ctx, File perCellTsv, Readset rs, Readset htoOrCiteReadset, int genomeId, RecordedAction action, List commandArgs, boolean writeLoupe, String category) throws PipelineJobException - { - return processBarcodeFile(ctx, perCellTsv, rs, htoOrCiteReadset, genomeId, action, commandArgs, writeLoupe, category, true); - } - - public static File processBarcodeFile(SequenceOutputHandler.JobContext ctx, File perCellTsv, Readset rs, Readset htoOrCiteReadset, int genomeId, RecordedAction action, List commandArgs, boolean writeLoupe, String category, boolean generateHtoCalls) throws PipelineJobException - { - CellRangerVDJUtils utils = new CellRangerVDJUtils(ctx.getLogger(), ctx.getSourceDirectory()); - return processBarcodeFile(ctx, perCellTsv, rs, htoOrCiteReadset, genomeId, action, commandArgs, writeLoupe, category, true, utils.getValidHashingBarcodeFile(), generateHtoCalls); - } - - public static File processBarcodeFile(SequenceOutputHandler.JobContext ctx, File perCellTsv, Readset rs, Readset htoOrCiteReadset, int genomeId, RecordedAction action, List commandArgs, boolean writeLoupe, String category, boolean createOutputFiles, File htoBarcodeWhitelist, boolean generateHtoCalls) throws PipelineJobException - { - ctx.getLogger().debug("inspecting file: " + perCellTsv.getPath()); - - CellRangerVDJUtils utils = new CellRangerVDJUtils(ctx.getLogger(), ctx.getSourceDirectory()); - - //prepare whitelist of cell indexes - File cellBarcodeWhitelist = utils.getValidCellIndexFile(); - Set uniqueBarcodes = new HashSet<>(); - ctx.getLogger().debug("writing cell barcodes, using file: " + perCellTsv.getPath()); - try (CSVWriter writer = new CSVWriter(PrintWriters.getPrintWriter(cellBarcodeWhitelist), ',', CSVWriter.NO_QUOTE_CHARACTER);CSVReader reader = new CSVReader(IOUtil.openFileForBufferedUtf8Reading(perCellTsv), '\t')) - { - int rowIdx = 0; - String[] row; - while ((row = reader.readNext()) != null) - { - //skip header - rowIdx++; - if (rowIdx > 1) - { - String barcode = row[0]; - - //NOTE: 10x appends "-1" to barcodes - if (barcode.contains("-")) - { - barcode = barcode.split("-")[0]; - } - - //This format is written out by the seurat pipeline - if (barcode.contains("_")) - { - barcode = barcode.split("_")[1]; - } - - if (!uniqueBarcodes.contains(barcode)) - { - writer.writeNext(new String[]{barcode}); - uniqueBarcodes.add(barcode); - } - } - } - - ctx.getLogger().debug("rows inspected: " + (rowIdx - 1)); - ctx.getLogger().debug("unique cell barcodes: " + uniqueBarcodes.size()); - ctx.getFileManager().addIntermediateFile(cellBarcodeWhitelist); - } - catch (IOException e) - { - throw new PipelineJobException(e); - } - - //prepare whitelist of barcodes, based on cDNA records - if (!htoBarcodeWhitelist.exists()) - { - throw new PipelineJobException("Unable to find file: " + htoBarcodeWhitelist.getPath()); - } - ctx.getFileManager().addIntermediateFile(htoBarcodeWhitelist); - - //run CiteSeqCount. - List extraParams = new ArrayList<>(); - extraParams.addAll(commandArgs); - - boolean scanEditDistances = ctx.getParams().optBoolean("scanEditDistances", false); - int editDistance = ctx.getParams().optInt("editDistance", 3); - int minCountPerCell = ctx.getParams().optInt("minCountPerCell", 3); - boolean useSeurat = ctx.getParams().optBoolean("useSeurat", true); - boolean useMultiSeq = ctx.getParams().optBoolean("useMultiSeq", true); - - PipelineStepOutput output = new DefaultPipelineStepOutput(); - String basename = FileUtil.makeLegalName(rs.getName()); - File cellToHto = SequencePipelineService.get().runCiteSeqCount(output, category, htoOrCiteReadset, htoBarcodeWhitelist, cellBarcodeWhitelist, ctx.getWorkingDirectory(), basename, ctx.getLogger(), extraParams, false, minCountPerCell, ctx.getSourceDirectory(), editDistance, scanEditDistances, rs, genomeId, generateHtoCalls, createOutputFiles, useSeurat, useMultiSeq); - ctx.getFileManager().addStepOutputs(action, output); - - ctx.getFileManager().addOutput(action, category, cellToHto); - File html = new File(cellToHto.getParentFile(), FileUtil.getBaseName(cellToHto.getName()) + ".html"); - if (html.exists()) - { - ctx.getFileManager().addOutput(action, "Cell Hashing Report", html); - } - - File citeSeqCountUnknownOutput = new File(cellToHto.getParentFile(), "citeSeqUnknownBarcodes.txt"); - ctx.getFileManager().addOutput(action,"CiteSeqCount Unknown Barcodes", citeSeqCountUnknownOutput); - - if (writeLoupe) - { - File forLoupe = new File(ctx.getSourceDirectory(), rs.getName() + "-CiteSeqCalls.csv"); - try (CSVReader reader = new CSVReader(Readers.getReader(cellToHto), '\t'); CSVWriter writer = new CSVWriter(PrintWriters.getPrintWriter(forLoupe), ',', CSVWriter.NO_QUOTE_CHARACTER)) - { - String[] line; - int idx = 0; - while ((line = reader.readNext()) != null) - { - idx++; - - if (idx > 1) - { - line[0] = line[0] + "-1"; - } - - writer.writeNext(new String[]{line[0], line[1]}); - } - } - catch (IOException e) - { - throw new PipelineJobException(e); - } - - if (createOutputFiles) - { - ctx.getFileManager().addSequenceOutput(forLoupe, rs.getName() + ": Cell Hashing Calls", "10x GEX Cell Hashing Calls (Loupe)", rs.getReadsetId(), null, genomeId, null); - } - else - { - ctx.getLogger().debug("Output file creation will be skipped"); - } - } - - return cellToHto; - } -} \ No newline at end of file diff --git a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerSeuratHandler.java b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerSeuratHandler.java deleted file mode 100644 index b927c37f3..000000000 --- a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerSeuratHandler.java +++ /dev/null @@ -1,1061 +0,0 @@ -package org.labkey.tcrdb.pipeline; - -import au.com.bytecode.opencsv.CSVReader; -import au.com.bytecode.opencsv.CSVWriter; -import htsjdk.samtools.util.IOUtil; -import org.apache.commons.io.FileUtils; -import org.apache.commons.io.IOUtils; -import org.apache.commons.lang3.StringUtils; -import org.apache.logging.log4j.Logger; -import org.apache.commons.lang3.math.NumberUtils; -import org.json.JSONObject; -import org.labkey.api.data.CompareType; -import org.labkey.api.data.DbSchema; -import org.labkey.api.data.DbSchemaType; -import org.labkey.api.data.SimpleFilter; -import org.labkey.api.data.Table; -import org.labkey.api.data.TableInfo; -import org.labkey.api.data.TableSelector; -import org.labkey.api.exp.api.ExpData; -import org.labkey.api.exp.api.ExpRun; -import org.labkey.api.exp.api.ExperimentService; -import org.labkey.api.module.ModuleLoader; -import org.labkey.api.pipeline.PipelineJob; -import org.labkey.api.pipeline.PipelineJobException; -import org.labkey.api.pipeline.PipelineService; -import org.labkey.api.pipeline.PipelineStatusFile; -import org.labkey.api.pipeline.RecordedAction; -import org.labkey.api.query.FieldKey; -import org.labkey.api.reader.Readers; -import org.labkey.api.sequenceanalysis.SequenceAnalysisService; -import org.labkey.api.sequenceanalysis.SequenceOutputFile; -import org.labkey.api.sequenceanalysis.model.Readset; -import org.labkey.api.sequenceanalysis.pipeline.AbstractParameterizedOutputHandler; -import org.labkey.api.sequenceanalysis.pipeline.SequenceAnalysisJobSupport; -import org.labkey.api.sequenceanalysis.pipeline.SequenceOutputHandler; -import org.labkey.api.sequenceanalysis.pipeline.SequencePipelineService; -import org.labkey.api.sequenceanalysis.pipeline.ToolParameterDescriptor; -import org.labkey.api.sequenceanalysis.run.SimpleScriptWrapper; -import org.labkey.api.util.FileType; -import org.labkey.api.util.FileUtil; -import org.labkey.api.util.PageFlowUtil; -import org.labkey.api.writer.PrintWriters; -import org.labkey.tcrdb.TCRdbModule; - -import java.io.BufferedReader; -import java.io.File; -import java.io.IOException; -import java.io.PrintWriter; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; - -public class CellRangerSeuratHandler extends AbstractParameterizedOutputHandler -{ - private FileType _fileType = new FileType("cloupe", false); - public static final String SEURAT_MAX_THREADS = "seuratMaxThreads"; - private static final String GTF_FILE_ID = "gtfFileId"; - - public CellRangerSeuratHandler() - { - super(ModuleLoader.getInstance().getModule(TCRdbModule.class), "Run Seurat", "This will run a standard seurat-based pipeline on the selected 10x/cellranger data and save the resulting Seurat object as an rds file for external use.", new LinkedHashSet<>(PageFlowUtil.set("sequenceanalysis/field/GenomeFileSelectorField.js")), getDefaultParams()); - } - - private static List getDefaultParams() - { - List ret = new ArrayList<>(Arrays.asList( - ToolParameterDescriptor.create("projectName", "Output Name", "This will be used as the final sample/file name. If blank, the readset name will be used. The latter cannot be used when merging multiple inputs.", "textfield", new JSONObject(){{ - - }}, null), - ToolParameterDescriptor.create("doSplitJobs", "Run Separately", "If checked, each input dataset will be run separately. Otherwise they will be merged", "checkbox", new JSONObject(){{ - put("checked", true); - }}, false), - ToolParameterDescriptor.create("skipProcessing", "Skip Processing", "If checked, the initial merge and EmptyDrops processing will be run, but PCA, DimRux, etc. will be skipped. The primary use of this is to created a merged seurat object for manual downstream processing", "checkbox", new JSONObject(){{ - put("checked", false); - }}, false), - ToolParameterDescriptor.create("useOutputFileContainer", "Submit to Source File Workbook", "If checked, each job will be submitted to the same workbook as the input file, as opposed to submitting all jobs to the same workbook. This is primarily useful if submitting a large batch of files to process separately. This only applies if 'Run Separately' is selected.", "checkbox", new JSONObject(){{ - put("checked", true); - }}, false), - ToolParameterDescriptor.create("dimsToUse", "PCs To Use", "If non-blank, this is the number of PCs that seurat will use for dim reduction steps.", "ldk-integerfield", new JSONObject(){{ - - }}, null), - ToolParameterDescriptor.create("minDimsToUse", "Minimum PCs To Use", "If non-blank, the pipeline will attempt to infer the number of PCs to use for dim reduction, but will not use fewer than this value.", "ldk-integerfield", new JSONObject(){{ - - }}, 12), - ToolParameterDescriptor.create("doCellFilter", "Perform Cell Filtering", "If selected, cells will be filtered on pct.mito and number of unique genes.", "checkbox", new JSONObject(){{ - put("checked", true); - }}, true), - ToolParameterDescriptor.create("doCellCycle", "Perform Cell Cycle Correction", "If selected, the pipeline will attempt to correct for cell cycle.", "checkbox", new JSONObject(){{ - put("checked", true); - }}, true), - ToolParameterDescriptor.create("useSCTransform", "Use SCTransform", "If selected, the pipeline will use the newer SCtransform method instead of the standard Seurat pipeline.", "checkbox", new JSONObject(){{ - put("checked", false); - }}, false), - ToolParameterDescriptor.create("runSingleR", "Run SingleR", "If selected, SingleR will be run after Seurat processing.", "checkbox", new JSONObject(){{ - put("checked", true); - }}, true), - ToolParameterDescriptor.create("mergeMethod", "Merge Method", "This determines whether any batch correction will be applied when merging datasets.", "ldk-simplecombo", new JSONObject(){{ - put("storeValues", "simple;cca"); - }}, "simple"), - ToolParameterDescriptor.create(SEURAT_MAX_THREADS, "Seurat Max Threads", "Because seurat can behave badly with multiple threads, this allows a separate cap to be used from the main job. This will allow CITE-Seq-Count and other tools to run with more threads.", "ldk-integerfield", null, 1), - ToolParameterDescriptor.createExpDataParam(GTF_FILE_ID, "Gene File", "This is the ID of a GTF file containing genes from this genome.", "sequenceanalysis-genomefileselectorfield", new JSONObject() - {{ - put("extensions", Arrays.asList("gtf")); - put("width", 400); - put("allowBlank", true); - }}, null) - )); - - ret.addAll(CellRangerCellHashingHandler.getDefaultHashingParams(false)); - - return ret; - } - - @Override - public boolean canProcess(SequenceOutputFile o) - { - return o.getFile() != null && _fileType.isType(o.getFile()); - } - - @Override - public List validateParameters(List outputFiles, JSONObject params) - { - if (!params.optBoolean("doSplitJobs", false) && StringUtils.trimToNull(params.optString("projectName")) == null && outputFiles.size() > 1) - { - return Collections.singletonList("Must provide the output name when merging multiple inputs"); - } - - return null; - } - - @Override - public boolean doRunRemote() - { - return true; - } - - @Override - public boolean doRunLocal() - { - return false; - } - - @Override - public boolean requiresSingleGenome() - { - return false; - } - - @Override - public SequenceOutputProcessor getProcessor() - { - return new CellRangerSeuratHandler.Processor(); - } - - @Override - public boolean doSplitJobs() - { - return false; - } - - public class Processor implements SequenceOutputProcessor - { - @Override - public void init(PipelineJob job, SequenceAnalysisJobSupport support, List inputFiles, JSONObject params, File outputDir, List actions, List outputsToCreate) throws UnsupportedOperationException, PipelineJobException - { - for (SequenceOutputFile so : inputFiles) - { - if (so.getReadset() != null) - { - support.cacheReadset(so.getReadset(), job.getUser()); - } - else - { - job.getLogger().error("Output file lacks a readset and will be skipped: " + so.getRowid()); - } - } - - new CellRangerVDJUtils(job.getLogger(), outputDir).prepareHashingAndCiteSeqFilesIfNeeded(job, support,"readsetId", params.optBoolean("excludeFailedcDNA", true), false, false); - - if (params.get(GTF_FILE_ID) == null) - { - job.getLogger().info("attempting to infer GTF:"); - - //TODO: collapse by filepath - Set gtfIds = new HashSet<>(); - for (SequenceOutputFile so : inputFiles) - { - ExpData gtf = null; - ExpRun run = ExperimentService.get().getExpRun(so.getRunId()); - if (run != null) - { - //Because existing runs didnt explicitly track GTF as an input, try to infer: - PipelineStatusFile sf = PipelineService.get().getStatusFile(run.getJobId()); - if (sf != null) - { - File log = new File(sf.getFilePath()); - File paramFile = new File(log.getParentFile(), "sequenceAnalysis.json"); - if (paramFile.exists()) - { - try (BufferedReader reader = Readers.getReader(paramFile)) - { - List lines = IOUtils.readLines(reader); - - JSONObject json = lines.isEmpty() ? new JSONObject() : new JSONObject(StringUtils.join(lines, '\n')); - Integer expData = json.optInt("alignment.CellRanger.gtfFile", -1); - if (expData == -1) - { - - } - - gtf = ExperimentService.get().getExpData(expData); - } - catch (IOException e) - { - throw new PipelineJobException(e); - } - } - } - } - - if (gtf == null) - { - throw new PipelineJobException("Unable to find GTF for output: " + so.getRowid()); - } - - gtfIds.add(gtf.getRowId()); - } - - if (gtfIds.size() != 1) - { - throw new PipelineJobException("All inputs must use the same GTF file, found: " + StringUtils.join(gtfIds, ",")); - } - - support.cacheExpData(ExperimentService.get().getExpData(gtfIds.iterator().next())); - support.cacheObject(GTF_FILE_ID, gtfIds.iterator().next()); - } - } - - @Override - public void processFilesOnWebserver(PipelineJob job, SequenceAnalysisJobSupport support, List inputFiles, JSONObject params, File outputDir, List actions, List outputsToCreate) throws UnsupportedOperationException, PipelineJobException - { - - } - - @Override - public void processFilesRemote(List inputFiles, JobContext ctx) throws UnsupportedOperationException, PipelineJobException - { - RecordedAction action = new RecordedAction(getName()); - ctx.addActions(action); - - int gtfId = ctx.getParams().optInt(GTF_FILE_ID, -1); - if (gtfId == -1) - { - ctx.getLogger().debug("GTF file was not specified, defaulting to inferred file"); - gtfId = ctx.getSequenceSupport().getCachedObject(GTF_FILE_ID, Integer.class); - } - - File gtfFile = ctx.getSequenceSupport().getCachedData(gtfId); - if (!gtfFile.exists()) - { - throw new PipelineJobException("Unable to find GTF file: " + gtfFile.getPath()); - } - ctx.getFileManager().addInput(action, "GTF File", gtfFile); - - Set rsNames = new HashSet<>(); - for (SequenceOutputFile so : inputFiles) - { - ctx.getFileManager().addInput(action, "CellRanger Loupe", so.getFile()); - if (so.getReadset() != null) - { - rsNames.add(ctx.getSequenceSupport().getCachedReadset(so.getReadset()).getName()); - } - } - - String outPrefix = StringUtils.trimToNull(ctx.getParams().getString("projectName")); - if (outPrefix == null) - { - if (rsNames.size() == 1) - { - outPrefix = rsNames.iterator().next(); - } - else - { - throw new PipelineJobException("Must provide the output prefix when merging more than one output file"); - } - } - outPrefix = FileUtil.makeLegalName(outPrefix); - - File seuratObj = new File(ctx.getWorkingDirectory(), outPrefix + ".seurat.rds"); - File doneFile = new File(seuratObj.getPath() + ".done"); - boolean seuratHasRun = doneFile.exists(); - if (seuratHasRun) - { - ctx.getLogger().info("Seurat has already run, will not repeat"); - } - - Map dataMap = new HashMap<>(); - - File pr = ctx.getFolderPipeRoot().getRootPath().getParentFile(); //drop the @files or @pipeline - for (SequenceOutputFile so : inputFiles) - { - //start with seurat 3 - File subDir = new File(so.getFile().getParentFile(), "raw_feature_bc_matrix"); - if (!subDir.exists()) - { - //try 2 - subDir = new File(so.getFile().getParentFile(), "raw_gene_bc_matrices"); - if (subDir.exists()) - { - //now infer subdir: - for (File f : subDir.listFiles()) - { - if (f.isDirectory()) - { - subDir = f; - break; - } - } - } - } - - if (!subDir.exists()) - { - throw new PipelineJobException("Unable to find raw data for input: " + so.getFile().getPath()); - } - - try - { - String subDirRel = FileUtil.relativize(pr, subDir, true); - ctx.getLogger().debug("pipe root: " + pr.getPath()); - ctx.getLogger().debug("file path: " + subDir.getPath()); - ctx.getLogger().debug("relative path: " + subDirRel); - - //Copy raw data directory locally to avoid docker permission issues - String dirName = so.getRowid() + "_RawData"; - File copyDir = new File(ctx.getWorkingDirectory(), dirName); - if (!seuratHasRun) - { - if (copyDir.exists()) - { - ctx.getLogger().debug("Deleting directory: " + copyDir.getPath()); - FileUtils.deleteDirectory(copyDir); - } - - ctx.getLogger().debug("Copying raw data directory: " + subDir.getPath()); - ctx.getLogger().debug("To: " + copyDir.getPath()); - FileUtils.copyDirectory(subDir, copyDir); - } - ctx.getFileManager().addIntermediateFile(copyDir); - - dataMap.put(so, dirName); - } - catch (IOException e) - { - throw new PipelineJobException(e); - } - } - - File rmdScript = new File(SequenceAnalysisService.get().getScriptPath(TCRdbModule.NAME, "external/scRNAseq/Seurat3.rmd")); - if (!rmdScript.exists()) - { - throw new PipelineJobException("Unable to find script: " + rmdScript.getPath()); - } - - File wrapperScript = new File(SequenceAnalysisService.get().getScriptPath(TCRdbModule.NAME, "external/scRNAseq/seuratWrapper.sh")); - if (!wrapperScript.exists()) - { - throw new PipelineJobException("Unable to find script: " + wrapperScript.getPath()); - } - - File tmpScript = new File(ctx.getWorkingDirectory(), "script.R"); - File outHtml = new File(ctx.getWorkingDirectory(), outPrefix + ".html"); - boolean skipProcessing = ctx.getParams().optBoolean("skipProcessing", false); - - try (PrintWriter writer = PrintWriters.getPrintWriter(tmpScript)) - { - File scriptCopy = new File(ctx.getWorkingDirectory(), rmdScript.getName()); - if (scriptCopy.exists()) - { - scriptCopy.delete(); - } - - IOUtil.copyFile(rmdScript, scriptCopy); - rmdScript = scriptCopy; - ctx.getFileManager().addIntermediateFile(rmdScript); - - scriptCopy = new File(ctx.getWorkingDirectory(), wrapperScript.getName()); - if (scriptCopy.exists()) - { - scriptCopy.delete(); - } - - IOUtil.copyFile(wrapperScript, scriptCopy); - ctx.getFileManager().addIntermediateFile(scriptCopy); - - writer.println("outPrefix <- '" + outPrefix + "'"); - writer.println("resolutionToUse <- 0.6"); - for (String v : new String[]{"dimsToUse", "minDimsToUse"}) - { - String val = StringUtils.trimToNull(ctx.getParams().optString(v)); - val = val == null ? "NULL" : val; - - writer.println(v + " <- " + val); - } - - //GTF file: - File gtfCopy = new File(ctx.getWorkingDirectory(), gtfId + ".gtf"); - if (gtfCopy.exists()) - { - gtfCopy.delete(); - } - IOUtil.copyFile(gtfFile, gtfCopy); - ctx.getFileManager().addIntermediateFile(gtfCopy); - - writer.println("gtfFile <- '" + gtfCopy.getName() + "'"); - - String mergeMethod = StringUtils.trimToNull(ctx.getParams().optString("mergeMethod")); - mergeMethod = mergeMethod == null ? "NULL" : "'" + mergeMethod + "'"; - writer.println("mergeMethod <- " + mergeMethod); - - boolean doCellFilter = ctx.getParams().optBoolean("doCellFilter", true); - writer.println("doCellFilter <- " + String.valueOf(doCellFilter).toUpperCase()); - - writer.println("skipProcessing <- " + String.valueOf(skipProcessing).toUpperCase()); - - boolean runSingleR = ctx.getParams().optBoolean("runSingleR", true); - writer.println("runSingleR <- " + String.valueOf(runSingleR).toUpperCase()); - - boolean doCellCycle = ctx.getParams().optBoolean("doCellCycle", true); - writer.println("doCellCycle <- " + String.valueOf(doCellCycle).toUpperCase()); - - boolean useSCTransform = ctx.getParams().optBoolean("useSCTransform", false); - writer.println("useSCTransform <- " + String.valueOf(useSCTransform).toUpperCase()); - - writer.println("data <- list("); - String delim = ""; - for (SequenceOutputFile so : dataMap.keySet()) - { - writer.println("\t" + delim + "'" + so.getRowid() + "'='" + dataMap.get(so) + "'"); - delim = ","; - } - writer.println(")"); - writer.println(); - writer.println(); - writer.println("setwd('/work')"); - - writer.println("rmarkdown::render('" + rmdScript.getName() + "', clean=TRUE, output_format = 'html_document', output_file='" + outHtml.getName() + "')"); - } - catch (IOException e) - { - throw new PipelineJobException(e); - } - - if (!seuratHasRun) - { - SimpleScriptWrapper wrapper = new SimpleScriptWrapper(ctx.getLogger()); - wrapper.setWorkingDir(ctx.getWorkingDirectory()); - - Integer maxThreads = SequencePipelineService.get().getMaxThreads(ctx.getLogger()); - if (maxThreads != null) - { - if (ctx.getParams().get(SEURAT_MAX_THREADS) != null) - { - maxThreads = Math.min(ctx.getParams().getInt(SEURAT_MAX_THREADS), maxThreads); - wrapper.addToEnvironment("SEQUENCEANALYSIS_MAX_THREADS", maxThreads.toString()); - } - } - - wrapper.execute(Arrays.asList("/bin/bash", wrapperScript.getName(), pr.getPath())); - - try - { - FileUtils.touch(doneFile); - ctx.getFileManager().addIntermediateFile(doneFile); - } - catch (IOException e) - { - throw new PipelineJobException(e); - } - } - - if (!seuratObj.exists()) - { - throw new PipelineJobException("Unable to find expected file: " + seuratObj.getPath()); - } - - String dimsToUse = StringUtils.trimToNull(ctx.getParams().optString("dimsToUse")); - String minDimsToUse = StringUtils.trimToNull(ctx.getParams().optString("minDimsToUse")); - String mergeMethod = StringUtils.trimToNull(ctx.getParams().optString("mergeMethod")); - - String description = StringUtils.join(new String[]{ - "Correct Cell Cycle: " + ctx.getParams().optBoolean("doCellCycle", true), - "Perform Cell Filtering: " + ctx.getParams().optBoolean("doCellFilter", true), - "Min. Dims To Use: " + (minDimsToUse == null ? "NA" : minDimsToUse), - "Dims To Use: " + (dimsToUse == null ? "automatic" : dimsToUse), - "Use SCTransform: " + ctx.getParams().optBoolean("useSCTransform", false), - "Merge method: " + mergeMethod - }, "\n"); - - if (skipProcessing) - { - ctx.getFileManager().addSequenceOutput(seuratObj, "Seurat Raw Counts: " + outPrefix, "Seurat Unprocessed Data", (inputFiles.size() == 1 ? inputFiles.iterator().next().getReadset() : null), null, getGenomeId(inputFiles), "Unprocessed Data"); - } - else - { - ctx.getFileManager().addSequenceOutput(seuratObj, "Seurat Object: " + outPrefix, "Seurat Data", (inputFiles.size() == 1 ? inputFiles.iterator().next().getReadset() : null), null, getGenomeId(inputFiles), description); - } - - ctx.getFileManager().addOutput(action, "Seurat Object", seuratObj); - - if (!outHtml.exists()) - { - throw new PipelineJobException("Unable to find summary report"); - } - ctx.getFileManager().addOutput(action, "Seurat Report", outHtml); - - if (skipProcessing) - { - ctx.getFileManager().addSequenceOutput(outHtml, "Seurat Report: " + outPrefix, "Seurat Report", (inputFiles.size() == 1 ? inputFiles.iterator().next().getReadset() : null), null, getGenomeId(inputFiles), "Unprocessed Data"); - } - else - { - ctx.getFileManager().addSequenceOutput(outHtml, "Seurat Report: " + outPrefix, "Seurat Report", (inputFiles.size() == 1 ? inputFiles.iterator().next().getReadset() : null), null, getGenomeId(inputFiles), description); - } - - File seuratObjRaw = new File(ctx.getWorkingDirectory(), outPrefix + ".rawData.rds"); - if (seuratObjRaw.exists()) - { - ctx.getFileManager().addIntermediateFile(seuratObjRaw); - } - - CellRangerVDJUtils utils = new CellRangerVDJUtils(ctx.getLogger(), ctx.getSourceDirectory()); - if (utils.useCellHashing(ctx.getSequenceSupport())) - { - runCellHashing(ctx, inputFiles, seuratObj, action, utils); - } - else - { - ctx.getLogger().info("Cell hashing was not used"); - } - - if (utils.useCiteSeq(ctx.getSequenceSupport(), inputFiles)) - { - runCiteSeq(ctx, inputFiles, seuratObj, action, outPrefix); - } - else - { - ctx.getLogger().info("CITE-seq was not used"); - } - } - - private File getAllCellBarcodesFile(File seuratObj) throws PipelineJobException - { - File allCellBarcodes = new File(seuratObj.getParentFile(), seuratObj.getName().replaceAll("seurat.rds", "cellBarcodes.csv")); - if (!allCellBarcodes.exists()) - { - throw new PipelineJobException("Unable to find expected cell barcodes file. This might indicate the seurat object was created with an older version of the pipeline. Expected: " + allCellBarcodes.getPath()); - } - - return allCellBarcodes; - } - - private void runCiteSeq(JobContext ctx, List inputFiles, File seuratObj, RecordedAction action, String outPrefix) throws PipelineJobException - { - ctx.getLogger().info("Adding CITE-seq"); - - Map citeSeqData = new HashMap<>(); - Map markerMetadata = new HashMap<>(); - File allCellBarcodes = getAllCellBarcodesFile(seuratObj); - - for (SequenceOutputFile so : inputFiles) - { - //This is the loupe file at this point - String barcodePrefix = so.getRowid().toString(); - Readset rs = ctx.getSequenceSupport().getCachedReadset(so.getReadset()); - if (rs == null) - { - throw new PipelineJobException("Unable to find readset for outputfile: " + so.getRowid()); - } - else if (rs.getReadsetId() == null) - { - throw new PipelineJobException("Readset lacks a rowId for outputfile: " + so.getRowid()); - } - - File barcodes = subsetBarcodes(allCellBarcodes, barcodePrefix); - ctx.getFileManager().addIntermediateFile(barcodes); - - // write readset-specific HTO list - Integer citeseqReadsetId = CellRangerVDJUtils.getCachedCiteSeqReadsetMap(ctx.getSequenceSupport()).get(rs.getReadsetId()); - if (citeseqReadsetId == null) - { - ctx.getLogger().info("No cite-seq readset for: " + rs.getReadsetId() + ", this probably indicates either hashing is not used or the hashing data is not available."); - continue; - } - - Readset citeseqReadset = ctx.getSequenceSupport().getCachedReadset(citeseqReadsetId); - if (citeseqReadset == null) - { - throw new PipelineJobException("Unable to find Cite-seq readset for GEX readset: " + rs.getReadsetId()); - } - - File perReadsetAdts = CellRangerVDJUtils.getValidCiteSeqBarcodeFile(ctx.getSourceDirectory(), rs.getReadsetId()); - long adtsForReadset = !perReadsetAdts.exists() ? 0 : SequencePipelineService.get().getLineCount(perReadsetAdts) - 1; - - if (adtsForReadset > 0) - { - ctx.getLogger().info("Total ADTs for readset: " + adtsForReadset); - File countMatrix = CellRangerCellHashingHandler.processBarcodeFile(ctx, barcodes, rs, citeseqReadset, so.getLibrary_id(), action, getClientCommandArgs(ctx.getParams()), false, SeuratCiteSeqHandler.CATEGORY, true, perReadsetAdts, false); - citeSeqData.put(barcodePrefix, countMatrix.getParentFile()); - File perReadsetAdtMetadata = CellRangerVDJUtils.getValidCiteSeqBarcodeMetadataFile(ctx.getSourceDirectory(), rs.getReadsetId()); - markerMetadata.put(barcodePrefix, perReadsetAdtMetadata); - } - else - { - ctx.getLogger().info("No ADTs found for readset: " + rs.getReadsetId()); - } - } - - if (!citeSeqData.isEmpty()) - { - ctx.getLogger().info("Storing cite-seq data in seurat object"); - File outHtml = appendCiteSeqToSeurat(ctx, seuratObj, citeSeqData, markerMetadata); - - ctx.getFileManager().addSequenceOutput(outHtml, "CITE-Seq Report: " + outPrefix, "CITE-Seq Report", (inputFiles.size() == 1 ? inputFiles.iterator().next().getReadset() : null), null, getGenomeId(inputFiles), null); - } - else - { - ctx.getLogger().info("CITE-seq was not used. Will not append to seurat"); - } - } - - private void runCellHashing(JobContext ctx, List inputFiles, File seuratObj, RecordedAction action, CellRangerVDJUtils utils) throws PipelineJobException - { - ctx.getLogger().info("Adding cell hashing"); - - Map finalCalls = new HashMap<>(); - File allCellBarcodes = getAllCellBarcodesFile(seuratObj); - - for (SequenceOutputFile so : inputFiles) - { - //This is the loupe file at this point - String barcodePrefix = so.getRowid().toString(); - Readset rs = ctx.getSequenceSupport().getCachedReadset(so.getReadset()); - if (rs == null) - { - throw new PipelineJobException("Unable to find readset for outputfile: " + so.getRowid()); - } - else if (rs.getReadsetId() == null) - { - throw new PipelineJobException("Readset lacks a rowId for outputfile: " + so.getRowid()); - } - - File barcodes = subsetBarcodes(allCellBarcodes, barcodePrefix); - ctx.getFileManager().addIntermediateFile(barcodes); - - // write readset-specific HTO list - Integer hashingReadsetId = CellRangerVDJUtils.getCachedHashingReadsetMap(ctx.getSequenceSupport()).get(rs.getReadsetId()); - if (hashingReadsetId == null) - { - ctx.getLogger().info("No hashing readset for: " + rs.getReadsetId() + ", this probably indicates either hashing is not used or the hashing data is not available."); - continue; - } - - Readset htoReadset = ctx.getSequenceSupport().getCachedReadset(hashingReadsetId); - if (htoReadset == null) - { - throw new PipelineJobException("Unable to find hashing readset for GEX readset: " + rs.getReadsetId()); - } - - File perReadsetHtos = new File(allCellBarcodes.getParentFile(), "allowableHtos." + barcodePrefix + ".txt"); - int htosForReadset = 0; - try (CSVReader reader = new CSVReader(Readers.getReader(utils.getCDNAInfoFile()), '\t'); CSVWriter bcWriter = new CSVWriter(PrintWriters.getPrintWriter(perReadsetHtos), ',', CSVWriter.NO_QUOTE_CHARACTER)) - { - String[] line; - while ((line = reader.readNext()) != null) - { - if (hashingReadsetId.toString().equals(line[5])) - { - htosForReadset++; - bcWriter.writeNext(new String[]{line[8], line[7]}); - } - } - } - catch (IOException e) - { - throw new PipelineJobException(e); - } - - if (htosForReadset > 1) - { - ctx.getLogger().info("Total HTOs for readset: " + htosForReadset); - finalCalls.put(barcodePrefix, CellRangerCellHashingHandler.processBarcodeFile(ctx, barcodes, rs, htoReadset, so.getLibrary_id(), action, getClientCommandArgs(ctx.getParams()), false, SeuratCellHashingHandler.CATEGORY, true, perReadsetHtos, true)); - } - else if (htosForReadset == 1) - { - ctx.getLogger().info("Only single HTO used for lane, skipping cell hashing calling"); - } - else - { - ctx.getLogger().info("No HTOs found for readset"); - } - } - - if (!finalCalls.isEmpty()) - { - ctx.getLogger().info("Storing cell hashing calls in seurat object"); - appendHashingCallsToSeurat(ctx, seuratObj, finalCalls); - } - else - { - ctx.getLogger().info("Cell hashing was not used. will not append to seurat"); - } - } - - private File subsetBarcodes(File allCellBarcodes, String barcodePrefix) throws PipelineJobException - { - //Subset barcodes by dataset: - File barcodes = new File(allCellBarcodes.getParentFile(), "cellBarcodeWhitelist." + barcodePrefix + ".txt"); - try (CSVReader reader = new CSVReader(Readers.getReader(allCellBarcodes), '\t'); CSVWriter writer = new CSVWriter(PrintWriters.getPrintWriter(barcodes), '\t', CSVWriter.NO_QUOTE_CHARACTER)) - { - String[] line; - while ((line = reader.readNext()) != null) - { - String barcode = line[0]; - if (barcode.startsWith(barcodePrefix + "_")) - { - barcode = barcode.split("_")[1]; - writer.writeNext(new String[]{barcode}); - } - } - } - catch (IOException e) - { - throw new PipelineJobException(e); - } - - return barcodes; - } - - private File appendCiteSeqToSeurat(JobContext ctx, File seuratObj, Map citeseqData, Map perReadsetAdtMap) throws PipelineJobException - { - File rScript = new File(seuratObj.getParentFile(), "appendCiteSeq.Rmd"); - File bashScript = new File(seuratObj.getParentFile(), "runDockerForCiteSeq.sh"); - - File localRoot = seuratObj.getParentFile(); - - File outputHtml = new File(localRoot, FileUtil.getBaseName(seuratObj) + ".citeseq.html"); - - Set toDelete = new HashSet<>(); - try (PrintWriter rWriter = PrintWriters.getPrintWriter(rScript); PrintWriter bashWriter = PrintWriters.getPrintWriter(bashScript)) - { - rWriter.println("---"); - rWriter.println(" title: 'CITE-seq'"); - rWriter.println("---"); - - rWriter.println("```{r setup}"); - rWriter.println("library(OOSAP)"); - rWriter.println("```"); - rWriter.println(""); - - rWriter.println("```{r citeseq}"); - rWriter.println("seuratObj <- readRDS('" + seuratObj.getName() + "')"); - rWriter.println("initialCells <- ncol(seuratObj)"); - rWriter.println("citeSeq <- list("); - int idx = 0; - for (String barcodePrefix : citeseqData.keySet()) { - idx++; - String localCopy = ensureLocalCopy(localRoot, toDelete, citeseqData.get(barcodePrefix), ctx.getLogger()); - rWriter.println("'" + barcodePrefix + "' = '" + localCopy + "'" + (idx < citeseqData.size() ? "," : "")); - } - rWriter.println(")"); - rWriter.println(""); - - rWriter.println("perReadsetAdtMap <- list("); - idx = 0; - for (String barcodePrefix : perReadsetAdtMap.keySet()) { - idx++; - String localCopy = ensureLocalCopy(localRoot, toDelete, perReadsetAdtMap.get(barcodePrefix), ctx.getLogger()); - rWriter.println("'" + barcodePrefix + "' = '" + localCopy + "'" + (idx < citeseqData.size() ? "," : "")); - } - rWriter.println(")"); - rWriter.println(""); - - rWriter.println("for (barcodePrefix in names(citeSeq)) {"); - rWriter.println(" seuratObj <- OOSAP:::AppendCiteSeq(seuratObj = seuratObj, countMatrixDir = citeSeq[[barcodePrefix]], barcodePrefix = barcodePrefix, featureLabelTable = perReadsetAdtMap[[barcodePrefix]])"); - rWriter.println("}"); - rWriter.println("if (ncol(seuratObj) != initialCells) { stop('Cell count not equal after appending cite-seq calls!') }"); - rWriter.println("saveRDS(seuratObj, file = '" + seuratObj.getName() + "')"); - rWriter.println("```"); - - rWriter.println("```{r Plot}"); - rWriter.println("OOSAP:::.PlotCiteSeqCountData(seuratObj)"); - rWriter.println("```"); - - rWriter.println("```{r SessionInfo}"); - rWriter.println("sessionInfo()"); - rWriter.println("```"); - - - bashWriter.println("#!/bin/bash"); - bashWriter.println("set -e"); - bashWriter.println("set -x"); - bashWriter.println("DOCKER=/opt/acc/sbin/exadocker"); - bashWriter.println("WD=`pwd`"); - bashWriter.println("HOME=`echo ~/`"); - - Integer maxRam = SequencePipelineService.get().getMaxRam(); - String ramOpts = ""; - if (maxRam != null) - { - ramOpts = " --memory=" +maxRam +"g "; - } - - bashWriter.println("sudo $DOCKER pull bimberlab/oosap"); - bashWriter.println("sudo $DOCKER run --rm=true " + ramOpts + "-v \"${WD}:/work\" -v \"${HOME}:/homeDir\" -u $UID -e USERID=$UID -w /work -e HOME=/homeDir bimberlab/oosap Rscript -e \"" + "rmarkdown::render('" + rScript.getName() + "', output_file = '" + outputHtml.getName() + "')\""); - - } - catch (IOException e) - { - throw new PipelineJobException(e); - } - - try - { - SimpleScriptWrapper wrapper = new SimpleScriptWrapper(ctx.getLogger()); - wrapper.setWorkingDir(seuratObj.getParentFile()); - wrapper.execute(Arrays.asList("/bin/bash", bashScript.getName())); - - for (File f : toDelete) - { - ctx.getLogger().debug("deleting local copy: " + f.getPath()); - if (f.isDirectory()) - { - FileUtils.deleteDirectory(f); - } - else - { - f.delete(); - } - } - } - catch (IOException e) - { - throw new PipelineJobException(e); - } - - return outputHtml; - } - - private String ensureLocalCopy(File localRoot, Set toDelete, File toCopy, Logger log) throws PipelineJobException - { - log.info("copying file locally: " + toCopy.getPath()); - - if (toCopy.getPath().startsWith(localRoot.getPath())) - { - return FileUtil.relativePath(localRoot.getPath(), toCopy.getPath()); - } - - try - { - File localCopy; - if (toCopy.isDirectory()) - { - localCopy = new File(localRoot, toCopy.getName()); - - File umiDir = new File(toCopy, "umi_count"); - if (!umiDir.exists()) - { - throw new PipelineJobException("Missing umi_count dir: " + umiDir.getPath()); - } - - if (localCopy.exists()) - { - log.info("local copy exists, skipping: " + localCopy.getPath()); - } - else - { - FileUtils.copyDirectory(umiDir, localCopy); - } - } - else - { - localCopy = new File(localRoot, toCopy.getName()); - - if (localCopy.exists()) - { - log.info("local copy exists, skipping: " + localCopy.getPath()); - } - else - { - FileUtils.copyFile(toCopy, localCopy); - } - } - - log.debug("destination: " + localCopy.getPath()); - toDelete.add(localCopy); - - return FileUtil.relativePath(localRoot.getPath(), localCopy.getPath()); - } - catch (IOException e) - { - throw new PipelineJobException(e); - } - } - - private void appendHashingCallsToSeurat(JobContext ctx, File seuratObj, Map finalCalls) throws PipelineJobException - { - File rScript = new File(seuratObj.getParentFile(), "appendHashing.R"); - File bashScript = new File(seuratObj.getParentFile(), "runDockerForHashing.sh"); - - try (PrintWriter rWriter = PrintWriters.getPrintWriter(rScript); PrintWriter bashWriter = PrintWriters.getPrintWriter(bashScript)) - { - rWriter.println("library(OOSAP)"); - rWriter.println("seuratObj <- readRDS('" + seuratObj.getName() + "')"); - rWriter.println("initialCells <- ncol(seuratObj)"); - rWriter.println("callsFiles <- list("); - int idx = 0; - for (String barcodePrefix : finalCalls.keySet()) - { - idx++; - rWriter.println("'" + barcodePrefix + "' = '" + finalCalls.get(barcodePrefix).getName() + "'" + (idx < finalCalls.size() ? "," : "")); - } - - rWriter.println(")"); - rWriter.println(""); - rWriter.println("for (barcodePrefix in names(callsFiles)) {"); - rWriter.println(" seuratObj <- OOSAP:::AppendCellHashing(seuratObj = seuratObj, barcodeCallFile = callsFiles[[barcodePrefix]], barcodePrefix = barcodePrefix)"); - rWriter.println("}"); - rWriter.println("if (ncol(seuratObj) != initialCells) { stop('Cell count not equal after appending cell hashing calls!') }"); - rWriter.println("saveRDS(seuratObj, file = '" + seuratObj.getName() + "')"); - - bashWriter.println("#!/bin/bash"); - bashWriter.println("set -e"); - bashWriter.println("set -x"); - bashWriter.println("DOCKER=/opt/acc/sbin/exadocker"); - bashWriter.println("WD=`pwd`"); - bashWriter.println("HOME=`echo ~/`"); - - Integer maxRam = SequencePipelineService.get().getMaxRam(); - String ramOpts = ""; - if (maxRam != null) - { - ramOpts = " --memory=" + maxRam + "g "; - } - - bashWriter.println("sudo $DOCKER pull bimberlab/oosap"); - bashWriter.println("sudo $DOCKER run --rm=true " + ramOpts + "-v \"${WD}:/work\" -v \"${HOME}:/homeDir\" -u $UID -e USERID=$UID -w /work -e HOME=/homeDir bimberlab/oosap Rscript --vanilla " + rScript.getName()); - } - catch (IOException e) - { - throw new PipelineJobException(e); - } - - SimpleScriptWrapper wrapper = new SimpleScriptWrapper(ctx.getLogger()); - wrapper.setWorkingDir(seuratObj.getParentFile()); - wrapper.execute(Arrays.asList("/bin/bash", bashScript.getName())); - } - - @Override - public void complete(PipelineJob job, List inputs, List outputsCreated, SequenceAnalysisJobSupport support) throws PipelineJobException - { - for (SequenceOutputFile so : outputsCreated) - { - if (so.getFile() != null && so.getFile().getPath().endsWith(".seurat.rds")) - { - File metrics = new File(so.getFile().getPath().replaceAll(".seurat.rds", ".summary.txt")); - if (metrics.exists()) - { - processMetricsFile(job, metrics, so); - } - else - { - job.getLogger().warn("Unable to find metrics file: " + metrics.getPath()); - } - } - else if (so.getFile() != null && so.getFile().getPath().endsWith(".calls.txt")) - { - File metrics = new File(so.getFile().getPath().replaceAll(".calls.txt", ".metrics.txt")); - if (metrics.exists()) - { - processMetricsFile(job, metrics, so); - } - else - { - job.getLogger().warn("Unable to find metrics file: " + metrics.getPath()); - } - } - } - } - } - - private void processMetricsFile(PipelineJob job, File metrics, SequenceOutputFile so) throws PipelineJobException - { - job.getLogger().info("Loading metrics"); - TableInfo ti = DbSchema.get("sequenceanalysis", DbSchemaType.Module).getTable("quality_metrics"); - - //NOTE: if this job errored and restarted, we may have duplicate records: - SimpleFilter filter = new SimpleFilter(FieldKey.fromString("readset"), so.getReadset()); - filter.addCondition(FieldKey.fromString("analysis_id"), so.getAnalysis_id(), CompareType.EQUAL); - filter.addCondition(FieldKey.fromString("dataid"), so.getDataId(), CompareType.EQUAL); - filter.addCondition(FieldKey.fromString("container"), job.getContainer().getId(), CompareType.EQUAL); - TableSelector ts = new TableSelector(ti, PageFlowUtil.set("rowid"), filter, null); - if (ts.exists()) - { - job.getLogger().info("Deleting existing QC metrics (probably from prior restarted job)"); - ts.getArrayList(Integer.class).forEach(rowid -> { - Table.delete(ti, rowid); - }); - } - - int total = 0; - try (CSVReader reader = new CSVReader(Readers.getReader(metrics), '\t')) - { - String[] line; - while ((line = reader.readNext()) != null) - { - if ("Category".equals(line[0])) - { - continue; - } - - Map r = new HashMap<>(); - r.put("category", line[0]); - r.put("metricname", line[1]); - - String fieldName = NumberUtils.isCreatable(line[2]) ? "metricvalue" : "qualvalue"; - r.put(fieldName, line[2]); - r.put("analysis_id", so.getAnalysis_id()); - r.put("dataid", so.getDataId()); - r.put("readset", so.getReadset()); - r.put("container", job.getContainer()); - r.put("createdby", job.getUser().getUserId()); - - Table.insert(job.getUser(), ti, r); - total++; - } - } - catch (IOException e) - { - throw new PipelineJobException(e); - } - - job.getLogger().info("total metrics: " + total); - } - - private Integer getGenomeId(List inputFiles) - { - Set genomeIds = new HashSet<>(); - inputFiles.forEach(x -> { - genomeIds.add(x.getLibrary_id()); - }); - - return genomeIds.size() == 1 ? genomeIds.iterator().next() : null; - } -} diff --git a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java index 19a3fdb61..be242b0fd 100644 --- a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java +++ b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java @@ -1,24 +1,12 @@ package org.labkey.tcrdb.pipeline; -import au.com.bytecode.opencsv.CSVReader; -import org.apache.commons.beanutils.ConversionException; import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.math.NumberUtils; import org.json.JSONObject; -import org.labkey.api.data.CompareType; import org.labkey.api.data.ConvertHelper; -import org.labkey.api.data.DbSchema; -import org.labkey.api.data.DbSchemaType; -import org.labkey.api.data.SimpleFilter; -import org.labkey.api.data.Table; -import org.labkey.api.data.TableInfo; -import org.labkey.api.data.TableSelector; import org.labkey.api.module.ModuleLoader; import org.labkey.api.pipeline.PipelineJob; import org.labkey.api.pipeline.PipelineJobException; import org.labkey.api.pipeline.RecordedAction; -import org.labkey.api.query.FieldKey; -import org.labkey.api.reader.Readers; import org.labkey.api.sequenceanalysis.SequenceOutputFile; import org.labkey.api.sequenceanalysis.model.AnalysisModel; import org.labkey.api.sequenceanalysis.model.Readset; @@ -27,28 +15,25 @@ import org.labkey.api.sequenceanalysis.pipeline.SequenceAnalysisJobSupport; import org.labkey.api.sequenceanalysis.pipeline.SequenceOutputHandler; import org.labkey.api.sequenceanalysis.pipeline.ToolParameterDescriptor; +import org.labkey.api.singlecell.CellHashingService; import org.labkey.api.util.FileType; import org.labkey.api.util.PageFlowUtil; import org.labkey.tcrdb.TCRdbModule; import java.io.File; -import java.io.IOException; -import java.text.DecimalFormat; import java.util.ArrayList; import java.util.Arrays; -import java.util.HashMap; import java.util.LinkedHashSet; import java.util.List; -import java.util.Map; - -import static org.labkey.tcrdb.pipeline.CellRangerVDJWrapper.DELETE_EXISTING_ASSAY_DATA; -import static org.labkey.tcrdb.pipeline.CellRangerVDJWrapper.TARGET_ASSAY; public class CellRangerVDJCellHashingHandler extends AbstractParameterizedOutputHandler { private FileType _fileType = new FileType("vloupe", false); public static final String CATEGORY = "Cell Hashing Calls (VDJ)"; + public static final String TARGET_ASSAY = "targetAssay"; + public static final String DELETE_EXISTING_ASSAY_DATA = "deleteExistingAssayData"; + public CellRangerVDJCellHashingHandler() { super(ModuleLoader.getInstance().getModule(TCRdbModule.class), "CellRanger VDJ Import", "This will either directly import data (if cell hashing is not used), or run CiteSeqCount/MultiSeqClassifier to generate a sample-to-cellbarcode TSV based on the filtered barcodes from CellRanger VDJ and then import.", new LinkedHashSet<>(PageFlowUtil.set("tcrdb/field/AssaySelectorField.js")), getDefaultParams()); @@ -66,7 +51,7 @@ private static List getDefaultParams() }}, false) )); - ret.addAll(CellRangerCellHashingHandler.getDefaultHashingParams(true)); + ret.addAll(CellHashingService.get().getDefaultHashingParams(true)); return ret; } @@ -112,10 +97,8 @@ public class Processor implements SequenceOutputHandler.SequenceOutputProcessor @Override public void init(PipelineJob job, SequenceAnalysisJobSupport support, List inputFiles, JSONObject params, File outputDir, List actions, List outputsToCreate) throws UnsupportedOperationException, PipelineJobException { - CellRangerVDJUtils utils = new CellRangerVDJUtils(job.getLogger(), outputDir); - //NOTE: this is the pathway to import assay data, whether hashing is used or not - utils.prepareHashingAndCiteSeqFilesIfNeeded(job, support, "enrichedReadsetId", params.optBoolean("excludeFailedcDNA", true), false, false); + CellHashingService.get().prepareHashingAndCiteSeqFilesIfNeeded(outputDir, job, support, "tcrReadsetId", params.optBoolean("excludeFailedcDNA", true), false, false); } @Override @@ -131,11 +114,10 @@ public void complete(PipelineJob job, List inputFiles, List< { if (CATEGORY.equals(so.getCategory())) { - processMetrics(so, job, true); + CellHashingService.get().processMetrics(so, job, true); } } - CellRangerVDJUtils utils = new CellRangerVDJUtils(job.getLogger(), job.getLogFile().getParentFile()); if (StringUtils.trimToNull(job.getParameters().get(TARGET_ASSAY)) == null) { job.getLogger().info("No assay selected, will not import"); @@ -157,7 +139,7 @@ public void complete(PipelineJob job, List inputFiles, List< for (SequenceOutputFile so : inputFiles) { AnalysisModel model = support.getCachedAnalysis(so.getAnalysis_id()); - utils.importAssayData(job, model, so.getFile().getParentFile(), assayId, null, deleteExistingData); + new CellRangerVDJUtils(job.getLogger()).importAssayData(job, model, so.getFile().getParentFile(), assayId, null, deleteExistingData); } } } @@ -165,15 +147,13 @@ public void complete(PipelineJob job, List inputFiles, List< @Override public void processFilesRemote(List inputFiles, JobContext ctx) throws UnsupportedOperationException, PipelineJobException { - CellRangerVDJUtils utils = new CellRangerVDJUtils(ctx.getLogger(), ctx.getSourceDirectory()); RecordedAction action = new RecordedAction(getName()); - for (SequenceOutputFile so : inputFiles) { ctx.getLogger().info("processing file: " + so.getName()); //find TSV: - File perCellTsv = utils.getPerCellCsv(so.getFile().getParentFile()); + File perCellTsv = CellRangerVDJUtils.getPerCellCsv(so.getFile().getParentFile()); if (!perCellTsv.exists()) { throw new PipelineJobException("Unable to find file: " + perCellTsv.getPath()); @@ -197,8 +177,6 @@ else if (rs.getReadsetId() == null) private void processVloupeFile(JobContext ctx, File perCellTsv, Readset rs, RecordedAction action, Integer genomeId) throws PipelineJobException { - CellRangerVDJUtils utils = new CellRangerVDJUtils(ctx.getLogger(), ctx.getSourceDirectory()); - List extraParams = new ArrayList<>(); extraParams.addAll(getClientCommandArgs(ctx.getParams())); @@ -210,8 +188,8 @@ private void processVloupeFile(JobContext ctx, File perCellTsv, Readset rs, Reco int minCountPerCell = ctx.getParams().optInt("minCountPerCell", 3); int editDistance = ctx.getParams().optInt("editDistance", 2); - File cellToHto = utils.runRemoteVdjCellHashingTasks(output, CATEGORY, perCellTsv, rs, ctx.getSequenceSupport(), extraParams, ctx.getWorkingDirectory(), ctx.getSourceDirectory(), editDistance, scanEditDistances, genomeId, minCountPerCell, useSeurat, useMultiSeq); - if (utils.useCellHashing(ctx.getSequenceSupport()) && cellToHto == null) + File cellToHto = CellHashingService.get().runRemoteVdjCellHashingTasks(output, CATEGORY, perCellTsv, rs, ctx.getSequenceSupport(), extraParams, ctx.getWorkingDirectory(), ctx.getSourceDirectory(), editDistance, scanEditDistances, genomeId, minCountPerCell, useSeurat, useMultiSeq); + if (CellHashingService.get().usesCellHashing(ctx.getSequenceSupport(), ctx.getSourceDirectory()) && cellToHto == null) { throw new PipelineJobException("Missing cell to HTO file"); @@ -221,131 +199,4 @@ private void processVloupeFile(JobContext ctx, File perCellTsv, Readset rs, Reco } } - - private static File getMetricsFile(File callFile) - { - return new File(callFile.getPath().replaceAll(".calls.txt", ".metrics.txt")); - } - - public static void processMetrics(SequenceOutputFile so, PipelineJob job, boolean updateDescription) throws PipelineJobException - { - if (so.getFile() != null) - { - Map valueMap = new HashMap<>(); - - File metrics = getMetricsFile(so.getFile()); - if (metrics.exists()) - { - job.getLogger().info("Loading metrics"); - int total = 0; - TableInfo ti = DbSchema.get("sequenceanalysis", DbSchemaType.Module).getTable("quality_metrics"); - - //NOTE: if this job errored and restarted, we may have duplicate records: - SimpleFilter filter = new SimpleFilter(FieldKey.fromString("readset"), so.getReadset()); - filter.addCondition(FieldKey.fromString("analysis_id"), so.getAnalysis_id(), CompareType.EQUAL); - filter.addCondition(FieldKey.fromString("dataid"), so.getDataId(), CompareType.EQUAL); - filter.addCondition(FieldKey.fromString("container"), job.getContainer().getId(), CompareType.EQUAL); - TableSelector ts = new TableSelector(ti, PageFlowUtil.set("rowid"), filter, null); - if (ts.exists()) - { - job.getLogger().info("Deleting existing QC metrics (probably from prior restarted job)"); - ts.getArrayList(Integer.class).forEach(rowid -> { - Table.delete(ti, rowid); - }); - } - - try (CSVReader reader = new CSVReader(Readers.getReader(metrics), '\t')) - { - String[] line; - while ((line = reader.readNext()) != null) - { - if ("Category".equals(line[0])) - { - continue; - } - - Map r = new HashMap<>(); - r.put("category", line[0]); - r.put("metricname", line[1]); - - //NOTE: R saves NaN as NA. This is fixed in the R code, but add this check here to let existing jobs import - String value = line[2]; - if ("NA".equals(value)) - { - value = "0"; - } - - String fieldName = NumberUtils.isCreatable(value) ? "metricvalue" : "qualvalue"; - r.put(fieldName, value); - - r.put("analysis_id", so.getAnalysis_id()); - r.put("dataid", so.getDataId()); - r.put("readset", so.getReadset()); - r.put("container", job.getContainer()); - r.put("createdby", job.getUser().getUserId()); - - Table.insert(job.getUser(), ti, r); - total++; - - valueMap.put(line[1], value); - } - - job.getLogger().info("total metrics: " + total); - - if (updateDescription) - { - job.getLogger().debug("Updating description"); - StringBuilder description = new StringBuilder(); - if (StringUtils.trimToNull(so.getDescription()) != null) - { - description.append(StringUtils.trimToNull(so.getDescription())); - } - - String delim = description.length() > 0 ? "\n" : ""; - - DecimalFormat fmt = new DecimalFormat("##.##%"); - for (String metricName : Arrays.asList("InputBarcodes", "TotalCalled", "TotalCounts", "TotalSinglet", "FractionOfInputCalled", "FractionOfInputSinglet", "FractionOfInputDoublet", "FractionOfInputDiscordant", "FractionCalledNotInInput", "SeuratNonNegative", "MultiSeqNonNegative", "UniqueHtos", "UnknownTagMatchingKnown")) - { - if (valueMap.get(metricName) != null) - { - Double d = null; - if (metricName.startsWith("Fraction")) - { - try - { - d = ConvertHelper.convert(valueMap.get(metricName), Double.class); - } - catch (ConversionException | IllegalArgumentException e) - { - job.getLogger().error("Unable to convert to double: " + valueMap.get(metricName)); - throw e; - } - } - - description.append(delim).append(metricName).append(": ").append(d == null ? valueMap.get(metricName) : fmt.format(d)); - delim = ",\n"; - } - } - - so.setDescription(description.toString()); - - TableInfo tableOutputs = DbSchema.get("sequenceanalysis", DbSchemaType.Module).getTable("outputfiles"); - Table.update(job.getUser(), tableOutputs, so, so.getRowid()); - } - } - catch (IOException e) - { - throw new PipelineJobException(e); - } - } - else - { - job.getLogger().warn("Unable to find metrics file: " + metrics.getPath()); - } - } - else - { - job.getLogger().warn("Unable to update metrics, file id is null: " + so.getName()); - } - } } \ No newline at end of file diff --git a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJUtils.java b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJUtils.java index 4c376e533..b6b9640c3 100644 --- a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJUtils.java +++ b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJUtils.java @@ -1,7 +1,6 @@ package org.labkey.tcrdb.pipeline; import au.com.bytecode.opencsv.CSVReader; -import au.com.bytecode.opencsv.CSVWriter; import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.Logger; import org.jetbrains.annotations.Nullable; @@ -10,8 +9,6 @@ import org.labkey.api.assay.AssayProvider; import org.labkey.api.assay.AssayService; import org.labkey.api.collections.CaseInsensitiveHashMap; -import org.labkey.api.data.ColumnInfo; -import org.labkey.api.data.CompareType; import org.labkey.api.data.Container; import org.labkey.api.data.SimpleFilter; import org.labkey.api.data.TableInfo; @@ -29,24 +26,19 @@ import org.labkey.api.query.InvalidKeyException; import org.labkey.api.query.QueryService; import org.labkey.api.query.QueryUpdateServiceException; -import org.labkey.api.query.UserSchema; import org.labkey.api.query.ValidationException; import org.labkey.api.reader.FastaDataLoader; import org.labkey.api.reader.FastaLoader; import org.labkey.api.reader.Readers; import org.labkey.api.security.User; -import org.labkey.api.sequenceanalysis.SequenceAnalysisService; -import org.labkey.api.sequenceanalysis.SequenceOutputFile; import org.labkey.api.sequenceanalysis.model.AnalysisModel; -import org.labkey.api.sequenceanalysis.model.Readset; -import org.labkey.api.sequenceanalysis.pipeline.PipelineStepOutput; -import org.labkey.api.sequenceanalysis.pipeline.SequenceAnalysisJobSupport; import org.labkey.api.sequenceanalysis.pipeline.SequencePipelineService; +import org.labkey.api.singlecell.CellHashingService; +import org.labkey.api.singlecell.model.CDNA_Library; import org.labkey.api.util.FileUtil; import org.labkey.api.util.PageFlowUtil; import org.labkey.api.view.ViewBackgroundInfo; import org.labkey.api.view.ViewContext; -import org.labkey.api.writer.PrintWriters; import org.labkey.tcrdb.TCRdbSchema; import java.io.File; @@ -56,416 +48,19 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; -import java.util.Date; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; public class CellRangerVDJUtils { private Logger _log; - private File _sourceDir; - public static final String READSET_TO_HASHING_MAP = "readsetToHashingMap"; - public static final String READSET_TO_CITESEQ_MAP = "readsetToCiteSeqMap"; - private static final String HASHING_CALLS = "Cell Hashing TCR Calls"; - - public CellRangerVDJUtils(Logger log, File sourceDir) + public CellRangerVDJUtils(Logger log) { _log = log; - _sourceDir = sourceDir; - } - - public void prepareHashingAndCiteSeqFilesIfNeeded(PipelineJob job, SequenceAnalysisJobSupport support, String filterField, final boolean skipFailedCdna, boolean failIfNoHashing, boolean failIfNoCiteSeq) throws PipelineJobException - { - Container target = job.getContainer().isWorkbook() ? job.getContainer().getParent() : job.getContainer(); - UserSchema tcr = QueryService.get().getUserSchema(job.getUser(), target, TCRdbSchema.NAME); - TableInfo cDNAs = tcr.getTable(TCRdbSchema.TABLE_CDNAS, null); - - _log.debug("preparing cDNA and cell hashing files"); - - SequenceAnalysisService.get().writeAllCellHashingBarcodes(_sourceDir, job.getUser(), job.getContainer()); - SequenceAnalysisService.get().writeAllCiteSeqBarcodes(_sourceDir, job.getUser(), job.getContainer()); - - Map colMap = QueryService.get().getColumns(cDNAs, PageFlowUtil.set( - FieldKey.fromString("rowid"), - FieldKey.fromString("sortId/stimId/animalId"), - FieldKey.fromString("sortId/stimId/stim"), - FieldKey.fromString("sortId/population"), - FieldKey.fromString("sortId/hto"), - FieldKey.fromString("sortId/hto/sequence"), - FieldKey.fromString("hashingReadsetId"), - FieldKey.fromString("hashingReadsetId/totalFiles"), - FieldKey.fromString("citeseqReadsetId"), - FieldKey.fromString("citeseqReadsetId/totalFiles"), - FieldKey.fromString("citeseqPanel"), - FieldKey.fromString("status")) - ); - - File output = getCDNAInfoFile(); - File barcodeOutput = getValidHashingBarcodeFile(); - HashMap readsetToHashingMap = new HashMap<>(); - HashMap readsetToCiteSeqMap = new HashMap<>(); - HashMap> gexToPanels = new HashMap<>(); - - try (CSVWriter writer = new CSVWriter(PrintWriters.getPrintWriter(output), '\t', CSVWriter.NO_QUOTE_CHARACTER); CSVWriter bcWriter = new CSVWriter(PrintWriters.getPrintWriter(barcodeOutput), ',', CSVWriter.NO_QUOTE_CHARACTER)) - { - writer.writeNext(new String[]{"ReadsetId", "CDNA_ID", "AnimalId", "Stim", "Population", "HashingReadsetId", "HasHashingReads", "HTO_Name", "HTO_Seq", "CiteSeqReadsetId", "HasCiteSeqReads", "CiteSeqPanel"}); - List cachedReadsets = support.getCachedReadsets(); - Set distinctHTOs = new HashSet<>(); - Set hashingStatus = new HashSet<>(); - Set citeseqStatus = new HashSet<>(); - AtomicInteger totalWritten = new AtomicInteger(0); - for (Readset rs : cachedReadsets) - { - AtomicBoolean hasError = new AtomicBoolean(false); - //find cDNA records using this readset - new TableSelector(cDNAs, colMap.values(), new SimpleFilter(FieldKey.fromString(filterField), rs.getRowId()), null).forEachResults(results -> { - if (skipFailedCdna && results.getObject(FieldKey.fromString("status")) != null) - { - _log.info("skipping cDNA with non-null status: " + results.getString(FieldKey.fromString("rowid"))); - return; - } - - writer.writeNext(new String[]{ - String.valueOf(rs.getRowId()), - results.getString(FieldKey.fromString("rowid")), - results.getString(FieldKey.fromString("sortId/stimId/animalId")), - results.getString(FieldKey.fromString("sortId/stimId/stim")), - results.getString(FieldKey.fromString("sortId/population")), - String.valueOf(results.getObject(FieldKey.fromString("hashingReadsetId")) == null ? "" : results.getInt(FieldKey.fromString("hashingReadsetId"))), - String.valueOf(results.getObject(FieldKey.fromString("hashingReadsetId/totalFiles")) != null && results.getInt(FieldKey.fromString("hashingReadsetId/totalFiles")) > 0), - results.getString(FieldKey.fromString("sortId/hto")), - results.getString(FieldKey.fromString("sortId/hto/sequence")), - String.valueOf(results.getObject(FieldKey.fromString("citeseqReadsetId")) == null ? "" : results.getInt(FieldKey.fromString("citeseqReadsetId"))), - String.valueOf(results.getObject(FieldKey.fromString("citeseqReadsetId/totalFiles")) != null && results.getInt(FieldKey.fromString("citeseqReadsetId/totalFiles")) > 0), - results.getString(FieldKey.fromString("citeseqPanel")) - }); - totalWritten.getAndIncrement(); - - boolean useCellHashing = results.getObject(FieldKey.fromString("sortId/hto")) != null; - hashingStatus.add(useCellHashing); - if (useCellHashing) - { - if (results.getObject(FieldKey.fromString("hashingReadsetId")) == null) - { - job.getLogger().error("cDNA specifies HTO, but does not list a hashing readset: " + results.getString(FieldKey.fromString("rowid"))); - hasError.set(true); - } - else - { - readsetToHashingMap.put(rs.getReadsetId(), results.getInt(FieldKey.fromString("hashingReadsetId"))); - - String hto = results.getString(FieldKey.fromString("sortId/hto")) + "<>" + results.getString(FieldKey.fromString("sortId/hto/sequence")); - if (!distinctHTOs.contains(hto) && !StringUtils.isEmpty(results.getString(FieldKey.fromString("sortId/hto/sequence")))) - { - distinctHTOs.add(hto); - bcWriter.writeNext(new String[]{results.getString(FieldKey.fromString("sortId/hto/sequence")), results.getString(FieldKey.fromString("sortId/hto"))}); - } - - if (results.getObject(FieldKey.fromString("sortId/hto/sequence")) == null) - { - job.getLogger().error("Unable to find sequence for HTO: " + results.getString(FieldKey.fromString("sortId/hto"))); - hasError.set(true); - } - } - } - - boolean useCiteSeq = results.getObject(FieldKey.fromString("citeseqPanel")) != null; - citeseqStatus.add(useCiteSeq); - if (useCiteSeq) - { - if (results.getObject(FieldKey.fromString("citeseqReadsetId")) == null) - { - job.getLogger().error("cDNA specifies cite-seq readset but does not list panel: " + results.getString(FieldKey.fromString("rowid"))); - hasError.set(true); - } - else - { - Set panels = gexToPanels.getOrDefault(rs.getRowId(), new HashSet<>()); - panels.add(results.getString(FieldKey.fromString("citeseqPanel"))); - gexToPanels.put(rs.getRowId(), panels); - - readsetToCiteSeqMap.put(rs.getReadsetId(), results.getInt(FieldKey.fromString("citeseqReadsetId"))); - } - } - }); - - if (hasError.get()) - { - throw new PipelineJobException("No cell hashing readset or HTO found for one or more cDNAs. see the file: " + output.getName()); - } - - if (hashingStatus.size() > 1) - { - _log.info("The selected readsets/cDNA records use a mixture of cell hashing and non-hashing."); - } - - //NOTE: hashingStatus.isEmpty() indicates there are no cDNA records associated with the data - } - - // if distinct HTOs is 1, no point in running hashing. note: presence of hashing readsets is a trigger downstream - if (distinctHTOs.size() > 1) - { - readsetToHashingMap.forEach((readsetId, hashingReadsetId) -> support.cacheReadset(hashingReadsetId, job.getUser())); - } - else if (distinctHTOs.size() == 1) - { - job.getLogger().info("There is only a single HTO in this pool, will not use hashing"); - } - - if (totalWritten.get() == 0) - { - throw new PipelineJobException("No matching cDNA records found"); - } - - boolean useCellHashing = hashingStatus.isEmpty() ? false : hashingStatus.size() > 1 ? true : hashingStatus.iterator().next(); - if (useCellHashing && distinctHTOs.isEmpty()) - { - throw new PipelineJobException("Cell hashing was selected, but no HTOs were found"); - } - else - { - _log.info("distinct HTOs: " + distinctHTOs.size()); - } - - support.cacheObject(READSET_TO_HASHING_MAP, readsetToHashingMap); - support.cacheObject(READSET_TO_CITESEQ_MAP, readsetToCiteSeqMap); - readsetToCiteSeqMap.forEach((readsetId, citeseqReadsetId) -> support.cacheReadset(citeseqReadsetId, job.getUser())); - } - catch (IOException e) - { - throw new PipelineJobException(e); - } - - writeCiteSeqBarcodes(job, gexToPanels, _sourceDir); - - if (failIfNoHashing && readsetToHashingMap.isEmpty()) - { - throw new PipelineJobException("Readsets do not use cell hashing"); - } - - if (failIfNoCiteSeq && readsetToCiteSeqMap.isEmpty()) - { - throw new PipelineJobException("Readsets do not use CITE-seq"); - } - } - - public static File getValidCiteSeqBarcodeFile(File sourceDir, int gexReadsetId) - { - return new File(sourceDir, "validADTS." + gexReadsetId + ".csv"); - } - - public static File getValidCiteSeqBarcodeMetadataFile(File sourceDir, int gexReadsetId) - { - return new File(sourceDir, "validADTS." + gexReadsetId + ".metadata.txt"); - } - - private void writeCiteSeqBarcodes(PipelineJob job, Map> gexToPanels, File outputDir) throws PipelineJobException - { - Container target = job.getContainer().isWorkbook() ? job.getContainer().getParent() : job.getContainer(); - UserSchema tcr = QueryService.get().getUserSchema(job.getUser(), target, TCRdbSchema.NAME); - TableInfo panels = tcr.getTable(TCRdbSchema.TABLE_CITE_SEQ_PANELS, null); - - Map barcodeColMap = QueryService.get().getColumns(panels, PageFlowUtil.set( - FieldKey.fromString("antibody"), - FieldKey.fromString("antibody/markerName"), - FieldKey.fromString("antibody/markerLabel"), - FieldKey.fromString("markerLabel"), - FieldKey.fromString("antibody/adaptersequence") - )); - - for (int gexReadsetId : gexToPanels.keySet()) - { - job.getLogger().info("Writing all unique ADTs for readset: " + gexReadsetId); - File barcodeOutput = getValidCiteSeqBarcodeFile(outputDir, gexReadsetId); - File metadataOutput = getValidCiteSeqBarcodeMetadataFile(outputDir, gexReadsetId); - try (CSVWriter writer = new CSVWriter(PrintWriters.getPrintWriter(barcodeOutput), ',', CSVWriter.NO_QUOTE_CHARACTER);CSVWriter metaWriter = new CSVWriter(PrintWriters.getPrintWriter(metadataOutput), '\t', CSVWriter.NO_QUOTE_CHARACTER)) - { - metaWriter.writeNext(new String[]{"tagname", "sequence", "markername", "markerlabel"}); - AtomicInteger barcodeCount = new AtomicInteger(); - Set found = new HashSet<>(); - new TableSelector(panels, barcodeColMap.values(), new SimpleFilter(FieldKey.fromString("name"), gexToPanels.get(gexReadsetId), CompareType.IN), new org.labkey.api.data.Sort("antibody")).forEachResults(results -> { - if (found.contains(results.getString(FieldKey.fromString("antibody/adaptersequence")))) - { - return; - } - - found.add(results.getString(FieldKey.fromString("antibody/adaptersequence"))); - barcodeCount.getAndIncrement(); - - writer.writeNext(new String[]{results.getString(FieldKey.fromString("antibody/adaptersequence")), results.getString(FieldKey.fromString("antibody"))}); - - //allow aliasing based on DB - String label = StringUtils.trimToNull(results.getString(FieldKey.fromString("markerLabel"))) == null ? results.getString(FieldKey.fromString("antibody/markerLabel")) : results.getString(FieldKey.fromString("markerLabel")); - String name = StringUtils.trimToNull(results.getString(FieldKey.fromString("markerLabel"))) != null ? results.getString(FieldKey.fromString("markerLabel")) : - StringUtils.trimToNull(results.getString(FieldKey.fromString("antibody/markerName"))) != null ? results.getString(FieldKey.fromString("antibody/markerName")) : results.getString(FieldKey.fromString("antibody")); - metaWriter.writeNext(new String[]{results.getString(FieldKey.fromString("antibody")), results.getString(FieldKey.fromString("antibody/adaptersequence")), name, label}); - }); - - job.getLogger().info("Total CITE-seq barcodes written: " + barcodeCount.get()); - } - catch (IOException e) - { - throw new PipelineJobException(e); - } - } - } - - public File getCDNAInfoFile() - { - return getCDNAInfoFile(_sourceDir); - } - - public static File getCDNAInfoFile(File sourceDir) - { - return new File(sourceDir, "cDNAInfo.txt"); - } - - public File getValidHashingBarcodeFile() - { - return getValidHashingBarcodeFile(_sourceDir); - } - - public static File getValidHashingBarcodeFile(File sourceDir) - { - return new File(sourceDir, "validHashingBarcodes.csv"); - } - - public File getValidCellIndexFile() - { - return new File(_sourceDir, "validCellIndexes.csv"); - } - - public File getPerCellCsv(File outDir) - { - return new File(outDir, "all_contig_annotations.csv"); - } - - public File runRemoteVdjCellHashingTasks(PipelineStepOutput output, String outputCategory, File perCellTsv, Readset rs, SequenceAnalysisJobSupport support, List extraParams, File workingDir, File sourceDir, Integer editDistance, boolean scanEditDistances, Integer genomeId, Integer minCountPerCell, boolean useSeurat, boolean useMultiSeq) throws PipelineJobException - { - Map readsetToHashing = getCachedHashingReadsetMap(support); - if (readsetToHashing.isEmpty()) - { - _log.info("No cached hashing readsets, skipping"); - return null; - } - - //prepare whitelist of barcodes, based on cDNA records - File htoBarcodeWhitelist = getValidHashingBarcodeFile(); - if (!htoBarcodeWhitelist.exists()) - { - throw new PipelineJobException("Unable to find file: " + htoBarcodeWhitelist.getPath()); - } - - long lineCount = SequencePipelineService.get().getLineCount(htoBarcodeWhitelist); - if (lineCount == 1) - { - _log.info("Only one HTO is used, will not use hashing"); - return null; - } - - _log.debug("total cached readset/hashing readset pairs: " + readsetToHashing.size()); - _log.debug("unique HTOs: " + lineCount); - - //prepare whitelist of cell indexes - File cellBarcodeWhitelist = getValidCellIndexFile(); - Set uniqueBarcodes = new HashSet<>(); - Set uniqueBarcodesIncludingNoCDR3 = new HashSet<>(); - _log.debug("writing cell barcodes, using file: " + perCellTsv.getPath()); - try (CSVWriter writer = new CSVWriter(PrintWriters.getPrintWriter(cellBarcodeWhitelist), ',', CSVWriter.NO_QUOTE_CHARACTER); CSVReader reader = new CSVReader(Readers.getReader(perCellTsv), ',')) - { - int rowIdx = 0; - int noCallRows = 0; - int nonCell = 0; - String[] row; - while ((row = reader.readNext()) != null) - { - //skip header - rowIdx++; - if (rowIdx > 1) - { - if ("False".equalsIgnoreCase(row[1])) - { - nonCell++; - continue; - } - - //NOTE: allow these to pass for cell-hashing under some conditions - boolean hasCDR3 = !"None".equals(row[12]); - if (!hasCDR3) - { - noCallRows++; - } - - //NOTE: 10x appends "-1" to barcodes - String barcode = row[0].split("-")[0]; - if (hasCDR3 && !uniqueBarcodes.contains(barcode)) - { - writer.writeNext(new String[]{barcode}); - uniqueBarcodes.add(barcode); - } - - uniqueBarcodesIncludingNoCDR3.add(barcode); - } - } - - _log.debug("rows inspected: " + (rowIdx - 1)); - _log.debug("rows without CDR3: " + noCallRows); - _log.debug("rows not called as cells: " + nonCell); - _log.debug("unique cell barcodes (with CDR3): " + uniqueBarcodes.size()); - _log.debug("unique cell barcodes (including no CDR3): " + uniqueBarcodesIncludingNoCDR3.size()); - output.addIntermediateFile(cellBarcodeWhitelist); - } - catch (IOException e) - { - throw new PipelineJobException(e); - } - - if (uniqueBarcodes.size() < 500 && uniqueBarcodesIncludingNoCDR3.size() > uniqueBarcodes.size()) - { - _log.info("Total cell barcodes with CDR3s is low, so cell hashing will be performing using an input that includes valid cells that lacked CDR3 data."); - try (CSVWriter writer = new CSVWriter(PrintWriters.getPrintWriter(cellBarcodeWhitelist), ',', CSVWriter.NO_QUOTE_CHARACTER)) - { - for (String barcode : uniqueBarcodesIncludingNoCDR3) - { - writer.writeNext(new String[]{barcode}); - } - } - catch (IOException e) - { - throw new PipelineJobException(e); - } - } - - Readset htoReadset = support.getCachedReadset(readsetToHashing.get(rs.getReadsetId())); - if (htoReadset == null) - { - throw new PipelineJobException("Unable to find HTO readset for readset: " + rs.getRowId()); - } - - //run CiteSeqCount. this will use Multiseq to make calls per cell - String basename = FileUtil.makeLegalName(rs.getName()); - File hashtagCalls = SequencePipelineService.get().runCiteSeqCount(output, outputCategory, htoReadset, htoBarcodeWhitelist, cellBarcodeWhitelist, workingDir, basename, _log, extraParams, false, minCountPerCell, sourceDir, editDistance, scanEditDistances, rs, genomeId, true, true, useSeurat, useMultiSeq); - if (!hashtagCalls.exists()) - { - throw new PipelineJobException("Unable to find expected file: " + hashtagCalls.getPath()); - } - output.addOutput(hashtagCalls, HASHING_CALLS); - - File html = new File(hashtagCalls.getParentFile(), FileUtil.getBaseName(FileUtil.getBaseName(hashtagCalls.getName())) + ".html"); - if (!html.exists()) - { - throw new PipelineJobException("Unable to find HTML file: " + html.getPath()); - } - - output.addOutput(html, "Cell Hashing TCR Report"); - - return hashtagCalls; } public void importAssayData(PipelineJob job, AnalysisModel model, File outDir, Integer assayId, @Nullable Integer runId, boolean deleteExisting) throws PipelineJobException @@ -521,9 +116,9 @@ public void importAssayData(PipelineJob job, AnalysisModel model, File outDir, I job.getLogger().debug("Using supplied runId: " + runId); } - File cDNAFile = getCDNAInfoFile(); - Map htoNameToCDNAMap = new HashMap<>(); - Map cDNAMap = new HashMap<>(); + File cDNAFile = CellHashingService.get().getCDNAInfoFile(outDir); + Map htoNameToCDNAMap = new HashMap<>(); + Map cDNAMap = new HashMap<>(); if (cDNAFile.exists()) { try (CSVReader reader = new CSVReader(Readers.getReader(cDNAFile), '\t')) @@ -539,7 +134,7 @@ public void importAssayData(PipelineJob job, AnalysisModel model, File outDir, I String htoName = StringUtils.trimToNull(line[7]); - CDNA cdna = CDNA.getRowId(Integer.parseInt(line[1])); + CDNA_Library cdna = CellHashingService.get().getLibraryById(Integer.parseInt(line[1])); cDNAMap.put(Integer.parseInt(line[1]), cdna); if (htoName != null) { @@ -627,7 +222,7 @@ else if ("Negative".equals(hto)) continue; } - CDNA cDNA = htoNameToCDNAMap.get(hto); + CDNA_Library cDNA = htoNameToCDNAMap.get(hto); if (cDNA == null) { _log.warn("Unable to find cDNA record for hto: " + hto); @@ -865,7 +460,7 @@ private AssayModel createForRow(String[] line, String sequenceContigName, Intege private File getCellToHtoFile(ExpRun run) throws PipelineJobException { - List datas = run.getInputDatas(HASHING_CALLS, ExpProtocol.ApplicationType.ExperimentRunOutput); + List datas = run.getInputDatas(CellHashingService.HASHING_CALLS, ExpProtocol.ApplicationType.ExperimentRunOutput); if (datas.isEmpty()) { throw new PipelineJobException("Unable to find hashing calls output"); @@ -901,9 +496,9 @@ private static class AssayModel private String sequenceContigName; } - private Map processRow(AssayModel assayModel, AnalysisModel model, Map cDNAMap, Integer runId, Map> totalCellsBySample, Map sequenceMap) throws PipelineJobException + private Map processRow(AssayModel assayModel, AnalysisModel model, Map cDNAMap, Integer runId, Map> totalCellsBySample, Map sequenceMap) throws PipelineJobException { - CDNA cDNARecord = cDNAMap.get(assayModel.cdna); + CDNA_Library cDNARecord = cDNAMap.get(assayModel.cdna); if (cDNARecord == null) { throw new PipelineJobException("Unable to find cDNA for ID: " + assayModel.cdna); @@ -912,8 +507,8 @@ private Map processRow(AssayModel assayModel, AnalysisModel mode Map row = new CaseInsensitiveHashMap<>(); row.put("sampleName", cDNARecord.getAssaySampleName()); - row.put("subjectId", cDNARecord.getSortRecord().getStimRecord().getAnimalId()); - row.put("sampleDate", cDNARecord.getSortRecord().getStimRecord().getDate()); + row.put("subjectId", cDNARecord.getSortRecord().getSampleRecord().getSubjectId()); + row.put("sampleDate", cDNARecord.getSortRecord().getSampleRecord().getSampledate()); row.put("cDNA", assayModel.cdna); row.put("alignmentId", model.getAlignmentFile()); @@ -963,7 +558,7 @@ private void saveRun(PipelineJob job, ExpProtocol protocol, AnalysisModel model, if (model.getLibraryId() != null) { - TableSelector ts = new TableSelector(TCRdbSchema.getInstance().getSchema().getTable(TCRdbSchema.TABLE_LIBRARIES), PageFlowUtil.set("rowid"), new SimpleFilter(FieldKey.fromString("libraryId"), model.getLibraryId()), null); + TableSelector ts = new TableSelector(TCRdbSchema.getInstance().getSchema().getTable(TCRdbSchema.TABLE_MIXCR_LIBRARIES), PageFlowUtil.set("rowid"), new SimpleFilter(FieldKey.fromString("libraryId"), model.getLibraryId()), null); if (ts.exists()) { int mixcrId = ts.getObject(Integer.class); @@ -1052,307 +647,8 @@ public static void deleteExistingData(AssayProvider ap, ExpProtocol protocol, Co } } - public static Map getCachedHashingReadsetMap(SequenceAnalysisJobSupport support) throws PipelineJobException - { - return support.getCachedObject(CellRangerVDJUtils.READSET_TO_HASHING_MAP, PipelineJob.createObjectMapper().getTypeFactory().constructParametricType(Map.class, Integer.class, Integer.class)); - } - - public static Map getCachedCiteSeqReadsetMap(SequenceAnalysisJobSupport support) throws PipelineJobException - { - return support.getCachedObject(CellRangerVDJUtils.READSET_TO_CITESEQ_MAP, PipelineJob.createObjectMapper().getTypeFactory().constructParametricType(Map.class, Integer.class, Integer.class)); - } - - //NOTE: if readset ID is null, this will be interpreted as any readset using hashing - public boolean useCellHashing(SequenceAnalysisJobSupport support) throws PipelineJobException - { - Map gexToHashingMap = getCachedHashingReadsetMap(support); - if (gexToHashingMap == null || gexToHashingMap.isEmpty()) - return false; - - File htoBarcodeWhitelist = getValidHashingBarcodeFile(); - if (!htoBarcodeWhitelist.exists()) - { - throw new PipelineJobException("Unable to find file: " + htoBarcodeWhitelist.getPath()); - } - - return SequencePipelineService.get().getLineCount(htoBarcodeWhitelist) > 1; - } - - //NOTE: if readset ID is null, this will be interpreted as any readset using hashing - public boolean useCiteSeq(SequenceAnalysisJobSupport support, List inputFiles) throws PipelineJobException - { - Map gexToCiteMap = getCachedCiteSeqReadsetMap(support); - if (gexToCiteMap == null || gexToCiteMap.isEmpty()) - return false; - - for (SequenceOutputFile so : inputFiles) - { - if (gexToCiteMap.containsKey(so.getReadset())) - { - return true; - } - } - - return false; - } - - public static class CDNA + public static File getPerCellCsv(File outDir) { - private int _rowId; - private Integer _sortId; - private String _chemistry; - private Double _concentration; - private String _plateId; - private String _well; - - private Integer _readsetId; - private Integer _enrichedReadsetId; - private Integer _hashingReadsetId; - private String _container; - - private Sort _sortRecord; - - public int getRowId() - { - return _rowId; - } - - public void setRowId(int rowId) - { - _rowId = rowId; - } - - public Integer getSortId() - { - return _sortId; - } - - public void setSortId(Integer sortId) - { - _sortId = sortId; - } - - public String getChemistry() - { - return _chemistry; - } - - public void setChemistry(String chemistry) - { - _chemistry = chemistry; - } - - public Double getConcentration() - { - return _concentration; - } - - public void setConcentration(Double concentration) - { - _concentration = concentration; - } - - public String getPlateId() - { - return _plateId; - } - - public void setPlateId(String plateId) - { - _plateId = plateId; - } - - public String getWell() - { - return _well; - } - - public void setWell(String well) - { - _well = well; - } - - public Integer getReadsetId() - { - return _readsetId; - } - - public void setReadsetId(Integer readsetId) - { - _readsetId = readsetId; - } - - public Integer getEnrichedReadsetId() - { - return _enrichedReadsetId; - } - - public void setEnrichedReadsetId(Integer enrichedReadsetId) - { - _enrichedReadsetId = enrichedReadsetId; - } - - public Integer getHashingReadsetId() - { - return _hashingReadsetId; - } - - public void setHashingReadsetId(Integer hashingReadsetId) - { - _hashingReadsetId = hashingReadsetId; - } - - public String getContainer() - { - return _container; - } - - public void setContainer(String container) - { - _container = container; - } - - public Sort getSortRecord() - { - if (_sortRecord == null) - { - _sortRecord = Sort.getRowId(_sortId); - } - - return _sortRecord; - } - - public String getAssaySampleName() - { - return getPlateId() + "_" + getWell() + "_" + getSortRecord().getStimRecord().getAnimalId() + "_" + getSortRecord().getStimRecord().getStim() + "_" + getSortRecord().getPopulation() + (getSortRecord().getHto() == null ? "" : "_" + getSortRecord().getHto()); - } - - public static CDNA getRowId(int rowId) - { - return new TableSelector(TCRdbSchema.getInstance().getSchema().getTable(TCRdbSchema.TABLE_CDNAS)).getObject(rowId, CDNA.class); - } - } - - public static class Sort - { - private int _rowId; - private Integer _stimId; - private String _population; - private String _hto; - - private Stim _stimRecord; - - public Stim getStimRecord() - { - if (_stimRecord == null) - { - _stimRecord = Stim.getRowId(_stimId); - } - - return _stimRecord; - } - - public int getRowId() - { - return _rowId; - } - - public void setRowId(int rowId) - { - _rowId = rowId; - } - - public Integer getStimId() - { - return _stimId; - } - - public void setStimId(Integer stimId) - { - _stimId = stimId; - } - - public String getPopulation() - { - return _population; - } - - public void setPopulation(String population) - { - _population = population; - } - - public String getHto() - { - return _hto; - } - - public void setHto(String hto) - { - _hto = hto; - } - - public void setStimRecord(Stim stimRecord) - { - _stimRecord = stimRecord; - } - - public static Sort getRowId(int rowId) - { - return new TableSelector(TCRdbSchema.getInstance().getSchema().getTable(TCRdbSchema.TABLE_SORTS)).getObject(rowId, Sort.class); - } - } - - public static class Stim - { - private int _rowId; - private String _animalId; - private String _stim; - private Date _date; - - public int getRowId() - { - return _rowId; - } - - public void setRowId(int rowId) - { - _rowId = rowId; - } - - public String getAnimalId() - { - return _animalId; - } - - public void setAnimalId(String animalId) - { - _animalId = animalId; - } - - public String getStim() - { - return _stim; - } - - public void setStim(String stim) - { - _stim = stim; - } - - public Date getDate() - { - return _date; - } - - public void setDate(Date date) - { - _date = date; - } - - public static Stim getRowId(int rowId) - { - return new TableSelector(TCRdbSchema.getInstance().getSchema().getTable(TCRdbSchema.TABLE_STIMS)).getObject(rowId, Stim.class); - } + return new File(outDir, "all_contig_annotations.csv"); } } diff --git a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJWrapper.java b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJWrapper.java deleted file mode 100644 index 19fe214ca..000000000 --- a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJWrapper.java +++ /dev/null @@ -1,727 +0,0 @@ -package org.labkey.tcrdb.pipeline; - -import au.com.bytecode.opencsv.CSVReader; -import org.apache.commons.io.FileUtils; -import org.apache.commons.lang3.StringUtils; -import org.apache.logging.log4j.Logger; -import org.jetbrains.annotations.Nullable; -import org.json.JSONObject; -import org.labkey.api.collections.CaseInsensitiveHashMap; -import org.labkey.api.data.CompareType; -import org.labkey.api.data.ConvertHelper; -import org.labkey.api.data.DbSchema; -import org.labkey.api.data.DbSchemaType; -import org.labkey.api.data.SimpleFilter; -import org.labkey.api.data.Table; -import org.labkey.api.data.TableInfo; -import org.labkey.api.data.TableSelector; -import org.labkey.api.pipeline.PipelineJob; -import org.labkey.api.pipeline.PipelineJobException; -import org.labkey.api.query.FieldKey; -import org.labkey.api.query.QueryService; -import org.labkey.api.query.UserSchema; -import org.labkey.api.reader.Readers; -import org.labkey.api.sequenceanalysis.RefNtSequenceModel; -import org.labkey.api.sequenceanalysis.model.AnalysisModel; -import org.labkey.api.sequenceanalysis.model.ReadData; -import org.labkey.api.sequenceanalysis.model.Readset; -import org.labkey.api.sequenceanalysis.pipeline.AbstractAlignmentStepProvider; -import org.labkey.api.sequenceanalysis.pipeline.AlignerIndexUtil; -import org.labkey.api.sequenceanalysis.pipeline.AlignmentOutputImpl; -import org.labkey.api.sequenceanalysis.pipeline.AlignmentStep; -import org.labkey.api.sequenceanalysis.pipeline.AlignmentStepProvider; -import org.labkey.api.sequenceanalysis.pipeline.CommandLineParam; -import org.labkey.api.sequenceanalysis.pipeline.IndexOutputImpl; -import org.labkey.api.sequenceanalysis.pipeline.PipelineContext; -import org.labkey.api.sequenceanalysis.pipeline.ReferenceGenome; -import org.labkey.api.sequenceanalysis.pipeline.SequenceAnalysisJobSupport; -import org.labkey.api.sequenceanalysis.pipeline.SequencePipelineService; -import org.labkey.api.sequenceanalysis.pipeline.ToolParameterDescriptor; -import org.labkey.api.sequenceanalysis.run.AbstractAlignmentPipelineStep; -import org.labkey.api.sequenceanalysis.run.AbstractCommandWrapper; -import org.labkey.api.sequenceanalysis.run.SimpleScriptWrapper; -import org.labkey.api.util.FileUtil; -import org.labkey.api.util.PageFlowUtil; -import org.labkey.api.writer.PrintWriters; - -import java.io.File; -import java.io.IOException; -import java.io.PrintWriter; -import java.nio.file.Files; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Date; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -public class CellRangerVDJWrapper extends AbstractCommandWrapper -{ - public CellRangerVDJWrapper(@Nullable Logger logger) - { - super(logger); - } - - public static final String TARGET_ASSAY = "targetAssay"; - public static final String DELETE_EXISTING_ASSAY_DATA = "deleteExistingAssayData"; - public static final String INNER_ENRICHMENT_PRIMERS = "innerEnrichmentPrimers"; - - public static class VDJProvider extends AbstractAlignmentStepProvider - { - public VDJProvider() - { - super("CellRanger VDJ", "Cell Ranger is an alignment/analysis pipeline specific to 10x genomic data, and this can only be used on fastqs generated by 10x.", Arrays.asList( - //--sample - - ToolParameterDescriptor.create("id", "Run ID Suffix", "If provided, this will be appended to the ID of this run (readset name will be first).", "textfield", new JSONObject(){{ - put("allowBlank", true); - }}, null), - ToolParameterDescriptor.createCommandLineParam(CommandLineParam.create("--force-cells"), "force-cells", "Force Cells", "Force pipeline to use this number of cells, bypassing the cell detection algorithm. Use this if the number of cells estimated by Cell Ranger is not consistent with the barcode rank plot.", "ldk-integerfield", new JSONObject(){{ - put("minValue", 0); - }}, null), - ToolParameterDescriptor.createCommandLineParam(CommandLineParam.createSwitch("--disable-ui"), "disable-ui", "Disable UI", "If checked, this will run cellranger with the optional web-based UI disabled.", "checkbox", new JSONObject(){{ - put("checked", true); - }}, true), - ToolParameterDescriptor.create(INNER_ENRICHMENT_PRIMERS, "Inner Enrichment Primers", "An option comma-separated list of the inner primers used for TCR enrichment. These will be used for trimming.", "textarea", new JSONObject(){{ - put("height", 100); - put("width", 400); - }}, null), - ToolParameterDescriptor.create(TARGET_ASSAY, "Target Assay", "Results will be loaded into this assay. If no assay is selected, a table will be created with nothing in the DB.", "tcr-assayselectorfield", new JSONObject(){{ - put("autoSelectAssay", false); - }}, null), - ToolParameterDescriptor.create(DELETE_EXISTING_ASSAY_DATA, "Delete Any Existing Assay Data", "If selected, prior to importing assay data, and existing assay runs in the target container from this readset will be deleted.", "checkbox", new JSONObject(){{ - put("checked", true); - }}, true), - ToolParameterDescriptor.create("excludeFailedcDNA", "Exclude Failed cDNA", "If selected, cDNAs with non-blank status fields will be omitted", "checkbox", null, true) - - ), PageFlowUtil.set("tcrdb/field/AssaySelectorField.js"), "https://support.10xgenomics.com/single-cell-gene-expression/software/pipelines/latest/what-is-cell-ranger", true, false, false, ALIGNMENT_MODE.MERGE_THEN_ALIGN); - } - - @Override - public boolean shouldRunIdxstats() - { - return false; - } - - public String getName() - { - return "CellRanger-VDJ"; - } - - public String getDescription() - { - return null; - } - - public AlignmentStep create(PipelineContext context) - { - return new CellRangerVDJAlignmentStep(this, context, new CellRangerVDJWrapper(context.getLogger())); - } - } - - public static class CellRangerVDJAlignmentStep extends AbstractAlignmentPipelineStep implements AlignmentStep - { - private CellRangerVDJUtils _utils = null; - - private CellRangerVDJUtils getUtils() - { - if (_utils == null) - { - _utils = new CellRangerVDJUtils(getPipelineCtx().getLogger(), getPipelineCtx().getSourceDirectory()); - } - - return _utils; - } - - public CellRangerVDJAlignmentStep(AlignmentStepProvider provider, PipelineContext ctx, CellRangerVDJWrapper wrapper) - { - super(provider, ctx, wrapper); - } - - @Override - public boolean supportsMetrics() - { - return false; - } - - @Override - public void init(SequenceAnalysisJobSupport support) throws PipelineJobException - { - ReferenceGenome referenceGenome = support.getCachedGenomes().iterator().next(); - boolean hasCachedIndex = AlignerIndexUtil.hasCachedIndex(this.getPipelineCtx(), getIndexCachedDirName(getPipelineCtx().getJob()), referenceGenome); - if (!hasCachedIndex) - { - getPipelineCtx().getLogger().info("Creating FASTA for CellRanger VDJ Index for genome: " + referenceGenome.getName()); - File fasta = getGenomeFasta(); - try (PrintWriter writer = PrintWriters.getPrintWriter(fasta)) - { - final AtomicInteger i = new AtomicInteger(0); - UserSchema us = QueryService.get().getUserSchema(getPipelineCtx().getJob().getUser(), getPipelineCtx().getJob().getContainer(), "sequenceanalysis"); - List seqIds = new TableSelector(us.getTable("reference_library_members", null), PageFlowUtil.set("ref_nt_id"), new SimpleFilter(FieldKey.fromString("library_id"), referenceGenome.getGenomeId()), null).getArrayList(Integer.class); - new TableSelector(us.getTable("ref_nt_sequences", null), new SimpleFilter(FieldKey.fromString("rowid"), seqIds, CompareType.IN), null).forEach(RefNtSequenceModel.class, nt -> { - - if (nt.getLocus() == null) - { - throw new IllegalArgumentException("Locus was empty for NT with ID: " + nt.getRowid()); - } - - //NOTE: this allows dual TRA/TRD segments - String[] loci = nt.getLocus().split("/"); - for (String locus : loci) - { - i.getAndIncrement(); //cant use sequenceId since sequences might be represented multiple times across loci - - String seq = nt.getSequence(); - - //example: >1|TRAV41*01 TRAV41|TRAV41|L-REGION+V-REGION|TR|TRA|None|None - StringBuilder header = new StringBuilder(); - header.append(">").append(i.get()).append("|").append(nt.getName()).append(" ").append(nt.getLineage()).append("|").append(nt.getLineage()).append("|"); - //translate into V_Region - String type; - if (nt.getLineage().contains("J")) - { - type = "J-REGION"; - } - else if (nt.getLineage().contains("V")) - { - if (seq.length() < 300) - { - getPipelineCtx().getLogger().info("Using V-REGION due to short length: " + nt.getName() + " / " + nt.getSeqLength()); - type = "V-REGION"; - } - else - { - type = "L-REGION+V-REGION"; - } - } - else if (nt.getLineage().contains("C")) - { - type = "C-REGION"; - } - else if (nt.getLineage().contains("D")) - { - type = "D-REGION"; - } - else - { - throw new RuntimeException("Unknown lineage: " + nt.getLineage()); - } - - header.append(type).append("|TR|").append(locus).append("|None|None"); - - writer.write(header + "\n"); - writer.write(seq + "\n"); - } - nt.clearCachedSequence(); - }); - } - catch (IllegalArgumentException | IOException e) - { - throw new PipelineJobException(e); - } - } - - boolean excludeFailedcDNA = getProvider().getParameterByName("excludeFailedcDNA").extractValue(getPipelineCtx().getJob(), getProvider(), getStepIdx(), Boolean.class, true); - getUtils().prepareHashingAndCiteSeqFilesIfNeeded(getPipelineCtx().getJob(), getPipelineCtx().getSequenceSupport(), "enrichedReadsetId", excludeFailedcDNA, false, false); - } - - private File getGenomeFasta() - { - return new File(getPipelineCtx().getSourceDirectory(), "cellRangerVDJ.fasta"); - } - - @Override - public String getIndexCachedDirName(PipelineJob job) - { - return getProvider().getName(); - } - - @Override - public AlignmentStep.IndexOutput createIndex(ReferenceGenome referenceGenome, File outputDir) throws PipelineJobException - { - IndexOutputImpl output = new IndexOutputImpl(referenceGenome); - - File indexDir = new File(outputDir, getIndexCachedDirName(getPipelineCtx().getJob())); - boolean hasCachedIndex = AlignerIndexUtil.hasCachedIndex(this.getPipelineCtx(), getIndexCachedDirName(getPipelineCtx().getJob()), referenceGenome); - if (!hasCachedIndex) - { - getPipelineCtx().getLogger().info("Creating CellRanger VDJ Index"); - getPipelineCtx().getLogger().info("using file: " + getGenomeFasta().getPath()); - output.addIntermediateFile(getGenomeFasta()); - - //remove if directory exists - if (indexDir.exists()) - { - try - { - FileUtils.deleteDirectory(indexDir); - } - catch (IOException e) - { - throw new PipelineJobException(e); - } - } - - output.addInput(getGenomeFasta(), "Input FASTA"); - - List args = new ArrayList<>(); - args.add(getWrapper().getExe().getPath()); - args.add("mkvdjref"); - args.add("--seqs=" + getGenomeFasta().getPath()); - args.add("--genome=" + indexDir.getName()); - - getWrapper().setWorkingDir(indexDir.getParentFile()); - getWrapper().execute(args); - - output.appendOutputs(referenceGenome.getWorkingFastaFile(), indexDir); - - //recache if not already - AlignerIndexUtil.saveCachedIndex(hasCachedIndex, getPipelineCtx(), indexDir, getIndexCachedDirName(getPipelineCtx().getJob()), referenceGenome); - - } - - return output; - } - - @Override - public AlignmentStep.AlignmentOutput performAlignment(Readset rs, File inputFastq1, @Nullable File inputFastq2, File outputDirectory, ReferenceGenome referenceGenome, String basename, String readGroupId, @Nullable String platformUnit) throws PipelineJobException - { - AlignmentOutputImpl output = new AlignmentOutputImpl(); - - List args = new ArrayList<>(); - args.add(getWrapper().getExe().getPath()); - args.add("vdj"); - - String idParam = StringUtils.trimToNull(getProvider().getParameterByName("id").extractValue(getPipelineCtx().getJob(), getProvider(), getStepIdx(), String.class)); - String id = FileUtil.makeLegalName(rs.getName()) + (idParam == null ? "" : "-" + idParam); - id = id.replaceAll("[^a-zA-z0-9_\\-]", "_"); - args.add("--id=" + id); - - File indexDir = AlignerIndexUtil.getIndexDir(referenceGenome, getIndexCachedDirName(getPipelineCtx().getJob())); - args.add("--reference=" + indexDir.getPath()); - - String primers = StringUtils.trimToNull(getProvider().getParameterByName(INNER_ENRICHMENT_PRIMERS).extractValue(getPipelineCtx().getJob(), getProvider(), getStepIdx(), String.class, null)); - if (primers != null) - { - primers = primers.replaceAll("\\s+", ","); - primers = primers.replaceAll(",+", ","); - - File primerFile = new File(outputDirectory, "primers.txt"); - try (PrintWriter writer = PrintWriters.getPrintWriter(primerFile)) - { - Arrays.stream(primers.split(",")).forEach(x -> { - x = StringUtils.trimToNull(x); - if (x != null) - { - writer.println(x); - } - }); - } - catch (IOException e) - { - throw new PipelineJobException(e); - } - - output.addIntermediateFile(primerFile); - args.add("--inner-enrichment-primers=" + primerFile.getPath()); - } - - args.addAll(getClientCommandArgs("=")); - - Integer maxThreads = SequencePipelineService.get().getMaxThreads(getPipelineCtx().getLogger()); - if (maxThreads != null) - { - args.add("--localcores=" + maxThreads.toString()); - } - - Integer maxRam = SequencePipelineService.get().getMaxRam(); - if (maxRam != null) - { - args.add("--localmem=" + maxRam.toString()); - } - - File localFqDir = new File(outputDirectory, "localFq"); - output.addIntermediateFile(localFqDir); - Set sampleNames = prepareFastqSymlinks(rs, localFqDir); - args.add("--fastqs=" + localFqDir.getPath()); - - getPipelineCtx().getLogger().debug("Sample names: [" + StringUtils.join(sampleNames, ",") + "]"); - if (sampleNames.size() > 1) - { - args.add("--sample=" + StringUtils.join(sampleNames, ",")); - } - - getWrapper().setWorkingDir(outputDirectory); - - //Note: we can safely assume only this server is working on these files, so if the _lock file exists, it was from a previous failed job. - File lockFile = new File(outputDirectory, id + "/_lock"); - if (lockFile.exists()) - { - getPipelineCtx().getLogger().info("Lock file exists, deleting: " + lockFile.getPath()); - lockFile.delete(); - } - - getWrapper().execute(args); - - File outdir = new File(outputDirectory, id); - outdir = new File(outdir, "outs"); - - File bam = new File(outdir, "all_contig.bam"); - if (!bam.exists()) - { - throw new PipelineJobException("Unable to find file: " + bam.getPath()); - } - output.setBAM(bam); - - //NOTE: run these before cleanup in case of failure - Integer assayId = getProvider().getParameterByName(TARGET_ASSAY).extractValue(getPipelineCtx().getJob(), getProvider(), getStepIdx(), Integer.class); - if (assayId != null) - { - boolean scanEditDistances = getProvider().getParameterByName("scanEditDistances").extractValue(getPipelineCtx().getJob(), getProvider(), getStepIdx(), Boolean.class, false); - int editDistance = getProvider().getParameterByName("editDistance").extractValue(getPipelineCtx().getJob(), getProvider(), getStepIdx(), Integer.class, 2); - int minCountPerCell = getProvider().getParameterByName("minCountPerCell").extractValue(getPipelineCtx().getJob(), getProvider(), getStepIdx(), Integer.class, 3); - boolean useSeurat = getProvider().getParameterByName("useSeurat").extractValue(getPipelineCtx().getJob(), getProvider(), getStepIdx(), Boolean.class, true); - boolean useMultiSeq = getProvider().getParameterByName("useMultiSeq").extractValue(getPipelineCtx().getJob(), getProvider(), getStepIdx(), Boolean.class, true); - - getUtils().runRemoteVdjCellHashingTasks(output, CellRangerVDJCellHashingHandler.CATEGORY, getUtils().getPerCellCsv(output.getBAM().getParentFile()), rs, getPipelineCtx().getSequenceSupport(), null, getPipelineCtx().getWorkingDirectory(), getPipelineCtx().getSourceDirectory(), editDistance, scanEditDistances, referenceGenome.getGenomeId(), minCountPerCell, useSeurat, useMultiSeq); - } - else - { - getPipelineCtx().getLogger().debug("No target assay selected, skipping cell hashing steps"); - } - - //now do cleanup/rename: - try - { - String prefix = FileUtil.makeLegalName(rs.getName() + "_"); - File outputHtml = new File(outdir, "web_summary.html"); - if (!outputHtml.exists()) - { - throw new PipelineJobException("Unable to find file: " + outputHtml.getPath()); - } - - File outputHtmlRename = new File(outdir, prefix + outputHtml.getName()); - if (outputHtmlRename.exists()) - { - outputHtmlRename.delete(); - } - FileUtils.moveFile(outputHtml, outputHtmlRename); - - output.addSequenceOutput(outputHtmlRename, rs.getName() + " 10x VDJ Summary", "10x Run Summary", rs.getRowId(), null, referenceGenome.getGenomeId(), null); - - File outputVloupe = new File(outdir, "vloupe.vloupe"); - if (!outputVloupe.exists()) - { - throw new PipelineJobException("Unable to find file: " + outputVloupe.getPath()); - } - - File outputVloupeRename = new File(outdir, prefix + outputVloupe.getName()); - if (outputVloupeRename.exists()) - { - outputVloupeRename.delete(); - } - FileUtils.moveFile(outputVloupe, outputVloupeRename); - output.addSequenceOutput(outputVloupeRename, rs.getName() + " 10x VLoupe", "10x VLoupe", rs.getRowId(), null, referenceGenome.getGenomeId(), null); - } - catch (IOException e) - { - throw new PipelineJobException(e); - } - - //NOTE: this folder has many unnecessary files and symlinks that get corrupted when we rename the main outputs - File directory = new File(outdir.getParentFile(), "SC_VDJ_ASSEMBLER_CS"); - if (directory.exists()) - { - //NOTE: this will have lots of symlinks, including corrupted ones, which java handles badly - new SimpleScriptWrapper(getPipelineCtx().getLogger()).execute(Arrays.asList("rm", "-Rf", directory.getPath())); - } - else - { - getPipelineCtx().getLogger().warn("Unable to find folder: " + directory.getPath()); - } - - deleteSymlinks(localFqDir); - - return output; - } - - @Override - public boolean doAddReadGroups() - { - return false; - } - - @Override - public boolean doSortIndexBam() - { - return false; - } - - @Override - public boolean alwaysCopyIndexToWorkingDir() - { - return false; - } - - @Override - public boolean supportsGzipFastqs() - { - return true; - } - - private String getSymlinkFileName(String fileName, boolean doRename, String sampleName, int idx, boolean isReversed) - { - //NOTE: cellranger is very picky about file name formatting - if (doRename) - { - sampleName = FileUtil.makeLegalName(sampleName.replaceAll("_", "-")).replaceAll(" ", "-").replaceAll("\\.", "-");; - return sampleName + "_S1_L001_R" + (isReversed ? "2" : "1") + "_" + StringUtils.leftPad(String.valueOf(idx), 3, "0") + ".fastq.gz"; - } - else - { - Matcher m = FILE_PATTERN.matcher(fileName); - if (m.matches()) - { - if (!StringUtils.isEmpty(m.group(7))) - { - return m.group(1).replaceAll("_", "-") + StringUtils.trimToEmpty(m.group(2)) + "_L" + StringUtils.trimToEmpty(m.group(3)) + "_" + StringUtils.trimToEmpty(m.group(4)) + StringUtils.trimToEmpty(m.group(5)) + StringUtils.trimToEmpty(m.group(6)) + ".fastq.gz"; - } - else if (m.group(1).contains("_")) - { - getPipelineCtx().getLogger().info("replacing underscores in file/sample name"); - return m.group(1).replaceAll("_", "-") + StringUtils.trimToEmpty(m.group(2)) + "_L" + StringUtils.trimToEmpty(m.group(3)) + "_" + StringUtils.trimToEmpty(m.group(4)) + StringUtils.trimToEmpty(m.group(5)) + StringUtils.trimToEmpty(m.group(6)) + ".fastq.gz"; - } - else - { - getPipelineCtx().getLogger().info("no additional characters found"); - } - } - else - { - getPipelineCtx().getLogger().warn("filename does not match Illumina formatting: " + fileName); - } - } - - return FileUtil.makeLegalName(fileName); - } - - public Set prepareFastqSymlinks(Readset rs, File localFqDir) throws PipelineJobException - { - Set ret = new HashSet<>(); - if (!localFqDir.exists()) - { - localFqDir.mkdirs(); - } - - String[] files = localFqDir.list(); - if (files != null && files.length > 0) - { - deleteSymlinks(localFqDir); - } - - int idx = 0; - boolean doRename = true; //cellranger is too picky - simply rename files all the time - for (ReadData rd : rs.getReadData()) - { - idx++; - try - { - File target1 = new File(localFqDir, getSymlinkFileName(rd.getFile1().getName(), doRename, rs.getName(), idx, false)); - getPipelineCtx().getLogger().debug("file: " + rd.getFile1().getPath()); - getPipelineCtx().getLogger().debug("target: " + target1.getPath()); - if (target1.exists()) - { - getPipelineCtx().getLogger().debug("deleting existing symlink: " + target1.getName()); - Files.delete(target1.toPath()); - } - - Files.createSymbolicLink(target1.toPath(), rd.getFile1().toPath()); - ret.add(getSampleName(target1.getName())); - - if (rd.getFile2() != null) - { - File target2 = new File(localFqDir, getSymlinkFileName(rd.getFile2().getName(), doRename, rs.getName(), idx, true)); - getPipelineCtx().getLogger().debug("file: " + rd.getFile2().getPath()); - getPipelineCtx().getLogger().debug("target: " + target2.getPath()); - if (target2.exists()) - { - getPipelineCtx().getLogger().debug("deleting existing symlink: " + target2.getName()); - Files.delete(target2.toPath()); - } - Files.createSymbolicLink(target2.toPath(), rd.getFile2().toPath()); - ret.add(getSampleName(target2.getName())); - } - } - catch (IOException e) - { - throw new PipelineJobException(e); - } - } - - return ret; - } - - public void deleteSymlinks(File localFqDir) throws PipelineJobException - { - for (File fq : localFqDir.listFiles()) - { - try - { - getPipelineCtx().getLogger().debug("deleting symlink: " + fq.getName()); - Files.delete(fq.toPath()); - } - catch (IOException e) - { - throw new PipelineJobException(e); - } - } - } - - public void addMetrics(AnalysisModel model) throws PipelineJobException - { - getPipelineCtx().getLogger().debug("adding 10x metrics"); - - File metrics = new File(model.getAlignmentFileObject().getParentFile(), "metrics_summary.csv"); - if (metrics.exists()) - { - try (CSVReader reader = new CSVReader(Readers.getReader(metrics))) - { - String[] line; - String[] header = null; - String[] metricValues = null; - - int i = 0; - while ((line = reader.readNext()) != null) - { - if (i == 0) - { - header = line; - } - else - { - metricValues = line; - break; - } - - i++; - } - - int totalAdded = 0; - TableInfo ti = DbSchema.get("sequenceanalysis", DbSchemaType.Module).getTable("quality_metrics"); - - //NOTE: if this job errored and restarted, we may have duplicate records: - SimpleFilter filter = new SimpleFilter(FieldKey.fromString("readset"), model.getReadset()); - filter.addCondition(FieldKey.fromString("analysis_id"), model.getRowId(), CompareType.EQUAL); - filter.addCondition(FieldKey.fromString("dataid"), model.getAlignmentFile(), CompareType.EQUAL); - filter.addCondition(FieldKey.fromString("category"), "Cell Ranger VDJ", CompareType.EQUAL); - filter.addCondition(FieldKey.fromString("container"), getPipelineCtx().getJob().getContainer().getId(), CompareType.EQUAL); - TableSelector ts = new TableSelector(ti, PageFlowUtil.set("rowid"), filter, null); - if (ts.exists()) - { - getPipelineCtx().getLogger().info("Deleting existing QC metrics (probably from prior restarted job)"); - ts.getArrayList(Integer.class).forEach(rowid -> { - Table.delete(ti, rowid); - }); - } - - for (int j = 0; j < header.length; j++) - { - Map toInsert = new CaseInsensitiveHashMap<>(); - toInsert.put("container", getPipelineCtx().getJob().getContainer().getId()); - toInsert.put("createdby", getPipelineCtx().getJob().getUser().getUserId()); - toInsert.put("created", new Date()); - toInsert.put("readset", model.getReadset()); - toInsert.put("analysis_id", model.getRowId()); - toInsert.put("dataid", model.getAlignmentFile()); - - toInsert.put("category", "Cell Ranger VDJ"); - toInsert.put("metricname", header[j]); - - metricValues[j] = metricValues[j].replaceAll(",", ""); - Object val = metricValues[j]; - if (metricValues[j].contains("%")) - { - metricValues[j] = metricValues[j].replaceAll("%", ""); - Double d = ConvertHelper.convert(metricValues[j], Double.class); - d = d / 100.0; - val = d; - } - - toInsert.put("metricvalue", val); - - Table.insert(getPipelineCtx().getJob().getUser(), ti, toInsert); - totalAdded++; - } - - getPipelineCtx().getLogger().info("total metrics added: " + totalAdded); - } - catch (IOException e) - { - throw new PipelineJobException(e); - } - } - else - { - getPipelineCtx().getLogger().warn("unable to find metrics file: " + metrics.getPath()); - } - } - - public void complete(SequenceAnalysisJobSupport support, AnalysisModel model) throws PipelineJobException - { - addMetrics(model); - - File bam = model.getAlignmentData().getFile(); - if (bam.exists()) - { - Integer assayId = getProvider().getParameterByName(TARGET_ASSAY).extractValue(getPipelineCtx().getJob(), getProvider(), getStepIdx(), Integer.class); - Boolean deleteExisting = getProvider().getParameterByName(DELETE_EXISTING_ASSAY_DATA).extractValue(getPipelineCtx().getJob(), getProvider(), getStepIdx(), Boolean.class, false); - getUtils().importAssayData(getPipelineCtx().getJob(), model, bam.getParentFile(), assayId, null, deleteExisting); - } - else - { - getPipelineCtx().getLogger().warn("BAM not found, expected: " + bam.getPath()); - } - } - - private static Pattern FILE_PATTERN = Pattern.compile("^(.+?)(_S[0-9]+){0,1}_L(.+?)_(R){0,1}([0-9])(_[0-9]+){0,1}(.*?)(\\.f(ast){0,1}q)(\\.gz)?$"); - private static Pattern SAMPLE_PATTERN = Pattern.compile("^(.+)_S[0-9]+(.*)$"); - - private String getSampleName(String fn) - { - Matcher matcher = FILE_PATTERN.matcher(fn); - if (matcher.matches()) - { - String ret = matcher.group(1); - Matcher matcher2 = SAMPLE_PATTERN.matcher(ret); - if (matcher2.matches()) - { - ret = matcher2.group(1); - } - else - { - getPipelineCtx().getLogger().debug("_S not found in sample: [" + ret + "]"); - } - - ret = ret.replaceAll("_", "-"); - - return ret; - } - else - { - getPipelineCtx().getLogger().debug("file does not match illumina pattern: [" + fn + "]"); - } - - throw new IllegalArgumentException("Unable to infer Illumina sample name: " + fn); - } - } - - protected File getExe() - { - //NOTE: cellranger 4 doesnt work w/ custom libraries currently. update to CR4 when fixed - return SequencePipelineService.get().getExeForPackage("CELLRANGERPATH", "cellranger-31"); - } -} diff --git a/tcrdb/src/org/labkey/tcrdb/pipeline/MiXCRAnalysis.java b/tcrdb/src/org/labkey/tcrdb/pipeline/MiXCRAnalysis.java index 09b89b5c8..a9738b517 100644 --- a/tcrdb/src/org/labkey/tcrdb/pipeline/MiXCRAnalysis.java +++ b/tcrdb/src/org/labkey/tcrdb/pipeline/MiXCRAnalysis.java @@ -13,6 +13,7 @@ import org.labkey.api.assay.AssayService; import org.labkey.api.collections.CaseInsensitiveHashMap; import org.labkey.api.data.CompareType; +import org.labkey.api.data.Container; import org.labkey.api.data.SimpleFilter; import org.labkey.api.data.TableInfo; import org.labkey.api.data.TableSelector; @@ -25,6 +26,7 @@ import org.labkey.api.module.ModuleLoader; import org.labkey.api.pipeline.PipelineJobException; import org.labkey.api.query.FieldKey; +import org.labkey.api.query.QueryService; import org.labkey.api.query.ValidationException; import org.labkey.api.reader.Readers; import org.labkey.api.resource.FileResource; @@ -76,8 +78,6 @@ import java.util.TreeSet; import java.util.zip.GZIPInputStream; -import static org.labkey.tcrdb.pipeline.CellRangerVDJWrapper.DELETE_EXISTING_ASSAY_DATA; - /** * Created by bimber on 5/10/2016. @@ -126,7 +126,7 @@ public Provider() {{ put("value", "ALL"); }}, true), - ToolParameterDescriptor.create(DELETE_EXISTING_ASSAY_DATA, "Delete Any Existing Assay Data", "If selected, prior to importing assay data, and existing assay runs in the target container from this readset will be deleted.", "checkbox", new JSONObject(){{ + ToolParameterDescriptor.create(CellRangerVDJCellHashingHandler.DELETE_EXISTING_ASSAY_DATA, "Delete Any Existing Assay Data", "If selected, prior to importing assay data, and existing assay runs in the target container from this readset will be deleted.", "checkbox", new JSONObject(){{ put("checked", true); }}, true), ToolParameterDescriptor.create(FLAG_MISSENSE, "Flag Missense CDR3", "If checked, if a sample has duplicate CDR3 clones from the same locus, and and one of these is missense, that clone will be flagged and excluded from many reports.", "checkbox", new JSONObject() @@ -1215,6 +1215,12 @@ private void inspectForOrphanAlignment(Readset rs, String[] line, Set } } + private TableInfo getCdnaTable() + { + Container target = getPipelineCtx().getJob().getContainer().isWorkbook() ? getPipelineCtx().getJob().getContainer().getParent() : getPipelineCtx().getJob().getContainer(); + return QueryService.get().getUserSchema(getPipelineCtx().getJob().getUser(), target, TCRdbSchema.SINGLE_CELL).getTable(TCRdbSchema.TABLE_CDNAS); + } + private void parseCloneOutput(Map runMap, File table, AnalysisModel model, File inputBam) throws PipelineJobException { Integer runId = SequencePipelineService.get().getExpRunIdForJob(getPipelineCtx().getJob()); @@ -1223,6 +1229,7 @@ private void parseCloneOutput(Map runMap, File table, AnalysisM List cloneDatas = run.getInputDatas(CLONES_FILE, ExpProtocol.ApplicationType.ExperimentRunOutput); List vdjDatas = run.getInputDatas(FINAL_VDJ_FILE, ExpProtocol.ApplicationType.ExperimentRunOutput); + TableInfo cDNATable = getCdnaTable(); try (CSVReader reader = new CSVReader(Readers.getReader(table), '\t')) { int lineNo = 0; @@ -1235,7 +1242,7 @@ private void parseCloneOutput(Map runMap, File table, AnalysisM continue; } - Map row = getBaseRow(model, runId); + Map row = getBaseRow(model, runId, cDNATable); if (line.length != (FIELDS.size() + TOTAL_EXPORTED_FIELDS_NOT_IN_DB)) //this includes one additional field appended to the end { @@ -1323,7 +1330,7 @@ private void parseCloneOutput(Map runMap, File table, AnalysisM } } - private Map getBaseRow(AnalysisModel model, Integer runId) throws PipelineJobException + private Map getBaseRow(AnalysisModel model, Integer runId, TableInfo cDNATable) throws PipelineJobException { Map row = new CaseInsensitiveHashMap<>(); if (model.getReadset() != null) @@ -1356,13 +1363,12 @@ private Map getBaseRow(AnalysisModel model, Integer runId) throw row.put("analysisId", model.getRowId()); //attempt to locate cDNA: - TableInfo cDNATable = TCRdbSchema.getInstance().getSchema().getTable(TCRdbSchema.TABLE_CDNAS); if (model.getReadset() != null) { SimpleFilter filter = new SimpleFilter(); filter.addClause(new SimpleFilter.OrClause( new CompareType.CompareClause(FieldKey.fromString("readsetId"), CompareType.EQUAL, model.getReadset()), - new CompareType.CompareClause(FieldKey.fromString("enrichedReadsetId"), CompareType.EQUAL, model.getReadset()) + new CompareType.CompareClause(FieldKey.fromString("tcrReadsetId"), CompareType.EQUAL, model.getReadset()) )); TableSelector ts1 = new TableSelector(cDNATable, PageFlowUtil.set("rowId"), filter, null); @@ -1592,7 +1598,8 @@ private void importRun(RunData rd, File outDir, AnalysisModel model, ExpProtocol if (rd.rows.isEmpty()) { //NOTE: we need to add a placeholder row since assay import will die w/ a run-only import: - Map row = getBaseRow(model, runId); + TableInfo cDNATable = getCdnaTable(); + Map row = getBaseRow(model, runId, cDNATable); row.put("species", rd.species); row.put("libraryId", rd.libraryId); row.put("locus", "None"); @@ -1602,7 +1609,7 @@ private void importRun(RunData rd, File outDir, AnalysisModel model, ExpProtocol getPipelineCtx().getLogger().debug("saving assay file to: " + assayTmp.getPath()); AssayProvider ap = AssayService.get().getProvider(protocol); - boolean deleteExistingAssayData = getProvider().getParameterByName(DELETE_EXISTING_ASSAY_DATA).extractValue(getPipelineCtx().getJob(), getProvider(), getStepIdx(), Boolean.class, false); + boolean deleteExistingAssayData = getProvider().getParameterByName(CellRangerVDJCellHashingHandler.DELETE_EXISTING_ASSAY_DATA).extractValue(getPipelineCtx().getJob(), getProvider(), getStepIdx(), Boolean.class, false); if (deleteExistingAssayData) { if (model.getReadset() == null) diff --git a/tcrdb/src/org/labkey/tcrdb/pipeline/SeuratCellHashingHandler.java b/tcrdb/src/org/labkey/tcrdb/pipeline/SeuratCellHashingHandler.java deleted file mode 100644 index 8e526dfed..000000000 --- a/tcrdb/src/org/labkey/tcrdb/pipeline/SeuratCellHashingHandler.java +++ /dev/null @@ -1,153 +0,0 @@ -package org.labkey.tcrdb.pipeline; - -import org.json.JSONObject; -import org.labkey.api.module.ModuleLoader; -import org.labkey.api.pipeline.PipelineJob; -import org.labkey.api.pipeline.PipelineJobException; -import org.labkey.api.pipeline.RecordedAction; -import org.labkey.api.sequenceanalysis.SequenceOutputFile; -import org.labkey.api.sequenceanalysis.model.Readset; -import org.labkey.api.sequenceanalysis.pipeline.AbstractParameterizedOutputHandler; -import org.labkey.api.sequenceanalysis.pipeline.SequenceAnalysisJobSupport; -import org.labkey.api.sequenceanalysis.pipeline.SequenceOutputHandler; -import org.labkey.api.sequenceanalysis.pipeline.ToolParameterDescriptor; -import org.labkey.api.util.FileType; -import org.labkey.tcrdb.TCRdbModule; - -import java.io.File; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -public class SeuratCellHashingHandler extends AbstractParameterizedOutputHandler -{ - private FileType _fileType = new FileType(".seurat.rds", false); - public static final String CATEGORY = "Seurat Cell Hashing Calls"; - - public SeuratCellHashingHandler() - { - super(ModuleLoader.getInstance().getModule(TCRdbModule.class), "Seurat GEX/Cell Hashing", "This will run CiteSeqCount/MultiSeqClassifier to generate a sample-to-cellbarcode TSV based on the cell barcodes present in the saved Seurat object.", null, getDefaultParams()); - } - - private static List getDefaultParams() - { - List ret = new ArrayList<>(); - ret.add(ToolParameterDescriptor.create("useOutputFileContainer", "Submit to Source File Workbook", "If checked, each job will be submitted to the same workbook as the input file, as opposed to submitting all jobs to the same workbook. This is primarily useful if submitting a large batch of files to process separately..", "checkbox", new JSONObject() - {{ - put("checked", true); - }}, false)); - - ret.addAll(CellRangerCellHashingHandler.getDefaultHashingParams(true)); - - return ret; - } - - @Override - public boolean canProcess(SequenceOutputFile o) - { - return o.getFile() != null && _fileType.isType(o.getFile()); - } - - @Override - public boolean doRunRemote() - { - return true; - } - - @Override - public boolean doRunLocal() - { - return false; - } - - @Override - public SequenceOutputProcessor getProcessor() - { - return new Processor(); - } - - @Override - public boolean doSplitJobs() - { - return true; - } - - @Override - public boolean requiresSingleGenome() - { - return false; - } - - public class Processor implements SequenceOutputHandler.SequenceOutputProcessor - { - @Override - public void init(PipelineJob job, SequenceAnalysisJobSupport support, List inputFiles, JSONObject params, File outputDir, List actions, List outputsToCreate) throws UnsupportedOperationException, PipelineJobException - { - new CellRangerVDJUtils(job.getLogger(), outputDir).prepareHashingAndCiteSeqFilesIfNeeded(job, support, "readsetId", params.optBoolean("excludeFailedcDNA", true), true, false); - } - - @Override - public void processFilesOnWebserver(PipelineJob job, SequenceAnalysisJobSupport support, List inputFiles, JSONObject params, File outputDir, List actions, List outputsToCreate) throws UnsupportedOperationException, PipelineJobException - { - - } - - @Override - public void processFilesRemote(List inputFiles, SequenceOutputHandler.JobContext ctx) throws UnsupportedOperationException, PipelineJobException - { - RecordedAction action = new RecordedAction(getName()); - Map readsetToHashing = CellRangerVDJUtils.getCachedHashingReadsetMap(ctx.getSequenceSupport()); - ctx.getLogger().debug("total cached readset to hashing pairs: " + readsetToHashing.size()); - - for (SequenceOutputFile so : inputFiles) - { - ctx.getLogger().info("processing file: " + so.getName()); - - File barcodes = getBarcodesFromSeurat(so.getFile()); - - Readset rs = ctx.getSequenceSupport().getCachedReadset(so.getReadset()); - if (rs == null) - { - throw new PipelineJobException("Unable to find readset for outputfile: " + so.getRowid()); - } - else if (rs.getReadsetId() == null) - { - throw new PipelineJobException("Readset lacks a rowId for outputfile: " + so.getRowid()); - } - - Readset htoReadset = ctx.getSequenceSupport().getCachedReadset(readsetToHashing.get(rs.getReadsetId())); - if (htoReadset == null) - { - throw new PipelineJobException("Unable to find Hashing/Cite-seq readset for GEX readset: " + rs.getReadsetId()); - } - - CellRangerCellHashingHandler.processBarcodeFile(ctx, barcodes, rs, htoReadset, so.getLibrary_id(), action, getClientCommandArgs(ctx.getParams()), false, CATEGORY); - } - - ctx.addActions(action); - } - - @Override - public void complete(PipelineJob job, List inputs, List outputsCreated, SequenceAnalysisJobSupport support) throws PipelineJobException - { - for (SequenceOutputFile so : outputsCreated) - { - if (so.getCategory().equals(CATEGORY)) - { - CellRangerVDJCellHashingHandler.processMetrics(so, job, true); - } - } - } - } - - public static File getBarcodesFromSeurat(File seuratObj) throws PipelineJobException - { - File barcodes = new File(seuratObj.getParentFile(), seuratObj.getName().replaceAll("seurat.rds", "cellBarcodes.csv")); - if (!barcodes.exists()) - { - throw new PipelineJobException("Unable to find expected cell barcodes file. This might indicate the seurat object was created with an older version of the pipeline. Expected: " + barcodes.getPath()); - } - - return barcodes; - } -} diff --git a/tcrdb/src/org/labkey/tcrdb/pipeline/SeuratCiteSeqHandler.java b/tcrdb/src/org/labkey/tcrdb/pipeline/SeuratCiteSeqHandler.java deleted file mode 100644 index 582b5dd92..000000000 --- a/tcrdb/src/org/labkey/tcrdb/pipeline/SeuratCiteSeqHandler.java +++ /dev/null @@ -1,131 +0,0 @@ -package org.labkey.tcrdb.pipeline; - -import org.json.JSONObject; -import org.labkey.api.module.ModuleLoader; -import org.labkey.api.pipeline.PipelineJob; -import org.labkey.api.pipeline.PipelineJobException; -import org.labkey.api.pipeline.RecordedAction; -import org.labkey.api.sequenceanalysis.SequenceOutputFile; -import org.labkey.api.sequenceanalysis.model.Readset; -import org.labkey.api.sequenceanalysis.pipeline.AbstractParameterizedOutputHandler; -import org.labkey.api.sequenceanalysis.pipeline.SequenceAnalysisJobSupport; -import org.labkey.api.sequenceanalysis.pipeline.SequenceOutputHandler; -import org.labkey.api.sequenceanalysis.pipeline.ToolParameterDescriptor; -import org.labkey.api.util.FileType; -import org.labkey.tcrdb.TCRdbModule; - -import java.io.File; -import java.util.Arrays; -import java.util.List; -import java.util.Map; - -public class SeuratCiteSeqHandler extends AbstractParameterizedOutputHandler -{ - protected FileType _fileType = new FileType(".seurat.rds", false); - public static final String CATEGORY = "Seurat CITE-Seq Count Matrix"; - - public SeuratCiteSeqHandler() - { - super(ModuleLoader.getInstance().getModule(TCRdbModule.class), "Seurat GEX/CITE-seq Counts", "This will run CiteSeqCount to generate a sample-to-cellbarcode TSV based on the cell barcodes present in the saved Seurat object.", null, Arrays.asList( - ToolParameterDescriptor.create("editDistance", "Edit Distance", null, "ldk-integerfield", null, 2), - ToolParameterDescriptor.create("excludeFailedcDNA", "Exclude Failed cDNA", "If selected, cDNAs with non-blank status fields will be omitted", "checkbox", null, true), - ToolParameterDescriptor.create("minCountPerCell", "Min Reads/Cell (Cell Hashing)", null, "ldk-integerfield", null, 5), - ToolParameterDescriptor.create("useOutputFileContainer", "Submit to Source File Workbook", "If checked, each job will be submitted to the same workbook as the input file, as opposed to submitting all jobs to the same workbook. This is primarily useful if submitting a large batch of files to process separately..", "checkbox", new JSONObject() - {{ - put("checked", true); - }}, false) - )); - } - - @Override - public boolean canProcess(SequenceOutputFile o) - { - return o.getFile() != null && _fileType.isType(o.getFile()); - } - - @Override - public boolean doRunRemote() - { - return true; - } - - @Override - public boolean doRunLocal() - { - return false; - } - - @Override - public SequenceOutputProcessor getProcessor() - { - return new Processor(); - } - - @Override - public boolean doSplitJobs() - { - return true; - } - - @Override - public boolean requiresSingleGenome() - { - return false; - } - - public class Processor implements SequenceOutputHandler.SequenceOutputProcessor - { - @Override - public void init(PipelineJob job, SequenceAnalysisJobSupport support, List inputFiles, JSONObject params, File outputDir, List actions, List outputsToCreate) throws UnsupportedOperationException, PipelineJobException - { - new CellRangerVDJUtils(job.getLogger(), outputDir).prepareHashingAndCiteSeqFilesIfNeeded(job, support,"readsetId", params.optBoolean("excludeFailedcDNA", true), false, true); - } - - @Override - public void processFilesOnWebserver(PipelineJob job, SequenceAnalysisJobSupport support, List inputFiles, JSONObject params, File outputDir, List actions, List outputsToCreate) throws UnsupportedOperationException, PipelineJobException - { - - } - - @Override - public void processFilesRemote(List inputFiles, SequenceOutputHandler.JobContext ctx) throws UnsupportedOperationException, PipelineJobException - { - RecordedAction action = new RecordedAction(getName()); - - Map readsetToCiteSeq = CellRangerVDJUtils.getCachedCiteSeqReadsetMap(ctx.getSequenceSupport()); - ctx.getLogger().debug("total cached readset to GEX/citeseq pairs: " + readsetToCiteSeq.size()); - - for (SequenceOutputFile so : inputFiles) - { - ctx.getLogger().info("processing file: " + so.getName()); - - File barcodes = SeuratCellHashingHandler.getBarcodesFromSeurat(so.getFile()); - - Readset rs = ctx.getSequenceSupport().getCachedReadset(so.getReadset()); - if (rs == null) - { - throw new PipelineJobException("Unable to find readset for outputfile: " + so.getRowid()); - } - else if (rs.getReadsetId() == null) - { - throw new PipelineJobException("Readset lacks a rowId for outputfile: " + so.getRowid()); - } - - Readset citeseqReadset = ctx.getSequenceSupport().getCachedReadset(readsetToCiteSeq.get(rs.getReadsetId())); - if (citeseqReadset == null) - { - throw new PipelineJobException("Unable to find Cite-seq readset for GEX readset: " + rs.getReadsetId()); - } - - File adtWhitelist = CellRangerVDJUtils.getValidCiteSeqBarcodeFile(ctx.getSourceDirectory(), so.getReadset()); - File citeSeqMatrix = CellRangerCellHashingHandler.processBarcodeFile(ctx, barcodes, rs, citeseqReadset, so.getLibrary_id(), action, getClientCommandArgs(ctx.getParams()), false, CATEGORY, true, adtWhitelist, false); - if (!citeSeqMatrix.exists()) - { - throw new PipelineJobException("Unable to find expected file: " + citeSeqMatrix.getPath()); - } - } - - ctx.addActions(action); - } - } -} From c0480ede029a80bf2ab1a07cef6c33122d952335 Mon Sep 17 00:00:00 2001 From: bbimber Date: Wed, 30 Dec 2020 09:50:04 -0800 Subject: [PATCH 30/98] Add ManageVersion: false to modules --- LabPurchasing/module.properties | 1 + mcc/module.properties | 1 + 2 files changed, 2 insertions(+) diff --git a/LabPurchasing/module.properties b/LabPurchasing/module.properties index 577c924b4..cf4adca31 100644 --- a/LabPurchasing/module.properties +++ b/LabPurchasing/module.properties @@ -3,3 +3,4 @@ Label: Lab Purchacing Description: A module designed to assist with purchasing supplies for academic labs License: Apache 2.0 LicenseURL: http://www.apache.org/licenses/LICENSE-2.0 +ManageVersion: false \ No newline at end of file diff --git a/mcc/module.properties b/mcc/module.properties index 0ddf5e2dd..ab8619788 100644 --- a/mcc/module.properties +++ b/mcc/module.properties @@ -3,3 +3,4 @@ Label: Maromoset Coordinating Center Description: This module is used by the BRAIN Initiative Maromoset Coordinating Center License: Apache 2.0 LicenseURL: http://www.apache.org/licenses/LICENSE-2.0 +ManageVersion: false \ No newline at end of file From 8fc2d4b5352ece4267727f37e15fa1e687f64eef Mon Sep 17 00:00:00 2001 From: bbimber Date: Thu, 31 Dec 2020 08:56:11 -0800 Subject: [PATCH 31/98] Use outputfileId, not dataId --- mGAP/src/org/labkey/mgap/pipeline/mGapReleaseGenerator.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mGAP/src/org/labkey/mgap/pipeline/mGapReleaseGenerator.java b/mGAP/src/org/labkey/mgap/pipeline/mGapReleaseGenerator.java index 7e4df8200..df399f6b3 100644 --- a/mGAP/src/org/labkey/mgap/pipeline/mGapReleaseGenerator.java +++ b/mGAP/src/org/labkey/mgap/pipeline/mGapReleaseGenerator.java @@ -188,7 +188,8 @@ public void init(PipelineJob job, SequenceAnalysisJobSupport support, List Date: Sat, 2 Jan 2021 15:34:10 -0800 Subject: [PATCH 32/98] Issue query against parent --- mGAP/src/org/labkey/mgap/pipeline/mGapReleaseGenerator.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mGAP/src/org/labkey/mgap/pipeline/mGapReleaseGenerator.java b/mGAP/src/org/labkey/mgap/pipeline/mGapReleaseGenerator.java index df399f6b3..0fa095374 100644 --- a/mGAP/src/org/labkey/mgap/pipeline/mGapReleaseGenerator.java +++ b/mGAP/src/org/labkey/mgap/pipeline/mGapReleaseGenerator.java @@ -239,7 +239,8 @@ public void init(PipelineJob job, SequenceAnalysisJobSupport support, List idsWithRecord = new TableSelector(ti, PageFlowUtil.set("subjectname"), new SimpleFilter(FieldKey.fromString("subjectname"), ids, CompareType.IN), null).getArrayList(String.class); ids.removeAll(idsWithRecord); From e90c0c192c4cd4d4af3eb472049efc1b67073f80 Mon Sep 17 00:00:00 2001 From: bbimber Date: Sun, 3 Jan 2021 07:44:46 -0800 Subject: [PATCH 33/98] Clean up container logic --- mGAP/src/org/labkey/mgap/pipeline/mGapReleaseGenerator.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mGAP/src/org/labkey/mgap/pipeline/mGapReleaseGenerator.java b/mGAP/src/org/labkey/mgap/pipeline/mGapReleaseGenerator.java index 0fa095374..ce5ad8e2c 100644 --- a/mGAP/src/org/labkey/mgap/pipeline/mGapReleaseGenerator.java +++ b/mGAP/src/org/labkey/mgap/pipeline/mGapReleaseGenerator.java @@ -166,7 +166,8 @@ public Processor() public void init(PipelineJob job, SequenceAnalysisJobSupport support, List inputFiles, JSONObject params, File outputDir, List actions, List outputsToCreate) throws UnsupportedOperationException, PipelineJobException { job.getLogger().info("writing track/subset data to file"); - TableInfo releaseTracks = QueryService.get().getUserSchema(job.getUser(), (job.getContainer().isWorkbook() ? job.getContainer().getParent() : job.getContainer()), mGAPSchema.NAME).getTable(mGAPSchema.TABLE_RELEASE_TRACKS); + Container target = job.getContainer().isWorkbook() ? job.getContainer().getParent() : job.getContainer(); + TableInfo releaseTracks = QueryService.get().getUserSchema(job.getUser(), target, mGAPSchema.NAME).getTable(mGAPSchema.TABLE_RELEASE_TRACKS); Set toSelect = new HashSet<>(); toSelect.add(FieldKey.fromString("trackName")); @@ -239,8 +240,7 @@ public void init(PipelineJob job, SequenceAnalysisJobSupport support, List idsWithRecord = new TableSelector(ti, PageFlowUtil.set("subjectname"), new SimpleFilter(FieldKey.fromString("subjectname"), ids, CompareType.IN), null).getArrayList(String.class); ids.removeAll(idsWithRecord); From 255b6c2815d2e0ff06fdd789e18f66c5848d89b5 Mon Sep 17 00:00:00 2001 From: bbimber Date: Tue, 12 Jan 2021 11:04:24 -0800 Subject: [PATCH 34/98] Add dependencies, re-create outputfile if attributes differ --- mcc/build.gradle | 16 ++++++++++++++++ primeseq/build.gradle | 3 +++ 2 files changed, 19 insertions(+) create mode 100644 mcc/build.gradle diff --git a/mcc/build.gradle b/mcc/build.gradle new file mode 100644 index 000000000..87943e447 --- /dev/null +++ b/mcc/build.gradle @@ -0,0 +1,16 @@ +import org.labkey.gradle.util.BuildUtils; + +dependencies { + BuildUtils.addLabKeyDependency(project: project, config: "implementation", depProjectPath: ":server:modules:DiscvrLabKeyModules:jbrowse", depProjectConfig: "apiJarFile") + BuildUtils.addLabKeyDependency(project: project, config: "implementation", depProjectPath: ":server:modules:DiscvrLabKeyModules:SequenceAnalysis", depProjectConfig: "apiJarFile") + BuildUtils.addLabKeyDependency(project: project, config: "implementation", depProjectPath: ":server:modules:LabDevKitModules:laboratory", depProjectConfig: "apiJarFile") + BuildUtils.addLabKeyDependency(project: project, config: "implementation", depProjectPath: ":server:modules:LabDevKitModules:LDK", depProjectConfig: "apiJarFile") + BuildUtils.addLabKeyDependency(project: project, config: "implementation", depProjectPath: ":server:modules:dataintegration", depProjectConfig: "apiJarFile") + external "com.github.samtools:htsjdk:${htsjdkVersion}" + implementation "net.sf.opencsv:opencsv:${opencsvVersion}" + + BuildUtils.addLabKeyDependency(project: project, config: "modules", depProjectPath: ":server:modules:dataintegration", depProjectConfig: "published", depExtension: "module") + BuildUtils.addLabKeyDependency(project: project, config: "modules", depProjectPath: ":server:modules:LabDevKitModules:LDK", depProjectConfig: "published", depExtension: "module") + BuildUtils.addLabKeyDependency(project: project, config: "modules", depProjectPath: ":server:modules:DiscvrLabKeyModules:SequenceAnalysis", depProjectConfig: "published", depExtension: "module") + BuildUtils.addLabKeyDependency(project: project, config: "modules", depProjectPath: ":server:modules:LabDevKitModules:laboratory", depProjectConfig: "published", depExtension: "module") +} diff --git a/primeseq/build.gradle b/primeseq/build.gradle index bd248fbf7..b172c8848 100644 --- a/primeseq/build.gradle +++ b/primeseq/build.gradle @@ -14,4 +14,7 @@ dependencies { BuildUtils.addLabKeyDependency(project: project, config: "modules", depProjectPath: ":server:modules:DiscvrLabKeyModules:SequenceAnalysis", depProjectConfig: "published", depExtension: "module") BuildUtils.addLabKeyDependency(project: project, config: "modules", depProjectPath: ":server:modules:DiscvrLabKeyModules:cluster", depProjectConfig: "published", depExtension: "module") BuildUtils.addLabKeyDependency(project: project, config: "modules", depProjectPath: ":server:modules:DiscvrLabKeyModules:jbrowse", depProjectConfig: "published", depExtension: "module") + + BuildUtils.addLabKeyDependency(project: project, config: "implementation", depProjectPath: ":server:modules:dataintegration", depProjectConfig: "apiJarFile") + BuildUtils.addLabKeyDependency(project: project, config: "modules", depProjectPath: ":server:modules:dataintegration", depProjectConfig: "published", depExtension: "module") } From 4ddf5fd0ea030dade94cba59264a4d0e77590584 Mon Sep 17 00:00:00 2001 From: bbimber Date: Tue, 12 Jan 2021 11:17:42 -0800 Subject: [PATCH 35/98] Add credits --- mcc/resources/credits/dependencies.txt | 2 ++ mcc/resources/credits/jars.txt | 4 ++++ 2 files changed, 6 insertions(+) create mode 100644 mcc/resources/credits/dependencies.txt create mode 100644 mcc/resources/credits/jars.txt diff --git a/mcc/resources/credits/dependencies.txt b/mcc/resources/credits/dependencies.txt new file mode 100644 index 000000000..b3f521155 --- /dev/null +++ b/mcc/resources/credits/dependencies.txt @@ -0,0 +1,2 @@ +# direct external dependencies for project :server:modules:BimberLabKeyModules:mcc +htsjdk-2.21.3.jar diff --git a/mcc/resources/credits/jars.txt b/mcc/resources/credits/jars.txt new file mode 100644 index 000000000..0a2c7d2f9 --- /dev/null +++ b/mcc/resources/credits/jars.txt @@ -0,0 +1,4 @@ +{table} +Filename|Component|Version|Source|License|LabKey Dev|Purpose +htsjdk-2.21.3.jar|htsjdk|2.21.3|{link:htsjdk|http://samtools.github.io/htsjdk/}|{link:MIT License|http://opensource.org/licenses/MIT}|bbimber|A Java API for high-throughput sequencing data (HTS) formats +{table} \ No newline at end of file From 87b3da1a1dc48362dd9e498a6e775f81b9a9cbb1 Mon Sep 17 00:00:00 2001 From: bbimber Date: Wed, 13 Jan 2021 09:52:41 -0800 Subject: [PATCH 36/98] Add santity checks --- mGAP/src/org/labkey/mgap/pipeline/mGapReleaseGenerator.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/mGAP/src/org/labkey/mgap/pipeline/mGapReleaseGenerator.java b/mGAP/src/org/labkey/mgap/pipeline/mGapReleaseGenerator.java index ce5ad8e2c..25dc72490 100644 --- a/mGAP/src/org/labkey/mgap/pipeline/mGapReleaseGenerator.java +++ b/mGAP/src/org/labkey/mgap/pipeline/mGapReleaseGenerator.java @@ -307,6 +307,11 @@ else if (so.getCategory().endsWith("Release Track")) boolean testOnly = StringUtils.isEmpty(job.getParameters().get("testOnly")) ? false : ConvertHelper.convert(job.getParameters().get("testOnly"), boolean.class); + if (outputVCFMap.isEmpty()) + { + throw new PipelineJobException("No releases were found"); + } + String releaseId = new GUID().toString(); for (String release : outputVCFMap.keySet()) { From b2ff8bc6664d5a3daebd073775e2f359568f9d1f Mon Sep 17 00:00:00 2001 From: bbimber Date: Wed, 13 Jan 2021 21:07:55 -0800 Subject: [PATCH 37/98] Checkpoint toward streamlining cellhashing/citeseq code --- .../CellRangerVDJCellHashingHandler.java | 102 ++++++++++++++++-- 1 file changed, 92 insertions(+), 10 deletions(-) diff --git a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java index be242b0fd..5159a0294 100644 --- a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java +++ b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java @@ -1,5 +1,7 @@ package org.labkey.tcrdb.pipeline; +import au.com.bytecode.opencsv.CSVReader; +import au.com.bytecode.opencsv.CSVWriter; import org.apache.commons.lang3.StringUtils; import org.json.JSONObject; import org.labkey.api.data.ConvertHelper; @@ -7,6 +9,7 @@ import org.labkey.api.pipeline.PipelineJob; import org.labkey.api.pipeline.PipelineJobException; import org.labkey.api.pipeline.RecordedAction; +import org.labkey.api.reader.Readers; import org.labkey.api.sequenceanalysis.SequenceOutputFile; import org.labkey.api.sequenceanalysis.model.AnalysisModel; import org.labkey.api.sequenceanalysis.model.Readset; @@ -17,14 +20,19 @@ import org.labkey.api.sequenceanalysis.pipeline.ToolParameterDescriptor; import org.labkey.api.singlecell.CellHashingService; import org.labkey.api.util.FileType; +import org.labkey.api.util.FileUtil; import org.labkey.api.util.PageFlowUtil; +import org.labkey.api.writer.PrintWriters; import org.labkey.tcrdb.TCRdbModule; import java.io.File; +import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; +import java.util.HashSet; import java.util.LinkedHashSet; import java.util.List; +import java.util.Set; public class CellRangerVDJCellHashingHandler extends AbstractParameterizedOutputHandler { @@ -177,18 +185,15 @@ else if (rs.getReadsetId() == null) private void processVloupeFile(JobContext ctx, File perCellTsv, Readset rs, RecordedAction action, Integer genomeId) throws PipelineJobException { - List extraParams = new ArrayList<>(); - extraParams.addAll(getClientCommandArgs(ctx.getParams())); - - //prepare whitelist of cell indexes AlignmentOutputImpl output = new AlignmentOutputImpl(); - boolean scanEditDistances = ctx.getParams().optBoolean("scanEditDistances", false); - boolean useSeurat = ctx.getParams().optBoolean("useSeurat", true); - boolean useMultiSeq = ctx.getParams().optBoolean("useMultiSeq", true); - int minCountPerCell = ctx.getParams().optInt("minCountPerCell", 3); - int editDistance = ctx.getParams().optInt("editDistance", 2); - File cellToHto = CellHashingService.get().runRemoteVdjCellHashingTasks(output, CATEGORY, perCellTsv, rs, ctx.getSequenceSupport(), extraParams, ctx.getWorkingDirectory(), ctx.getSourceDirectory(), editDistance, scanEditDistances, genomeId, minCountPerCell, useSeurat, useMultiSeq); + CellHashingService.CellHashingParameters parameters = CellHashingService.CellHashingParameters.createFromJson(CellHashingService.BARCODE_TYPE.hashing, ctx.getParams(), null, rs); + parameters.cellBarcodeWhitelistFile = createCellbarcodeWhitelist(ctx, perCellTsv, true); + parameters.genomeId = genomeId; + parameters.outputCategory = CATEGORY; + parameters.basename = FileUtil.makeLegalName(rs.getName()); + + File cellToHto = CellHashingService.get().processCellHashingOrCiteSeqForParent(rs, output, ctx, parameters); if (CellHashingService.get().usesCellHashing(ctx.getSequenceSupport(), ctx.getSourceDirectory()) && cellToHto == null) { throw new PipelineJobException("Missing cell to HTO file"); @@ -196,7 +201,84 @@ private void processVloupeFile(JobContext ctx, File perCellTsv, Readset rs, Reco } ctx.getFileManager().addStepOutputs(action, output); + } + + private File createCellbarcodeWhitelist(JobContext ctx, File perCellTsv, boolean allowCellsLackingCDR3) throws PipelineJobException + { + //prepare whitelist of cell indexes based on TCR calls: + File cellBarcodeWhitelist = new File(ctx.getSourceDirectory(), "validCellIndexes.csv"); + Set uniqueBarcodes = new HashSet<>(); + Set uniqueBarcodesIncludingNoCDR3 = new HashSet<>(); + ctx.getLogger().debug("writing cell barcodes, using file: " + perCellTsv.getPath()); + ctx.getLogger().debug("allow cells lacking CDR3: " + allowCellsLackingCDR3); + try (CSVWriter writer = new CSVWriter(PrintWriters.getPrintWriter(cellBarcodeWhitelist), ',', CSVWriter.NO_QUOTE_CHARACTER); CSVReader reader = new CSVReader(Readers.getReader(perCellTsv), ',')) + { + int rowIdx = 0; + int noCallRows = 0; + int nonCell = 0; + String[] row; + while ((row = reader.readNext()) != null) + { + //skip header + rowIdx++; + if (rowIdx > 1) + { + if ("False".equalsIgnoreCase(row[1])) + { + nonCell++; + continue; + } + + //NOTE: allow these to pass for cell-hashing under some conditions + boolean hasCDR3 = !"None".equals(row[12]); + if (!hasCDR3) + { + noCallRows++; + } + + //NOTE: 10x appends "-1" to barcodes + String barcode = row[0].split("-")[0]; + if (hasCDR3 && !uniqueBarcodes.contains(barcode)) + { + writer.writeNext(new String[]{barcode}); + uniqueBarcodes.add(barcode); + } + + uniqueBarcodesIncludingNoCDR3.add(barcode); + } + } + + ctx.getLogger().debug("rows inspected: " + (rowIdx - 1)); + ctx.getLogger().debug("rows without CDR3: " + noCallRows); + ctx.getLogger().debug("rows not called as cells: " + nonCell); + ctx.getLogger().debug("unique cell barcodes (with CDR3): " + uniqueBarcodes.size()); + ctx.getLogger().debug("unique cell barcodes (including no CDR3): " + uniqueBarcodesIncludingNoCDR3.size()); + ctx.getFileManager().addIntermediateFile(cellBarcodeWhitelist); + } + catch (IOException e) + { + throw new PipelineJobException(e); + } + + if (uniqueBarcodes.size() < 500 && uniqueBarcodesIncludingNoCDR3.size() > uniqueBarcodes.size()) + { + ctx.getLogger().info("Total cell barcodes with CDR3s is low, so cell hashing will be performing using an input that includes valid cells that lacked CDR3 data."); + try (CSVWriter writer = new CSVWriter(PrintWriters.getPrintWriter(cellBarcodeWhitelist), ',', CSVWriter.NO_QUOTE_CHARACTER)) + { + for (String barcode : uniqueBarcodesIncludingNoCDR3) + { + writer.writeNext(new String[]{barcode}); + } + } + catch (IOException e) + { + throw new PipelineJobException(e); + } + } + + //TODO: consider looking up GEX data? + return cellBarcodeWhitelist; } } } \ No newline at end of file From 002f2963a7d2c06e4c6aab372fc8f05d2fa5a882 Mon Sep 17 00:00:00 2001 From: bbimber Date: Thu, 14 Jan 2021 12:26:15 -0800 Subject: [PATCH 38/98] Fix typo --- mcc/resources/etls/wnprc.xml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/mcc/resources/etls/wnprc.xml b/mcc/resources/etls/wnprc.xml index f0d8a115f..e5f1ee6d1 100644 --- a/mcc/resources/etls/wnprc.xml +++ b/mcc/resources/etls/wnprc.xml @@ -1,7 +1,7 @@ - SNPRC_Data - SNPRC Clinical/Demographics Data + WNPRC_Data + WNPRC Clinical/Demographics Data Copy to target @@ -13,7 +13,7 @@ - - + + From 4034b84ba29397e6e3b3b79d190d3cb828573e52 Mon Sep 17 00:00:00 2001 From: bbimber Date: Thu, 14 Jan 2021 13:42:20 -0800 Subject: [PATCH 39/98] Bugfix allowable set of barcodes for hashing --- .../CellRangerVDJCellHashingHandler.java | 39 +++++++++++++------ 1 file changed, 27 insertions(+), 12 deletions(-) diff --git a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java index 5159a0294..9ddc89d4e 100644 --- a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java +++ b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java @@ -187,20 +187,35 @@ private void processVloupeFile(JobContext ctx, File perCellTsv, Readset rs, Reco { AlignmentOutputImpl output = new AlignmentOutputImpl(); - CellHashingService.CellHashingParameters parameters = CellHashingService.CellHashingParameters.createFromJson(CellHashingService.BARCODE_TYPE.hashing, ctx.getParams(), null, rs); - parameters.cellBarcodeWhitelistFile = createCellbarcodeWhitelist(ctx, perCellTsv, true); - parameters.genomeId = genomeId; - parameters.outputCategory = CATEGORY; - parameters.basename = FileUtil.makeLegalName(rs.getName()); - - File cellToHto = CellHashingService.get().processCellHashingOrCiteSeqForParent(rs, output, ctx, parameters); - if (CellHashingService.get().usesCellHashing(ctx.getSequenceSupport(), ctx.getSourceDirectory()) && cellToHto == null) + List htosPerReadset = CellHashingService.get().getHtosForParentReadset(rs.getReadsetId(), ctx.getSourceDirectory(), ctx.getSequenceSupport()); + if (htosPerReadset.size() > 1) { - throw new PipelineJobException("Missing cell to HTO file"); + ctx.getLogger().info("Total HTOs for readset: " + htosPerReadset.size()); - } + CellHashingService.CellHashingParameters parameters = CellHashingService.CellHashingParameters.createFromJson(CellHashingService.BARCODE_TYPE.hashing, ctx.getSourceDirectory(), ctx.getParams(), null, rs, null); + parameters.cellBarcodeWhitelistFile = createCellbarcodeWhitelist(ctx, perCellTsv, true); + parameters.genomeId = genomeId; + parameters.outputCategory = CATEGORY; + parameters.basename = FileUtil.makeLegalName(rs.getName()); + parameters.allowableHtoOrCiteseqBarcodes = htosPerReadset; + + File cellToHto = CellHashingService.get().processCellHashingOrCiteSeqForParent(rs, output, ctx, parameters); + if (CellHashingService.get().usesCellHashing(ctx.getSequenceSupport(), ctx.getSourceDirectory()) && cellToHto == null) + { + throw new PipelineJobException("Missing cell to HTO file"); - ctx.getFileManager().addStepOutputs(action, output); + } + + ctx.getFileManager().addStepOutputs(action, output); + } + else if (htosPerReadset.size() == 1) + { + ctx.getLogger().info("Only single HTO used for lane, skipping cell hashing calling"); + } + else + { + ctx.getLogger().info("No HTOs found for readset"); + } } private File createCellbarcodeWhitelist(JobContext ctx, File perCellTsv, boolean allowCellsLackingCDR3) throws PipelineJobException @@ -238,7 +253,7 @@ private File createCellbarcodeWhitelist(JobContext ctx, File perCellTsv, boolean //NOTE: 10x appends "-1" to barcodes String barcode = row[0].split("-")[0]; - if (hasCDR3 && !uniqueBarcodes.contains(barcode)) + if ((allowCellsLackingCDR3 || hasCDR3) && !uniqueBarcodes.contains(barcode)) { writer.writeNext(new String[]{barcode}); uniqueBarcodes.add(barcode); From bfde6cd3f5db7be77b20e3d8bf21efc954f93bfe Mon Sep 17 00:00:00 2001 From: bbimber Date: Thu, 14 Jan 2021 14:42:33 -0800 Subject: [PATCH 40/98] Use job-specific logger --- mcc/resources/views/_footer.html | 3 +++ mcc/src/org/labkey/mcc/MccModule.java | 30 ++++++++++++++++++++++++--- 2 files changed, 30 insertions(+), 3 deletions(-) create mode 100644 mcc/resources/views/_footer.html diff --git a/mcc/resources/views/_footer.html b/mcc/resources/views/_footer.html new file mode 100644 index 000000000..93de7faf0 --- /dev/null +++ b/mcc/resources/views/_footer.html @@ -0,0 +1,3 @@ +

+ The MCC is supported by NIH/BRAIN Initiative U24 XXXXXXXXX +

\ No newline at end of file diff --git a/mcc/src/org/labkey/mcc/MccModule.java b/mcc/src/org/labkey/mcc/MccModule.java index 29361b628..1bf6e817a 100644 --- a/mcc/src/org/labkey/mcc/MccModule.java +++ b/mcc/src/org/labkey/mcc/MccModule.java @@ -19,15 +19,20 @@ import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import org.labkey.api.data.Container; +import org.labkey.api.ehr.EHRService; +import org.labkey.api.ldk.ExtendedSimpleModule; import org.labkey.api.module.DefaultModule; import org.labkey.api.module.ModuleContext; +import org.labkey.api.query.DetailsURL; +import org.labkey.api.resource.Resource; import org.labkey.api.view.WebPartFactory; +import org.labkey.api.view.template.ClientDependency; import java.util.Collection; import java.util.Collections; import java.util.Set; -public class MccModule extends DefaultModule +public class MccModule extends ExtendedSimpleModule { public static final String NAME = "MCC"; @@ -63,9 +68,9 @@ protected void init() } @Override - public void doStartup(ModuleContext moduleContext) + protected void doStartupAfterSpringConfig(ModuleContext moduleContext) { - + registerEHRResources(); } @Override @@ -81,4 +86,23 @@ public Set getSchemaNames() { return Collections.singleton(MccSchema.NAME); } + + private void registerEHRResources() + { + EHRService.get().registerModule(this); + //EHRService.get().registerTableCustomizer(this, ONPRC_EHRCustomizer.class); + + //Resource r = getModuleResource("/scripts/mcc/mcc_triggers.js"); + //assert r != null; + //EHRService.get().registerTriggerScript(this, r); + + //EHRService.get().registerClientDependency(ClientDependency.supplierFromPath("Ext4"), this); + //EHRService.get().registerClientDependency(ClientDependency.supplierFromPath("onprc_ehr/panel/BloodSummaryPanel.js"), this); + + //EHRService.get().registerReportLink(EHRService.REPORT_LINK_TYPE.housing, "List Single Housed Animals", this, DetailsURL.fromString("/query/executeQuery.view?schemaName=study&query.queryName=demographicsPaired&query.viewName=Single Housed"), "Commonly Used Queries"); + //EHRService.get().registerReportLink(EHRService.REPORT_LINK_TYPE.moreReports, "Clinical Snapshot Printable Report", this, DetailsURL.fromString("/onprc_ehr/SnapshotPrintableReport.view"), "Clinical"); + + //EHRService.get().registerDemographicsProvider(new ActiveCasesDemographicsProvider(this)); + //EHRService.get().registerHistoryDataSource(new DefaultSustainedReleaseDatasource(this)); + } } \ No newline at end of file From 469e5fd6fe9f76500c9860702d313d2ae8ee4f3d Mon Sep 17 00:00:00 2001 From: bbimber Date: Thu, 14 Jan 2021 15:21:48 -0800 Subject: [PATCH 41/98] Back out EHR dependency to allow TeamCity to build --- mcc/build.gradle | 2 ++ mcc/src/org/labkey/mcc/MccModule.java | 7 +------ 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/mcc/build.gradle b/mcc/build.gradle index 87943e447..feada3412 100644 --- a/mcc/build.gradle +++ b/mcc/build.gradle @@ -6,6 +6,7 @@ dependencies { BuildUtils.addLabKeyDependency(project: project, config: "implementation", depProjectPath: ":server:modules:LabDevKitModules:laboratory", depProjectConfig: "apiJarFile") BuildUtils.addLabKeyDependency(project: project, config: "implementation", depProjectPath: ":server:modules:LabDevKitModules:LDK", depProjectConfig: "apiJarFile") BuildUtils.addLabKeyDependency(project: project, config: "implementation", depProjectPath: ":server:modules:dataintegration", depProjectConfig: "apiJarFile") + //BuildUtils.addLabKeyDependency(project: project, config: "implementation", depProjectPath: ":server:modules:ehrModules:ehr", depProjectConfig: "apiJarFile") external "com.github.samtools:htsjdk:${htsjdkVersion}" implementation "net.sf.opencsv:opencsv:${opencsvVersion}" @@ -13,4 +14,5 @@ dependencies { BuildUtils.addLabKeyDependency(project: project, config: "modules", depProjectPath: ":server:modules:LabDevKitModules:LDK", depProjectConfig: "published", depExtension: "module") BuildUtils.addLabKeyDependency(project: project, config: "modules", depProjectPath: ":server:modules:DiscvrLabKeyModules:SequenceAnalysis", depProjectConfig: "published", depExtension: "module") BuildUtils.addLabKeyDependency(project: project, config: "modules", depProjectPath: ":server:modules:LabDevKitModules:laboratory", depProjectConfig: "published", depExtension: "module") + //BuildUtils.addLabKeyDependency(project: project, config: "modules", depProjectPath: ":server:modules:ehrModules:ehr", depProjectConfig: 'published', depExtension: 'module') } diff --git a/mcc/src/org/labkey/mcc/MccModule.java b/mcc/src/org/labkey/mcc/MccModule.java index 1bf6e817a..fac12f3d2 100644 --- a/mcc/src/org/labkey/mcc/MccModule.java +++ b/mcc/src/org/labkey/mcc/MccModule.java @@ -19,14 +19,9 @@ import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import org.labkey.api.data.Container; -import org.labkey.api.ehr.EHRService; import org.labkey.api.ldk.ExtendedSimpleModule; -import org.labkey.api.module.DefaultModule; import org.labkey.api.module.ModuleContext; -import org.labkey.api.query.DetailsURL; -import org.labkey.api.resource.Resource; import org.labkey.api.view.WebPartFactory; -import org.labkey.api.view.template.ClientDependency; import java.util.Collection; import java.util.Collections; @@ -89,7 +84,7 @@ public Set getSchemaNames() private void registerEHRResources() { - EHRService.get().registerModule(this); +// EHRService.get().registerModule(this); //EHRService.get().registerTableCustomizer(this, ONPRC_EHRCustomizer.class); //Resource r = getModuleResource("/scripts/mcc/mcc_triggers.js"); From 08fae7ddd2d167fdd046a89a86598dfdd18245a1 Mon Sep 17 00:00:00 2001 From: bbimber Date: Fri, 15 Jan 2021 09:37:07 -0800 Subject: [PATCH 42/98] Re-add MCC dependency --- mcc/build.gradle | 4 ++-- mcc/src/org/labkey/mcc/MccModule.java | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/mcc/build.gradle b/mcc/build.gradle index feada3412..128b7d630 100644 --- a/mcc/build.gradle +++ b/mcc/build.gradle @@ -6,7 +6,7 @@ dependencies { BuildUtils.addLabKeyDependency(project: project, config: "implementation", depProjectPath: ":server:modules:LabDevKitModules:laboratory", depProjectConfig: "apiJarFile") BuildUtils.addLabKeyDependency(project: project, config: "implementation", depProjectPath: ":server:modules:LabDevKitModules:LDK", depProjectConfig: "apiJarFile") BuildUtils.addLabKeyDependency(project: project, config: "implementation", depProjectPath: ":server:modules:dataintegration", depProjectConfig: "apiJarFile") - //BuildUtils.addLabKeyDependency(project: project, config: "implementation", depProjectPath: ":server:modules:ehrModules:ehr", depProjectConfig: "apiJarFile") + BuildUtils.addLabKeyDependency(project: project, config: "implementation", depProjectPath: ":server:modules:ehrModules:ehr", depProjectConfig: "apiJarFile") external "com.github.samtools:htsjdk:${htsjdkVersion}" implementation "net.sf.opencsv:opencsv:${opencsvVersion}" @@ -14,5 +14,5 @@ dependencies { BuildUtils.addLabKeyDependency(project: project, config: "modules", depProjectPath: ":server:modules:LabDevKitModules:LDK", depProjectConfig: "published", depExtension: "module") BuildUtils.addLabKeyDependency(project: project, config: "modules", depProjectPath: ":server:modules:DiscvrLabKeyModules:SequenceAnalysis", depProjectConfig: "published", depExtension: "module") BuildUtils.addLabKeyDependency(project: project, config: "modules", depProjectPath: ":server:modules:LabDevKitModules:laboratory", depProjectConfig: "published", depExtension: "module") - //BuildUtils.addLabKeyDependency(project: project, config: "modules", depProjectPath: ":server:modules:ehrModules:ehr", depProjectConfig: 'published', depExtension: 'module') + BuildUtils.addLabKeyDependency(project: project, config: "modules", depProjectPath: ":server:modules:ehrModules:ehr", depProjectConfig: 'published', depExtension: 'module') } diff --git a/mcc/src/org/labkey/mcc/MccModule.java b/mcc/src/org/labkey/mcc/MccModule.java index fac12f3d2..03182cb68 100644 --- a/mcc/src/org/labkey/mcc/MccModule.java +++ b/mcc/src/org/labkey/mcc/MccModule.java @@ -19,6 +19,7 @@ import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import org.labkey.api.data.Container; +import org.labkey.api.ehr.EHRService; import org.labkey.api.ldk.ExtendedSimpleModule; import org.labkey.api.module.ModuleContext; import org.labkey.api.view.WebPartFactory; @@ -84,7 +85,7 @@ public Set getSchemaNames() private void registerEHRResources() { -// EHRService.get().registerModule(this); + EHRService.get().registerModule(this); //EHRService.get().registerTableCustomizer(this, ONPRC_EHRCustomizer.class); //Resource r = getModuleResource("/scripts/mcc/mcc_triggers.js"); From a01fc6c83e98d43c81b4e71dab718db3d78865c4 Mon Sep 17 00:00:00 2001 From: bbimber Date: Fri, 15 Jan 2021 12:37:59 -0800 Subject: [PATCH 43/98] First round of MCC EHR resources --- .../queries/study/animalGroupMembership.xml | 46 + .../study/animalGroupMembership/.qview.xml | 18 + .../Active Members.qview.xml | 25 + .../queries/study/assignment.query.xml | 97 ++ .../queries/study/assignment/.qview.xml | 18 + mcc/resources/queries/study/birth/.qview.xml | 19 + mcc/resources/queries/study/clinremarks.xml | 22 + .../queries/study/clinremarks/.qview.xml | 19 + mcc/resources/queries/study/deaths.query.xml | 93 ++ mcc/resources/queries/study/deaths/.qview.xml | 16 + .../queries/study/demographics.query.xml | 242 ++++ .../queries/study/demographics/.qview.xml | 30 + .../demographics/Alive, at Center.qview.xml | 92 ++ .../queries/study/encounters.query.xml | 103 ++ .../queries/study/encounters/.qview.xml | 24 + mcc/resources/queries/study/flags.query.xml | 54 + mcc/resources/queries/study/flags/.qview.xml | 16 + .../study/flags/Active Flags.qview.xml | 8 + mcc/resources/queries/study/labwork.query.xml | 136 ++ .../queries/study/labwork/.qview.xml | 19 + .../queries/study/labwork/Requests.qview.xml | 25 + .../queries/study/labworkResults.query.xml | 67 + .../queries/study/labworkResults/.qview.xml | 21 + .../study/medicationAdministration.query.xml | 194 +++ .../study/medicationAdministration/.qview.xml | 29 + .../queries/study/medicationOrders.query.xml | 181 +++ .../queries/study/medicationOrders/.qview.xml | 28 + .../Active Treatments.qview.xml | 9 + .../queries/study/parentage.query.xml | 45 + .../queries/study/parentage/.qview.xml | 14 + .../study/parentage/Active Calls.qview.xml | 8 + .../study/parentageConflicts.query.xml | 31 + .../queries/study/parentageConflicts.sql | 47 + .../queries/study/parentageSummary.query.xml | 25 + .../queries/study/parentageSummary.sql | 36 + mcc/resources/queries/study/samples.query.xml | 70 + .../queries/study/samples/.qview.xml | 20 + mcc/resources/queries/study/weight.query.xml | 48 + mcc/resources/queries/study/weight/.qview.xml | 16 + mcc/resources/referenceStudy/README | 12 + .../PrimateElectronicHealthRecord.dataset | 19 + .../datasets/datasets_manifest.xml | 55 + .../datasets/datasets_metadata.xml | 1163 +++++++++++++++++ mcc/resources/referenceStudy/study.xml | 18 + mcc/resources/referenceStudy/studyPolicy.xml | 10 + 45 files changed, 3288 insertions(+) create mode 100644 mcc/resources/queries/study/animalGroupMembership.xml create mode 100644 mcc/resources/queries/study/animalGroupMembership/.qview.xml create mode 100644 mcc/resources/queries/study/animalGroupMembership/Active Members.qview.xml create mode 100644 mcc/resources/queries/study/assignment.query.xml create mode 100644 mcc/resources/queries/study/assignment/.qview.xml create mode 100644 mcc/resources/queries/study/birth/.qview.xml create mode 100644 mcc/resources/queries/study/clinremarks.xml create mode 100644 mcc/resources/queries/study/clinremarks/.qview.xml create mode 100644 mcc/resources/queries/study/deaths.query.xml create mode 100644 mcc/resources/queries/study/deaths/.qview.xml create mode 100644 mcc/resources/queries/study/demographics.query.xml create mode 100644 mcc/resources/queries/study/demographics/.qview.xml create mode 100644 mcc/resources/queries/study/demographics/Alive, at Center.qview.xml create mode 100644 mcc/resources/queries/study/encounters.query.xml create mode 100644 mcc/resources/queries/study/encounters/.qview.xml create mode 100644 mcc/resources/queries/study/flags.query.xml create mode 100644 mcc/resources/queries/study/flags/.qview.xml create mode 100644 mcc/resources/queries/study/flags/Active Flags.qview.xml create mode 100644 mcc/resources/queries/study/labwork.query.xml create mode 100644 mcc/resources/queries/study/labwork/.qview.xml create mode 100644 mcc/resources/queries/study/labwork/Requests.qview.xml create mode 100644 mcc/resources/queries/study/labworkResults.query.xml create mode 100644 mcc/resources/queries/study/labworkResults/.qview.xml create mode 100644 mcc/resources/queries/study/medicationAdministration.query.xml create mode 100644 mcc/resources/queries/study/medicationAdministration/.qview.xml create mode 100644 mcc/resources/queries/study/medicationOrders.query.xml create mode 100644 mcc/resources/queries/study/medicationOrders/.qview.xml create mode 100644 mcc/resources/queries/study/medicationOrders/Active Treatments.qview.xml create mode 100644 mcc/resources/queries/study/parentage.query.xml create mode 100644 mcc/resources/queries/study/parentage/.qview.xml create mode 100644 mcc/resources/queries/study/parentage/Active Calls.qview.xml create mode 100644 mcc/resources/queries/study/parentageConflicts.query.xml create mode 100644 mcc/resources/queries/study/parentageConflicts.sql create mode 100644 mcc/resources/queries/study/parentageSummary.query.xml create mode 100644 mcc/resources/queries/study/parentageSummary.sql create mode 100644 mcc/resources/queries/study/samples.query.xml create mode 100644 mcc/resources/queries/study/samples/.qview.xml create mode 100644 mcc/resources/queries/study/weight.query.xml create mode 100644 mcc/resources/queries/study/weight/.qview.xml create mode 100644 mcc/resources/referenceStudy/README create mode 100644 mcc/resources/referenceStudy/datasets/PrimateElectronicHealthRecord.dataset create mode 100644 mcc/resources/referenceStudy/datasets/datasets_manifest.xml create mode 100644 mcc/resources/referenceStudy/datasets/datasets_metadata.xml create mode 100644 mcc/resources/referenceStudy/study.xml create mode 100644 mcc/resources/referenceStudy/studyPolicy.xml diff --git a/mcc/resources/queries/study/animalGroupMembership.xml b/mcc/resources/queries/study/animalGroupMembership.xml new file mode 100644 index 000000000..d87426e0c --- /dev/null +++ b/mcc/resources/queries/study/animalGroupMembership.xml @@ -0,0 +1,46 @@ + + + + + Animal Group Members + + + + + + Date Added + + + Date Removed + false + + + + ehr + animal_groups + rowid + name + + + + + ehr_lookups + animalGroupReleaseType + value + + + + + core + qcstate + rowid + + + + true + + +
+ + + diff --git a/mcc/resources/queries/study/animalGroupMembership/.qview.xml b/mcc/resources/queries/study/animalGroupMembership/.qview.xml new file mode 100644 index 000000000..c7a202751 --- /dev/null +++ b/mcc/resources/queries/study/animalGroupMembership/.qview.xml @@ -0,0 +1,18 @@ + + + + + + + + + + + + + + + + + + diff --git a/mcc/resources/queries/study/animalGroupMembership/Active Members.qview.xml b/mcc/resources/queries/study/animalGroupMembership/Active Members.qview.xml new file mode 100644 index 000000000..b0c30966e --- /dev/null +++ b/mcc/resources/queries/study/animalGroupMembership/Active Members.qview.xml @@ -0,0 +1,25 @@ + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/mcc/resources/queries/study/assignment.query.xml b/mcc/resources/queries/study/assignment.query.xml new file mode 100644 index 000000000..a210b253e --- /dev/null +++ b/mcc/resources/queries/study/assignment.query.xml @@ -0,0 +1,97 @@ + + + + + + + + + + + + + + ehr + project + project + + + + Assign Date + Date + + + Projected Release Date + + + Release Date + false + true + true + Date + + + true + + + + + + + + + + Condition At Assignment + + ehr_lookups + animal_condition + code + + + + Projected Release Condition + + ehr_lookups + animal_condition + code + + + + Condition At Release + + ehr_lookups + animal_condition + code + + + + true + + + Release Type + + ehr_lookups + assignmentReleaseType + value + + + + Date Assignment End Entered + This records the date the end of the assignment was actually entered, which may differ from the enddate itself + true + + + CoAssignments + false + true + + study + assignmentTotalCoAssigned + lsid + + + +
+ + + \ No newline at end of file diff --git a/mcc/resources/queries/study/assignment/.qview.xml b/mcc/resources/queries/study/assignment/.qview.xml new file mode 100644 index 000000000..4c0c2fdb7 --- /dev/null +++ b/mcc/resources/queries/study/assignment/.qview.xml @@ -0,0 +1,18 @@ + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/mcc/resources/queries/study/birth/.qview.xml b/mcc/resources/queries/study/birth/.qview.xml new file mode 100644 index 000000000..f1449329f --- /dev/null +++ b/mcc/resources/queries/study/birth/.qview.xml @@ -0,0 +1,19 @@ + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/mcc/resources/queries/study/clinremarks.xml b/mcc/resources/queries/study/clinremarks.xml new file mode 100644 index 000000000..0bda2a18e --- /dev/null +++ b/mcc/resources/queries/study/clinremarks.xml @@ -0,0 +1,22 @@ + + + + + Clinical Remarks + + + + + + + + + + + + + +
+ + + diff --git a/mcc/resources/queries/study/clinremarks/.qview.xml b/mcc/resources/queries/study/clinremarks/.qview.xml new file mode 100644 index 000000000..78d39cdfc --- /dev/null +++ b/mcc/resources/queries/study/clinremarks/.qview.xml @@ -0,0 +1,19 @@ + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/mcc/resources/queries/study/deaths.query.xml b/mcc/resources/queries/study/deaths.query.xml new file mode 100644 index 000000000..25e309e6e --- /dev/null +++ b/mcc/resources/queries/study/deaths.query.xml @@ -0,0 +1,93 @@ + + + + + + + + + + + + + + + Time of Death + + + Type of Death + + ehr_lookups + death_cause + value + + + + Manner of Death + + ehr_lookups + death_manner + value + + + + Necropsy Case No + /query/executeQuery.view?schemaName=study& + query.queryName=Necropsies& + query.caseno~eq=${necropsy}& + + + + + 110 + textarea + + + Cage At Time + + + Key + false + false + false + true + + + + Tattoo/Tag Number + + + Dam (infants only) + + + Entered By + + + Room At Time + + + + + + ehr_lookups + rooms + room + + + + Death Was Not At Center + + + Final Condition + true + + ehr_lookups + animal_condition + code + + + +
+ + + \ No newline at end of file diff --git a/mcc/resources/queries/study/deaths/.qview.xml b/mcc/resources/queries/study/deaths/.qview.xml new file mode 100644 index 000000000..fa7420819 --- /dev/null +++ b/mcc/resources/queries/study/deaths/.qview.xml @@ -0,0 +1,16 @@ + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/mcc/resources/queries/study/demographics.query.xml b/mcc/resources/queries/study/demographics.query.xml new file mode 100644 index 000000000..48d852b41 --- /dev/null +++ b/mcc/resources/queries/study/demographics.query.xml @@ -0,0 +1,242 @@ + + + + + + + + + + + + + + + true + + + true + + + Gender + + ehr_lookups + gender_codes + code + + + + Species + + ehr_lookups + species + common + + + + + Geographic Origin + + ehr_lookups + geographic_origins + meaning + + + + + + Date + Birth + /query/executeQuery.view? + schemaName=study& + query.queryName=Birth& + query.Id~eq=${Id} + + + + + Date + Death + /query/executeQuery.view? + schemaName=study& + query.queryName=Deaths& + query.Id~eq=${Id} + + + + false + true + Status + + ehr_lookups + status_codes + value + + + + false + Status + + ehr_lookups + calculated_status_codes + code + + + + + + + Record Status + true + + + + + true + Availability + + + + + + + + Hold + + + + + + + + Dam + + + + + + /ehr/participantView.view?participantId=${dam} + + + Sire + + + + + + /ehr/participantView.view?participantId=${sire} + + + Origin + + ehr_lookups + source + code + + + + + + true + DateTime + false + Arrival Date + + + true + DateTime + false + Departure Date + + + true + false + Room + /ehr/cageDetails.view? + room=${room}& + + + ehr_lookups + rooms + room + + + + true + false + Cage + /ehr/cageDetails.view? + room=${room}& + cage=${cage}& + + + + true + false + 30 + Condition + + ehr_lookups + housing_condition_codes + value + + + + true + false + Current Weight (kg) + /query/executeQuery.view?schemaName=study& + query.queryName=Weight& + query.id~eq=${id} + + + + + false + true + DateTime + Weight Date + /query/executeQuery.view?schemaName=study& + query.queryName=Weight& + query.id~eq=${id}& + query.date~eq=${wdate} + + + + false + true + Date + Last TB Date + /query/executeQuery.view?schemaName=study& + query.queryName=TB Tests& + query.id~eq=${id}& + query.sort=-Date + + + + Medical + + + Replacement Prepaid By + + + Viral Status + true + + ehr_lookups + viral_status + value + + + + + + + + + + +
+ + + \ No newline at end of file diff --git a/mcc/resources/queries/study/demographics/.qview.xml b/mcc/resources/queries/study/demographics/.qview.xml new file mode 100644 index 000000000..6ffca2250 --- /dev/null +++ b/mcc/resources/queries/study/demographics/.qview.xml @@ -0,0 +1,30 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/mcc/resources/queries/study/demographics/Alive, at Center.qview.xml b/mcc/resources/queries/study/demographics/Alive, at Center.qview.xml new file mode 100644 index 000000000..90efd4415 --- /dev/null +++ b/mcc/resources/queries/study/demographics/Alive, at Center.qview.xml @@ -0,0 +1,92 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/mcc/resources/queries/study/encounters.query.xml b/mcc/resources/queries/study/encounters.query.xml new file mode 100644 index 000000000..972f02566 --- /dev/null +++ b/mcc/resources/queries/study/encounters.query.xml @@ -0,0 +1,103 @@ + + + + + /ehr/encounterDetails.view?objectid=${objectid}&formtype=${taskid/formtype}&taskid=${taskid} + + + + + + + + + + Date + + + End Time + false + + + + + + Type + + ehr_lookups + encounter_types + value + + + + Charge Unit + + ehr_lookups + procedureChargeType + value + + + + Assisting Staff + false + + ehr_lookups + procedureChargeType + value + + + + Case Number + + + Procedure + + ehr_lookups + procedures + rowid + name + + + + Major Surgery? + + ehr_lookups + yesno + value + + + + Service Requested + + + Special Instructions + textarea + + + Title + + + Restraint + + ehr_lookups + restraint_type + type + + + + Time Restrained + + ehr_lookups + restraint_duration + value + + + + Date Requested + true + + +
+ + + \ No newline at end of file diff --git a/mcc/resources/queries/study/encounters/.qview.xml b/mcc/resources/queries/study/encounters/.qview.xml new file mode 100644 index 000000000..f1da9f3a4 --- /dev/null +++ b/mcc/resources/queries/study/encounters/.qview.xml @@ -0,0 +1,24 @@ + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/mcc/resources/queries/study/flags.query.xml b/mcc/resources/queries/study/flags.query.xml new file mode 100644 index 000000000..e1e38b743 --- /dev/null +++ b/mcc/resources/queries/study/flags.query.xml @@ -0,0 +1,54 @@ + + + + + + + + + + + + + Date Added + Date + + + Date Removed + false + Date + + + true + + + Category + true + + ehr_lookups + flag_categories + category + + + + + Flag + + ehr_lookups + flag_values + objectid + value + + + + Value + true + + + Entered By + + +
+ + + diff --git a/mcc/resources/queries/study/flags/.qview.xml b/mcc/resources/queries/study/flags/.qview.xml new file mode 100644 index 000000000..05349f54b --- /dev/null +++ b/mcc/resources/queries/study/flags/.qview.xml @@ -0,0 +1,16 @@ + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/mcc/resources/queries/study/flags/Active Flags.qview.xml b/mcc/resources/queries/study/flags/Active Flags.qview.xml new file mode 100644 index 000000000..19142d6b5 --- /dev/null +++ b/mcc/resources/queries/study/flags/Active Flags.qview.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/mcc/resources/queries/study/labwork.query.xml b/mcc/resources/queries/study/labwork.query.xml new file mode 100644 index 000000000..54592fece --- /dev/null +++ b/mcc/resources/queries/study/labwork.query.xml @@ -0,0 +1,136 @@ + + + + + + + + + + servicerequested + + + + + + Collection Date + + + + + + Service Requested + + ehr_lookups + Labwork_services + servicename + + + + Charge Unit + + ehr_lookups + labworkChargeType + value + + + + Sample Type + + ehr_lookups + clinpath_sampletype + value + + + + Tissue + + ehr_lookups + snomed + code + + + + Sample Quantity + true + + + Quantity Units + true + + + Sample Units + true + + + Collected By + + + Collection Method + + ehr_lookups + clinpath_collection_method + value + + + + Method + + + Remark + + + Category + + ehr_lookups + clinpath_types + value + + + + + Special Instructions + textarea + + + Reviewed By + + + Date Reviewed + + + true + + + true + + + true + + + true + + + true + + + true + + + Units + + + Clinical Remark + + + Sample Id + true + + + Condition + + +
+ + + \ No newline at end of file diff --git a/mcc/resources/queries/study/labwork/.qview.xml b/mcc/resources/queries/study/labwork/.qview.xml new file mode 100644 index 000000000..a781baa24 --- /dev/null +++ b/mcc/resources/queries/study/labwork/.qview.xml @@ -0,0 +1,19 @@ + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/mcc/resources/queries/study/labwork/Requests.qview.xml b/mcc/resources/queries/study/labwork/Requests.qview.xml new file mode 100644 index 000000000..133c818ba --- /dev/null +++ b/mcc/resources/queries/study/labwork/Requests.qview.xml @@ -0,0 +1,25 @@ + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/mcc/resources/queries/study/labworkResults.query.xml b/mcc/resources/queries/study/labworkResults.query.xml new file mode 100644 index 000000000..9955e946b --- /dev/null +++ b/mcc/resources/queries/study/labworkResults.query.xml @@ -0,0 +1,67 @@ + + + + + Labwork Results + + + + + + + + + + + + + true + + + Test Id + 120 + + ehr_lookups + misc_tests + testid + + + + + Category + + + Numeric Result + + + Units + 60 + + ehr_lookups + lab_test_units + units + + + + + Text Result + + + Method + + + Sample Type + + ehr_lookups + snomed + code + + + + true + + +
+ + + \ No newline at end of file diff --git a/mcc/resources/queries/study/labworkResults/.qview.xml b/mcc/resources/queries/study/labworkResults/.qview.xml new file mode 100644 index 000000000..e333545d1 --- /dev/null +++ b/mcc/resources/queries/study/labworkResults/.qview.xml @@ -0,0 +1,21 @@ + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/mcc/resources/queries/study/medicationAdministration.query.xml b/mcc/resources/queries/study/medicationAdministration.query.xml new file mode 100644 index 000000000..81252806f --- /dev/null +++ b/mcc/resources/queries/study/medicationAdministration.query.xml @@ -0,0 +1,194 @@ + + + + + /ehr/drugDetails.view?lsid=${lsid} + + + + + + + + + + Begin Date + yyyy-MM-dd HH:mm + + + Header Date + yyyy-MM-dd H:mm + + + End Time + yyyy-MM-dd H:mm + false + + + Charge To + + ehr + project + project + + + + Credit To + + ehr_lookups + medicationChargeType + value + + + No Charge + + + Code + + ehr_lookups + snomed + code + + + + Is Billable + + ehr_lookups + yesno + value + + + No + + + Qualifier + + + Reason + + ehr_lookups + drugReason + value + + + + Route + + ehr_lookups + routes + route + + + + + Drug Conc + + + Conc Units + + ehr_lookups + conc_units + unit + + + + + Dosage + + + Dosage Units + + ehr_lookups + dosage_units + unit + + + + + Volume + + + Vol Units + + ehr_lookups + volume_units + unit + + + + + Amount + + + Amount Units + + ehr_lookups + amount_units + unit + + + + + Restraint + + ehr_lookups + restraint_type + type + + + + Time Restrained + + ehr_lookups + restraint_duration + value + + + + Outcome + + ehr_lookups + drugOutcome + value + + + Normal + + + Lot + + + Remark + + + Category + + ehr_lookups + drug_categories + value + + + + + Begin Date + + + Treatment Id + true + + + Time Ordered + For drugs that were ordered using the Treatment Orders table, this stores the original time this administration was scheduled to be administered. + true + + + + + + true + + +
+ + + \ No newline at end of file diff --git a/mcc/resources/queries/study/medicationAdministration/.qview.xml b/mcc/resources/queries/study/medicationAdministration/.qview.xml new file mode 100644 index 000000000..b99c9a3c6 --- /dev/null +++ b/mcc/resources/queries/study/medicationAdministration/.qview.xml @@ -0,0 +1,29 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/mcc/resources/queries/study/medicationOrders.query.xml b/mcc/resources/queries/study/medicationOrders.query.xml new file mode 100644 index 000000000..a9da4a138 --- /dev/null +++ b/mcc/resources/queries/study/medicationOrders.query.xml @@ -0,0 +1,181 @@ + + + + + /EHR/treatmentDetails.view?key=${lsid} + + + + + + + + + + Begin Date + yyyy-MM-dd HH:mm + + + End Date + false + yyyy-MM-dd HH:mm + + + + Charge To + + ehr + project + project + + + + Category + + ehr_lookups + drug_categories + value + + + Depending on what is selected, the treatment will appear on a different schedule (ie. Clinical, Surgical, etc) + + + Reason + + + + Credit To + + ehr_lookups + medicationChargeType + value + + + No Charge + + + Short Name + + + false + Treatment + + ehr_lookups + snomed + code + + + + Is Billable + + ehr_lookups + yesno + value + + + No + + + Qualifier + + + Frequency + + ehr_lookups + treatment_frequency + rowid + + + + Route + + ehr_lookups + routes + route + + + 40 + + + Drug Conc + + + Conc Units + + ehr_lookups + conc_units + unit + + + + + Dosage + + + Dosage Units + + ehr_lookups + dosage_units + unit + + + + + Volume + + + Volume Units + + ehr_lookups + volume_units + unit + + + + + Amount + + + Amount Units + + ehr_lookups + amount_units + unit + + + + + false + Ordered By + + + false + Modified By + + + false + Modified Date + + + false + Last Administered + /query/executeQuery.view?schemaName=study& + query.queryName=Drug%20Administration& + query.parentid~eq=${objectid}& + + + + + + + + + + + Ordered By + + +
+ + + \ No newline at end of file diff --git a/mcc/resources/queries/study/medicationOrders/.qview.xml b/mcc/resources/queries/study/medicationOrders/.qview.xml new file mode 100644 index 000000000..f3854a2f8 --- /dev/null +++ b/mcc/resources/queries/study/medicationOrders/.qview.xml @@ -0,0 +1,28 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/mcc/resources/queries/study/medicationOrders/Active Treatments.qview.xml b/mcc/resources/queries/study/medicationOrders/Active Treatments.qview.xml new file mode 100644 index 000000000..97f1b7be9 --- /dev/null +++ b/mcc/resources/queries/study/medicationOrders/Active Treatments.qview.xml @@ -0,0 +1,9 @@ + + + + + + + + + \ No newline at end of file diff --git a/mcc/resources/queries/study/parentage.query.xml b/mcc/resources/queries/study/parentage.query.xml new file mode 100644 index 000000000..a108e2772 --- /dev/null +++ b/mcc/resources/queries/study/parentage.query.xml @@ -0,0 +1,45 @@ + + + + + + + + + + + + Parent + false + + study + animal + id + + + + Relationship + false + + ehr_lookups + parentageRelationship + value + + + + Method + false + + ehr_lookups + parentageMethod + value + + + + true + + +
+ + + diff --git a/mcc/resources/queries/study/parentage/.qview.xml b/mcc/resources/queries/study/parentage/.qview.xml new file mode 100644 index 000000000..ca535b042 --- /dev/null +++ b/mcc/resources/queries/study/parentage/.qview.xml @@ -0,0 +1,14 @@ + + + + + + + + + + + + + + \ No newline at end of file diff --git a/mcc/resources/queries/study/parentage/Active Calls.qview.xml b/mcc/resources/queries/study/parentage/Active Calls.qview.xml new file mode 100644 index 000000000..7d9e28d9d --- /dev/null +++ b/mcc/resources/queries/study/parentage/Active Calls.qview.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/mcc/resources/queries/study/parentageConflicts.query.xml b/mcc/resources/queries/study/parentageConflicts.query.xml new file mode 100644 index 000000000..e5bed3219 --- /dev/null +++ b/mcc/resources/queries/study/parentageConflicts.query.xml @@ -0,0 +1,31 @@ + + + + + Parentage Conflicts + + + Parent(s) + + + Relationship + + + Method(s) + + + Total Records + /query/executeQuery.view?schemaName=study& + query.queryName=Parentage& + query.Id~eq=${Id}& + query.relationship~eq=${relationship}& + query.relationship~neq=Foster Dam& + query.enddate~isblank& + query.relationship~neq=Surrogate Dam + + + +
+ + + diff --git a/mcc/resources/queries/study/parentageConflicts.sql b/mcc/resources/queries/study/parentageConflicts.sql new file mode 100644 index 000000000..dd879cac8 --- /dev/null +++ b/mcc/resources/queries/study/parentageConflicts.sql @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2013-2014 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +SELECT + p.Id, + p.relationship, + + group_concat(distinct p.parent) as parents, + group_concat(distinct p.method) as method, + 'Duplicate Parents With Same Relationship' as type, + count(p.Id) as totalRecords + +FROM study.parentage p +WHERE p.qcstate.publicdata = true and p.enddateCoalesced <= now() +AND p.relationship != 'Surrogate Dam' and p.relationship != 'Foster Dam' and p.enddateCoalesced >= curdate() + +GROUP BY p.Id, p.relationship +HAVING COUNT(DISTINCT p.parent) > 1 + +-- UNION ALL +-- +-- SELECT +-- p.Id, +-- p.relationship, +-- p.parent as parents, +-- group_concat(distinct p.method) as method, +-- 'Duplicate Methods For The Same Parent' as type, +-- count(p.Id) as totalRecords +-- +-- FROM study.parentage p +-- WHERE p.qcstate.publicdata = true and p.enddateCoalesced <= now() +-- AND p.relationship != 'Surrogate Dam' and p.relationship != 'Foster Dam' and p.enddateCoalesced >= curdate() +-- +-- GROUP BY p.Id, p.relationship, p.parent +-- HAVING COUNT(distinct p.method) > 1 \ No newline at end of file diff --git a/mcc/resources/queries/study/parentageSummary.query.xml b/mcc/resources/queries/study/parentageSummary.query.xml new file mode 100644 index 000000000..1efefbcd2 --- /dev/null +++ b/mcc/resources/queries/study/parentageSummary.query.xml @@ -0,0 +1,25 @@ + + + + + Parentage Summary + + + Parent + + study + animal + id + + + + Relationship + + + Method + + +
+ + + diff --git a/mcc/resources/queries/study/parentageSummary.sql b/mcc/resources/queries/study/parentageSummary.sql new file mode 100644 index 000000000..46086a185 --- /dev/null +++ b/mcc/resources/queries/study/parentageSummary.sql @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2013-2017 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +SELECT + p.Id, + p.date, + p.parent, + p.relationship, + p.method + +FROM study.parentage p +WHERE p.qcstate.publicdata = true and p.enddateCoalesced <= now() + +UNION ALL + +SELECT + b.Id, + b.date, + b.dam, + 'Dam' as relationship, + 'Observed' as method + +FROM study.birth b +WHERE b.dam is not null and b.qcstate.publicdata = true \ No newline at end of file diff --git a/mcc/resources/queries/study/samples.query.xml b/mcc/resources/queries/study/samples.query.xml new file mode 100644 index 000000000..9e0f58f4c --- /dev/null +++ b/mcc/resources/queries/study/samples.query.xml @@ -0,0 +1,70 @@ + + + + + + + + + + + + + + + + + + true + + + Organ/Tissue + + ehr_lookups + snomed + code + + + + Qualifier + + ehr_lookups + snomed_qualifiers + value + + + + Tissue Condition + true + + ehr_lookups + tissue_condition + value + + + + Preparation + + ehr_lookups + tissue_preparation + value + + + + Quantity + true + + + Weight + + + No Weight + + + + + +
+ + + \ No newline at end of file diff --git a/mcc/resources/queries/study/samples/.qview.xml b/mcc/resources/queries/study/samples/.qview.xml new file mode 100644 index 000000000..a2a93675a --- /dev/null +++ b/mcc/resources/queries/study/samples/.qview.xml @@ -0,0 +1,20 @@ + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/mcc/resources/queries/study/weight.query.xml b/mcc/resources/queries/study/weight.query.xml new file mode 100644 index 000000000..29c16ce25 --- /dev/null +++ b/mcc/resources/queries/study/weight.query.xml @@ -0,0 +1,48 @@ + + + + + + + + + + + + + + + + + + + + + Percent Change + false + true + + study + weightPctChange + lsid + + + + Relative Change + false + true + + study + weightRelChange + lsid + + + + Weight (kg) + 0.### + + +
+ + + \ No newline at end of file diff --git a/mcc/resources/queries/study/weight/.qview.xml b/mcc/resources/queries/study/weight/.qview.xml new file mode 100644 index 000000000..74ac7f98d --- /dev/null +++ b/mcc/resources/queries/study/weight/.qview.xml @@ -0,0 +1,16 @@ + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/mcc/resources/referenceStudy/README b/mcc/resources/referenceStudy/README new file mode 100644 index 000000000..8710e0b71 --- /dev/null +++ b/mcc/resources/referenceStudy/README @@ -0,0 +1,12 @@ +This folder contains the reference study for the MCC EHR. It should be generated by performing +a folder export from the production server, then copying the datasets_manifest.xml, datasets_metadata.xml, +PrimateElectronicHealthRecord.dataset and study.xml files. After copying, the following can be used to find/replace +metadata in datasets_metadata.xml to remove unwanted information: + +Replace the following regex expressions with empty string: + +( )*(.*)\n +( )*(.*)\n|( )*(.*)\n +( )*(.*)\n|( )*(.*)\n|( )*(.*)\n|( )*(.*)\n +( )*(.*)\n|( )*(.*)\n|( )*(.)*\n( )*(.)*\n +( )*(.*)\n|( )*(.*)\n|( )*(.*)\n diff --git a/mcc/resources/referenceStudy/datasets/PrimateElectronicHealthRecord.dataset b/mcc/resources/referenceStudy/datasets/PrimateElectronicHealthRecord.dataset new file mode 100644 index 000000000..8e5970288 --- /dev/null +++ b/mcc/resources/referenceStudy/datasets/PrimateElectronicHealthRecord.dataset @@ -0,0 +1,19 @@ +# default group can be used to avoid repeating definitions for each dataset +# +# action=[REPLACE,APPEND,DELETE] (default:REPLACE) +# deleteAfterImport=[TRUE|FALSE] (default:FALSE) + +default.action=REPLACE +default.deleteAfterImport=FALSE + +# map a source tsv column (right side) to a property name or full propertyURI (left) +# predefined properties: ParticipantId, SiteId, VisitId, Created +default.property.ParticipantId=ptid +default.property.Created=dfcreate + +# use to map from filename->datasetid +# NOTE: if there are NO explicit import definitions, we will try to import all files matching pattern +# NOTE: if there are ANY explicit mapping, we will only import listed datasets + +default.filePattern=dataset(\\d*).tsv +default.importAllMatches=TRUE diff --git a/mcc/resources/referenceStudy/datasets/datasets_manifest.xml b/mcc/resources/referenceStudy/datasets/datasets_manifest.xml new file mode 100644 index 000000000..3600f8261 --- /dev/null +++ b/mcc/resources/referenceStudy/datasets/datasets_manifest.xml @@ -0,0 +1,55 @@ + + + + ClinPath + Colony Management + Clinical + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/mcc/resources/referenceStudy/datasets/datasets_metadata.xml b/mcc/resources/referenceStudy/datasets/datasets_metadata.xml new file mode 100644 index 000000000..8f19b36f9 --- /dev/null +++ b/mcc/resources/referenceStudy/datasets/datasets_metadata.xml @@ -0,0 +1,1163 @@ + + + + + + varchar + urn:ehr.labkey.org/#TaskId + + + varchar + urn:ehr.labkey.org/#ParentId + + + varchar + urn:ehr.labkey.org/#RequestId + + + varchar + urn:ehr.labkey.org/#PerformedBy + + + varchar + urn:ehr.labkey.org/#Description + + + varchar + urn:ehr.labkey.org/#Remark + + + + + + + varchar + http://cpas.labkey.com/Study#ParticipantId + + ptid + + + + timestamp + http://cpas.labkey.com/Study#VisitDate + http://cpas.labkey.com/Study#VisitDate + + + varchar + + + timestamp + urn:ehr.labkey.org/#EndDate + + + integer + urn:ehr.labkey.org/#Project + + + entityid + urn:ehr.labkey.org/#ObjectId + true + + + Animal Record Flags +
+ + + + varchar + http://cpas.labkey.com/Study#ParticipantId + + ptid + + + + timestamp + http://cpas.labkey.com/Study#VisitDate + http://cpas.labkey.com/Study#VisitDate + + + varchar + + + varchar + + + varchar + + + varchar + urn:ehr.labkey.org/#EndDate + + + integer + urn:ehr.labkey.org/#Project + + + entityid + urn:ehr.labkey.org/#ObjectId + true + + + Parentage +
+ + + + varchar + http://cpas.labkey.com/Study#ParticipantId + + ptid + + + + timestamp + http://cpas.labkey.com/Study#VisitDate + http://cpas.labkey.com/Study#VisitDate + + + varchar + + + varchar + + + varchar + + + double + + + timestamp + + + varchar + + + varchar + + + varchar + + + varchar + + + varchar + + + varchar + + + varchar + + + timestamp + + + integer + + + varchar + + + timestamp + + + varchar + urn:ehr.labkey.org/#ObjectId + + + integer + urn:ehr.labkey.org/#Project + + + timestamp + urn:ehr.labkey.org/#EndDate + + + Birth +
+ + + + varchar + http://cpas.labkey.com/Study#ParticipantId + + ptid + + + + timestamp + http://cpas.labkey.com/Study#VisitDate + http://cpas.labkey.com/Study#VisitDate + + + timestamp + urn:ehr.labkey.org/#EndDate + + + integer + urn:ehr.labkey.org/#Project + + + varchar + + + entityid + urn:ehr.labkey.org/#ObjectId + true + + + varchar + urn:ehr.labkey.org/#VetReview + + + timestamp + urn:ehr.labkey.org/#VetReviewDate + + + varchar + + + varchar + + + varchar + + + varchar + + + timestamp + urn:ehr.labkey.org/#DateRequested + + + varchar + + + varchar + + + integer + + + timestamp + + + Clinical Encounters +
+ + + + varchar + http://cpas.labkey.com/Study#ParticipantId + + ptid + + + + timestamp + http://cpas.labkey.com/Study#VisitDate + http://cpas.labkey.com/Study#VisitDate + + + integer + urn:ehr.labkey.org/#Project + + + varchar + + + entityid + urn:ehr.labkey.org/#ObjectId + true + + + varchar + urn:ehr.labkey.org/#VetReview + + + timestamp + urn:ehr.labkey.org/#VetReviewDate + + + varchar + + + varchar + + + varchar + + + varchar + + + varchar + + + varchar + + + varchar + + + timestamp + urn:ehr.labkey.org/#EndDate + + + timestamp + + + varchar + + + varchar + + + Clinical Remarks +
+ + + + varchar + http://cpas.labkey.com/Study#ParticipantId + + ptid + + + + timestamp + http://cpas.labkey.com/Study#VisitDate + http://cpas.labkey.com/Study#VisitDate + + + integer + urn:ehr.labkey.org/#Project + + + varchar + + + varchar + + + double + + + varchar + + + double + + + varchar + + + double + + + varchar + + + double + + + varchar + + + varchar + + + timestamp + + + timestamp + urn:ehr.labkey.org/#EndDate + + + varchar + + + entityid + urn:ehr.labkey.org/#ObjectId + true + + + varchar + + + varchar + + + timestamp + + + varchar + + + Medication Administration +
+ + + + varchar + http://cpas.labkey.com/Study#ParticipantId + + ptid + + + + timestamp + http://cpas.labkey.com/Study#VisitDate + http://cpas.labkey.com/Study#VisitDate + + + integer + urn:ehr.labkey.org/#Project + + + varchar + + + double + + + varchar + + + double + + + varchar + + + double + + + varchar + + + varchar + + + timestamp + urn:ehr.labkey.org/#EndDate + + + integer + + + entityid + urn:ehr.labkey.org/#ObjectId + true + + + double + + + varchar + + + varchar + + + varchar + + + varchar + + + Medication/Treatment Orders +
+ + + + varchar + http://cpas.labkey.com/Study#ParticipantId + + ptid + + + + timestamp + http://cpas.labkey.com/Study#VisitDate + http://cpas.labkey.com/Study#VisitDate + + + double + + + entityid + urn:ehr.labkey.org/#ObjectId + true + + + integer + urn:ehr.labkey.org/#Project + + + timestamp + urn:ehr.labkey.org/#EndDate + + + Weight +
+ + + + varchar + http://cpas.labkey.com/Study#ParticipantId + + ptid + + + + timestamp + http://cpas.labkey.com/Study#VisitDate + http://cpas.labkey.com/Study#VisitDate + + + timestamp + urn:ehr.labkey.org/#EndDate + + + varchar + + + varchar + + + integer + urn:ehr.labkey.org/#Project + + + varchar + + + entityid + urn:ehr.labkey.org/#ObjectId + true + + + varchar + + + varchar + + + varchar + + + varchar + + + varchar + + + varchar + + + varchar + + + double + + + varchar + + + varchar + + + timestamp + + + Clinpath Runs +
+ + + + varchar + http://cpas.labkey.com/Study#ParticipantId + + ptid + + + + timestamp + http://cpas.labkey.com/Study#VisitDate + http://cpas.labkey.com/Study#VisitDate + + + varchar + + + double + + + varchar + + + double + + + double + + + varchar + + + varchar + + + entityid + urn:ehr.labkey.org/#ObjectId + true + + + varchar + + + integer + urn:ehr.labkey.org/#Project + + + timestamp + urn:ehr.labkey.org/#EndDate + + + varchar + + + Hematology Results +
+ + + + varchar + http://cpas.labkey.com/Study#ParticipantId + + ptid + + + + timestamp + http://cpas.labkey.com/Study#VisitDate + http://cpas.labkey.com/Study#VisitDate + + + varchar + + + entityid + urn:ehr.labkey.org/#ObjectId + true + + + varchar + + + integer + urn:ehr.labkey.org/#Project + + + timestamp + urn:ehr.labkey.org/#EndDate + + + double + + + varchar + + + varchar + + + varchar + + + varchar + + + Parasitology Results +
+ + + + varchar + http://cpas.labkey.com/Study#ParticipantId + + ptid + + + + timestamp + http://cpas.labkey.com/Study#VisitDate + http://cpas.labkey.com/Study#VisitDate + + + varchar + + + double + + + varchar + + + double + + + double + + + double + + + varchar + + + varchar + + + entityid + urn:ehr.labkey.org/#ObjectId + true + + + integer + urn:ehr.labkey.org/#Project + + + timestamp + urn:ehr.labkey.org/#EndDate + + + varchar + + + double + + + varchar + + + double + + + double + + + varchar + + + Urinalysis Results +
+ + + + varchar + http://cpas.labkey.com/Study#ParticipantId + + ptid + + + + timestamp + http://cpas.labkey.com/Study#VisitDate + http://cpas.labkey.com/Study#VisitDate + + + varchar + + + entityid + urn:ehr.labkey.org/#ObjectId + true + + + integer + urn:ehr.labkey.org/#Project + + + timestamp + urn:ehr.labkey.org/#EndDate + + + timestamp + + + boolean + + + varchar + + + varchar + + + varchar + + + varchar + + + varchar + + + varchar + + + varchar + + + integer + + + integer + + + varchar + + + varchar + + + varchar + + + timestamp + + + Arrival +
+ + + + varchar + http://cpas.labkey.com/Study#ParticipantId + + ptid + + + + timestamp + http://cpas.labkey.com/Study#VisitDate + http://cpas.labkey.com/Study#VisitDate + + + timestamp + urn:ehr.labkey.org/#EndDate + + + integer + urn:ehr.labkey.org/#Project + + + entityid + urn:ehr.labkey.org/#ObjectId + true + + + timestamp + + + integer + + + integer + + + integer + + + varchar + + + timestamp + + + timestamp + + + varchar + + + Assignment +
+ + + + varchar + http://cpas.labkey.com/Study#ParticipantId + + ptid + + + + timestamp + http://cpas.labkey.com/Study#VisitDate + http://cpas.labkey.com/Study#VisitDate + + + timestamp + urn:ehr.labkey.org/#EndDate + + + integer + + + entityid + urn:ehr.labkey.org/#ObjectId + true + + + varchar + + + Animal Group Members +
+ + + + varchar + http://cpas.labkey.com/Study#ParticipantId + + ptid + + + + timestamp + http://cpas.labkey.com/Study#VisitDate + http://cpas.labkey.com/Study#VisitDate + + + varchar + + + varchar + + + varchar + urn:ehr.labkey.org/#ObjectId + + + integer + urn:ehr.labkey.org/#Project + + + timestamp + urn:ehr.labkey.org/#EndDate + + + varchar + + + varchar + + + varchar + + + varchar + + + boolean + + + Deaths +
+ + + + varchar + http://cpas.labkey.com/Study#ParticipantId + + ptid + + + + timestamp + http://cpas.labkey.com/Study#VisitDate + http://cpas.labkey.com/Study#VisitDate + + + varchar + + + varchar + + + timestamp + + + timestamp + + + varchar + + + varchar + urn:ehr.labkey.org/#ObjectId + + + integer + urn:ehr.labkey.org/#Project + + + timestamp + urn:ehr.labkey.org/#EndDate + + + varchar + + + Demographics +
+ + + + varchar + http://cpas.labkey.com/Study#ParticipantId + + ptid + + + + timestamp + http://cpas.labkey.com/Study#VisitDate + http://cpas.labkey.com/Study#VisitDate + + + varchar + + + varchar + + + entityid + urn:ehr.labkey.org/#ObjectId + true + + + integer + urn:ehr.labkey.org/#Project + + + timestamp + urn:ehr.labkey.org/#EndDate + + + Departure +
+ + + + varchar + http://cpas.labkey.com/Study#ParticipantId + + ptid + + + + timestamp + http://cpas.labkey.com/Study#VisitDate + http://cpas.labkey.com/Study#VisitDate + + + timestamp + urn:ehr.labkey.org/#EndDate + + + varchar + + + varchar + + + integer + + + integer + + + integer + + + entityid + urn:ehr.labkey.org/#ObjectId + true + + + varchar + + + integer + urn:ehr.labkey.org/#Project + + + Housing +
+ + + + varchar + http://cpas.labkey.com/Study#ParticipantId + + ptid + + + + timestamp + http://cpas.labkey.com/Study#VisitDate + http://cpas.labkey.com/Study#VisitDate + + + varchar + + + varchar + + + entityid + urn:ehr.labkey.org/#ObjectId + true + + + integer + urn:ehr.labkey.org/#Project + + + timestamp + urn:ehr.labkey.org/#EndDate + + + + + + + + + varchar + + + varchar + + + varchar + + + + + + double + + + boolean + + + Tissue Samples +
+ diff --git a/mcc/resources/referenceStudy/study.xml b/mcc/resources/referenceStudy/study.xml new file mode 100644 index 000000000..8f84d3259 --- /dev/null +++ b/mcc/resources/referenceStudy/study.xml @@ -0,0 +1,18 @@ + + + + + + + + + + + + + + + + + + diff --git a/mcc/resources/referenceStudy/studyPolicy.xml b/mcc/resources/referenceStudy/studyPolicy.xml new file mode 100644 index 000000000..61b6c973c --- /dev/null +++ b/mcc/resources/referenceStudy/studyPolicy.xml @@ -0,0 +1,10 @@ + + + ADVANCED_WRITE + + + + + + + \ No newline at end of file From 3b6a8fa7ec0b7afaee182332e3b0214fffa68f5e Mon Sep 17 00:00:00 2001 From: bbimber Date: Sun, 17 Jan 2021 20:32:00 -0800 Subject: [PATCH 44/98] Look for cDNA file in job root --- .../labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java index 9ddc89d4e..354b9a45a 100644 --- a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java +++ b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java @@ -147,7 +147,7 @@ public void complete(PipelineJob job, List inputFiles, List< for (SequenceOutputFile so : inputFiles) { AnalysisModel model = support.getCachedAnalysis(so.getAnalysis_id()); - new CellRangerVDJUtils(job.getLogger()).importAssayData(job, model, so.getFile().getParentFile(), assayId, null, deleteExistingData); + new CellRangerVDJUtils(job.getLogger()).importAssayData(job, model, job.getLogFile().getParentFile(), assayId, null, deleteExistingData); } } } From 58dd3e7e8e01c029aba2c4a17bb3aaaba1af48f9 Mon Sep 17 00:00:00 2001 From: bbimber Date: Mon, 18 Jan 2021 13:30:27 -0800 Subject: [PATCH 45/98] Prepare for single-cell pipeline --- .../src/org/labkey/primeseq/pipeline/BismarkWrapper.java | 2 ++ .../primeseq/pipeline/SequenceJobResourceAllocator.java | 8 ++++---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/primeseq/src/org/labkey/primeseq/pipeline/BismarkWrapper.java b/primeseq/src/org/labkey/primeseq/pipeline/BismarkWrapper.java index f533fd7f2..fe49c917c 100644 --- a/primeseq/src/org/labkey/primeseq/pipeline/BismarkWrapper.java +++ b/primeseq/src/org/labkey/primeseq/pipeline/BismarkWrapper.java @@ -14,6 +14,7 @@ import org.labkey.api.jbrowse.JBrowseService; import org.labkey.api.module.Module; import org.labkey.api.module.ModuleLoader; +import org.labkey.api.pipeline.PipelineJob; import org.labkey.api.pipeline.PipelineJobException; import org.labkey.api.reader.Readers; import org.labkey.api.resource.FileResource; @@ -550,6 +551,7 @@ public Output performAnalysisPerSampleLocal(AnalysisModel model, File inputBam, try { getPipelineCtx().getLogger().debug("preparing for JBrowse"); + getPipelineCtx().getJob().setStatus(PipelineJob.TaskStatus.running, "Preparing for JBrowse"); JBrowseService.get().prepareOutputFile(getPipelineCtx().getJob().getUser(), getPipelineCtx().getLogger(), so.getRowid(), true, additionalConfig); } catch (IOException e) diff --git a/primeseq/src/org/labkey/primeseq/pipeline/SequenceJobResourceAllocator.java b/primeseq/src/org/labkey/primeseq/pipeline/SequenceJobResourceAllocator.java index 1d6507ecb..3cea82534 100644 --- a/primeseq/src/org/labkey/primeseq/pipeline/SequenceJobResourceAllocator.java +++ b/primeseq/src/org/labkey/primeseq/pipeline/SequenceJobResourceAllocator.java @@ -203,22 +203,22 @@ public Integer getMaxRequestMemory(PipelineJob job) Map params = job.getParameters(); if (params != null) { - if (params.containsKey(PipelineStep.StepType.analysis.name()) && params.get(PipelineStep.StepType.analysis.name()).contains("HaplotypeCallerAnalysis")) + if (params.containsKey(PipelineStep.CorePipelineStepTypes.analysis.name()) && params.get(PipelineStep.CorePipelineStepTypes.analysis.name()).contains("HaplotypeCallerAnalysis")) { hasHaplotypeCaller = true; } - if (params.containsKey(PipelineStep.StepType.alignment.name()) && params.get(PipelineStep.StepType.alignment.name()).contains("STAR")) + if (params.containsKey(PipelineStep.CorePipelineStepTypes.alignment.name()) && params.get(PipelineStep.CorePipelineStepTypes.alignment.name()).contains("STAR")) { hasStar = true; } - if (params.containsKey(PipelineStep.StepType.alignment.name()) && params.get(PipelineStep.StepType.alignment.name()).contains("Bismark")) + if (params.containsKey(PipelineStep.CorePipelineStepTypes.alignment.name()) && params.get(PipelineStep.CorePipelineStepTypes.alignment.name()).contains("Bismark")) { hasBismark = true; } - if (params.containsKey(PipelineStep.StepType.alignment.name()) && params.get(PipelineStep.StepType.alignment.name()).contains("Bowtie2")) + if (params.containsKey(PipelineStep.CorePipelineStepTypes.alignment.name()) && params.get(PipelineStep.CorePipelineStepTypes.alignment.name()).contains("Bowtie2")) { hasBowtie2 = true; } From e170f5412846b963a9c7fdd4e486e22d4891eed0 Mon Sep 17 00:00:00 2001 From: bbimber Date: Mon, 18 Jan 2021 16:23:50 -0800 Subject: [PATCH 46/98] Refactor output handler to use JobContext --- .../mgap/pipeline/mGapReleaseGenerator.java | 24 +++++----- .../CombineMethylationRatesHandler.java | 6 --- .../analysis/MethylationRateComparison.java | 6 --- .../MethylationRateComparisonHandler.java | 6 --- .../CellRangerVDJCellHashingHandler.java | 2 +- .../analysis/GBSAnalysisHandler.java | 10 ++-- .../analysis/ImputationAnalysis.java | 46 +++++++++---------- .../analysis/MendelianAnalysisHandler.java | 10 ++-- 8 files changed, 46 insertions(+), 64 deletions(-) diff --git a/mGAP/src/org/labkey/mgap/pipeline/mGapReleaseGenerator.java b/mGAP/src/org/labkey/mgap/pipeline/mGapReleaseGenerator.java index 25dc72490..848a0a09d 100644 --- a/mGAP/src/org/labkey/mgap/pipeline/mGapReleaseGenerator.java +++ b/mGAP/src/org/labkey/mgap/pipeline/mGapReleaseGenerator.java @@ -163,11 +163,11 @@ public Processor() } @Override - public void init(PipelineJob job, SequenceAnalysisJobSupport support, List inputFiles, JSONObject params, File outputDir, List actions, List outputsToCreate) throws UnsupportedOperationException, PipelineJobException + public void init(JobContext ctx, List inputFiles, List actions, List outputsToCreate) throws UnsupportedOperationException, PipelineJobException { - job.getLogger().info("writing track/subset data to file"); - Container target = job.getContainer().isWorkbook() ? job.getContainer().getParent() : job.getContainer(); - TableInfo releaseTracks = QueryService.get().getUserSchema(job.getUser(), target, mGAPSchema.NAME).getTable(mGAPSchema.TABLE_RELEASE_TRACKS); + ctx.getJob().getLogger().info("writing track/subset data to file"); + Container target = ctx.getJob().getContainer().isWorkbook() ? ctx.getJob().getContainer().getParent() : ctx.getJob().getContainer(); + TableInfo releaseTracks = QueryService.get().getUserSchema(ctx.getJob().getUser(), target, mGAPSchema.NAME).getTable(mGAPSchema.TABLE_RELEASE_TRACKS); Set toSelect = new HashSet<>(); toSelect.add(FieldKey.fromString("trackName")); @@ -180,7 +180,7 @@ public void init(PipelineJob job, SequenceAnalysisJobSupport support, List allVcfs = new HashSet<>(); Set distinctTracks = new HashSet<>(); - File trackFile = getTrackListFile(outputDir); + File trackFile = getTrackListFile(ctx.getOutputDir()); try (CSVWriter writer = new CSVWriter(PrintWriters.getPrintWriter(trackFile), '\t', CSVWriter.NO_QUOTE_CHARACTER)) { new TableSelector(releaseTracks, colMap.values(), null, null).forEachResults(rs -> { @@ -191,7 +191,7 @@ public void init(PipelineJob job, SequenceAnalysisJobSupport support, List genomeIds = new HashSet<>(); @@ -223,10 +223,10 @@ public void init(PipelineJob job, SequenceAnalysisJobSupport support, List ids = new HashSet<>(); @@ -240,7 +240,7 @@ public void init(PipelineJob job, SequenceAnalysisJobSupport support, List idsWithRecord = new TableSelector(ti, PageFlowUtil.set("subjectname"), new SimpleFilter(FieldKey.fromString("subjectname"), ids, CompareType.IN), null).getArrayList(String.class); ids.removeAll(idsWithRecord); diff --git a/primeseq/src/org/labkey/primeseq/analysis/CombineMethylationRatesHandler.java b/primeseq/src/org/labkey/primeseq/analysis/CombineMethylationRatesHandler.java index e680124a3..8be4d6678 100644 --- a/primeseq/src/org/labkey/primeseq/analysis/CombineMethylationRatesHandler.java +++ b/primeseq/src/org/labkey/primeseq/analysis/CombineMethylationRatesHandler.java @@ -83,12 +83,6 @@ public SequenceOutputProcessor getProcessor() public class Processor implements SequenceOutputProcessor { - @Override - public void init(PipelineJob job, SequenceAnalysisJobSupport support, List inputFiles, JSONObject params, File outputDir, List actions, List outputsToCreate) throws UnsupportedOperationException, PipelineJobException - { - - } - @Override public void processFilesOnWebserver(PipelineJob job, SequenceAnalysisJobSupport support, List inputFiles, JSONObject params, File outputDir, List actions, List outputsToCreate) throws UnsupportedOperationException, PipelineJobException { diff --git a/primeseq/src/org/labkey/primeseq/analysis/MethylationRateComparison.java b/primeseq/src/org/labkey/primeseq/analysis/MethylationRateComparison.java index 595e58873..80ecfed3a 100644 --- a/primeseq/src/org/labkey/primeseq/analysis/MethylationRateComparison.java +++ b/primeseq/src/org/labkey/primeseq/analysis/MethylationRateComparison.java @@ -119,12 +119,6 @@ public boolean doSplitJobs() public class Processor implements SequenceOutputProcessor { - @Override - public void init(PipelineJob job, SequenceAnalysisJobSupport support, List inputFiles, JSONObject params, File outputDir, List actions, List outputsToCreate) throws UnsupportedOperationException, PipelineJobException - { - - } - @Override public void processFilesRemote(List inputFiles, JobContext ctx) throws UnsupportedOperationException, PipelineJobException { diff --git a/primeseq/src/org/labkey/primeseq/analysis/MethylationRateComparisonHandler.java b/primeseq/src/org/labkey/primeseq/analysis/MethylationRateComparisonHandler.java index 6419e9772..6ce67f819 100644 --- a/primeseq/src/org/labkey/primeseq/analysis/MethylationRateComparisonHandler.java +++ b/primeseq/src/org/labkey/primeseq/analysis/MethylationRateComparisonHandler.java @@ -142,12 +142,6 @@ public SequenceOutputProcessor getProcessor() public class Processor implements SequenceOutputProcessor { - @Override - public void init(PipelineJob job, SequenceAnalysisJobSupport support, List inputFiles, JSONObject params, File outputDir, List actions, List outputsToCreate) throws UnsupportedOperationException, PipelineJobException - { - - } - @Override public void complete(PipelineJob job, List inputs, List outputsCreated, SequenceAnalysisJobSupport support) throws PipelineJobException { diff --git a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java index 354b9a45a..97881348b 100644 --- a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java +++ b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java @@ -103,7 +103,7 @@ public boolean requiresSingleGenome() public class Processor implements SequenceOutputHandler.SequenceOutputProcessor { @Override - public void init(PipelineJob job, SequenceAnalysisJobSupport support, List inputFiles, JSONObject params, File outputDir, List actions, List outputsToCreate) throws UnsupportedOperationException, PipelineJobException + public void init(JobContext ctx, List inputFiles, List actions, List outputsToCreate) throws UnsupportedOperationException, PipelineJobException { //NOTE: this is the pathway to import assay data, whether hashing is used or not CellHashingService.get().prepareHashingAndCiteSeqFilesIfNeeded(outputDir, job, support, "tcrReadsetId", params.optBoolean("excludeFailedcDNA", true), false, false); diff --git a/variantdb/src/org/labkey/variantdb/analysis/GBSAnalysisHandler.java b/variantdb/src/org/labkey/variantdb/analysis/GBSAnalysisHandler.java index 773f99e05..1356bec46 100644 --- a/variantdb/src/org/labkey/variantdb/analysis/GBSAnalysisHandler.java +++ b/variantdb/src/org/labkey/variantdb/analysis/GBSAnalysisHandler.java @@ -80,16 +80,16 @@ public SequenceOutputProcessor getProcessor() public class Processor implements SequenceOutputProcessor { @Override - public void init(PipelineJob job, SequenceAnalysisJobSupport support, List inputFiles, JSONObject params, File outputDir, List actions, List outputsToCreate) throws UnsupportedOperationException, PipelineJobException + public void init(JobContext ctx, List inputFiles, List actions, List outputsToCreate) throws UnsupportedOperationException, PipelineJobException { for (ToolParameterDescriptor pd : getParameters()) { - if (params.containsKey(pd.getName()) && !StringUtils.isEmpty(params.getString(pd.getName()))) + if (ctx.getParams().containsKey(pd.getName()) && !StringUtils.isEmpty(ctx.getParams().getString(pd.getName()))) { - ExpData d = ExperimentService.get().getExpData(params.getInt(pd.getName())); + ExpData d = ExperimentService.get().getExpData(ctx.getParams().getInt(pd.getName())); if (d != null) { - support.cacheExpData(d); + ctx.getSequenceSupport().cacheExpData(d); } } } @@ -98,7 +98,7 @@ public void init(PipelineJob job, SequenceAnalysisJobSupport support, List inputFiles, JSONObject params, File outputDir, List actions, List outputsToCreate) throws UnsupportedOperationException, PipelineJobException + public void init(JobContext ctx, List inputFiles, List actions, List outputsToCreate) throws UnsupportedOperationException, PipelineJobException { //find genome Set ids = new HashSet<>(); @@ -166,7 +166,7 @@ public void init(PipelineJob job, SequenceAnalysisJobSupport support, List pedigreeRecords = generatePedigree(job, params); + List pedigreeRecords = generatePedigree(ctx.getJob(), ctx.getParams()); - File gatkPed = new File(job.getJobSupport(FileAnalysisJobSupport.class).getAnalysisDirectory(), "gatkPed.ped"); - File morganPed = new File(job.getJobSupport(FileAnalysisJobSupport.class).getAnalysisDirectory(), "morgan.ped"); + File gatkPed = new File(ctx.getJob().getJobSupport(FileAnalysisJobSupport.class).getAnalysisDirectory(), "gatkPed.ped"); + File morganPed = new File(ctx.getSourceDirectory(), "morgan.ped"); try (PrintWriter gatkWriter = PrintWriters.getPrintWriter(gatkPed); PrintWriter morganWriter = PrintWriters.getPrintWriter(morganPed)) { morganWriter.write("input pedigree size " + pedigreeRecords.size() + '\n'); @@ -204,39 +204,39 @@ public void init(PipelineJob job, SequenceAnalysisJobSupport support, List inputFiles, JSONObject params, File outputDir, List actions, List outputsToCreate) throws UnsupportedOperationException, PipelineJobException + public void init(JobContext ctx, List inputFiles, List actions, List outputsToCreate) throws UnsupportedOperationException, PipelineJobException { for (ToolParameterDescriptor pd : getParameters()) { - if (params.containsKey(pd.getName()) && !StringUtils.isEmpty(params.getString(pd.getName()))) + if (ctx.getParams().containsKey(pd.getName()) && !StringUtils.isEmpty(ctx.getParams().getString(pd.getName()))) { - ExpData d = ExperimentService.get().getExpData(params.getInt(pd.getName())); + ExpData d = ExperimentService.get().getExpData(ctx.getParams().getInt(pd.getName())); if (d != null) { - support.cacheExpData(d); + ctx.getSequenceSupport().cacheExpData(d); } } } @@ -95,7 +95,7 @@ public void init(PipelineJob job, SequenceAnalysisJobSupport support, List Date: Mon, 18 Jan 2021 16:32:30 -0800 Subject: [PATCH 47/98] Add file missed in last commit --- .../CellRangerVDJCellHashingHandler.java | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java index 97881348b..71d1c1820 100644 --- a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java +++ b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java @@ -29,9 +29,11 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; +import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashSet; import java.util.List; +import java.util.Map; import java.util.Set; public class CellRangerVDJCellHashingHandler extends AbstractParameterizedOutputHandler @@ -41,6 +43,7 @@ public class CellRangerVDJCellHashingHandler extends AbstractParameterizedOutput public static final String TARGET_ASSAY = "targetAssay"; public static final String DELETE_EXISTING_ASSAY_DATA = "deleteExistingAssayData"; + public static final String USE_GEX_BARCODES = "useGexBarcodes"; public CellRangerVDJCellHashingHandler() { @@ -56,6 +59,9 @@ private static List getDefaultParams() }}, true), ToolParameterDescriptor.create("useOutputFileContainer", "Submit to Source File Workbook", "If checked, each job will be submitted to the same workbook as the input file, as opposed to submitting all jobs to the same workbook. This is primarily useful if submitting a large batch of files to process separately. This only applies if 'Run Separately' is selected.", "checkbox", new JSONObject(){{ put("checked", true); + }}, false), + ToolParameterDescriptor.create(USE_GEX_BARCODES, "Use GEX and TCR Cell Barcodes", "If checked, the cell barcode whitelist used for cell hashing will be the union of TCR and GEX cell barcodes. If T-cells are a rare component of total cells, this might enhance the effectiveness of the callers by providing more positive signal.", "checkbox", new JSONObject(){{ + put("checked", true); }}, false) )); @@ -106,7 +112,14 @@ public class Processor implements SequenceOutputHandler.SequenceOutputProcessor public void init(JobContext ctx, List inputFiles, List actions, List outputsToCreate) throws UnsupportedOperationException, PipelineJobException { //NOTE: this is the pathway to import assay data, whether hashing is used or not - CellHashingService.get().prepareHashingAndCiteSeqFilesIfNeeded(outputDir, job, support, "tcrReadsetId", params.optBoolean("excludeFailedcDNA", true), false, false); + CellHashingService.get().prepareHashingAndCiteSeqFilesIfNeeded(ctx.getOutputDir(), ctx.getJob(), ctx.getSequenceSupport(), "tcrReadsetId", ctx.getParams().optBoolean("excludeFailedcDNA", true), false, false); + + if (ctx.getParams().optBoolean(USE_GEX_BARCODES, false)) + { + ctx.getJob().getLogger().info("The union of TCR and GEX cell barcodes will be used for calling"); + Map vLoupeIdToGexBarcodeDir = new HashMap<>(); + + } } @Override @@ -192,6 +205,8 @@ private void processVloupeFile(JobContext ctx, File perCellTsv, Readset rs, Reco { ctx.getLogger().info("Total HTOs for readset: " + htosPerReadset.size()); + //TODO: allow union of GEX and TCR cell barcodes for whitelist! + CellHashingService.CellHashingParameters parameters = CellHashingService.CellHashingParameters.createFromJson(CellHashingService.BARCODE_TYPE.hashing, ctx.getSourceDirectory(), ctx.getParams(), null, rs, null); parameters.cellBarcodeWhitelistFile = createCellbarcodeWhitelist(ctx, perCellTsv, true); parameters.genomeId = genomeId; From a3c253fd6588186c9a6463e3ef73d1c2bfafb30f Mon Sep 17 00:00:00 2001 From: bbimber Date: Tue, 19 Jan 2021 12:16:00 -0800 Subject: [PATCH 48/98] No longer need to use ensembl archive build --- .../labkey/mgap/columnTransforms/JBrowseSessionTransform.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mGAP/src/org/labkey/mgap/columnTransforms/JBrowseSessionTransform.java b/mGAP/src/org/labkey/mgap/columnTransforms/JBrowseSessionTransform.java index 8493d435c..a9a8d57ae 100644 --- a/mGAP/src/org/labkey/mgap/columnTransforms/JBrowseSessionTransform.java +++ b/mGAP/src/org/labkey/mgap/columnTransforms/JBrowseSessionTransform.java @@ -309,6 +309,6 @@ protected String getDatabaseName() protected String getTrackJson() { - return "{\"category\":\"mGAP Variant Catalog\",\"visibleByDefault\": true,\"ensemblUrl\":\"jul2019.archive.ensembl.org\",\"ensemblId\":\"Macaca_mulatta\",\"additionalFeatureMsg\":\"

**The annotations below are primarily derived from human data sources (not macaque), and must be viewed in that context.

\"}"; + return "{\"category\":\"mGAP Variant Catalog\",\"visibleByDefault\": true,\"ensemblId\":\"Macaca_mulatta\",\"additionalFeatureMsg\":\"

**The annotations below are primarily derived from human data sources (not macaque), and must be viewed in that context.

\"}"; } } From 7dc44fe529e5e873e3617ba493a57224dcbbe2e8 Mon Sep 17 00:00:00 2001 From: bbimber Date: Tue, 19 Jan 2021 12:33:29 -0800 Subject: [PATCH 49/98] Bugfix search panel --- mGAP/resources/views/geneSearch.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mGAP/resources/views/geneSearch.html b/mGAP/resources/views/geneSearch.html index 7aaa583a2..515e90b9e 100644 --- a/mGAP/resources/views/geneSearch.html +++ b/mGAP/resources/views/geneSearch.html @@ -170,7 +170,7 @@ } }); - url = Object.keys(unique)[0] + ':' + minStart + '..' + maxStop; + url = Object.keys(uniqueRef)[0] + ':' + minStart + '..' + maxStop; } if (!url) { From 845a0d88b66abb02063644b443e27548ee1dc45c Mon Sep 17 00:00:00 2001 From: bbimber Date: Tue, 19 Jan 2021 14:42:44 -0800 Subject: [PATCH 50/98] Bugfix JBrowse demographics provider --- mGAP/resources/referenceStudy/datasets/datasets_metadata.xml | 2 +- mGAP/resources/schemas/mgap.xml | 2 +- mGAP/src/org/labkey/mgap/mGAPDemographicsSource.java | 4 +++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/mGAP/resources/referenceStudy/datasets/datasets_metadata.xml b/mGAP/resources/referenceStudy/datasets/datasets_metadata.xml index 6c5f22112..6375977ad 100644 --- a/mGAP/resources/referenceStudy/datasets/datasets_metadata.xml +++ b/mGAP/resources/referenceStudy/datasets/datasets_metadata.xml @@ -241,7 +241,7 @@ http://cpas.labkey.com/Study#VisitDate - Gender + Sex varchar diff --git a/mGAP/resources/schemas/mgap.xml b/mGAP/resources/schemas/mgap.xml index fd75ef2c4..e55e9ffca 100644 --- a/mGAP/resources/schemas/mgap.xml +++ b/mGAP/resources/schemas/mgap.xml @@ -1000,7 +1000,7 @@ - Gender + Sex laboratory genders diff --git a/mGAP/src/org/labkey/mgap/mGAPDemographicsSource.java b/mGAP/src/org/labkey/mgap/mGAPDemographicsSource.java index 52539e61d..4f243d06f 100644 --- a/mGAP/src/org/labkey/mgap/mGAPDemographicsSource.java +++ b/mGAP/src/org/labkey/mgap/mGAPDemographicsSource.java @@ -68,6 +68,8 @@ public Map> resolveSubjects(List subjects, C map.put(field, rs.getObject(FieldKey.fromString(field))); } } + + ret.put(subject, map); }); return ret; @@ -77,7 +79,7 @@ public Map> resolveSubjects(List subjects, C public LinkedHashMap getFields() { LinkedHashMap ret = new LinkedHashMap<>(); - ret.put("gender", "Gender"); + ret.put("gender", "Sex"); ret.put("species", "Species"); ret.put("center", "Center"); ret.put("geographic_origin", "Geographic Origin"); From 57886a3aaad1da5a9a345fa26a1ce381ae7b24aa Mon Sep 17 00:00:00 2001 From: bbimber Date: Tue, 19 Jan 2021 15:24:47 -0800 Subject: [PATCH 51/98] Add delimiter --- mGAP/src/org/labkey/mgap/mGAPDemographicsSource.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mGAP/src/org/labkey/mgap/mGAPDemographicsSource.java b/mGAP/src/org/labkey/mgap/mGAPDemographicsSource.java index 4f243d06f..97047cd49 100644 --- a/mGAP/src/org/labkey/mgap/mGAPDemographicsSource.java +++ b/mGAP/src/org/labkey/mgap/mGAPDemographicsSource.java @@ -61,7 +61,7 @@ public Map> resolveSubjects(List subjects, C { if ("datatypes".equalsIgnoreCase(field)) { - map.put("datatypes", (dataTypeMap.containsKey(subject) ? StringUtils.join(dataTypeMap.get(subject)) : null)); + map.put("datatypes", (dataTypeMap.containsKey(subject) ? StringUtils.join(dataTypeMap.get(subject), ",") : null)); } else { From 39725fbc9c6eccb16c660442f0d2484d32cc83f1 Mon Sep 17 00:00:00 2001 From: bbimber Date: Mon, 25 Jan 2021 12:22:29 -0800 Subject: [PATCH 52/98] Fix comment --- mcc/resources/etls/wnprc.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mcc/resources/etls/wnprc.xml b/mcc/resources/etls/wnprc.xml index e5f1ee6d1..de5850d40 100644 --- a/mcc/resources/etls/wnprc.xml +++ b/mcc/resources/etls/wnprc.xml @@ -13,7 +13,7 @@ - + From ed4935d3e9a8d21ed75a913ce5d2b4e3481b9d1e Mon Sep 17 00:00:00 2001 From: bbimber Date: Mon, 25 Jan 2021 12:30:55 -0800 Subject: [PATCH 53/98] Fix markdown syntax --- mcc/src/org/labkey/mcc/MccModule.java | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/mcc/src/org/labkey/mcc/MccModule.java b/mcc/src/org/labkey/mcc/MccModule.java index 03182cb68..96e462c0f 100644 --- a/mcc/src/org/labkey/mcc/MccModule.java +++ b/mcc/src/org/labkey/mcc/MccModule.java @@ -86,19 +86,5 @@ public Set getSchemaNames() private void registerEHRResources() { EHRService.get().registerModule(this); - //EHRService.get().registerTableCustomizer(this, ONPRC_EHRCustomizer.class); - - //Resource r = getModuleResource("/scripts/mcc/mcc_triggers.js"); - //assert r != null; - //EHRService.get().registerTriggerScript(this, r); - - //EHRService.get().registerClientDependency(ClientDependency.supplierFromPath("Ext4"), this); - //EHRService.get().registerClientDependency(ClientDependency.supplierFromPath("onprc_ehr/panel/BloodSummaryPanel.js"), this); - - //EHRService.get().registerReportLink(EHRService.REPORT_LINK_TYPE.housing, "List Single Housed Animals", this, DetailsURL.fromString("/query/executeQuery.view?schemaName=study&query.queryName=demographicsPaired&query.viewName=Single Housed"), "Commonly Used Queries"); - //EHRService.get().registerReportLink(EHRService.REPORT_LINK_TYPE.moreReports, "Clinical Snapshot Printable Report", this, DetailsURL.fromString("/onprc_ehr/SnapshotPrintableReport.view"), "Clinical"); - - //EHRService.get().registerDemographicsProvider(new ActiveCasesDemographicsProvider(this)); - //EHRService.get().registerHistoryDataSource(new DefaultSustainedReleaseDatasource(this)); } } \ No newline at end of file From 5cc59ff833eb4c357b07eb8733ffea6f2bce1734 Mon Sep 17 00:00:00 2001 From: bbimber Date: Tue, 26 Jan 2021 12:42:10 -0800 Subject: [PATCH 54/98] Bugfix TCR import --- .../CellRangerVDJCellHashingHandler.java | 2 +- .../tcrdb/pipeline/CellRangerVDJUtils.java | 25 +++++++++---------- 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java index 71d1c1820..d2443b30f 100644 --- a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java +++ b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java @@ -160,7 +160,7 @@ public void complete(PipelineJob job, List inputFiles, List< for (SequenceOutputFile so : inputFiles) { AnalysisModel model = support.getCachedAnalysis(so.getAnalysis_id()); - new CellRangerVDJUtils(job.getLogger()).importAssayData(job, model, job.getLogFile().getParentFile(), assayId, null, deleteExistingData); + new CellRangerVDJUtils(job.getLogger()).importAssayData(job, model, so.getFile(), job.getLogFile().getParentFile(), assayId, null, deleteExistingData); } } } diff --git a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJUtils.java b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJUtils.java index b6b9640c3..e09c0e871 100644 --- a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJUtils.java +++ b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJUtils.java @@ -63,8 +63,10 @@ public CellRangerVDJUtils(Logger log) _log = log; } - public void importAssayData(PipelineJob job, AnalysisModel model, File outDir, Integer assayId, @Nullable Integer runId, boolean deleteExisting) throws PipelineJobException + public void importAssayData(PipelineJob job, AnalysisModel model, File vLoupeFile, File outDir, Integer assayId, @Nullable Integer runId, boolean deleteExisting) throws PipelineJobException { + File cellRangerOutDir = vLoupeFile.getParentFile(); + if (assayId == null) { _log.info("No assay selected, will not import"); @@ -77,32 +79,29 @@ public void importAssayData(PipelineJob job, AnalysisModel model, File outDir, I throw new PipelineJobException("Unable to find protocol: " + assayId); } - File allCsv = getPerCellCsv(outDir); + File allCsv = getPerCellCsv(cellRangerOutDir); if (!allCsv.exists()) { _log.warn("unable to find consensus contigs: " + allCsv .getPath()); return; } - File consensusCsv = new File(outDir, "consensus_annotations.csv"); + File consensusCsv = new File(cellRangerOutDir, "consensus_annotations.csv"); if (!consensusCsv .exists()) { - _log.warn("unable to find consensus contigs: " + consensusCsv .getPath()); - return; + throw new PipelineJobException("unable to find consensus contigs: " + consensusCsv .getPath()); } - File consensusFasta = new File(outDir, "consensus.fasta"); + File consensusFasta = new File(cellRangerOutDir, "consensus.fasta"); if (!consensusFasta.exists()) { - _log.warn("unable to find FASTA: " + consensusFasta.getPath()); - return; + throw new PipelineJobException("unable to find FASTA: " + consensusFasta.getPath()); } - File allFasta = new File(outDir, "all_contig.fasta"); + File allFasta = new File(cellRangerOutDir, "all_contig.fasta"); if (!allFasta.exists()) { - _log.warn("unable to find FASTA: " + allFasta.getPath()); - return; + throw new PipelineJobException("unable to find FASTA: " + allFasta.getPath()); } _log.info("loading results into assay: " + assayId); @@ -647,8 +646,8 @@ public static void deleteExistingData(AssayProvider ap, ExpProtocol protocol, Co } } - public static File getPerCellCsv(File outDir) + public static File getPerCellCsv(File cellRangerOutDir) { - return new File(outDir, "all_contig_annotations.csv"); + return new File(cellRangerOutDir, "all_contig_annotations.csv"); } } From 60f7ce1a2fd91686fd8839a2410305008628fdbf Mon Sep 17 00:00:00 2001 From: bbimber Date: Tue, 26 Jan 2021 14:05:06 -0800 Subject: [PATCH 55/98] Bugfix TCR hashing --- .../tcrdb/pipeline/CellRangerVDJCellHashingHandler.java | 1 + tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJUtils.java | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java index d2443b30f..a54aea590 100644 --- a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java +++ b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java @@ -221,6 +221,7 @@ private void processVloupeFile(JobContext ctx, File perCellTsv, Readset rs, Reco } + action.addOutput(cellToHto, CellRangerVDJUtils.TCR_HASHING_CALLS, false); ctx.getFileManager().addStepOutputs(action, output); } else if (htosPerReadset.size() == 1) diff --git a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJUtils.java b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJUtils.java index e09c0e871..425d940b1 100644 --- a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJUtils.java +++ b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJUtils.java @@ -56,6 +56,8 @@ public class CellRangerVDJUtils { + public static final String TCR_HASHING_CALLS = "Cell Hashing TCR Calls"; + private Logger _log; public CellRangerVDJUtils(Logger log) @@ -459,7 +461,7 @@ private AssayModel createForRow(String[] line, String sequenceContigName, Intege private File getCellToHtoFile(ExpRun run) throws PipelineJobException { - List datas = run.getInputDatas(CellHashingService.HASHING_CALLS, ExpProtocol.ApplicationType.ExperimentRunOutput); + List datas = run.getInputDatas(TCR_HASHING_CALLS, ExpProtocol.ApplicationType.ExperimentRunOutput); if (datas.isEmpty()) { throw new PipelineJobException("Unable to find hashing calls output"); From 4f0fefbfa7b7e77323dc7ff6e855d609b7934c34 Mon Sep 17 00:00:00 2001 From: bbimber Date: Tue, 26 Jan 2021 18:51:56 -0800 Subject: [PATCH 56/98] Allow CDR3s from rows lacking C-Gene --- .../tcrdb/pipeline/CellRangerVDJUtils.java | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJUtils.java b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJUtils.java index 425d940b1..b52974def 100644 --- a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJUtils.java +++ b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJUtils.java @@ -292,10 +292,15 @@ else if ("Negative".equals(hto)) continue; } - if ("None".equals(line[9])) + String cGene = removeNone(line[9]); + if (cGene == null) { - noCGene++; - continue; + // Only discard these if chain type doesnt match between JGene and VGene. + if (!line[8].substring(0, 3).equals(line[6].substring(0,3))) + { + noCGene++; + continue; + } } if ("False".equals(line[10])) @@ -343,9 +348,9 @@ else if (discordantBarcodes.contains(barcode)) //NOTE: chimeras with a TRDV / TRAJ / TRAC are relatively common. categorize as TRA for reporting ease String locus = line[5]; - if (locus.equals("Multi") && removeNone(line[9]) != null && removeNone(line[8]) != null && removeNone(line[6]) != null) + if (locus.equals("Multi") && cGene != null && removeNone(line[8]) != null && removeNone(line[6]) != null) { - if (removeNone(line[9]).contains("TRAC") && removeNone(line[8]).contains("TRAJ") && removeNone(line[6]).contains("TRDV")) + if (cGene.contains("TRAC") && removeNone(line[8]).contains("TRAJ") && removeNone(line[6]).contains("TRDV")) { locus = "TRA"; multiChainConverted++; @@ -353,7 +358,7 @@ else if (discordantBarcodes.contains(barcode)) } // Aggregate by: cDNA_ID, cdr3, chain, raw_clonotype_id, sequenceContigName, vHit, dHit, jHit, cHit, cdr3_nt - String key = StringUtils.join(new String[]{cDNA.toString(), line[12], locus, clonotypeId, sequenceContigName, removeNone(line[6]), removeNone(line[7]), removeNone(line[8]), removeNone(line[9]), removeNone(line[13])}, "<>"); + String key = StringUtils.join(new String[]{cDNA.toString(), line[12], locus, clonotypeId, sequenceContigName, removeNone(line[6]), removeNone(line[7]), removeNone(line[8]), cGene, removeNone(line[13])}, "<>"); AssayModel am; if (!rows.containsKey(key)) { From 79ac3e623dc70fe630abb62582cf37e69e1e6539 Mon Sep 17 00:00:00 2001 From: bbimber Date: Wed, 27 Jan 2021 17:08:19 -0800 Subject: [PATCH 57/98] Update field name --- .../assay/TCRdb/queries/Data/cDNA Info.qview.xml | 6 +++--- .../singlecell/cdna_libraries/Assay Info.qview.xml | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/tcrdb/resources/assay/TCRdb/queries/Data/cDNA Info.qview.xml b/tcrdb/resources/assay/TCRdb/queries/Data/cDNA Info.qview.xml index d8518ac12..2b3104ee9 100644 --- a/tcrdb/resources/assay/TCRdb/queries/Data/cDNA Info.qview.xml +++ b/tcrdb/resources/assay/TCRdb/queries/Data/cDNA Info.qview.xml @@ -2,9 +2,9 @@ - - - + + + diff --git a/tcrdb/resources/queries/singlecell/cdna_libraries/Assay Info.qview.xml b/tcrdb/resources/queries/singlecell/cdna_libraries/Assay Info.qview.xml index 8a69ed8a5..109d35ad5 100644 --- a/tcrdb/resources/queries/singlecell/cdna_libraries/Assay Info.qview.xml +++ b/tcrdb/resources/queries/singlecell/cdna_libraries/Assay Info.qview.xml @@ -2,10 +2,10 @@ - - - - + + + + From 539a1965aa892d9dbc3fdbc7956bb9796954ac4c Mon Sep 17 00:00:00 2001 From: bbimber Date: Thu, 28 Jan 2021 13:56:55 -0800 Subject: [PATCH 58/98] Checkpoint for MHC migration code --- .../src/org/labkey/primeseq/MhcMigration.java | 1009 +++++++++++++++++ .../labkey/primeseq/PrimeseqController.java | 43 + 2 files changed, 1052 insertions(+) create mode 100644 primeseq/src/org/labkey/primeseq/MhcMigration.java diff --git a/primeseq/src/org/labkey/primeseq/MhcMigration.java b/primeseq/src/org/labkey/primeseq/MhcMigration.java new file mode 100644 index 000000000..76246381b --- /dev/null +++ b/primeseq/src/org/labkey/primeseq/MhcMigration.java @@ -0,0 +1,1009 @@ +package org.labkey.primeseq; + +import org.apache.commons.io.FileUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.labkey.api.collections.CaseInsensitiveHashMap; +import org.labkey.api.data.CompareType; +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerManager; +import org.labkey.api.data.DbSchema; +import org.labkey.api.data.DbSchemaType; +import org.labkey.api.data.DbScope; +import org.labkey.api.data.SimpleFilter; +import org.labkey.api.data.Sort; +import org.labkey.api.data.Table; +import org.labkey.api.data.TableInfo; +import org.labkey.api.data.TableSelector; +import org.labkey.api.data.WorkbookContainerType; +import org.labkey.api.di.DataIntegrationService; +import org.labkey.api.exp.api.DataType; +import org.labkey.api.exp.api.ExpData; +import org.labkey.api.exp.api.ExpRun; +import org.labkey.api.exp.api.ExperimentService; +import org.labkey.api.pipeline.PipelineService; +import org.labkey.api.pipeline.PipelineStatusFile; +import org.labkey.api.pipeline.RecordedActionSet; +import org.labkey.api.query.BatchValidationException; +import org.labkey.api.query.FieldKey; +import org.labkey.api.query.QueryService; +import org.labkey.api.query.UserSchema; +import org.labkey.api.security.User; +import org.labkey.api.sequenceanalysis.SequenceAnalysisService; +import org.labkey.api.sequenceanalysis.model.Readset; +import org.labkey.api.util.PageFlowUtil; +import org.labkey.remoteapi.CommandException; +import org.labkey.remoteapi.Connection; +import org.labkey.remoteapi.query.Filter; +import org.labkey.remoteapi.query.SelectRowsCommand; +import org.labkey.remoteapi.query.SelectRowsResponse; + +import java.io.File; +import java.io.IOException; +import java.net.URI; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; + +public class MhcMigration +{ + private static final Logger _log = LogManager.getLogger(MhcMigration.class); + + private final String remoteServerFolder; + private final String remoteConnectionName; + + private final User user; + private final Container target; + + public MhcMigration(Container c, User u, String remoteConnectionName, String remoteServerFolder) + { + this.target = c; + this.user = u; + this.remoteConnectionName = remoteConnectionName; + this.remoteServerFolder = remoteServerFolder; + } + + private Connection getConnection() + { + DataIntegrationService.RemoteConnection rc = DataIntegrationService.get().getRemoteConnection(remoteConnectionName, target, _log); + + return(rc.connection); + } + + public void doWork() + { + try (DbScope.Transaction transaction = DbScope.getLabKeyScope().ensureTransaction()) + { + createWorkbooks(); + + createLibraries(); + createLibraryMembers(); + + createReadsets(); + transaction.commitAndKeepConnection(); + + createReaddata(); + + createAnalyses(); + createOutputFiles(); + + //TODO: + //samples + //alignment_summary + //alignment_summary_junction + //quality_metrics + //subjects + //WaNPRC + + //sequenceanalysis.haplotypes + //sequenceanalysis.haplotype_types + //sequenceanalysis.haplotype_sequences + + //Create assay runs, including data and haplotypes + + transaction.commit(); + } + } + + private void replaceEntireTable(String schema, String query, List columns, String workbookColName, boolean truncateExisting) throws Exception + { + SelectRowsCommand sr = new SelectRowsCommand(schema, query); + sr.setColumns(columns); + SelectRowsResponse srr = sr.execute(getConnection(), remoteServerFolder); + + List> toInsert = new ArrayList<>(); + srr.getRowset().forEach(r -> { + Map row = new CaseInsensitiveHashMap<>(); + srr.getColumnModel().forEach(col -> { + String colName = (String)col.get("Name"); + Object val = r.getValue(colName); + if ("readset".equals(colName) || "readsetid".equals(colName)) + { + if (!readsetMap.containsKey((int)val)) + { + throw new IllegalStateException("Unable to find readset: " + val); + } + + val = readsetMap.get((int)val); + } + else if ("library_id".equals(colName)) + { + if (!libraryMap.containsKey((int)val)) + { + throw new IllegalStateException("Unable to find library: " + val); + } + + val = libraryMap.get((int)val); + + } + else if ("ref_nt_id".equals(colName)) + { + if (!sequenceMap.containsKey((int)val)) + { + throw new IllegalStateException("Unable to find sequence: " + val); + } + + val = sequenceMap.get((int)val); + } + else if ("analysis_id".equals(colName)) + { + if (!analysisMap.containsKey((int)val)) + { + throw new IllegalStateException("Unable to find analysis: " + val); + } + + val = analysisMap.get((int)val); + } + + row.put(colName, val); + }); + + if (workbookColName != null) + { + Object workbookId = r.getValue(workbookColName); + if (workbookId != null) + { + row.put("container", workbookMap.get(Integer.parseInt(String.valueOf(workbookId))).getId()); + } + } + + toInsert.add(row); + }); + + + + } + + //All of these map remote Id to local Id + private final Map workbookMap = new HashMap<>(); + private final Map readsetMap = new HashMap<>(); + private final Map readdataMap = new HashMap<>(); + private final Map analysisMap = new HashMap<>(); + private final Map libraryMap = new HashMap<>(); + private final Map outputFileMap = new HashMap<>(); + private final Map sequenceMap = new HashMap<>(); + private final Map runIdMap = new HashMap<>(); + private final Map jobIdMap = new HashMap<>(); + + private void createLibraryMembers() + { + _log.info("Creating library members"); + + final UserSchema us = QueryService.get().getUserSchema(user, target, "sequenceanalysis"); + final TableInfo ti = us.getTable("reference_library_members"); + final TableInfo refNtTable = us.getTable("ref_nt_sequences"); + + try + { + SelectRowsCommand sr = new SelectRowsCommand("sequenceanalysis", "reference_library_members"); + sr.setColumns(Arrays.asList("rowid", "library_id", "ref_nt_id", "ref_nt_id/name", "ref_nt_id/seqLength", "workbook/workbookId")); + + SelectRowsResponse srr = sr.execute(getConnection(), remoteServerFolder); + + srr.getRowset().forEach(rd -> { + int remoteId = Integer.parseInt(String.valueOf(rd.getValue("rowid"))); + int seqLength = Integer.parseInt(String.valueOf(rd.getValue("ref_nt_id/seqLength"))); + + int remoteSeqId = Integer.parseInt(String.valueOf(rd.getValue("ref_nt_id"))); + String name = String.valueOf(rd.getValue("ref_nt_id/name")); + int localSeqId = getOrCreateSequence(remoteSeqId, name, seqLength, refNtTable); + + int remoteLibraryId = Integer.parseInt(String.valueOf(rd.getValue("library_id"))); + Integer localLibraryId = libraryMap.get(remoteLibraryId); + if (localLibraryId == null) + { + throw new IllegalStateException("Unable to find library id: " + remoteLibraryId); + } + + SimpleFilter filter = new SimpleFilter(FieldKey.fromString("library_id"), localLibraryId); + filter.addCondition(FieldKey.fromString("ref_nt_id"), localSeqId); + + if (new TableSelector(ti, PageFlowUtil.set("rowid"), filter, null).exists()) + { + //Already exists: + return; + } + + Map toCreate = new CaseInsensitiveHashMap<>(); + toCreate.put("library_id", localLibraryId); + toCreate.put("ref_nt_id", localSeqId); + + try + { + BatchValidationException bve = new BatchValidationException(); + List> created = ti.getUpdateService().insertRows(user, target, Arrays.asList(toCreate), bve, null, null); + if (bve.hasErrors()) + { + throw new RuntimeException(bve); + } + } + catch (Exception e) + { + _log.error(e.getMessage(), e); + throw new RuntimeException(e); + } + }); + } + catch (Exception e) + { + _log.error(e.getMessage(), e); + throw new RuntimeException(e); + } + } + + private int getOrCreateSequence(int remoteSeqId, String name, int seqLength, TableInfo refNtTable) + { + if (sequenceMap.containsKey(remoteSeqId)) + { + return sequenceMap.get(remoteSeqId); + } + else + { + SimpleFilter filter = new SimpleFilter(FieldKey.fromString("name"), name); + filter.addCondition(FieldKey.fromString("datedisabled"), null, CompareType.ISBLANK); + TableSelector ts = new TableSelector(refNtTable, PageFlowUtil.set("rowid", "seqLength"), filter, new Sort("rowid")); + if (ts.exists()) + { + if (ts.getRowCount() > 1) + { + _log.info("Duplicate ref name: " + name); + } + + AtomicInteger localId = new AtomicInteger(-1); + ts.forEachResults(rs -> { + if (rs.getInt(FieldKey.fromString("seqLength")) < seqLength) + { + _log.warn("length doesnt match for " + name + ", expected: " + seqLength); + return; + } + + localId.set(rs.getInt(FieldKey.fromString("rowid"))); + }); + + if (localId.get() != -1) + { + sequenceMap.put(remoteSeqId, localId.get()); + return localId.get(); + } + } + + //TODO: Create sequence? + //throw new IllegalStateException("Expected sequence to exist: " + name); + _log.error("Sequence missing: " + name); + return -1; + } + } + + public String getParent(String path) { + final char separatorChar = '/'; + + int index = path.lastIndexOf(separatorChar); + + return path.substring(0, index); + } + + private void createLibraries() + { + _log.info("Creating libraries"); + try + { + final TableInfo libraryTable = QueryService.get().getUserSchema(user, target, "sequenceanalysis").getTable("reference_libraries"); + + SelectRowsCommand sr = new SelectRowsCommand("sequenceanalysis", "reference_libraries"); + sr.setColumns(Arrays.asList("rowid", "name", "description", "fasta_file", "datedisabled", "assemblyId", "fasta_file/DataFileUrl", "workbook/workbookId")); + + SelectRowsResponse srr = sr.execute(getConnection(), remoteServerFolder); + + srr.getRowset().forEach(rd -> { + int remoteId = Integer.parseInt(String.valueOf(rd.getValue("rowid"))); + + Integer remoteWorkbook = rd.getValue("workbook/workbookId") == null ? null : Integer.parseInt(String.valueOf(rd.getValue("workbook/workbookId"))); + Container targetContainer = remoteWorkbook == null ? target : workbookMap.get(remoteWorkbook); + + SimpleFilter filter = new SimpleFilter(FieldKey.fromString("name"), rd.getValue("name")); + TableSelector ts = new TableSelector(libraryTable, PageFlowUtil.set("rowid"), filter, null); + if (ts.exists()) + { + libraryMap.put(remoteId, ts.getObject(Integer.class)); + } + else + { + Map toCreate = new CaseInsensitiveHashMap<>(); + toCreate.put("name", rd.getValue("name")); + toCreate.put("description", rd.getValue("description")); + toCreate.put("datedisabled", rd.getValue("datedisabled")); + toCreate.put("assemblyId", rd.getValue("assemblyId")); + try + { + String remoteJobRoot = getParent(URI.create(String.valueOf(rd.getValue("fasta_file/DatafileUrl"))).getPath()); + URI localJobRoot = PipelineService.get().getPipelineRootSetting(targetContainer).getRootPath().toURI(); + URI localFasta = translateURI(String.valueOf(rd.getValue("fasta_file/DatafileUrl")), remoteJobRoot, localJobRoot.getPath()); + toCreate.put("fasta_file", getOrCreateExpData(localFasta, targetContainer)); + + //Ensure parent folder exists: + File localJobRootFile = new File(localFasta).getParentFile(); + if (!localJobRootFile.getParentFile().exists()) + { + localJobRootFile.getParentFile().mkdirs(); + } + + _log.info(remoteJobRoot); + _log.info(localJobRoot.getPath()); + File remoteJobRootFile = new File(remoteJobRoot); + if (remoteJobRootFile.exists()) + { + FileUtils.copyDirectory(remoteJobRootFile, localJobRootFile); + } + + BatchValidationException bve = new BatchValidationException(); + List> created = libraryTable.getUpdateService().insertRows(user, target, Arrays.asList(toCreate), bve, null, null); + if (bve.hasErrors()) + { + throw new RuntimeException(bve); + } + + libraryMap.put(remoteId, Integer.parseInt(String.valueOf(created.get(0).get("rowid")))); + } + catch (Exception e) + { + throw new RuntimeException(e); + } + } + }); + } + catch (Exception e) + { + _log.error(e.getMessage(), e); + throw new RuntimeException(e); + } + } + + private void createOutputFiles() + { + _log.info("Creating outputfiles"); + try + { + final TableInfo outputTable = QueryService.get().getUserSchema(user, target, "sequenceanalysis").getTable("outputfiles"); + + SelectRowsCommand sr = new SelectRowsCommand("sequenceanalysis", "outputfiles"); + sr.setColumns(Arrays.asList("rowid", "name", "description", "dataid", "library_id", "readset", "analysis_id", "category", "sra_accession", "dataid/DataFileUrl", "runid/jobid", "runid/Name", "workbook/workbookId", "runid/JobId", "runid/Name", "runid/JobId/FilePath")); + + SelectRowsResponse srr = sr.execute(getConnection(), remoteServerFolder); + + srr.getRowset().forEach(rd -> { + int remoteId = Integer.parseInt(String.valueOf(rd.getValue("rowid"))); + int remoteReadset = Integer.parseInt(String.valueOf(rd.getValue("readset"))); + Integer localReadset = readsetMap.get(remoteReadset); + if (localReadset == null) + { + throw new IllegalArgumentException("Unable to find readset for remote id: " + remoteReadset); + } + + int remoteLibrary = Integer.parseInt(String.valueOf(rd.getValue("library_id"))); + Integer localLibrary = libraryMap.get(remoteLibrary); + if (localLibrary == null) + { + throw new IllegalArgumentException("Unable to find genome for remote id: " + remoteLibrary); + } + + int remoteAnalysis = Integer.parseInt(String.valueOf(rd.getValue("analysis_id"))); + Integer localAnalysis = analysisMap.get(remoteAnalysis); + if (localAnalysis == null) + { + throw new IllegalArgumentException("Unable to find analysis for remote id: " + remoteAnalysis); + } + + Readset rs = SequenceAnalysisService.get().getReadset(localReadset, user); + Container targetWorkbook = ContainerManager.getForId(rs.getContainer()); + + SimpleFilter filter = new SimpleFilter(FieldKey.fromString("readset"), rs.getRowId()); + filter.addCondition(FieldKey.fromString("name"), rd.getValue("name")); + filter.addCondition(FieldKey.fromString("category"), rd.getValue("category")); + filter.addCondition(FieldKey.fromString("analysis_id"), localAnalysis); + filter.addCondition(FieldKey.fromString("container"), targetWorkbook.getId(), CompareType.EQUAL); + + TableSelector tsOutputFiles = new TableSelector(outputTable, PageFlowUtil.set("rowid"), filter, null); + if (tsOutputFiles.exists()) + { + outputFileMap.put(remoteId, tsOutputFiles.getObject(Integer.class)); + } + else + { + Map toCreate = new CaseInsensitiveHashMap<>(); + toCreate.put("readset", rs.getRowId()); + toCreate.put("analysis_id", localAnalysis); + toCreate.put("description", rd.getValue("description")); + toCreate.put("sra_accession", rd.getValue("sra_accession")); + toCreate.put("library_id", localLibrary); + toCreate.put("name", rd.getValue("name")); + toCreate.put("category", rd.getValue("category")); + + try + { + int remoteJobId = Integer.parseInt(String.valueOf(rd.getValue("runid/JobId"))); + int jobId = getOrCreateJob(remoteJobId, targetWorkbook); + PipelineStatusFile sf = PipelineService.get().getStatusFile(jobId); + + String localJobRoot = getParent(sf.getFilePath()); + String remoteJobRoot = getParent(URI.create(String.valueOf(rd.getValue("runid/JobId/FilePath")).replaceAll(" ", "_")).getPath()); + + URI newFileAlignment = translateURI(String.valueOf(rd.getValue("dataid/DatafileUrl")), remoteJobRoot, localJobRoot); + toCreate.put("dataid", getOrCreateExpData(newFileAlignment, targetWorkbook)); + + //Create run: + if (rd.getValue("runid") != null && rd.getValue("runid/JobId") != null) + { + int runId = createExpRun(Integer.parseInt(String.valueOf(rd.getValue("runid"))), targetWorkbook, String.valueOf(rd.getValue("runid/Name")), jobId); + toCreate.put("runid", runId); + } + else + { + _log.error("output missing runid: " + remoteId); + } + + BatchValidationException bve = new BatchValidationException(); + List> created = outputTable.getUpdateService().insertRows(user, target, Arrays.asList(toCreate), bve, null, null); + if (bve.hasErrors()) + { + throw new RuntimeException(bve); + } + + outputFileMap.put(remoteId, Integer.parseInt(String.valueOf(created.get(0).get("rowid")))); + } + catch (Exception e) + { + throw new RuntimeException(e); + } + } + }); + } + catch (Exception e) + { + _log.error(e.getMessage(), e); + throw new RuntimeException(e); + } + } + + private void createAnalyses() + { + _log.info("Creating analyses"); + try + { + final TableInfo analysisTable = QueryService.get().getUserSchema(user, target, "sequenceanalysis").getTable("sequence_analyses"); + + SelectRowsCommand sr = new SelectRowsCommand("sequenceanalysis", "sequence_analyses"); + sr.setColumns(Arrays.asList("rowid", "type", "description", "synopsis", "runid", "readset", "alignmentfile", "reference_library", "library_id", "sra_accession", "alignmentfile/DataFileUrl", "alignmentfile/Name", "reference_library", "reference_library/DataFileUrl", "runid/jobid", "runid/Name", "workbook/workbookId", "runid/JobId", "runid/Name", "runid/JobId/FilePath", "runid/JobId/Description")); + + SelectRowsResponse srr = sr.execute(getConnection(), remoteServerFolder); + + srr.getRowset().forEach(rd -> { + int remoteId = Integer.parseInt(String.valueOf(rd.getValue("rowid"))); + if (rd.getValue("readset") == null) + { + _log.warn("analysis lacks readset, skipping: " + remoteId); + return; + } + + int remoteReadset = Integer.parseInt(String.valueOf(rd.getValue("readset"))); + Integer localReadset = readsetMap.get(remoteReadset); + if (localReadset == null) + { + throw new IllegalArgumentException("Unable to find readset for remote id: " + remoteReadset); + } + + Integer localLibrary = null; + if (rd.getValue("library_id") != null) + { + int remoteLibrary = Integer.parseInt(String.valueOf(rd.getValue("library_id"))); + localLibrary = libraryMap.get(remoteLibrary); + if (localLibrary == null) + { + throw new IllegalArgumentException("Unable to find genome for remote id: " + remoteLibrary); + } + } + + Readset rs = SequenceAnalysisService.get().getReadset(localReadset, user); + Container targetWorkbook = ContainerManager.getForId(rs.getContainer()); + + SimpleFilter filter = new SimpleFilter(FieldKey.fromString("readset"), rs.getRowId()); + filter.addCondition(FieldKey.fromString("runid/JobId/Description"), rd.getValue("runid/JobId/Description")); + filter.addCondition(FieldKey.fromString("container"), targetWorkbook.getId(), CompareType.EQUAL); + + TableSelector tsAnalyses = new TableSelector(analysisTable, PageFlowUtil.set("rowid"), filter, null); + if (tsAnalyses.exists()) + { + analysisMap.put(remoteId, tsAnalyses.getObject(Integer.class)); + } + else + { + Map toCreate = new CaseInsensitiveHashMap<>(); + + toCreate.put("readset", rs.getRowId()); + toCreate.put("synopsis", rd.getValue("synopsis")); + toCreate.put("centerName", rd.getValue("centerName")); + toCreate.put("type", rd.getValue("type")); + toCreate.put("description", rd.getValue("description")); + toCreate.put("sra_accession", rd.getValue("sra_accession")); + if (localLibrary != null) + { + toCreate.put("library_id", localLibrary); + } + + try + { + if (rd.getValue("runid/JobId") == null) + { + _log.info("skipping analysis without runid: " + remoteId); + return; + } + + int remoteJobId = Integer.parseInt(String.valueOf(rd.getValue("runid/JobId"))); + int jobId = getOrCreateJob(remoteJobId, targetWorkbook); + PipelineStatusFile sf = PipelineService.get().getStatusFile(jobId); + + String localJobRoot = getParent(sf.getFilePath()); + String remoteJobRoot = getParent(URI.create(String.valueOf(rd.getValue("runid/JobId/FilePath")).replaceAll(" ", "_")).getPath()); + + URI newFileAlignment = translateURI(String.valueOf(rd.getValue("alignmentfile/DatafileUrl")), remoteJobRoot, localJobRoot); + toCreate.put("alignmentfile", getOrCreateExpData(newFileAlignment, targetWorkbook)); + + if (rd.getValue("reference_library") != null) + { + URI newFile2 = translateURI(String.valueOf(rd.getValue("reference_library/DatafileUrl")), remoteJobRoot, localJobRoot); + toCreate.put("reference_library", getOrCreateExpData(newFile2, targetWorkbook)); + } + + //Create run: + if (rd.getValue("runid") != null && rd.getValue("runid/JobId") != null) + { + int runId = createExpRun(Integer.parseInt(String.valueOf(rd.getValue("runid"))), targetWorkbook, String.valueOf(rd.getValue("runid/Name")), jobId); + toCreate.put("runid", runId); + } + else + { + _log.error("analysis missing runid: " + remoteId); + } + + BatchValidationException bve = new BatchValidationException(); + List> created = analysisTable.getUpdateService().insertRows(user, target, Arrays.asList(toCreate), bve, null, null); + if (bve.hasErrors()) + { + throw new RuntimeException(bve); + } + + analysisMap.put(remoteId, Integer.parseInt(String.valueOf(created.get(0).get("rowid")))); + } + catch (Exception e) + { + throw new RuntimeException(e); + } + } + }); + } + catch (Exception e) + { + _log.error(e.getMessage(), e); + throw new RuntimeException(e); + } + } + + private void createReaddata() + { + _log.info("Creating read data"); + try + { + final TableInfo readdataTable = QueryService.get().getUserSchema(user, target, "sequenceanalysis").getTable("readdata"); + + SelectRowsCommand sr = new SelectRowsCommand("sequenceanalysis", "readdata"); + sr.setColumns(Arrays.asList("rowid", "readset", "platformUnit", "centerName", "date", "fileid1", "fileid1/DataFileUrl", "fileid2", "fileid2/DataFileUrl", "fileid1/Name", "description", "sra_accession", "runid", "runid/jobid", "runid/Name", "readset/workbook/workbookId", "runid/JobId", "runid/Name", "runid/JobId/FilePath", "runid/JobId/Description")); + + SelectRowsResponse srr = sr.execute(getConnection(), remoteServerFolder); + + srr.getRowset().forEach(rd -> { + int remoteId = Integer.parseInt(String.valueOf(rd.getValue("rowid"))); + int remoteReadset = Integer.parseInt(String.valueOf(rd.getValue("readset"))); + Integer localReadset = readsetMap.get(remoteReadset); + if (localReadset == null) + { + throw new IllegalArgumentException("Unable to find readset for remote id: " + remoteReadset); + } + + Readset rs = SequenceAnalysisService.get().getReadset(localReadset, user); + Container targetWorkbook = ContainerManager.getForId(rs.getContainer()); + + SimpleFilter rdFilter = new SimpleFilter(FieldKey.fromString("readset"), rs.getRowId()); + rdFilter.addCondition(FieldKey.fromString("runid/JobId/Description"), rd.getValue("runid/JobId/Description")); + rdFilter.addCondition(FieldKey.fromString("fileid1/Name"), rd.getValue("fileid1/Name")); + rdFilter.addCondition(FieldKey.fromString("container"), targetWorkbook.getId(), CompareType.EQUAL); + + if (rd.getValue("platformUnit") != null) + { + rdFilter.addCondition(FieldKey.fromString("platformUnit"), rd.getValue("platformUnit")); + } + + TableSelector tsReaddata = new TableSelector(readdataTable, PageFlowUtil.set("rowid"), rdFilter, null); + if (tsReaddata.exists()) + { + readdataMap.put(remoteId, tsReaddata.getObject(Integer.class)); + } + else + { + Map toCreate = new CaseInsensitiveHashMap<>(); + toCreate.put("readset", rs.getRowId()); + toCreate.put("platformUnit", rd.getValue("platformUnit")); + toCreate.put("centerName", rd.getValue("centerName")); + toCreate.put("date", rd.getValue("date")); + toCreate.put("description", rd.getValue("description")); + toCreate.put("sra_accession", rd.getValue("sra_accession")); + try + { + if (rd.getValue("runid/JobId") != null) + { + int remoteJobId = Integer.parseInt(String.valueOf(rd.getValue("runid/JobId"))); + int jobId = getOrCreateJob(remoteJobId, targetWorkbook); + PipelineStatusFile sf = PipelineService.get().getStatusFile(jobId); + + String localJobRoot = getParent(sf.getFilePath()); + String remoteJobRoot = getParent(URI.create(String.valueOf(rd.getValue("runid/JobId/FilePath")).replaceAll(" ", "_")).getPath()); + + if (rd.getValue("fileid1/DataFileUrl") != null) + { + URI newFile1 = translateURI(String.valueOf(rd.getValue("fileid1/DataFileUrl")), remoteJobRoot, localJobRoot); + toCreate.put("fileid1", getOrCreateExpData(newFile1, targetWorkbook)); + } + + if (rd.getValue("fileid2/DataFileUrl") != null) + { + URI newFile2 = translateURI(String.valueOf(rd.getValue("fileid2/DatafileUrl")), remoteJobRoot, localJobRoot); + toCreate.put("fileid2", getOrCreateExpData(newFile2, targetWorkbook)); + } + } + else + { + _log.error("readddata missing jobid: " + remoteId); + } + + //Create run: + if (rd.getValue("runid") != null && rd.getValue("runid/JobId") != null) + { + int remoteJobId = Integer.parseInt(String.valueOf(rd.getValue("runid/JobId"))); + int jobId = getOrCreateJob(remoteJobId, targetWorkbook); + int runId = createExpRun(Integer.parseInt(String.valueOf(rd.getValue("runid"))), targetWorkbook, String.valueOf(rd.getValue("runid/Name")), jobId); + toCreate.put("runid", runId); + } + else + { + _log.error("readddata missing runid: " + remoteId); + } + + BatchValidationException bve = new BatchValidationException(); + List> created = readdataTable.getUpdateService().insertRows(user, target, Arrays.asList(toCreate), bve, null, null); + if (bve.hasErrors()) + { + throw new RuntimeException(bve); + } + + readdataMap.put(remoteId, Integer.parseInt(String.valueOf(created.get(0).get("rowid")))); + } + catch (Exception e) + { + throw new RuntimeException(e); + } + } + }); + } + catch (Exception e) + { + _log.error(e.getMessage(), e); + throw new RuntimeException(e); + } + } + + private int getOrCreateExpData(URI file, Container workbook) + { + ExpData ret = ExperimentService.get().getExpDataByURL(new File(file), workbook); + if (ret == null) + { + ret = ExperimentService.get().createData(workbook, new DataType("Data")); + ret.setDataFileURI(file); + ret.save(user); + } + + return ret.getRowId(); + } + + private void createReadsets() + { + _log.info("Creating readsets"); + try + { + final UserSchema us = QueryService.get().getUserSchema(user, target, "sequenceanalysis"); + final TableInfo readsetTable = us.getTable("sequence_readsets"); + + SelectRowsCommand sr = new SelectRowsCommand("sequenceanalysis", "sequence_readsets"); + sr.setColumns(Arrays.asList("rowid", "name", "platform", "application", "librarytype", "chemistry", "comments", "status", "subjectid", "subjectdate", "sampletype", "sampleid", "barcode5", "barcode3", "runid", "runid/jobid", "runid/Name", "workbook/workbookId", "runid/JobId", "runid/Name", "runid/JobId/FilePath")); + + SelectRowsResponse srr = sr.execute(getConnection(), remoteServerFolder); + + srr.getRowset().forEach(rs -> { + int remoteId = Integer.parseInt(String.valueOf(rs.getValue("rowid"))); + int sourceWorkbook = Integer.parseInt(String.valueOf(rs.getValue("workbook/workbookId"))); + Container targetWorkbook = workbookMap.get(sourceWorkbook); + if (targetWorkbook == null) + { + throw new IllegalArgumentException("Unable to find local workbook for source: " + sourceWorkbook); + } + + SimpleFilter rsFilter = new SimpleFilter(FieldKey.fromString("name"), rs.getValue("name")); + rsFilter.addCondition(FieldKey.fromString("container"), targetWorkbook.getId(), CompareType.EQUAL); + if (rs.getValue("subjectid") != null) + { + rsFilter.addCondition(FieldKey.fromString("subjectid"), rs.getValue("subjectid"), CompareType.EQUAL); + } + + TableSelector tsReadset = new TableSelector(readsetTable, PageFlowUtil.set("rowid"), rsFilter, null); + if (tsReadset.exists()) + { + readsetMap.put(remoteId, tsReadset.getObject(Integer.class)); + } + else + { + Map toCreate = new CaseInsensitiveHashMap<>(); + toCreate.put("name", rs.getValue("name")); + toCreate.put("platform", rs.getValue("platform")); + toCreate.put("application", rs.getValue("application")); + toCreate.put("barcode5", rs.getValue("barcode5")); + toCreate.put("barcode3", rs.getValue("barcode3")); + toCreate.put("subjectid", rs.getValue("subjectid")); + + toCreate.put("sampleid", rs.getValue("sampleid")); + toCreate.put("sampledate", rs.getValue("sampledate")); + toCreate.put("librarytype", rs.getValue("librarytype")); + toCreate.put("sampletype", rs.getValue("sampletype")); + toCreate.put("chemistry", rs.getValue("chemistry")); + toCreate.put("comments", rs.getValue("comments")); + toCreate.put("status", rs.getValue("status")); + + toCreate.put("container", targetWorkbook.getId()); + + try + { + //Create run: + if (rs.getValue("runid") != null && rs.getValue("runid/JobId") != null) + { + int remoteJobId = Integer.parseInt(String.valueOf(rs.getValue("runid/JobId"))); + int jobId = getOrCreateJob(remoteJobId, targetWorkbook); + int runid = createExpRun(Integer.parseInt(String.valueOf(rs.getValue("runid"))), targetWorkbook, String.valueOf(rs.getValue("runid/Name")), jobId); + toCreate.put("runid", runid); + } + else + { + _log.error("readset missing run id: " + remoteId); + } + + BatchValidationException bve = new BatchValidationException(); + List> created = readsetTable.getUpdateService().insertRows(user, target, Arrays.asList(toCreate), bve, null, null); + if (bve.hasErrors()) + { + throw new RuntimeException(bve); + } + + readsetMap.put(remoteId, Integer.parseInt(String.valueOf(created.get(0).get("rowid")))); + } + catch (Exception e) + { + throw new RuntimeException(e); + } + } + }); + } + catch (Exception e) + { + _log.error(e.getMessage(), e); + throw new RuntimeException(e); + } + } + + private int getOrCreateJob(int remoteJobId, Container targetWorkbook) + { + if (jobIdMap.containsKey(remoteJobId)) + { + return jobIdMap.get(remoteJobId); + } + + TableInfo ti = DbSchema.get("pipeline", DbSchemaType.Module).getTable("StatusFiles"); + + try + { + SelectRowsCommand sr = new SelectRowsCommand("pipeline", "job"); + sr.addFilter(new Filter("rowid", remoteJobId, Filter.Operator.EQUAL)); + sr.setColumns(Arrays.asList("RowId", "Info", "FilePath", "Email", "Description", "DataUrl", "Job", "Provider", "HadError", "ActiveTaskId")); + + SelectRowsResponse srr = sr.execute(getConnection(), remoteServerFolder); + + File fr = PipelineService.get().getPipelineRootSetting(targetWorkbook).getRootPath(); + + AtomicInteger ret = new AtomicInteger(); + srr.getRowset().forEach(pj -> { + String filepath = String.valueOf(pj.getValue("FilePath")); + if (!filepath.contains("@files")) + { + //This appears to be an error in PRIMe's data: + if (filepath.contains("illuminaImport")) + { + filepath = filepath.replace("illuminaImport", "@files/illuminaImport"); + } + else + { + _log.error("Unexpected filepath: " + pj.getValue("FilePath")); + } + } + + File remoteDir = new File(URI.create(filepath.replaceAll(" ", "_")).getPath()); + File localDir = new File(fr, filepath.split("@files")[1]); + + //Check for existing row: + TableSelector ts = new TableSelector(ti, PageFlowUtil.set("RowId"), new SimpleFilter(FieldKey.fromString("Job"), pj.getValue("Job")), null); + if (ts.exists()) + { + ret.set(ts.getObject(Integer.class)); + } + else + { + Map toCreate = new CaseInsensitiveHashMap<>(); + toCreate.put("Info", pj.getValue("Info")); + toCreate.put("FilePath", localDir.getPath()); + toCreate.put("Email", pj.getValue("Email")); + toCreate.put("Description", pj.getValue("Description")); + toCreate.put("DataUrl", pj.getValue("DataUrl")); + toCreate.put("Job", pj.getValue("Job")); + toCreate.put("Provider", pj.getValue("Provider")); + toCreate.put("HadError", pj.getValue("HadError")); + toCreate.put("ActiveTaskId", pj.getValue("ActiveTaskId")); + toCreate.put("Container", targetWorkbook.getId()); + + toCreate = Table.insert(user, ti, toCreate); + + ret.set((int) toCreate.get("RowId")); + } + + if (localDir.exists()) + { + _log.info("Directory exists, will not re-copy: " + localDir.getPath()); + return; + } + + try + { + _log.info(remoteDir.getPath()); + _log.info(localDir.getPath()); + + if (!localDir.getParentFile().exists()) + { + localDir.getParentFile().mkdirs(); + } + + if (remoteDir.exists()) + { + FileUtils.copyDirectory(remoteDir, localDir); + } + else + { + _log.error("source folder not found: " + remoteDir.getPath()); + } + } + catch (Exception e) + { + throw new RuntimeException(e); + } + }); + + jobIdMap.put(remoteJobId, ret.get()); + + return ret.get(); + } + catch (Exception e) + { + _log.error(e.getMessage(), e); + throw new RuntimeException(e); + } + } + + private int createExpRun(int remoteId, Container c, String name, int localJobId) throws Exception + { + if (runIdMap.containsKey(remoteId)) + { + return runIdMap.get(remoteId); + } + else + { + ExpRun ret = ExperimentService.get().createRunForProvenanceRecording(c, user, new RecordedActionSet(), name, localJobId); + runIdMap.put(remoteId, ret.getRowId()); + + return ret.getRowId(); + } + } + + private void createWorkbooks() + { + _log.info("Creating workbooks"); + try + { + TableInfo containers = QueryService.get().getUserSchema(user, target, "core").getTable("containers"); + + SelectRowsCommand sr = new SelectRowsCommand("core", "workbooks"); + sr.setColumns(Arrays.asList("Name", "Title", "Description")); + SelectRowsResponse srr = sr.execute(getConnection(), remoteServerFolder); + + srr.getRowset().forEach(wb -> { + String localTitle = (String)wb.getValue("Title"); + + TableSelector ts = new TableSelector(containers, PageFlowUtil.set("RowId"), new SimpleFilter(FieldKey.fromString("Title"), localTitle), null); + if (ts.exists()) + { + Container workbook = ContainerManager.getForRowId(ts.getObject(Integer.class)); + workbookMap.put(Integer.parseInt(String.valueOf(wb.getValue("Name"))), workbook); + } + else + { + String description = String.valueOf(wb.getValue("Description")); + if (description != null) + { + description = description + ". "; + } + else + { + description = ""; + } + + description = description + "Originally PRIMe workbook: " + wb.getValue("Name"); + + Container workbook = ContainerManager.createContainer(target, null, localTitle, description, WorkbookContainerType.NAME, user); + workbookMap.put(Integer.parseInt(String.valueOf(wb.getValue("Name"))), workbook); + } + }); + } + catch (CommandException | IOException e) + { + throw new RuntimeException(e); + } + } + + private URI translateURI(String databaseURI, String remoteFolderRoot, String localFolderRoot) + { + databaseURI = databaseURI.replace("\\", "/"); + remoteFolderRoot = remoteFolderRoot.replace("\\", "/").split("@files")[0]; + localFolderRoot = localFolderRoot.replace("\\", "/").split("@files")[0]; + if (localFolderRoot.startsWith("C:")) + { + localFolderRoot = localFolderRoot.replaceAll("^C:", ""); + } + + databaseURI = databaseURI.replace(remoteFolderRoot, localFolderRoot); + + return URI.create(databaseURI); + } +} diff --git a/primeseq/src/org/labkey/primeseq/PrimeseqController.java b/primeseq/src/org/labkey/primeseq/PrimeseqController.java index bc680d82c..f077e2267 100644 --- a/primeseq/src/org/labkey/primeseq/PrimeseqController.java +++ b/primeseq/src/org/labkey/primeseq/PrimeseqController.java @@ -203,4 +203,47 @@ public URLHelper getSuccessURL(Object o) } } + @RequiresSiteAdmin + public class SyncMhcAction extends ConfirmAction + { + @Override + public ModelAndView getConfirmView(Object o, BindException errors) throws Exception + { + setTitle("Sync MHC Data from PRIMe"); + + return new HtmlView(HtmlString.of("This will attempt to sync MHC typing data from PRIMe to the current folder, creating all sequence records and workbooks. Do you want to continue?")); + } + + @Override + public boolean handlePost(Object o, BindException errors) throws Exception + { + try + { + MhcMigration mhc = new MhcMigration(getContainer(), getUser(), "PRIMe", "ONPRC/Core Facilities/Genetics Core/MHC_Typing/"); + mhc.doWork(); + } + catch (Exception e) + { + _log.error(e); + errors.reject(ERROR_MSG, e.getMessage()); + return false; + + } + + return true; + } + + @Override + public void validateCommand(Object o, Errors errors) + { + + } + + @NotNull + @Override + public URLHelper getSuccessURL(Object o) + { + return PageFlowUtil.urlProvider(PipelineUrls.class).urlBegin(getContainer()); + } + } } \ No newline at end of file From 14215142f6bc391966ca8c0119ed9bd850878e53 Mon Sep 17 00:00:00 2001 From: bbimber Date: Fri, 29 Jan 2021 07:07:27 -0800 Subject: [PATCH 59/98] Add validation --- primeseq/src/org/labkey/primeseq/MhcMigration.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/primeseq/src/org/labkey/primeseq/MhcMigration.java b/primeseq/src/org/labkey/primeseq/MhcMigration.java index 76246381b..b17a87fe2 100644 --- a/primeseq/src/org/labkey/primeseq/MhcMigration.java +++ b/primeseq/src/org/labkey/primeseq/MhcMigration.java @@ -856,6 +856,10 @@ private int getOrCreateJob(int remoteJobId, Container targetWorkbook) { filepath = filepath.replace("illuminaImport", "@files/illuminaImport"); } + else if (filepath.contains("sequenceAnalysis")) + { + filepath = filepath.replace("sequenceAnalysis", "@files/sequenceAnalysis"); + } else { _log.error("Unexpected filepath: " + pj.getValue("FilePath")); From 29cdf6dbbcbd0ca366743ad8579135d15ab973b6 Mon Sep 17 00:00:00 2001 From: bbimber Date: Fri, 29 Jan 2021 09:30:54 -0800 Subject: [PATCH 60/98] Convert MHC migration code to a pipeline job --- .../src/org/labkey/primeseq/MhcMigration.java | 1013 --------------- .../labkey/primeseq/PrimeseqController.java | 8 +- .../org/labkey/primeseq/PrimeseqModule.java | 4 + .../pipeline/MhcMigrationPipelineJob.java | 1133 +++++++++++++++++ primeseq/webapp/WEB-INF/primeseqContext.xml | 22 + 5 files changed, 1165 insertions(+), 1015 deletions(-) delete mode 100644 primeseq/src/org/labkey/primeseq/MhcMigration.java create mode 100644 primeseq/src/org/labkey/primeseq/pipeline/MhcMigrationPipelineJob.java diff --git a/primeseq/src/org/labkey/primeseq/MhcMigration.java b/primeseq/src/org/labkey/primeseq/MhcMigration.java deleted file mode 100644 index b17a87fe2..000000000 --- a/primeseq/src/org/labkey/primeseq/MhcMigration.java +++ /dev/null @@ -1,1013 +0,0 @@ -package org.labkey.primeseq; - -import org.apache.commons.io.FileUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.labkey.api.collections.CaseInsensitiveHashMap; -import org.labkey.api.data.CompareType; -import org.labkey.api.data.Container; -import org.labkey.api.data.ContainerManager; -import org.labkey.api.data.DbSchema; -import org.labkey.api.data.DbSchemaType; -import org.labkey.api.data.DbScope; -import org.labkey.api.data.SimpleFilter; -import org.labkey.api.data.Sort; -import org.labkey.api.data.Table; -import org.labkey.api.data.TableInfo; -import org.labkey.api.data.TableSelector; -import org.labkey.api.data.WorkbookContainerType; -import org.labkey.api.di.DataIntegrationService; -import org.labkey.api.exp.api.DataType; -import org.labkey.api.exp.api.ExpData; -import org.labkey.api.exp.api.ExpRun; -import org.labkey.api.exp.api.ExperimentService; -import org.labkey.api.pipeline.PipelineService; -import org.labkey.api.pipeline.PipelineStatusFile; -import org.labkey.api.pipeline.RecordedActionSet; -import org.labkey.api.query.BatchValidationException; -import org.labkey.api.query.FieldKey; -import org.labkey.api.query.QueryService; -import org.labkey.api.query.UserSchema; -import org.labkey.api.security.User; -import org.labkey.api.sequenceanalysis.SequenceAnalysisService; -import org.labkey.api.sequenceanalysis.model.Readset; -import org.labkey.api.util.PageFlowUtil; -import org.labkey.remoteapi.CommandException; -import org.labkey.remoteapi.Connection; -import org.labkey.remoteapi.query.Filter; -import org.labkey.remoteapi.query.SelectRowsCommand; -import org.labkey.remoteapi.query.SelectRowsResponse; - -import java.io.File; -import java.io.IOException; -import java.net.URI; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.atomic.AtomicInteger; - -public class MhcMigration -{ - private static final Logger _log = LogManager.getLogger(MhcMigration.class); - - private final String remoteServerFolder; - private final String remoteConnectionName; - - private final User user; - private final Container target; - - public MhcMigration(Container c, User u, String remoteConnectionName, String remoteServerFolder) - { - this.target = c; - this.user = u; - this.remoteConnectionName = remoteConnectionName; - this.remoteServerFolder = remoteServerFolder; - } - - private Connection getConnection() - { - DataIntegrationService.RemoteConnection rc = DataIntegrationService.get().getRemoteConnection(remoteConnectionName, target, _log); - - return(rc.connection); - } - - public void doWork() - { - try (DbScope.Transaction transaction = DbScope.getLabKeyScope().ensureTransaction()) - { - createWorkbooks(); - - createLibraries(); - createLibraryMembers(); - - createReadsets(); - transaction.commitAndKeepConnection(); - - createReaddata(); - - createAnalyses(); - createOutputFiles(); - - //TODO: - //samples - //alignment_summary - //alignment_summary_junction - //quality_metrics - //subjects - //WaNPRC - - //sequenceanalysis.haplotypes - //sequenceanalysis.haplotype_types - //sequenceanalysis.haplotype_sequences - - //Create assay runs, including data and haplotypes - - transaction.commit(); - } - } - - private void replaceEntireTable(String schema, String query, List columns, String workbookColName, boolean truncateExisting) throws Exception - { - SelectRowsCommand sr = new SelectRowsCommand(schema, query); - sr.setColumns(columns); - SelectRowsResponse srr = sr.execute(getConnection(), remoteServerFolder); - - List> toInsert = new ArrayList<>(); - srr.getRowset().forEach(r -> { - Map row = new CaseInsensitiveHashMap<>(); - srr.getColumnModel().forEach(col -> { - String colName = (String)col.get("Name"); - Object val = r.getValue(colName); - if ("readset".equals(colName) || "readsetid".equals(colName)) - { - if (!readsetMap.containsKey((int)val)) - { - throw new IllegalStateException("Unable to find readset: " + val); - } - - val = readsetMap.get((int)val); - } - else if ("library_id".equals(colName)) - { - if (!libraryMap.containsKey((int)val)) - { - throw new IllegalStateException("Unable to find library: " + val); - } - - val = libraryMap.get((int)val); - - } - else if ("ref_nt_id".equals(colName)) - { - if (!sequenceMap.containsKey((int)val)) - { - throw new IllegalStateException("Unable to find sequence: " + val); - } - - val = sequenceMap.get((int)val); - } - else if ("analysis_id".equals(colName)) - { - if (!analysisMap.containsKey((int)val)) - { - throw new IllegalStateException("Unable to find analysis: " + val); - } - - val = analysisMap.get((int)val); - } - - row.put(colName, val); - }); - - if (workbookColName != null) - { - Object workbookId = r.getValue(workbookColName); - if (workbookId != null) - { - row.put("container", workbookMap.get(Integer.parseInt(String.valueOf(workbookId))).getId()); - } - } - - toInsert.add(row); - }); - - - - } - - //All of these map remote Id to local Id - private final Map workbookMap = new HashMap<>(); - private final Map readsetMap = new HashMap<>(); - private final Map readdataMap = new HashMap<>(); - private final Map analysisMap = new HashMap<>(); - private final Map libraryMap = new HashMap<>(); - private final Map outputFileMap = new HashMap<>(); - private final Map sequenceMap = new HashMap<>(); - private final Map runIdMap = new HashMap<>(); - private final Map jobIdMap = new HashMap<>(); - - private void createLibraryMembers() - { - _log.info("Creating library members"); - - final UserSchema us = QueryService.get().getUserSchema(user, target, "sequenceanalysis"); - final TableInfo ti = us.getTable("reference_library_members"); - final TableInfo refNtTable = us.getTable("ref_nt_sequences"); - - try - { - SelectRowsCommand sr = new SelectRowsCommand("sequenceanalysis", "reference_library_members"); - sr.setColumns(Arrays.asList("rowid", "library_id", "ref_nt_id", "ref_nt_id/name", "ref_nt_id/seqLength", "workbook/workbookId")); - - SelectRowsResponse srr = sr.execute(getConnection(), remoteServerFolder); - - srr.getRowset().forEach(rd -> { - int remoteId = Integer.parseInt(String.valueOf(rd.getValue("rowid"))); - int seqLength = Integer.parseInt(String.valueOf(rd.getValue("ref_nt_id/seqLength"))); - - int remoteSeqId = Integer.parseInt(String.valueOf(rd.getValue("ref_nt_id"))); - String name = String.valueOf(rd.getValue("ref_nt_id/name")); - int localSeqId = getOrCreateSequence(remoteSeqId, name, seqLength, refNtTable); - - int remoteLibraryId = Integer.parseInt(String.valueOf(rd.getValue("library_id"))); - Integer localLibraryId = libraryMap.get(remoteLibraryId); - if (localLibraryId == null) - { - throw new IllegalStateException("Unable to find library id: " + remoteLibraryId); - } - - SimpleFilter filter = new SimpleFilter(FieldKey.fromString("library_id"), localLibraryId); - filter.addCondition(FieldKey.fromString("ref_nt_id"), localSeqId); - - if (new TableSelector(ti, PageFlowUtil.set("rowid"), filter, null).exists()) - { - //Already exists: - return; - } - - Map toCreate = new CaseInsensitiveHashMap<>(); - toCreate.put("library_id", localLibraryId); - toCreate.put("ref_nt_id", localSeqId); - - try - { - BatchValidationException bve = new BatchValidationException(); - List> created = ti.getUpdateService().insertRows(user, target, Arrays.asList(toCreate), bve, null, null); - if (bve.hasErrors()) - { - throw new RuntimeException(bve); - } - } - catch (Exception e) - { - _log.error(e.getMessage(), e); - throw new RuntimeException(e); - } - }); - } - catch (Exception e) - { - _log.error(e.getMessage(), e); - throw new RuntimeException(e); - } - } - - private int getOrCreateSequence(int remoteSeqId, String name, int seqLength, TableInfo refNtTable) - { - if (sequenceMap.containsKey(remoteSeqId)) - { - return sequenceMap.get(remoteSeqId); - } - else - { - SimpleFilter filter = new SimpleFilter(FieldKey.fromString("name"), name); - filter.addCondition(FieldKey.fromString("datedisabled"), null, CompareType.ISBLANK); - TableSelector ts = new TableSelector(refNtTable, PageFlowUtil.set("rowid", "seqLength"), filter, new Sort("rowid")); - if (ts.exists()) - { - if (ts.getRowCount() > 1) - { - _log.info("Duplicate ref name: " + name); - } - - AtomicInteger localId = new AtomicInteger(-1); - ts.forEachResults(rs -> { - if (rs.getInt(FieldKey.fromString("seqLength")) < seqLength) - { - _log.warn("length doesnt match for " + name + ", expected: " + seqLength); - return; - } - - localId.set(rs.getInt(FieldKey.fromString("rowid"))); - }); - - if (localId.get() != -1) - { - sequenceMap.put(remoteSeqId, localId.get()); - return localId.get(); - } - } - - //TODO: Create sequence? - //throw new IllegalStateException("Expected sequence to exist: " + name); - _log.error("Sequence missing: " + name); - return -1; - } - } - - public String getParent(String path) { - final char separatorChar = '/'; - - int index = path.lastIndexOf(separatorChar); - - return path.substring(0, index); - } - - private void createLibraries() - { - _log.info("Creating libraries"); - try - { - final TableInfo libraryTable = QueryService.get().getUserSchema(user, target, "sequenceanalysis").getTable("reference_libraries"); - - SelectRowsCommand sr = new SelectRowsCommand("sequenceanalysis", "reference_libraries"); - sr.setColumns(Arrays.asList("rowid", "name", "description", "fasta_file", "datedisabled", "assemblyId", "fasta_file/DataFileUrl", "workbook/workbookId")); - - SelectRowsResponse srr = sr.execute(getConnection(), remoteServerFolder); - - srr.getRowset().forEach(rd -> { - int remoteId = Integer.parseInt(String.valueOf(rd.getValue("rowid"))); - - Integer remoteWorkbook = rd.getValue("workbook/workbookId") == null ? null : Integer.parseInt(String.valueOf(rd.getValue("workbook/workbookId"))); - Container targetContainer = remoteWorkbook == null ? target : workbookMap.get(remoteWorkbook); - - SimpleFilter filter = new SimpleFilter(FieldKey.fromString("name"), rd.getValue("name")); - TableSelector ts = new TableSelector(libraryTable, PageFlowUtil.set("rowid"), filter, null); - if (ts.exists()) - { - libraryMap.put(remoteId, ts.getObject(Integer.class)); - } - else - { - Map toCreate = new CaseInsensitiveHashMap<>(); - toCreate.put("name", rd.getValue("name")); - toCreate.put("description", rd.getValue("description")); - toCreate.put("datedisabled", rd.getValue("datedisabled")); - toCreate.put("assemblyId", rd.getValue("assemblyId")); - try - { - String remoteJobRoot = getParent(URI.create(String.valueOf(rd.getValue("fasta_file/DatafileUrl"))).getPath()); - URI localJobRoot = PipelineService.get().getPipelineRootSetting(targetContainer).getRootPath().toURI(); - URI localFasta = translateURI(String.valueOf(rd.getValue("fasta_file/DatafileUrl")), remoteJobRoot, localJobRoot.getPath()); - toCreate.put("fasta_file", getOrCreateExpData(localFasta, targetContainer)); - - //Ensure parent folder exists: - File localJobRootFile = new File(localFasta).getParentFile(); - if (!localJobRootFile.getParentFile().exists()) - { - localJobRootFile.getParentFile().mkdirs(); - } - - _log.info(remoteJobRoot); - _log.info(localJobRoot.getPath()); - File remoteJobRootFile = new File(remoteJobRoot); - if (remoteJobRootFile.exists()) - { - FileUtils.copyDirectory(remoteJobRootFile, localJobRootFile); - } - - BatchValidationException bve = new BatchValidationException(); - List> created = libraryTable.getUpdateService().insertRows(user, target, Arrays.asList(toCreate), bve, null, null); - if (bve.hasErrors()) - { - throw new RuntimeException(bve); - } - - libraryMap.put(remoteId, Integer.parseInt(String.valueOf(created.get(0).get("rowid")))); - } - catch (Exception e) - { - throw new RuntimeException(e); - } - } - }); - } - catch (Exception e) - { - _log.error(e.getMessage(), e); - throw new RuntimeException(e); - } - } - - private void createOutputFiles() - { - _log.info("Creating outputfiles"); - try - { - final TableInfo outputTable = QueryService.get().getUserSchema(user, target, "sequenceanalysis").getTable("outputfiles"); - - SelectRowsCommand sr = new SelectRowsCommand("sequenceanalysis", "outputfiles"); - sr.setColumns(Arrays.asList("rowid", "name", "description", "dataid", "library_id", "readset", "analysis_id", "category", "sra_accession", "dataid/DataFileUrl", "runid/jobid", "runid/Name", "workbook/workbookId", "runid/JobId", "runid/Name", "runid/JobId/FilePath")); - - SelectRowsResponse srr = sr.execute(getConnection(), remoteServerFolder); - - srr.getRowset().forEach(rd -> { - int remoteId = Integer.parseInt(String.valueOf(rd.getValue("rowid"))); - int remoteReadset = Integer.parseInt(String.valueOf(rd.getValue("readset"))); - Integer localReadset = readsetMap.get(remoteReadset); - if (localReadset == null) - { - throw new IllegalArgumentException("Unable to find readset for remote id: " + remoteReadset); - } - - int remoteLibrary = Integer.parseInt(String.valueOf(rd.getValue("library_id"))); - Integer localLibrary = libraryMap.get(remoteLibrary); - if (localLibrary == null) - { - throw new IllegalArgumentException("Unable to find genome for remote id: " + remoteLibrary); - } - - int remoteAnalysis = Integer.parseInt(String.valueOf(rd.getValue("analysis_id"))); - Integer localAnalysis = analysisMap.get(remoteAnalysis); - if (localAnalysis == null) - { - throw new IllegalArgumentException("Unable to find analysis for remote id: " + remoteAnalysis); - } - - Readset rs = SequenceAnalysisService.get().getReadset(localReadset, user); - Container targetWorkbook = ContainerManager.getForId(rs.getContainer()); - - SimpleFilter filter = new SimpleFilter(FieldKey.fromString("readset"), rs.getRowId()); - filter.addCondition(FieldKey.fromString("name"), rd.getValue("name")); - filter.addCondition(FieldKey.fromString("category"), rd.getValue("category")); - filter.addCondition(FieldKey.fromString("analysis_id"), localAnalysis); - filter.addCondition(FieldKey.fromString("container"), targetWorkbook.getId(), CompareType.EQUAL); - - TableSelector tsOutputFiles = new TableSelector(outputTable, PageFlowUtil.set("rowid"), filter, null); - if (tsOutputFiles.exists()) - { - outputFileMap.put(remoteId, tsOutputFiles.getObject(Integer.class)); - } - else - { - Map toCreate = new CaseInsensitiveHashMap<>(); - toCreate.put("readset", rs.getRowId()); - toCreate.put("analysis_id", localAnalysis); - toCreate.put("description", rd.getValue("description")); - toCreate.put("sra_accession", rd.getValue("sra_accession")); - toCreate.put("library_id", localLibrary); - toCreate.put("name", rd.getValue("name")); - toCreate.put("category", rd.getValue("category")); - - try - { - int remoteJobId = Integer.parseInt(String.valueOf(rd.getValue("runid/JobId"))); - int jobId = getOrCreateJob(remoteJobId, targetWorkbook); - PipelineStatusFile sf = PipelineService.get().getStatusFile(jobId); - - String localJobRoot = getParent(sf.getFilePath()); - String remoteJobRoot = getParent(URI.create(String.valueOf(rd.getValue("runid/JobId/FilePath")).replaceAll(" ", "_")).getPath()); - - URI newFileAlignment = translateURI(String.valueOf(rd.getValue("dataid/DatafileUrl")), remoteJobRoot, localJobRoot); - toCreate.put("dataid", getOrCreateExpData(newFileAlignment, targetWorkbook)); - - //Create run: - if (rd.getValue("runid") != null && rd.getValue("runid/JobId") != null) - { - int runId = createExpRun(Integer.parseInt(String.valueOf(rd.getValue("runid"))), targetWorkbook, String.valueOf(rd.getValue("runid/Name")), jobId); - toCreate.put("runid", runId); - } - else - { - _log.error("output missing runid: " + remoteId); - } - - BatchValidationException bve = new BatchValidationException(); - List> created = outputTable.getUpdateService().insertRows(user, target, Arrays.asList(toCreate), bve, null, null); - if (bve.hasErrors()) - { - throw new RuntimeException(bve); - } - - outputFileMap.put(remoteId, Integer.parseInt(String.valueOf(created.get(0).get("rowid")))); - } - catch (Exception e) - { - throw new RuntimeException(e); - } - } - }); - } - catch (Exception e) - { - _log.error(e.getMessage(), e); - throw new RuntimeException(e); - } - } - - private void createAnalyses() - { - _log.info("Creating analyses"); - try - { - final TableInfo analysisTable = QueryService.get().getUserSchema(user, target, "sequenceanalysis").getTable("sequence_analyses"); - - SelectRowsCommand sr = new SelectRowsCommand("sequenceanalysis", "sequence_analyses"); - sr.setColumns(Arrays.asList("rowid", "type", "description", "synopsis", "runid", "readset", "alignmentfile", "reference_library", "library_id", "sra_accession", "alignmentfile/DataFileUrl", "alignmentfile/Name", "reference_library", "reference_library/DataFileUrl", "runid/jobid", "runid/Name", "workbook/workbookId", "runid/JobId", "runid/Name", "runid/JobId/FilePath", "runid/JobId/Description")); - - SelectRowsResponse srr = sr.execute(getConnection(), remoteServerFolder); - - srr.getRowset().forEach(rd -> { - int remoteId = Integer.parseInt(String.valueOf(rd.getValue("rowid"))); - if (rd.getValue("readset") == null) - { - _log.warn("analysis lacks readset, skipping: " + remoteId); - return; - } - - int remoteReadset = Integer.parseInt(String.valueOf(rd.getValue("readset"))); - Integer localReadset = readsetMap.get(remoteReadset); - if (localReadset == null) - { - throw new IllegalArgumentException("Unable to find readset for remote id: " + remoteReadset); - } - - Integer localLibrary = null; - if (rd.getValue("library_id") != null) - { - int remoteLibrary = Integer.parseInt(String.valueOf(rd.getValue("library_id"))); - localLibrary = libraryMap.get(remoteLibrary); - if (localLibrary == null) - { - throw new IllegalArgumentException("Unable to find genome for remote id: " + remoteLibrary); - } - } - - Readset rs = SequenceAnalysisService.get().getReadset(localReadset, user); - Container targetWorkbook = ContainerManager.getForId(rs.getContainer()); - - SimpleFilter filter = new SimpleFilter(FieldKey.fromString("readset"), rs.getRowId()); - filter.addCondition(FieldKey.fromString("runid/JobId/Description"), rd.getValue("runid/JobId/Description")); - filter.addCondition(FieldKey.fromString("container"), targetWorkbook.getId(), CompareType.EQUAL); - - TableSelector tsAnalyses = new TableSelector(analysisTable, PageFlowUtil.set("rowid"), filter, null); - if (tsAnalyses.exists()) - { - analysisMap.put(remoteId, tsAnalyses.getObject(Integer.class)); - } - else - { - Map toCreate = new CaseInsensitiveHashMap<>(); - - toCreate.put("readset", rs.getRowId()); - toCreate.put("synopsis", rd.getValue("synopsis")); - toCreate.put("centerName", rd.getValue("centerName")); - toCreate.put("type", rd.getValue("type")); - toCreate.put("description", rd.getValue("description")); - toCreate.put("sra_accession", rd.getValue("sra_accession")); - if (localLibrary != null) - { - toCreate.put("library_id", localLibrary); - } - - try - { - if (rd.getValue("runid/JobId") == null) - { - _log.info("skipping analysis without runid: " + remoteId); - return; - } - - int remoteJobId = Integer.parseInt(String.valueOf(rd.getValue("runid/JobId"))); - int jobId = getOrCreateJob(remoteJobId, targetWorkbook); - PipelineStatusFile sf = PipelineService.get().getStatusFile(jobId); - - String localJobRoot = getParent(sf.getFilePath()); - String remoteJobRoot = getParent(URI.create(String.valueOf(rd.getValue("runid/JobId/FilePath")).replaceAll(" ", "_")).getPath()); - - URI newFileAlignment = translateURI(String.valueOf(rd.getValue("alignmentfile/DatafileUrl")), remoteJobRoot, localJobRoot); - toCreate.put("alignmentfile", getOrCreateExpData(newFileAlignment, targetWorkbook)); - - if (rd.getValue("reference_library") != null) - { - URI newFile2 = translateURI(String.valueOf(rd.getValue("reference_library/DatafileUrl")), remoteJobRoot, localJobRoot); - toCreate.put("reference_library", getOrCreateExpData(newFile2, targetWorkbook)); - } - - //Create run: - if (rd.getValue("runid") != null && rd.getValue("runid/JobId") != null) - { - int runId = createExpRun(Integer.parseInt(String.valueOf(rd.getValue("runid"))), targetWorkbook, String.valueOf(rd.getValue("runid/Name")), jobId); - toCreate.put("runid", runId); - } - else - { - _log.error("analysis missing runid: " + remoteId); - } - - BatchValidationException bve = new BatchValidationException(); - List> created = analysisTable.getUpdateService().insertRows(user, target, Arrays.asList(toCreate), bve, null, null); - if (bve.hasErrors()) - { - throw new RuntimeException(bve); - } - - analysisMap.put(remoteId, Integer.parseInt(String.valueOf(created.get(0).get("rowid")))); - } - catch (Exception e) - { - throw new RuntimeException(e); - } - } - }); - } - catch (Exception e) - { - _log.error(e.getMessage(), e); - throw new RuntimeException(e); - } - } - - private void createReaddata() - { - _log.info("Creating read data"); - try - { - final TableInfo readdataTable = QueryService.get().getUserSchema(user, target, "sequenceanalysis").getTable("readdata"); - - SelectRowsCommand sr = new SelectRowsCommand("sequenceanalysis", "readdata"); - sr.setColumns(Arrays.asList("rowid", "readset", "platformUnit", "centerName", "date", "fileid1", "fileid1/DataFileUrl", "fileid2", "fileid2/DataFileUrl", "fileid1/Name", "description", "sra_accession", "runid", "runid/jobid", "runid/Name", "readset/workbook/workbookId", "runid/JobId", "runid/Name", "runid/JobId/FilePath", "runid/JobId/Description")); - - SelectRowsResponse srr = sr.execute(getConnection(), remoteServerFolder); - - srr.getRowset().forEach(rd -> { - int remoteId = Integer.parseInt(String.valueOf(rd.getValue("rowid"))); - int remoteReadset = Integer.parseInt(String.valueOf(rd.getValue("readset"))); - Integer localReadset = readsetMap.get(remoteReadset); - if (localReadset == null) - { - throw new IllegalArgumentException("Unable to find readset for remote id: " + remoteReadset); - } - - Readset rs = SequenceAnalysisService.get().getReadset(localReadset, user); - Container targetWorkbook = ContainerManager.getForId(rs.getContainer()); - - SimpleFilter rdFilter = new SimpleFilter(FieldKey.fromString("readset"), rs.getRowId()); - rdFilter.addCondition(FieldKey.fromString("runid/JobId/Description"), rd.getValue("runid/JobId/Description")); - rdFilter.addCondition(FieldKey.fromString("fileid1/Name"), rd.getValue("fileid1/Name")); - rdFilter.addCondition(FieldKey.fromString("container"), targetWorkbook.getId(), CompareType.EQUAL); - - if (rd.getValue("platformUnit") != null) - { - rdFilter.addCondition(FieldKey.fromString("platformUnit"), rd.getValue("platformUnit")); - } - - TableSelector tsReaddata = new TableSelector(readdataTable, PageFlowUtil.set("rowid"), rdFilter, null); - if (tsReaddata.exists()) - { - readdataMap.put(remoteId, tsReaddata.getObject(Integer.class)); - } - else - { - Map toCreate = new CaseInsensitiveHashMap<>(); - toCreate.put("readset", rs.getRowId()); - toCreate.put("platformUnit", rd.getValue("platformUnit")); - toCreate.put("centerName", rd.getValue("centerName")); - toCreate.put("date", rd.getValue("date")); - toCreate.put("description", rd.getValue("description")); - toCreate.put("sra_accession", rd.getValue("sra_accession")); - try - { - if (rd.getValue("runid/JobId") != null) - { - int remoteJobId = Integer.parseInt(String.valueOf(rd.getValue("runid/JobId"))); - int jobId = getOrCreateJob(remoteJobId, targetWorkbook); - PipelineStatusFile sf = PipelineService.get().getStatusFile(jobId); - - String localJobRoot = getParent(sf.getFilePath()); - String remoteJobRoot = getParent(URI.create(String.valueOf(rd.getValue("runid/JobId/FilePath")).replaceAll(" ", "_")).getPath()); - - if (rd.getValue("fileid1/DataFileUrl") != null) - { - URI newFile1 = translateURI(String.valueOf(rd.getValue("fileid1/DataFileUrl")), remoteJobRoot, localJobRoot); - toCreate.put("fileid1", getOrCreateExpData(newFile1, targetWorkbook)); - } - - if (rd.getValue("fileid2/DataFileUrl") != null) - { - URI newFile2 = translateURI(String.valueOf(rd.getValue("fileid2/DatafileUrl")), remoteJobRoot, localJobRoot); - toCreate.put("fileid2", getOrCreateExpData(newFile2, targetWorkbook)); - } - } - else - { - _log.error("readddata missing jobid: " + remoteId); - } - - //Create run: - if (rd.getValue("runid") != null && rd.getValue("runid/JobId") != null) - { - int remoteJobId = Integer.parseInt(String.valueOf(rd.getValue("runid/JobId"))); - int jobId = getOrCreateJob(remoteJobId, targetWorkbook); - int runId = createExpRun(Integer.parseInt(String.valueOf(rd.getValue("runid"))), targetWorkbook, String.valueOf(rd.getValue("runid/Name")), jobId); - toCreate.put("runid", runId); - } - else - { - _log.error("readddata missing runid: " + remoteId); - } - - BatchValidationException bve = new BatchValidationException(); - List> created = readdataTable.getUpdateService().insertRows(user, target, Arrays.asList(toCreate), bve, null, null); - if (bve.hasErrors()) - { - throw new RuntimeException(bve); - } - - readdataMap.put(remoteId, Integer.parseInt(String.valueOf(created.get(0).get("rowid")))); - } - catch (Exception e) - { - throw new RuntimeException(e); - } - } - }); - } - catch (Exception e) - { - _log.error(e.getMessage(), e); - throw new RuntimeException(e); - } - } - - private int getOrCreateExpData(URI file, Container workbook) - { - ExpData ret = ExperimentService.get().getExpDataByURL(new File(file), workbook); - if (ret == null) - { - ret = ExperimentService.get().createData(workbook, new DataType("Data")); - ret.setDataFileURI(file); - ret.save(user); - } - - return ret.getRowId(); - } - - private void createReadsets() - { - _log.info("Creating readsets"); - try - { - final UserSchema us = QueryService.get().getUserSchema(user, target, "sequenceanalysis"); - final TableInfo readsetTable = us.getTable("sequence_readsets"); - - SelectRowsCommand sr = new SelectRowsCommand("sequenceanalysis", "sequence_readsets"); - sr.setColumns(Arrays.asList("rowid", "name", "platform", "application", "librarytype", "chemistry", "comments", "status", "subjectid", "subjectdate", "sampletype", "sampleid", "barcode5", "barcode3", "runid", "runid/jobid", "runid/Name", "workbook/workbookId", "runid/JobId", "runid/Name", "runid/JobId/FilePath")); - - SelectRowsResponse srr = sr.execute(getConnection(), remoteServerFolder); - - srr.getRowset().forEach(rs -> { - int remoteId = Integer.parseInt(String.valueOf(rs.getValue("rowid"))); - int sourceWorkbook = Integer.parseInt(String.valueOf(rs.getValue("workbook/workbookId"))); - Container targetWorkbook = workbookMap.get(sourceWorkbook); - if (targetWorkbook == null) - { - throw new IllegalArgumentException("Unable to find local workbook for source: " + sourceWorkbook); - } - - SimpleFilter rsFilter = new SimpleFilter(FieldKey.fromString("name"), rs.getValue("name")); - rsFilter.addCondition(FieldKey.fromString("container"), targetWorkbook.getId(), CompareType.EQUAL); - if (rs.getValue("subjectid") != null) - { - rsFilter.addCondition(FieldKey.fromString("subjectid"), rs.getValue("subjectid"), CompareType.EQUAL); - } - - TableSelector tsReadset = new TableSelector(readsetTable, PageFlowUtil.set("rowid"), rsFilter, null); - if (tsReadset.exists()) - { - readsetMap.put(remoteId, tsReadset.getObject(Integer.class)); - } - else - { - Map toCreate = new CaseInsensitiveHashMap<>(); - toCreate.put("name", rs.getValue("name")); - toCreate.put("platform", rs.getValue("platform")); - toCreate.put("application", rs.getValue("application")); - toCreate.put("barcode5", rs.getValue("barcode5")); - toCreate.put("barcode3", rs.getValue("barcode3")); - toCreate.put("subjectid", rs.getValue("subjectid")); - - toCreate.put("sampleid", rs.getValue("sampleid")); - toCreate.put("sampledate", rs.getValue("sampledate")); - toCreate.put("librarytype", rs.getValue("librarytype")); - toCreate.put("sampletype", rs.getValue("sampletype")); - toCreate.put("chemistry", rs.getValue("chemistry")); - toCreate.put("comments", rs.getValue("comments")); - toCreate.put("status", rs.getValue("status")); - - toCreate.put("container", targetWorkbook.getId()); - - try - { - //Create run: - if (rs.getValue("runid") != null && rs.getValue("runid/JobId") != null) - { - int remoteJobId = Integer.parseInt(String.valueOf(rs.getValue("runid/JobId"))); - int jobId = getOrCreateJob(remoteJobId, targetWorkbook); - int runid = createExpRun(Integer.parseInt(String.valueOf(rs.getValue("runid"))), targetWorkbook, String.valueOf(rs.getValue("runid/Name")), jobId); - toCreate.put("runid", runid); - } - else - { - _log.error("readset missing run id: " + remoteId); - } - - BatchValidationException bve = new BatchValidationException(); - List> created = readsetTable.getUpdateService().insertRows(user, target, Arrays.asList(toCreate), bve, null, null); - if (bve.hasErrors()) - { - throw new RuntimeException(bve); - } - - readsetMap.put(remoteId, Integer.parseInt(String.valueOf(created.get(0).get("rowid")))); - } - catch (Exception e) - { - throw new RuntimeException(e); - } - } - }); - } - catch (Exception e) - { - _log.error(e.getMessage(), e); - throw new RuntimeException(e); - } - } - - private int getOrCreateJob(int remoteJobId, Container targetWorkbook) - { - if (jobIdMap.containsKey(remoteJobId)) - { - return jobIdMap.get(remoteJobId); - } - - TableInfo ti = DbSchema.get("pipeline", DbSchemaType.Module).getTable("StatusFiles"); - - try - { - SelectRowsCommand sr = new SelectRowsCommand("pipeline", "job"); - sr.addFilter(new Filter("rowid", remoteJobId, Filter.Operator.EQUAL)); - sr.setColumns(Arrays.asList("RowId", "Info", "FilePath", "Email", "Description", "DataUrl", "Job", "Provider", "HadError", "ActiveTaskId")); - - SelectRowsResponse srr = sr.execute(getConnection(), remoteServerFolder); - - File fr = PipelineService.get().getPipelineRootSetting(targetWorkbook).getRootPath(); - - AtomicInteger ret = new AtomicInteger(); - srr.getRowset().forEach(pj -> { - String filepath = String.valueOf(pj.getValue("FilePath")); - if (!filepath.contains("@files")) - { - //This appears to be an error in PRIMe's data: - if (filepath.contains("illuminaImport")) - { - filepath = filepath.replace("illuminaImport", "@files/illuminaImport"); - } - else if (filepath.contains("sequenceAnalysis")) - { - filepath = filepath.replace("sequenceAnalysis", "@files/sequenceAnalysis"); - } - else - { - _log.error("Unexpected filepath: " + pj.getValue("FilePath")); - } - } - - File remoteDir = new File(URI.create(filepath.replaceAll(" ", "_")).getPath()); - File localDir = new File(fr, filepath.split("@files")[1]); - - //Check for existing row: - TableSelector ts = new TableSelector(ti, PageFlowUtil.set("RowId"), new SimpleFilter(FieldKey.fromString("Job"), pj.getValue("Job")), null); - if (ts.exists()) - { - ret.set(ts.getObject(Integer.class)); - } - else - { - Map toCreate = new CaseInsensitiveHashMap<>(); - toCreate.put("Info", pj.getValue("Info")); - toCreate.put("FilePath", localDir.getPath()); - toCreate.put("Email", pj.getValue("Email")); - toCreate.put("Description", pj.getValue("Description")); - toCreate.put("DataUrl", pj.getValue("DataUrl")); - toCreate.put("Job", pj.getValue("Job")); - toCreate.put("Provider", pj.getValue("Provider")); - toCreate.put("HadError", pj.getValue("HadError")); - toCreate.put("ActiveTaskId", pj.getValue("ActiveTaskId")); - toCreate.put("Container", targetWorkbook.getId()); - - toCreate = Table.insert(user, ti, toCreate); - - ret.set((int) toCreate.get("RowId")); - } - - if (localDir.exists()) - { - _log.info("Directory exists, will not re-copy: " + localDir.getPath()); - return; - } - - try - { - _log.info(remoteDir.getPath()); - _log.info(localDir.getPath()); - - if (!localDir.getParentFile().exists()) - { - localDir.getParentFile().mkdirs(); - } - - if (remoteDir.exists()) - { - FileUtils.copyDirectory(remoteDir, localDir); - } - else - { - _log.error("source folder not found: " + remoteDir.getPath()); - } - } - catch (Exception e) - { - throw new RuntimeException(e); - } - }); - - jobIdMap.put(remoteJobId, ret.get()); - - return ret.get(); - } - catch (Exception e) - { - _log.error(e.getMessage(), e); - throw new RuntimeException(e); - } - } - - private int createExpRun(int remoteId, Container c, String name, int localJobId) throws Exception - { - if (runIdMap.containsKey(remoteId)) - { - return runIdMap.get(remoteId); - } - else - { - ExpRun ret = ExperimentService.get().createRunForProvenanceRecording(c, user, new RecordedActionSet(), name, localJobId); - runIdMap.put(remoteId, ret.getRowId()); - - return ret.getRowId(); - } - } - - private void createWorkbooks() - { - _log.info("Creating workbooks"); - try - { - TableInfo containers = QueryService.get().getUserSchema(user, target, "core").getTable("containers"); - - SelectRowsCommand sr = new SelectRowsCommand("core", "workbooks"); - sr.setColumns(Arrays.asList("Name", "Title", "Description")); - SelectRowsResponse srr = sr.execute(getConnection(), remoteServerFolder); - - srr.getRowset().forEach(wb -> { - String localTitle = (String)wb.getValue("Title"); - - TableSelector ts = new TableSelector(containers, PageFlowUtil.set("RowId"), new SimpleFilter(FieldKey.fromString("Title"), localTitle), null); - if (ts.exists()) - { - Container workbook = ContainerManager.getForRowId(ts.getObject(Integer.class)); - workbookMap.put(Integer.parseInt(String.valueOf(wb.getValue("Name"))), workbook); - } - else - { - String description = String.valueOf(wb.getValue("Description")); - if (description != null) - { - description = description + ". "; - } - else - { - description = ""; - } - - description = description + "Originally PRIMe workbook: " + wb.getValue("Name"); - - Container workbook = ContainerManager.createContainer(target, null, localTitle, description, WorkbookContainerType.NAME, user); - workbookMap.put(Integer.parseInt(String.valueOf(wb.getValue("Name"))), workbook); - } - }); - } - catch (CommandException | IOException e) - { - throw new RuntimeException(e); - } - } - - private URI translateURI(String databaseURI, String remoteFolderRoot, String localFolderRoot) - { - databaseURI = databaseURI.replace("\\", "/"); - remoteFolderRoot = remoteFolderRoot.replace("\\", "/").split("@files")[0]; - localFolderRoot = localFolderRoot.replace("\\", "/").split("@files")[0]; - if (localFolderRoot.startsWith("C:")) - { - localFolderRoot = localFolderRoot.replaceAll("^C:", ""); - } - - databaseURI = databaseURI.replace(remoteFolderRoot, localFolderRoot); - - return URI.create(databaseURI); - } -} diff --git a/primeseq/src/org/labkey/primeseq/PrimeseqController.java b/primeseq/src/org/labkey/primeseq/PrimeseqController.java index f077e2267..66d1de408 100644 --- a/primeseq/src/org/labkey/primeseq/PrimeseqController.java +++ b/primeseq/src/org/labkey/primeseq/PrimeseqController.java @@ -30,6 +30,8 @@ import org.labkey.api.data.ContainerType; import org.labkey.api.module.Module; import org.labkey.api.module.ModuleLoader; +import org.labkey.api.pipeline.PipeRoot; +import org.labkey.api.pipeline.PipelineService; import org.labkey.api.pipeline.PipelineUrls; import org.labkey.api.security.RequiresPermission; import org.labkey.api.security.RequiresSiteAdmin; @@ -39,6 +41,7 @@ import org.labkey.api.util.URLHelper; import org.labkey.api.view.ActionURL; import org.labkey.api.view.HtmlView; +import org.labkey.primeseq.pipeline.MhcMigrationPipelineJob; import org.springframework.validation.BindException; import org.springframework.validation.Errors; import org.springframework.web.servlet.ModelAndView; @@ -219,8 +222,9 @@ public boolean handlePost(Object o, BindException errors) throws Exception { try { - MhcMigration mhc = new MhcMigration(getContainer(), getUser(), "PRIMe", "ONPRC/Core Facilities/Genetics Core/MHC_Typing/"); - mhc.doWork(); + PipeRoot pipelineRoot = PipelineService.get().findPipelineRoot(getContainer()); + MhcMigrationPipelineJob job = new MhcMigrationPipelineJob(getContainer(), getUser(), getViewContext().getActionURL(), pipelineRoot, "PRIMe", "ONPRC/Core Facilities/Genetics Core/MHC_Typing/"); + PipelineService.get().queueJob(job); } catch (Exception e) { diff --git a/primeseq/src/org/labkey/primeseq/PrimeseqModule.java b/primeseq/src/org/labkey/primeseq/PrimeseqModule.java index bc65a9262..bd36014d1 100644 --- a/primeseq/src/org/labkey/primeseq/PrimeseqModule.java +++ b/primeseq/src/org/labkey/primeseq/PrimeseqModule.java @@ -23,6 +23,7 @@ import org.labkey.api.data.Container; import org.labkey.api.ldk.ExtendedSimpleModule; import org.labkey.api.module.ModuleContext; +import org.labkey.api.pipeline.PipelineService; import org.labkey.api.sequenceanalysis.SequenceAnalysisService; import org.labkey.api.sequenceanalysis.pipeline.SequencePipelineService; import org.labkey.api.util.PageFlowUtil; @@ -36,6 +37,7 @@ import org.labkey.primeseq.pipeline.BlastPipelineJobResourceAllocator; import org.labkey.primeseq.pipeline.ClusterMaintenanceTask; import org.labkey.primeseq.pipeline.ExacloudResourceSettings; +import org.labkey.primeseq.pipeline.MhcMigrationPipelineJob; import org.labkey.primeseq.pipeline.SequenceJobResourceAllocator; import java.util.Collection; @@ -75,6 +77,8 @@ protected void doStartupAfterSpringConfig(ModuleContext moduleContext) ClusterService.get().registerResourceAllocator(new BlastPipelineJobResourceAllocator.Factory()); ClusterService.get().registerResourceAllocator(new SequenceJobResourceAllocator.Factory()); + PipelineService.get().registerPipelineProvider(new MhcMigrationPipelineJob.Provider(this)); + //register resources new PipelineStartup(); diff --git a/primeseq/src/org/labkey/primeseq/pipeline/MhcMigrationPipelineJob.java b/primeseq/src/org/labkey/primeseq/pipeline/MhcMigrationPipelineJob.java new file mode 100644 index 000000000..dfd668352 --- /dev/null +++ b/primeseq/src/org/labkey/primeseq/pipeline/MhcMigrationPipelineJob.java @@ -0,0 +1,1133 @@ +package org.labkey.primeseq.pipeline; + +import org.apache.commons.io.FileUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.labkey.api.collections.CaseInsensitiveHashMap; +import org.labkey.api.data.CompareType; +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerManager; +import org.labkey.api.data.DbSchema; +import org.labkey.api.data.DbSchemaType; +import org.labkey.api.data.DbScope; +import org.labkey.api.data.SimpleFilter; +import org.labkey.api.data.Sort; +import org.labkey.api.data.Table; +import org.labkey.api.data.TableInfo; +import org.labkey.api.data.TableSelector; +import org.labkey.api.data.WorkbookContainerType; +import org.labkey.api.di.DataIntegrationService; +import org.labkey.api.exp.api.DataType; +import org.labkey.api.exp.api.ExpData; +import org.labkey.api.exp.api.ExpRun; +import org.labkey.api.exp.api.ExperimentService; +import org.labkey.api.files.FileUrls; +import org.labkey.api.module.Module; +import org.labkey.api.pipeline.AbstractTaskFactory; +import org.labkey.api.pipeline.AbstractTaskFactorySettings; +import org.labkey.api.pipeline.PipeRoot; +import org.labkey.api.pipeline.PipelineDirectory; +import org.labkey.api.pipeline.PipelineJob; +import org.labkey.api.pipeline.PipelineJobException; +import org.labkey.api.pipeline.PipelineJobService; +import org.labkey.api.pipeline.PipelineProvider; +import org.labkey.api.pipeline.PipelineService; +import org.labkey.api.pipeline.PipelineStatusFile; +import org.labkey.api.pipeline.RecordedActionSet; +import org.labkey.api.pipeline.TaskId; +import org.labkey.api.pipeline.TaskPipeline; +import org.labkey.api.query.BatchValidationException; +import org.labkey.api.query.FieldKey; +import org.labkey.api.query.QueryService; +import org.labkey.api.query.UserSchema; +import org.labkey.api.security.User; +import org.labkey.api.sequenceanalysis.SequenceAnalysisService; +import org.labkey.api.sequenceanalysis.model.Readset; +import org.labkey.api.util.FileType; +import org.labkey.api.util.FileUtil; +import org.labkey.api.util.PageFlowUtil; +import org.labkey.api.view.ActionURL; +import org.labkey.api.view.ViewBackgroundInfo; +import org.labkey.api.view.ViewContext; +import org.labkey.remoteapi.CommandException; +import org.labkey.remoteapi.Connection; +import org.labkey.remoteapi.query.Filter; +import org.labkey.remoteapi.query.SelectRowsCommand; +import org.labkey.remoteapi.query.SelectRowsResponse; + +import java.io.File; +import java.io.IOException; +import java.net.URI; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; + +public class MhcMigrationPipelineJob extends PipelineJob +{ + private static final Logger _log = LogManager.getLogger(MhcMigrationPipelineJob.class); + + private String remoteServerFolder; + private String remoteConnectionName; + + private Container targetContainer; + + public static class Provider extends PipelineProvider + { + public static final String NAME = "mhcMigrationPipeline"; + + public Provider(Module owningModule) + { + super(NAME, owningModule); + } + + @Override + public void updateFileProperties(ViewContext context, PipeRoot pr, PipelineDirectory directory, boolean includeAll) + { + + } + } + + // Default constructor for serialization + protected MhcMigrationPipelineJob() + { + } + + public MhcMigrationPipelineJob(Container c, User u, ActionURL url, PipeRoot pipeRoot, String remoteConnectionName, String remoteServerFolder) + { + super(Provider.NAME, new ViewBackgroundInfo(c, u, url), pipeRoot); + + this.targetContainer = c; + this.remoteConnectionName = remoteConnectionName; + this.remoteServerFolder = remoteServerFolder; + + File subdir = new File(pipeRoot.getRootPath(), Provider.NAME); + if (!subdir.exists()) + { + subdir.mkdirs(); + } + + setLogFile(new File(subdir, FileUtil.makeFileNameWithTimestamp("mhcMigration", "log"))); + + } + + @Override + public ActionURL getStatusHref() + { + return PageFlowUtil.urlProvider(FileUrls.class).urlBegin(getContainer()); + } + + @Override + public String getDescription() + { + return "Find Orphan Sequence Files"; + } + + @Override + public TaskPipeline getTaskPipeline() + { + return PipelineJobService.get().getTaskPipeline(new TaskId(MhcMigrationPipelineJob.class)); + } + + public static class Task extends PipelineJob.Task + { + protected Task(Factory factory, PipelineJob job) + { + super(factory, job); + } + + public static class Factory extends AbstractTaskFactory + { + public Factory() + { + super(Task.class); + } + + @Override + public List getInputTypes() + { + return Collections.emptyList(); + } + + @Override + public String getStatusName() + { + return PipelineJob.TaskStatus.running.toString(); + } + + @Override + public List getProtocolActionNames() + { + return Arrays.asList("Migrate MHC Data"); + } + + @Override + public PipelineJob.Task createTask(PipelineJob job) + { + return new Task(this, job); + } + + @Override + public boolean isJobComplete(PipelineJob job) + { + return false; + } + } + + private MhcMigrationPipelineJob getPipelineJob() + { + return (MhcMigrationPipelineJob)getJob(); + } + + private Connection getConnection() + { + DataIntegrationService.RemoteConnection rc = DataIntegrationService.get().getRemoteConnection(getPipelineJob().remoteConnectionName, getPipelineJob().targetContainer, _log); + + return(rc.connection); + } + + @Override + public RecordedActionSet run() throws PipelineJobException + { + try (DbScope.Transaction transaction = DbScope.getLabKeyScope().ensureTransaction()) + { + createWorkbooks(); + + createLibraries(); + createLibraryMembers(); + + createReadsets(); + transaction.commitAndKeepConnection(); + + createReaddata(); + + createAnalyses(); + createOutputFiles(); + + //TODO: + //samples + //alignment_summary + //alignment_summary_junction + //quality_metrics + //subjects + //WaNPRC + + //sequenceanalysis.haplotypes + //sequenceanalysis.haplotype_types + //sequenceanalysis.haplotype_sequences + + //Create assay runs, including data and haplotypes + + transaction.commit(); + } + + return new RecordedActionSet(); + } + + private void replaceEntireTable(String schema, String query, List columns, String workbookColName, boolean truncateExisting) throws Exception + { + SelectRowsCommand sr = new SelectRowsCommand(schema, query); + sr.setColumns(columns); + SelectRowsResponse srr = sr.execute(getConnection(), getPipelineJob().remoteServerFolder); + + List> toInsert = new ArrayList<>(); + srr.getRowset().forEach(r -> { + Map row = new CaseInsensitiveHashMap<>(); + srr.getColumnModel().forEach(col -> { + String colName = (String) col.get("Name"); + Object val = r.getValue(colName); + if ("readset".equals(colName) || "readsetid".equals(colName)) + { + if (!readsetMap.containsKey((int) val)) + { + throw new IllegalStateException("Unable to find readset: " + val); + } + + val = readsetMap.get((int) val); + } + else if ("library_id".equals(colName)) + { + if (!libraryMap.containsKey((int) val)) + { + throw new IllegalStateException("Unable to find library: " + val); + } + + val = libraryMap.get((int) val); + + } + else if ("ref_nt_id".equals(colName)) + { + if (!sequenceMap.containsKey((int) val)) + { + throw new IllegalStateException("Unable to find sequence: " + val); + } + + val = sequenceMap.get((int) val); + } + else if ("analysis_id".equals(colName)) + { + if (!analysisMap.containsKey((int) val)) + { + throw new IllegalStateException("Unable to find analysis: " + val); + } + + val = analysisMap.get((int) val); + } + + row.put(colName, val); + }); + + if (workbookColName != null) + { + Object workbookId = r.getValue(workbookColName); + if (workbookId != null) + { + row.put("container", workbookMap.get(Integer.parseInt(String.valueOf(workbookId))).getId()); + } + } + + toInsert.add(row); + }); + + + } + + //All of these map remote Id to local Id + private final Map workbookMap = new HashMap<>(); + private final Map readsetMap = new HashMap<>(); + private final Map readdataMap = new HashMap<>(); + private final Map analysisMap = new HashMap<>(); + private final Map libraryMap = new HashMap<>(); + private final Map outputFileMap = new HashMap<>(); + private final Map sequenceMap = new HashMap<>(); + private final Map runIdMap = new HashMap<>(); + private final Map jobIdMap = new HashMap<>(); + + private void createLibraryMembers() + { + _log.info("Creating library members"); + + final UserSchema us = QueryService.get().getUserSchema(getJob().getUser(), getPipelineJob().targetContainer, "sequenceanalysis"); + final TableInfo ti = us.getTable("reference_library_members"); + final TableInfo refNtTable = us.getTable("ref_nt_sequences"); + + try + { + SelectRowsCommand sr = new SelectRowsCommand("sequenceanalysis", "reference_library_members"); + sr.setColumns(Arrays.asList("rowid", "library_id", "ref_nt_id", "ref_nt_id/name", "ref_nt_id/seqLength", "workbook/workbookId")); + + SelectRowsResponse srr = sr.execute(getConnection(), getPipelineJob().remoteServerFolder); + + srr.getRowset().forEach(rd -> { + int remoteId = Integer.parseInt(String.valueOf(rd.getValue("rowid"))); + int seqLength = Integer.parseInt(String.valueOf(rd.getValue("ref_nt_id/seqLength"))); + + int remoteSeqId = Integer.parseInt(String.valueOf(rd.getValue("ref_nt_id"))); + String name = String.valueOf(rd.getValue("ref_nt_id/name")); + int localSeqId = getOrCreateSequence(remoteSeqId, name, seqLength, refNtTable); + + int remoteLibraryId = Integer.parseInt(String.valueOf(rd.getValue("library_id"))); + Integer localLibraryId = libraryMap.get(remoteLibraryId); + if (localLibraryId == null) + { + throw new IllegalStateException("Unable to find library id: " + remoteLibraryId); + } + + SimpleFilter filter = new SimpleFilter(FieldKey.fromString("library_id"), localLibraryId); + filter.addCondition(FieldKey.fromString("ref_nt_id"), localSeqId); + + if (new TableSelector(ti, PageFlowUtil.set("rowid"), filter, null).exists()) + { + //Already exists: + return; + } + + Map toCreate = new CaseInsensitiveHashMap<>(); + toCreate.put("library_id", localLibraryId); + toCreate.put("ref_nt_id", localSeqId); + + try + { + BatchValidationException bve = new BatchValidationException(); + List> created = ti.getUpdateService().insertRows(getJob().getUser(), getPipelineJob().targetContainer, Arrays.asList(toCreate), bve, null, null); + if (bve.hasErrors()) + { + throw new RuntimeException(bve); + } + } + catch (Exception e) + { + _log.error(e.getMessage(), e); + throw new RuntimeException(e); + } + }); + } + catch (Exception e) + { + _log.error(e.getMessage(), e); + throw new RuntimeException(e); + } + } + + private int getOrCreateSequence(int remoteSeqId, String name, int seqLength, TableInfo refNtTable) + { + if (sequenceMap.containsKey(remoteSeqId)) + { + return sequenceMap.get(remoteSeqId); + } + else + { + SimpleFilter filter = new SimpleFilter(FieldKey.fromString("name"), name); + filter.addCondition(FieldKey.fromString("datedisabled"), null, CompareType.ISBLANK); + TableSelector ts = new TableSelector(refNtTable, PageFlowUtil.set("rowid", "seqLength"), filter, new Sort("rowid")); + if (ts.exists()) + { + if (ts.getRowCount() > 1) + { + _log.info("Duplicate ref name: " + name); + } + + AtomicInteger localId = new AtomicInteger(-1); + ts.forEachResults(rs -> { + if (rs.getInt(FieldKey.fromString("seqLength")) < seqLength) + { + _log.warn("length doesnt match for " + name + ", expected: " + seqLength); + return; + } + + localId.set(rs.getInt(FieldKey.fromString("rowid"))); + }); + + if (localId.get() != -1) + { + sequenceMap.put(remoteSeqId, localId.get()); + return localId.get(); + } + } + + //TODO: Create sequence? + //throw new IllegalStateException("Expected sequence to exist: " + name); + _log.error("Sequence missing: " + name); + return -1; + } + } + + public String getParent(String path) + { + final char separatorChar = '/'; + + int index = path.lastIndexOf(separatorChar); + + return path.substring(0, index); + } + + private void createLibraries() + { + _log.info("Creating libraries"); + try + { + final TableInfo libraryTable = QueryService.get().getUserSchema(getJob().getUser(), getPipelineJob().targetContainer, "sequenceanalysis").getTable("reference_libraries"); + + SelectRowsCommand sr = new SelectRowsCommand("sequenceanalysis", "reference_libraries"); + sr.setColumns(Arrays.asList("rowid", "name", "description", "fasta_file", "datedisabled", "assemblyId", "fasta_file/DataFileUrl", "workbook/workbookId")); + + SelectRowsResponse srr = sr.execute(getConnection(), getPipelineJob().remoteServerFolder); + + srr.getRowset().forEach(rd -> { + int remoteId = Integer.parseInt(String.valueOf(rd.getValue("rowid"))); + + Integer remoteWorkbook = rd.getValue("workbook/workbookId") == null ? null : Integer.parseInt(String.valueOf(rd.getValue("workbook/workbookId"))); + Container targetContainer = remoteWorkbook == null ? getPipelineJob().targetContainer : workbookMap.get(remoteWorkbook); + + SimpleFilter filter = new SimpleFilter(FieldKey.fromString("name"), rd.getValue("name")); + TableSelector ts = new TableSelector(libraryTable, PageFlowUtil.set("rowid"), filter, null); + if (ts.exists()) + { + libraryMap.put(remoteId, ts.getObject(Integer.class)); + } + else + { + Map toCreate = new CaseInsensitiveHashMap<>(); + toCreate.put("name", rd.getValue("name")); + toCreate.put("description", rd.getValue("description")); + toCreate.put("datedisabled", rd.getValue("datedisabled")); + toCreate.put("assemblyId", rd.getValue("assemblyId")); + try + { + String remoteJobRoot = getParent(URI.create(String.valueOf(rd.getValue("fasta_file/DatafileUrl"))).getPath()); + URI localJobRoot = PipelineService.get().getPipelineRootSetting(targetContainer).getRootPath().toURI(); + URI localFasta = translateURI(String.valueOf(rd.getValue("fasta_file/DatafileUrl")), remoteJobRoot, localJobRoot.getPath()); + toCreate.put("fasta_file", getOrCreateExpData(localFasta, targetContainer)); + + //Ensure parent folder exists: + File localJobRootFile = new File(localFasta).getParentFile(); + if (!localJobRootFile.getParentFile().exists()) + { + localJobRootFile.getParentFile().mkdirs(); + } + + _log.info(remoteJobRoot); + _log.info(localJobRoot.getPath()); + File remoteJobRootFile = new File(remoteJobRoot); + if (remoteJobRootFile.exists()) + { + FileUtils.copyDirectory(remoteJobRootFile, localJobRootFile); + } + + BatchValidationException bve = new BatchValidationException(); + List> created = libraryTable.getUpdateService().insertRows(getJob().getUser(), getPipelineJob().targetContainer, Arrays.asList(toCreate), bve, null, null); + if (bve.hasErrors()) + { + throw new RuntimeException(bve); + } + + libraryMap.put(remoteId, Integer.parseInt(String.valueOf(created.get(0).get("rowid")))); + } + catch (Exception e) + { + throw new RuntimeException(e); + } + } + }); + } + catch (Exception e) + { + _log.error(e.getMessage(), e); + throw new RuntimeException(e); + } + } + + private void createOutputFiles() + { + _log.info("Creating outputfiles"); + try + { + final TableInfo outputTable = QueryService.get().getUserSchema(getJob().getUser(), getPipelineJob().targetContainer, "sequenceanalysis").getTable("outputfiles"); + + SelectRowsCommand sr = new SelectRowsCommand("sequenceanalysis", "outputfiles"); + sr.setColumns(Arrays.asList("rowid", "name", "description", "dataid", "library_id", "readset", "analysis_id", "category", "sra_accession", "dataid/DataFileUrl", "runid/jobid", "runid/Name", "workbook/workbookId", "runid/JobId", "runid/Name", "runid/JobId/FilePath")); + + SelectRowsResponse srr = sr.execute(getConnection(), getPipelineJob().remoteServerFolder); + + srr.getRowset().forEach(rd -> { + int remoteId = Integer.parseInt(String.valueOf(rd.getValue("rowid"))); + int remoteReadset = Integer.parseInt(String.valueOf(rd.getValue("readset"))); + Integer localReadset = readsetMap.get(remoteReadset); + if (localReadset == null) + { + throw new IllegalArgumentException("Unable to find readset for remote id: " + remoteReadset); + } + + int remoteLibrary = Integer.parseInt(String.valueOf(rd.getValue("library_id"))); + Integer localLibrary = libraryMap.get(remoteLibrary); + if (localLibrary == null) + { + throw new IllegalArgumentException("Unable to find genome for remote id: " + remoteLibrary); + } + + int remoteAnalysis = Integer.parseInt(String.valueOf(rd.getValue("analysis_id"))); + Integer localAnalysis = analysisMap.get(remoteAnalysis); + if (localAnalysis == null) + { + throw new IllegalArgumentException("Unable to find analysis for remote id: " + remoteAnalysis); + } + + Readset rs = SequenceAnalysisService.get().getReadset(localReadset, getJob().getUser()); + Container targetWorkbook = ContainerManager.getForId(rs.getContainer()); + + SimpleFilter filter = new SimpleFilter(FieldKey.fromString("readset"), rs.getRowId()); + filter.addCondition(FieldKey.fromString("name"), rd.getValue("name")); + filter.addCondition(FieldKey.fromString("category"), rd.getValue("category")); + filter.addCondition(FieldKey.fromString("analysis_id"), localAnalysis); + filter.addCondition(FieldKey.fromString("container"), targetWorkbook.getId(), CompareType.EQUAL); + + TableSelector tsOutputFiles = new TableSelector(outputTable, PageFlowUtil.set("rowid"), filter, null); + if (tsOutputFiles.exists()) + { + outputFileMap.put(remoteId, tsOutputFiles.getObject(Integer.class)); + } + else + { + Map toCreate = new CaseInsensitiveHashMap<>(); + toCreate.put("readset", rs.getRowId()); + toCreate.put("analysis_id", localAnalysis); + toCreate.put("description", rd.getValue("description")); + toCreate.put("sra_accession", rd.getValue("sra_accession")); + toCreate.put("library_id", localLibrary); + toCreate.put("name", rd.getValue("name")); + toCreate.put("category", rd.getValue("category")); + + try + { + int remoteJobId = Integer.parseInt(String.valueOf(rd.getValue("runid/JobId"))); + int jobId = getOrCreateJob(remoteJobId, targetWorkbook); + PipelineStatusFile sf = PipelineService.get().getStatusFile(jobId); + + String localJobRoot = getParent(sf.getFilePath()); + String remoteJobRoot = getParent(URI.create(String.valueOf(rd.getValue("runid/JobId/FilePath")).replaceAll(" ", "_")).getPath()); + + URI newFileAlignment = translateURI(String.valueOf(rd.getValue("dataid/DatafileUrl")), remoteJobRoot, localJobRoot); + toCreate.put("dataid", getOrCreateExpData(newFileAlignment, targetWorkbook)); + + //Create run: + if (rd.getValue("runid") != null && rd.getValue("runid/JobId") != null) + { + int runId = createExpRun(Integer.parseInt(String.valueOf(rd.getValue("runid"))), targetWorkbook, String.valueOf(rd.getValue("runid/Name")), jobId); + toCreate.put("runid", runId); + } + else + { + _log.error("output missing runid: " + remoteId); + } + + BatchValidationException bve = new BatchValidationException(); + List> created = outputTable.getUpdateService().insertRows(getJob().getUser(), getPipelineJob().targetContainer, Arrays.asList(toCreate), bve, null, null); + if (bve.hasErrors()) + { + throw new RuntimeException(bve); + } + + outputFileMap.put(remoteId, Integer.parseInt(String.valueOf(created.get(0).get("rowid")))); + } + catch (Exception e) + { + throw new RuntimeException(e); + } + } + }); + } + catch (Exception e) + { + _log.error(e.getMessage(), e); + throw new RuntimeException(e); + } + } + + private void createAnalyses() + { + _log.info("Creating analyses"); + try + { + final TableInfo analysisTable = QueryService.get().getUserSchema(getJob().getUser(), getPipelineJob().targetContainer, "sequenceanalysis").getTable("sequence_analyses"); + + SelectRowsCommand sr = new SelectRowsCommand("sequenceanalysis", "sequence_analyses"); + sr.setColumns(Arrays.asList("rowid", "type", "description", "synopsis", "runid", "readset", "alignmentfile", "reference_library", "library_id", "sra_accession", "alignmentfile/DataFileUrl", "alignmentfile/Name", "reference_library", "reference_library/DataFileUrl", "runid/jobid", "runid/Name", "workbook/workbookId", "runid/JobId", "runid/Name", "runid/JobId/FilePath", "runid/JobId/Description")); + + SelectRowsResponse srr = sr.execute(getConnection(), getPipelineJob().remoteServerFolder); + + srr.getRowset().forEach(rd -> { + int remoteId = Integer.parseInt(String.valueOf(rd.getValue("rowid"))); + if (rd.getValue("readset") == null) + { + _log.warn("analysis lacks readset, skipping: " + remoteId); + return; + } + + int remoteReadset = Integer.parseInt(String.valueOf(rd.getValue("readset"))); + Integer localReadset = readsetMap.get(remoteReadset); + if (localReadset == null) + { + throw new IllegalArgumentException("Unable to find readset for remote id: " + remoteReadset); + } + + Integer localLibrary = null; + if (rd.getValue("library_id") != null) + { + int remoteLibrary = Integer.parseInt(String.valueOf(rd.getValue("library_id"))); + localLibrary = libraryMap.get(remoteLibrary); + if (localLibrary == null) + { + throw new IllegalArgumentException("Unable to find genome for remote id: " + remoteLibrary); + } + } + + Readset rs = SequenceAnalysisService.get().getReadset(localReadset, getJob().getUser()); + Container targetWorkbook = ContainerManager.getForId(rs.getContainer()); + + SimpleFilter filter = new SimpleFilter(FieldKey.fromString("readset"), rs.getRowId()); + filter.addCondition(FieldKey.fromString("runid/JobId/Description"), rd.getValue("runid/JobId/Description")); + filter.addCondition(FieldKey.fromString("container"), targetWorkbook.getId(), CompareType.EQUAL); + + TableSelector tsAnalyses = new TableSelector(analysisTable, PageFlowUtil.set("rowid"), filter, null); + if (tsAnalyses.exists()) + { + analysisMap.put(remoteId, tsAnalyses.getObject(Integer.class)); + } + else + { + Map toCreate = new CaseInsensitiveHashMap<>(); + + toCreate.put("readset", rs.getRowId()); + toCreate.put("synopsis", rd.getValue("synopsis")); + toCreate.put("centerName", rd.getValue("centerName")); + toCreate.put("type", rd.getValue("type")); + toCreate.put("description", rd.getValue("description")); + toCreate.put("sra_accession", rd.getValue("sra_accession")); + if (localLibrary != null) + { + toCreate.put("library_id", localLibrary); + } + + try + { + if (rd.getValue("runid/JobId") == null) + { + _log.info("skipping analysis without runid: " + remoteId); + return; + } + + int remoteJobId = Integer.parseInt(String.valueOf(rd.getValue("runid/JobId"))); + int jobId = getOrCreateJob(remoteJobId, targetWorkbook); + PipelineStatusFile sf = PipelineService.get().getStatusFile(jobId); + + String localJobRoot = getParent(sf.getFilePath()); + String remoteJobRoot = getParent(URI.create(String.valueOf(rd.getValue("runid/JobId/FilePath")).replaceAll(" ", "_")).getPath()); + + URI newFileAlignment = translateURI(String.valueOf(rd.getValue("alignmentfile/DatafileUrl")), remoteJobRoot, localJobRoot); + toCreate.put("alignmentfile", getOrCreateExpData(newFileAlignment, targetWorkbook)); + + if (rd.getValue("reference_library") != null) + { + URI newFile2 = translateURI(String.valueOf(rd.getValue("reference_library/DatafileUrl")), remoteJobRoot, localJobRoot); + toCreate.put("reference_library", getOrCreateExpData(newFile2, targetWorkbook)); + } + + //Create run: + if (rd.getValue("runid") != null && rd.getValue("runid/JobId") != null) + { + int runId = createExpRun(Integer.parseInt(String.valueOf(rd.getValue("runid"))), targetWorkbook, String.valueOf(rd.getValue("runid/Name")), jobId); + toCreate.put("runid", runId); + } + else + { + _log.error("analysis missing runid: " + remoteId); + } + + BatchValidationException bve = new BatchValidationException(); + List> created = analysisTable.getUpdateService().insertRows(getJob().getUser(), getPipelineJob().targetContainer, Arrays.asList(toCreate), bve, null, null); + if (bve.hasErrors()) + { + throw new RuntimeException(bve); + } + + analysisMap.put(remoteId, Integer.parseInt(String.valueOf(created.get(0).get("rowid")))); + } + catch (Exception e) + { + throw new RuntimeException(e); + } + } + }); + } + catch (Exception e) + { + _log.error(e.getMessage(), e); + throw new RuntimeException(e); + } + } + + private void createReaddata() + { + _log.info("Creating read data"); + try + { + final TableInfo readdataTable = QueryService.get().getUserSchema(getJob().getUser(), getPipelineJob().targetContainer, "sequenceanalysis").getTable("readdata"); + + SelectRowsCommand sr = new SelectRowsCommand("sequenceanalysis", "readdata"); + sr.setColumns(Arrays.asList("rowid", "readset", "platformUnit", "centerName", "date", "fileid1", "fileid1/DataFileUrl", "fileid2", "fileid2/DataFileUrl", "fileid1/Name", "description", "sra_accession", "runid", "runid/jobid", "runid/Name", "readset/workbook/workbookId", "runid/JobId", "runid/Name", "runid/JobId/FilePath", "runid/JobId/Description")); + + SelectRowsResponse srr = sr.execute(getConnection(), getPipelineJob().remoteServerFolder); + + srr.getRowset().forEach(rd -> { + int remoteId = Integer.parseInt(String.valueOf(rd.getValue("rowid"))); + int remoteReadset = Integer.parseInt(String.valueOf(rd.getValue("readset"))); + Integer localReadset = readsetMap.get(remoteReadset); + if (localReadset == null) + { + throw new IllegalArgumentException("Unable to find readset for remote id: " + remoteReadset); + } + + Readset rs = SequenceAnalysisService.get().getReadset(localReadset, getJob().getUser()); + Container targetWorkbook = ContainerManager.getForId(rs.getContainer()); + + SimpleFilter rdFilter = new SimpleFilter(FieldKey.fromString("readset"), rs.getRowId()); + rdFilter.addCondition(FieldKey.fromString("runid/JobId/Description"), rd.getValue("runid/JobId/Description")); + rdFilter.addCondition(FieldKey.fromString("fileid1/Name"), rd.getValue("fileid1/Name")); + rdFilter.addCondition(FieldKey.fromString("container"), targetWorkbook.getId(), CompareType.EQUAL); + + if (rd.getValue("platformUnit") != null) + { + rdFilter.addCondition(FieldKey.fromString("platformUnit"), rd.getValue("platformUnit")); + } + + TableSelector tsReaddata = new TableSelector(readdataTable, PageFlowUtil.set("rowid"), rdFilter, null); + if (tsReaddata.exists()) + { + readdataMap.put(remoteId, tsReaddata.getObject(Integer.class)); + } + else + { + Map toCreate = new CaseInsensitiveHashMap<>(); + toCreate.put("readset", rs.getRowId()); + toCreate.put("platformUnit", rd.getValue("platformUnit")); + toCreate.put("centerName", rd.getValue("centerName")); + toCreate.put("date", rd.getValue("date")); + toCreate.put("description", rd.getValue("description")); + toCreate.put("sra_accession", rd.getValue("sra_accession")); + try + { + if (rd.getValue("runid/JobId") != null) + { + int remoteJobId = Integer.parseInt(String.valueOf(rd.getValue("runid/JobId"))); + int jobId = getOrCreateJob(remoteJobId, targetWorkbook); + PipelineStatusFile sf = PipelineService.get().getStatusFile(jobId); + + String localJobRoot = getParent(sf.getFilePath()); + String remoteJobRoot = getParent(URI.create(String.valueOf(rd.getValue("runid/JobId/FilePath")).replaceAll(" ", "_")).getPath()); + + if (rd.getValue("fileid1/DataFileUrl") != null) + { + URI newFile1 = translateURI(String.valueOf(rd.getValue("fileid1/DataFileUrl")), remoteJobRoot, localJobRoot); + toCreate.put("fileid1", getOrCreateExpData(newFile1, targetWorkbook)); + } + + if (rd.getValue("fileid2/DataFileUrl") != null) + { + URI newFile2 = translateURI(String.valueOf(rd.getValue("fileid2/DatafileUrl")), remoteJobRoot, localJobRoot); + toCreate.put("fileid2", getOrCreateExpData(newFile2, targetWorkbook)); + } + } + else + { + _log.error("readddata missing jobid: " + remoteId); + } + + //Create run: + if (rd.getValue("runid") != null && rd.getValue("runid/JobId") != null) + { + int remoteJobId = Integer.parseInt(String.valueOf(rd.getValue("runid/JobId"))); + int jobId = getOrCreateJob(remoteJobId, targetWorkbook); + int runId = createExpRun(Integer.parseInt(String.valueOf(rd.getValue("runid"))), targetWorkbook, String.valueOf(rd.getValue("runid/Name")), jobId); + toCreate.put("runid", runId); + } + else + { + _log.error("readddata missing runid: " + remoteId); + } + + BatchValidationException bve = new BatchValidationException(); + List> created = readdataTable.getUpdateService().insertRows(getJob().getUser(), getPipelineJob().targetContainer, Arrays.asList(toCreate), bve, null, null); + if (bve.hasErrors()) + { + throw new RuntimeException(bve); + } + + readdataMap.put(remoteId, Integer.parseInt(String.valueOf(created.get(0).get("rowid")))); + } + catch (Exception e) + { + throw new RuntimeException(e); + } + } + }); + } + catch (Exception e) + { + _log.error(e.getMessage(), e); + throw new RuntimeException(e); + } + } + + private int getOrCreateExpData(URI file, Container workbook) + { + ExpData ret = ExperimentService.get().getExpDataByURL(new File(file), workbook); + if (ret == null) + { + ret = ExperimentService.get().createData(workbook, new DataType("Data")); + ret.setDataFileURI(file); + ret.save(getJob().getUser()); + } + + return ret.getRowId(); + } + + private void createReadsets() + { + _log.info("Creating readsets"); + try + { + final UserSchema us = QueryService.get().getUserSchema(getJob().getUser(), getPipelineJob().targetContainer, "sequenceanalysis"); + final TableInfo readsetTable = us.getTable("sequence_readsets"); + + SelectRowsCommand sr = new SelectRowsCommand("sequenceanalysis", "sequence_readsets"); + sr.setColumns(Arrays.asList("rowid", "name", "platform", "application", "librarytype", "chemistry", "comments", "status", "subjectid", "subjectdate", "sampletype", "sampleid", "barcode5", "barcode3", "runid", "runid/jobid", "runid/Name", "workbook/workbookId", "runid/JobId", "runid/Name", "runid/JobId/FilePath")); + + SelectRowsResponse srr = sr.execute(getConnection(), getPipelineJob().remoteServerFolder); + + srr.getRowset().forEach(rs -> { + int remoteId = Integer.parseInt(String.valueOf(rs.getValue("rowid"))); + int sourceWorkbook = Integer.parseInt(String.valueOf(rs.getValue("workbook/workbookId"))); + Container targetWorkbook = workbookMap.get(sourceWorkbook); + if (targetWorkbook == null) + { + throw new IllegalArgumentException("Unable to find local workbook for source: " + sourceWorkbook); + } + + SimpleFilter rsFilter = new SimpleFilter(FieldKey.fromString("name"), rs.getValue("name")); + rsFilter.addCondition(FieldKey.fromString("container"), targetWorkbook.getId(), CompareType.EQUAL); + if (rs.getValue("subjectid") != null) + { + rsFilter.addCondition(FieldKey.fromString("subjectid"), rs.getValue("subjectid"), CompareType.EQUAL); + } + + TableSelector tsReadset = new TableSelector(readsetTable, PageFlowUtil.set("rowid"), rsFilter, null); + if (tsReadset.exists()) + { + readsetMap.put(remoteId, tsReadset.getObject(Integer.class)); + } + else + { + Map toCreate = new CaseInsensitiveHashMap<>(); + toCreate.put("name", rs.getValue("name")); + toCreate.put("platform", rs.getValue("platform")); + toCreate.put("application", rs.getValue("application")); + toCreate.put("barcode5", rs.getValue("barcode5")); + toCreate.put("barcode3", rs.getValue("barcode3")); + toCreate.put("subjectid", rs.getValue("subjectid")); + + toCreate.put("sampleid", rs.getValue("sampleid")); + toCreate.put("sampledate", rs.getValue("sampledate")); + toCreate.put("librarytype", rs.getValue("librarytype")); + toCreate.put("sampletype", rs.getValue("sampletype")); + toCreate.put("chemistry", rs.getValue("chemistry")); + toCreate.put("comments", rs.getValue("comments")); + toCreate.put("status", rs.getValue("status")); + + toCreate.put("container", targetWorkbook.getId()); + + try + { + //Create run: + if (rs.getValue("runid") != null && rs.getValue("runid/JobId") != null) + { + int remoteJobId = Integer.parseInt(String.valueOf(rs.getValue("runid/JobId"))); + int jobId = getOrCreateJob(remoteJobId, targetWorkbook); + int runid = createExpRun(Integer.parseInt(String.valueOf(rs.getValue("runid"))), targetWorkbook, String.valueOf(rs.getValue("runid/Name")), jobId); + toCreate.put("runid", runid); + } + else + { + _log.error("readset missing run id: " + remoteId); + } + + BatchValidationException bve = new BatchValidationException(); + List> created = readsetTable.getUpdateService().insertRows(getJob().getUser(), getPipelineJob().targetContainer, Arrays.asList(toCreate), bve, null, null); + if (bve.hasErrors()) + { + throw new RuntimeException(bve); + } + + readsetMap.put(remoteId, Integer.parseInt(String.valueOf(created.get(0).get("rowid")))); + } + catch (Exception e) + { + throw new RuntimeException(e); + } + } + }); + } + catch (Exception e) + { + _log.error(e.getMessage(), e); + throw new RuntimeException(e); + } + } + + private int getOrCreateJob(int remoteJobId, Container targetWorkbook) + { + if (jobIdMap.containsKey(remoteJobId)) + { + return jobIdMap.get(remoteJobId); + } + + TableInfo ti = DbSchema.get("pipeline", DbSchemaType.Module).getTable("StatusFiles"); + + try + { + SelectRowsCommand sr = new SelectRowsCommand("pipeline", "job"); + sr.addFilter(new Filter("rowid", remoteJobId, Filter.Operator.EQUAL)); + sr.setColumns(Arrays.asList("RowId", "Info", "FilePath", "Email", "Description", "DataUrl", "Job", "Provider", "HadError", "ActiveTaskId")); + + SelectRowsResponse srr = sr.execute(getConnection(), getPipelineJob().remoteServerFolder); + + File fr = PipelineService.get().getPipelineRootSetting(targetWorkbook).getRootPath(); + + AtomicInteger ret = new AtomicInteger(); + srr.getRowset().forEach(pj -> { + String filepath = String.valueOf(pj.getValue("FilePath")); + if (!filepath.contains("@files")) + { + //This appears to be an error in PRIMe's data: + if (filepath.contains("illuminaImport")) + { + filepath = filepath.replace("illuminaImport", "@files/illuminaImport"); + } + else if (filepath.contains("sequenceAnalysis")) + { + filepath = filepath.replace("sequenceAnalysis", "@files/sequenceAnalysis"); + } + else + { + _log.error("Unexpected filepath: " + pj.getValue("FilePath")); + } + } + + File remoteDir = new File(URI.create(filepath.replaceAll(" ", "_")).getPath()); + File localDir = new File(fr, filepath.split("@files")[1]); + + //Check for existing row: + TableSelector ts = new TableSelector(ti, PageFlowUtil.set("RowId"), new SimpleFilter(FieldKey.fromString("Job"), pj.getValue("Job")), null); + if (ts.exists()) + { + ret.set(ts.getObject(Integer.class)); + } + else + { + Map toCreate = new CaseInsensitiveHashMap<>(); + toCreate.put("Info", pj.getValue("Info")); + toCreate.put("FilePath", localDir.getPath()); + toCreate.put("Email", pj.getValue("Email")); + toCreate.put("Description", pj.getValue("Description")); + toCreate.put("DataUrl", pj.getValue("DataUrl")); + toCreate.put("Job", pj.getValue("Job")); + toCreate.put("Provider", pj.getValue("Provider")); + toCreate.put("HadError", pj.getValue("HadError")); + toCreate.put("ActiveTaskId", pj.getValue("ActiveTaskId")); + toCreate.put("Container", targetWorkbook.getId()); + + toCreate = Table.insert(getJob().getUser(), ti, toCreate); + + ret.set((int) toCreate.get("RowId")); + } + + if (localDir.exists()) + { + _log.info("Directory exists, will not re-copy: " + localDir.getPath()); + return; + } + + try + { + _log.info(remoteDir.getPath()); + _log.info(localDir.getPath()); + + if (!localDir.getParentFile().exists()) + { + localDir.getParentFile().mkdirs(); + } + + if (remoteDir.exists()) + { + FileUtils.copyDirectory(remoteDir, localDir); + } + else + { + _log.error("source folder not found: " + remoteDir.getPath()); + } + } + catch (Exception e) + { + throw new RuntimeException(e); + } + }); + + jobIdMap.put(remoteJobId, ret.get()); + + return ret.get(); + } + catch (Exception e) + { + _log.error(e.getMessage(), e); + throw new RuntimeException(e); + } + } + + private int createExpRun(int remoteId, Container c, String name, int localJobId) throws Exception + { + if (runIdMap.containsKey(remoteId)) + { + return runIdMap.get(remoteId); + } + else + { + ExpRun ret = ExperimentService.get().createRunForProvenanceRecording(c, getJob().getUser(), new RecordedActionSet(), name, localJobId); + runIdMap.put(remoteId, ret.getRowId()); + + return ret.getRowId(); + } + } + + private void createWorkbooks() + { + _log.info("Creating workbooks"); + try + { + TableInfo containers = QueryService.get().getUserSchema(getJob().getUser(), getPipelineJob().targetContainer, "core").getTable("containers"); + + SelectRowsCommand sr = new SelectRowsCommand("core", "workbooks"); + sr.setColumns(Arrays.asList("Name", "Title", "Description")); + SelectRowsResponse srr = sr.execute(getConnection(), getPipelineJob().remoteServerFolder); + + srr.getRowset().forEach(wb -> { + String localTitle = (String) wb.getValue("Title"); + + TableSelector ts = new TableSelector(containers, PageFlowUtil.set("RowId"), new SimpleFilter(FieldKey.fromString("Title"), localTitle), null); + if (ts.exists()) + { + Container workbook = ContainerManager.getForRowId(ts.getObject(Integer.class)); + workbookMap.put(Integer.parseInt(String.valueOf(wb.getValue("Name"))), workbook); + } + else + { + String description = String.valueOf(wb.getValue("Description")); + if (description != null) + { + description = description + ". "; + } + else + { + description = ""; + } + + description = description + "Originally PRIMe workbook: " + wb.getValue("Name"); + + Container workbook = ContainerManager.createContainer(getPipelineJob().targetContainer, null, localTitle, description, WorkbookContainerType.NAME, getJob().getUser()); + workbookMap.put(Integer.parseInt(String.valueOf(wb.getValue("Name"))), workbook); + } + }); + } + catch (CommandException | IOException e) + { + throw new RuntimeException(e); + } + } + + private URI translateURI(String databaseURI, String remoteFolderRoot, String localFolderRoot) + { + databaseURI = databaseURI.replace("\\", "/"); + remoteFolderRoot = remoteFolderRoot.replace("\\", "/").split("@files")[0]; + localFolderRoot = localFolderRoot.replace("\\", "/").split("@files")[0]; + if (localFolderRoot.startsWith("C:")) + { + localFolderRoot = localFolderRoot.replaceAll("^C:", ""); + } + + databaseURI = databaseURI.replace(remoteFolderRoot, localFolderRoot); + + return URI.create(databaseURI); + } + } +} diff --git a/primeseq/webapp/WEB-INF/primeseqContext.xml b/primeseq/webapp/WEB-INF/primeseqContext.xml index 411bb99a8..b91d3dfb3 100644 --- a/primeseq/webapp/WEB-INF/primeseqContext.xml +++ b/primeseq/webapp/WEB-INF/primeseqContext.xml @@ -3,6 +3,28 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-2.0.xsd"> + + + + + + + + + + + + + + + org.labkey.primeseq.pipeline.MhcMigrationPipelineJob.Task + + + + + + + From 15819417e8ebdceaac6202c1199e686dcda7dbe0 Mon Sep 17 00:00:00 2001 From: bbimber Date: Fri, 29 Jan 2021 15:26:18 -0800 Subject: [PATCH 61/98] Store barcodes as list to enforce uniqueness --- .../labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java index a54aea590..d2fc05b73 100644 --- a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java +++ b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java @@ -200,7 +200,7 @@ private void processVloupeFile(JobContext ctx, File perCellTsv, Readset rs, Reco { AlignmentOutputImpl output = new AlignmentOutputImpl(); - List htosPerReadset = CellHashingService.get().getHtosForParentReadset(rs.getReadsetId(), ctx.getSourceDirectory(), ctx.getSequenceSupport()); + Set htosPerReadset = CellHashingService.get().getHtosForParentReadset(rs.getReadsetId(), ctx.getSourceDirectory(), ctx.getSequenceSupport()); if (htosPerReadset.size() > 1) { ctx.getLogger().info("Total HTOs for readset: " + htosPerReadset.size()); From ae386b5542a5fbdacd2600d53902263867fde35e Mon Sep 17 00:00:00 2001 From: bbimber Date: Sat, 30 Jan 2021 16:29:50 -0800 Subject: [PATCH 62/98] Include default cite-seq-count params for CiteSeq handler --- .../labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java index d2fc05b73..4e121453e 100644 --- a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java +++ b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java @@ -112,7 +112,7 @@ public class Processor implements SequenceOutputHandler.SequenceOutputProcessor public void init(JobContext ctx, List inputFiles, List actions, List outputsToCreate) throws UnsupportedOperationException, PipelineJobException { //NOTE: this is the pathway to import assay data, whether hashing is used or not - CellHashingService.get().prepareHashingAndCiteSeqFilesIfNeeded(ctx.getOutputDir(), ctx.getJob(), ctx.getSequenceSupport(), "tcrReadsetId", ctx.getParams().optBoolean("excludeFailedcDNA", true), false, false); + CellHashingService.get().prepareHashingAndCiteSeqFilesIfNeeded(ctx.getOutputDir(), ctx.getJob(), ctx.getSequenceSupport(), "tcrReadsetId", ctx.getParams().optBoolean("excludeFailedcDNA", false), false, false); if (ctx.getParams().optBoolean(USE_GEX_BARCODES, false)) { From 67f89f6877c153de4084a0bdee11b30edc9a9cfc Mon Sep 17 00:00:00 2001 From: bbimber Date: Mon, 1 Feb 2021 13:12:32 -0800 Subject: [PATCH 63/98] Add default trigger scripts --- .../queries/study/animalGroupMembership.js | 14 +++ mcc/resources/queries/study/deaths.js | 97 +++++++++++++++++++ mcc/resources/queries/study/demographics.js | 23 +++++ mcc/resources/queries/study/encounters.js | 13 +++ mcc/resources/queries/study/flags.js | 50 ++++++++++ mcc/resources/queries/study/labworkResults.js | 14 +++ mcc/resources/queries/study/parentage.js | 13 +++ mcc/resources/queries/study/samples.js | 7 ++ mcc/resources/queries/study/weight.js | 89 +++++++++++++++++ 9 files changed, 320 insertions(+) create mode 100644 mcc/resources/queries/study/animalGroupMembership.js create mode 100644 mcc/resources/queries/study/deaths.js create mode 100644 mcc/resources/queries/study/demographics.js create mode 100644 mcc/resources/queries/study/encounters.js create mode 100644 mcc/resources/queries/study/flags.js create mode 100644 mcc/resources/queries/study/labworkResults.js create mode 100644 mcc/resources/queries/study/parentage.js create mode 100644 mcc/resources/queries/study/samples.js create mode 100644 mcc/resources/queries/study/weight.js diff --git a/mcc/resources/queries/study/animalGroupMembership.js b/mcc/resources/queries/study/animalGroupMembership.js new file mode 100644 index 000000000..c8510ccb2 --- /dev/null +++ b/mcc/resources/queries/study/animalGroupMembership.js @@ -0,0 +1,14 @@ +/* + * Copyright (c) 2011-2014 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0 + */ + +require("ehr/triggers").initScript(this); + +function onInit(event, helper){ + helper.setScriptOptions({ + allowFutureDates: true, + removeTimeFromDate: true + }); +} \ No newline at end of file diff --git a/mcc/resources/queries/study/deaths.js b/mcc/resources/queries/study/deaths.js new file mode 100644 index 000000000..9dda9861c --- /dev/null +++ b/mcc/resources/queries/study/deaths.js @@ -0,0 +1,97 @@ +/* + * Copyright (c) 2018-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0 + */ + +require("ehr/triggers").initScript(this); +EHR.Server.Utils = require("ehr/utils").EHR.Server.Utils; + +var demographicsUpdates = []; +var validIds = []; + +function onInit(event, helper){ + helper.setScriptOptions({ + requiresStatusRecalc: true + }); + + helper.decodeExtraContextProperty('deathsInTransaction'); + + // Cache valid Ids for check on each row + LABKEY.Query.selectRows({ + requiredVersion: 9.1, + schemaName: 'study', + queryName: 'demographics', + columns: ['Id'], + scope: this, + success: function (results) { + if (!results || !results.rows || results.rows.length < 1) + return; + + for(var i=0; i 0) { + console.log('updating demographics death date for ' + demographicsUpdates.length + " animals"); + helper.getJavaHelper().updateDemographicsRecord(demographicsUpdates); + } + + var deaths = helper.getDeaths(); + if (deaths){ + var ids = []; + for (var id in deaths){ + ids.push(id); + } + + if (!helper.isETL()) { + console.log('sending death notification'); + helper.getJavaHelper().sendDeathNotification(ids); + } + } +} \ No newline at end of file diff --git a/mcc/resources/queries/study/demographics.js b/mcc/resources/queries/study/demographics.js new file mode 100644 index 000000000..992138d43 --- /dev/null +++ b/mcc/resources/queries/study/demographics.js @@ -0,0 +1,23 @@ +/* + * Copyright (c) 2010-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0 + */ + +require("ehr/triggers").initScript(this); + +function onInit(event, helper){ + helper.setScriptOptions({ + allowAnyId: true, + requiresStatusRecalc: false, + allowDatesInDistantPast: true + }); +} + +function onUpsert(helper, scriptErrors, row, oldRow){ + //NOTE: this should be getting set by the birth, death, arrival & departure tables + //ALSO: it should be rare to insert directly into this table. usually this record will be created by inserting into either birth or arrival + if (!row.calculated_status && !helper.isETL()){ + row.calculated_status = helper.getJavaHelper().getCalculatedStatusValue(row.Id); + } +} \ No newline at end of file diff --git a/mcc/resources/queries/study/encounters.js b/mcc/resources/queries/study/encounters.js new file mode 100644 index 000000000..fbb17ad72 --- /dev/null +++ b/mcc/resources/queries/study/encounters.js @@ -0,0 +1,13 @@ +/* + * Copyright (c) 2018-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0 + */ + +require("ehr/triggers").initScript(this); + +function onUpsert(helper, scriptErrors, row, oldRow){ + if (!helper.isETL() && row.date && !row.requestdate){ + row.requestdate = row.date; + } +} \ No newline at end of file diff --git a/mcc/resources/queries/study/flags.js b/mcc/resources/queries/study/flags.js new file mode 100644 index 000000000..454371e4c --- /dev/null +++ b/mcc/resources/queries/study/flags.js @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2012-2018 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0 + */ + +require("ehr/triggers").initScript(this); + +function onInit(event, helper){ + helper.setScriptOptions({ + allowFutureDates: true, + removeTimeFromDate: true, + removeTimeFromEndDate: true + }); +} + +function onUpsert(helper, scriptErrors, row, oldRow){ + //if the animal is not at the center, automatically set the enddate + if (!helper.isETL() && row.Id && !row.enddate){ + EHR.Server.Utils.findDemographics({ + participant: row.Id, + helper: helper, + scope: this, + callback: function(data){ + if (!data) + return; + + if (data && data.calculated_status && data.calculated_status != 'Alive'){ + row.enddate = data.death || data.departure; + } + } + }); + + } + + if (!helper.isETL() && row.Id && row.date && row.flag){ + var active = helper.getJavaHelper().getOverlappingFlags(row.Id, row.flag, row.objectid || null, row.date); + if (active > 0){ + EHR.Server.Utils.addError(scriptErrors, 'flag', 'There are already ' + active + ' active flag(s) of the same type spanning this date.', 'INFO'); + } + } +} + +function onAfterInsert(helper, errors, row){ + //if this category enforces only a single active flag at once, enforce it + //note: if this flag has a future date, preemptively set enddate on flags, since isActive should handle this + if (!helper.isETL() && row.Id && row.flag && !row.enddate && row.date){ + helper.getJavaHelper().ensureSingleFlagCategoryActive(row.Id, row.flag, row.objectId, row.date); + } +} \ No newline at end of file diff --git a/mcc/resources/queries/study/labworkResults.js b/mcc/resources/queries/study/labworkResults.js new file mode 100644 index 000000000..2004fb6cf --- /dev/null +++ b/mcc/resources/queries/study/labworkResults.js @@ -0,0 +1,14 @@ +/* + * Copyright (c) 2012-2018 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0 + */ + +require("ehr/triggers").initScript(this); + +function onInit(event, helper){ + helper.setScriptOptions({ + removeTimeFromDate: false, + allowDatesInDistantPast: true + }); +} \ No newline at end of file diff --git a/mcc/resources/queries/study/parentage.js b/mcc/resources/queries/study/parentage.js new file mode 100644 index 000000000..0cea7a1c5 --- /dev/null +++ b/mcc/resources/queries/study/parentage.js @@ -0,0 +1,13 @@ +/* + * Copyright (c) 2013 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0 + */ + +require("ehr/triggers").initScript(this); + +function onInit(event, helper){ + helper.setScriptOptions({ + lookupValidationFields: ['relationship', 'method'] + }); +} diff --git a/mcc/resources/queries/study/samples.js b/mcc/resources/queries/study/samples.js new file mode 100644 index 000000000..64a117a3e --- /dev/null +++ b/mcc/resources/queries/study/samples.js @@ -0,0 +1,7 @@ +/* + * Copyright (c) 2011-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0 + */ + +require("ehr/triggers").initScript(this); \ No newline at end of file diff --git a/mcc/resources/queries/study/weight.js b/mcc/resources/queries/study/weight.js new file mode 100644 index 000000000..a515311e1 --- /dev/null +++ b/mcc/resources/queries/study/weight.js @@ -0,0 +1,89 @@ +/* + * Copyright (c) 2010-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0 + */ + +require("ehr/triggers").initScript(this); + +function onInit(event, helper){ + helper.setScriptOptions({ + allowAnyId: true, + allowDeadIds: true, + skipIdFormatCheck: true + }); + + helper.registerRowProcessor(function(helper, row){ + if (!row) + return; + + if (!row.Id || !row.weight){ + return; + } + + var weightInTransaction = helper.getProperty('weightInTransaction'); + weightInTransaction = weightInTransaction || {}; + weightInTransaction[row.Id] = weightInTransaction[row.Id] || []; + + var shouldAdd = true; + if (row.objectid){ + LABKEY.ExtAdapter.each(weightInTransaction[row.Id], function(r){ + if (r.objectid === row.objectid){ + if (r.weight !== row.weight){ + r.weight = row.weight; + } + else { + shouldAdd = false; + return false; + } + } + }, this); + } + + if (shouldAdd){ + weightInTransaction[row.Id].push({ + objectid: row.objectid, + date: row.date, + qcstate: row.QCState, + weight: row.weight + }); + } + + helper.setProperty('weightInTransaction', weightInTransaction); + }); +} + +function onUpsert(helper, scriptErrors, row, oldRow){ + if (!row.weight){ + EHR.Server.Utils.addError(scriptErrors, 'weight', 'This field is required', 'WARN'); + } + + // warn if more than 10% different from last weight + // the highest error this can produce is WARN. therefore skip this check if we would ignore it anyway in order to save the overhead. + // this would normally occur when finalizing a form + if (!helper.isETL() && row.Id && row.weight && EHR.Server.Utils.shouldIncludeError('WARN', helper.getErrorThreshold(), helper)){ + EHR.Server.Utils.findDemographics({ + participant: row.Id, + helper: helper, + scope: this, + callback: function(data){ + if (!data) + return; + + if (data.mostRecentWeight && (row.weight <= data.mostRecentWeight * 0.9)){ + EHR.Server.Utils.addError(scriptErrors, 'weight', 'Weight drop of >10%. Last weight ' + data.mostRecentWeight + ' kg', 'INFO'); + } + else if (data.mostRecentWeight && (row.weight >= data.mostRecentWeight / 0.9)){ + EHR.Server.Utils.addError(scriptErrors, 'weight', 'Weight gain of >10%. Last weight ' + data.mostRecentWeight + ' kg', 'INFO'); + } + + if (data && data.species){ + var msg = helper.getJavaHelper().verifyWeightRange(row.id, row.weight, data.species); + if (msg != null){ + EHR.Server.Utils.addError(scriptErrors, 'weight', msg, 'WARN'); + } + } + } + }); + } +} \ No newline at end of file From 5a559830c7dc11cad60cd7b6c138221936e413fb Mon Sep 17 00:00:00 2001 From: bbimber Date: Thu, 4 Feb 2021 08:54:38 -0800 Subject: [PATCH 64/98] Dont enforce calling methods for cite-seq only --- .../labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java index 4e121453e..85ac41dea 100644 --- a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java +++ b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java @@ -65,7 +65,7 @@ private static List getDefaultParams() }}, false) )); - ret.addAll(CellHashingService.get().getDefaultHashingParams(true)); + ret.addAll(CellHashingService.get().getDefaultHashingParams(true, CellHashingService.BARCODE_TYPE.hashing)); return ret; } From c17cbfc6acf4341668b34b60de7a09b7676013aa Mon Sep 17 00:00:00 2001 From: bbimber Date: Thu, 4 Feb 2021 12:40:43 -0800 Subject: [PATCH 65/98] Update reference study --- .../datasets/datasets_manifest.xml | 10 +- .../datasets/datasets_metadata.xml | 482 +----------------- 2 files changed, 17 insertions(+), 475 deletions(-) diff --git a/mcc/resources/referenceStudy/datasets/datasets_manifest.xml b/mcc/resources/referenceStudy/datasets/datasets_manifest.xml index 3600f8261..1d43fa147 100644 --- a/mcc/resources/referenceStudy/datasets/datasets_manifest.xml +++ b/mcc/resources/referenceStudy/datasets/datasets_manifest.xml @@ -18,7 +18,10 @@ - + + + + @@ -30,7 +33,7 @@ - + @@ -45,9 +48,6 @@ - - - diff --git a/mcc/resources/referenceStudy/datasets/datasets_metadata.xml b/mcc/resources/referenceStudy/datasets/datasets_metadata.xml index 8f19b36f9..6a154bcc1 100644 --- a/mcc/resources/referenceStudy/datasets/datasets_metadata.xml +++ b/mcc/resources/referenceStudy/datasets/datasets_metadata.xml @@ -135,33 +135,12 @@ varchar - - varchar - - - varchar - varchar - - varchar - - - varchar - - - timestamp - - - integer - varchar - - timestamp - varchar urn:ehr.labkey.org/#ObjectId @@ -207,36 +186,6 @@ urn:ehr.labkey.org/#ObjectId true - - varchar - urn:ehr.labkey.org/#VetReview - - - timestamp - urn:ehr.labkey.org/#VetReviewDate - - - varchar - - - varchar - - - varchar - - - varchar - - - timestamp - urn:ehr.labkey.org/#DateRequested - - - varchar - - - varchar - integer @@ -264,6 +213,9 @@ integer urn:ehr.labkey.org/#Project + + varchar + varchar @@ -272,35 +224,6 @@ urn:ehr.labkey.org/#ObjectId true - - varchar - urn:ehr.labkey.org/#VetReview - - - timestamp - urn:ehr.labkey.org/#VetReviewDate - - - varchar - - - varchar - - - varchar - - - varchar - - - varchar - - - varchar - - - varchar - timestamp urn:ehr.labkey.org/#EndDate @@ -308,16 +231,10 @@ timestamp - - varchar - - - varchar - Clinical Remarks - +
varchar @@ -389,12 +306,6 @@ varchar - - timestamp - - - varchar - Medication Administration
@@ -503,7 +414,7 @@ Weight - +
varchar @@ -524,33 +435,15 @@ varchar - - varchar - integer urn:ehr.labkey.org/#Project - - varchar - entityid urn:ehr.labkey.org/#ObjectId true - - varchar - - - varchar - - - varchar - - - varchar - varchar @@ -566,16 +459,13 @@ varchar - - varchar - timestamp - Clinpath Runs + Labwork
- +
varchar @@ -630,132 +520,9 @@ varchar - Hematology Results -
- - - - varchar - http://cpas.labkey.com/Study#ParticipantId - - ptid - - - - timestamp - http://cpas.labkey.com/Study#VisitDate - http://cpas.labkey.com/Study#VisitDate - - - varchar - - - entityid - urn:ehr.labkey.org/#ObjectId - true - - - varchar - - - integer - urn:ehr.labkey.org/#Project - - - timestamp - urn:ehr.labkey.org/#EndDate - - - double - - - varchar - - - varchar - - - varchar - - - varchar - - - Parasitology Results -
- - - - varchar - http://cpas.labkey.com/Study#ParticipantId - - ptid - - - - timestamp - http://cpas.labkey.com/Study#VisitDate - http://cpas.labkey.com/Study#VisitDate - - - varchar - - - double - - - varchar - - - double - - - double - - - double - - - varchar - - - varchar - - - entityid - urn:ehr.labkey.org/#ObjectId - true - - - integer - urn:ehr.labkey.org/#Project - - - timestamp - urn:ehr.labkey.org/#EndDate - - - varchar - - - double - - - varchar - - - double - - - double - - - varchar - - - Urinalysis Results + Lab Results
- +
varchar @@ -791,45 +558,6 @@ boolean - - varchar - - - varchar - - - varchar - - - varchar - - - varchar - - - varchar - - - varchar - - - integer - - - integer - - - varchar - - - varchar - - - varchar - - - timestamp - Arrival
@@ -860,34 +588,10 @@ urn:ehr.labkey.org/#ObjectId true - - timestamp - - - integer - - - integer - - - integer - - - varchar - - - timestamp - - - timestamp - - - varchar - Assignment - +
varchar @@ -913,9 +617,6 @@ urn:ehr.labkey.org/#ObjectId true - - varchar - Animal Group Members
@@ -936,9 +637,6 @@ varchar - - varchar - varchar urn:ehr.labkey.org/#ObjectId @@ -951,21 +649,6 @@ timestamp urn:ehr.labkey.org/#EndDate - - varchar - - - varchar - - - varchar - - - varchar - - - boolean - Deaths @@ -1013,151 +696,10 @@ varchar - - Demographics - - - - - varchar - http://cpas.labkey.com/Study#ParticipantId - - ptid - - - - timestamp - http://cpas.labkey.com/Study#VisitDate - http://cpas.labkey.com/Study#VisitDate - - - varchar - - - varchar - - - entityid - urn:ehr.labkey.org/#ObjectId - true - - - integer - urn:ehr.labkey.org/#Project - - - timestamp - urn:ehr.labkey.org/#EndDate - - - Departure -
- - - - varchar - http://cpas.labkey.com/Study#ParticipantId - - ptid - - - - timestamp - http://cpas.labkey.com/Study#VisitDate - http://cpas.labkey.com/Study#VisitDate - - - timestamp - urn:ehr.labkey.org/#EndDate - - - varchar - - + varchar - - integer - - - integer - - - integer - - - entityid - urn:ehr.labkey.org/#ObjectId - true - - - varchar - - - integer - urn:ehr.labkey.org/#Project - - - Housing -
- - - - varchar - http://cpas.labkey.com/Study#ParticipantId - - ptid - - - - timestamp - http://cpas.labkey.com/Study#VisitDate - http://cpas.labkey.com/Study#VisitDate - - - varchar - - - varchar - - - entityid - urn:ehr.labkey.org/#ObjectId - true - - - integer - urn:ehr.labkey.org/#Project - - - timestamp - urn:ehr.labkey.org/#EndDate - - - - - - - - - varchar - - - varchar - - - varchar - - - - - - double - - - boolean - - Tissue Samples + Demographics
From b18d2a1e9614a5518ea5719f33a5709218dba8e2 Mon Sep 17 00:00:00 2001 From: bbimber Date: Thu, 4 Feb 2021 14:28:22 -0800 Subject: [PATCH 66/98] Add stubs of various files for MCC --- mGAP/resources/views/contact.html | 2 +- mcc/resources/module.xml | 21 + .../queries/wnprcSource/demographics.sql | 6 + .../postgresql/mcc-20.000-20.001.sql | 18 + .../dbscripts/sqlserver/mcc-20.000-20.001.sql | 18 + mcc/resources/schemas/mcc.xml | 90 +++- mcc/resources/views/about.html | 3 + mcc/resources/views/about.view.xml | 8 + mcc/resources/views/about.webpart.xml | 6 + mcc/resources/views/contact.html | 75 +++ mcc/resources/views/contact.view.xml | 8 + mcc/resources/views/helpMenu.html | 1 + mcc/resources/views/helpMenu.view.xml | 8 + mcc/resources/views/helpMenu.webpart.xml | 6 + mcc/resources/views/login.html | 40 ++ mcc/resources/views/login.view.xml | 9 + mcc/resources/views/overview.html | 1 + mcc/resources/views/overview.view.xml | 10 + mcc/resources/views/overview.webpart.xml | 6 + mcc/resources/views/requestLogin.html | 117 +++++ mcc/resources/views/requestLogin.view.xml | 8 + mcc/resources/web/mcc/Security.js | 49 ++ mcc/src/org/labkey/mcc/MccController.java | 494 +++++++++++++++++- mcc/src/org/labkey/mcc/MccManager.java | 70 +++ mcc/src/org/labkey/mcc/MccModule.java | 2 +- mcc/src/org/labkey/mcc/MccSchema.java | 2 + .../mcc/query/UserRequestCustomizer.java | 44 ++ 27 files changed, 1115 insertions(+), 7 deletions(-) create mode 100644 mcc/resources/module.xml create mode 100644 mcc/resources/queries/wnprcSource/demographics.sql create mode 100644 mcc/resources/schemas/dbscripts/postgresql/mcc-20.000-20.001.sql create mode 100644 mcc/resources/schemas/dbscripts/sqlserver/mcc-20.000-20.001.sql create mode 100644 mcc/resources/views/about.html create mode 100644 mcc/resources/views/about.view.xml create mode 100644 mcc/resources/views/about.webpart.xml create mode 100644 mcc/resources/views/contact.html create mode 100644 mcc/resources/views/contact.view.xml create mode 100644 mcc/resources/views/helpMenu.html create mode 100644 mcc/resources/views/helpMenu.view.xml create mode 100644 mcc/resources/views/helpMenu.webpart.xml create mode 100644 mcc/resources/views/login.html create mode 100644 mcc/resources/views/login.view.xml create mode 100644 mcc/resources/views/overview.html create mode 100644 mcc/resources/views/overview.view.xml create mode 100644 mcc/resources/views/overview.webpart.xml create mode 100644 mcc/resources/views/requestLogin.html create mode 100644 mcc/resources/views/requestLogin.view.xml create mode 100644 mcc/resources/web/mcc/Security.js create mode 100644 mcc/src/org/labkey/mcc/query/UserRequestCustomizer.java diff --git a/mGAP/resources/views/contact.html b/mGAP/resources/views/contact.html index 1b110d8cc..e1f2f6b53 100644 --- a/mGAP/resources/views/contact.html +++ b/mGAP/resources/views/contact.html @@ -49,7 +49,7 @@ success: function(response){ console.log(response); - Ext4.Msg.alert('Success', 'An account has been requested. You should receive a reply shortly.', function(){ + Ext4.Msg.alert('Success', 'Your request has been sent. You should receive a reply shortly.', function(){ window.location = LABKEY.ActionURL.getContextPath() + '/'; }); }, diff --git a/mcc/resources/module.xml b/mcc/resources/module.xml new file mode 100644 index 000000000..b07899cb8 --- /dev/null +++ b/mcc/resources/module.xml @@ -0,0 +1,21 @@ + + + + false + This is the path to the container holding the primary MCC Study. Use of slashes is very important - it should be in the format '/myProject/mcc' + + ADMIN + + + + false + This is a comma separated list of LabKey user names of users that should be notified by email when requests are submitted through MCC. + + ADMIN + + + + + + + diff --git a/mcc/resources/queries/wnprcSource/demographics.sql b/mcc/resources/queries/wnprcSource/demographics.sql new file mode 100644 index 000000000..c63412b14 --- /dev/null +++ b/mcc/resources/queries/wnprcSource/demographics.sql @@ -0,0 +1,6 @@ +SELECT + +Id, date, gender, geographic_origin, birth, death, species, objectid + +FROM "/WNPRC/EHR/".study.demographics +WHERE species = 'Marmoset'; \ No newline at end of file diff --git a/mcc/resources/schemas/dbscripts/postgresql/mcc-20.000-20.001.sql b/mcc/resources/schemas/dbscripts/postgresql/mcc-20.000-20.001.sql new file mode 100644 index 000000000..567bfef27 --- /dev/null +++ b/mcc/resources/schemas/dbscripts/postgresql/mcc-20.000-20.001.sql @@ -0,0 +1,18 @@ +CREATE TABLE mcc.userRequests ( + rowid serial, + email varchar(1000), + firstName varchar(1000), + lastName varchar(1000), + title varchar(1000), + institution varchar(1000), + reason varchar(4000), + userid userid, + + container entityid, + created timestamp, + createdby userid, + modified timestamp, + modifiedby userid, + + CONSTRAINT PK_userRequests PRIMARY KEY (rowid) +); \ No newline at end of file diff --git a/mcc/resources/schemas/dbscripts/sqlserver/mcc-20.000-20.001.sql b/mcc/resources/schemas/dbscripts/sqlserver/mcc-20.000-20.001.sql new file mode 100644 index 000000000..de27af5ef --- /dev/null +++ b/mcc/resources/schemas/dbscripts/sqlserver/mcc-20.000-20.001.sql @@ -0,0 +1,18 @@ +CREATE TABLE mcc.userRequests ( + rowid int identity(1,1), + email varchar(1000), + firstName varchar(1000), + lastName varchar(1000), + title varchar(1000), + institution varchar(1000), + reason varchar(4000), + userid userid, + + container entityid, + created datetime, + createdby userid, + modified datetime, + modifiedby userid, + + CONSTRAINT PK_userRequests PRIMARY KEY (rowid) +); \ No newline at end of file diff --git a/mcc/resources/schemas/mcc.xml b/mcc/resources/schemas/mcc.xml index 2bba6c71d..e69524513 100644 --- a/mcc/resources/schemas/mcc.xml +++ b/mcc/resources/schemas/mcc.xml @@ -17,4 +17,92 @@ --> \ No newline at end of file + xmlns="http://labkey.org/data/xml" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"> + + + + + + + + rowid + Requests For Logins + DETAILED + + + true + false + false + false + false + Request Id + + + Email + false + + + First Name + false + + + Last Name + false + + + Title + false + + + Institution + false + + + Reason For Request + false + + + false + + core + Users + UserId + + + + true + + + true + + + false + false + false + true + true + + + true + + + false + false + false + true + true + + + + ldk.context + /mcc/Security.js + + MCC.Security.approveUserRequests(dataRegionName); + + + +
+ + \ No newline at end of file diff --git a/mcc/resources/views/about.html b/mcc/resources/views/about.html new file mode 100644 index 000000000..436f532b2 --- /dev/null +++ b/mcc/resources/views/about.html @@ -0,0 +1,3 @@ +MCC is supported by NIH U24 xxxxxxx. +

+Please remember to cite this funding source in all publications that make use of MCC data. \ No newline at end of file diff --git a/mcc/resources/views/about.view.xml b/mcc/resources/views/about.view.xml new file mode 100644 index 000000000..48ca9ced8 --- /dev/null +++ b/mcc/resources/views/about.view.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/mcc/resources/views/about.webpart.xml b/mcc/resources/views/about.webpart.xml new file mode 100644 index 000000000..f40c189c7 --- /dev/null +++ b/mcc/resources/views/about.webpart.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/mcc/resources/views/contact.html b/mcc/resources/views/contact.html new file mode 100644 index 000000000..714f95170 --- /dev/null +++ b/mcc/resources/views/contact.html @@ -0,0 +1,75 @@ + \ No newline at end of file diff --git a/mcc/resources/views/contact.view.xml b/mcc/resources/views/contact.view.xml new file mode 100644 index 000000000..0fc3bfa31 --- /dev/null +++ b/mcc/resources/views/contact.view.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/mcc/resources/views/helpMenu.html b/mcc/resources/views/helpMenu.html new file mode 100644 index 000000000..932f848cc --- /dev/null +++ b/mcc/resources/views/helpMenu.html @@ -0,0 +1 @@ +We'd love to hear your feedback! Click here to send a help or feature request, or email mcc@ohsu.edu for any questions. \ No newline at end of file diff --git a/mcc/resources/views/helpMenu.view.xml b/mcc/resources/views/helpMenu.view.xml new file mode 100644 index 000000000..3960a69c1 --- /dev/null +++ b/mcc/resources/views/helpMenu.view.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/mcc/resources/views/helpMenu.webpart.xml b/mcc/resources/views/helpMenu.webpart.xml new file mode 100644 index 000000000..3f4624d6a --- /dev/null +++ b/mcc/resources/views/helpMenu.webpart.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/mcc/resources/views/login.html b/mcc/resources/views/login.html new file mode 100644 index 000000000..71bcbf7d3 --- /dev/null +++ b/mcc/resources/views/login.html @@ -0,0 +1,40 @@ + +
+
Sign In
+
+
+ + + + + Remember my email address + + +
+ + + + or + Request an account + +
+ + + +
+
+ \ No newline at end of file diff --git a/mcc/resources/views/login.view.xml b/mcc/resources/views/login.view.xml new file mode 100644 index 000000000..7ce47abe2 --- /dev/null +++ b/mcc/resources/views/login.view.xml @@ -0,0 +1,9 @@ + + + + + + + + + \ No newline at end of file diff --git a/mcc/resources/views/overview.html b/mcc/resources/views/overview.html new file mode 100644 index 000000000..f03b886bc --- /dev/null +++ b/mcc/resources/views/overview.html @@ -0,0 +1 @@ +This will hold the overview information for MCC. \ No newline at end of file diff --git a/mcc/resources/views/overview.view.xml b/mcc/resources/views/overview.view.xml new file mode 100644 index 000000000..f9e80a68a --- /dev/null +++ b/mcc/resources/views/overview.view.xml @@ -0,0 +1,10 @@ + + + + + + + + + + \ No newline at end of file diff --git a/mcc/resources/views/overview.webpart.xml b/mcc/resources/views/overview.webpart.xml new file mode 100644 index 000000000..c81f56bb6 --- /dev/null +++ b/mcc/resources/views/overview.webpart.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/mcc/resources/views/requestLogin.html b/mcc/resources/views/requestLogin.html new file mode 100644 index 000000000..6c3b1fb7f --- /dev/null +++ b/mcc/resources/views/requestLogin.html @@ -0,0 +1,117 @@ + \ No newline at end of file diff --git a/mcc/resources/views/requestLogin.view.xml b/mcc/resources/views/requestLogin.view.xml new file mode 100644 index 000000000..13c485f9f --- /dev/null +++ b/mcc/resources/views/requestLogin.view.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/mcc/resources/web/mcc/Security.js b/mcc/resources/web/mcc/Security.js new file mode 100644 index 000000000..5ab385833 --- /dev/null +++ b/mcc/resources/web/mcc/Security.js @@ -0,0 +1,49 @@ +Ext4.namespace('MCC.Security'); + +MCC.Security = new function(){ + return { + approveUserRequests: function(dataRegionName){ + var dr = LABKEY.DataRegions[dataRegionName]; + if (!dr){ + alert('Unable to find DataRegion with name: ' + dataRegionName); + return; + } + + var rowIds = dr.getChecked(); + if (!rowIds.length){ + alert('Must select one or more rows'); + return; + } + + Ext4.Msg.confirm('Approve Requests', 'You are able to approve ' + rowIds.length + ' user requests. Continue?', function(val){ + if (val === 'yes'){ + Ext4.Msg.wait('Loading...'); + LABKEY.Ajax.request({ + method: 'POST', + url: LABKEY.ActionURL.buildURL('mcc', 'approveUserRequests'), + params: { + requestIds: rowIds + }, + success: function(){ + Ext4.Msg.hide(); + Ext4.Msg.alert('Success', 'Requests approved!', function(){ + //note: drop view, so we see the newly added user(s) + LABKEY.DataRegions[dataRegionName].changeView(null); + }); + }, + failure: LDK.Utils.getErrorCallback({ + showAlertOnError: false, + scope: this, + callback: function(responseObj){ + if (responseObj.errorMsg){ + Ext4.Msg.alert('Error', responseObj.errorMsg); + } + } + }) + }); + + } + }, this); + } + } +}; \ No newline at end of file diff --git a/mcc/src/org/labkey/mcc/MccController.java b/mcc/src/org/labkey/mcc/MccController.java index 0967188da..ea84cef04 100644 --- a/mcc/src/org/labkey/mcc/MccController.java +++ b/mcc/src/org/labkey/mcc/MccController.java @@ -16,22 +16,508 @@ package org.labkey.mcc; -import org.labkey.api.action.SimpleViewAction; +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.labkey.api.action.ApiSimpleResponse; +import org.labkey.api.action.MutatingApiAction; import org.labkey.api.action.SpringActionController; +import org.labkey.api.data.Container; +import org.labkey.api.data.CoreSchema; +import org.labkey.api.data.DbScope; +import org.labkey.api.data.SimpleFilter; +import org.labkey.api.data.Table; +import org.labkey.api.data.TableInfo; +import org.labkey.api.data.TableSelector; +import org.labkey.api.module.AllowedDuringUpgrade; +import org.labkey.api.query.DetailsURL; +import org.labkey.api.query.FieldKey; +import org.labkey.api.security.IgnoresTermsOfUse; +import org.labkey.api.security.MutableSecurityPolicy; +import org.labkey.api.security.RequiresNoPermission; import org.labkey.api.security.RequiresPermission; +import org.labkey.api.security.SecurityManager; +import org.labkey.api.security.SecurityPolicyManager; +import org.labkey.api.security.User; +import org.labkey.api.security.UserManager; +import org.labkey.api.security.ValidEmail; +import org.labkey.api.security.permissions.AdminPermission; import org.labkey.api.security.permissions.ReadPermission; -import org.labkey.api.view.JspView; -import org.labkey.api.view.NavTree; +import org.labkey.api.security.roles.ReaderRole; +import org.labkey.api.settings.AppProps; +import org.labkey.api.settings.LookAndFeelProperties; +import org.labkey.api.util.ConfigurationException; +import org.labkey.api.util.ExceptionUtil; +import org.labkey.api.util.MailHelper; +import org.labkey.api.util.PageFlowUtil; import org.springframework.validation.BindException; -import org.springframework.web.servlet.ModelAndView; +import org.springframework.validation.Errors; + +import javax.mail.Address; +import javax.mail.Message; +import javax.mail.internet.InternetAddress; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; public class MccController extends SpringActionController { private static final DefaultActionResolver _actionResolver = new DefaultActionResolver(MccController.class); public static final String NAME = "mcc"; + private static final Logger _log = LogManager.getLogger(MccController.class); + public MccController() { setActionResolver(_actionResolver); } + + @RequiresNoPermission + @IgnoresTermsOfUse + @AllowedDuringUpgrade + public class RequestUserAction extends MutatingApiAction + { + @Override + public void validateForm(RequestUserForm form, Errors errors) + { + Container mccContainer = MccManager.get().getMCCContainer(); + if (mccContainer == null) + { + errors.reject(ERROR_MSG, "The MCC project has not been set on this server. This is an administrator error."); + return; + } + + if (StringUtils.isEmpty(form.getEmail()) || StringUtils.isEmpty(form.getEmailConfirmation())) + { + errors.reject(ERROR_REQUIRED, "No email address provided"); + } + else if (StringUtils.isEmpty(form.getFirstName()) || StringUtils.isEmpty(form.getLastName()) || StringUtils.isEmpty(form.getTitle()) || StringUtils.isEmpty(form.getInstitution()) || StringUtils.isEmpty(form.getReason())) + { + errors.reject(ERROR_REQUIRED, "You must provide your first and last name, title, institution, and reason for requesting access"); + } + else + { + try + { + ValidEmail email = new ValidEmail(form.getEmail()); + if (!form.getEmail().equals(form.getEmailConfirmation())) + { + errors.reject(ERROR_MSG, "The email addresses you have entered do not match. Please verify your email addresses below."); + } + + TableInfo ti = MccSchema.getInstance().getSchema().getTable(MccSchema.TABLE_USER_REQUESTS); + + //first check if this email exists: + SimpleFilter filter = new SimpleFilter(FieldKey.fromString("email"), form.getEmail()); + filter.addCondition(FieldKey.fromString("container"), mccContainer.getId()); + if (new TableSelector(ti, filter, null).exists()) + { + errors.reject(ERROR_MSG, "A login has already been requested for this email. You should receive a reply shortly from the site administrator."); + } + } + catch (ValidEmail.InvalidEmailException e) + { + errors.reject(ERROR_MSG, "Your email address is not valid. Please verify your email address below."); + } + } + } + + @Override + public Object execute(RequestUserForm form, BindException errors) throws Exception + { + ApiSimpleResponse response = new ApiSimpleResponse(); + + try + { + TableInfo ti = MccSchema.getInstance().getSchema().getTable(MccSchema.TABLE_USER_REQUESTS); + Map row = new HashMap<>(); + row.put("email", form.getEmail()); + row.put("firstName", form.getFirstName()); + row.put("lastName", form.getLastName()); + row.put("title", form.getTitle()); + row.put("institution", form.getInstitution()); + row.put("reason", form.getReason()); + row.put("container", MccManager.get().getMCCContainer().getId()); + + Table.insert(UserManager.getGuestUser(), ti, row); + + Set users = MccManager.get().getNotificationUsers(); + if (users != null && !users.isEmpty()) + { + try + { + Set
emails = new HashSet<>(); + for (User u : users) + { + emails.add(new InternetAddress(u.getEmail())); + } + + MailHelper.MultipartMessage mail = MailHelper.createMultipartMessage(); + Container c = MccManager.get().getMCCContainer(); + if (c == null) + { + c = getContainer(); + _log.warn("MCC container was not set, using: " + c.getPath()); + } + + DetailsURL url = DetailsURL.fromString("/query/executeQuery.view?schemaName=mcc&query.queryName=userRequests&query.viewName=Pending Requests", c); + mail.setEncodedHtmlContent("A user requested an account on MCC. Click here to view/approve this request"); + mail.setFrom(getReplyEmail(getContainer())); + mail.setSubject("MCC Account Request"); + mail.addRecipients(Message.RecipientType.TO, emails.toArray(new Address[0])); + + MailHelper.send(mail, getUser(), c); + } + catch (Exception e) + { + ExceptionUtil.logExceptionToMothership(null, e); + } + } + + + } + catch (ConfigurationException e) + { + errors.reject(ERROR_MSG, "There was a problem sending the registration email. Please contact your administrator."); + _log.error("Error adding self registered user", e); + } + + response.put("success", !errors.hasErrors()); + if (!errors.hasErrors()) + response.put("email", form.getEmail()); + + return response; + } + } + + public static class RequestUserForm extends Object + { + private String email; + private String emailConfirmation; + private String firstName; + private String lastName; + private String title; + private String institution; + private String reason; + + public void setEmail(String email) + { + this.email = email; + } + + public String getEmail() + { + return this.email; + } + + public void setEmailConfirmation(String email) + { + this.emailConfirmation = email; + } + + public String getEmailConfirmation() + { + return this.emailConfirmation; + } + + public String getFirstName() + { + return firstName; + } + + public void setFirstName(String firstName) + { + this.firstName = firstName; + } + + public String getLastName() + { + return lastName; + } + + public void setLastName(String lastName) + { + this.lastName = lastName; + } + + public String getTitle() + { + return title; + } + + public void setTitle(String title) + { + this.title = title; + } + + public String getInstitution() + { + return institution; + } + + public void setInstitution(String institution) + { + this.institution = institution; + } + + public String getReason() + { + return reason; + } + + public void setReason(String reason) + { + this.reason = reason; + } + } + + @RequiresPermission(AdminPermission.class) + public class ApproveUserRequestsAction extends MutatingApiAction + { + @Override + public void validateForm(ApproveUserRequestsForm form, Errors errors) + { + Container mccContainer = MccManager.get().getMCCContainer(); + if (mccContainer == null) + { + errors.reject(ERROR_MSG, "The MCC project has not been set on this server. This is an administrator error."); + return; + } + + if (form.getRequestIds() == null || form.getRequestIds().length == 0) + { + errors.reject(ERROR_MSG, "No request IDs provided"); + } + + TableInfo ti = MccSchema.getInstance().getSchema().getTable(MccSchema.TABLE_USER_REQUESTS); + for (int requestId : form.getRequestIds()) + { + TableSelector ts = new TableSelector(ti, PageFlowUtil.set("userId"), new SimpleFilter(FieldKey.fromString("rowId"), requestId), null); + if (!ts.exists()) + { + errors.reject(ERROR_MSG, "No request found for request ID: " + requestId); + break; + } + } + } + + @Override + public Object execute(ApproveUserRequestsForm form, BindException errors) throws Exception + { + ApiSimpleResponse response = new ApiSimpleResponse(); + MutableSecurityPolicy policy = new MutableSecurityPolicy(MccManager.get().getMCCContainer().getPolicy()); + List newUserStatusList = new ArrayList<>(); + List existingUsersGivenAccess = new ArrayList<>(); + try (DbScope.Transaction transaction = CoreSchema.getInstance().getScope().ensureTransaction()) + { + TableInfo ti = MccSchema.getInstance().getSchema().getTable(MccSchema.TABLE_USER_REQUESTS); + for (int requestId : form.getRequestIds()) + { + TableSelector ts = new TableSelector(ti, new SimpleFilter(FieldKey.fromString("rowId"), requestId), null); + Map map = ts.getMap(requestId); + + User u; + if (map.get("userId") != null) + { + Integer userId = (Integer)map.get("userId"); + u = UserManager.getUser(userId); + existingUsersGivenAccess.add(u); + } + else + { + ValidEmail ve = new ValidEmail((String)map.get("email")); + u = UserManager.getUser(ve); + if (u != null) + { + existingUsersGivenAccess.add(u); + } + else + { + SecurityManager.NewUserStatus st = SecurityManager.addUser(ve, getUser()); + u = st.getUser(); + u.setFirstName((String)map.get("firstName")); + u.setLastName((String)map.get("lastName")); + UserManager.updateUser(getUser(), u); + + if (st.isLdapEmail()) + { + existingUsersGivenAccess.add(st.getUser()); + } + else + { + newUserStatusList.add(st); + } + } + } + + Map row = new HashMap<>(); + row.put("rowId", requestId); + row.put("userId", u.getUserId()); + Table.update(getUser(), ti, row, requestId); + + if (!policy.hasPermission(u, ReadPermission.class)) + { + policy.addRoleAssignment(u, ReaderRole.class); + } + else + { + _log.info("user already has read permission on MCC container: " + u.getDisplayName(getUser())); + } + } + + SecurityPolicyManager.savePolicy(policy); + + transaction.commit(); + } + + //send emails: + for (SecurityManager.NewUserStatus st : newUserStatusList) + { + SecurityManager.sendRegistrationEmail(getViewContext(), st.getEmail(), null, st, null); + } + + for (User u : existingUsersGivenAccess) + { + Container mccContainer = MccManager.get().getMCCContainer(); + boolean isLDAP = SecurityManager.isLdapEmail(new ValidEmail(u.getEmail())); + + MailHelper.MultipartMessage mail = MailHelper.createMultipartMessage(); + mail.setEncodedHtmlContent("Your account request has been approved for MCC! " + "Click here to access the site." + (isLDAP ? " Use your normal OHSU email/password to login." : "")); + mail.setFrom(getReplyEmail(getContainer())); + mail.setSubject("MCC Account Request"); + mail.addRecipients(Message.RecipientType.TO, u.getEmail()); + + MailHelper.send(mail, getUser(), getContainer()); + } + + response.put("success", !errors.hasErrors()); + + return response; + } + } + + private String getReplyEmail(Container c) + { + LookAndFeelProperties lfp = LookAndFeelProperties.getInstance(getContainer()); + String email = lfp.getSystemEmailAddress(); + if (email == null) + { + return AppProps.getInstance().getAdministratorContactEmail(true); + } + + return email; + } + + public static class ApproveUserRequestsForm + { + private int[] requestIds; + + public int[] getRequestIds() + { + return requestIds; + } + + public void setRequestIds(int[] requestIds) + { + this.requestIds = requestIds; + } + } + + @RequiresNoPermission + @IgnoresTermsOfUse + @AllowedDuringUpgrade + public class RequestHelpAction extends MutatingApiAction + { + @Override + public void validateForm(RequestHelpForm form, Errors errors) + { + Container mccContainer = MccManager.get().getMCCContainer(); + if (mccContainer == null) + { + errors.reject(ERROR_MSG, "The MCC project has not been set on this server. This is an administrator error."); + return; + } + + if (StringUtils.isEmpty(form.getEmail()) || StringUtils.isEmpty(form.getComment())) + { + errors.reject(ERROR_REQUIRED, "Must provide both an email address and question/comment"); + } + else + { + try + { + new ValidEmail(form.getEmail()); + } + catch (ValidEmail.InvalidEmailException e) + { + errors.reject(ERROR_MSG, "Your email address is not valid. Please verify your email address below."); + } + } + } + + @Override + public Object execute(RequestHelpForm form, BindException errors) throws Exception + { + Set users = MccManager.get().getNotificationUsers(); + if (users != null && !users.isEmpty()) + { + try + { + Set
emails = new HashSet<>(); + for (User u : users) + { + emails.add(new InternetAddress(u.getEmail())); + } + + MailHelper.MultipartMessage mail = MailHelper.createMultipartMessage(); + mail.setEncodedHtmlContent("A support request was submitted from MCC by: " + form.getEmail() + "

Message:
" + form.getComment()); + mail.setFrom(form.getEmail()); + mail.setSubject("MCC Help Request"); + mail.addRecipients(Message.RecipientType.TO, emails.toArray(new Address[0])); + + MailHelper.send(mail, getUser(), getContainer()); + } + catch (Exception e) + { + ExceptionUtil.logExceptionToMothership(null, e); + } + } + else + { + _log.error("A help request was received by MCC, but the admin emails have not been configured. The request from: " + form.getEmail()); + _log.error(form.getComment()); + } + + return new ApiSimpleResponse("success", true); + } + } + + public static class RequestHelpForm + { + private String _email; + private String _comment; + + public String getEmail() + { + return _email; + } + + public void setEmail(String email) + { + _email = email; + } + + public String getComment() + { + return _comment; + } + + public void setComment(String comment) + { + _comment = comment; + } + } } diff --git a/mcc/src/org/labkey/mcc/MccManager.java b/mcc/src/org/labkey/mcc/MccManager.java index d6417b79d..d0f657de9 100644 --- a/mcc/src/org/labkey/mcc/MccManager.java +++ b/mcc/src/org/labkey/mcc/MccManager.java @@ -16,8 +16,28 @@ package org.labkey.mcc; +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerManager; +import org.labkey.api.module.Module; +import org.labkey.api.module.ModuleLoader; +import org.labkey.api.module.ModuleProperty; +import org.labkey.api.security.User; +import org.labkey.api.security.UserManager; +import org.labkey.api.security.ValidEmail; + +import java.util.HashSet; +import java.util.Set; + public class MccManager { + private static final Logger _log = LogManager.getLogger(MccManager.class); + + public static final String ContainerPropName = "MCCContainer"; + public static final String NotifyPropName = "MCCContactUsers"; + private static final MccManager _instance = new MccManager(); private MccManager() @@ -29,4 +49,54 @@ public static MccManager get() { return _instance; } + + public Container getMCCContainer() + { + Module m = ModuleLoader.getInstance().getModule(MccModule.NAME); + ModuleProperty mp = m.getModuleProperties().get(MccManager.ContainerPropName); + String path = mp.getEffectiveValue(ContainerManager.getRoot()); + if (path == null) + return null; + + return ContainerManager.getForPath(path); + } + + public Set getNotificationUsers() + { + Module m = ModuleLoader.getInstance().getModule(MccModule.NAME); + ModuleProperty mp = m.getModuleProperties().get(MccManager.NotifyPropName); + String userNames = mp.getEffectiveValue(ContainerManager.getRoot()); + userNames = StringUtils.trimToNull(userNames); + if (userNames == null) + return null; + + Set ret = new HashSet<>(); + for (String username : userNames.split(",")) + { + User u = UserManager.getUserByDisplayName(username); + if (u == null) + { + try + { + u = UserManager.getUser(new ValidEmail(username)); + } + catch (ValidEmail.InvalidEmailException e) + { + //ignore + } + } + + if (u == null) + { + _log.error("Unknown user registered for MCC notifcations: " + username); + } + + if (u != null) + { + ret.add(u); + } + } + + return ret; + } } \ No newline at end of file diff --git a/mcc/src/org/labkey/mcc/MccModule.java b/mcc/src/org/labkey/mcc/MccModule.java index 96e462c0f..ea28010a7 100644 --- a/mcc/src/org/labkey/mcc/MccModule.java +++ b/mcc/src/org/labkey/mcc/MccModule.java @@ -41,7 +41,7 @@ public String getName() @Override public @Nullable Double getSchemaVersion() { - return 20.000; + return 20.001; } @Override diff --git a/mcc/src/org/labkey/mcc/MccSchema.java b/mcc/src/org/labkey/mcc/MccSchema.java index d072b94a9..be31ef6f1 100644 --- a/mcc/src/org/labkey/mcc/MccSchema.java +++ b/mcc/src/org/labkey/mcc/MccSchema.java @@ -25,6 +25,8 @@ public class MccSchema private static final MccSchema _instance = new MccSchema(); public static final String NAME = "mcc"; + public static final String TABLE_USER_REQUESTS = "userRequests"; + public static MccSchema getInstance() { return _instance; diff --git a/mcc/src/org/labkey/mcc/query/UserRequestCustomizer.java b/mcc/src/org/labkey/mcc/query/UserRequestCustomizer.java new file mode 100644 index 000000000..7f789d464 --- /dev/null +++ b/mcc/src/org/labkey/mcc/query/UserRequestCustomizer.java @@ -0,0 +1,44 @@ +package org.labkey.mcc.query; + +import org.labkey.api.data.AbstractTableInfo; +import org.labkey.api.data.JdbcType; +import org.labkey.api.data.SQLFragment; +import org.labkey.api.data.TableCustomizer; +import org.labkey.api.data.TableInfo; +import org.labkey.api.ldk.LDKService; +import org.labkey.api.query.ExprColumn; + +public class UserRequestCustomizer implements TableCustomizer +{ + @Override + public void customize(TableInfo tableInfo) + { + LDKService.get().getDefaultTableCustomizer().customize(tableInfo); + + if (tableInfo instanceof AbstractTableInfo) + { + addUserCol((AbstractTableInfo)tableInfo); + } + } + + public void addUserCol(AbstractTableInfo ti) + { + String colName = "hasAccess"; + if (ti.getColumn(colName) != null) + { + return; + } + + ExprColumn col = new ExprColumn(ti, colName, new SQLFragment("(CASE WHEN (exists (" + + "select u.rowid from mcc.userrequests u " + + "left join core.RoleAssignments ra " + + "on (u.userid = ra.UserId AND u.container = ra.ResourceId) " + + "WHERE ra.Role = 'org.labkey.api.security.roles.ReaderRole' AND u.rowid = " + ExprColumn.STR_TABLE_ALIAS + ".rowid " + + ")) THEN " + ti.getSqlDialect().getBooleanTRUE() + " ELSE " + ti.getSqlDialect().getBooleanFALSE() + " END)"), JdbcType.BOOLEAN, ti.getColumn("userId")); + col.setLabel("Has MCC Access?"); + col.setReadOnly(true); + col.setIsUnselectable(true); + col.setUserEditable(false); + ti.addColumn(col); + } +} From c3cf85e99b4f189afbe20b0b8aadac0a65e93d94 Mon Sep 17 00:00:00 2001 From: bbimber Date: Thu, 4 Feb 2021 15:16:49 -0800 Subject: [PATCH 67/98] Add release notes page --- mGAP/resources/folderTypes/mGAP.folderType.xml | 4 ++++ mGAP/resources/views/releaseNotes.html | 12 ++++++++++++ mGAP/resources/views/releaseNotes.view.xml | 8 ++++++++ mGAP/resources/views/releaseNotes.webpart.xml | 6 ++++++ mGAP/resources/views/variants.html | 2 +- .../labkey/mgap/pipeline/mGapReleaseGenerator.java | 4 ++-- 6 files changed, 33 insertions(+), 3 deletions(-) create mode 100644 mGAP/resources/views/releaseNotes.html create mode 100644 mGAP/resources/views/releaseNotes.view.xml create mode 100644 mGAP/resources/views/releaseNotes.webpart.xml diff --git a/mGAP/resources/folderTypes/mGAP.folderType.xml b/mGAP/resources/folderTypes/mGAP.folderType.xml index bee390d74..bf7576130 100644 --- a/mGAP/resources/folderTypes/mGAP.folderType.xml +++ b/mGAP/resources/folderTypes/mGAP.folderType.xml @@ -84,6 +84,10 @@ mGAP Variant Releases body + + mGAP Release Notes + body + mGAP Gene Search right diff --git a/mGAP/resources/views/releaseNotes.html b/mGAP/resources/views/releaseNotes.html new file mode 100644 index 000000000..110244e37 --- /dev/null +++ b/mGAP/resources/views/releaseNotes.html @@ -0,0 +1,12 @@ +

Release 2.0:

+
    +
  • Substantial revamp of all data. All samples have been realigned to the MMul_10 reference genome, followed by our standard GenotypeGVCFs pipeline. The MMul_10 is the most complete rhesus macaque assembly to date, and we expect this should improve accuracy of variant calls. Further, because our data are now aligned to the same assembly as NCBI/Ensembl, it should be easier to translate between mGAP and other databases.
    • +
  • Our internal variant calling process has switched to use GATK's GenomicsDB to pre-aggregate data prior to calling with GenotypeGVCFs, as opposed to CombineGVCFs, which was used in prior releases. This should be a purely technical difference with no change in the resulting data
    • +
+ +

Future Plans:

+
    +
  • We expect to upgrade the genome browser to use the redesigned JBrowse 2 browser. This should provide general performance improvements and will make future mGAP-specific customization easier.
    • +
  • We will support other modes of viewing and downloading variant data, in particular tabular views by gene.
    • +
  • We recognize that the mGAP release VCF can be enormous, particularly because of all the site-specific functional annotation. To support different types of users, upcoming releases will include 'slim' versions of the data, which will be downloadable files with certain information removed to save file size.
    • +
\ No newline at end of file diff --git a/mGAP/resources/views/releaseNotes.view.xml b/mGAP/resources/views/releaseNotes.view.xml new file mode 100644 index 000000000..c9a2b3e33 --- /dev/null +++ b/mGAP/resources/views/releaseNotes.view.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/mGAP/resources/views/releaseNotes.webpart.xml b/mGAP/resources/views/releaseNotes.webpart.xml new file mode 100644 index 000000000..f7f58d800 --- /dev/null +++ b/mGAP/resources/views/releaseNotes.webpart.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/mGAP/resources/views/variants.html b/mGAP/resources/views/variants.html index 3d66a6667..f59586d79 100644 --- a/mGAP/resources/views/variants.html +++ b/mGAP/resources/views/variants.html @@ -9,7 +9,7 @@ title: 'Variant Catalog Releases', schemaName: 'mgap', queryName: 'variantCatalogReleases', - maxRows: 50, + maxRows: 3, showRecordSelectors: false, showDetailsColumn: false, buttonBar: {position: 'none', includeStandardButtons: false, items: []} diff --git a/mGAP/src/org/labkey/mgap/pipeline/mGapReleaseGenerator.java b/mGAP/src/org/labkey/mgap/pipeline/mGapReleaseGenerator.java index 848a0a09d..a3fd61c17 100644 --- a/mGAP/src/org/labkey/mgap/pipeline/mGapReleaseGenerator.java +++ b/mGAP/src/org/labkey/mgap/pipeline/mGapReleaseGenerator.java @@ -953,7 +953,7 @@ private void inspectAndSummarizeVcf(JobContext ctx, File vcfInput, GeneToNameTra File interestingVariantTable = getVariantTableName(ctx, vcfInput); try (VCFFileReader reader = new VCFFileReader(vcfInput); CloseableIterator it = reader.iterator(); CSVWriter writer = new CSVWriter(PrintWriters.getPrintWriter(interestingVariantTable), '\t', CSVWriter.NO_QUOTE_CHARACTER)) { - writer.writeNext(new String[]{"Chromosome", "Position", "Reference", "Allele", "Source", "Reason", "Description", "Overlapping Gene(s)", "OMIM Entries", "OMIM Phenotypes", "AF", "CADD_PH"}); + writer.writeNext(new String[]{"Chromosome", "Position", "Reference", "Allele", "Source", "Reason", "Description", "Overlapping Gene(s)", "OMIM Entries", "OMIM Phenotypes", "AF", "Identifier", "CADD_PH"}); while (it.hasNext()) { Set> queuedLines = new LinkedHashSet<>(); @@ -1113,7 +1113,7 @@ private void inspectAndSummarizeVcf(JobContext ctx, File vcfInput, GeneToNameTra try { String allele = clnAlleles.get(i); - maybeWriteVariantLine(queuedLines, vc, allele, "ClinVar", diseaseSplit.get(j), description, overlappingGenes, omims, omimds, ctx.getLogger(), "ClinVar:" + clnAlleleIds.get(j)); + maybeWriteVariantLine(queuedLines, vc, allele, "ClinVar", diseaseSplit.get(j), description, overlappingGenes, omims, omimds, ctx.getLogger(), "ClinVar:" + clnAlleleIds.get(i)); } catch (IndexOutOfBoundsException e) From 8ca15eef8fb6cc0e7c68fb6f67a975f421a2e078 Mon Sep 17 00:00:00 2001 From: bbimber Date: Thu, 4 Feb 2021 16:02:07 -0800 Subject: [PATCH 68/98] Add more WNPRC ETL code --- mcc/resources/etls/wnprc.xml | 92 ++++++++++++++++++- mcc/resources/queries/wnprcSource/birth.sql | 6 ++ mcc/resources/queries/wnprcSource/deaths.sql | 8 ++ .../queries/wnprcSource/demographics.sql | 2 +- .../queries/wnprcSource/parentage.sql | 27 ++++++ mcc/resources/queries/wnprcSource/weight.sql | 8 ++ .../datasets/datasets_manifest.xml | 6 -- .../datasets/datasets_metadata.xml | 59 ------------ 8 files changed, 138 insertions(+), 70 deletions(-) create mode 100644 mcc/resources/queries/wnprcSource/birth.sql create mode 100644 mcc/resources/queries/wnprcSource/deaths.sql create mode 100644 mcc/resources/queries/wnprcSource/parentage.sql create mode 100644 mcc/resources/queries/wnprcSource/weight.sql diff --git a/mcc/resources/etls/wnprc.xml b/mcc/resources/etls/wnprc.xml index de5850d40..d9d4bf593 100644 --- a/mcc/resources/etls/wnprc.xml +++ b/mcc/resources/etls/wnprc.xml @@ -3,14 +3,98 @@ WNPRC_Data WNPRC Clinical/Demographics Data - + Copy to target - - + + + Id + date + gender + geographic_origin + birth + death + species + objectid + + + + + + + + + + Copy to target + + + Id + date + parent + relationship + method + objectid + + + + + + + + + + Copy to target + + + Id + date + gender + species + geographic_origin + dam + sire + objectid + + + + + + + + + + Copy to target + + + Id + date + weight + objectid + + + + + + + + + + Copy to target + + + Id + date + cause + objectid + + + + + - + + diff --git a/mcc/resources/queries/wnprcSource/birth.sql b/mcc/resources/queries/wnprcSource/birth.sql new file mode 100644 index 000000000..06808b9af --- /dev/null +++ b/mcc/resources/queries/wnprcSource/birth.sql @@ -0,0 +1,6 @@ +SELECT + +Id, date, gender, species, geographic_origin, dam, sire, objectid, modified + +FROM "/WNPRC/EHR/".study.birth +WHERE species = 'Marmoset'; \ No newline at end of file diff --git a/mcc/resources/queries/wnprcSource/deaths.sql b/mcc/resources/queries/wnprcSource/deaths.sql new file mode 100644 index 000000000..94c301bb7 --- /dev/null +++ b/mcc/resources/queries/wnprcSource/deaths.sql @@ -0,0 +1,8 @@ +SELECT + + Id, date, + cause, + objectid, modified + +FROM "/WNPRC/EHR/".study.weight +WHERE Id.demographics.species = 'Marmoset'; \ No newline at end of file diff --git a/mcc/resources/queries/wnprcSource/demographics.sql b/mcc/resources/queries/wnprcSource/demographics.sql index c63412b14..7a860fef3 100644 --- a/mcc/resources/queries/wnprcSource/demographics.sql +++ b/mcc/resources/queries/wnprcSource/demographics.sql @@ -1,6 +1,6 @@ SELECT -Id, date, gender, geographic_origin, birth, death, species, objectid +Id, date, gender, geographic_origin, birth, death, species, objectid, modified FROM "/WNPRC/EHR/".study.demographics WHERE species = 'Marmoset'; \ No newline at end of file diff --git a/mcc/resources/queries/wnprcSource/parentage.sql b/mcc/resources/queries/wnprcSource/parentage.sql new file mode 100644 index 000000000..a8fe2e80c --- /dev/null +++ b/mcc/resources/queries/wnprcSource/parentage.sql @@ -0,0 +1,27 @@ +SELECT + + Id, + date, + sire as parent, + 'Sire' as relationship, + 'Observed' as method, + cast(objectid as varchar) || '-Sire' as objectid, + modified + +FROM "/WNPRC/EHR/".study.demographics +WHERE species = 'Marmoset' and sire is not null + +UNION ALL + +SELECT + + Id, + date, + sire as parent, + 'Dam' as relationship, + 'Observed' as method, + cast(objectid as varchar) || '-Dam' as objectid, + modified + +FROM "/WNPRC/EHR/".study.demographics +WHERE species = 'Marmoset' and dam is not null \ No newline at end of file diff --git a/mcc/resources/queries/wnprcSource/weight.sql b/mcc/resources/queries/wnprcSource/weight.sql new file mode 100644 index 000000000..db7bce107 --- /dev/null +++ b/mcc/resources/queries/wnprcSource/weight.sql @@ -0,0 +1,8 @@ +SELECT + + Id, date, + weight, + objectid, modified + +FROM "/WNPRC/EHR/".study.weight +WHERE Id.demographics.species = 'Marmoset'; \ No newline at end of file diff --git a/mcc/resources/referenceStudy/datasets/datasets_manifest.xml b/mcc/resources/referenceStudy/datasets/datasets_manifest.xml index 1d43fa147..47d75b91c 100644 --- a/mcc/resources/referenceStudy/datasets/datasets_manifest.xml +++ b/mcc/resources/referenceStudy/datasets/datasets_manifest.xml @@ -39,17 +39,11 @@ - - - - - - diff --git a/mcc/resources/referenceStudy/datasets/datasets_metadata.xml b/mcc/resources/referenceStudy/datasets/datasets_metadata.xml index 6a154bcc1..480c6a89d 100644 --- a/mcc/resources/referenceStudy/datasets/datasets_metadata.xml +++ b/mcc/resources/referenceStudy/datasets/datasets_metadata.xml @@ -561,65 +561,6 @@ Arrival - - - - varchar - http://cpas.labkey.com/Study#ParticipantId - - ptid - - - - timestamp - http://cpas.labkey.com/Study#VisitDate - http://cpas.labkey.com/Study#VisitDate - - - timestamp - urn:ehr.labkey.org/#EndDate - - - integer - urn:ehr.labkey.org/#Project - - - entityid - urn:ehr.labkey.org/#ObjectId - true - - - Assignment -
- - - - varchar - http://cpas.labkey.com/Study#ParticipantId - - ptid - - - - timestamp - http://cpas.labkey.com/Study#VisitDate - http://cpas.labkey.com/Study#VisitDate - - - timestamp - urn:ehr.labkey.org/#EndDate - - - integer - - - entityid - urn:ehr.labkey.org/#ObjectId - true - - - Animal Group Members -
From a578266fb27f75c453f445a4365fe1ea0bb129c3 Mon Sep 17 00:00:00 2001 From: bbimber Date: Fri, 5 Feb 2021 14:44:24 -0800 Subject: [PATCH 69/98] Update ETLs --- mcc/resources/etls/snprc.xml | 103 ++++++++++++++++++++++++++++++++++- mcc/resources/etls/wnprc.xml | 15 +++-- 2 files changed, 107 insertions(+), 11 deletions(-) diff --git a/mcc/resources/etls/snprc.xml b/mcc/resources/etls/snprc.xml index 127947d46..adadf3468 100644 --- a/mcc/resources/etls/snprc.xml +++ b/mcc/resources/etls/snprc.xml @@ -1,14 +1,111 @@ SNPRC_Data + + SNPRC Clinical/Demographics Data - + Copy to target - - + + + AnimalId + date + gender + geographic_origin + birth + death + species + objectid + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Copy to target + + + AnimalId + birth + gender + species + dam + sire + objectid + + + + + + + + + + + + + Copy to target + + + AnimalId + date + weight + objectid + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/mcc/resources/etls/wnprc.xml b/mcc/resources/etls/wnprc.xml index d9d4bf593..9b3aac007 100644 --- a/mcc/resources/etls/wnprc.xml +++ b/mcc/resources/etls/wnprc.xml @@ -17,7 +17,7 @@ objectid - + @@ -35,7 +35,7 @@ objectid - + @@ -55,7 +55,7 @@ objectid - + @@ -71,7 +71,7 @@ objectid - + @@ -87,17 +87,16 @@ objectid - + - - - + + From 871f4c4d3d5ed41daaff334f055b8055930aeec3 Mon Sep 17 00:00:00 2001 From: bbimber Date: Sun, 7 Feb 2021 11:09:23 -0800 Subject: [PATCH 70/98] Allow cellranger VDJ to finish when there are no a/b hits --- .../tcrdb/pipeline/CellRangerVDJCellHashingHandler.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java index 85ac41dea..8e52ec939 100644 --- a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java +++ b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java @@ -38,7 +38,9 @@ public class CellRangerVDJCellHashingHandler extends AbstractParameterizedOutputHandler { - private FileType _fileType = new FileType("vloupe", false); + private FileType _vloupeFileType = new FileType("vloupe", false); + private FileType _htmlFileType = new FileType("html", false); + public static final String CATEGORY = "Cell Hashing Calls (VDJ)"; public static final String TARGET_ASSAY = "targetAssay"; @@ -73,7 +75,7 @@ private static List getDefaultParams() @Override public boolean canProcess(SequenceOutputFile o) { - return o.getFile() != null && _fileType.isType(o.getFile()); + return o.getFile() != null && (_vloupeFileType.isType(o.getFile()) || (_htmlFileType.isType(o.getFile()) && "10x Run Summary".equals(o.getCategory()) && o.getName().contains("VDJ Summary"))); } @Override From daaf73d87b2be6a770f67a28c5dc606633b9ee87 Mon Sep 17 00:00:00 2001 From: bbimber Date: Mon, 8 Feb 2021 08:42:24 -0800 Subject: [PATCH 71/98] Fail more clearly if no cell barcodes found --- .../tcrdb/pipeline/CellRangerVDJCellHashingHandler.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java index 8e52ec939..e7ae5b2c1 100644 --- a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java +++ b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java @@ -244,6 +244,8 @@ private File createCellbarcodeWhitelist(JobContext ctx, File perCellTsv, boolean Set uniqueBarcodesIncludingNoCDR3 = new HashSet<>(); ctx.getLogger().debug("writing cell barcodes, using file: " + perCellTsv.getPath()); ctx.getLogger().debug("allow cells lacking CDR3: " + allowCellsLackingCDR3); + + int totalBarcodeWritten = 0; try (CSVWriter writer = new CSVWriter(PrintWriters.getPrintWriter(cellBarcodeWhitelist), ',', CSVWriter.NO_QUOTE_CHARACTER); CSVReader reader = new CSVReader(Readers.getReader(perCellTsv), ',')) { int rowIdx = 0; @@ -275,6 +277,7 @@ private File createCellbarcodeWhitelist(JobContext ctx, File perCellTsv, boolean { writer.writeNext(new String[]{barcode}); uniqueBarcodes.add(barcode); + totalBarcodeWritten++; } uniqueBarcodesIncludingNoCDR3.add(barcode); @@ -301,6 +304,7 @@ private File createCellbarcodeWhitelist(JobContext ctx, File perCellTsv, boolean for (String barcode : uniqueBarcodesIncludingNoCDR3) { writer.writeNext(new String[]{barcode}); + totalBarcodeWritten++; } } catch (IOException e) @@ -309,6 +313,11 @@ private File createCellbarcodeWhitelist(JobContext ctx, File perCellTsv, boolean } } + if (totalBarcodeWritten == 0) + { + throw new PipelineJobException("No valid cell barcodes found!"); + } + //TODO: consider looking up GEX data? return cellBarcodeWhitelist; From 3bf50badf6213000c919d921dc7fd990543e2dd1 Mon Sep 17 00:00:00 2001 From: bbimber Date: Tue, 9 Feb 2021 14:38:37 -0800 Subject: [PATCH 72/98] Refactor citeseq/hashing to use pre-computed count matrix --- mcc/resources/etls/snprc.xml | 2 +- mcc/resources/etls/wnprc.xml | 2 +- .../pipeline/CellRangerVDJCellHashingHandler.java | 10 +++++----- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/mcc/resources/etls/snprc.xml b/mcc/resources/etls/snprc.xml index adadf3468..103417512 100644 --- a/mcc/resources/etls/snprc.xml +++ b/mcc/resources/etls/snprc.xml @@ -7,7 +7,7 @@ Copy to target - + AnimalId date diff --git a/mcc/resources/etls/wnprc.xml b/mcc/resources/etls/wnprc.xml index 9b3aac007..c4433e54c 100644 --- a/mcc/resources/etls/wnprc.xml +++ b/mcc/resources/etls/wnprc.xml @@ -5,7 +5,7 @@ Copy to target - + Id date diff --git a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java index e7ae5b2c1..79204c155 100644 --- a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java +++ b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java @@ -67,7 +67,7 @@ private static List getDefaultParams() }}, false) )); - ret.addAll(CellHashingService.get().getDefaultHashingParams(true, CellHashingService.BARCODE_TYPE.hashing)); + ret.addAll(CellHashingService.get().getHashingCallingParams()); return ret; } @@ -114,7 +114,7 @@ public class Processor implements SequenceOutputHandler.SequenceOutputProcessor public void init(JobContext ctx, List inputFiles, List actions, List outputsToCreate) throws UnsupportedOperationException, PipelineJobException { //NOTE: this is the pathway to import assay data, whether hashing is used or not - CellHashingService.get().prepareHashingAndCiteSeqFilesIfNeeded(ctx.getOutputDir(), ctx.getJob(), ctx.getSequenceSupport(), "tcrReadsetId", ctx.getParams().optBoolean("excludeFailedcDNA", false), false, false); + CellHashingService.get().prepareHashingAndCiteSeqFilesIfNeeded(ctx.getOutputDir(), ctx.getJob(), ctx.getSequenceSupport(), "tcrReadsetId", false, false); if (ctx.getParams().optBoolean(USE_GEX_BARCODES, false)) { @@ -210,17 +210,17 @@ private void processVloupeFile(JobContext ctx, File perCellTsv, Readset rs, Reco //TODO: allow union of GEX and TCR cell barcodes for whitelist! CellHashingService.CellHashingParameters parameters = CellHashingService.CellHashingParameters.createFromJson(CellHashingService.BARCODE_TYPE.hashing, ctx.getSourceDirectory(), ctx.getParams(), null, rs, null); - parameters.cellBarcodeWhitelistFile = createCellbarcodeWhitelist(ctx, perCellTsv, true); parameters.genomeId = genomeId; parameters.outputCategory = CATEGORY; parameters.basename = FileUtil.makeLegalName(rs.getName()); parameters.allowableHtoOrCiteseqBarcodes = htosPerReadset; + parameters.cellBarcodeWhitelistFile = createCellbarcodeWhitelist(ctx, perCellTsv, true); + File existingCountMatrixUmiDir = CellHashingService.get().getExistingFeatureBarcodeCountDir(rs, CellHashingService.BARCODE_TYPE.hashing, ctx.getSequenceSupport()); - File cellToHto = CellHashingService.get().processCellHashingOrCiteSeqForParent(rs, output, ctx, parameters); + File cellToHto = CellHashingService.get().generateHashingCallsForRawMatrix(rs, output, ctx, parameters, existingCountMatrixUmiDir); if (CellHashingService.get().usesCellHashing(ctx.getSequenceSupport(), ctx.getSourceDirectory()) && cellToHto == null) { throw new PipelineJobException("Missing cell to HTO file"); - } action.addOutput(cellToHto, CellRangerVDJUtils.TCR_HASHING_CALLS, false); From 8c41f3f72a5881901a152c11e319e7e7663c3aa3 Mon Sep 17 00:00:00 2001 From: bbimber Date: Wed, 10 Feb 2021 17:17:47 -0800 Subject: [PATCH 73/98] Prepare MCC examples --- mcc/resources/queries/study/labwork.js | 8 ++++++++ mcc/resources/queries/study/medicationAdministration.js | 8 ++++++++ mcc/resources/queries/study/medicationOrders.js | 8 ++++++++ mcc/resources/views/dashboard.html | 9 +++++++++ mcc/resources/views/dashboard.view.xml | 9 +++++++++ mcc/resources/views/dashboard.webpart.xml | 6 ++++++ mcc/resources/web/mcc/dashboard.js | 0 7 files changed, 48 insertions(+) create mode 100644 mcc/resources/queries/study/labwork.js create mode 100644 mcc/resources/queries/study/medicationAdministration.js create mode 100644 mcc/resources/queries/study/medicationOrders.js create mode 100644 mcc/resources/views/dashboard.html create mode 100644 mcc/resources/views/dashboard.view.xml create mode 100644 mcc/resources/views/dashboard.webpart.xml create mode 100644 mcc/resources/web/mcc/dashboard.js diff --git a/mcc/resources/queries/study/labwork.js b/mcc/resources/queries/study/labwork.js new file mode 100644 index 000000000..d4075c56c --- /dev/null +++ b/mcc/resources/queries/study/labwork.js @@ -0,0 +1,8 @@ +/* + * Copyright (c) 2018-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0 + */ + +require("ehr/triggers").initScript(this); + diff --git a/mcc/resources/queries/study/medicationAdministration.js b/mcc/resources/queries/study/medicationAdministration.js new file mode 100644 index 000000000..d4075c56c --- /dev/null +++ b/mcc/resources/queries/study/medicationAdministration.js @@ -0,0 +1,8 @@ +/* + * Copyright (c) 2018-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0 + */ + +require("ehr/triggers").initScript(this); + diff --git a/mcc/resources/queries/study/medicationOrders.js b/mcc/resources/queries/study/medicationOrders.js new file mode 100644 index 000000000..d4075c56c --- /dev/null +++ b/mcc/resources/queries/study/medicationOrders.js @@ -0,0 +1,8 @@ +/* + * Copyright (c) 2018-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0 + */ + +require("ehr/triggers").initScript(this); + diff --git a/mcc/resources/views/dashboard.html b/mcc/resources/views/dashboard.html new file mode 100644 index 000000000..014a425a7 --- /dev/null +++ b/mcc/resources/views/dashboard.html @@ -0,0 +1,9 @@ + \ No newline at end of file diff --git a/mcc/resources/views/dashboard.view.xml b/mcc/resources/views/dashboard.view.xml new file mode 100644 index 000000000..252242cd6 --- /dev/null +++ b/mcc/resources/views/dashboard.view.xml @@ -0,0 +1,9 @@ + + + + + + + + + \ No newline at end of file diff --git a/mcc/resources/views/dashboard.webpart.xml b/mcc/resources/views/dashboard.webpart.xml new file mode 100644 index 000000000..4e56bf8c1 --- /dev/null +++ b/mcc/resources/views/dashboard.webpart.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/mcc/resources/web/mcc/dashboard.js b/mcc/resources/web/mcc/dashboard.js new file mode 100644 index 000000000..e69de29bb From bfb90055898f3df53c58da1a6f48bea3b40b80a5 Mon Sep 17 00:00:00 2001 From: bbimber Date: Thu, 11 Feb 2021 09:46:58 -0800 Subject: [PATCH 74/98] Fix case in ETL --- mcc/resources/etls/snprc.xml | 8 ++++---- mcc/resources/etls/wnprc.xml | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/mcc/resources/etls/snprc.xml b/mcc/resources/etls/snprc.xml index 103417512..9eccdb1aa 100644 --- a/mcc/resources/etls/snprc.xml +++ b/mcc/resources/etls/snprc.xml @@ -29,7 +29,7 @@ - + @@ -49,7 +49,7 @@ Copy to target - + AnimalId birth @@ -71,7 +71,7 @@ Copy to target - + AnimalId date @@ -89,7 +89,7 @@ - + diff --git a/mcc/resources/etls/wnprc.xml b/mcc/resources/etls/wnprc.xml index c4433e54c..3147ab99d 100644 --- a/mcc/resources/etls/wnprc.xml +++ b/mcc/resources/etls/wnprc.xml @@ -25,7 +25,7 @@ Copy to target - + Id date @@ -43,7 +43,7 @@ Copy to target - + Id date @@ -63,7 +63,7 @@ Copy to target - + Id date @@ -79,7 +79,7 @@ Copy to target - + Id date From 4bfda5679138fc0fe64771d0eb43f80b22c0ec05 Mon Sep 17 00:00:00 2001 From: bbimber Date: Thu, 11 Feb 2021 11:03:21 -0800 Subject: [PATCH 75/98] Remove ETL step --- mcc/resources/etls/snprc.xml | 41 ++++++++++++++++++------------------ 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/mcc/resources/etls/snprc.xml b/mcc/resources/etls/snprc.xml index 9eccdb1aa..9f856c935 100644 --- a/mcc/resources/etls/snprc.xml +++ b/mcc/resources/etls/snprc.xml @@ -47,27 +47,28 @@ - - Copy to target - - - AnimalId - birth - gender - species - dam - sire - objectid - - - - - - - - + + + + + + + + + + + + + + + + + + + + - + Copy to target From e68be3401bed407f8b8d3ce860c7ab3d32de60bf Mon Sep 17 00:00:00 2001 From: bbimber Date: Thu, 11 Feb 2021 13:22:16 -0800 Subject: [PATCH 76/98] Further simplify hashing params code --- .../tcrdb/pipeline/CellRangerVDJCellHashingHandler.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java index 79204c155..26b1e01fa 100644 --- a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java +++ b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java @@ -49,7 +49,7 @@ public class CellRangerVDJCellHashingHandler extends AbstractParameterizedOutput public CellRangerVDJCellHashingHandler() { - super(ModuleLoader.getInstance().getModule(TCRdbModule.class), "CellRanger VDJ Import", "This will either directly import data (if cell hashing is not used), or run CiteSeqCount/MultiSeqClassifier to generate a sample-to-cellbarcode TSV based on the filtered barcodes from CellRanger VDJ and then import.", new LinkedHashSet<>(PageFlowUtil.set("tcrdb/field/AssaySelectorField.js")), getDefaultParams()); + super(ModuleLoader.getInstance().getModule(TCRdbModule.class), "CellRanger VDJ Import", "This will either directly import data (if cell hashing is not used), or run cellhashR on the hashing count matrix to generate a sample-to-cellbarcode TSV based on the filtered barcodes from CellRanger VDJ and then import.", new LinkedHashSet<>(PageFlowUtil.set("tcrdb/field/AssaySelectorField.js")), getDefaultParams()); } private static List getDefaultParams() @@ -209,11 +209,11 @@ private void processVloupeFile(JobContext ctx, File perCellTsv, Readset rs, Reco //TODO: allow union of GEX and TCR cell barcodes for whitelist! - CellHashingService.CellHashingParameters parameters = CellHashingService.CellHashingParameters.createFromJson(CellHashingService.BARCODE_TYPE.hashing, ctx.getSourceDirectory(), ctx.getParams(), null, rs, null); + CellHashingService.CellHashingParameters parameters = CellHashingService.CellHashingParameters.createFromJson(CellHashingService.BARCODE_TYPE.hashing, ctx.getSourceDirectory(), ctx.getParams(), null, rs); parameters.genomeId = genomeId; parameters.outputCategory = CATEGORY; parameters.basename = FileUtil.makeLegalName(rs.getName()); - parameters.allowableHtoOrCiteseqBarcodes = htosPerReadset; + parameters.allowableHtoBarcodes = htosPerReadset; parameters.cellBarcodeWhitelistFile = createCellbarcodeWhitelist(ctx, perCellTsv, true); File existingCountMatrixUmiDir = CellHashingService.get().getExistingFeatureBarcodeCountDir(rs, CellHashingService.BARCODE_TYPE.hashing, ctx.getSequenceSupport()); From b4ef0c8bdbbd966c9e27370ab909d2d7a47059d5 Mon Sep 17 00:00:00 2001 From: bbimber Date: Thu, 11 Feb 2021 14:07:07 -0800 Subject: [PATCH 77/98] Add alternate keys --- mcc/resources/etls/snprc.xml | 9 +++++++++ mcc/resources/etls/wnprc.xml | 22 +++++++++++++++------- 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/mcc/resources/etls/snprc.xml b/mcc/resources/etls/snprc.xml index 9f856c935..4a959b289 100644 --- a/mcc/resources/etls/snprc.xml +++ b/mcc/resources/etls/snprc.xml @@ -23,6 +23,9 @@ + + + @@ -84,6 +87,9 @@ + + + @@ -102,6 +108,9 @@ + + + diff --git a/mcc/resources/etls/wnprc.xml b/mcc/resources/etls/wnprc.xml index 3147ab99d..f8efab248 100644 --- a/mcc/resources/etls/wnprc.xml +++ b/mcc/resources/etls/wnprc.xml @@ -18,7 +18,9 @@ - + + + @@ -36,7 +38,9 @@ - + + + @@ -56,7 +60,9 @@ - + + + @@ -72,9 +78,10 @@ - + + + - @@ -88,9 +95,10 @@ - + + + - From ed9495b51110637d18122d6c5ad6fb65cfd7ce47 Mon Sep 17 00:00:00 2001 From: bbimber Date: Thu, 11 Feb 2021 14:28:44 -0800 Subject: [PATCH 78/98] Add dummy data loading --- mcc/resources/web/mcc/dashboard.js | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/mcc/resources/web/mcc/dashboard.js b/mcc/resources/web/mcc/dashboard.js index e69de29bb..4c94d7e87 100644 --- a/mcc/resources/web/mcc/dashboard.js +++ b/mcc/resources/web/mcc/dashboard.js @@ -0,0 +1,18 @@ +var MCC = {}; + +MCC.Dashboard = new function() { + return { + loadData: function () { + LABKEY.Query.selectRows({ + schemaName: 'study', + queryName: 'demographics', + columns: 'Id,birth,death,gender,species,Id/age/AgeFriendly', + success: function(results) { + console.log(results.rows); + }, + error: LDK.Utils.getErrorCallback(), + scope: this + }); + } + } +}; From 098d19f1749a77c5aa7e0115a5708121490d9636 Mon Sep 17 00:00:00 2001 From: bbimber Date: Fri, 12 Feb 2021 06:18:34 -0800 Subject: [PATCH 79/98] Save metadata table from Seurat objects --- primeseq/resources/views/geneticsMenu.html | 6 +++--- primeseq/src/org/labkey/primeseq/PrimeseqController.java | 8 +------- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/primeseq/resources/views/geneticsMenu.html b/primeseq/resources/views/geneticsMenu.html index 3ff3df451..4b6308609 100644 --- a/primeseq/resources/views/geneticsMenu.html +++ b/primeseq/resources/views/geneticsMenu.html @@ -88,15 +88,15 @@ items: [{ title: 'Public Resources', itemId: 'public' - },{ - itemId: 'collaborations', - title: 'Collaborations' },{ title: 'Labs', itemId: 'labs' },{ itemId: 'internal', title: 'Internal Projects' + },{ + itemId: 'collaborations', + title: 'Bimber Lab Collaborations' }] }); diff --git a/primeseq/src/org/labkey/primeseq/PrimeseqController.java b/primeseq/src/org/labkey/primeseq/PrimeseqController.java index 66d1de408..de1835eab 100644 --- a/primeseq/src/org/labkey/primeseq/PrimeseqController.java +++ b/primeseq/src/org/labkey/primeseq/PrimeseqController.java @@ -71,7 +71,7 @@ public ApiResponse execute(Object form, BindException errors) { Map resultProperties = new HashMap<>(); - resultProperties.put("collaborations", getSection("/Public/Collaborations")); + resultProperties.put("collaborations", getSection("/Labs/Bimber/Collaborations")); resultProperties.put("internal", getSection("/Internal")); resultProperties.put("labs", getSection("/Labs")); @@ -128,12 +128,6 @@ private List getSection(String path) { for (Container c : mainContainer.getChildren()) { - //NOTE: unlike EHR, omit children if the current user cannot read them - if (!c.hasPermission(getUser(), ReadPermission.class)) - { - continue; - } - JSONObject json = new JSONObject(); json.put("name", c.getName()); json.put("title", c.getTitle()); From 8a6033bb6f57054bad1187042de4ffaead334e3f Mon Sep 17 00:00:00 2001 From: bbimber Date: Fri, 12 Feb 2021 06:50:34 -0800 Subject: [PATCH 80/98] Left align text --- primeseq/resources/views/geneticsMenu.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/primeseq/resources/views/geneticsMenu.html b/primeseq/resources/views/geneticsMenu.html index 4b6308609..5177cd413 100644 --- a/primeseq/resources/views/geneticsMenu.html +++ b/primeseq/resources/views/geneticsMenu.html @@ -44,7 +44,7 @@ 'data-qtip="You do not have permission to view this page"', 'style="width: 300px;height: auto;" class="thumb-wrap thumb-wrap-side">', '', - '{title:htmlEncode}', + '{title:htmlEncode}', '', '', '', From ed0f1d689f3de36cec5d80533a059cd698482591 Mon Sep 17 00:00:00 2001 From: bbimber Date: Thu, 18 Feb 2021 13:28:53 -0800 Subject: [PATCH 81/98] Use pipeline job's logger --- .../userRequests/Pending Requests.qview.xml | 8 +++ .../pipeline/MhcMigrationPipelineJob.java | 70 +++++++++---------- 2 files changed, 42 insertions(+), 36 deletions(-) create mode 100644 mcc/resources/queries/mcc/userRequests/Pending Requests.qview.xml diff --git a/mcc/resources/queries/mcc/userRequests/Pending Requests.qview.xml b/mcc/resources/queries/mcc/userRequests/Pending Requests.qview.xml new file mode 100644 index 000000000..b849a22a8 --- /dev/null +++ b/mcc/resources/queries/mcc/userRequests/Pending Requests.qview.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/primeseq/src/org/labkey/primeseq/pipeline/MhcMigrationPipelineJob.java b/primeseq/src/org/labkey/primeseq/pipeline/MhcMigrationPipelineJob.java index dfd668352..0539a3761 100644 --- a/primeseq/src/org/labkey/primeseq/pipeline/MhcMigrationPipelineJob.java +++ b/primeseq/src/org/labkey/primeseq/pipeline/MhcMigrationPipelineJob.java @@ -68,8 +68,6 @@ public class MhcMigrationPipelineJob extends PipelineJob { - private static final Logger _log = LogManager.getLogger(MhcMigrationPipelineJob.class); - private String remoteServerFolder; private String remoteConnectionName; @@ -123,7 +121,7 @@ public ActionURL getStatusHref() @Override public String getDescription() { - return "Find Orphan Sequence Files"; + return "Migrate MHC Data"; } @Override @@ -184,7 +182,7 @@ private MhcMigrationPipelineJob getPipelineJob() private Connection getConnection() { - DataIntegrationService.RemoteConnection rc = DataIntegrationService.get().getRemoteConnection(getPipelineJob().remoteConnectionName, getPipelineJob().targetContainer, _log); + DataIntegrationService.RemoteConnection rc = DataIntegrationService.get().getRemoteConnection(getPipelineJob().remoteConnectionName, getPipelineJob().targetContainer, getJob().getLogger()); return(rc.connection); } @@ -308,7 +306,7 @@ else if ("analysis_id".equals(colName)) private void createLibraryMembers() { - _log.info("Creating library members"); + getJob().getLogger().info("Creating library members"); final UserSchema us = QueryService.get().getUserSchema(getJob().getUser(), getPipelineJob().targetContainer, "sequenceanalysis"); final TableInfo ti = us.getTable("reference_library_members"); @@ -360,14 +358,14 @@ private void createLibraryMembers() } catch (Exception e) { - _log.error(e.getMessage(), e); + getJob().getLogger().error(e.getMessage(), e); throw new RuntimeException(e); } }); } catch (Exception e) { - _log.error(e.getMessage(), e); + getJob().getLogger().error(e.getMessage(), e); throw new RuntimeException(e); } } @@ -387,14 +385,14 @@ private int getOrCreateSequence(int remoteSeqId, String name, int seqLength, Tab { if (ts.getRowCount() > 1) { - _log.info("Duplicate ref name: " + name); + getJob().getLogger().info("Duplicate ref name: " + name); } AtomicInteger localId = new AtomicInteger(-1); ts.forEachResults(rs -> { if (rs.getInt(FieldKey.fromString("seqLength")) < seqLength) { - _log.warn("length doesnt match for " + name + ", expected: " + seqLength); + getJob().getLogger().warn("length doesnt match for " + name + ", expected: " + seqLength); return; } @@ -410,7 +408,7 @@ private int getOrCreateSequence(int remoteSeqId, String name, int seqLength, Tab //TODO: Create sequence? //throw new IllegalStateException("Expected sequence to exist: " + name); - _log.error("Sequence missing: " + name); + getJob().getLogger().error("Sequence missing: " + name); return -1; } } @@ -426,7 +424,7 @@ public String getParent(String path) private void createLibraries() { - _log.info("Creating libraries"); + getJob().getLogger().info("Creating libraries"); try { final TableInfo libraryTable = QueryService.get().getUserSchema(getJob().getUser(), getPipelineJob().targetContainer, "sequenceanalysis").getTable("reference_libraries"); @@ -469,8 +467,8 @@ private void createLibraries() localJobRootFile.getParentFile().mkdirs(); } - _log.info(remoteJobRoot); - _log.info(localJobRoot.getPath()); + getJob().getLogger().info(remoteJobRoot); + getJob().getLogger().info(localJobRoot.getPath()); File remoteJobRootFile = new File(remoteJobRoot); if (remoteJobRootFile.exists()) { @@ -495,14 +493,14 @@ private void createLibraries() } catch (Exception e) { - _log.error(e.getMessage(), e); + getJob().getLogger().error(e.getMessage(), e); throw new RuntimeException(e); } } private void createOutputFiles() { - _log.info("Creating outputfiles"); + getJob().getLogger().info("Creating outputfiles"); try { final TableInfo outputTable = QueryService.get().getUserSchema(getJob().getUser(), getPipelineJob().targetContainer, "sequenceanalysis").getTable("outputfiles"); @@ -580,7 +578,7 @@ private void createOutputFiles() } else { - _log.error("output missing runid: " + remoteId); + getJob().getLogger().error("output missing runid: " + remoteId); } BatchValidationException bve = new BatchValidationException(); @@ -601,14 +599,14 @@ private void createOutputFiles() } catch (Exception e) { - _log.error(e.getMessage(), e); + getJob().getLogger().error(e.getMessage(), e); throw new RuntimeException(e); } } private void createAnalyses() { - _log.info("Creating analyses"); + getJob().getLogger().info("Creating analyses"); try { final TableInfo analysisTable = QueryService.get().getUserSchema(getJob().getUser(), getPipelineJob().targetContainer, "sequenceanalysis").getTable("sequence_analyses"); @@ -622,7 +620,7 @@ private void createAnalyses() int remoteId = Integer.parseInt(String.valueOf(rd.getValue("rowid"))); if (rd.getValue("readset") == null) { - _log.warn("analysis lacks readset, skipping: " + remoteId); + getJob().getLogger().warn("analysis lacks readset, skipping: " + remoteId); return; } @@ -675,7 +673,7 @@ private void createAnalyses() { if (rd.getValue("runid/JobId") == null) { - _log.info("skipping analysis without runid: " + remoteId); + getJob().getLogger().info("skipping analysis without runid: " + remoteId); return; } @@ -703,7 +701,7 @@ private void createAnalyses() } else { - _log.error("analysis missing runid: " + remoteId); + getJob().getLogger().error("analysis missing runid: " + remoteId); } BatchValidationException bve = new BatchValidationException(); @@ -724,14 +722,14 @@ private void createAnalyses() } catch (Exception e) { - _log.error(e.getMessage(), e); + getJob().getLogger().error(e.getMessage(), e); throw new RuntimeException(e); } } private void createReaddata() { - _log.info("Creating read data"); + getJob().getLogger().info("Creating read data"); try { final TableInfo readdataTable = QueryService.get().getUserSchema(getJob().getUser(), getPipelineJob().targetContainer, "sequenceanalysis").getTable("readdata"); @@ -802,7 +800,7 @@ private void createReaddata() } else { - _log.error("readddata missing jobid: " + remoteId); + getJob().getLogger().error("readddata missing jobid: " + remoteId); } //Create run: @@ -815,7 +813,7 @@ private void createReaddata() } else { - _log.error("readddata missing runid: " + remoteId); + getJob().getLogger().error("readddata missing runid: " + remoteId); } BatchValidationException bve = new BatchValidationException(); @@ -836,7 +834,7 @@ private void createReaddata() } catch (Exception e) { - _log.error(e.getMessage(), e); + getJob().getLogger().error(e.getMessage(), e); throw new RuntimeException(e); } } @@ -856,7 +854,7 @@ private int getOrCreateExpData(URI file, Container workbook) private void createReadsets() { - _log.info("Creating readsets"); + getJob().getLogger().info("Creating readsets"); try { final UserSchema us = QueryService.get().getUserSchema(getJob().getUser(), getPipelineJob().targetContainer, "sequenceanalysis"); @@ -920,7 +918,7 @@ private void createReadsets() } else { - _log.error("readset missing run id: " + remoteId); + getJob().getLogger().error("readset missing run id: " + remoteId); } BatchValidationException bve = new BatchValidationException(); @@ -941,7 +939,7 @@ private void createReadsets() } catch (Exception e) { - _log.error(e.getMessage(), e); + getJob().getLogger().error(e.getMessage(), e); throw new RuntimeException(e); } } @@ -981,7 +979,7 @@ else if (filepath.contains("sequenceAnalysis")) } else { - _log.error("Unexpected filepath: " + pj.getValue("FilePath")); + getJob().getLogger().error("Unexpected filepath: " + pj.getValue("FilePath")); } } @@ -1015,14 +1013,14 @@ else if (filepath.contains("sequenceAnalysis")) if (localDir.exists()) { - _log.info("Directory exists, will not re-copy: " + localDir.getPath()); + getJob().getLogger().info("Directory exists, will not re-copy: " + localDir.getPath()); return; } try { - _log.info(remoteDir.getPath()); - _log.info(localDir.getPath()); + getJob().getLogger().info(remoteDir.getPath()); + getJob().getLogger().info(localDir.getPath()); if (!localDir.getParentFile().exists()) { @@ -1035,7 +1033,7 @@ else if (filepath.contains("sequenceAnalysis")) } else { - _log.error("source folder not found: " + remoteDir.getPath()); + getJob().getLogger().error("source folder not found: " + remoteDir.getPath()); } } catch (Exception e) @@ -1050,7 +1048,7 @@ else if (filepath.contains("sequenceAnalysis")) } catch (Exception e) { - _log.error(e.getMessage(), e); + getJob().getLogger().error(e.getMessage(), e); throw new RuntimeException(e); } } @@ -1072,7 +1070,7 @@ private int createExpRun(int remoteId, Container c, String name, int localJobId) private void createWorkbooks() { - _log.info("Creating workbooks"); + getJob().getLogger().info("Creating workbooks"); try { TableInfo containers = QueryService.get().getUserSchema(getJob().getUser(), getPipelineJob().targetContainer, "core").getTable("containers"); From 654076f74d8caf8968a3a5cc6bdcda65aa976c3d Mon Sep 17 00:00:00 2001 From: bbimber Date: Thu, 18 Feb 2021 15:10:20 -0800 Subject: [PATCH 82/98] allow greater decimal precision for filtering --- .../pipeline/MhcMigrationPipelineJob.java | 109 ++++++++++++++++-- 1 file changed, 99 insertions(+), 10 deletions(-) diff --git a/primeseq/src/org/labkey/primeseq/pipeline/MhcMigrationPipelineJob.java b/primeseq/src/org/labkey/primeseq/pipeline/MhcMigrationPipelineJob.java index 0539a3761..fe6007851 100644 --- a/primeseq/src/org/labkey/primeseq/pipeline/MhcMigrationPipelineJob.java +++ b/primeseq/src/org/labkey/primeseq/pipeline/MhcMigrationPipelineJob.java @@ -1,8 +1,6 @@ package org.labkey.primeseq.pipeline; import org.apache.commons.io.FileUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; import org.labkey.api.collections.CaseInsensitiveHashMap; import org.labkey.api.data.CompareType; import org.labkey.api.data.Container; @@ -10,6 +8,7 @@ import org.labkey.api.data.DbSchema; import org.labkey.api.data.DbSchemaType; import org.labkey.api.data.DbScope; +import org.labkey.api.data.Results; import org.labkey.api.data.SimpleFilter; import org.labkey.api.data.Sort; import org.labkey.api.data.Table; @@ -58,6 +57,7 @@ import java.io.File; import java.io.IOException; import java.net.URI; +import java.sql.SQLException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -205,9 +205,11 @@ public RecordedActionSet run() throws PipelineJobException createAnalyses(); createOutputFiles(); + createAlignmentSummary(); + //TODO: //samples - //alignment_summary + //alignment_summary_junction //quality_metrics //subjects @@ -225,6 +227,75 @@ public RecordedActionSet run() throws PipelineJobException return new RecordedActionSet(); } + private void createAlignmentSummary() throws PipelineJobException + { + try + { + TableInfo alignmentSummary = DbSchema.get("sequenceanalysis", DbSchemaType.Module).getTable("alignment_summary"); + TableInfo alignmentSummaryJunction = DbSchema.get("sequenceanalysis", DbSchemaType.Module).getTable("alignment_summary_junction"); + + SelectRowsCommand sr = new SelectRowsCommand("sequenceanalysis", "alignment_summary"); + sr.setColumns(Arrays.asList("rowid", "analysis_id", "file_id", "total", "total_forward", "total_reverse", "valid_pairs", "workbook/workbookId")); + + SelectRowsResponse srr = sr.execute(getConnection(), getPipelineJob().remoteServerFolder); + + Map alignmentSummaryMap = new HashMap<>(); + srr.getRowset().forEach(rs -> { + CaseInsensitiveHashMap map = new CaseInsensitiveHashMap<>(); + Integer localId = analysisMap.get(rs.getValue("analysis_id")); + if (localId == null) + { + throw new RuntimeException("Unable to find analysis: " + rs.getValue("analysis_id")); + } + map.put("analysis_id", localId); + map.put("file_id", analysisToFileMap.get(localId)); + + map.put("total", rs.getValue("total")); + map.put("total_forward", rs.getValue("total_forward")); + map.put("total_reverse", rs.getValue("total_reverse")); + map.put("valid_pairs", rs.getValue("valid_pairs")); + + Container c = workbookMap.get((int)rs.getValue("workbook/workbookId")); + map.put("container", c.getId()); + + map = Table.insert(getJob().getUser(), alignmentSummary, map); + alignmentSummaryMap.put((int)rs.getValue("rowid"), (int)map.get("rowid")); + }); + + SelectRowsCommand sr2 = new SelectRowsCommand("sequenceanalysis", "alignment_summary_junction"); + sr2.setColumns(Arrays.asList("analysis_id", "alignment_id", "ref_nt_id", "analysis_id/workbook/workbookId")); + + SelectRowsResponse srr2 = sr2.execute(getConnection(), getPipelineJob().remoteServerFolder); + srr2.getRowset().forEach(rs -> { + CaseInsensitiveHashMap map = new CaseInsensitiveHashMap<>(); + Integer localId = analysisMap.get(rs.getValue("analysis_id")); + if (localId == null) + { + throw new RuntimeException("Unable to find analysis: " + rs.getValue("analysis_id")); + } + map.put("analysis_id", localId); + + Integer localNT = sequenceMap.get(rs.getValue("ref_nt_id")); + if (localNT == null) + { + throw new RuntimeException("Unable to find ref_nt_id: " + rs.getValue("ref_nt_id")); + } + map.put("ref_nt_id", localNT); + + map.put("alignment_id", alignmentSummaryMap.get("alignment_id")); + + Container c = workbookMap.get((int)rs.getValue("analysis_id/workbook/workbookId")); + map.put("container", c.getId()); + + map = Table.insert(getJob().getUser(), alignmentSummaryJunction, map); + }); + } + catch (CommandException | IOException e) + { + throw new PipelineJobException(e); + } + } + private void replaceEntireTable(String schema, String query, List columns, String workbookColName, boolean truncateExisting) throws Exception { SelectRowsCommand sr = new SelectRowsCommand(schema, query); @@ -298,6 +369,7 @@ else if ("analysis_id".equals(colName)) private final Map readsetMap = new HashMap<>(); private final Map readdataMap = new HashMap<>(); private final Map analysisMap = new HashMap<>(); + private final Map analysisToFileMap = new HashMap<>(); //local analysis_id -> alignment file private final Map libraryMap = new HashMap<>(); private final Map outputFileMap = new HashMap<>(); private final Map sequenceMap = new HashMap<>(); @@ -392,7 +464,7 @@ private int getOrCreateSequence(int remoteSeqId, String name, int seqLength, Tab ts.forEachResults(rs -> { if (rs.getInt(FieldKey.fromString("seqLength")) < seqLength) { - getJob().getLogger().warn("length doesnt match for " + name + ", expected: " + seqLength); + getJob().getLogger().warn("length doesnt match for " + name + ", expected: " + seqLength + ", was: " + rs.getInt(FieldKey.fromString("seqLength"))); return; } @@ -526,11 +598,19 @@ private void createOutputFiles() throw new IllegalArgumentException("Unable to find genome for remote id: " + remoteLibrary); } - int remoteAnalysis = Integer.parseInt(String.valueOf(rd.getValue("analysis_id"))); - Integer localAnalysis = analysisMap.get(remoteAnalysis); - if (localAnalysis == null) + Integer localAnalysis; + if (rd.getValue("analysis_id") != null) { - throw new IllegalArgumentException("Unable to find analysis for remote id: " + remoteAnalysis); + int remoteAnalysis = Integer.parseInt(String.valueOf(rd.getValue("analysis_id"))); + localAnalysis = analysisMap.get(remoteAnalysis); + if (localAnalysis == null) + { + throw new IllegalArgumentException("Unable to find analysis for remote id: " + remoteAnalysis); + } + } + else + { + localAnalysis = null; } Readset rs = SequenceAnalysisService.get().getReadset(localReadset, getJob().getUser()); @@ -649,10 +729,19 @@ private void createAnalyses() filter.addCondition(FieldKey.fromString("runid/JobId/Description"), rd.getValue("runid/JobId/Description")); filter.addCondition(FieldKey.fromString("container"), targetWorkbook.getId(), CompareType.EQUAL); - TableSelector tsAnalyses = new TableSelector(analysisTable, PageFlowUtil.set("rowid"), filter, null); + TableSelector tsAnalyses = new TableSelector(analysisTable, PageFlowUtil.set("rowid", "alignmentfile"), filter, null); if (tsAnalyses.exists()) { - analysisMap.put(remoteId, tsAnalyses.getObject(Integer.class)); + Results results = tsAnalyses.getResults(); + try + { + analysisMap.put(remoteId, results.getInt("rowid")); + analysisToFileMap.put(results.getInt("rowid"), results.getInt("alignmentfile")); + } + catch (SQLException e) + { + throw new RuntimeException(e); + } } else { From b83db1cfa9d1bddd0af7261f8571b5ec407164e5 Mon Sep 17 00:00:00 2001 From: bbimber Date: Fri, 19 Feb 2021 12:20:55 -0800 Subject: [PATCH 83/98] Improve reporting for GenomicsDBImport --- .../labkey/primeseq/pipeline/MhcMigrationPipelineJob.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/primeseq/src/org/labkey/primeseq/pipeline/MhcMigrationPipelineJob.java b/primeseq/src/org/labkey/primeseq/pipeline/MhcMigrationPipelineJob.java index fe6007851..c979ea32e 100644 --- a/primeseq/src/org/labkey/primeseq/pipeline/MhcMigrationPipelineJob.java +++ b/primeseq/src/org/labkey/primeseq/pipeline/MhcMigrationPipelineJob.java @@ -640,6 +640,11 @@ private void createOutputFiles() try { + if (rd.getValue("runid/JobId") == null) + { + throw new PipelineJobException("Output missing runId"); + } + int remoteJobId = Integer.parseInt(String.valueOf(rd.getValue("runid/JobId"))); int jobId = getOrCreateJob(remoteJobId, targetWorkbook); PipelineStatusFile sf = PipelineService.get().getStatusFile(jobId); From cc57cc656a561ea6df61db063eb53b2714093b51 Mon Sep 17 00:00:00 2001 From: bbimber Date: Fri, 19 Feb 2021 12:55:33 -0800 Subject: [PATCH 84/98] Update Seurat/subset step --- .../labkey/primeseq/pipeline/MhcMigrationPipelineJob.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/primeseq/src/org/labkey/primeseq/pipeline/MhcMigrationPipelineJob.java b/primeseq/src/org/labkey/primeseq/pipeline/MhcMigrationPipelineJob.java index c979ea32e..b535c27d5 100644 --- a/primeseq/src/org/labkey/primeseq/pipeline/MhcMigrationPipelineJob.java +++ b/primeseq/src/org/labkey/primeseq/pipeline/MhcMigrationPipelineJob.java @@ -397,6 +397,13 @@ private void createLibraryMembers() int remoteSeqId = Integer.parseInt(String.valueOf(rd.getValue("ref_nt_id"))); String name = String.valueOf(rd.getValue("ref_nt_id/name")); + + //Skip all pigtail MHC. + if (name.startsWith("Mane")) + { + return; + } + int localSeqId = getOrCreateSequence(remoteSeqId, name, seqLength, refNtTable); int remoteLibraryId = Integer.parseInt(String.valueOf(rd.getValue("library_id"))); From ee708049d0d442c5030c7899fb60be12e3b68a54 Mon Sep 17 00:00:00 2001 From: bbimber Date: Wed, 3 Mar 2021 09:30:27 -0800 Subject: [PATCH 85/98] Update dashboard example --- mcc/resources/views/dashboard.html | 3 +++ mcc/resources/views/dashboard.view.xml | 2 ++ 2 files changed, 5 insertions(+) diff --git a/mcc/resources/views/dashboard.html b/mcc/resources/views/dashboard.html index 014a425a7..a56ee8d76 100644 --- a/mcc/resources/views/dashboard.html +++ b/mcc/resources/views/dashboard.html @@ -6,4 +6,7 @@ // This is an automatically generated DIV where we should output content: document.getElementById(webpart.wrapperDivId).innerHTML = 'Hello!'; + // This is just a dummy example to show the query API. It should use some kind of onReady() function. See which frameworks OCTRI wants to use. + MCC.Dashboard.loadData(); + \ No newline at end of file diff --git a/mcc/resources/views/dashboard.view.xml b/mcc/resources/views/dashboard.view.xml index 252242cd6..6779a9525 100644 --- a/mcc/resources/views/dashboard.view.xml +++ b/mcc/resources/views/dashboard.view.xml @@ -3,6 +3,8 @@ + + From 40980315b392acfd6363b17a7bcfe8609f9d0308 Mon Sep 17 00:00:00 2001 From: bbimber Date: Wed, 3 Mar 2021 12:34:36 -0800 Subject: [PATCH 86/98] Bugfixed to hashing/VDJ calls --- mcc/resources/etls/snprc.xml | 8 +++---- mcc/resources/views/dashboard.html | 23 +++++++++++++++++-- mcc/resources/web/mcc/dashboard.js | 6 +++-- .../CellRangerVDJCellHashingHandler.java | 2 +- .../tcrdb/pipeline/CellRangerVDJUtils.java | 18 +++++++++++++-- 5 files changed, 46 insertions(+), 11 deletions(-) diff --git a/mcc/resources/etls/snprc.xml b/mcc/resources/etls/snprc.xml index 4a959b289..f61f1b9bc 100644 --- a/mcc/resources/etls/snprc.xml +++ b/mcc/resources/etls/snprc.xml @@ -19,7 +19,7 @@ objectid - + @@ -64,7 +64,7 @@ - + @@ -83,7 +83,7 @@ objectid - + @@ -115,7 +115,7 @@ - + diff --git a/mcc/resources/views/dashboard.html b/mcc/resources/views/dashboard.html index a56ee8d76..a50a225e3 100644 --- a/mcc/resources/views/dashboard.html +++ b/mcc/resources/views/dashboard.html @@ -1,3 +1,6 @@ + + + \ No newline at end of file diff --git a/mcc/resources/web/mcc/dashboard.js b/mcc/resources/web/mcc/dashboard.js index 4c94d7e87..1a387e0c7 100644 --- a/mcc/resources/web/mcc/dashboard.js +++ b/mcc/resources/web/mcc/dashboard.js @@ -2,7 +2,7 @@ var MCC = {}; MCC.Dashboard = new function() { return { - loadData: function () { + loadDataAndRender: function (wrapperDivId) { LABKEY.Query.selectRows({ schemaName: 'study', queryName: 'demographics', @@ -10,7 +10,9 @@ MCC.Dashboard = new function() { success: function(results) { console.log(results.rows); }, - error: LDK.Utils.getErrorCallback(), + failure: function(response) { + alert('It didnt work!'); + }, scope: this }); } diff --git a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java index 26b1e01fa..cab9554ae 100644 --- a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java +++ b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java @@ -114,7 +114,7 @@ public class Processor implements SequenceOutputHandler.SequenceOutputProcessor public void init(JobContext ctx, List inputFiles, List actions, List outputsToCreate) throws UnsupportedOperationException, PipelineJobException { //NOTE: this is the pathway to import assay data, whether hashing is used or not - CellHashingService.get().prepareHashingAndCiteSeqFilesIfNeeded(ctx.getOutputDir(), ctx.getJob(), ctx.getSequenceSupport(), "tcrReadsetId", false, false); + CellHashingService.get().prepareHashingIfNeeded(ctx.getOutputDir(), ctx.getJob(), ctx.getSequenceSupport(), "tcrReadsetId", false); if (ctx.getParams().optBoolean(USE_GEX_BARCODES, false)) { diff --git a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJUtils.java b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJUtils.java index b52974def..28a5eee9c 100644 --- a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJUtils.java +++ b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJUtils.java @@ -196,15 +196,29 @@ public void importAssayData(PipelineJob job, AnalysisModel model, File vLoupeFil int doublet = 0; int discordant = 0; int negative = 0; + + int consensusIdx = -1; + while ((line = reader.readNext()) != null) { + if (line.length == 0) + { + throw new PipelineJobException("Line was empty"); + } + //header - if ("CellBarcode".equals(line[0])) + if ("cellbarcode".equalsIgnoreCase(line[0])) { + consensusIdx = Arrays.asList(line).indexOf("consensuscall"); + if (consensusIdx == -1) + { + throw new PipelineJobException("consensuscall column not found"); + } + continue; } - String hto = line[1]; + String hto = line[consensusIdx]; if ("Doublet".equals(hto)) { doublet++; From 2bf3af2135308b88320c053b8cdc041599c44a5b Mon Sep 17 00:00:00 2001 From: bbimber Date: Wed, 3 Mar 2021 13:17:25 -0800 Subject: [PATCH 87/98] Improved validation --- .../labkey/tcrdb/pipeline/CellRangerVDJUtils.java | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJUtils.java b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJUtils.java index 28a5eee9c..109117429 100644 --- a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJUtils.java +++ b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJUtils.java @@ -198,26 +198,25 @@ public void importAssayData(PipelineJob job, AnalysisModel model, File vLoupeFil int negative = 0; int consensusIdx = -1; - while ((line = reader.readNext()) != null) { - if (line.length == 0) + if (line.length < 3) { - throw new PipelineJobException("Line was empty"); + throw new PipelineJobException("Line too short"); } //header if ("cellbarcode".equalsIgnoreCase(line[0])) { consensusIdx = Arrays.asList(line).indexOf("consensuscall"); - if (consensusIdx == -1) - { - throw new PipelineJobException("consensuscall column not found"); - } - continue; } + if (consensusIdx == -1) + { + throw new PipelineJobException("consensuscall column not found"); + } + String hto = line[consensusIdx]; if ("Doublet".equals(hto)) { From 7171cc88cfd155460adffd988c9c379929fcfc46 Mon Sep 17 00:00:00 2001 From: bbimber Date: Sun, 7 Mar 2021 11:52:23 -0800 Subject: [PATCH 88/98] More permissive regex for Illumina import --- .../pipeline/MhcMigrationPipelineJob.java | 338 ++++++++++++------ 1 file changed, 219 insertions(+), 119 deletions(-) diff --git a/primeseq/src/org/labkey/primeseq/pipeline/MhcMigrationPipelineJob.java b/primeseq/src/org/labkey/primeseq/pipeline/MhcMigrationPipelineJob.java index b535c27d5..e3e1acb21 100644 --- a/primeseq/src/org/labkey/primeseq/pipeline/MhcMigrationPipelineJob.java +++ b/primeseq/src/org/labkey/primeseq/pipeline/MhcMigrationPipelineJob.java @@ -62,9 +62,11 @@ import java.util.Arrays; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; public class MhcMigrationPipelineJob extends PipelineJob { @@ -194,6 +196,9 @@ public RecordedActionSet run() throws PipelineJobException { createWorkbooks(); + replaceEntireTable("laboratory", "samples", Arrays.asList("samplename", "subjectid", "sampledate", "sampletype", "samplesubtype", "samplesource", "location", "freezer", "cane", "box", "box_row", "box_column", "comment", "workbook/workbookId", "samplespecies", "processdate", "concentration", "concentration_units", "quantity", "quantity_units", "ratio"), "workbook/workbookId", true, getJob().getContainer(), getPipelineJob().remoteServerFolder); + replaceEntireTable("laboratory", "subjects", Arrays.asList("subjectname", "species"), null, true, getJob().getContainer(), getPipelineJob().remoteServerFolder); + createLibraries(); createLibraryMembers(); @@ -207,19 +212,9 @@ public RecordedActionSet run() throws PipelineJobException createAlignmentSummary(); - //TODO: - //samples - - //alignment_summary_junction - //quality_metrics - //subjects - //WaNPRC + createQualityMetrics(); - //sequenceanalysis.haplotypes - //sequenceanalysis.haplotype_types - //sequenceanalysis.haplotype_sequences - - //Create assay runs, including data and haplotypes + //TODO: create assay runs, including data and haplotypes transaction.commit(); } @@ -227,6 +222,14 @@ public RecordedActionSet run() throws PipelineJobException return new RecordedActionSet(); } + private void createQualityMetrics() throws PipelineJobException + { + for (int workbook : workbookMap.keySet()) + { + replaceEntireTable("sequenceanalysis", "quanlity_metrics", Arrays.asList("dataid", "dataid/DatafileUrl", "category", "metricname", "metricvalue", "qualvalue", "analysis_id", "readset"), null, true, workbookMap.get(workbook), getPipelineJob().remoteServerFolder + workbook + "/"); + } + } + private void createAlignmentSummary() throws PipelineJobException { try @@ -234,61 +237,77 @@ private void createAlignmentSummary() throws PipelineJobException TableInfo alignmentSummary = DbSchema.get("sequenceanalysis", DbSchemaType.Module).getTable("alignment_summary"); TableInfo alignmentSummaryJunction = DbSchema.get("sequenceanalysis", DbSchemaType.Module).getTable("alignment_summary_junction"); - SelectRowsCommand sr = new SelectRowsCommand("sequenceanalysis", "alignment_summary"); - sr.setColumns(Arrays.asList("rowid", "analysis_id", "file_id", "total", "total_forward", "total_reverse", "valid_pairs", "workbook/workbookId")); + //NOTE: split by workbook to avoid huge API calls: + SelectRowsCommand srWB = new SelectRowsCommand("core", "workbooks"); + srWB.setColumns(Arrays.asList("workbookId")); - SelectRowsResponse srr = sr.execute(getConnection(), getPipelineJob().remoteServerFolder); + SelectRowsResponse srrWB = srWB.execute(getConnection(), getPipelineJob().remoteServerFolder); + List workbooks = srrWB.getRows().stream().map(x -> x.get("workbookId")).collect(Collectors.toList()); + for (Object workbook : workbooks) + { + getJob().getLogger().info("importing alignments for workbook: " + workbook); - Map alignmentSummaryMap = new HashMap<>(); - srr.getRowset().forEach(rs -> { - CaseInsensitiveHashMap map = new CaseInsensitiveHashMap<>(); - Integer localId = analysisMap.get(rs.getValue("analysis_id")); - if (localId == null) - { - throw new RuntimeException("Unable to find analysis: " + rs.getValue("analysis_id")); - } - map.put("analysis_id", localId); - map.put("file_id", analysisToFileMap.get(localId)); + SelectRowsCommand sr = new SelectRowsCommand("sequenceanalysis", "alignment_summary"); + sr.setColumns(Arrays.asList("rowid", "analysis_id", "file_id", "total", "total_forward", "total_reverse", "valid_pairs", "workbook/workbookId")); - map.put("total", rs.getValue("total")); - map.put("total_forward", rs.getValue("total_forward")); - map.put("total_reverse", rs.getValue("total_reverse")); - map.put("valid_pairs", rs.getValue("valid_pairs")); + SelectRowsResponse srr = sr.execute(getConnection(), getPipelineJob().remoteServerFolder + workbook + "/"); - Container c = workbookMap.get((int)rs.getValue("workbook/workbookId")); - map.put("container", c.getId()); + Map alignmentSummaryMap = new HashMap<>(); + srr.getRowset().forEach(rs -> { + CaseInsensitiveHashMap map = new CaseInsensitiveHashMap<>(); + Integer localId = analysisMap.get(rs.getValue("analysis_id")); + if (localId == null) + { + throw new RuntimeException("Unable to find analysis: " + rs.getValue("analysis_id")); + } + map.put("analysis_id", localId); + map.put("file_id", analysisToFileMap.get(localId)); - map = Table.insert(getJob().getUser(), alignmentSummary, map); - alignmentSummaryMap.put((int)rs.getValue("rowid"), (int)map.get("rowid")); - }); + map.put("total", rs.getValue("total")); + map.put("total_forward", rs.getValue("total_forward")); + map.put("total_reverse", rs.getValue("total_reverse")); + map.put("valid_pairs", rs.getValue("valid_pairs")); - SelectRowsCommand sr2 = new SelectRowsCommand("sequenceanalysis", "alignment_summary_junction"); - sr2.setColumns(Arrays.asList("analysis_id", "alignment_id", "ref_nt_id", "analysis_id/workbook/workbookId")); + Container c = workbookMap.get((int) rs.getValue("workbook/workbookId")); + map.put("container", c.getId()); - SelectRowsResponse srr2 = sr2.execute(getConnection(), getPipelineJob().remoteServerFolder); - srr2.getRowset().forEach(rs -> { - CaseInsensitiveHashMap map = new CaseInsensitiveHashMap<>(); - Integer localId = analysisMap.get(rs.getValue("analysis_id")); - if (localId == null) - { - throw new RuntimeException("Unable to find analysis: " + rs.getValue("analysis_id")); - } - map.put("analysis_id", localId); + map = Table.insert(getJob().getUser(), alignmentSummary, map); + if (map.get("rowid") == null) + { + throw new RuntimeException("RowId was null after insert!"); + } - Integer localNT = sequenceMap.get(rs.getValue("ref_nt_id")); - if (localNT == null) - { - throw new RuntimeException("Unable to find ref_nt_id: " + rs.getValue("ref_nt_id")); - } - map.put("ref_nt_id", localNT); + alignmentSummaryMap.put((int) rs.getValue("rowid"), (int) map.get("rowid")); + }); - map.put("alignment_id", alignmentSummaryMap.get("alignment_id")); + SelectRowsCommand sr2 = new SelectRowsCommand("sequenceanalysis", "alignment_summary_junction"); + sr2.setColumns(Arrays.asList("analysis_id", "alignment_id", "ref_nt_id", "analysis_id/workbook/workbookId")); - Container c = workbookMap.get((int)rs.getValue("analysis_id/workbook/workbookId")); - map.put("container", c.getId()); + SelectRowsResponse srr2 = sr2.execute(getConnection(), getPipelineJob().remoteServerFolder + workbook + "/"); + srr2.getRowset().forEach(rs -> { + CaseInsensitiveHashMap map = new CaseInsensitiveHashMap<>(); + Integer localId = analysisMap.get(rs.getValue("analysis_id")); + if (localId == null) + { + throw new RuntimeException("Unable to find analysis: " + rs.getValue("analysis_id")); + } + map.put("analysis_id", localId); - map = Table.insert(getJob().getUser(), alignmentSummaryJunction, map); - }); + Integer localNT = sequenceMap.get(rs.getValue("ref_nt_id")); + if (localNT == null) + { + throw new RuntimeException("Unable to find ref_nt_id: " + rs.getValue("ref_nt_id")); + } + map.put("ref_nt_id", localNT); + + map.put("alignment_id", alignmentSummaryMap.get("alignment_id")); + + Container c = workbookMap.get((int) rs.getValue("analysis_id/workbook/workbookId")); + map.put("container", c.getId()); + + Table.insert(getJob().getUser(), alignmentSummaryJunction, map); + }); + } } catch (CommandException | IOException e) { @@ -296,72 +315,119 @@ private void createAlignmentSummary() throws PipelineJobException } } - private void replaceEntireTable(String schema, String query, List columns, String workbookColName, boolean truncateExisting) throws Exception + private void replaceEntireTable(String schema, String query, List columns, String workbookColName, boolean truncateExisting, Container targetContainer, String remoteServerFolder) throws PipelineJobException { - SelectRowsCommand sr = new SelectRowsCommand(schema, query); - sr.setColumns(columns); - SelectRowsResponse srr = sr.execute(getConnection(), getPipelineJob().remoteServerFolder); - - List> toInsert = new ArrayList<>(); - srr.getRowset().forEach(r -> { - Map row = new CaseInsensitiveHashMap<>(); - srr.getColumnModel().forEach(col -> { - String colName = (String) col.get("Name"); - Object val = r.getValue(colName); - if ("readset".equals(colName) || "readsetid".equals(colName)) - { - if (!readsetMap.containsKey((int) val)) - { - throw new IllegalStateException("Unable to find readset: " + val); - } + getJob().getLogger().info("replacing table: " + query + " for container: " + targetContainer.getPath()); + try + { + SelectRowsCommand sr = new SelectRowsCommand(schema, query); + sr.setColumns(columns); + SelectRowsResponse srr = sr.execute(getConnection(), remoteServerFolder); - val = readsetMap.get((int) val); - } - else if ("library_id".equals(colName)) - { - if (!libraryMap.containsKey((int) val)) + TableInfo ti = QueryService.get().getUserSchema(getJob().getUser(), targetContainer, schema).getTable(query); + long existing = new TableSelector(ti).getRowCount(); + if (srr.getRowCount().longValue() == existing) + { + getJob().getLogger().info("Row counts identical, assuming has been synced: " + query); + } + + List> toInsert = new ArrayList<>(); + srr.getRowset().forEach(r -> { + Map row = new CaseInsensitiveHashMap<>(); + srr.getColumnModel().forEach(col -> { + String colName = (String) col.get("dataIndex"); + if (ti.getColumn(colName) != null) { - throw new IllegalStateException("Unable to find library: " + val); - } + Object val = r.getValue(colName); + if ("readset".equals(colName) || "readsetid".equals(colName)) + { + if (!readsetMap.containsKey((int) val)) + { + throw new IllegalStateException("Unable to find readset: " + val); + } - val = libraryMap.get((int) val); + val = readsetMap.get((int) val); + } + else if ("library_id".equals(colName)) + { + if (!libraryMap.containsKey((int) val)) + { + throw new IllegalStateException("Unable to find library: " + val); + } - } - else if ("ref_nt_id".equals(colName)) - { - if (!sequenceMap.containsKey((int) val)) + val = libraryMap.get((int) val); + + } + else if ("ref_nt_id".equals(colName)) + { + if (!sequenceMap.containsKey((int) val)) + { + throw new IllegalStateException("Unable to find sequence: " + val); + } + + val = sequenceMap.get((int) val); + } + else if ("analysis_id".equals(colName)) + { + if (!analysisMap.containsKey((int) val)) + { + throw new IllegalStateException("Unable to find analysis: " + val); + } + + val = analysisMap.get((int) val); + } + + row.put(colName, val); + } + else if ("dataid/DatafileUrl".equalsIgnoreCase(colName)) { - throw new IllegalStateException("Unable to find sequence: " + val); + String remoteJobRoot = getParent(URI.create(String.valueOf(r.getValue("dataid/FilePath")).replaceAll(" ", "_")).getPath()); + + URI localFileRoot = PipelineService.get().getPipelineRootSetting(targetContainer).getRootPath().toURI(); + URI localPath = translateURI(String.valueOf(r.getValue("dataid/DatafileUrl")), remoteJobRoot, localFileRoot.getPath()); + row.put("dataid", getOrCreateExpData(localPath, targetContainer)); } + }); - val = sequenceMap.get((int) val); - } - else if ("analysis_id".equals(colName)) + if (workbookColName != null) { - if (!analysisMap.containsKey((int) val)) + Object workbookId = r.getValue(workbookColName); + if (workbookId != null) { - throw new IllegalStateException("Unable to find analysis: " + val); + row.put("container", workbookMap.get(Integer.parseInt(String.valueOf(workbookId))).getId()); } - - val = analysisMap.get((int) val); } - row.put(colName, val); + toInsert.add(row); }); - if (workbookColName != null) + if (truncateExisting) { - Object workbookId = r.getValue(workbookColName); - if (workbookId != null) + List toDelete = new TableSelector(ti, new HashSet<>(ti.getPkColumnNames())).getArrayList(Object.class); + if (!toDelete.isEmpty()) { - row.put("container", workbookMap.get(Integer.parseInt(String.valueOf(workbookId))).getId()); + final List> rowsToDelete = new ArrayList<>(); + toDelete.forEach(x -> { + Map map = new CaseInsensitiveHashMap<>(); + map.put(ti.getPkColumnNames().get(0), x); + rowsToDelete.add(map); + }); + + ti.getUpdateService().deleteRows(getJob().getUser(), targetContainer, rowsToDelete, null, null); } } - toInsert.add(row); - }); - - + BatchValidationException bve = new BatchValidationException(); + ti.getUpdateService().insertRows(getJob().getUser(), targetContainer, toInsert, bve, null, null); + if (bve.hasErrors()) + { + throw bve; + } + } + catch (Exception e) + { + throw new PipelineJobException(e); + } } //All of these map remote Id to local Id @@ -551,7 +617,10 @@ private void createLibraries() File remoteJobRootFile = new File(remoteJobRoot); if (remoteJobRootFile.exists()) { - FileUtils.copyDirectory(remoteJobRootFile, localJobRootFile); + if (!localJobRootFile.exists()) + { + throw new PipelineJobException("Expected folder to have been copied: " + remoteJobRootFile.getPath() + " to " + localJobRootFile.getPath()); + } } BatchValidationException bve = new BatchValidationException(); @@ -1095,21 +1164,29 @@ else if (filepath.contains("sequenceAnalysis")) } else { - Map toCreate = new CaseInsensitiveHashMap<>(); - toCreate.put("Info", pj.getValue("Info")); - toCreate.put("FilePath", localDir.getPath()); - toCreate.put("Email", pj.getValue("Email")); - toCreate.put("Description", pj.getValue("Description")); - toCreate.put("DataUrl", pj.getValue("DataUrl")); - toCreate.put("Job", pj.getValue("Job")); - toCreate.put("Provider", pj.getValue("Provider")); - toCreate.put("HadError", pj.getValue("HadError")); - toCreate.put("ActiveTaskId", pj.getValue("ActiveTaskId")); - toCreate.put("Container", targetWorkbook.getId()); - - toCreate = Table.insert(getJob().getUser(), ti, toCreate); - - ret.set((int) toCreate.get("RowId")); + ts = new TableSelector(ti, PageFlowUtil.set("RowId"), new SimpleFilter(FieldKey.fromString("FilePath"), localDir.getPath()), null); + if (ts.exists()) + { + ret.set(ts.getObject(Integer.class)); + } + else + { + Map toCreate = new CaseInsensitiveHashMap<>(); + toCreate.put("Info", pj.getValue("Info")); + toCreate.put("FilePath", localDir.getPath()); + toCreate.put("Email", pj.getValue("Email")); + toCreate.put("Description", pj.getValue("Description")); + toCreate.put("DataUrl", pj.getValue("DataUrl")); + toCreate.put("Job", pj.getValue("Job")); + toCreate.put("Provider", pj.getValue("Provider")); + toCreate.put("HadError", pj.getValue("HadError")); + toCreate.put("ActiveTaskId", pj.getValue("ActiveTaskId")); + toCreate.put("Container", targetWorkbook.getId()); + + toCreate = Table.insert(getJob().getUser(), ti, toCreate); + + ret.set((int) toCreate.get("RowId")); + } } if (localDir.exists()) @@ -1130,7 +1207,10 @@ else if (filepath.contains("sequenceAnalysis")) if (remoteDir.exists()) { - FileUtils.copyDirectory(remoteDir, localDir); + if (!localDir.exists()) + { + throw new PipelineJobException("Expected folder to have been copied: " + remoteDir.getPath() + " to " + localDir.getPath()); + } } else { @@ -1191,6 +1271,8 @@ private void createWorkbooks() } else { + PipeRoot pr = PipelineService.get().getPipelineRootSetting(getJob().getContainer()); + String description = String.valueOf(wb.getValue("Description")); if (description != null) { @@ -1205,6 +1287,24 @@ private void createWorkbooks() Container workbook = ContainerManager.createContainer(getPipelineJob().targetContainer, null, localTitle, description, WorkbookContainerType.NAME, getJob().getUser()); workbookMap.put(Integer.parseInt(String.valueOf(wb.getValue("Name"))), workbook); + + File sourceDir = new File("/home/groups/miSeqLK/Production/MHC_Typing", wb.getValue("Name") + "/@files"); + File targetDir = pr.getRootPath(); + if (sourceDir.exists()) + { + try + { + FileUtils.copyDirectory(sourceDir, targetDir); + } + catch (Exception e) + { + throw new RuntimeException(e); + } + } + else + { + getJob().getLogger().error("source folder not found: " + sourceDir.getPath()); + } } }); } From 88f813a4f5d7afb71a33eb6e5fa5fcdd0a2eb5f8 Mon Sep 17 00:00:00 2001 From: bbimber Date: Sun, 7 Mar 2021 17:37:50 -0800 Subject: [PATCH 89/98] Add action to bulk update filepaths --- .../labkey/primeseq/PrimeseqController.java | 194 ++++++++++++++++++ 1 file changed, 194 insertions(+) diff --git a/primeseq/src/org/labkey/primeseq/PrimeseqController.java b/primeseq/src/org/labkey/primeseq/PrimeseqController.java index de1835eab..ee6e01d94 100644 --- a/primeseq/src/org/labkey/primeseq/PrimeseqController.java +++ b/primeseq/src/org/labkey/primeseq/PrimeseqController.java @@ -28,6 +28,9 @@ import org.labkey.api.data.Container; import org.labkey.api.data.ContainerManager; import org.labkey.api.data.ContainerType; +import org.labkey.api.data.DbScope; +import org.labkey.api.data.SQLFragment; +import org.labkey.api.data.SqlExecutor; import org.labkey.api.module.Module; import org.labkey.api.module.ModuleLoader; import org.labkey.api.pipeline.PipeRoot; @@ -244,4 +247,195 @@ public URLHelper getSuccessURL(Object o) return PageFlowUtil.urlProvider(PipelineUrls.class).urlBegin(getContainer()); } } + + @RequiresSiteAdmin + public static class UpdateFilePathsAction extends ConfirmAction + { + @Override + public ModelAndView getConfirmView(UpdateFilePathsForm form, BindException errors) throws Exception + { + StringBuilder html = new StringBuilder(); + if (form.getReplacementPrefix() == null) + { + html.append("This action is designed to bulk update filepaths stored in the database, such as when a folder's file root is updated. This circumvents LabKey's normal file update listeners, and should only be performed if you are certain this is what you want. A reason for this is because the default codepath can be slow with extremely large moves."); + html.append("

"); + html.append("Enter the following:
"); + html.append("
"); + html.append(""); + html.append(""); + html.append(""); + html.append(""); + html.append(""); + html.append("
"); + html.append("
"); + html.append("When you hit confirm, you will be given an intermediate page summarizing changes before any changes are actually committed. Note: this could potentially make changes site-wide. Continue?"); + + return new HtmlView(HtmlString.unsafe(html.toString())); + } + else + { + return new HtmlView(HtmlString.unsafe(generateChangeSummary(form))); + } + } + + private String generateChangeSummary(UpdateFilePathsForm form) + { + StringBuilder ret = new StringBuilder(); + ret.append("You entered the following values:"); + ret.append("
"); + ret.append(""); + ret.append(""); + ret.append(""); + ret.append(""); + ret.append(""); + ret.append(""); + ret.append("
" + HtmlString.of(form.getSourcePrefix()) + "
" + HtmlString.of(form.getReplacementPrefix()) + "
"); + ret.append(""); + ret.append("
");
+            ret.append(getSql(form, true));
+            ret.append("
"); + ret.append("
"); + ret.append("Note: if the URL of the folder changed you may also want to execute something like the following (manually):
"); + ret.append("
");
+            ret.append(HtmlString.of("UPDATE pipeline.StatusFiles SET DataUrl = replace(DataUrl, '', '') "));
+            ret.append(HtmlString.of("WHERE DataUrl like '%%';"));
+            ret.append("
"); + + ret.append("
"); + + return ret.toString(); + } + + private String ensureSlashes(String input) + { + if (input == null) + { + return input; + } + + if (!input.startsWith("/")) + { + input = "/" + input; + } + + if (!input.endsWith("/")) + { + input = input + "/"; + } + + return input; + } + + private String getSql(UpdateFilePathsForm form, boolean calculateCounts) + { + // Ensure start/end with slash: + String sourcePrefix = ensureSlashes(form.getSourcePrefix()); + String replacementPrefix = ensureSlashes(form.getReplacementPrefix()); + + StringBuilder sql = new StringBuilder(); + + if (calculateCounts) + { + int count = new SqlExecutor(DbScope.getLabKeyScope()).execute(new SQLFragment("SELECT count(*) FROM Exp.Data WHERE DataFileUrl like 'file://" + sourcePrefix + "%'")); + sql.append("--Matching rows: " + count + "\n"); + } + + sql.append("UPDATE Exp.Data SET DataFileUrl = replace(DataFileUrl, 'file://" + sourcePrefix + "', 'file://" + replacementPrefix + "') "); + sql.append("WHERE DataFileUrl like 'file://" + sourcePrefix + "%';\n"); + + if (calculateCounts) + { + int count = new SqlExecutor(DbScope.getLabKeyScope()).execute(new SQLFragment("SELECT count(*) FROM pipeline.StatusFiles WHERE FilePath like '" + sourcePrefix + "%'")); + sql.append("--Matching rows: " + count + "\n"); + } + sql.append("UPDATE pipeline.StatusFiles SET FilePath = replace(FilePath, '" + sourcePrefix + "', '" + replacementPrefix + "') "); + sql.append("WHERE FilePath like '" + sourcePrefix + "%';"); + + return sql.toString(); + } + @Override + public boolean handlePost(UpdateFilePathsForm form, BindException errors) throws Exception + { + if (form.isUpdateDatabase()) + { + String sql = getSql(form, false); + + SqlExecutor se = new SqlExecutor(DbScope.getLabKeyScope()); + se.execute(new SQLFragment(sql)); + } + + return true; + } + + @Override + public void validateCommand(UpdateFilePathsForm form, Errors errors) + { + if (form.isUpdateDatabase() && form.getReplacementPrefix() == null) + { + errors.reject(ERROR_MSG, "Missing replacementPrefix"); + } + + if (form.isUpdateDatabase() && form.getSourcePrefix() == null) + { + errors.reject(ERROR_MSG, "Missing sourcePrefix"); + } + } + + @Override + public @NotNull URLHelper getSuccessURL(UpdateFilePathsForm form) + { + if (!form.isUpdateDatabase()) + { + ActionURL url = new ActionURL(UpdateFilePathsAction.class, getContainer()); + url.addParameter("sourcePrefix", form.getSourcePrefix()); + url.addParameter("replacementPrefix", form.getReplacementPrefix()); + + return url; + } + else + { + return getContainer().getStartURL(getUser()); + } + } + } + + public static class UpdateFilePathsForm + { + private String _sourcePrefix; + private String _replacementPrefix; + private boolean _updateDatabase = false; + + public String getSourcePrefix() + { + return _sourcePrefix; + } + + public void setSourcePrefix(String sourcePrefix) + { + _sourcePrefix = sourcePrefix; + } + + public String getReplacementPrefix() + { + return _replacementPrefix; + } + + public void setReplacementPrefix(String replacementPrefix) + { + _replacementPrefix = replacementPrefix; + } + + public boolean isUpdateDatabase() + { + return _updateDatabase; + } + + public void setUpdateDatabase(boolean updateDatabase) + { + _updateDatabase = updateDatabase; + } + } } \ No newline at end of file From e05cc496d73a6007dbe94542757d6e5a34b5c3de Mon Sep 17 00:00:00 2001 From: bbimber Date: Wed, 10 Mar 2021 09:03:01 -0800 Subject: [PATCH 90/98] Need to actually print plots --- mGAP/resources/etls/prime-seq.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mGAP/resources/etls/prime-seq.xml b/mGAP/resources/etls/prime-seq.xml index da78c21a3..fa21db466 100644 --- a/mGAP/resources/etls/prime-seq.xml +++ b/mGAP/resources/etls/prime-seq.xml @@ -143,7 +143,7 @@ af - + @@ -167,6 +167,6 @@ - + From 35ba3d10fa8ec184bb18cfedc7107ac03c8ed2f0 Mon Sep 17 00:00:00 2001 From: bbimber Date: Tue, 30 Mar 2021 09:54:13 -0700 Subject: [PATCH 91/98] Add sql script for the purposes of allowing lastDayAtCenter to work --- mcc/resources/queries/study/departure.sql | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 mcc/resources/queries/study/departure.sql diff --git a/mcc/resources/queries/study/departure.sql b/mcc/resources/queries/study/departure.sql new file mode 100644 index 000000000..f50c54bc4 --- /dev/null +++ b/mcc/resources/queries/study/departure.sql @@ -0,0 +1,7 @@ +--NOTE: this is created so that the calculated lastDayAtCenter column will work: +SELECT + +Id, +date + +FROM study.transfers \ No newline at end of file From 4055cdfaf60bc9c405af848b0ca13cb2872963bc Mon Sep 17 00:00:00 2001 From: bbimber Date: Tue, 30 Mar 2021 16:22:20 -0700 Subject: [PATCH 92/98] Another iteration on MHC data migration --- .../pipeline/MhcMigrationPipelineJob.java | 231 +++++++++++++++--- 1 file changed, 191 insertions(+), 40 deletions(-) diff --git a/primeseq/src/org/labkey/primeseq/pipeline/MhcMigrationPipelineJob.java b/primeseq/src/org/labkey/primeseq/pipeline/MhcMigrationPipelineJob.java index e3e1acb21..e265f0f19 100644 --- a/primeseq/src/org/labkey/primeseq/pipeline/MhcMigrationPipelineJob.java +++ b/primeseq/src/org/labkey/primeseq/pipeline/MhcMigrationPipelineJob.java @@ -1,6 +1,9 @@ package org.labkey.primeseq.pipeline; import org.apache.commons.io.FileUtils; +import org.json.JSONObject; +import org.labkey.api.assay.AssayProvider; +import org.labkey.api.assay.AssayService; import org.labkey.api.collections.CaseInsensitiveHashMap; import org.labkey.api.data.CompareType; import org.labkey.api.data.Container; @@ -18,9 +21,12 @@ import org.labkey.api.di.DataIntegrationService; import org.labkey.api.exp.api.DataType; import org.labkey.api.exp.api.ExpData; +import org.labkey.api.exp.api.ExpProtocol; import org.labkey.api.exp.api.ExpRun; import org.labkey.api.exp.api.ExperimentService; import org.labkey.api.files.FileUrls; +import org.labkey.api.laboratory.LaboratoryService; +import org.labkey.api.module.FolderTypeManager; import org.labkey.api.module.Module; import org.labkey.api.pipeline.AbstractTaskFactory; import org.labkey.api.pipeline.AbstractTaskFactorySettings; @@ -39,6 +45,7 @@ import org.labkey.api.query.FieldKey; import org.labkey.api.query.QueryService; import org.labkey.api.query.UserSchema; +import org.labkey.api.query.ValidationException; import org.labkey.api.security.User; import org.labkey.api.sequenceanalysis.SequenceAnalysisService; import org.labkey.api.sequenceanalysis.model.Readset; @@ -65,6 +72,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; @@ -200,21 +208,24 @@ public RecordedActionSet run() throws PipelineJobException replaceEntireTable("laboratory", "subjects", Arrays.asList("subjectname", "species"), null, true, getJob().getContainer(), getPipelineJob().remoteServerFolder); createLibraries(); - createLibraryMembers(); + //createLibraryMembers(); //TODO: restore this createReadsets(); transaction.commitAndKeepConnection(); - createReaddata(); + //createReaddata(); //TODO: restore this createAnalyses(); createOutputFiles(); - createAlignmentSummary(); - createQualityMetrics(); + transaction.commitAndKeepConnection(); + + //create assay runs, including data and haplotypes + syncAssay("GenotypeAssay", "Genotype", Arrays.asList("Name", "Comments", "performedBy", "runDate", "instrument", "assayType", "barcode"), Arrays.asList("subjectId", "date", "marker", "result", "qual_result", "sampleId", "category", "plate", "well", "parentId", "comment", "requestid", "qcflag", "analysisId", "DataId", "sampleType", "statusflag", "rawResult")); + transaction.commitAndKeepConnection(); - //TODO: create assay runs, including data and haplotypes + createAlignmentSummary(); transaction.commit(); } @@ -222,11 +233,95 @@ public RecordedActionSet run() throws PipelineJobException return new RecordedActionSet(); } + private void syncAssay(String providerName, String assayName, List runColumns, List resultColumns) throws PipelineJobException + { + getJob().getLogger().info("syncing assay: " + providerName + " / " + assayName); + AssayProvider ap = AssayService.get().getProvider(providerName); + for (Integer wb : workbookMap.keySet()) + { + List protocols = AssayService.get().getAssayProtocols(workbookMap.get(wb), ap); + ExpProtocol protocol = protocols.get(0); + + SelectRowsCommand sr1 = new SelectRowsCommand("assay." + ap.getName() + "." + protocol.getName(), "Runs"); + sr1.setColumns(runColumns); + + try + { + SelectRowsResponse srr = sr1.execute(getConnection(), getPipelineJob().remoteServerFolder + wb); + if (srr.getRowCount().intValue() == 0) + { + continue; + } + + //Existing runs: + List existingRunNames = new TableSelector(AssayService.get().createRunTable(protocol, ap, getJob().getUser(), workbookMap.get(wb), null), PageFlowUtil.set("Name")).getArrayList(String.class); + if (existingRunNames.size() == srr.getRowCount().intValue()) + { + getJob().getLogger().info("Run count matches, skipping: " + wb); + continue; + } + + File assayTmp = File.createTempFile("assay-upload", ".txt").getAbsoluteFile(); + ViewBackgroundInfo info = getJob().getInfo(); + ViewContext vc = ViewContext.getMockViewContext(info.getUser(), workbookMap.get(wb), info.getURL(), false); + + srr.getRows().forEach(run -> { + if (existingRunNames.contains(run.get("Name"))) + { + getJob().getLogger().info("Run exists, skipping: " + run.get("Name")); + return; + } + + JSONObject json = new JSONObject(); + json.put("Run", run); + + SelectRowsCommand sr2 = new SelectRowsCommand("assay." + ap.getName() + "." + protocol.getName(), "Data"); + sr2.setColumns(resultColumns); + + try + { + SelectRowsResponse srr2 = sr2.execute(getConnection(), getPipelineJob().remoteServerFolder + wb); + List> resultRows = srr2.getRows(); + final Set missingAnalyses = new HashSet<>(); + resultRows.forEach(x -> { + if (x.get("analysisId") != null) + { + if (analysisMap.containsKey(x.get("analysisId"))) + { + x.put("analysisId", analysisMap.get(x.get("analysisId"))); + } + else + { + if (!missingAnalyses.contains(x.get("analysisId"))) + { + getJob().getLogger().error("Unable to find analysis to match: " + x.get("analysisId")); + missingAnalyses.add(x.get("analysisId")); + } + } + } + }); + + LaboratoryService.get().saveAssayBatch(resultRows, json, assayTmp, vc, ap, protocol); + } + catch (ValidationException | CommandException | IOException e) + { + throw new RuntimeException(e); + } + }); + + } + catch (IOException | CommandException e) + { + throw new PipelineJobException(e); + } + } + } + private void createQualityMetrics() throws PipelineJobException { for (int workbook : workbookMap.keySet()) { - replaceEntireTable("sequenceanalysis", "quanlity_metrics", Arrays.asList("dataid", "dataid/DatafileUrl", "category", "metricname", "metricvalue", "qualvalue", "analysis_id", "readset"), null, true, workbookMap.get(workbook), getPipelineJob().remoteServerFolder + workbook + "/"); + replaceEntireTable("sequenceanalysis", "quality_metrics", Arrays.asList("dataid", "dataid/DatafileUrl", "runid/JobId/FilePath", "category", "metricname", "metricvalue", "qualvalue", "analysis_id", "readset", "readset/runid/JobId", "readset/runid/JobId/FilePath", "dataid/Run/JobId/FilePath"), null, true, workbookMap.get(workbook), getPipelineJob().remoteServerFolder + workbook + "/"); } } @@ -239,10 +334,10 @@ private void createAlignmentSummary() throws PipelineJobException //NOTE: split by workbook to avoid huge API calls: SelectRowsCommand srWB = new SelectRowsCommand("core", "workbooks"); - srWB.setColumns(Arrays.asList("workbookId")); + srWB.setColumns(Arrays.asList("Name")); SelectRowsResponse srrWB = srWB.execute(getConnection(), getPipelineJob().remoteServerFolder); - List workbooks = srrWB.getRows().stream().map(x -> x.get("workbookId")).collect(Collectors.toList()); + List workbooks = srrWB.getRows().stream().map(x -> x.get("Name")).collect(Collectors.toList()); for (Object workbook : workbooks) { getJob().getLogger().info("importing alignments for workbook: " + workbook); @@ -252,7 +347,7 @@ private void createAlignmentSummary() throws PipelineJobException SelectRowsResponse srr = sr.execute(getConnection(), getPipelineJob().remoteServerFolder + workbook + "/"); - Map alignmentSummaryMap = new HashMap<>(); + final Map alignmentSummaryMap = new HashMap<>(srr.getRowCount().intValue()); srr.getRowset().forEach(rs -> { CaseInsensitiveHashMap map = new CaseInsensitiveHashMap<>(); Integer localId = analysisMap.get(rs.getValue("analysis_id")); @@ -329,6 +424,12 @@ private void replaceEntireTable(String schema, String query, List column if (srr.getRowCount().longValue() == existing) { getJob().getLogger().info("Row counts identical, assuming has been synced: " + query); + return; + } + else if (srr.getRowCount().equals(0)) + { + getJob().getLogger().info("No rows, skipping: " + query); + return; } List> toInsert = new ArrayList<>(); @@ -341,51 +442,92 @@ private void replaceEntireTable(String schema, String query, List column Object val = r.getValue(colName); if ("readset".equals(colName) || "readsetid".equals(colName)) { - if (!readsetMap.containsKey((int) val)) + if (val != null && !readsetMap.containsKey(val)) { throw new IllegalStateException("Unable to find readset: " + val); } - val = readsetMap.get((int) val); + val = readsetMap.get(val); } else if ("library_id".equals(colName)) { - if (!libraryMap.containsKey((int) val)) + if (val != null && !libraryMap.containsKey(val)) { throw new IllegalStateException("Unable to find library: " + val); } - val = libraryMap.get((int) val); + val = libraryMap.get(val); } else if ("ref_nt_id".equals(colName)) { - if (!sequenceMap.containsKey((int) val)) + if (val != null && !sequenceMap.containsKey(val)) { throw new IllegalStateException("Unable to find sequence: " + val); } - val = sequenceMap.get((int) val); + val = sequenceMap.get(val); } else if ("analysis_id".equals(colName)) { - if (!analysisMap.containsKey((int) val)) + if (val != null && !analysisMap.containsKey(val)) { throw new IllegalStateException("Unable to find analysis: " + val); } - val = analysisMap.get((int) val); + val = analysisMap.get(val); } row.put(colName, val); } - else if ("dataid/DatafileUrl".equalsIgnoreCase(colName)) + else if ("dataid/DatafileUrl".equalsIgnoreCase(colName) && r.getValue("dataid/DatafileUrl") != null) { - String remoteJobRoot = getParent(URI.create(String.valueOf(r.getValue("dataid/FilePath")).replaceAll(" ", "_")).getPath()); + String remoteJobRoot; + if (r.getValue("runid/JobId/FilePath") == null) + { + if (r.getValue("analysis_id") != null) + { + Integer localAnalysisId = analysisMap.get((int) r.getValue("analysis_id")); + if (analysisToJobPath.containsKey(localAnalysisId)) + { + remoteJobRoot = analysisToJobPath.get(localAnalysisId); + } + else + { + getJob().getLogger().error("Missing path: " + r.getValue("dataid/DatafileUrl")); + return; + } + } + else if (r.getValue("readset/runid/JobId/FilePath") != null) + { + remoteJobRoot = getParent(URI.create(String.valueOf(r.getValue("readset/runid/JobId/FilePath")).replaceAll(" ", "%20")).getPath()); + } + else if (r.getValue("dataid/Run/JobId/FilePath") != null) + { + remoteJobRoot = getParent(URI.create(String.valueOf(r.getValue("dataid/Run/JobId/FilePath")).replaceAll(" ", "%20")).getPath()); + } + else + { + getJob().getLogger().error("Missing path: " + r.getValue("dataid/DatafileUrl")); + return; + } + } + else + { + remoteJobRoot = getParent(URI.create(String.valueOf(r.getValue("runid/JobId/FilePath")).replaceAll(" ", "%20")).getPath()); + } - URI localFileRoot = PipelineService.get().getPipelineRootSetting(targetContainer).getRootPath().toURI(); - URI localPath = translateURI(String.valueOf(r.getValue("dataid/DatafileUrl")), remoteJobRoot, localFileRoot.getPath()); - row.put("dataid", getOrCreateExpData(localPath, targetContainer)); + if (remoteJobRoot != null) + { + URI localFileRoot = PipelineService.get().getPipelineRootSetting(targetContainer).getRootPath().toURI(); + URI localPath = translateURI(String.valueOf(r.getValue("dataid/DatafileUrl")), remoteJobRoot, localFileRoot.getPath()); + row.put("dataid", getOrCreateExpData(localPath, targetContainer)); + } + else + { + getJob().getLogger().error("Unable to find job root: " + r.getValue("dataid/DatafileUrl")); + return; + } } }); @@ -436,6 +578,7 @@ else if ("dataid/DatafileUrl".equalsIgnoreCase(colName)) private final Map readdataMap = new HashMap<>(); private final Map analysisMap = new HashMap<>(); private final Map analysisToFileMap = new HashMap<>(); //local analysis_id -> alignment file + private final Map analysisToJobPath = new HashMap<>(); private final Map libraryMap = new HashMap<>(); private final Map outputFileMap = new HashMap<>(); private final Map sequenceMap = new HashMap<>(); @@ -551,7 +694,6 @@ private int getOrCreateSequence(int remoteSeqId, String name, int seqLength, Tab } } - //TODO: Create sequence? //throw new IllegalStateException("Expected sequence to exist: " + name); getJob().getLogger().error("Sequence missing: " + name); return -1; @@ -563,6 +705,10 @@ public String getParent(String path) final char separatorChar = '/'; int index = path.lastIndexOf(separatorChar); + if (index == -1) + { + throw new IllegalArgumentException("Missing slash"); + } return path.substring(0, index); } @@ -654,7 +800,7 @@ private void createOutputFiles() final TableInfo outputTable = QueryService.get().getUserSchema(getJob().getUser(), getPipelineJob().targetContainer, "sequenceanalysis").getTable("outputfiles"); SelectRowsCommand sr = new SelectRowsCommand("sequenceanalysis", "outputfiles"); - sr.setColumns(Arrays.asList("rowid", "name", "description", "dataid", "library_id", "readset", "analysis_id", "category", "sra_accession", "dataid/DataFileUrl", "runid/jobid", "runid/Name", "workbook/workbookId", "runid/JobId", "runid/Name", "runid/JobId/FilePath")); + sr.setColumns(Arrays.asList("rowid", "name", "description", "dataid", "library_id", "readset", "analysis_id", "category", "sra_accession", "dataid/DataFileUrl", "runid", "runid/JobId", "runid/Name", "workbook/workbookId", "runid/Name", "runid/JobId/FilePath")); SelectRowsResponse srr = sr.execute(getConnection(), getPipelineJob().remoteServerFolder); @@ -726,7 +872,7 @@ private void createOutputFiles() PipelineStatusFile sf = PipelineService.get().getStatusFile(jobId); String localJobRoot = getParent(sf.getFilePath()); - String remoteJobRoot = getParent(URI.create(String.valueOf(rd.getValue("runid/JobId/FilePath")).replaceAll(" ", "_")).getPath()); + String remoteJobRoot = getParent(URI.create(String.valueOf(rd.getValue("runid/JobId/FilePath")).replaceAll(" ", "%20")).getPath()); URI newFileAlignment = translateURI(String.valueOf(rd.getValue("dataid/DatafileUrl")), remoteJobRoot, localJobRoot); toCreate.put("dataid", getOrCreateExpData(newFileAlignment, targetWorkbook)); @@ -852,7 +998,7 @@ private void createAnalyses() PipelineStatusFile sf = PipelineService.get().getStatusFile(jobId); String localJobRoot = getParent(sf.getFilePath()); - String remoteJobRoot = getParent(URI.create(String.valueOf(rd.getValue("runid/JobId/FilePath")).replaceAll(" ", "_")).getPath()); + String remoteJobRoot = getParent(URI.create(String.valueOf(rd.getValue("runid/JobId/FilePath")).replaceAll(" ", "%20")).getPath()); URI newFileAlignment = translateURI(String.valueOf(rd.getValue("alignmentfile/DatafileUrl")), remoteJobRoot, localJobRoot); toCreate.put("alignmentfile", getOrCreateExpData(newFileAlignment, targetWorkbook)); @@ -882,6 +1028,7 @@ private void createAnalyses() } analysisMap.put(remoteId, Integer.parseInt(String.valueOf(created.get(0).get("rowid")))); + analysisToJobPath.put(Integer.parseInt(String.valueOf(created.get(0).get("rowid"))), remoteJobRoot); } catch (Exception e) { @@ -938,6 +1085,12 @@ private void createReaddata() } else { + if (rd.getValue("fileid1/DataFileUrl") == null) + { + getJob().getLogger().warn("readddata missing files, skipping: " + remoteId); + return; + } + Map toCreate = new CaseInsensitiveHashMap<>(); toCreate.put("readset", rs.getRowId()); toCreate.put("platformUnit", rd.getValue("platformUnit")); @@ -954,7 +1107,7 @@ private void createReaddata() PipelineStatusFile sf = PipelineService.get().getStatusFile(jobId); String localJobRoot = getParent(sf.getFilePath()); - String remoteJobRoot = getParent(URI.create(String.valueOf(rd.getValue("runid/JobId/FilePath")).replaceAll(" ", "_")).getPath()); + String remoteJobRoot = getParent(URI.create(String.valueOf(rd.getValue("runid/JobId/FilePath")).replaceAll(" ", "%20")).getPath()); if (rd.getValue("fileid1/DataFileUrl") != null) { @@ -970,7 +1123,10 @@ private void createReaddata() } else { - getJob().getLogger().error("readddata missing jobid: " + remoteId); + if (rd.getValue("fileid1/DataFileUrl") != null) + { + getJob().getLogger().error("readddata missing jobid: " + remoteId); + } } //Create run: @@ -983,7 +1139,10 @@ private void createReaddata() } else { - getJob().getLogger().error("readddata missing runid: " + remoteId); + if (rd.getValue("fileid1/DataFileUrl") != null) + { + getJob().getLogger().error("readddata missing runid: " + remoteId); + } } BatchValidationException bve = new BatchValidationException(); @@ -1153,7 +1312,7 @@ else if (filepath.contains("sequenceAnalysis")) } } - File remoteDir = new File(URI.create(filepath.replaceAll(" ", "_")).getPath()); + File remoteDir = new File(URI.create(filepath.replaceAll(" ", "%20")).getPath()); File localDir = new File(fr, filepath.split("@files")[1]); //Check for existing row: @@ -1273,19 +1432,11 @@ private void createWorkbooks() { PipeRoot pr = PipelineService.get().getPipelineRootSetting(getJob().getContainer()); - String description = String.valueOf(wb.getValue("Description")); - if (description != null) - { - description = description + ". "; - } - else - { - description = ""; - } - + String description = wb.getValue("Description") != null ? String.valueOf(wb.getValue("Description")) + ". " : ""; description = description + "Originally PRIMe workbook: " + wb.getValue("Name"); Container workbook = ContainerManager.createContainer(getPipelineJob().targetContainer, null, localTitle, description, WorkbookContainerType.NAME, getJob().getUser()); + workbook.setFolderType(FolderTypeManager.get().getFolderType("Expt Workbook"), getJob().getUser()); workbookMap.put(Integer.parseInt(String.valueOf(wb.getValue("Name"))), workbook); File sourceDir = new File("/home/groups/miSeqLK/Production/MHC_Typing", wb.getValue("Name") + "/@files"); From 7125a796ee6859c3836f162ae685d7953b8ce447 Mon Sep 17 00:00:00 2001 From: bbimber Date: Wed, 31 Mar 2021 14:54:46 -0700 Subject: [PATCH 93/98] Improve MHC sync pipeline job --- .../queries/study/departure.query.xml | 28 +++ .../pipeline/MhcMigrationPipelineJob.java | 219 ++++++++++-------- 2 files changed, 146 insertions(+), 101 deletions(-) create mode 100644 mcc/resources/queries/study/departure.query.xml diff --git a/mcc/resources/queries/study/departure.query.xml b/mcc/resources/queries/study/departure.query.xml new file mode 100644 index 000000000..0b7bbb463 --- /dev/null +++ b/mcc/resources/queries/study/departure.query.xml @@ -0,0 +1,28 @@ + + + + + + + + + + + + + + + true + + + true + + + Record Status + true + + +
+ + + \ No newline at end of file diff --git a/primeseq/src/org/labkey/primeseq/pipeline/MhcMigrationPipelineJob.java b/primeseq/src/org/labkey/primeseq/pipeline/MhcMigrationPipelineJob.java index e265f0f19..bfef3e1dc 100644 --- a/primeseq/src/org/labkey/primeseq/pipeline/MhcMigrationPipelineJob.java +++ b/primeseq/src/org/labkey/primeseq/pipeline/MhcMigrationPipelineJob.java @@ -203,26 +203,31 @@ public RecordedActionSet run() throws PipelineJobException try (DbScope.Transaction transaction = DbScope.getLabKeyScope().ensureTransaction()) { createWorkbooks(); + transaction.commitAndKeepConnection(); replaceEntireTable("laboratory", "samples", Arrays.asList("samplename", "subjectid", "sampledate", "sampletype", "samplesubtype", "samplesource", "location", "freezer", "cane", "box", "box_row", "box_column", "comment", "workbook/workbookId", "samplespecies", "processdate", "concentration", "concentration_units", "quantity", "quantity_units", "ratio"), "workbook/workbookId", true, getJob().getContainer(), getPipelineJob().remoteServerFolder); replaceEntireTable("laboratory", "subjects", Arrays.asList("subjectname", "species"), null, true, getJob().getContainer(), getPipelineJob().remoteServerFolder); + transaction.commitAndKeepConnection(); createLibraries(); - //createLibraryMembers(); //TODO: restore this + createLibraryMembers(); + transaction.commitAndKeepConnection(); createReadsets(); transaction.commitAndKeepConnection(); - //createReaddata(); //TODO: restore this + createReaddata(); + transaction.commitAndKeepConnection(); createAnalyses(); createOutputFiles(); + transaction.commitAndKeepConnection(); createQualityMetrics(); transaction.commitAndKeepConnection(); //create assay runs, including data and haplotypes - syncAssay("GenotypeAssay", "Genotype", Arrays.asList("Name", "Comments", "performedBy", "runDate", "instrument", "assayType", "barcode"), Arrays.asList("subjectId", "date", "marker", "result", "qual_result", "sampleId", "category", "plate", "well", "parentId", "comment", "requestid", "qcflag", "analysisId", "DataId", "sampleType", "statusflag", "rawResult")); + syncAssay("GenotypeAssay", "Genotype", Arrays.asList("RowId", "Name", "Comments", "performedBy", "runDate", "instrument", "assayType", "barcode"), Arrays.asList("subjectId", "date", "marker", "result", "qual_result", "sampleId", "category", "plate", "well", "parentId", "comment", "requestid", "qcflag", "analysisId", "DataId", "sampleType", "statusflag", "rawResult")); transaction.commitAndKeepConnection(); createAlignmentSummary(); @@ -264,7 +269,7 @@ private void syncAssay(String providerName, String assayName, List runCo File assayTmp = File.createTempFile("assay-upload", ".txt").getAbsoluteFile(); ViewBackgroundInfo info = getJob().getInfo(); ViewContext vc = ViewContext.getMockViewContext(info.getUser(), workbookMap.get(wb), info.getURL(), false); - + final Set missingAnalyses = new HashSet<>(); srr.getRows().forEach(run -> { if (existingRunNames.contains(run.get("Name"))) { @@ -277,12 +282,12 @@ private void syncAssay(String providerName, String assayName, List runCo SelectRowsCommand sr2 = new SelectRowsCommand("assay." + ap.getName() + "." + protocol.getName(), "Data"); sr2.setColumns(resultColumns); + sr2.addFilter(new Filter("Run", run.get("RowId"))); try { SelectRowsResponse srr2 = sr2.execute(getConnection(), getPipelineJob().remoteServerFolder + wb); List> resultRows = srr2.getRows(); - final Set missingAnalyses = new HashSet<>(); resultRows.forEach(x -> { if (x.get("analysisId") != null) { @@ -600,8 +605,8 @@ private void createLibraryMembers() SelectRowsResponse srr = sr.execute(getConnection(), getPipelineJob().remoteServerFolder); + List> sequencesToCreate = new ArrayList<>(); srr.getRowset().forEach(rd -> { - int remoteId = Integer.parseInt(String.valueOf(rd.getValue("rowid"))); int seqLength = Integer.parseInt(String.valueOf(rd.getValue("ref_nt_id/seqLength"))); int remoteSeqId = Integer.parseInt(String.valueOf(rd.getValue("ref_nt_id"))); @@ -634,22 +639,19 @@ private void createLibraryMembers() Map toCreate = new CaseInsensitiveHashMap<>(); toCreate.put("library_id", localLibraryId); toCreate.put("ref_nt_id", localSeqId); + sequencesToCreate.add(toCreate); + }); - try - { - BatchValidationException bve = new BatchValidationException(); - List> created = ti.getUpdateService().insertRows(getJob().getUser(), getPipelineJob().targetContainer, Arrays.asList(toCreate), bve, null, null); - if (bve.hasErrors()) - { - throw new RuntimeException(bve); - } - } - catch (Exception e) + getJob().getLogger().info("Total sequences to create: " + sequencesToCreate.size()); + if (!sequencesToCreate.isEmpty()) + { + BatchValidationException bve = new BatchValidationException(); + List> created = ti.getUpdateService().insertRows(getJob().getUser(), getPipelineJob().targetContainer, sequencesToCreate, bve, null, null); + if (bve.hasErrors()) { - getJob().getLogger().error(e.getMessage(), e); - throw new RuntimeException(e); + throw new RuntimeException(bve); } - }); + } } catch (Exception e) { @@ -1049,119 +1051,134 @@ private void createReaddata() getJob().getLogger().info("Creating read data"); try { - final TableInfo readdataTable = QueryService.get().getUserSchema(getJob().getUser(), getPipelineJob().targetContainer, "sequenceanalysis").getTable("readdata"); + for (Integer workbookId : workbookMap.keySet()) + { + final TableInfo readdataTable = QueryService.get().getUserSchema(getJob().getUser(), workbookMap.get(workbookId), "sequenceanalysis").getTable("readdata"); - SelectRowsCommand sr = new SelectRowsCommand("sequenceanalysis", "readdata"); - sr.setColumns(Arrays.asList("rowid", "readset", "platformUnit", "centerName", "date", "fileid1", "fileid1/DataFileUrl", "fileid2", "fileid2/DataFileUrl", "fileid1/Name", "description", "sra_accession", "runid", "runid/jobid", "runid/Name", "readset/workbook/workbookId", "runid/JobId", "runid/Name", "runid/JobId/FilePath", "runid/JobId/Description")); + SelectRowsCommand sr = new SelectRowsCommand("sequenceanalysis", "readdata"); + sr.setColumns(Arrays.asList("rowid", "readset", "platformUnit", "centerName", "date", "fileid1", "fileid1/DataFileUrl", "fileid2", "fileid2/DataFileUrl", "fileid1/Name", "description", "sra_accession", "runid", "runid/jobid", "runid/Name", "readset/workbook/workbookId", "runid/JobId", "runid/Name", "runid/JobId/FilePath", "runid/JobId/Description")); - SelectRowsResponse srr = sr.execute(getConnection(), getPipelineJob().remoteServerFolder); + SelectRowsResponse srr = sr.execute(getConnection(), getPipelineJob().remoteServerFolder + workbookId + "/"); - srr.getRowset().forEach(rd -> { - int remoteId = Integer.parseInt(String.valueOf(rd.getValue("rowid"))); - int remoteReadset = Integer.parseInt(String.valueOf(rd.getValue("readset"))); - Integer localReadset = readsetMap.get(remoteReadset); - if (localReadset == null) + long existing = new TableSelector(readdataTable).getRowCount(); + if (srr.getRowCount().longValue() == existing) { - throw new IllegalArgumentException("Unable to find readset for remote id: " + remoteReadset); + getJob().getLogger().info("Readdata count identical, skipping: " + workbookId); + continue; + } + else if (srr.getRowCount().intValue() == 0) + { + getJob().getLogger().info("No readdata records, skipping: " + workbookId); + continue; } - Readset rs = SequenceAnalysisService.get().getReadset(localReadset, getJob().getUser()); - Container targetWorkbook = ContainerManager.getForId(rs.getContainer()); + srr.getRowset().forEach(rd -> { + int remoteId = Integer.parseInt(String.valueOf(rd.getValue("rowid"))); + int remoteReadset = Integer.parseInt(String.valueOf(rd.getValue("readset"))); + Integer localReadset = readsetMap.get(remoteReadset); + if (localReadset == null) + { + throw new IllegalArgumentException("Unable to find readset for remote id: " + remoteReadset); + } - SimpleFilter rdFilter = new SimpleFilter(FieldKey.fromString("readset"), rs.getRowId()); - rdFilter.addCondition(FieldKey.fromString("runid/JobId/Description"), rd.getValue("runid/JobId/Description")); - rdFilter.addCondition(FieldKey.fromString("fileid1/Name"), rd.getValue("fileid1/Name")); - rdFilter.addCondition(FieldKey.fromString("container"), targetWorkbook.getId(), CompareType.EQUAL); + Readset rs = SequenceAnalysisService.get().getReadset(localReadset, getJob().getUser()); + Container targetWorkbook = ContainerManager.getForId(rs.getContainer()); - if (rd.getValue("platformUnit") != null) - { - rdFilter.addCondition(FieldKey.fromString("platformUnit"), rd.getValue("platformUnit")); - } + SimpleFilter rdFilter = new SimpleFilter(FieldKey.fromString("readset"), rs.getRowId()); + rdFilter.addCondition(FieldKey.fromString("runid/JobId/Description"), rd.getValue("runid/JobId/Description")); + rdFilter.addCondition(FieldKey.fromString("fileid1/Name"), rd.getValue("fileid1/Name")); + rdFilter.addCondition(FieldKey.fromString("container"), targetWorkbook.getId(), CompareType.EQUAL); - TableSelector tsReaddata = new TableSelector(readdataTable, PageFlowUtil.set("rowid"), rdFilter, null); - if (tsReaddata.exists()) - { - readdataMap.put(remoteId, tsReaddata.getObject(Integer.class)); - } - else - { - if (rd.getValue("fileid1/DataFileUrl") == null) + if (rd.getValue("platformUnit") != null) { - getJob().getLogger().warn("readddata missing files, skipping: " + remoteId); - return; + rdFilter.addCondition(FieldKey.fromString("platformUnit"), rd.getValue("platformUnit")); } - Map toCreate = new CaseInsensitiveHashMap<>(); - toCreate.put("readset", rs.getRowId()); - toCreate.put("platformUnit", rd.getValue("platformUnit")); - toCreate.put("centerName", rd.getValue("centerName")); - toCreate.put("date", rd.getValue("date")); - toCreate.put("description", rd.getValue("description")); - toCreate.put("sra_accession", rd.getValue("sra_accession")); - try + TableSelector tsReaddata = new TableSelector(readdataTable, PageFlowUtil.set("rowid"), rdFilter, null); + if (tsReaddata.exists()) + { + readdataMap.put(remoteId, tsReaddata.getObject(Integer.class)); + } + else { - if (rd.getValue("runid/JobId") != null) + if (rd.getValue("fileid1/DataFileUrl") == null) { - int remoteJobId = Integer.parseInt(String.valueOf(rd.getValue("runid/JobId"))); - int jobId = getOrCreateJob(remoteJobId, targetWorkbook); - PipelineStatusFile sf = PipelineService.get().getStatusFile(jobId); + getJob().getLogger().warn("readddata missing files, skipping: " + remoteId); + return; + } - String localJobRoot = getParent(sf.getFilePath()); - String remoteJobRoot = getParent(URI.create(String.valueOf(rd.getValue("runid/JobId/FilePath")).replaceAll(" ", "%20")).getPath()); + Map toCreate = new CaseInsensitiveHashMap<>(); + toCreate.put("readset", rs.getRowId()); + toCreate.put("platformUnit", rd.getValue("platformUnit")); + toCreate.put("centerName", rd.getValue("centerName")); + toCreate.put("date", rd.getValue("date")); + toCreate.put("description", rd.getValue("description")); + toCreate.put("sra_accession", rd.getValue("sra_accession")); + try + { + if (rd.getValue("runid/JobId") != null) + { + int remoteJobId = Integer.parseInt(String.valueOf(rd.getValue("runid/JobId"))); + int jobId = getOrCreateJob(remoteJobId, targetWorkbook); + PipelineStatusFile sf = PipelineService.get().getStatusFile(jobId); - if (rd.getValue("fileid1/DataFileUrl") != null) + String localJobRoot = getParent(sf.getFilePath()); + String remoteJobRoot = getParent(URI.create(String.valueOf(rd.getValue("runid/JobId/FilePath")).replaceAll(" ", "%20")).getPath()); + + if (rd.getValue("fileid1/DataFileUrl") != null) + { + URI newFile1 = translateURI(String.valueOf(rd.getValue("fileid1/DataFileUrl")), remoteJobRoot, localJobRoot); + toCreate.put("fileid1", getOrCreateExpData(newFile1, targetWorkbook)); + } + + if (rd.getValue("fileid2/DataFileUrl") != null) + { + URI newFile2 = translateURI(String.valueOf(rd.getValue("fileid2/DatafileUrl")), remoteJobRoot, localJobRoot); + toCreate.put("fileid2", getOrCreateExpData(newFile2, targetWorkbook)); + } + } + else { - URI newFile1 = translateURI(String.valueOf(rd.getValue("fileid1/DataFileUrl")), remoteJobRoot, localJobRoot); - toCreate.put("fileid1", getOrCreateExpData(newFile1, targetWorkbook)); + if (rd.getValue("fileid1/DataFileUrl") != null) + { + getJob().getLogger().error("readddata missing jobid: " + remoteId); + } } - if (rd.getValue("fileid2/DataFileUrl") != null) + //Create run: + if (rd.getValue("runid") != null && rd.getValue("runid/JobId") != null) { - URI newFile2 = translateURI(String.valueOf(rd.getValue("fileid2/DatafileUrl")), remoteJobRoot, localJobRoot); - toCreate.put("fileid2", getOrCreateExpData(newFile2, targetWorkbook)); + int remoteJobId = Integer.parseInt(String.valueOf(rd.getValue("runid/JobId"))); + int jobId = getOrCreateJob(remoteJobId, targetWorkbook); + int runId = createExpRun(Integer.parseInt(String.valueOf(rd.getValue("runid"))), targetWorkbook, String.valueOf(rd.getValue("runid/Name")), jobId); + toCreate.put("runid", runId); } - } - else - { - if (rd.getValue("fileid1/DataFileUrl") != null) + else { - getJob().getLogger().error("readddata missing jobid: " + remoteId); + if (rd.getValue("fileid1/DataFileUrl") != null) + { + getJob().getLogger().error("readddata missing runid: " + remoteId); + } } - } - //Create run: - if (rd.getValue("runid") != null && rd.getValue("runid/JobId") != null) - { - int remoteJobId = Integer.parseInt(String.valueOf(rd.getValue("runid/JobId"))); - int jobId = getOrCreateJob(remoteJobId, targetWorkbook); - int runId = createExpRun(Integer.parseInt(String.valueOf(rd.getValue("runid"))), targetWorkbook, String.valueOf(rd.getValue("runid/Name")), jobId); - toCreate.put("runid", runId); - } - else - { - if (rd.getValue("fileid1/DataFileUrl") != null) + BatchValidationException bve = new BatchValidationException(); + List> created = readdataTable.getUpdateService().insertRows(getJob().getUser(), getPipelineJob().targetContainer, Arrays.asList(toCreate), bve, null, null); + if (bve.hasErrors()) { - getJob().getLogger().error("readddata missing runid: " + remoteId); + throw new RuntimeException(bve); } - } - BatchValidationException bve = new BatchValidationException(); - List> created = readdataTable.getUpdateService().insertRows(getJob().getUser(), getPipelineJob().targetContainer, Arrays.asList(toCreate), bve, null, null); - if (bve.hasErrors()) + readdataMap.put(remoteId, Integer.parseInt(String.valueOf(created.get(0).get("rowid")))); + } + catch (Exception e) { - throw new RuntimeException(bve); + throw new RuntimeException(e); } - - readdataMap.put(remoteId, Integer.parseInt(String.valueOf(created.get(0).get("rowid")))); - } - catch (Exception e) - { - throw new RuntimeException(e); } - } - }); + }); + } } - catch (Exception e) + catch(Exception e) { getJob().getLogger().error(e.getMessage(), e); throw new RuntimeException(e); From 477ed9181ecb82411a308e93efc49a02b8fec160 Mon Sep 17 00:00:00 2001 From: bbimber Date: Wed, 31 Mar 2021 15:05:21 -0700 Subject: [PATCH 94/98] Support nVariableFeatures in seurat normalization --- .../labkey/primeseq/pipeline/MhcMigrationPipelineJob.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/primeseq/src/org/labkey/primeseq/pipeline/MhcMigrationPipelineJob.java b/primeseq/src/org/labkey/primeseq/pipeline/MhcMigrationPipelineJob.java index bfef3e1dc..205944e5f 100644 --- a/primeseq/src/org/labkey/primeseq/pipeline/MhcMigrationPipelineJob.java +++ b/primeseq/src/org/labkey/primeseq/pipeline/MhcMigrationPipelineJob.java @@ -244,6 +244,8 @@ private void syncAssay(String providerName, String assayName, List runCo AssayProvider ap = AssayService.get().getProvider(providerName); for (Integer wb : workbookMap.keySet()) { + getJob().getLogger().info("processing workbook: " + wb); + List protocols = AssayService.get().getAssayProtocols(workbookMap.get(wb), ap); ExpProtocol protocol = protocols.get(0); @@ -288,6 +290,12 @@ private void syncAssay(String providerName, String assayName, List runCo { SelectRowsResponse srr2 = sr2.execute(getConnection(), getPipelineJob().remoteServerFolder + wb); List> resultRows = srr2.getRows(); + if (resultRows.isEmpty()) + { + getJob().getLogger().info("No results, skipping: " + run.get("Name")); + return; + } + resultRows.forEach(x -> { if (x.get("analysisId") != null) { From e5e1a6cc83e4630434ad41b6b67f27ef80ef6b72 Mon Sep 17 00:00:00 2001 From: bbimber Date: Thu, 1 Apr 2021 09:55:58 -0700 Subject: [PATCH 95/98] Update MCC reference study --- .../queries/study/departure.query.xml | 28 ---------------- mcc/resources/queries/study/departure.sql | 7 ---- .../datasets/datasets_manifest.xml | 3 ++ .../datasets/datasets_metadata.xml | 33 +++++++++++++++++++ 4 files changed, 36 insertions(+), 35 deletions(-) delete mode 100644 mcc/resources/queries/study/departure.query.xml delete mode 100644 mcc/resources/queries/study/departure.sql diff --git a/mcc/resources/queries/study/departure.query.xml b/mcc/resources/queries/study/departure.query.xml deleted file mode 100644 index 0b7bbb463..000000000 --- a/mcc/resources/queries/study/departure.query.xml +++ /dev/null @@ -1,28 +0,0 @@ - - - - - - - - - - - - - - - true - - - true - - - Record Status - true - - -
- - - \ No newline at end of file diff --git a/mcc/resources/queries/study/departure.sql b/mcc/resources/queries/study/departure.sql deleted file mode 100644 index f50c54bc4..000000000 --- a/mcc/resources/queries/study/departure.sql +++ /dev/null @@ -1,7 +0,0 @@ ---NOTE: this is created so that the calculated lastDayAtCenter column will work: -SELECT - -Id, -date - -FROM study.transfers \ No newline at end of file diff --git a/mcc/resources/referenceStudy/datasets/datasets_manifest.xml b/mcc/resources/referenceStudy/datasets/datasets_manifest.xml index 47d75b91c..352b7da94 100644 --- a/mcc/resources/referenceStudy/datasets/datasets_manifest.xml +++ b/mcc/resources/referenceStudy/datasets/datasets_manifest.xml @@ -45,5 +45,8 @@ + + + diff --git a/mcc/resources/referenceStudy/datasets/datasets_metadata.xml b/mcc/resources/referenceStudy/datasets/datasets_metadata.xml index 480c6a89d..9738324b3 100644 --- a/mcc/resources/referenceStudy/datasets/datasets_metadata.xml +++ b/mcc/resources/referenceStudy/datasets/datasets_metadata.xml @@ -643,4 +643,37 @@ Demographics + + + + varchar + http://cpas.labkey.com/Study#ParticipantId + + ptid + + + + timestamp + http://cpas.labkey.com/Study#VisitDate + http://cpas.labkey.com/Study#VisitDate + + + varchar + + + entityid + urn:ehr.labkey.org/#ObjectId + true + + + integer + urn:ehr.labkey.org/#Project + + + timestamp + urn:ehr.labkey.org/#EndDate + + + Departure +
From df8e16cd54ec28d77ee44d56a3d09b91ece07cb5 Mon Sep 17 00:00:00 2001 From: bbimber Date: Fri, 2 Apr 2021 06:02:37 -0700 Subject: [PATCH 96/98] MHC ETL improvements --- .../pipeline/MhcMigrationPipelineJob.java | 61 +++++++++++++------ 1 file changed, 43 insertions(+), 18 deletions(-) diff --git a/primeseq/src/org/labkey/primeseq/pipeline/MhcMigrationPipelineJob.java b/primeseq/src/org/labkey/primeseq/pipeline/MhcMigrationPipelineJob.java index 205944e5f..a2f5e4e08 100644 --- a/primeseq/src/org/labkey/primeseq/pipeline/MhcMigrationPipelineJob.java +++ b/primeseq/src/org/labkey/primeseq/pipeline/MhcMigrationPipelineJob.java @@ -230,6 +230,9 @@ public RecordedActionSet run() throws PipelineJobException syncAssay("GenotypeAssay", "Genotype", Arrays.asList("RowId", "Name", "Comments", "performedBy", "runDate", "instrument", "assayType", "barcode"), Arrays.asList("subjectId", "date", "marker", "result", "qual_result", "sampleId", "category", "plate", "well", "parentId", "comment", "requestid", "qcflag", "analysisId", "DataId", "sampleType", "statusflag", "rawResult")); transaction.commitAndKeepConnection(); + syncAssay("SSP_assay", "SSP", Arrays.asList("RowId", "Name", "Comments", "performedBy", "runDate"), Arrays.asList("subjectId", "date", "laneNumber", "method", "sampleType", "primerPair", "result", "comment", "qcflag", "statusflag")); + transaction.commitAndKeepConnection(); + createAlignmentSummary(); transaction.commit(); @@ -334,7 +337,7 @@ private void createQualityMetrics() throws PipelineJobException { for (int workbook : workbookMap.keySet()) { - replaceEntireTable("sequenceanalysis", "quality_metrics", Arrays.asList("dataid", "dataid/DatafileUrl", "runid/JobId/FilePath", "category", "metricname", "metricvalue", "qualvalue", "analysis_id", "readset", "readset/runid/JobId", "readset/runid/JobId/FilePath", "dataid/Run/JobId/FilePath"), null, true, workbookMap.get(workbook), getPipelineJob().remoteServerFolder + workbook + "/"); + replaceEntireTable("sequenceanalysis", "quality_metrics", Arrays.asList("dataid", "dataid/DatafileUrl", "dataid/Name", "runid/JobId/FilePath", "category", "metricname", "metricvalue", "qualvalue", "analysis_id", "readset", "readset/runid/JobId", "readset/runid/JobId/FilePath", "dataid/Run/JobId/FilePath"), null, true, workbookMap.get(workbook), getPipelineJob().remoteServerFolder + workbook + "/"); } } @@ -359,6 +362,18 @@ private void createAlignmentSummary() throws PipelineJobException sr.setColumns(Arrays.asList("rowid", "analysis_id", "file_id", "total", "total_forward", "total_reverse", "valid_pairs", "workbook/workbookId")); SelectRowsResponse srr = sr.execute(getConnection(), getPipelineJob().remoteServerFolder + workbook + "/"); + getJob().getLogger().info("total alignment_summary records: " + srr.getRowCount()); + if (srr.getRowCount().intValue() == 0) + { + continue; + } + + long existing = new TableSelector(alignmentSummary, new SimpleFilter(FieldKey.fromString("container"), workbookMap.get(workbook).getId()), null).getRowCount(); + if (srr.getRowCount().longValue() == existing) + { + getJob().getLogger().info("alignment_summary row count identical, skipping: " + workbook); + continue; + } final Map alignmentSummaryMap = new HashMap<>(srr.getRowCount().intValue()); srr.getRowset().forEach(rs -> { @@ -389,9 +404,11 @@ private void createAlignmentSummary() throws PipelineJobException }); SelectRowsCommand sr2 = new SelectRowsCommand("sequenceanalysis", "alignment_summary_junction"); - sr2.setColumns(Arrays.asList("analysis_id", "alignment_id", "ref_nt_id", "analysis_id/workbook/workbookId")); + sr2.setColumns(Arrays.asList("analysis_id", "alignment_id", "ref_nt_id", "status", "analysis_id/workbook/workbookId", "ref_nt_id/name")); + sr2.addFilter(new Filter("analysis_id/workbook/workbookId", workbook)); SelectRowsResponse srr2 = sr2.execute(getConnection(), getPipelineJob().remoteServerFolder + workbook + "/"); + getJob().getLogger().info("total alignment_summary_junction records: " + srr2.getRowCount()); srr2.getRowset().forEach(rs -> { CaseInsensitiveHashMap map = new CaseInsensitiveHashMap<>(); Integer localId = analysisMap.get(rs.getValue("analysis_id")); @@ -404,9 +421,10 @@ private void createAlignmentSummary() throws PipelineJobException Integer localNT = sequenceMap.get(rs.getValue("ref_nt_id")); if (localNT == null) { - throw new RuntimeException("Unable to find ref_nt_id: " + rs.getValue("ref_nt_id")); + throw new RuntimeException("Unable to find ref_nt_id: " + rs.getValue("ref_nt_id") + " / " + rs.getValue("ref_nt_id/name")); } map.put("ref_nt_id", localNT); + map.put("status", rs.getValue("status")); map.put("alignment_id", alignmentSummaryMap.get("alignment_id")); @@ -534,7 +552,7 @@ else if (r.getValue("dataid/Run/JobId/FilePath") != null) { URI localFileRoot = PipelineService.get().getPipelineRootSetting(targetContainer).getRootPath().toURI(); URI localPath = translateURI(String.valueOf(r.getValue("dataid/DatafileUrl")), remoteJobRoot, localFileRoot.getPath()); - row.put("dataid", getOrCreateExpData(localPath, targetContainer)); + row.put("dataid", getOrCreateExpData(localPath, targetContainer, String.valueOf(r.getValue("dataid/Name")))); } else { @@ -627,6 +645,10 @@ private void createLibraryMembers() } int localSeqId = getOrCreateSequence(remoteSeqId, name, seqLength, refNtTable); + if (localSeqId == -1) + { + return; + } int remoteLibraryId = Integer.parseInt(String.valueOf(rd.getValue("library_id"))); Integer localLibraryId = libraryMap.get(remoteLibraryId); @@ -690,8 +712,8 @@ private int getOrCreateSequence(int remoteSeqId, String name, int seqLength, Tab ts.forEachResults(rs -> { if (rs.getInt(FieldKey.fromString("seqLength")) < seqLength) { + //NOTE: accept these as most are trimmed getJob().getLogger().warn("length doesnt match for " + name + ", expected: " + seqLength + ", was: " + rs.getInt(FieldKey.fromString("seqLength"))); - return; } localId.set(rs.getInt(FieldKey.fromString("rowid"))); @@ -704,7 +726,6 @@ private int getOrCreateSequence(int remoteSeqId, String name, int seqLength, Tab } } - //throw new IllegalStateException("Expected sequence to exist: " + name); getJob().getLogger().error("Sequence missing: " + name); return -1; } @@ -731,7 +752,7 @@ private void createLibraries() final TableInfo libraryTable = QueryService.get().getUserSchema(getJob().getUser(), getPipelineJob().targetContainer, "sequenceanalysis").getTable("reference_libraries"); SelectRowsCommand sr = new SelectRowsCommand("sequenceanalysis", "reference_libraries"); - sr.setColumns(Arrays.asList("rowid", "name", "description", "fasta_file", "datedisabled", "assemblyId", "fasta_file/DataFileUrl", "workbook/workbookId")); + sr.setColumns(Arrays.asList("rowid", "name", "description", "fasta_file", "datedisabled", "assemblyId", "fasta_file/DataFileUrl", "fasta_file/Name", "workbook/workbookId")); SelectRowsResponse srr = sr.execute(getConnection(), getPipelineJob().remoteServerFolder); @@ -759,7 +780,7 @@ private void createLibraries() String remoteJobRoot = getParent(URI.create(String.valueOf(rd.getValue("fasta_file/DatafileUrl"))).getPath()); URI localJobRoot = PipelineService.get().getPipelineRootSetting(targetContainer).getRootPath().toURI(); URI localFasta = translateURI(String.valueOf(rd.getValue("fasta_file/DatafileUrl")), remoteJobRoot, localJobRoot.getPath()); - toCreate.put("fasta_file", getOrCreateExpData(localFasta, targetContainer)); + toCreate.put("fasta_file", getOrCreateExpData(localFasta, targetContainer, String.valueOf(rd.getValue("fasta_file/Name")))); //Ensure parent folder exists: File localJobRootFile = new File(localFasta).getParentFile(); @@ -810,7 +831,7 @@ private void createOutputFiles() final TableInfo outputTable = QueryService.get().getUserSchema(getJob().getUser(), getPipelineJob().targetContainer, "sequenceanalysis").getTable("outputfiles"); SelectRowsCommand sr = new SelectRowsCommand("sequenceanalysis", "outputfiles"); - sr.setColumns(Arrays.asList("rowid", "name", "description", "dataid", "library_id", "readset", "analysis_id", "category", "sra_accession", "dataid/DataFileUrl", "runid", "runid/JobId", "runid/Name", "workbook/workbookId", "runid/Name", "runid/JobId/FilePath")); + sr.setColumns(Arrays.asList("rowid", "name", "description", "dataid", "library_id", "readset", "analysis_id", "category", "sra_accession", "dataid/DataFileUrl", "dataid/Name", "runid", "runid/JobId", "runid/Name", "workbook/workbookId", "runid/Name", "runid/JobId/FilePath")); SelectRowsResponse srr = sr.execute(getConnection(), getPipelineJob().remoteServerFolder); @@ -885,7 +906,7 @@ private void createOutputFiles() String remoteJobRoot = getParent(URI.create(String.valueOf(rd.getValue("runid/JobId/FilePath")).replaceAll(" ", "%20")).getPath()); URI newFileAlignment = translateURI(String.valueOf(rd.getValue("dataid/DatafileUrl")), remoteJobRoot, localJobRoot); - toCreate.put("dataid", getOrCreateExpData(newFileAlignment, targetWorkbook)); + toCreate.put("dataid", getOrCreateExpData(newFileAlignment, targetWorkbook, String.valueOf(rd.getValue("dataid/Name")))); //Create run: if (rd.getValue("runid") != null && rd.getValue("runid/JobId") != null) @@ -929,7 +950,7 @@ private void createAnalyses() final TableInfo analysisTable = QueryService.get().getUserSchema(getJob().getUser(), getPipelineJob().targetContainer, "sequenceanalysis").getTable("sequence_analyses"); SelectRowsCommand sr = new SelectRowsCommand("sequenceanalysis", "sequence_analyses"); - sr.setColumns(Arrays.asList("rowid", "type", "description", "synopsis", "runid", "readset", "alignmentfile", "reference_library", "library_id", "sra_accession", "alignmentfile/DataFileUrl", "alignmentfile/Name", "reference_library", "reference_library/DataFileUrl", "runid/jobid", "runid/Name", "workbook/workbookId", "runid/JobId", "runid/Name", "runid/JobId/FilePath", "runid/JobId/Description")); + sr.setColumns(Arrays.asList("rowid", "type", "description", "synopsis", "runid", "readset", "alignmentfile", "reference_library", "library_id", "sra_accession", "alignmentfile/DataFileUrl", "alignmentfile/Name", "alignmentfile/Name", "reference_library", "reference_library/DataFileUrl", "reference_library/Name", "runid/jobid", "runid/Name", "workbook/workbookId", "runid/JobId", "runid/Name", "runid/JobId/FilePath", "runid/JobId/Description")); SelectRowsResponse srr = sr.execute(getConnection(), getPipelineJob().remoteServerFolder); @@ -990,6 +1011,8 @@ private void createAnalyses() toCreate.put("type", rd.getValue("type")); toCreate.put("description", rd.getValue("description")); toCreate.put("sra_accession", rd.getValue("sra_accession")); + Container workbook = workbookMap.get(rd.getValue("workbook/workbookId")); + toCreate.put("container", workbook.getId()); if (localLibrary != null) { toCreate.put("library_id", localLibrary); @@ -1011,12 +1034,12 @@ private void createAnalyses() String remoteJobRoot = getParent(URI.create(String.valueOf(rd.getValue("runid/JobId/FilePath")).replaceAll(" ", "%20")).getPath()); URI newFileAlignment = translateURI(String.valueOf(rd.getValue("alignmentfile/DatafileUrl")), remoteJobRoot, localJobRoot); - toCreate.put("alignmentfile", getOrCreateExpData(newFileAlignment, targetWorkbook)); + toCreate.put("alignmentfile", getOrCreateExpData(newFileAlignment, targetWorkbook, String.valueOf(rd.getValue("alignmentfile/Name")))); if (rd.getValue("reference_library") != null) { URI newFile2 = translateURI(String.valueOf(rd.getValue("reference_library/DatafileUrl")), remoteJobRoot, localJobRoot); - toCreate.put("reference_library", getOrCreateExpData(newFile2, targetWorkbook)); + toCreate.put("reference_library", getOrCreateExpData(newFile2, targetWorkbook, String.valueOf(rd.getValue("reference_library/Name")))); } //Create run: @@ -1064,7 +1087,7 @@ private void createReaddata() final TableInfo readdataTable = QueryService.get().getUserSchema(getJob().getUser(), workbookMap.get(workbookId), "sequenceanalysis").getTable("readdata"); SelectRowsCommand sr = new SelectRowsCommand("sequenceanalysis", "readdata"); - sr.setColumns(Arrays.asList("rowid", "readset", "platformUnit", "centerName", "date", "fileid1", "fileid1/DataFileUrl", "fileid2", "fileid2/DataFileUrl", "fileid1/Name", "description", "sra_accession", "runid", "runid/jobid", "runid/Name", "readset/workbook/workbookId", "runid/JobId", "runid/Name", "runid/JobId/FilePath", "runid/JobId/Description")); + sr.setColumns(Arrays.asList("rowid", "readset", "platformUnit", "centerName", "date", "fileid1", "fileid1/DataFileUrl", "fileid1/Name", "fileid2", "fileid2/DataFileUrl", "fileid2/Name", "description", "sra_accession", "runid", "runid/jobid", "runid/Name", "readset/workbook/workbookId", "runid/JobId", "runid/Name", "runid/JobId/FilePath", "runid/JobId/Description")); SelectRowsResponse srr = sr.execute(getConnection(), getPipelineJob().remoteServerFolder + workbookId + "/"); @@ -1117,6 +1140,8 @@ else if (srr.getRowCount().intValue() == 0) Map toCreate = new CaseInsensitiveHashMap<>(); toCreate.put("readset", rs.getRowId()); + Container workbook = workbookMap.get(rd.getValue("readset/workbook/workbookId")); + toCreate.put("container", workbook.getId()); toCreate.put("platformUnit", rd.getValue("platformUnit")); toCreate.put("centerName", rd.getValue("centerName")); toCreate.put("date", rd.getValue("date")); @@ -1136,13 +1161,13 @@ else if (srr.getRowCount().intValue() == 0) if (rd.getValue("fileid1/DataFileUrl") != null) { URI newFile1 = translateURI(String.valueOf(rd.getValue("fileid1/DataFileUrl")), remoteJobRoot, localJobRoot); - toCreate.put("fileid1", getOrCreateExpData(newFile1, targetWorkbook)); + toCreate.put("fileid1", getOrCreateExpData(newFile1, targetWorkbook, String.valueOf(rd.getValue("fileid1/Name")))); } if (rd.getValue("fileid2/DataFileUrl") != null) { URI newFile2 = translateURI(String.valueOf(rd.getValue("fileid2/DatafileUrl")), remoteJobRoot, localJobRoot); - toCreate.put("fileid2", getOrCreateExpData(newFile2, targetWorkbook)); + toCreate.put("fileid2", getOrCreateExpData(newFile2, targetWorkbook, String.valueOf(rd.getValue("fileid2/Name")))); } } else @@ -1193,12 +1218,12 @@ else if (srr.getRowCount().intValue() == 0) } } - private int getOrCreateExpData(URI file, Container workbook) + private int getOrCreateExpData(URI file, Container workbook, String fileName) { ExpData ret = ExperimentService.get().getExpDataByURL(new File(file), workbook); if (ret == null) { - ret = ExperimentService.get().createData(workbook, new DataType("Data")); + ret = ExperimentService.get().createData(workbook, new DataType("Data"), fileName); ret.setDataFileURI(file); ret.save(getJob().getUser()); } From 97673a5ccb5a313818e1df065984845e192bcc3d Mon Sep 17 00:00:00 2001 From: bbimber Date: Fri, 2 Apr 2021 13:01:21 -0700 Subject: [PATCH 97/98] Save Seurat variableFeatures to text file --- .../pipeline/MhcMigrationPipelineJob.java | 110 ++++++++++++++++-- 1 file changed, 99 insertions(+), 11 deletions(-) diff --git a/primeseq/src/org/labkey/primeseq/pipeline/MhcMigrationPipelineJob.java b/primeseq/src/org/labkey/primeseq/pipeline/MhcMigrationPipelineJob.java index a2f5e4e08..a2a58ae30 100644 --- a/primeseq/src/org/labkey/primeseq/pipeline/MhcMigrationPipelineJob.java +++ b/primeseq/src/org/labkey/primeseq/pipeline/MhcMigrationPipelineJob.java @@ -205,12 +205,14 @@ public RecordedActionSet run() throws PipelineJobException createWorkbooks(); transaction.commitAndKeepConnection(); + replaceEntireTable("genotypeassays", "primer_pairs", Arrays.asList("primername", "ref_nt_name", "ref_nt_id", "shortname"), null, true, getJob().getContainer(), getPipelineJob().remoteServerFolder); + replaceEntireTable("laboratory", "samples", Arrays.asList("samplename", "subjectid", "sampledate", "sampletype", "samplesubtype", "samplesource", "location", "freezer", "cane", "box", "box_row", "box_column", "comment", "workbook/workbookId", "samplespecies", "processdate", "concentration", "concentration_units", "quantity", "quantity_units", "ratio"), "workbook/workbookId", true, getJob().getContainer(), getPipelineJob().remoteServerFolder); replaceEntireTable("laboratory", "subjects", Arrays.asList("subjectname", "species"), null, true, getJob().getContainer(), getPipelineJob().remoteServerFolder); transaction.commitAndKeepConnection(); - createLibraries(); - createLibraryMembers(); + Set preExisting = createLibraries(); + createLibraryMembers(preExisting); transaction.commitAndKeepConnection(); createReadsets(); @@ -616,9 +618,11 @@ else if (r.getValue("dataid/Run/JobId/FilePath") != null) private final Map runIdMap = new HashMap<>(); private final Map jobIdMap = new HashMap<>(); - private void createLibraryMembers() + private void createLibraryMembers(Set preExisting) { getJob().getLogger().info("Creating library members"); + AtomicInteger totalCreated = new AtomicInteger(0); + AtomicInteger totalExisting = new AtomicInteger(0); final UserSchema us = QueryService.get().getUserSchema(getJob().getUser(), getPipelineJob().targetContainer, "sequenceanalysis"); final TableInfo ti = us.getTable("reference_library_members"); @@ -663,6 +667,7 @@ private void createLibraryMembers() if (new TableSelector(ti, PageFlowUtil.set("rowid"), filter, null).exists()) { //Already exists: + totalExisting.getAndIncrement(); return; } @@ -677,6 +682,7 @@ private void createLibraryMembers() { BatchValidationException bve = new BatchValidationException(); List> created = ti.getUpdateService().insertRows(getJob().getUser(), getPipelineJob().targetContainer, sequencesToCreate, bve, null, null); + totalCreated.getAndAdd(sequencesToCreate.size()); if (bve.hasErrors()) { throw new RuntimeException(bve); @@ -688,6 +694,8 @@ private void createLibraryMembers() getJob().getLogger().error(e.getMessage(), e); throw new RuntimeException(e); } + + getJob().getLogger().info("total created: " + totalCreated.get() + ", total existing: " + totalExisting.get()); } private int getOrCreateSequence(int remoteSeqId, String name, int seqLength, TableInfo refNtTable) @@ -744,9 +752,13 @@ public String getParent(String path) return path.substring(0, index); } - private void createLibraries() + private Set createLibraries() { getJob().getLogger().info("Creating libraries"); + AtomicInteger totalCreated = new AtomicInteger(0); + AtomicInteger totalExisting = new AtomicInteger(0); + Set preExisting = new HashSet<>(); + try { final TableInfo libraryTable = QueryService.get().getUserSchema(getJob().getUser(), getPipelineJob().targetContainer, "sequenceanalysis").getTable("reference_libraries"); @@ -766,7 +778,10 @@ private void createLibraries() TableSelector ts = new TableSelector(libraryTable, PageFlowUtil.set("rowid"), filter, null); if (ts.exists()) { + getJob().getLogger().info("Library exists: " + rd.getValue("name")); + preExisting.add(String.valueOf(rd.getValue("name"))); libraryMap.put(remoteId, ts.getObject(Integer.class)); + totalExisting.getAndIncrement(); } else { @@ -808,6 +823,7 @@ private void createLibraries() } libraryMap.put(remoteId, Integer.parseInt(String.valueOf(created.get(0).get("rowid")))); + totalCreated.getAndIncrement(); } catch (Exception e) { @@ -821,11 +837,18 @@ private void createLibraries() getJob().getLogger().error(e.getMessage(), e); throw new RuntimeException(e); } + + getJob().getLogger().info("total created: " + totalCreated.get() + ", total existing: " + totalExisting.get()); + + return preExisting; } private void createOutputFiles() { getJob().getLogger().info("Creating outputfiles"); + AtomicInteger totalCreated = new AtomicInteger(0); + AtomicInteger totalExisting = new AtomicInteger(0); + try { final TableInfo outputTable = QueryService.get().getUserSchema(getJob().getUser(), getPipelineJob().targetContainer, "sequenceanalysis").getTable("outputfiles"); @@ -879,6 +902,7 @@ private void createOutputFiles() if (tsOutputFiles.exists()) { outputFileMap.put(remoteId, tsOutputFiles.getObject(Integer.class)); + totalExisting.getAndIncrement(); } else { @@ -927,6 +951,7 @@ private void createOutputFiles() } outputFileMap.put(remoteId, Integer.parseInt(String.valueOf(created.get(0).get("rowid")))); + totalCreated.getAndIncrement(); } catch (Exception e) { @@ -940,11 +965,16 @@ private void createOutputFiles() getJob().getLogger().error(e.getMessage(), e); throw new RuntimeException(e); } + + getJob().getLogger().info("total created: " + totalCreated.get() + ", total existing: " + totalExisting.get()); } private void createAnalyses() { getJob().getLogger().info("Creating analyses"); + AtomicInteger totalCreated = new AtomicInteger(0); + AtomicInteger totalExisting = new AtomicInteger(0); + try { final TableInfo analysisTable = QueryService.get().getUserSchema(getJob().getUser(), getPipelineJob().targetContainer, "sequenceanalysis").getTable("sequence_analyses"); @@ -990,13 +1020,26 @@ private void createAnalyses() TableSelector tsAnalyses = new TableSelector(analysisTable, PageFlowUtil.set("rowid", "alignmentfile"), filter, null); if (tsAnalyses.exists()) { - Results results = tsAnalyses.getResults(); try { - analysisMap.put(remoteId, results.getInt("rowid")); - analysisToFileMap.put(results.getInt("rowid"), results.getInt("alignmentfile")); + tsAnalyses.forEachResults(results -> { + try + { + analysisMap.put(remoteId, results.getInt("rowid")); + if (results.getObject("alignmentfile") != null) + { + analysisToFileMap.put(results.getInt("rowid"), results.getInt("alignmentfile")); + } + } + catch (IndexOutOfBoundsException e) + { + throw new RuntimeException(e); + } + }); + + totalExisting.getAndIncrement(); } - catch (SQLException e) + catch (Exception e) { throw new RuntimeException(e); } @@ -1062,6 +1105,11 @@ private void createAnalyses() analysisMap.put(remoteId, Integer.parseInt(String.valueOf(created.get(0).get("rowid")))); analysisToJobPath.put(Integer.parseInt(String.valueOf(created.get(0).get("rowid"))), remoteJobRoot); + if (toCreate.get("alignmentfile") != null) + { + analysisToFileMap.put(remoteId, (int)toCreate.get("alignmentfile")); + } + totalCreated.getAndIncrement(); } catch (Exception e) { @@ -1075,11 +1123,16 @@ private void createAnalyses() getJob().getLogger().error(e.getMessage(), e); throw new RuntimeException(e); } + + getJob().getLogger().info("total created: " + totalCreated.get() + ", total existing: " + totalExisting.get()); } private void createReaddata() { getJob().getLogger().info("Creating read data"); + AtomicInteger totalCreated = new AtomicInteger(0); + AtomicInteger totalExisting = new AtomicInteger(0); + try { for (Integer workbookId : workbookMap.keySet()) @@ -1088,6 +1141,7 @@ private void createReaddata() SelectRowsCommand sr = new SelectRowsCommand("sequenceanalysis", "readdata"); sr.setColumns(Arrays.asList("rowid", "readset", "platformUnit", "centerName", "date", "fileid1", "fileid1/DataFileUrl", "fileid1/Name", "fileid2", "fileid2/DataFileUrl", "fileid2/Name", "description", "sra_accession", "runid", "runid/jobid", "runid/Name", "readset/workbook/workbookId", "runid/JobId", "runid/Name", "runid/JobId/FilePath", "runid/JobId/Description")); + sr.addFilter(new Filter("fileid1/DataFileUrl", null, Filter.Operator.NONBLANK)); SelectRowsResponse srr = sr.execute(getConnection(), getPipelineJob().remoteServerFolder + workbookId + "/"); @@ -1095,6 +1149,7 @@ private void createReaddata() if (srr.getRowCount().longValue() == existing) { getJob().getLogger().info("Readdata count identical, skipping: " + workbookId); + totalExisting.getAndAdd(srr.getRowCount().intValue()); continue; } else if (srr.getRowCount().intValue() == 0) @@ -1129,6 +1184,7 @@ else if (srr.getRowCount().intValue() == 0) if (tsReaddata.exists()) { readdataMap.put(remoteId, tsReaddata.getObject(Integer.class)); + totalExisting.getAndIncrement(); } else { @@ -1202,6 +1258,7 @@ else if (srr.getRowCount().intValue() == 0) } readdataMap.put(remoteId, Integer.parseInt(String.valueOf(created.get(0).get("rowid")))); + totalCreated.getAndIncrement(); } catch (Exception e) { @@ -1216,6 +1273,8 @@ else if (srr.getRowCount().intValue() == 0) getJob().getLogger().error(e.getMessage(), e); throw new RuntimeException(e); } + + getJob().getLogger().info("total created: " + totalCreated.get() + ", total existing: " + totalExisting.get()); } private int getOrCreateExpData(URI file, Container workbook, String fileName) @@ -1223,9 +1282,24 @@ private int getOrCreateExpData(URI file, Container workbook, String fileName) ExpData ret = ExperimentService.get().getExpDataByURL(new File(file), workbook); if (ret == null) { - ret = ExperimentService.get().createData(workbook, new DataType("Data"), fileName); - ret.setDataFileURI(file); - ret.save(getJob().getUser()); + String lsid = ExperimentService.get().generateLSID(workbook, new DataType("Data"), file.getPath()); + List datas = ExperimentService.get().getExpDatasByLSID(Collections.singleton(lsid)); + if (!datas.isEmpty()) + { + ret = datas.get(0); + if (!workbook.equals(ret.getContainer())) + { + throw new IllegalArgumentException("Expected datas to be from the same container: " + lsid); + } + } + + if (ret == null) + { + ret = ExperimentService.get().createData(workbook, new DataType("Data"), fileName); + ret.setDataFileURI(file); + ret.setLSID(lsid); + ret.save(getJob().getUser()); + } } return ret.getRowId(); @@ -1234,6 +1308,9 @@ private int getOrCreateExpData(URI file, Container workbook, String fileName) private void createReadsets() { getJob().getLogger().info("Creating readsets"); + AtomicInteger totalCreated = new AtomicInteger(0); + AtomicInteger totalExisting = new AtomicInteger(0); + try { final UserSchema us = QueryService.get().getUserSchema(getJob().getUser(), getPipelineJob().targetContainer, "sequenceanalysis"); @@ -1264,6 +1341,7 @@ private void createReadsets() if (tsReadset.exists()) { readsetMap.put(remoteId, tsReadset.getObject(Integer.class)); + totalExisting.getAndIncrement(); } else { @@ -1308,6 +1386,7 @@ private void createReadsets() } readsetMap.put(remoteId, Integer.parseInt(String.valueOf(created.get(0).get("rowid")))); + totalCreated.getAndIncrement(); } catch (Exception e) { @@ -1315,6 +1394,8 @@ private void createReadsets() } } }); + + getJob().getLogger().info("total created: " + totalCreated.get() + ", total existing: " + totalExisting.get()); } catch (Exception e) { @@ -1461,6 +1542,9 @@ private int createExpRun(int remoteId, Container c, String name, int localJobId) private void createWorkbooks() { getJob().getLogger().info("Creating workbooks"); + AtomicInteger totalCreated = new AtomicInteger(0); + AtomicInteger totalExisting = new AtomicInteger(0); + try { TableInfo containers = QueryService.get().getUserSchema(getJob().getUser(), getPipelineJob().targetContainer, "core").getTable("containers"); @@ -1477,6 +1561,7 @@ private void createWorkbooks() { Container workbook = ContainerManager.getForRowId(ts.getObject(Integer.class)); workbookMap.put(Integer.parseInt(String.valueOf(wb.getValue("Name"))), workbook); + totalExisting.getAndIncrement(); } else { @@ -1488,6 +1573,7 @@ private void createWorkbooks() Container workbook = ContainerManager.createContainer(getPipelineJob().targetContainer, null, localTitle, description, WorkbookContainerType.NAME, getJob().getUser()); workbook.setFolderType(FolderTypeManager.get().getFolderType("Expt Workbook"), getJob().getUser()); workbookMap.put(Integer.parseInt(String.valueOf(wb.getValue("Name"))), workbook); + totalCreated.getAndIncrement(); File sourceDir = new File("/home/groups/miSeqLK/Production/MHC_Typing", wb.getValue("Name") + "/@files"); File targetDir = pr.getRootPath(); @@ -1513,6 +1599,8 @@ private void createWorkbooks() { throw new RuntimeException(e); } + + getJob().getLogger().info("total created: " + totalCreated.get() + ", total existing: " + totalExisting.get()); } private URI translateURI(String databaseURI, String remoteFolderRoot, String localFolderRoot) From 2321e59068499eee3117b8b76c2b7e71b515e254 Mon Sep 17 00:00:00 2001 From: bbimber Date: Fri, 2 Apr 2021 14:49:58 -0700 Subject: [PATCH 98/98] Update ELISPOT import to reflect alternate format --- .../elispot_assay/assay/AIDImportMethod.java | 22 +++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/elispot_assay/src/org/labkey/elispot_assay/assay/AIDImportMethod.java b/elispot_assay/src/org/labkey/elispot_assay/assay/AIDImportMethod.java index 6606c1593..f99393759 100644 --- a/elispot_assay/src/org/labkey/elispot_assay/assay/AIDImportMethod.java +++ b/elispot_assay/src/org/labkey/elispot_assay/assay/AIDImportMethod.java @@ -22,6 +22,8 @@ import java.io.IOException; import java.io.StringWriter; import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; import java.util.LinkedHashMap; import java.util.List; import java.util.ListIterator; @@ -291,17 +293,19 @@ protected List> processRows(List> rows, private enum PLATE { - spot("Spot counts:", "spots"), - saturation("Well's saturation values (%)", "saturation"), - cytokine("Cytokine Activities:", "cytokine"); + spot("Spot counts:", "spots", Collections.singletonList("Number of Spots:")), + saturation("Well's saturation values (%)", "saturation", Collections.singletonList("Well's saturation values (%)")), + cytokine("Cytokine Activities:", "cytokine", Collections.singletonList("Activity:")); private String description; private String field; + private Collection aliases; - PLATE(String description, String field) + PLATE(String description, String field, Collection aliases) { this.description = description; this.field = field; + this.aliases = aliases; } public static PLATE getByDescription(String description) @@ -310,6 +314,16 @@ public static PLATE getByDescription(String description) { if (t.description.equalsIgnoreCase(description)) return t; + else if (t.aliases != null) + { + for (String alias : t.aliases) + { + if (alias.equals(description)) + { + return t; + } + } + } } return null; }