diff --git a/mGAP/resources/credits/dependencies.txt b/mGAP/resources/credits/dependencies.txt new file mode 100644 index 000000000..67fa439e0 --- /dev/null +++ b/mGAP/resources/credits/dependencies.txt @@ -0,0 +1,2 @@ +# direct external dependencies for project :server:modules:BimberLabKeyModules:mGAP +htsjdk-2.14.3.jar diff --git a/mGAP/resources/etls/prime-seq.xml b/mGAP/resources/etls/prime-seq.xml index 5cb301f89..1c8e2d3aa 100644 --- a/mGAP/resources/etls/prime-seq.xml +++ b/mGAP/resources/etls/prime-seq.xml @@ -11,9 +11,10 @@ gender species geographic_origin + center - + diff --git a/mGAP/resources/queries/mGAP/subjectsSource.query.xml b/mGAP/resources/queries/mGAP/subjectsSource.query.xml new file mode 100644 index 000000000..6185f7741 --- /dev/null +++ b/mGAP/resources/queries/mGAP/subjectsSource.query.xml @@ -0,0 +1,9 @@ + + + + + subjectName +
+ + + \ No newline at end of file diff --git a/mGAP/resources/queries/mGAP/subjectsSource.sql b/mGAP/resources/queries/mGAP/subjectsSource.sql index d57ede557..861373ce5 100644 --- a/mGAP/resources/queries/mGAP/subjectsSource.sql +++ b/mGAP/resources/queries/mGAP/subjectsSource.sql @@ -1,8 +1,15 @@ SELECT m.externalAlias as subjectName, - s.gender, - s.species, - s.geographic_origin + coalesce(s.gender, d.gender) as gender, + coalesce(s.species, d.species) as species, + coalesce(s.geographic_origin, d.geographic_origin) as geographic_origin, + CASE + WHEN d.center IS NOT NULL THEN d.center + WHEN s.subjectname IS NOT NULL THEN 'ONPRC' + ELSE NULL END as center, + d.status as status FROM mgap.animalMapping m -JOIN laboratory.subjects s ON (m.subjectname = s.subjectname) \ No newline at end of file +LEFT JOIN laboratory.subjects s ON (m.subjectname = s.subjectname) +LEFT JOIN mgap.demographics d ON (m.subjectname = d.subjectname) +WHERE (s.subjectname IS NOT NULL OR d.subjectname IS NOT NULL) \ No newline at end of file diff --git a/mGAP/resources/schemas/dbscripts/postgresql/mgap-16.55-16.56.sql b/mGAP/resources/schemas/dbscripts/postgresql/mgap-16.55-16.56.sql new file mode 100644 index 000000000..0d8173feb --- /dev/null +++ b/mGAP/resources/schemas/dbscripts/postgresql/mgap-16.55-16.56.sql @@ -0,0 +1,17 @@ +CREATE TABLE mGAP.demographics ( + rowid serial, + subjectname varchar(100), + species varchar(100), + gender varchar(100), + geographic_origin varchar(100), + center varchar(1000), + status varchar(1000), + + container entityid, + created timestamp, + createdby userid, + modified timestamp, + modifiedby userid, + + CONSTRAINT PK_demographics PRIMARY KEY (rowid) +); \ No newline at end of file diff --git a/mGAP/resources/schemas/dbscripts/sqlserver/mgap-16.55-16.56.sql b/mGAP/resources/schemas/dbscripts/sqlserver/mgap-16.55-16.56.sql new file mode 100644 index 000000000..6480e5012 --- /dev/null +++ b/mGAP/resources/schemas/dbscripts/sqlserver/mgap-16.55-16.56.sql @@ -0,0 +1,17 @@ +CREATE TABLE mGAP.demographics ( + rowid int identity(1,1), + subjectname varchar(100), + species varchar(100), + gender varchar(100), + geographic_origin varchar(100), + center varchar(1000), + status varchar(1000), + + container entityid, + created datetime, + createdby userid, + modified datetime, + modifiedby userid, + + CONSTRAINT PK_demographics PRIMARY KEY (rowid) +); \ No newline at end of file diff --git a/mGAP/resources/schemas/mgap.xml b/mGAP/resources/schemas/mgap.xml index f97c13248..bdcfa044e 100644 --- a/mGAP/resources/schemas/mgap.xml +++ b/mGAP/resources/schemas/mgap.xml @@ -962,4 +962,80 @@ + + + rowid + Supplemental Demographics + DETAILED + + + true + false + false + false + Row Id + + + Subject Name + false + + + Species + + laboratory + species + common_name + + + + + Gender + + laboratory + genders + code + + + + + Geographic Origin + + laboratory + geographic_origins + origin + + + + + Center + + + Status + + + true + + + true + + + false + false + false + true + true + + + true + + + false + false + false + true + true + + +
+ \ No newline at end of file diff --git a/mGAP/resources/views/admin.html b/mGAP/resources/views/admin.html index 9b0301e38..e08d39564 100644 --- a/mGAP/resources/views/admin.html +++ b/mGAP/resources/views/admin.html @@ -22,10 +22,6 @@ xtype: 'ldk-linkbutton', text: 'Manage JBrowse Tracks', href: LABKEY.ActionURL.buildURL('query', 'executeQuery', null, {schemaName: 'mgap', queryName: 'releaseTracks'}) - },{ - xtype: 'ldk-linkbutton', - text: 'Manage Animal/Track Subsets', - href: LABKEY.ActionURL.buildURL('query', 'executeQuery', null, {schemaName: 'mgap', queryName: 'releaseTrackSubsets'}) }] }); diff --git a/mGAP/src/org/labkey/mgap/columnTransforms/JBrowseSessionTransform.java b/mGAP/src/org/labkey/mgap/columnTransforms/JBrowseSessionTransform.java index 2caaf0181..c70766cd3 100644 --- a/mGAP/src/org/labkey/mgap/columnTransforms/JBrowseSessionTransform.java +++ b/mGAP/src/org/labkey/mgap/columnTransforms/JBrowseSessionTransform.java @@ -19,7 +19,6 @@ import org.labkey.api.query.QueryService; import org.labkey.api.query.UserSchema; import org.labkey.api.util.GUID; -import org.labkey.api.util.JobRunner; import org.labkey.api.util.PageFlowUtil; import org.labkey.mgap.mGAPSchema; @@ -310,6 +309,6 @@ protected String getDatabaseName() protected String getTrackJson() { - return "{\"category\":\"mGAP Variant Catalog\",\"visibleByDefault\": true,\"ensemblId\":\"Macaca_mulatta\",\"additionalFeatureMsg\":\"

**The annotations below are primarily derived from human data sources (not macaque), and must be viewed in that context.

\"}"; + return "{\"category\":\"mGAP Variant Catalog\",\"visibleByDefault\": true,\",\"ensemblUrl\":\"jul2019.archive.ensembl.org\",\"ensemblId\":\"Macaca_mulatta\",\"additionalFeatureMsg\":\"

**The annotations below are primarily derived from human data sources (not macaque), and must be viewed in that context.

\"}"; } } diff --git a/mGAP/src/org/labkey/mgap/mGAPDemographicsSource.java b/mGAP/src/org/labkey/mgap/mGAPDemographicsSource.java new file mode 100644 index 000000000..df1cdc01d --- /dev/null +++ b/mGAP/src/org/labkey/mgap/mGAPDemographicsSource.java @@ -0,0 +1,65 @@ +package org.labkey.mgap; + +import org.labkey.api.collections.CaseInsensitiveHashMap; +import org.labkey.api.data.CompareType; +import org.labkey.api.data.Container; +import org.labkey.api.data.SimpleFilter; +import org.labkey.api.data.TableInfo; +import org.labkey.api.data.TableSelector; +import org.labkey.api.jbrowse.DemographicsSource; +import org.labkey.api.module.ModuleLoader; +import org.labkey.api.query.FieldKey; +import org.labkey.api.query.QueryService; +import org.labkey.api.security.User; + +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +public class mGAPDemographicsSource implements DemographicsSource +{ + + @Override + public Map> resolveSubjects(List subjects, Container c, User u) + { + Map> ret = new HashMap<>(); + + TableInfo ti = QueryService.get().getUserSchema(u, c, mGAPSchema.NAME).getTable(mGAPSchema.TABLE_DEMOGRAPHICS); + Set fields = new LinkedHashSet<>(getFields().keySet()); + SimpleFilter filter = new SimpleFilter(FieldKey.fromString("subjectname"), subjects, CompareType.IN); + fields.add("subjectname"); + new TableSelector(ti, fields, filter, null).forEachResults(rs -> { + Map map = new CaseInsensitiveHashMap<>(); + for (String field : getFields().keySet()) + { + map.put(field, rs.getObject(FieldKey.fromString(field))); + } + + ret.put(rs.getString(FieldKey.fromString("subjectname")), map); + }); + + return ret; + } + + @Override + public LinkedHashMap getFields() + { + LinkedHashMap ret = new LinkedHashMap(); + ret.put("gender", "Gender"); + ret.put("species", "Species"); + ret.put("center", "Center"); + ret.put("geographic_origin", "Geographic Origin"); + ret.put("status", "Status"); + + return ret; + } + + @Override + public boolean isAvailable(Container c, User u) + { + return c.getActiveModules().contains(ModuleLoader.getInstance().getModule(mGAPModule.class)); + } +} diff --git a/mGAP/src/org/labkey/mgap/mGAPModule.java b/mGAP/src/org/labkey/mgap/mGAPModule.java index 684c6464b..84c880582 100644 --- a/mGAP/src/org/labkey/mgap/mGAPModule.java +++ b/mGAP/src/org/labkey/mgap/mGAPModule.java @@ -24,6 +24,7 @@ import org.labkey.api.data.SimpleFilter; import org.labkey.api.data.Sort; import org.labkey.api.data.TableSelector; +import org.labkey.api.jbrowse.JBrowseService; import org.labkey.api.ldk.ExtendedSimpleModule; import org.labkey.api.ldk.LDKService; import org.labkey.api.ldk.buttons.ShowBulkEditButton; @@ -57,7 +58,7 @@ public String getName() @Override public double getVersion() { - return 16.55; + return 16.56; } @Override @@ -78,6 +79,8 @@ public void doStartupAfterSpringConfig(ModuleContext moduleContext) NotificationService.get().registerNotification(new mGAPUserNotification(this)); + JBrowseService.get().registerDemographicsSource(new mGAPDemographicsSource()); + new PipelineStartup(); } diff --git a/mGAP/src/org/labkey/mgap/mGAPSchema.java b/mGAP/src/org/labkey/mgap/mGAPSchema.java index 50f461b6d..01eb54cf8 100644 --- a/mGAP/src/org/labkey/mgap/mGAPSchema.java +++ b/mGAP/src/org/labkey/mgap/mGAPSchema.java @@ -34,6 +34,8 @@ public class mGAPSchema public static final String TABLE_TRACKS_PER_RELEASE = "tracksPerRelease"; public static final String TABLE_PHENOTYPES = "phenotypes"; public static final String TABLE_PEDIGREE_OVERRIDES = "pedigreeOverrides"; + public static final String TABLE_DEMOGRAPHICS = "demographics"; + public static final String TABLE_SUBJECT_SOURCE = "subjectsSource"; public static mGAPSchema getInstance() diff --git a/mGAP/src/org/labkey/mgap/pipeline/mGapReleaseGenerator.java b/mGAP/src/org/labkey/mgap/pipeline/mGapReleaseGenerator.java index f5058b6c9..d2cd93867 100644 --- a/mGAP/src/org/labkey/mgap/pipeline/mGapReleaseGenerator.java +++ b/mGAP/src/org/labkey/mgap/pipeline/mGapReleaseGenerator.java @@ -165,7 +165,7 @@ public Processor() public void init(PipelineJob job, SequenceAnalysisJobSupport support, List inputFiles, JSONObject params, File outputDir, List actions, List outputsToCreate) throws UnsupportedOperationException, PipelineJobException { job.getLogger().info("writing track/subset data to file"); - TableInfo releaseTrackSubsets = QueryService.get().getUserSchema(job.getUser(), (job.getContainer().isWorkbook() ? job.getContainer().getParent() : job.getContainer()), mGAPSchema.NAME).getTable(mGAPSchema.TABLE_RELEASE_TRACKS); + TableInfo releaseTracks = QueryService.get().getUserSchema(job.getUser(), (job.getContainer().isWorkbook() ? job.getContainer().getParent() : job.getContainer()), mGAPSchema.NAME).getTable(mGAPSchema.TABLE_RELEASE_TRACKS); Set toSelect = new HashSet<>(); toSelect.add(FieldKey.fromString("trackName")); @@ -174,13 +174,13 @@ public void init(PipelineJob job, SequenceAnalysisJobSupport support, List colMap = QueryService.get().getColumns(releaseTrackSubsets, toSelect); + Map colMap = QueryService.get().getColumns(releaseTracks, toSelect); Set distinctTracks = new HashSet<>(); File trackFile = getTrackListFile(outputDir); try (CSVWriter writer = new CSVWriter(PrintWriters.getPrintWriter(trackFile), '\t', CSVWriter.NO_QUOTE_CHARACTER)) { - new TableSelector(releaseTrackSubsets, colMap.values(), null, null).forEachResults(rs -> { + new TableSelector(releaseTracks, colMap.values(), null, null).forEachResults(rs -> { if (rs.getObject(FieldKey.fromString("vcfId")) == null) { boolean isPrimary = rs.getObject(FieldKey.fromString("isprimarytrack")) != null && rs.getBoolean(FieldKey.fromString("isprimarytrack")); diff --git a/mGAP/src/org/labkey/mgap/query/SourceDisplayColumnFactory.java b/mGAP/src/org/labkey/mgap/query/SourceDisplayColumnFactory.java index be204834b..4b91754b3 100644 --- a/mGAP/src/org/labkey/mgap/query/SourceDisplayColumnFactory.java +++ b/mGAP/src/org/labkey/mgap/query/SourceDisplayColumnFactory.java @@ -1,6 +1,7 @@ package org.labkey.mgap.query; import org.apache.commons.lang3.StringUtils; +import org.apache.log4j.Logger; import org.labkey.api.data.ColumnInfo; import org.labkey.api.data.DataColumn; import org.labkey.api.data.DisplayColumn; @@ -14,6 +15,8 @@ public class SourceDisplayColumnFactory implements DisplayColumnFactory { + private static final Logger _log = Logger.getLogger(SourceDisplayColumnFactory.class); + @Override public DisplayColumn createRenderer(ColumnInfo colInfo) { @@ -40,14 +43,21 @@ public void renderGridCellContents(RenderContext ctx, Writer out) throws IOExcep if (identifier != null && identifier.contains(":")) { String[] parts = identifier.split(":"); - switch (parts[0]) + if (parts.length != 2) + { + _log.error("Invalid variant identifier: " + val, new Exception()); + } + else { - case "ClinVar": - if (!StringUtils.isEmpty(parts[1])) - { - url = "https://www.ncbi.nlm.nih.gov/clinvar/variation/" + parts[1] + "/"; - } - break; + switch (parts[0]) + { + case "ClinVar": + if (!StringUtils.isEmpty(parts[1])) + { + url = "https://www.ncbi.nlm.nih.gov/clinvar/variation/" + parts[1] + "/"; + } + break; + } } } diff --git a/mGAP/src/org/labkey/mgap/query/VariantListJBrowseDisplayColumnFactory.java b/mGAP/src/org/labkey/mgap/query/VariantListJBrowseDisplayColumnFactory.java index 8b6ae3ec3..f4c18a5b0 100644 --- a/mGAP/src/org/labkey/mgap/query/VariantListJBrowseDisplayColumnFactory.java +++ b/mGAP/src/org/labkey/mgap/query/VariantListJBrowseDisplayColumnFactory.java @@ -99,7 +99,7 @@ public void renderGridCellContents(RenderContext ctx, Writer out) throws IOExcep //Ensembl does use chr or padded names. String contigE = contig.replaceAll("chr", ""); contigE = contigE.replaceAll("^0", ""); - String url = "https://www.ensembl.org/Macaca_mulatta/Location/View?db=core;r=" + contigE + ":" + start +"-" + stop; + String url = "https://jul2019.archive.ensembl.org/Macaca_mulatta/Location/View?db=core;r=" + contigE + ":" + start +"-" + stop; out.write(delim); out.write("View Region in Ensembl"); } diff --git a/mGAP/src/org/labkey/mgap/query/mGAPUserSchema.java b/mGAP/src/org/labkey/mgap/query/mGAPUserSchema.java index 4e5053908..c2e8923b2 100644 --- a/mGAP/src/org/labkey/mgap/query/mGAPUserSchema.java +++ b/mGAP/src/org/labkey/mgap/query/mGAPUserSchema.java @@ -47,6 +47,10 @@ protected TableInfo createWrappedTable(String name, @NotNull TableInfo sourceTab // TODO: assert cf is null or not default? return new ContainerScopedTable<>(this, sourceTable, cf, "subjectname").init(); } + else if (mGAPSchema.TABLE_DEMOGRAPHICS.equalsIgnoreCase(name)) + { + return new ContainerScopedTable<>(this, sourceTable, cf, "subjectname").init(); + } else if (mGAPSchema.TABLE_VARIANT_CATALOG_RELEASES.equalsIgnoreCase(name)) { return createWrappedVariantTable(name, sourceTable, cf); diff --git a/primeseq/src/org/labkey/primeseq/pipeline/BismarkWrapper.java b/primeseq/src/org/labkey/primeseq/pipeline/BismarkWrapper.java index cbfbd11de..a3577324b 100644 --- a/primeseq/src/org/labkey/primeseq/pipeline/BismarkWrapper.java +++ b/primeseq/src/org/labkey/primeseq/pipeline/BismarkWrapper.java @@ -27,6 +27,7 @@ import org.labkey.api.sequenceanalysis.pipeline.AlignerIndexUtil; import org.labkey.api.sequenceanalysis.pipeline.AlignmentOutputImpl; import org.labkey.api.sequenceanalysis.pipeline.AlignmentStep; +import org.labkey.api.sequenceanalysis.pipeline.AlignmentStepProvider; import org.labkey.api.sequenceanalysis.pipeline.AnalysisOutputImpl; import org.labkey.api.sequenceanalysis.pipeline.AnalysisStep; import org.labkey.api.sequenceanalysis.pipeline.CommandLineParam; @@ -39,6 +40,7 @@ import org.labkey.api.sequenceanalysis.pipeline.SequenceOutputTracker; import org.labkey.api.sequenceanalysis.pipeline.SequencePipelineService; import org.labkey.api.sequenceanalysis.pipeline.ToolParameterDescriptor; +import org.labkey.api.sequenceanalysis.run.AbstractAlignmentPipelineStep; import org.labkey.api.sequenceanalysis.run.AbstractCommandPipelineStep; import org.labkey.api.sequenceanalysis.run.AbstractCommandWrapper; import org.labkey.api.util.FileUtil; @@ -78,9 +80,9 @@ public BismarkWrapper(@Nullable Logger logger) super(logger); } - public static class BismarkAlignmentStep extends AbstractCommandPipelineStep implements AlignmentStep + public static class BismarkAlignmentStep extends AbstractAlignmentPipelineStep implements AlignmentStep { - public BismarkAlignmentStep(PipelineStepProvider provider, PipelineContext ctx) + public BismarkAlignmentStep(AlignmentStepProvider provider, PipelineContext ctx) { super(provider, ctx, new BismarkWrapper(ctx.getLogger())); } diff --git a/primeseq/src/org/labkey/primeseq/pipeline/ExacloudResourceSettings.java b/primeseq/src/org/labkey/primeseq/pipeline/ExacloudResourceSettings.java index 8be1e055e..68331d0cf 100644 --- a/primeseq/src/org/labkey/primeseq/pipeline/ExacloudResourceSettings.java +++ b/primeseq/src/org/labkey/primeseq/pipeline/ExacloudResourceSettings.java @@ -1,10 +1,10 @@ package org.labkey.primeseq.pipeline; -import org.labkey.primeseq.PrimeseqModule; import org.labkey.api.data.Container; import org.labkey.api.module.ModuleLoader; import org.labkey.api.sequenceanalysis.pipeline.JobResourceSettings; import org.labkey.api.sequenceanalysis.pipeline.ToolParameterDescriptor; +import org.labkey.primeseq.PrimeseqModule; import java.util.Arrays; import java.util.List; @@ -22,7 +22,7 @@ public List getParams() ToolParameterDescriptor.create("ram", "RAM (GB)", "The RAM requested for this job", "ldk-integerfield", null, null), ToolParameterDescriptor.create("weekLongJob", "Expect To Run More Than 24H", "Check this if you expect the job to run more than 24H. This will add the long_jobs flag to the submit script", "checkbox", null, null), ToolParameterDescriptor.create("veryLongJob", "Expect To Run More Than 10 Days", "Check this if you expect the job to run more than 10 Days. This will add the very_long_jobs flag to the submit script", "checkbox", null, null), - //ToolParameterDescriptor.create("highio", "Use The HighIO Queue", "If this is checked, the job will be submitted to the high IO queue, which is a way to titrate the maximum number of concurrent jobs to 60. This is superseded by WEEK_LONG_JOBS", "checkbox", null, null), + ToolParameterDescriptor.create("time", "Requested Time (days/hours)", "If provided, this is passed to the --time argument. This cannot be higher than the limit for your requested partition. Examples are: '2124', '1-0' (1 day, 0 hours), '10-0' (10 days), and '0-72' (72 hours). If left blank, this will be automatically assigned.", "textfield", null, null), ToolParameterDescriptor.create("javaProcessXmx", "Java Process Xmx (GB)", "The value to be used as -Xmx for the LabKey remote java process. Unless you have a good reason, do not change this", "ldk-integerfield", null, null) ); } diff --git a/primeseq/src/org/labkey/primeseq/pipeline/SequenceJobResourceAllocator.java b/primeseq/src/org/labkey/primeseq/pipeline/SequenceJobResourceAllocator.java index da33ce341..21d9a7b21 100644 --- a/primeseq/src/org/labkey/primeseq/pipeline/SequenceJobResourceAllocator.java +++ b/primeseq/src/org/labkey/primeseq/pipeline/SequenceJobResourceAllocator.java @@ -280,7 +280,7 @@ public void addExtraSubmitScriptLines(PipelineJob job, RemoteExecutionEngine eng } } - private void removeExistingPartition(List lines, boolean removeTime) + private void removeLines(List lines, boolean removeTime) { lines.removeIf(line -> line.contains("#SBATCH --qos=")); @@ -290,8 +290,28 @@ private void removeExistingPartition(List lines, boolean removeTime) } } + private String getTime(PipelineJob job) + { + Map params = ((HasJobParams)job).getJobParams(); + if (params.get("resourceSettings.resourceSettings.time") != null) + { + return StringUtils.trimToNull(params.get("resourceSettings.resourceSettings.time")); + } + + return null; + } + private void possiblyAddWeekLongLines(PipelineJob job, RemoteExecutionEngine engine, List lines) { + String time = getTime(job); + if (time != null) + { + job.getLogger().debug("adding user-supplied time to job: " + time); + removeLines(lines, true); + + lines.add("#SBATCH --time=" + time); + } + Map params = ((HasJobParams)job).getJobParams(); if (params.get("resourceSettings.resourceSettings.weekLongJob") != null) { @@ -312,11 +332,11 @@ else if (engine.getType().equals("SlurmEngine")) //Note: consider supporting --time, which allows request of a shorter duration job //first remove existing - removeExistingPartition(lines, true); + removeLines(lines, true); //then add lines.add("#SBATCH --qos=long_jobs"); - lines.add("#SBATCH --time=10-0"); //10 days + lines.add("#SBATCH --time=" + (time == null ? "10-0" : time)); //10 days } } } @@ -330,72 +350,16 @@ else if (engine.getType().equals("SlurmEngine")) if (engine.getType().equals("SlurmEngine")) { //first remove existing - removeExistingPartition(lines, true); + removeLines(lines, true); //then add lines.add("#SBATCH --qos=very_long_jobs"); - lines.add("#SBATCH --time=30-0"); //30 days + lines.add("#SBATCH --time=" + (time == null ? "30-0" : time)); //30 days } } } } - private boolean jobProvidedCpusOrRam(PipelineJob job) - { - if (job instanceof HasJobParams) - { - Map params = ((HasJobParams) job).getJobParams(); - if (StringUtils.trimToNull(params.get("resourceSettings.resourceSettings.cpus")) != null) - { - return true; - } - else if (StringUtils.trimToNull(params.get("resourceSettings.resourceSettings.ram")) != null) - { - return true; - } - } - - return false; - } - -// private boolean getHighIOValue(PipelineJob job) -// { -// Map params = ((HasJobParams) job).getJobParams(); -// if (params.get("resourceSettings.resourceSettings.highio") != null) -// { -// return ConvertHelper.convert(params.get("resourceSettings.resourceSettings.highio"), Boolean.class); -// } -// -// return false; -// } -// -// private void possiblyAddHighIoFlag(PipelineJob job, RemoteExecutionEngine engine, List lines) -// { -// if (job instanceof HasJobParams) -// { -// boolean highio = getHighIOValue(job); -// if (highio) -// { -// job.getLogger().debug("adding highio as supplied by job"); -// if (engine.getType().equals("HTCondorEngine")) -// { -// lines.add("concurrency_limits = highio"); -// return; -// } -// else if (engine.getType().equals("SlurmEngine")) -// { -// removeExistingPartition(lines, false); -// -// lines.add("#SBATCH --partition=highio"); -// } -// else -// { -// job.getLogger().debug("HighIO was selected, but it is not supported on this cluster type: " + engine.getType()); -// } -// } -// } -// } - private Long getFileSize(PipelineJob job) { if (_totalFileSize != null) diff --git a/tcrdb/build.gradle b/tcrdb/build.gradle index d21832355..1fce9e48e 100644 --- a/tcrdb/build.gradle +++ b/tcrdb/build.gradle @@ -4,7 +4,7 @@ dependencies { BuildUtils.addLabKeyDependency(project: project, config: "compile", depProjectPath: ":server:modules:DiscvrLabKeyModules:SequenceAnalysis", depProjectConfig: "apiCompile") BuildUtils.addLabKeyDependency(project: project, config: "compile", depProjectPath: ":server:modules:LabDevKitModules:laboratory", depProjectConfig: "apiCompile") BuildUtils.addLabKeyDependency(project: project, config: "compile", depProjectPath: ":server:modules:LabDevKitModules:LDK", depProjectConfig: "apiCompile") - BuildUtils.addLabKeyDependency(project: project, config: "compile", depProjectPath: BuildUtils.getPlatformModuleProjectPath(project.gradle, "assay"), depProjectConfig: "apiCompile") + BuildUtils.addLabKeyDependency(project: project, config: "compile", depProjectPath: BuildUtils.getPlatformModuleProjectPath(project.gradle, "assay"), depProjectConfig: "apiCompile") external "io.repseq:repseqio:${repseqVersion}" } diff --git a/tcrdb/gradle.properties b/tcrdb/gradle.properties index b0625b077..3bf722a8c 100644 --- a/tcrdb/gradle.properties +++ b/tcrdb/gradle.properties @@ -1,2 +1,3 @@ +milibVersion=1.11 mixcrVersion=2.0.3 repseqVersion=1.3.2 \ No newline at end of file diff --git a/tcrdb/resources/assay/TCRdb/queries/SegmentsByClone.sql b/tcrdb/resources/assay/TCRdb/queries/SegmentsByClone.sql new file mode 100644 index 000000000..93c383212 --- /dev/null +++ b/tcrdb/resources/assay/TCRdb/queries/SegmentsByClone.sql @@ -0,0 +1,58 @@ +SELECT + c.rowid, + c.cloneName, + c.chain, + c.cdr3, + + v.hits as vHits, + v.hitCount as vHitCount, + + j.hits as jHits, + j.hitCount as jHitCount, + + ct.hits as cHits, + ct.hitCount as cHitCount + + +FROM tcrdb.clones c + +LEFT JOIN ( + SELECT t.cdr3, group_concat(distinct t.hit, char(10)) as hits, group_concat(distinct (t.hit || ': ' || cast(t.total as varchar(100))), char(10)) as hitCount + FROM ( + SELECT d.cdr3, + d.vHit as hit, + sum("count") as total + FROM Data d + GROUP BY d.cdr3, d.vHit + HAVING count(*) > 1 AND sum("count") > 100 + ) t GROUP BY t.cdr3 +) v ON (v.cdr3 = c.cdr3) + +LEFT JOIN ( + SELECT t.cdr3, group_concat(distinct t.hit, char(10)) as hits, group_concat(distinct (t.hit || ': ' || cast(t.total as varchar(100))), char(10)) as hitCount + FROM ( + SELECT d.cdr3, + d.jHit as hit, + sum("count") as total + FROM Data d + GROUP BY d.cdr3, d.jHit + HAVING count(*) > 1 AND sum("count") > 100 + ) t GROUP BY t.cdr3 +) j ON (j.cdr3 = c.cdr3) + +LEFT JOIN ( + SELECT t.cdr3, group_concat(distinct t.hit, char(10)) as hits, group_concat(distinct (t.hit || ': ' || cast(t.total as varchar(100))), char(10)) as hitCount + FROM ( + SELECT d.cdr3, + d.cHit as hit, + sum("count") as total + FROM Data d + GROUP BY d.cdr3, d.cHit + HAVING count(*) > 1 AND sum("count") > 100 + ) t GROUP BY t.cdr3 +) ct ON (ct.cdr3 = c.cdr3) + + + + + diff --git a/tcrdb/resources/credits/dependencies.txt b/tcrdb/resources/credits/dependencies.txt new file mode 100644 index 000000000..44527e2e7 --- /dev/null +++ b/tcrdb/resources/credits/dependencies.txt @@ -0,0 +1,2 @@ +# direct external dependencies for project :server:modules:BimberLabKeyModules:tcrdb +repseqio-1.3.2.jar diff --git a/tcrdb/resources/schemas/dbscripts/postgresql/tcrdb-15.46-15.47.sql b/tcrdb/resources/schemas/dbscripts/postgresql/tcrdb-15.46-15.47.sql new file mode 100644 index 000000000..a98ba4268 --- /dev/null +++ b/tcrdb/resources/schemas/dbscripts/postgresql/tcrdb-15.46-15.47.sql @@ -0,0 +1 @@ +ALTER TABLE tcrdb.mixcr_libraries ADD libraryId int; \ No newline at end of file diff --git a/tcrdb/resources/schemas/dbscripts/sqlserver/tcrdb-15.46-15.47.sql b/tcrdb/resources/schemas/dbscripts/sqlserver/tcrdb-15.46-15.47.sql new file mode 100644 index 000000000..a98ba4268 --- /dev/null +++ b/tcrdb/resources/schemas/dbscripts/sqlserver/tcrdb-15.46-15.47.sql @@ -0,0 +1 @@ +ALTER TABLE tcrdb.mixcr_libraries ADD libraryId int; \ No newline at end of file diff --git a/tcrdb/resources/schemas/tcrdb.xml b/tcrdb/resources/schemas/tcrdb.xml index 304980ef9..796d0edbc 100644 --- a/tcrdb/resources/schemas/tcrdb.xml +++ b/tcrdb/resources/schemas/tcrdb.xml @@ -37,6 +37,15 @@ Additional Params A semi-colon delimited list of additional command line arguments to pass to mixcr align + + Reference Genome + true + + sequenceanalysis + reference_libraries + rowid + + Date Disabled diff --git a/tcrdb/resources/web/tcrdb/panel/PoolImportPanel.js b/tcrdb/resources/web/tcrdb/panel/PoolImportPanel.js index d12a14a08..3cd9aa48b 100644 --- a/tcrdb/resources/web/tcrdb/panel/PoolImportPanel.js +++ b/tcrdb/resources/web/tcrdb/panel/PoolImportPanel.js @@ -82,7 +82,7 @@ Ext4.define('TCRdb.panel.PoolImportPanel', { transform: 'htoIndex' },{ name: 'hto_library_conc', - labels: ['HTO Library Conc', 'HTO Library Conc (ng/uL)', 'HTO (qubit) ng/uL'], + labels: ['HTO Library Conc', 'HTO Library Conc (ng/uL)', 'HTO (qubit) ng/uL', 'HTO (quibit) ng/uL'], allowRowSpan: true },{ name: 'gex_library_index', @@ -91,7 +91,7 @@ Ext4.define('TCRdb.panel.PoolImportPanel', { transform: 'tenXBarcode' },{ name: 'gex_library_conc', - labels: ['5\' GEX Library Conc', 'GEX Library Conc', 'GEX Library Conc (ng/uL)', '5\' GEX Conc', 'GEX Conc', 'GEX Conc (ng/uL)', '5\' GEX (qubit) ng/uL'], + labels: ['5\' GEX Library Conc', 'GEX Library Conc', 'GEX Library Conc (ng/uL)', '5\' GEX Conc', 'GEX Conc', 'GEX Conc (ng/uL)', '5\' GEX (qubit) ng/uL', '5\' GEX Library (qubit) ng/uL'], allowRowSpan: true },{ name: 'gex_library_fragment', @@ -116,7 +116,7 @@ Ext4.define('TCRdb.panel.PoolImportPanel', { transforms: { stim: function(val, panel) { - if (val && val === '--') { + if (val && (val === '--' || val === '-')) { val = 'NoStim'; } @@ -334,8 +334,13 @@ Ext4.define('TCRdb.panel.PoolImportPanel', { checked: true },{ xtype: 'checkbox', - fieldLabel: 'Skip cDNA Libraries', - itemId: 'skipCDNA', + fieldLabel: 'Require Library Concentrations', + itemId: 'requireConc', + checked: true + }, { + xtype: 'checkbox', + fieldLabel: 'Skip Readsets', + itemId: 'skipReadsets', checked: false, listeners: { scope: this, @@ -359,8 +364,8 @@ Ext4.define('TCRdb.panel.PoolImportPanel', { scope: this, handler: this.onPreview },{ - style: 'margin-top: 20px;margin-bottom: 10px;', itemId: 'previewArea', + style: 'margin-top: 20px;margin-bottom: 10px;', autoEl: 'table', cls: 'stripe hover' }]; @@ -581,24 +586,35 @@ Ext4.define('TCRdb.panel.PoolImportPanel', { var readsetGUIDs = {}; var requireHTO = this.down('#requireHTO').getValue(); - readsetGUIDs.hashingReadsetGUID = this.processReadsetForGroup(poolName, rowArr, ret.readsetRows, 'hto', 'HTO', 'Cell Hashing', null); - if (requireHTO && readsetGUIDs.hashingReadsetGUID === false){ + var rs = this.processReadsetForGroup(poolName, rowArr, ret.readsetRows, 'hto', 'HTO', 'Cell Hashing', null); + if (Ext4.isString(rs)) { + readsetGUIDs.hashingReadsetGUID = rs; + } + else if (requireHTO){ errorsMsgs.push('Missing HTO library'); + errorsMsgs = errorsMsgs.concat(rs); return false; } var requireGEX = this.down('#requireGEX').getValue(); - readsetGUIDs.readsetGUID = this.processReadsetForGroup(poolName, rowArr, ret.readsetRows, 'gex', 'GEX', 'RNA-seq, Single Cell', '10x 5\' GEX'); - // always expect a GEX readset - if (requireGEX && readsetGUIDs.readsetGUID === false){ + rs = this.processReadsetForGroup(poolName, rowArr, ret.readsetRows, 'gex', 'GEX', 'RNA-seq, Single Cell', '10x 5\' GEX'); + if (Ext4.isString(rs)) { + readsetGUIDs.readsetGUID = rs; + } + else if (requireGEX){ errorsMsgs.push('Missing GEX library'); + errorsMsgs = errorsMsgs.concat(rs); return false; } var requireTCR = this.down('#requireTCR').getValue(); - readsetGUIDs.enrichedReadsetGUID = this.processReadsetForGroup(poolName, rowArr, ret.readsetRows, 'tcr', 'TCR', 'RNA-seq, Single Cell', '10x 5\' VDJ (Rhesus A/B/G)'); - if (requireTCR && readsetGUIDs.enrichedReadsetGUID === false){ + rs = this.processReadsetForGroup(poolName, rowArr, ret.readsetRows, 'tcr', 'TCR', 'RNA-seq, Single Cell', '10x 5\' VDJ (Rhesus A/B/G)'); + if (Ext4.isString(rs)) { + readsetGUIDs.enrichedReadsetGUID = rs; + } + else if (requireTCR){ errorsMsgs.push('Missing TCR library'); + errorsMsgs = errorsMsgs.concat(rs); return false; } @@ -619,7 +635,7 @@ Ext4.define('TCRdb.panel.PoolImportPanel', { if (errorsMsgs.length) { errorsMsgs = Ext4.unique(errorsMsgs); - Ext4.Msg.alert('Error', errorsMsgs.join('\n')); + Ext4.Msg.alert('Error', errorsMsgs.join('
')); return null; } @@ -633,7 +649,13 @@ Ext4.define('TCRdb.panel.PoolImportPanel', { var subjectid = this.getUniqueValues(rowArr, 'animalId'); subjectid = subjectid.length === 1 ? subjectid[0] : null; + var requireConc = this.down('#requireConc').getValue(); + if (idxValues.length === 1){ + if (requireConc && !conc[0]) { + return ['Pool ' + poolName + ': did not provide concentration for library: ' + type]; + } + var guid = LABKEY.Utils.generateUUID(); readsetRows.push({ name: poolName + '-' + type, @@ -652,14 +674,12 @@ Ext4.define('TCRdb.panel.PoolImportPanel', { return guid; } else if (idxValues.length > 1) { - Ext4.Msg.alert('Error', 'Pool ' + poolName + ' uses more than one ' + type + ' index'); - return false; + return ['Error', 'Pool ' + poolName + ' uses more than one ' + type + ' index']; } else if (idxValues.length === 0) { var required = this.down('#require' + type).getValue(); if (required) { - Ext4.Msg.alert('Error', 'No index found for pool: ' + poolName + ', for library type: ' + type); - return false; + return ['Error', 'No index found for pool: ' + poolName + ', for library type: ' + type]; } } }, @@ -675,8 +695,8 @@ Ext4.define('TCRdb.panel.PoolImportPanel', { }, renderPreview: function(colArray, parsedRows, groupedRows){ - var target = this.down('#previewArea'); - target.removeAll(); + var previewArea = this.down('#previewArea'); + previewArea.removeAll(); var columns = [{title: 'Row #'}]; var colIdxs = []; @@ -708,7 +728,7 @@ Ext4.define('TCRdb.panel.PoolImportPanel', { data.push(toAdd); }, this); - var id = '#' + target.getId(); + var id = '#' + previewArea.getId(); if ( jQuery.fn.dataTable.isDataTable(id) ) { jQuery(id).DataTable().destroy(); } @@ -730,7 +750,7 @@ Ext4.define('TCRdb.panel.PoolImportPanel', { columns: columns }); - target.doLayout(); + previewArea.doLayout(); if (missingValues){ Ext4.Msg.alert('Error', 'One or more rows is missing data. Any required cells without values are marked MISSING'); @@ -740,7 +760,7 @@ Ext4.define('TCRdb.panel.PoolImportPanel', { onSubmit: function(e, dt, node, config){ Ext4.Msg.wait('Saving...'); LABKEY.Ajax.request({ - url: LABKEY.ActionURL.buildURL('tcrdb', 'importTenx', Laboratory.Utils.getQueryContainerPath()), + url: LABKEY.ActionURL.buildURL('tcrdb', 'importTenx'), method: 'POST', jsonData: config.rowData.groupedRows, scope: this, diff --git a/tcrdb/resources/web/tcrdb/panel/StimPanel.js b/tcrdb/resources/web/tcrdb/panel/StimPanel.js index 5b4f868fb..6378de21d 100644 --- a/tcrdb/resources/web/tcrdb/panel/StimPanel.js +++ b/tcrdb/resources/web/tcrdb/panel/StimPanel.js @@ -1887,13 +1887,13 @@ Ext4.define('TCRdb.panel.StimPanel', { var sampleName = getSampleName(simpleSampleNames, r.readsetId, r['readsetId/name']); var data = [sampleName, (instrument === 'Novogene' ? '' : cleanedName), bc, '']; if (instrument === 'Novogene') { - data = idx === 0 ? [sampleName] : ['']; //only add once per group + data = [sampleName]; if (r.plateAlias) { data.unshift(r.plateAlias); data.push('G' + r.plateId.replace(/-/g, '_')); } else { - data.unshift('G' + r.plateId.replace(/-/g, '_')); + data.unshift(idx === 0 ? 'G' + r.plateId.replace(/-/g, '_') : ''); } data.push('Macaca mulatta'); @@ -1928,13 +1928,13 @@ Ext4.define('TCRdb.panel.StimPanel', { var sampleName = getSampleName(simpleSampleNames, r.enrichedReadsetId, r['enrichedReadsetId/name'], (instrument === 'Novogene' ? '' : '-TCR')); var data = [sampleName, (instrument === 'Novogene' ? '' : cleanedName), bc, '']; if (instrument === 'Novogene') { - data = idx === 0 ? [sampleName] : ['']; //only add once per group + data = [sampleName]; if (r.plateAlias) { data.unshift(r.plateAlias); data.push('T' + r.plateId.replace(/-/g, '_')); } else { - data.push('T' + r.plateId.replace(/-/g, '_')); + data.unshift(idx === 0 ? 'T' + r.plateId.replace(/-/g, '_') : ''); } data.push('Macaca mulatta'); diff --git a/tcrdb/resources/web/tcrdb/panel/cDNAImportPanel.js b/tcrdb/resources/web/tcrdb/panel/cDNAImportPanel.js index 7b8befb71..e4b41ae27 100644 --- a/tcrdb/resources/web/tcrdb/panel/cDNAImportPanel.js +++ b/tcrdb/resources/web/tcrdb/panel/cDNAImportPanel.js @@ -120,6 +120,11 @@ Ext4.define('TCRdb.panel.cDNAImportPanel', { fieldLabel: 'Require HTO Library', itemId: 'requireHTO', checked: true + }, { + xtype: 'checkbox', + fieldLabel: 'Require Library Concentrations', + itemId: 'requireConc', + checked: true }, { xtype: 'textarea', fieldLabel: 'Paste Data Below', diff --git a/tcrdb/src/org/labkey/tcrdb/TCRdbController.java b/tcrdb/src/org/labkey/tcrdb/TCRdbController.java index 45bee32db..5e7179db1 100644 --- a/tcrdb/src/org/labkey/tcrdb/TCRdbController.java +++ b/tcrdb/src/org/labkey/tcrdb/TCRdbController.java @@ -32,6 +32,7 @@ import org.labkey.api.action.SimpleViewAction; import org.labkey.api.action.SpringActionController; import org.labkey.api.collections.CaseInsensitiveHashMap; +import org.labkey.api.data.ColumnInfo; import org.labkey.api.data.CompareType; import org.labkey.api.data.Container; import org.labkey.api.data.DbScope; @@ -544,18 +545,38 @@ public void export(DownloadCloneMaterialsForm form, HttpServletResponse response } StringBuilder fasta = new StringBuilder(); + Map> segmentsByLibrary = new HashMap<>(); //find assay records SimpleFilter assayFilter = new SimpleFilter(FieldKey.fromString("rowId"), rowIds, CompareType.IN); - TableSelector ts = new TableSelector(assayData, PageFlowUtil.set("samplename", "sequence", "cdr3", "vHit", "jHit", "dHit", "cHit"), assayFilter, null); - Set primarySegments = new HashSet<>(); + Map cols = QueryService.get().getColumns(assayData, PageFlowUtil.set( + FieldKey.fromString("samplename"), + FieldKey.fromString("sequence"), + FieldKey.fromString("cdr3"), + FieldKey.fromString("vHit"), + FieldKey.fromString("jHit"), + FieldKey.fromString("dHit"), + FieldKey.fromString("cHit"), + FieldKey.fromString("libraryId/libraryId"))); + TableSelector ts = new TableSelector(assayData, cols.values(), assayFilter, null); + Set segmentsByName = new HashSet<>(); final String[] segmentFields = new String[]{"vHit", "jHit", "cHit"}; ts.forEachResults(rs -> { + Integer libraryId = rs.getObject(FieldKey.fromString("libraryId/libraryId")) == null ? null : rs.getInt(FieldKey.fromString("libraryId/libraryId")); for (String fn : segmentFields) { if (rs.getString(FieldKey.fromString(fn)) != null) { - primarySegments.add(StringUtils.trimToNull(rs.getString(FieldKey.fromString(fn)))); + if (libraryId != null) + { + Set map = segmentsByLibrary.getOrDefault(libraryId, new HashSet<>()); + map.add(StringUtils.trimToNull(rs.getString(FieldKey.fromString(fn)))); + segmentsByLibrary.put(libraryId, map); + } + else + { + segmentsByName.add(StringUtils.trimToNull(rs.getString(FieldKey.fromString(fn)))); + } } } @@ -571,14 +592,38 @@ public void export(DownloadCloneMaterialsForm form, HttpServletResponse response }); // look up segments in NT table - SimpleFilter ntFilter = new SimpleFilter(FieldKey.fromString("name"), primarySegments, CompareType.IN); - ntFilter.addCondition(FieldKey.fromString("datedisabled"), null, CompareType.ISBLANK); - Set missingSegments = new HashSet<>(primarySegments); - new TableSelector(QueryService.get().getUserSchema(getUser(), target, "sequenceanalysis").getTable("ref_nt_sequences"), PageFlowUtil.set("rowid"), ntFilter, null).forEachResults(rs -> { - RefNtSequenceModel nt = RefNtSequenceModel.getForRowId(rs.getInt(FieldKey.fromString("rowid"))); - fasta.append(">").append(nt.getName() + (nt.getSpecies() != null ? "-" + nt.getSpecies() : "")).append('\n').append(nt.getSequence()).append('\n'); - missingSegments.remove(nt.getName()); - }); + Set missingSegments = new HashSet<>(segmentsByName); + for (int libraryId : segmentsByLibrary.keySet()) + { + missingSegments.addAll(segmentsByLibrary.get(libraryId)); + } + + if (!segmentsByLibrary.isEmpty()) + { + for (int libraryId : segmentsByLibrary.keySet()) + { + SimpleFilter ntFilter = new SimpleFilter(FieldKey.fromString("ref_nt_id/name"), segmentsByLibrary.get(libraryId), CompareType.IN); + ntFilter.addCondition(FieldKey.fromString("ref_nt_id/datedisabled"), null, CompareType.ISBLANK); + ntFilter.addCondition(FieldKey.fromString("library_id"), libraryId, CompareType.EQUAL); + new TableSelector(QueryService.get().getUserSchema(getUser(), target, "sequenceanalysis").getTable("reference_library_members"), PageFlowUtil.set("ref_nt_id"), ntFilter, null).forEachResults(rs -> { + RefNtSequenceModel nt = RefNtSequenceModel.getForRowId(rs.getInt(FieldKey.fromString("ref_nt_id"))); + fasta.append(">").append(nt.getName() + (nt.getSpecies() != null ? "-" + nt.getSpecies() : "")).append('\n').append(nt.getSequence()).append('\n'); + missingSegments.remove(nt.getName()); + }); + } + } + + if (!segmentsByName.isEmpty()) + { + SimpleFilter ntFilter = new SimpleFilter(FieldKey.fromString("name"), segmentsByName, CompareType.IN); + ntFilter.addCondition(FieldKey.fromString("datedisabled"), null, CompareType.ISBLANK); + + new TableSelector(QueryService.get().getUserSchema(getUser(), target, "sequenceanalysis").getTable("ref_nt_sequences"), PageFlowUtil.set("rowid"), ntFilter, null).forEachResults(rs -> { + RefNtSequenceModel nt = RefNtSequenceModel.getForRowId(rs.getInt(FieldKey.fromString("rowid"))); + fasta.append(">").append(nt.getName() + (nt.getSpecies() != null ? "-" + nt.getSpecies() : "")).append('\n').append(nt.getSequence()).append('\n'); + missingSegments.remove(nt.getName()); + }); + } if (!missingSegments.isEmpty()) { @@ -627,19 +672,45 @@ public void export(DownloadCloneMaterialsForm form, HttpServletResponse response // find distinct analyses for assay rows and primary segments SimpleFilter assayFilter = new SimpleFilter(FieldKey.fromString("rowId"), rowIds, CompareType.IN); - TableSelector ts = new TableSelector(assayData, PageFlowUtil.set("analysisId", "vHit", "jHit", "dHit", "cHit", "cloneId", "sequence", "sampleName", "clonesFile", "cdr3"), assayFilter, null); - Set primarySegments = new HashSet<>(); + Map cols = QueryService.get().getColumns(assayData, PageFlowUtil.set( + FieldKey.fromString("analysisId"), + FieldKey.fromString("samplename"), + FieldKey.fromString("sequence"), + FieldKey.fromString("cdr3"), + FieldKey.fromString("vHit"), + FieldKey.fromString("jHit"), + FieldKey.fromString("dHit"), + FieldKey.fromString("cHit"), + FieldKey.fromString("cloneId"), + FieldKey.fromString("sequence"), + FieldKey.fromString("clonesFile"), + FieldKey.fromString("libraryId/libraryId"))); + + TableSelector ts = new TableSelector(assayData, cols.values(), assayFilter, null); + Set segmentsByName = new HashSet<>(); + Map> segmentsByLibrary = new HashMap<>(); + Map> clnaToCloneMap = new HashMap<>(); Map clnaToCDR3Map = new HashMap<>(); StringBuilder imputedSequences = new StringBuilder(); Set analyses = new HashSet<>(); final String[] segmentFields = new String[]{"vHit", "jHit", "cHit"}; ts.forEachResults(rs -> { + Integer libraryId = rs.getObject(FieldKey.fromString("libraryId/libraryId")) == null ? null : rs.getInt(FieldKey.fromString("libraryId/libraryId")); for (String fn : segmentFields) { if (rs.getString(FieldKey.fromString(fn)) != null) { - primarySegments.add(rs.getString(FieldKey.fromString(fn))); + if (libraryId != null) + { + Set map = segmentsByLibrary.getOrDefault(libraryId, new HashSet<>()); + map.add(StringUtils.trimToNull(rs.getString(FieldKey.fromString(fn)))); + segmentsByLibrary.put(libraryId, map); + } + else + { + segmentsByName.add(StringUtils.trimToNull(rs.getString(FieldKey.fromString(fn)))); + } } } @@ -673,28 +744,60 @@ public void export(DownloadCloneMaterialsForm form, HttpServletResponse response } // then find all segments from these analyses - Set allSegments = new HashSet<>(primarySegments); SimpleFilter assayFilter2 = new SimpleFilter(FieldKey.fromString("analysisId"), analyses, CompareType.IN); - new TableSelector(assayData, PageFlowUtil.set("vHit", "jHit", "dHit", "cHit"), assayFilter2, null).forEachResults(rs -> { + new TableSelector(assayData, cols.values(), assayFilter2, null).forEachResults(rs -> { + Integer libraryId = rs.getObject(FieldKey.fromString("libraryId/libraryId")) == null ? null : rs.getInt(FieldKey.fromString("libraryId/libraryId")); for (String fn : segmentFields) { if (rs.getString(FieldKey.fromString(fn)) != null) { - allSegments.add(StringUtils.trimToNull(rs.getString(FieldKey.fromString(fn)))); + if (libraryId != null) + { + Set map = segmentsByLibrary.getOrDefault(libraryId, new HashSet<>()); + map.add(StringUtils.trimToNull(rs.getString(FieldKey.fromString(fn)))); + segmentsByLibrary.put(libraryId, map); + } + else + { + segmentsByName.add(StringUtils.trimToNull(rs.getString(FieldKey.fromString(fn)))); + } } } }); // look up segments in NT table + Set missingSegments = new HashSet<>(segmentsByName); + for (int libraryId : segmentsByLibrary.keySet()) + { + missingSegments.addAll(segmentsByLibrary.get(libraryId)); + } + StringBuilder fasta = new StringBuilder(); - SimpleFilter ntFilter = new SimpleFilter(FieldKey.fromString("name"), allSegments, CompareType.IN); - ntFilter.addCondition(FieldKey.fromString("datedisabled"), null, CompareType.ISBLANK); - Set missingSegments = new HashSet<>(allSegments); - new TableSelector(QueryService.get().getUserSchema(getUser(), target, "sequenceanalysis").getTable("ref_nt_sequences"), PageFlowUtil.set("rowid"), ntFilter, null).forEachResults(rs -> { - RefNtSequenceModel nt = RefNtSequenceModel.getForRowId(rs.getInt(FieldKey.fromString("rowid"))); - fasta.append(">").append(nt.getName() + (nt.getSpecies() != null ? "-" + nt.getSpecies() : "")).append('\n').append(nt.getSequence()).append('\n'); - missingSegments.remove(nt.getName()); - }); + if (!segmentsByLibrary.isEmpty()) + { + for (int libraryId : segmentsByLibrary.keySet()) + { + SimpleFilter ntFilter = new SimpleFilter(FieldKey.fromString("ref_nt_id/name"), segmentsByLibrary.get(libraryId), CompareType.IN); + ntFilter.addCondition(FieldKey.fromString("ref_nt_id/datedisabled"), null, CompareType.ISBLANK); + ntFilter.addCondition(FieldKey.fromString("library_id"), libraryId, CompareType.EQUAL); + new TableSelector(QueryService.get().getUserSchema(getUser(), target, "sequenceanalysis").getTable("reference_library_members"), PageFlowUtil.set("ref_nt_id"), ntFilter, null).forEachResults(rs -> { + RefNtSequenceModel nt = RefNtSequenceModel.getForRowId(rs.getInt(FieldKey.fromString("ref_nt_id"))); + fasta.append(">").append(nt.getName() + (nt.getSpecies() != null ? "-" + nt.getSpecies() : "")).append('\n').append(nt.getSequence()).append('\n'); + missingSegments.remove(nt.getName()); + }); + } + } + + if (!segmentsByName.isEmpty()) + { + SimpleFilter ntFilter = new SimpleFilter(FieldKey.fromString("name"), segmentsByName, CompareType.IN); + ntFilter.addCondition(FieldKey.fromString("datedisabled"), null, CompareType.ISBLANK); + new TableSelector(QueryService.get().getUserSchema(getUser(), target, "sequenceanalysis").getTable("ref_nt_sequences"), PageFlowUtil.set("rowid"), ntFilter, null).forEachResults(rs -> { + RefNtSequenceModel nt = RefNtSequenceModel.getForRowId(rs.getInt(FieldKey.fromString("rowid"))); + fasta.append(">").append(nt.getName() + (nt.getSpecies() != null ? "-" + nt.getSpecies() : "")).append('\n').append(nt.getSequence()).append('\n'); + missingSegments.remove(nt.getName()); + }); + } if (!missingSegments.isEmpty()) { diff --git a/tcrdb/src/org/labkey/tcrdb/TCRdbManager.java b/tcrdb/src/org/labkey/tcrdb/TCRdbManager.java index d465bb04e..43724ac12 100644 --- a/tcrdb/src/org/labkey/tcrdb/TCRdbManager.java +++ b/tcrdb/src/org/labkey/tcrdb/TCRdbManager.java @@ -29,16 +29,22 @@ import org.labkey.api.data.SimpleFilter; import org.labkey.api.data.TableInfo; import org.labkey.api.data.TableSelector; +import org.labkey.api.module.ModuleLoader; import org.labkey.api.query.BatchValidationException; import org.labkey.api.query.FieldKey; +import org.labkey.api.query.InvalidKeyException; import org.labkey.api.query.QueryService; +import org.labkey.api.query.QueryUpdateServiceException; import org.labkey.api.query.UserSchema; import org.labkey.api.security.User; +import org.labkey.api.sequenceanalysis.GenomeTrigger; import org.labkey.api.sequenceanalysis.RefNtSequenceModel; import org.labkey.api.sequenceanalysis.SequenceAnalysisService; +import org.labkey.api.util.PageFlowUtil; import org.labkey.tcrdb.query.MixcrLibrary; import java.io.File; +import java.sql.SQLException; import java.util.ArrayList; import java.util.Arrays; import java.util.LinkedHashMap; @@ -74,6 +80,11 @@ public void createGenomeFromMixcrDb(int mixcrRowId, User u, Container c) throws throw new IllegalArgumentException("Unable to find JSON for MiXCR library: " + mixcrRowId); } + if (lib.getLibraryId() != null) + { + throw new IllegalArgumentException("MiXCR library already has a genome associated with it: " + mixcrRowId); + } + Container target = c.isWorkbookOrTab() ? c.getParent() : c; UserSchema us = QueryService.get().getUserSchema(u, target, TCRdbSchema.SEQUENCE_ANALYSIS); TableInfo refNt = us.getTable("ref_nt_sequences"); @@ -174,6 +185,16 @@ public void createGenomeFromMixcrDb(int mixcrRowId, User u, Container c) throws } + if (ref.getDatedisabled() != null) + { + toUpdate.put("datedisabled", null); + } + + if (ref.getDisabledby() != null) + { + toUpdate.put("disabledby", null); + } + if (!gene.getGeneName().equals(ref.getSubset())) { toUpdate.put("subset", gene.getGeneName()); @@ -235,7 +256,7 @@ public void createGenomeFromMixcrDb(int mixcrRowId, User u, Container c) throws if (!sequences.isEmpty()) { _log.info("Creating mixcr genome with " + sequences.size() + " sequences"); - SequenceAnalysisService.get().createReferenceLibrary(sequences, ContainerManager.getForId(lib.getContainer()), u, lib.getLabel(), null, "Created from MiXCR library: " + lib.getLibraryName(), true, true); + SequenceAnalysisService.get().createReferenceLibrary(sequences, ContainerManager.getForId(lib.getContainer()), u, lib.getLabel(), null, "Created from MiXCR library: " + lib.getLibraryName(), true, true, PageFlowUtil.set(new MiXCRGenomeTrigger(mixcrRowId))); } else { @@ -250,4 +271,81 @@ public void createGenomeFromMixcrDb(int mixcrRowId, User u, Container c) throws } } + + public static class MiXCRGenomeTrigger implements GenomeTrigger + { + private Integer _mixcrId = null; + + public MiXCRGenomeTrigger() + { + + } + + public MiXCRGenomeTrigger(int mixcrId) + { + _mixcrId = mixcrId; + } + + public Integer getMixcrId() + { + return _mixcrId; + } + + public void setMixcrId(Integer mixcrId) + { + _mixcrId = mixcrId; + } + + @Override + public String getName() + { + return "MiXCR Library Update"; + } + + @Override + public void onCreate(Container c, User u, Logger log, int genomeId) + { + if (_mixcrId != null) + { + TableInfo ti = QueryService.get().getUserSchema(u, c, TCRdbSchema.NAME).getTable(TCRdbSchema.TABLE_LIBRARIES); + List> rows = new ArrayList<>(); + List> oldKeys = new ArrayList<>(); + Map row = new CaseInsensitiveHashMap<>(); + row.put("rowid", _mixcrId); + row.put("libraryId", genomeId); + rows.add(row); + + Map keyRow = new CaseInsensitiveHashMap<>(); + keyRow.put("rowid", _mixcrId); + oldKeys.add(keyRow); + + try + { + ti.getUpdateService().updateRows(u, c, rows, oldKeys, null, null); + } + catch (QueryUpdateServiceException | InvalidKeyException | BatchValidationException | SQLException e) + { + _log.error(e.getMessage(), e); + } + } + } + + @Override + public void onRecreate(Container c, User u, Logger log, int genomeId) + { + + } + + @Override + public void onDelete(Container c, User u, Logger log, int genomeId) + { + + } + + @Override + public boolean isAvailable(Container c) + { + return c.getActiveModules().contains(ModuleLoader.getInstance().getModule(TCRdbModule.class)); + } + } } \ No newline at end of file diff --git a/tcrdb/src/org/labkey/tcrdb/TCRdbModule.java b/tcrdb/src/org/labkey/tcrdb/TCRdbModule.java index 0e04b87af..c88761935 100644 --- a/tcrdb/src/org/labkey/tcrdb/TCRdbModule.java +++ b/tcrdb/src/org/labkey/tcrdb/TCRdbModule.java @@ -28,7 +28,6 @@ import org.labkey.api.sequenceanalysis.pipeline.SequencePipelineService; import org.labkey.tcrdb.pipeline.CellRangerCellHashingHandler; import org.labkey.tcrdb.pipeline.CellRangerVDJCellHashingHandler; -import org.labkey.tcrdb.pipeline.CellRangerVDJResultHandler; import org.labkey.tcrdb.pipeline.CellRangerVDJWrapper; import org.labkey.tcrdb.pipeline.MiXCRAnalysis; import org.labkey.tcrdb.pipeline.SeuratCellHashingHandler; @@ -49,7 +48,7 @@ public String getName() @Override public double getVersion() { - return 15.46; + return 15.47; } @Override @@ -120,7 +119,6 @@ public PipelineStartup() SequenceAnalysisService.get().registerFileHandler(new CellRangerCellHashingHandler()); SequenceAnalysisService.get().registerFileHandler(new CellRangerVDJCellHashingHandler()); - SequenceAnalysisService.get().registerFileHandler(new CellRangerVDJResultHandler()); SequenceAnalysisService.get().registerFileHandler(new SeuratCellHashingHandler()); _hasRegistered = true; diff --git a/tcrdb/src/org/labkey/tcrdb/TCRdbProvider.java b/tcrdb/src/org/labkey/tcrdb/TCRdbProvider.java index 7deb6cf31..0093e4f93 100644 --- a/tcrdb/src/org/labkey/tcrdb/TCRdbProvider.java +++ b/tcrdb/src/org/labkey/tcrdb/TCRdbProvider.java @@ -73,11 +73,11 @@ public List getDataNavItems(Container c, User u) item.setQueryCache(cache); items.add(item); - TCRdbBulkImportNavItem item2 = new TCRdbBulkImportNavItem(this, "TCR/10x Import", LaboratoryService.NavItemCategory.data, "TCRdb", "tcrdb/poolImport.view"); + TCRdbBulkImportNavItem item2 = new TCRdbBulkImportNavItem(this, "TCR/10x Stim/cDNA Import", LaboratoryService.NavItemCategory.data, "TCRdb", "tcrdb/poolImport.view"); item2.setQueryCache(cache); items.add(item2); - TCRdbBulkImportNavItem item3 = new TCRdbBulkImportNavItem(this, "TCR/10x cDNA Import", LaboratoryService.NavItemCategory.data, "TCRdb", "tcrdb/cDNAImport.view"); + TCRdbBulkImportNavItem item3 = new TCRdbBulkImportNavItem(this, "TCR/10x Readset Import", LaboratoryService.NavItemCategory.data, "TCRdb", "tcrdb/cDNAImport.view"); item3.setQueryCache(cache); items.add(item3); } diff --git a/tcrdb/src/org/labkey/tcrdb/TCRdbTableCustomizer.java b/tcrdb/src/org/labkey/tcrdb/TCRdbTableCustomizer.java index 26a285592..0fb60ac50 100644 --- a/tcrdb/src/org/labkey/tcrdb/TCRdbTableCustomizer.java +++ b/tcrdb/src/org/labkey/tcrdb/TCRdbTableCustomizer.java @@ -1,5 +1,9 @@ package org.labkey.tcrdb; +import org.labkey.api.assay.AssayProtocolSchema; +import org.labkey.api.assay.AssayProvider; +import org.labkey.api.assay.AssayResultTable; +import org.labkey.api.assay.AssayService; import org.labkey.api.data.AbstractTableInfo; import org.labkey.api.data.CompareType; import org.labkey.api.data.Container; @@ -18,10 +22,6 @@ import org.labkey.api.query.QueryForeignKey; import org.labkey.api.query.QueryService; import org.labkey.api.query.UserSchema; -import org.labkey.api.assay.AssayProtocolSchema; -import org.labkey.api.assay.AssayProvider; -import org.labkey.api.assay.AssayResultTable; -import org.labkey.api.assay.AssayService; import java.util.Arrays; import java.util.List; @@ -199,7 +199,7 @@ private void addAssayFieldsToCDnas(AbstractTableInfo ti) { addAssayFieldsToTable(ti, "analysisId/readset", "LEFT JOIN sequenceanalysis.sequence_analyses a2 ON (a.analysisId = a2.rowId) WHERE a2.readset = " + ExprColumn.STR_TABLE_ALIAS + ".readsetId", "readsetId", "FullTranscript", " (Full Transcriptome)"); - addAssayFieldsToTable(ti, "cdna", " WHERE a.cDNA = " + ExprColumn.STR_TABLE_ALIAS + ".rowid", "enrichedReadsetId", "TCREnriched", " (TCR Enriched)", false); + addAssayFieldsToTable(ti, "cdna", " WHERE a.cDNA = " + ExprColumn.STR_TABLE_ALIAS + ".rowid", "rowid", "TCREnriched", " (TCR Enriched)", false); } private void addAssayFieldsToTable(AbstractTableInfo ti, String urlField, String whereClause, String urlSourceCol) diff --git a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerCellHashingHandler.java b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerCellHashingHandler.java index 21d450c16..b52b144ce 100644 --- a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerCellHashingHandler.java +++ b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerCellHashingHandler.java @@ -198,7 +198,7 @@ public static File processBarcodeFile(SequenceOutputHandler.JobContext ctx, File CellRangerVDJUtils utils = new CellRangerVDJUtils(ctx.getLogger(), ctx.getSourceDirectory()); Map readsetToHashing = CellRangerVDJUtils.getCachedReadsetMap(ctx.getSequenceSupport()); - ctx.getLogger().debug("total cashed readset/HTO pairs: " + readsetToHashing.size()); + ctx.getLogger().debug("total cached readset/HTO pairs: " + readsetToHashing.size()); //prepare whitelist of cell indexes File cellBarcodeWhitelist = utils.getValidCellIndexFile(); diff --git a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java index c189b4b2d..dac76f329 100644 --- a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java +++ b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java @@ -48,7 +48,7 @@ public class CellRangerVDJCellHashingHandler extends AbstractParameterizedOutput public CellRangerVDJCellHashingHandler() { - super(ModuleLoader.getInstance().getModule(TCRdbModule.class), "CellRanger VDJ/Cell Hashing", "This will run CiteSeqCount/MultiSeqClassifier to generate a sample-to-cellbarcode TSV based on the filtered barcodes from CellRanger VDJ. Results will be imported into the selected assay.", new LinkedHashSet<>(PageFlowUtil.set("tcrdb/field/AssaySelectorField.js")), Arrays.asList( + super(ModuleLoader.getInstance().getModule(TCRdbModule.class), "CellRanger VDJ Import", "This will either directly import data (if cell hashing is not used), or run CiteSeqCount/MultiSeqClassifier to generate a sample-to-cellbarcode TSV based on the filtered barcodes from CellRanger VDJ and then import.", new LinkedHashSet<>(PageFlowUtil.set("tcrdb/field/AssaySelectorField.js")), Arrays.asList( ToolParameterDescriptor.create(TARGET_ASSAY, "Target Assay", "Results will be loaded into this assay. If no assay is selected, a table will be created with nothing in the DB.", "tcr-assayselectorfield", null, null), ToolParameterDescriptor.create(DELETE_EXISTING_ASSAY_DATA, "Delete Any Existing Assay Data", "If selected, prior to importing assay data, and existing assay runs in the target container from this readset will be deleted.", "checkbox", new JSONObject(){{ put("checked", true); @@ -218,6 +218,7 @@ public static void processMetrics(SequenceOutputFile so, PipelineJob job, boolea if (metrics.exists()) { job.getLogger().info("Loading metrics"); + int total = 0; TableInfo ti = DbSchema.get("sequenceanalysis", DbSchemaType.Module).getTable("quality_metrics"); try (CSVReader reader = new CSVReader(Readers.getReader(metrics), '\t')) { @@ -243,16 +244,20 @@ public static void processMetrics(SequenceOutputFile so, PipelineJob job, boolea String fieldName = NumberUtils.isCreatable(value) ? "metricvalue" : "qualvalue"; r.put(fieldName, value); + r.put("analysis_id", so.getAnalysis_id()); r.put("dataid", so.getDataId()); r.put("readset", so.getReadset()); r.put("container", job.getContainer()); r.put("createdby", job.getUser().getUserId()); Table.insert(job.getUser(), ti, r); + total++; valueMap.put(line[1], value); } + job.getLogger().info("total metrics: " + total); + if (updateDescription) { job.getLogger().debug("Updating description"); @@ -301,8 +306,12 @@ public static void processMetrics(SequenceOutputFile so, PipelineJob job, boolea } else { - job.getLogger().info("Unable to find metrics file: " + metrics.getPath()); + job.getLogger().warn("Unable to find metrics file: " + metrics.getPath()); } } + else + { + job.getLogger().warn("Unable to update metrics, file id is null: " + so.getName()); + } } } \ No newline at end of file diff --git a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJResultHandler.java b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJResultHandler.java deleted file mode 100644 index 758e22a32..000000000 --- a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJResultHandler.java +++ /dev/null @@ -1,125 +0,0 @@ -package org.labkey.tcrdb.pipeline; - -import org.apache.commons.lang3.StringUtils; -import org.json.JSONObject; -import org.labkey.api.data.ConvertHelper; -import org.labkey.api.module.ModuleLoader; -import org.labkey.api.pipeline.PipelineJob; -import org.labkey.api.pipeline.PipelineJobException; -import org.labkey.api.pipeline.RecordedAction; -import org.labkey.api.sequenceanalysis.SequenceOutputFile; -import org.labkey.api.sequenceanalysis.model.AnalysisModel; -import org.labkey.api.sequenceanalysis.pipeline.AbstractParameterizedOutputHandler; -import org.labkey.api.sequenceanalysis.pipeline.SequenceAnalysisJobSupport; -import org.labkey.api.sequenceanalysis.pipeline.SequenceOutputHandler; -import org.labkey.api.sequenceanalysis.pipeline.ToolParameterDescriptor; -import org.labkey.api.util.FileType; -import org.labkey.api.util.PageFlowUtil; -import org.labkey.tcrdb.TCRdbModule; - -import java.io.File; -import java.util.Arrays; -import java.util.LinkedHashSet; -import java.util.List; - -import static org.labkey.tcrdb.pipeline.CellRangerVDJWrapper.DELETE_EXISTING_ASSAY_DATA; -import static org.labkey.tcrdb.pipeline.CellRangerVDJWrapper.TARGET_ASSAY; - -public class CellRangerVDJResultHandler extends AbstractParameterizedOutputHandler -{ - private FileType _fileType = new FileType("vloupe", false); - - public CellRangerVDJResultHandler() - { - super(ModuleLoader.getInstance().getModule(TCRdbModule.class), "CellRanger VDJ/Cell Result Import", "This will run parse the results of Cell Ranger VDJ and import the results into the target assay.", new LinkedHashSet<>(PageFlowUtil.set("tcrdb/field/AssaySelectorField.js")), Arrays.asList( - ToolParameterDescriptor.create(TARGET_ASSAY, "Target Assay", "Results will be loaded into this assay. If no assay is selected, a table will be created with nothing in the DB.", "tcr-assayselectorfield", null, null), - ToolParameterDescriptor.create(DELETE_EXISTING_ASSAY_DATA, "Delete Any Existing Assay Data", "If selected, prior to importing assay data, and existing assay runs in the target container from this readset will be deleted.", "checkbox", new JSONObject(){{ - put("checked", true); - }}, true), - ToolParameterDescriptor.create("useOutputFileContainer", "Submit to Source File Workbook", "If checked, each job will be submitted to the same workbook as the input file, as opposed to submitting all jobs to the same workbook. This is primarily useful if submitting a large batch of files.", "checkbox", new JSONObject(){{ - put("checked", false); - }}, false) - )); - } - - @Override - public boolean canProcess(SequenceOutputFile o) - { - return o.getFile() != null && _fileType.isType(o.getFile()); - } - - @Override - public boolean doRunRemote() - { - return false; - } - - @Override - public boolean doRunLocal() - { - return true; - } - - @Override - public SequenceOutputProcessor getProcessor() - { - return new CellRangerVDJResultHandler.Processor(); - } - - @Override - public boolean doSplitJobs() - { - return true; - } - - @Override - public boolean requiresSingleGenome() - { - return false; - } - - public class Processor implements SequenceOutputProcessor - { - @Override - public void init(PipelineJob job, SequenceAnalysisJobSupport support, List inputFiles, JSONObject params, File outputDir, List actions, List outputsToCreate) throws UnsupportedOperationException, PipelineJobException - { - - } - - @Override - public void processFilesOnWebserver(PipelineJob job, SequenceAnalysisJobSupport support, List inputFiles, JSONObject params, File outputDir, List actions, List outputsToCreate) throws UnsupportedOperationException, PipelineJobException - { - CellRangerVDJUtils utils = new CellRangerVDJUtils(job.getLogger(), job.getLogFile().getParentFile()); - if (StringUtils.trimToNull(job.getParameters().get(TARGET_ASSAY)) == null) - { - job.getLogger().info("No assay selected, will not import"); - } - else - { - Integer assayId = ConvertHelper.convert(job.getParameters().get(TARGET_ASSAY), Integer.class); - if (assayId == null) - { - throw new PipelineJobException("Invalid assay Id, cannot import: " + job.getParameters().get(TARGET_ASSAY)); - } - - Boolean deleteExistingData = false; - if (job.getParameters().get(DELETE_EXISTING_ASSAY_DATA) != null) - { - deleteExistingData = ConvertHelper.convert(job.getParameters().get(DELETE_EXISTING_ASSAY_DATA), Boolean.class); - } - - for (SequenceOutputFile so : inputFiles) - { - AnalysisModel model = support.getCachedAnalysis(so.getAnalysis_id()); - utils.importAssayData(job, model, so.getFile().getParentFile(), assayId, model.getRunId(), deleteExistingData); - } - } - } - - @Override - public void processFilesRemote(List inputFiles, JobContext ctx) throws UnsupportedOperationException, PipelineJobException - { - - } - } -} \ No newline at end of file diff --git a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJUtils.java b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJUtils.java index 521a3c3e7..994dbef92 100644 --- a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJUtils.java +++ b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJUtils.java @@ -197,7 +197,7 @@ public File runRemoteCellHashingTasks(AlignmentOutputImpl output, File perCellTs return null; } - _log.debug("total cashed readset/HTO pairs: " + readsetToHashing.size()); + _log.debug("total cached readset/HTO pairs: " + readsetToHashing.size()); //prepare whitelist of cell indexes File cellBarcodeWhitelist = getValidCellIndexFile(); @@ -618,6 +618,24 @@ private void saveRun(PipelineJob job, ExpProtocol protocol, AnalysisModel model, runProps.put("analysisId", model.getAnalysisId()); runProps.put("pipelineRunId", runId); + if (model.getLibraryId() != null) + { + TableSelector ts = new TableSelector(TCRdbSchema.getInstance().getSchema().getTable(TCRdbSchema.TABLE_LIBRARIES), PageFlowUtil.set("rowid"), new SimpleFilter(FieldKey.fromString("libraryId"), model.getLibraryId()), null); + if (ts.exists()) + { + int mixcrId = ts.getObject(Integer.class); + _log.debug("adding mixcr library id: " + mixcrId); + for (Map row : rows) + { + row.put("libraryId", mixcrId); + } + } + else + { + _log.debug("Unable to find MiXCR library for genome: " + model.getLibraryId()); + } + } + JSONObject json = new JSONObject(); json.put("Run", runProps); @@ -657,13 +675,14 @@ private void saveRun(PipelineJob job, ExpProtocol protocol, AnalysisModel model, public static void deleteExistingData(AssayProvider ap, ExpProtocol protocol, Container c, User u, Logger log, int readsetId) throws PipelineJobException { - log.info("Preparing to delete any existing runs from this container for the same readset:" + readsetId); + log.info("Preparing to delete any existing runs from this container for the same readset: " + readsetId); SimpleFilter filter = new SimpleFilter(FieldKey.fromString("analysisId/readset"), readsetId); filter.addCondition(FieldKey.fromString("Folder"), c.getId()); AssayProtocolSchema aps = ap.createProtocolSchema(u, c, protocol, null); - TableInfo runsTable = aps.createRunsTable(null); + TableInfo runsTable = QueryService.get().getUserSchema(u, c, aps.getSchemaPath()).getTable(AssayProtocolSchema.RUNS_TABLE_NAME); + TableSelector ts = new TableSelector(runsTable, PageFlowUtil.set("RowId"), filter, null); if (ts.exists()) { @@ -722,6 +741,7 @@ public static void prepareCellHashingFiles(PipelineJob job, SequenceAnalysisJobS try (CSVWriter bcWriter = new CSVWriter(PrintWriters.getPrintWriter(barcodeOutput), ',', CSVWriter.NO_QUOTE_CHARACTER)) { List cachedReadsets = support.getCachedReadsets(); + job.getLogger().debug("total cached readsets: " + cachedReadsets.size()); Set distinctHTOs = new HashSet<>(); Set hashingStatus = new HashSet<>(); for (Readset rs : cachedReadsets) @@ -767,6 +787,10 @@ public static void prepareCellHashingFiles(PipelineJob job, SequenceAnalysisJobS { throw new PipelineJobException("The selected readsets/cDNA records use a mixture of cell hashing and non-hashing."); } + else if (hashingStatus.isEmpty()) + { + throw new PipelineJobException("There were no readsets found."); + } boolean useCellHashing = hashingStatus.iterator().next(); if (useCellHashing && distinctHTOs.isEmpty()) @@ -912,7 +936,7 @@ public Sort getSortRecord() public String getAssaySampleName() { - return getPlateId() + "_" + getWell() + "_" + getSortRecord().getStimRecord().getAnimalId() + "_" + getSortRecord().getStimRecord().getStim() + "_" + getSortRecord().getPopulation() + "_" + getSortRecord().getHto(); + return getPlateId() + "_" + getWell() + "_" + getSortRecord().getStimRecord().getAnimalId() + "_" + getSortRecord().getStimRecord().getStim() + "_" + getSortRecord().getPopulation() + (getSortRecord().getHto() == null ? "" : "_" + getSortRecord().getHto()); } public static CDNA getRowId(int rowId) diff --git a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJWrapper.java b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJWrapper.java index 1860e3395..eb676c1d9 100644 --- a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJWrapper.java +++ b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJWrapper.java @@ -29,15 +29,15 @@ import org.labkey.api.sequenceanalysis.pipeline.AlignerIndexUtil; import org.labkey.api.sequenceanalysis.pipeline.AlignmentOutputImpl; import org.labkey.api.sequenceanalysis.pipeline.AlignmentStep; +import org.labkey.api.sequenceanalysis.pipeline.AlignmentStepProvider; import org.labkey.api.sequenceanalysis.pipeline.CommandLineParam; import org.labkey.api.sequenceanalysis.pipeline.IndexOutputImpl; import org.labkey.api.sequenceanalysis.pipeline.PipelineContext; -import org.labkey.api.sequenceanalysis.pipeline.PipelineStepProvider; import org.labkey.api.sequenceanalysis.pipeline.ReferenceGenome; import org.labkey.api.sequenceanalysis.pipeline.SequenceAnalysisJobSupport; import org.labkey.api.sequenceanalysis.pipeline.SequencePipelineService; import org.labkey.api.sequenceanalysis.pipeline.ToolParameterDescriptor; -import org.labkey.api.sequenceanalysis.run.AbstractCommandPipelineStep; +import org.labkey.api.sequenceanalysis.run.AbstractAlignmentPipelineStep; import org.labkey.api.sequenceanalysis.run.AbstractCommandWrapper; import org.labkey.api.sequenceanalysis.run.SimpleScriptWrapper; import org.labkey.api.util.FileUtil; @@ -92,6 +92,12 @@ public VDJProvider() ), PageFlowUtil.set("tcrdb/field/AssaySelectorField.js"), "https://support.10xgenomics.com/single-cell-gene-expression/software/pipelines/latest/what-is-cell-ranger", true, false, false, ALIGNMENT_MODE.MERGE_THEN_ALIGN); } + @Override + public boolean shouldRunIdxstats() + { + return false; + } + public String getName() { return "CellRanger-VDJ"; @@ -108,7 +114,7 @@ public AlignmentStep create(PipelineContext context) } } - public static class CellRangerVDJAlignmentStep extends AbstractCommandPipelineStep implements AlignmentStep + public static class CellRangerVDJAlignmentStep extends AbstractAlignmentPipelineStep implements AlignmentStep { private CellRangerVDJUtils _utils = null; @@ -122,7 +128,7 @@ private CellRangerVDJUtils getUtils() return _utils; } - public CellRangerVDJAlignmentStep(PipelineStepProvider provider, PipelineContext ctx, CellRangerVDJWrapper wrapper) + public CellRangerVDJAlignmentStep(AlignmentStepProvider provider, PipelineContext ctx, CellRangerVDJWrapper wrapper) { super(provider, ctx, wrapper); } @@ -336,7 +342,15 @@ public AlignmentStep.AlignmentOutput performAlignment(Readset rs, File inputFast output.setBAM(bam); //NOTE: run these before cleanup in case of failure - getUtils().runRemoteCellHashingTasks(output, getUtils().getPerCellCsv(output.getBAM().getParentFile()), rs, getPipelineCtx().getSequenceSupport(), null, getPipelineCtx().getWorkingDirectory(), getPipelineCtx().getSourceDirectory()); + Integer assayId = getProvider().getParameterByName(TARGET_ASSAY).extractValue(getPipelineCtx().getJob(), getProvider(), getStepIdx(), Integer.class); + if (assayId != null) + { + getUtils().runRemoteCellHashingTasks(output, getUtils().getPerCellCsv(output.getBAM().getParentFile()), rs, getPipelineCtx().getSequenceSupport(), null, getPipelineCtx().getWorkingDirectory(), getPipelineCtx().getSourceDirectory()); + } + else + { + getPipelineCtx().getLogger().debug("No target assay selected, skipping cell hashing steps"); + } //now do cleanup/rename: try @@ -417,32 +431,40 @@ public boolean supportsGzipFastqs() return true; } - private String getSymlinkFileName(String fileName) + private String getSymlinkFileName(String fileName, boolean doRename, String sampleName, int idx, boolean isReversed) { //NOTE: cellranger is very picky about file name formatting - Matcher m = FILE_PATTERN.matcher(fileName); - if (m.matches()) + if (doRename) { - if (!StringUtils.isEmpty(m.group(7))) - { - return m.group(1).replaceAll("_", "-") + StringUtils.trimToEmpty(m.group(2)) + "_L" + StringUtils.trimToEmpty(m.group(3)) + "_" + StringUtils.trimToEmpty(m.group(4)) + StringUtils.trimToEmpty(m.group(5)) + StringUtils.trimToEmpty(m.group(6)) + ".fastq.gz"; - } - else if (m.group(1).contains("_")) + sampleName = FileUtil.makeLegalName(sampleName.replaceAll("_", "-")).replaceAll(" ", "-").replaceAll("\\.", "-");; + return sampleName + "_S1_L001_R" + (isReversed ? "2" : "1") + "_" + StringUtils.leftPad(String.valueOf(idx), 3, "0") + ".fastq.gz"; + } + else + { + Matcher m = FILE_PATTERN.matcher(fileName); + if (m.matches()) { - getPipelineCtx().getLogger().info("replacing underscores in file/sample name"); - return m.group(1).replaceAll("_", "-") + StringUtils.trimToEmpty(m.group(2)) + "_L" + StringUtils.trimToEmpty(m.group(3)) + "_" + StringUtils.trimToEmpty(m.group(4)) + StringUtils.trimToEmpty(m.group(5)) + StringUtils.trimToEmpty(m.group(6)) + ".fastq.gz"; + if (!StringUtils.isEmpty(m.group(7))) + { + return m.group(1).replaceAll("_", "-") + StringUtils.trimToEmpty(m.group(2)) + "_L" + StringUtils.trimToEmpty(m.group(3)) + "_" + StringUtils.trimToEmpty(m.group(4)) + StringUtils.trimToEmpty(m.group(5)) + StringUtils.trimToEmpty(m.group(6)) + ".fastq.gz"; + } + else if (m.group(1).contains("_")) + { + getPipelineCtx().getLogger().info("replacing underscores in file/sample name"); + return m.group(1).replaceAll("_", "-") + StringUtils.trimToEmpty(m.group(2)) + "_L" + StringUtils.trimToEmpty(m.group(3)) + "_" + StringUtils.trimToEmpty(m.group(4)) + StringUtils.trimToEmpty(m.group(5)) + StringUtils.trimToEmpty(m.group(6)) + ".fastq.gz"; + } + else + { + getPipelineCtx().getLogger().info("no additional characters found"); + } } else { - getPipelineCtx().getLogger().info("no additional characters found"); + getPipelineCtx().getLogger().warn("filename does not match Illumina formatting: " + fileName); } } - else - { - getPipelineCtx().getLogger().warn("filename does not match Illumina formatting: " + fileName); - } - return fileName; + return FileUtil.makeLegalName(fileName); } public Set prepareFastqSymlinks(Readset rs, File localFqDir) throws PipelineJobException @@ -453,11 +475,20 @@ public Set prepareFastqSymlinks(Readset rs, File localFqDir) throws Pipe localFqDir.mkdirs(); } + String[] files = localFqDir.list(); + if (files != null && files.length > 0) + { + deleteSymlinks(localFqDir); + } + + int idx = 0; + boolean doRename = true; //cellranger is too picky - simply rename files all the time for (ReadData rd : rs.getReadData()) { + idx++; try { - File target1 = new File(localFqDir, getSymlinkFileName(rd.getFile1().getName())); + File target1 = new File(localFqDir, getSymlinkFileName(rd.getFile1().getName(), doRename, rs.getName(), idx, false)); getPipelineCtx().getLogger().debug("file: " + rd.getFile1().getPath()); getPipelineCtx().getLogger().debug("target: " + target1.getPath()); if (target1.exists()) @@ -471,7 +502,7 @@ public Set prepareFastqSymlinks(Readset rs, File localFqDir) throws Pipe if (rd.getFile2() != null) { - File target2 = new File(localFqDir, getSymlinkFileName(rd.getFile2().getName())); + File target2 = new File(localFqDir, getSymlinkFileName(rd.getFile2().getName(), doRename, rs.getName(), idx, true)); getPipelineCtx().getLogger().debug("file: " + rd.getFile2().getPath()); getPipelineCtx().getLogger().debug("target: " + target2.getPath()); if (target2.exists()) diff --git a/tcrdb/src/org/labkey/tcrdb/pipeline/MiXCRAnalysis.java b/tcrdb/src/org/labkey/tcrdb/pipeline/MiXCRAnalysis.java index 0e701a113..f28fd8d26 100644 --- a/tcrdb/src/org/labkey/tcrdb/pipeline/MiXCRAnalysis.java +++ b/tcrdb/src/org/labkey/tcrdb/pipeline/MiXCRAnalysis.java @@ -9,6 +9,8 @@ import org.apache.commons.lang3.time.DateUtils; import org.json.JSONArray; import org.json.JSONObject; +import org.labkey.api.assay.AssayProvider; +import org.labkey.api.assay.AssayService; import org.labkey.api.collections.CaseInsensitiveHashMap; import org.labkey.api.data.CompareType; import org.labkey.api.data.SimpleFilter; @@ -42,8 +44,6 @@ import org.labkey.api.sequenceanalysis.pipeline.ToolParameterDescriptor; import org.labkey.api.sequenceanalysis.run.PicardWrapper; import org.labkey.api.sequenceanalysis.run.SimpleScriptWrapper; -import org.labkey.api.assay.AssayProvider; -import org.labkey.api.assay.AssayService; import org.labkey.api.util.FileUtil; import org.labkey.api.util.PageFlowUtil; import org.labkey.api.util.Pair; @@ -1380,7 +1380,7 @@ private Map getBaseRow(AnalysisModel model, Integer runId) throw } } - if (!row.containsKey("cDNA")) + if (!row.containsKey("cDNA") || row.get("cDNA") == null) { getPipelineCtx().getLogger().warn("Unable to find cDNA for: " + model.getReadset()); } diff --git a/tcrdb/src/org/labkey/tcrdb/query/MixcrLibrary.java b/tcrdb/src/org/labkey/tcrdb/query/MixcrLibrary.java index c201b3e2a..e2a606304 100644 --- a/tcrdb/src/org/labkey/tcrdb/query/MixcrLibrary.java +++ b/tcrdb/src/org/labkey/tcrdb/query/MixcrLibrary.java @@ -12,6 +12,7 @@ public class MixcrLibrary private String _libraryName; private String _species; private String _additionalParams; + private Integer _libraryId; private Date _dateDisabled; private String _container; private Date _created; @@ -162,4 +163,14 @@ public File getJsonFile() return null; } + + public Integer getLibraryId() + { + return _libraryId; + } + + public void setLibraryId(Integer libraryId) + { + _libraryId = libraryId; + } }