From e05e48462f915ffdfd57f5a18c7ca97fbabdad89 Mon Sep 17 00:00:00 2001 From: bbimber Date: Fri, 20 May 2022 10:22:34 -0700 Subject: [PATCH 01/10] Rename AppendNimble for consistency with name --- .../resources/chunks/{NimbleAppend.R => AppendNimble.R} | 0 .../src/org/labkey/singlecell/SingleCellModule.java | 2 +- .../singlecell/{NimbleAppend.java => AppendNimble.java} | 8 ++++---- 3 files changed, 5 insertions(+), 5 deletions(-) rename singlecell/resources/chunks/{NimbleAppend.R => AppendNimble.R} (100%) rename singlecell/src/org/labkey/singlecell/pipeline/singlecell/{NimbleAppend.java => AppendNimble.java} (93%) diff --git a/singlecell/resources/chunks/NimbleAppend.R b/singlecell/resources/chunks/AppendNimble.R similarity index 100% rename from singlecell/resources/chunks/NimbleAppend.R rename to singlecell/resources/chunks/AppendNimble.R diff --git a/singlecell/src/org/labkey/singlecell/SingleCellModule.java b/singlecell/src/org/labkey/singlecell/SingleCellModule.java index 6c2b35a02..821328730 100644 --- a/singlecell/src/org/labkey/singlecell/SingleCellModule.java +++ b/singlecell/src/org/labkey/singlecell/SingleCellModule.java @@ -188,7 +188,7 @@ public static void registerPipelineSteps() SequencePipelineService.get().registerPipelineStep(new CheckExpectations.Provider()); SequencePipelineService.get().registerPipelineStep(new CommonFilters.Provider()); SequencePipelineService.get().registerPipelineStep(new RunVision.Provider()); - SequencePipelineService.get().registerPipelineStep(new NimbleAppend.Provider()); + SequencePipelineService.get().registerPipelineStep(new AppendNimble.Provider()); SequencePipelineService.get().registerPipelineStep(new AppendTcr.Provider()); SequencePipelineService.get().registerPipelineStep(new TcrFilter.Provider()); diff --git a/singlecell/src/org/labkey/singlecell/pipeline/singlecell/NimbleAppend.java b/singlecell/src/org/labkey/singlecell/pipeline/singlecell/AppendNimble.java similarity index 93% rename from singlecell/src/org/labkey/singlecell/pipeline/singlecell/NimbleAppend.java rename to singlecell/src/org/labkey/singlecell/pipeline/singlecell/AppendNimble.java index bcf8822f6..871abaf7e 100644 --- a/singlecell/src/org/labkey/singlecell/pipeline/singlecell/NimbleAppend.java +++ b/singlecell/src/org/labkey/singlecell/pipeline/singlecell/AppendNimble.java @@ -15,9 +15,9 @@ import java.util.List; import java.util.Set; -public class NimbleAppend extends AbstractRDiscvrStep +public class AppendNimble extends AbstractRDiscvrStep { - public NimbleAppend(PipelineContext ctx, NimbleAppend.Provider provider) + public AppendNimble(PipelineContext ctx, AppendNimble.Provider provider) { super(provider, ctx); } @@ -35,9 +35,9 @@ public Provider() @Override - public NimbleAppend create(PipelineContext ctx) + public AppendNimble create(PipelineContext ctx) { - return new NimbleAppend(ctx, this); + return new AppendNimble(ctx, this); } } From a4185d66fd6f114e769d0fcfd0f69fa8ea3b1284 Mon Sep 17 00:00:00 2001 From: bbimber Date: Fri, 20 May 2022 10:24:44 -0700 Subject: [PATCH 02/10] More accurate error --- .../singlecell/pipeline/singlecell/CheckExpectations.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/singlecell/src/org/labkey/singlecell/pipeline/singlecell/CheckExpectations.java b/singlecell/src/org/labkey/singlecell/pipeline/singlecell/CheckExpectations.java index c98270562..dd9a5b15c 100644 --- a/singlecell/src/org/labkey/singlecell/pipeline/singlecell/CheckExpectations.java +++ b/singlecell/src/org/labkey/singlecell/pipeline/singlecell/CheckExpectations.java @@ -72,10 +72,15 @@ protected Chunk createParamChunk(SequenceOutputHandler.JobContext ctx, List htosPerReadset = CellHashingServiceImpl.get().getHtosForParentReadset(parentReadset.getReadsetId(), ctx.getSourceDirectory(), ctx.getSequenceSupport(), false); From c0085d219239cfbf2a3a83b948ff44f1607e5ec9 Mon Sep 17 00:00:00 2001 From: bbimber Date: Sat, 21 May 2022 07:12:47 -0700 Subject: [PATCH 03/10] Bugfix to Nimble/Append --- .../labkey/singlecell/pipeline/singlecell/AppendNimble.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/singlecell/src/org/labkey/singlecell/pipeline/singlecell/AppendNimble.java b/singlecell/src/org/labkey/singlecell/pipeline/singlecell/AppendNimble.java index 871abaf7e..93b82a091 100644 --- a/singlecell/src/org/labkey/singlecell/pipeline/singlecell/AppendNimble.java +++ b/singlecell/src/org/labkey/singlecell/pipeline/singlecell/AppendNimble.java @@ -62,6 +62,7 @@ protected Chunk createParamChunk(SequenceOutputHandler.JobContext ctx, List Date: Sat, 21 May 2022 08:40:19 -0700 Subject: [PATCH 04/10] Fix syntax in nimble markdown --- .../org/labkey/singlecell/pipeline/singlecell/AppendNimble.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/singlecell/src/org/labkey/singlecell/pipeline/singlecell/AppendNimble.java b/singlecell/src/org/labkey/singlecell/pipeline/singlecell/AppendNimble.java index 93b82a091..7088332c1 100644 --- a/singlecell/src/org/labkey/singlecell/pipeline/singlecell/AppendNimble.java +++ b/singlecell/src/org/labkey/singlecell/pipeline/singlecell/AppendNimble.java @@ -73,7 +73,7 @@ protected Chunk createParamChunk(SequenceOutputHandler.JobContext ctx, List Date: Mon, 23 May 2022 13:38:16 -0700 Subject: [PATCH 05/10] R cannot handle extremely long strings when assigning variables --- .../AbstractSingleCellPipelineStep.java | 35 +++++++++++++------ .../pipeline/SeuratToolParameter.java | 2 +- 2 files changed, 25 insertions(+), 12 deletions(-) diff --git a/singlecell/api-src/org/labkey/api/singlecell/pipeline/AbstractSingleCellPipelineStep.java b/singlecell/api-src/org/labkey/api/singlecell/pipeline/AbstractSingleCellPipelineStep.java index add9ee2da..1fee55f0c 100644 --- a/singlecell/api-src/org/labkey/api/singlecell/pipeline/AbstractSingleCellPipelineStep.java +++ b/singlecell/api-src/org/labkey/api/singlecell/pipeline/AbstractSingleCellPipelineStep.java @@ -348,38 +348,51 @@ public static void executeR(SequenceOutputHandler.JobContext ctx, String dockerC localBashScript.delete(); } - protected String prepareValueForR(SeuratToolParameter pd) + protected void addParameterVariables(SeuratToolParameter pd, List body) { String val = StringUtils.trimToNull(pd.extractValue(getPipelineCtx().getJob(), getProvider(), getStepIdx())); if (val == null) { - return "NULL"; + body.add((pd.getVariableName() + " <- NULL")); } else if ("false".equals(val)) { - return "FALSE"; + body.add((pd.getVariableName() + " <- FALSE")); } else if ("true".equals(val)) { - return "TRUE"; + body.add((pd.getVariableName() + " <- TRUE")); } else if (NumberUtils.isCreatable(val)) { - return val; + body.add((pd.getVariableName() + " <- " + val)); } else if ("sequenceanalysis-trimmingtextarea".equals(pd.getFieldXtype())) { val = val.replace("'", "\\\'"); - String[] vals = val.split(pd.getDelimiter()); - return "c('" + StringUtils.join(vals, "','") + "')"; + serializeMultiValueParam(pd, body, val); + } else if (pd.isMultiValue()) { - String[] vals = val.split(pd.getDelimiter()); - return "c('" + StringUtils.join(vals, "','") + "')"; + serializeMultiValueParam(pd, body, val); } - return "'" + val + "'"; + body.add((pd.getVariableName() + " <- '" + val + "'")); + } + + private void serializeMultiValueParam(SeuratToolParameter pd, List body, String val) + { + String[] vals = val.split(pd.getDelimiter()); + final int batchSize = 75; + int numBatches = (int)Math.ceil((double)vals.length / batchSize); + + for (int i=0;i loadChunkFromFile() throws PipelineJobException @@ -423,7 +436,7 @@ protected Chunk createParamChunk(SequenceOutputHandler.JobContext ctx, List provider, int stepIdx) { if (!_includeIfEmptyOrNull) { From b52803aa7afa8d3cf6da998b8ecc03c3c1e50681 Mon Sep 17 00:00:00 2001 From: bbimber Date: Mon, 23 May 2022 14:07:53 -0700 Subject: [PATCH 06/10] Better handling of NAs --- singlecell/resources/chunks/TcrFilter.R | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/singlecell/resources/chunks/TcrFilter.R b/singlecell/resources/chunks/TcrFilter.R index 7da1c4e08..925eeef46 100644 --- a/singlecell/resources/chunks/TcrFilter.R +++ b/singlecell/resources/chunks/TcrFilter.R @@ -18,6 +18,10 @@ for (datasetId in names(seuratObjects)) { cdr3ForLocus <- gsub(cdr3ForLocus, pattern = paste0(fieldName, ':'), replacement = '') matchingCells <- sapply(seuratObj@meta.data[[fieldName]], function(x){ + if (is.na(x)) { + return(FALSE) + } + values <- unlist(strsplit(x, split = ',')) return(length(intersect(values, cdr3ForLocus)) != 0) }) From fa69c4b310f6e4514959ca92ba6795602fba5b81 Mon Sep 17 00:00:00 2001 From: bbimber Date: Mon, 23 May 2022 14:38:42 -0700 Subject: [PATCH 07/10] Bugfix to TCR filtering --- singlecell/resources/chunks/TcrFilter.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/singlecell/resources/chunks/TcrFilter.R b/singlecell/resources/chunks/TcrFilter.R index 925eeef46..237cf109d 100644 --- a/singlecell/resources/chunks/TcrFilter.R +++ b/singlecell/resources/chunks/TcrFilter.R @@ -22,7 +22,7 @@ for (datasetId in names(seuratObjects)) { return(FALSE) } - values <- unlist(strsplit(x, split = ',')) + values <- unlist(strsplit(as.character(x), split = ',')) return(length(intersect(values, cdr3ForLocus)) != 0) }) From 5e101117769561cd1a89c8914d5f11fef3731c71 Mon Sep 17 00:00:00 2001 From: bbimber Date: Mon, 23 May 2022 14:42:18 -0700 Subject: [PATCH 08/10] Add logging to TCR filtering --- singlecell/resources/chunks/TcrFilter.R | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/singlecell/resources/chunks/TcrFilter.R b/singlecell/resources/chunks/TcrFilter.R index 237cf109d..3a5066254 100644 --- a/singlecell/resources/chunks/TcrFilter.R +++ b/singlecell/resources/chunks/TcrFilter.R @@ -41,8 +41,9 @@ for (datasetId in names(seuratObjects)) { if (all(is.null(cellsToKeep))) { print('There were no matching cells') } else { + print(paste0('Total passing cells: ', length(cellsToKeep))) seuratObj <- subset(seuratObj, cells = cellsToKeep) - #saveData(seuratObj, datasetId) + saveData(seuratObj, datasetId) totalPassed <- totalPassed + 1 } From 6baf568d979b461bb9268343ef43f3031e2f32ea Mon Sep 17 00:00:00 2001 From: bbimber Date: Tue, 24 May 2022 07:47:37 -0700 Subject: [PATCH 09/10] Bugfix support for long-string multi-value params --- .../api/singlecell/pipeline/AbstractSingleCellPipelineStep.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/singlecell/api-src/org/labkey/api/singlecell/pipeline/AbstractSingleCellPipelineStep.java b/singlecell/api-src/org/labkey/api/singlecell/pipeline/AbstractSingleCellPipelineStep.java index 1fee55f0c..0f4f26459 100644 --- a/singlecell/api-src/org/labkey/api/singlecell/pipeline/AbstractSingleCellPipelineStep.java +++ b/singlecell/api-src/org/labkey/api/singlecell/pipeline/AbstractSingleCellPipelineStep.java @@ -371,11 +371,13 @@ else if ("sequenceanalysis-trimmingtextarea".equals(pd.getFieldXtype())) { val = val.replace("'", "\\\'"); serializeMultiValueParam(pd, body, val); + return; } else if (pd.isMultiValue()) { serializeMultiValueParam(pd, body, val); + return; } body.add((pd.getVariableName() + " <- '" + val + "'")); From 778ef907826dd0c6a52debbc3a3367195f6c913d Mon Sep 17 00:00:00 2001 From: bbimber Date: Tue, 24 May 2022 07:51:48 -0700 Subject: [PATCH 10/10] Second bugfix to support for long-string multi-value params --- .../pipeline/AbstractSingleCellPipelineStep.java | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/singlecell/api-src/org/labkey/api/singlecell/pipeline/AbstractSingleCellPipelineStep.java b/singlecell/api-src/org/labkey/api/singlecell/pipeline/AbstractSingleCellPipelineStep.java index 0f4f26459..b10ba3191 100644 --- a/singlecell/api-src/org/labkey/api/singlecell/pipeline/AbstractSingleCellPipelineStep.java +++ b/singlecell/api-src/org/labkey/api/singlecell/pipeline/AbstractSingleCellPipelineStep.java @@ -371,16 +371,15 @@ else if ("sequenceanalysis-trimmingtextarea".equals(pd.getFieldXtype())) { val = val.replace("'", "\\\'"); serializeMultiValueParam(pd, body, val); - return; - } else if (pd.isMultiValue()) { serializeMultiValueParam(pd, body, val); - return; } - - body.add((pd.getVariableName() + " <- '" + val + "'")); + else + { + body.add((pd.getVariableName() + " <- '" + val + "'")); + } } private void serializeMultiValueParam(SeuratToolParameter pd, List body, String val)