Merge discvr 19.3

.github/workflows/sync-develop.yml

@@ -0,0 +1,15 @@
+# Designed to keep develop branch as a perfect copy of LabKey fork
+on:
+ schedule:
+ - cron: "*/15 * * * *"
+jobs:
+ sync-develop:
+ runs-on: ubuntu-latest
+ steps:
+ - name: "Sync Develop Branch"
+ uses: bimberlabinternal/DevOps/githubActions/git-sync@master
+ with:
+ source_repo: "labkey/BimberLabKeyModules"
+ source_branch: "develop"
+ destination_branch: "develop"
+ github_token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/sync-release-branches.yml

@@ -0,0 +1,16 @@
+# Designed to keep develop branch as a perfect copy of LabKey fork
+on:
+ schedule:
+ - cron: "*/15 * * * *"
+jobs:
+ sync-release-branches:
+ runs-on: ubuntu-latest
+ steps:
+ - name: "Sync Release Branches"
+ uses: bimberlabinternal/DevOps/githubActions/branch-create@master
+ with:
+ source_repo: "labkey/BimberLabKeyModules"
+ source_branch_prefix: "release"
+ destination_repo: "BimberLabInternal/BimberLabKeyModules"
+ destination_branch_prefix: "discvr-"
+ github_token: ${{ secrets.GITHUB_TOKEN }}

mGAP/src/org/labkey/mgap/pipeline/AnnotationStep.java

@@ -121,7 +121,7 @@ public void init(PipelineJob job, SequenceAnalysisJobSupport support, List<Seque
  }
 
  @Override
- public Output processVariants(File inputVCF, File outputDirectory, ReferenceGenome genome, @Nullable Interval interval) throws PipelineJobException
+ public Output processVariants(File inputVCF, File outputDirectory, ReferenceGenome genome, @Nullable List<Interval> intervals) throws PipelineJobException
  {
  VariantProcessingStepOutputImpl output = new VariantProcessingStepOutputImpl();
 
@@ -144,7 +144,7 @@ public Output processVariants(File inputVCF, File outputDirectory, ReferenceGeno
  totalSubjects = reader.getFileHeader().getSampleNamesInOrder().size();
  }
 
- boolean needToSubsetToInterval = interval != null;
+ boolean needToSubsetToInterval = intervals != null && !intervals.isEmpty();
  boolean dropGenotypes = totalSubjects > 10;
  boolean dropFiltered = getProvider().getParameterByName("dropFiltered").extractValue(getPipelineCtx().getJob(), getProvider(), getStepIdx(), Boolean.class);
 
@@ -170,8 +170,11 @@ public Output processVariants(File inputVCF, File outputDirectory, ReferenceGeno
 
  if (needToSubsetToInterval)
  {
- selectArgs.add("-L");
- selectArgs.add(interval.getContig() + ":" + interval.getStart() + "-" + interval.getEnd());
+ for (Interval interval : intervals)
+ {
+ selectArgs.add("-L");
+ selectArgs.add(interval.getContig() + ":" + interval.getStart() + "-" + interval.getEnd());
+ }
  needToSubsetToInterval = false;
  }
 
@@ -202,26 +205,29 @@ public Output processVariants(File inputVCF, File outputDirectory, ReferenceGeno
  {
  List<String> selectArgs = new ArrayList<>();
  getPipelineCtx().getLogger().info("subsetting VCF by interval");
- selectArgs.add("-L");
- selectArgs.add(interval.getContig() + ":" + interval.getStart() + "-" + interval.getEnd());
+ for (Interval interval : intervals)
+ {
+ selectArgs.add("-L");
+ selectArgs.add(interval.getContig() + ":" + interval.getStart() + "-" + interval.getEnd());
+ }
  needToSubsetToInterval = false;
 
- File subset = new File(outputDirectory, SequenceAnalysisService.get().getUnzippedBaseName(inputVCF.getName()) + "." + interval.getContig() + ".subset.vcf.gz");
- if (!indexExists(subset))
+ File intervalSubset = new File(outputDirectory, SequenceAnalysisService.get().getUnzippedBaseName(inputVCF.getName()) + ".intervalSubset.vcf.gz");
+ if (!indexExists(intervalSubset))
  {
  SelectVariantsWrapper wrapper = new SelectVariantsWrapper(getPipelineCtx().getLogger());
- wrapper.execute(originalGenome.getWorkingFastaFile(), inputVCF, subset, selectArgs);
+ wrapper.execute(originalGenome.getWorkingFastaFile(), inputVCF, intervalSubset, selectArgs);
  }
  else
  {
- getPipelineCtx().getLogger().info("resuming with existing file: " + subset.getPath());
+ getPipelineCtx().getLogger().info("resuming with existing file: " + intervalSubset.getPath());
  }
 
- output.addOutput(subset, "VCF Subset");
- output.addIntermediateFile(subset);
- output.addIntermediateFile(new File(subset.getPath() + ".tbi"));
+ output.addOutput(intervalSubset, "VCF Subset");
+ output.addIntermediateFile(intervalSubset);
+ output.addIntermediateFile(new File(intervalSubset.getPath() + ".tbi"));
 
- currentVcf = subset;
+ currentVcf = intervalSubset;
 
  getPipelineCtx().getJob().getLogger().info("total variants: " + SequenceAnalysisService.get().getVCFLineCount(currentVcf, getPipelineCtx().getJob().getLogger(), false));
  getPipelineCtx().getJob().getLogger().info("passing variants: " + SequenceAnalysisService.get().getVCFLineCount(currentVcf, getPipelineCtx().getJob().getLogger(), true));
@@ -333,8 +339,11 @@ public Output processVariants(File inputVCF, File outputDirectory, ReferenceGeno
  List<String> options = new ArrayList<>();
  if (needToSubsetToInterval)
  {
- options.add("-L");
- options.add(interval.getContig() + ":" + interval.getStart() + "-" + interval.getEnd());
+ for (Interval interval : intervals)
+ {
+ options.add("-L");
+ options.add(interval.getContig() + ":" + interval.getStart() + "-" + interval.getEnd());
+ }
  needToSubsetToInterval = false;
  }
 

mGAP/src/org/labkey/mgap/pipeline/CassandraRunner.java

@@ -104,7 +104,7 @@ private void correctHeaderAndBGzip(File inputUnzip, File outputGzip) throws Pipe
  writer.write("set -x\n");
  writer.write("set -e\n");
  writer.write("{\n");
- writer.write("cat " + inputUnzip.getPath() + " | head -n 50000 | grep -e '^#' | sed 's/Number=0,Type=String/Number=1,Type=String/';\n");
+ writer.write("cat " + inputUnzip.getPath() + " | head -n 50000 | grep -e '^#' | grep -v '^##META' | sed 's/Number=0,Type=String/Number=1,Type=String/';\n");
  writer.write("cat " + inputUnzip.getPath() + " | grep -v '^#';\n");
  writer.write("} | bgzip > " + outputGzip + "\n");
  }

mGAP/src/org/labkey/mgap/pipeline/RemoveAnnotationsForMgapStep.java

@@ -47,7 +47,7 @@ public PipelineStep create(PipelineContext context)
  }
 
  @Override
- public Output processVariants(File inputVCF, File outputDirectory, ReferenceGenome genome, @Nullable Interval interval) throws PipelineJobException
+ public Output processVariants(File inputVCF, File outputDirectory, ReferenceGenome genome, @Nullable List<Interval> intervals) throws PipelineJobException
  {
  VariantProcessingStepOutputImpl output = new VariantProcessingStepOutputImpl();
 
@@ -58,7 +58,7 @@ public Output processVariants(File inputVCF, File outputDirectory, ReferenceGeno
  }
  else
  {
- getWrapper().execute(inputVCF, outputFile, genome.getWorkingFastaFile(), interval);
+ getWrapper().execute(inputVCF, outputFile, genome.getWorkingFastaFile(), intervals);
  }
 
  output.setVcf(outputFile);
@@ -80,7 +80,7 @@ public RemoveAnnotationsWrapper(Logger log)
  super(log);
  }
 
- public void execute(File input, File outputFile, File referenceFasta, @Nullable Interval interval) throws PipelineJobException
+ public void execute(File input, File outputFile, File referenceFasta, @Nullable List<Interval> intervals) throws PipelineJobException
  {
  List<String> args = new ArrayList<>(getBaseArgs());
  args.add("RemoveAnnotations");
@@ -91,10 +91,12 @@ public void execute(File input, File outputFile, File referenceFasta, @Nullable
  args.add("-O");
  args.add(outputFile.getPath());
 
- if (interval != null)
+ if (intervals != null)
  {
- args.add("-L");
- args.add(interval.getContig() + ":" + interval.getStart() + "-" + interval.getEnd());
+ intervals.forEach(interval -> {
+ args.add("-L");
+ args.add(interval.getContig() + ":" + interval.getStart() + "-" + interval.getEnd());
+ });
  }
 
  for (String key : ALLOWABLE_ANNOTATIONS)

mGAP/src/org/labkey/mgap/pipeline/RenameSamplesForMgapStep.java

@@ -83,11 +83,11 @@ public void init(PipelineJob job, SequenceAnalysisJobSupport support, List<Seque
  }
 
  @Override
- public Output processVariants(File inputVCF, File outputDirectory, ReferenceGenome genome, @Nullable Interval interval) throws PipelineJobException
+ public Output processVariants(File inputVCF, File outputDirectory, ReferenceGenome genome, @Nullable List<Interval> intervals) throws PipelineJobException
  {
  VariantProcessingStepOutputImpl output = new VariantProcessingStepOutputImpl();
 
- File outputFile = renameSamples(inputVCF, genome, interval);
+ File outputFile = renameSamples(inputVCF, genome, intervals);
 
  output.setVcf(outputFile);
  output.addIntermediateFile(outputFile);
@@ -106,7 +106,7 @@ private File getSampleNameFile(File outputDir)
  return new File(outputDir, "sampleMapping.txt");
  }
 
- private File renameSamples(File currentVCF, ReferenceGenome genome, @Nullable Interval interval) throws PipelineJobException
+ private File renameSamples(File currentVCF, ReferenceGenome genome, @Nullable List<Interval> intervals) throws PipelineJobException
  {
  getPipelineCtx().getLogger().info("renaming samples in VCF");
 
@@ -154,11 +154,28 @@ else if (!allSamples.contains(sample))
  }
 
  writer.writeHeader(new VCFHeader(header.getMetaDataInInputOrder(), remappedSamples));
-try (CloseableIterator<VariantContext> it = (interval == null ? reader.iterator() : reader.query(interval.getContig(), interval.getStart(), interval.getEnd())))
+if (intervals == null)
  {
-while (it.hasNext())
+try (CloseableIterator<VariantContext> it = reader.iterator())
  {
- writer.add(it.next());
+ while (it.hasNext())
+ {
+ writer.add(it.next());
+ }
+ }
+
+ }
+ else
+ {
+ for (Interval interval : intervals)
+ {
+ try (CloseableIterator<VariantContext> it = reader.query(interval.getContig(), interval.getStart(), interval.getEnd()))
+ {
+ while (it.hasNext())
+ {
+ writer.add(it.next());
+ }
+ }
  }
  }
  }

primeseq/tools/installLabkey.sh

@@ -10,23 +10,39 @@ labkey_home=/usr/local/labkey
 cd /usr/local/src
 
 #NOTE: corresponding changes must be made in javaWrapper.sh
-MAJOR=18
-MINOR=2
-BRANCH=Discvr${MAJOR}${MINOR}_Installers
-ARTIFACT=LabKey${MAJOR}.${MINOR}
-MODULE_DIST_NAME=prime-seq-modules
-PREMIUM=premium-${MAJOR}.${MINOR}.module
+MAJOR=19
+MINOR_FULL="3.4"
+MINOR_SHORT=3
+BRANCH=LabKey_Discvr_Discvr${MAJOR}${MINOR_SHORT}_Premuim_Installers
 TOMCAT_HOME=/usr/share/tomcat
+TEAMCITY_USERNAME=username
+MODULE_DIST_NAME=prime-seq-modules
+
+ARTIFACT=LabKey${MAJOR}.${MINOR_FULL}
+PREMIUM=premium-${MAJOR}.${MINOR_SHORT}.module
+DATAINTEGRATION=dataintegration-${MAJOR}.${MINOR_SHORT}.module
+
+isGzZip() {
+	RET=`file $1 | grep -E 'gzip compressed|Zip archive data' | wc -l`
+	if [ $RET == 0 ];then
+ echo "Not GZIP!"
+ exit 1
+	else
+ echo "Is GZIP!"
+	fi
+}
 
 #first download
 DATE=$(date +"%Y%m%d%H%M")
 MODULE_ZIP=${ARTIFACT}-ExtraModules-${DATE}.zip
 rm -Rf $MODULE_ZIP
-wget --trust-server-names --no-check-certificate -O $MODULE_ZIP http://teamcity.labkey.org/guestAuth/repository/download/LabKey_${BRANCH}/.lastSuccessful/${MODULE_DIST_NAME}/${ARTIFACT}-{build.number}-ExtraModules.zip
+wget -O $MODULE_ZIP https://${TEAMCITY_USERNAME}@teamcity.labkey.org/repository/download/${BRANCH}/.lastSuccessful/${MODULE_DIST_NAME}/${ARTIFACT}-{build.number}-ExtraModules.zip
+isGzZip $MODULE_ZIP
 
 GZ=${ARTIFACT}-${DATE}-discvr-bin.tar.gz
 rm -Rf $GZ
-wget --trust-server-names --no-check-certificate -O $GZ http://teamcity.labkey.org/guestAuth/repository/download/Labkey_${BRANCH}/.lastSuccessful/discvr/${ARTIFACT}-{build.number}-discvr-bin.tar.gz
+wget -O $GZ https://${TEAMCITY_USERNAME}@teamcity.labkey.org/repository/download/${BRANCH}/.lastSuccessful/discvr/${ARTIFACT}-{build.number}-discvr-bin.tar.gz
+isGzZip $GZ
 
 #extract, find name
 tar -xf $GZ
@@ -55,6 +71,11 @@ if [ -e $PREMIUM ];then
  cp $PREMIUM ${labkey_home}/externalModules
 fi
 
+#DataIntegration
+if [ -e $DATAINTEGRATION ];then
+ cp $DATAINTEGRATION ${labkey_home}/externalModules
+fi
+
 #main server
 echo "Installing LabKey using: $GZ"
 cd $DIR