diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 000000000..583decfd1 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,7 @@ +version: 2 +updates: + # Maintain dependencies for GitHub Actions + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "daily" \ No newline at end of file diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 000000000..aca42609f --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,19 @@ +name: Build DISCVR +on: + push: + pull_request: +jobs: + build-modules: + # See: https://help.github.com/en/actions/reference/contexts-and-expression-syntax-for-github-actions#github-context + # https://help.github.com/en/actions/configuring-and-managing-workflows/using-environment-variables#default-environment-variables + if: github.repository == 'BimberLabInternal/BimberLabKeyModules' + runs-on: ubuntu-latest + steps: + - name: "Build DISCVR" + uses: bimberlabinternal/DevOps/githubActions/discvr-build@master + with: + artifactory_user: ${{secrets.artifactory_user}} + artifactory_password: ${{secrets.artifactory_password}} + # NOTE: permissions are limited on the default secrets.GITHUB_TOKEN, including updating workflows, so use a personal access token + github_token: ${{ secrets.PAT }} + diff --git a/.github/workflows/sync-repos.yml b/.github/workflows/sync-repos.yml index 199e5f75a..e5201d176 100644 --- a/.github/workflows/sync-repos.yml +++ b/.github/workflows/sync-repos.yml @@ -31,4 +31,4 @@ jobs: source_repo: "labkey/BimberLabKeyModules" source_branch: "develop" destination_branch: "develop" - github_token: ${{ secrets.GITHUB_TOKEN }} + github_token: ${{ secrets.PAT }} diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 4f4ef89f4..000000000 --- a/.travis.yml +++ /dev/null @@ -1,22 +0,0 @@ -language: java -dist: trusty -git: - depth: 9999999 -jdk: - - openjdk13 - -before_cache: - - rm -f $HOME/.gradle/caches/modules-2/modules-2.lock - - rm -fr $HOME/.gradle/caches/*/plugin-resolution/ - -cache: - directories: - - $HOME/.gradle/caches/ - - $HOME/.gradle/wrapper/ - - $HOME/.m2 - - $HOME/site-library - -install: skip -script: - - wget -O ./travis.sh https://github.com/bimberlabinternal/DevOps/raw/master/travisci/travis.sh - - bash ./travis.sh \ No newline at end of file diff --git a/GenotypeAssays/module.properties b/GenotypeAssays/module.properties index 9a8f3c517..26b7527df 100644 --- a/GenotypeAssays/module.properties +++ b/GenotypeAssays/module.properties @@ -1,3 +1,2 @@ ModuleClass: org.labkey.genotypeassays.GenotypeAssaysModule -ConsolidateScripts: false ManageVersion: false diff --git a/LabPurchasing/module.properties b/LabPurchasing/module.properties new file mode 100644 index 000000000..cf4adca31 --- /dev/null +++ b/LabPurchasing/module.properties @@ -0,0 +1,6 @@ +ModuleClass: org.labkey.labpurchasing.LabPurchasingModule +Label: Lab Purchacing +Description: A module designed to assist with purchasing supplies for academic labs +License: Apache 2.0 +LicenseURL: http://www.apache.org/licenses/LICENSE-2.0 +ManageVersion: false \ No newline at end of file diff --git a/LabPurchasing/resources/schemas/dbscripts/postgresql/labpurchasing-0.00-20.000.sql b/LabPurchasing/resources/schemas/dbscripts/postgresql/labpurchasing-0.00-20.000.sql new file mode 100644 index 000000000..fe2e39c0f --- /dev/null +++ b/LabPurchasing/resources/schemas/dbscripts/postgresql/labpurchasing-0.00-20.000.sql @@ -0,0 +1,19 @@ +/* + * Copyright (c) 2020 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +-- Create schema, tables, indexes, and constraints used for LabPurchasing module here +-- All SQL VIEW definitions should be created in labpurchasing-create.sql and dropped in labpurchasing-drop.sql +CREATE SCHEMA labpurchasing; diff --git a/LabPurchasing/resources/schemas/dbscripts/sqlserver/labpurchasing-0.00-20.000.sql b/LabPurchasing/resources/schemas/dbscripts/sqlserver/labpurchasing-0.00-20.000.sql new file mode 100644 index 000000000..35a4a8574 --- /dev/null +++ b/LabPurchasing/resources/schemas/dbscripts/sqlserver/labpurchasing-0.00-20.000.sql @@ -0,0 +1,20 @@ +/* + * Copyright (c) 2020 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +-- Create schema, tables, indexes, and constraints used for LabPurchasing module here +-- All SQL VIEW definitions should be created in labpurchasing-create.sql and dropped in labpurchasing-drop.sql +CREATE SCHEMA labpurchasing; +GO \ No newline at end of file diff --git a/LabPurchasing/resources/schemas/labpurchasing.xml b/LabPurchasing/resources/schemas/labpurchasing.xml new file mode 100644 index 000000000..2bba6c71d --- /dev/null +++ b/LabPurchasing/resources/schemas/labpurchasing.xml @@ -0,0 +1,20 @@ + + + \ No newline at end of file diff --git a/LabPurchasing/src/org/labkey/labpurchasing/LabPurchasingController.java b/LabPurchasing/src/org/labkey/labpurchasing/LabPurchasingController.java new file mode 100644 index 000000000..e53cbc6c5 --- /dev/null +++ b/LabPurchasing/src/org/labkey/labpurchasing/LabPurchasingController.java @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2020 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.labpurchasing; + +import org.labkey.api.action.SimpleViewAction; +import org.labkey.api.action.SpringActionController; +import org.labkey.api.security.RequiresPermission; +import org.labkey.api.security.permissions.ReadPermission; +import org.labkey.api.view.JspView; +import org.labkey.api.view.NavTree; +import org.springframework.validation.BindException; +import org.springframework.web.servlet.ModelAndView; + +public class LabPurchasingController extends SpringActionController +{ + private static final DefaultActionResolver _actionResolver = new DefaultActionResolver(LabPurchasingController.class); + public static final String NAME = "labpurchasing"; + + public LabPurchasingController() + { + setActionResolver(_actionResolver); + } +} diff --git a/LabPurchasing/src/org/labkey/labpurchasing/LabPurchasingManager.java b/LabPurchasing/src/org/labkey/labpurchasing/LabPurchasingManager.java new file mode 100644 index 000000000..43b0d9543 --- /dev/null +++ b/LabPurchasing/src/org/labkey/labpurchasing/LabPurchasingManager.java @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2020 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.labpurchasing; + +public class LabPurchasingManager +{ + private static final LabPurchasingManager _instance = new LabPurchasingManager(); + + private LabPurchasingManager() + { + // prevent external construction with a private default constructor + } + + public static LabPurchasingManager get() + { + return _instance; + } +} \ No newline at end of file diff --git a/LabPurchasing/src/org/labkey/labpurchasing/LabPurchasingModule.java b/LabPurchasing/src/org/labkey/labpurchasing/LabPurchasingModule.java new file mode 100644 index 000000000..230a72130 --- /dev/null +++ b/LabPurchasing/src/org/labkey/labpurchasing/LabPurchasingModule.java @@ -0,0 +1,85 @@ +/* + * Copyright (c) 2020 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.labpurchasing; + +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerManager; +import org.labkey.api.module.DefaultModule; +import org.labkey.api.module.ModuleContext; +import org.labkey.api.view.WebPartFactory; + +import java.util.Collection; +import java.util.Collections; +import java.util.Set; + +public class LabPurchasingModule extends DefaultModule +{ + public static final String NAME = "LabPurchasing"; + + @Override + public String getName() + { + return NAME; + } + + @Override + public @Nullable Double getSchemaVersion() + { + return 20.000; + } + + @Override + public boolean hasScripts() + { + return true; + } + + @Override + @NotNull + protected Collection createWebPartFactories() + { + return Collections.emptyList(); + } + + @Override + protected void init() + { + addController(LabPurchasingController.NAME, LabPurchasingController.class); + } + + @Override + public void doStartup(ModuleContext moduleContext) + { + + } + + @Override + @NotNull + public Collection getSummary(Container c) + { + return Collections.emptyList(); + } + + @Override + @NotNull + public Set getSchemaNames() + { + return Collections.singleton(LabPurchasingSchema.NAME); + } +} \ No newline at end of file diff --git a/LabPurchasing/src/org/labkey/labpurchasing/LabPurchasingSchema.java b/LabPurchasing/src/org/labkey/labpurchasing/LabPurchasingSchema.java new file mode 100644 index 000000000..d4425241d --- /dev/null +++ b/LabPurchasing/src/org/labkey/labpurchasing/LabPurchasingSchema.java @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2020 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.labpurchasing; + +import org.labkey.api.data.DbSchema; +import org.labkey.api.data.DbSchemaType; +import org.labkey.api.data.dialect.SqlDialect; + +public class LabPurchasingSchema +{ + private static final LabPurchasingSchema _instance = new LabPurchasingSchema(); + public static final String NAME = "labpurchasing"; + + public static LabPurchasingSchema getInstance() + { + return _instance; + } + + private LabPurchasingSchema() + { + // private constructor to prevent instantiation from + // outside this class: this singleton should only be + // accessed via org.labkey.labpurchasing.LabPurchasingSchema.getInstance() + } + + public DbSchema getSchema() + { + return DbSchema.get(NAME, DbSchemaType.Module); + } + + public SqlDialect getSqlDialect() + { + return getSchema().getSqlDialect(); + } +} diff --git a/LabPurchasing/test/src/org/labkey/test/components/labpurchasing/LabPurchasingWebPart.java b/LabPurchasing/test/src/org/labkey/test/components/labpurchasing/LabPurchasingWebPart.java new file mode 100644 index 000000000..eaa9965df --- /dev/null +++ b/LabPurchasing/test/src/org/labkey/test/components/labpurchasing/LabPurchasingWebPart.java @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2020 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.test.components.labpurchasing; + +import org.labkey.test.Locator; +import org.labkey.test.components.BodyWebPart; +import org.labkey.test.components.html.Input; +import org.labkey.test.pages.LabKeyPage; +import org.openqa.selenium.WebDriver; +import org.openqa.selenium.WebElement; + +import static org.labkey.test.components.html.Input.Input; + +/** + * TODO: Component for a hypothetical webpart containing an input and a save button + * Component classes should handle all timing and functionality for a component + */ +public class LabPurchasingWebPart extends BodyWebPart +{ + public LabPurchasingWebPart(WebDriver driver) + { + this(driver, 0); + } + + public LabPurchasingWebPart(WebDriver driver, int index) + { + super(driver, "LabPurchasing", index); + } + + public LabPurchasingWebPart setInput(String value) + { + elementCache().input.set(value); + // TODO: Methods that don't navigate should return this object + return this; + } + + public LabKeyPage clickSave() + { + getWrapper().clickAndWait(elementCache().button); + // TODO: Methods that navigate should return an appropriate page object + return new LabKeyPage(getDriver()); + } + + @Override + protected ElementCache newElementCache() + { + return new ElementCache(); + } + + protected class ElementCache extends BodyWebPart.ElementCache + { + protected final WebElement button = Locator.tag("button").withText("Save").findWhenNeeded(this); + protected final Input input = Input(Locator.tag("input"), getDriver()).findWhenNeeded(this); + } +} \ No newline at end of file diff --git a/LabPurchasing/test/src/org/labkey/test/pages/labpurchasing/BeginPage.java b/LabPurchasing/test/src/org/labkey/test/pages/labpurchasing/BeginPage.java new file mode 100644 index 000000000..7e36f3e45 --- /dev/null +++ b/LabPurchasing/test/src/org/labkey/test/pages/labpurchasing/BeginPage.java @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2020 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.test.pages.labpurchasing; + +import org.labkey.test.BaseWebDriverTest; +import org.labkey.test.WebDriverWrapper; +import org.labkey.test.Locator; +import org.labkey.test.WebTestHelper; +import org.labkey.test.pages.LabKeyPage; +import org.openqa.selenium.WebElement; + +public class BeginPage extends LabKeyPage +{ + public BeginPage(WebDriverWrapper driver) + { + super(driver); + } + + public static BeginPage beginAt(WebDriverWrapper driver) + { + return beginAt(driver, driver.getCurrentContainerPath()); + } + + public static BeginPage beginAt(WebDriverWrapper driver, String containerPath) + { + driver.beginAt(WebTestHelper.buildURL("labpurchasing", containerPath, "begin")); + return new BeginPage(driver); + } + + public String getHelloMessage() + { + return elementCache().helloMessage.getText(); + } + + @Override + protected ElementCache newElementCache() + { + return new ElementCache(); + } + + protected class ElementCache extends LabKeyPage.ElementCache + { + protected final WebElement helloMessage = Locator.tagWithName("div", "helloMessage").findWhenNeeded(this); + } +} diff --git a/LabPurchasing/test/src/org/labkey/test/tests/labpurchasing/LabPurchasingTest.java b/LabPurchasing/test/src/org/labkey/test/tests/labpurchasing/LabPurchasingTest.java new file mode 100644 index 000000000..1fa717924 --- /dev/null +++ b/LabPurchasing/test/src/org/labkey/test/tests/labpurchasing/LabPurchasingTest.java @@ -0,0 +1,88 @@ +/* + * Copyright (c) 2020 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.test.tests.labpurchasing; + +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.labkey.test.BaseWebDriverTest; +import org.labkey.test.TestTimeoutException; +import org.labkey.test.categories.InDevelopment; +import org.labkey.test.pages.labpurchasing.BeginPage; + +import java.util.Collections; +import java.util.List; + +import static org.junit.Assert.*; + +@Category({InDevelopment.class}) +public class LabPurchasingTest extends BaseWebDriverTest +{ + @Override + protected void doCleanup(boolean afterTest) throws TestTimeoutException + { + _containerHelper.deleteProject(getProjectName(), afterTest); + } + + @BeforeClass + public static void setupProject() + { + LabPurchasingTest init = (LabPurchasingTest)getCurrentTest(); + + init.doSetup(); + } + + private void doSetup() + { + _containerHelper.createProject(getProjectName(), null); + } + + @Before + public void preTest() + { + goToProjectHome(); + } + + @Test + public void testLabPurchasingModule() + { + _containerHelper.enableModule("LabPurchasing"); + BeginPage beginPage = BeginPage.beginAt(this, getProjectName()); + assertEquals(200, getResponseCode()); + final String expectedHello = "Hello, and welcome to the LabPurchasing module."; + assertEquals("Wrong hello message", expectedHello, beginPage.getHelloMessage()); + } + + @Override + protected BrowserType bestBrowser() + { + return BrowserType.CHROME; + } + + @Override + protected String getProjectName() + { + return "LabPurchasingTest Project"; + } + + @Override + public List getAssociatedModules() + { + return Collections.singletonList("LabPurchasing"); + } +} \ No newline at end of file diff --git a/README.md b/README.md index c498f9a50..b5f6334a8 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -[![Build Status](https://api.travis-ci.com/BimberLab/DiscvrLabKeyModules.svg)](https://travis-ci.com/BimberLab/DiscvrLabKeyModules) +![Build DISCVR](https://github.com/bimberlabinternal/BimberLabKeyModules/workflows/Build%20DISCVR/badge.svg) ## Overview diff --git a/elispot_assay/module.properties b/elispot_assay/module.properties index 77a02d564..4f2d4f3f5 100644 --- a/elispot_assay/module.properties +++ b/elispot_assay/module.properties @@ -1,3 +1,2 @@ ModuleClass: org.labkey.elispot_assay.ELISPOT_AssayModule -ConsolidateScripts: false ManageVersion: false diff --git a/elispot_assay/resources/credits/dependencies.txt b/elispot_assay/resources/credits/dependencies.txt new file mode 100644 index 000000000..2b459f3fc --- /dev/null +++ b/elispot_assay/resources/credits/dependencies.txt @@ -0,0 +1,2 @@ +# direct external dependencies for project :server:modules:BimberLabKeyModules:elispot_assay +commons-math3-3.6.1.jar diff --git a/elispot_assay/src/org/labkey/elispot_assay/assay/AIDImportMethod.java b/elispot_assay/src/org/labkey/elispot_assay/assay/AIDImportMethod.java index 6606c1593..f99393759 100644 --- a/elispot_assay/src/org/labkey/elispot_assay/assay/AIDImportMethod.java +++ b/elispot_assay/src/org/labkey/elispot_assay/assay/AIDImportMethod.java @@ -22,6 +22,8 @@ import java.io.IOException; import java.io.StringWriter; import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; import java.util.LinkedHashMap; import java.util.List; import java.util.ListIterator; @@ -291,17 +293,19 @@ protected List> processRows(List> rows, private enum PLATE { - spot("Spot counts:", "spots"), - saturation("Well's saturation values (%)", "saturation"), - cytokine("Cytokine Activities:", "cytokine"); + spot("Spot counts:", "spots", Collections.singletonList("Number of Spots:")), + saturation("Well's saturation values (%)", "saturation", Collections.singletonList("Well's saturation values (%)")), + cytokine("Cytokine Activities:", "cytokine", Collections.singletonList("Activity:")); private String description; private String field; + private Collection aliases; - PLATE(String description, String field) + PLATE(String description, String field, Collection aliases) { this.description = description; this.field = field; + this.aliases = aliases; } public static PLATE getByDescription(String description) @@ -310,6 +314,16 @@ public static PLATE getByDescription(String description) { if (t.description.equalsIgnoreCase(description)) return t; + else if (t.aliases != null) + { + for (String alias : t.aliases) + { + if (alias.equals(description)) + { + return t; + } + } + } } return null; } diff --git a/flowassays/module.properties b/flowassays/module.properties index 384050e17..df44cb72e 100644 --- a/flowassays/module.properties +++ b/flowassays/module.properties @@ -1,3 +1,2 @@ ModuleClass: org.labkey.flowassays.FlowAssaysModule -ConsolidateScripts: false ManageVersion: false diff --git a/mGAP/module.properties b/mGAP/module.properties index 5378792a1..18b50ca5b 100644 --- a/mGAP/module.properties +++ b/mGAP/module.properties @@ -1,5 +1,4 @@ ModuleClass: org.labkey.mgap.mGAPModule License: Apache 2.0 LicenseURL: http://www.apache.org/licenses/LICENSE-2.0 -ConsolidateScripts: false ManageVersion: false diff --git a/mGAP/resources/credits/jars.txt b/mGAP/resources/credits/jars.txt index b3fbb0b53..0a2c7d2f9 100644 --- a/mGAP/resources/credits/jars.txt +++ b/mGAP/resources/credits/jars.txt @@ -1,4 +1,4 @@ {table} Filename|Component|Version|Source|License|LabKey Dev|Purpose -htsjdk-2.21.3.jar|htsjdk|2.21.3|{link:htsjdk|http://samtools.github.io/htsjdk/}|{link:MIT License|http://opensource.org/licenses/MIT}|bbimber|Description A Java API for high-throughput sequencing data (HTS) formats +htsjdk-2.21.3.jar|htsjdk|2.21.3|{link:htsjdk|http://samtools.github.io/htsjdk/}|{link:MIT License|http://opensource.org/licenses/MIT}|bbimber|A Java API for high-throughput sequencing data (HTS) formats {table} \ No newline at end of file diff --git a/mGAP/resources/etls/prime-seq.xml b/mGAP/resources/etls/prime-seq.xml index da78c21a3..fa21db466 100644 --- a/mGAP/resources/etls/prime-seq.xml +++ b/mGAP/resources/etls/prime-seq.xml @@ -143,7 +143,7 @@ af - + @@ -167,6 +167,6 @@ - + diff --git a/mGAP/resources/folderTypes/mGAP.folderType.xml b/mGAP/resources/folderTypes/mGAP.folderType.xml index bee390d74..bf7576130 100644 --- a/mGAP/resources/folderTypes/mGAP.folderType.xml +++ b/mGAP/resources/folderTypes/mGAP.folderType.xml @@ -84,6 +84,10 @@ mGAP Variant Releases body + + mGAP Release Notes + body + mGAP Gene Search right diff --git a/mGAP/resources/referenceStudy/datasets/datasets_metadata.xml b/mGAP/resources/referenceStudy/datasets/datasets_metadata.xml index 6c5f22112..6375977ad 100644 --- a/mGAP/resources/referenceStudy/datasets/datasets_metadata.xml +++ b/mGAP/resources/referenceStudy/datasets/datasets_metadata.xml @@ -241,7 +241,7 @@ http://cpas.labkey.com/Study#VisitDate - Gender + Sex varchar diff --git a/mGAP/resources/schemas/mgap.xml b/mGAP/resources/schemas/mgap.xml index fd75ef2c4..e55e9ffca 100644 --- a/mGAP/resources/schemas/mgap.xml +++ b/mGAP/resources/schemas/mgap.xml @@ -1000,7 +1000,7 @@ - Gender + Sex laboratory genders diff --git a/mGAP/resources/views/contact.html b/mGAP/resources/views/contact.html index 1b110d8cc..e1f2f6b53 100644 --- a/mGAP/resources/views/contact.html +++ b/mGAP/resources/views/contact.html @@ -49,7 +49,7 @@ success: function(response){ console.log(response); - Ext4.Msg.alert('Success', 'An account has been requested. You should receive a reply shortly.', function(){ + Ext4.Msg.alert('Success', 'Your request has been sent. You should receive a reply shortly.', function(){ window.location = LABKEY.ActionURL.getContextPath() + '/'; }); }, diff --git a/mGAP/resources/views/geneSearch.html b/mGAP/resources/views/geneSearch.html index 7aaa583a2..515e90b9e 100644 --- a/mGAP/resources/views/geneSearch.html +++ b/mGAP/resources/views/geneSearch.html @@ -170,7 +170,7 @@ } }); - url = Object.keys(unique)[0] + ':' + minStart + '..' + maxStop; + url = Object.keys(uniqueRef)[0] + ':' + minStart + '..' + maxStop; } if (!url) { diff --git a/mGAP/resources/views/releaseNotes.html b/mGAP/resources/views/releaseNotes.html new file mode 100644 index 000000000..110244e37 --- /dev/null +++ b/mGAP/resources/views/releaseNotes.html @@ -0,0 +1,12 @@ +

Release 2.0:

+
    +
  • Substantial revamp of all data. All samples have been realigned to the MMul_10 reference genome, followed by our standard GenotypeGVCFs pipeline. The MMul_10 is the most complete rhesus macaque assembly to date, and we expect this should improve accuracy of variant calls. Further, because our data are now aligned to the same assembly as NCBI/Ensembl, it should be easier to translate between mGAP and other databases.
  • +
  • Our internal variant calling process has switched to use GATK's GenomicsDB to pre-aggregate data prior to calling with GenotypeGVCFs, as opposed to CombineGVCFs, which was used in prior releases. This should be a purely technical difference with no change in the resulting data
  • +
+ +

Future Plans:

+
    +
  • We expect to upgrade the genome browser to use the redesigned JBrowse 2 browser. This should provide general performance improvements and will make future mGAP-specific customization easier.
  • +
  • We will support other modes of viewing and downloading variant data, in particular tabular views by gene.
  • +
  • We recognize that the mGAP release VCF can be enormous, particularly because of all the site-specific functional annotation. To support different types of users, upcoming releases will include 'slim' versions of the data, which will be downloadable files with certain information removed to save file size.
  • +
\ No newline at end of file diff --git a/mGAP/resources/views/releaseNotes.view.xml b/mGAP/resources/views/releaseNotes.view.xml new file mode 100644 index 000000000..c9a2b3e33 --- /dev/null +++ b/mGAP/resources/views/releaseNotes.view.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/mGAP/resources/views/releaseNotes.webpart.xml b/mGAP/resources/views/releaseNotes.webpart.xml new file mode 100644 index 000000000..f7f58d800 --- /dev/null +++ b/mGAP/resources/views/releaseNotes.webpart.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/mGAP/resources/views/variants.html b/mGAP/resources/views/variants.html index 3d66a6667..f59586d79 100644 --- a/mGAP/resources/views/variants.html +++ b/mGAP/resources/views/variants.html @@ -9,7 +9,7 @@ title: 'Variant Catalog Releases', schemaName: 'mgap', queryName: 'variantCatalogReleases', - maxRows: 50, + maxRows: 3, showRecordSelectors: false, showDetailsColumn: false, buttonBar: {position: 'none', includeStandardButtons: false, items: []} diff --git a/mGAP/src/org/labkey/mgap/columnTransforms/JBrowseSessionTransform.java b/mGAP/src/org/labkey/mgap/columnTransforms/JBrowseSessionTransform.java index 8493d435c..a9a8d57ae 100644 --- a/mGAP/src/org/labkey/mgap/columnTransforms/JBrowseSessionTransform.java +++ b/mGAP/src/org/labkey/mgap/columnTransforms/JBrowseSessionTransform.java @@ -309,6 +309,6 @@ protected String getDatabaseName() protected String getTrackJson() { - return "{\"category\":\"mGAP Variant Catalog\",\"visibleByDefault\": true,\"ensemblUrl\":\"jul2019.archive.ensembl.org\",\"ensemblId\":\"Macaca_mulatta\",\"additionalFeatureMsg\":\"

**The annotations below are primarily derived from human data sources (not macaque), and must be viewed in that context.

\"}"; + return "{\"category\":\"mGAP Variant Catalog\",\"visibleByDefault\": true,\"ensemblId\":\"Macaca_mulatta\",\"additionalFeatureMsg\":\"

**The annotations below are primarily derived from human data sources (not macaque), and must be viewed in that context.

\"}"; } } diff --git a/mGAP/src/org/labkey/mgap/mGAPDemographicsSource.java b/mGAP/src/org/labkey/mgap/mGAPDemographicsSource.java index 52539e61d..97047cd49 100644 --- a/mGAP/src/org/labkey/mgap/mGAPDemographicsSource.java +++ b/mGAP/src/org/labkey/mgap/mGAPDemographicsSource.java @@ -61,13 +61,15 @@ public Map> resolveSubjects(List subjects, C { if ("datatypes".equalsIgnoreCase(field)) { - map.put("datatypes", (dataTypeMap.containsKey(subject) ? StringUtils.join(dataTypeMap.get(subject)) : null)); + map.put("datatypes", (dataTypeMap.containsKey(subject) ? StringUtils.join(dataTypeMap.get(subject), ",") : null)); } else { map.put(field, rs.getObject(FieldKey.fromString(field))); } } + + ret.put(subject, map); }); return ret; @@ -77,7 +79,7 @@ public Map> resolveSubjects(List subjects, C public LinkedHashMap getFields() { LinkedHashMap ret = new LinkedHashMap<>(); - ret.put("gender", "Gender"); + ret.put("gender", "Sex"); ret.put("species", "Species"); ret.put("center", "Center"); ret.put("geographic_origin", "Geographic Origin"); diff --git a/mGAP/src/org/labkey/mgap/pipeline/mGapReleaseGenerator.java b/mGAP/src/org/labkey/mgap/pipeline/mGapReleaseGenerator.java index 7e4df8200..a3fd61c17 100644 --- a/mGAP/src/org/labkey/mgap/pipeline/mGapReleaseGenerator.java +++ b/mGAP/src/org/labkey/mgap/pipeline/mGapReleaseGenerator.java @@ -163,10 +163,11 @@ public Processor() } @Override - public void init(PipelineJob job, SequenceAnalysisJobSupport support, List inputFiles, JSONObject params, File outputDir, List actions, List outputsToCreate) throws UnsupportedOperationException, PipelineJobException + public void init(JobContext ctx, List inputFiles, List actions, List outputsToCreate) throws UnsupportedOperationException, PipelineJobException { - job.getLogger().info("writing track/subset data to file"); - TableInfo releaseTracks = QueryService.get().getUserSchema(job.getUser(), (job.getContainer().isWorkbook() ? job.getContainer().getParent() : job.getContainer()), mGAPSchema.NAME).getTable(mGAPSchema.TABLE_RELEASE_TRACKS); + ctx.getJob().getLogger().info("writing track/subset data to file"); + Container target = ctx.getJob().getContainer().isWorkbook() ? ctx.getJob().getContainer().getParent() : ctx.getJob().getContainer(); + TableInfo releaseTracks = QueryService.get().getUserSchema(ctx.getJob().getUser(), target, mGAPSchema.NAME).getTable(mGAPSchema.TABLE_RELEASE_TRACKS); Set toSelect = new HashSet<>(); toSelect.add(FieldKey.fromString("trackName")); @@ -179,7 +180,7 @@ public void init(PipelineJob job, SequenceAnalysisJobSupport support, List allVcfs = new HashSet<>(); Set distinctTracks = new HashSet<>(); - File trackFile = getTrackListFile(outputDir); + File trackFile = getTrackListFile(ctx.getOutputDir()); try (CSVWriter writer = new CSVWriter(PrintWriters.getPrintWriter(trackFile), '\t', CSVWriter.NO_QUOTE_CHARACTER)) { new TableSelector(releaseTracks, colMap.values(), null, null).forEachResults(rs -> { @@ -188,8 +189,9 @@ public void init(PipelineJob job, SequenceAnalysisJobSupport support, List genomeIds = new HashSet<>(); @@ -221,10 +223,10 @@ public void init(PipelineJob job, SequenceAnalysisJobSupport support, List ids = new HashSet<>(); @@ -238,7 +240,7 @@ public void init(PipelineJob job, SequenceAnalysisJobSupport support, List idsWithRecord = new TableSelector(ti, PageFlowUtil.set("subjectname"), new SimpleFilter(FieldKey.fromString("subjectname"), ids, CompareType.IN), null).getArrayList(String.class); ids.removeAll(idsWithRecord); @@ -305,6 +307,11 @@ else if (so.getCategory().endsWith("Release Track")) boolean testOnly = StringUtils.isEmpty(job.getParameters().get("testOnly")) ? false : ConvertHelper.convert(job.getParameters().get("testOnly"), boolean.class); + if (outputVCFMap.isEmpty()) + { + throw new PipelineJobException("No releases were found"); + } + String releaseId = new GUID().toString(); for (String release : outputVCFMap.keySet()) { @@ -946,7 +953,7 @@ private void inspectAndSummarizeVcf(JobContext ctx, File vcfInput, GeneToNameTra File interestingVariantTable = getVariantTableName(ctx, vcfInput); try (VCFFileReader reader = new VCFFileReader(vcfInput); CloseableIterator it = reader.iterator(); CSVWriter writer = new CSVWriter(PrintWriters.getPrintWriter(interestingVariantTable), '\t', CSVWriter.NO_QUOTE_CHARACTER)) { - writer.writeNext(new String[]{"Chromosome", "Position", "Reference", "Allele", "Source", "Reason", "Description", "Overlapping Gene(s)", "OMIM Entries", "OMIM Phenotypes", "AF", "CADD_PH"}); + writer.writeNext(new String[]{"Chromosome", "Position", "Reference", "Allele", "Source", "Reason", "Description", "Overlapping Gene(s)", "OMIM Entries", "OMIM Phenotypes", "AF", "Identifier", "CADD_PH"}); while (it.hasNext()) { Set> queuedLines = new LinkedHashSet<>(); @@ -1106,7 +1113,7 @@ private void inspectAndSummarizeVcf(JobContext ctx, File vcfInput, GeneToNameTra try { String allele = clnAlleles.get(i); - maybeWriteVariantLine(queuedLines, vc, allele, "ClinVar", diseaseSplit.get(j), description, overlappingGenes, omims, omimds, ctx.getLogger(), "ClinVar:" + clnAlleleIds.get(j)); + maybeWriteVariantLine(queuedLines, vc, allele, "ClinVar", diseaseSplit.get(j), description, overlappingGenes, omims, omimds, ctx.getLogger(), "ClinVar:" + clnAlleleIds.get(i)); } catch (IndexOutOfBoundsException e) diff --git a/mcc/build.gradle b/mcc/build.gradle new file mode 100644 index 000000000..128b7d630 --- /dev/null +++ b/mcc/build.gradle @@ -0,0 +1,18 @@ +import org.labkey.gradle.util.BuildUtils; + +dependencies { + BuildUtils.addLabKeyDependency(project: project, config: "implementation", depProjectPath: ":server:modules:DiscvrLabKeyModules:jbrowse", depProjectConfig: "apiJarFile") + BuildUtils.addLabKeyDependency(project: project, config: "implementation", depProjectPath: ":server:modules:DiscvrLabKeyModules:SequenceAnalysis", depProjectConfig: "apiJarFile") + BuildUtils.addLabKeyDependency(project: project, config: "implementation", depProjectPath: ":server:modules:LabDevKitModules:laboratory", depProjectConfig: "apiJarFile") + BuildUtils.addLabKeyDependency(project: project, config: "implementation", depProjectPath: ":server:modules:LabDevKitModules:LDK", depProjectConfig: "apiJarFile") + BuildUtils.addLabKeyDependency(project: project, config: "implementation", depProjectPath: ":server:modules:dataintegration", depProjectConfig: "apiJarFile") + BuildUtils.addLabKeyDependency(project: project, config: "implementation", depProjectPath: ":server:modules:ehrModules:ehr", depProjectConfig: "apiJarFile") + external "com.github.samtools:htsjdk:${htsjdkVersion}" + implementation "net.sf.opencsv:opencsv:${opencsvVersion}" + + BuildUtils.addLabKeyDependency(project: project, config: "modules", depProjectPath: ":server:modules:dataintegration", depProjectConfig: "published", depExtension: "module") + BuildUtils.addLabKeyDependency(project: project, config: "modules", depProjectPath: ":server:modules:LabDevKitModules:LDK", depProjectConfig: "published", depExtension: "module") + BuildUtils.addLabKeyDependency(project: project, config: "modules", depProjectPath: ":server:modules:DiscvrLabKeyModules:SequenceAnalysis", depProjectConfig: "published", depExtension: "module") + BuildUtils.addLabKeyDependency(project: project, config: "modules", depProjectPath: ":server:modules:LabDevKitModules:laboratory", depProjectConfig: "published", depExtension: "module") + BuildUtils.addLabKeyDependency(project: project, config: "modules", depProjectPath: ":server:modules:ehrModules:ehr", depProjectConfig: 'published', depExtension: 'module') +} diff --git a/mcc/module.properties b/mcc/module.properties new file mode 100644 index 000000000..ab8619788 --- /dev/null +++ b/mcc/module.properties @@ -0,0 +1,6 @@ +ModuleClass: org.labkey.mcc.MccModule +Label: Maromoset Coordinating Center +Description: This module is used by the BRAIN Initiative Maromoset Coordinating Center +License: Apache 2.0 +LicenseURL: http://www.apache.org/licenses/LICENSE-2.0 +ManageVersion: false \ No newline at end of file diff --git a/mcc/resources/credits/dependencies.txt b/mcc/resources/credits/dependencies.txt new file mode 100644 index 000000000..b3f521155 --- /dev/null +++ b/mcc/resources/credits/dependencies.txt @@ -0,0 +1,2 @@ +# direct external dependencies for project :server:modules:BimberLabKeyModules:mcc +htsjdk-2.21.3.jar diff --git a/mcc/resources/credits/jars.txt b/mcc/resources/credits/jars.txt new file mode 100644 index 000000000..0a2c7d2f9 --- /dev/null +++ b/mcc/resources/credits/jars.txt @@ -0,0 +1,4 @@ +{table} +Filename|Component|Version|Source|License|LabKey Dev|Purpose +htsjdk-2.21.3.jar|htsjdk|2.21.3|{link:htsjdk|http://samtools.github.io/htsjdk/}|{link:MIT License|http://opensource.org/licenses/MIT}|bbimber|A Java API for high-throughput sequencing data (HTS) formats +{table} \ No newline at end of file diff --git a/mcc/resources/etls/snprc.xml b/mcc/resources/etls/snprc.xml new file mode 100644 index 000000000..f61f1b9bc --- /dev/null +++ b/mcc/resources/etls/snprc.xml @@ -0,0 +1,124 @@ + + + SNPRC_Data + + + SNPRC Clinical/Demographics Data + + + Copy to target + + + AnimalId + date + gender + geographic_origin + birth + death + species + objectid + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Copy to target + + + AnimalId + date + weight + objectid + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/mcc/resources/etls/wnprc.xml b/mcc/resources/etls/wnprc.xml new file mode 100644 index 000000000..f8efab248 --- /dev/null +++ b/mcc/resources/etls/wnprc.xml @@ -0,0 +1,110 @@ + + + WNPRC_Data + WNPRC Clinical/Demographics Data + + + Copy to target + + + Id + date + gender + geographic_origin + birth + death + species + objectid + + + + + + + + + + + + Copy to target + + + Id + date + parent + relationship + method + objectid + + + + + + + + + + + + Copy to target + + + Id + date + gender + species + geographic_origin + dam + sire + objectid + + + + + + + + + + + + Copy to target + + + Id + date + weight + objectid + + + + + + + + + + + Copy to target + + + Id + date + cause + objectid + + + + + + + + + + + + + + + + diff --git a/mcc/resources/module.xml b/mcc/resources/module.xml new file mode 100644 index 000000000..b07899cb8 --- /dev/null +++ b/mcc/resources/module.xml @@ -0,0 +1,21 @@ + + + + false + This is the path to the container holding the primary MCC Study. Use of slashes is very important - it should be in the format '/myProject/mcc' + + ADMIN + + + + false + This is a comma separated list of LabKey user names of users that should be notified by email when requests are submitted through MCC. + + ADMIN + + + + + + + diff --git a/mcc/resources/queries/mcc/userRequests/Pending Requests.qview.xml b/mcc/resources/queries/mcc/userRequests/Pending Requests.qview.xml new file mode 100644 index 000000000..b849a22a8 --- /dev/null +++ b/mcc/resources/queries/mcc/userRequests/Pending Requests.qview.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/mcc/resources/queries/study/animalGroupMembership.js b/mcc/resources/queries/study/animalGroupMembership.js new file mode 100644 index 000000000..c8510ccb2 --- /dev/null +++ b/mcc/resources/queries/study/animalGroupMembership.js @@ -0,0 +1,14 @@ +/* + * Copyright (c) 2011-2014 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0 + */ + +require("ehr/triggers").initScript(this); + +function onInit(event, helper){ + helper.setScriptOptions({ + allowFutureDates: true, + removeTimeFromDate: true + }); +} \ No newline at end of file diff --git a/mcc/resources/queries/study/animalGroupMembership.xml b/mcc/resources/queries/study/animalGroupMembership.xml new file mode 100644 index 000000000..d87426e0c --- /dev/null +++ b/mcc/resources/queries/study/animalGroupMembership.xml @@ -0,0 +1,46 @@ + + + + + Animal Group Members + + + + + + Date Added + + + Date Removed + false + + + + ehr + animal_groups + rowid + name + + + + + ehr_lookups + animalGroupReleaseType + value + + + + + core + qcstate + rowid + + + + true + + +
+
+
+
diff --git a/mcc/resources/queries/study/animalGroupMembership/.qview.xml b/mcc/resources/queries/study/animalGroupMembership/.qview.xml new file mode 100644 index 000000000..c7a202751 --- /dev/null +++ b/mcc/resources/queries/study/animalGroupMembership/.qview.xml @@ -0,0 +1,18 @@ + + + + + + + + + + + + + + + + + + diff --git a/mcc/resources/queries/study/animalGroupMembership/Active Members.qview.xml b/mcc/resources/queries/study/animalGroupMembership/Active Members.qview.xml new file mode 100644 index 000000000..b0c30966e --- /dev/null +++ b/mcc/resources/queries/study/animalGroupMembership/Active Members.qview.xml @@ -0,0 +1,25 @@ + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/mcc/resources/queries/study/assignment.query.xml b/mcc/resources/queries/study/assignment.query.xml new file mode 100644 index 000000000..a210b253e --- /dev/null +++ b/mcc/resources/queries/study/assignment.query.xml @@ -0,0 +1,97 @@ + + + + + + + + + + + + + + ehr + project + project + + + + Assign Date + Date + + + Projected Release Date + + + Release Date + false + true + true + Date + + + true + + + + + + + + + + Condition At Assignment + + ehr_lookups + animal_condition + code + + + + Projected Release Condition + + ehr_lookups + animal_condition + code + + + + Condition At Release + + ehr_lookups + animal_condition + code + + + + true + + + Release Type + + ehr_lookups + assignmentReleaseType + value + + + + Date Assignment End Entered + This records the date the end of the assignment was actually entered, which may differ from the enddate itself + true + + + CoAssignments + false + true + + study + assignmentTotalCoAssigned + lsid + + + +
+
+
+
\ No newline at end of file diff --git a/mcc/resources/queries/study/assignment/.qview.xml b/mcc/resources/queries/study/assignment/.qview.xml new file mode 100644 index 000000000..4c0c2fdb7 --- /dev/null +++ b/mcc/resources/queries/study/assignment/.qview.xml @@ -0,0 +1,18 @@ + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/mcc/resources/queries/study/birth/.qview.xml b/mcc/resources/queries/study/birth/.qview.xml new file mode 100644 index 000000000..f1449329f --- /dev/null +++ b/mcc/resources/queries/study/birth/.qview.xml @@ -0,0 +1,19 @@ + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/mcc/resources/queries/study/clinremarks.xml b/mcc/resources/queries/study/clinremarks.xml new file mode 100644 index 000000000..0bda2a18e --- /dev/null +++ b/mcc/resources/queries/study/clinremarks.xml @@ -0,0 +1,22 @@ + + + + + Clinical Remarks + + + + + + + + + + + + + +
+
+
+
diff --git a/mcc/resources/queries/study/clinremarks/.qview.xml b/mcc/resources/queries/study/clinremarks/.qview.xml new file mode 100644 index 000000000..78d39cdfc --- /dev/null +++ b/mcc/resources/queries/study/clinremarks/.qview.xml @@ -0,0 +1,19 @@ + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/mcc/resources/queries/study/deaths.js b/mcc/resources/queries/study/deaths.js new file mode 100644 index 000000000..9dda9861c --- /dev/null +++ b/mcc/resources/queries/study/deaths.js @@ -0,0 +1,97 @@ +/* + * Copyright (c) 2018-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0 + */ + +require("ehr/triggers").initScript(this); +EHR.Server.Utils = require("ehr/utils").EHR.Server.Utils; + +var demographicsUpdates = []; +var validIds = []; + +function onInit(event, helper){ + helper.setScriptOptions({ + requiresStatusRecalc: true + }); + + helper.decodeExtraContextProperty('deathsInTransaction'); + + // Cache valid Ids for check on each row + LABKEY.Query.selectRows({ + requiredVersion: 9.1, + schemaName: 'study', + queryName: 'demographics', + columns: ['Id'], + scope: this, + success: function (results) { + if (!results || !results.rows || results.rows.length < 1) + return; + + for(var i=0; i 0) { + console.log('updating demographics death date for ' + demographicsUpdates.length + " animals"); + helper.getJavaHelper().updateDemographicsRecord(demographicsUpdates); + } + + var deaths = helper.getDeaths(); + if (deaths){ + var ids = []; + for (var id in deaths){ + ids.push(id); + } + + if (!helper.isETL()) { + console.log('sending death notification'); + helper.getJavaHelper().sendDeathNotification(ids); + } + } +} \ No newline at end of file diff --git a/mcc/resources/queries/study/deaths.query.xml b/mcc/resources/queries/study/deaths.query.xml new file mode 100644 index 000000000..25e309e6e --- /dev/null +++ b/mcc/resources/queries/study/deaths.query.xml @@ -0,0 +1,93 @@ + + + + + + + + + + + + + + + Time of Death + + + Type of Death + + ehr_lookups + death_cause + value + + + + Manner of Death + + ehr_lookups + death_manner + value + + + + Necropsy Case No + /query/executeQuery.view?schemaName=study& + query.queryName=Necropsies& + query.caseno~eq=${necropsy}& + + + + + 110 + textarea + + + Cage At Time + + + Key + false + false + false + true + + + + Tattoo/Tag Number + + + Dam (infants only) + + + Entered By + + + Room At Time + + + + + + ehr_lookups + rooms + room + + + + Death Was Not At Center + + + Final Condition + true + + ehr_lookups + animal_condition + code + + + +
+
+
+
\ No newline at end of file diff --git a/mcc/resources/queries/study/deaths/.qview.xml b/mcc/resources/queries/study/deaths/.qview.xml new file mode 100644 index 000000000..fa7420819 --- /dev/null +++ b/mcc/resources/queries/study/deaths/.qview.xml @@ -0,0 +1,16 @@ + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/mcc/resources/queries/study/demographics.js b/mcc/resources/queries/study/demographics.js new file mode 100644 index 000000000..992138d43 --- /dev/null +++ b/mcc/resources/queries/study/demographics.js @@ -0,0 +1,23 @@ +/* + * Copyright (c) 2010-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0 + */ + +require("ehr/triggers").initScript(this); + +function onInit(event, helper){ + helper.setScriptOptions({ + allowAnyId: true, + requiresStatusRecalc: false, + allowDatesInDistantPast: true + }); +} + +function onUpsert(helper, scriptErrors, row, oldRow){ + //NOTE: this should be getting set by the birth, death, arrival & departure tables + //ALSO: it should be rare to insert directly into this table. usually this record will be created by inserting into either birth or arrival + if (!row.calculated_status && !helper.isETL()){ + row.calculated_status = helper.getJavaHelper().getCalculatedStatusValue(row.Id); + } +} \ No newline at end of file diff --git a/mcc/resources/queries/study/demographics.query.xml b/mcc/resources/queries/study/demographics.query.xml new file mode 100644 index 000000000..48d852b41 --- /dev/null +++ b/mcc/resources/queries/study/demographics.query.xml @@ -0,0 +1,242 @@ + + + + + + + + + + + + + + + true + + + true + + + Gender + + ehr_lookups + gender_codes + code + + + + Species + + ehr_lookups + species + common + + + + + Geographic Origin + + ehr_lookups + geographic_origins + meaning + + + + + + Date + Birth + /query/executeQuery.view? + schemaName=study& + query.queryName=Birth& + query.Id~eq=${Id} + + + + + Date + Death + /query/executeQuery.view? + schemaName=study& + query.queryName=Deaths& + query.Id~eq=${Id} + + + + false + true + Status + + ehr_lookups + status_codes + value + + + + false + Status + + ehr_lookups + calculated_status_codes + code + + + + + + + Record Status + true + + + + + true + Availability + + + + + + + + Hold + + + + + + + + Dam + + + + + + /ehr/participantView.view?participantId=${dam} + + + Sire + + + + + + /ehr/participantView.view?participantId=${sire} + + + Origin + + ehr_lookups + source + code + + + + + + true + DateTime + false + Arrival Date + + + true + DateTime + false + Departure Date + + + true + false + Room + /ehr/cageDetails.view? + room=${room}& + + + ehr_lookups + rooms + room + + + + true + false + Cage + /ehr/cageDetails.view? + room=${room}& + cage=${cage}& + + + + true + false + 30 + Condition + + ehr_lookups + housing_condition_codes + value + + + + true + false + Current Weight (kg) + /query/executeQuery.view?schemaName=study& + query.queryName=Weight& + query.id~eq=${id} + + + + + false + true + DateTime + Weight Date + /query/executeQuery.view?schemaName=study& + query.queryName=Weight& + query.id~eq=${id}& + query.date~eq=${wdate} + + + + false + true + Date + Last TB Date + /query/executeQuery.view?schemaName=study& + query.queryName=TB Tests& + query.id~eq=${id}& + query.sort=-Date + + + + Medical + + + Replacement Prepaid By + + + Viral Status + true + + ehr_lookups + viral_status + value + + + + + + + + + + +
+
+
+
\ No newline at end of file diff --git a/mcc/resources/queries/study/demographics/.qview.xml b/mcc/resources/queries/study/demographics/.qview.xml new file mode 100644 index 000000000..6ffca2250 --- /dev/null +++ b/mcc/resources/queries/study/demographics/.qview.xml @@ -0,0 +1,30 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/mcc/resources/queries/study/demographics/Alive, at Center.qview.xml b/mcc/resources/queries/study/demographics/Alive, at Center.qview.xml new file mode 100644 index 000000000..90efd4415 --- /dev/null +++ b/mcc/resources/queries/study/demographics/Alive, at Center.qview.xml @@ -0,0 +1,92 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/mcc/resources/queries/study/encounters.js b/mcc/resources/queries/study/encounters.js new file mode 100644 index 000000000..fbb17ad72 --- /dev/null +++ b/mcc/resources/queries/study/encounters.js @@ -0,0 +1,13 @@ +/* + * Copyright (c) 2018-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0 + */ + +require("ehr/triggers").initScript(this); + +function onUpsert(helper, scriptErrors, row, oldRow){ + if (!helper.isETL() && row.date && !row.requestdate){ + row.requestdate = row.date; + } +} \ No newline at end of file diff --git a/mcc/resources/queries/study/encounters.query.xml b/mcc/resources/queries/study/encounters.query.xml new file mode 100644 index 000000000..972f02566 --- /dev/null +++ b/mcc/resources/queries/study/encounters.query.xml @@ -0,0 +1,103 @@ + + + + + /ehr/encounterDetails.view?objectid=${objectid}&formtype=${taskid/formtype}&taskid=${taskid} + + + + + + + + + + Date + + + End Time + false + + + + + + Type + + ehr_lookups + encounter_types + value + + + + Charge Unit + + ehr_lookups + procedureChargeType + value + + + + Assisting Staff + false + + ehr_lookups + procedureChargeType + value + + + + Case Number + + + Procedure + + ehr_lookups + procedures + rowid + name + + + + Major Surgery? + + ehr_lookups + yesno + value + + + + Service Requested + + + Special Instructions + textarea + + + Title + + + Restraint + + ehr_lookups + restraint_type + type + + + + Time Restrained + + ehr_lookups + restraint_duration + value + + + + Date Requested + true + + +
+
+
+
\ No newline at end of file diff --git a/mcc/resources/queries/study/encounters/.qview.xml b/mcc/resources/queries/study/encounters/.qview.xml new file mode 100644 index 000000000..f1da9f3a4 --- /dev/null +++ b/mcc/resources/queries/study/encounters/.qview.xml @@ -0,0 +1,24 @@ + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/mcc/resources/queries/study/flags.js b/mcc/resources/queries/study/flags.js new file mode 100644 index 000000000..454371e4c --- /dev/null +++ b/mcc/resources/queries/study/flags.js @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2012-2018 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0 + */ + +require("ehr/triggers").initScript(this); + +function onInit(event, helper){ + helper.setScriptOptions({ + allowFutureDates: true, + removeTimeFromDate: true, + removeTimeFromEndDate: true + }); +} + +function onUpsert(helper, scriptErrors, row, oldRow){ + //if the animal is not at the center, automatically set the enddate + if (!helper.isETL() && row.Id && !row.enddate){ + EHR.Server.Utils.findDemographics({ + participant: row.Id, + helper: helper, + scope: this, + callback: function(data){ + if (!data) + return; + + if (data && data.calculated_status && data.calculated_status != 'Alive'){ + row.enddate = data.death || data.departure; + } + } + }); + + } + + if (!helper.isETL() && row.Id && row.date && row.flag){ + var active = helper.getJavaHelper().getOverlappingFlags(row.Id, row.flag, row.objectid || null, row.date); + if (active > 0){ + EHR.Server.Utils.addError(scriptErrors, 'flag', 'There are already ' + active + ' active flag(s) of the same type spanning this date.', 'INFO'); + } + } +} + +function onAfterInsert(helper, errors, row){ + //if this category enforces only a single active flag at once, enforce it + //note: if this flag has a future date, preemptively set enddate on flags, since isActive should handle this + if (!helper.isETL() && row.Id && row.flag && !row.enddate && row.date){ + helper.getJavaHelper().ensureSingleFlagCategoryActive(row.Id, row.flag, row.objectId, row.date); + } +} \ No newline at end of file diff --git a/mcc/resources/queries/study/flags.query.xml b/mcc/resources/queries/study/flags.query.xml new file mode 100644 index 000000000..e1e38b743 --- /dev/null +++ b/mcc/resources/queries/study/flags.query.xml @@ -0,0 +1,54 @@ + + + + + + + + + + + + + Date Added + Date + + + Date Removed + false + Date + + + true + + + Category + true + + ehr_lookups + flag_categories + category + + + + + Flag + + ehr_lookups + flag_values + objectid + value + + + + Value + true + + + Entered By + + +
+
+
+
diff --git a/mcc/resources/queries/study/flags/.qview.xml b/mcc/resources/queries/study/flags/.qview.xml new file mode 100644 index 000000000..05349f54b --- /dev/null +++ b/mcc/resources/queries/study/flags/.qview.xml @@ -0,0 +1,16 @@ + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/mcc/resources/queries/study/flags/Active Flags.qview.xml b/mcc/resources/queries/study/flags/Active Flags.qview.xml new file mode 100644 index 000000000..19142d6b5 --- /dev/null +++ b/mcc/resources/queries/study/flags/Active Flags.qview.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/mcc/resources/queries/study/labwork.js b/mcc/resources/queries/study/labwork.js new file mode 100644 index 000000000..d4075c56c --- /dev/null +++ b/mcc/resources/queries/study/labwork.js @@ -0,0 +1,8 @@ +/* + * Copyright (c) 2018-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0 + */ + +require("ehr/triggers").initScript(this); + diff --git a/mcc/resources/queries/study/labwork.query.xml b/mcc/resources/queries/study/labwork.query.xml new file mode 100644 index 000000000..54592fece --- /dev/null +++ b/mcc/resources/queries/study/labwork.query.xml @@ -0,0 +1,136 @@ + + + + + + + + + + servicerequested + + + + + + Collection Date + + + + + + Service Requested + + ehr_lookups + Labwork_services + servicename + + + + Charge Unit + + ehr_lookups + labworkChargeType + value + + + + Sample Type + + ehr_lookups + clinpath_sampletype + value + + + + Tissue + + ehr_lookups + snomed + code + + + + Sample Quantity + true + + + Quantity Units + true + + + Sample Units + true + + + Collected By + + + Collection Method + + ehr_lookups + clinpath_collection_method + value + + + + Method + + + Remark + + + Category + + ehr_lookups + clinpath_types + value + + + + + Special Instructions + textarea + + + Reviewed By + + + Date Reviewed + + + true + + + true + + + true + + + true + + + true + + + true + + + Units + + + Clinical Remark + + + Sample Id + true + + + Condition + + +
+
+
+
\ No newline at end of file diff --git a/mcc/resources/queries/study/labwork/.qview.xml b/mcc/resources/queries/study/labwork/.qview.xml new file mode 100644 index 000000000..a781baa24 --- /dev/null +++ b/mcc/resources/queries/study/labwork/.qview.xml @@ -0,0 +1,19 @@ + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/mcc/resources/queries/study/labwork/Requests.qview.xml b/mcc/resources/queries/study/labwork/Requests.qview.xml new file mode 100644 index 000000000..133c818ba --- /dev/null +++ b/mcc/resources/queries/study/labwork/Requests.qview.xml @@ -0,0 +1,25 @@ + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/mcc/resources/queries/study/labworkResults.js b/mcc/resources/queries/study/labworkResults.js new file mode 100644 index 000000000..2004fb6cf --- /dev/null +++ b/mcc/resources/queries/study/labworkResults.js @@ -0,0 +1,14 @@ +/* + * Copyright (c) 2012-2018 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0 + */ + +require("ehr/triggers").initScript(this); + +function onInit(event, helper){ + helper.setScriptOptions({ + removeTimeFromDate: false, + allowDatesInDistantPast: true + }); +} \ No newline at end of file diff --git a/mcc/resources/queries/study/labworkResults.query.xml b/mcc/resources/queries/study/labworkResults.query.xml new file mode 100644 index 000000000..9955e946b --- /dev/null +++ b/mcc/resources/queries/study/labworkResults.query.xml @@ -0,0 +1,67 @@ + + + + + Labwork Results + + + + + + + + + + + + + true + + + Test Id + 120 + + ehr_lookups + misc_tests + testid + + + + + Category + + + Numeric Result + + + Units + 60 + + ehr_lookups + lab_test_units + units + + + + + Text Result + + + Method + + + Sample Type + + ehr_lookups + snomed + code + + + + true + + +
+
+
+
\ No newline at end of file diff --git a/mcc/resources/queries/study/labworkResults/.qview.xml b/mcc/resources/queries/study/labworkResults/.qview.xml new file mode 100644 index 000000000..e333545d1 --- /dev/null +++ b/mcc/resources/queries/study/labworkResults/.qview.xml @@ -0,0 +1,21 @@ + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/mcc/resources/queries/study/medicationAdministration.js b/mcc/resources/queries/study/medicationAdministration.js new file mode 100644 index 000000000..d4075c56c --- /dev/null +++ b/mcc/resources/queries/study/medicationAdministration.js @@ -0,0 +1,8 @@ +/* + * Copyright (c) 2018-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0 + */ + +require("ehr/triggers").initScript(this); + diff --git a/mcc/resources/queries/study/medicationAdministration.query.xml b/mcc/resources/queries/study/medicationAdministration.query.xml new file mode 100644 index 000000000..81252806f --- /dev/null +++ b/mcc/resources/queries/study/medicationAdministration.query.xml @@ -0,0 +1,194 @@ + + + + + /ehr/drugDetails.view?lsid=${lsid} + + + + + + + + + + Begin Date + yyyy-MM-dd HH:mm + + + Header Date + yyyy-MM-dd H:mm + + + End Time + yyyy-MM-dd H:mm + false + + + Charge To + + ehr + project + project + + + + Credit To + + ehr_lookups + medicationChargeType + value + + + No Charge + + + Code + + ehr_lookups + snomed + code + + + + Is Billable + + ehr_lookups + yesno + value + + + No + + + Qualifier + + + Reason + + ehr_lookups + drugReason + value + + + + Route + + ehr_lookups + routes + route + + + + + Drug Conc + + + Conc Units + + ehr_lookups + conc_units + unit + + + + + Dosage + + + Dosage Units + + ehr_lookups + dosage_units + unit + + + + + Volume + + + Vol Units + + ehr_lookups + volume_units + unit + + + + + Amount + + + Amount Units + + ehr_lookups + amount_units + unit + + + + + Restraint + + ehr_lookups + restraint_type + type + + + + Time Restrained + + ehr_lookups + restraint_duration + value + + + + Outcome + + ehr_lookups + drugOutcome + value + + + Normal + + + Lot + + + Remark + + + Category + + ehr_lookups + drug_categories + value + + + + + Begin Date + + + Treatment Id + true + + + Time Ordered + For drugs that were ordered using the Treatment Orders table, this stores the original time this administration was scheduled to be administered. + true + + + + + + true + + +
+
+
+
\ No newline at end of file diff --git a/mcc/resources/queries/study/medicationAdministration/.qview.xml b/mcc/resources/queries/study/medicationAdministration/.qview.xml new file mode 100644 index 000000000..b99c9a3c6 --- /dev/null +++ b/mcc/resources/queries/study/medicationAdministration/.qview.xml @@ -0,0 +1,29 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/mcc/resources/queries/study/medicationOrders.js b/mcc/resources/queries/study/medicationOrders.js new file mode 100644 index 000000000..d4075c56c --- /dev/null +++ b/mcc/resources/queries/study/medicationOrders.js @@ -0,0 +1,8 @@ +/* + * Copyright (c) 2018-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0 + */ + +require("ehr/triggers").initScript(this); + diff --git a/mcc/resources/queries/study/medicationOrders.query.xml b/mcc/resources/queries/study/medicationOrders.query.xml new file mode 100644 index 000000000..a9da4a138 --- /dev/null +++ b/mcc/resources/queries/study/medicationOrders.query.xml @@ -0,0 +1,181 @@ + + + + + /EHR/treatmentDetails.view?key=${lsid} + + + + + + + + + + Begin Date + yyyy-MM-dd HH:mm + + + End Date + false + yyyy-MM-dd HH:mm + + + + Charge To + + ehr + project + project + + + + Category + + ehr_lookups + drug_categories + value + + + Depending on what is selected, the treatment will appear on a different schedule (ie. Clinical, Surgical, etc) + + + Reason + + + + Credit To + + ehr_lookups + medicationChargeType + value + + + No Charge + + + Short Name + + + false + Treatment + + ehr_lookups + snomed + code + + + + Is Billable + + ehr_lookups + yesno + value + + + No + + + Qualifier + + + Frequency + + ehr_lookups + treatment_frequency + rowid + + + + Route + + ehr_lookups + routes + route + + + 40 + + + Drug Conc + + + Conc Units + + ehr_lookups + conc_units + unit + + + + + Dosage + + + Dosage Units + + ehr_lookups + dosage_units + unit + + + + + Volume + + + Volume Units + + ehr_lookups + volume_units + unit + + + + + Amount + + + Amount Units + + ehr_lookups + amount_units + unit + + + + + false + Ordered By + + + false + Modified By + + + false + Modified Date + + + false + Last Administered + /query/executeQuery.view?schemaName=study& + query.queryName=Drug%20Administration& + query.parentid~eq=${objectid}& + + + + + + + + + + + Ordered By + + +
+
+
+
\ No newline at end of file diff --git a/mcc/resources/queries/study/medicationOrders/.qview.xml b/mcc/resources/queries/study/medicationOrders/.qview.xml new file mode 100644 index 000000000..f3854a2f8 --- /dev/null +++ b/mcc/resources/queries/study/medicationOrders/.qview.xml @@ -0,0 +1,28 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/mcc/resources/queries/study/medicationOrders/Active Treatments.qview.xml b/mcc/resources/queries/study/medicationOrders/Active Treatments.qview.xml new file mode 100644 index 000000000..97f1b7be9 --- /dev/null +++ b/mcc/resources/queries/study/medicationOrders/Active Treatments.qview.xml @@ -0,0 +1,9 @@ + + + + + + + + + \ No newline at end of file diff --git a/mcc/resources/queries/study/parentage.js b/mcc/resources/queries/study/parentage.js new file mode 100644 index 000000000..0cea7a1c5 --- /dev/null +++ b/mcc/resources/queries/study/parentage.js @@ -0,0 +1,13 @@ +/* + * Copyright (c) 2013 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0 + */ + +require("ehr/triggers").initScript(this); + +function onInit(event, helper){ + helper.setScriptOptions({ + lookupValidationFields: ['relationship', 'method'] + }); +} diff --git a/mcc/resources/queries/study/parentage.query.xml b/mcc/resources/queries/study/parentage.query.xml new file mode 100644 index 000000000..a108e2772 --- /dev/null +++ b/mcc/resources/queries/study/parentage.query.xml @@ -0,0 +1,45 @@ + + + + + + + + + + + + Parent + false + + study + animal + id + + + + Relationship + false + + ehr_lookups + parentageRelationship + value + + + + Method + false + + ehr_lookups + parentageMethod + value + + + + true + + +
+
+
+
diff --git a/mcc/resources/queries/study/parentage/.qview.xml b/mcc/resources/queries/study/parentage/.qview.xml new file mode 100644 index 000000000..ca535b042 --- /dev/null +++ b/mcc/resources/queries/study/parentage/.qview.xml @@ -0,0 +1,14 @@ + + + + + + + + + + + + + + \ No newline at end of file diff --git a/mcc/resources/queries/study/parentage/Active Calls.qview.xml b/mcc/resources/queries/study/parentage/Active Calls.qview.xml new file mode 100644 index 000000000..7d9e28d9d --- /dev/null +++ b/mcc/resources/queries/study/parentage/Active Calls.qview.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/mcc/resources/queries/study/parentageConflicts.query.xml b/mcc/resources/queries/study/parentageConflicts.query.xml new file mode 100644 index 000000000..e5bed3219 --- /dev/null +++ b/mcc/resources/queries/study/parentageConflicts.query.xml @@ -0,0 +1,31 @@ + + + + + Parentage Conflicts + + + Parent(s) + + + Relationship + + + Method(s) + + + Total Records + /query/executeQuery.view?schemaName=study& + query.queryName=Parentage& + query.Id~eq=${Id}& + query.relationship~eq=${relationship}& + query.relationship~neq=Foster Dam& + query.enddate~isblank& + query.relationship~neq=Surrogate Dam + + + +
+
+
+
diff --git a/mcc/resources/queries/study/parentageConflicts.sql b/mcc/resources/queries/study/parentageConflicts.sql new file mode 100644 index 000000000..dd879cac8 --- /dev/null +++ b/mcc/resources/queries/study/parentageConflicts.sql @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2013-2014 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +SELECT + p.Id, + p.relationship, + + group_concat(distinct p.parent) as parents, + group_concat(distinct p.method) as method, + 'Duplicate Parents With Same Relationship' as type, + count(p.Id) as totalRecords + +FROM study.parentage p +WHERE p.qcstate.publicdata = true and p.enddateCoalesced <= now() +AND p.relationship != 'Surrogate Dam' and p.relationship != 'Foster Dam' and p.enddateCoalesced >= curdate() + +GROUP BY p.Id, p.relationship +HAVING COUNT(DISTINCT p.parent) > 1 + +-- UNION ALL +-- +-- SELECT +-- p.Id, +-- p.relationship, +-- p.parent as parents, +-- group_concat(distinct p.method) as method, +-- 'Duplicate Methods For The Same Parent' as type, +-- count(p.Id) as totalRecords +-- +-- FROM study.parentage p +-- WHERE p.qcstate.publicdata = true and p.enddateCoalesced <= now() +-- AND p.relationship != 'Surrogate Dam' and p.relationship != 'Foster Dam' and p.enddateCoalesced >= curdate() +-- +-- GROUP BY p.Id, p.relationship, p.parent +-- HAVING COUNT(distinct p.method) > 1 \ No newline at end of file diff --git a/mcc/resources/queries/study/parentageSummary.query.xml b/mcc/resources/queries/study/parentageSummary.query.xml new file mode 100644 index 000000000..1efefbcd2 --- /dev/null +++ b/mcc/resources/queries/study/parentageSummary.query.xml @@ -0,0 +1,25 @@ + + + + + Parentage Summary + + + Parent + + study + animal + id + + + + Relationship + + + Method + + +
+
+
+
diff --git a/mcc/resources/queries/study/parentageSummary.sql b/mcc/resources/queries/study/parentageSummary.sql new file mode 100644 index 000000000..46086a185 --- /dev/null +++ b/mcc/resources/queries/study/parentageSummary.sql @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2013-2017 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +SELECT + p.Id, + p.date, + p.parent, + p.relationship, + p.method + +FROM study.parentage p +WHERE p.qcstate.publicdata = true and p.enddateCoalesced <= now() + +UNION ALL + +SELECT + b.Id, + b.date, + b.dam, + 'Dam' as relationship, + 'Observed' as method + +FROM study.birth b +WHERE b.dam is not null and b.qcstate.publicdata = true \ No newline at end of file diff --git a/mcc/resources/queries/study/samples.js b/mcc/resources/queries/study/samples.js new file mode 100644 index 000000000..64a117a3e --- /dev/null +++ b/mcc/resources/queries/study/samples.js @@ -0,0 +1,7 @@ +/* + * Copyright (c) 2011-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0 + */ + +require("ehr/triggers").initScript(this); \ No newline at end of file diff --git a/mcc/resources/queries/study/samples.query.xml b/mcc/resources/queries/study/samples.query.xml new file mode 100644 index 000000000..9e0f58f4c --- /dev/null +++ b/mcc/resources/queries/study/samples.query.xml @@ -0,0 +1,70 @@ + + + + + + + + + + + + + + + + + + true + + + Organ/Tissue + + ehr_lookups + snomed + code + + + + Qualifier + + ehr_lookups + snomed_qualifiers + value + + + + Tissue Condition + true + + ehr_lookups + tissue_condition + value + + + + Preparation + + ehr_lookups + tissue_preparation + value + + + + Quantity + true + + + Weight + + + No Weight + + + + + +
+
+
+
\ No newline at end of file diff --git a/mcc/resources/queries/study/samples/.qview.xml b/mcc/resources/queries/study/samples/.qview.xml new file mode 100644 index 000000000..a2a93675a --- /dev/null +++ b/mcc/resources/queries/study/samples/.qview.xml @@ -0,0 +1,20 @@ + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/mcc/resources/queries/study/weight.js b/mcc/resources/queries/study/weight.js new file mode 100644 index 000000000..a515311e1 --- /dev/null +++ b/mcc/resources/queries/study/weight.js @@ -0,0 +1,89 @@ +/* + * Copyright (c) 2010-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0 + */ + +require("ehr/triggers").initScript(this); + +function onInit(event, helper){ + helper.setScriptOptions({ + allowAnyId: true, + allowDeadIds: true, + skipIdFormatCheck: true + }); + + helper.registerRowProcessor(function(helper, row){ + if (!row) + return; + + if (!row.Id || !row.weight){ + return; + } + + var weightInTransaction = helper.getProperty('weightInTransaction'); + weightInTransaction = weightInTransaction || {}; + weightInTransaction[row.Id] = weightInTransaction[row.Id] || []; + + var shouldAdd = true; + if (row.objectid){ + LABKEY.ExtAdapter.each(weightInTransaction[row.Id], function(r){ + if (r.objectid === row.objectid){ + if (r.weight !== row.weight){ + r.weight = row.weight; + } + else { + shouldAdd = false; + return false; + } + } + }, this); + } + + if (shouldAdd){ + weightInTransaction[row.Id].push({ + objectid: row.objectid, + date: row.date, + qcstate: row.QCState, + weight: row.weight + }); + } + + helper.setProperty('weightInTransaction', weightInTransaction); + }); +} + +function onUpsert(helper, scriptErrors, row, oldRow){ + if (!row.weight){ + EHR.Server.Utils.addError(scriptErrors, 'weight', 'This field is required', 'WARN'); + } + + // warn if more than 10% different from last weight + // the highest error this can produce is WARN. therefore skip this check if we would ignore it anyway in order to save the overhead. + // this would normally occur when finalizing a form + if (!helper.isETL() && row.Id && row.weight && EHR.Server.Utils.shouldIncludeError('WARN', helper.getErrorThreshold(), helper)){ + EHR.Server.Utils.findDemographics({ + participant: row.Id, + helper: helper, + scope: this, + callback: function(data){ + if (!data) + return; + + if (data.mostRecentWeight && (row.weight <= data.mostRecentWeight * 0.9)){ + EHR.Server.Utils.addError(scriptErrors, 'weight', 'Weight drop of >10%. Last weight ' + data.mostRecentWeight + ' kg', 'INFO'); + } + else if (data.mostRecentWeight && (row.weight >= data.mostRecentWeight / 0.9)){ + EHR.Server.Utils.addError(scriptErrors, 'weight', 'Weight gain of >10%. Last weight ' + data.mostRecentWeight + ' kg', 'INFO'); + } + + if (data && data.species){ + var msg = helper.getJavaHelper().verifyWeightRange(row.id, row.weight, data.species); + if (msg != null){ + EHR.Server.Utils.addError(scriptErrors, 'weight', msg, 'WARN'); + } + } + } + }); + } +} \ No newline at end of file diff --git a/mcc/resources/queries/study/weight.query.xml b/mcc/resources/queries/study/weight.query.xml new file mode 100644 index 000000000..29c16ce25 --- /dev/null +++ b/mcc/resources/queries/study/weight.query.xml @@ -0,0 +1,48 @@ + + + + + + + + + + + + + + + + + + + + + Percent Change + false + true + + study + weightPctChange + lsid + + + + Relative Change + false + true + + study + weightRelChange + lsid + + + + Weight (kg) + 0.### + + +
+
+
+
\ No newline at end of file diff --git a/mcc/resources/queries/study/weight/.qview.xml b/mcc/resources/queries/study/weight/.qview.xml new file mode 100644 index 000000000..74ac7f98d --- /dev/null +++ b/mcc/resources/queries/study/weight/.qview.xml @@ -0,0 +1,16 @@ + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/mcc/resources/queries/wnprcSource/birth.sql b/mcc/resources/queries/wnprcSource/birth.sql new file mode 100644 index 000000000..06808b9af --- /dev/null +++ b/mcc/resources/queries/wnprcSource/birth.sql @@ -0,0 +1,6 @@ +SELECT + +Id, date, gender, species, geographic_origin, dam, sire, objectid, modified + +FROM "/WNPRC/EHR/".study.birth +WHERE species = 'Marmoset'; \ No newline at end of file diff --git a/mcc/resources/queries/wnprcSource/deaths.sql b/mcc/resources/queries/wnprcSource/deaths.sql new file mode 100644 index 000000000..94c301bb7 --- /dev/null +++ b/mcc/resources/queries/wnprcSource/deaths.sql @@ -0,0 +1,8 @@ +SELECT + + Id, date, + cause, + objectid, modified + +FROM "/WNPRC/EHR/".study.weight +WHERE Id.demographics.species = 'Marmoset'; \ No newline at end of file diff --git a/mcc/resources/queries/wnprcSource/demographics.sql b/mcc/resources/queries/wnprcSource/demographics.sql new file mode 100644 index 000000000..7a860fef3 --- /dev/null +++ b/mcc/resources/queries/wnprcSource/demographics.sql @@ -0,0 +1,6 @@ +SELECT + +Id, date, gender, geographic_origin, birth, death, species, objectid, modified + +FROM "/WNPRC/EHR/".study.demographics +WHERE species = 'Marmoset'; \ No newline at end of file diff --git a/mcc/resources/queries/wnprcSource/parentage.sql b/mcc/resources/queries/wnprcSource/parentage.sql new file mode 100644 index 000000000..a8fe2e80c --- /dev/null +++ b/mcc/resources/queries/wnprcSource/parentage.sql @@ -0,0 +1,27 @@ +SELECT + + Id, + date, + sire as parent, + 'Sire' as relationship, + 'Observed' as method, + cast(objectid as varchar) || '-Sire' as objectid, + modified + +FROM "/WNPRC/EHR/".study.demographics +WHERE species = 'Marmoset' and sire is not null + +UNION ALL + +SELECT + + Id, + date, + sire as parent, + 'Dam' as relationship, + 'Observed' as method, + cast(objectid as varchar) || '-Dam' as objectid, + modified + +FROM "/WNPRC/EHR/".study.demographics +WHERE species = 'Marmoset' and dam is not null \ No newline at end of file diff --git a/mcc/resources/queries/wnprcSource/weight.sql b/mcc/resources/queries/wnprcSource/weight.sql new file mode 100644 index 000000000..db7bce107 --- /dev/null +++ b/mcc/resources/queries/wnprcSource/weight.sql @@ -0,0 +1,8 @@ +SELECT + + Id, date, + weight, + objectid, modified + +FROM "/WNPRC/EHR/".study.weight +WHERE Id.demographics.species = 'Marmoset'; \ No newline at end of file diff --git a/mcc/resources/referenceStudy/README b/mcc/resources/referenceStudy/README new file mode 100644 index 000000000..8710e0b71 --- /dev/null +++ b/mcc/resources/referenceStudy/README @@ -0,0 +1,12 @@ +This folder contains the reference study for the MCC EHR. It should be generated by performing +a folder export from the production server, then copying the datasets_manifest.xml, datasets_metadata.xml, +PrimateElectronicHealthRecord.dataset and study.xml files. After copying, the following can be used to find/replace +metadata in datasets_metadata.xml to remove unwanted information: + +Replace the following regex expressions with empty string: + +( )*(.*)\n +( )*(.*)\n|( )*(.*)\n +( )*(.*)\n|( )*(.*)\n|( )*(.*)\n|( )*(.*)\n +( )*(.*)\n|( )*(.*)\n|( )*(.)*\n( )*(.)*\n +( )*(.*)\n|( )*(.*)\n|( )*(.*)\n diff --git a/mcc/resources/referenceStudy/datasets/PrimateElectronicHealthRecord.dataset b/mcc/resources/referenceStudy/datasets/PrimateElectronicHealthRecord.dataset new file mode 100644 index 000000000..8e5970288 --- /dev/null +++ b/mcc/resources/referenceStudy/datasets/PrimateElectronicHealthRecord.dataset @@ -0,0 +1,19 @@ +# default group can be used to avoid repeating definitions for each dataset +# +# action=[REPLACE,APPEND,DELETE] (default:REPLACE) +# deleteAfterImport=[TRUE|FALSE] (default:FALSE) + +default.action=REPLACE +default.deleteAfterImport=FALSE + +# map a source tsv column (right side) to a property name or full propertyURI (left) +# predefined properties: ParticipantId, SiteId, VisitId, Created +default.property.ParticipantId=ptid +default.property.Created=dfcreate + +# use to map from filename->datasetid +# NOTE: if there are NO explicit import definitions, we will try to import all files matching pattern +# NOTE: if there are ANY explicit mapping, we will only import listed datasets + +default.filePattern=dataset(\\d*).tsv +default.importAllMatches=TRUE diff --git a/mcc/resources/referenceStudy/datasets/datasets_manifest.xml b/mcc/resources/referenceStudy/datasets/datasets_manifest.xml new file mode 100644 index 000000000..352b7da94 --- /dev/null +++ b/mcc/resources/referenceStudy/datasets/datasets_manifest.xml @@ -0,0 +1,52 @@ + + + + ClinPath + Colony Management + Clinical + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/mcc/resources/referenceStudy/datasets/datasets_metadata.xml b/mcc/resources/referenceStudy/datasets/datasets_metadata.xml new file mode 100644 index 000000000..9738324b3 --- /dev/null +++ b/mcc/resources/referenceStudy/datasets/datasets_metadata.xml @@ -0,0 +1,679 @@ + + + + + + varchar + urn:ehr.labkey.org/#TaskId + + + varchar + urn:ehr.labkey.org/#ParentId + + + varchar + urn:ehr.labkey.org/#RequestId + + + varchar + urn:ehr.labkey.org/#PerformedBy + + + varchar + urn:ehr.labkey.org/#Description + + + varchar + urn:ehr.labkey.org/#Remark + + + + + + + varchar + http://cpas.labkey.com/Study#ParticipantId + + ptid + + + + timestamp + http://cpas.labkey.com/Study#VisitDate + http://cpas.labkey.com/Study#VisitDate + + + varchar + + + timestamp + urn:ehr.labkey.org/#EndDate + + + integer + urn:ehr.labkey.org/#Project + + + entityid + urn:ehr.labkey.org/#ObjectId + true + + + Animal Record Flags +
+ + + + varchar + http://cpas.labkey.com/Study#ParticipantId + + ptid + + + + timestamp + http://cpas.labkey.com/Study#VisitDate + http://cpas.labkey.com/Study#VisitDate + + + varchar + + + varchar + + + varchar + + + varchar + urn:ehr.labkey.org/#EndDate + + + integer + urn:ehr.labkey.org/#Project + + + entityid + urn:ehr.labkey.org/#ObjectId + true + + + Parentage +
+ + + + varchar + http://cpas.labkey.com/Study#ParticipantId + + ptid + + + + timestamp + http://cpas.labkey.com/Study#VisitDate + http://cpas.labkey.com/Study#VisitDate + + + varchar + + + varchar + + + varchar + + + double + + + timestamp + + + varchar + + + varchar + + + varchar + + + varchar + + + varchar + urn:ehr.labkey.org/#ObjectId + + + integer + urn:ehr.labkey.org/#Project + + + timestamp + urn:ehr.labkey.org/#EndDate + + + Birth +
+ + + + varchar + http://cpas.labkey.com/Study#ParticipantId + + ptid + + + + timestamp + http://cpas.labkey.com/Study#VisitDate + http://cpas.labkey.com/Study#VisitDate + + + timestamp + urn:ehr.labkey.org/#EndDate + + + integer + urn:ehr.labkey.org/#Project + + + varchar + + + entityid + urn:ehr.labkey.org/#ObjectId + true + + + integer + + + timestamp + + + Clinical Encounters +
+ + + + varchar + http://cpas.labkey.com/Study#ParticipantId + + ptid + + + + timestamp + http://cpas.labkey.com/Study#VisitDate + http://cpas.labkey.com/Study#VisitDate + + + integer + urn:ehr.labkey.org/#Project + + + varchar + + + varchar + + + entityid + urn:ehr.labkey.org/#ObjectId + true + + + timestamp + urn:ehr.labkey.org/#EndDate + + + timestamp + + + Clinical Remarks +
+ + + + varchar + http://cpas.labkey.com/Study#ParticipantId + + ptid + + + + timestamp + http://cpas.labkey.com/Study#VisitDate + http://cpas.labkey.com/Study#VisitDate + + + integer + urn:ehr.labkey.org/#Project + + + varchar + + + varchar + + + double + + + varchar + + + double + + + varchar + + + double + + + varchar + + + double + + + varchar + + + varchar + + + timestamp + + + timestamp + urn:ehr.labkey.org/#EndDate + + + varchar + + + entityid + urn:ehr.labkey.org/#ObjectId + true + + + varchar + + + varchar + + + Medication Administration +
+ + + + varchar + http://cpas.labkey.com/Study#ParticipantId + + ptid + + + + timestamp + http://cpas.labkey.com/Study#VisitDate + http://cpas.labkey.com/Study#VisitDate + + + integer + urn:ehr.labkey.org/#Project + + + varchar + + + double + + + varchar + + + double + + + varchar + + + double + + + varchar + + + varchar + + + timestamp + urn:ehr.labkey.org/#EndDate + + + integer + + + entityid + urn:ehr.labkey.org/#ObjectId + true + + + double + + + varchar + + + varchar + + + varchar + + + varchar + + + Medication/Treatment Orders +
+ + + + varchar + http://cpas.labkey.com/Study#ParticipantId + + ptid + + + + timestamp + http://cpas.labkey.com/Study#VisitDate + http://cpas.labkey.com/Study#VisitDate + + + double + + + entityid + urn:ehr.labkey.org/#ObjectId + true + + + integer + urn:ehr.labkey.org/#Project + + + timestamp + urn:ehr.labkey.org/#EndDate + + + Weight +
+ + + + varchar + http://cpas.labkey.com/Study#ParticipantId + + ptid + + + + timestamp + http://cpas.labkey.com/Study#VisitDate + http://cpas.labkey.com/Study#VisitDate + + + timestamp + urn:ehr.labkey.org/#EndDate + + + varchar + + + integer + urn:ehr.labkey.org/#Project + + + entityid + urn:ehr.labkey.org/#ObjectId + true + + + varchar + + + varchar + + + varchar + + + double + + + varchar + + + timestamp + + + Labwork +
+ + + + varchar + http://cpas.labkey.com/Study#ParticipantId + + ptid + + + + timestamp + http://cpas.labkey.com/Study#VisitDate + http://cpas.labkey.com/Study#VisitDate + + + varchar + + + double + + + varchar + + + double + + + double + + + varchar + + + varchar + + + entityid + urn:ehr.labkey.org/#ObjectId + true + + + varchar + + + integer + urn:ehr.labkey.org/#Project + + + timestamp + urn:ehr.labkey.org/#EndDate + + + varchar + + + Lab Results +
+ + + + varchar + http://cpas.labkey.com/Study#ParticipantId + + ptid + + + + timestamp + http://cpas.labkey.com/Study#VisitDate + http://cpas.labkey.com/Study#VisitDate + + + varchar + + + entityid + urn:ehr.labkey.org/#ObjectId + true + + + integer + urn:ehr.labkey.org/#Project + + + timestamp + urn:ehr.labkey.org/#EndDate + + + timestamp + + + boolean + + + Arrival +
+ + + + varchar + http://cpas.labkey.com/Study#ParticipantId + + ptid + + + + timestamp + http://cpas.labkey.com/Study#VisitDate + http://cpas.labkey.com/Study#VisitDate + + + varchar + + + varchar + urn:ehr.labkey.org/#ObjectId + + + integer + urn:ehr.labkey.org/#Project + + + timestamp + urn:ehr.labkey.org/#EndDate + + + Deaths +
+ + + + varchar + http://cpas.labkey.com/Study#ParticipantId + + ptid + + + + timestamp + http://cpas.labkey.com/Study#VisitDate + http://cpas.labkey.com/Study#VisitDate + + + varchar + + + varchar + + + timestamp + + + timestamp + + + varchar + + + varchar + urn:ehr.labkey.org/#ObjectId + + + integer + urn:ehr.labkey.org/#Project + + + timestamp + urn:ehr.labkey.org/#EndDate + + + varchar + + + varchar + + + Demographics +
+ + + + varchar + http://cpas.labkey.com/Study#ParticipantId + + ptid + + + + timestamp + http://cpas.labkey.com/Study#VisitDate + http://cpas.labkey.com/Study#VisitDate + + + varchar + + + entityid + urn:ehr.labkey.org/#ObjectId + true + + + integer + urn:ehr.labkey.org/#Project + + + timestamp + urn:ehr.labkey.org/#EndDate + + + Departure +
+
diff --git a/mcc/resources/referenceStudy/study.xml b/mcc/resources/referenceStudy/study.xml new file mode 100644 index 000000000..8f84d3259 --- /dev/null +++ b/mcc/resources/referenceStudy/study.xml @@ -0,0 +1,18 @@ + + + + + + + + + + + + + + + + + + diff --git a/mcc/resources/referenceStudy/studyPolicy.xml b/mcc/resources/referenceStudy/studyPolicy.xml new file mode 100644 index 000000000..61b6c973c --- /dev/null +++ b/mcc/resources/referenceStudy/studyPolicy.xml @@ -0,0 +1,10 @@ + + + ADVANCED_WRITE + + + + + + + \ No newline at end of file diff --git a/mcc/resources/schemas/dbscripts/postgresql/mcc-0.00-20.000.sql b/mcc/resources/schemas/dbscripts/postgresql/mcc-0.00-20.000.sql new file mode 100644 index 000000000..53628e9d3 --- /dev/null +++ b/mcc/resources/schemas/dbscripts/postgresql/mcc-0.00-20.000.sql @@ -0,0 +1,19 @@ +/* + * Copyright (c) 2020 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +-- Create schema, tables, indexes, and constraints used for Mcc module here +-- All SQL VIEW definitions should be created in mcc-create.sql and dropped in mcc-drop.sql +CREATE SCHEMA mcc; diff --git a/mcc/resources/schemas/dbscripts/postgresql/mcc-20.000-20.001.sql b/mcc/resources/schemas/dbscripts/postgresql/mcc-20.000-20.001.sql new file mode 100644 index 000000000..567bfef27 --- /dev/null +++ b/mcc/resources/schemas/dbscripts/postgresql/mcc-20.000-20.001.sql @@ -0,0 +1,18 @@ +CREATE TABLE mcc.userRequests ( + rowid serial, + email varchar(1000), + firstName varchar(1000), + lastName varchar(1000), + title varchar(1000), + institution varchar(1000), + reason varchar(4000), + userid userid, + + container entityid, + created timestamp, + createdby userid, + modified timestamp, + modifiedby userid, + + CONSTRAINT PK_userRequests PRIMARY KEY (rowid) +); \ No newline at end of file diff --git a/mcc/resources/schemas/dbscripts/sqlserver/mcc-0.00-20.000.sql b/mcc/resources/schemas/dbscripts/sqlserver/mcc-0.00-20.000.sql new file mode 100644 index 000000000..5609630ff --- /dev/null +++ b/mcc/resources/schemas/dbscripts/sqlserver/mcc-0.00-20.000.sql @@ -0,0 +1,20 @@ +/* + * Copyright (c) 2020 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +-- Create schema, tables, indexes, and constraints used for Mcc module here +-- All SQL VIEW definitions should be created in mcc-create.sql and dropped in mcc-drop.sql +CREATE SCHEMA mcc; +GO \ No newline at end of file diff --git a/mcc/resources/schemas/dbscripts/sqlserver/mcc-20.000-20.001.sql b/mcc/resources/schemas/dbscripts/sqlserver/mcc-20.000-20.001.sql new file mode 100644 index 000000000..de27af5ef --- /dev/null +++ b/mcc/resources/schemas/dbscripts/sqlserver/mcc-20.000-20.001.sql @@ -0,0 +1,18 @@ +CREATE TABLE mcc.userRequests ( + rowid int identity(1,1), + email varchar(1000), + firstName varchar(1000), + lastName varchar(1000), + title varchar(1000), + institution varchar(1000), + reason varchar(4000), + userid userid, + + container entityid, + created datetime, + createdby userid, + modified datetime, + modifiedby userid, + + CONSTRAINT PK_userRequests PRIMARY KEY (rowid) +); \ No newline at end of file diff --git a/mcc/resources/schemas/mcc.xml b/mcc/resources/schemas/mcc.xml new file mode 100644 index 000000000..e69524513 --- /dev/null +++ b/mcc/resources/schemas/mcc.xml @@ -0,0 +1,108 @@ + + + + + + + + + + + rowid + Requests For Logins + DETAILED + + + true + false + false + false + false + Request Id + + + Email + false + + + First Name + false + + + Last Name + false + + + Title + false + + + Institution + false + + + Reason For Request + false + + + false + + core + Users + UserId + + + + true + + + true + + + false + false + false + true + true + + + true + + + false + false + false + true + true + + + + ldk.context + /mcc/Security.js + + MCC.Security.approveUserRequests(dataRegionName); + + + +
+ +
\ No newline at end of file diff --git a/mcc/resources/views/_footer.html b/mcc/resources/views/_footer.html new file mode 100644 index 000000000..93de7faf0 --- /dev/null +++ b/mcc/resources/views/_footer.html @@ -0,0 +1,3 @@ +

+ The MCC is supported by NIH/BRAIN Initiative U24 XXXXXXXXX +

\ No newline at end of file diff --git a/mcc/resources/views/about.html b/mcc/resources/views/about.html new file mode 100644 index 000000000..436f532b2 --- /dev/null +++ b/mcc/resources/views/about.html @@ -0,0 +1,3 @@ +MCC is supported by NIH U24 xxxxxxx. +

+Please remember to cite this funding source in all publications that make use of MCC data. \ No newline at end of file diff --git a/mcc/resources/views/about.view.xml b/mcc/resources/views/about.view.xml new file mode 100644 index 000000000..48ca9ced8 --- /dev/null +++ b/mcc/resources/views/about.view.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/mcc/resources/views/about.webpart.xml b/mcc/resources/views/about.webpart.xml new file mode 100644 index 000000000..f40c189c7 --- /dev/null +++ b/mcc/resources/views/about.webpart.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/mcc/resources/views/contact.html b/mcc/resources/views/contact.html new file mode 100644 index 000000000..714f95170 --- /dev/null +++ b/mcc/resources/views/contact.html @@ -0,0 +1,75 @@ + \ No newline at end of file diff --git a/mcc/resources/views/contact.view.xml b/mcc/resources/views/contact.view.xml new file mode 100644 index 000000000..0fc3bfa31 --- /dev/null +++ b/mcc/resources/views/contact.view.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/mcc/resources/views/dashboard.html b/mcc/resources/views/dashboard.html new file mode 100644 index 000000000..a50a225e3 --- /dev/null +++ b/mcc/resources/views/dashboard.html @@ -0,0 +1,31 @@ + + + + \ No newline at end of file diff --git a/mcc/resources/views/dashboard.view.xml b/mcc/resources/views/dashboard.view.xml new file mode 100644 index 000000000..6779a9525 --- /dev/null +++ b/mcc/resources/views/dashboard.view.xml @@ -0,0 +1,11 @@ + + + + + + + + + + + \ No newline at end of file diff --git a/mcc/resources/views/dashboard.webpart.xml b/mcc/resources/views/dashboard.webpart.xml new file mode 100644 index 000000000..4e56bf8c1 --- /dev/null +++ b/mcc/resources/views/dashboard.webpart.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/mcc/resources/views/helpMenu.html b/mcc/resources/views/helpMenu.html new file mode 100644 index 000000000..932f848cc --- /dev/null +++ b/mcc/resources/views/helpMenu.html @@ -0,0 +1 @@ +We'd love to hear your feedback! Click here to send a help or feature request, or email mcc@ohsu.edu for any questions. \ No newline at end of file diff --git a/mcc/resources/views/helpMenu.view.xml b/mcc/resources/views/helpMenu.view.xml new file mode 100644 index 000000000..3960a69c1 --- /dev/null +++ b/mcc/resources/views/helpMenu.view.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/mcc/resources/views/helpMenu.webpart.xml b/mcc/resources/views/helpMenu.webpart.xml new file mode 100644 index 000000000..3f4624d6a --- /dev/null +++ b/mcc/resources/views/helpMenu.webpart.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/mcc/resources/views/login.html b/mcc/resources/views/login.html new file mode 100644 index 000000000..71bcbf7d3 --- /dev/null +++ b/mcc/resources/views/login.html @@ -0,0 +1,40 @@ + +
+
Sign In
+
+
+ + + + + Remember my email address + + +
+ + + + or + Request an account + +
+ + + +
+
+ \ No newline at end of file diff --git a/mcc/resources/views/login.view.xml b/mcc/resources/views/login.view.xml new file mode 100644 index 000000000..7ce47abe2 --- /dev/null +++ b/mcc/resources/views/login.view.xml @@ -0,0 +1,9 @@ + + + + + + + + + \ No newline at end of file diff --git a/mcc/resources/views/overview.html b/mcc/resources/views/overview.html new file mode 100644 index 000000000..f03b886bc --- /dev/null +++ b/mcc/resources/views/overview.html @@ -0,0 +1 @@ +This will hold the overview information for MCC. \ No newline at end of file diff --git a/mcc/resources/views/overview.view.xml b/mcc/resources/views/overview.view.xml new file mode 100644 index 000000000..f9e80a68a --- /dev/null +++ b/mcc/resources/views/overview.view.xml @@ -0,0 +1,10 @@ + + + + + + + + + + \ No newline at end of file diff --git a/mcc/resources/views/overview.webpart.xml b/mcc/resources/views/overview.webpart.xml new file mode 100644 index 000000000..c81f56bb6 --- /dev/null +++ b/mcc/resources/views/overview.webpart.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/mcc/resources/views/requestLogin.html b/mcc/resources/views/requestLogin.html new file mode 100644 index 000000000..6c3b1fb7f --- /dev/null +++ b/mcc/resources/views/requestLogin.html @@ -0,0 +1,117 @@ + \ No newline at end of file diff --git a/mcc/resources/views/requestLogin.view.xml b/mcc/resources/views/requestLogin.view.xml new file mode 100644 index 000000000..13c485f9f --- /dev/null +++ b/mcc/resources/views/requestLogin.view.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/mcc/resources/web/mcc/Security.js b/mcc/resources/web/mcc/Security.js new file mode 100644 index 000000000..5ab385833 --- /dev/null +++ b/mcc/resources/web/mcc/Security.js @@ -0,0 +1,49 @@ +Ext4.namespace('MCC.Security'); + +MCC.Security = new function(){ + return { + approveUserRequests: function(dataRegionName){ + var dr = LABKEY.DataRegions[dataRegionName]; + if (!dr){ + alert('Unable to find DataRegion with name: ' + dataRegionName); + return; + } + + var rowIds = dr.getChecked(); + if (!rowIds.length){ + alert('Must select one or more rows'); + return; + } + + Ext4.Msg.confirm('Approve Requests', 'You are able to approve ' + rowIds.length + ' user requests. Continue?', function(val){ + if (val === 'yes'){ + Ext4.Msg.wait('Loading...'); + LABKEY.Ajax.request({ + method: 'POST', + url: LABKEY.ActionURL.buildURL('mcc', 'approveUserRequests'), + params: { + requestIds: rowIds + }, + success: function(){ + Ext4.Msg.hide(); + Ext4.Msg.alert('Success', 'Requests approved!', function(){ + //note: drop view, so we see the newly added user(s) + LABKEY.DataRegions[dataRegionName].changeView(null); + }); + }, + failure: LDK.Utils.getErrorCallback({ + showAlertOnError: false, + scope: this, + callback: function(responseObj){ + if (responseObj.errorMsg){ + Ext4.Msg.alert('Error', responseObj.errorMsg); + } + } + }) + }); + + } + }, this); + } + } +}; \ No newline at end of file diff --git a/mcc/resources/web/mcc/dashboard.js b/mcc/resources/web/mcc/dashboard.js new file mode 100644 index 000000000..1a387e0c7 --- /dev/null +++ b/mcc/resources/web/mcc/dashboard.js @@ -0,0 +1,20 @@ +var MCC = {}; + +MCC.Dashboard = new function() { + return { + loadDataAndRender: function (wrapperDivId) { + LABKEY.Query.selectRows({ + schemaName: 'study', + queryName: 'demographics', + columns: 'Id,birth,death,gender,species,Id/age/AgeFriendly', + success: function(results) { + console.log(results.rows); + }, + failure: function(response) { + alert('It didnt work!'); + }, + scope: this + }); + } + } +}; diff --git a/mcc/src/org/labkey/mcc/MccController.java b/mcc/src/org/labkey/mcc/MccController.java new file mode 100644 index 000000000..ea84cef04 --- /dev/null +++ b/mcc/src/org/labkey/mcc/MccController.java @@ -0,0 +1,523 @@ +/* + * Copyright (c) 2020 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.mcc; + +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.labkey.api.action.ApiSimpleResponse; +import org.labkey.api.action.MutatingApiAction; +import org.labkey.api.action.SpringActionController; +import org.labkey.api.data.Container; +import org.labkey.api.data.CoreSchema; +import org.labkey.api.data.DbScope; +import org.labkey.api.data.SimpleFilter; +import org.labkey.api.data.Table; +import org.labkey.api.data.TableInfo; +import org.labkey.api.data.TableSelector; +import org.labkey.api.module.AllowedDuringUpgrade; +import org.labkey.api.query.DetailsURL; +import org.labkey.api.query.FieldKey; +import org.labkey.api.security.IgnoresTermsOfUse; +import org.labkey.api.security.MutableSecurityPolicy; +import org.labkey.api.security.RequiresNoPermission; +import org.labkey.api.security.RequiresPermission; +import org.labkey.api.security.SecurityManager; +import org.labkey.api.security.SecurityPolicyManager; +import org.labkey.api.security.User; +import org.labkey.api.security.UserManager; +import org.labkey.api.security.ValidEmail; +import org.labkey.api.security.permissions.AdminPermission; +import org.labkey.api.security.permissions.ReadPermission; +import org.labkey.api.security.roles.ReaderRole; +import org.labkey.api.settings.AppProps; +import org.labkey.api.settings.LookAndFeelProperties; +import org.labkey.api.util.ConfigurationException; +import org.labkey.api.util.ExceptionUtil; +import org.labkey.api.util.MailHelper; +import org.labkey.api.util.PageFlowUtil; +import org.springframework.validation.BindException; +import org.springframework.validation.Errors; + +import javax.mail.Address; +import javax.mail.Message; +import javax.mail.internet.InternetAddress; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +public class MccController extends SpringActionController +{ + private static final DefaultActionResolver _actionResolver = new DefaultActionResolver(MccController.class); + public static final String NAME = "mcc"; + + private static final Logger _log = LogManager.getLogger(MccController.class); + + public MccController() + { + setActionResolver(_actionResolver); + } + + @RequiresNoPermission + @IgnoresTermsOfUse + @AllowedDuringUpgrade + public class RequestUserAction extends MutatingApiAction + { + @Override + public void validateForm(RequestUserForm form, Errors errors) + { + Container mccContainer = MccManager.get().getMCCContainer(); + if (mccContainer == null) + { + errors.reject(ERROR_MSG, "The MCC project has not been set on this server. This is an administrator error."); + return; + } + + if (StringUtils.isEmpty(form.getEmail()) || StringUtils.isEmpty(form.getEmailConfirmation())) + { + errors.reject(ERROR_REQUIRED, "No email address provided"); + } + else if (StringUtils.isEmpty(form.getFirstName()) || StringUtils.isEmpty(form.getLastName()) || StringUtils.isEmpty(form.getTitle()) || StringUtils.isEmpty(form.getInstitution()) || StringUtils.isEmpty(form.getReason())) + { + errors.reject(ERROR_REQUIRED, "You must provide your first and last name, title, institution, and reason for requesting access"); + } + else + { + try + { + ValidEmail email = new ValidEmail(form.getEmail()); + if (!form.getEmail().equals(form.getEmailConfirmation())) + { + errors.reject(ERROR_MSG, "The email addresses you have entered do not match. Please verify your email addresses below."); + } + + TableInfo ti = MccSchema.getInstance().getSchema().getTable(MccSchema.TABLE_USER_REQUESTS); + + //first check if this email exists: + SimpleFilter filter = new SimpleFilter(FieldKey.fromString("email"), form.getEmail()); + filter.addCondition(FieldKey.fromString("container"), mccContainer.getId()); + if (new TableSelector(ti, filter, null).exists()) + { + errors.reject(ERROR_MSG, "A login has already been requested for this email. You should receive a reply shortly from the site administrator."); + } + } + catch (ValidEmail.InvalidEmailException e) + { + errors.reject(ERROR_MSG, "Your email address is not valid. Please verify your email address below."); + } + } + } + + @Override + public Object execute(RequestUserForm form, BindException errors) throws Exception + { + ApiSimpleResponse response = new ApiSimpleResponse(); + + try + { + TableInfo ti = MccSchema.getInstance().getSchema().getTable(MccSchema.TABLE_USER_REQUESTS); + Map row = new HashMap<>(); + row.put("email", form.getEmail()); + row.put("firstName", form.getFirstName()); + row.put("lastName", form.getLastName()); + row.put("title", form.getTitle()); + row.put("institution", form.getInstitution()); + row.put("reason", form.getReason()); + row.put("container", MccManager.get().getMCCContainer().getId()); + + Table.insert(UserManager.getGuestUser(), ti, row); + + Set users = MccManager.get().getNotificationUsers(); + if (users != null && !users.isEmpty()) + { + try + { + Set
emails = new HashSet<>(); + for (User u : users) + { + emails.add(new InternetAddress(u.getEmail())); + } + + MailHelper.MultipartMessage mail = MailHelper.createMultipartMessage(); + Container c = MccManager.get().getMCCContainer(); + if (c == null) + { + c = getContainer(); + _log.warn("MCC container was not set, using: " + c.getPath()); + } + + DetailsURL url = DetailsURL.fromString("/query/executeQuery.view?schemaName=mcc&query.queryName=userRequests&query.viewName=Pending Requests", c); + mail.setEncodedHtmlContent("A user requested an account on MCC. Click here to view/approve this request"); + mail.setFrom(getReplyEmail(getContainer())); + mail.setSubject("MCC Account Request"); + mail.addRecipients(Message.RecipientType.TO, emails.toArray(new Address[0])); + + MailHelper.send(mail, getUser(), c); + } + catch (Exception e) + { + ExceptionUtil.logExceptionToMothership(null, e); + } + } + + + } + catch (ConfigurationException e) + { + errors.reject(ERROR_MSG, "There was a problem sending the registration email. Please contact your administrator."); + _log.error("Error adding self registered user", e); + } + + response.put("success", !errors.hasErrors()); + if (!errors.hasErrors()) + response.put("email", form.getEmail()); + + return response; + } + } + + public static class RequestUserForm extends Object + { + private String email; + private String emailConfirmation; + private String firstName; + private String lastName; + private String title; + private String institution; + private String reason; + + public void setEmail(String email) + { + this.email = email; + } + + public String getEmail() + { + return this.email; + } + + public void setEmailConfirmation(String email) + { + this.emailConfirmation = email; + } + + public String getEmailConfirmation() + { + return this.emailConfirmation; + } + + public String getFirstName() + { + return firstName; + } + + public void setFirstName(String firstName) + { + this.firstName = firstName; + } + + public String getLastName() + { + return lastName; + } + + public void setLastName(String lastName) + { + this.lastName = lastName; + } + + public String getTitle() + { + return title; + } + + public void setTitle(String title) + { + this.title = title; + } + + public String getInstitution() + { + return institution; + } + + public void setInstitution(String institution) + { + this.institution = institution; + } + + public String getReason() + { + return reason; + } + + public void setReason(String reason) + { + this.reason = reason; + } + } + + @RequiresPermission(AdminPermission.class) + public class ApproveUserRequestsAction extends MutatingApiAction + { + @Override + public void validateForm(ApproveUserRequestsForm form, Errors errors) + { + Container mccContainer = MccManager.get().getMCCContainer(); + if (mccContainer == null) + { + errors.reject(ERROR_MSG, "The MCC project has not been set on this server. This is an administrator error."); + return; + } + + if (form.getRequestIds() == null || form.getRequestIds().length == 0) + { + errors.reject(ERROR_MSG, "No request IDs provided"); + } + + TableInfo ti = MccSchema.getInstance().getSchema().getTable(MccSchema.TABLE_USER_REQUESTS); + for (int requestId : form.getRequestIds()) + { + TableSelector ts = new TableSelector(ti, PageFlowUtil.set("userId"), new SimpleFilter(FieldKey.fromString("rowId"), requestId), null); + if (!ts.exists()) + { + errors.reject(ERROR_MSG, "No request found for request ID: " + requestId); + break; + } + } + } + + @Override + public Object execute(ApproveUserRequestsForm form, BindException errors) throws Exception + { + ApiSimpleResponse response = new ApiSimpleResponse(); + MutableSecurityPolicy policy = new MutableSecurityPolicy(MccManager.get().getMCCContainer().getPolicy()); + List newUserStatusList = new ArrayList<>(); + List existingUsersGivenAccess = new ArrayList<>(); + try (DbScope.Transaction transaction = CoreSchema.getInstance().getScope().ensureTransaction()) + { + TableInfo ti = MccSchema.getInstance().getSchema().getTable(MccSchema.TABLE_USER_REQUESTS); + for (int requestId : form.getRequestIds()) + { + TableSelector ts = new TableSelector(ti, new SimpleFilter(FieldKey.fromString("rowId"), requestId), null); + Map map = ts.getMap(requestId); + + User u; + if (map.get("userId") != null) + { + Integer userId = (Integer)map.get("userId"); + u = UserManager.getUser(userId); + existingUsersGivenAccess.add(u); + } + else + { + ValidEmail ve = new ValidEmail((String)map.get("email")); + u = UserManager.getUser(ve); + if (u != null) + { + existingUsersGivenAccess.add(u); + } + else + { + SecurityManager.NewUserStatus st = SecurityManager.addUser(ve, getUser()); + u = st.getUser(); + u.setFirstName((String)map.get("firstName")); + u.setLastName((String)map.get("lastName")); + UserManager.updateUser(getUser(), u); + + if (st.isLdapEmail()) + { + existingUsersGivenAccess.add(st.getUser()); + } + else + { + newUserStatusList.add(st); + } + } + } + + Map row = new HashMap<>(); + row.put("rowId", requestId); + row.put("userId", u.getUserId()); + Table.update(getUser(), ti, row, requestId); + + if (!policy.hasPermission(u, ReadPermission.class)) + { + policy.addRoleAssignment(u, ReaderRole.class); + } + else + { + _log.info("user already has read permission on MCC container: " + u.getDisplayName(getUser())); + } + } + + SecurityPolicyManager.savePolicy(policy); + + transaction.commit(); + } + + //send emails: + for (SecurityManager.NewUserStatus st : newUserStatusList) + { + SecurityManager.sendRegistrationEmail(getViewContext(), st.getEmail(), null, st, null); + } + + for (User u : existingUsersGivenAccess) + { + Container mccContainer = MccManager.get().getMCCContainer(); + boolean isLDAP = SecurityManager.isLdapEmail(new ValidEmail(u.getEmail())); + + MailHelper.MultipartMessage mail = MailHelper.createMultipartMessage(); + mail.setEncodedHtmlContent("Your account request has been approved for MCC! " + "Click here to access the site." + (isLDAP ? " Use your normal OHSU email/password to login." : "")); + mail.setFrom(getReplyEmail(getContainer())); + mail.setSubject("MCC Account Request"); + mail.addRecipients(Message.RecipientType.TO, u.getEmail()); + + MailHelper.send(mail, getUser(), getContainer()); + } + + response.put("success", !errors.hasErrors()); + + return response; + } + } + + private String getReplyEmail(Container c) + { + LookAndFeelProperties lfp = LookAndFeelProperties.getInstance(getContainer()); + String email = lfp.getSystemEmailAddress(); + if (email == null) + { + return AppProps.getInstance().getAdministratorContactEmail(true); + } + + return email; + } + + public static class ApproveUserRequestsForm + { + private int[] requestIds; + + public int[] getRequestIds() + { + return requestIds; + } + + public void setRequestIds(int[] requestIds) + { + this.requestIds = requestIds; + } + } + + @RequiresNoPermission + @IgnoresTermsOfUse + @AllowedDuringUpgrade + public class RequestHelpAction extends MutatingApiAction + { + @Override + public void validateForm(RequestHelpForm form, Errors errors) + { + Container mccContainer = MccManager.get().getMCCContainer(); + if (mccContainer == null) + { + errors.reject(ERROR_MSG, "The MCC project has not been set on this server. This is an administrator error."); + return; + } + + if (StringUtils.isEmpty(form.getEmail()) || StringUtils.isEmpty(form.getComment())) + { + errors.reject(ERROR_REQUIRED, "Must provide both an email address and question/comment"); + } + else + { + try + { + new ValidEmail(form.getEmail()); + } + catch (ValidEmail.InvalidEmailException e) + { + errors.reject(ERROR_MSG, "Your email address is not valid. Please verify your email address below."); + } + } + } + + @Override + public Object execute(RequestHelpForm form, BindException errors) throws Exception + { + Set users = MccManager.get().getNotificationUsers(); + if (users != null && !users.isEmpty()) + { + try + { + Set
emails = new HashSet<>(); + for (User u : users) + { + emails.add(new InternetAddress(u.getEmail())); + } + + MailHelper.MultipartMessage mail = MailHelper.createMultipartMessage(); + mail.setEncodedHtmlContent("A support request was submitted from MCC by: " + form.getEmail() + "

Message:
" + form.getComment()); + mail.setFrom(form.getEmail()); + mail.setSubject("MCC Help Request"); + mail.addRecipients(Message.RecipientType.TO, emails.toArray(new Address[0])); + + MailHelper.send(mail, getUser(), getContainer()); + } + catch (Exception e) + { + ExceptionUtil.logExceptionToMothership(null, e); + } + } + else + { + _log.error("A help request was received by MCC, but the admin emails have not been configured. The request from: " + form.getEmail()); + _log.error(form.getComment()); + } + + return new ApiSimpleResponse("success", true); + } + } + + public static class RequestHelpForm + { + private String _email; + private String _comment; + + public String getEmail() + { + return _email; + } + + public void setEmail(String email) + { + _email = email; + } + + public String getComment() + { + return _comment; + } + + public void setComment(String comment) + { + _comment = comment; + } + } +} diff --git a/mcc/src/org/labkey/mcc/MccManager.java b/mcc/src/org/labkey/mcc/MccManager.java new file mode 100644 index 000000000..d0f657de9 --- /dev/null +++ b/mcc/src/org/labkey/mcc/MccManager.java @@ -0,0 +1,102 @@ +/* + * Copyright (c) 2020 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.mcc; + +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerManager; +import org.labkey.api.module.Module; +import org.labkey.api.module.ModuleLoader; +import org.labkey.api.module.ModuleProperty; +import org.labkey.api.security.User; +import org.labkey.api.security.UserManager; +import org.labkey.api.security.ValidEmail; + +import java.util.HashSet; +import java.util.Set; + +public class MccManager +{ + private static final Logger _log = LogManager.getLogger(MccManager.class); + + public static final String ContainerPropName = "MCCContainer"; + public static final String NotifyPropName = "MCCContactUsers"; + + private static final MccManager _instance = new MccManager(); + + private MccManager() + { + // prevent external construction with a private default constructor + } + + public static MccManager get() + { + return _instance; + } + + public Container getMCCContainer() + { + Module m = ModuleLoader.getInstance().getModule(MccModule.NAME); + ModuleProperty mp = m.getModuleProperties().get(MccManager.ContainerPropName); + String path = mp.getEffectiveValue(ContainerManager.getRoot()); + if (path == null) + return null; + + return ContainerManager.getForPath(path); + } + + public Set getNotificationUsers() + { + Module m = ModuleLoader.getInstance().getModule(MccModule.NAME); + ModuleProperty mp = m.getModuleProperties().get(MccManager.NotifyPropName); + String userNames = mp.getEffectiveValue(ContainerManager.getRoot()); + userNames = StringUtils.trimToNull(userNames); + if (userNames == null) + return null; + + Set ret = new HashSet<>(); + for (String username : userNames.split(",")) + { + User u = UserManager.getUserByDisplayName(username); + if (u == null) + { + try + { + u = UserManager.getUser(new ValidEmail(username)); + } + catch (ValidEmail.InvalidEmailException e) + { + //ignore + } + } + + if (u == null) + { + _log.error("Unknown user registered for MCC notifcations: " + username); + } + + if (u != null) + { + ret.add(u); + } + } + + return ret; + } +} \ No newline at end of file diff --git a/mcc/src/org/labkey/mcc/MccModule.java b/mcc/src/org/labkey/mcc/MccModule.java new file mode 100644 index 000000000..ea28010a7 --- /dev/null +++ b/mcc/src/org/labkey/mcc/MccModule.java @@ -0,0 +1,90 @@ +/* + * Copyright (c) 2020 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.mcc; + +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.labkey.api.data.Container; +import org.labkey.api.ehr.EHRService; +import org.labkey.api.ldk.ExtendedSimpleModule; +import org.labkey.api.module.ModuleContext; +import org.labkey.api.view.WebPartFactory; + +import java.util.Collection; +import java.util.Collections; +import java.util.Set; + +public class MccModule extends ExtendedSimpleModule +{ + public static final String NAME = "MCC"; + + @Override + public String getName() + { + return NAME; + } + + @Override + public @Nullable Double getSchemaVersion() + { + return 20.001; + } + + @Override + public boolean hasScripts() + { + return true; + } + + @Override + @NotNull + protected Collection createWebPartFactories() + { + return Collections.emptyList(); + } + + @Override + protected void init() + { + addController(MccController.NAME, MccController.class); + } + + @Override + protected void doStartupAfterSpringConfig(ModuleContext moduleContext) + { + registerEHRResources(); + } + + @Override + @NotNull + public Collection getSummary(Container c) + { + return Collections.emptyList(); + } + + @Override + @NotNull + public Set getSchemaNames() + { + return Collections.singleton(MccSchema.NAME); + } + + private void registerEHRResources() + { + EHRService.get().registerModule(this); + } +} \ No newline at end of file diff --git a/mcc/src/org/labkey/mcc/MccSchema.java b/mcc/src/org/labkey/mcc/MccSchema.java new file mode 100644 index 000000000..be31ef6f1 --- /dev/null +++ b/mcc/src/org/labkey/mcc/MccSchema.java @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2020 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.mcc; + +import org.labkey.api.data.DbSchema; +import org.labkey.api.data.DbSchemaType; +import org.labkey.api.data.dialect.SqlDialect; + +public class MccSchema +{ + private static final MccSchema _instance = new MccSchema(); + public static final String NAME = "mcc"; + + public static final String TABLE_USER_REQUESTS = "userRequests"; + + public static MccSchema getInstance() + { + return _instance; + } + + private MccSchema() + { + // private constructor to prevent instantiation from + // outside this class: this singleton should only be + // accessed via org.labkey.mcc.MccSchema.getInstance() + } + + public DbSchema getSchema() + { + return DbSchema.get(NAME, DbSchemaType.Module); + } + + public SqlDialect getSqlDialect() + { + return getSchema().getSqlDialect(); + } +} diff --git a/mcc/src/org/labkey/mcc/query/UserRequestCustomizer.java b/mcc/src/org/labkey/mcc/query/UserRequestCustomizer.java new file mode 100644 index 000000000..7f789d464 --- /dev/null +++ b/mcc/src/org/labkey/mcc/query/UserRequestCustomizer.java @@ -0,0 +1,44 @@ +package org.labkey.mcc.query; + +import org.labkey.api.data.AbstractTableInfo; +import org.labkey.api.data.JdbcType; +import org.labkey.api.data.SQLFragment; +import org.labkey.api.data.TableCustomizer; +import org.labkey.api.data.TableInfo; +import org.labkey.api.ldk.LDKService; +import org.labkey.api.query.ExprColumn; + +public class UserRequestCustomizer implements TableCustomizer +{ + @Override + public void customize(TableInfo tableInfo) + { + LDKService.get().getDefaultTableCustomizer().customize(tableInfo); + + if (tableInfo instanceof AbstractTableInfo) + { + addUserCol((AbstractTableInfo)tableInfo); + } + } + + public void addUserCol(AbstractTableInfo ti) + { + String colName = "hasAccess"; + if (ti.getColumn(colName) != null) + { + return; + } + + ExprColumn col = new ExprColumn(ti, colName, new SQLFragment("(CASE WHEN (exists (" + + "select u.rowid from mcc.userrequests u " + + "left join core.RoleAssignments ra " + + "on (u.userid = ra.UserId AND u.container = ra.ResourceId) " + + "WHERE ra.Role = 'org.labkey.api.security.roles.ReaderRole' AND u.rowid = " + ExprColumn.STR_TABLE_ALIAS + ".rowid " + + ")) THEN " + ti.getSqlDialect().getBooleanTRUE() + " ELSE " + ti.getSqlDialect().getBooleanFALSE() + " END)"), JdbcType.BOOLEAN, ti.getColumn("userId")); + col.setLabel("Has MCC Access?"); + col.setReadOnly(true); + col.setIsUnselectable(true); + col.setUserEditable(false); + ti.addColumn(col); + } +} diff --git a/mcc/test/src/org/labkey/test/components/mcc/MccWebPart.java b/mcc/test/src/org/labkey/test/components/mcc/MccWebPart.java new file mode 100644 index 000000000..a48615354 --- /dev/null +++ b/mcc/test/src/org/labkey/test/components/mcc/MccWebPart.java @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2020 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.test.components.mcc; + +import org.labkey.test.Locator; +import org.labkey.test.components.BodyWebPart; +import org.labkey.test.components.html.Input; +import org.labkey.test.pages.LabKeyPage; +import org.openqa.selenium.WebDriver; +import org.openqa.selenium.WebElement; + +import static org.labkey.test.components.html.Input.Input; + +/** + * TODO: Component for a hypothetical webpart containing an input and a save button + * Component classes should handle all timing and functionality for a component + */ +public class MccWebPart extends BodyWebPart +{ + public MccWebPart(WebDriver driver) + { + this(driver, 0); + } + + public MccWebPart(WebDriver driver, int index) + { + super(driver, "Mcc", index); + } + + public MccWebPart setInput(String value) + { + elementCache().input.set(value); + // TODO: Methods that don't navigate should return this object + return this; + } + + public LabKeyPage clickSave() + { + getWrapper().clickAndWait(elementCache().button); + // TODO: Methods that navigate should return an appropriate page object + return new LabKeyPage(getDriver()); + } + + @Override + protected ElementCache newElementCache() + { + return new ElementCache(); + } + + protected class ElementCache extends BodyWebPart.ElementCache + { + protected final WebElement button = Locator.tag("button").withText("Save").findWhenNeeded(this); + protected final Input input = Input(Locator.tag("input"), getDriver()).findWhenNeeded(this); + } +} \ No newline at end of file diff --git a/mcc/test/src/org/labkey/test/pages/mcc/BeginPage.java b/mcc/test/src/org/labkey/test/pages/mcc/BeginPage.java new file mode 100644 index 000000000..3baa4087e --- /dev/null +++ b/mcc/test/src/org/labkey/test/pages/mcc/BeginPage.java @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2020 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.test.pages.mcc; + +import org.labkey.test.BaseWebDriverTest; +import org.labkey.test.WebDriverWrapper; +import org.labkey.test.Locator; +import org.labkey.test.WebTestHelper; +import org.labkey.test.pages.LabKeyPage; +import org.openqa.selenium.WebElement; + +public class BeginPage extends LabKeyPage +{ + public BeginPage(WebDriverWrapper driver) + { + super(driver); + } + + public static BeginPage beginAt(WebDriverWrapper driver) + { + return beginAt(driver, driver.getCurrentContainerPath()); + } + + public static BeginPage beginAt(WebDriverWrapper driver, String containerPath) + { + driver.beginAt(WebTestHelper.buildURL("mcc", containerPath, "begin")); + return new BeginPage(driver); + } + + public String getHelloMessage() + { + return elementCache().helloMessage.getText(); + } + + @Override + protected ElementCache newElementCache() + { + return new ElementCache(); + } + + protected class ElementCache extends LabKeyPage.ElementCache + { + protected final WebElement helloMessage = Locator.tagWithName("div", "helloMessage").findWhenNeeded(this); + } +} diff --git a/mcc/test/src/org/labkey/test/tests/mcc/MccTest.java b/mcc/test/src/org/labkey/test/tests/mcc/MccTest.java new file mode 100644 index 000000000..1c53ac721 --- /dev/null +++ b/mcc/test/src/org/labkey/test/tests/mcc/MccTest.java @@ -0,0 +1,88 @@ +/* + * Copyright (c) 2020 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.test.tests.mcc; + +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.labkey.test.BaseWebDriverTest; +import org.labkey.test.TestTimeoutException; +import org.labkey.test.categories.InDevelopment; +import org.labkey.test.pages.mcc.BeginPage; + +import java.util.Collections; +import java.util.List; + +import static org.junit.Assert.*; + +@Category({InDevelopment.class}) +public class MccTest extends BaseWebDriverTest +{ + @Override + protected void doCleanup(boolean afterTest) throws TestTimeoutException + { + _containerHelper.deleteProject(getProjectName(), afterTest); + } + + @BeforeClass + public static void setupProject() + { + MccTest init = (MccTest)getCurrentTest(); + + init.doSetup(); + } + + private void doSetup() + { + _containerHelper.createProject(getProjectName(), null); + } + + @Before + public void preTest() + { + goToProjectHome(); + } + + @Test + public void testMccModule() + { + _containerHelper.enableModule("Mcc"); + BeginPage beginPage = BeginPage.beginAt(this, getProjectName()); + assertEquals(200, getResponseCode()); + final String expectedHello = "Hello, and welcome to the Mcc module."; + assertEquals("Wrong hello message", expectedHello, beginPage.getHelloMessage()); + } + + @Override + protected BrowserType bestBrowser() + { + return BrowserType.CHROME; + } + + @Override + protected String getProjectName() + { + return "MccTest Project"; + } + + @Override + public List getAssociatedModules() + { + return Collections.singletonList("Mcc"); + } +} \ No newline at end of file diff --git a/primeseq/build.gradle b/primeseq/build.gradle index bd248fbf7..b172c8848 100644 --- a/primeseq/build.gradle +++ b/primeseq/build.gradle @@ -14,4 +14,7 @@ dependencies { BuildUtils.addLabKeyDependency(project: project, config: "modules", depProjectPath: ":server:modules:DiscvrLabKeyModules:SequenceAnalysis", depProjectConfig: "published", depExtension: "module") BuildUtils.addLabKeyDependency(project: project, config: "modules", depProjectPath: ":server:modules:DiscvrLabKeyModules:cluster", depProjectConfig: "published", depExtension: "module") BuildUtils.addLabKeyDependency(project: project, config: "modules", depProjectPath: ":server:modules:DiscvrLabKeyModules:jbrowse", depProjectConfig: "published", depExtension: "module") + + BuildUtils.addLabKeyDependency(project: project, config: "implementation", depProjectPath: ":server:modules:dataintegration", depProjectConfig: "apiJarFile") + BuildUtils.addLabKeyDependency(project: project, config: "modules", depProjectPath: ":server:modules:dataintegration", depProjectConfig: "published", depExtension: "module") } diff --git a/primeseq/module.properties b/primeseq/module.properties index 6e8b00db6..47985036b 100644 --- a/primeseq/module.properties +++ b/primeseq/module.properties @@ -3,5 +3,4 @@ Label: PRIMe-Seq Description: This module contains code related to our internal server, PRIMe-Seq License: Apache 2.0 LicenseURL: http://www.apache.org/licenses/LICENSE-2.0 -ConsolidateScripts: false ManageVersion: false \ No newline at end of file diff --git a/primeseq/resources/views/geneticsMenu.html b/primeseq/resources/views/geneticsMenu.html index 3ff3df451..5177cd413 100644 --- a/primeseq/resources/views/geneticsMenu.html +++ b/primeseq/resources/views/geneticsMenu.html @@ -44,7 +44,7 @@ 'data-qtip="You do not have permission to view this page"', 'style="width: 300px;height: auto;" class="thumb-wrap thumb-wrap-side">', '', - '{title:htmlEncode}', + '{title:htmlEncode}', '', '', '', @@ -88,15 +88,15 @@ items: [{ title: 'Public Resources', itemId: 'public' - },{ - itemId: 'collaborations', - title: 'Collaborations' },{ title: 'Labs', itemId: 'labs' },{ itemId: 'internal', title: 'Internal Projects' + },{ + itemId: 'collaborations', + title: 'Bimber Lab Collaborations' }] }); diff --git a/primeseq/src/org/labkey/primeseq/PrimeseqController.java b/primeseq/src/org/labkey/primeseq/PrimeseqController.java index bc680d82c..ee6e01d94 100644 --- a/primeseq/src/org/labkey/primeseq/PrimeseqController.java +++ b/primeseq/src/org/labkey/primeseq/PrimeseqController.java @@ -28,8 +28,13 @@ import org.labkey.api.data.Container; import org.labkey.api.data.ContainerManager; import org.labkey.api.data.ContainerType; +import org.labkey.api.data.DbScope; +import org.labkey.api.data.SQLFragment; +import org.labkey.api.data.SqlExecutor; import org.labkey.api.module.Module; import org.labkey.api.module.ModuleLoader; +import org.labkey.api.pipeline.PipeRoot; +import org.labkey.api.pipeline.PipelineService; import org.labkey.api.pipeline.PipelineUrls; import org.labkey.api.security.RequiresPermission; import org.labkey.api.security.RequiresSiteAdmin; @@ -39,6 +44,7 @@ import org.labkey.api.util.URLHelper; import org.labkey.api.view.ActionURL; import org.labkey.api.view.HtmlView; +import org.labkey.primeseq.pipeline.MhcMigrationPipelineJob; import org.springframework.validation.BindException; import org.springframework.validation.Errors; import org.springframework.web.servlet.ModelAndView; @@ -68,7 +74,7 @@ public ApiResponse execute(Object form, BindException errors) { Map resultProperties = new HashMap<>(); - resultProperties.put("collaborations", getSection("/Public/Collaborations")); + resultProperties.put("collaborations", getSection("/Labs/Bimber/Collaborations")); resultProperties.put("internal", getSection("/Internal")); resultProperties.put("labs", getSection("/Labs")); @@ -125,12 +131,6 @@ private List getSection(String path) { for (Container c : mainContainer.getChildren()) { - //NOTE: unlike EHR, omit children if the current user cannot read them - if (!c.hasPermission(getUser(), ReadPermission.class)) - { - continue; - } - JSONObject json = new JSONObject(); json.put("name", c.getName()); json.put("title", c.getTitle()); @@ -203,4 +203,239 @@ public URLHelper getSuccessURL(Object o) } } + @RequiresSiteAdmin + public class SyncMhcAction extends ConfirmAction + { + @Override + public ModelAndView getConfirmView(Object o, BindException errors) throws Exception + { + setTitle("Sync MHC Data from PRIMe"); + + return new HtmlView(HtmlString.of("This will attempt to sync MHC typing data from PRIMe to the current folder, creating all sequence records and workbooks. Do you want to continue?")); + } + + @Override + public boolean handlePost(Object o, BindException errors) throws Exception + { + try + { + PipeRoot pipelineRoot = PipelineService.get().findPipelineRoot(getContainer()); + MhcMigrationPipelineJob job = new MhcMigrationPipelineJob(getContainer(), getUser(), getViewContext().getActionURL(), pipelineRoot, "PRIMe", "ONPRC/Core Facilities/Genetics Core/MHC_Typing/"); + PipelineService.get().queueJob(job); + } + catch (Exception e) + { + _log.error(e); + errors.reject(ERROR_MSG, e.getMessage()); + return false; + + } + + return true; + } + + @Override + public void validateCommand(Object o, Errors errors) + { + + } + + @NotNull + @Override + public URLHelper getSuccessURL(Object o) + { + return PageFlowUtil.urlProvider(PipelineUrls.class).urlBegin(getContainer()); + } + } + + @RequiresSiteAdmin + public static class UpdateFilePathsAction extends ConfirmAction + { + @Override + public ModelAndView getConfirmView(UpdateFilePathsForm form, BindException errors) throws Exception + { + StringBuilder html = new StringBuilder(); + if (form.getReplacementPrefix() == null) + { + html.append("This action is designed to bulk update filepaths stored in the database, such as when a folder's file root is updated. This circumvents LabKey's normal file update listeners, and should only be performed if you are certain this is what you want. A reason for this is because the default codepath can be slow with extremely large moves."); + html.append("

"); + html.append("Enter the following:
"); + html.append(""); + html.append(""); + html.append(""); + html.append(""); + html.append(""); + html.append(""); + html.append("
"); + html.append("
"); + html.append("When you hit confirm, you will be given an intermediate page summarizing changes before any changes are actually committed. Note: this could potentially make changes site-wide. Continue?"); + + return new HtmlView(HtmlString.unsafe(html.toString())); + } + else + { + return new HtmlView(HtmlString.unsafe(generateChangeSummary(form))); + } + } + + private String generateChangeSummary(UpdateFilePathsForm form) + { + StringBuilder ret = new StringBuilder(); + ret.append("You entered the following values:"); + ret.append("
"); + ret.append(""); + ret.append(""); + ret.append(""); + ret.append(""); + ret.append(""); + ret.append(""); + ret.append("
" + HtmlString.of(form.getSourcePrefix()) + "
" + HtmlString.of(form.getReplacementPrefix()) + "
"); + ret.append(""); + ret.append("
");
+            ret.append(getSql(form, true));
+            ret.append("
"); + ret.append("
"); + ret.append("Note: if the URL of the folder changed you may also want to execute something like the following (manually):
"); + ret.append("
");
+            ret.append(HtmlString.of("UPDATE pipeline.StatusFiles SET DataUrl = replace(DataUrl, '', '') "));
+            ret.append(HtmlString.of("WHERE DataUrl like '%%';"));
+            ret.append("
"); + + ret.append("
"); + + return ret.toString(); + } + + private String ensureSlashes(String input) + { + if (input == null) + { + return input; + } + + if (!input.startsWith("/")) + { + input = "/" + input; + } + + if (!input.endsWith("/")) + { + input = input + "/"; + } + + return input; + } + + private String getSql(UpdateFilePathsForm form, boolean calculateCounts) + { + // Ensure start/end with slash: + String sourcePrefix = ensureSlashes(form.getSourcePrefix()); + String replacementPrefix = ensureSlashes(form.getReplacementPrefix()); + + StringBuilder sql = new StringBuilder(); + + if (calculateCounts) + { + int count = new SqlExecutor(DbScope.getLabKeyScope()).execute(new SQLFragment("SELECT count(*) FROM Exp.Data WHERE DataFileUrl like 'file://" + sourcePrefix + "%'")); + sql.append("--Matching rows: " + count + "\n"); + } + + sql.append("UPDATE Exp.Data SET DataFileUrl = replace(DataFileUrl, 'file://" + sourcePrefix + "', 'file://" + replacementPrefix + "') "); + sql.append("WHERE DataFileUrl like 'file://" + sourcePrefix + "%';\n"); + + if (calculateCounts) + { + int count = new SqlExecutor(DbScope.getLabKeyScope()).execute(new SQLFragment("SELECT count(*) FROM pipeline.StatusFiles WHERE FilePath like '" + sourcePrefix + "%'")); + sql.append("--Matching rows: " + count + "\n"); + } + sql.append("UPDATE pipeline.StatusFiles SET FilePath = replace(FilePath, '" + sourcePrefix + "', '" + replacementPrefix + "') "); + sql.append("WHERE FilePath like '" + sourcePrefix + "%';"); + + return sql.toString(); + } + @Override + public boolean handlePost(UpdateFilePathsForm form, BindException errors) throws Exception + { + if (form.isUpdateDatabase()) + { + String sql = getSql(form, false); + + SqlExecutor se = new SqlExecutor(DbScope.getLabKeyScope()); + se.execute(new SQLFragment(sql)); + } + + return true; + } + + @Override + public void validateCommand(UpdateFilePathsForm form, Errors errors) + { + if (form.isUpdateDatabase() && form.getReplacementPrefix() == null) + { + errors.reject(ERROR_MSG, "Missing replacementPrefix"); + } + + if (form.isUpdateDatabase() && form.getSourcePrefix() == null) + { + errors.reject(ERROR_MSG, "Missing sourcePrefix"); + } + } + + @Override + public @NotNull URLHelper getSuccessURL(UpdateFilePathsForm form) + { + if (!form.isUpdateDatabase()) + { + ActionURL url = new ActionURL(UpdateFilePathsAction.class, getContainer()); + url.addParameter("sourcePrefix", form.getSourcePrefix()); + url.addParameter("replacementPrefix", form.getReplacementPrefix()); + + return url; + } + else + { + return getContainer().getStartURL(getUser()); + } + } + } + + public static class UpdateFilePathsForm + { + private String _sourcePrefix; + private String _replacementPrefix; + private boolean _updateDatabase = false; + + public String getSourcePrefix() + { + return _sourcePrefix; + } + + public void setSourcePrefix(String sourcePrefix) + { + _sourcePrefix = sourcePrefix; + } + + public String getReplacementPrefix() + { + return _replacementPrefix; + } + + public void setReplacementPrefix(String replacementPrefix) + { + _replacementPrefix = replacementPrefix; + } + + public boolean isUpdateDatabase() + { + return _updateDatabase; + } + + public void setUpdateDatabase(boolean updateDatabase) + { + _updateDatabase = updateDatabase; + } + } } \ No newline at end of file diff --git a/primeseq/src/org/labkey/primeseq/PrimeseqModule.java b/primeseq/src/org/labkey/primeseq/PrimeseqModule.java index bc65a9262..bd36014d1 100644 --- a/primeseq/src/org/labkey/primeseq/PrimeseqModule.java +++ b/primeseq/src/org/labkey/primeseq/PrimeseqModule.java @@ -23,6 +23,7 @@ import org.labkey.api.data.Container; import org.labkey.api.ldk.ExtendedSimpleModule; import org.labkey.api.module.ModuleContext; +import org.labkey.api.pipeline.PipelineService; import org.labkey.api.sequenceanalysis.SequenceAnalysisService; import org.labkey.api.sequenceanalysis.pipeline.SequencePipelineService; import org.labkey.api.util.PageFlowUtil; @@ -36,6 +37,7 @@ import org.labkey.primeseq.pipeline.BlastPipelineJobResourceAllocator; import org.labkey.primeseq.pipeline.ClusterMaintenanceTask; import org.labkey.primeseq.pipeline.ExacloudResourceSettings; +import org.labkey.primeseq.pipeline.MhcMigrationPipelineJob; import org.labkey.primeseq.pipeline.SequenceJobResourceAllocator; import java.util.Collection; @@ -75,6 +77,8 @@ protected void doStartupAfterSpringConfig(ModuleContext moduleContext) ClusterService.get().registerResourceAllocator(new BlastPipelineJobResourceAllocator.Factory()); ClusterService.get().registerResourceAllocator(new SequenceJobResourceAllocator.Factory()); + PipelineService.get().registerPipelineProvider(new MhcMigrationPipelineJob.Provider(this)); + //register resources new PipelineStartup(); diff --git a/primeseq/src/org/labkey/primeseq/analysis/CombineMethylationRatesHandler.java b/primeseq/src/org/labkey/primeseq/analysis/CombineMethylationRatesHandler.java index e680124a3..8be4d6678 100644 --- a/primeseq/src/org/labkey/primeseq/analysis/CombineMethylationRatesHandler.java +++ b/primeseq/src/org/labkey/primeseq/analysis/CombineMethylationRatesHandler.java @@ -83,12 +83,6 @@ public SequenceOutputProcessor getProcessor() public class Processor implements SequenceOutputProcessor { - @Override - public void init(PipelineJob job, SequenceAnalysisJobSupport support, List inputFiles, JSONObject params, File outputDir, List actions, List outputsToCreate) throws UnsupportedOperationException, PipelineJobException - { - - } - @Override public void processFilesOnWebserver(PipelineJob job, SequenceAnalysisJobSupport support, List inputFiles, JSONObject params, File outputDir, List actions, List outputsToCreate) throws UnsupportedOperationException, PipelineJobException { diff --git a/primeseq/src/org/labkey/primeseq/analysis/MethylationRateComparison.java b/primeseq/src/org/labkey/primeseq/analysis/MethylationRateComparison.java index 595e58873..80ecfed3a 100644 --- a/primeseq/src/org/labkey/primeseq/analysis/MethylationRateComparison.java +++ b/primeseq/src/org/labkey/primeseq/analysis/MethylationRateComparison.java @@ -119,12 +119,6 @@ public boolean doSplitJobs() public class Processor implements SequenceOutputProcessor { - @Override - public void init(PipelineJob job, SequenceAnalysisJobSupport support, List inputFiles, JSONObject params, File outputDir, List actions, List outputsToCreate) throws UnsupportedOperationException, PipelineJobException - { - - } - @Override public void processFilesRemote(List inputFiles, JobContext ctx) throws UnsupportedOperationException, PipelineJobException { diff --git a/primeseq/src/org/labkey/primeseq/analysis/MethylationRateComparisonHandler.java b/primeseq/src/org/labkey/primeseq/analysis/MethylationRateComparisonHandler.java index 6419e9772..6ce67f819 100644 --- a/primeseq/src/org/labkey/primeseq/analysis/MethylationRateComparisonHandler.java +++ b/primeseq/src/org/labkey/primeseq/analysis/MethylationRateComparisonHandler.java @@ -142,12 +142,6 @@ public SequenceOutputProcessor getProcessor() public class Processor implements SequenceOutputProcessor { - @Override - public void init(PipelineJob job, SequenceAnalysisJobSupport support, List inputFiles, JSONObject params, File outputDir, List actions, List outputsToCreate) throws UnsupportedOperationException, PipelineJobException - { - - } - @Override public void complete(PipelineJob job, List inputs, List outputsCreated, SequenceAnalysisJobSupport support) throws PipelineJobException { diff --git a/primeseq/src/org/labkey/primeseq/pipeline/BismarkWrapper.java b/primeseq/src/org/labkey/primeseq/pipeline/BismarkWrapper.java index f533fd7f2..fe49c917c 100644 --- a/primeseq/src/org/labkey/primeseq/pipeline/BismarkWrapper.java +++ b/primeseq/src/org/labkey/primeseq/pipeline/BismarkWrapper.java @@ -14,6 +14,7 @@ import org.labkey.api.jbrowse.JBrowseService; import org.labkey.api.module.Module; import org.labkey.api.module.ModuleLoader; +import org.labkey.api.pipeline.PipelineJob; import org.labkey.api.pipeline.PipelineJobException; import org.labkey.api.reader.Readers; import org.labkey.api.resource.FileResource; @@ -550,6 +551,7 @@ public Output performAnalysisPerSampleLocal(AnalysisModel model, File inputBam, try { getPipelineCtx().getLogger().debug("preparing for JBrowse"); + getPipelineCtx().getJob().setStatus(PipelineJob.TaskStatus.running, "Preparing for JBrowse"); JBrowseService.get().prepareOutputFile(getPipelineCtx().getJob().getUser(), getPipelineCtx().getLogger(), so.getRowid(), true, additionalConfig); } catch (IOException e) diff --git a/primeseq/src/org/labkey/primeseq/pipeline/MhcMigrationPipelineJob.java b/primeseq/src/org/labkey/primeseq/pipeline/MhcMigrationPipelineJob.java new file mode 100644 index 000000000..a2a58ae30 --- /dev/null +++ b/primeseq/src/org/labkey/primeseq/pipeline/MhcMigrationPipelineJob.java @@ -0,0 +1,1621 @@ +package org.labkey.primeseq.pipeline; + +import org.apache.commons.io.FileUtils; +import org.json.JSONObject; +import org.labkey.api.assay.AssayProvider; +import org.labkey.api.assay.AssayService; +import org.labkey.api.collections.CaseInsensitiveHashMap; +import org.labkey.api.data.CompareType; +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerManager; +import org.labkey.api.data.DbSchema; +import org.labkey.api.data.DbSchemaType; +import org.labkey.api.data.DbScope; +import org.labkey.api.data.Results; +import org.labkey.api.data.SimpleFilter; +import org.labkey.api.data.Sort; +import org.labkey.api.data.Table; +import org.labkey.api.data.TableInfo; +import org.labkey.api.data.TableSelector; +import org.labkey.api.data.WorkbookContainerType; +import org.labkey.api.di.DataIntegrationService; +import org.labkey.api.exp.api.DataType; +import org.labkey.api.exp.api.ExpData; +import org.labkey.api.exp.api.ExpProtocol; +import org.labkey.api.exp.api.ExpRun; +import org.labkey.api.exp.api.ExperimentService; +import org.labkey.api.files.FileUrls; +import org.labkey.api.laboratory.LaboratoryService; +import org.labkey.api.module.FolderTypeManager; +import org.labkey.api.module.Module; +import org.labkey.api.pipeline.AbstractTaskFactory; +import org.labkey.api.pipeline.AbstractTaskFactorySettings; +import org.labkey.api.pipeline.PipeRoot; +import org.labkey.api.pipeline.PipelineDirectory; +import org.labkey.api.pipeline.PipelineJob; +import org.labkey.api.pipeline.PipelineJobException; +import org.labkey.api.pipeline.PipelineJobService; +import org.labkey.api.pipeline.PipelineProvider; +import org.labkey.api.pipeline.PipelineService; +import org.labkey.api.pipeline.PipelineStatusFile; +import org.labkey.api.pipeline.RecordedActionSet; +import org.labkey.api.pipeline.TaskId; +import org.labkey.api.pipeline.TaskPipeline; +import org.labkey.api.query.BatchValidationException; +import org.labkey.api.query.FieldKey; +import org.labkey.api.query.QueryService; +import org.labkey.api.query.UserSchema; +import org.labkey.api.query.ValidationException; +import org.labkey.api.security.User; +import org.labkey.api.sequenceanalysis.SequenceAnalysisService; +import org.labkey.api.sequenceanalysis.model.Readset; +import org.labkey.api.util.FileType; +import org.labkey.api.util.FileUtil; +import org.labkey.api.util.PageFlowUtil; +import org.labkey.api.view.ActionURL; +import org.labkey.api.view.ViewBackgroundInfo; +import org.labkey.api.view.ViewContext; +import org.labkey.remoteapi.CommandException; +import org.labkey.remoteapi.Connection; +import org.labkey.remoteapi.query.Filter; +import org.labkey.remoteapi.query.SelectRowsCommand; +import org.labkey.remoteapi.query.SelectRowsResponse; + +import java.io.File; +import java.io.IOException; +import java.net.URI; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; + +public class MhcMigrationPipelineJob extends PipelineJob +{ + private String remoteServerFolder; + private String remoteConnectionName; + + private Container targetContainer; + + public static class Provider extends PipelineProvider + { + public static final String NAME = "mhcMigrationPipeline"; + + public Provider(Module owningModule) + { + super(NAME, owningModule); + } + + @Override + public void updateFileProperties(ViewContext context, PipeRoot pr, PipelineDirectory directory, boolean includeAll) + { + + } + } + + // Default constructor for serialization + protected MhcMigrationPipelineJob() + { + } + + public MhcMigrationPipelineJob(Container c, User u, ActionURL url, PipeRoot pipeRoot, String remoteConnectionName, String remoteServerFolder) + { + super(Provider.NAME, new ViewBackgroundInfo(c, u, url), pipeRoot); + + this.targetContainer = c; + this.remoteConnectionName = remoteConnectionName; + this.remoteServerFolder = remoteServerFolder; + + File subdir = new File(pipeRoot.getRootPath(), Provider.NAME); + if (!subdir.exists()) + { + subdir.mkdirs(); + } + + setLogFile(new File(subdir, FileUtil.makeFileNameWithTimestamp("mhcMigration", "log"))); + + } + + @Override + public ActionURL getStatusHref() + { + return PageFlowUtil.urlProvider(FileUrls.class).urlBegin(getContainer()); + } + + @Override + public String getDescription() + { + return "Migrate MHC Data"; + } + + @Override + public TaskPipeline getTaskPipeline() + { + return PipelineJobService.get().getTaskPipeline(new TaskId(MhcMigrationPipelineJob.class)); + } + + public static class Task extends PipelineJob.Task + { + protected Task(Factory factory, PipelineJob job) + { + super(factory, job); + } + + public static class Factory extends AbstractTaskFactory + { + public Factory() + { + super(Task.class); + } + + @Override + public List getInputTypes() + { + return Collections.emptyList(); + } + + @Override + public String getStatusName() + { + return PipelineJob.TaskStatus.running.toString(); + } + + @Override + public List getProtocolActionNames() + { + return Arrays.asList("Migrate MHC Data"); + } + + @Override + public PipelineJob.Task createTask(PipelineJob job) + { + return new Task(this, job); + } + + @Override + public boolean isJobComplete(PipelineJob job) + { + return false; + } + } + + private MhcMigrationPipelineJob getPipelineJob() + { + return (MhcMigrationPipelineJob)getJob(); + } + + private Connection getConnection() + { + DataIntegrationService.RemoteConnection rc = DataIntegrationService.get().getRemoteConnection(getPipelineJob().remoteConnectionName, getPipelineJob().targetContainer, getJob().getLogger()); + + return(rc.connection); + } + + @Override + public RecordedActionSet run() throws PipelineJobException + { + try (DbScope.Transaction transaction = DbScope.getLabKeyScope().ensureTransaction()) + { + createWorkbooks(); + transaction.commitAndKeepConnection(); + + replaceEntireTable("genotypeassays", "primer_pairs", Arrays.asList("primername", "ref_nt_name", "ref_nt_id", "shortname"), null, true, getJob().getContainer(), getPipelineJob().remoteServerFolder); + + replaceEntireTable("laboratory", "samples", Arrays.asList("samplename", "subjectid", "sampledate", "sampletype", "samplesubtype", "samplesource", "location", "freezer", "cane", "box", "box_row", "box_column", "comment", "workbook/workbookId", "samplespecies", "processdate", "concentration", "concentration_units", "quantity", "quantity_units", "ratio"), "workbook/workbookId", true, getJob().getContainer(), getPipelineJob().remoteServerFolder); + replaceEntireTable("laboratory", "subjects", Arrays.asList("subjectname", "species"), null, true, getJob().getContainer(), getPipelineJob().remoteServerFolder); + transaction.commitAndKeepConnection(); + + Set preExisting = createLibraries(); + createLibraryMembers(preExisting); + transaction.commitAndKeepConnection(); + + createReadsets(); + transaction.commitAndKeepConnection(); + + createReaddata(); + transaction.commitAndKeepConnection(); + + createAnalyses(); + createOutputFiles(); + transaction.commitAndKeepConnection(); + + createQualityMetrics(); + transaction.commitAndKeepConnection(); + + //create assay runs, including data and haplotypes + syncAssay("GenotypeAssay", "Genotype", Arrays.asList("RowId", "Name", "Comments", "performedBy", "runDate", "instrument", "assayType", "barcode"), Arrays.asList("subjectId", "date", "marker", "result", "qual_result", "sampleId", "category", "plate", "well", "parentId", "comment", "requestid", "qcflag", "analysisId", "DataId", "sampleType", "statusflag", "rawResult")); + transaction.commitAndKeepConnection(); + + syncAssay("SSP_assay", "SSP", Arrays.asList("RowId", "Name", "Comments", "performedBy", "runDate"), Arrays.asList("subjectId", "date", "laneNumber", "method", "sampleType", "primerPair", "result", "comment", "qcflag", "statusflag")); + transaction.commitAndKeepConnection(); + + createAlignmentSummary(); + + transaction.commit(); + } + + return new RecordedActionSet(); + } + + private void syncAssay(String providerName, String assayName, List runColumns, List resultColumns) throws PipelineJobException + { + getJob().getLogger().info("syncing assay: " + providerName + " / " + assayName); + AssayProvider ap = AssayService.get().getProvider(providerName); + for (Integer wb : workbookMap.keySet()) + { + getJob().getLogger().info("processing workbook: " + wb); + + List protocols = AssayService.get().getAssayProtocols(workbookMap.get(wb), ap); + ExpProtocol protocol = protocols.get(0); + + SelectRowsCommand sr1 = new SelectRowsCommand("assay." + ap.getName() + "." + protocol.getName(), "Runs"); + sr1.setColumns(runColumns); + + try + { + SelectRowsResponse srr = sr1.execute(getConnection(), getPipelineJob().remoteServerFolder + wb); + if (srr.getRowCount().intValue() == 0) + { + continue; + } + + //Existing runs: + List existingRunNames = new TableSelector(AssayService.get().createRunTable(protocol, ap, getJob().getUser(), workbookMap.get(wb), null), PageFlowUtil.set("Name")).getArrayList(String.class); + if (existingRunNames.size() == srr.getRowCount().intValue()) + { + getJob().getLogger().info("Run count matches, skipping: " + wb); + continue; + } + + File assayTmp = File.createTempFile("assay-upload", ".txt").getAbsoluteFile(); + ViewBackgroundInfo info = getJob().getInfo(); + ViewContext vc = ViewContext.getMockViewContext(info.getUser(), workbookMap.get(wb), info.getURL(), false); + final Set missingAnalyses = new HashSet<>(); + srr.getRows().forEach(run -> { + if (existingRunNames.contains(run.get("Name"))) + { + getJob().getLogger().info("Run exists, skipping: " + run.get("Name")); + return; + } + + JSONObject json = new JSONObject(); + json.put("Run", run); + + SelectRowsCommand sr2 = new SelectRowsCommand("assay." + ap.getName() + "." + protocol.getName(), "Data"); + sr2.setColumns(resultColumns); + sr2.addFilter(new Filter("Run", run.get("RowId"))); + + try + { + SelectRowsResponse srr2 = sr2.execute(getConnection(), getPipelineJob().remoteServerFolder + wb); + List> resultRows = srr2.getRows(); + if (resultRows.isEmpty()) + { + getJob().getLogger().info("No results, skipping: " + run.get("Name")); + return; + } + + resultRows.forEach(x -> { + if (x.get("analysisId") != null) + { + if (analysisMap.containsKey(x.get("analysisId"))) + { + x.put("analysisId", analysisMap.get(x.get("analysisId"))); + } + else + { + if (!missingAnalyses.contains(x.get("analysisId"))) + { + getJob().getLogger().error("Unable to find analysis to match: " + x.get("analysisId")); + missingAnalyses.add(x.get("analysisId")); + } + } + } + }); + + LaboratoryService.get().saveAssayBatch(resultRows, json, assayTmp, vc, ap, protocol); + } + catch (ValidationException | CommandException | IOException e) + { + throw new RuntimeException(e); + } + }); + + } + catch (IOException | CommandException e) + { + throw new PipelineJobException(e); + } + } + } + + private void createQualityMetrics() throws PipelineJobException + { + for (int workbook : workbookMap.keySet()) + { + replaceEntireTable("sequenceanalysis", "quality_metrics", Arrays.asList("dataid", "dataid/DatafileUrl", "dataid/Name", "runid/JobId/FilePath", "category", "metricname", "metricvalue", "qualvalue", "analysis_id", "readset", "readset/runid/JobId", "readset/runid/JobId/FilePath", "dataid/Run/JobId/FilePath"), null, true, workbookMap.get(workbook), getPipelineJob().remoteServerFolder + workbook + "/"); + } + } + + private void createAlignmentSummary() throws PipelineJobException + { + try + { + TableInfo alignmentSummary = DbSchema.get("sequenceanalysis", DbSchemaType.Module).getTable("alignment_summary"); + TableInfo alignmentSummaryJunction = DbSchema.get("sequenceanalysis", DbSchemaType.Module).getTable("alignment_summary_junction"); + + //NOTE: split by workbook to avoid huge API calls: + SelectRowsCommand srWB = new SelectRowsCommand("core", "workbooks"); + srWB.setColumns(Arrays.asList("Name")); + + SelectRowsResponse srrWB = srWB.execute(getConnection(), getPipelineJob().remoteServerFolder); + List workbooks = srrWB.getRows().stream().map(x -> x.get("Name")).collect(Collectors.toList()); + for (Object workbook : workbooks) + { + getJob().getLogger().info("importing alignments for workbook: " + workbook); + + SelectRowsCommand sr = new SelectRowsCommand("sequenceanalysis", "alignment_summary"); + sr.setColumns(Arrays.asList("rowid", "analysis_id", "file_id", "total", "total_forward", "total_reverse", "valid_pairs", "workbook/workbookId")); + + SelectRowsResponse srr = sr.execute(getConnection(), getPipelineJob().remoteServerFolder + workbook + "/"); + getJob().getLogger().info("total alignment_summary records: " + srr.getRowCount()); + if (srr.getRowCount().intValue() == 0) + { + continue; + } + + long existing = new TableSelector(alignmentSummary, new SimpleFilter(FieldKey.fromString("container"), workbookMap.get(workbook).getId()), null).getRowCount(); + if (srr.getRowCount().longValue() == existing) + { + getJob().getLogger().info("alignment_summary row count identical, skipping: " + workbook); + continue; + } + + final Map alignmentSummaryMap = new HashMap<>(srr.getRowCount().intValue()); + srr.getRowset().forEach(rs -> { + CaseInsensitiveHashMap map = new CaseInsensitiveHashMap<>(); + Integer localId = analysisMap.get(rs.getValue("analysis_id")); + if (localId == null) + { + throw new RuntimeException("Unable to find analysis: " + rs.getValue("analysis_id")); + } + map.put("analysis_id", localId); + map.put("file_id", analysisToFileMap.get(localId)); + + map.put("total", rs.getValue("total")); + map.put("total_forward", rs.getValue("total_forward")); + map.put("total_reverse", rs.getValue("total_reverse")); + map.put("valid_pairs", rs.getValue("valid_pairs")); + + Container c = workbookMap.get((int) rs.getValue("workbook/workbookId")); + map.put("container", c.getId()); + + map = Table.insert(getJob().getUser(), alignmentSummary, map); + if (map.get("rowid") == null) + { + throw new RuntimeException("RowId was null after insert!"); + } + + alignmentSummaryMap.put((int) rs.getValue("rowid"), (int) map.get("rowid")); + }); + + SelectRowsCommand sr2 = new SelectRowsCommand("sequenceanalysis", "alignment_summary_junction"); + sr2.setColumns(Arrays.asList("analysis_id", "alignment_id", "ref_nt_id", "status", "analysis_id/workbook/workbookId", "ref_nt_id/name")); + sr2.addFilter(new Filter("analysis_id/workbook/workbookId", workbook)); + + SelectRowsResponse srr2 = sr2.execute(getConnection(), getPipelineJob().remoteServerFolder + workbook + "/"); + getJob().getLogger().info("total alignment_summary_junction records: " + srr2.getRowCount()); + srr2.getRowset().forEach(rs -> { + CaseInsensitiveHashMap map = new CaseInsensitiveHashMap<>(); + Integer localId = analysisMap.get(rs.getValue("analysis_id")); + if (localId == null) + { + throw new RuntimeException("Unable to find analysis: " + rs.getValue("analysis_id")); + } + map.put("analysis_id", localId); + + Integer localNT = sequenceMap.get(rs.getValue("ref_nt_id")); + if (localNT == null) + { + throw new RuntimeException("Unable to find ref_nt_id: " + rs.getValue("ref_nt_id") + " / " + rs.getValue("ref_nt_id/name")); + } + map.put("ref_nt_id", localNT); + map.put("status", rs.getValue("status")); + + map.put("alignment_id", alignmentSummaryMap.get("alignment_id")); + + Container c = workbookMap.get((int) rs.getValue("analysis_id/workbook/workbookId")); + map.put("container", c.getId()); + + Table.insert(getJob().getUser(), alignmentSummaryJunction, map); + }); + } + } + catch (CommandException | IOException e) + { + throw new PipelineJobException(e); + } + } + + private void replaceEntireTable(String schema, String query, List columns, String workbookColName, boolean truncateExisting, Container targetContainer, String remoteServerFolder) throws PipelineJobException + { + getJob().getLogger().info("replacing table: " + query + " for container: " + targetContainer.getPath()); + try + { + SelectRowsCommand sr = new SelectRowsCommand(schema, query); + sr.setColumns(columns); + SelectRowsResponse srr = sr.execute(getConnection(), remoteServerFolder); + + TableInfo ti = QueryService.get().getUserSchema(getJob().getUser(), targetContainer, schema).getTable(query); + long existing = new TableSelector(ti).getRowCount(); + if (srr.getRowCount().longValue() == existing) + { + getJob().getLogger().info("Row counts identical, assuming has been synced: " + query); + return; + } + else if (srr.getRowCount().equals(0)) + { + getJob().getLogger().info("No rows, skipping: " + query); + return; + } + + List> toInsert = new ArrayList<>(); + srr.getRowset().forEach(r -> { + Map row = new CaseInsensitiveHashMap<>(); + srr.getColumnModel().forEach(col -> { + String colName = (String) col.get("dataIndex"); + if (ti.getColumn(colName) != null) + { + Object val = r.getValue(colName); + if ("readset".equals(colName) || "readsetid".equals(colName)) + { + if (val != null && !readsetMap.containsKey(val)) + { + throw new IllegalStateException("Unable to find readset: " + val); + } + + val = readsetMap.get(val); + } + else if ("library_id".equals(colName)) + { + if (val != null && !libraryMap.containsKey(val)) + { + throw new IllegalStateException("Unable to find library: " + val); + } + + val = libraryMap.get(val); + + } + else if ("ref_nt_id".equals(colName)) + { + if (val != null && !sequenceMap.containsKey(val)) + { + throw new IllegalStateException("Unable to find sequence: " + val); + } + + val = sequenceMap.get(val); + } + else if ("analysis_id".equals(colName)) + { + if (val != null && !analysisMap.containsKey(val)) + { + throw new IllegalStateException("Unable to find analysis: " + val); + } + + val = analysisMap.get(val); + } + + row.put(colName, val); + } + else if ("dataid/DatafileUrl".equalsIgnoreCase(colName) && r.getValue("dataid/DatafileUrl") != null) + { + String remoteJobRoot; + if (r.getValue("runid/JobId/FilePath") == null) + { + if (r.getValue("analysis_id") != null) + { + Integer localAnalysisId = analysisMap.get((int) r.getValue("analysis_id")); + if (analysisToJobPath.containsKey(localAnalysisId)) + { + remoteJobRoot = analysisToJobPath.get(localAnalysisId); + } + else + { + getJob().getLogger().error("Missing path: " + r.getValue("dataid/DatafileUrl")); + return; + } + } + else if (r.getValue("readset/runid/JobId/FilePath") != null) + { + remoteJobRoot = getParent(URI.create(String.valueOf(r.getValue("readset/runid/JobId/FilePath")).replaceAll(" ", "%20")).getPath()); + } + else if (r.getValue("dataid/Run/JobId/FilePath") != null) + { + remoteJobRoot = getParent(URI.create(String.valueOf(r.getValue("dataid/Run/JobId/FilePath")).replaceAll(" ", "%20")).getPath()); + } + else + { + getJob().getLogger().error("Missing path: " + r.getValue("dataid/DatafileUrl")); + return; + } + } + else + { + remoteJobRoot = getParent(URI.create(String.valueOf(r.getValue("runid/JobId/FilePath")).replaceAll(" ", "%20")).getPath()); + } + + if (remoteJobRoot != null) + { + URI localFileRoot = PipelineService.get().getPipelineRootSetting(targetContainer).getRootPath().toURI(); + URI localPath = translateURI(String.valueOf(r.getValue("dataid/DatafileUrl")), remoteJobRoot, localFileRoot.getPath()); + row.put("dataid", getOrCreateExpData(localPath, targetContainer, String.valueOf(r.getValue("dataid/Name")))); + } + else + { + getJob().getLogger().error("Unable to find job root: " + r.getValue("dataid/DatafileUrl")); + return; + } + } + }); + + if (workbookColName != null) + { + Object workbookId = r.getValue(workbookColName); + if (workbookId != null) + { + row.put("container", workbookMap.get(Integer.parseInt(String.valueOf(workbookId))).getId()); + } + } + + toInsert.add(row); + }); + + if (truncateExisting) + { + List toDelete = new TableSelector(ti, new HashSet<>(ti.getPkColumnNames())).getArrayList(Object.class); + if (!toDelete.isEmpty()) + { + final List> rowsToDelete = new ArrayList<>(); + toDelete.forEach(x -> { + Map map = new CaseInsensitiveHashMap<>(); + map.put(ti.getPkColumnNames().get(0), x); + rowsToDelete.add(map); + }); + + ti.getUpdateService().deleteRows(getJob().getUser(), targetContainer, rowsToDelete, null, null); + } + } + + BatchValidationException bve = new BatchValidationException(); + ti.getUpdateService().insertRows(getJob().getUser(), targetContainer, toInsert, bve, null, null); + if (bve.hasErrors()) + { + throw bve; + } + } + catch (Exception e) + { + throw new PipelineJobException(e); + } + } + + //All of these map remote Id to local Id + private final Map workbookMap = new HashMap<>(); + private final Map readsetMap = new HashMap<>(); + private final Map readdataMap = new HashMap<>(); + private final Map analysisMap = new HashMap<>(); + private final Map analysisToFileMap = new HashMap<>(); //local analysis_id -> alignment file + private final Map analysisToJobPath = new HashMap<>(); + private final Map libraryMap = new HashMap<>(); + private final Map outputFileMap = new HashMap<>(); + private final Map sequenceMap = new HashMap<>(); + private final Map runIdMap = new HashMap<>(); + private final Map jobIdMap = new HashMap<>(); + + private void createLibraryMembers(Set preExisting) + { + getJob().getLogger().info("Creating library members"); + AtomicInteger totalCreated = new AtomicInteger(0); + AtomicInteger totalExisting = new AtomicInteger(0); + + final UserSchema us = QueryService.get().getUserSchema(getJob().getUser(), getPipelineJob().targetContainer, "sequenceanalysis"); + final TableInfo ti = us.getTable("reference_library_members"); + final TableInfo refNtTable = us.getTable("ref_nt_sequences"); + + try + { + SelectRowsCommand sr = new SelectRowsCommand("sequenceanalysis", "reference_library_members"); + sr.setColumns(Arrays.asList("rowid", "library_id", "ref_nt_id", "ref_nt_id/name", "ref_nt_id/seqLength", "workbook/workbookId")); + + SelectRowsResponse srr = sr.execute(getConnection(), getPipelineJob().remoteServerFolder); + + List> sequencesToCreate = new ArrayList<>(); + srr.getRowset().forEach(rd -> { + int seqLength = Integer.parseInt(String.valueOf(rd.getValue("ref_nt_id/seqLength"))); + + int remoteSeqId = Integer.parseInt(String.valueOf(rd.getValue("ref_nt_id"))); + String name = String.valueOf(rd.getValue("ref_nt_id/name")); + + //Skip all pigtail MHC. + if (name.startsWith("Mane")) + { + return; + } + + int localSeqId = getOrCreateSequence(remoteSeqId, name, seqLength, refNtTable); + if (localSeqId == -1) + { + return; + } + + int remoteLibraryId = Integer.parseInt(String.valueOf(rd.getValue("library_id"))); + Integer localLibraryId = libraryMap.get(remoteLibraryId); + if (localLibraryId == null) + { + throw new IllegalStateException("Unable to find library id: " + remoteLibraryId); + } + + SimpleFilter filter = new SimpleFilter(FieldKey.fromString("library_id"), localLibraryId); + filter.addCondition(FieldKey.fromString("ref_nt_id"), localSeqId); + + if (new TableSelector(ti, PageFlowUtil.set("rowid"), filter, null).exists()) + { + //Already exists: + totalExisting.getAndIncrement(); + return; + } + + Map toCreate = new CaseInsensitiveHashMap<>(); + toCreate.put("library_id", localLibraryId); + toCreate.put("ref_nt_id", localSeqId); + sequencesToCreate.add(toCreate); + }); + + getJob().getLogger().info("Total sequences to create: " + sequencesToCreate.size()); + if (!sequencesToCreate.isEmpty()) + { + BatchValidationException bve = new BatchValidationException(); + List> created = ti.getUpdateService().insertRows(getJob().getUser(), getPipelineJob().targetContainer, sequencesToCreate, bve, null, null); + totalCreated.getAndAdd(sequencesToCreate.size()); + if (bve.hasErrors()) + { + throw new RuntimeException(bve); + } + } + } + catch (Exception e) + { + getJob().getLogger().error(e.getMessage(), e); + throw new RuntimeException(e); + } + + getJob().getLogger().info("total created: " + totalCreated.get() + ", total existing: " + totalExisting.get()); + } + + private int getOrCreateSequence(int remoteSeqId, String name, int seqLength, TableInfo refNtTable) + { + if (sequenceMap.containsKey(remoteSeqId)) + { + return sequenceMap.get(remoteSeqId); + } + else + { + SimpleFilter filter = new SimpleFilter(FieldKey.fromString("name"), name); + filter.addCondition(FieldKey.fromString("datedisabled"), null, CompareType.ISBLANK); + TableSelector ts = new TableSelector(refNtTable, PageFlowUtil.set("rowid", "seqLength"), filter, new Sort("rowid")); + if (ts.exists()) + { + if (ts.getRowCount() > 1) + { + getJob().getLogger().info("Duplicate ref name: " + name); + } + + AtomicInteger localId = new AtomicInteger(-1); + ts.forEachResults(rs -> { + if (rs.getInt(FieldKey.fromString("seqLength")) < seqLength) + { + //NOTE: accept these as most are trimmed + getJob().getLogger().warn("length doesnt match for " + name + ", expected: " + seqLength + ", was: " + rs.getInt(FieldKey.fromString("seqLength"))); + } + + localId.set(rs.getInt(FieldKey.fromString("rowid"))); + }); + + if (localId.get() != -1) + { + sequenceMap.put(remoteSeqId, localId.get()); + return localId.get(); + } + } + + getJob().getLogger().error("Sequence missing: " + name); + return -1; + } + } + + public String getParent(String path) + { + final char separatorChar = '/'; + + int index = path.lastIndexOf(separatorChar); + if (index == -1) + { + throw new IllegalArgumentException("Missing slash"); + } + + return path.substring(0, index); + } + + private Set createLibraries() + { + getJob().getLogger().info("Creating libraries"); + AtomicInteger totalCreated = new AtomicInteger(0); + AtomicInteger totalExisting = new AtomicInteger(0); + Set preExisting = new HashSet<>(); + + try + { + final TableInfo libraryTable = QueryService.get().getUserSchema(getJob().getUser(), getPipelineJob().targetContainer, "sequenceanalysis").getTable("reference_libraries"); + + SelectRowsCommand sr = new SelectRowsCommand("sequenceanalysis", "reference_libraries"); + sr.setColumns(Arrays.asList("rowid", "name", "description", "fasta_file", "datedisabled", "assemblyId", "fasta_file/DataFileUrl", "fasta_file/Name", "workbook/workbookId")); + + SelectRowsResponse srr = sr.execute(getConnection(), getPipelineJob().remoteServerFolder); + + srr.getRowset().forEach(rd -> { + int remoteId = Integer.parseInt(String.valueOf(rd.getValue("rowid"))); + + Integer remoteWorkbook = rd.getValue("workbook/workbookId") == null ? null : Integer.parseInt(String.valueOf(rd.getValue("workbook/workbookId"))); + Container targetContainer = remoteWorkbook == null ? getPipelineJob().targetContainer : workbookMap.get(remoteWorkbook); + + SimpleFilter filter = new SimpleFilter(FieldKey.fromString("name"), rd.getValue("name")); + TableSelector ts = new TableSelector(libraryTable, PageFlowUtil.set("rowid"), filter, null); + if (ts.exists()) + { + getJob().getLogger().info("Library exists: " + rd.getValue("name")); + preExisting.add(String.valueOf(rd.getValue("name"))); + libraryMap.put(remoteId, ts.getObject(Integer.class)); + totalExisting.getAndIncrement(); + } + else + { + Map toCreate = new CaseInsensitiveHashMap<>(); + toCreate.put("name", rd.getValue("name")); + toCreate.put("description", rd.getValue("description")); + toCreate.put("datedisabled", rd.getValue("datedisabled")); + toCreate.put("assemblyId", rd.getValue("assemblyId")); + try + { + String remoteJobRoot = getParent(URI.create(String.valueOf(rd.getValue("fasta_file/DatafileUrl"))).getPath()); + URI localJobRoot = PipelineService.get().getPipelineRootSetting(targetContainer).getRootPath().toURI(); + URI localFasta = translateURI(String.valueOf(rd.getValue("fasta_file/DatafileUrl")), remoteJobRoot, localJobRoot.getPath()); + toCreate.put("fasta_file", getOrCreateExpData(localFasta, targetContainer, String.valueOf(rd.getValue("fasta_file/Name")))); + + //Ensure parent folder exists: + File localJobRootFile = new File(localFasta).getParentFile(); + if (!localJobRootFile.getParentFile().exists()) + { + localJobRootFile.getParentFile().mkdirs(); + } + + getJob().getLogger().info(remoteJobRoot); + getJob().getLogger().info(localJobRoot.getPath()); + File remoteJobRootFile = new File(remoteJobRoot); + if (remoteJobRootFile.exists()) + { + if (!localJobRootFile.exists()) + { + throw new PipelineJobException("Expected folder to have been copied: " + remoteJobRootFile.getPath() + " to " + localJobRootFile.getPath()); + } + } + + BatchValidationException bve = new BatchValidationException(); + List> created = libraryTable.getUpdateService().insertRows(getJob().getUser(), getPipelineJob().targetContainer, Arrays.asList(toCreate), bve, null, null); + if (bve.hasErrors()) + { + throw new RuntimeException(bve); + } + + libraryMap.put(remoteId, Integer.parseInt(String.valueOf(created.get(0).get("rowid")))); + totalCreated.getAndIncrement(); + } + catch (Exception e) + { + throw new RuntimeException(e); + } + } + }); + } + catch (Exception e) + { + getJob().getLogger().error(e.getMessage(), e); + throw new RuntimeException(e); + } + + getJob().getLogger().info("total created: " + totalCreated.get() + ", total existing: " + totalExisting.get()); + + return preExisting; + } + + private void createOutputFiles() + { + getJob().getLogger().info("Creating outputfiles"); + AtomicInteger totalCreated = new AtomicInteger(0); + AtomicInteger totalExisting = new AtomicInteger(0); + + try + { + final TableInfo outputTable = QueryService.get().getUserSchema(getJob().getUser(), getPipelineJob().targetContainer, "sequenceanalysis").getTable("outputfiles"); + + SelectRowsCommand sr = new SelectRowsCommand("sequenceanalysis", "outputfiles"); + sr.setColumns(Arrays.asList("rowid", "name", "description", "dataid", "library_id", "readset", "analysis_id", "category", "sra_accession", "dataid/DataFileUrl", "dataid/Name", "runid", "runid/JobId", "runid/Name", "workbook/workbookId", "runid/Name", "runid/JobId/FilePath")); + + SelectRowsResponse srr = sr.execute(getConnection(), getPipelineJob().remoteServerFolder); + + srr.getRowset().forEach(rd -> { + int remoteId = Integer.parseInt(String.valueOf(rd.getValue("rowid"))); + int remoteReadset = Integer.parseInt(String.valueOf(rd.getValue("readset"))); + Integer localReadset = readsetMap.get(remoteReadset); + if (localReadset == null) + { + throw new IllegalArgumentException("Unable to find readset for remote id: " + remoteReadset); + } + + int remoteLibrary = Integer.parseInt(String.valueOf(rd.getValue("library_id"))); + Integer localLibrary = libraryMap.get(remoteLibrary); + if (localLibrary == null) + { + throw new IllegalArgumentException("Unable to find genome for remote id: " + remoteLibrary); + } + + Integer localAnalysis; + if (rd.getValue("analysis_id") != null) + { + int remoteAnalysis = Integer.parseInt(String.valueOf(rd.getValue("analysis_id"))); + localAnalysis = analysisMap.get(remoteAnalysis); + if (localAnalysis == null) + { + throw new IllegalArgumentException("Unable to find analysis for remote id: " + remoteAnalysis); + } + } + else + { + localAnalysis = null; + } + + Readset rs = SequenceAnalysisService.get().getReadset(localReadset, getJob().getUser()); + Container targetWorkbook = ContainerManager.getForId(rs.getContainer()); + + SimpleFilter filter = new SimpleFilter(FieldKey.fromString("readset"), rs.getRowId()); + filter.addCondition(FieldKey.fromString("name"), rd.getValue("name")); + filter.addCondition(FieldKey.fromString("category"), rd.getValue("category")); + filter.addCondition(FieldKey.fromString("analysis_id"), localAnalysis); + filter.addCondition(FieldKey.fromString("container"), targetWorkbook.getId(), CompareType.EQUAL); + + TableSelector tsOutputFiles = new TableSelector(outputTable, PageFlowUtil.set("rowid"), filter, null); + if (tsOutputFiles.exists()) + { + outputFileMap.put(remoteId, tsOutputFiles.getObject(Integer.class)); + totalExisting.getAndIncrement(); + } + else + { + Map toCreate = new CaseInsensitiveHashMap<>(); + toCreate.put("readset", rs.getRowId()); + toCreate.put("analysis_id", localAnalysis); + toCreate.put("description", rd.getValue("description")); + toCreate.put("sra_accession", rd.getValue("sra_accession")); + toCreate.put("library_id", localLibrary); + toCreate.put("name", rd.getValue("name")); + toCreate.put("category", rd.getValue("category")); + + try + { + if (rd.getValue("runid/JobId") == null) + { + throw new PipelineJobException("Output missing runId"); + } + + int remoteJobId = Integer.parseInt(String.valueOf(rd.getValue("runid/JobId"))); + int jobId = getOrCreateJob(remoteJobId, targetWorkbook); + PipelineStatusFile sf = PipelineService.get().getStatusFile(jobId); + + String localJobRoot = getParent(sf.getFilePath()); + String remoteJobRoot = getParent(URI.create(String.valueOf(rd.getValue("runid/JobId/FilePath")).replaceAll(" ", "%20")).getPath()); + + URI newFileAlignment = translateURI(String.valueOf(rd.getValue("dataid/DatafileUrl")), remoteJobRoot, localJobRoot); + toCreate.put("dataid", getOrCreateExpData(newFileAlignment, targetWorkbook, String.valueOf(rd.getValue("dataid/Name")))); + + //Create run: + if (rd.getValue("runid") != null && rd.getValue("runid/JobId") != null) + { + int runId = createExpRun(Integer.parseInt(String.valueOf(rd.getValue("runid"))), targetWorkbook, String.valueOf(rd.getValue("runid/Name")), jobId); + toCreate.put("runid", runId); + } + else + { + getJob().getLogger().error("output missing runid: " + remoteId); + } + + BatchValidationException bve = new BatchValidationException(); + List> created = outputTable.getUpdateService().insertRows(getJob().getUser(), getPipelineJob().targetContainer, Arrays.asList(toCreate), bve, null, null); + if (bve.hasErrors()) + { + throw new RuntimeException(bve); + } + + outputFileMap.put(remoteId, Integer.parseInt(String.valueOf(created.get(0).get("rowid")))); + totalCreated.getAndIncrement(); + } + catch (Exception e) + { + throw new RuntimeException(e); + } + } + }); + } + catch (Exception e) + { + getJob().getLogger().error(e.getMessage(), e); + throw new RuntimeException(e); + } + + getJob().getLogger().info("total created: " + totalCreated.get() + ", total existing: " + totalExisting.get()); + } + + private void createAnalyses() + { + getJob().getLogger().info("Creating analyses"); + AtomicInteger totalCreated = new AtomicInteger(0); + AtomicInteger totalExisting = new AtomicInteger(0); + + try + { + final TableInfo analysisTable = QueryService.get().getUserSchema(getJob().getUser(), getPipelineJob().targetContainer, "sequenceanalysis").getTable("sequence_analyses"); + + SelectRowsCommand sr = new SelectRowsCommand("sequenceanalysis", "sequence_analyses"); + sr.setColumns(Arrays.asList("rowid", "type", "description", "synopsis", "runid", "readset", "alignmentfile", "reference_library", "library_id", "sra_accession", "alignmentfile/DataFileUrl", "alignmentfile/Name", "alignmentfile/Name", "reference_library", "reference_library/DataFileUrl", "reference_library/Name", "runid/jobid", "runid/Name", "workbook/workbookId", "runid/JobId", "runid/Name", "runid/JobId/FilePath", "runid/JobId/Description")); + + SelectRowsResponse srr = sr.execute(getConnection(), getPipelineJob().remoteServerFolder); + + srr.getRowset().forEach(rd -> { + int remoteId = Integer.parseInt(String.valueOf(rd.getValue("rowid"))); + if (rd.getValue("readset") == null) + { + getJob().getLogger().warn("analysis lacks readset, skipping: " + remoteId); + return; + } + + int remoteReadset = Integer.parseInt(String.valueOf(rd.getValue("readset"))); + Integer localReadset = readsetMap.get(remoteReadset); + if (localReadset == null) + { + throw new IllegalArgumentException("Unable to find readset for remote id: " + remoteReadset); + } + + Integer localLibrary = null; + if (rd.getValue("library_id") != null) + { + int remoteLibrary = Integer.parseInt(String.valueOf(rd.getValue("library_id"))); + localLibrary = libraryMap.get(remoteLibrary); + if (localLibrary == null) + { + throw new IllegalArgumentException("Unable to find genome for remote id: " + remoteLibrary); + } + } + + Readset rs = SequenceAnalysisService.get().getReadset(localReadset, getJob().getUser()); + Container targetWorkbook = ContainerManager.getForId(rs.getContainer()); + + SimpleFilter filter = new SimpleFilter(FieldKey.fromString("readset"), rs.getRowId()); + filter.addCondition(FieldKey.fromString("runid/JobId/Description"), rd.getValue("runid/JobId/Description")); + filter.addCondition(FieldKey.fromString("container"), targetWorkbook.getId(), CompareType.EQUAL); + + TableSelector tsAnalyses = new TableSelector(analysisTable, PageFlowUtil.set("rowid", "alignmentfile"), filter, null); + if (tsAnalyses.exists()) + { + try + { + tsAnalyses.forEachResults(results -> { + try + { + analysisMap.put(remoteId, results.getInt("rowid")); + if (results.getObject("alignmentfile") != null) + { + analysisToFileMap.put(results.getInt("rowid"), results.getInt("alignmentfile")); + } + } + catch (IndexOutOfBoundsException e) + { + throw new RuntimeException(e); + } + }); + + totalExisting.getAndIncrement(); + } + catch (Exception e) + { + throw new RuntimeException(e); + } + } + else + { + Map toCreate = new CaseInsensitiveHashMap<>(); + + toCreate.put("readset", rs.getRowId()); + toCreate.put("synopsis", rd.getValue("synopsis")); + toCreate.put("centerName", rd.getValue("centerName")); + toCreate.put("type", rd.getValue("type")); + toCreate.put("description", rd.getValue("description")); + toCreate.put("sra_accession", rd.getValue("sra_accession")); + Container workbook = workbookMap.get(rd.getValue("workbook/workbookId")); + toCreate.put("container", workbook.getId()); + if (localLibrary != null) + { + toCreate.put("library_id", localLibrary); + } + + try + { + if (rd.getValue("runid/JobId") == null) + { + getJob().getLogger().info("skipping analysis without runid: " + remoteId); + return; + } + + int remoteJobId = Integer.parseInt(String.valueOf(rd.getValue("runid/JobId"))); + int jobId = getOrCreateJob(remoteJobId, targetWorkbook); + PipelineStatusFile sf = PipelineService.get().getStatusFile(jobId); + + String localJobRoot = getParent(sf.getFilePath()); + String remoteJobRoot = getParent(URI.create(String.valueOf(rd.getValue("runid/JobId/FilePath")).replaceAll(" ", "%20")).getPath()); + + URI newFileAlignment = translateURI(String.valueOf(rd.getValue("alignmentfile/DatafileUrl")), remoteJobRoot, localJobRoot); + toCreate.put("alignmentfile", getOrCreateExpData(newFileAlignment, targetWorkbook, String.valueOf(rd.getValue("alignmentfile/Name")))); + + if (rd.getValue("reference_library") != null) + { + URI newFile2 = translateURI(String.valueOf(rd.getValue("reference_library/DatafileUrl")), remoteJobRoot, localJobRoot); + toCreate.put("reference_library", getOrCreateExpData(newFile2, targetWorkbook, String.valueOf(rd.getValue("reference_library/Name")))); + } + + //Create run: + if (rd.getValue("runid") != null && rd.getValue("runid/JobId") != null) + { + int runId = createExpRun(Integer.parseInt(String.valueOf(rd.getValue("runid"))), targetWorkbook, String.valueOf(rd.getValue("runid/Name")), jobId); + toCreate.put("runid", runId); + } + else + { + getJob().getLogger().error("analysis missing runid: " + remoteId); + } + + BatchValidationException bve = new BatchValidationException(); + List> created = analysisTable.getUpdateService().insertRows(getJob().getUser(), getPipelineJob().targetContainer, Arrays.asList(toCreate), bve, null, null); + if (bve.hasErrors()) + { + throw new RuntimeException(bve); + } + + analysisMap.put(remoteId, Integer.parseInt(String.valueOf(created.get(0).get("rowid")))); + analysisToJobPath.put(Integer.parseInt(String.valueOf(created.get(0).get("rowid"))), remoteJobRoot); + if (toCreate.get("alignmentfile") != null) + { + analysisToFileMap.put(remoteId, (int)toCreate.get("alignmentfile")); + } + totalCreated.getAndIncrement(); + } + catch (Exception e) + { + throw new RuntimeException(e); + } + } + }); + } + catch (Exception e) + { + getJob().getLogger().error(e.getMessage(), e); + throw new RuntimeException(e); + } + + getJob().getLogger().info("total created: " + totalCreated.get() + ", total existing: " + totalExisting.get()); + } + + private void createReaddata() + { + getJob().getLogger().info("Creating read data"); + AtomicInteger totalCreated = new AtomicInteger(0); + AtomicInteger totalExisting = new AtomicInteger(0); + + try + { + for (Integer workbookId : workbookMap.keySet()) + { + final TableInfo readdataTable = QueryService.get().getUserSchema(getJob().getUser(), workbookMap.get(workbookId), "sequenceanalysis").getTable("readdata"); + + SelectRowsCommand sr = new SelectRowsCommand("sequenceanalysis", "readdata"); + sr.setColumns(Arrays.asList("rowid", "readset", "platformUnit", "centerName", "date", "fileid1", "fileid1/DataFileUrl", "fileid1/Name", "fileid2", "fileid2/DataFileUrl", "fileid2/Name", "description", "sra_accession", "runid", "runid/jobid", "runid/Name", "readset/workbook/workbookId", "runid/JobId", "runid/Name", "runid/JobId/FilePath", "runid/JobId/Description")); + sr.addFilter(new Filter("fileid1/DataFileUrl", null, Filter.Operator.NONBLANK)); + + SelectRowsResponse srr = sr.execute(getConnection(), getPipelineJob().remoteServerFolder + workbookId + "/"); + + long existing = new TableSelector(readdataTable).getRowCount(); + if (srr.getRowCount().longValue() == existing) + { + getJob().getLogger().info("Readdata count identical, skipping: " + workbookId); + totalExisting.getAndAdd(srr.getRowCount().intValue()); + continue; + } + else if (srr.getRowCount().intValue() == 0) + { + getJob().getLogger().info("No readdata records, skipping: " + workbookId); + continue; + } + + srr.getRowset().forEach(rd -> { + int remoteId = Integer.parseInt(String.valueOf(rd.getValue("rowid"))); + int remoteReadset = Integer.parseInt(String.valueOf(rd.getValue("readset"))); + Integer localReadset = readsetMap.get(remoteReadset); + if (localReadset == null) + { + throw new IllegalArgumentException("Unable to find readset for remote id: " + remoteReadset); + } + + Readset rs = SequenceAnalysisService.get().getReadset(localReadset, getJob().getUser()); + Container targetWorkbook = ContainerManager.getForId(rs.getContainer()); + + SimpleFilter rdFilter = new SimpleFilter(FieldKey.fromString("readset"), rs.getRowId()); + rdFilter.addCondition(FieldKey.fromString("runid/JobId/Description"), rd.getValue("runid/JobId/Description")); + rdFilter.addCondition(FieldKey.fromString("fileid1/Name"), rd.getValue("fileid1/Name")); + rdFilter.addCondition(FieldKey.fromString("container"), targetWorkbook.getId(), CompareType.EQUAL); + + if (rd.getValue("platformUnit") != null) + { + rdFilter.addCondition(FieldKey.fromString("platformUnit"), rd.getValue("platformUnit")); + } + + TableSelector tsReaddata = new TableSelector(readdataTable, PageFlowUtil.set("rowid"), rdFilter, null); + if (tsReaddata.exists()) + { + readdataMap.put(remoteId, tsReaddata.getObject(Integer.class)); + totalExisting.getAndIncrement(); + } + else + { + if (rd.getValue("fileid1/DataFileUrl") == null) + { + getJob().getLogger().warn("readddata missing files, skipping: " + remoteId); + return; + } + + Map toCreate = new CaseInsensitiveHashMap<>(); + toCreate.put("readset", rs.getRowId()); + Container workbook = workbookMap.get(rd.getValue("readset/workbook/workbookId")); + toCreate.put("container", workbook.getId()); + toCreate.put("platformUnit", rd.getValue("platformUnit")); + toCreate.put("centerName", rd.getValue("centerName")); + toCreate.put("date", rd.getValue("date")); + toCreate.put("description", rd.getValue("description")); + toCreate.put("sra_accession", rd.getValue("sra_accession")); + try + { + if (rd.getValue("runid/JobId") != null) + { + int remoteJobId = Integer.parseInt(String.valueOf(rd.getValue("runid/JobId"))); + int jobId = getOrCreateJob(remoteJobId, targetWorkbook); + PipelineStatusFile sf = PipelineService.get().getStatusFile(jobId); + + String localJobRoot = getParent(sf.getFilePath()); + String remoteJobRoot = getParent(URI.create(String.valueOf(rd.getValue("runid/JobId/FilePath")).replaceAll(" ", "%20")).getPath()); + + if (rd.getValue("fileid1/DataFileUrl") != null) + { + URI newFile1 = translateURI(String.valueOf(rd.getValue("fileid1/DataFileUrl")), remoteJobRoot, localJobRoot); + toCreate.put("fileid1", getOrCreateExpData(newFile1, targetWorkbook, String.valueOf(rd.getValue("fileid1/Name")))); + } + + if (rd.getValue("fileid2/DataFileUrl") != null) + { + URI newFile2 = translateURI(String.valueOf(rd.getValue("fileid2/DatafileUrl")), remoteJobRoot, localJobRoot); + toCreate.put("fileid2", getOrCreateExpData(newFile2, targetWorkbook, String.valueOf(rd.getValue("fileid2/Name")))); + } + } + else + { + if (rd.getValue("fileid1/DataFileUrl") != null) + { + getJob().getLogger().error("readddata missing jobid: " + remoteId); + } + } + + //Create run: + if (rd.getValue("runid") != null && rd.getValue("runid/JobId") != null) + { + int remoteJobId = Integer.parseInt(String.valueOf(rd.getValue("runid/JobId"))); + int jobId = getOrCreateJob(remoteJobId, targetWorkbook); + int runId = createExpRun(Integer.parseInt(String.valueOf(rd.getValue("runid"))), targetWorkbook, String.valueOf(rd.getValue("runid/Name")), jobId); + toCreate.put("runid", runId); + } + else + { + if (rd.getValue("fileid1/DataFileUrl") != null) + { + getJob().getLogger().error("readddata missing runid: " + remoteId); + } + } + + BatchValidationException bve = new BatchValidationException(); + List> created = readdataTable.getUpdateService().insertRows(getJob().getUser(), getPipelineJob().targetContainer, Arrays.asList(toCreate), bve, null, null); + if (bve.hasErrors()) + { + throw new RuntimeException(bve); + } + + readdataMap.put(remoteId, Integer.parseInt(String.valueOf(created.get(0).get("rowid")))); + totalCreated.getAndIncrement(); + } + catch (Exception e) + { + throw new RuntimeException(e); + } + } + }); + } + } + catch(Exception e) + { + getJob().getLogger().error(e.getMessage(), e); + throw new RuntimeException(e); + } + + getJob().getLogger().info("total created: " + totalCreated.get() + ", total existing: " + totalExisting.get()); + } + + private int getOrCreateExpData(URI file, Container workbook, String fileName) + { + ExpData ret = ExperimentService.get().getExpDataByURL(new File(file), workbook); + if (ret == null) + { + String lsid = ExperimentService.get().generateLSID(workbook, new DataType("Data"), file.getPath()); + List datas = ExperimentService.get().getExpDatasByLSID(Collections.singleton(lsid)); + if (!datas.isEmpty()) + { + ret = datas.get(0); + if (!workbook.equals(ret.getContainer())) + { + throw new IllegalArgumentException("Expected datas to be from the same container: " + lsid); + } + } + + if (ret == null) + { + ret = ExperimentService.get().createData(workbook, new DataType("Data"), fileName); + ret.setDataFileURI(file); + ret.setLSID(lsid); + ret.save(getJob().getUser()); + } + } + + return ret.getRowId(); + } + + private void createReadsets() + { + getJob().getLogger().info("Creating readsets"); + AtomicInteger totalCreated = new AtomicInteger(0); + AtomicInteger totalExisting = new AtomicInteger(0); + + try + { + final UserSchema us = QueryService.get().getUserSchema(getJob().getUser(), getPipelineJob().targetContainer, "sequenceanalysis"); + final TableInfo readsetTable = us.getTable("sequence_readsets"); + + SelectRowsCommand sr = new SelectRowsCommand("sequenceanalysis", "sequence_readsets"); + sr.setColumns(Arrays.asList("rowid", "name", "platform", "application", "librarytype", "chemistry", "comments", "status", "subjectid", "subjectdate", "sampletype", "sampleid", "barcode5", "barcode3", "runid", "runid/jobid", "runid/Name", "workbook/workbookId", "runid/JobId", "runid/Name", "runid/JobId/FilePath")); + + SelectRowsResponse srr = sr.execute(getConnection(), getPipelineJob().remoteServerFolder); + + srr.getRowset().forEach(rs -> { + int remoteId = Integer.parseInt(String.valueOf(rs.getValue("rowid"))); + int sourceWorkbook = Integer.parseInt(String.valueOf(rs.getValue("workbook/workbookId"))); + Container targetWorkbook = workbookMap.get(sourceWorkbook); + if (targetWorkbook == null) + { + throw new IllegalArgumentException("Unable to find local workbook for source: " + sourceWorkbook); + } + + SimpleFilter rsFilter = new SimpleFilter(FieldKey.fromString("name"), rs.getValue("name")); + rsFilter.addCondition(FieldKey.fromString("container"), targetWorkbook.getId(), CompareType.EQUAL); + if (rs.getValue("subjectid") != null) + { + rsFilter.addCondition(FieldKey.fromString("subjectid"), rs.getValue("subjectid"), CompareType.EQUAL); + } + + TableSelector tsReadset = new TableSelector(readsetTable, PageFlowUtil.set("rowid"), rsFilter, null); + if (tsReadset.exists()) + { + readsetMap.put(remoteId, tsReadset.getObject(Integer.class)); + totalExisting.getAndIncrement(); + } + else + { + Map toCreate = new CaseInsensitiveHashMap<>(); + toCreate.put("name", rs.getValue("name")); + toCreate.put("platform", rs.getValue("platform")); + toCreate.put("application", rs.getValue("application")); + toCreate.put("barcode5", rs.getValue("barcode5")); + toCreate.put("barcode3", rs.getValue("barcode3")); + toCreate.put("subjectid", rs.getValue("subjectid")); + + toCreate.put("sampleid", rs.getValue("sampleid")); + toCreate.put("sampledate", rs.getValue("sampledate")); + toCreate.put("librarytype", rs.getValue("librarytype")); + toCreate.put("sampletype", rs.getValue("sampletype")); + toCreate.put("chemistry", rs.getValue("chemistry")); + toCreate.put("comments", rs.getValue("comments")); + toCreate.put("status", rs.getValue("status")); + + toCreate.put("container", targetWorkbook.getId()); + + try + { + //Create run: + if (rs.getValue("runid") != null && rs.getValue("runid/JobId") != null) + { + int remoteJobId = Integer.parseInt(String.valueOf(rs.getValue("runid/JobId"))); + int jobId = getOrCreateJob(remoteJobId, targetWorkbook); + int runid = createExpRun(Integer.parseInt(String.valueOf(rs.getValue("runid"))), targetWorkbook, String.valueOf(rs.getValue("runid/Name")), jobId); + toCreate.put("runid", runid); + } + else + { + getJob().getLogger().error("readset missing run id: " + remoteId); + } + + BatchValidationException bve = new BatchValidationException(); + List> created = readsetTable.getUpdateService().insertRows(getJob().getUser(), getPipelineJob().targetContainer, Arrays.asList(toCreate), bve, null, null); + if (bve.hasErrors()) + { + throw new RuntimeException(bve); + } + + readsetMap.put(remoteId, Integer.parseInt(String.valueOf(created.get(0).get("rowid")))); + totalCreated.getAndIncrement(); + } + catch (Exception e) + { + throw new RuntimeException(e); + } + } + }); + + getJob().getLogger().info("total created: " + totalCreated.get() + ", total existing: " + totalExisting.get()); + } + catch (Exception e) + { + getJob().getLogger().error(e.getMessage(), e); + throw new RuntimeException(e); + } + } + + private int getOrCreateJob(int remoteJobId, Container targetWorkbook) + { + if (jobIdMap.containsKey(remoteJobId)) + { + return jobIdMap.get(remoteJobId); + } + + TableInfo ti = DbSchema.get("pipeline", DbSchemaType.Module).getTable("StatusFiles"); + + try + { + SelectRowsCommand sr = new SelectRowsCommand("pipeline", "job"); + sr.addFilter(new Filter("rowid", remoteJobId, Filter.Operator.EQUAL)); + sr.setColumns(Arrays.asList("RowId", "Info", "FilePath", "Email", "Description", "DataUrl", "Job", "Provider", "HadError", "ActiveTaskId")); + + SelectRowsResponse srr = sr.execute(getConnection(), getPipelineJob().remoteServerFolder); + + File fr = PipelineService.get().getPipelineRootSetting(targetWorkbook).getRootPath(); + + AtomicInteger ret = new AtomicInteger(); + srr.getRowset().forEach(pj -> { + String filepath = String.valueOf(pj.getValue("FilePath")); + if (!filepath.contains("@files")) + { + //This appears to be an error in PRIMe's data: + if (filepath.contains("illuminaImport")) + { + filepath = filepath.replace("illuminaImport", "@files/illuminaImport"); + } + else if (filepath.contains("sequenceAnalysis")) + { + filepath = filepath.replace("sequenceAnalysis", "@files/sequenceAnalysis"); + } + else + { + getJob().getLogger().error("Unexpected filepath: " + pj.getValue("FilePath")); + } + } + + File remoteDir = new File(URI.create(filepath.replaceAll(" ", "%20")).getPath()); + File localDir = new File(fr, filepath.split("@files")[1]); + + //Check for existing row: + TableSelector ts = new TableSelector(ti, PageFlowUtil.set("RowId"), new SimpleFilter(FieldKey.fromString("Job"), pj.getValue("Job")), null); + if (ts.exists()) + { + ret.set(ts.getObject(Integer.class)); + } + else + { + ts = new TableSelector(ti, PageFlowUtil.set("RowId"), new SimpleFilter(FieldKey.fromString("FilePath"), localDir.getPath()), null); + if (ts.exists()) + { + ret.set(ts.getObject(Integer.class)); + } + else + { + Map toCreate = new CaseInsensitiveHashMap<>(); + toCreate.put("Info", pj.getValue("Info")); + toCreate.put("FilePath", localDir.getPath()); + toCreate.put("Email", pj.getValue("Email")); + toCreate.put("Description", pj.getValue("Description")); + toCreate.put("DataUrl", pj.getValue("DataUrl")); + toCreate.put("Job", pj.getValue("Job")); + toCreate.put("Provider", pj.getValue("Provider")); + toCreate.put("HadError", pj.getValue("HadError")); + toCreate.put("ActiveTaskId", pj.getValue("ActiveTaskId")); + toCreate.put("Container", targetWorkbook.getId()); + + toCreate = Table.insert(getJob().getUser(), ti, toCreate); + + ret.set((int) toCreate.get("RowId")); + } + } + + if (localDir.exists()) + { + getJob().getLogger().info("Directory exists, will not re-copy: " + localDir.getPath()); + return; + } + + try + { + getJob().getLogger().info(remoteDir.getPath()); + getJob().getLogger().info(localDir.getPath()); + + if (!localDir.getParentFile().exists()) + { + localDir.getParentFile().mkdirs(); + } + + if (remoteDir.exists()) + { + if (!localDir.exists()) + { + throw new PipelineJobException("Expected folder to have been copied: " + remoteDir.getPath() + " to " + localDir.getPath()); + } + } + else + { + getJob().getLogger().error("source folder not found: " + remoteDir.getPath()); + } + } + catch (Exception e) + { + throw new RuntimeException(e); + } + }); + + jobIdMap.put(remoteJobId, ret.get()); + + return ret.get(); + } + catch (Exception e) + { + getJob().getLogger().error(e.getMessage(), e); + throw new RuntimeException(e); + } + } + + private int createExpRun(int remoteId, Container c, String name, int localJobId) throws Exception + { + if (runIdMap.containsKey(remoteId)) + { + return runIdMap.get(remoteId); + } + else + { + ExpRun ret = ExperimentService.get().createRunForProvenanceRecording(c, getJob().getUser(), new RecordedActionSet(), name, localJobId); + runIdMap.put(remoteId, ret.getRowId()); + + return ret.getRowId(); + } + } + + private void createWorkbooks() + { + getJob().getLogger().info("Creating workbooks"); + AtomicInteger totalCreated = new AtomicInteger(0); + AtomicInteger totalExisting = new AtomicInteger(0); + + try + { + TableInfo containers = QueryService.get().getUserSchema(getJob().getUser(), getPipelineJob().targetContainer, "core").getTable("containers"); + + SelectRowsCommand sr = new SelectRowsCommand("core", "workbooks"); + sr.setColumns(Arrays.asList("Name", "Title", "Description")); + SelectRowsResponse srr = sr.execute(getConnection(), getPipelineJob().remoteServerFolder); + + srr.getRowset().forEach(wb -> { + String localTitle = (String) wb.getValue("Title"); + + TableSelector ts = new TableSelector(containers, PageFlowUtil.set("RowId"), new SimpleFilter(FieldKey.fromString("Title"), localTitle), null); + if (ts.exists()) + { + Container workbook = ContainerManager.getForRowId(ts.getObject(Integer.class)); + workbookMap.put(Integer.parseInt(String.valueOf(wb.getValue("Name"))), workbook); + totalExisting.getAndIncrement(); + } + else + { + PipeRoot pr = PipelineService.get().getPipelineRootSetting(getJob().getContainer()); + + String description = wb.getValue("Description") != null ? String.valueOf(wb.getValue("Description")) + ". " : ""; + description = description + "Originally PRIMe workbook: " + wb.getValue("Name"); + + Container workbook = ContainerManager.createContainer(getPipelineJob().targetContainer, null, localTitle, description, WorkbookContainerType.NAME, getJob().getUser()); + workbook.setFolderType(FolderTypeManager.get().getFolderType("Expt Workbook"), getJob().getUser()); + workbookMap.put(Integer.parseInt(String.valueOf(wb.getValue("Name"))), workbook); + totalCreated.getAndIncrement(); + + File sourceDir = new File("/home/groups/miSeqLK/Production/MHC_Typing", wb.getValue("Name") + "/@files"); + File targetDir = pr.getRootPath(); + if (sourceDir.exists()) + { + try + { + FileUtils.copyDirectory(sourceDir, targetDir); + } + catch (Exception e) + { + throw new RuntimeException(e); + } + } + else + { + getJob().getLogger().error("source folder not found: " + sourceDir.getPath()); + } + } + }); + } + catch (CommandException | IOException e) + { + throw new RuntimeException(e); + } + + getJob().getLogger().info("total created: " + totalCreated.get() + ", total existing: " + totalExisting.get()); + } + + private URI translateURI(String databaseURI, String remoteFolderRoot, String localFolderRoot) + { + databaseURI = databaseURI.replace("\\", "/"); + remoteFolderRoot = remoteFolderRoot.replace("\\", "/").split("@files")[0]; + localFolderRoot = localFolderRoot.replace("\\", "/").split("@files")[0]; + if (localFolderRoot.startsWith("C:")) + { + localFolderRoot = localFolderRoot.replaceAll("^C:", ""); + } + + databaseURI = databaseURI.replace(remoteFolderRoot, localFolderRoot); + + return URI.create(databaseURI); + } + } +} diff --git a/primeseq/src/org/labkey/primeseq/pipeline/SequenceJobResourceAllocator.java b/primeseq/src/org/labkey/primeseq/pipeline/SequenceJobResourceAllocator.java index 1d6507ecb..3cea82534 100644 --- a/primeseq/src/org/labkey/primeseq/pipeline/SequenceJobResourceAllocator.java +++ b/primeseq/src/org/labkey/primeseq/pipeline/SequenceJobResourceAllocator.java @@ -203,22 +203,22 @@ public Integer getMaxRequestMemory(PipelineJob job) Map params = job.getParameters(); if (params != null) { - if (params.containsKey(PipelineStep.StepType.analysis.name()) && params.get(PipelineStep.StepType.analysis.name()).contains("HaplotypeCallerAnalysis")) + if (params.containsKey(PipelineStep.CorePipelineStepTypes.analysis.name()) && params.get(PipelineStep.CorePipelineStepTypes.analysis.name()).contains("HaplotypeCallerAnalysis")) { hasHaplotypeCaller = true; } - if (params.containsKey(PipelineStep.StepType.alignment.name()) && params.get(PipelineStep.StepType.alignment.name()).contains("STAR")) + if (params.containsKey(PipelineStep.CorePipelineStepTypes.alignment.name()) && params.get(PipelineStep.CorePipelineStepTypes.alignment.name()).contains("STAR")) { hasStar = true; } - if (params.containsKey(PipelineStep.StepType.alignment.name()) && params.get(PipelineStep.StepType.alignment.name()).contains("Bismark")) + if (params.containsKey(PipelineStep.CorePipelineStepTypes.alignment.name()) && params.get(PipelineStep.CorePipelineStepTypes.alignment.name()).contains("Bismark")) { hasBismark = true; } - if (params.containsKey(PipelineStep.StepType.alignment.name()) && params.get(PipelineStep.StepType.alignment.name()).contains("Bowtie2")) + if (params.containsKey(PipelineStep.CorePipelineStepTypes.alignment.name()) && params.get(PipelineStep.CorePipelineStepTypes.alignment.name()).contains("Bowtie2")) { hasBowtie2 = true; } diff --git a/primeseq/webapp/WEB-INF/primeseqContext.xml b/primeseq/webapp/WEB-INF/primeseqContext.xml index 411bb99a8..b91d3dfb3 100644 --- a/primeseq/webapp/WEB-INF/primeseqContext.xml +++ b/primeseq/webapp/WEB-INF/primeseqContext.xml @@ -3,6 +3,28 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-2.0.xsd"> + + + + + + + + + + + + + + + org.labkey.primeseq.pipeline.MhcMigrationPipelineJob.Task + + + + + + + diff --git a/tcrdb/build.gradle b/tcrdb/build.gradle index c1117ff51..03b7b0bac 100644 --- a/tcrdb/build.gradle +++ b/tcrdb/build.gradle @@ -1,6 +1,7 @@ import org.labkey.gradle.util.BuildUtils; dependencies { + BuildUtils.addLabKeyDependency(project: project, config: "implementation", depProjectPath: ":server:modules:DiscvrLabKeyModules:singlecell", depProjectConfig: "apiJarFile") BuildUtils.addLabKeyDependency(project: project, config: "implementation", depProjectPath: ":server:modules:DiscvrLabKeyModules:SequenceAnalysis", depProjectConfig: "apiJarFile") BuildUtils.addLabKeyDependency(project: project, config: "implementation", depProjectPath: ":server:modules:DiscvrLabKeyModules:SequenceAnalysis", depProjectConfig: "apiElements") BuildUtils.addLabKeyDependency(project: project, config: "implementation", depProjectPath: ":server:modules:LabDevKitModules:laboratory", depProjectConfig: "apiJarFile") @@ -14,5 +15,6 @@ dependencies { BuildUtils.addLabKeyDependency(project: project, config: "modules", depProjectPath: ":server:modules:LabDevKitModules:laboratory", depProjectConfig: "published", depExtension: "module") BuildUtils.addLabKeyDependency(project: project, config: "modules", depProjectPath: ":server:modules:LabDevKitModules:LDK", depProjectConfig: "published", depExtension: "module") BuildUtils.addLabKeyDependency(project: project, config: "modules", depProjectPath: ":server:modules:DiscvrLabKeyModules:SequenceAnalysis", depProjectConfig: "published", depExtension: "module") + BuildUtils.addLabKeyDependency(project: project, config: "modules", depProjectPath: ":server:modules:DiscvrLabKeyModules:singlecell", depProjectConfig: "published", depExtension: "module") } diff --git a/tcrdb/module.properties b/tcrdb/module.properties index 7ee0ae800..b282909c1 100644 --- a/tcrdb/module.properties +++ b/tcrdb/module.properties @@ -3,5 +3,4 @@ Label: TCRdb Description: The TCRdb module is designed to manage TCR sequence from either clones or populations. License: Apache 2.0 LicenseURL: http://www.apache.org/licenses/LICENSE-2.0 -ConsolidateScripts: false ManageVersion: false diff --git a/tcrdb/resources/assay/TCRdb/queries/Data.query.xml b/tcrdb/resources/assay/TCRdb/queries/Data.query.xml index f46107a76..20a2fb77e 100644 --- a/tcrdb/resources/assay/TCRdb/queries/Data.query.xml +++ b/tcrdb/resources/assay/TCRdb/queries/Data.query.xml @@ -69,8 +69,8 @@ - tcrdb - cdnas + singlecell + cdna_libraries rowid rowid diff --git a/tcrdb/resources/assay/TCRdb/queries/Data/cDNA Info.qview.xml b/tcrdb/resources/assay/TCRdb/queries/Data/cDNA Info.qview.xml index d8518ac12..2b3104ee9 100644 --- a/tcrdb/resources/assay/TCRdb/queries/Data/cDNA Info.qview.xml +++ b/tcrdb/resources/assay/TCRdb/queries/Data/cDNA Info.qview.xml @@ -2,9 +2,9 @@ - - - + + + diff --git a/tcrdb/resources/external/install.R b/tcrdb/resources/external/install.R deleted file mode 100644 index 8000b3313..000000000 --- a/tcrdb/resources/external/install.R +++ /dev/null @@ -1 +0,0 @@ -install.packages(c("reshape2", "FField", "reshape", "gplots", "gridExtra", "circlize", "ggplot2", "grid", "VennDiagram", "ape", "MASS", "plotrix", "RColorBrewer", "scales"), dependencies=TRUE, repos='http://cran.rstudio.com') \ No newline at end of file diff --git a/tcrdb/resources/external/installCiteSeqCount.sh b/tcrdb/resources/external/installCiteSeqCount.sh deleted file mode 100644 index 286748329..000000000 --- a/tcrdb/resources/external/installCiteSeqCount.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -scl enable rh-python36 bash -virtualenv /home/groups/prime-seq/pipeline_tools/bin/primeseq-python -source /home/groups/prime-seq/pipeline_tools/bin/primeseq-python/bin/activate -pip install --upgrade pip -pip install CITE-seq-Count - - diff --git a/tcrdb/resources/external/scRNAseq/Seurat3.rmd b/tcrdb/resources/external/scRNAseq/Seurat3.rmd deleted file mode 100644 index a3911a6c7..000000000 --- a/tcrdb/resources/external/scRNAseq/Seurat3.rmd +++ /dev/null @@ -1,183 +0,0 @@ ---- -title: 'Seurat scRNA-seq Analysis' ---- - -```{r Setup} - -knitr::opts_chunk$set(message=FALSE, warning=FALSE,echo=TRUE,error = FALSE) -library(knitr) -library(OOSAP) - -cores <- Sys.getenv('SEQUENCEANALYSIS_MAX_THREADS') -if (cores != ''){ - print(paste0('Setting future::plan to ', cores, ' cores')) - future::plan("multiprocess", workers = as.integer(cores)) - Sys.setenv('OMP_NUM_THREADS' = cores) -} else { - print('SEQUENCEANALYSIS_MAX_THREADS not set, will not set cores') -} - -print('Updating future.globals.maxSize') -options(future.globals.maxSize = Inf) - -print('Global variables: ') -for (v in c('outPrefix', 'resolutionToUse', 'dimsToUse', 'minDimsToUse', 'doCellFilter', 'doCellCycle', 'useSCTransform', 'runSingleR', 'mergeMethod', 'skipProcessing', 'gtfFile')){ - if (exists(v)){ - print(paste0(v, ': ', get(v))) - } else { - print(paste0(v, ': not defined')) - } -} - -``` - -## Prepare data - -```{r PreparingData, fig.width=12} - -rawDataSaveFile <- paste0(outPrefix, '.rawData.rds') -seuratObjs <- list() -if (file.exists(rawDataSaveFile)) { - print('resuming from file') - seuratObjs <- readRDS(rawDataSaveFile) -} else { - for (datasetName in names(data)) { - print(paste0('Loading dataset: ', datasetName)) - seuratObjs[[datasetName]] <- ReadAndFilter10xData(dataDir = data[[datasetName]], datasetName = datasetName, gtfFile = gtfFile) - - print(seuratObjs[[datasetName]]) - } - - saveRDS(seuratObjs, file = rawDataSaveFile) -} - -``` - -## Merge data - -```{r MergeDatasets} - -seuratObj <- NULL -saveFile <- paste0(outPrefix, '.seurat.rds') -if (file.exists(saveFile)) { - print('resuming from file') - seuratObj <- readRDS(saveFile) -} else { - seuratObj <- MergeSeuratObjs(seuratObjs, metadata = data, method = mergeMethod) - saveRDS(seuratObj, file = saveFile) - rm(seuratObjs) -} - -print(seuratObj) - -``` - -## Initial Processing - -```{r InitialProcessing, fig.width=12} - -if (!skipProcessing) { - seuratObj <- ProcessSeurat1(seuratObj, variableGeneTable = paste0(outPrefix, '.variableGenes.txt'), doCellFilter = doCellFilter, doCellCycle = doCellCycle, useSCTransform = useSCTransform, saveFile = saveFile) - - print(seuratObj) -} else { - print('Downstream processing will be skipped') -} - -``` - -## DimRedux - -```{r DimRedux, fig.width=12} - -if (!skipProcessing) { - seuratObj <- FindClustersAndDimRedux(seuratObj, dimsToUse = dimsToUse, minDimsToUse = minDimsToUse, saveFile = saveFile) - - Find_Markers(seuratObj, resolutionToUse = resolutionToUse, outFile = paste0(outPrefix, '.markers.txt'), saveFileMarkers = paste0(outPrefix, '.markers.rds')) - - print(seuratObj) -} - -``` - -## SingleR - -```{r SingleR, fig.width=12} - -if (!skipProcessing && runSingleR) { - tryCatch({ - seuratObj <- RunSingleR(seuratObj = seuratObj, resultTableFile = paste0(outPrefix, '.singleR.txt')) - saveRDS(seuratObj, file = saveFile) - - DimPlot_SingleRClassLabs(seuratObj, plotIndividually = T) - - Tabulate_SingleRClassLabs(seuratObj, plotIndividually = T) - }, error = function(e){ - print('There was an error in SingleR') - - saveRDS(e, file = 'error.rds') - }) - - print(seuratObj) -} else { - print('SingleR will not be run') -} - -``` - -## Phenotypes - -```{r Phenotypes, fig.width=12} - -if ( !skipProcessing ) { - PlotImmuneMarkers(seuratObj, reductions = c('tsne', 'umap')) - - if (length(unique(seuratObj$BarcodePrefix)) > 1) { - print(Seurat::DimPlot(seuratObj, reduction = 'pca', group.by = 'BarcodePrefix', label = T)) - print(Seurat::DimPlot(seuratObj, reduction = 'tsne', group.by = 'BarcodePrefix', label = T)) - print(Seurat::DimPlot(seuratObj, reduction = 'umap', group.by = 'BarcodePrefix', label = T)) - - t <- table(Cluster = Seurat::Idents(seuratObj), Dataset = seuratObj$BarcodePrefix) - t <- round(t / colSums(t), 2) - knitr::kable(t) - } -} - -``` - -## Activation - -```{r ActivationScore} - -if ( !skipProcessing ) { - seuratObj <- ClassifySGSAndApply(seuratObj = seuratObj, geneSetName = 'HighlyActivated', geneList = OOSAP::Phenotyping_GeneList()$HighlyActivated, positivityThreshold = 0.5, saveFilePath = paste0(outPrefix, '.ha.txt')) - saveRDS(seuratObj, file = saveFile) -} - -``` - -## Write Summary - -```{r Summary} - -saveRDS(seuratObj, file = saveFile) -unlink(rawDataSaveFile) - -WriteSummaryMetrics(seuratObj, file = paste0(outPrefix, '.summary.txt')) - -if ( !skipProcessing ) { - SaveDimRedux(seuratObj, file = paste0(outPrefix, '.DimReduxComps.csv')) -} - -WriteCellBarcodes(seuratObj, file = paste0(outPrefix, '.cellBarcodes.csv')) - -``` - -## Print Session Info - -```{r SessionInfo} - -sessionInfo() - -``` - diff --git a/tcrdb/resources/external/scRNAseq/seuratWrapper.sh b/tcrdb/resources/external/scRNAseq/seuratWrapper.sh deleted file mode 100644 index 5a6f2d81b..000000000 --- a/tcrdb/resources/external/scRNAseq/seuratWrapper.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash - -set -e -set -u -set -x - -WD=`pwd` -HOME=`echo ~/` - -DOCKER=/opt/acc/sbin/exadocker -LK_ROOT=$1 - -RAM_OPTS="" -ENV_OPTS="" -if [ ! -z $SEQUENCEANALYSIS_MAX_RAM ];then - RAM_OPTS=" --memory=${SEQUENCEANALYSIS_MAX_RAM}g" - - ENV_OPTS=" -e SEQUENCEANALYSIS_MAX_RAM" -fi - -if [ ! -z SEQUENCEANALYSIS_MAX_THREADS ];then - ENV_OPTS=${ENV_OPTS}" -e SEQUENCEANALYSIS_MAX_THREADS="${SEQUENCEANALYSIS_MAX_THREADS} -fi - -sudo $DOCKER pull bimberlab/oosap - -sudo $DOCKER run --rm=true $RAM_OPTS $ENV_OPTS -v "${WD}:/work" -v "${HOME}:/homeDir" -u $UID -e USERID=$UID -w /work -e HOME=/homeDir bimberlab/oosap Rscript --vanilla script.R \ No newline at end of file diff --git a/tcrdb/resources/queries/tcrdb/cdnas/Assay Info.qview.xml b/tcrdb/resources/queries/singlecell/cdna_libraries/Assay Info.qview.xml similarity index 80% rename from tcrdb/resources/queries/tcrdb/cdnas/Assay Info.qview.xml rename to tcrdb/resources/queries/singlecell/cdna_libraries/Assay Info.qview.xml index 482b26d5d..109d35ad5 100644 --- a/tcrdb/resources/queries/tcrdb/cdnas/Assay Info.qview.xml +++ b/tcrdb/resources/queries/singlecell/cdna_libraries/Assay Info.qview.xml @@ -2,10 +2,10 @@ - - - - + + + + @@ -13,7 +13,7 @@ - + diff --git a/tcrdb/resources/queries/tcrdb/cdnas.js b/tcrdb/resources/queries/tcrdb/cdnas.js deleted file mode 100644 index e465f71e1..000000000 --- a/tcrdb/resources/queries/tcrdb/cdnas.js +++ /dev/null @@ -1,39 +0,0 @@ -var console = require("console"); -var LABKEY = require("labkey"); -var importHelper = org.labkey.tcrdb.ImportHelper.create(LABKEY.Security.currentContainer.id, LABKEY.Security.currentUser.id, 'cdnas'); - -var wellMap = importHelper.getInitialWells(); - -function beforeInsert(row, errors){ - beforeUpsert(row, null, errors); -} - -function beforeUpdate(row, oldRow, errors){ - beforeUpsert(row, oldRow, errors); -} - -var rowIdx = -1; - -function beforeUpsert(row, oldRow, errors){ - //check for duplicate plate/well - oldRow = oldRow || {}; - var rowId = row.rowId || oldRow.rowId || rowIdx; - rowIdx--; - - var well = row.well || oldRow.well || ''; - if ('pool' !== well.toLowerCase()) { - var wellArr = [(row.plateId || oldRow.plateId), well]; - var wellKey = wellArr.join('<>').toUpperCase(); - if (wellMap[wellKey] && wellMap[wellKey] !== rowId) { - errors.well = 'Duplicate entry for plate/well: ' + wellArr.join('/'); - } - else { - wellMap[wellKey] = rowId; - } - } - - //Note: this will only work if the incoming row has a container property - //if (row.sortId && !row.container){ - // row.container = importHelper.getContainerForSort(row.sortId); - //} -} \ No newline at end of file diff --git a/tcrdb/resources/queries/tcrdb/cdnas/.qview.xml b/tcrdb/resources/queries/tcrdb/cdnas/.qview.xml deleted file mode 100644 index fafeb6227..000000000 --- a/tcrdb/resources/queries/tcrdb/cdnas/.qview.xml +++ /dev/null @@ -1,29 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/tcrdb/resources/queries/tcrdb/citeseq_panel_names.query.xml b/tcrdb/resources/queries/tcrdb/citeseq_panel_names.query.xml deleted file mode 100644 index bf5d41a17..000000000 --- a/tcrdb/resources/queries/tcrdb/citeseq_panel_names.query.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - true - - -
-
-
-
\ No newline at end of file diff --git a/tcrdb/resources/queries/tcrdb/citeseq_panel_names.sql b/tcrdb/resources/queries/tcrdb/citeseq_panel_names.sql deleted file mode 100644 index c4140f37f..000000000 --- a/tcrdb/resources/queries/tcrdb/citeseq_panel_names.sql +++ /dev/null @@ -1,6 +0,0 @@ -SELECT - distinct name, - count(*) as totalMarkers - -FROM tcrdb.citeseq_panels -GROUP BY name \ No newline at end of file diff --git a/tcrdb/resources/queries/tcrdb/citeseq_panels/.qview.xml b/tcrdb/resources/queries/tcrdb/citeseq_panels/.qview.xml deleted file mode 100644 index 570773573..000000000 --- a/tcrdb/resources/queries/tcrdb/citeseq_panels/.qview.xml +++ /dev/null @@ -1,12 +0,0 @@ - - - - - - - - - - - - \ No newline at end of file diff --git a/tcrdb/resources/queries/tcrdb/hashtag_oligos.sql b/tcrdb/resources/queries/tcrdb/hashtag_oligos.sql deleted file mode 100644 index 16e695b4c..000000000 --- a/tcrdb/resources/queries/tcrdb/hashtag_oligos.sql +++ /dev/null @@ -1,7 +0,0 @@ -SELECT - b.tag_name, - b.sequence, - b.group_name - -FROM sequenceanalysis.barcodes b -WHERE group_name IN ('5p-HTOs', 'MultiSeq Barcodes') \ No newline at end of file diff --git a/tcrdb/resources/queries/tcrdb/sortStatusByPlate.query.xml b/tcrdb/resources/queries/tcrdb/sortStatusByPlate.query.xml deleted file mode 100644 index 6ea09e41f..000000000 --- a/tcrdb/resources/queries/tcrdb/sortStatusByPlate.query.xml +++ /dev/null @@ -1,21 +0,0 @@ - - - - - TCR Status By Plate - - - true - /query/executeQuery.view?schemaName=tcrdb&query.queryName=sort&query.plateId~eq=${plateId} - - - true - - - Plate Complete - - -
-
-
-
\ No newline at end of file diff --git a/tcrdb/resources/queries/tcrdb/sortStatusByPlate.sql b/tcrdb/resources/queries/tcrdb/sortStatusByPlate.sql deleted file mode 100644 index ae28bf06a..000000000 --- a/tcrdb/resources/queries/tcrdb/sortStatusByPlate.sql +++ /dev/null @@ -1,88 +0,0 @@ -SELECT - t3.sortPlateId, - t3.container, - t3.workbook, - t3.sortType, - t3.processingRequested, - t3.animals, - t3.stims, - t3.sampleDates, - t3.totalSorts, - t3.totalBulkSorts, - t3.totalSingleCells, - t3.totalLibraries, - t3.totalLibrariesWithData, - t3.totalLibrariesWithBulkData, - t3.totalLibrariesWithEnrichedData, - t3.librariesComplete, - t3.sequencingComplete, - CASE - WHEN (t3.librariesComplete = TRUE AND t3.sequencingComplete = TRUE) THEN TRUE - ELSE FALSE - END as isComplete - -FROM ( -SELECT - t2.sortPlateId, - t2.container, - t2.workbook, - t2.animals, - t2.stims, - t2.sampleDates, - t2.totalSorts, - t2.totalBulkSorts, - t2.totalSingleCells, - t2.totalLibraries, - t2.totalLibrariesWithData, - t2.totalLibrariesWithBulkData, - t2.totalLibrariesWithEnrichedData, - CASE WHEN t2.totalSorts - t2.totalLibraries <= 0 THEN true ELSE false END as librariesComplete, - CASE - WHEN (t2.processingRequested LIKE '%Whole Transcriptome%' AND t2.totalSorts - t2.totalLibrariesWithBulkData > 0) THEN FALSE - WHEN (t2.processingRequested LIKE '%Enriched%' AND t2.totalSorts - t2.totalLibrariesWithEnrichedData > 0) THEN FALSE - WHEN (t2.totalSorts - t2.totalLibrariesWithData > 0) THEN FALSE - ELSE TRUE - END as sequencingComplete, - CASE - WHEN (t2.totalBulkSorts > 0 AND t2.totalSingleCells > 0) THEN 'MIXED' - WHEN (t2.totalBulkSorts > 0) THEN 'BULK' - WHEN (t2.totalSingleCells > 0) THEN 'SINGLE' - END as sortType, - t2.processingRequested -FROM ( -SELECT - t.plateId as sortPlateId, - t.totalSorts, - t.totalBulkSorts, - t.totalSingleCells, - t.animals, - t.stims, - t.sampleDates, - t.container, - t.workbook, - t.processingRequested, - (SELECT count(*) as expr from tcrdb.cdnas c1 WHERE c1.sortId.plateId = t.plateId) as totalLibraries, - (SELECT count(*) as expr from tcrdb.cdnas c2 WHERE c2.sortId.plateId = t.plateId AND c2.hasReadsetWithData = true) as totalLibrariesWithData, - (SELECT count(*) as expr from tcrdb.cdnas c3 WHERE c3.sortId.plateId = t.plateId AND c3.readsetId.totalFiles > 0) as totalLibrariesWithBulkData, - (SELECT count(*) as expr from tcrdb.cdnas c4 WHERE c4.sortId.plateId = t.plateId AND c4.enrichedReadsetId.totalFiles > 0) as totalLibrariesWithEnrichedData, - (SELECT group_concat(distinct c3.plateId, chr(10)) as expr from tcrdb.cdnas c3 WHERE c3.sortId.plateId = t.plateId) as libraryPlates - -FROM ( - SELECT - s.plateId, - s.container, - s.workbook, - count(*) AS totalSorts, - SUM(CASE WHEN s.cells > 1 THEN 1 ELSE 0 END) AS totalBulkSorts, - SUM(CASE WHEN s.cells = 1 THEN 1 ELSE 0 END) AS totalSingleCells, - group_concat(distinct s.stimId.animalId) as animals, - group_concat(distinct s.stimId.stim) as stims, - group_concat(distinct ((year(s.stimId.date) || '-' || month(s.stimId.date) || '-' || dayofmonth(s.stimId.date)))) as sampleDates, - group_concat(distinct s.processingRequested) as processingRequested - - FROM tcrdb.sorts s - GROUP BY s.plateId, s.container, s.workbook - -) t -) t2 -) t3 \ No newline at end of file diff --git a/tcrdb/resources/queries/tcrdb/sortStatusByPlate/.qview.xml b/tcrdb/resources/queries/tcrdb/sortStatusByPlate/.qview.xml deleted file mode 100644 index 366339a29..000000000 --- a/tcrdb/resources/queries/tcrdb/sortStatusByPlate/.qview.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/tcrdb/resources/queries/tcrdb/sortStatusByPlateAndSample.query.xml b/tcrdb/resources/queries/tcrdb/sortStatusByPlateAndSample.query.xml deleted file mode 100644 index 6ea09e41f..000000000 --- a/tcrdb/resources/queries/tcrdb/sortStatusByPlateAndSample.query.xml +++ /dev/null @@ -1,21 +0,0 @@ - - - - - TCR Status By Plate - - - true - /query/executeQuery.view?schemaName=tcrdb&query.queryName=sort&query.plateId~eq=${plateId} - - - true - - - Plate Complete - - -
-
-
-
\ No newline at end of file diff --git a/tcrdb/resources/queries/tcrdb/sortStatusByPlateAndSample.sql b/tcrdb/resources/queries/tcrdb/sortStatusByPlateAndSample.sql deleted file mode 100644 index e8aa0bb97..000000000 --- a/tcrdb/resources/queries/tcrdb/sortStatusByPlateAndSample.sql +++ /dev/null @@ -1,72 +0,0 @@ -SELECT - t3.sortPlateId, - t3.container, - t3.workbook, - t3.sortType, - t3.animalId, - t3.stim, - t3.date, - t3.totalSorts, - t3.totalBulkSorts, - t3.totalSingleCells, - t3.totalLibraries, - t3.totalLibrariesWithData, - t3.totalLibrariesWithBulkData, - t3.totalLibrariesWithEnrichedData, - t3.librariesComplete - -FROM ( -SELECT - t2.sortPlateId, - t2.container, - t2.workbook, - t2.animalId, - t2.stim, - t2.date, - t2.totalSorts, - t2.totalBulkSorts, - t2.totalSingleCells, - t2.totalLibraries, - t2.totalLibrariesWithData, - t2.totalLibrariesWithBulkData, - t2.totalLibrariesWithEnrichedData, - CASE WHEN t2.totalSorts - t2.totalLibraries <= 0 THEN true ELSE false END as librariesComplete, - CASE - WHEN (t2.totalBulkSorts > 0 AND t2.totalSingleCells > 0) THEN 'MIXED' - WHEN (t2.totalBulkSorts > 0) THEN 'BULK' - WHEN (t2.totalSingleCells > 0) THEN 'SINGLE' - END as sortType -FROM ( -SELECT - t.plateId as sortPlateId, - t.totalSorts, - t.totalBulkSorts, - t.totalSingleCells, - t.animalId, - t.stim, - t.date, - t.container, - t.workbook, - (SELECT count(*) as expr from tcrdb.cdnas c1 WHERE c1.sortId.plateId = t.plateId) as totalLibraries, - (SELECT count(*) as expr from tcrdb.cdnas c2 WHERE c2.sortId.plateId = t.plateId AND c2.hasReadsetWithData = true) as totalLibrariesWithData, - (SELECT count(*) as expr from tcrdb.cdnas c3 WHERE c3.sortId.plateId = t.plateId AND c3.readsetId.totalFiles > 0) as totalLibrariesWithBulkData, - (SELECT count(*) as expr from tcrdb.cdnas c4 WHERE c4.sortId.plateId = t.plateId AND c4.enrichedReadsetId.totalFiles > 0) as totalLibrariesWithEnrichedData, - (SELECT group_concat(distinct c3.plateId, chr(10)) as expr from tcrdb.cdnas c3 WHERE c3.sortId.plateId = t.plateId) as libraryPlates - -FROM ( - SELECT - s.plateId, - s.container, - s.workbook, - s.stimId.animalId, - s.stimId.stim, - s.stimId.date, - count(*) AS totalSorts, - SUM(CASE WHEN s.cells > 1 THEN 1 ELSE 0 END) AS totalBulkSorts, - SUM(CASE WHEN s.cells = 1 THEN 1 ELSE 0 END) AS totalSingleCells - FROM tcrdb.sorts s - GROUP BY s.plateId, s.container, s.workbook, s.stimId.animalId, s.stimId.stim, s.stimId.date - -) t -) t2 -) t3 \ No newline at end of file diff --git a/tcrdb/resources/queries/tcrdb/sortStatusByPlateAndSample/.qview.xml b/tcrdb/resources/queries/tcrdb/sortStatusByPlateAndSample/.qview.xml deleted file mode 100644 index f6e7f6a7f..000000000 --- a/tcrdb/resources/queries/tcrdb/sortStatusByPlateAndSample/.qview.xml +++ /dev/null @@ -1,7 +0,0 @@ - - - - - - - \ No newline at end of file diff --git a/tcrdb/resources/queries/tcrdb/sorts.js b/tcrdb/resources/queries/tcrdb/sorts.js deleted file mode 100644 index 7c62e725f..000000000 --- a/tcrdb/resources/queries/tcrdb/sorts.js +++ /dev/null @@ -1,82 +0,0 @@ -var console = require("console"); -var LABKEY = require("labkey"); -var helper = org.labkey.ldk.query.LookupValidationHelper.create(LABKEY.Security.currentContainer.id, LABKEY.Security.currentUser.id, 'tcrdb', 'sorts'); -var wellHelper = org.labkey.tcrdb.ImportHelper.create(LABKEY.Security.currentContainer.id, LABKEY.Security.currentUser.id, 'sorts'); - -var wellMap = wellHelper.getInitialWells(); - -function beforeInsert(row, errors){ - beforeUpsert(row, null, errors); -} - -function beforeUpdate(row, oldRow, errors){ - beforeUpsert(row, oldRow, errors); -} - -var rowIdx = -1; - -function beforeUpsert(row, oldRow, errors){ - if (row.well){ - row.well = row.well.toUpperCase(); - } - - if (['TNF+', 'TNF Pos', 'CD69/TNF', 'CD69+/TNF+', 'CD69-Pos/TNF-Pos', 'CD69/TNFa', 'TNF+/CD69+'].indexOf(row.population) !== -1){ - row.population = 'TNF-Pos'; - } - else if (['TNF-', 'CD69-/TNF-', 'TNF Neg', 'CD69-Neg/TNF-Neg'].indexOf(row.population) !== -1){ - row.population = 'TNF-Neg'; - } - else if (['Bulk CD8', 'Bulk CD8 T-cells', 'Bulk-CD8', 'CD8+', 'CD8', 'CD8s'].indexOf(row.population) !== -1){ - row.population = 'Bulk CD8s'; - } - else if (['CD8-CD69-Pos', 'CD69-Pos/TNF-Neg', 'TNF-/CD69+', 'CD69+', 'CD69+/TNF-'].indexOf(row.population) !== -1){ - row.population = 'CD69-Pos'; - } - - //Naive cells - if (row.population && row.population.match(/ï/)){ - row.population = row.population.replace(/ï/g, 'i'); - } - - //Tetramer/spaces: - if (row.population && row.population.match(/ Tet$/)){ - row.population = row.population.replace(/ /g, '-'); - } - - //check for duplicate plate/well - oldRow = oldRow || {}; - var rowId = row.rowId || oldRow.rowId || rowIdx; - rowIdx--; - - var lookupFields = ['stimId']; - - //for 10x-style pooled expts, support 'pool' as a special-case for well name - var well = row.well || oldRow.well || ''; - if ('pool' !== well.toLowerCase()) { - var wellArr = [(row.plateId || oldRow.plateId), well]; - var wellKey = wellArr.join('<>').toUpperCase(); - if (wellMap[wellKey] && wellMap[wellKey] !== rowId) { - errors.well = 'Duplicate entry for plate/well: ' + wellArr.join('/'); - } - else { - wellMap[wellKey] = rowId; - } - - lookupFields.push('well'); - } - - for (var i=0;i - - - - - - -
-
-
- \ No newline at end of file diff --git a/tcrdb/resources/queries/tcrdb/sorts/.qview.xml b/tcrdb/resources/queries/tcrdb/sorts/.qview.xml deleted file mode 100644 index b561d0bb4..000000000 --- a/tcrdb/resources/queries/tcrdb/sorts/.qview.xml +++ /dev/null @@ -1,26 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/tcrdb/resources/queries/tcrdb/stims.js b/tcrdb/resources/queries/tcrdb/stims.js deleted file mode 100644 index bca243881..000000000 --- a/tcrdb/resources/queries/tcrdb/stims.js +++ /dev/null @@ -1,36 +0,0 @@ -var console = require("console"); -var LABKEY = require("labkey"); -var helper = org.labkey.ldk.query.LookupValidationHelper.create(LABKEY.Security.currentContainer.id, LABKEY.Security.currentUser.id, 'tcrdb', 'stims'); - -function beforeInsert(row, errors){ - beforeUpsert(row, null, errors); -} - -function beforeUpdate(row, oldRow, errors){ - beforeUpsert(row, oldRow, errors); -} - -function beforeUpsert(row, oldRow, errors){ - if (['IE1', 'IE-1', 'IE1 Pool', 'IE-1 Pool', 'CMV IE-1 Pool', 'CMV IE1 Pool'].indexOf(row.stim) !== -1){ - row.stim = 'CMV IE-1'; - } - else if (['IE2', 'IE-2', 'IE2 Pool', 'IE-2 Pool', 'CMV IE-2 Pool', 'CMV IE2 Pool'].indexOf(row.stim) !== -1){ - row.stim = 'CMV IE-2'; - } - else if (['IE-1|IE-2', 'IE1/IE2', 'IE-1/IE-2', 'IE1IE2', 'CMV IE-2 Pool', 'CMV IE2 Pool'].indexOf(row.stim) !== -1){ - row.stim = 'IE-1/IE-2'; - } - - var lookupFields = ['stim']; - for (var i=0;i - - Ext4.onReady(function(){ - var webpart = <%=webpartContext%>; - Ext4.create('TCRdb.panel.cDNAImportPanel').render(webpart.wrapperDivId); - }); - - \ No newline at end of file diff --git a/tcrdb/resources/views/cDNAImport.view.xml b/tcrdb/resources/views/cDNAImport.view.xml deleted file mode 100644 index 7164ffd11..000000000 --- a/tcrdb/resources/views/cDNAImport.view.xml +++ /dev/null @@ -1,31 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/tcrdb/resources/views/libraryExport.html b/tcrdb/resources/views/libraryExport.html deleted file mode 100644 index 77fe3e9c7..000000000 --- a/tcrdb/resources/views/libraryExport.html +++ /dev/null @@ -1,8 +0,0 @@ - \ No newline at end of file diff --git a/tcrdb/resources/views/libraryExport.view.xml b/tcrdb/resources/views/libraryExport.view.xml deleted file mode 100644 index 0a2d0b33e..000000000 --- a/tcrdb/resources/views/libraryExport.view.xml +++ /dev/null @@ -1,7 +0,0 @@ - - - - - - - \ No newline at end of file diff --git a/tcrdb/resources/views/poolImport.html b/tcrdb/resources/views/poolImport.html deleted file mode 100644 index aca7a6e09..000000000 --- a/tcrdb/resources/views/poolImport.html +++ /dev/null @@ -1,8 +0,0 @@ - \ No newline at end of file diff --git a/tcrdb/resources/views/poolImport.view.xml b/tcrdb/resources/views/poolImport.view.xml deleted file mode 100644 index ea2101326..000000000 --- a/tcrdb/resources/views/poolImport.view.xml +++ /dev/null @@ -1,30 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/tcrdb/resources/views/stimDashboard.html b/tcrdb/resources/views/stimDashboard.html deleted file mode 100644 index 111c5d08e..000000000 --- a/tcrdb/resources/views/stimDashboard.html +++ /dev/null @@ -1,8 +0,0 @@ - \ No newline at end of file diff --git a/tcrdb/resources/views/stimDashboard.view.xml b/tcrdb/resources/views/stimDashboard.view.xml deleted file mode 100644 index f452ee3c7..000000000 --- a/tcrdb/resources/views/stimDashboard.view.xml +++ /dev/null @@ -1,7 +0,0 @@ - - - - - - - \ No newline at end of file diff --git a/tcrdb/resources/web/tcrdb/buttons.js b/tcrdb/resources/web/tcrdb/buttons.js index 2db248537..a59d12a5a 100644 --- a/tcrdb/resources/web/tcrdb/buttons.js +++ b/tcrdb/resources/web/tcrdb/buttons.js @@ -1,9 +1,6 @@ Ext4.ns('TCRdb.buttons'); - TCRdb.buttons = new function(){ - - return { createMixcrGenome: function(dataRegionName) { var dataRegion = LABKEY.DataRegions[dataRegionName]; diff --git a/tcrdb/resources/web/tcrdb/exampleData/ImportExample.xlsx b/tcrdb/resources/web/tcrdb/exampleData/ImportExample.xlsx deleted file mode 100644 index 4f5d287cf..000000000 Binary files a/tcrdb/resources/web/tcrdb/exampleData/ImportExample.xlsx and /dev/null differ diff --git a/tcrdb/resources/web/tcrdb/exampleData/ImportReadsetTemplate.xlsx b/tcrdb/resources/web/tcrdb/exampleData/ImportReadsetTemplate.xlsx deleted file mode 100644 index e59199073..000000000 Binary files a/tcrdb/resources/web/tcrdb/exampleData/ImportReadsetTemplate.xlsx and /dev/null differ diff --git a/tcrdb/resources/web/tcrdb/exampleData/ImportTemplate.xlsx b/tcrdb/resources/web/tcrdb/exampleData/ImportTemplate.xlsx deleted file mode 100644 index 490e520cd..000000000 Binary files a/tcrdb/resources/web/tcrdb/exampleData/ImportTemplate.xlsx and /dev/null differ diff --git a/tcrdb/resources/web/tcrdb/panel/LibraryExportPanel.js b/tcrdb/resources/web/tcrdb/panel/LibraryExportPanel.js deleted file mode 100644 index 4988c82e5..000000000 --- a/tcrdb/resources/web/tcrdb/panel/LibraryExportPanel.js +++ /dev/null @@ -1,822 +0,0 @@ -Ext4.define('TCRdb.panel.LibraryExportPanel', { - extend: 'Ext.panel.Panel', - alias: 'widget.tcrdb-libraryexportpanel', - - statics: { - BARCODES5: ['N701', 'N702', 'N703', 'N704', 'N705', 'N706', 'N707', 'N708', 'N709', 'N710', 'N711', 'N712'], - - BARCODES3: ['S517', 'S502', 'S503', 'S504', 'S505', 'S506', 'S507', 'S508'], - - TENX_BARCODES: ['SI-GA-A1','SI-GA-A2','SI-GA-A3','SI-GA-A4','SI-GA-A5','SI-GA-A6','SI-GA-A7','SI-GA-A8','SI-GA-A9','SI-GA-A10','SI-GA-A11','SI-GA-A12','SI-GA-B1','SI-GA-B2','SI-GA-B3','SI-GA-B4','SI-GA-B5','SI-GA-B6','SI-GA-B7','SI-GA-B8','SI-GA-B9','SI-GA-B10','SI-GA-B11','SI-GA-B12','SI-GA-C1','SI-GA-C2','SI-GA-C3','SI-GA-C4','SI-GA-C5','SI-GA-C6','SI-GA-C7','SI-GA-C8','SI-GA-C9','SI-GA-C10','SI-GA-C11','SI-GA-C12','SI-GA-D1','SI-GA-D2','SI-GA-D3','SI-GA-D4','SI-GA-D5','SI-GA-D6','SI-GA-D7','SI-GA-D8','SI-GA-D9','SI-GA-D10','SI-GA-D11','SI-GA-D12','SI-GA-E1','SI-GA-E2','SI-GA-E3','SI-GA-E4','SI-GA-E5','SI-GA-E6','SI-GA-E7','SI-GA-E8','SI-GA-E9','SI-GA-E10','SI-GA-E11','SI-GA-E12','SI-GA-F1','SI-GA-F2','SI-GA-F3','SI-GA-F4','SI-GA-F5','SI-GA-F6','SI-GA-F7','SI-GA-F8','SI-GA-F9','SI-GA-F10','SI-GA-F11','SI-GA-F12','SI-GA-G1','SI-GA-G2','SI-GA-G3','SI-GA-G4','SI-GA-G5','SI-GA-G6','SI-GA-G7','SI-GA-G8','SI-GA-G9','SI-GA-G10','SI-GA-G11','SI-GA-G12','SI-GA-H1','SI-GA-H2','SI-GA-H3','SI-GA-H4','SI-GA-H5','SI-GA-H6','SI-GA-H7','SI-GA-H8','SI-GA-H9','SI-GA-H10','SI-GA-H11','SI-GA-H12'] - }, - - initComponent: function () { - Ext4.apply(this, { - title: null, - border: false, - defaults: { - border: false - }, - items: [{ - xtype: 'radiogroup', - name: 'importType', - columns: 1, - items: [{ - boxLabel: 'Novogene/Plate List', - inputValue: 'plateList', - name: 'importType', - checked: true - },{ - boxLabel: 'Other', - inputValue: 'other', - name: 'importType' - }], - listeners: { - scope: this, - afterrender: function(field) { - field.fireEvent('change', field, field.getValue()); - }, - change: function(field, val) { - val = val.importType; - var target = field.up('panel').down('#importArea'); - target.removeAll(); - if (val === 'other') { - target.add([{ - xtype: 'ldk-simplecombo', - itemId: 'instrument', - fieldLabel: 'Instrument/Core', - forceSelection: true, - editable: false, - labelWidth: 160, - storeValues: ['NextSeq (MPSSR)', 'MiSeq (ONPRC)', 'Basic List (MedGenome)', '10x Sample Sheet', 'Novogene'] - },{ - xtype: 'ldk-simplecombo', - itemId: 'application', - fieldLabel: 'Application/Type', - forceSelection: true, - editable: true, - labelWidth: 160, - allowBlank: true, - storeValues: ['Whole Transcriptome RNA-Seq', 'TCR Enriched', '10x GEX', '10x VDJ'] - },{ - xtype: 'labkey-combo', - forceSelection: true, - multiSelect: true, - displayField: 'plateId', - valueField: 'plateId', - itemId: 'sourcePlates', - fieldLabel: 'Source Plate Id', - store: { - type: 'labkey-store', - schemaName: 'tcrdb', - sql: 'SELECT distinct plateId as plateId from tcrdb.cdnas c WHERE c.allReadsetsHaveData = false', - autoLoad: true - }, - labelWidth: 160 - },{ - xtype: 'textfield', - itemId: 'adapter', - fieldLabel: 'Adapter', - labelWidth: 160, - value: 'CTGTCTCTTATACACATCT' - }]); - } - else { - target.add({ - border: false, - defaults: { - border: false - }, - items: [{ - html: 'Add an ordered list of plates, using tab-delimited columns. The first column(s) are plate ID and library type (GEX, VDJ, CITE, or HTO). These can either be one column (i.e. G234-1, C234-1, H234-1, or T234-1), or as two columns (234-1 GEX or 234-1 HTO). An optional next column is the lane assignment (i.e. Novaseq1, HiSeq1, HiSeq2). Finally, an optional final column can be used to provide the alias for this pool. This is mostly used for CITE-Seq/HTOs, where multiple libraries are pre-pooled. See these examples:
' + - '
' +
-                                                '234-2\tGEX
' + - '234-2\tVDJ
' + - 'G233-2
' + - 'T235-2
' + - '234-2\tVDJ\tNovaSeq1
' + - 'G233-2\tNovaSeq1
' + - '235-2\tHTO\tHiSeq1\tBNB-HTO-1
' + - 'H235-2\tHiSeq1\tBNB-HTO-1
' + - '235-2\tHTO\tHiSeq2\tBNB-HTO-1
' + - 'H235-2\tHiSeq1\tBNB-HTO-1
' + - 'C235-2\tHiSeq1\tBNB-HTO-1' + - '
', - border: false - },{ - xtype: 'hidden', - itemId: 'instrument', - value: 'Novogene' - },{ - xtype: 'textarea', - itemId: 'plateList', - fieldLabel: 'Plate List', - labelAlign: 'top', - width: 270, - height: 200, - enableKeyEvents: true, - listeners: { - specialkey: function (field, e) { - if (e.getKey() === e.TAB) { - field.setValue(field.getValue() + '\t'); - e.preventDefault(); - } - } - }, - },{ - xtype: 'ldk-numberfield', - itemId: 'defaultVolume', - fieldLabel: 'Default Volume (uL)', - value: 10 - }], - buttonAlign: 'left', - buttons: [{ - text: 'Add', - scope: this, - handler: function (btn) { - var text = btn.up('panel').down('#plateList').getValue(); - if (!text) { - Ext4.Msg.alert('Error', 'Must enter a list of plates'); - return; - } - - text = LDK.Utils.CSVToArray(Ext4.String.trim(text), '\t'); - Ext4.Array.forEach(text, function(r, idx){ - var val = r[0]; - if (val.startsWith('G')){ - val = val.substr(1); - val = val.replace('_', '-'); - r[0] = 'GEX'; - r.unshift(val); - - } - else if (val.startsWith('T')){ - val = val.substr(1); - val = val.replace('_', '-'); - r[0] = 'VDJ'; - r.unshift(val); - } - else if (val.startsWith('H')){ - val = val.substr(1); - val = val.replace('_', '-'); - r[0] = 'HTO'; - r.unshift(val); - } - else if (val.startsWith('C')){ - val = val.substr(1); - val = val.replace('_', '-'); - r[0] = 'CITE'; - r.unshift(val); - } - }, this); - - var hadError = false; - Ext4.Array.forEach(text, function(r){ - if (r.length < 2){ - hadError = true; - } - - //ensure all rows are of length 4 - if (r.length !== 4) { - for (i=0;i<(4-r.length);i++) { - r.push(''); - } - } - - Ext4.Array.forEach(r, function(val, idx){ - r[idx] = Ext4.String.trim(val); - }, this); - }, this); - - if (hadError) { - Ext4.Msg.alert('Error', 'All rows must have at least 2 values'); - return; - } - - this.onSubmit(btn, text); - } - }] - }); - } - } - } - }, { - bodyStyle: 'padding: 5px;', - itemId: 'importArea', - border: false, - defaults: { - border: false - } - },{ - xtype: 'checkbox', - boxLabel: 'Allow Duplicate Barcodes', - checked: false, - itemId: 'allowDuplicates' - },{ - xtype: 'checkbox', - boxLabel: 'Use Simple Sample Names', - checked: true, - itemId: 'simpleSampleNames' - },{ - xtype: 'checkbox', - boxLabel: 'Include Blanks', - checked: true, - itemId: 'includeBlanks' - },{ - xtype: 'checkbox', - boxLabel: 'Include Libraries With Data', - checked: false, - itemId: 'includeWithData', - listeners: { - change: function (field, val) { - var target = field.up('tcrdb-libraryexportpanel').down('#sourcePlates'); - if (target) { - var sql = 'SELECT distinct plateId as plateId from tcrdb.cdnas ' + (val ? '' : 'c WHERE c.allReadsetsHaveData = false'); - target.store.sql = sql; - target.store.removeAll(); - target.store.load(function () { - if (target.getPicker()) { - target.getPicker().refresh(); - } - }, this); - } - } - } - },{ - xtype: 'textarea', - itemId: 'outputArea', - fieldLabel: 'Output', - labelAlign: 'top', - width: 1000, - height: 400 - }], - buttonAlign: 'left', - buttons: [{ - text: 'Submit', - scope: this, - handler: function(btn){ - this.onSubmit(btn); - } - }, { - text: 'Download Data', - itemId: 'downloadData', - disabled: true, - handler: function (btn) { - var instrument = btn.up('tcrdb-libraryexportpanel').down('#instrument').getValue(); - var plateId = btn.up('tcrdb-libraryexportpanel').down('#sourcePlates').getValue(); - var delim = 'TAB'; - var extention = 'txt'; - var split = '\t'; - if (instrument !== 'NextSeq (MPSSR)') { - delim = 'COMMA'; - extention = 'csv'; - split = ','; - } - - var val = btn.up('tcrdb-libraryexportpanel').down('#outputArea').getValue(); - var rows = LDK.Utils.CSVToArray(Ext4.String.trim(val), split); - - LABKEY.Utils.convertToTable({ - fileName: plateId + '.' + extention, - rows: rows, - delim: delim - }); - } - },{ - text: 'Assign Readsets To Batch', - itemId: 'readsetBatch', - disabled: true, - handler: function (btn) { - var panel = btn.up('tcrdb-libraryexportpanel'); - var readsetIds = btn.readsetIds; - if (!readsetIds) { - Ext4.Msg.alert('Error', 'No Readset IDs Found'); - return; - } - - Ext4.Msg.wait('Loading...'); - LABKEY.Query.selectRows({ - containerPath: Laboratory.Utils.getQueryContainerPath(), - schemaName: 'sequenceanalysis', - queryName: 'sequence_readsets', - columns: 'rowid,container', - filterArray: [LABKEY.Filter.create('rowid', readsetIds.join(';'), LABKEY.Filter.Types.IN)], - scope: this, - failure: LDK.Utils.getErrorCallback(), - success: function (results) { - Ext4.Msg.hide(); - - if (!results || !results.rows || !results.rows.length) { - Ext4.Msg.hide(); - Ext4.Msg.alert('Error', 'Readsets not found: ' + readsetIds.join(';')); - return; - } - - var readsetRows = results.rows; - - Ext4.create('Ext.window.Window', { - title: 'Assign Readsets To Batch', - width: 800, - bodyStyle: 'padding: 10px;', - readsetIds: readsetIds, - items: [{ - html: 'The following readsets will be assigned to an instrument run/batch: ' + readsetIds.join(', '), - style: 'padding-bottom: 10px;', - border: false - }, { - xtype: 'textfield', - fieldLabel: 'Run/Batch Name', - labelWidth: 150, - itemId: 'batchName' - }, { - xtype: 'ldk-integerfield', - fieldLabel: 'Target Workbook', - labelWidth: 150, - itemId: 'targetWorkbook' - }], - buttons: [{ - text: 'Submit', - scope: this, - handler: function (btn) { - var win = btn.up('window'); - var batchId = win.down('#batchName').getValue(); - if (!batchId) { - Ext4.Msg.alert('Error', 'Must enter a batch name'); - return; - } - - var workbook = win.down('#targetWorkbook').getValue(); - if (workbook) { - Ext4.Msg.wait('Loading...'); - LABKEY.Query.selectRows({ - containerPath: Laboratory.Utils.getQueryContainerPath(), - schemaName: 'core', - queryName: 'workbooks', - columns: 'EntityId', - filterArray: [LABKEY.Filter.create('workbookId/workbookId', workbook, LABKEY.Filter.Types.EQUAL)], - scope: this, - failure: LDK.Utils.getErrorCallback(), - success: function (results) { - if (!results || !results.rows || !results.rows.length) { - Ext4.Msg.hide(); - Ext4.Msg.alert('Error', 'Workbook not found: ' + workbook); - return; - } - - LDK.Assert.assertEquality('Expected single workbook to be returned', results.rows.length, 1); - - win.close(); - panel.createInstrumentRun(readsetRows, batchId, results.rows[0].EntityId); - } - }); - } - else { - panel.createInstrumentRun(readsetRows, batchId); - } - } - }, { - text: 'Cancel', - handler: function (btn) { - btn.up('window').close(); - } - }] - }).show(); - } - }); - } - }] - }); - - this.callParent(arguments); - - Ext4.Msg.wait('Loading...'); - LABKEY.Query.selectRows({ - schemaName: 'sequenceanalysis', - queryName: 'barcodes', - sort: 'group_name,tag_name', - scope: this, - failure: LDK.Utils.getErrorCallback(), - success: function(results){ - this.barcodeMap = {}; - - Ext4.Array.forEach(results.rows, function(r){ - this.barcodeMap[r.group_name] = this.barcodeMap[r.group_name] || {}; - this.barcodeMap[r.group_name][r.tag_name] = r.sequence; - }, this); - - Ext4.Msg.hide(); - } - }); - }, - - createInstrumentRun: function (readsetRows, batchId, containerId) { - containerId = containerId || Laboratory.Utils.getQueryContainerPath(); - LABKEY.Query.insertRows({ - containerPath: containerId, - schemaName: 'sequenceanalysis', - queryName: 'instrument_runs', - scope: this, - rows: [{ - name: batchId - }], - failure: LDK.Utils.getErrorCallback(), - success: function (results) { - var runId = results.rows[0].rowId; - LDK.Assert.assertNotEmpty('Error creating instrument run', runId); - - Ext4.Array.forEach(readsetRows, function (rs) { - rs.instrument_run_id = runId - }, this); - - LABKEY.Query.updateRows({ - containerPath: Laboratory.Utils.getQueryContainerPath(), - schemaName: 'sequenceanalysis', - queryName: 'sequence_readsets', - rows: readsetRows, - failure: LDK.Utils.getErrorCallback(), - success: function (results) { - Ext4.Msg.hide(); - Ext4.Msg.alert('Success', 'Readsets updated'); - } - }); - } - }); - }, - - onSubmit: function(btn, expectedPairs){ - var plateIds = []; - - if (expectedPairs) { - var hadError = false; - Ext4.Array.forEach(expectedPairs, function(p){ - plateIds.push(Ext4.String.trim(p[0])); - }, this); - } - else { - plateIds = btn.up('tcrdb-libraryexportpanel').down('#sourcePlates').getValue(); - } - - if (!plateIds || !plateIds.length){ - Ext4.Msg.alert('Error', 'Must provide the plate Id(s)'); - return; - } - - plateIds = Ext4.unique(plateIds); - - var instrument = btn.up('tcrdb-libraryexportpanel').down('#instrument').getValue(); - var application = btn.up('tcrdb-libraryexportpanel').down('#application') ? btn.up('tcrdb-libraryexportpanel').down('#application').getValue() : null; - var defaultVolume = btn.up('tcrdb-libraryexportpanel').down('#defaultVolume') ? btn.up('tcrdb-libraryexportpanel').down('#defaultVolume').getValue() : ''; - var adapter = btn.up('tcrdb-libraryexportpanel').down('#adapter') ? btn.up('tcrdb-libraryexportpanel').down('#adapter').getValue() : null; - var includeWithData = btn.up('tcrdb-libraryexportpanel').down('#includeWithData').getValue(); - var allowDuplicates = btn.up('tcrdb-libraryexportpanel').down('#allowDuplicates').getValue(); - var simpleSampleNames = btn.up('tcrdb-libraryexportpanel').down('#simpleSampleNames').getValue(); - var includeBlanks = btn.up('tcrdb-libraryexportpanel').down('#includeBlanks').getValue(); - var doReverseComplement = btn.up('tcrdb-libraryexportpanel').doReverseComplement; - - var isMatchingApplication = function(application, libraryType, readsetApplication, rowLevelApplication){ - if (!application && !rowLevelApplication){ - return true; - } - - if (application === 'Whole Transcriptome RNA-Seq'){ - return readsetApplication === 'RNA-seq' || readsetApplication === 'RNA-seq, Single Cell'; - } - else if (application === 'TCR Enriched'){ - return readsetApplication === 'RNA-seq + Enrichment'; - } - else if (readsetApplication === 'RNA-seq, Single Cell'){ - application = rowLevelApplication || application; - return (libraryType.match(/^10x [35]\' GEX/) && application === '10x GEX') || (libraryType.match(/^10x 5' VDJ/) && application === '10x VDJ'); - } - else if (readsetApplication === 'Cell Hashing'){ - application = rowLevelApplication || application; - return (application === '10x HTO'); - } - else if (readsetApplication === 'CITE-Seq'){ - application = rowLevelApplication || application; - return (application === '10x CITE-Seq'); - } - }; - - var getSampleName = function(simpleSampleNames, readsetId, readsetName, suffix){ - return (simpleSampleNames ? 's_' + readsetId : readsetId + '_' + readsetName) + (suffix ? '_' + suffix : ''); - }; - - Ext4.Msg.wait('Loading cDNA data'); - LABKEY.Query.selectRows({ - method: 'POST', - containerPath: Laboratory.Utils.getQueryContainerPath(), - schemaName: 'tcrdb', - queryName: 'cdnas', - sort: 'plateId,well/addressByColumn', - columns: 'rowid,plateid' + - ',readsetId,readsetId/name,readsetId/application,readsetId/librarytype,readsetId/barcode5,readsetId/barcode5/sequence,readsetId/barcode3,readsetId/barcode3/sequence,readsetId/totalFiles,readsetId/concentration' + - ',enrichedReadsetId,enrichedReadsetId/name,enrichedReadsetId/application,enrichedReadsetId/librarytype,enrichedReadsetId/barcode5,enrichedReadsetId/barcode5/sequence,enrichedReadsetId/barcode3,enrichedReadsetId/barcode3/sequence,enrichedReadsetId/totalFiles,enrichedReadsetId/concentration' + - ',hashingReadsetId,hashingReadsetId/name,hashingReadsetId/application,hashingReadsetId/librarytype,hashingReadsetId/barcode5,hashingReadsetId/barcode5/sequence,hashingReadsetId/barcode3,hashingReadsetId/barcode3/sequence,hashingReadsetId/totalFiles,hashingReadsetId/concentration' + - ',citeseqReadsetId,citeseqReadsetId/name,citeseqReadsetId/application,citeseqReadsetId/librarytype,citeseqReadsetId/barcode5,citeseqReadsetId/barcode5/sequence,citeseqReadsetId/barcode3,citeseqReadsetId/barcode3/sequence,citeseqReadsetId/totalFiles,citeseqReadsetId/concentration', - scope: this, - filterArray: [LABKEY.Filter.create('plateId', plateIds.join(';'), LABKEY.Filter.Types.IN)], - failure: LDK.Utils.getErrorCallback(), - success: function (results) { - Ext4.Msg.hide(); - - if (!results || !results.rows || !results.rows.length) { - Ext4.Msg.alert('Error', 'No libraries found for the selected plates'); - return; - } - - var sortedRows = results.rows; - if (expectedPairs) { - sortedRows = []; - var missingRows = []; - Ext4.Array.forEach(expectedPairs, function(p){ - var found = false; - Ext4.Array.forEach(results.rows, function(row){ - if (row.plateId === p[0]) { - if (p[1] === 'GEX') { - if (includeWithData || row['readsetId/totalFiles'] === 0) { - if (row['readsetId'] && row['readsetId/librarytype'] && row['readsetId/librarytype'].match('GEX')) { - sortedRows.push(Ext4.apply({targetApplication: '10x GEX', laneAssignment: (p.length > 2 ? p[2] : null), plateAlias: (p.length > 3 ? p[3] : null)}, row)); - found = true; - return false; - } - } - } - else if (p[1] === 'HTO') { - if (includeWithData || row['hashingReadsetId/totalFiles'] === 0) { - if (row['hashingReadsetId'] && row['hashingReadsetId/application'] && row['hashingReadsetId/application'].match('Cell Hashing')) { - sortedRows.push(Ext4.apply({targetApplication: '10x HTO', laneAssignment: (p.length > 2 ? p[2] : null), plateAlias: (p.length > 3 ? p[3] : null)}, row)); - found = true; - return false; - } - } - } - else if (p[1] === 'CITE') { - if (includeWithData || row['citeseqReadsetId/totalFiles'] === 0) { - if (row['citeseqReadsetId'] && row['citeseqReadsetId/application'] && row['citeseqReadsetId/application'].match('CITE-Seq')) { - sortedRows.push(Ext4.apply({targetApplication: '10x CITE-Seq', laneAssignment: (p.length > 2 ? p[2] : null), plateAlias: (p.length > 3 ? p[3] : null)}, row)); - found = true; - return false; - } - } - } - else if (p[1] === 'VDJ') { - if (includeWithData || row['enrichedReadsetId/totalFiles'] === 0) { - if (row['enrichedReadsetId'] && row['enrichedReadsetId/librarytype'].match('VDJ')) { - sortedRows.push(Ext4.apply({targetApplication: '10x VDJ', laneAssignment: (p.length > 2 ? p[2] : null), plateAlias: (p.length > 3 ? p[3] : null)}, row)); - found = true; - return false; - } - } - } - } - }, this); - - if (!found) { - missingRows.push(p[0] + '/' + p[1]); - } - }, this); - - if (missingRows.length){ - Ext4.Msg.alert('Error', 'The following plates were not found:
' + missingRows.join('
')); - return; - } - } - - var barcodes = 'Illumina'; - var readsetIds = {}; - var barcodeCombosUsed = []; - if (instrument === 'NextSeq (MPSSR)' || instrument === 'Basic List (MedGenome)') { - var rc5 = (instrument === 'NextSeq (MPSSR)'); - var rc3 = (instrument === 'NextSeq (MPSSR)'); - - var rows = [['Name', 'Adapter', 'I7_Index_ID', 'I7_Seq', 'I5_Index_ID', 'I5_Seq'].join('\t')]; - Ext4.Array.forEach(sortedRows, function (r) { - //only include readsets without existing data - var processSample = function(rows, r, fieldName) { - if (!readsetIds[r[fieldName]] && r[fieldName] && (includeWithData || r[fieldName + '/totalFiles'] === 0) && isMatchingApplication(application, r[fieldName + '/librarytype'], r[fieldName + '/application'], r.targetApplication)) { - //allow for cell hashing / shared readsets - readsetIds[r[fieldName]] = true; - - //reverse complement both barcodes: - var barcode5 = rc5 ? doReverseComplement(r[fieldName + '/barcode5/sequence']) : r[fieldName + '/barcode5/sequence']; - var barcode3 = rc3 ? doReverseComplement(r[fieldName + '/barcode3/sequence']) : r[fieldName + '/barcode3/sequence']; - barcodeCombosUsed.push(r[fieldName + '/barcode5'] + '/' + r[fieldName + '/barcode3']); - rows.push([getSampleName(simpleSampleNames, r[fieldName], r[fieldName + '/name']), adapter, r[fieldName + '/barcode5'], barcode5, r[fieldName + '/barcode3'], barcode3].join('\t')); - } - }; - - processSample(rows, r, 'readsetId'); - processSample(rows, r, 'enrichedReadsetId'); - processSample(rows, r, 'hashingReadsetId'); - processSample(rows, r, 'citeseqReadsetId'); - }, this); - - //add missing barcodes: - if (includeBlanks) { - var blankIdx = 0; - Ext4.Array.forEach(TCRdb.panel.LibraryExportPanel.BARCODES5, function (barcode5) { - Ext4.Array.forEach(TCRdb.panel.LibraryExportPanel.BARCODES3, function (barcode3) { - var combo = barcode5 + '/' + barcode3; - if (barcodeCombosUsed.indexOf(combo) === -1) { - blankIdx++; - var barcode5Seq = rc5 ? doReverseComplement(this.barcodeMap[barcodes][barcode5]) : this.barcodeMap[barcodes][barcode5]; - var barcode3Seq = rc3 ? doReverseComplement(this.barcodeMap[barcodes][barcode3]) : this.barcodeMap[barcodes][barcode3]; - - var name = simpleSampleNames ? 's_Blank' + blankIdx : plateIds.join(';').replace(/\//g, '-') + '_Blank' + blankIdx; - rows.push([name, adapter, barcode5, barcode5Seq, barcode3, barcode3Seq].join('\t')); - } - }, this); - }, this); - } - } - else if (instrument === 'MiSeq (ONPRC)') { - var rows = []; - rows.push('[Header]'); - rows.push('IEMFileVersion,4'); - rows.push('Investigator Name,Bimber'); - rows.push('Experiment Name,' + plateIds.join(';')); - rows.push('Date,11/16/2017'); - rows.push('Workflow,GenerateFASTQ'); - rows.push('Application,FASTQ Only'); - rows.push('Assay,Nextera XT'); - rows.push('Description,'); - rows.push('Chemistry,Amplicon'); - rows.push(''); - rows.push('[Reads]'); - rows.push('251'); - rows.push('251'); - rows.push(''); - rows.push('[Settings]'); - rows.push('ReverseComplement,0'); - rows.push('Adapter,' + adapter); - rows.push(''); - rows.push('[Data]'); - rows.push('Sample_ID,Sample_Name,Sample_Plate,Sample_Well,I7_Index_ID,index,I5_Index_ID,index2,Sample_Project,Description'); - - Ext4.Array.forEach(sortedRows, function (r) { - //only include readsets without existing data - if (!readsetIds[r.readsetId] && r.readsetId && (includeWithData || r['readsetId/totalFiles'] === 0) && isMatchingApplication(application, r['readsetId/librarytype'], r['readsetId/application'], r.targetApplication)) { - //allow for cell hashing / shared readsets - readsetIds[r.readsetId] = true; - - //reverse complement both barcodes: - var barcode5 = doReverseComplement(r['readsetId/barcode5/sequence']); - var barcode3 = r['readsetId/barcode3/sequence']; - var cleanedName = r.readsetId + '_' + r['readsetId/name'].replace(/ /g, '_'); - cleanedName = cleanedName.replace(/\//g, '-'); - - barcodeCombosUsed.push(r['readsetId/barcode5'] + '/' + r['readsetId/barcode3']); - rows.push([r.readsetId, cleanedName, '', '', r['readsetId/barcode5'], barcode5, r['readsetId/barcode3'], barcode3].join(',')); - } - - if (!readsetIds[r.enrichedReadsetId] && r.enrichedReadsetId && (includeWithData || r['enrichedReadsetId/totalFiles'] == 0) && isMatchingApplication(application, r['enrichedReadsetId/librarytype'], r['enrichedReadsetId/application'], r.targetApplication)) { - //allow for cell hashing / shared readsets - readsetIds[r.enrichedReadsetId] = true; - - var barcode5 = doReverseComplement(r['enrichedReadsetId/barcode5/sequence']); - var barcode3 = r['enrichedReadsetId/barcode3/sequence']; - var cleanedName = r.enrichedReadsetId + '_' + r['enrichedReadsetId/name'].replace(/ /g, '_'); - cleanedName = cleanedName.replace(/\//g, '-'); - - barcodeCombosUsed.push(r['enrichedReadsetId/barcode5'] + '/' + r['enrichedReadsetId/barcode3']); - rows.push([r.enrichedReadsetId, cleanedName, '', '', r['enrichedReadsetId/barcode5'], barcode5, r['enrichedReadsetId/barcode3'], barcode3].join(',')) - } - }, this); - - //add missing barcodes: - if (includeBlanks) { - var blankIdx = 0; - Ext4.Array.forEach(TCRdb.panel.LibraryExportPanel.BARCODES5, function (barcode5) { - Ext4.Array.forEach(TCRdb.panel.LibraryExportPanel.BARCODES3, function (barcode3) { - var combo = barcode5 + '/' + barcode3; - if (barcodeCombosUsed.indexOf(combo) === -1) { - blankIdx++; - var barcode5Seq = doReverseComplement(this.barcodeMap[barcodes][barcode5]); - var barcode3Seq = this.barcodeMap[barcodes][barcode3]; - rows.push([plateIds.join(';').replace(/\//g, '-') + '_Blank' + blankIdx, null, null, null, barcode5, barcode5Seq, barcode3, barcode3Seq].join(',')); - } - }, this); - }, this); - } - } - else if (instrument === '10x Sample Sheet' || instrument === 'Novogene') { - //we make the default assumption that we're using 10x primers, which are listed in the sample-sheet orientation - var doRC = false; - var rows = []; - var barcodes = '10x Chromium Single Cell v2'; - - if (instrument === '10x Sample Sheet') { - rows.push('Sample_ID,Sample_Name,index,Sample_Project'); - } - - //only include readsets without existing data - var processType = function(readsetIds, rows, r, fieldName, suffix, size, phiX, samplePrefix, comment, doRC) { - if (!readsetIds[r[fieldName]] && r[fieldName] && (includeWithData || r[fieldName + '/totalFiles'] === 0) && isMatchingApplication(application, r[fieldName + '/librarytype'], r[fieldName + '/application'], r.targetApplication)) { - //allow for shared readsets across cDNAs (hashing, etc.) - readsetIds[r[fieldName]] = true; - - var cleanedName = r[fieldName] + '_' + r[fieldName + '/name'].replace(/ /g, '_'); - cleanedName = cleanedName.replace(/\//g, '-'); - var sampleName = getSampleName(simpleSampleNames, r[fieldName], r[fieldName + '/name']) + (suffix && instrument === 'Novogene' ? '' : '-' + suffix); - - var barcode5s = r[fieldName + '/barcode5/sequence'] ? r[fieldName + '/barcode5/sequence'].split(',') : []; - if (!barcode5s) { - LDK.Utils.logError('Sample missing barcode: ' + sampleName); - } - - barcodeCombosUsed.push([r[fieldName + '/barcode5'], '', r.laneAssignment || ''].join('/')); - Ext4.Array.forEach(barcode5s, function (bc, idx) { - bc = doRC ? doReverseComplement(bc) : bc; - - var data = [sampleName, (instrument === 'Novogene' ? '' : cleanedName), bc, '']; - if (instrument === 'Novogene') { - data = [sampleName]; - if (r.plateAlias) { - data.unshift(r.plateAlias); - } - else { - data.unshift(samplePrefix + r.plateId.replace(/-/g, '_')); - } - - data.push('Macaca mulatta'); - data.push(bc); - data.push(''); - data.push(r[fieldName + '/concentration'] || ''); - data.push(defaultVolume); - data.push(''); - data.push(size); - data.push(phiX); //PhiX - data.push(r.laneAssignment || ''); - data.push(comment || 'Please QC individually and pool in equal amounts per lane'); - } - rows.push(data.join(delim)); - }, this); - } - }; - - var delim = instrument === 'Novogene' ? '\t' : ','; - Ext4.Array.forEach(sortedRows, function (r) { - processType(readsetIds, rows, r, 'readsetId', 'GEX', 500, 1, 'G', null, false); - processType(readsetIds, rows, r, 'enrichedReadsetId', 'TCR', 700, 1, 'T', null, false); - processType(readsetIds, rows, r, 'hashingReadsetId', 'HTO', 182, 5, 'H', 'Cell hashing, 190bp amplicon. Please QC individually and pool in equal amounts per lane', true); - processType(readsetIds, rows, r, 'citeseqReadsetId', 'CITE', 182, 5, 'C', 'CITE-Seq, 190bp amplicon. Please QC individually and pool in equal amounts per lane', false); - }, this); - - //add missing barcodes: - if (includeBlanks && instrument !== 'Novogene') { - var blankIdx = 0; - Ext4.Array.forEach(TCRdb.panel.LibraryExportPanel.TENX_BARCODES, function (barcode5) { - if (barcodeCombosUsed.indexOf(barcode5) === -1) { - blankIdx++; - var barcode5Seq = this.barcodeMap[barcodes][barcode5].split(','); - Ext4.Array.forEach(barcode5Seq, function (seq, idx) { - seq = doRC ? doReverseComplement(seq) : seq; - rows.push([barcode5 + '_' + (idx + 1), plateIds.join(';').replace(/\//g, '-') + '_Blank' + blankIdx, seq, ''].join(delim)); - }, this); - } - }, this); - } - } - - //check for unique barcodes - var sorted = barcodeCombosUsed.slice().sort(); - var duplicates = []; - for (var i = 0; i < sorted.length - 1; i++) { - if (sorted[i + 1] === sorted[i]) { - duplicates.push(sorted[i]); - } - } - - duplicates = Ext4.unique(duplicates); - if (!allowDuplicates && duplicates.length){ - Ext4.Msg.alert('Error', 'Duplicate barcodes: ' + duplicates.join(', ')); - btn.up('tcrdb-libraryexportpanel').down('#outputArea').setValue(null); - btn.up('tcrdb-libraryexportpanel').down('#downloadData').setDisabled(true); - } - else { - btn.up('tcrdb-libraryexportpanel').down('#outputArea').setValue(rows.join('\n')); - btn.up('tcrdb-libraryexportpanel').down('#downloadData').setDisabled(false); - - var rsBtn = btn.up('tcrdb-libraryexportpanel').down('#readsetBatch'); - rsBtn.readsetIds = Ext4.Object.getKeys(readsetIds); - rsBtn.setDisabled(false); - } - } - }); - }, - - doReverseComplement: function(seq){ - if (!seq){ - return seq; - } - var match={'a': 'T', 'A': 'T', 't': 'A', 'T': 'A', 'g': 'C', 'G': 'C', 'c': 'G', 'C': 'G'}; - var o = ''; - for (var i = seq.length - 1; i >= 0; i--) { - if (match[seq[i]] === undefined) break; - o += match[seq[i]]; - } - - return o; - } -}); diff --git a/tcrdb/resources/web/tcrdb/panel/PoolImportPanel.js b/tcrdb/resources/web/tcrdb/panel/PoolImportPanel.js deleted file mode 100644 index fc4a4dbf8..000000000 --- a/tcrdb/resources/web/tcrdb/panel/PoolImportPanel.js +++ /dev/null @@ -1,1017 +0,0 @@ -Ext4.define('TCRdb.panel.PoolImportPanel', { - extend: 'Ext.panel.Panel', - - COLUMNS: [{ - name: 'workbook', - labels: ['Experiment/Workbook', 'Expt', 'Expt #', 'Experiment', 'Exp#', 'Exp #', 'Workbook', 'Workbook #'], - allowRowSpan: true, - alwaysShow: true, - transform: 'expt', - allowBlank: false - },{ - name: 'plateId', - labels: ['Pool/Tube', 'Pool', 'Pool Num', 'Pool #', 'Tube #', 'Tube#'], - allowRowSpan: true, - alwaysShow: true, - transform: 'pool', - allowBlank: false - },{ - name: 'stimId', - labels: ['Stim Id'], - allowRowSpan: false, - allowBlank: true, - alwaysShow: true - },{ - name: 'animalId', - labels: ['Animal', 'Animal Id', 'SubjectId', 'Subject Id'], - allowRowSpan: true, - allowBlank: false, - transform: 'animal' - },{ - name: 'sampleDate', - labels: ['Sample Date', 'Date'], - alwaysShow: true, - allowRowSpan: true, - transform: 'sampleDate', - allowBlank: false - },{ - name: 'effector', - labels: ['Effectors', 'Effector'], - alwaysShow: true, - allowRowSpan: true, - transform: 'effector', - allowBlank: false - },{ - name: 'tissue', - labels: ['Tissue', 'Tissue Sample'], - alwaysShow: false, - allowRowSpan: true, - allowBlank: true - },{ - name: 'stim', - labels: ['Stim', 'Peptide Only Conditions'], - allowRowSpan: false, - allowBlank: false, - transform: 'stim' - },{ - name: 'stim_num', - labels: ['Stim #'], - allowRowSpan: false, - alwaysShow: true - },{ - name: 'population', - labels: ['Population', 'Target Population', 'Target Pop'], - allowRowSpan: true, - allowBlank: false, - transform: 'population' - },{ - name: 'tetramer', - labels: ['Tetramer'], - allowRowSpan: false, - allowBlank: true, - transform: 'tetramer' - },{ - name: 'sortId', - labels: ['Sort Id'], - allowRowSpan: false, - allowBlank: true, - alwaysShow: true - },{ - name: 'hto', - labels: ['HTO', 'HTO Oligo', 'HTO-Oligo', 'HTO barcode', 'Barcode'], - allowRowSpan: false, - transform: 'hto' - },{ - name: 'cells', - labels: ['Cells', 'Cell #', 'Sort', 'Sort Cell Count'], - allowRowSpan: false, - allowBlank: false, - transform: 'cells' - },{ - name: 'hto_library_index', - labels: ['HTO Library Index', 'HTO Index', 'MultiSeq Index', 'MultiSeq Library Index'], - allowRowSpan: true, - transform: 'htoIndex' - },{ - name: 'hto_library_conc', - labels: ['HTO Library Conc', 'HTO Library Conc (ng/uL)', 'HTO (qubit) ng/uL', 'HTO (quibit) ng/uL', 'MultiSeq Library Conc', 'MultiSeq Library (qubit) ng/uL', 'MultiSeq Library Conc (qubit) ng/uL'], - allowRowSpan: true - },{ - name: 'citeseqpanel', - labels: ['Cite-Seq Panel', 'Cite-Seq Panel Name', 'CiteSeq Panel'], - allowRowSpan: true - },{ - name: 'citeseq_library_index', - labels: ['Cite-Seq Library Index', 'Cite-Seq Index', 'CiteSeq Library Index', 'CiteSeq Index', 'Cite-Seq Library Index', 'Cite-Seq Index', 'CiteSeq Library (qubit) ng/uL'], - allowRowSpan: true, - transform: 'citeSeqTenXBarcode' - },{ - name: 'citeseq_library_conc', - labels: ['Cite-Seq Library Conc', 'Cite-Seq Library Conc (ng/uL)', 'Cite-Seq (qubit) ng/uL', 'Cite-Seq (quibit) ng/uL'], - allowRowSpan: true - },{ - name: 'gex_library_index', - labels: ['5\' GEX Library Index', '5\' GEX Index', 'GEX Index', 'GEX Library Index', '5-GEX Index', '5\'GEX Library Index'], - allowRowSpan: true, - transform: 'tenXBarcode' - },{ - name: 'gex_library_conc', - labels: ['5\' GEX Library Conc', 'GEX Library Conc', 'GEX Library Conc (ng/uL)', '5\' GEX Conc', 'GEX Conc', 'GEX Conc (ng/uL)', '5\' GEX (qubit) ng/uL', '5\' GEX Library (qubit) ng/uL'], - allowRowSpan: true - },{ - name: 'gex_library_fragment', - labels: ['5\' GEX Library Fragment Size', 'GEX Library Fragment Size', '5\' GEX Fragment Size', 'GEX Fragment Size', 'GEX Library Fragment Size (bp)'], - allowRowSpan: true - },{ - name: 'tcr_library_index', - labels: ['TCR Library Index', 'TCR Index', 'TCR Libray Index'], - allowRowSpan: true, - transform: 'tenXBarcode' - },{ - name: 'tcr_library_conc', - labels: ['TCR Library Conc', 'TCR Library Conc (ng/uL)', 'TCR (qubit) ng/uL', 'TCR library (qubit) ng/uL'], - allowRowSpan: true - },{ - name: 'tcr_library_fragment', - labels: ['TCR Library Fragment Size', 'TCR Library Fragment Size (bp)'], - allowRowSpan: true - }], - - IGNORED_COLUMNS: [], - - transforms: { - stim: function(val, panel) { - if (val && (val === '--' || val === '-')) { - val = 'NoStim'; - } - - return val; - }, - - animal: function(val, panel) { - if (val) { - val = val.replace(/ PBMC/, ''); - } - - return val; - }, - - htoIndex: function(val, panel) { - if (Ext4.isNumeric(val)) { - //indexes are named D7XX. accept rows named '1', '12', etc. - var type = panel.down('#hashingType').getValue(); - if (type === 'CD298') { - val = parseInt(val); - if (val < 100) { - val = val + 700; - } - return 'D' + val; - } - else if (type === 'MultiSeq') { - val = parseInt(val); - - return 'MultiSeq-Idx-RP' + val; - } - else { - LDK.Utils.logError('Unknown or missing hashingType: ' + type); - } - } - else if (val) { - var type = panel.down('#hashingType').getValue(); - if (type === 'MultiSeq') { - val = String(val); - if (val.match(/^MS-[0-9]+$/i)) { - val = val.replace(/^MS(-)*/ig, 'MultiSeq-Idx-RP'); - } - - val = val.replace(/^MS[- ]Idx/ig, 'MultiSeq-Idx'); - val = val.replace(/^MultiSeq[- ]Idx[- ]RP/ig, 'MultiSeq-Idx-RP'); - - return val; - } - } - - return val; - }, - - citeSeqTenXBarcode: function(val, panel){ - if (!val){ - return; - } - - var barcodeSeries = panel.down('#citeseqBarcodeSeries').getValue(); - val = val.toUpperCase(); - var re = new RegExp('^' + barcodeSeries + '-', 'i'); - if (!val.match(re)) { - if (val.length > 3) { - //errorMsgs.push('Every row must have name, application and proper barcodes'); - } - else { - val = barcodeSeries + '-' + val; - } - } - - return val; - }, - - tenXBarcode: function(val, panel){ - if (!val){ - return; - } - - var barcodeSeries = panel.down('#barcodeSeries').getValue(); - val = val.toUpperCase(); - var re = new RegExp('^' + barcodeSeries + '-', 'i'); - if (!val.match(re)) { - if (val.length > 3) { - //errorMsgs.push('Every row must have name, application and proper barcodes'); - } - else { - val = barcodeSeries + '-' + val; - } - } - - return val; - }, - - hto: function(val, panel){ - if (Ext4.isNumeric(val)){ - var type = panel.down('#hashingType').getValue(); - if (type === 'CD298') { - return 'HTO-' + val; - } - else if (type === 'MultiSeq') { - return 'MS-' + val; - } - } - else if (val) { - //Normalize hyphen use - val = String(val); - val = val.replace(/^MS(-)*/, 'MS-'); - val = val.replace(/^HTO(-)*/, 'HTO-'); - } - - return val; - }, - - expt: function(val, panel){ - return val || panel.EXPERIMENT; - }, - - cells: function(val, panel){ - return val ? Ext4.data.Types.INTEGER.convert(val) : val; - }, - - pool: function(val, panel, row){ - var workbook = row.workbook || panel.EXPERIMENT; - //Note: convert values like 2B -> 2 - if (val && !Ext4.isNumeric(val)) { - val = val.replace(/[^0-9]+/, ''); - } - if (workbook && Ext4.isNumeric(val) && workbook !== val){ - return workbook + '-' + val; - } - - return val; - }, - - tetramer: function(val, panel, row){ - if (val) { - if (['Tet+', 'Tetramer+', 'Tetramer'].indexOf(val) > -1) { - row.population = null; - } - - row.population = row.population || val; - } - - return val; - }, - - population: function(val, panel, row){ - if (val && ['Tet+', 'Tetramer+', 'Tetramer'].indexOf(val) > -1) { - val = row.tetramer; - } - - return val; - }, - - sampleDate: function(val, panel){ - return val || panel.SAMPLE_DATE; - }, - - effector: function(val, panel, row){ - if (val && val.endsWith('PBMC')) { - var tmp = val.replace(/( )+PBMC$/,''); - row.animalId = tmp; - val = 'PBMC'; - } - return val || panel.EFFECTOR; - } - }, - - COLUMN_MAP: null, - - initComponent: function () { - this.COLUMN_MAP = {}; - Ext4.Array.forEach(this.COLUMNS, function(col){ - this.COLUMN_MAP[col.name.toLowerCase()] = col; - Ext4.Array.forEach(col.labels, function(alias){ - this.COLUMN_MAP[alias.toLowerCase()] = col; - }, this); - }, this); - - Ext4.apply(this, { - title: null, - border: false, - defaults: { - border: false - }, - items: this.getPanelItems() - }); - - this.callParent(arguments); - }, - - getPanelItems: function(){ - return [{ - layout: { - type: 'hbox' - }, - items: [{ - xtype: 'ldk-integerfield', - style: 'margin-right: 5px;', - fieldLabel: 'Current Folder/Workbook', - labelWidth: 200, - minValue: 1, - value: LABKEY.Security.currentContainer.type === 'workbook' ? LABKEY.Security.currentContainer.name : null, - emptyText: LABKEY.Security.currentContainer.type === 'workbook' ? null : 'Showing All', - listeners: { - afterRender: function(field){ - new Ext4.util.KeyNav(field.getEl(), { - enter : function(e){ - var btn = field.up('panel').down('#goButton'); - btn.handler(btn); - }, - scope : this - }); - } - } - },{ - xtype: 'button', - itemId: 'goButton', - scope: this, - text: 'Go', - handler: function(btn){ - var wb = btn.up('panel').down('ldk-integerfield').getValue(); - if (!wb){ - wb = ''; - } - - var container = LABKEY.Security.currentContainer.type === 'workbook' ? LABKEY.Security.currentContainer.parentPath + '/' + wb : LABKEY.Security.currentContainer.path + '/' + wb; - window.location = LABKEY.ActionURL.buildURL('tcrdb', 'poolImport', container); - } - },{ - xtype: 'button', - scope: this, - hidden: !LABKEY.Security.currentUser.canInsert, - text: 'Create Workbook', - handler: function(btn){ - Ext4.create('Laboratory.window.WorkbookCreationWindow', { - abortIfContainerIsWorkbook: false, - canAddToExistingExperiment: false, - controller: 'tcrdb', - action: 'poolImport', - title: 'Create Workbook' - }).show(); - } - }] - }, { - style: 'padding-top: 10px;', - html: 'This page is designed to help import TCR/10x data, including pooled samples. Each sample tends to create many libraries with many indexes/barcodes to track. Use the fields below to download the excel template and paste data to import.

' - },{ - layout: 'hbox', - items: [{ - xtype: 'button', - text: 'Download Template', - border: true, - scope: this, - href: LABKEY.ActionURL.getContextPath() + '/tcrdb/exampleData/ImportTemplate.xlsx' - },{ - xtype: 'button', - text: 'Download Example Import', - border: true, - scope: this, - href: LABKEY.ActionURL.getContextPath() + '/tcrdb/exampleData/ImportExample.xlsx' - }] - }, { - xtype: 'ldk-linkbutton', - text: 'Manage Allowable Values for Stims', - linkCls: 'labkey-text-link', - href: LABKEY.ActionURL.buildURL('query', 'executeQuery', Laboratory.Utils.getQueryContainerPath(), {schemaName: 'tcrdb', 'query.queryName': 'peptides'}), - style: 'margin-top: 10px;' - },{ - xtype: 'textfield', - style: 'margin-top: 20px;', - fieldLabel: 'Expt Number', - itemId: 'exptNum', - value: LABKEY.Security.currentContainer.type === 'workbook' ? LABKEY.Security.currentContainer.name : null - },{ - xtype: 'datefield', - fieldLabel: 'Sample Date', - itemId: 'sampleDate' - },{ - xtype: 'textfield', - fieldLabel: 'Effectors', - itemId: 'effector', - value: 'PBMC' - },{ - xtype: 'checkbox', - fieldLabel: 'Require HTO', - itemId: 'requireHashTag', - checked: true - },{ - xtype: 'checkbox', - fieldLabel: 'Require GEX Library', - itemId: 'requireGEX', - checked: false - },{ - xtype: 'checkbox', - fieldLabel: 'Require TCR Library', - itemId: 'requireTCR', - checked: false - },{ - xtype: 'checkbox', - fieldLabel: 'Require HTO Library', - itemId: 'requireHTO', - checked: false - },{ - xtype: 'checkbox', - fieldLabel: 'Require Cite-Seq Library', - itemId: 'requireCITE', - checked: false - },{ - xtype: 'checkbox', - fieldLabel: 'Require Library Concentrations', - itemId: 'requireConc', - checked: false - }, { - xtype: 'checkbox', - fieldLabel: 'Skip Readsets', - itemId: 'skipReadsets', - checked: true, - listeners: { - scope: this, - change: function(field, val) { - field.up('panel').down('#requireGEX').setValue(!val); - field.up('panel').down('#requireTCR').setValue(!val); - field.up('panel').down('#requireHTO').setValue(!val); - field.up('panel').down('#requireCITE').setValue(!val); - } - } - },{ - xtype: 'ldk-simplecombo', - fieldLabel: '10x GEX/TCR Barcode Series', - itemId: 'barcodeSeries', - forceSelection: true, - storeValues: ['SI-GA'], - value: 'SI-GA' - },{ - xtype: 'ldk-simplecombo', - fieldLabel: '10x Cite-Seq Barcode Series', - itemId: 'citeseqBarcodeSeries', - forceSelection: true, - storeValues: ['SI-NA'], - value: 'SI-NA' - },{ - xtype: 'ldk-simplecombo', - fieldLabel: 'Hashing Type', - itemId: 'hashingType', - forceSelection: true, - storeValues: ['CD298', 'MultiSeq'], - value: 'MultiSeq' - },{ - xtype: 'textarea', - fieldLabel: 'Paste Data Below', - labelAlign: 'top', - itemId: 'data', - width: 1000, - height: 300 - },{ - xtype: 'button', - text: 'Preview', - border: true, - scope: this, - handler: this.onPreview - },{ - itemId: 'previewArea', - style: 'margin-top: 20px;margin-bottom: 10px;', - autoEl: 'table', - cls: 'stripe hover' - }]; - }, - - onPreview: function(btn) { - var text = this.down('#data').getValue(); - if (!text) { - Ext4.Msg.alert('Error', 'Must provide the table of data'); - return; - } - - this.EXPERIMENT = this.down('#exptNum').getValue(); - this.SAMPLE_DATE = this.down('#sampleDate').getValue(); - if (this.SAMPLE_DATE) { - this.SAMPLE_DATE = Ext4.Date.format(this.SAMPLE_DATE, 'Y-m-d'); - } - this.EFFECTOR = this.down('#effector').getValue(); - - //this is a special case. if the first character is Tab, this indicates a blank field. Add a placeholder so it's not trimmed: - if (text .startsWith("\t")) { - text = 'Column1' + text; - } - text = Ext4.String.trim(text); - - var rows = LDK.Utils.CSVToArray(text, '\t'); - var colArray = this.parseHeader(rows.shift()); - var parsedRows = this.parseRows(colArray, rows); - var stimRows = []; - Ext4.Array.forEach(parsedRows, function(r){ - LDK.Assert.assertNotEmpty('Expected non-null workbook', r.workbook); - stimRows.push({ - animalId: r.animalId, - date: r.sampleDate, - stim: r.stim, - treatment: r.treatment || 'None', - tissue: r.tissue, - objectId: r.objectId, - population: r.population, - workbook: r.workbook - }); - }, this); - - Ext4.Msg.wait('Looking for matching stims'); - LABKEY.Ajax.request({ - url: LABKEY.ActionURL.buildURL('tcrdb', 'getMatchingStims', Laboratory.Utils.getQueryContainerPath(), null), - timeout: 99999, - method: 'POST', - jsonData: { - stimRows: stimRows - }, - scope: this, - success: LABKEY.Utils.getCallbackWrapper(function(results){ - Ext4.Msg.hide(); - - Ext4.Array.forEach(parsedRows, function(r){ - if (results.stimMap[r.objectId]){ - r.stimId = results.stimMap[r.objectId]; - } - - if (results.sortMap[r.objectId]){ - r.sortId = results.sortMap[r.objectId]; - } - }, this); - - var groupedRows = this.groupForImport(colArray, parsedRows); - if (!groupedRows){ - console.log('No rows after grouping'); - return; - } - - this.renderPreview(colArray, parsedRows, groupedRows); - - if (results.recordErrors) { - Ext4.Array.forEach(results.recordErrors, function(e){ - console.error(e); - }, this); - } - }, this), - failure: LDK.Utils.getErrorCallback() - }); - }, - - parseHeader: function(headerRow){ - var colArray = []; - var colNames = {}; - Ext4.Array.forEach(headerRow, function(headerText, idx){ - //replace common terms: - if (headerText.match(/ng\/ul/i) || headerText.match(/qubit/i)) { - headerText = headerText.replace(/( )+(\()*ng\/ul(\))*/i, ''); - headerText = headerText.replace(/( )+(\()*qubit(\))*/i, ''); - headerText = Ext4.String.trim(headerText); - if (!headerText.match(/Conc/i)) { - headerText = headerText + ' Conc'; - } - } - headerText = headerText.replace(/CiteSeq/i, 'Cite-Seq'); - headerText = headerText.replace(/Cite Seq/i, 'Cite-Seq'); - headerText = headerText.replace(/^MS /i, 'MultiSeq '); - headerText = headerText.replace(/Multi Seq/i, 'MultiSeq'); - headerText = headerText.replace(/Multi-Seq/i, 'MultiSeq'); - headerText = headerText.replace(/Library Index/i, 'Index'); - headerText = headerText.replace(/ RP#/i, ''); - headerText = headerText.replace(/:( )+10X Plate N Set A/i, ''); - headerText = headerText.replace(/:( )+10X Plate T Kit A/i, ''); - - headerText = headerText.replace(/5'[- ]*GEX/i, 'GEX'); - headerText = headerText.replace(/5[- ]GEX/i, 'GEX'); - headerText = Ext4.String.trim(headerText); - - var colData = this.COLUMN_MAP[headerText.toLowerCase()]; - if (colData){ - colNames[colData.name] = idx; - } - }, this); - - Ext4.Array.forEach(this.COLUMNS, function(colData, idx){ - if (this.IGNORED_COLUMNS.indexOf(colData.name) > -1) { - return; - } - - if (colData.alwaysShow || colData.allowBlank === false || colNames[colData.name]){ - colData = Ext4.apply({}, colData); - colData.dataIdx = colNames[colData.name]; - - colArray.push(colData); - } - },this); - - return colArray; - }, - - parseRows: function(colArray, rows){ - var lastValueByCol = new Array(colArray.length); - var ret = []; - - var doSplitCellsByPool = false; - Ext4.Array.forEach(rows, function(row, rowIdx){ - var data = { - objectId: LABKEY.Utils.generateUUID() - }; - - Ext4.Array.forEach(colArray, function(col, colIdx){ - var cell = Ext4.isDefined(col.dataIdx) ? row[col.dataIdx] : ''; - if (cell){ - if (col.transform && this.transforms[col.transform]){ - cell = this.transforms[col.transform](cell, this, data); - } - - data[col.name] = cell; - lastValueByCol[colIdx] = cell; - } - else if (col.allowRowSpan && lastValueByCol[colIdx]){ - data[col.name] = lastValueByCol[colIdx]; - } - else { - //allow transform even if value is null - if (col.transform && this.transforms[col.transform]){ - cell = this.transforms[col.transform](cell, this, data); - } - - data[col.name] = cell; - - if (!cell && col.name === 'cells' && lastValueByCol[colIdx]) { - doSplitCellsByPool = true; - } - } - }, this); - - ret.push(data); - }, this); - - //split cells across rows - if (doSplitCellsByPool) { - var cellCountMap = {}; - Ext4.Array.forEach(ret, function(data) { - if (data.plateId) { - cellCountMap[data.plateId] = cellCountMap[data.plateId] || []; - cellCountMap[data.plateId].push(data.cells); - } - }, this); - - Ext4.Array.forEach(Ext4.Object.getKeys(cellCountMap), function(plateId) { - var arr = cellCountMap[plateId]; - var size = arr.length; - arr = Ext4.Array.remove(arr, null); - arr = Ext4.Array.remove(arr, ''); - if (arr.length === 1) { - cellCountMap[plateId] = arr[0] / size; - } - else { - delete cellCountMap[plateId]; - } - }, this); - - Ext4.Array.forEach(ret, function(data) { - if (data.plateId && cellCountMap[data.plateId]) { - data.cells = cellCountMap[data.plateId]; - } - }, this); - } - - return ret; - }, - - groupForImport: function(colArray, parsedRows){ - var ret = { - stimRows: [], - sortRows: [], - cDNARows: [], - readsetRows: [] - }; - - var errorsMsgs = []; - - //stims: - var stimMap = {}; - var stimIdxs = {}; - var stimIdx = 0; - Ext4.Array.forEach(parsedRows, function(row){ - var key = this.getStimKey(row); - if (!stimMap[key]){ - var guid = LABKEY.Utils.generateUUID(); - stimIdx++; - - stimMap[key] = guid; - stimIdxs[key] = stimIdx; - LDK.Assert.assertNotEmpty('Expected non-null workbook', row.workbook); - ret.stimRows.push({ - rowId: row.stimId || null, - animalId: row.animalId, - date: row.sampleDate, - stim: row.stim, - effector: row.effector, - tissue: row.tissue, - treatment: row.treatment || 'None', - objectId: guid, - workbook: row.workbook - }); - } - - row.stim_num = row.stim_num || stimIdxs[key]; - }, this); - - //sorts: - var sortMap = {}; - Ext4.Array.forEach(parsedRows, function(row){ - LDK.Assert.assertNotEmpty('Expected non-null workbook', row.workbook); - var stimGUID = stimMap[this.getStimKey(row)]; - var key = this.getSortKey(row); - if (!sortMap[key]){ - var guid = LABKEY.Utils.generateUUID(); - sortMap[key] = guid; - ret.sortRows.push({ - rowId: row.sortId || null, - stimGUID: stimGUID, - population: row.population, - replicate: row.replicate, - cells: row.cells, - well: row.well || 'Pool', - hto: row.hto, - buffer: row.buffer, - objectId: guid, - workbook: row.workbook - }); - } - }, this); - - //cDNA/readsets: group by pool - var poolMap = {}; - Ext4.Array.forEach(parsedRows, function(row) { - poolMap[row.plateId] = poolMap[row.plateId] || []; - poolMap[row.plateId].push(row); - }, this); - - Ext4.Object.each(poolMap, function(poolName, rowArr){ - var readsetGUIDs = {}; - - var requireHTO = this.down('#requireHTO').getValue(); - var hashingType = this.down('#hashingType').getValue(); - var libraryType = null; - if (hashingType === 'CD298'){ - libraryType = 'CD298 Hashing'; - } - else if (hashingType === 'MultiSeq'){ - libraryType = 'MultiSeq'; - } - - var rs = this.processReadsetForGroup(poolName, rowArr, ret.readsetRows, 'hto', 'HTO', 'Cell Hashing', libraryType); - if (Ext4.isString(rs)) { - readsetGUIDs.hashingReadsetGUID = rs; - } - else if (requireHTO){ - errorsMsgs.push('Missing HTO library'); - errorsMsgs = errorsMsgs.concat(rs); - return false; - } - - var requireCITE = this.down('#requireCITE').getValue(); - var rs = this.processReadsetForGroup(poolName, rowArr, ret.readsetRows, 'citeseq', 'CITE', 'CITE-Seq', null); - if (Ext4.isString(rs)) { - readsetGUIDs.citeseqReadsetGUID = rs; - } - else if (requireCITE){ - errorsMsgs.push('Missing CITE-Seq library'); - errorsMsgs = errorsMsgs.concat(rs); - return false; - } - - var requireGEX = this.down('#requireGEX').getValue(); - rs = this.processReadsetForGroup(poolName, rowArr, ret.readsetRows, 'gex', 'GEX', 'RNA-seq, Single Cell', '10x 5\' GEX'); - if (Ext4.isString(rs)) { - readsetGUIDs.readsetGUID = rs; - } - else if (requireGEX){ - errorsMsgs.push('Missing GEX library'); - errorsMsgs = errorsMsgs.concat(rs); - return false; - } - - var requireTCR = this.down('#requireTCR').getValue(); - rs = this.processReadsetForGroup(poolName, rowArr, ret.readsetRows, 'tcr', 'TCR', 'RNA-seq, Single Cell', '10x 5\' VDJ (Rhesus A/B/G)'); - if (Ext4.isString(rs)) { - readsetGUIDs.enrichedReadsetGUID = rs; - } - else if (requireTCR){ - errorsMsgs.push('Missing TCR library'); - errorsMsgs = errorsMsgs.concat(rs); - return false; - } - - Ext4.Array.forEach(rowArr, function(row) { - var sortKey = this.getSortKey(row); - - LDK.Assert.assertNotEmpty('Expected non-null workbook', row.workbook); - var cDNA = Ext4.apply({ - sortGUID: sortMap[sortKey], - chemistry: null, - plateId: row.plateId, - well: row.well || 'Pool', - citeseqpanel: row.citeseqpanel, - workbook: row.workbook - }, readsetGUIDs); - - ret.cDNARows.push(cDNA); - }, this); - }, this); - - if (errorsMsgs.length) { - errorsMsgs = Ext4.unique(errorsMsgs); - Ext4.Msg.alert('Error', errorsMsgs.join('
')); - return null; - } - - return ret; - }, - - processReadsetForGroup: function(poolName, rowArr, readsetRows, prefix, type, application, librarytype){ - var idxValues = this.getUniqueValues(rowArr, prefix + '_library_index'); - var conc = this.getUniqueValues(rowArr, prefix + '_library_conc'); - var fragment = this.getUniqueValues(rowArr, prefix + '_library_fragment'); - var workbook = this.getUniqueValues(rowArr, 'workbook'); - if (workbook.length > 1) { - return ['Error', 'Pool ' + poolName + ' uses more workbook ' + workbook.join(';')]; - } - workbook = workbook.length === 1 ? workbook[0] : null; - - var subjectid = this.getUniqueValues(rowArr, 'animalId'); - subjectid = subjectid.length === 1 ? subjectid[0] : null; - - var requireConc = this.down('#requireConc').getValue(); - - if (idxValues.length === 1){ - if (requireConc && !conc[0]) { - return ['Pool ' + poolName + ': did not provide concentration for library: ' + type]; - } - - var guid = LABKEY.Utils.generateUUID(); - LDK.Assert.assertNotEmpty('Expected non-null workbook', workbook); - readsetRows.push({ - name: poolName + '-' + type, - barcode5: idxValues[0], - concentration: conc[0], - fragmentSize: fragment[0], - platform: 'ILLUMINA', - application: application, - librarytype: librarytype, - subjectid: subjectid, - sampleType: 'mRNA', - objectId: guid, - workbook: workbook - }); - - return guid; - } - else if (idxValues.length > 1) { - return ['Error', 'Pool ' + poolName + ' uses more than one ' + type + ' index']; - } - else if (idxValues.length === 0) { - var required = this.down('#require' + type).getValue(); - if (required) { - return ['Error', 'No index found for pool: ' + poolName + ', for library type: ' + type]; - } - } - }, - - getUniqueValues: function(rowArr, colName){ - var ret = []; - Ext4.Array.forEach(rowArr, function(row){ - if (row[colName]) - ret.push(row[colName]); - }, this); - - return Ext4.unique(ret); - }, - - renderPreview: function(colArray, parsedRows, groupedRows){ - var previewArea = this.down('#previewArea'); - previewArea.removeAll(); - - var columns = [{title: 'Row #'}]; - var colIdxs = []; - Ext4.Array.forEach(colArray, function(col, idx){ - if (col){ - columns.push({title: col.labels[0], className: 'dt-center'}); - colIdxs.push(idx); - } - }, this); - - var data = []; - var missingValues = false; - var requireHTO = this.down('#requireHTO').getValue() || (this.down('#requireHashTag') && this.down('#requireHashTag').getValue()); - Ext4.Array.forEach(parsedRows, function(row, rowIdx){ - var toAdd = [rowIdx + 1]; - Ext4.Array.forEach(colIdxs, function(colIdx){ - var colDef = colArray[colIdx]; - var propName = colDef.name; - - var allowBlank = colDef.allowBlank; - if (requireHTO && colDef.name === 'hto') { - allowBlank = false; - } - - if (allowBlank === false && Ext4.isEmpty(row[propName])){ - missingValues = true; - toAdd.push('MISSING'); - } - else { - toAdd.push(row[propName] || 'ND'); - } - - }, this); - - data.push(toAdd); - }, this); - - var id = '#' + previewArea.getId(); - if ( jQuery.fn.dataTable.isDataTable(id) ) { - jQuery(id).DataTable().destroy(); - } - - jQuery(id).DataTable({ - data: data, - pageLength: 500, - dom: 'rt<"bottom"BS><"clear">', - buttons: missingValues ? [] : [{ - text: 'Submit', - action: this.onSubmit, - rowData: { - colArray: colArray, - parsedRows: parsedRows, - groupedRows: groupedRows, - panel: this - } - }], - columns: columns - }); - - previewArea.doLayout(); - - if (missingValues){ - Ext4.Msg.alert('Error', 'One or more rows is missing data. Any required cells without values are marked MISSING'); - } - }, - - onSubmit: function(e, dt, node, config){ - Ext4.Msg.wait('Saving...'); - LABKEY.Ajax.request({ - url: LABKEY.ActionURL.buildURL('tcrdb', 'importTenx', Laboratory.Utils.getQueryContainerPath()), - method: 'POST', - jsonData: config.rowData.groupedRows, - scope: this, - success: function(){ - Ext4.Msg.hide(); - Ext4.Msg.alert('Success', 'Data Saved', function(){ - window.location = LABKEY.ActionURL.buildURL('query', 'executeQuery.view', Laboratory.Utils.getQueryContainerPath(), {'query.queryName': 'cdnas', schemaName: 'tcrdb', 'query.sort': '-created'}) - }, this); - }, - failure: LDK.Utils.getErrorCallback() - }); - }, - - getStimKey: function(data){ - return [data.stimId, data.animalId, data.stim, data.treatment, data.tissue, (Ext4.isDate(data.sampleDate) ? Ext4.Date.format(data.sampleDate, 'Y-m-d') : data.sampleDate)].join('|'); - }, - - getSortKey: function(data){ - return [this.getStimKey(data), data.sortId, data.population, data.hto].join('|'); - } -}); \ No newline at end of file diff --git a/tcrdb/resources/web/tcrdb/panel/StimPanel.js b/tcrdb/resources/web/tcrdb/panel/StimPanel.js deleted file mode 100644 index 62e1f2af9..000000000 --- a/tcrdb/resources/web/tcrdb/panel/StimPanel.js +++ /dev/null @@ -1,1515 +0,0 @@ -Ext4.define('TCRdb.panel.StimPanel', { - extend: 'Ext.panel.Panel', - alias: 'widget.tcrdb-stimpanel', - - initComponent: function(){ - Ext4.apply(this, { - title: null, - border: false, - defaults: { - border: false - }, - items: [{ - layout: { - type: 'hbox' - }, - items: [{ - xtype: 'ldk-integerfield', - style: 'margin-right: 5px;', - fieldLabel: 'Current Folder/Workbook', - labelWidth: 200, - minValue: 1, - value: LABKEY.Security.currentContainer.type === 'workbook' ? LABKEY.Security.currentContainer.name : null, - emptyText: LABKEY.Security.currentContainer.type === 'workbook' ? null : 'Showing All', - listeners: { - afterRender: function(field){ - new Ext4.util.KeyNav(field.getEl(), { - enter : function(e){ - var btn = field.up('panel').down('#goButton'); - btn.handler(btn); - }, - scope : this - }); - } - } - },{ - xtype: 'button', - itemId: 'goButton', - scope: this, - text: 'Go', - handler: function(btn){ - var wb = btn.up('panel').down('ldk-integerfield').getValue(); - if (!wb){ - wb = ''; - } - - var container = LABKEY.Security.currentContainer.type === 'workbook' ? LABKEY.Security.currentContainer.parentPath + '/' + wb : LABKEY.Security.currentContainer.path + '/' + wb; - window.location = LABKEY.ActionURL.buildURL('tcrdb', 'stimDashboard', container); - } - },{ - xtype: 'button', - scope: this, - hidden: !LABKEY.Security.currentUser.canInsert, - text: 'Create Workbook', - handler: function(btn){ - Ext4.create('Laboratory.window.WorkbookCreationWindow', { - abortIfContainerIsWorkbook: false, - canAddToExistingExperiment: false, - controller: 'tcrdb', - action: 'stimDashboard', - title: 'Create Workbook' - }).show(); - } - }] - },{ - style: 'padding-top: 10px;', - html: 'This page is designed to help manage samples for the TCR sequencing project. Where possible we try to carry sample information from to step to step; however, each step often generates new info we need to track, and sometimes samples and plates generated at different times are combined for later steps. The basic steps are:

' - }] - }); - - this.callParent(arguments); - - Ext4.Msg.wait('Loading...'); - this.loadData(); - }, - - getFolderSummaryConfig: function(){ - - }, - - loadData: function(){ - var multi = new LABKEY.MultiRequest(); - multi.add(LABKEY.Query.selectRows, { - schemaName: 'laboratory', - queryName: 'well_layout', - columns: 'well_96,addressbycolumn_96', - filterArray: [LABKEY.Filter.create('plate', 1)], - sort: 'addressbycolumn_96', - scope: this, - failure: LDK.Utils.getErrorCallback(), - success: function(results){ - this.wellNames96 = []; - - Ext4.Array.forEach(results.rows, function(r){ - this.wellNames96.push(r.well_96); - }, this); - } - }); - - multi.add(LABKEY.Query.selectRows, { - schemaName: 'sequenceanalysis', - queryName: 'barcodes', - sort: 'group_name,tag_name', - scope: this, - failure: LDK.Utils.getErrorCallback(), - success: function(results){ - this.barcodeMap = {}; - - Ext4.Array.forEach(results.rows, function(r){ - this.barcodeMap[r.group_name] = this.barcodeMap[r.group_name] || {}; - this.barcodeMap[r.group_name][r.tag_name] = r.sequence; - }, this); - } - }); - - multi.add(LABKEY.Query.selectRows, { - schemaName: 'tcrdb', - queryName: 'stims', - columns: 'rowid,tubeNum,animalId,effector,effectors,date,stim,treatment,costim,background,activated,comment,numSorts,status', - scope: this, - failure: LDK.Utils.getErrorCallback(), - success: function(results){ - this.stimRows = results.rows; - this.stimStats = { - totalStims: 0, - lackingSort: 0, - hasStatus: 0 - }; - - Ext4.Array.forEach(results.rows, function(r){ - this.stimStats.totalStims++; - if (!r.numSorts && !r.status){ - this.stimStats.lackingSort++; - } - - if (r.status){ - this.stimStats.hasStatus++; - } - }, this); - } - }); - - multi.add(LABKEY.Query.selectRows, { - schemaName: 'tcrdb', - queryName: 'sorts', - columns: 'rowid,stimId,stimId/animalId,stimId/effector,stimId/date,stimId/treatment,population,replicate,cells,plateId,well,well/addressByColumn,numLibraries,maxCellsForPlate,container', - scope: this, - failure: LDK.Utils.getErrorCallback(), - success: function(results){ - this.sortRows = results.rows; - this.sortStats = { - totalSorts: 0, - totalPlates: [], - lackingLibraries: 0, - bulkLackingLibraries: 0, - totalPlatesLackingLibraries: [], - totalBulkPlatesLackingLibraries: [] - }; - - Ext4.Array.forEach(results.rows, function(r){ - this.sortStats.totalSorts++; - if (!r.numLibraries){ - this.sortStats.lackingLibraries++; - - if (r.cells > 1) { - this.sortStats.bulkLackingLibraries++; - } - - if (r.plateId){ - this.sortStats.totalPlatesLackingLibraries.push(r.plateId); - - if (r.maxCellsForPlate > 1){ - this.sortStats.totalBulkPlatesLackingLibraries.push(r.plateId); - } - } - } - - if (r.plateId){ - this.sortStats.totalPlates.push(r.plateId); - } - }, this); - - this.sortStats.totalPlates = Ext4.unique(this.sortStats.totalPlates); - this.sortStats.totalPlatesLackingLibraries = Ext4.unique(this.sortStats.totalPlatesLackingLibraries); - this.sortStats.totalBulkPlatesLackingLibraries = Ext4.unique(this.sortStats.totalBulkPlatesLackingLibraries); - } - }); - - multi.add(LABKEY.Query.selectRows, { - schemaName: 'tcrdb', - queryName: 'cdnas', - columns: 'rowid,sortId,cells,plateId,well,well/addressByColumn,readsetId,readsetId/totalFiles,enrichedReadsetId,enrichedReadsetId/totalFiles,sortId/stimId,sortId/stimId/animalId,sortId/stimId/effector,sortId/stimId/date,sortId/stimId/treatment,sortId/population,sortId/replicate,sortId/cells,sortId/plateId,sortId/sortId/well,sortId/well/addressByColumn,sortId/stimId/stim', - sort: 'plateId,well/addressByColumn', - scope: this, - failure: LDK.Utils.getErrorCallback(), - success: function(results){ - this.libraryRows = results.rows; - this.libraryStats = { - totalLibraries: 0, - totalPlates: [], - lackingAnyReadset: 0, - totalPlatesLackingAnyReadset: [], - withReadset: 0, - totalPlatesWithReadset: [], - withTCRReadset: 0, - totalPlatesWithTCRReadset: [], - lackingBarcodes: 0 - - }; - - Ext4.Array.forEach(results.rows, function(r){ - this.libraryStats.totalLibraries++; - if (r.readsetId){ - this.libraryStats.withReadset++; - if (r.plateId){ - this.libraryStats.totalPlatesWithReadset.push(r.plateId); - } - } - - if (r.enrichedReadsetId) { - this.libraryStats.withTCRReadset++; - if (r.plateId){ - this.libraryStats.totalPlatesWithTCRReadset.push(r.plateId); - } - } - - if (!r.enrichedReadsetId && !r.readsetId) { - this.libraryStats.lackingAnyReadset++; - if (r.plateId){ - this.libraryStats.totalPlatesLackingAnyReadset.push(r.plateId); - } - } - - if (r.plateId){ - this.libraryStats.totalPlates.push(r.plateId); - } - }, this); - - this.libraryStats.totalPlates = Ext4.unique(this.libraryStats.totalPlates); - this.libraryStats.totalPlatesWithReadset = Ext4.unique(this.libraryStats.totalPlatesWithReadset); - this.libraryStats.totalPlatesWithTCRReadset = Ext4.unique(this.libraryStats.totalPlatesWithTCRReadset); - this.libraryStats.totalPlatesLackingAnyReadset = Ext4.unique(this.libraryStats.totalPlatesLackingAnyReadset); - } - }); - - multi.add(LABKEY.Query.selectRows, { - schemaName: 'sequenceanalysis', - queryName: 'sequence_readsets', - columns: 'rowid,name,application,totalFiles', - scope: this, - failure: LDK.Utils.getErrorCallback(), - success: function(results){ - this.readsetRows = results.rows; - this.readsetStats = { - totalReadsets: 0, - lackingData: 0, - dataImported: 0 - }; - - Ext4.Array.forEach(results.rows, function(r){ - this.readsetStats.totalReadsets++; - if (!r.totalFiles){ - this.readsetStats.lackingData++; - } - else { - this.readsetStats.dataImported++; - } - }, this); - } - }); - - multi.send(this.onDataLoad, this); - }, - - onDataLoad: function(){ - this.add(this.getItemConfig()); - - Ext4.Msg.hide(); - }, - - getItemConfig: function(){ - return { - defaults: { - border: true, - style: 'padding-bottom: 10px;', - bodyStyle: 'padding: 5px;' - }, - items: [{ - defaults: { - border: false - }, - title: 'Step 1: Stims/Blood Draws', - layout: { - type: 'table', - columns: 2, - tdAttrs: { style: 'padding-right: 10px;' } - }, - items: [{ - html: 'Total Stims:' - },{ - html: '' + this.stimStats.totalStims + '' - },{ - html: 'Lacking Sorts:' - },{ - html: '' + this.stimStats.lackingSort + '' - },{ - html: 'Non-passing Status:' - },{ - html: '' + this.stimStats.hasStatus + '' - },{ - xtype: 'ldk-linkbutton', - text: 'Import Stims', - href: 'javascript:void(0);', - linkCls: 'labkey-text-link', - handler: function(btn){ - if (LABKEY.Security.currentContainer.type === 'workbook'){ - Ext4.define('TCRdb.window.StimUploadWindow', { - extend: 'Ext.window.Window', - initComponent: function(){ - Ext4.apply(this, { - title: 'Import Stims', - items: [{ - xtype: 'labkey-exceluploadpanel', - bubbleEvents: ['uploadexception', 'uploadcomplete'], - itemId: 'theForm', - title: null, - buttons: null, - containerPath: Laboratory.Utils.getQueryContainerPath(), - schemaName: 'tcrdb', - queryName: 'stims', - populateTemplates: function(meta){ - Ext4.Msg.hide(); - var toAdd = []; - - toAdd.push({ - html: 'Use the button below to download an excel template for uploading stims.', - border: false, - style: 'padding-bottom: 10px;', - width: 700 - }); - - toAdd.push({ - xtype: 'ldk-integerfield', - itemId: 'templateRows', - fieldLabel: 'Total Stims', - labelWidth: 120, - value: 10 - }); - - toAdd.push({ - xtype: 'textfield', - itemId: 'treatment', - fieldLabel: 'Treatment', - labelWidth: 120, - value: 'TAPI-0' - }); - - toAdd.push({ - xtype: 'textfield', - itemId: 'coStim', - fieldLabel: 'Co-Stim', - labelWidth: 120, - value: 'CD28/CD49d' - }); - - toAdd.push({ - xtype: 'textfield', - itemId: 'effectors', - fieldLabel: 'Effector', - labelWidth: 120, - value: 'PBMC' - }); - - toAdd.push({ - xtype: 'ldk-numberfield', - itemId: 'numEffectors', - fieldLabel: '# Effectors', - labelWidth: 120, - value: 1000000 - }); - - toAdd.push({ - xtype: 'textfield', - itemId: 'apc', - fieldLabel: 'APCs', - labelWidth: 120, - value: 'PBMC' - }); - - toAdd.push({ - xtype: 'ldk-numberfield', - itemId: 'numAPC', - fieldLabel: '# APCs', - labelWidth: 120, - value: null - }); - - toAdd.push({ - xtype: 'button', - style: 'margin-bottom: 10px;', - text: 'Download Template', - border: true, - handler: this.generateExcelTemplate - }); - - this.down('#templateArea').add(toAdd); - }, - generateExcelTemplate: function(){ - var win = this.up('window'); - var numRows = win.down('#templateRows').getValue() || 1; - var effectors = win.down('#effectors').getValue(); - var numEffectors = win.down('#numEffectors').getValue(); - var apc = win.down('#apc').getValue(); - var numAPC = win.down('#numAPC').getValue(); - var treatment = win.down('#treatment').getValue(); - var coStim = win.down('#coStim').getValue(); - - var data = []; - data.push(['Tube #', 'Animal/Cell', 'Sample Date', 'Effectors', '# Effectors', 'APCs', '# APCs', 'Treatment', 'Co-stim', 'Peptide/Stim', 'Comment']); - for (var i=0;i' + this.sortStats.totalSorts + '' - }, { - xtype: 'ldk-linkbutton', - text: '(' + this.sortStats.totalPlates.length + ' plates)', - href: 'javascript:void(0);', - handler: this.getPlateCallback(this.sortStats.totalPlates) - }, { - html: 'Lacking cDNA Libraries (All):' - }, { - html: '' + this.sortStats.lackingLibraries + '' - }, { - xtype: 'ldk-linkbutton', - text: '(' + this.sortStats.totalPlatesLackingLibraries.length + ' plates)', - href: 'javascript:void(0);', - handler: this.getPlateCallback(this.sortStats.totalPlatesLackingLibraries) - }, { - html: 'Lacking cDNA Libraries (Bulk):' - }, { - html: '' + this.sortStats.bulkLackingLibraries + '' - }, { - xtype: 'ldk-linkbutton', - text: '(' + this.sortStats.totalBulkPlatesLackingLibraries.length + ' plates)', - href: 'javascript:void(0);', - handler: this.getPlateCallback(this.sortStats.totalBulkPlatesLackingLibraries) - }] - },{ - xtype: 'ldk-linkbutton', - text: 'Import Sort Data For Stims', - href: 'javascript:void(0);', - linkCls: 'labkey-text-link', - handler: function (btn) { - if (LABKEY.Security.currentContainer.type === 'workbook') { - Ext4.define('TCRdb.window.SortUploadWindow', { - extend: 'Ext.window.Window', - initComponent: function () { - Ext4.apply(this, { - title: 'Import Sorts for Stims', - items: [{ - xtype: 'labkey-exceluploadpanel', - bubbleEvents: ['uploadexception', 'uploadcomplete'], - itemId: 'theForm', - title: null, - buttons: null, - containerPath: Laboratory.Utils.getQueryContainerPath(), - schemaName: 'tcrdb', - queryName: 'sorts', - populateTemplates: function (meta) { - Ext4.Msg.hide(); - var toAdd = []; - - toAdd.push({ - html: 'Use the button below to download an excel template pre-populated with data from the sorts imported into this workbook.', - border: false, - style: 'padding-bottom: 10px;', - width: 700 - }); - - toAdd.push({ - xtype: 'textfield', - itemId: 'buffer', - fieldLabel: 'Sort Buffer', - labelWidth: 120, - value: 'Takara Buffer' - }); - - toAdd.push({ - xtype: 'checkbox', - itemId: 'skipWithData', - fieldLabel: 'Skip Stims With Sorts Imported', - labelWidth: 120, - helpPopup: 'If checked, stims with sort records already importd will be skipped', - checked: true - }); - - toAdd.push({ - xtype: 'ldk-integerfield', - itemId: 'templateRows', - fieldLabel: 'Rows Per Stim', - labelWidth: 120, - helpPopup: 'For each stim, the template will include this many rows', - value: 2 - }); - - toAdd.push({ - xtype: 'button', - style: 'margin-bottom: 10px;', - text: 'Download Template', - border: true, - handler: this.generateExcelTemplate - }); - - this.down('#templateArea').add(toAdd); - }, - generateExcelTemplate: function () { - var win = this.up('window'); - var rowsPer = win.down('#templateRows').getValue() || 1; - var skipWithData = win.down('#skipWithData').getValue(); - var buffer = win.down('#buffer').getValue(); - - var data = []; - data.push(['TubeNum', 'StimId', 'AnimalId', 'SampleDate', 'Peptide/Stim', 'Treatment', 'Buffer', 'Population', 'Replicate', 'Cells', 'PlateId', 'Well', 'Comment']); - Ext4.Array.forEach(win.stimRows, function (r) { - if (skipWithData && r.numSorts) { - return; - } - - for (var i = 0; i < rowsPer; i++) { - data.push([r.tubeNum, r.rowid, r.animalId, r.date, r.stim, r.treatment, buffer, null, null, null, null, null, null, null]); - } - }, this); - - LABKEY.Utils.convertToExcel({ - fileName: 'SortImport_' + Ext4.Date.format(new Date(), 'Y-m-d H_i_s') + '.xls', - sheets: [{ - name: 'Sorts', - data: data - }] - }); - }, - listeners: { - uploadcomplete: function (panel, response) { - Ext4.Msg.alert('Success', 'Upload Complete!', function (btn) { - this.up('window').close(); - location.reload(); - }, this); - } - } - }] - }); - - this.callParent(); - }, - buttons: [{ - text: 'Upload', - width: 50, - handler: function (btn) { - var form = btn.up('window').down('#theForm'); - form.formSubmit.call(form, btn); - }, - scope: this, - formBind: true - }, { - text: 'Close', - width: 50, - handler: function (btn) { - btn.up('window').close(); - } - }] - }); - - Ext4.create('TCRdb.window.SortUploadWindow', { - stimRows: this.up('tcrdb-stimpanel').stimRows - }).show(); - } - else { - Ext4.Msg.alert('Error', 'This is only allowed when in a specific workbook. Please enter the workbook into the box at the top of the page and hit \'Go\''); - } - } - }] - },{ - title: 'Step 3: cDNA Synthesis / Library Prep', - defaults: { - border: false - }, - items: [{ - layout: { - type: 'table', - columns: 3, - tdAttrs: { style: 'padding-right: 10px;' } - }, - defaults: { - border: false - }, - items: [{ - html: 'Total cDNA Libraries:' - },{ - html: '' + this.libraryStats.totalLibraries + '' - },{ - xtype: 'ldk-linkbutton', - text: '(' + this.libraryStats.totalPlates.length + ' plates)', - href: 'javascript:void(0);', - handler: this.getPlateCallback(this.libraryStats.totalPlates) - },{ - html: 'Lacking Any Readset:' - },{ - html: '' + this.libraryStats.lackingAnyReadset + '' - },{ - xtype: 'ldk-linkbutton', - text: '(' + this.libraryStats.totalPlatesLackingAnyReadset.length + ' plates)', - href: 'javascript:void(0);', - handler: this.getPlateCallback(this.libraryStats.totalPlatesLackingAnyReadset) - },{ - html: 'With Whole Transcriptome Readset:' - },{ - html: '' + this.libraryStats.withReadset + '' - },{ - xtype: 'ldk-linkbutton', - text: '(' + this.libraryStats.totalPlatesWithReadset.length + ' plates)', - href: 'javascript:void(0);', - handler: this.getPlateCallback(this.libraryStats.totalPlatesWithReadset) - },{ - html: 'With TCR Enriched Readset:' - },{ - html: '' + this.libraryStats.withTCRReadset + '' - },{ - xtype: 'ldk-linkbutton', - text: '(' + this.libraryStats.totalPlatesWithTCRReadset.length + ' plates)', - href: 'javascript:void(0);', - handler: this.getPlateCallback(this.libraryStats.totalPlatesWithTCRReadset) - }] - },{ - xtype: 'ldk-linkbutton', - text: 'Create cDNA Libraries From Sorts', - href: 'javascript:void(0);', - scope: this, - linkCls: 'labkey-text-link', - handler: function(){ - if (LABKEY.Security.currentContainer.type === 'workbook'){ - Ext4.define('TCRdb.window.cDNAUploadWindow', { - extend: 'Ext.window.Window', - initComponent: function(){ - Ext4.apply(this, { - title: 'Create cDNA Libraries From Sorts', - items: [{ - xtype: 'labkey-exceluploadpanel', - bubbleEvents: ['uploadexception', 'uploadcomplete'], - itemId: 'theForm', - title: null, - buttons: null, - containerPath: Laboratory.Utils.getQueryContainerPath(), - schemaName: 'tcrdb', - queryName: 'cdnas', - populateTemplates: function(meta){ - Ext4.Msg.hide(); - var toAdd = []; - - toAdd.push({ - html: 'Use the button below to download an excel template pre-populated with data from the selected plate IDs.', - border: false, - style: 'padding-bottom: 10px;', - width: 700 - }); - - toAdd.push({ - xtype: 'textfield', - itemId: 'destPlate', - fieldLabel: 'Destination Plate ID', - labelWidth: 160 - }); - - toAdd.push({ - xtype: 'ldk-simplecombo', - itemId: 'chemistry', - fieldLabel: 'Chemistry', - labelWidth: 160, - storeValues: ['SMART-Seq2', 'Takara SMART-Seq HT', '10x GEX/VDJ'], - value: 'SMART-Seq2' - }); - - toAdd.push({ - xtype: 'textarea', - itemId: 'plates', - fieldLabel: 'Source Plates', - labelWidth: 160, - //width: 200, - height: 100 - }); - - var win = this.up('window'); - toAdd.push({ - xtype: 'ldk-linkbutton', - itemId: 'showIds', - style: 'margin-left: 165px;', - text: 'Show Plate IDs', - scope: this, - handler: win.getPlateCallback(win.sortStats.totalPlatesLackingLibraries, 'plates'), - linkCls: 'labkey-text-link' - }); - - toAdd.push({ - xtype: 'button', - style: 'margin-bottom: 10px;', - text: 'Download Template', - border: true, - handler: this.generateExcelTemplate - }); - - toAdd.push({ - xtype: 'checkbox', - itemId: 'keepWell', - checked: true, - fieldLabel: 'Keep Original Well', - labelWidth: 160 - }); - - this.down('#templateArea').add(toAdd); - }, - - generateExcelTemplate: function(btn) { - var win = btn.up('window'); - var chemistry = win.down('#chemistry').getValue(); - var destPlate = win.down('#destPlate').getValue(); - var keepWell = win.down('#keepWell').getValue(); - - if (!destPlate) { - Ext4.Msg.alert('Error', 'Must provide destination plate IDs'); - return; - } - - var plates = Ext4.String.trim(btn.up('window').down('textarea').getValue()); - if (!plates) { - Ext4.Msg.alert('Error', 'Must provide source plate IDs'); - return; - } - - plates = plates.replace(/[\r\n]+/g, '\n'); - plates = plates.replace(/[\n]+/g, '\n'); - plates = Ext4.String.trim(plates); - if (plates){ - plates = plates.split('\n'); - } - - Ext4.Msg.wait('Loading...'); - LABKEY.Query.selectRows({ - containerPath: Laboratory.Utils.getQueryContainerPath(), - schemaName: 'tcrdb', - queryName: 'sorts', - sort: 'well/addressByColumn', - columns: 'rowid,stimId,stimId/animalId,stimId/effector,stimId/stim,stimId/date,stimId/treatment,population,replicate,cells,plateId,well,well/addressByColumn,numLibraries', - scope: win, - filterArray: [LABKEY.Filter.create('plateId', plates.join(';'), LABKEY.Filter.Types.IN)], - failure: LDK.Utils.getErrorCallback(), - success: function (results) { - Ext4.Msg.hide(); - - if (!results || !results.rows || !results.rows.length) { - Ext4.Msg.alert('Error', 'No sorts found for the selected plates'); - return; - } - - var data = []; - data.push(['Source Plate', 'Source Well', 'SortId', 'Plate Id', 'Well', 'Name', 'Chemistry', 'Comments']); - var wellIdx = 0; - var wellsUsed = {}; - var errors = []; - Ext4.Array.forEach(plates, function (sourcePlateId) { - Ext4.Array.forEach(results.rows, function (r) { - if (r.plateId !== sourcePlateId){ - return; - } - - var name = TCRdb.panel.StimPanel.getNameFromSort(r); - var targetWell = keepWell ? r.well : this.wellNames96[wellIdx]; - if (wellsUsed[targetWell]){ - errors.push('Duplicate well: ' + targetWell); - } - wellsUsed[targetWell] = true; - data.push([r.plateId, r.well, r.rowid, destPlate, targetWell, name, chemistry, null]); - wellIdx++; - }, this); - }, this); - - for (var i=0;i'), function(){ - LABKEY.Utils.convertToExcel({ - fileName: 'cDNAImport_' + Ext4.Date.format(new Date(), 'Y-m-d H_i_s') + '.xls', - sheets: [{ - name: 'cDNA Libraries', - data: data - }] - }); - }, this); - } - else { - LABKEY.Utils.convertToExcel({ - fileName: 'cDNAImport_' + Ext4.Date.format(new Date(), 'Y-m-d H_i_s') + '.xls', - sheets: [{ - name: 'cDNA Libraries', - data: data - }] - }); - } - } - }); - }, - listeners: { - uploadcomplete: function(panel, response){ - Ext4.Msg.alert('Success', 'Upload Complete!', function(btn){ - this.up('window').close(); - location.reload(); - }, this); - } - } - }] - }); - - this.callParent(); - }, - getWellSort: function(wellNames96){ - return function(a, b){ - var idx1 = wellNames96.indexOf(a[4]); - var idx2 = wellNames96.indexOf(b[4]); - - return idx1 - idx2; - } - }, - buttons: [{ - text: 'Upload', - width: 50, - handler: function(btn){ - var form = btn.up('window').down('#theForm'); - form.formSubmit.call(form, btn); - }, - scope: this, - formBind: true - },{ - text: 'Close', - width: 50, - handler: function(btn){ - btn.up('window').close(); - } - }] - }); - - Ext4.create('TCRdb.window.cDNAUploadWindow', { - wellNames96: this.wellNames96, - sortStats: this.sortStats, - getPlateCallback: this.getPlateCallback - }).show(); - } - else { - Ext4.Msg.alert('Error', 'This is only allowed when in a specific workbook. Please enter the workbook into the box at the top of the page and hit \'Go\''); - } - } - },{ - xtype: 'ldk-linkbutton', - text: 'Download Library Prep Template (box)', - linkCls: 'labkey-text-link', - href: 'https://ohsu.box.com/s/6kncrzm4ba9mxjput12v8u500tjlsip7', - linkTarget: '_blank' - },{ - xtype: 'ldk-linkbutton', - text: 'Download TCR Enrichment Library Prep Template (box)', - linkCls: 'labkey-text-link', - href: 'https://ohsu.box.com/s/js55a347q5mioqxwowe1dk3prkvn4d29', - linkTarget: '_blank' - },{ - xtype: 'ldk-linkbutton', - text: 'Download Names To Use In Protocols', - href: 'javascript:void(0);', - linkCls: 'labkey-text-link', - handler: function(){ - if (LABKEY.Security.currentContainer.type === 'workbook'){ - Ext4.Msg.wait('Loading...'); - LABKEY.Query.selectRows({ - containerPath: Laboratory.Utils.getQueryContainerPath(), - schemaName: 'tcrdb', - queryName: 'cdnas', - sort: 'well/addressByColumn', - scope: this, - failure: LDK.Utils.getErrorCallback(), - success: function(results){ - Ext4.Msg.hide(); - if (!results || !results.rows || !results.rows.length){ - Ext4.Msg.alert('Error', 'No cDNA libraries found'); - return; - } - - var rows = []; - rows.push(['Well', 'Name'].join('\t')); - Ext4.Array.forEach(results.rows, function(r){ - var name = TCRdb.panel.StimPanel.getNameFromCDNAs(r); - rows.push([r.well, name].join('\t')); - }, this); - - Ext4.create('Ext.window.Window', { - bodyStyle: 'padding: 5px;', - items: [{ - html: 'Please use the following as names for the sorts in the folder', - border: false, - style: 'padding-bottom: 10px;' - },{ - xtype: 'textarea', - width: 500, - height: 200, - value: rows.join('\n') - }], - buttons: [{ - text: 'Close', - handler: function(btn){ - btn.up('window').close(); - } - }] - }).show(); - } - }); - } - else { - Ext4.Msg.alert('Error', 'This is only allowed when in a specific workbook. Please enter the workbook into the box at the top of the page and hit \'Go\''); - } - } - }] - },{ - title: 'Step 4: Create Readsets / Template for Sequencing', - defaults: { - border: false - }, - items: [{ - layout: { - type: 'table', - columns: 2, - tdAttrs: { style: 'padding-right: 10px;' } - }, - defaults: { - border: false - }, - items: [{ - html: 'Total Readsets:' - },{ - html: '' + this.readsetStats.totalReadsets + '' - },{ - html: 'Data Not Imported:' - },{ - html: '' + this.readsetStats.lackingData + '' - }] - },{ - xtype: 'ldk-linkbutton', - text: 'Create Readsets From cDNA Libraries', - href: 'javascript:void(0);', - linkCls: 'labkey-text-link', - handler: function(){ - if (LABKEY.Security.currentContainer.type === 'workbook'){ - Ext4.define('TCRdb.window.ReadsetUploadWindow', { - extend: 'Ext.window.Window', - initComponent: function(){ - Ext4.apply(this, { - title: 'Create Readsets From cDNAs', - bodyStyle: 'padding: 5px;', - items: [{ - html: 'Use the button below to download an excel template pre-populated with data from the sorts imported into this workbook.', - border: false, - style: 'padding-bottom: 10px;', - width: 700 - },{ - xtype: 'textarea', - itemId: 'plateIds', - fieldLabel: 'Plate Id(s)', - labelWidth: 120, - height: 100 - },{ - xtype: 'ldk-linkbutton', - itemId: 'showIds', - text: 'Show Plate IDs', - style: 'margin-left: 125px;', - scope: this, - handler: this.getPlateCallback(this.libraryStats.totalPlatesLackingAnyReadset, 'plateIds'), - linkCls: 'labkey-text-link' - },{ - xtype: 'ldk-simplecombo', - itemId: 'application', - fieldLabel: 'Application', - labelWidth: 120, - storeValues: ['Whole Transcriptome RNA-Seq', 'TCR Enrichment', '10x GEX Only', '10x GEX/TCR'], - forceSelection: true, - multiSelect: true - },{ - xtype: 'labkey-combo', - itemId: 'chemistry', - fieldLabel: 'Chemistry', - labelWidth: 120, - store: { - type: 'labkey-store', - schemaName: 'sequenceanalysis', - queryName: 'sequence_chemistries', - autoLoad: true - }, - displayField: 'chemistry', - valueField: 'chemistry', - value: 'Illumina HiSeq3000', - forceSelection: true - },{ - xtype: 'checkbox', - itemId: 'includeImported', - fieldLabel: 'Include Those With Existing Readsets' - },{ - xtype: 'button', - style: 'margin-bottom: 10px;', - text: 'Download Template', - border: true, - handler: this.generateExcelTemplate - },{ - xtype: 'textarea', - height: 350, - width: 700, - itemId: 'template' - }] - }); - - this.callParent(); - }, - generateExcelTemplate: function(){ - var win = this.up('window'); - - //'Whole Transcriptome RNA-Seq', 'TCR Enrichment', 10x GEX Only, 10x GEX/TCR - var types = win.down('#application').getValue(); - var chemistry = win.down('#chemistry').getValue(); - var includeImported = win.down('#includeImported').getValue(); - var plates = Ext4.String.trim(win.down('textarea').getValue()); - if (!plates) { - Ext4.Msg.alert('Error', 'Must provide source plate IDs'); - return; - } - - plates = plates.replace(/[\r\n]+/g, '\n'); - plates = plates.replace(/[\n]+/g, '\n'); - plates = Ext4.String.trim(plates); - if (plates){ - plates = plates.split('\n'); - } - - if (!types || !types.length){ - Ext4.Msg.alert('Error', 'Must choose the application(s)'); - return; - } - - var applications = []; - Ext4.Array.forEach(types, function(type) { - switch (type) { - case 'Whole Transcriptome RNA-Seq': - applications.push('RNA-seq'); - break; - case 'TCR Enrichment': - applications.push('RNA-seq + Enrichment'); - break; - case '10x GEX Only': - applications.push('10x GEX'); - break; - case '10x GEX/TCR': - applications.push('10x GEX'); - applications.push('10x VDJ'); - } - }, this); - applications = Ext4.unique(applications); - - var data = []; - data.push(['LibraryId', 'PlateId', 'Source Well', 'Name', 'Subject Id', 'Sample Date', '5-Barcode', '3-Barcode', 'Sample Type', 'Sequencing Platform', 'Application', 'Chemistry', 'Library Type', 'Comments']); - Ext4.Array.forEach(win.libraryRows, function(r){ - Ext4.Array.forEach(applications, function(application){ - if (plates.indexOf(r.plateId) > -1) { - if (includeImported || (['RNA-seq + Enrichment', '10x VDJ'].indexOf(application) > -1 && !r.enrichedReadsetId) || (['RNA-seq', '10x GEX'].indexOf(application) > -1 && !r.readsetId)) { - var applicationValue = application; - if (application === 'RNA-seq' && r.cells === 1) { - applicationValue = 'RNA-seq, Single Cell'; - } - else if (['10x GEX', '10x VDJ'].indexOf(application) > -1){ - applicationValue = 'RNA-seq, Single Cell'; - } - - var libraryType = null; - switch (application){ - case 'RNA-seq': - libraryType = 'SMART-Seq2'; - break; - case '10x VDJ': - libraryType = '10x 5\' VDJ (Rhesus A/B/G)'; - break; - case '10x GEX': - libraryType = '10x 5\' GEX'; - } - - var name = TCRdb.panel.StimPanel.getNameFromCDNAs(r); - data.push([r.rowid, r.plateId, r.well, name, r['sortId/stimId/animalId'], r['sortId/stimId/date'], null, null, 'mRNA', 'ILLUMINA', applicationValue, chemistry, libraryType, null]); - } - } - }, this); - }, this); - - if (data.length === 1){ - Ext4.Msg.alert('Error', 'No matching rows found'); - return; - } - - LABKEY.Utils.convertToExcel({ - fileName: 'ReadsetImport_' + Ext4.Date.format(new Date(), 'Y-m-d H_i_s') + '.xls', - sheets: [{ - name: 'Readsets', - data: data - }] - }); - }, - buttons: [{ - text: 'Upload', - width: 50, - handler: function(btn){ - btn.up('window').processUpload(); - }, - scope: this, - formBind: true - },{ - text: 'Close', - width: 50, - handler: function(btn){ - btn.up('window').close(); - } - }], - processUpload: function(){ - var text = this.down('#template').getValue(); - if (!text){ - Ext4.Msg.alert('Error', 'No rows provided'); - return; - } - text = LDK.Utils.CSVToArray(Ext4.String.trim(text), '\t'); - - var header = text.shift(); - var headerToField = { - Name: 'name', - 'Subject Id': 'subjectid', - 'Sample Date': 'sampledate', - '5-Barcode': 'barcode5', - '3-Barcode': 'barcode3', - 'Sample Type': 'sampletype', - 'Sequencing Platform': 'platform', - 'Application': 'application', - 'Chemistry': 'chemsitry', - 'Library Type': 'libraryType', - 'Comments': 'comments', - 'LibraryId': 'libraryId' - - }; - - var readsetToInsert = []; - var cDNAsToUpdate = {}; - var errorMsgs = []; - - Ext4.Array.forEach(text, function (row, rowIdx) { - var r = {}; - for (var headerName in headerToField){ - var idx = header.indexOf(headerName); - if (idx !== -1 && row.length > idx){ - r[headerToField[headerName]] = row[idx]; - } - } - - var hasBarcodes; - switch (r.libraryType){ - case '10x 5\' GEX': - case '10x 5\' VDJ (Rhesus A/B/G)': - hasBarcodes = !!r.barcode5 && !r.barcode3; - if (!hasBarcodes) {errorMsgs.push('10x data must have the 5\' barcode but not 3\'')}; - break; - default: - hasBarcodes = !!r.barcode5 && !!r.barcode3; - } - - if (!r.name || !r.application || !hasBarcodes || !r.libraryId){ - errorMsgs.push('Every row must have name, application and proper barcodes'); - return; - } - - //TODO: set container to match sort - // if (row.libraryId && containerMap[row.libraryId]){ - // row.container = containerMap[row.libraryId]; - // } - // - // if (!row.container){ - // //TODO - // } - - if (['10x 5\' GEX', '10x 5\' VDJ (Rhesus A/B/G)'].indexOf(r.libraryType) > -1 ) { - r.barcode5 = r.barcode5.toUpperCase(); - if (!r.barcode5.match(/^SI-GA-/)) { - if (r.barcode5.length > 3) { - errorMsgs.push('Every row must have name, application and proper barcodes'); - } - else { - r.barcode5 = 'SI-GA-' + r.barcode5; - } - } - } - - readsetToInsert.push(r); - - cDNAsToUpdate[r.libraryId] = cDNAsToUpdate[r.libraryId] || {}; - cDNAsToUpdate[r.libraryId].container = row.container; - if ('rna-seq' === r.application.toLowerCase()){ - cDNAsToUpdate[r.libraryId].readsetIdx = rowIdx; - } - else if ('rna-seq, single cell' === r.application.toLowerCase() && r.libraryType === '10x 5\' GEX'){ - cDNAsToUpdate[r.libraryId].readsetIdx = rowIdx; - } - else if ('rna-seq + enrichment' === r.application.toLowerCase()){ - cDNAsToUpdate[r.libraryId].enrichedReadsetIdx = rowIdx; - } - else if ('rna-seq, single cell' === r.application.toLowerCase() && r.libraryType === '10x 5\' VDJ (Rhesus A/B/G)'){ - cDNAsToUpdate[r.libraryId].enrichedReadsetIdx = rowIdx; - } - else { - errorMsgs.push('Unknown application/libraryType: ' + r.application + ' / ' + r.libraryType); - } - }, this); - - if (errorMsgs.length){ - errorMsgs = Ext4.unique(errorMsgs); - Ext4.Msg.alert('Error', errorMsgs.join('
')); - return; - } - - Ext4.Msg.wait('Saving...'); - LABKEY.Query.insertRows({ - //containerPath: Laboratory.Utils.getQueryContainerPath(), - schemaName: 'sequenceanalysis', - queryName: 'sequence_readsets', - rows: readsetToInsert, - scope: this, - failure: LDK.Utils.getErrorCallback(), - success: function (results) { - var toUpdate = []; - for (var libraryId in cDNAsToUpdate){ - //TODO: add container - var r = {rowid: libraryId}; - if (Ext4.isDefined(cDNAsToUpdate[libraryId].readsetIdx)){ - r.readsetId = results.rows[cDNAsToUpdate[libraryId].readsetIdx].rowId - } - - if (Ext4.isDefined(cDNAsToUpdate[libraryId].enrichedReadsetIdx)){ - r.enrichedReadsetId = results.rows[cDNAsToUpdate[libraryId].enrichedReadsetIdx].rowId - } - - if (Ext4.isDefined(cDNAsToUpdate[libraryId].container)){ - r.container = cDNAsToUpdate[libraryId].container; - } - - if (r.readsetId || r.enrichedReadsetId){ - toUpdate.push(r); - } - } - - if (toUpdate.length){ - LABKEY.Query.updateRows({ - //containerPath: Laboratory.Utils.getQueryContainerPath(), - schemaName: 'tcrdb', - queryName: 'cdnas', - rows: toUpdate, - scope: this, - failure: LDK.Utils.getErrorCallback(), - success: function (results) { - Ext4.Msg.hide(); - - Ext4.Msg.alert('Success', 'Rows saved', function(){ - window.location.reload(); - }); - } - }); - } - else { - Ext4.Msg.hide(); - Ext4.Msg.alert('Error', 'There were no readsets to update'); - } - } - }); - } - }); - - Ext4.create('TCRdb.window.ReadsetUploadWindow', { - libraryRows: this.up('tcrdb-stimpanel').libraryRows, - libraryStats: this.up('tcrdb-stimpanel').libraryStats, - getPlateCallback: this.up('tcrdb-stimpanel').getPlateCallback - }).show(); - } - else { - Ext4.Msg.alert('Error', 'This is only allowed when in a specific workbook. Please enter the workbook into the box at the top of the page and hit \'Go\''); - } - } - },{ - xtype: 'ldk-linkbutton', - text: 'Download Blank MPSSR Template (box)', - linkCls: 'labkey-text-link', - href: 'https://ohsu.box.com/s/awhkmncp3gphs60inlu0mnts1yd22z25' - },{ - xtype: 'ldk-linkbutton', - text: 'Request Runs From MPSSR (iLABS)', - linkCls: 'labkey-text-link', - href: 'https://ohsu.corefacilities.org/account/pending/ohsu' - },{ - xtype: 'ldk-linkbutton', - text: 'MedGenome Information', - linkCls: 'labkey-text-link', - href: 'https://prime-seq.ohsu.edu/wiki/Internal/Bimber/page.view?name=tcrSequenceShipping' - },{ - xtype: 'ldk-linkbutton', - text: 'Shipment List', - linkCls: 'labkey-text-link', - href: LABKEY.ActionURL.buildURL('query', 'executeQuery', Laboratory.Utils.getQueryContainerPath(), {schemaName: 'lists', queryName: 'MedGenomeShipments'}) - }] - },{ - title: 'Plate Summary', - defaults: { - border: false - }, - items: [{ - xtype: 'ldk-linkbutton', - text: 'View Summary of Sorts By Plate (this workbook)', - linkCls: 'labkey-text-link', - hidden: LABKEY.Security.currentContainer.type != 'workbook', - href: LABKEY.ActionURL.buildURL('query', 'executeQuery', null, {schemaName: 'tcrdb', queryName: 'sortStatusByPlate', 'query.isComplete~eq': false}) - },{ - xtype: 'ldk-linkbutton', - text: 'View Summary of Sorts By Plate (entire folder)', - linkCls: 'labkey-text-link', - href: LABKEY.ActionURL.buildURL('query', 'executeQuery', Laboratory.Utils.getQueryContainerPath(), {schemaName: 'tcrdb', queryName: 'sortStatusByPlate', 'query.isComplete~eq': false}) - },{ - xtype: 'ldk-linkbutton', - text: 'View Summary of Sorts By Animal/Plate (this workbook)', - linkCls: 'labkey-text-link', - hidden: LABKEY.Security.currentContainer.type !== 'workbook', - href: LABKEY.ActionURL.buildURL('query', 'executeQuery', null, {schemaName: 'tcrdb', queryName: 'sortStatusByPlateAndSample'}) - },{ - xtype: 'ldk-linkbutton', - text: 'View Summary of Sorts By Animal/Plate (entire folder)', - linkCls: 'labkey-text-link', - href: LABKEY.ActionURL.buildURL('query', 'executeQuery', Laboratory.Utils.getQueryContainerPath(), {schemaName: 'tcrdb', queryName: 'sortStatusByPlateAndSample'}) - }] - }] - } - }, - - getPlateCallback: function(plateIds, fieldId){ - return function(f){ - var target; - if (fieldId){ - target = f.up('window').down('#' + fieldId); - } - - var items = []; - Ext4.Array.forEach(plateIds, function(id){ - var listener = target ? { - scope: this, - afterrender: function(panel){ - panel.mon(panel.getEl(), 'click', function(){ - target.setValue(target.getValue() + (target.getValue() ? '\n' : '') + id); - }, this); - } - } : null; - items.push({ - html: id, - bodyStyle: target ? 'text-decoration: underline;cursor: pointer;' : null, - border: false, - listeners: listener - }); - }, this); - - if (plateIds.length === 0){ - items.push({html: 'There are no plates in this folder', border: false}); - } - - Ext4.create('Ext.window.Window', { - modal: true, - title: 'Plate IDs', - maxHeight: '400', - autoScroll: true, - width: 300, - bodyStyle: 'padding: 5px;', - items: [{ - xtype: 'container', - items: items - }], - buttons: [{ - text: 'Close', - handler: function(btn){ - btn.up('window').close(); - } - }] - }).show(); - } - }, - - statics: { - getNameFromSort: function(r){ - return [ - r['plateId'], - r['well'], - r['stimId/animalId'], - r['stimId/stim'], - r['stimId/treatment'], - r.population + (r.replicate ? '_' + r.replicate : '') - ].join('_').replace(/ /g, '-'); - }, - - getNameFromCDNAs: function(r){ - return [ - //NOTE: preferentially retain the original sort plate name, in case of combined cDNA plates - r['sortId/plateId'] || r['plateId'], - r['well'], - r['sortId/stimId/animalId'], - r['sortId/stimId/stim'], - r['sortId/stimId/treatment'], - r['sortId/population'] + (r['sortId/cells'] === 1 ? '_Clone' : '') + (r['sortId/replicate'] ? '_' + r['sortId/replicate'] : '') - ].join('_').replace(/ /g, '-').replace(/\(/g, '').replace(/\)/g, '').replace(/\+/g, 'Pos'); - } - } -}); \ No newline at end of file diff --git a/tcrdb/resources/web/tcrdb/panel/cDNAImportPanel.js b/tcrdb/resources/web/tcrdb/panel/cDNAImportPanel.js deleted file mode 100644 index 28c8ef451..000000000 --- a/tcrdb/resources/web/tcrdb/panel/cDNAImportPanel.js +++ /dev/null @@ -1,378 +0,0 @@ -Ext4.define('TCRdb.panel.cDNAImportPanel', { - extend: 'TCRdb.panel.PoolImportPanel', - - IGNORED_COLUMNS: ['animalId', 'sampleDate', 'stimId', 'population', 'sortId', 'hto', 'cells', 'stim', 'effector', 'tissue', 'stim_num'], - - initComponent: function () { - this.COLUMN_MAP = {}; - Ext4.Array.forEach(this.COLUMNS, function (col) { - if (this.IGNORED_COLUMNS.indexOf(col.name) > -1) { - return; - } - - //Do not allow rowspan for this type of import - col.allowRowSpan = false; - - this.COLUMN_MAP[col.name.toLowerCase()] = col; - Ext4.Array.forEach(col.labels, function (alias) { - this.COLUMN_MAP[alias.toLowerCase()] = col; - }, this); - }, this); - - Ext4.apply(this, { - title: null, - border: false, - defaults: { - border: false - }, - items: this.getPanelItems() - }); - - this.callParent(arguments); - }, - - getPanelItems: function(){ - return [{ - layout: { - type: 'hbox' - }, - items: [{ - xtype: 'ldk-integerfield', - style: 'margin-right: 5px;', - fieldLabel: 'Current Folder/Workbook', - labelWidth: 200, - minValue: 1, - value: LABKEY.Security.currentContainer.type === 'workbook' ? LABKEY.Security.currentContainer.name : null, - emptyText: LABKEY.Security.currentContainer.type === 'workbook' ? null : 'Showing All', - listeners: { - afterRender: function (field) { - new Ext4.util.KeyNav(field.getEl(), { - enter: function (e) { - var btn = field.up('panel').down('#goButton'); - btn.handler(btn); - }, - scope: this - }); - } - } - }, { - xtype: 'button', - itemId: 'goButton', - scope: this, - text: 'Go', - handler: function (btn) { - var wb = btn.up('panel').down('ldk-integerfield').getValue(); - if (!wb) { - wb = ''; - } - - var container = LABKEY.Security.currentContainer.type === 'workbook' ? LABKEY.Security.currentContainer.parentPath + '/' + wb : LABKEY.Security.currentContainer.path + '/' + wb; - window.location = LABKEY.ActionURL.buildURL('tcrdb', 'poolImport', container); - } - }, { - xtype: 'button', - scope: this, - hidden: !LABKEY.Security.currentUser.canInsert, - text: 'Create Workbook', - handler: function (btn) { - Ext4.create('Laboratory.window.WorkbookCreationWindow', { - abortIfContainerIsWorkbook: false, - canAddToExistingExperiment: false, - controller: 'tcrdb', - action: 'poolImport', - title: 'Create Workbook' - }).show(); - } - }] - }, { - style: 'padding-top: 10px;', - html: 'This page is designed to help import the readset/index information for 10x libraries, after the stim/sort data has already been imported.

' - }, { - layout: 'hbox', - items: [{ - xtype: 'button', - text: 'Download Template', - border: true, - scope: this, - href: LABKEY.ActionURL.getContextPath() + '/tcrdb/exampleData/ImportReadsetTemplate.xlsx' - }] - }, { - xtype: 'textfield', - style: 'margin-top: 20px;', - fieldLabel: 'Expt Number', - itemId: 'exptNum', - value: LABKEY.Security.currentContainer.type === 'workbook' ? LABKEY.Security.currentContainer.name : null - }, { - xtype: 'checkbox', - fieldLabel: 'Require GEX Library', - itemId: 'requireGEX', - checked: true - }, { - xtype: 'checkbox', - fieldLabel: 'Require TCR Library', - itemId: 'requireTCR', - checked: true - }, { - xtype: 'checkbox', - fieldLabel: 'Require HTO Library', - itemId: 'requireHTO', - checked: true - },{ - xtype: 'checkbox', - fieldLabel: 'Require Cite-Seq Library', - itemId: 'requireCITE', - checked: false - }, { - xtype: 'checkbox', - fieldLabel: 'Require Library Concentrations', - itemId: 'requireConc', - checked: true - },{ - xtype: 'ldk-simplecombo', - fieldLabel: '10x GEX/TCR Barcode Series', - itemId: 'barcodeSeries', - forceSelection: true, - storeValues: ['SI-GA'], - value: 'SI-GA' - },{ - xtype: 'ldk-simplecombo', - fieldLabel: '10x Cite-Seq Barcode Series', - itemId: 'citeseqBarcodeSeries', - forceSelection: true, - storeValues: ['SI-NA'], - value: 'SI-NA' - },{ - xtype: 'ldk-simplecombo', - fieldLabel: 'Hashing Type', - itemId: 'hashingType', - forceSelection: true, - storeValues: ['CD298', 'MultiSeq'], - value: 'MultiSeq' - }, { - xtype: 'textarea', - fieldLabel: 'Paste Data Below', - labelAlign: 'top', - itemId: 'data', - width: 1000, - height: 300 - }, { - xtype: 'button', - text: 'Preview', - border: true, - scope: this, - handler: this.onPreview - }, { - style: 'margin-top: 20px;margin-bottom: 10px;', - itemId: 'previewArea', - autoEl: 'table', - cls: 'stripe hover' - }]; - }, - - onPreview: function (btn) { - var text = this.down('#data').getValue(); - if (!text) { - Ext4.Msg.alert('Error', 'Must provide the table of data'); - return; - } - - this.EXPERIMENT = this.down('#exptNum').getValue(); - - text = Ext4.String.trim(text); - - var rows = LDK.Utils.CSVToArray(text, '\t'); - var colArray = this.parseHeader(rows.shift()); - var parsedRows = this.parseRows(colArray, rows); - - var groupedRows = this.groupForImport(colArray, parsedRows); - if (!groupedRows) { - console.log('No rows after grouping'); - return; - } - - var workbooks = []; - var hadError = false; - Ext4.Array.forEach(parsedRows, function(row){ - if (!row.workbook) { - hadError = true; - } - else { - workbooks.push(row.workbook); - } - }, this); - - if (hadError) { - Ext4.Msg.alert('Error', 'One or more rows missing a workbook ID'); - return; - } - - Ext4.Msg.wait('Loading workbooks'); - LABKEY.Query.selectRows({ - containerPath: Laboratory.Utils.getQueryContainerPath(), - schemaName: 'core', - queryName: 'workbooks', - columns: 'Name,EntityId', - filterArray: [LABKEY.Filter.create('Name', Ext4.unique(workbooks).join(';'), LABKEY.Filter.Types.IN)], - scope: this, - success: function(results) { - Ext4.Msg.hide(); - - var workbookMap = {}; - Ext4.Array.forEach(results.rows, function(r){ - workbookMap[r.Name] = r.EntityId; - }, this); - - Ext4.Array.forEach(groupedRows.cDNARows, function(r){ - LDK.Assert.assertNotEmpty('Unable to find workbook in map: ' + r.workbook, workbookMap[r.workbook]); - r.container = workbookMap[r.workbook]; - }, this); - - Ext4.Array.forEach(groupedRows.readsetRows, function(r){ - LDK.Assert.assertNotEmpty('Unable to find workbook in map: ' + r.workbook, workbookMap[r.workbook]); - r.container = workbookMap[r.workbook]; - }, this); - - this.onWorkbookQueryLoad(colArray, parsedRows, groupedRows); - }, - failure: LDK.Utils.getErrorCallback() - }); - }, - - onWorkbookQueryLoad: function(colArray, parsedRows, groupedRows) { - var plateIDs = []; - var hadError = false; - Ext4.Array.forEach(parsedRows, function(row){ - if (!row.plateId) { - hadError = true; - } - else { - plateIDs.push(row.plateId); - } - }, this); - - if (hadError) { - Ext4.Msg.alert('Error', 'One or more rows missing plate ID'); - return; - } - - plateIDs = Ext4.unique(plateIDs); - - Ext4.Msg.wait('Looking for matching cDNA'); - LABKEY.Query.selectRows({ - containerPath: Laboratory.Utils.getQueryContainerPath(), - schemaName: 'tcrdb', - queryName: 'cdnas', - columns: 'rowid,plateid,readsetid,enrichedreadsetid,hashingreadsetid,citeseqreadsetid,sortid/population,sortid,sortid/stimid,citeseqpanel', - filterArray: [LABKEY.Filter.create('plateId', plateIDs.join(';'), LABKEY.Filter.Types.IN)], - scope: this, - success: function(results) { - Ext4.Msg.hide(); - - if (!results.rows || !results.rows.length) { - Ext4.Msg.alert('Error', 'No matching rows found'); - return; - } - - var plateToCDNAMap = {}; - Ext4.Array.forEach(results.rows, function(row) { - plateToCDNAMap[row.plateId] = plateToCDNAMap[row.plateId] || []; - plateToCDNAMap[row.plateId].push(row.rowid); - }, this); - - var missing = []; - Ext4.Array.forEach(plateIDs, function (r) { - if (!plateToCDNAMap[r]) { - missing.push(r); - } - }, this); - - if (missing.length) { - Ext4.Msg.alert('Error', 'No cDNA records found for plates: ' + missing.join(', ')); - return; - } - - Ext4.Array.forEach(groupedRows.cDNARows, function(r){ - r.rowIds = plateToCDNAMap[r.plateId]; - }, this); - - this.renderPreview(colArray, parsedRows, groupedRows); - }, - failure: LDK.Utils.getErrorCallback() - }); - }, - - onSubmit: function (e, dt, node, config) { - Ext4.Msg.wait('Saving...'); - - var data = config.rowData.groupedRows; - - LABKEY.Query.insertRows({ - containerPath: Laboratory.Utils.getQueryContainerPath(), - schemaName: 'sequenceanalysis', - queryName: 'sequence_readsets', - rows: data.readsetRows, - success: function(results){ - var readsetMap = {}; - Ext4.Array.forEach(results.rows, function(row){ - readsetMap[row.name] = row.rowId; - }, this); - - var toUpdate = []; - Ext4.Array.forEach(data.cDNARows, function(row){ - var baseRow = {}; - - var gexReadsetId = readsetMap[row.plateId + '-GEX']; - if (gexReadsetId) { - baseRow.readsetId = gexReadsetId; - } - - var tcrReadsetId = readsetMap[row.plateId + '-TCR']; - if (tcrReadsetId) { - baseRow.enrichedReadsetId = tcrReadsetId; - } - - var htoReadsetId = readsetMap[row.plateId + '-HTO']; - if (htoReadsetId) { - baseRow.hashingReadsetId = htoReadsetId; - } - - var citeseqReadsetId = readsetMap[row.plateId + '-CITE']; - if (citeseqReadsetId) { - baseRow.citeseqReadsetId = citeseqReadsetId; - } - - baseRow.container = row.container; - - if (row.rowIds) { - Ext4.Array.forEach(row.rowIds, function(r){ - var toAdd = Ext4.apply({ - rowId: r - }, baseRow); - - toUpdate.push(toAdd); - }, this); - } - }, this); - - if (toUpdate.length) { - LABKEY.Query.updateRows({ - containerPath: Laboratory.Utils.getQueryContainerPath(), - schemaName: 'tcrdb', - queryName: 'cdnas', - rows: toUpdate, - success: function (results) { - Ext4.Msg.hide(); - Ext4.Msg.alert('Success', 'Data Saved', function(){ - window.location = LABKEY.ActionURL.buildURL('query', 'executeQuery.view', Laboratory.Utils.getQueryContainerPath(), {'query.queryName': 'cdnas', schemaName: 'tcrdb', 'query.sort': '-created'}); - }, this); - }, - failure: LDK.Utils.getErrorCallback(), - scope: this - }); - } - }, - failure: LDK.Utils.getErrorCallback(), - scope: this - }); - } -}); \ No newline at end of file diff --git a/tcrdb/src/org/labkey/tcrdb/ImportHelper.java b/tcrdb/src/org/labkey/tcrdb/ImportHelper.java deleted file mode 100644 index 4ac075926..000000000 --- a/tcrdb/src/org/labkey/tcrdb/ImportHelper.java +++ /dev/null @@ -1,110 +0,0 @@ -package org.labkey.tcrdb; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.labkey.api.data.Container; -import org.labkey.api.data.ContainerManager; -import org.labkey.api.data.TableInfo; -import org.labkey.api.data.TableSelector; -import org.labkey.api.query.FieldKey; -import org.labkey.api.query.QueryService; -import org.labkey.api.query.UserSchema; -import org.labkey.api.security.User; -import org.labkey.api.security.UserManager; -import org.labkey.api.util.MemTracker; -import org.labkey.api.util.PageFlowUtil; - -import java.util.HashMap; -import java.util.Map; - -public class ImportHelper -{ - private Container _container; - private User _user; - private TableInfo _table; - private TableInfo _sortTable; - - private static final Logger _log = LogManager.getLogger(ImportHelper.class); - - private Map _userSchemaMap = new HashMap<>(); - - private ImportHelper(String containerId, int userId, String queryName) - { - String schemaName = "tcrdb"; - - _container = ContainerManager.getForId(containerId); - if (_container == null) - throw new IllegalArgumentException("Unknown container: " + containerId); - - _container = _container.isWorkbook() ? _container.getParent() : _container; - - _user = UserManager.getUser(userId); - if (_user == null) - throw new IllegalArgumentException("Unknown user: " + userId); - - UserSchema us = getUserSchema(schemaName); - if (us == null) - throw new IllegalArgumentException("Unknown schema: " + schemaName); - - _table = us.getTable(queryName); - if (_table == null) - throw new IllegalArgumentException("Unknown table: " + schemaName + "." + queryName); - - _sortTable = us.getTable(TCRdbSchema.TABLE_SORTS); - if (_sortTable == null) - throw new IllegalArgumentException("Unknown table: " + schemaName + "." + queryName); - - MemTracker.getInstance().put(this); - } - - public static ImportHelper create(String containerId, int userId, String queryName) - { - return new ImportHelper(containerId, userId, queryName); - } - - private UserSchema getUserSchema(String name) - { - if (_userSchemaMap.containsKey(name)) - return _userSchemaMap.get(name); - - UserSchema us = QueryService.get().getUserSchema(_user, _container, name); - _userSchemaMap.put(name, us); - - return us; - } - - public Map getInitialWells() - { - TableSelector ts = new TableSelector(_table, PageFlowUtil.set("plateId", "well", "rowid")); - final Map ret = new HashMap<>(); - ts.forEachResults(rs -> { - String key = (rs.getString(FieldKey.fromString("plateId")) + "<>" + rs.getString(FieldKey.fromString("well"))).toUpperCase(); - ret.put(key, rs.getInt(FieldKey.fromString("rowId"))); - }); - - return ret; - } - - private Map sortToContainer = null; - - public String getContainerForSort(int sortId) - { - if (sortToContainer == null) - { - sortToContainer = new HashMap<>(); - new TableSelector(_sortTable, PageFlowUtil.set("rowId", "container")).forEachResults(rs -> { - sortToContainer.put(rs.getInt(FieldKey.fromString("rowId")), rs.getString(FieldKey.fromString("container"))); - }); - } - - if (sortToContainer != null && sortToContainer.containsKey(sortId)) - { - return sortToContainer.get(sortId); - } - - String containerId = new TableSelector(_sortTable, PageFlowUtil.set("container")).getObject(sortId, String.class); - sortToContainer.put(sortId, containerId); - - return containerId; - } -} diff --git a/tcrdb/src/org/labkey/tcrdb/TCRdbBulkImportNavItem.java b/tcrdb/src/org/labkey/tcrdb/TCRdbBulkImportNavItem.java deleted file mode 100644 index 4203f5097..000000000 --- a/tcrdb/src/org/labkey/tcrdb/TCRdbBulkImportNavItem.java +++ /dev/null @@ -1,53 +0,0 @@ -package org.labkey.tcrdb; - -import org.labkey.api.data.Container; -import org.labkey.api.laboratory.AbstractImportingNavItem; -import org.labkey.api.laboratory.DataProvider; -import org.labkey.api.laboratory.LaboratoryService; -import org.labkey.api.module.ModuleLoader; -import org.labkey.api.query.DetailsURL; -import org.labkey.api.security.User; -import org.labkey.api.view.ActionURL; - -public class TCRdbBulkImportNavItem extends AbstractImportingNavItem -{ - public static final String NAME = "TCR/10x Import"; - - private String _url; - - public TCRdbBulkImportNavItem(DataProvider provider, String label, LaboratoryService.NavItemCategory itemType, String reportCategory, String url) - { - super(provider, NAME, label, itemType, (reportCategory == null ? "TCRdb" : reportCategory)); - _url = url; - } - - @Override - public ActionURL getImportUrl(Container c, User u) - { - return DetailsURL.fromString(_url).getActionURL(); - } - - @Override - public ActionURL getSearchUrl(Container c, User u) - { - return null; - } - - @Override - public ActionURL getBrowseUrl(Container c, User u) - { - return null; - } - - @Override - public boolean isImportIntoWorkbooks(Container c, User u) - { - return true; - } - - @Override - public boolean getDefaultVisibility(Container c, User u) - { - return getTargetContainer(c).getActiveModules().contains(ModuleLoader.getInstance().getModule(TCRdbModule.NAME)); - } -} diff --git a/tcrdb/src/org/labkey/tcrdb/TCRdbController.java b/tcrdb/src/org/labkey/tcrdb/TCRdbController.java index 59b0acc6b..7647b1d63 100644 --- a/tcrdb/src/org/labkey/tcrdb/TCRdbController.java +++ b/tcrdb/src/org/labkey/tcrdb/TCRdbController.java @@ -22,32 +22,19 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.jetbrains.annotations.NotNull; -import org.json.JSONArray; -import org.labkey.api.action.ApiSimpleResponse; -import org.labkey.api.action.ApiUsageException; import org.labkey.api.action.ConfirmAction; import org.labkey.api.action.ExportAction; -import org.labkey.api.action.MutatingApiAction; -import org.labkey.api.action.ReadOnlyApiAction; -import org.labkey.api.action.SimpleApiJsonForm; import org.labkey.api.action.SimpleViewAction; import org.labkey.api.action.SpringActionController; -import org.labkey.api.collections.CaseInsensitiveHashMap; import org.labkey.api.data.ColumnInfo; import org.labkey.api.data.CompareType; import org.labkey.api.data.Container; -import org.labkey.api.data.ContainerManager; -import org.labkey.api.data.ContainerType; -import org.labkey.api.data.DbScope; -import org.labkey.api.data.Selector; import org.labkey.api.data.SimpleFilter; -import org.labkey.api.data.StopIteratingException; import org.labkey.api.data.TableInfo; import org.labkey.api.data.TableSelector; import org.labkey.api.exp.api.ExpData; import org.labkey.api.exp.api.ExperimentService; import org.labkey.api.pipeline.PipelineJobException; -import org.labkey.api.query.BatchValidationException; import org.labkey.api.query.FieldKey; import org.labkey.api.query.QueryAction; import org.labkey.api.query.QueryService; @@ -56,7 +43,6 @@ import org.labkey.api.security.IgnoresTermsOfUse; import org.labkey.api.security.RequiresPermission; import org.labkey.api.security.permissions.AdminPermission; -import org.labkey.api.security.permissions.InsertPermission; import org.labkey.api.security.permissions.ReadPermission; import org.labkey.api.sequenceanalysis.RefNtSequenceModel; import org.labkey.api.sequenceanalysis.SequenceAnalysisService; @@ -78,7 +64,6 @@ import java.io.File; import java.io.FileInputStream; import java.io.StringWriter; -import java.sql.SQLException; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Arrays; @@ -136,44 +121,38 @@ public ModelAndView getView(ExportAlignmentsForm form, BindException errors) thr TableInfo ti = us.getTable("data"); final Map> VDJMap = new HashMap<>(); - List rowIds = new ArrayList<>(); - rowIds.addAll(Arrays.asList(form.getAssayRowIds())); + List rowIds = new ArrayList<>(Arrays.asList(form.getAssayRowIds())); TableSelector ts = new TableSelector(ti, new SimpleFilter(FieldKey.fromString("rowid"), rowIds, CompareType.IN), null); final StringWriter writer = new StringWriter(); - ts.forEach(new Selector.ForEachBlock() - { - @Override - public void exec(AssayRecord r) throws SQLException, StopIteratingException + ts.forEach(AssayRecord.class, r -> { + if (r.getVdjFile() == null) { - if (r.getVdjFile() == null) - { - writer.write("ERROR: Row lacks VDJCA file: " + r.getRowId() + "\n"); - return; - } - - ExpData d = ExperimentService.get().getExpData(r.getVdjFile()); - if (d == null) - { - writer.write("ERROR: Unable to find VDJCA file for row: " + r.getRowId() + ", ExpData: " + r.getVdjFile() + "\n"); - return; - } + writer.write("ERROR: Row lacks VDJCA file: " + r.getRowId() + "\n"); + return; + } - if (!d.getFile().exists()) - { - writer.write("ERROR: Unable to find VDJCA file for row: " + r.getRowId() + ", file does not exist: " + d.getFile().getPath() + "\n"); - return; - } + ExpData d = ExperimentService.get().getExpData(r.getVdjFile()); + if (d == null) + { + writer.write("ERROR: Unable to find VDJCA file for row: " + r.getRowId() + ", ExpData: " + r.getVdjFile() + "\n"); + return; + } - if (!VDJMap.containsKey(d.getFile())) - { - VDJMap.put(d.getFile(), new ArrayList<>()); - } + if (!d.getFile().exists()) + { + writer.write("ERROR: Unable to find VDJCA file for row: " + r.getRowId() + ", file does not exist: " + d.getFile().getPath() + "\n"); + return; + } - VDJMap.get(d.getFile()).add(r); + if (!VDJMap.containsKey(d.getFile())) + { + VDJMap.put(d.getFile(), new ArrayList<>()); } - }, AssayRecord.class); + + VDJMap.get(d.getFile()).add(r); + }); if (VDJMap.isEmpty()) { @@ -973,239 +952,6 @@ public void setQueryName(String queryName) } } - @RequiresPermission(InsertPermission.class) - public static class ImportTenXAction extends MutatingApiAction - { - @Override - public Object execute(SimpleApiJsonForm form, BindException errors) throws Exception - { - List> stimRows = parseRows(form, "stimRows", getContainer()); - List> sortRows = parseRows(form, "sortRows", getContainer()); - List> readsetRows = parseRows(form, "readsetRows", getContainer()); - List> cDNARows = parseRows(form, "cDNARows", getContainer()); - - UserSchema tcrdb = QueryService.get().getUserSchema(getUser(), getContainer(), TCRdbSchema.NAME); - UserSchema sequenceAnalysis = QueryService.get().getUserSchema(getUser(), getContainer(), "sequenceanalysis"); - - try (DbScope.Transaction transaction = DbScope.getLabKeyScope().ensureTransaction()) - { - BatchValidationException bve = new BatchValidationException(); - - Map stimMap = new HashMap<>(); - final List> stimRowsToInsert = new ArrayList<>(); - stimRows.forEach(r -> { - if (r.get("objectId") == null) - { - throw new ApiUsageException("Missing objectId for stim row"); - } - - if (r.get("rowId") != null && StringUtils.trimToNull(r.get("rowId").toString()) != null) - { - stimMap.put((String) r.get("objectId"), (Integer) r.get("rowId")); - } - else - { - stimRowsToInsert.add(r); - } - }); - - - List> insertedStimRows = tcrdb.getTable(TCRdbSchema.TABLE_STIMS, null).getUpdateService().insertRows(getUser(), getContainer(), stimRowsToInsert, bve, null, new HashMap<>()); - if (bve.hasErrors()) - { - throw bve; - } - - insertedStimRows.forEach(r -> { - if (r.get("rowId") == null) - { - throw new ApiUsageException("Missing rowId for inserted stim row"); - } - - stimMap.put((String) r.get("objectId"), (Integer) r.get("rowId")); - }); - - Map sortMap = new HashMap<>(); - final List> sortRowsToInsert = new ArrayList<>(); - sortRows.forEach(r -> { - if (stimMap.get(r.get("stimGUID")) == null) - { - throw new ApiUsageException("Unable to find stimId for row"); - } - - if (r.get("rowId") != null && StringUtils.trimToNull(r.get("rowId").toString()) != null) - { - sortMap.put((String) r.get("objectId"), (Integer) r.get("rowId")); - } - else - { - r.put("stimId", stimMap.get(r.get("stimGUID"))); - sortRowsToInsert.add(r); - } - }); - - sortRows = tcrdb.getTable(TCRdbSchema.TABLE_SORTS, null).getUpdateService().insertRows(getUser(), getContainer(), sortRowsToInsert, bve, null, new HashMap<>()); - if (bve.hasErrors()) - { - throw bve; - } - - sortRows.forEach(r -> { - if (r.get("objectId") == null) - { - throw new ApiUsageException("Missing objectId for sort row"); - } - - sortMap.put((String) r.get("objectId"), (Integer) r.get("rowId")); - }); - - readsetRows = sequenceAnalysis.getTable("sequence_readsets", null).getUpdateService().insertRows(getUser(), getContainer(), readsetRows, bve, null, new HashMap<>()); - if (bve.hasErrors()) - { - throw bve; - } - - Map readsetMap = new HashMap<>(); - readsetRows.forEach(r -> { - if (r.get("objectId") == null) - { - throw new ApiUsageException("Missing objectId for readset row"); - } - - readsetMap.put((String)r.get("objectId"), (Integer)r.get("rowId")); - }); - - cDNARows.forEach(r -> { - if (sortMap.get(r.get("sortGUID")) == null) - { - throw new ApiUsageException("Unable to find sortId for row"); - } - r.put("sortId", sortMap.get((String)r.get("sortGUID"))); - }); - cDNARows.forEach(r -> r.put("readsetId", readsetMap.get((String)r.get("readsetGUID")))); - cDNARows.forEach(r -> r.put("hashingReadsetId", readsetMap.get((String)r.get("hashingReadsetGUID")))); - cDNARows.forEach(r -> r.put("enrichedReadsetId", readsetMap.get((String)r.get("enrichedReadsetGUID")))); - cDNARows.forEach(r -> r.put("citeseqReadsetId", readsetMap.get((String)r.get("citeseqReadsetGUID")))); - tcrdb.getTable(TCRdbSchema.TABLE_CDNAS, null).getUpdateService().insertRows(getUser(), getContainer(), cDNARows, bve, null, new HashMap<>()); - if (bve.hasErrors()) - { - throw bve; - } - - transaction.commit(); - } - - return new ApiSimpleResponse("success", true); - } - } - - private static List> parseRows(SimpleApiJsonForm form, String propName, Container container) throws ApiUsageException - { - if (!form.getJsonObject().containsKey(propName)) - { - throw new ApiUsageException("Missing property: " + propName); - } - - JSONArray arr = form.getJsonObject().getJSONArray(propName); - - List> ret = new ArrayList<>(); - Arrays.stream(arr.toJSONObjectArray()).forEach(m -> { - Map map = new CaseInsensitiveHashMap<>(); - map.putAll(m); - - if (map.containsKey("workbook")) - { - Container parent = container.getContainerFor(ContainerType.DataType.folderManagement); - Container workbook = ContainerManager.getForPath(parent.getPath() + "/" + map.get("workbook").toString()); - if (workbook == null) - { - throw new IllegalArgumentException("Unable to identify matching workbook for: " + map.get("workbook")); - } - - map.put("container", workbook == null ? null : workbook.getId()); - } - ret.add(map); - }); - - return ret; - } - - @RequiresPermission(ReadPermission.class) - public static class GetMatchingStimsAction extends ReadOnlyApiAction - { - final List FIELDS = Arrays.asList("animalId", "date", "stim", "treatment", "tissue"); - - @Override - public Object execute(SimpleApiJsonForm form, BindException errors) throws Exception - { - ApiSimpleResponse resp = new ApiSimpleResponse(); - - List> stimRows = parseRows(form, "stimRows", getContainer()); - - UserSchema us = QueryService.get().getUserSchema(getUser(), getContainer(), TCRdbSchema.NAME); - if (us == null) - { - throw new ApiUsageException("Unable to find schema: " + TCRdbSchema.NAME); - } - - TableInfo ti = us.getTable(TCRdbSchema.TABLE_STIMS, null); - TableInfo tiSort = us.getTable(TCRdbSchema.TABLE_SORTS, null); - - List retErrors = new ArrayList<>(); - Map stimRowMap = new HashMap<>(); - Map sortRowMap = new HashMap<>(); - stimRows.forEach(r -> { - List keys = new ArrayList<>(); - SimpleFilter filter = new SimpleFilter(); - FIELDS.forEach(f -> { - if (r.get(f) != null) - { - filter.addCondition(FieldKey.fromString(f), r.get(f), ("date".equals(f) ? CompareType.DATE_EQUAL : CompareType.EQUAL)); - keys.add(r.get(f)); - } - }); - - if (!filter.isEmpty()) - { - TableSelector ts = new TableSelector(ti, PageFlowUtil.set("rowId"), filter, null); - long count = ts.getRowCount(); - if (count == 1) - { - int rowId = ts.getObject(Integer.class); - stimRowMap.put(r.get("objectId"), rowId); - - if (r.get("population") != null) - { - SimpleFilter sortFilter = new SimpleFilter(FieldKey.fromString("stimId"), rowId); - sortFilter.addCondition(FieldKey.fromString("population"), r.get("population")); - TableSelector tsSort = new TableSelector(tiSort, PageFlowUtil.set("rowId"), sortFilter, null); - long countSort = tsSort.getRowCount(); - if (countSort == 1) - { - int sortRowId = tsSort.getObject(Integer.class); - sortRowMap.put(r.get("objectId"), sortRowId); - } - else if (countSort > 1) - { - retErrors.add("More than one matching sort found: " + StringUtils.join(keys, "|") + "|" + r.get("population")); - } - } - } - else if (count > 1 && filter.getClauses().size() == FIELDS.size()) - { - retErrors.add("More than one matching stim found: " + StringUtils.join(keys, "|")); - } - } - }); - - resp.put("stimMap", stimRowMap); - resp.put("sortMap", sortRowMap); - resp.put("recordErrors", retErrors); - - return resp; - } - } - @RequiresPermission(AdminPermission.class) public class CreateGenomeFromMixcrAction extends ConfirmAction { @@ -1247,7 +993,7 @@ public void validateCommand(CreateGenomeFromMixcrForm form, Errors errors) @Override public URLHelper getSuccessURL(CreateGenomeFromMixcrForm form) { - return QueryService.get().urlFor(getUser(), getContainer(), QueryAction.executeQuery, TCRdbSchema.NAME, TCRdbSchema.TABLE_LIBRARIES); + return QueryService.get().urlFor(getUser(), getContainer(), QueryAction.executeQuery, TCRdbSchema.NAME, TCRdbSchema.TABLE_MIXCR_LIBRARIES); } } diff --git a/tcrdb/src/org/labkey/tcrdb/TCRdbImportNavItem.java b/tcrdb/src/org/labkey/tcrdb/TCRdbImportNavItem.java deleted file mode 100644 index dc74a4fbe..000000000 --- a/tcrdb/src/org/labkey/tcrdb/TCRdbImportNavItem.java +++ /dev/null @@ -1,50 +0,0 @@ -package org.labkey.tcrdb; - -import org.labkey.api.data.Container; -import org.labkey.api.laboratory.AbstractImportingNavItem; -import org.labkey.api.laboratory.DataProvider; -import org.labkey.api.laboratory.LaboratoryService; -import org.labkey.api.module.ModuleLoader; -import org.labkey.api.query.DetailsURL; -import org.labkey.api.security.User; -import org.labkey.api.view.ActionURL; - -public class TCRdbImportNavItem extends AbstractImportingNavItem -{ - public static final String NAME = "TCR Sorts/Stims"; - - public TCRdbImportNavItem(DataProvider provider, String label, LaboratoryService.NavItemCategory itemType, String reportCategory) - { - super(provider, NAME, label, itemType, (reportCategory == null ? "TCRdb" : reportCategory)); - } - - @Override - public ActionURL getImportUrl(Container c, User u) - { - return DetailsURL.fromString("tcrdb/stimDashboard.view").getActionURL(); - } - - @Override - public ActionURL getSearchUrl(Container c, User u) - { - return null; - } - - @Override - public ActionURL getBrowseUrl(Container c, User u) - { - return DetailsURL.fromString("tcrdb/stimDashboard.view").getActionURL(); - } - - @Override - public boolean isImportIntoWorkbooks(Container c, User u) - { - return true; - } - - @Override - public boolean getDefaultVisibility(Container c, User u) - { - return getTargetContainer(c).getActiveModules().contains(ModuleLoader.getInstance().getModule(TCRdbModule.NAME)); - } -} diff --git a/tcrdb/src/org/labkey/tcrdb/TCRdbManager.java b/tcrdb/src/org/labkey/tcrdb/TCRdbManager.java index 79b3e06ac..efc318e7e 100644 --- a/tcrdb/src/org/labkey/tcrdb/TCRdbManager.java +++ b/tcrdb/src/org/labkey/tcrdb/TCRdbManager.java @@ -69,7 +69,7 @@ public static TCRdbManager get() public void createGenomeFromMixcrDb(int mixcrRowId, User u, Container c) throws Exception { - MixcrLibrary lib = new TableSelector(TCRdbSchema.getInstance().getSchema().getTable(TCRdbSchema.TABLE_LIBRARIES)).getObject(mixcrRowId, MixcrLibrary.class); + MixcrLibrary lib = new TableSelector(TCRdbSchema.getInstance().getSchema().getTable(TCRdbSchema.TABLE_MIXCR_LIBRARIES)).getObject(mixcrRowId, MixcrLibrary.class); if (lib == null) { throw new IllegalArgumentException("Unable to find MiXCR library: " + mixcrRowId); @@ -308,7 +308,7 @@ public void onCreate(Container c, User u, Logger log, int genomeId) { if (_mixcrId != null) { - TableInfo ti = QueryService.get().getUserSchema(u, c, TCRdbSchema.NAME).getTable(TCRdbSchema.TABLE_LIBRARIES); + TableInfo ti = QueryService.get().getUserSchema(u, c, TCRdbSchema.NAME).getTable(TCRdbSchema.TABLE_MIXCR_LIBRARIES); List> rows = new ArrayList<>(); List> oldKeys = new ArrayList<>(); Map row = new CaseInsensitiveHashMap<>(); diff --git a/tcrdb/src/org/labkey/tcrdb/TCRdbModule.java b/tcrdb/src/org/labkey/tcrdb/TCRdbModule.java index 0b1796b56..6d5f552f6 100644 --- a/tcrdb/src/org/labkey/tcrdb/TCRdbModule.java +++ b/tcrdb/src/org/labkey/tcrdb/TCRdbModule.java @@ -27,13 +27,8 @@ import org.labkey.api.module.ModuleContext; import org.labkey.api.sequenceanalysis.SequenceAnalysisService; import org.labkey.api.sequenceanalysis.pipeline.SequencePipelineService; -import org.labkey.tcrdb.pipeline.CellRangerCellHashingHandler; -import org.labkey.tcrdb.pipeline.CellRangerSeuratHandler; import org.labkey.tcrdb.pipeline.CellRangerVDJCellHashingHandler; -import org.labkey.tcrdb.pipeline.CellRangerVDJWrapper; import org.labkey.tcrdb.pipeline.MiXCRAnalysis; -import org.labkey.tcrdb.pipeline.SeuratCellHashingHandler; -import org.labkey.tcrdb.pipeline.SeuratCiteSeqHandler; import java.util.Collection; import java.util.Collections; @@ -51,7 +46,7 @@ public String getName() @Override public Double getSchemaVersion() { - return 15.51; + return 15.52; } @Override @@ -74,17 +69,13 @@ protected void doStartupAfterSpringConfig(ModuleContext moduleContext) LaboratoryService.get().registerDataProvider(new TCRdbProvider(this)); SequenceAnalysisService.get().registerDataProvider(new TCRdbProvider(this)); - LaboratoryService.get().registerTableCustomizer(this, TCRdbTableCustomizer.class, "sequenceanalysis", "sequence_readsets"); - LaboratoryService.get().registerTableCustomizer(this, TCRdbTableCustomizer.class, "sequenceanalysis", "sequence_analyses"); - LaboratoryService.get().registerTableCustomizer(this, TCRdbTableCustomizer.class, TCRdbSchema.NAME, TCRdbSchema.TABLE_STIMS); - LaboratoryService.get().registerTableCustomizer(this, TCRdbTableCustomizer.class, TCRdbSchema.NAME, TCRdbSchema.TABLE_SORTS); - LaboratoryService.get().registerTableCustomizer(this, TCRdbTableCustomizer.class, TCRdbSchema.NAME, TCRdbSchema.TABLE_CDNAS); + LaboratoryService.get().registerTableCustomizer(this, TCRdbTableCustomizer.class, TCRdbSchema.SEQUENCE_ANALYSIS, "sequence_readsets"); + LaboratoryService.get().registerTableCustomizer(this, TCRdbTableCustomizer.class, TCRdbSchema.SEQUENCE_ANALYSIS, "sequence_analyses"); LaboratoryService.get().registerTableCustomizer(this, TCRdbTableCustomizer.class, TCRdbSchema.NAME, TCRdbSchema.TABLE_CLONES); + LaboratoryService.get().registerTableCustomizer(this, TCRdbTableCustomizer.class, TCRdbSchema.SINGLE_CELL, TCRdbSchema.TABLE_CDNAS); - LDKService.get().registerQueryButton(new ChangeStatusButton(), "tcrdb", "stims"); + LDKService.get().registerQueryButton(new ChangeStatusButton(), TCRdbSchema.SINGLE_CELL, "samples"); - LDKService.get().registerQueryButton(new ShowBulkEditButton(this, TCRdbSchema.NAME, TCRdbSchema.TABLE_CDNAS), TCRdbSchema.NAME, TCRdbSchema.TABLE_CDNAS); - LDKService.get().registerQueryButton(new ShowBulkEditButton(this, TCRdbSchema.NAME, TCRdbSchema.TABLE_SORTS), TCRdbSchema.NAME, TCRdbSchema.TABLE_SORTS); LDKService.get().registerQueryButton(new ShowBulkEditButton(this, TCRdbSchema.NAME, TCRdbSchema.TABLE_CLONES), TCRdbSchema.NAME, TCRdbSchema.TABLE_CLONES); //register resources @@ -118,13 +109,7 @@ public PipelineStartup() else { SequencePipelineService.get().registerPipelineStep(new MiXCRAnalysis.Provider()); - SequencePipelineService.get().registerPipelineStep(new CellRangerVDJWrapper.VDJProvider()); - - SequenceAnalysisService.get().registerFileHandler(new CellRangerCellHashingHandler()); SequenceAnalysisService.get().registerFileHandler(new CellRangerVDJCellHashingHandler()); - SequenceAnalysisService.get().registerFileHandler(new SeuratCellHashingHandler()); - SequenceAnalysisService.get().registerFileHandler(new SeuratCiteSeqHandler()); - SequenceAnalysisService.get().registerFileHandler(new CellRangerSeuratHandler()); _hasRegistered = true; } diff --git a/tcrdb/src/org/labkey/tcrdb/TCRdbProvider.java b/tcrdb/src/org/labkey/tcrdb/TCRdbProvider.java index 1e3f0d31c..0359e45a7 100644 --- a/tcrdb/src/org/labkey/tcrdb/TCRdbProvider.java +++ b/tcrdb/src/org/labkey/tcrdb/TCRdbProvider.java @@ -3,20 +3,13 @@ import org.json.JSONObject; import org.labkey.api.data.Container; import org.labkey.api.data.ContainerManager; -import org.labkey.api.laboratory.DetailsUrlWithoutLabelNavItem; import org.labkey.api.laboratory.LaboratoryService; import org.labkey.api.laboratory.NavItem; import org.labkey.api.laboratory.QueryCountNavItem; import org.labkey.api.laboratory.QueryImportNavItem; -import org.labkey.api.laboratory.QueryTabbedReportItem; -import org.labkey.api.laboratory.SimpleSettingsItem; import org.labkey.api.laboratory.SummaryNavItem; -import org.labkey.api.laboratory.TabbedReportItem; import org.labkey.api.ldk.table.QueryCache; import org.labkey.api.module.Module; -import org.labkey.api.module.ModuleLoader; -import org.labkey.api.query.DetailsURL; -import org.labkey.api.query.FieldKey; import org.labkey.api.security.User; import org.labkey.api.sequenceanalysis.AbstractSequenceDataProvider; import org.labkey.api.view.ActionURL; @@ -60,15 +53,9 @@ public ActionURL getInstructionsUrl(Container c, User u) } @Override - public List getMiscItems(Container c, User u) + public List getSubjectIdSummary(Container c, User u, String subjectId) { - List items = new ArrayList<>(); - if (c.getActiveModules().contains(ModuleLoader.getInstance().getModule(TCRdbModule.class))) - { - items.add(new DetailsUrlWithoutLabelNavItem(this, "Export 10x Library Information", DetailsURL.fromString("tcrdb/libraryExport.view"), LaboratoryService.NavItemCategory.misc, NAME)); - } - - return items; + return Collections.emptyList(); } @Override @@ -81,27 +68,7 @@ public List getDataNavItems(Container c, User u) return Collections.emptyList(); } - TCRdbImportNavItem item = new TCRdbImportNavItem(this, "TCR Stims/Sorts (SMART-seq)", LaboratoryService.NavItemCategory.data, NAME); - item.setQueryCache(cache); - items.add(item); - - TCRdbBulkImportNavItem item2 = new TCRdbBulkImportNavItem(this, "TCR/10x Import 1: Stims/cDNA", LaboratoryService.NavItemCategory.data, NAME, "tcrdb/poolImport.view"); - item2.setQueryCache(cache); - items.add(item2); - - TCRdbBulkImportNavItem item3 = new TCRdbBulkImportNavItem(this, "TCR/10x Import 2: Libraries/Readsets", LaboratoryService.NavItemCategory.data, NAME, "tcrdb/cDNAImport.view"); - item3.setQueryCache(cache); - items.add(item3); - items.add(new QueryImportNavItem(this, TCRdbSchema.NAME, TCRdbSchema.TABLE_CLONES, "TCR Clones", LaboratoryService.NavItemCategory.data, NAME, cache)); - items.add(new QueryImportNavItem(this, TCRdbSchema.NAME, TCRdbSchema.TABLE_CDNAS, "TCR cDNA Libraries", LaboratoryService.NavItemCategory.data, NAME, cache){ - @Override - public ActionURL getImportUrl(Container c, User u) - { - return null; - } - }); - return Collections.unmodifiableList(items); } @@ -117,11 +84,7 @@ public List getSettingsItems(Container c, User u) List items = new ArrayList<>(); if (ContainerManager.getSharedContainer().equals(c)) { - items.add(new QueryImportNavItem(this, ContainerManager.getSharedContainer(), TCRdbSchema.NAME, TCRdbSchema.TABLE_LIBRARIES, LaboratoryService.NavItemCategory.settings, "MiXCR Libraries", NAME)); - } - else - { - items.add(new SimpleSettingsItem(this, TCRdbSchema.NAME, "peptides", NAME, "Peptides/Stims")); + items.add(new QueryImportNavItem(this, ContainerManager.getSharedContainer(), TCRdbSchema.NAME, TCRdbSchema.TABLE_MIXCR_LIBRARIES, LaboratoryService.NavItemCategory.settings, "MiXCR Libraries", NAME)); } return items; @@ -148,56 +111,6 @@ public Module getOwningModule() @Override public List getSummary(Container c, User u) { - List items = new ArrayList<>(); - - items.add(new QueryCountNavItem(this, TCRdbSchema.NAME, "stims", LaboratoryService.NavItemCategory.data, LaboratoryService.NavItemCategory.data.name(), "TCR Stims")); - items.add(new QueryCountNavItem(this, TCRdbSchema.NAME, "sorts", LaboratoryService.NavItemCategory.data, LaboratoryService.NavItemCategory.data.name(), "TCR Sorts")); - items.add(new QueryCountNavItem(this, TCRdbSchema.NAME, "cdnas", LaboratoryService.NavItemCategory.data, LaboratoryService.NavItemCategory.data.name(), "TCR cDNA Libraries")); - items.add(new QueryCountNavItem(this, TCRdbSchema.NAME, "clones", LaboratoryService.NavItemCategory.data, LaboratoryService.NavItemCategory.data.name(), "TCR Clones")); - - return Collections.unmodifiableList(items); - } - - @Override - public List getSubjectIdSummary(Container c, User u, String subjectId) - { - return Collections.emptyList(); - } - - @Override - public List getTabbedReportItems(Container c, User u) - { - if (!c.getActiveModules().contains(getOwningModule())) - { - return Collections.emptyList(); - } - - List items = new ArrayList<>(); - - NavItem owner = getDataNavItems(c, u).get(0); - String category = NAME; - QueryCache cache = new QueryCache(); - - TabbedReportItem stims = new QueryTabbedReportItem(cache, this, TCRdbSchema.NAME, TCRdbSchema.TABLE_STIMS, "TCR Stims/Blood Draws", category); - stims.setOwnerKey(owner.getPropertyManagerKey()); - items.add(stims); - - TabbedReportItem sorts = new QueryTabbedReportItem(cache, this, TCRdbSchema.NAME, TCRdbSchema.TABLE_SORTS, "TCR Sorts", category); - sorts.setSubjectIdFieldKey(FieldKey.fromString("stimId/animalId")); - sorts.setSampleDateFieldKey(FieldKey.fromString("stimId/date")); - sorts.setAllProjectsFieldKey(FieldKey.fromString("stimId/allProjectsPivot")); - sorts.setOverlappingProjectsFieldKey(FieldKey.fromString("stimId/overlappingProjectsPivot")); - sorts.setOwnerKey(owner.getPropertyManagerKey()); - items.add(sorts); - - TabbedReportItem cdnas = new QueryTabbedReportItem(cache, this, TCRdbSchema.NAME, TCRdbSchema.TABLE_CDNAS, "TCR cDNA Libraries", category); - cdnas.setSubjectIdFieldKey(FieldKey.fromString("sortId/stimId/animalId")); - cdnas.setSampleDateFieldKey(FieldKey.fromString("sortId/stimId/date")); - cdnas.setAllProjectsFieldKey(FieldKey.fromString("sortId/stimId/allProjectsPivot")); - cdnas.setOverlappingProjectsFieldKey(FieldKey.fromString("sortId/stimId/overlappingProjectsPivot")); - cdnas.setOwnerKey(owner.getPropertyManagerKey()); - items.add(cdnas); - - return items; + return Collections.singletonList(new QueryCountNavItem(this, TCRdbSchema.NAME, TCRdbSchema.TABLE_CLONES, LaboratoryService.NavItemCategory.data, LaboratoryService.NavItemCategory.data.name(), "TCR Clones")); } } diff --git a/tcrdb/src/org/labkey/tcrdb/TCRdbSchema.java b/tcrdb/src/org/labkey/tcrdb/TCRdbSchema.java index 0336e338c..32a0ca98f 100644 --- a/tcrdb/src/org/labkey/tcrdb/TCRdbSchema.java +++ b/tcrdb/src/org/labkey/tcrdb/TCRdbSchema.java @@ -24,16 +24,13 @@ public class TCRdbSchema { private static final TCRdbSchema _instance = new TCRdbSchema(); public static final String NAME = "tcrdb"; - public static final String SEQUENCE_ANALYSIS = "sequenceanalysis"; - public static final String TABLE_LIBRARIES = "mixcr_libraries"; - public static final String TABLE_SORTS = "sorts"; - public static final String TABLE_STIMS = "stims"; - public static final String TABLE_CDNAS = "cdnas"; + public static final String TABLE_MIXCR_LIBRARIES = "mixcr_libraries"; public static final String TABLE_CLONES = "clones"; - public static final String TABLE_CITE_SEQ_ANTIBODIES = "citeseq_antibodies"; - public static final String TABLE_CITE_SEQ_PANELS = "citeseq_panels"; - public static final String TABLE_PROCESSING = "plate_processing"; + + public static final String SEQUENCE_ANALYSIS = "sequenceanalysis"; + public static final String SINGLE_CELL = "singlecell"; + public static final String TABLE_CDNAS = "cdna_libraries"; public static TCRdbSchema getInstance() { diff --git a/tcrdb/src/org/labkey/tcrdb/TCRdbTableCustomizer.java b/tcrdb/src/org/labkey/tcrdb/TCRdbTableCustomizer.java index 9c7059a83..cf1a8e14e 100644 --- a/tcrdb/src/org/labkey/tcrdb/TCRdbTableCustomizer.java +++ b/tcrdb/src/org/labkey/tcrdb/TCRdbTableCustomizer.java @@ -19,9 +19,7 @@ import org.labkey.api.query.DetailsURL; import org.labkey.api.query.ExprColumn; import org.labkey.api.query.FieldKey; -import org.labkey.api.query.QueryForeignKey; import org.labkey.api.query.QueryService; -import org.labkey.api.query.UserSchema; import java.util.Arrays; import java.util.List; @@ -34,27 +32,19 @@ public void customize(TableInfo table) if (table instanceof AbstractTableInfo) { AbstractTableInfo ti = (AbstractTableInfo) table; - if (matches(ti, "sequenceanalysis", "sequence_analyses")) + if (matches(ti, TCRdbSchema.SEQUENCE_ANALYSIS, "sequence_analyses")) { addAssayFieldsToAnalyses(ti); } - else if (matches(ti, "sequenceanalysis", "sequence_readsets")) + else if (matches(ti, TCRdbSchema.SEQUENCE_ANALYSIS, "sequence_readsets")) { customizeReadsets(ti); } - else if (matches(ti, "tcrdb", "stims")) - { - customizeStims(ti); - } - else if (matches(ti, "tcrdb", "sorts")) - { - customizeSorts(ti); - } - else if (matches(ti, "tcrdb", "cdnas")) + else if (matches(ti, TCRdbSchema.SINGLE_CELL, TCRdbSchema.TABLE_CDNAS)) { customizeCdnas(ti); } - else if (matches(ti, "tcrdb", "clones")) + else if (matches(ti, TCRdbSchema.NAME, TCRdbSchema.TABLE_CLONES)) { customizeClones(ti); } @@ -67,130 +57,12 @@ else if (ti instanceof AssayResultTable) private void customizeCdnas(AbstractTableInfo ti) { - String name = "hasReadsetWithData"; - if (ti.getColumn(name) == null) - { - SQLFragment sql = new SQLFragment("CASE " + - " WHEN (select count(*) as expr FROM sequenceanalysis.sequence_readsets r JOIN sequenceanalysis.readdata d ON (r.rowid = d.readset) WHERE r.rowid = " + ExprColumn.STR_TABLE_ALIAS + ".readsetId) > 0 THEN " + ti.getSqlDialect().getBooleanTRUE() + - " WHEN (select count(*) as expr FROM sequenceanalysis.sequence_readsets r JOIN sequenceanalysis.readdata d ON (r.rowid = d.readset) WHERE r.rowid = " + ExprColumn.STR_TABLE_ALIAS + ".enrichedReadsetId) > 0 THEN " + ti.getSqlDialect().getBooleanTRUE() + - " ELSE " + ti.getSqlDialect().getBooleanFALSE() + " END"); - - ExprColumn newCol = new ExprColumn(ti, name, sql, JdbcType.BOOLEAN, ti.getColumn("readsetId"), ti.getColumn("enrichedReadsetId")); - newCol.setLabel("Has Any Readset With Data?"); - ti.addColumn(newCol); - } - - String name2 = "allReadsetsHaveData"; - if (ti.getColumn(name2) == null) - { - SQLFragment sql = new SQLFragment("CASE " + - " WHEN (" + ExprColumn.STR_TABLE_ALIAS + ".readsetId IS NOT NULL AND (select count(*) as expr FROM sequenceanalysis.sequence_readsets r JOIN sequenceanalysis.readdata d ON (r.rowid = d.readset) WHERE r.rowid = " + ExprColumn.STR_TABLE_ALIAS + ".readsetId) = 0) THEN " + ti.getSqlDialect().getBooleanFALSE() + - " WHEN (" + ExprColumn.STR_TABLE_ALIAS + ".enrichedReadsetId IS NOT NULL AND (select count(*) as expr FROM sequenceanalysis.sequence_readsets r JOIN sequenceanalysis.readdata d ON (r.rowid = d.readset) WHERE r.rowid = " + ExprColumn.STR_TABLE_ALIAS + ".enrichedReadsetId) = 0) THEN " + ti.getSqlDialect().getBooleanFALSE() + - " ELSE " + ti.getSqlDialect().getBooleanTRUE() + " END"); - - ExprColumn newCol = new ExprColumn(ti, name2, sql, JdbcType.BOOLEAN, ti.getColumn("readsetId"), ti.getColumn("enrichedReadsetId")); - newCol.setLabel("All Readsets Have Data?"); - ti.addColumn(newCol); - } - addAssayFieldsToCDnas(ti); - - LDKService.get().applyNaturalSort(ti, "plateId"); - } - - private void customizeSorts(AbstractTableInfo ti) - { - LDKService.get().applyNaturalSort(ti, "plateId"); - LDKService.get().applyNaturalSort(ti, "hto"); - - String name = "numLibraries"; - if (ti.getColumn(name) == null) - { - DetailsURL details = DetailsURL.fromString("/query/executeQuery.view?schemaName=tcrdb&query.queryName=cdnas&query.sortId~eq=${rowid}", (ti.getUserSchema().getContainer().isWorkbook() ? ti.getUserSchema().getContainer().getParent() : ti.getUserSchema().getContainer())); - - SQLFragment sql = new SQLFragment("(select count(*) as expr FROM " + TCRdbSchema.NAME + "." + TCRdbSchema.TABLE_CDNAS + " s WHERE s.sortId = " + ExprColumn.STR_TABLE_ALIAS + ".rowid)"); - ExprColumn newCol = new ExprColumn(ti, name, sql, JdbcType.INTEGER, ti.getColumn("rowid")); - newCol.setLabel("# cDNA Libraries"); - newCol.setURL(details); - ti.addColumn(newCol); - } - - name = "maxCellsForPlate"; - if (ti.getColumn(name) == null) - { - SQLFragment sql = new SQLFragment("(select count(*) as expr FROM " + TCRdbSchema.NAME + "." + TCRdbSchema.TABLE_SORTS + " s WHERE s.plateId = " + ExprColumn.STR_TABLE_ALIAS + ".plateId AND s.container = " + ExprColumn.STR_TABLE_ALIAS + ".container)"); - ExprColumn newCol = new ExprColumn(ti, name, sql, JdbcType.INTEGER, ti.getColumn("plateId"), ti.getColumn("container")); - newCol.setLabel("Max Cells/Well In Plate"); - ti.addColumn(newCol); - } - - name = "processingRequested"; - if (ti.getColumn(name) == null) - { - SQLFragment sql = new SQLFragment("(select ").append(ti.getSqlDialect().getGroupConcat(new SQLFragment("p.type"), true, true)).append(new SQLFragment(" as expr FROM " + TCRdbSchema.NAME + "." + TCRdbSchema.TABLE_PROCESSING + " p WHERE p.plateId = " + ExprColumn.STR_TABLE_ALIAS + ".plateId AND p.container = " + ExprColumn.STR_TABLE_ALIAS + ".container)")); - ExprColumn newCol = new ExprColumn(ti, name, sql, JdbcType.VARCHAR, ti.getColumn("plateId"), ti.getColumn("container")); - newCol.setLabel("Processing Requested"); - ti.addColumn(newCol); - } - } - - private void customizeStims(AbstractTableInfo ti) - { - String name = "numSorts"; - if (ti.getColumn(name) == null) - { - DetailsURL details = DetailsURL.fromString("/query/executeQuery.view?schemaName=tcrdb&query.queryName=sorts&query.stimId~eq=${rowid}", (ti.getUserSchema().getContainer().isWorkbook() ? ti.getUserSchema().getContainer().getParent() : ti.getUserSchema().getContainer())); - - SQLFragment sql = new SQLFragment("(select count(*) as expr FROM " + TCRdbSchema.NAME + "." + TCRdbSchema.TABLE_SORTS + " s WHERE s.stimId = " + ExprColumn.STR_TABLE_ALIAS + ".rowid)"); - ExprColumn newCol = new ExprColumn(ti, "numSorts", sql, JdbcType.INTEGER, ti.getColumn("rowid")); - newCol.setLabel("# Sorts"); - newCol.setURL(details); - ti.addColumn(newCol); - } - - name = "numLibraries"; - if (ti.getColumn(name) == null) - { - DetailsURL details = DetailsURL.fromString("/query/executeQuery.view?schemaName=tcrdb&query.queryName=cdnas&query.sortId/stimId~eq=${rowid}", (ti.getUserSchema().getContainer().isWorkbook() ? ti.getUserSchema().getContainer().getParent() : ti.getUserSchema().getContainer())); - - SQLFragment sql = new SQLFragment("(select count(c.rowid) as expr FROM " + TCRdbSchema.NAME + "." + TCRdbSchema.TABLE_SORTS + " so JOIN " + TCRdbSchema.NAME + "." + TCRdbSchema.TABLE_CDNAS + " c ON (so.rowid = c.sortId) WHERE so.stimId = " + ExprColumn.STR_TABLE_ALIAS + ".rowid)"); - ExprColumn newCol = new ExprColumn(ti, "numLibraries", sql, JdbcType.INTEGER, ti.getColumn("rowid")); - newCol.setLabel("# cDNA Libraries"); - newCol.setURL(details); - ti.addColumn(newCol); - } } private void customizeReadsets(AbstractTableInfo ti) { addAssayFieldsToTable(ti, "analysisId/readset", "LEFT JOIN sequenceanalysis.sequence_analyses a2 ON (a.analysisId = a2.rowId) WHERE a2.readset = " + ExprColumn.STR_TABLE_ALIAS + ".rowid", "rowid"); - - String name = "numTCRLibraries"; - if (ti.getColumn(name) == null) - { - SQLFragment sql = new SQLFragment("(select count(*) as expr FROM " + TCRdbSchema.NAME + "." + TCRdbSchema.TABLE_CDNAS + " c WHERE c.readsetId = " + ExprColumn.STR_TABLE_ALIAS + ".rowid OR c.enrichedReadsetId = " + ExprColumn.STR_TABLE_ALIAS + ".rowid)"); - ExprColumn newCol = new ExprColumn(ti, name, sql, JdbcType.INTEGER, ti.getColumn("rowid")); - newCol.setLabel("# TCR Libraries"); - ti.addColumn(newCol); - } - - String cDNA = "cDNA"; - if (ti.getColumn(cDNA) == null) - { - SQLFragment sql = new SQLFragment("(CASE" + - " WHEN ((select count(*) as expr FROM " + TCRdbSchema.NAME + "." + TCRdbSchema.TABLE_CDNAS + " c WHERE c.readsetId = " + ExprColumn.STR_TABLE_ALIAS + ".rowid OR c.enrichedReadsetId = " + ExprColumn.STR_TABLE_ALIAS + ".rowid) > 0) " + - " THEN (select max(c.rowid) FROM " + TCRdbSchema.NAME + "." + TCRdbSchema.TABLE_CDNAS + " c WHERE c.readsetId = " + ExprColumn.STR_TABLE_ALIAS + ".rowid OR c.enrichedReadsetId = " + ExprColumn.STR_TABLE_ALIAS + ".rowid) " + - " ELSE null " + - "END)"); - ExprColumn newCol = new ExprColumn(ti, cDNA, sql, JdbcType.INTEGER, ti.getColumn("rowid")); - newCol.setLabel("cDNA Library"); - UserSchema us = QueryService.get().getUserSchema(ti.getUserSchema().getUser(), (ti.getUserSchema().getContainer().isWorkbook() ? ti.getUserSchema().getContainer().getParent() : ti.getUserSchema().getContainer()), TCRdbSchema.NAME); - newCol.setFk(QueryForeignKey.from(us, ti.getContainerFilter()) - .table(TCRdbSchema.TABLE_CDNAS) - .key("rowid") - .display("rowid")); - ti.addColumn(newCol); - } } private void addAssayFieldsToAnalyses(AbstractTableInfo ti) diff --git a/tcrdb/src/org/labkey/tcrdb/TCRdbUserSchema.java b/tcrdb/src/org/labkey/tcrdb/TCRdbUserSchema.java index 402e11a85..188fcc112 100644 --- a/tcrdb/src/org/labkey/tcrdb/TCRdbUserSchema.java +++ b/tcrdb/src/org/labkey/tcrdb/TCRdbUserSchema.java @@ -6,7 +6,6 @@ import org.labkey.api.data.ContainerFilter; import org.labkey.api.data.DbSchema; import org.labkey.api.data.TableInfo; -import org.labkey.api.ldk.table.ContainerScopedTable; import org.labkey.api.ldk.table.SharedDataTable; import org.labkey.api.module.Module; import org.labkey.api.query.DefaultSchema; @@ -42,15 +41,11 @@ public QuerySchema createSchema(final DefaultSchema schema, Module module) @Nullable protected TableInfo createWrappedTable(String name, @NotNull TableInfo sourceTable, ContainerFilter cf) { - if (TCRdbSchema.TABLE_LIBRARIES.equalsIgnoreCase(name)) + if (TCRdbSchema.TABLE_MIXCR_LIBRARIES.equalsIgnoreCase(name)) { // TODO: assert cf is null or not default? return new SharedDataTable<>(this, sourceTable).init(); } - else if (TCRdbSchema.TABLE_CITE_SEQ_ANTIBODIES.equalsIgnoreCase(name)) - { - return new ContainerScopedTable<>(this, sourceTable, cf, "antibodyName").init(); - } return super.createWrappedTable(name, sourceTable, cf); } diff --git a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerCellHashingHandler.java b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerCellHashingHandler.java deleted file mode 100644 index 00be83f8c..000000000 --- a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerCellHashingHandler.java +++ /dev/null @@ -1,361 +0,0 @@ -package org.labkey.tcrdb.pipeline; - -import au.com.bytecode.opencsv.CSVReader; -import au.com.bytecode.opencsv.CSVWriter; -import htsjdk.samtools.util.IOUtil; -import org.json.JSONObject; -import org.labkey.api.module.ModuleLoader; -import org.labkey.api.pipeline.PipelineJob; -import org.labkey.api.pipeline.PipelineJobException; -import org.labkey.api.pipeline.RecordedAction; -import org.labkey.api.reader.Readers; -import org.labkey.api.sequenceanalysis.SequenceOutputFile; -import org.labkey.api.sequenceanalysis.model.Readset; -import org.labkey.api.sequenceanalysis.pipeline.AbstractParameterizedOutputHandler; -import org.labkey.api.sequenceanalysis.pipeline.DefaultPipelineStepOutput; -import org.labkey.api.sequenceanalysis.pipeline.PipelineStepOutput; -import org.labkey.api.sequenceanalysis.pipeline.SequenceAnalysisJobSupport; -import org.labkey.api.sequenceanalysis.pipeline.SequenceOutputHandler; -import org.labkey.api.sequenceanalysis.pipeline.SequencePipelineService; -import org.labkey.api.sequenceanalysis.pipeline.ToolParameterDescriptor; -import org.labkey.api.util.FileType; -import org.labkey.api.util.FileUtil; -import org.labkey.api.util.PageFlowUtil; -import org.labkey.api.writer.PrintWriters; -import org.labkey.tcrdb.TCRdbModule; - -import java.io.File; -import java.io.FileFilter; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashSet; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; - -public class CellRangerCellHashingHandler extends AbstractParameterizedOutputHandler -{ - private FileType _fileType = new FileType("cloupe", false); - public static String CATEGORY = "10x GEX Cell Hashing Calls"; - - public CellRangerCellHashingHandler() - { - super(ModuleLoader.getInstance().getModule(TCRdbModule.class), "CellRanger GEX/Cell Hashing", "This will run CiteSeqCount/MultiSeqClassifier to generate a sample-to-cellbarcode TSV based on the filtered barcodes from CellRanger.", new LinkedHashSet<>(PageFlowUtil.set("sequenceanalysis/field/CellRangerAggrTextarea.js")), getDefaultParams()); - } - - private static List getDefaultParams() - { - List ret = new ArrayList<>(getDefaultHashingParams(true)); - ret.add( - ToolParameterDescriptor.create("useOutputFileContainer", "Submit to Source File Workbook", "If checked, each job will be submitted to the same workbook as the input file, as opposed to submitting all jobs to the same workbook. This is primarily useful if submitting a large batch of files to process separately. This only applies if 'Run Separately' is selected.", "checkbox", new JSONObject(){{ - put("checked", true); - }}, true) - ); - - return ret; - } - - public static List getDefaultHashingParams(boolean includeExcludeFailedcDNA) - { - List ret = new ArrayList<>(Arrays.asList( - ToolParameterDescriptor.create("scanEditDistances", "Scan Edit Distances", "If checked, CITE-seq-count will be run using edit distances from 0-3 and the iteration with the highest singlets will be used.", "checkbox", new JSONObject(){{ - put("checked", false); - }}, false), - ToolParameterDescriptor.create("editDistance", "Edit Distance", null, "ldk-integerfield", null, 3), - ToolParameterDescriptor.create("minCountPerCell", "Min Reads/Cell", null, "ldk-integerfield", null, 5), - ToolParameterDescriptor.create("useSeurat", "Use Seurat Calling", "If checked, the seurat HTO calling algorithm will be used.", "checkbox", null, true), - ToolParameterDescriptor.create("useMultiSeq", "Use MultiSeq Calling", "If checked, the MultiSeq HTO calling algorithm will be used.", "checkbox", null, true) - )); - - if (includeExcludeFailedcDNA) - { - ret.add(ToolParameterDescriptor.create("excludeFailedcDNA", "Exclude Failed cDNA", "If selected, cDNAs with non-blank status fields will be omitted", "checkbox", null, true)); - } - - return ret; - } - - @Override - public boolean canProcess(SequenceOutputFile o) - { - return o.getFile() != null && _fileType.isType(o.getFile()); - } - - @Override - public boolean doRunRemote() - { - return true; - } - - @Override - public boolean doRunLocal() - { - return false; - } - - @Override - public SequenceOutputProcessor getProcessor() - { - return new CellRangerCellHashingHandler.Processor(); - } - - @Override - public boolean doSplitJobs() - { - return true; - } - - @Override - public boolean requiresSingleGenome() - { - return false; - } - - public class Processor implements SequenceOutputHandler.SequenceOutputProcessor - { - @Override - public void init(PipelineJob job, SequenceAnalysisJobSupport support, List inputFiles, JSONObject params, File outputDir, List actions, List outputsToCreate) throws UnsupportedOperationException, PipelineJobException - { - new CellRangerVDJUtils(job.getLogger(), outputDir).prepareHashingAndCiteSeqFilesIfNeeded(job, support, "readsetId", params.optBoolean("excludeFailedcDNA", true), true, false); - } - - @Override - public void processFilesOnWebserver(PipelineJob job, SequenceAnalysisJobSupport support, List inputFiles, JSONObject params, File outputDir, List actions, List outputsToCreate) throws UnsupportedOperationException, PipelineJobException - { - - } - - @Override - public void processFilesRemote(List inputFiles, SequenceOutputHandler.JobContext ctx) throws UnsupportedOperationException, PipelineJobException - { - RecordedAction action = new RecordedAction(getName()); - Map readsetToHashing = CellRangerVDJUtils.getCachedHashingReadsetMap(ctx.getSequenceSupport()); - ctx.getLogger().debug("total cached readset to hashing pairs: " + readsetToHashing.size()); - - for (SequenceOutputFile so : inputFiles) - { - ctx.getLogger().info("processing file: " + so.getName()); - - //find TSV: - File perCellTsv; - File barcodeDir = null; - for (String dirName : Arrays.asList("filtered_gene_bc_matrices", "filtered_feature_bc_matrix")) - { - File f = new File(so.getFile().getParentFile(), dirName); - if (f.exists()) - { - barcodeDir = f; - break; - } - } - - if (barcodeDir == null) - { - //this might be a re-analysis loupe directory. in this case, use the tsne projection.csv as the whitelist: - File dir = new File(so.getFile().getParentFile(), "analysis"); - dir = new File(dir, "tsne"); - dir = new File(dir, "2_components"); - if (!dir.exists()) - { - throw new PipelineJobException("Unable to find barcode or analysis directory: " + dir.getPath()); - } - - perCellTsv = new File(dir, "projection.csv"); - } - //cellranger 2 format - else if ("filtered_gene_bc_matrices".equals(barcodeDir.getName())) - { - File[] children = barcodeDir.listFiles(new FileFilter() - { - @Override - public boolean accept(File pathname) - { - return pathname.isDirectory(); - } - }); - - if (children == null || children.length != 1) - { - throw new PipelineJobException("Expected to find a single subfolder under: " + barcodeDir.getPath()); - } - - perCellTsv = new File(children[0], "barcodes.tsv"); - } - else - { - perCellTsv = new File(barcodeDir, "barcodes.tsv.gz"); - } - - if (!perCellTsv.exists()) - { - throw new PipelineJobException("Unable to find file: " + perCellTsv.getPath()); - } - - Readset rs = ctx.getSequenceSupport().getCachedReadset(so.getReadset()); - if (rs == null) - { - throw new PipelineJobException("Unable to find readset for outputfile: " + so.getRowid()); - } - else if (rs.getReadsetId() == null) - { - throw new PipelineJobException("Readset lacks a rowId for outputfile: " + so.getRowid()); - } - - Readset htoReadset = ctx.getSequenceSupport().getCachedReadset(readsetToHashing.get(rs.getReadsetId())); - if (htoReadset == null) - { - throw new PipelineJobException("Unable to find Hashing/Cite-seq readset for GEX readset: " + rs.getReadsetId()); - } - - processBarcodeFile(ctx, perCellTsv, rs, htoReadset, so.getLibrary_id(), action, getClientCommandArgs(ctx.getParams()), true, CATEGORY); - } - - ctx.addActions(action); - } - - @Override - public void complete(PipelineJob job, List inputs, List outputsCreated, SequenceAnalysisJobSupport support) throws PipelineJobException - { - for (SequenceOutputFile so : outputsCreated) - { - if (so.getCategory().equals(CATEGORY)) - { - CellRangerVDJCellHashingHandler.processMetrics(so, job, true); - } - } - } - } - - public static File processBarcodeFile(SequenceOutputHandler.JobContext ctx, File perCellTsv, Readset rs, Readset htoOrCiteReadset, int genomeId, RecordedAction action, List commandArgs, boolean writeLoupe, String category) throws PipelineJobException - { - return processBarcodeFile(ctx, perCellTsv, rs, htoOrCiteReadset, genomeId, action, commandArgs, writeLoupe, category, true); - } - - public static File processBarcodeFile(SequenceOutputHandler.JobContext ctx, File perCellTsv, Readset rs, Readset htoOrCiteReadset, int genomeId, RecordedAction action, List commandArgs, boolean writeLoupe, String category, boolean generateHtoCalls) throws PipelineJobException - { - CellRangerVDJUtils utils = new CellRangerVDJUtils(ctx.getLogger(), ctx.getSourceDirectory()); - return processBarcodeFile(ctx, perCellTsv, rs, htoOrCiteReadset, genomeId, action, commandArgs, writeLoupe, category, true, utils.getValidHashingBarcodeFile(), generateHtoCalls); - } - - public static File processBarcodeFile(SequenceOutputHandler.JobContext ctx, File perCellTsv, Readset rs, Readset htoOrCiteReadset, int genomeId, RecordedAction action, List commandArgs, boolean writeLoupe, String category, boolean createOutputFiles, File htoBarcodeWhitelist, boolean generateHtoCalls) throws PipelineJobException - { - ctx.getLogger().debug("inspecting file: " + perCellTsv.getPath()); - - CellRangerVDJUtils utils = new CellRangerVDJUtils(ctx.getLogger(), ctx.getSourceDirectory()); - - //prepare whitelist of cell indexes - File cellBarcodeWhitelist = utils.getValidCellIndexFile(); - Set uniqueBarcodes = new HashSet<>(); - ctx.getLogger().debug("writing cell barcodes"); - try (CSVWriter writer = new CSVWriter(PrintWriters.getPrintWriter(cellBarcodeWhitelist), ',', CSVWriter.NO_QUOTE_CHARACTER);CSVReader reader = new CSVReader(IOUtil.openFileForBufferedUtf8Reading(perCellTsv), '\t')) - { - int rowIdx = 0; - String[] row; - while ((row = reader.readNext()) != null) - { - //skip header - rowIdx++; - if (rowIdx > 1) - { - String barcode = row[0]; - - //NOTE: 10x appends "-1" to barcodes - if (barcode.contains("-")) - { - barcode = barcode.split("-")[0]; - } - - //This format is written out by the seurat pipeline - if (barcode.contains("_")) - { - barcode = barcode.split("_")[1]; - } - - if (!uniqueBarcodes.contains(barcode)) - { - writer.writeNext(new String[]{barcode}); - uniqueBarcodes.add(barcode); - } - } - } - - ctx.getLogger().debug("rows inspected: " + (rowIdx - 1)); - ctx.getLogger().debug("unique cell barcodes: " + uniqueBarcodes.size()); - ctx.getFileManager().addIntermediateFile(cellBarcodeWhitelist); - } - catch (IOException e) - { - throw new PipelineJobException(e); - } - - //prepare whitelist of barcodes, based on cDNA records - if (!htoBarcodeWhitelist.exists()) - { - throw new PipelineJobException("Unable to find file: " + htoBarcodeWhitelist.getPath()); - } - ctx.getFileManager().addIntermediateFile(htoBarcodeWhitelist); - - //run CiteSeqCount. - List extraParams = new ArrayList<>(); - extraParams.addAll(commandArgs); - - boolean scanEditDistances = ctx.getParams().optBoolean("scanEditDistances", false); - int editDistance = ctx.getParams().optInt("editDistance", 3); - int minCountPerCell = ctx.getParams().optInt("minCountPerCell", 3); - boolean useSeurat = ctx.getParams().optBoolean("useSeurat", true); - boolean useMultiSeq = ctx.getParams().optBoolean("useMultiSeq", true); - - PipelineStepOutput output = new DefaultPipelineStepOutput(); - String basename = FileUtil.makeLegalName(rs.getName()); - File cellToHto = SequencePipelineService.get().runCiteSeqCount(output, category, htoOrCiteReadset, htoBarcodeWhitelist, cellBarcodeWhitelist, ctx.getWorkingDirectory(), basename, ctx.getLogger(), extraParams, false, minCountPerCell, ctx.getSourceDirectory(), editDistance, scanEditDistances, rs, genomeId, generateHtoCalls, createOutputFiles, useSeurat, useMultiSeq); - ctx.getFileManager().addStepOutputs(action, output); - - ctx.getFileManager().addOutput(action, category, cellToHto); - File html = new File(cellToHto.getParentFile(), FileUtil.getBaseName(cellToHto.getName()) + ".html"); - if (html.exists()) - { - ctx.getFileManager().addOutput(action, "Cell Hashing Report", html); - } - - File citeSeqCountUnknownOutput = new File(cellToHto.getParentFile(), "citeSeqUnknownBarcodes.txt"); - ctx.getFileManager().addOutput(action,"CiteSeqCount Unknown Barcodes", citeSeqCountUnknownOutput); - - if (writeLoupe) - { - File forLoupe = new File(ctx.getSourceDirectory(), rs.getName() + "-CiteSeqCalls.csv"); - try (CSVReader reader = new CSVReader(Readers.getReader(cellToHto), '\t'); CSVWriter writer = new CSVWriter(PrintWriters.getPrintWriter(forLoupe), ',', CSVWriter.NO_QUOTE_CHARACTER)) - { - String[] line; - int idx = 0; - while ((line = reader.readNext()) != null) - { - idx++; - - if (idx > 1) - { - line[0] = line[0] + "-1"; - } - - writer.writeNext(new String[]{line[0], line[1]}); - } - } - catch (IOException e) - { - throw new PipelineJobException(e); - } - - if (createOutputFiles) - { - ctx.getFileManager().addSequenceOutput(forLoupe, rs.getName() + ": Cell Hashing Calls", "10x GEX Cell Hashing Calls (Loupe)", rs.getReadsetId(), null, genomeId, null); - } - else - { - ctx.getLogger().debug("Output file creation will be skipped"); - } - } - - return cellToHto; - } -} \ No newline at end of file diff --git a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerSeuratHandler.java b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerSeuratHandler.java deleted file mode 100644 index 7d1fd5930..000000000 --- a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerSeuratHandler.java +++ /dev/null @@ -1,1054 +0,0 @@ -package org.labkey.tcrdb.pipeline; - -import au.com.bytecode.opencsv.CSVReader; -import au.com.bytecode.opencsv.CSVWriter; -import htsjdk.samtools.util.IOUtil; -import org.apache.commons.io.FileUtils; -import org.apache.commons.io.IOUtils; -import org.apache.commons.lang3.StringUtils; -import org.apache.logging.log4j.Logger; -import org.apache.commons.lang3.math.NumberUtils; -import org.json.JSONObject; -import org.labkey.api.data.CompareType; -import org.labkey.api.data.DbSchema; -import org.labkey.api.data.DbSchemaType; -import org.labkey.api.data.SimpleFilter; -import org.labkey.api.data.Table; -import org.labkey.api.data.TableInfo; -import org.labkey.api.data.TableSelector; -import org.labkey.api.exp.api.ExpData; -import org.labkey.api.exp.api.ExpRun; -import org.labkey.api.exp.api.ExperimentService; -import org.labkey.api.module.ModuleLoader; -import org.labkey.api.pipeline.PipelineJob; -import org.labkey.api.pipeline.PipelineJobException; -import org.labkey.api.pipeline.PipelineService; -import org.labkey.api.pipeline.PipelineStatusFile; -import org.labkey.api.pipeline.RecordedAction; -import org.labkey.api.query.FieldKey; -import org.labkey.api.reader.Readers; -import org.labkey.api.sequenceanalysis.SequenceAnalysisService; -import org.labkey.api.sequenceanalysis.SequenceOutputFile; -import org.labkey.api.sequenceanalysis.model.Readset; -import org.labkey.api.sequenceanalysis.pipeline.AbstractParameterizedOutputHandler; -import org.labkey.api.sequenceanalysis.pipeline.SequenceAnalysisJobSupport; -import org.labkey.api.sequenceanalysis.pipeline.SequenceOutputHandler; -import org.labkey.api.sequenceanalysis.pipeline.SequencePipelineService; -import org.labkey.api.sequenceanalysis.pipeline.ToolParameterDescriptor; -import org.labkey.api.sequenceanalysis.run.SimpleScriptWrapper; -import org.labkey.api.util.FileType; -import org.labkey.api.util.FileUtil; -import org.labkey.api.util.PageFlowUtil; -import org.labkey.api.writer.PrintWriters; -import org.labkey.tcrdb.TCRdbModule; - -import java.io.BufferedReader; -import java.io.File; -import java.io.IOException; -import java.io.PrintWriter; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; - -public class CellRangerSeuratHandler extends AbstractParameterizedOutputHandler -{ - private FileType _fileType = new FileType("cloupe", false); - public static final String SEURAT_MAX_THREADS = "seuratMaxThreads"; - - public CellRangerSeuratHandler() - { - super(ModuleLoader.getInstance().getModule(TCRdbModule.class), "Run Seurat", "This will run a standard seurat-based pipeline on the selected 10x/cellranger data and save the resulting Seurat object as an rds file for external use.", new LinkedHashSet<>(PageFlowUtil.set("sequenceanalysis/field/GenomeFileSelectorField.js")), getDefaultParams()); - } - - private static List getDefaultParams() - { - List ret = new ArrayList<>(Arrays.asList( - ToolParameterDescriptor.create("projectName", "Output Name", "This will be used as the final sample/file name. If blank, the readset name will be used. The latter cannot be used when merging multiple inputs.", "textfield", new JSONObject(){{ - - }}, null), - ToolParameterDescriptor.create("doSplitJobs", "Run Separately", "If checked, each input dataset will be run separately. Otherwise they will be merged", "checkbox", new JSONObject(){{ - put("checked", true); - }}, false), - ToolParameterDescriptor.create("skipProcessing", "Skip Processing", "If checked, the initial merge and EmptyDrops processing will be run, but PCA, DimRux, etc. will be skipped. The primary use of this is to created a merged seurat object for manual downstream processing", "checkbox", new JSONObject(){{ - put("checked", false); - }}, false), - ToolParameterDescriptor.create("useOutputFileContainer", "Submit to Source File Workbook", "If checked, each job will be submitted to the same workbook as the input file, as opposed to submitting all jobs to the same workbook. This is primarily useful if submitting a large batch of files to process separately. This only applies if 'Run Separately' is selected.", "checkbox", new JSONObject(){{ - put("checked", true); - }}, false), - ToolParameterDescriptor.create("dimsToUse", "PCs To Use", "If non-blank, this is the number of PCs that seurat will use for dim reduction steps.", "ldk-integerfield", new JSONObject(){{ - - }}, null), - ToolParameterDescriptor.create("minDimsToUse", "Minimum PCs To Use", "If non-blank, the pipeline will attempt to infer the number of PCs to use for dim reduction, but will not use fewer than this value.", "ldk-integerfield", new JSONObject(){{ - - }}, 12), - ToolParameterDescriptor.create("doCellFilter", "Perform Cell Filtering", "If selected, cells will be filtered on pct.mito and number of unique genes.", "checkbox", new JSONObject(){{ - put("checked", true); - }}, true), - ToolParameterDescriptor.create("doCellCycle", "Perform Cell Cycle Correction", "If selected, the pipeline will attempt to correct for cell cycle.", "checkbox", new JSONObject(){{ - put("checked", true); - }}, true), - ToolParameterDescriptor.create("useSCTransform", "Use SCTransform", "If selected, the pipeline will use the newer SCtransform method instead of the standard Seurat pipeline.", "checkbox", new JSONObject(){{ - put("checked", false); - }}, false), - ToolParameterDescriptor.create("runSingleR", "Run SingleR", "If selected, SingleR will be run after Seurat processing.", "checkbox", new JSONObject(){{ - put("checked", true); - }}, true), - ToolParameterDescriptor.create("mergeMethod", "Merge Method", "This determines whether any batch correction will be applied when merging datasets.", "ldk-simplecombo", new JSONObject(){{ - put("storeValues", "simple;cca"); - }}, "simple"), - ToolParameterDescriptor.create(SEURAT_MAX_THREADS, "Seurat Max Threads", "Because seurat can behave badly with multiple threads, this allows a separate cap to be used from the main job. This will allow CITE-Seq-Count and other tools to run with more threads.", "ldk-integerfield", null, 1), - ToolParameterDescriptor.createExpDataParam("gtfFile", "Gene File", "This is the ID of a GTF file containing genes from this genome.", "sequenceanalysis-genomefileselectorfield", new JSONObject() - {{ - put("extensions", Arrays.asList("gtf")); - put("width", 400); - put("allowBlank", false); - }}, null) - )); - - ret.addAll(CellRangerCellHashingHandler.getDefaultHashingParams(false)); - - return ret; - } - - @Override - public boolean canProcess(SequenceOutputFile o) - { - return o.getFile() != null && _fileType.isType(o.getFile()); - } - - @Override - public List validateParameters(List outputFiles, JSONObject params) - { - if (!params.optBoolean("doSplitJobs", false) && StringUtils.trimToNull(params.optString("projectName")) == null && outputFiles.size() > 1) - { - return Collections.singletonList("Must provide the output name when merging multiple inputs"); - } - - return null; - } - - @Override - public boolean doRunRemote() - { - return true; - } - - @Override - public boolean doRunLocal() - { - return false; - } - - @Override - public boolean requiresSingleGenome() - { - return false; - } - - @Override - public SequenceOutputProcessor getProcessor() - { - return new CellRangerSeuratHandler.Processor(); - } - - @Override - public boolean doSplitJobs() - { - return false; - } - - public class Processor implements SequenceOutputProcessor - { - private static final String GTF_FILE_ID = "gtfFileIf"; - - @Override - public void init(PipelineJob job, SequenceAnalysisJobSupport support, List inputFiles, JSONObject params, File outputDir, List actions, List outputsToCreate) throws UnsupportedOperationException, PipelineJobException - { - for (SequenceOutputFile so : inputFiles) - { - if (so.getReadset() != null) - { - support.cacheReadset(so.getReadset(), job.getUser()); - } - else - { - job.getLogger().error("Output file lacks a readset and will be skipped: " + so.getRowid()); - } - } - - new CellRangerVDJUtils(job.getLogger(), outputDir).prepareHashingAndCiteSeqFilesIfNeeded(job, support,"readsetId", params.optBoolean("excludeFailedcDNA", true), false, false); - - Set gtfIds = new HashSet<>(); - for (SequenceOutputFile so : inputFiles) - { - ExpData gtf = null; - ExpRun run = ExperimentService.get().getExpRun(so.getRunId()); - if (run != null) - { - List gtfDatas = run.getInputDatas("GTF File", null); - if (!gtfDatas.isEmpty()) - { - gtf = gtfDatas.get(0); - } - else - { - //Because existing runs didnt explicitly track GTF as an input, try to infer: - PipelineStatusFile sf = PipelineService.get().getStatusFile(run.getJobId()); - if (sf != null) - { - File log = new File(sf.getFilePath()); - File paramFile = new File(log.getParentFile(), "sequenceAnalysis.json"); - if (paramFile.exists()) - { - try (BufferedReader reader = Readers.getReader(paramFile)) - { - List lines = IOUtils.readLines(reader); - - JSONObject json = lines.isEmpty() ? new JSONObject() : new JSONObject(StringUtils.join(lines, '\n')); - Integer expData = json.optInt("alignment.CellRanger.gtfFile", -1); - if (expData == -1) - { - - } - - gtf = ExperimentService.get().getExpData(expData); - } - catch (IOException e) - { - throw new PipelineJobException(e); - } - } - } - } - } - - if (gtf == null) - { - throw new PipelineJobException("Unable to find GTF for output: " + so.getRowid()); - } - - gtfIds.add(gtf.getRowId()); - } - - if (gtfIds.size() != 1) - { - throw new PipelineJobException("All inputs must use the same GTF file, found: " + StringUtils.join(gtfIds, ",")); - } - - support.cacheExpData(ExperimentService.get().getExpData(gtfIds.iterator().next())); - support.cacheObject(GTF_FILE_ID, gtfIds.iterator().next()); - } - - @Override - public void processFilesOnWebserver(PipelineJob job, SequenceAnalysisJobSupport support, List inputFiles, JSONObject params, File outputDir, List actions, List outputsToCreate) throws UnsupportedOperationException, PipelineJobException - { - - } - - @Override - public void processFilesRemote(List inputFiles, JobContext ctx) throws UnsupportedOperationException, PipelineJobException - { - RecordedAction action = new RecordedAction(getName()); - ctx.addActions(action); - - int gtfId = ctx.getSequenceSupport().getCachedObject(GTF_FILE_ID, Integer.class); - File gtfFile = ctx.getSequenceSupport().getCachedData(gtfId); - if (!gtfFile.exists()) - { - throw new PipelineJobException("Unable to find GTF file: " + gtfFile.getPath()); - } - ctx.getFileManager().addInput(action, "GTF File", gtfFile); - - Set rsNames = new HashSet<>(); - for (SequenceOutputFile so : inputFiles) - { - ctx.getFileManager().addInput(action, "CellRanger Loupe", so.getFile()); - if (so.getReadset() != null) - { - rsNames.add(ctx.getSequenceSupport().getCachedReadset(so.getReadset()).getName()); - } - } - - String outPrefix = StringUtils.trimToNull(ctx.getParams().getString("projectName")); - if (outPrefix == null) - { - if (rsNames.size() == 1) - { - outPrefix = rsNames.iterator().next(); - } - else - { - throw new PipelineJobException("Must provide the output prefix when merging more than one output file"); - } - } - outPrefix = FileUtil.makeLegalName(outPrefix); - - File seuratObj = new File(ctx.getWorkingDirectory(), outPrefix + ".seurat.rds"); - File doneFile = new File(seuratObj.getPath() + ".done"); - boolean seuratHasRun = doneFile.exists(); - if (seuratHasRun) - { - ctx.getLogger().info("Seurat has already run, will not repeat"); - } - - Map dataMap = new HashMap<>(); - - File pr = ctx.getFolderPipeRoot().getRootPath().getParentFile(); //drop the @files or @pipeline - for (SequenceOutputFile so : inputFiles) - { - //start with seurat 3 - File subDir = new File(so.getFile().getParentFile(), "raw_feature_bc_matrix"); - if (!subDir.exists()) - { - //try 2 - subDir = new File(so.getFile().getParentFile(), "raw_gene_bc_matrices"); - if (subDir.exists()) - { - //now infer subdir: - for (File f : subDir.listFiles()) - { - if (f.isDirectory()) - { - subDir = f; - break; - } - } - } - } - - if (!subDir.exists()) - { - throw new PipelineJobException("Unable to find raw data for input: " + so.getFile().getPath()); - } - - try - { - String subDirRel = FileUtil.relativize(pr, subDir, true); - ctx.getLogger().debug("pipe root: " + pr.getPath()); - ctx.getLogger().debug("file path: " + subDir.getPath()); - ctx.getLogger().debug("relative path: " + subDirRel); - - //Copy raw data directory locally to avoid docker permission issues - String dirName = so.getRowid() + "_RawData"; - File copyDir = new File(ctx.getWorkingDirectory(), dirName); - if (!seuratHasRun) - { - if (copyDir.exists()) - { - ctx.getLogger().debug("Deleting directory: " + copyDir.getPath()); - FileUtils.deleteDirectory(copyDir); - } - - ctx.getLogger().debug("Copying raw data directory: " + subDir.getPath()); - ctx.getLogger().debug("To: " + copyDir.getPath()); - FileUtils.copyDirectory(subDir, copyDir); - } - ctx.getFileManager().addIntermediateFile(copyDir); - - dataMap.put(so, dirName); - } - catch (IOException e) - { - throw new PipelineJobException(e); - } - } - - File rmdScript = new File(SequenceAnalysisService.get().getScriptPath(TCRdbModule.NAME, "external/scRNAseq/Seurat3.rmd")); - if (!rmdScript.exists()) - { - throw new PipelineJobException("Unable to find script: " + rmdScript.getPath()); - } - - File wrapperScript = new File(SequenceAnalysisService.get().getScriptPath(TCRdbModule.NAME, "external/scRNAseq/seuratWrapper.sh")); - if (!wrapperScript.exists()) - { - throw new PipelineJobException("Unable to find script: " + wrapperScript.getPath()); - } - - File tmpScript = new File(ctx.getWorkingDirectory(), "script.R"); - File outHtml = new File(ctx.getWorkingDirectory(), outPrefix + ".html"); - boolean skipProcessing = ctx.getParams().optBoolean("skipProcessing", false); - - try (PrintWriter writer = PrintWriters.getPrintWriter(tmpScript)) - { - File scriptCopy = new File(ctx.getWorkingDirectory(), rmdScript.getName()); - if (scriptCopy.exists()) - { - scriptCopy.delete(); - } - - IOUtil.copyFile(rmdScript, scriptCopy); - rmdScript = scriptCopy; - ctx.getFileManager().addIntermediateFile(rmdScript); - - scriptCopy = new File(ctx.getWorkingDirectory(), wrapperScript.getName()); - if (scriptCopy.exists()) - { - scriptCopy.delete(); - } - - IOUtil.copyFile(wrapperScript, scriptCopy); - ctx.getFileManager().addIntermediateFile(scriptCopy); - - writer.println("outPrefix <- '" + outPrefix + "'"); - writer.println("resolutionToUse <- 0.6"); - for (String v : new String[]{"dimsToUse", "minDimsToUse"}) - { - String val = StringUtils.trimToNull(ctx.getParams().optString(v)); - val = val == null ? "NULL" : val; - - writer.println(v + " <- " + val); - } - - //GTF file: - File gtfCopy = new File(ctx.getWorkingDirectory(), gtfId + ".gtf"); - if (gtfCopy.exists()) - { - gtfCopy.delete(); - } - IOUtil.copyFile(gtfFile, gtfCopy); - ctx.getFileManager().addIntermediateFile(gtfCopy); - - writer.println("gtfFile <- '" + gtfCopy.getName() + "'"); - - String mergeMethod = StringUtils.trimToNull(ctx.getParams().optString("mergeMethod")); - mergeMethod = mergeMethod == null ? "NULL" : "'" + mergeMethod + "'"; - writer.println("mergeMethod <- " + mergeMethod); - - boolean doCellFilter = ctx.getParams().optBoolean("doCellFilter", true); - writer.println("doCellFilter <- " + String.valueOf(doCellFilter).toUpperCase()); - - writer.println("skipProcessing <- " + String.valueOf(skipProcessing).toUpperCase()); - - boolean runSingleR = ctx.getParams().optBoolean("runSingleR", true); - writer.println("runSingleR <- " + String.valueOf(runSingleR).toUpperCase()); - - boolean doCellCycle = ctx.getParams().optBoolean("doCellCycle", true); - writer.println("doCellCycle <- " + String.valueOf(doCellCycle).toUpperCase()); - - boolean useSCTransform = ctx.getParams().optBoolean("useSCTransform", false); - writer.println("useSCTransform <- " + String.valueOf(useSCTransform).toUpperCase()); - - writer.println("data <- list("); - String delim = ""; - for (SequenceOutputFile so : dataMap.keySet()) - { - writer.println("\t" + delim + "'" + so.getRowid() + "'='" + dataMap.get(so) + "'"); - delim = ","; - } - writer.println(")"); - writer.println(); - writer.println(); - writer.println("setwd('/work')"); - - writer.println("rmarkdown::render('" + rmdScript.getName() + "', clean=TRUE, output_file='" + outHtml.getName() + "')"); - } - catch (IOException e) - { - throw new PipelineJobException(e); - } - - if (!seuratHasRun) - { - SimpleScriptWrapper wrapper = new SimpleScriptWrapper(ctx.getLogger()); - wrapper.setWorkingDir(ctx.getWorkingDirectory()); - - Integer maxThreads = SequencePipelineService.get().getMaxThreads(ctx.getLogger()); - if (maxThreads != null) - { - if (ctx.getParams().get(SEURAT_MAX_THREADS) != null) - { - maxThreads = Math.min(ctx.getParams().getInt(SEURAT_MAX_THREADS), maxThreads); - wrapper.addToEnvironment("SEQUENCEANALYSIS_MAX_THREADS", maxThreads.toString()); - } - } - - wrapper.execute(Arrays.asList("/bin/bash", wrapperScript.getName(), pr.getPath())); - - try - { - FileUtils.touch(doneFile); - ctx.getFileManager().addIntermediateFile(doneFile); - } - catch (IOException e) - { - throw new PipelineJobException(e); - } - } - - if (!seuratObj.exists()) - { - throw new PipelineJobException("Unable to find expected file: " + seuratObj.getPath()); - } - - String dimsToUse = StringUtils.trimToNull(ctx.getParams().optString("dimsToUse")); - String minDimsToUse = StringUtils.trimToNull(ctx.getParams().optString("minDimsToUse")); - String mergeMethod = StringUtils.trimToNull(ctx.getParams().optString("mergeMethod")); - - String description = StringUtils.join(new String[]{ - "Correct Cell Cycle: " + ctx.getParams().optBoolean("doCellCycle", true), - "Perform Cell Filtering: " + ctx.getParams().optBoolean("doCellFilter", true), - "Min. Dims To Use: " + (minDimsToUse == null ? "NA" : minDimsToUse), - "Dims To Use: " + (dimsToUse == null ? "automatic" : dimsToUse), - "Use SCTransform: " + ctx.getParams().optBoolean("useSCTransform", false), - "Merge method: " + mergeMethod - }, "\n"); - - if (skipProcessing) - { - ctx.getFileManager().addSequenceOutput(seuratObj, "Seurat Raw Counts: " + outPrefix, "Seurat Unprocessed Data", (inputFiles.size() == 1 ? inputFiles.iterator().next().getReadset() : null), null, getGenomeId(inputFiles), "Unprocessed Data"); - } - else - { - ctx.getFileManager().addSequenceOutput(seuratObj, "Seurat Object: " + outPrefix, "Seurat Data", (inputFiles.size() == 1 ? inputFiles.iterator().next().getReadset() : null), null, getGenomeId(inputFiles), description); - } - - ctx.getFileManager().addOutput(action, "Seurat Object", seuratObj); - - if (!outHtml.exists()) - { - throw new PipelineJobException("Unable to find summary report"); - } - ctx.getFileManager().addOutput(action, "Seurat Report", outHtml); - - if (skipProcessing) - { - ctx.getFileManager().addSequenceOutput(outHtml, "Seurat Report: " + outPrefix, "Seurat Report", (inputFiles.size() == 1 ? inputFiles.iterator().next().getReadset() : null), null, getGenomeId(inputFiles), "Unprocessed Data"); - } - else - { - ctx.getFileManager().addSequenceOutput(outHtml, "Seurat Report: " + outPrefix, "Seurat Report", (inputFiles.size() == 1 ? inputFiles.iterator().next().getReadset() : null), null, getGenomeId(inputFiles), description); - } - - File seuratObjRaw = new File(ctx.getWorkingDirectory(), outPrefix + ".rawData.rds"); - if (seuratObjRaw.exists()) - { - ctx.getFileManager().addIntermediateFile(seuratObjRaw); - } - - CellRangerVDJUtils utils = new CellRangerVDJUtils(ctx.getLogger(), ctx.getSourceDirectory()); - if (utils.useCellHashing(ctx.getSequenceSupport())) - { - runCellHashing(ctx, inputFiles, seuratObj, action, utils); - } - else - { - ctx.getLogger().info("Cell hashing was not used"); - } - - if (utils.useCiteSeq(ctx.getSequenceSupport(), inputFiles)) - { - runCiteSeq(ctx, inputFiles, seuratObj, action, outPrefix); - } - else - { - ctx.getLogger().info("CITE-seq was not used"); - } - } - - private File getAllCellBarcodesFile(File seuratObj) throws PipelineJobException - { - File allCellBarcodes = new File(seuratObj.getParentFile(), seuratObj.getName().replaceAll("seurat.rds", "cellBarcodes.csv")); - if (!allCellBarcodes.exists()) - { - throw new PipelineJobException("Unable to find expected cell barcodes file. This might indicate the seurat object was created with an older version of the pipeline. Expected: " + allCellBarcodes.getPath()); - } - - return allCellBarcodes; - } - - private void runCiteSeq(JobContext ctx, List inputFiles, File seuratObj, RecordedAction action, String outPrefix) throws PipelineJobException - { - ctx.getLogger().info("Adding CITE-seq"); - - Map citeSeqData = new HashMap<>(); - Map markerMetadata = new HashMap<>(); - File allCellBarcodes = getAllCellBarcodesFile(seuratObj); - - for (SequenceOutputFile so : inputFiles) - { - //This is the loupe file at this point - String barcodePrefix = so.getRowid().toString(); - Readset rs = ctx.getSequenceSupport().getCachedReadset(so.getReadset()); - if (rs == null) - { - throw new PipelineJobException("Unable to find readset for outputfile: " + so.getRowid()); - } - else if (rs.getReadsetId() == null) - { - throw new PipelineJobException("Readset lacks a rowId for outputfile: " + so.getRowid()); - } - - File barcodes = subsetBarcodes(allCellBarcodes, barcodePrefix); - ctx.getFileManager().addIntermediateFile(barcodes); - - // write readset-specific HTO list - Integer citeseqReadsetId = CellRangerVDJUtils.getCachedCiteSeqReadsetMap(ctx.getSequenceSupport()).get(rs.getReadsetId()); - if (citeseqReadsetId == null) - { - ctx.getLogger().info("No cite-seq readset for: " + rs.getReadsetId() + ", this probably indicates either hashing is not used or the hashing data is not available."); - continue; - } - - Readset citeseqReadset = ctx.getSequenceSupport().getCachedReadset(citeseqReadsetId); - if (citeseqReadset == null) - { - throw new PipelineJobException("Unable to find Cite-seq readset for GEX readset: " + rs.getReadsetId()); - } - - File perReadsetAdts = CellRangerVDJUtils.getValidCiteSeqBarcodeFile(ctx.getSourceDirectory(), rs.getReadsetId()); - long adtsForReadset = !perReadsetAdts.exists() ? 0 : SequencePipelineService.get().getLineCount(perReadsetAdts) - 1; - - if (adtsForReadset > 0) - { - ctx.getLogger().info("Total ADTs for readset: " + adtsForReadset); - File countMatrix = CellRangerCellHashingHandler.processBarcodeFile(ctx, barcodes, rs, citeseqReadset, so.getLibrary_id(), action, getClientCommandArgs(ctx.getParams()), false, SeuratCiteSeqHandler.CATEGORY, true, perReadsetAdts, false); - citeSeqData.put(barcodePrefix, countMatrix.getParentFile()); - File perReadsetAdtMetadata = CellRangerVDJUtils.getValidCiteSeqBarcodeMetadataFile(ctx.getSourceDirectory(), rs.getReadsetId()); - markerMetadata.put(barcodePrefix, perReadsetAdtMetadata); - } - else - { - ctx.getLogger().info("No ADTs found for readset: " + rs.getReadsetId()); - } - } - - if (!citeSeqData.isEmpty()) - { - ctx.getLogger().info("Storing cite-seq data in seurat object"); - File outHtml = appendCiteSeqToSeurat(ctx, seuratObj, citeSeqData, markerMetadata); - - ctx.getFileManager().addSequenceOutput(outHtml, "CITE-Seq Report: " + outPrefix, "CITE-Seq Report", (inputFiles.size() == 1 ? inputFiles.iterator().next().getReadset() : null), null, getGenomeId(inputFiles), null); - } - else - { - ctx.getLogger().info("CITE-seq was not used. Will not append to seurat"); - } - } - - private void runCellHashing(JobContext ctx, List inputFiles, File seuratObj, RecordedAction action, CellRangerVDJUtils utils) throws PipelineJobException - { - ctx.getLogger().info("Adding cell hashing"); - - Map finalCalls = new HashMap<>(); - File allCellBarcodes = getAllCellBarcodesFile(seuratObj); - - for (SequenceOutputFile so : inputFiles) - { - //This is the loupe file at this point - String barcodePrefix = so.getRowid().toString(); - Readset rs = ctx.getSequenceSupport().getCachedReadset(so.getReadset()); - if (rs == null) - { - throw new PipelineJobException("Unable to find readset for outputfile: " + so.getRowid()); - } - else if (rs.getReadsetId() == null) - { - throw new PipelineJobException("Readset lacks a rowId for outputfile: " + so.getRowid()); - } - - File barcodes = subsetBarcodes(allCellBarcodes, barcodePrefix); - ctx.getFileManager().addIntermediateFile(barcodes); - - // write readset-specific HTO list - Integer hashingReadsetId = CellRangerVDJUtils.getCachedHashingReadsetMap(ctx.getSequenceSupport()).get(rs.getReadsetId()); - if (hashingReadsetId == null) - { - ctx.getLogger().info("No hashing readset for: " + rs.getReadsetId() + ", this probably indicates either hashing is not used or the hashing data is not available."); - continue; - } - - Readset htoReadset = ctx.getSequenceSupport().getCachedReadset(hashingReadsetId); - if (htoReadset == null) - { - throw new PipelineJobException("Unable to find hashing readset for GEX readset: " + rs.getReadsetId()); - } - - File perReadsetHtos = new File(allCellBarcodes.getParentFile(), "allowableHtos." + barcodePrefix + ".txt"); - int htosForReadset = 0; - try (CSVReader reader = new CSVReader(Readers.getReader(utils.getCDNAInfoFile()), '\t'); CSVWriter bcWriter = new CSVWriter(PrintWriters.getPrintWriter(perReadsetHtos), ',', CSVWriter.NO_QUOTE_CHARACTER)) - { - String[] line; - while ((line = reader.readNext()) != null) - { - if (hashingReadsetId.toString().equals(line[5])) - { - htosForReadset++; - bcWriter.writeNext(new String[]{line[8], line[7]}); - } - } - } - catch (IOException e) - { - throw new PipelineJobException(e); - } - - if (htosForReadset > 0) - { - ctx.getLogger().info("Total HTOs for readset: " + htosForReadset); - finalCalls.put(barcodePrefix, CellRangerCellHashingHandler.processBarcodeFile(ctx, barcodes, rs, htoReadset, so.getLibrary_id(), action, getClientCommandArgs(ctx.getParams()), false, SeuratCellHashingHandler.CATEGORY, true, perReadsetHtos, true)); - } - else - { - ctx.getLogger().info("No HTOs found for readset"); - } - } - - if (!finalCalls.isEmpty()) - { - ctx.getLogger().info("Storing cell hashing calls in seurat object"); - appendHashingCallsToSeurat(ctx, seuratObj, finalCalls); - } - else - { - ctx.getLogger().info("Cell hashing was not used. will not append to seurat"); - } - } - - private File subsetBarcodes(File allCellBarcodes, String barcodePrefix) throws PipelineJobException - { - //Subset barcodes by dataset: - File barcodes = new File(allCellBarcodes.getParentFile(), "cellBarcodeWhitelist." + barcodePrefix + ".txt"); - try (CSVReader reader = new CSVReader(Readers.getReader(allCellBarcodes), '\t'); CSVWriter writer = new CSVWriter(PrintWriters.getPrintWriter(barcodes), '\t', CSVWriter.NO_QUOTE_CHARACTER)) - { - String[] line; - while ((line = reader.readNext()) != null) - { - String barcode = line[0]; - if (barcode.startsWith(barcodePrefix + "_")) - { - barcode = barcode.split("_")[1]; - writer.writeNext(new String[]{barcode}); - } - } - } - catch (IOException e) - { - throw new PipelineJobException(e); - } - - return barcodes; - } - - private File appendCiteSeqToSeurat(JobContext ctx, File seuratObj, Map citeseqData, Map perReadsetAdtMap) throws PipelineJobException - { - File rScript = new File(seuratObj.getParentFile(), "appendCiteSeq.Rmd"); - File bashScript = new File(seuratObj.getParentFile(), "runDockerForCiteSeq.sh"); - - File localRoot = seuratObj.getParentFile(); - - File outputHtml = new File(localRoot, FileUtil.getBaseName(seuratObj) + ".citeseq.html"); - - Set toDelete = new HashSet<>(); - try (PrintWriter rWriter = PrintWriters.getPrintWriter(rScript); PrintWriter bashWriter = PrintWriters.getPrintWriter(bashScript)) - { - rWriter.println("---"); - rWriter.println(" title: 'CITE-seq'"); - rWriter.println("---"); - - rWriter.println("```{r setup}"); - rWriter.println("library(OOSAP)"); - rWriter.println("```"); - rWriter.println(""); - - rWriter.println("```{r citeseq}"); - rWriter.println("seuratObj <- readRDS('" + seuratObj.getName() + "')"); - rWriter.println("initialCells <- ncol(seuratObj)"); - rWriter.println("citeSeq <- list("); - int idx = 0; - for (String barcodePrefix : citeseqData.keySet()) { - idx++; - String localCopy = ensureLocalCopy(localRoot, toDelete, citeseqData.get(barcodePrefix), ctx.getLogger()); - rWriter.println("'" + barcodePrefix + "' = '" + localCopy + "'" + (idx < citeseqData.size() ? "," : "")); - } - rWriter.println(")"); - rWriter.println(""); - - rWriter.println("perReadsetAdtMap <- list("); - idx = 0; - for (String barcodePrefix : perReadsetAdtMap.keySet()) { - idx++; - String localCopy = ensureLocalCopy(localRoot, toDelete, perReadsetAdtMap.get(barcodePrefix), ctx.getLogger()); - rWriter.println("'" + barcodePrefix + "' = '" + localCopy + "'" + (idx < citeseqData.size() ? "," : "")); - } - rWriter.println(")"); - rWriter.println(""); - - rWriter.println("for (barcodePrefix in names(citeSeq)) {"); - rWriter.println(" seuratObj <- OOSAP:::AppendCiteSeq(seuratObj = seuratObj, countMatrixDir = citeSeq[[barcodePrefix]], barcodePrefix = barcodePrefix, featureLabelTable = perReadsetAdtMap[[barcodePrefix]])"); - rWriter.println("}"); - rWriter.println("if (ncol(seuratObj) != initialCells) { stop('Cell count not equal after appending cite-seq calls!') }"); - rWriter.println("saveRDS(seuratObj, file = '" + seuratObj.getName() + "')"); - rWriter.println("```"); - - rWriter.println("```{r Plot}"); - rWriter.println("OOSAP:::.PlotCiteSeqCountData(seuratObj)"); - rWriter.println("```"); - - rWriter.println("```{r SessionInfo}"); - rWriter.println("sessionInfo()"); - rWriter.println("```"); - - - bashWriter.println("#!/bin/bash"); - bashWriter.println("set -e"); - bashWriter.println("set -x"); - bashWriter.println("DOCKER=/opt/acc/sbin/exadocker"); - bashWriter.println("WD=`pwd`"); - bashWriter.println("HOME=`echo ~/`"); - - Integer maxRam = SequencePipelineService.get().getMaxRam(); - String ramOpts = ""; - if (maxRam != null) - { - ramOpts = " --memory=" +maxRam +"g "; - } - - bashWriter.println("sudo $DOCKER pull bimberlab/oosap"); - bashWriter.println("sudo $DOCKER run --rm=true " + ramOpts + "-v \"${WD}:/work\" -v \"${HOME}:/homeDir\" -u $UID -e USERID=$UID -w /work -e HOME=/homeDir bimberlab/oosap Rscript -e \"" + "rmarkdown::render('" + rScript.getName() + "', output_file = '" + outputHtml.getName() + "')\""); - - } - catch (IOException e) - { - throw new PipelineJobException(e); - } - - try - { - SimpleScriptWrapper wrapper = new SimpleScriptWrapper(ctx.getLogger()); - wrapper.setWorkingDir(seuratObj.getParentFile()); - wrapper.execute(Arrays.asList("/bin/bash", bashScript.getName())); - - for (File f : toDelete) - { - ctx.getLogger().debug("deleting local copy: " + f.getPath()); - if (f.isDirectory()) - { - FileUtils.deleteDirectory(f); - } - else - { - f.delete(); - } - } - } - catch (IOException e) - { - throw new PipelineJobException(e); - } - - return outputHtml; - } - - private String ensureLocalCopy(File localRoot, Set toDelete, File toCopy, Logger log) throws PipelineJobException - { - log.info("copying file locally: " + toCopy.getPath()); - - if (toCopy.getPath().startsWith(localRoot.getPath())) - { - return FileUtil.relativePath(localRoot.getPath(), toCopy.getPath()); - } - - try - { - File localCopy; - if (toCopy.isDirectory()) - { - localCopy = new File(localRoot, toCopy.getName()); - - File umiDir = new File(toCopy, "umi_count"); - if (!umiDir.exists()) - { - throw new PipelineJobException("Missing umi_count dir: " + umiDir.getPath()); - } - - if (localCopy.exists()) - { - log.info("local copy exists, skipping: " + localCopy.getPath()); - } - else - { - FileUtils.copyDirectory(umiDir, localCopy); - } - } - else - { - localCopy = new File(localRoot, toCopy.getName()); - - if (localCopy.exists()) - { - log.info("local copy exists, skipping: " + localCopy.getPath()); - } - else - { - FileUtils.copyFile(toCopy, localCopy); - } - } - - log.debug("destination: " + localCopy.getPath()); - toDelete.add(localCopy); - - return FileUtil.relativePath(localRoot.getPath(), localCopy.getPath()); - } - catch (IOException e) - { - throw new PipelineJobException(e); - } - } - - private void appendHashingCallsToSeurat(JobContext ctx, File seuratObj, Map finalCalls) throws PipelineJobException - { - File rScript = new File(seuratObj.getParentFile(), "appendHashing.R"); - File bashScript = new File(seuratObj.getParentFile(), "runDockerForHashing.sh"); - - try (PrintWriter rWriter = PrintWriters.getPrintWriter(rScript); PrintWriter bashWriter = PrintWriters.getPrintWriter(bashScript)) - { - rWriter.println("library(OOSAP)"); - rWriter.println("seuratObj <- readRDS('" + seuratObj.getName() + "')"); - rWriter.println("initialCells <- ncol(seuratObj)"); - rWriter.println("callsFiles <- list("); - int idx = 0; - for (String barcodePrefix : finalCalls.keySet()) - { - idx++; - rWriter.println("'" + barcodePrefix + "' = '" + finalCalls.get(barcodePrefix).getName() + "'" + (idx < finalCalls.size() ? "," : "")); - } - - rWriter.println(")"); - rWriter.println(""); - rWriter.println("for (barcodePrefix in names(callsFiles)) {"); - rWriter.println(" seuratObj <- OOSAP:::AppendCellHashing(seuratObj = seuratObj, barcodeCallFile = callsFiles[[barcodePrefix]], barcodePrefix = barcodePrefix)"); - rWriter.println("}"); - rWriter.println("if (ncol(seuratObj) != initialCells) { stop('Cell count not equal after appending cell hashing calls!') }"); - rWriter.println("saveRDS(seuratObj, file = '" + seuratObj.getName() + "')"); - - bashWriter.println("#!/bin/bash"); - bashWriter.println("set -e"); - bashWriter.println("set -x"); - bashWriter.println("DOCKER=/opt/acc/sbin/exadocker"); - bashWriter.println("WD=`pwd`"); - bashWriter.println("HOME=`echo ~/`"); - - Integer maxRam = SequencePipelineService.get().getMaxRam(); - String ramOpts = ""; - if (maxRam != null) - { - ramOpts = " --memory=" + maxRam + "g "; - } - - bashWriter.println("sudo $DOCKER pull bimberlab/oosap"); - bashWriter.println("sudo $DOCKER run --rm=true " + ramOpts + "-v \"${WD}:/work\" -v \"${HOME}:/homeDir\" -u $UID -e USERID=$UID -w /work -e HOME=/homeDir bimberlab/oosap Rscript --vanilla " + rScript.getName()); - } - catch (IOException e) - { - throw new PipelineJobException(e); - } - - SimpleScriptWrapper wrapper = new SimpleScriptWrapper(ctx.getLogger()); - wrapper.setWorkingDir(seuratObj.getParentFile()); - wrapper.execute(Arrays.asList("/bin/bash", bashScript.getName())); - } - - @Override - public void complete(PipelineJob job, List inputs, List outputsCreated, SequenceAnalysisJobSupport support) throws PipelineJobException - { - for (SequenceOutputFile so : outputsCreated) - { - if (so.getFile() != null && so.getFile().getPath().endsWith(".seurat.rds")) - { - File metrics = new File(so.getFile().getPath().replaceAll(".seurat.rds", ".summary.txt")); - if (metrics.exists()) - { - processMetricsFile(job, metrics, so); - } - else - { - job.getLogger().warn("Unable to find metrics file: " + metrics.getPath()); - } - } - else if (so.getFile() != null && so.getFile().getPath().endsWith(".calls.txt")) - { - File metrics = new File(so.getFile().getPath().replaceAll(".calls.txt", ".metrics.txt")); - if (metrics.exists()) - { - processMetricsFile(job, metrics, so); - } - else - { - job.getLogger().warn("Unable to find metrics file: " + metrics.getPath()); - } - } - } - } - } - - private void processMetricsFile(PipelineJob job, File metrics, SequenceOutputFile so) throws PipelineJobException - { - job.getLogger().info("Loading metrics"); - TableInfo ti = DbSchema.get("sequenceanalysis", DbSchemaType.Module).getTable("quality_metrics"); - - //NOTE: if this job errored and restarted, we may have duplicate records: - SimpleFilter filter = new SimpleFilter(FieldKey.fromString("readset"), so.getReadset()); - filter.addCondition(FieldKey.fromString("analysis_id"), so.getAnalysis_id(), CompareType.EQUAL); - filter.addCondition(FieldKey.fromString("dataid"), so.getDataId(), CompareType.EQUAL); - filter.addCondition(FieldKey.fromString("container"), job.getContainer().getId(), CompareType.EQUAL); - TableSelector ts = new TableSelector(ti, PageFlowUtil.set("rowid"), filter, null); - if (ts.exists()) - { - job.getLogger().info("Deleting existing QC metrics (probably from prior restarted job)"); - ts.getArrayList(Integer.class).forEach(rowid -> { - Table.delete(ti, rowid); - }); - } - - int total = 0; - try (CSVReader reader = new CSVReader(Readers.getReader(metrics), '\t')) - { - String[] line; - while ((line = reader.readNext()) != null) - { - if ("Category".equals(line[0])) - { - continue; - } - - Map r = new HashMap<>(); - r.put("category", line[0]); - r.put("metricname", line[1]); - - String fieldName = NumberUtils.isCreatable(line[2]) ? "metricvalue" : "qualvalue"; - r.put(fieldName, line[2]); - r.put("analysis_id", so.getAnalysis_id()); - r.put("dataid", so.getDataId()); - r.put("readset", so.getReadset()); - r.put("container", job.getContainer()); - r.put("createdby", job.getUser().getUserId()); - - Table.insert(job.getUser(), ti, r); - total++; - } - } - catch (IOException e) - { - throw new PipelineJobException(e); - } - - job.getLogger().info("total metrics: " + total); - } - - private Integer getGenomeId(List inputFiles) - { - Set genomeIds = new HashSet<>(); - inputFiles.forEach(x -> { - genomeIds.add(x.getLibrary_id()); - }); - - return genomeIds.size() == 1 ? genomeIds.iterator().next() : null; - } -} diff --git a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java index 19a3fdb61..cab9554ae 100644 --- a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java +++ b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java @@ -1,23 +1,14 @@ package org.labkey.tcrdb.pipeline; import au.com.bytecode.opencsv.CSVReader; -import org.apache.commons.beanutils.ConversionException; +import au.com.bytecode.opencsv.CSVWriter; import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.math.NumberUtils; import org.json.JSONObject; -import org.labkey.api.data.CompareType; import org.labkey.api.data.ConvertHelper; -import org.labkey.api.data.DbSchema; -import org.labkey.api.data.DbSchemaType; -import org.labkey.api.data.SimpleFilter; -import org.labkey.api.data.Table; -import org.labkey.api.data.TableInfo; -import org.labkey.api.data.TableSelector; import org.labkey.api.module.ModuleLoader; import org.labkey.api.pipeline.PipelineJob; import org.labkey.api.pipeline.PipelineJobException; import org.labkey.api.pipeline.RecordedAction; -import org.labkey.api.query.FieldKey; import org.labkey.api.reader.Readers; import org.labkey.api.sequenceanalysis.SequenceOutputFile; import org.labkey.api.sequenceanalysis.model.AnalysisModel; @@ -27,31 +18,38 @@ import org.labkey.api.sequenceanalysis.pipeline.SequenceAnalysisJobSupport; import org.labkey.api.sequenceanalysis.pipeline.SequenceOutputHandler; import org.labkey.api.sequenceanalysis.pipeline.ToolParameterDescriptor; +import org.labkey.api.singlecell.CellHashingService; import org.labkey.api.util.FileType; +import org.labkey.api.util.FileUtil; import org.labkey.api.util.PageFlowUtil; +import org.labkey.api.writer.PrintWriters; import org.labkey.tcrdb.TCRdbModule; import java.io.File; import java.io.IOException; -import java.text.DecimalFormat; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; +import java.util.HashSet; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; - -import static org.labkey.tcrdb.pipeline.CellRangerVDJWrapper.DELETE_EXISTING_ASSAY_DATA; -import static org.labkey.tcrdb.pipeline.CellRangerVDJWrapper.TARGET_ASSAY; +import java.util.Set; public class CellRangerVDJCellHashingHandler extends AbstractParameterizedOutputHandler { - private FileType _fileType = new FileType("vloupe", false); + private FileType _vloupeFileType = new FileType("vloupe", false); + private FileType _htmlFileType = new FileType("html", false); + public static final String CATEGORY = "Cell Hashing Calls (VDJ)"; + public static final String TARGET_ASSAY = "targetAssay"; + public static final String DELETE_EXISTING_ASSAY_DATA = "deleteExistingAssayData"; + public static final String USE_GEX_BARCODES = "useGexBarcodes"; + public CellRangerVDJCellHashingHandler() { - super(ModuleLoader.getInstance().getModule(TCRdbModule.class), "CellRanger VDJ Import", "This will either directly import data (if cell hashing is not used), or run CiteSeqCount/MultiSeqClassifier to generate a sample-to-cellbarcode TSV based on the filtered barcodes from CellRanger VDJ and then import.", new LinkedHashSet<>(PageFlowUtil.set("tcrdb/field/AssaySelectorField.js")), getDefaultParams()); + super(ModuleLoader.getInstance().getModule(TCRdbModule.class), "CellRanger VDJ Import", "This will either directly import data (if cell hashing is not used), or run cellhashR on the hashing count matrix to generate a sample-to-cellbarcode TSV based on the filtered barcodes from CellRanger VDJ and then import.", new LinkedHashSet<>(PageFlowUtil.set("tcrdb/field/AssaySelectorField.js")), getDefaultParams()); } private static List getDefaultParams() @@ -63,10 +61,13 @@ private static List getDefaultParams() }}, true), ToolParameterDescriptor.create("useOutputFileContainer", "Submit to Source File Workbook", "If checked, each job will be submitted to the same workbook as the input file, as opposed to submitting all jobs to the same workbook. This is primarily useful if submitting a large batch of files to process separately. This only applies if 'Run Separately' is selected.", "checkbox", new JSONObject(){{ put("checked", true); + }}, false), + ToolParameterDescriptor.create(USE_GEX_BARCODES, "Use GEX and TCR Cell Barcodes", "If checked, the cell barcode whitelist used for cell hashing will be the union of TCR and GEX cell barcodes. If T-cells are a rare component of total cells, this might enhance the effectiveness of the callers by providing more positive signal.", "checkbox", new JSONObject(){{ + put("checked", true); }}, false) )); - ret.addAll(CellRangerCellHashingHandler.getDefaultHashingParams(true)); + ret.addAll(CellHashingService.get().getHashingCallingParams()); return ret; } @@ -74,7 +75,7 @@ private static List getDefaultParams() @Override public boolean canProcess(SequenceOutputFile o) { - return o.getFile() != null && _fileType.isType(o.getFile()); + return o.getFile() != null && (_vloupeFileType.isType(o.getFile()) || (_htmlFileType.isType(o.getFile()) && "10x Run Summary".equals(o.getCategory()) && o.getName().contains("VDJ Summary"))); } @Override @@ -110,12 +111,17 @@ public boolean requiresSingleGenome() public class Processor implements SequenceOutputHandler.SequenceOutputProcessor { @Override - public void init(PipelineJob job, SequenceAnalysisJobSupport support, List inputFiles, JSONObject params, File outputDir, List actions, List outputsToCreate) throws UnsupportedOperationException, PipelineJobException + public void init(JobContext ctx, List inputFiles, List actions, List outputsToCreate) throws UnsupportedOperationException, PipelineJobException { - CellRangerVDJUtils utils = new CellRangerVDJUtils(job.getLogger(), outputDir); - //NOTE: this is the pathway to import assay data, whether hashing is used or not - utils.prepareHashingAndCiteSeqFilesIfNeeded(job, support, "enrichedReadsetId", params.optBoolean("excludeFailedcDNA", true), false, false); + CellHashingService.get().prepareHashingIfNeeded(ctx.getOutputDir(), ctx.getJob(), ctx.getSequenceSupport(), "tcrReadsetId", false); + + if (ctx.getParams().optBoolean(USE_GEX_BARCODES, false)) + { + ctx.getJob().getLogger().info("The union of TCR and GEX cell barcodes will be used for calling"); + Map vLoupeIdToGexBarcodeDir = new HashMap<>(); + + } } @Override @@ -131,11 +137,10 @@ public void complete(PipelineJob job, List inputFiles, List< { if (CATEGORY.equals(so.getCategory())) { - processMetrics(so, job, true); + CellHashingService.get().processMetrics(so, job, true); } } - CellRangerVDJUtils utils = new CellRangerVDJUtils(job.getLogger(), job.getLogFile().getParentFile()); if (StringUtils.trimToNull(job.getParameters().get(TARGET_ASSAY)) == null) { job.getLogger().info("No assay selected, will not import"); @@ -157,7 +162,7 @@ public void complete(PipelineJob job, List inputFiles, List< for (SequenceOutputFile so : inputFiles) { AnalysisModel model = support.getCachedAnalysis(so.getAnalysis_id()); - utils.importAssayData(job, model, so.getFile().getParentFile(), assayId, null, deleteExistingData); + new CellRangerVDJUtils(job.getLogger()).importAssayData(job, model, so.getFile(), job.getLogFile().getParentFile(), assayId, null, deleteExistingData); } } } @@ -165,15 +170,13 @@ public void complete(PipelineJob job, List inputFiles, List< @Override public void processFilesRemote(List inputFiles, JobContext ctx) throws UnsupportedOperationException, PipelineJobException { - CellRangerVDJUtils utils = new CellRangerVDJUtils(ctx.getLogger(), ctx.getSourceDirectory()); RecordedAction action = new RecordedAction(getName()); - for (SequenceOutputFile so : inputFiles) { ctx.getLogger().info("processing file: " + so.getName()); //find TSV: - File perCellTsv = utils.getPerCellCsv(so.getFile().getParentFile()); + File perCellTsv = CellRangerVDJUtils.getPerCellCsv(so.getFile().getParentFile()); if (!perCellTsv.exists()) { throw new PipelineJobException("Unable to find file: " + perCellTsv.getPath()); @@ -197,140 +200,111 @@ else if (rs.getReadsetId() == null) private void processVloupeFile(JobContext ctx, File perCellTsv, Readset rs, RecordedAction action, Integer genomeId) throws PipelineJobException { - CellRangerVDJUtils utils = new CellRangerVDJUtils(ctx.getLogger(), ctx.getSourceDirectory()); - - List extraParams = new ArrayList<>(); - extraParams.addAll(getClientCommandArgs(ctx.getParams())); - - //prepare whitelist of cell indexes AlignmentOutputImpl output = new AlignmentOutputImpl(); - boolean scanEditDistances = ctx.getParams().optBoolean("scanEditDistances", false); - boolean useSeurat = ctx.getParams().optBoolean("useSeurat", true); - boolean useMultiSeq = ctx.getParams().optBoolean("useMultiSeq", true); - int minCountPerCell = ctx.getParams().optInt("minCountPerCell", 3); - int editDistance = ctx.getParams().optInt("editDistance", 2); - - File cellToHto = utils.runRemoteVdjCellHashingTasks(output, CATEGORY, perCellTsv, rs, ctx.getSequenceSupport(), extraParams, ctx.getWorkingDirectory(), ctx.getSourceDirectory(), editDistance, scanEditDistances, genomeId, minCountPerCell, useSeurat, useMultiSeq); - if (utils.useCellHashing(ctx.getSequenceSupport()) && cellToHto == null) + + Set htosPerReadset = CellHashingService.get().getHtosForParentReadset(rs.getReadsetId(), ctx.getSourceDirectory(), ctx.getSequenceSupport()); + if (htosPerReadset.size() > 1) { - throw new PipelineJobException("Missing cell to HTO file"); + ctx.getLogger().info("Total HTOs for readset: " + htosPerReadset.size()); - } + //TODO: allow union of GEX and TCR cell barcodes for whitelist! - ctx.getFileManager().addStepOutputs(action, output); + CellHashingService.CellHashingParameters parameters = CellHashingService.CellHashingParameters.createFromJson(CellHashingService.BARCODE_TYPE.hashing, ctx.getSourceDirectory(), ctx.getParams(), null, rs); + parameters.genomeId = genomeId; + parameters.outputCategory = CATEGORY; + parameters.basename = FileUtil.makeLegalName(rs.getName()); + parameters.allowableHtoBarcodes = htosPerReadset; + parameters.cellBarcodeWhitelistFile = createCellbarcodeWhitelist(ctx, perCellTsv, true); + File existingCountMatrixUmiDir = CellHashingService.get().getExistingFeatureBarcodeCountDir(rs, CellHashingService.BARCODE_TYPE.hashing, ctx.getSequenceSupport()); - } - } + File cellToHto = CellHashingService.get().generateHashingCallsForRawMatrix(rs, output, ctx, parameters, existingCountMatrixUmiDir); + if (CellHashingService.get().usesCellHashing(ctx.getSequenceSupport(), ctx.getSourceDirectory()) && cellToHto == null) + { + throw new PipelineJobException("Missing cell to HTO file"); + } - private static File getMetricsFile(File callFile) - { - return new File(callFile.getPath().replaceAll(".calls.txt", ".metrics.txt")); - } + action.addOutput(cellToHto, CellRangerVDJUtils.TCR_HASHING_CALLS, false); + ctx.getFileManager().addStepOutputs(action, output); + } + else if (htosPerReadset.size() == 1) + { + ctx.getLogger().info("Only single HTO used for lane, skipping cell hashing calling"); + } + else + { + ctx.getLogger().info("No HTOs found for readset"); + } + } - public static void processMetrics(SequenceOutputFile so, PipelineJob job, boolean updateDescription) throws PipelineJobException - { - if (so.getFile() != null) + private File createCellbarcodeWhitelist(JobContext ctx, File perCellTsv, boolean allowCellsLackingCDR3) throws PipelineJobException { - Map valueMap = new HashMap<>(); - - File metrics = getMetricsFile(so.getFile()); - if (metrics.exists()) + //prepare whitelist of cell indexes based on TCR calls: + File cellBarcodeWhitelist = new File(ctx.getSourceDirectory(), "validCellIndexes.csv"); + Set uniqueBarcodes = new HashSet<>(); + Set uniqueBarcodesIncludingNoCDR3 = new HashSet<>(); + ctx.getLogger().debug("writing cell barcodes, using file: " + perCellTsv.getPath()); + ctx.getLogger().debug("allow cells lacking CDR3: " + allowCellsLackingCDR3); + + int totalBarcodeWritten = 0; + try (CSVWriter writer = new CSVWriter(PrintWriters.getPrintWriter(cellBarcodeWhitelist), ',', CSVWriter.NO_QUOTE_CHARACTER); CSVReader reader = new CSVReader(Readers.getReader(perCellTsv), ',')) { - job.getLogger().info("Loading metrics"); - int total = 0; - TableInfo ti = DbSchema.get("sequenceanalysis", DbSchemaType.Module).getTable("quality_metrics"); - - //NOTE: if this job errored and restarted, we may have duplicate records: - SimpleFilter filter = new SimpleFilter(FieldKey.fromString("readset"), so.getReadset()); - filter.addCondition(FieldKey.fromString("analysis_id"), so.getAnalysis_id(), CompareType.EQUAL); - filter.addCondition(FieldKey.fromString("dataid"), so.getDataId(), CompareType.EQUAL); - filter.addCondition(FieldKey.fromString("container"), job.getContainer().getId(), CompareType.EQUAL); - TableSelector ts = new TableSelector(ti, PageFlowUtil.set("rowid"), filter, null); - if (ts.exists()) + int rowIdx = 0; + int noCallRows = 0; + int nonCell = 0; + String[] row; + while ((row = reader.readNext()) != null) { - job.getLogger().info("Deleting existing QC metrics (probably from prior restarted job)"); - ts.getArrayList(Integer.class).forEach(rowid -> { - Table.delete(ti, rowid); - }); - } - - try (CSVReader reader = new CSVReader(Readers.getReader(metrics), '\t')) - { - String[] line; - while ((line = reader.readNext()) != null) + //skip header + rowIdx++; + if (rowIdx > 1) { - if ("Category".equals(line[0])) + if ("False".equalsIgnoreCase(row[1])) { + nonCell++; continue; } - Map r = new HashMap<>(); - r.put("category", line[0]); - r.put("metricname", line[1]); - - //NOTE: R saves NaN as NA. This is fixed in the R code, but add this check here to let existing jobs import - String value = line[2]; - if ("NA".equals(value)) + //NOTE: allow these to pass for cell-hashing under some conditions + boolean hasCDR3 = !"None".equals(row[12]); + if (!hasCDR3) { - value = "0"; + noCallRows++; } - String fieldName = NumberUtils.isCreatable(value) ? "metricvalue" : "qualvalue"; - r.put(fieldName, value); - - r.put("analysis_id", so.getAnalysis_id()); - r.put("dataid", so.getDataId()); - r.put("readset", so.getReadset()); - r.put("container", job.getContainer()); - r.put("createdby", job.getUser().getUserId()); - - Table.insert(job.getUser(), ti, r); - total++; - - valueMap.put(line[1], value); - } - - job.getLogger().info("total metrics: " + total); - - if (updateDescription) - { - job.getLogger().debug("Updating description"); - StringBuilder description = new StringBuilder(); - if (StringUtils.trimToNull(so.getDescription()) != null) + //NOTE: 10x appends "-1" to barcodes + String barcode = row[0].split("-")[0]; + if ((allowCellsLackingCDR3 || hasCDR3) && !uniqueBarcodes.contains(barcode)) { - description.append(StringUtils.trimToNull(so.getDescription())); + writer.writeNext(new String[]{barcode}); + uniqueBarcodes.add(barcode); + totalBarcodeWritten++; } - String delim = description.length() > 0 ? "\n" : ""; - - DecimalFormat fmt = new DecimalFormat("##.##%"); - for (String metricName : Arrays.asList("InputBarcodes", "TotalCalled", "TotalCounts", "TotalSinglet", "FractionOfInputCalled", "FractionOfInputSinglet", "FractionOfInputDoublet", "FractionOfInputDiscordant", "FractionCalledNotInInput", "SeuratNonNegative", "MultiSeqNonNegative", "UniqueHtos", "UnknownTagMatchingKnown")) - { - if (valueMap.get(metricName) != null) - { - Double d = null; - if (metricName.startsWith("Fraction")) - { - try - { - d = ConvertHelper.convert(valueMap.get(metricName), Double.class); - } - catch (ConversionException | IllegalArgumentException e) - { - job.getLogger().error("Unable to convert to double: " + valueMap.get(metricName)); - throw e; - } - } - - description.append(delim).append(metricName).append(": ").append(d == null ? valueMap.get(metricName) : fmt.format(d)); - delim = ",\n"; - } - } + uniqueBarcodesIncludingNoCDR3.add(barcode); + } + } - so.setDescription(description.toString()); + ctx.getLogger().debug("rows inspected: " + (rowIdx - 1)); + ctx.getLogger().debug("rows without CDR3: " + noCallRows); + ctx.getLogger().debug("rows not called as cells: " + nonCell); + ctx.getLogger().debug("unique cell barcodes (with CDR3): " + uniqueBarcodes.size()); + ctx.getLogger().debug("unique cell barcodes (including no CDR3): " + uniqueBarcodesIncludingNoCDR3.size()); + ctx.getFileManager().addIntermediateFile(cellBarcodeWhitelist); + } + catch (IOException e) + { + throw new PipelineJobException(e); + } - TableInfo tableOutputs = DbSchema.get("sequenceanalysis", DbSchemaType.Module).getTable("outputfiles"); - Table.update(job.getUser(), tableOutputs, so, so.getRowid()); + if (uniqueBarcodes.size() < 500 && uniqueBarcodesIncludingNoCDR3.size() > uniqueBarcodes.size()) + { + ctx.getLogger().info("Total cell barcodes with CDR3s is low, so cell hashing will be performing using an input that includes valid cells that lacked CDR3 data."); + try (CSVWriter writer = new CSVWriter(PrintWriters.getPrintWriter(cellBarcodeWhitelist), ',', CSVWriter.NO_QUOTE_CHARACTER)) + { + for (String barcode : uniqueBarcodesIncludingNoCDR3) + { + writer.writeNext(new String[]{barcode}); + totalBarcodeWritten++; } } catch (IOException e) @@ -338,14 +312,15 @@ public static void processMetrics(SequenceOutputFile so, PipelineJob job, boolea throw new PipelineJobException(e); } } - else + + if (totalBarcodeWritten == 0) { - job.getLogger().warn("Unable to find metrics file: " + metrics.getPath()); + throw new PipelineJobException("No valid cell barcodes found!"); } - } - else - { - job.getLogger().warn("Unable to update metrics, file id is null: " + so.getName()); + + //TODO: consider looking up GEX data? + + return cellBarcodeWhitelist; } } } \ No newline at end of file diff --git a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJUtils.java b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJUtils.java index a71cf5bac..109117429 100644 --- a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJUtils.java +++ b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJUtils.java @@ -1,7 +1,6 @@ package org.labkey.tcrdb.pipeline; import au.com.bytecode.opencsv.CSVReader; -import au.com.bytecode.opencsv.CSVWriter; import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.Logger; import org.jetbrains.annotations.Nullable; @@ -10,8 +9,6 @@ import org.labkey.api.assay.AssayProvider; import org.labkey.api.assay.AssayService; import org.labkey.api.collections.CaseInsensitiveHashMap; -import org.labkey.api.data.ColumnInfo; -import org.labkey.api.data.CompareType; import org.labkey.api.data.Container; import org.labkey.api.data.SimpleFilter; import org.labkey.api.data.TableInfo; @@ -29,24 +26,19 @@ import org.labkey.api.query.InvalidKeyException; import org.labkey.api.query.QueryService; import org.labkey.api.query.QueryUpdateServiceException; -import org.labkey.api.query.UserSchema; import org.labkey.api.query.ValidationException; import org.labkey.api.reader.FastaDataLoader; import org.labkey.api.reader.FastaLoader; import org.labkey.api.reader.Readers; import org.labkey.api.security.User; -import org.labkey.api.sequenceanalysis.SequenceAnalysisService; -import org.labkey.api.sequenceanalysis.SequenceOutputFile; import org.labkey.api.sequenceanalysis.model.AnalysisModel; -import org.labkey.api.sequenceanalysis.model.Readset; -import org.labkey.api.sequenceanalysis.pipeline.PipelineStepOutput; -import org.labkey.api.sequenceanalysis.pipeline.SequenceAnalysisJobSupport; import org.labkey.api.sequenceanalysis.pipeline.SequencePipelineService; +import org.labkey.api.singlecell.CellHashingService; +import org.labkey.api.singlecell.model.CDNA_Library; import org.labkey.api.util.FileUtil; import org.labkey.api.util.PageFlowUtil; import org.labkey.api.view.ViewBackgroundInfo; import org.labkey.api.view.ViewContext; -import org.labkey.api.writer.PrintWriters; import org.labkey.tcrdb.TCRdbSchema; import java.io.File; @@ -56,420 +48,27 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; -import java.util.Date; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; public class CellRangerVDJUtils { - private Logger _log; - private File _sourceDir; + public static final String TCR_HASHING_CALLS = "Cell Hashing TCR Calls"; - public static final String READSET_TO_HASHING_MAP = "readsetToHashingMap"; - public static final String READSET_TO_CITESEQ_MAP = "readsetToCiteSeqMap"; - private static final String HASHING_CALLS = "Cell Hashing TCR Calls"; + private Logger _log; - public CellRangerVDJUtils(Logger log, File sourceDir) + public CellRangerVDJUtils(Logger log) { _log = log; - _sourceDir = sourceDir; } - public void prepareHashingAndCiteSeqFilesIfNeeded(PipelineJob job, SequenceAnalysisJobSupport support, String filterField, final boolean skipFailedCdna, boolean failIfNoHashing, boolean failIfNoCiteSeq) throws PipelineJobException + public void importAssayData(PipelineJob job, AnalysisModel model, File vLoupeFile, File outDir, Integer assayId, @Nullable Integer runId, boolean deleteExisting) throws PipelineJobException { - Container target = job.getContainer().isWorkbook() ? job.getContainer().getParent() : job.getContainer(); - UserSchema tcr = QueryService.get().getUserSchema(job.getUser(), target, TCRdbSchema.NAME); - TableInfo cDNAs = tcr.getTable(TCRdbSchema.TABLE_CDNAS, null); - - _log.debug("preparing cDNA and cell hashing files"); - - SequenceAnalysisService.get().writeAllCellHashingBarcodes(_sourceDir, job.getUser(), job.getContainer()); - SequenceAnalysisService.get().writeAllCiteSeqBarcodes(_sourceDir, job.getUser(), job.getContainer()); - - Map colMap = QueryService.get().getColumns(cDNAs, PageFlowUtil.set( - FieldKey.fromString("rowid"), - FieldKey.fromString("sortId/stimId/animalId"), - FieldKey.fromString("sortId/stimId/stim"), - FieldKey.fromString("sortId/population"), - FieldKey.fromString("sortId/hto"), - FieldKey.fromString("sortId/hto/sequence"), - FieldKey.fromString("hashingReadsetId"), - FieldKey.fromString("hashingReadsetId/totalFiles"), - FieldKey.fromString("citeseqReadsetId"), - FieldKey.fromString("citeseqReadsetId/totalFiles"), - FieldKey.fromString("citeseqPanel"), - FieldKey.fromString("status")) - ); - - File output = getCDNAInfoFile(); - File barcodeOutput = getValidHashingBarcodeFile(); - HashMap readsetToHashingMap = new HashMap<>(); - HashMap readsetToCiteSeqMap = new HashMap<>(); - HashMap> gexToPanels = new HashMap<>(); - - try (CSVWriter writer = new CSVWriter(PrintWriters.getPrintWriter(output), '\t', CSVWriter.NO_QUOTE_CHARACTER); CSVWriter bcWriter = new CSVWriter(PrintWriters.getPrintWriter(barcodeOutput), ',', CSVWriter.NO_QUOTE_CHARACTER)) - { - writer.writeNext(new String[]{"ReadsetId", "CDNA_ID", "AnimalId", "Stim", "Population", "HashingReadsetId", "HasHashingReads", "HTO_Name", "HTO_Seq", "CiteSeqReadsetId", "HasCiteSeqReads", "CiteSeqPanel"}); - List cachedReadsets = support.getCachedReadsets(); - Set distinctHTOs = new HashSet<>(); - Set hashingStatus = new HashSet<>(); - Set citeseqStatus = new HashSet<>(); - AtomicInteger totalWritten = new AtomicInteger(0); - for (Readset rs : cachedReadsets) - { - AtomicBoolean hasError = new AtomicBoolean(false); - //find cDNA records using this readset - new TableSelector(cDNAs, colMap.values(), new SimpleFilter(FieldKey.fromString(filterField), rs.getRowId()), null).forEachResults(results -> { - if (skipFailedCdna && results.getObject(FieldKey.fromString("status")) != null) - { - _log.info("skipping cDNA with non-null status: " + results.getString(FieldKey.fromString("rowid"))); - return; - } - - writer.writeNext(new String[]{ - String.valueOf(rs.getRowId()), - results.getString(FieldKey.fromString("rowid")), - results.getString(FieldKey.fromString("sortId/stimId/animalId")), - results.getString(FieldKey.fromString("sortId/stimId/stim")), - results.getString(FieldKey.fromString("sortId/population")), - String.valueOf(results.getObject(FieldKey.fromString("hashingReadsetId")) == null ? "" : results.getInt(FieldKey.fromString("hashingReadsetId"))), - String.valueOf(results.getObject(FieldKey.fromString("hashingReadsetId/totalFiles")) != null && results.getInt(FieldKey.fromString("hashingReadsetId/totalFiles")) > 0), - results.getString(FieldKey.fromString("sortId/hto")), - results.getString(FieldKey.fromString("sortId/hto/sequence")), - String.valueOf(results.getObject(FieldKey.fromString("citeseqReadsetId")) == null ? "" : results.getInt(FieldKey.fromString("citeseqReadsetId"))), - String.valueOf(results.getObject(FieldKey.fromString("citeseqReadsetId/totalFiles")) != null && results.getInt(FieldKey.fromString("citeseqReadsetId/totalFiles")) > 0), - results.getString(FieldKey.fromString("citeseqPanel")) - }); - totalWritten.getAndIncrement(); - - boolean useCellHashing = results.getObject(FieldKey.fromString("sortId/hto")) != null; - hashingStatus.add(useCellHashing); - if (useCellHashing) - { - if (results.getObject(FieldKey.fromString("hashingReadsetId")) == null) - { - job.getLogger().error("cDNA specifies HTO, but does not list a hashing readset: " + results.getString(FieldKey.fromString("rowid"))); - hasError.set(true); - } - else - { - readsetToHashingMap.put(rs.getReadsetId(), results.getInt(FieldKey.fromString("hashingReadsetId"))); - - String hto = results.getString(FieldKey.fromString("sortId/hto")) + "<>" + results.getString(FieldKey.fromString("sortId/hto/sequence")); - if (!distinctHTOs.contains(hto) && !StringUtils.isEmpty(results.getString(FieldKey.fromString("sortId/hto/sequence")))) - { - distinctHTOs.add(hto); - bcWriter.writeNext(new String[]{results.getString(FieldKey.fromString("sortId/hto/sequence")), results.getString(FieldKey.fromString("sortId/hto"))}); - } - - if (results.getObject(FieldKey.fromString("sortId/hto/sequence")) == null) - { - job.getLogger().error("Unable to find sequence for HTO: " + results.getString(FieldKey.fromString("sortId/hto"))); - hasError.set(true); - } - } - } - - boolean useCiteSeq = results.getObject(FieldKey.fromString("citeseqPanel")) != null; - citeseqStatus.add(useCiteSeq); - if (useCiteSeq) - { - if (results.getObject(FieldKey.fromString("citeseqReadsetId")) == null) - { - job.getLogger().error("cDNA specifies cite-seq readset but does not list panel: " + results.getString(FieldKey.fromString("rowid"))); - hasError.set(true); - } - else - { - Set panels = gexToPanels.getOrDefault(rs.getRowId(), new HashSet<>()); - panels.add(results.getString(FieldKey.fromString("citeseqPanel"))); - gexToPanels.put(rs.getRowId(), panels); - - readsetToCiteSeqMap.put(rs.getReadsetId(), results.getInt(FieldKey.fromString("citeseqReadsetId"))); - } - } - }); - - if (hasError.get()) - { - throw new PipelineJobException("No cell hashing readset or HTO found for one or more cDNAs. see the file: " + output.getName()); - } - - if (hashingStatus.size() > 1) - { - _log.info("The selected readsets/cDNA records use a mixture of cell hashing and non-hashing."); - } - - //NOTE: hashingStatus.isEmpty() indicates there are no cDNA records associated with the data - } - - // if distinct HTOs is 1, no point in running hashing. note: presence of hashing readsets is a trigger downstream - if (distinctHTOs.size() > 1) - { - readsetToHashingMap.forEach((readsetId, hashingReadsetId) -> support.cacheReadset(hashingReadsetId, job.getUser())); - } - else if (distinctHTOs.size() == 1) - { - job.getLogger().info("There is only a single HTO in this pool, will not use hashing"); - } - - if (totalWritten.get() == 0) - { - throw new PipelineJobException("No matching cDNA records found"); - } - - boolean useCellHashing = hashingStatus.isEmpty() ? false : hashingStatus.size() > 1 ? true : hashingStatus.iterator().next(); - if (useCellHashing && distinctHTOs.isEmpty()) - { - throw new PipelineJobException("Cell hashing was selected, but no HTOs were found"); - } - else - { - _log.info("distinct HTOs: " + distinctHTOs.size()); - } - - support.cacheObject(READSET_TO_HASHING_MAP, readsetToHashingMap); - support.cacheObject(READSET_TO_CITESEQ_MAP, readsetToCiteSeqMap); - readsetToCiteSeqMap.forEach((readsetId, citeseqReadsetId) -> support.cacheReadset(citeseqReadsetId, job.getUser())); - } - catch (IOException e) - { - throw new PipelineJobException(e); - } - - writeCiteSeqBarcodes(job, gexToPanels, _sourceDir); - - if (failIfNoHashing && readsetToHashingMap.isEmpty()) - { - throw new PipelineJobException("Readsets do not use cell hashing"); - } - - if (failIfNoCiteSeq && readsetToCiteSeqMap.isEmpty()) - { - throw new PipelineJobException("Readsets do not use CITE-seq"); - } - } - - public static File getValidCiteSeqBarcodeFile(File sourceDir, int gexReadsetId) - { - return new File(sourceDir, "validADTS." + gexReadsetId + ".csv"); - } - - public static File getValidCiteSeqBarcodeMetadataFile(File sourceDir, int gexReadsetId) - { - return new File(sourceDir, "validADTS." + gexReadsetId + ".metadata.txt"); - } - - private void writeCiteSeqBarcodes(PipelineJob job, Map> gexToPanels, File outputDir) throws PipelineJobException - { - Container target = job.getContainer().isWorkbook() ? job.getContainer().getParent() : job.getContainer(); - UserSchema tcr = QueryService.get().getUserSchema(job.getUser(), target, TCRdbSchema.NAME); - TableInfo panels = tcr.getTable(TCRdbSchema.TABLE_CITE_SEQ_PANELS, null); - - Map barcodeColMap = QueryService.get().getColumns(panels, PageFlowUtil.set( - FieldKey.fromString("antibody"), - FieldKey.fromString("antibody/markerName"), - FieldKey.fromString("antibody/markerLabel"), - FieldKey.fromString("markerLabel"), - FieldKey.fromString("antibody/adaptersequence") - )); - - for (int gexReadsetId : gexToPanels.keySet()) - { - job.getLogger().info("Writing all unique ADTs for readset: " + gexReadsetId); - File barcodeOutput = getValidCiteSeqBarcodeFile(outputDir, gexReadsetId); - File metadataOutput = getValidCiteSeqBarcodeMetadataFile(outputDir, gexReadsetId); - try (CSVWriter writer = new CSVWriter(PrintWriters.getPrintWriter(barcodeOutput), ',', CSVWriter.NO_QUOTE_CHARACTER);CSVWriter metaWriter = new CSVWriter(PrintWriters.getPrintWriter(metadataOutput), '\t', CSVWriter.NO_QUOTE_CHARACTER)) - { - metaWriter.writeNext(new String[]{"tagname", "sequence", "markername", "markerlabel"}); - AtomicInteger barcodeCount = new AtomicInteger(); - Set found = new HashSet<>(); - new TableSelector(panels, barcodeColMap.values(), new SimpleFilter(FieldKey.fromString("name"), gexToPanels.get(gexReadsetId), CompareType.IN), new org.labkey.api.data.Sort("antibody")).forEachResults(results -> { - if (found.contains(results.getString(FieldKey.fromString("antibody/adaptersequence")))) - { - return; - } - - found.add(results.getString(FieldKey.fromString("antibody/adaptersequence"))); - barcodeCount.getAndIncrement(); - - writer.writeNext(new String[]{results.getString(FieldKey.fromString("antibody/adaptersequence")), results.getString(FieldKey.fromString("antibody"))}); - - //allow aliasing based on DB - String label = StringUtils.trimToNull(results.getString(FieldKey.fromString("markerLabel"))) == null ? results.getString(FieldKey.fromString("antibody/markerLabel")) : results.getString(FieldKey.fromString("markerLabel")); - String name = StringUtils.trimToNull(results.getString(FieldKey.fromString("markerLabel"))) != null ? results.getString(FieldKey.fromString("markerLabel")) : - StringUtils.trimToNull(results.getString(FieldKey.fromString("antibody/markerName"))) != null ? results.getString(FieldKey.fromString("antibody/markerName")) : results.getString(FieldKey.fromString("antibody")); - metaWriter.writeNext(new String[]{results.getString(FieldKey.fromString("antibody")), results.getString(FieldKey.fromString("antibody/adaptersequence")), name, label}); - }); - - job.getLogger().info("Total CITE-seq barcodes written: " + barcodeCount.get()); - } - catch (IOException e) - { - throw new PipelineJobException(e); - } - } - } - - public File getCDNAInfoFile() - { - return getCDNAInfoFile(_sourceDir); - } + File cellRangerOutDir = vLoupeFile.getParentFile(); - public static File getCDNAInfoFile(File sourceDir) - { - return new File(sourceDir, "cDNAInfo.txt"); - } - - public File getValidHashingBarcodeFile() - { - return getValidHashingBarcodeFile(_sourceDir); - } - - public static File getValidHashingBarcodeFile(File sourceDir) - { - return new File(sourceDir, "validHashingBarcodes.csv"); - } - - public File getValidCellIndexFile() - { - return new File(_sourceDir, "validCellIndexes.csv"); - } - - public File getPerCellCsv(File outDir) - { - return new File(outDir, "all_contig_annotations.csv"); - } - - public File runRemoteVdjCellHashingTasks(PipelineStepOutput output, String outputCategory, File perCellTsv, Readset rs, SequenceAnalysisJobSupport support, List extraParams, File workingDir, File sourceDir, Integer editDistance, boolean scanEditDistances, Integer genomeId, Integer minCountPerCell, boolean useSeurat, boolean useMultiSeq) throws PipelineJobException - { - Map readsetToHashing = getCachedHashingReadsetMap(support); - if (readsetToHashing.isEmpty()) - { - _log.info("No cached hashing readsets, skipping"); - return null; - } - - //prepare whitelist of barcodes, based on cDNA records - File htoBarcodeWhitelist = getValidHashingBarcodeFile(); - if (!htoBarcodeWhitelist.exists()) - { - throw new PipelineJobException("Unable to find file: " + htoBarcodeWhitelist.getPath()); - } - - long lineCount = SequencePipelineService.get().getLineCount(htoBarcodeWhitelist); - if (lineCount == 1) - { - _log.info("Only one HTO is used, will not use hashing"); - return null; - } - - _log.debug("total cached readset/hashing readset pairs: " + readsetToHashing.size()); - _log.debug("unique HTOs: " + lineCount); - - //prepare whitelist of cell indexes - File cellBarcodeWhitelist = getValidCellIndexFile(); - Set uniqueBarcodes = new HashSet<>(); - Set uniqueBarcodesIncludingNoCDR3 = new HashSet<>(); - _log.debug("writing cell barcodes"); - try (CSVWriter writer = new CSVWriter(PrintWriters.getPrintWriter(cellBarcodeWhitelist), ',', CSVWriter.NO_QUOTE_CHARACTER); CSVReader reader = new CSVReader(Readers.getReader(perCellTsv), ',')) - { - int rowIdx = 0; - int noCallRows = 0; - int nonCell = 0; - String[] row; - while ((row = reader.readNext()) != null) - { - //skip header - rowIdx++; - if (rowIdx > 1) - { - if ("False".equalsIgnoreCase(row[1])) - { - nonCell++; - continue; - } - - //NOTE: allow these to pass for cell-hashing under some conditions - boolean hasCDR3 = !"None".equals(row[12]); - if (!hasCDR3) - { - noCallRows++; - } - - //NOTE: 10x appends "-1" to barcodes - String barcode = row[0].split("-")[0]; - if (hasCDR3 && !uniqueBarcodes.contains(barcode)) - { - writer.writeNext(new String[]{barcode}); - uniqueBarcodes.add(barcode); - } - - uniqueBarcodesIncludingNoCDR3.add(barcode); - } - } - - _log.debug("rows inspected: " + (rowIdx - 1)); - _log.debug("rows without CDR3: " + noCallRows); - _log.debug("rows not called as cells: " + nonCell); - _log.debug("unique cell barcodes (with CDR3): " + uniqueBarcodes.size()); - _log.debug("unique cell barcodes (including no CDR3): " + uniqueBarcodesIncludingNoCDR3.size()); - output.addIntermediateFile(cellBarcodeWhitelist); - } - catch (IOException e) - { - throw new PipelineJobException(e); - } - - if (uniqueBarcodes.size() < 500 && uniqueBarcodesIncludingNoCDR3.size() > uniqueBarcodes.size()) - { - _log.info("Total cell barcodes with CDR3s is low, so cell hashing will be performing using an input that includes valid cells that lacked CDR3 data."); - try (CSVWriter writer = new CSVWriter(PrintWriters.getPrintWriter(cellBarcodeWhitelist), ',', CSVWriter.NO_QUOTE_CHARACTER)) - { - for (String barcode : uniqueBarcodesIncludingNoCDR3) - { - writer.writeNext(new String[]{barcode}); - } - } - catch (IOException e) - { - throw new PipelineJobException(e); - } - } - - Readset htoReadset = support.getCachedReadset(readsetToHashing.get(rs.getReadsetId())); - if (htoReadset == null) - { - throw new PipelineJobException("Unable to find HTO readset for readset: " + rs.getRowId()); - } - - //run CiteSeqCount. this will use Multiseq to make calls per cell - String basename = FileUtil.makeLegalName(rs.getName()); - File hashtagCalls = SequencePipelineService.get().runCiteSeqCount(output, outputCategory, htoReadset, htoBarcodeWhitelist, cellBarcodeWhitelist, workingDir, basename, _log, extraParams, false, minCountPerCell, sourceDir, editDistance, scanEditDistances, rs, genomeId, true, true, useSeurat, useMultiSeq); - if (!hashtagCalls.exists()) - { - throw new PipelineJobException("Unable to find expected file: " + hashtagCalls.getPath()); - } - output.addOutput(hashtagCalls, HASHING_CALLS); - - File html = new File(hashtagCalls.getParentFile(), FileUtil.getBaseName(FileUtil.getBaseName(hashtagCalls.getName())) + ".html"); - if (!html.exists()) - { - throw new PipelineJobException("Unable to find HTML file: " + html.getPath()); - } - - output.addOutput(html, "Cell Hashing TCR Report"); - - return hashtagCalls; - } - - public void importAssayData(PipelineJob job, AnalysisModel model, File outDir, Integer assayId, @Nullable Integer runId, boolean deleteExisting) throws PipelineJobException - { if (assayId == null) { _log.info("No assay selected, will not import"); @@ -482,32 +81,29 @@ public void importAssayData(PipelineJob job, AnalysisModel model, File outDir, I throw new PipelineJobException("Unable to find protocol: " + assayId); } - File allCsv = getPerCellCsv(outDir); + File allCsv = getPerCellCsv(cellRangerOutDir); if (!allCsv.exists()) { _log.warn("unable to find consensus contigs: " + allCsv .getPath()); return; } - File consensusCsv = new File(outDir, "consensus_annotations.csv"); + File consensusCsv = new File(cellRangerOutDir, "consensus_annotations.csv"); if (!consensusCsv .exists()) { - _log.warn("unable to find consensus contigs: " + consensusCsv .getPath()); - return; + throw new PipelineJobException("unable to find consensus contigs: " + consensusCsv .getPath()); } - File consensusFasta = new File(outDir, "consensus.fasta"); + File consensusFasta = new File(cellRangerOutDir, "consensus.fasta"); if (!consensusFasta.exists()) { - _log.warn("unable to find FASTA: " + consensusFasta.getPath()); - return; + throw new PipelineJobException("unable to find FASTA: " + consensusFasta.getPath()); } - File allFasta = new File(outDir, "all_contig.fasta"); + File allFasta = new File(cellRangerOutDir, "all_contig.fasta"); if (!allFasta.exists()) { - _log.warn("unable to find FASTA: " + allFasta.getPath()); - return; + throw new PipelineJobException("unable to find FASTA: " + allFasta.getPath()); } _log.info("loading results into assay: " + assayId); @@ -521,9 +117,9 @@ public void importAssayData(PipelineJob job, AnalysisModel model, File outDir, I job.getLogger().debug("Using supplied runId: " + runId); } - File cDNAFile = getCDNAInfoFile(); - Map htoNameToCDNAMap = new HashMap<>(); - Map cDNAMap = new HashMap<>(); + File cDNAFile = CellHashingService.get().getCDNAInfoFile(outDir); + Map htoNameToCDNAMap = new HashMap<>(); + Map cDNAMap = new HashMap<>(); if (cDNAFile.exists()) { try (CSVReader reader = new CSVReader(Readers.getReader(cDNAFile), '\t')) @@ -539,7 +135,7 @@ public void importAssayData(PipelineJob job, AnalysisModel model, File outDir, I String htoName = StringUtils.trimToNull(line[7]); - CDNA cdna = CDNA.getRowId(Integer.parseInt(line[1])); + CDNA_Library cdna = CellHashingService.get().getLibraryById(Integer.parseInt(line[1])); cDNAMap.put(Integer.parseInt(line[1]), cdna); if (htoName != null) { @@ -600,15 +196,28 @@ public void importAssayData(PipelineJob job, AnalysisModel model, File outDir, I int doublet = 0; int discordant = 0; int negative = 0; + + int consensusIdx = -1; while ((line = reader.readNext()) != null) { + if (line.length < 3) + { + throw new PipelineJobException("Line too short"); + } + //header - if ("CellBarcode".equals(line[0])) + if ("cellbarcode".equalsIgnoreCase(line[0])) { + consensusIdx = Arrays.asList(line).indexOf("consensuscall"); continue; } - String hto = line[1]; + if (consensusIdx == -1) + { + throw new PipelineJobException("consensuscall column not found"); + } + + String hto = line[consensusIdx]; if ("Doublet".equals(hto)) { doublet++; @@ -627,7 +236,7 @@ else if ("Negative".equals(hto)) continue; } - CDNA cDNA = htoNameToCDNAMap.get(hto); + CDNA_Library cDNA = htoNameToCDNAMap.get(hto); if (cDNA == null) { _log.warn("Unable to find cDNA record for hto: " + hto); @@ -696,10 +305,15 @@ else if ("Negative".equals(hto)) continue; } - if ("None".equals(line[9])) + String cGene = removeNone(line[9]); + if (cGene == null) { - noCGene++; - continue; + // Only discard these if chain type doesnt match between JGene and VGene. + if (!line[8].substring(0, 3).equals(line[6].substring(0,3))) + { + noCGene++; + continue; + } } if ("False".equals(line[10])) @@ -747,9 +361,9 @@ else if (discordantBarcodes.contains(barcode)) //NOTE: chimeras with a TRDV / TRAJ / TRAC are relatively common. categorize as TRA for reporting ease String locus = line[5]; - if (locus.equals("Multi") && removeNone(line[9]) != null && removeNone(line[8]) != null && removeNone(line[6]) != null) + if (locus.equals("Multi") && cGene != null && removeNone(line[8]) != null && removeNone(line[6]) != null) { - if (removeNone(line[9]).contains("TRAC") && removeNone(line[8]).contains("TRAJ") && removeNone(line[6]).contains("TRDV")) + if (cGene.contains("TRAC") && removeNone(line[8]).contains("TRAJ") && removeNone(line[6]).contains("TRDV")) { locus = "TRA"; multiChainConverted++; @@ -757,7 +371,7 @@ else if (discordantBarcodes.contains(barcode)) } // Aggregate by: cDNA_ID, cdr3, chain, raw_clonotype_id, sequenceContigName, vHit, dHit, jHit, cHit, cdr3_nt - String key = StringUtils.join(new String[]{cDNA.toString(), line[12], locus, clonotypeId, sequenceContigName, removeNone(line[6]), removeNone(line[7]), removeNone(line[8]), removeNone(line[9]), removeNone(line[13])}, "<>"); + String key = StringUtils.join(new String[]{cDNA.toString(), line[12], locus, clonotypeId, sequenceContigName, removeNone(line[6]), removeNone(line[7]), removeNone(line[8]), cGene, removeNone(line[13])}, "<>"); AssayModel am; if (!rows.containsKey(key)) { @@ -865,7 +479,7 @@ private AssayModel createForRow(String[] line, String sequenceContigName, Intege private File getCellToHtoFile(ExpRun run) throws PipelineJobException { - List datas = run.getInputDatas(HASHING_CALLS, ExpProtocol.ApplicationType.ExperimentRunOutput); + List datas = run.getInputDatas(TCR_HASHING_CALLS, ExpProtocol.ApplicationType.ExperimentRunOutput); if (datas.isEmpty()) { throw new PipelineJobException("Unable to find hashing calls output"); @@ -901,9 +515,9 @@ private static class AssayModel private String sequenceContigName; } - private Map processRow(AssayModel assayModel, AnalysisModel model, Map cDNAMap, Integer runId, Map> totalCellsBySample, Map sequenceMap) throws PipelineJobException + private Map processRow(AssayModel assayModel, AnalysisModel model, Map cDNAMap, Integer runId, Map> totalCellsBySample, Map sequenceMap) throws PipelineJobException { - CDNA cDNARecord = cDNAMap.get(assayModel.cdna); + CDNA_Library cDNARecord = cDNAMap.get(assayModel.cdna); if (cDNARecord == null) { throw new PipelineJobException("Unable to find cDNA for ID: " + assayModel.cdna); @@ -912,8 +526,8 @@ private Map processRow(AssayModel assayModel, AnalysisModel mode Map row = new CaseInsensitiveHashMap<>(); row.put("sampleName", cDNARecord.getAssaySampleName()); - row.put("subjectId", cDNARecord.getSortRecord().getStimRecord().getAnimalId()); - row.put("sampleDate", cDNARecord.getSortRecord().getStimRecord().getDate()); + row.put("subjectId", cDNARecord.getSortRecord().getSampleRecord().getSubjectId()); + row.put("sampleDate", cDNARecord.getSortRecord().getSampleRecord().getSampledate()); row.put("cDNA", assayModel.cdna); row.put("alignmentId", model.getAlignmentFile()); @@ -963,7 +577,7 @@ private void saveRun(PipelineJob job, ExpProtocol protocol, AnalysisModel model, if (model.getLibraryId() != null) { - TableSelector ts = new TableSelector(TCRdbSchema.getInstance().getSchema().getTable(TCRdbSchema.TABLE_LIBRARIES), PageFlowUtil.set("rowid"), new SimpleFilter(FieldKey.fromString("libraryId"), model.getLibraryId()), null); + TableSelector ts = new TableSelector(TCRdbSchema.getInstance().getSchema().getTable(TCRdbSchema.TABLE_MIXCR_LIBRARIES), PageFlowUtil.set("rowid"), new SimpleFilter(FieldKey.fromString("libraryId"), model.getLibraryId()), null); if (ts.exists()) { int mixcrId = ts.getObject(Integer.class); @@ -1052,307 +666,8 @@ public static void deleteExistingData(AssayProvider ap, ExpProtocol protocol, Co } } - public static Map getCachedHashingReadsetMap(SequenceAnalysisJobSupport support) throws PipelineJobException - { - return support.getCachedObject(CellRangerVDJUtils.READSET_TO_HASHING_MAP, PipelineJob.createObjectMapper().getTypeFactory().constructParametricType(Map.class, Integer.class, Integer.class)); - } - - public static Map getCachedCiteSeqReadsetMap(SequenceAnalysisJobSupport support) throws PipelineJobException - { - return support.getCachedObject(CellRangerVDJUtils.READSET_TO_CITESEQ_MAP, PipelineJob.createObjectMapper().getTypeFactory().constructParametricType(Map.class, Integer.class, Integer.class)); - } - - //NOTE: if readset ID is null, this will be interpreted as any readset using hashing - public boolean useCellHashing(SequenceAnalysisJobSupport support) throws PipelineJobException - { - Map gexToHashingMap = getCachedHashingReadsetMap(support); - if (gexToHashingMap == null || gexToHashingMap.isEmpty()) - return false; - - File htoBarcodeWhitelist = getValidHashingBarcodeFile(); - if (!htoBarcodeWhitelist.exists()) - { - throw new PipelineJobException("Unable to find file: " + htoBarcodeWhitelist.getPath()); - } - - return SequencePipelineService.get().getLineCount(htoBarcodeWhitelist) > 1; - } - - //NOTE: if readset ID is null, this will be interpreted as any readset using hashing - public boolean useCiteSeq(SequenceAnalysisJobSupport support, List inputFiles) throws PipelineJobException + public static File getPerCellCsv(File cellRangerOutDir) { - Map gexToCiteMap = getCachedCiteSeqReadsetMap(support); - if (gexToCiteMap == null || gexToCiteMap.isEmpty()) - return false; - - for (SequenceOutputFile so : inputFiles) - { - if (gexToCiteMap.containsKey(so.getReadset())) - { - return true; - } - } - - return false; - } - - public static class CDNA - { - private int _rowId; - private Integer _sortId; - private String _chemistry; - private Double _concentration; - private String _plateId; - private String _well; - - private Integer _readsetId; - private Integer _enrichedReadsetId; - private Integer _hashingReadsetId; - private String _container; - - private Sort _sortRecord; - - public int getRowId() - { - return _rowId; - } - - public void setRowId(int rowId) - { - _rowId = rowId; - } - - public Integer getSortId() - { - return _sortId; - } - - public void setSortId(Integer sortId) - { - _sortId = sortId; - } - - public String getChemistry() - { - return _chemistry; - } - - public void setChemistry(String chemistry) - { - _chemistry = chemistry; - } - - public Double getConcentration() - { - return _concentration; - } - - public void setConcentration(Double concentration) - { - _concentration = concentration; - } - - public String getPlateId() - { - return _plateId; - } - - public void setPlateId(String plateId) - { - _plateId = plateId; - } - - public String getWell() - { - return _well; - } - - public void setWell(String well) - { - _well = well; - } - - public Integer getReadsetId() - { - return _readsetId; - } - - public void setReadsetId(Integer readsetId) - { - _readsetId = readsetId; - } - - public Integer getEnrichedReadsetId() - { - return _enrichedReadsetId; - } - - public void setEnrichedReadsetId(Integer enrichedReadsetId) - { - _enrichedReadsetId = enrichedReadsetId; - } - - public Integer getHashingReadsetId() - { - return _hashingReadsetId; - } - - public void setHashingReadsetId(Integer hashingReadsetId) - { - _hashingReadsetId = hashingReadsetId; - } - - public String getContainer() - { - return _container; - } - - public void setContainer(String container) - { - _container = container; - } - - public Sort getSortRecord() - { - if (_sortRecord == null) - { - _sortRecord = Sort.getRowId(_sortId); - } - - return _sortRecord; - } - - public String getAssaySampleName() - { - return getPlateId() + "_" + getWell() + "_" + getSortRecord().getStimRecord().getAnimalId() + "_" + getSortRecord().getStimRecord().getStim() + "_" + getSortRecord().getPopulation() + (getSortRecord().getHto() == null ? "" : "_" + getSortRecord().getHto()); - } - - public static CDNA getRowId(int rowId) - { - return new TableSelector(TCRdbSchema.getInstance().getSchema().getTable(TCRdbSchema.TABLE_CDNAS)).getObject(rowId, CDNA.class); - } - } - - public static class Sort - { - private int _rowId; - private Integer _stimId; - private String _population; - private String _hto; - - private Stim _stimRecord; - - public Stim getStimRecord() - { - if (_stimRecord == null) - { - _stimRecord = Stim.getRowId(_stimId); - } - - return _stimRecord; - } - - public int getRowId() - { - return _rowId; - } - - public void setRowId(int rowId) - { - _rowId = rowId; - } - - public Integer getStimId() - { - return _stimId; - } - - public void setStimId(Integer stimId) - { - _stimId = stimId; - } - - public String getPopulation() - { - return _population; - } - - public void setPopulation(String population) - { - _population = population; - } - - public String getHto() - { - return _hto; - } - - public void setHto(String hto) - { - _hto = hto; - } - - public void setStimRecord(Stim stimRecord) - { - _stimRecord = stimRecord; - } - - public static Sort getRowId(int rowId) - { - return new TableSelector(TCRdbSchema.getInstance().getSchema().getTable(TCRdbSchema.TABLE_SORTS)).getObject(rowId, Sort.class); - } - } - - public static class Stim - { - private int _rowId; - private String _animalId; - private String _stim; - private Date _date; - - public int getRowId() - { - return _rowId; - } - - public void setRowId(int rowId) - { - _rowId = rowId; - } - - public String getAnimalId() - { - return _animalId; - } - - public void setAnimalId(String animalId) - { - _animalId = animalId; - } - - public String getStim() - { - return _stim; - } - - public void setStim(String stim) - { - _stim = stim; - } - - public Date getDate() - { - return _date; - } - - public void setDate(Date date) - { - _date = date; - } - - public static Stim getRowId(int rowId) - { - return new TableSelector(TCRdbSchema.getInstance().getSchema().getTable(TCRdbSchema.TABLE_STIMS)).getObject(rowId, Stim.class); - } + return new File(cellRangerOutDir, "all_contig_annotations.csv"); } } diff --git a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJWrapper.java b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJWrapper.java deleted file mode 100644 index 1fc0aaf0d..000000000 --- a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJWrapper.java +++ /dev/null @@ -1,693 +0,0 @@ -package org.labkey.tcrdb.pipeline; - -import au.com.bytecode.opencsv.CSVReader; -import org.apache.commons.io.FileUtils; -import org.apache.commons.lang3.StringUtils; -import org.apache.logging.log4j.Logger; -import org.jetbrains.annotations.Nullable; -import org.json.JSONObject; -import org.labkey.api.collections.CaseInsensitiveHashMap; -import org.labkey.api.data.CompareType; -import org.labkey.api.data.ConvertHelper; -import org.labkey.api.data.DbSchema; -import org.labkey.api.data.DbSchemaType; -import org.labkey.api.data.SimpleFilter; -import org.labkey.api.data.Table; -import org.labkey.api.data.TableInfo; -import org.labkey.api.data.TableSelector; -import org.labkey.api.pipeline.PipelineJob; -import org.labkey.api.pipeline.PipelineJobException; -import org.labkey.api.query.FieldKey; -import org.labkey.api.query.QueryService; -import org.labkey.api.query.UserSchema; -import org.labkey.api.reader.Readers; -import org.labkey.api.sequenceanalysis.RefNtSequenceModel; -import org.labkey.api.sequenceanalysis.model.AnalysisModel; -import org.labkey.api.sequenceanalysis.model.ReadData; -import org.labkey.api.sequenceanalysis.model.Readset; -import org.labkey.api.sequenceanalysis.pipeline.AbstractAlignmentStepProvider; -import org.labkey.api.sequenceanalysis.pipeline.AlignerIndexUtil; -import org.labkey.api.sequenceanalysis.pipeline.AlignmentOutputImpl; -import org.labkey.api.sequenceanalysis.pipeline.AlignmentStep; -import org.labkey.api.sequenceanalysis.pipeline.AlignmentStepProvider; -import org.labkey.api.sequenceanalysis.pipeline.CommandLineParam; -import org.labkey.api.sequenceanalysis.pipeline.IndexOutputImpl; -import org.labkey.api.sequenceanalysis.pipeline.PipelineContext; -import org.labkey.api.sequenceanalysis.pipeline.ReferenceGenome; -import org.labkey.api.sequenceanalysis.pipeline.SequenceAnalysisJobSupport; -import org.labkey.api.sequenceanalysis.pipeline.SequencePipelineService; -import org.labkey.api.sequenceanalysis.pipeline.ToolParameterDescriptor; -import org.labkey.api.sequenceanalysis.run.AbstractAlignmentPipelineStep; -import org.labkey.api.sequenceanalysis.run.AbstractCommandWrapper; -import org.labkey.api.sequenceanalysis.run.SimpleScriptWrapper; -import org.labkey.api.util.FileUtil; -import org.labkey.api.util.PageFlowUtil; -import org.labkey.api.writer.PrintWriters; - -import java.io.File; -import java.io.IOException; -import java.io.PrintWriter; -import java.nio.file.Files; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Date; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -public class CellRangerVDJWrapper extends AbstractCommandWrapper -{ - public CellRangerVDJWrapper(@Nullable Logger logger) - { - super(logger); - } - - public static final String TARGET_ASSAY = "targetAssay"; - public static final String DELETE_EXISTING_ASSAY_DATA = "deleteExistingAssayData"; - - public static class VDJProvider extends AbstractAlignmentStepProvider - { - public VDJProvider() - { - super("CellRanger VDJ", "Cell Ranger is an alignment/analysis pipeline specific to 10x genomic data, and this can only be used on fastqs generated by 10x.", Arrays.asList( - //--sample - - ToolParameterDescriptor.create("id", "Run ID Suffix", "If provided, this will be appended to the ID of this run (readset name will be first).", "textfield", new JSONObject(){{ - put("allowBlank", true); - }}, null), - ToolParameterDescriptor.createCommandLineParam(CommandLineParam.create("--force-cells"), "force-cells", "Force Cells", "Force pipeline to use this number of cells, bypassing the cell detection algorithm. Use this if the number of cells estimated by Cell Ranger is not consistent with the barcode rank plot.", "ldk-integerfield", new JSONObject(){{ - put("minValue", 0); - }}, null), - ToolParameterDescriptor.create(TARGET_ASSAY, "Target Assay", "Results will be loaded into this assay. If no assay is selected, a table will be created with nothing in the DB.", "tcr-assayselectorfield", new JSONObject(){{ - put("autoSelectAssay", false); - }}, null), - ToolParameterDescriptor.create(DELETE_EXISTING_ASSAY_DATA, "Delete Any Existing Assay Data", "If selected, prior to importing assay data, and existing assay runs in the target container from this readset will be deleted.", "checkbox", new JSONObject(){{ - put("checked", true); - }}, true), - ToolParameterDescriptor.create("excludeFailedcDNA", "Exclude Failed cDNA", "If selected, cDNAs with non-blank status fields will be omitted", "checkbox", null, true) - - ), PageFlowUtil.set("tcrdb/field/AssaySelectorField.js"), "https://support.10xgenomics.com/single-cell-gene-expression/software/pipelines/latest/what-is-cell-ranger", true, false, false, ALIGNMENT_MODE.MERGE_THEN_ALIGN); - } - - @Override - public boolean shouldRunIdxstats() - { - return false; - } - - public String getName() - { - return "CellRanger-VDJ"; - } - - public String getDescription() - { - return null; - } - - public AlignmentStep create(PipelineContext context) - { - return new CellRangerVDJAlignmentStep(this, context, new CellRangerVDJWrapper(context.getLogger())); - } - } - - public static class CellRangerVDJAlignmentStep extends AbstractAlignmentPipelineStep implements AlignmentStep - { - private CellRangerVDJUtils _utils = null; - - private CellRangerVDJUtils getUtils() - { - if (_utils == null) - { - _utils = new CellRangerVDJUtils(getPipelineCtx().getLogger(), getPipelineCtx().getSourceDirectory()); - } - - return _utils; - } - - public CellRangerVDJAlignmentStep(AlignmentStepProvider provider, PipelineContext ctx, CellRangerVDJWrapper wrapper) - { - super(provider, ctx, wrapper); - } - - @Override - public boolean supportsMetrics() - { - return false; - } - - @Override - public void init(SequenceAnalysisJobSupport support) throws PipelineJobException - { - ReferenceGenome referenceGenome = support.getCachedGenomes().iterator().next(); - boolean hasCachedIndex = AlignerIndexUtil.hasCachedIndex(this.getPipelineCtx(), getIndexCachedDirName(getPipelineCtx().getJob()), referenceGenome); - if (!hasCachedIndex) - { - getPipelineCtx().getLogger().info("Creating FASTA for CellRanger VDJ Index for genome: " + referenceGenome.getName()); - File fasta = getGenomeFasta(); - try (PrintWriter writer = PrintWriters.getPrintWriter(fasta)) - { - final AtomicInteger i = new AtomicInteger(0); - UserSchema us = QueryService.get().getUserSchema(getPipelineCtx().getJob().getUser(), getPipelineCtx().getJob().getContainer(), "sequenceanalysis"); - List seqIds = new TableSelector(us.getTable("reference_library_members", null), PageFlowUtil.set("ref_nt_id"), new SimpleFilter(FieldKey.fromString("library_id"), referenceGenome.getGenomeId()), null).getArrayList(Integer.class); - new TableSelector(us.getTable("ref_nt_sequences", null), new SimpleFilter(FieldKey.fromString("rowid"), seqIds, CompareType.IN), null).forEach(nt -> { - - if (nt.getLocus() == null) - { - throw new IllegalArgumentException("Locus was empty for NT with ID: " + nt.getRowid()); - } - - //NOTE: this allows dual TRA/TRD segments - String[] loci = nt.getLocus().split("/"); - for (String locus : loci) - { - i.getAndIncrement(); //cant use sequenceId since sequences might be represented multiple times across loci - - String seq = nt.getSequence(); - - //example: >1|TRAV41*01 TRAV41|TRAV41|L-REGION+V-REGION|TR|TRA|None|None - StringBuilder header = new StringBuilder(); - header.append(">").append(i.get()).append("|").append(nt.getName()).append(" ").append(nt.getLineage()).append("|").append(nt.getLineage()).append("|"); - //translate into V_Region - String type; - if (nt.getLineage().contains("J")) - { - type = "J-REGION"; - } - else if (nt.getLineage().contains("V")) - { - if (seq.length() < 300) - { - getPipelineCtx().getLogger().info("Using V-REGION due to short length: " + nt.getName() + " / " + nt.getSeqLength()); - type = "V-REGION"; - } - else - { - type = "L-REGION+V-REGION"; - } - } - else if (nt.getLineage().contains("C")) - { - type = "C-REGION"; - } - else if (nt.getLineage().contains("D")) - { - type = "D-REGION"; - } - else - { - throw new RuntimeException("Unknown lineage: " + nt.getLineage()); - } - - header.append(type).append("|TR|").append(locus).append("|None|None"); - - writer.write(header + "\n"); - writer.write(seq + "\n"); - } - nt.clearCachedSequence(); - }, RefNtSequenceModel.class); - } - catch (IllegalArgumentException | IOException e) - { - throw new PipelineJobException(e); - } - } - - boolean excludeFailedcDNA = getProvider().getParameterByName("excludeFailedcDNA").extractValue(getPipelineCtx().getJob(), getProvider(), getStepIdx(), Boolean.class, true); - getUtils().prepareHashingAndCiteSeqFilesIfNeeded(getPipelineCtx().getJob(), getPipelineCtx().getSequenceSupport(), "enrichedReadsetId", excludeFailedcDNA, false, false); - } - - private File getGenomeFasta() - { - return new File(getPipelineCtx().getSourceDirectory(), "cellRangerVDJ.fasta"); - } - - @Override - public String getIndexCachedDirName(PipelineJob job) - { - return getProvider().getName(); - } - - @Override - public AlignmentStep.IndexOutput createIndex(ReferenceGenome referenceGenome, File outputDir) throws PipelineJobException - { - IndexOutputImpl output = new IndexOutputImpl(referenceGenome); - - File indexDir = new File(outputDir, getIndexCachedDirName(getPipelineCtx().getJob())); - boolean hasCachedIndex = AlignerIndexUtil.hasCachedIndex(this.getPipelineCtx(), getIndexCachedDirName(getPipelineCtx().getJob()), referenceGenome); - if (!hasCachedIndex) - { - getPipelineCtx().getLogger().info("Creating CellRanger VDJ Index"); - getPipelineCtx().getLogger().info("using file: " + getGenomeFasta().getPath()); - output.addIntermediateFile(getGenomeFasta()); - - //remove if directory exists - if (indexDir.exists()) - { - try - { - FileUtils.deleteDirectory(indexDir); - } - catch (IOException e) - { - throw new PipelineJobException(e); - } - } - - output.addInput(getGenomeFasta(), "Input FASTA"); - - List args = new ArrayList<>(); - args.add(getWrapper().getExe().getPath()); - args.add("mkvdjref"); - args.add("--seqs=" + getGenomeFasta().getPath()); - args.add("--genome=" + indexDir.getName()); - - getWrapper().setWorkingDir(indexDir.getParentFile()); - getWrapper().execute(args); - - output.appendOutputs(referenceGenome.getWorkingFastaFile(), indexDir); - - //recache if not already - AlignerIndexUtil.saveCachedIndex(hasCachedIndex, getPipelineCtx(), indexDir, getIndexCachedDirName(getPipelineCtx().getJob()), referenceGenome); - - } - - return output; - } - - @Override - public AlignmentStep.AlignmentOutput performAlignment(Readset rs, File inputFastq1, @Nullable File inputFastq2, File outputDirectory, ReferenceGenome referenceGenome, String basename, String readGroupId, @Nullable String platformUnit) throws PipelineJobException - { - AlignmentOutputImpl output = new AlignmentOutputImpl(); - - List args = new ArrayList<>(); - args.add(getWrapper().getExe().getPath()); - args.add("vdj"); - - String idParam = StringUtils.trimToNull(getProvider().getParameterByName("id").extractValue(getPipelineCtx().getJob(), getProvider(), getStepIdx(), String.class)); - String id = FileUtil.makeLegalName(rs.getName()) + (idParam == null ? "" : "-" + idParam); - id = id.replaceAll("[^a-zA-z0-9_\\-]", "_"); - args.add("--id=" + id); - - File indexDir = AlignerIndexUtil.getIndexDir(referenceGenome, getIndexCachedDirName(getPipelineCtx().getJob())); - args.add("--reference=" + indexDir.getPath()); - - args.addAll(getClientCommandArgs("=")); - - Integer maxThreads = SequencePipelineService.get().getMaxThreads(getPipelineCtx().getLogger()); - if (maxThreads != null) - { - args.add("--localcores=" + maxThreads.toString()); - } - - Integer maxRam = SequencePipelineService.get().getMaxRam(); - if (maxRam != null) - { - args.add("--localmem=" + maxRam.toString()); - } - - File localFqDir = new File(outputDirectory, "localFq"); - output.addIntermediateFile(localFqDir); - Set sampleNames = prepareFastqSymlinks(rs, localFqDir); - args.add("--fastqs=" + localFqDir.getPath()); - - getPipelineCtx().getLogger().debug("Sample names: [" + StringUtils.join(sampleNames, ",") + "]"); - if (sampleNames.size() > 1) - { - args.add("--sample=" + StringUtils.join(sampleNames, ",")); - } - - getWrapper().setWorkingDir(outputDirectory); - - //Note: we can safely assume only this server is working on these files, so if the _lock file exists, it was from a previous failed job. - File lockFile = new File(outputDirectory, id + "/_lock"); - if (lockFile.exists()) - { - getPipelineCtx().getLogger().info("Lock file exists, deleting: " + lockFile.getPath()); - lockFile.delete(); - } - - getWrapper().execute(args); - - File outdir = new File(outputDirectory, id); - outdir = new File(outdir, "outs"); - - File bam = new File(outdir, "all_contig.bam"); - if (!bam.exists()) - { - throw new PipelineJobException("Unable to find file: " + bam.getPath()); - } - output.setBAM(bam); - - //NOTE: run these before cleanup in case of failure - Integer assayId = getProvider().getParameterByName(TARGET_ASSAY).extractValue(getPipelineCtx().getJob(), getProvider(), getStepIdx(), Integer.class); - if (assayId != null) - { - boolean scanEditDistances = getProvider().getParameterByName("scanEditDistances").extractValue(getPipelineCtx().getJob(), getProvider(), getStepIdx(), Boolean.class, false); - int editDistance = getProvider().getParameterByName("editDistance").extractValue(getPipelineCtx().getJob(), getProvider(), getStepIdx(), Integer.class, 2); - int minCountPerCell = getProvider().getParameterByName("minCountPerCell").extractValue(getPipelineCtx().getJob(), getProvider(), getStepIdx(), Integer.class, 3); - boolean useSeurat = getProvider().getParameterByName("useSeurat").extractValue(getPipelineCtx().getJob(), getProvider(), getStepIdx(), Boolean.class, true); - boolean useMultiSeq = getProvider().getParameterByName("useMultiSeq").extractValue(getPipelineCtx().getJob(), getProvider(), getStepIdx(), Boolean.class, true); - - getUtils().runRemoteVdjCellHashingTasks(output, CellRangerVDJCellHashingHandler.CATEGORY, getUtils().getPerCellCsv(output.getBAM().getParentFile()), rs, getPipelineCtx().getSequenceSupport(), null, getPipelineCtx().getWorkingDirectory(), getPipelineCtx().getSourceDirectory(), editDistance, scanEditDistances, referenceGenome.getGenomeId(), minCountPerCell, useSeurat, useMultiSeq); - } - else - { - getPipelineCtx().getLogger().debug("No target assay selected, skipping cell hashing steps"); - } - - //now do cleanup/rename: - try - { - String prefix = FileUtil.makeLegalName(rs.getName() + "_"); - File outputHtml = new File(outdir, "web_summary.html"); - if (!outputHtml.exists()) - { - throw new PipelineJobException("Unable to find file: " + outputHtml.getPath()); - } - - File outputHtmlRename = new File(outdir, prefix + outputHtml.getName()); - if (outputHtmlRename.exists()) - { - outputHtmlRename.delete(); - } - FileUtils.moveFile(outputHtml, outputHtmlRename); - - output.addSequenceOutput(outputHtmlRename, rs.getName() + " 10x VDJ Summary", "10x Run Summary", rs.getRowId(), null, referenceGenome.getGenomeId(), null); - - File outputVloupe = new File(outdir, "vloupe.vloupe"); - if (!outputVloupe.exists()) - { - throw new PipelineJobException("Unable to find file: " + outputVloupe.getPath()); - } - - File outputVloupeRename = new File(outdir, prefix + outputVloupe.getName()); - if (outputVloupeRename.exists()) - { - outputVloupeRename.delete(); - } - FileUtils.moveFile(outputVloupe, outputVloupeRename); - output.addSequenceOutput(outputVloupeRename, rs.getName() + " 10x VLoupe", "10x VLoupe", rs.getRowId(), null, referenceGenome.getGenomeId(), null); - } - catch (IOException e) - { - throw new PipelineJobException(e); - } - - //NOTE: this folder has many unnecessary files and symlinks that get corrupted when we rename the main outputs - File directory = new File(outdir.getParentFile(), "SC_VDJ_ASSEMBLER_CS"); - if (directory.exists()) - { - //NOTE: this will have lots of symlinks, including corrupted ones, which java handles badly - new SimpleScriptWrapper(getPipelineCtx().getLogger()).execute(Arrays.asList("rm", "-Rf", directory.getPath())); - } - else - { - getPipelineCtx().getLogger().warn("Unable to find folder: " + directory.getPath()); - } - - deleteSymlinks(localFqDir); - - return output; - } - - @Override - public boolean doAddReadGroups() - { - return false; - } - - @Override - public boolean doSortIndexBam() - { - return false; - } - - @Override - public boolean alwaysCopyIndexToWorkingDir() - { - return false; - } - - @Override - public boolean supportsGzipFastqs() - { - return true; - } - - private String getSymlinkFileName(String fileName, boolean doRename, String sampleName, int idx, boolean isReversed) - { - //NOTE: cellranger is very picky about file name formatting - if (doRename) - { - sampleName = FileUtil.makeLegalName(sampleName.replaceAll("_", "-")).replaceAll(" ", "-").replaceAll("\\.", "-");; - return sampleName + "_S1_L001_R" + (isReversed ? "2" : "1") + "_" + StringUtils.leftPad(String.valueOf(idx), 3, "0") + ".fastq.gz"; - } - else - { - Matcher m = FILE_PATTERN.matcher(fileName); - if (m.matches()) - { - if (!StringUtils.isEmpty(m.group(7))) - { - return m.group(1).replaceAll("_", "-") + StringUtils.trimToEmpty(m.group(2)) + "_L" + StringUtils.trimToEmpty(m.group(3)) + "_" + StringUtils.trimToEmpty(m.group(4)) + StringUtils.trimToEmpty(m.group(5)) + StringUtils.trimToEmpty(m.group(6)) + ".fastq.gz"; - } - else if (m.group(1).contains("_")) - { - getPipelineCtx().getLogger().info("replacing underscores in file/sample name"); - return m.group(1).replaceAll("_", "-") + StringUtils.trimToEmpty(m.group(2)) + "_L" + StringUtils.trimToEmpty(m.group(3)) + "_" + StringUtils.trimToEmpty(m.group(4)) + StringUtils.trimToEmpty(m.group(5)) + StringUtils.trimToEmpty(m.group(6)) + ".fastq.gz"; - } - else - { - getPipelineCtx().getLogger().info("no additional characters found"); - } - } - else - { - getPipelineCtx().getLogger().warn("filename does not match Illumina formatting: " + fileName); - } - } - - return FileUtil.makeLegalName(fileName); - } - - public Set prepareFastqSymlinks(Readset rs, File localFqDir) throws PipelineJobException - { - Set ret = new HashSet<>(); - if (!localFqDir.exists()) - { - localFqDir.mkdirs(); - } - - String[] files = localFqDir.list(); - if (files != null && files.length > 0) - { - deleteSymlinks(localFqDir); - } - - int idx = 0; - boolean doRename = true; //cellranger is too picky - simply rename files all the time - for (ReadData rd : rs.getReadData()) - { - idx++; - try - { - File target1 = new File(localFqDir, getSymlinkFileName(rd.getFile1().getName(), doRename, rs.getName(), idx, false)); - getPipelineCtx().getLogger().debug("file: " + rd.getFile1().getPath()); - getPipelineCtx().getLogger().debug("target: " + target1.getPath()); - if (target1.exists()) - { - getPipelineCtx().getLogger().debug("deleting existing symlink: " + target1.getName()); - Files.delete(target1.toPath()); - } - - Files.createSymbolicLink(target1.toPath(), rd.getFile1().toPath()); - ret.add(getSampleName(target1.getName())); - - if (rd.getFile2() != null) - { - File target2 = new File(localFqDir, getSymlinkFileName(rd.getFile2().getName(), doRename, rs.getName(), idx, true)); - getPipelineCtx().getLogger().debug("file: " + rd.getFile2().getPath()); - getPipelineCtx().getLogger().debug("target: " + target2.getPath()); - if (target2.exists()) - { - getPipelineCtx().getLogger().debug("deleting existing symlink: " + target2.getName()); - Files.delete(target2.toPath()); - } - Files.createSymbolicLink(target2.toPath(), rd.getFile2().toPath()); - ret.add(getSampleName(target2.getName())); - } - } - catch (IOException e) - { - throw new PipelineJobException(e); - } - } - - return ret; - } - - public void deleteSymlinks(File localFqDir) throws PipelineJobException - { - for (File fq : localFqDir.listFiles()) - { - try - { - getPipelineCtx().getLogger().debug("deleting symlink: " + fq.getName()); - Files.delete(fq.toPath()); - } - catch (IOException e) - { - throw new PipelineJobException(e); - } - } - } - - public void addMetrics(AnalysisModel model) throws PipelineJobException - { - getPipelineCtx().getLogger().debug("adding 10x metrics"); - - File metrics = new File(model.getAlignmentFileObject().getParentFile(), "metrics_summary.csv"); - if (metrics.exists()) - { - try (CSVReader reader = new CSVReader(Readers.getReader(metrics))) - { - String[] line; - String[] header = null; - String[] metricValues = null; - - int i = 0; - while ((line = reader.readNext()) != null) - { - if (i == 0) - { - header = line; - } - else - { - metricValues = line; - break; - } - - i++; - } - - int totalAdded = 0; - TableInfo ti = DbSchema.get("sequenceanalysis", DbSchemaType.Module).getTable("quality_metrics"); - - //NOTE: if this job errored and restarted, we may have duplicate records: - SimpleFilter filter = new SimpleFilter(FieldKey.fromString("readset"), model.getReadset()); - filter.addCondition(FieldKey.fromString("analysis_id"), model.getRowId(), CompareType.EQUAL); - filter.addCondition(FieldKey.fromString("dataid"), model.getAlignmentFile(), CompareType.EQUAL); - filter.addCondition(FieldKey.fromString("category"), "Cell Ranger VDJ", CompareType.EQUAL); - filter.addCondition(FieldKey.fromString("container"), getPipelineCtx().getJob().getContainer().getId(), CompareType.EQUAL); - TableSelector ts = new TableSelector(ti, PageFlowUtil.set("rowid"), filter, null); - if (ts.exists()) - { - getPipelineCtx().getLogger().info("Deleting existing QC metrics (probably from prior restarted job)"); - ts.getArrayList(Integer.class).forEach(rowid -> { - Table.delete(ti, rowid); - }); - } - - for (int j = 0; j < header.length; j++) - { - Map toInsert = new CaseInsensitiveHashMap<>(); - toInsert.put("container", getPipelineCtx().getJob().getContainer().getId()); - toInsert.put("createdby", getPipelineCtx().getJob().getUser().getUserId()); - toInsert.put("created", new Date()); - toInsert.put("readset", model.getReadset()); - toInsert.put("analysis_id", model.getRowId()); - toInsert.put("dataid", model.getAlignmentFile()); - - toInsert.put("category", "Cell Ranger VDJ"); - toInsert.put("metricname", header[j]); - - metricValues[j] = metricValues[j].replaceAll(",", ""); - Object val = metricValues[j]; - if (metricValues[j].contains("%")) - { - metricValues[j] = metricValues[j].replaceAll("%", ""); - Double d = ConvertHelper.convert(metricValues[j], Double.class); - d = d / 100.0; - val = d; - } - - toInsert.put("metricvalue", val); - - Table.insert(getPipelineCtx().getJob().getUser(), ti, toInsert); - totalAdded++; - } - - getPipelineCtx().getLogger().info("total metrics added: " + totalAdded); - } - catch (IOException e) - { - throw new PipelineJobException(e); - } - } - else - { - getPipelineCtx().getLogger().warn("unable to find metrics file: " + metrics.getPath()); - } - } - - public void complete(SequenceAnalysisJobSupport support, AnalysisModel model) throws PipelineJobException - { - addMetrics(model); - - File bam = model.getAlignmentData().getFile(); - if (bam.exists()) - { - Integer assayId = getProvider().getParameterByName(TARGET_ASSAY).extractValue(getPipelineCtx().getJob(), getProvider(), getStepIdx(), Integer.class); - Boolean deleteExisting = getProvider().getParameterByName(DELETE_EXISTING_ASSAY_DATA).extractValue(getPipelineCtx().getJob(), getProvider(), getStepIdx(), Boolean.class, false); - getUtils().importAssayData(getPipelineCtx().getJob(), model, bam.getParentFile(), assayId, null, deleteExisting); - } - else - { - getPipelineCtx().getLogger().warn("BAM not found, expected: " + bam.getPath()); - } - } - - private static Pattern FILE_PATTERN = Pattern.compile("^(.+?)(_S[0-9]+){0,1}_L(.+?)_(R){0,1}([0-9])(_[0-9]+){0,1}(.*?)(\\.f(ast){0,1}q)(\\.gz)?$"); - private static Pattern SAMPLE_PATTERN = Pattern.compile("^(.+)_S[0-9]+(.*)$"); - - private String getSampleName(String fn) - { - Matcher matcher = FILE_PATTERN.matcher(fn); - if (matcher.matches()) - { - String ret = matcher.group(1); - Matcher matcher2 = SAMPLE_PATTERN.matcher(ret); - if (matcher2.matches()) - { - ret = matcher2.group(1); - } - else - { - getPipelineCtx().getLogger().debug("_S not found in sample: [" + ret + "]"); - } - - ret = ret.replaceAll("_", "-"); - - return ret; - } - else - { - getPipelineCtx().getLogger().debug("file does not match illumina pattern: [" + fn + "]"); - } - - throw new IllegalArgumentException("Unable to infer Illumina sample name: " + fn); - } - } - - protected File getExe() - { - //NOTE: cellranger 4 doesnt work w/ custom libraries currently. update to CR4 when fixed - return SequencePipelineService.get().getExeForPackage("CELLRANGERPATH", "cellranger-31"); - } -} diff --git a/tcrdb/src/org/labkey/tcrdb/pipeline/MiXCRAnalysis.java b/tcrdb/src/org/labkey/tcrdb/pipeline/MiXCRAnalysis.java index 09b89b5c8..a9738b517 100644 --- a/tcrdb/src/org/labkey/tcrdb/pipeline/MiXCRAnalysis.java +++ b/tcrdb/src/org/labkey/tcrdb/pipeline/MiXCRAnalysis.java @@ -13,6 +13,7 @@ import org.labkey.api.assay.AssayService; import org.labkey.api.collections.CaseInsensitiveHashMap; import org.labkey.api.data.CompareType; +import org.labkey.api.data.Container; import org.labkey.api.data.SimpleFilter; import org.labkey.api.data.TableInfo; import org.labkey.api.data.TableSelector; @@ -25,6 +26,7 @@ import org.labkey.api.module.ModuleLoader; import org.labkey.api.pipeline.PipelineJobException; import org.labkey.api.query.FieldKey; +import org.labkey.api.query.QueryService; import org.labkey.api.query.ValidationException; import org.labkey.api.reader.Readers; import org.labkey.api.resource.FileResource; @@ -76,8 +78,6 @@ import java.util.TreeSet; import java.util.zip.GZIPInputStream; -import static org.labkey.tcrdb.pipeline.CellRangerVDJWrapper.DELETE_EXISTING_ASSAY_DATA; - /** * Created by bimber on 5/10/2016. @@ -126,7 +126,7 @@ public Provider() {{ put("value", "ALL"); }}, true), - ToolParameterDescriptor.create(DELETE_EXISTING_ASSAY_DATA, "Delete Any Existing Assay Data", "If selected, prior to importing assay data, and existing assay runs in the target container from this readset will be deleted.", "checkbox", new JSONObject(){{ + ToolParameterDescriptor.create(CellRangerVDJCellHashingHandler.DELETE_EXISTING_ASSAY_DATA, "Delete Any Existing Assay Data", "If selected, prior to importing assay data, and existing assay runs in the target container from this readset will be deleted.", "checkbox", new JSONObject(){{ put("checked", true); }}, true), ToolParameterDescriptor.create(FLAG_MISSENSE, "Flag Missense CDR3", "If checked, if a sample has duplicate CDR3 clones from the same locus, and and one of these is missense, that clone will be flagged and excluded from many reports.", "checkbox", new JSONObject() @@ -1215,6 +1215,12 @@ private void inspectForOrphanAlignment(Readset rs, String[] line, Set } } + private TableInfo getCdnaTable() + { + Container target = getPipelineCtx().getJob().getContainer().isWorkbook() ? getPipelineCtx().getJob().getContainer().getParent() : getPipelineCtx().getJob().getContainer(); + return QueryService.get().getUserSchema(getPipelineCtx().getJob().getUser(), target, TCRdbSchema.SINGLE_CELL).getTable(TCRdbSchema.TABLE_CDNAS); + } + private void parseCloneOutput(Map runMap, File table, AnalysisModel model, File inputBam) throws PipelineJobException { Integer runId = SequencePipelineService.get().getExpRunIdForJob(getPipelineCtx().getJob()); @@ -1223,6 +1229,7 @@ private void parseCloneOutput(Map runMap, File table, AnalysisM List cloneDatas = run.getInputDatas(CLONES_FILE, ExpProtocol.ApplicationType.ExperimentRunOutput); List vdjDatas = run.getInputDatas(FINAL_VDJ_FILE, ExpProtocol.ApplicationType.ExperimentRunOutput); + TableInfo cDNATable = getCdnaTable(); try (CSVReader reader = new CSVReader(Readers.getReader(table), '\t')) { int lineNo = 0; @@ -1235,7 +1242,7 @@ private void parseCloneOutput(Map runMap, File table, AnalysisM continue; } - Map row = getBaseRow(model, runId); + Map row = getBaseRow(model, runId, cDNATable); if (line.length != (FIELDS.size() + TOTAL_EXPORTED_FIELDS_NOT_IN_DB)) //this includes one additional field appended to the end { @@ -1323,7 +1330,7 @@ private void parseCloneOutput(Map runMap, File table, AnalysisM } } - private Map getBaseRow(AnalysisModel model, Integer runId) throws PipelineJobException + private Map getBaseRow(AnalysisModel model, Integer runId, TableInfo cDNATable) throws PipelineJobException { Map row = new CaseInsensitiveHashMap<>(); if (model.getReadset() != null) @@ -1356,13 +1363,12 @@ private Map getBaseRow(AnalysisModel model, Integer runId) throw row.put("analysisId", model.getRowId()); //attempt to locate cDNA: - TableInfo cDNATable = TCRdbSchema.getInstance().getSchema().getTable(TCRdbSchema.TABLE_CDNAS); if (model.getReadset() != null) { SimpleFilter filter = new SimpleFilter(); filter.addClause(new SimpleFilter.OrClause( new CompareType.CompareClause(FieldKey.fromString("readsetId"), CompareType.EQUAL, model.getReadset()), - new CompareType.CompareClause(FieldKey.fromString("enrichedReadsetId"), CompareType.EQUAL, model.getReadset()) + new CompareType.CompareClause(FieldKey.fromString("tcrReadsetId"), CompareType.EQUAL, model.getReadset()) )); TableSelector ts1 = new TableSelector(cDNATable, PageFlowUtil.set("rowId"), filter, null); @@ -1592,7 +1598,8 @@ private void importRun(RunData rd, File outDir, AnalysisModel model, ExpProtocol if (rd.rows.isEmpty()) { //NOTE: we need to add a placeholder row since assay import will die w/ a run-only import: - Map row = getBaseRow(model, runId); + TableInfo cDNATable = getCdnaTable(); + Map row = getBaseRow(model, runId, cDNATable); row.put("species", rd.species); row.put("libraryId", rd.libraryId); row.put("locus", "None"); @@ -1602,7 +1609,7 @@ private void importRun(RunData rd, File outDir, AnalysisModel model, ExpProtocol getPipelineCtx().getLogger().debug("saving assay file to: " + assayTmp.getPath()); AssayProvider ap = AssayService.get().getProvider(protocol); - boolean deleteExistingAssayData = getProvider().getParameterByName(DELETE_EXISTING_ASSAY_DATA).extractValue(getPipelineCtx().getJob(), getProvider(), getStepIdx(), Boolean.class, false); + boolean deleteExistingAssayData = getProvider().getParameterByName(CellRangerVDJCellHashingHandler.DELETE_EXISTING_ASSAY_DATA).extractValue(getPipelineCtx().getJob(), getProvider(), getStepIdx(), Boolean.class, false); if (deleteExistingAssayData) { if (model.getReadset() == null) diff --git a/tcrdb/src/org/labkey/tcrdb/pipeline/SeuratCellHashingHandler.java b/tcrdb/src/org/labkey/tcrdb/pipeline/SeuratCellHashingHandler.java deleted file mode 100644 index 8e526dfed..000000000 --- a/tcrdb/src/org/labkey/tcrdb/pipeline/SeuratCellHashingHandler.java +++ /dev/null @@ -1,153 +0,0 @@ -package org.labkey.tcrdb.pipeline; - -import org.json.JSONObject; -import org.labkey.api.module.ModuleLoader; -import org.labkey.api.pipeline.PipelineJob; -import org.labkey.api.pipeline.PipelineJobException; -import org.labkey.api.pipeline.RecordedAction; -import org.labkey.api.sequenceanalysis.SequenceOutputFile; -import org.labkey.api.sequenceanalysis.model.Readset; -import org.labkey.api.sequenceanalysis.pipeline.AbstractParameterizedOutputHandler; -import org.labkey.api.sequenceanalysis.pipeline.SequenceAnalysisJobSupport; -import org.labkey.api.sequenceanalysis.pipeline.SequenceOutputHandler; -import org.labkey.api.sequenceanalysis.pipeline.ToolParameterDescriptor; -import org.labkey.api.util.FileType; -import org.labkey.tcrdb.TCRdbModule; - -import java.io.File; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -public class SeuratCellHashingHandler extends AbstractParameterizedOutputHandler -{ - private FileType _fileType = new FileType(".seurat.rds", false); - public static final String CATEGORY = "Seurat Cell Hashing Calls"; - - public SeuratCellHashingHandler() - { - super(ModuleLoader.getInstance().getModule(TCRdbModule.class), "Seurat GEX/Cell Hashing", "This will run CiteSeqCount/MultiSeqClassifier to generate a sample-to-cellbarcode TSV based on the cell barcodes present in the saved Seurat object.", null, getDefaultParams()); - } - - private static List getDefaultParams() - { - List ret = new ArrayList<>(); - ret.add(ToolParameterDescriptor.create("useOutputFileContainer", "Submit to Source File Workbook", "If checked, each job will be submitted to the same workbook as the input file, as opposed to submitting all jobs to the same workbook. This is primarily useful if submitting a large batch of files to process separately..", "checkbox", new JSONObject() - {{ - put("checked", true); - }}, false)); - - ret.addAll(CellRangerCellHashingHandler.getDefaultHashingParams(true)); - - return ret; - } - - @Override - public boolean canProcess(SequenceOutputFile o) - { - return o.getFile() != null && _fileType.isType(o.getFile()); - } - - @Override - public boolean doRunRemote() - { - return true; - } - - @Override - public boolean doRunLocal() - { - return false; - } - - @Override - public SequenceOutputProcessor getProcessor() - { - return new Processor(); - } - - @Override - public boolean doSplitJobs() - { - return true; - } - - @Override - public boolean requiresSingleGenome() - { - return false; - } - - public class Processor implements SequenceOutputHandler.SequenceOutputProcessor - { - @Override - public void init(PipelineJob job, SequenceAnalysisJobSupport support, List inputFiles, JSONObject params, File outputDir, List actions, List outputsToCreate) throws UnsupportedOperationException, PipelineJobException - { - new CellRangerVDJUtils(job.getLogger(), outputDir).prepareHashingAndCiteSeqFilesIfNeeded(job, support, "readsetId", params.optBoolean("excludeFailedcDNA", true), true, false); - } - - @Override - public void processFilesOnWebserver(PipelineJob job, SequenceAnalysisJobSupport support, List inputFiles, JSONObject params, File outputDir, List actions, List outputsToCreate) throws UnsupportedOperationException, PipelineJobException - { - - } - - @Override - public void processFilesRemote(List inputFiles, SequenceOutputHandler.JobContext ctx) throws UnsupportedOperationException, PipelineJobException - { - RecordedAction action = new RecordedAction(getName()); - Map readsetToHashing = CellRangerVDJUtils.getCachedHashingReadsetMap(ctx.getSequenceSupport()); - ctx.getLogger().debug("total cached readset to hashing pairs: " + readsetToHashing.size()); - - for (SequenceOutputFile so : inputFiles) - { - ctx.getLogger().info("processing file: " + so.getName()); - - File barcodes = getBarcodesFromSeurat(so.getFile()); - - Readset rs = ctx.getSequenceSupport().getCachedReadset(so.getReadset()); - if (rs == null) - { - throw new PipelineJobException("Unable to find readset for outputfile: " + so.getRowid()); - } - else if (rs.getReadsetId() == null) - { - throw new PipelineJobException("Readset lacks a rowId for outputfile: " + so.getRowid()); - } - - Readset htoReadset = ctx.getSequenceSupport().getCachedReadset(readsetToHashing.get(rs.getReadsetId())); - if (htoReadset == null) - { - throw new PipelineJobException("Unable to find Hashing/Cite-seq readset for GEX readset: " + rs.getReadsetId()); - } - - CellRangerCellHashingHandler.processBarcodeFile(ctx, barcodes, rs, htoReadset, so.getLibrary_id(), action, getClientCommandArgs(ctx.getParams()), false, CATEGORY); - } - - ctx.addActions(action); - } - - @Override - public void complete(PipelineJob job, List inputs, List outputsCreated, SequenceAnalysisJobSupport support) throws PipelineJobException - { - for (SequenceOutputFile so : outputsCreated) - { - if (so.getCategory().equals(CATEGORY)) - { - CellRangerVDJCellHashingHandler.processMetrics(so, job, true); - } - } - } - } - - public static File getBarcodesFromSeurat(File seuratObj) throws PipelineJobException - { - File barcodes = new File(seuratObj.getParentFile(), seuratObj.getName().replaceAll("seurat.rds", "cellBarcodes.csv")); - if (!barcodes.exists()) - { - throw new PipelineJobException("Unable to find expected cell barcodes file. This might indicate the seurat object was created with an older version of the pipeline. Expected: " + barcodes.getPath()); - } - - return barcodes; - } -} diff --git a/tcrdb/src/org/labkey/tcrdb/pipeline/SeuratCiteSeqHandler.java b/tcrdb/src/org/labkey/tcrdb/pipeline/SeuratCiteSeqHandler.java deleted file mode 100644 index 853a68a0b..000000000 --- a/tcrdb/src/org/labkey/tcrdb/pipeline/SeuratCiteSeqHandler.java +++ /dev/null @@ -1,131 +0,0 @@ -package org.labkey.tcrdb.pipeline; - -import org.json.JSONObject; -import org.labkey.api.module.ModuleLoader; -import org.labkey.api.pipeline.PipelineJob; -import org.labkey.api.pipeline.PipelineJobException; -import org.labkey.api.pipeline.RecordedAction; -import org.labkey.api.sequenceanalysis.SequenceOutputFile; -import org.labkey.api.sequenceanalysis.model.Readset; -import org.labkey.api.sequenceanalysis.pipeline.AbstractParameterizedOutputHandler; -import org.labkey.api.sequenceanalysis.pipeline.SequenceAnalysisJobSupport; -import org.labkey.api.sequenceanalysis.pipeline.SequenceOutputHandler; -import org.labkey.api.sequenceanalysis.pipeline.ToolParameterDescriptor; -import org.labkey.api.util.FileType; -import org.labkey.tcrdb.TCRdbModule; - -import java.io.File; -import java.util.Arrays; -import java.util.List; -import java.util.Map; - -public class SeuratCiteSeqHandler extends AbstractParameterizedOutputHandler -{ - protected FileType _fileType = new FileType(".seurat.rds", false); - public static final String CATEGORY = "Seurat CITE-Seq Count Matrix"; - - public SeuratCiteSeqHandler() - { - super(ModuleLoader.getInstance().getModule(TCRdbModule.class), "Seurat GEX/CITE-seq Counts", "This will run CiteSeqCount to generate a sample-to-cellbarcode TSV based on the cell barcodes present in the saved Seurat object.", null, Arrays.asList( - ToolParameterDescriptor.create("editDistance", "Edit Distance", null, "ldk-integerfield", null, 3), - ToolParameterDescriptor.create("excludeFailedcDNA", "Exclude Failed cDNA", "If selected, cDNAs with non-blank status fields will be omitted", "checkbox", null, true), - ToolParameterDescriptor.create("minCountPerCell", "Min Reads/Cell (Cell Hashing)", null, "ldk-integerfield", null, 5), - ToolParameterDescriptor.create("useOutputFileContainer", "Submit to Source File Workbook", "If checked, each job will be submitted to the same workbook as the input file, as opposed to submitting all jobs to the same workbook. This is primarily useful if submitting a large batch of files to process separately..", "checkbox", new JSONObject() - {{ - put("checked", true); - }}, false) - )); - } - - @Override - public boolean canProcess(SequenceOutputFile o) - { - return o.getFile() != null && _fileType.isType(o.getFile()); - } - - @Override - public boolean doRunRemote() - { - return true; - } - - @Override - public boolean doRunLocal() - { - return false; - } - - @Override - public SequenceOutputProcessor getProcessor() - { - return new Processor(); - } - - @Override - public boolean doSplitJobs() - { - return true; - } - - @Override - public boolean requiresSingleGenome() - { - return false; - } - - public class Processor implements SequenceOutputHandler.SequenceOutputProcessor - { - @Override - public void init(PipelineJob job, SequenceAnalysisJobSupport support, List inputFiles, JSONObject params, File outputDir, List actions, List outputsToCreate) throws UnsupportedOperationException, PipelineJobException - { - new CellRangerVDJUtils(job.getLogger(), outputDir).prepareHashingAndCiteSeqFilesIfNeeded(job, support,"readsetId", params.optBoolean("excludeFailedcDNA", true), false, true); - } - - @Override - public void processFilesOnWebserver(PipelineJob job, SequenceAnalysisJobSupport support, List inputFiles, JSONObject params, File outputDir, List actions, List outputsToCreate) throws UnsupportedOperationException, PipelineJobException - { - - } - - @Override - public void processFilesRemote(List inputFiles, SequenceOutputHandler.JobContext ctx) throws UnsupportedOperationException, PipelineJobException - { - RecordedAction action = new RecordedAction(getName()); - - Map readsetToCiteSeq = CellRangerVDJUtils.getCachedCiteSeqReadsetMap(ctx.getSequenceSupport()); - ctx.getLogger().debug("total cached readset to GEX/citeseq pairs: " + readsetToCiteSeq.size()); - - for (SequenceOutputFile so : inputFiles) - { - ctx.getLogger().info("processing file: " + so.getName()); - - File barcodes = SeuratCellHashingHandler.getBarcodesFromSeurat(so.getFile()); - - Readset rs = ctx.getSequenceSupport().getCachedReadset(so.getReadset()); - if (rs == null) - { - throw new PipelineJobException("Unable to find readset for outputfile: " + so.getRowid()); - } - else if (rs.getReadsetId() == null) - { - throw new PipelineJobException("Readset lacks a rowId for outputfile: " + so.getRowid()); - } - - Readset citeseqReadset = ctx.getSequenceSupport().getCachedReadset(readsetToCiteSeq.get(rs.getReadsetId())); - if (citeseqReadset == null) - { - throw new PipelineJobException("Unable to find Cite-seq readset for GEX readset: " + rs.getReadsetId()); - } - - File adtWhitelist = CellRangerVDJUtils.getValidCiteSeqBarcodeFile(ctx.getSourceDirectory(), so.getReadset()); - File citeSeqMatrix = CellRangerCellHashingHandler.processBarcodeFile(ctx, barcodes, rs, citeseqReadset, so.getLibrary_id(), action, getClientCommandArgs(ctx.getParams()), false, CATEGORY, true, adtWhitelist, false); - if (!citeSeqMatrix.exists()) - { - throw new PipelineJobException("Unable to find expected file: " + citeSeqMatrix.getPath()); - } - } - - ctx.addActions(action); - } - } -} diff --git a/variantdb/module.properties b/variantdb/module.properties index 251fad2e6..b3faf5d00 100644 --- a/variantdb/module.properties +++ b/variantdb/module.properties @@ -1,3 +1,2 @@ ModuleClass: org.labkey.variantdb.VariantDBModule -ConsolidateScripts: false ManageVersion: false diff --git a/variantdb/resources/credits/dependencies.txt b/variantdb/resources/credits/dependencies.txt new file mode 100644 index 000000000..4e8ead70b --- /dev/null +++ b/variantdb/resources/credits/dependencies.txt @@ -0,0 +1,3 @@ +# direct external dependencies for project :server:modules:BimberLabKeyModules:variantdb +commons-net-3.5.jar +commons-math3-3.6.1.jar diff --git a/variantdb/src/org/labkey/variantdb/VariantDBManager.java b/variantdb/src/org/labkey/variantdb/VariantDBManager.java index 984551cb2..8a5445e20 100644 --- a/variantdb/src/org/labkey/variantdb/VariantDBManager.java +++ b/variantdb/src/org/labkey/variantdb/VariantDBManager.java @@ -149,71 +149,66 @@ public void exec(ResultSet rs) throws SQLException final Pair matches = Pair.of(0, 0); TableSelector variantTs = new TableSelector(VariantDBSchema.getInstance().getSchema().getTable(VariantDBSchema.TABLE_VARIANTS), variantFilter, null); - variantTs.forEach(new Selector.ForEachBlock() - { - @Override - public void exec(Variant v) throws SQLException + variantTs.forEach(Variant.class, v -> { + String name = resolveSequenceName(v.getSequenceId()); + if (name != null) { - String name = resolveSequenceName(v.getSequenceId()); - if (name != null) - { - matches.second++; + matches.second++; - //only delete once - deletePs.setString(1, v.getObjectid()); - deletePs.addBatch(); + //only delete once + deletePs.setString(1, v.getObjectid()); + deletePs.addBatch(); - v.setSequenceName(name); - for (Integer targetId : liftOverMap.keySet()) + v.setSequenceName(name); + for (Integer targetId : liftOverMap.keySet()) + { + LiftedVariant lv = VariantDBManager.get().liftOverVariant(liftOverMap.get(targetId), v, chainFileMap.get(targetId)); + if (lv.successfulLiftover()) { - LiftedVariant lv = VariantDBManager.get().liftOverVariant(liftOverMap.get(targetId), v, chainFileMap.get(targetId)); - if (lv.successfulLiftover()) - { - matches.first++; - } - - //variantid, sequenceid, startPosition, endPosition, reference, allele, referenceVariantId, referenceAlleleId, batchId, chainFile, created, createdBy, modified, modifiedBy - insertPs.setString(1, v.getObjectid()); - if (lv.successfulLiftover()) - { - insertPs.setInt(2, lv.getSequenceId()); - insertPs.setInt(3, lv.getStartPosition()); - insertPs.setInt(4, lv.getEndPosition()); - } - else - { - insertPs.setInt(2, -1); - insertPs.setInt(3, 0); - insertPs.setInt(4, 0); - } - insertPs.setString(5, null); - insertPs.setString(6, null); - - insertPs.setString(7, v.getReferenceVariantId()); - insertPs.setString(8, v.getReferenceAlleleId()); - insertPs.setString(9, batchId); - insertPs.setInt(10, lv.getChainFile()); - insertPs.setDate(11, new Date(System.currentTimeMillis())); - insertPs.setInt(12, u.getUserId()); - insertPs.setDate(13, new Date(System.currentTimeMillis())); - insertPs.setInt(14, u.getUserId()); - - insertPs.addBatch(); + matches.first++; } - if (matches.second % batchSize == 0) + //variantid, sequenceid, startPosition, endPosition, reference, allele, referenceVariantId, referenceAlleleId, batchId, chainFile, created, createdBy, modified, modifiedBy + insertPs.setString(1, v.getObjectid()); + if (lv.successfulLiftover()) + { + insertPs.setInt(2, lv.getSequenceId()); + insertPs.setInt(3, lv.getStartPosition()); + insertPs.setInt(4, lv.getEndPosition()); + } + else { - log.info("processed: " + matches.second + " variants"); - deletePs.executeBatch(); - insertPs.executeBatch(); + insertPs.setInt(2, -1); + insertPs.setInt(3, 0); + insertPs.setInt(4, 0); } + insertPs.setString(5, null); + insertPs.setString(6, null); + + insertPs.setString(7, v.getReferenceVariantId()); + insertPs.setString(8, v.getReferenceAlleleId()); + insertPs.setString(9, batchId); + insertPs.setInt(10, lv.getChainFile()); + insertPs.setDate(11, new Date(System.currentTimeMillis())); + insertPs.setInt(12, u.getUserId()); + insertPs.setDate(13, new Date(System.currentTimeMillis())); + insertPs.setInt(14, u.getUserId()); + + insertPs.addBatch(); } - else + + if (matches.second % batchSize == 0) { - log.error("unable to resolve sequenceId: " + v.getSequenceId()); + log.info("processed: " + matches.second + " variants"); + deletePs.executeBatch(); + insertPs.executeBatch(); } } - }, Variant.class); + else + { + log.error("unable to resolve sequenceId: " + v.getSequenceId()); + } + }); //execute any remaining commands log.info("processed: " + matches.second + " variants"); diff --git a/variantdb/src/org/labkey/variantdb/analysis/GBSAnalysisHandler.java b/variantdb/src/org/labkey/variantdb/analysis/GBSAnalysisHandler.java index 773f99e05..1356bec46 100644 --- a/variantdb/src/org/labkey/variantdb/analysis/GBSAnalysisHandler.java +++ b/variantdb/src/org/labkey/variantdb/analysis/GBSAnalysisHandler.java @@ -80,16 +80,16 @@ public SequenceOutputProcessor getProcessor() public class Processor implements SequenceOutputProcessor { @Override - public void init(PipelineJob job, SequenceAnalysisJobSupport support, List inputFiles, JSONObject params, File outputDir, List actions, List outputsToCreate) throws UnsupportedOperationException, PipelineJobException + public void init(JobContext ctx, List inputFiles, List actions, List outputsToCreate) throws UnsupportedOperationException, PipelineJobException { for (ToolParameterDescriptor pd : getParameters()) { - if (params.containsKey(pd.getName()) && !StringUtils.isEmpty(params.getString(pd.getName()))) + if (ctx.getParams().containsKey(pd.getName()) && !StringUtils.isEmpty(ctx.getParams().getString(pd.getName()))) { - ExpData d = ExperimentService.get().getExpData(params.getInt(pd.getName())); + ExpData d = ExperimentService.get().getExpData(ctx.getParams().getInt(pd.getName())); if (d != null) { - support.cacheExpData(d); + ctx.getSequenceSupport().cacheExpData(d); } } } @@ -98,7 +98,7 @@ public void init(PipelineJob job, SequenceAnalysisJobSupport support, List inputFiles, JSONObject params, File outputDir, List actions, List outputsToCreate) throws UnsupportedOperationException, PipelineJobException + public void init(JobContext ctx, List inputFiles, List actions, List outputsToCreate) throws UnsupportedOperationException, PipelineJobException { //find genome Set ids = new HashSet<>(); @@ -166,7 +166,7 @@ public void init(PipelineJob job, SequenceAnalysisJobSupport support, List pedigreeRecords = generatePedigree(job, params); + List pedigreeRecords = generatePedigree(ctx.getJob(), ctx.getParams()); - File gatkPed = new File(job.getJobSupport(FileAnalysisJobSupport.class).getAnalysisDirectory(), "gatkPed.ped"); - File morganPed = new File(job.getJobSupport(FileAnalysisJobSupport.class).getAnalysisDirectory(), "morgan.ped"); + File gatkPed = new File(ctx.getJob().getJobSupport(FileAnalysisJobSupport.class).getAnalysisDirectory(), "gatkPed.ped"); + File morganPed = new File(ctx.getSourceDirectory(), "morgan.ped"); try (PrintWriter gatkWriter = PrintWriters.getPrintWriter(gatkPed); PrintWriter morganWriter = PrintWriters.getPrintWriter(morganPed)) { morganWriter.write("input pedigree size " + pedigreeRecords.size() + '\n'); @@ -204,39 +204,39 @@ public void init(PipelineJob job, SequenceAnalysisJobSupport support, List inputFiles, JSONObject params, File outputDir, List actions, List outputsToCreate) throws UnsupportedOperationException, PipelineJobException + public void init(JobContext ctx, List inputFiles, List actions, List outputsToCreate) throws UnsupportedOperationException, PipelineJobException { for (ToolParameterDescriptor pd : getParameters()) { - if (params.containsKey(pd.getName()) && !StringUtils.isEmpty(params.getString(pd.getName()))) + if (ctx.getParams().containsKey(pd.getName()) && !StringUtils.isEmpty(ctx.getParams().getString(pd.getName()))) { - ExpData d = ExperimentService.get().getExpData(params.getInt(pd.getName())); + ExpData d = ExperimentService.get().getExpData(ctx.getParams().getInt(pd.getName())); if (d != null) { - support.cacheExpData(d); + ctx.getSequenceSupport().cacheExpData(d); } } } @@ -95,7 +95,7 @@ public void init(PipelineJob job, SequenceAnalysisJobSupport support, List