-
- Ext4.onReady(function(){
- var webpart = <%=webpartContext%>;
- Ext4.create('TCRdb.panel.cDNAImportPanel').render(webpart.wrapperDivId);
- });
-
-
\ No newline at end of file
diff --git a/tcrdb/resources/views/cDNAImport.view.xml b/tcrdb/resources/views/cDNAImport.view.xml
deleted file mode 100644
index 7164ffd11..000000000
--- a/tcrdb/resources/views/cDNAImport.view.xml
+++ /dev/null
@@ -1,31 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/tcrdb/resources/views/libraryExport.html b/tcrdb/resources/views/libraryExport.html
deleted file mode 100644
index 77fe3e9c7..000000000
--- a/tcrdb/resources/views/libraryExport.html
+++ /dev/null
@@ -1,8 +0,0 @@
-
\ No newline at end of file
diff --git a/tcrdb/resources/views/libraryExport.view.xml b/tcrdb/resources/views/libraryExport.view.xml
deleted file mode 100644
index 0a2d0b33e..000000000
--- a/tcrdb/resources/views/libraryExport.view.xml
+++ /dev/null
@@ -1,7 +0,0 @@
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/tcrdb/resources/views/poolImport.html b/tcrdb/resources/views/poolImport.html
deleted file mode 100644
index aca7a6e09..000000000
--- a/tcrdb/resources/views/poolImport.html
+++ /dev/null
@@ -1,8 +0,0 @@
-
\ No newline at end of file
diff --git a/tcrdb/resources/views/poolImport.view.xml b/tcrdb/resources/views/poolImport.view.xml
deleted file mode 100644
index ea2101326..000000000
--- a/tcrdb/resources/views/poolImport.view.xml
+++ /dev/null
@@ -1,30 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/tcrdb/resources/views/stimDashboard.html b/tcrdb/resources/views/stimDashboard.html
deleted file mode 100644
index 111c5d08e..000000000
--- a/tcrdb/resources/views/stimDashboard.html
+++ /dev/null
@@ -1,8 +0,0 @@
-
\ No newline at end of file
diff --git a/tcrdb/resources/views/stimDashboard.view.xml b/tcrdb/resources/views/stimDashboard.view.xml
deleted file mode 100644
index f452ee3c7..000000000
--- a/tcrdb/resources/views/stimDashboard.view.xml
+++ /dev/null
@@ -1,7 +0,0 @@
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/tcrdb/resources/web/tcrdb/buttons.js b/tcrdb/resources/web/tcrdb/buttons.js
index 2db248537..a59d12a5a 100644
--- a/tcrdb/resources/web/tcrdb/buttons.js
+++ b/tcrdb/resources/web/tcrdb/buttons.js
@@ -1,9 +1,6 @@
Ext4.ns('TCRdb.buttons');
-
TCRdb.buttons = new function(){
-
-
return {
createMixcrGenome: function(dataRegionName) {
var dataRegion = LABKEY.DataRegions[dataRegionName];
diff --git a/tcrdb/resources/web/tcrdb/exampleData/ImportExample.xlsx b/tcrdb/resources/web/tcrdb/exampleData/ImportExample.xlsx
deleted file mode 100644
index 4f5d287cf..000000000
Binary files a/tcrdb/resources/web/tcrdb/exampleData/ImportExample.xlsx and /dev/null differ
diff --git a/tcrdb/resources/web/tcrdb/exampleData/ImportReadsetTemplate.xlsx b/tcrdb/resources/web/tcrdb/exampleData/ImportReadsetTemplate.xlsx
deleted file mode 100644
index e59199073..000000000
Binary files a/tcrdb/resources/web/tcrdb/exampleData/ImportReadsetTemplate.xlsx and /dev/null differ
diff --git a/tcrdb/resources/web/tcrdb/exampleData/ImportTemplate.xlsx b/tcrdb/resources/web/tcrdb/exampleData/ImportTemplate.xlsx
deleted file mode 100644
index 490e520cd..000000000
Binary files a/tcrdb/resources/web/tcrdb/exampleData/ImportTemplate.xlsx and /dev/null differ
diff --git a/tcrdb/resources/web/tcrdb/panel/LibraryExportPanel.js b/tcrdb/resources/web/tcrdb/panel/LibraryExportPanel.js
deleted file mode 100644
index 4988c82e5..000000000
--- a/tcrdb/resources/web/tcrdb/panel/LibraryExportPanel.js
+++ /dev/null
@@ -1,822 +0,0 @@
-Ext4.define('TCRdb.panel.LibraryExportPanel', {
- extend: 'Ext.panel.Panel',
- alias: 'widget.tcrdb-libraryexportpanel',
-
- statics: {
- BARCODES5: ['N701', 'N702', 'N703', 'N704', 'N705', 'N706', 'N707', 'N708', 'N709', 'N710', 'N711', 'N712'],
-
- BARCODES3: ['S517', 'S502', 'S503', 'S504', 'S505', 'S506', 'S507', 'S508'],
-
- TENX_BARCODES: ['SI-GA-A1','SI-GA-A2','SI-GA-A3','SI-GA-A4','SI-GA-A5','SI-GA-A6','SI-GA-A7','SI-GA-A8','SI-GA-A9','SI-GA-A10','SI-GA-A11','SI-GA-A12','SI-GA-B1','SI-GA-B2','SI-GA-B3','SI-GA-B4','SI-GA-B5','SI-GA-B6','SI-GA-B7','SI-GA-B8','SI-GA-B9','SI-GA-B10','SI-GA-B11','SI-GA-B12','SI-GA-C1','SI-GA-C2','SI-GA-C3','SI-GA-C4','SI-GA-C5','SI-GA-C6','SI-GA-C7','SI-GA-C8','SI-GA-C9','SI-GA-C10','SI-GA-C11','SI-GA-C12','SI-GA-D1','SI-GA-D2','SI-GA-D3','SI-GA-D4','SI-GA-D5','SI-GA-D6','SI-GA-D7','SI-GA-D8','SI-GA-D9','SI-GA-D10','SI-GA-D11','SI-GA-D12','SI-GA-E1','SI-GA-E2','SI-GA-E3','SI-GA-E4','SI-GA-E5','SI-GA-E6','SI-GA-E7','SI-GA-E8','SI-GA-E9','SI-GA-E10','SI-GA-E11','SI-GA-E12','SI-GA-F1','SI-GA-F2','SI-GA-F3','SI-GA-F4','SI-GA-F5','SI-GA-F6','SI-GA-F7','SI-GA-F8','SI-GA-F9','SI-GA-F10','SI-GA-F11','SI-GA-F12','SI-GA-G1','SI-GA-G2','SI-GA-G3','SI-GA-G4','SI-GA-G5','SI-GA-G6','SI-GA-G7','SI-GA-G8','SI-GA-G9','SI-GA-G10','SI-GA-G11','SI-GA-G12','SI-GA-H1','SI-GA-H2','SI-GA-H3','SI-GA-H4','SI-GA-H5','SI-GA-H6','SI-GA-H7','SI-GA-H8','SI-GA-H9','SI-GA-H10','SI-GA-H11','SI-GA-H12']
- },
-
- initComponent: function () {
- Ext4.apply(this, {
- title: null,
- border: false,
- defaults: {
- border: false
- },
- items: [{
- xtype: 'radiogroup',
- name: 'importType',
- columns: 1,
- items: [{
- boxLabel: 'Novogene/Plate List',
- inputValue: 'plateList',
- name: 'importType',
- checked: true
- },{
- boxLabel: 'Other',
- inputValue: 'other',
- name: 'importType'
- }],
- listeners: {
- scope: this,
- afterrender: function(field) {
- field.fireEvent('change', field, field.getValue());
- },
- change: function(field, val) {
- val = val.importType;
- var target = field.up('panel').down('#importArea');
- target.removeAll();
- if (val === 'other') {
- target.add([{
- xtype: 'ldk-simplecombo',
- itemId: 'instrument',
- fieldLabel: 'Instrument/Core',
- forceSelection: true,
- editable: false,
- labelWidth: 160,
- storeValues: ['NextSeq (MPSSR)', 'MiSeq (ONPRC)', 'Basic List (MedGenome)', '10x Sample Sheet', 'Novogene']
- },{
- xtype: 'ldk-simplecombo',
- itemId: 'application',
- fieldLabel: 'Application/Type',
- forceSelection: true,
- editable: true,
- labelWidth: 160,
- allowBlank: true,
- storeValues: ['Whole Transcriptome RNA-Seq', 'TCR Enriched', '10x GEX', '10x VDJ']
- },{
- xtype: 'labkey-combo',
- forceSelection: true,
- multiSelect: true,
- displayField: 'plateId',
- valueField: 'plateId',
- itemId: 'sourcePlates',
- fieldLabel: 'Source Plate Id',
- store: {
- type: 'labkey-store',
- schemaName: 'tcrdb',
- sql: 'SELECT distinct plateId as plateId from tcrdb.cdnas c WHERE c.allReadsetsHaveData = false',
- autoLoad: true
- },
- labelWidth: 160
- },{
- xtype: 'textfield',
- itemId: 'adapter',
- fieldLabel: 'Adapter',
- labelWidth: 160,
- value: 'CTGTCTCTTATACACATCT'
- }]);
- }
- else {
- target.add({
- border: false,
- defaults: {
- border: false
- },
- items: [{
- html: 'Add an ordered list of plates, using tab-delimited columns. The first column(s) are plate ID and library type (GEX, VDJ, CITE, or HTO). These can either be one column (i.e. G234-1, C234-1, H234-1, or T234-1), or as two columns (234-1 GEX or 234-1 HTO). An optional next column is the lane assignment (i.e. Novaseq1, HiSeq1, HiSeq2). Finally, an optional final column can be used to provide the alias for this pool. This is mostly used for CITE-Seq/HTOs, where multiple libraries are pre-pooled. See these examples: ' +
- '' +
- '234-2\tGEX ' +
- '234-2\tVDJ ' +
- 'G233-2 ' +
- 'T235-2 ' +
- '234-2\tVDJ\tNovaSeq1 ' +
- 'G233-2\tNovaSeq1 ' +
- '235-2\tHTO\tHiSeq1\tBNB-HTO-1 ' +
- 'H235-2\tHiSeq1\tBNB-HTO-1 ' +
- '235-2\tHTO\tHiSeq2\tBNB-HTO-1 ' +
- 'H235-2\tHiSeq1\tBNB-HTO-1 ' +
- 'C235-2\tHiSeq1\tBNB-HTO-1' +
- ' ',
- border: false
- },{
- xtype: 'hidden',
- itemId: 'instrument',
- value: 'Novogene'
- },{
- xtype: 'textarea',
- itemId: 'plateList',
- fieldLabel: 'Plate List',
- labelAlign: 'top',
- width: 270,
- height: 200,
- enableKeyEvents: true,
- listeners: {
- specialkey: function (field, e) {
- if (e.getKey() === e.TAB) {
- field.setValue(field.getValue() + '\t');
- e.preventDefault();
- }
- }
- },
- },{
- xtype: 'ldk-numberfield',
- itemId: 'defaultVolume',
- fieldLabel: 'Default Volume (uL)',
- value: 10
- }],
- buttonAlign: 'left',
- buttons: [{
- text: 'Add',
- scope: this,
- handler: function (btn) {
- var text = btn.up('panel').down('#plateList').getValue();
- if (!text) {
- Ext4.Msg.alert('Error', 'Must enter a list of plates');
- return;
- }
-
- text = LDK.Utils.CSVToArray(Ext4.String.trim(text), '\t');
- Ext4.Array.forEach(text, function(r, idx){
- var val = r[0];
- if (val.startsWith('G')){
- val = val.substr(1);
- val = val.replace('_', '-');
- r[0] = 'GEX';
- r.unshift(val);
-
- }
- else if (val.startsWith('T')){
- val = val.substr(1);
- val = val.replace('_', '-');
- r[0] = 'VDJ';
- r.unshift(val);
- }
- else if (val.startsWith('H')){
- val = val.substr(1);
- val = val.replace('_', '-');
- r[0] = 'HTO';
- r.unshift(val);
- }
- else if (val.startsWith('C')){
- val = val.substr(1);
- val = val.replace('_', '-');
- r[0] = 'CITE';
- r.unshift(val);
- }
- }, this);
-
- var hadError = false;
- Ext4.Array.forEach(text, function(r){
- if (r.length < 2){
- hadError = true;
- }
-
- //ensure all rows are of length 4
- if (r.length !== 4) {
- for (i=0;i<(4-r.length);i++) {
- r.push('');
- }
- }
-
- Ext4.Array.forEach(r, function(val, idx){
- r[idx] = Ext4.String.trim(val);
- }, this);
- }, this);
-
- if (hadError) {
- Ext4.Msg.alert('Error', 'All rows must have at least 2 values');
- return;
- }
-
- this.onSubmit(btn, text);
- }
- }]
- });
- }
- }
- }
- }, {
- bodyStyle: 'padding: 5px;',
- itemId: 'importArea',
- border: false,
- defaults: {
- border: false
- }
- },{
- xtype: 'checkbox',
- boxLabel: 'Allow Duplicate Barcodes',
- checked: false,
- itemId: 'allowDuplicates'
- },{
- xtype: 'checkbox',
- boxLabel: 'Use Simple Sample Names',
- checked: true,
- itemId: 'simpleSampleNames'
- },{
- xtype: 'checkbox',
- boxLabel: 'Include Blanks',
- checked: true,
- itemId: 'includeBlanks'
- },{
- xtype: 'checkbox',
- boxLabel: 'Include Libraries With Data',
- checked: false,
- itemId: 'includeWithData',
- listeners: {
- change: function (field, val) {
- var target = field.up('tcrdb-libraryexportpanel').down('#sourcePlates');
- if (target) {
- var sql = 'SELECT distinct plateId as plateId from tcrdb.cdnas ' + (val ? '' : 'c WHERE c.allReadsetsHaveData = false');
- target.store.sql = sql;
- target.store.removeAll();
- target.store.load(function () {
- if (target.getPicker()) {
- target.getPicker().refresh();
- }
- }, this);
- }
- }
- }
- },{
- xtype: 'textarea',
- itemId: 'outputArea',
- fieldLabel: 'Output',
- labelAlign: 'top',
- width: 1000,
- height: 400
- }],
- buttonAlign: 'left',
- buttons: [{
- text: 'Submit',
- scope: this,
- handler: function(btn){
- this.onSubmit(btn);
- }
- }, {
- text: 'Download Data',
- itemId: 'downloadData',
- disabled: true,
- handler: function (btn) {
- var instrument = btn.up('tcrdb-libraryexportpanel').down('#instrument').getValue();
- var plateId = btn.up('tcrdb-libraryexportpanel').down('#sourcePlates').getValue();
- var delim = 'TAB';
- var extention = 'txt';
- var split = '\t';
- if (instrument !== 'NextSeq (MPSSR)') {
- delim = 'COMMA';
- extention = 'csv';
- split = ',';
- }
-
- var val = btn.up('tcrdb-libraryexportpanel').down('#outputArea').getValue();
- var rows = LDK.Utils.CSVToArray(Ext4.String.trim(val), split);
-
- LABKEY.Utils.convertToTable({
- fileName: plateId + '.' + extention,
- rows: rows,
- delim: delim
- });
- }
- },{
- text: 'Assign Readsets To Batch',
- itemId: 'readsetBatch',
- disabled: true,
- handler: function (btn) {
- var panel = btn.up('tcrdb-libraryexportpanel');
- var readsetIds = btn.readsetIds;
- if (!readsetIds) {
- Ext4.Msg.alert('Error', 'No Readset IDs Found');
- return;
- }
-
- Ext4.Msg.wait('Loading...');
- LABKEY.Query.selectRows({
- containerPath: Laboratory.Utils.getQueryContainerPath(),
- schemaName: 'sequenceanalysis',
- queryName: 'sequence_readsets',
- columns: 'rowid,container',
- filterArray: [LABKEY.Filter.create('rowid', readsetIds.join(';'), LABKEY.Filter.Types.IN)],
- scope: this,
- failure: LDK.Utils.getErrorCallback(),
- success: function (results) {
- Ext4.Msg.hide();
-
- if (!results || !results.rows || !results.rows.length) {
- Ext4.Msg.hide();
- Ext4.Msg.alert('Error', 'Readsets not found: ' + readsetIds.join(';'));
- return;
- }
-
- var readsetRows = results.rows;
-
- Ext4.create('Ext.window.Window', {
- title: 'Assign Readsets To Batch',
- width: 800,
- bodyStyle: 'padding: 10px;',
- readsetIds: readsetIds,
- items: [{
- html: 'The following readsets will be assigned to an instrument run/batch: ' + readsetIds.join(', '),
- style: 'padding-bottom: 10px;',
- border: false
- }, {
- xtype: 'textfield',
- fieldLabel: 'Run/Batch Name',
- labelWidth: 150,
- itemId: 'batchName'
- }, {
- xtype: 'ldk-integerfield',
- fieldLabel: 'Target Workbook',
- labelWidth: 150,
- itemId: 'targetWorkbook'
- }],
- buttons: [{
- text: 'Submit',
- scope: this,
- handler: function (btn) {
- var win = btn.up('window');
- var batchId = win.down('#batchName').getValue();
- if (!batchId) {
- Ext4.Msg.alert('Error', 'Must enter a batch name');
- return;
- }
-
- var workbook = win.down('#targetWorkbook').getValue();
- if (workbook) {
- Ext4.Msg.wait('Loading...');
- LABKEY.Query.selectRows({
- containerPath: Laboratory.Utils.getQueryContainerPath(),
- schemaName: 'core',
- queryName: 'workbooks',
- columns: 'EntityId',
- filterArray: [LABKEY.Filter.create('workbookId/workbookId', workbook, LABKEY.Filter.Types.EQUAL)],
- scope: this,
- failure: LDK.Utils.getErrorCallback(),
- success: function (results) {
- if (!results || !results.rows || !results.rows.length) {
- Ext4.Msg.hide();
- Ext4.Msg.alert('Error', 'Workbook not found: ' + workbook);
- return;
- }
-
- LDK.Assert.assertEquality('Expected single workbook to be returned', results.rows.length, 1);
-
- win.close();
- panel.createInstrumentRun(readsetRows, batchId, results.rows[0].EntityId);
- }
- });
- }
- else {
- panel.createInstrumentRun(readsetRows, batchId);
- }
- }
- }, {
- text: 'Cancel',
- handler: function (btn) {
- btn.up('window').close();
- }
- }]
- }).show();
- }
- });
- }
- }]
- });
-
- this.callParent(arguments);
-
- Ext4.Msg.wait('Loading...');
- LABKEY.Query.selectRows({
- schemaName: 'sequenceanalysis',
- queryName: 'barcodes',
- sort: 'group_name,tag_name',
- scope: this,
- failure: LDK.Utils.getErrorCallback(),
- success: function(results){
- this.barcodeMap = {};
-
- Ext4.Array.forEach(results.rows, function(r){
- this.barcodeMap[r.group_name] = this.barcodeMap[r.group_name] || {};
- this.barcodeMap[r.group_name][r.tag_name] = r.sequence;
- }, this);
-
- Ext4.Msg.hide();
- }
- });
- },
-
- createInstrumentRun: function (readsetRows, batchId, containerId) {
- containerId = containerId || Laboratory.Utils.getQueryContainerPath();
- LABKEY.Query.insertRows({
- containerPath: containerId,
- schemaName: 'sequenceanalysis',
- queryName: 'instrument_runs',
- scope: this,
- rows: [{
- name: batchId
- }],
- failure: LDK.Utils.getErrorCallback(),
- success: function (results) {
- var runId = results.rows[0].rowId;
- LDK.Assert.assertNotEmpty('Error creating instrument run', runId);
-
- Ext4.Array.forEach(readsetRows, function (rs) {
- rs.instrument_run_id = runId
- }, this);
-
- LABKEY.Query.updateRows({
- containerPath: Laboratory.Utils.getQueryContainerPath(),
- schemaName: 'sequenceanalysis',
- queryName: 'sequence_readsets',
- rows: readsetRows,
- failure: LDK.Utils.getErrorCallback(),
- success: function (results) {
- Ext4.Msg.hide();
- Ext4.Msg.alert('Success', 'Readsets updated');
- }
- });
- }
- });
- },
-
- onSubmit: function(btn, expectedPairs){
- var plateIds = [];
-
- if (expectedPairs) {
- var hadError = false;
- Ext4.Array.forEach(expectedPairs, function(p){
- plateIds.push(Ext4.String.trim(p[0]));
- }, this);
- }
- else {
- plateIds = btn.up('tcrdb-libraryexportpanel').down('#sourcePlates').getValue();
- }
-
- if (!plateIds || !plateIds.length){
- Ext4.Msg.alert('Error', 'Must provide the plate Id(s)');
- return;
- }
-
- plateIds = Ext4.unique(plateIds);
-
- var instrument = btn.up('tcrdb-libraryexportpanel').down('#instrument').getValue();
- var application = btn.up('tcrdb-libraryexportpanel').down('#application') ? btn.up('tcrdb-libraryexportpanel').down('#application').getValue() : null;
- var defaultVolume = btn.up('tcrdb-libraryexportpanel').down('#defaultVolume') ? btn.up('tcrdb-libraryexportpanel').down('#defaultVolume').getValue() : '';
- var adapter = btn.up('tcrdb-libraryexportpanel').down('#adapter') ? btn.up('tcrdb-libraryexportpanel').down('#adapter').getValue() : null;
- var includeWithData = btn.up('tcrdb-libraryexportpanel').down('#includeWithData').getValue();
- var allowDuplicates = btn.up('tcrdb-libraryexportpanel').down('#allowDuplicates').getValue();
- var simpleSampleNames = btn.up('tcrdb-libraryexportpanel').down('#simpleSampleNames').getValue();
- var includeBlanks = btn.up('tcrdb-libraryexportpanel').down('#includeBlanks').getValue();
- var doReverseComplement = btn.up('tcrdb-libraryexportpanel').doReverseComplement;
-
- var isMatchingApplication = function(application, libraryType, readsetApplication, rowLevelApplication){
- if (!application && !rowLevelApplication){
- return true;
- }
-
- if (application === 'Whole Transcriptome RNA-Seq'){
- return readsetApplication === 'RNA-seq' || readsetApplication === 'RNA-seq, Single Cell';
- }
- else if (application === 'TCR Enriched'){
- return readsetApplication === 'RNA-seq + Enrichment';
- }
- else if (readsetApplication === 'RNA-seq, Single Cell'){
- application = rowLevelApplication || application;
- return (libraryType.match(/^10x [35]\' GEX/) && application === '10x GEX') || (libraryType.match(/^10x 5' VDJ/) && application === '10x VDJ');
- }
- else if (readsetApplication === 'Cell Hashing'){
- application = rowLevelApplication || application;
- return (application === '10x HTO');
- }
- else if (readsetApplication === 'CITE-Seq'){
- application = rowLevelApplication || application;
- return (application === '10x CITE-Seq');
- }
- };
-
- var getSampleName = function(simpleSampleNames, readsetId, readsetName, suffix){
- return (simpleSampleNames ? 's_' + readsetId : readsetId + '_' + readsetName) + (suffix ? '_' + suffix : '');
- };
-
- Ext4.Msg.wait('Loading cDNA data');
- LABKEY.Query.selectRows({
- method: 'POST',
- containerPath: Laboratory.Utils.getQueryContainerPath(),
- schemaName: 'tcrdb',
- queryName: 'cdnas',
- sort: 'plateId,well/addressByColumn',
- columns: 'rowid,plateid' +
- ',readsetId,readsetId/name,readsetId/application,readsetId/librarytype,readsetId/barcode5,readsetId/barcode5/sequence,readsetId/barcode3,readsetId/barcode3/sequence,readsetId/totalFiles,readsetId/concentration' +
- ',enrichedReadsetId,enrichedReadsetId/name,enrichedReadsetId/application,enrichedReadsetId/librarytype,enrichedReadsetId/barcode5,enrichedReadsetId/barcode5/sequence,enrichedReadsetId/barcode3,enrichedReadsetId/barcode3/sequence,enrichedReadsetId/totalFiles,enrichedReadsetId/concentration' +
- ',hashingReadsetId,hashingReadsetId/name,hashingReadsetId/application,hashingReadsetId/librarytype,hashingReadsetId/barcode5,hashingReadsetId/barcode5/sequence,hashingReadsetId/barcode3,hashingReadsetId/barcode3/sequence,hashingReadsetId/totalFiles,hashingReadsetId/concentration' +
- ',citeseqReadsetId,citeseqReadsetId/name,citeseqReadsetId/application,citeseqReadsetId/librarytype,citeseqReadsetId/barcode5,citeseqReadsetId/barcode5/sequence,citeseqReadsetId/barcode3,citeseqReadsetId/barcode3/sequence,citeseqReadsetId/totalFiles,citeseqReadsetId/concentration',
- scope: this,
- filterArray: [LABKEY.Filter.create('plateId', plateIds.join(';'), LABKEY.Filter.Types.IN)],
- failure: LDK.Utils.getErrorCallback(),
- success: function (results) {
- Ext4.Msg.hide();
-
- if (!results || !results.rows || !results.rows.length) {
- Ext4.Msg.alert('Error', 'No libraries found for the selected plates');
- return;
- }
-
- var sortedRows = results.rows;
- if (expectedPairs) {
- sortedRows = [];
- var missingRows = [];
- Ext4.Array.forEach(expectedPairs, function(p){
- var found = false;
- Ext4.Array.forEach(results.rows, function(row){
- if (row.plateId === p[0]) {
- if (p[1] === 'GEX') {
- if (includeWithData || row['readsetId/totalFiles'] === 0) {
- if (row['readsetId'] && row['readsetId/librarytype'] && row['readsetId/librarytype'].match('GEX')) {
- sortedRows.push(Ext4.apply({targetApplication: '10x GEX', laneAssignment: (p.length > 2 ? p[2] : null), plateAlias: (p.length > 3 ? p[3] : null)}, row));
- found = true;
- return false;
- }
- }
- }
- else if (p[1] === 'HTO') {
- if (includeWithData || row['hashingReadsetId/totalFiles'] === 0) {
- if (row['hashingReadsetId'] && row['hashingReadsetId/application'] && row['hashingReadsetId/application'].match('Cell Hashing')) {
- sortedRows.push(Ext4.apply({targetApplication: '10x HTO', laneAssignment: (p.length > 2 ? p[2] : null), plateAlias: (p.length > 3 ? p[3] : null)}, row));
- found = true;
- return false;
- }
- }
- }
- else if (p[1] === 'CITE') {
- if (includeWithData || row['citeseqReadsetId/totalFiles'] === 0) {
- if (row['citeseqReadsetId'] && row['citeseqReadsetId/application'] && row['citeseqReadsetId/application'].match('CITE-Seq')) {
- sortedRows.push(Ext4.apply({targetApplication: '10x CITE-Seq', laneAssignment: (p.length > 2 ? p[2] : null), plateAlias: (p.length > 3 ? p[3] : null)}, row));
- found = true;
- return false;
- }
- }
- }
- else if (p[1] === 'VDJ') {
- if (includeWithData || row['enrichedReadsetId/totalFiles'] === 0) {
- if (row['enrichedReadsetId'] && row['enrichedReadsetId/librarytype'].match('VDJ')) {
- sortedRows.push(Ext4.apply({targetApplication: '10x VDJ', laneAssignment: (p.length > 2 ? p[2] : null), plateAlias: (p.length > 3 ? p[3] : null)}, row));
- found = true;
- return false;
- }
- }
- }
- }
- }, this);
-
- if (!found) {
- missingRows.push(p[0] + '/' + p[1]);
- }
- }, this);
-
- if (missingRows.length){
- Ext4.Msg.alert('Error', 'The following plates were not found: ' + missingRows.join(' '));
- return;
- }
- }
-
- var barcodes = 'Illumina';
- var readsetIds = {};
- var barcodeCombosUsed = [];
- if (instrument === 'NextSeq (MPSSR)' || instrument === 'Basic List (MedGenome)') {
- var rc5 = (instrument === 'NextSeq (MPSSR)');
- var rc3 = (instrument === 'NextSeq (MPSSR)');
-
- var rows = [['Name', 'Adapter', 'I7_Index_ID', 'I7_Seq', 'I5_Index_ID', 'I5_Seq'].join('\t')];
- Ext4.Array.forEach(sortedRows, function (r) {
- //only include readsets without existing data
- var processSample = function(rows, r, fieldName) {
- if (!readsetIds[r[fieldName]] && r[fieldName] && (includeWithData || r[fieldName + '/totalFiles'] === 0) && isMatchingApplication(application, r[fieldName + '/librarytype'], r[fieldName + '/application'], r.targetApplication)) {
- //allow for cell hashing / shared readsets
- readsetIds[r[fieldName]] = true;
-
- //reverse complement both barcodes:
- var barcode5 = rc5 ? doReverseComplement(r[fieldName + '/barcode5/sequence']) : r[fieldName + '/barcode5/sequence'];
- var barcode3 = rc3 ? doReverseComplement(r[fieldName + '/barcode3/sequence']) : r[fieldName + '/barcode3/sequence'];
- barcodeCombosUsed.push(r[fieldName + '/barcode5'] + '/' + r[fieldName + '/barcode3']);
- rows.push([getSampleName(simpleSampleNames, r[fieldName], r[fieldName + '/name']), adapter, r[fieldName + '/barcode5'], barcode5, r[fieldName + '/barcode3'], barcode3].join('\t'));
- }
- };
-
- processSample(rows, r, 'readsetId');
- processSample(rows, r, 'enrichedReadsetId');
- processSample(rows, r, 'hashingReadsetId');
- processSample(rows, r, 'citeseqReadsetId');
- }, this);
-
- //add missing barcodes:
- if (includeBlanks) {
- var blankIdx = 0;
- Ext4.Array.forEach(TCRdb.panel.LibraryExportPanel.BARCODES5, function (barcode5) {
- Ext4.Array.forEach(TCRdb.panel.LibraryExportPanel.BARCODES3, function (barcode3) {
- var combo = barcode5 + '/' + barcode3;
- if (barcodeCombosUsed.indexOf(combo) === -1) {
- blankIdx++;
- var barcode5Seq = rc5 ? doReverseComplement(this.barcodeMap[barcodes][barcode5]) : this.barcodeMap[barcodes][barcode5];
- var barcode3Seq = rc3 ? doReverseComplement(this.barcodeMap[barcodes][barcode3]) : this.barcodeMap[barcodes][barcode3];
-
- var name = simpleSampleNames ? 's_Blank' + blankIdx : plateIds.join(';').replace(/\//g, '-') + '_Blank' + blankIdx;
- rows.push([name, adapter, barcode5, barcode5Seq, barcode3, barcode3Seq].join('\t'));
- }
- }, this);
- }, this);
- }
- }
- else if (instrument === 'MiSeq (ONPRC)') {
- var rows = [];
- rows.push('[Header]');
- rows.push('IEMFileVersion,4');
- rows.push('Investigator Name,Bimber');
- rows.push('Experiment Name,' + plateIds.join(';'));
- rows.push('Date,11/16/2017');
- rows.push('Workflow,GenerateFASTQ');
- rows.push('Application,FASTQ Only');
- rows.push('Assay,Nextera XT');
- rows.push('Description,');
- rows.push('Chemistry,Amplicon');
- rows.push('');
- rows.push('[Reads]');
- rows.push('251');
- rows.push('251');
- rows.push('');
- rows.push('[Settings]');
- rows.push('ReverseComplement,0');
- rows.push('Adapter,' + adapter);
- rows.push('');
- rows.push('[Data]');
- rows.push('Sample_ID,Sample_Name,Sample_Plate,Sample_Well,I7_Index_ID,index,I5_Index_ID,index2,Sample_Project,Description');
-
- Ext4.Array.forEach(sortedRows, function (r) {
- //only include readsets without existing data
- if (!readsetIds[r.readsetId] && r.readsetId && (includeWithData || r['readsetId/totalFiles'] === 0) && isMatchingApplication(application, r['readsetId/librarytype'], r['readsetId/application'], r.targetApplication)) {
- //allow for cell hashing / shared readsets
- readsetIds[r.readsetId] = true;
-
- //reverse complement both barcodes:
- var barcode5 = doReverseComplement(r['readsetId/barcode5/sequence']);
- var barcode3 = r['readsetId/barcode3/sequence'];
- var cleanedName = r.readsetId + '_' + r['readsetId/name'].replace(/ /g, '_');
- cleanedName = cleanedName.replace(/\//g, '-');
-
- barcodeCombosUsed.push(r['readsetId/barcode5'] + '/' + r['readsetId/barcode3']);
- rows.push([r.readsetId, cleanedName, '', '', r['readsetId/barcode5'], barcode5, r['readsetId/barcode3'], barcode3].join(','));
- }
-
- if (!readsetIds[r.enrichedReadsetId] && r.enrichedReadsetId && (includeWithData || r['enrichedReadsetId/totalFiles'] == 0) && isMatchingApplication(application, r['enrichedReadsetId/librarytype'], r['enrichedReadsetId/application'], r.targetApplication)) {
- //allow for cell hashing / shared readsets
- readsetIds[r.enrichedReadsetId] = true;
-
- var barcode5 = doReverseComplement(r['enrichedReadsetId/barcode5/sequence']);
- var barcode3 = r['enrichedReadsetId/barcode3/sequence'];
- var cleanedName = r.enrichedReadsetId + '_' + r['enrichedReadsetId/name'].replace(/ /g, '_');
- cleanedName = cleanedName.replace(/\//g, '-');
-
- barcodeCombosUsed.push(r['enrichedReadsetId/barcode5'] + '/' + r['enrichedReadsetId/barcode3']);
- rows.push([r.enrichedReadsetId, cleanedName, '', '', r['enrichedReadsetId/barcode5'], barcode5, r['enrichedReadsetId/barcode3'], barcode3].join(','))
- }
- }, this);
-
- //add missing barcodes:
- if (includeBlanks) {
- var blankIdx = 0;
- Ext4.Array.forEach(TCRdb.panel.LibraryExportPanel.BARCODES5, function (barcode5) {
- Ext4.Array.forEach(TCRdb.panel.LibraryExportPanel.BARCODES3, function (barcode3) {
- var combo = barcode5 + '/' + barcode3;
- if (barcodeCombosUsed.indexOf(combo) === -1) {
- blankIdx++;
- var barcode5Seq = doReverseComplement(this.barcodeMap[barcodes][barcode5]);
- var barcode3Seq = this.barcodeMap[barcodes][barcode3];
- rows.push([plateIds.join(';').replace(/\//g, '-') + '_Blank' + blankIdx, null, null, null, barcode5, barcode5Seq, barcode3, barcode3Seq].join(','));
- }
- }, this);
- }, this);
- }
- }
- else if (instrument === '10x Sample Sheet' || instrument === 'Novogene') {
- //we make the default assumption that we're using 10x primers, which are listed in the sample-sheet orientation
- var doRC = false;
- var rows = [];
- var barcodes = '10x Chromium Single Cell v2';
-
- if (instrument === '10x Sample Sheet') {
- rows.push('Sample_ID,Sample_Name,index,Sample_Project');
- }
-
- //only include readsets without existing data
- var processType = function(readsetIds, rows, r, fieldName, suffix, size, phiX, samplePrefix, comment, doRC) {
- if (!readsetIds[r[fieldName]] && r[fieldName] && (includeWithData || r[fieldName + '/totalFiles'] === 0) && isMatchingApplication(application, r[fieldName + '/librarytype'], r[fieldName + '/application'], r.targetApplication)) {
- //allow for shared readsets across cDNAs (hashing, etc.)
- readsetIds[r[fieldName]] = true;
-
- var cleanedName = r[fieldName] + '_' + r[fieldName + '/name'].replace(/ /g, '_');
- cleanedName = cleanedName.replace(/\//g, '-');
- var sampleName = getSampleName(simpleSampleNames, r[fieldName], r[fieldName + '/name']) + (suffix && instrument === 'Novogene' ? '' : '-' + suffix);
-
- var barcode5s = r[fieldName + '/barcode5/sequence'] ? r[fieldName + '/barcode5/sequence'].split(',') : [];
- if (!barcode5s) {
- LDK.Utils.logError('Sample missing barcode: ' + sampleName);
- }
-
- barcodeCombosUsed.push([r[fieldName + '/barcode5'], '', r.laneAssignment || ''].join('/'));
- Ext4.Array.forEach(barcode5s, function (bc, idx) {
- bc = doRC ? doReverseComplement(bc) : bc;
-
- var data = [sampleName, (instrument === 'Novogene' ? '' : cleanedName), bc, ''];
- if (instrument === 'Novogene') {
- data = [sampleName];
- if (r.plateAlias) {
- data.unshift(r.plateAlias);
- }
- else {
- data.unshift(samplePrefix + r.plateId.replace(/-/g, '_'));
- }
-
- data.push('Macaca mulatta');
- data.push(bc);
- data.push('');
- data.push(r[fieldName + '/concentration'] || '');
- data.push(defaultVolume);
- data.push('');
- data.push(size);
- data.push(phiX); //PhiX
- data.push(r.laneAssignment || '');
- data.push(comment || 'Please QC individually and pool in equal amounts per lane');
- }
- rows.push(data.join(delim));
- }, this);
- }
- };
-
- var delim = instrument === 'Novogene' ? '\t' : ',';
- Ext4.Array.forEach(sortedRows, function (r) {
- processType(readsetIds, rows, r, 'readsetId', 'GEX', 500, 1, 'G', null, false);
- processType(readsetIds, rows, r, 'enrichedReadsetId', 'TCR', 700, 1, 'T', null, false);
- processType(readsetIds, rows, r, 'hashingReadsetId', 'HTO', 182, 5, 'H', 'Cell hashing, 190bp amplicon. Please QC individually and pool in equal amounts per lane', true);
- processType(readsetIds, rows, r, 'citeseqReadsetId', 'CITE', 182, 5, 'C', 'CITE-Seq, 190bp amplicon. Please QC individually and pool in equal amounts per lane', false);
- }, this);
-
- //add missing barcodes:
- if (includeBlanks && instrument !== 'Novogene') {
- var blankIdx = 0;
- Ext4.Array.forEach(TCRdb.panel.LibraryExportPanel.TENX_BARCODES, function (barcode5) {
- if (barcodeCombosUsed.indexOf(barcode5) === -1) {
- blankIdx++;
- var barcode5Seq = this.barcodeMap[barcodes][barcode5].split(',');
- Ext4.Array.forEach(barcode5Seq, function (seq, idx) {
- seq = doRC ? doReverseComplement(seq) : seq;
- rows.push([barcode5 + '_' + (idx + 1), plateIds.join(';').replace(/\//g, '-') + '_Blank' + blankIdx, seq, ''].join(delim));
- }, this);
- }
- }, this);
- }
- }
-
- //check for unique barcodes
- var sorted = barcodeCombosUsed.slice().sort();
- var duplicates = [];
- for (var i = 0; i < sorted.length - 1; i++) {
- if (sorted[i + 1] === sorted[i]) {
- duplicates.push(sorted[i]);
- }
- }
-
- duplicates = Ext4.unique(duplicates);
- if (!allowDuplicates && duplicates.length){
- Ext4.Msg.alert('Error', 'Duplicate barcodes: ' + duplicates.join(', '));
- btn.up('tcrdb-libraryexportpanel').down('#outputArea').setValue(null);
- btn.up('tcrdb-libraryexportpanel').down('#downloadData').setDisabled(true);
- }
- else {
- btn.up('tcrdb-libraryexportpanel').down('#outputArea').setValue(rows.join('\n'));
- btn.up('tcrdb-libraryexportpanel').down('#downloadData').setDisabled(false);
-
- var rsBtn = btn.up('tcrdb-libraryexportpanel').down('#readsetBatch');
- rsBtn.readsetIds = Ext4.Object.getKeys(readsetIds);
- rsBtn.setDisabled(false);
- }
- }
- });
- },
-
- doReverseComplement: function(seq){
- if (!seq){
- return seq;
- }
- var match={'a': 'T', 'A': 'T', 't': 'A', 'T': 'A', 'g': 'C', 'G': 'C', 'c': 'G', 'C': 'G'};
- var o = '';
- for (var i = seq.length - 1; i >= 0; i--) {
- if (match[seq[i]] === undefined) break;
- o += match[seq[i]];
- }
-
- return o;
- }
-});
diff --git a/tcrdb/resources/web/tcrdb/panel/PoolImportPanel.js b/tcrdb/resources/web/tcrdb/panel/PoolImportPanel.js
deleted file mode 100644
index fc4a4dbf8..000000000
--- a/tcrdb/resources/web/tcrdb/panel/PoolImportPanel.js
+++ /dev/null
@@ -1,1017 +0,0 @@
-Ext4.define('TCRdb.panel.PoolImportPanel', {
- extend: 'Ext.panel.Panel',
-
- COLUMNS: [{
- name: 'workbook',
- labels: ['Experiment/Workbook', 'Expt', 'Expt #', 'Experiment', 'Exp#', 'Exp #', 'Workbook', 'Workbook #'],
- allowRowSpan: true,
- alwaysShow: true,
- transform: 'expt',
- allowBlank: false
- },{
- name: 'plateId',
- labels: ['Pool/Tube', 'Pool', 'Pool Num', 'Pool #', 'Tube #', 'Tube#'],
- allowRowSpan: true,
- alwaysShow: true,
- transform: 'pool',
- allowBlank: false
- },{
- name: 'stimId',
- labels: ['Stim Id'],
- allowRowSpan: false,
- allowBlank: true,
- alwaysShow: true
- },{
- name: 'animalId',
- labels: ['Animal', 'Animal Id', 'SubjectId', 'Subject Id'],
- allowRowSpan: true,
- allowBlank: false,
- transform: 'animal'
- },{
- name: 'sampleDate',
- labels: ['Sample Date', 'Date'],
- alwaysShow: true,
- allowRowSpan: true,
- transform: 'sampleDate',
- allowBlank: false
- },{
- name: 'effector',
- labels: ['Effectors', 'Effector'],
- alwaysShow: true,
- allowRowSpan: true,
- transform: 'effector',
- allowBlank: false
- },{
- name: 'tissue',
- labels: ['Tissue', 'Tissue Sample'],
- alwaysShow: false,
- allowRowSpan: true,
- allowBlank: true
- },{
- name: 'stim',
- labels: ['Stim', 'Peptide Only Conditions'],
- allowRowSpan: false,
- allowBlank: false,
- transform: 'stim'
- },{
- name: 'stim_num',
- labels: ['Stim #'],
- allowRowSpan: false,
- alwaysShow: true
- },{
- name: 'population',
- labels: ['Population', 'Target Population', 'Target Pop'],
- allowRowSpan: true,
- allowBlank: false,
- transform: 'population'
- },{
- name: 'tetramer',
- labels: ['Tetramer'],
- allowRowSpan: false,
- allowBlank: true,
- transform: 'tetramer'
- },{
- name: 'sortId',
- labels: ['Sort Id'],
- allowRowSpan: false,
- allowBlank: true,
- alwaysShow: true
- },{
- name: 'hto',
- labels: ['HTO', 'HTO Oligo', 'HTO-Oligo', 'HTO barcode', 'Barcode'],
- allowRowSpan: false,
- transform: 'hto'
- },{
- name: 'cells',
- labels: ['Cells', 'Cell #', 'Sort', 'Sort Cell Count'],
- allowRowSpan: false,
- allowBlank: false,
- transform: 'cells'
- },{
- name: 'hto_library_index',
- labels: ['HTO Library Index', 'HTO Index', 'MultiSeq Index', 'MultiSeq Library Index'],
- allowRowSpan: true,
- transform: 'htoIndex'
- },{
- name: 'hto_library_conc',
- labels: ['HTO Library Conc', 'HTO Library Conc (ng/uL)', 'HTO (qubit) ng/uL', 'HTO (quibit) ng/uL', 'MultiSeq Library Conc', 'MultiSeq Library (qubit) ng/uL', 'MultiSeq Library Conc (qubit) ng/uL'],
- allowRowSpan: true
- },{
- name: 'citeseqpanel',
- labels: ['Cite-Seq Panel', 'Cite-Seq Panel Name', 'CiteSeq Panel'],
- allowRowSpan: true
- },{
- name: 'citeseq_library_index',
- labels: ['Cite-Seq Library Index', 'Cite-Seq Index', 'CiteSeq Library Index', 'CiteSeq Index', 'Cite-Seq Library Index', 'Cite-Seq Index', 'CiteSeq Library (qubit) ng/uL'],
- allowRowSpan: true,
- transform: 'citeSeqTenXBarcode'
- },{
- name: 'citeseq_library_conc',
- labels: ['Cite-Seq Library Conc', 'Cite-Seq Library Conc (ng/uL)', 'Cite-Seq (qubit) ng/uL', 'Cite-Seq (quibit) ng/uL'],
- allowRowSpan: true
- },{
- name: 'gex_library_index',
- labels: ['5\' GEX Library Index', '5\' GEX Index', 'GEX Index', 'GEX Library Index', '5-GEX Index', '5\'GEX Library Index'],
- allowRowSpan: true,
- transform: 'tenXBarcode'
- },{
- name: 'gex_library_conc',
- labels: ['5\' GEX Library Conc', 'GEX Library Conc', 'GEX Library Conc (ng/uL)', '5\' GEX Conc', 'GEX Conc', 'GEX Conc (ng/uL)', '5\' GEX (qubit) ng/uL', '5\' GEX Library (qubit) ng/uL'],
- allowRowSpan: true
- },{
- name: 'gex_library_fragment',
- labels: ['5\' GEX Library Fragment Size', 'GEX Library Fragment Size', '5\' GEX Fragment Size', 'GEX Fragment Size', 'GEX Library Fragment Size (bp)'],
- allowRowSpan: true
- },{
- name: 'tcr_library_index',
- labels: ['TCR Library Index', 'TCR Index', 'TCR Libray Index'],
- allowRowSpan: true,
- transform: 'tenXBarcode'
- },{
- name: 'tcr_library_conc',
- labels: ['TCR Library Conc', 'TCR Library Conc (ng/uL)', 'TCR (qubit) ng/uL', 'TCR library (qubit) ng/uL'],
- allowRowSpan: true
- },{
- name: 'tcr_library_fragment',
- labels: ['TCR Library Fragment Size', 'TCR Library Fragment Size (bp)'],
- allowRowSpan: true
- }],
-
- IGNORED_COLUMNS: [],
-
- transforms: {
- stim: function(val, panel) {
- if (val && (val === '--' || val === '-')) {
- val = 'NoStim';
- }
-
- return val;
- },
-
- animal: function(val, panel) {
- if (val) {
- val = val.replace(/ PBMC/, '');
- }
-
- return val;
- },
-
- htoIndex: function(val, panel) {
- if (Ext4.isNumeric(val)) {
- //indexes are named D7XX. accept rows named '1', '12', etc.
- var type = panel.down('#hashingType').getValue();
- if (type === 'CD298') {
- val = parseInt(val);
- if (val < 100) {
- val = val + 700;
- }
- return 'D' + val;
- }
- else if (type === 'MultiSeq') {
- val = parseInt(val);
-
- return 'MultiSeq-Idx-RP' + val;
- }
- else {
- LDK.Utils.logError('Unknown or missing hashingType: ' + type);
- }
- }
- else if (val) {
- var type = panel.down('#hashingType').getValue();
- if (type === 'MultiSeq') {
- val = String(val);
- if (val.match(/^MS-[0-9]+$/i)) {
- val = val.replace(/^MS(-)*/ig, 'MultiSeq-Idx-RP');
- }
-
- val = val.replace(/^MS[- ]Idx/ig, 'MultiSeq-Idx');
- val = val.replace(/^MultiSeq[- ]Idx[- ]RP/ig, 'MultiSeq-Idx-RP');
-
- return val;
- }
- }
-
- return val;
- },
-
- citeSeqTenXBarcode: function(val, panel){
- if (!val){
- return;
- }
-
- var barcodeSeries = panel.down('#citeseqBarcodeSeries').getValue();
- val = val.toUpperCase();
- var re = new RegExp('^' + barcodeSeries + '-', 'i');
- if (!val.match(re)) {
- if (val.length > 3) {
- //errorMsgs.push('Every row must have name, application and proper barcodes');
- }
- else {
- val = barcodeSeries + '-' + val;
- }
- }
-
- return val;
- },
-
- tenXBarcode: function(val, panel){
- if (!val){
- return;
- }
-
- var barcodeSeries = panel.down('#barcodeSeries').getValue();
- val = val.toUpperCase();
- var re = new RegExp('^' + barcodeSeries + '-', 'i');
- if (!val.match(re)) {
- if (val.length > 3) {
- //errorMsgs.push('Every row must have name, application and proper barcodes');
- }
- else {
- val = barcodeSeries + '-' + val;
- }
- }
-
- return val;
- },
-
- hto: function(val, panel){
- if (Ext4.isNumeric(val)){
- var type = panel.down('#hashingType').getValue();
- if (type === 'CD298') {
- return 'HTO-' + val;
- }
- else if (type === 'MultiSeq') {
- return 'MS-' + val;
- }
- }
- else if (val) {
- //Normalize hyphen use
- val = String(val);
- val = val.replace(/^MS(-)*/, 'MS-');
- val = val.replace(/^HTO(-)*/, 'HTO-');
- }
-
- return val;
- },
-
- expt: function(val, panel){
- return val || panel.EXPERIMENT;
- },
-
- cells: function(val, panel){
- return val ? Ext4.data.Types.INTEGER.convert(val) : val;
- },
-
- pool: function(val, panel, row){
- var workbook = row.workbook || panel.EXPERIMENT;
- //Note: convert values like 2B -> 2
- if (val && !Ext4.isNumeric(val)) {
- val = val.replace(/[^0-9]+/, '');
- }
- if (workbook && Ext4.isNumeric(val) && workbook !== val){
- return workbook + '-' + val;
- }
-
- return val;
- },
-
- tetramer: function(val, panel, row){
- if (val) {
- if (['Tet+', 'Tetramer+', 'Tetramer'].indexOf(val) > -1) {
- row.population = null;
- }
-
- row.population = row.population || val;
- }
-
- return val;
- },
-
- population: function(val, panel, row){
- if (val && ['Tet+', 'Tetramer+', 'Tetramer'].indexOf(val) > -1) {
- val = row.tetramer;
- }
-
- return val;
- },
-
- sampleDate: function(val, panel){
- return val || panel.SAMPLE_DATE;
- },
-
- effector: function(val, panel, row){
- if (val && val.endsWith('PBMC')) {
- var tmp = val.replace(/( )+PBMC$/,'');
- row.animalId = tmp;
- val = 'PBMC';
- }
- return val || panel.EFFECTOR;
- }
- },
-
- COLUMN_MAP: null,
-
- initComponent: function () {
- this.COLUMN_MAP = {};
- Ext4.Array.forEach(this.COLUMNS, function(col){
- this.COLUMN_MAP[col.name.toLowerCase()] = col;
- Ext4.Array.forEach(col.labels, function(alias){
- this.COLUMN_MAP[alias.toLowerCase()] = col;
- }, this);
- }, this);
-
- Ext4.apply(this, {
- title: null,
- border: false,
- defaults: {
- border: false
- },
- items: this.getPanelItems()
- });
-
- this.callParent(arguments);
- },
-
- getPanelItems: function(){
- return [{
- layout: {
- type: 'hbox'
- },
- items: [{
- xtype: 'ldk-integerfield',
- style: 'margin-right: 5px;',
- fieldLabel: 'Current Folder/Workbook',
- labelWidth: 200,
- minValue: 1,
- value: LABKEY.Security.currentContainer.type === 'workbook' ? LABKEY.Security.currentContainer.name : null,
- emptyText: LABKEY.Security.currentContainer.type === 'workbook' ? null : 'Showing All',
- listeners: {
- afterRender: function(field){
- new Ext4.util.KeyNav(field.getEl(), {
- enter : function(e){
- var btn = field.up('panel').down('#goButton');
- btn.handler(btn);
- },
- scope : this
- });
- }
- }
- },{
- xtype: 'button',
- itemId: 'goButton',
- scope: this,
- text: 'Go',
- handler: function(btn){
- var wb = btn.up('panel').down('ldk-integerfield').getValue();
- if (!wb){
- wb = '';
- }
-
- var container = LABKEY.Security.currentContainer.type === 'workbook' ? LABKEY.Security.currentContainer.parentPath + '/' + wb : LABKEY.Security.currentContainer.path + '/' + wb;
- window.location = LABKEY.ActionURL.buildURL('tcrdb', 'poolImport', container);
- }
- },{
- xtype: 'button',
- scope: this,
- hidden: !LABKEY.Security.currentUser.canInsert,
- text: 'Create Workbook',
- handler: function(btn){
- Ext4.create('Laboratory.window.WorkbookCreationWindow', {
- abortIfContainerIsWorkbook: false,
- canAddToExistingExperiment: false,
- controller: 'tcrdb',
- action: 'poolImport',
- title: 'Create Workbook'
- }).show();
- }
- }]
- }, {
- style: 'padding-top: 10px;',
- html: 'This page is designed to help import TCR/10x data, including pooled samples. Each sample tends to create many libraries with many indexes/barcodes to track. Use the fields below to download the excel template and paste data to import.'
- },{
- layout: 'hbox',
- items: [{
- xtype: 'button',
- text: 'Download Template',
- border: true,
- scope: this,
- href: LABKEY.ActionURL.getContextPath() + '/tcrdb/exampleData/ImportTemplate.xlsx'
- },{
- xtype: 'button',
- text: 'Download Example Import',
- border: true,
- scope: this,
- href: LABKEY.ActionURL.getContextPath() + '/tcrdb/exampleData/ImportExample.xlsx'
- }]
- }, {
- xtype: 'ldk-linkbutton',
- text: 'Manage Allowable Values for Stims',
- linkCls: 'labkey-text-link',
- href: LABKEY.ActionURL.buildURL('query', 'executeQuery', Laboratory.Utils.getQueryContainerPath(), {schemaName: 'tcrdb', 'query.queryName': 'peptides'}),
- style: 'margin-top: 10px;'
- },{
- xtype: 'textfield',
- style: 'margin-top: 20px;',
- fieldLabel: 'Expt Number',
- itemId: 'exptNum',
- value: LABKEY.Security.currentContainer.type === 'workbook' ? LABKEY.Security.currentContainer.name : null
- },{
- xtype: 'datefield',
- fieldLabel: 'Sample Date',
- itemId: 'sampleDate'
- },{
- xtype: 'textfield',
- fieldLabel: 'Effectors',
- itemId: 'effector',
- value: 'PBMC'
- },{
- xtype: 'checkbox',
- fieldLabel: 'Require HTO',
- itemId: 'requireHashTag',
- checked: true
- },{
- xtype: 'checkbox',
- fieldLabel: 'Require GEX Library',
- itemId: 'requireGEX',
- checked: false
- },{
- xtype: 'checkbox',
- fieldLabel: 'Require TCR Library',
- itemId: 'requireTCR',
- checked: false
- },{
- xtype: 'checkbox',
- fieldLabel: 'Require HTO Library',
- itemId: 'requireHTO',
- checked: false
- },{
- xtype: 'checkbox',
- fieldLabel: 'Require Cite-Seq Library',
- itemId: 'requireCITE',
- checked: false
- },{
- xtype: 'checkbox',
- fieldLabel: 'Require Library Concentrations',
- itemId: 'requireConc',
- checked: false
- }, {
- xtype: 'checkbox',
- fieldLabel: 'Skip Readsets',
- itemId: 'skipReadsets',
- checked: true,
- listeners: {
- scope: this,
- change: function(field, val) {
- field.up('panel').down('#requireGEX').setValue(!val);
- field.up('panel').down('#requireTCR').setValue(!val);
- field.up('panel').down('#requireHTO').setValue(!val);
- field.up('panel').down('#requireCITE').setValue(!val);
- }
- }
- },{
- xtype: 'ldk-simplecombo',
- fieldLabel: '10x GEX/TCR Barcode Series',
- itemId: 'barcodeSeries',
- forceSelection: true,
- storeValues: ['SI-GA'],
- value: 'SI-GA'
- },{
- xtype: 'ldk-simplecombo',
- fieldLabel: '10x Cite-Seq Barcode Series',
- itemId: 'citeseqBarcodeSeries',
- forceSelection: true,
- storeValues: ['SI-NA'],
- value: 'SI-NA'
- },{
- xtype: 'ldk-simplecombo',
- fieldLabel: 'Hashing Type',
- itemId: 'hashingType',
- forceSelection: true,
- storeValues: ['CD298', 'MultiSeq'],
- value: 'MultiSeq'
- },{
- xtype: 'textarea',
- fieldLabel: 'Paste Data Below',
- labelAlign: 'top',
- itemId: 'data',
- width: 1000,
- height: 300
- },{
- xtype: 'button',
- text: 'Preview',
- border: true,
- scope: this,
- handler: this.onPreview
- },{
- itemId: 'previewArea',
- style: 'margin-top: 20px;margin-bottom: 10px;',
- autoEl: 'table',
- cls: 'stripe hover'
- }];
- },
-
- onPreview: function(btn) {
- var text = this.down('#data').getValue();
- if (!text) {
- Ext4.Msg.alert('Error', 'Must provide the table of data');
- return;
- }
-
- this.EXPERIMENT = this.down('#exptNum').getValue();
- this.SAMPLE_DATE = this.down('#sampleDate').getValue();
- if (this.SAMPLE_DATE) {
- this.SAMPLE_DATE = Ext4.Date.format(this.SAMPLE_DATE, 'Y-m-d');
- }
- this.EFFECTOR = this.down('#effector').getValue();
-
- //this is a special case. if the first character is Tab, this indicates a blank field. Add a placeholder so it's not trimmed:
- if (text .startsWith("\t")) {
- text = 'Column1' + text;
- }
- text = Ext4.String.trim(text);
-
- var rows = LDK.Utils.CSVToArray(text, '\t');
- var colArray = this.parseHeader(rows.shift());
- var parsedRows = this.parseRows(colArray, rows);
- var stimRows = [];
- Ext4.Array.forEach(parsedRows, function(r){
- LDK.Assert.assertNotEmpty('Expected non-null workbook', r.workbook);
- stimRows.push({
- animalId: r.animalId,
- date: r.sampleDate,
- stim: r.stim,
- treatment: r.treatment || 'None',
- tissue: r.tissue,
- objectId: r.objectId,
- population: r.population,
- workbook: r.workbook
- });
- }, this);
-
- Ext4.Msg.wait('Looking for matching stims');
- LABKEY.Ajax.request({
- url: LABKEY.ActionURL.buildURL('tcrdb', 'getMatchingStims', Laboratory.Utils.getQueryContainerPath(), null),
- timeout: 99999,
- method: 'POST',
- jsonData: {
- stimRows: stimRows
- },
- scope: this,
- success: LABKEY.Utils.getCallbackWrapper(function(results){
- Ext4.Msg.hide();
-
- Ext4.Array.forEach(parsedRows, function(r){
- if (results.stimMap[r.objectId]){
- r.stimId = results.stimMap[r.objectId];
- }
-
- if (results.sortMap[r.objectId]){
- r.sortId = results.sortMap[r.objectId];
- }
- }, this);
-
- var groupedRows = this.groupForImport(colArray, parsedRows);
- if (!groupedRows){
- console.log('No rows after grouping');
- return;
- }
-
- this.renderPreview(colArray, parsedRows, groupedRows);
-
- if (results.recordErrors) {
- Ext4.Array.forEach(results.recordErrors, function(e){
- console.error(e);
- }, this);
- }
- }, this),
- failure: LDK.Utils.getErrorCallback()
- });
- },
-
- parseHeader: function(headerRow){
- var colArray = [];
- var colNames = {};
- Ext4.Array.forEach(headerRow, function(headerText, idx){
- //replace common terms:
- if (headerText.match(/ng\/ul/i) || headerText.match(/qubit/i)) {
- headerText = headerText.replace(/( )+(\()*ng\/ul(\))*/i, '');
- headerText = headerText.replace(/( )+(\()*qubit(\))*/i, '');
- headerText = Ext4.String.trim(headerText);
- if (!headerText.match(/Conc/i)) {
- headerText = headerText + ' Conc';
- }
- }
- headerText = headerText.replace(/CiteSeq/i, 'Cite-Seq');
- headerText = headerText.replace(/Cite Seq/i, 'Cite-Seq');
- headerText = headerText.replace(/^MS /i, 'MultiSeq ');
- headerText = headerText.replace(/Multi Seq/i, 'MultiSeq');
- headerText = headerText.replace(/Multi-Seq/i, 'MultiSeq');
- headerText = headerText.replace(/Library Index/i, 'Index');
- headerText = headerText.replace(/ RP#/i, '');
- headerText = headerText.replace(/:( )+10X Plate N Set A/i, '');
- headerText = headerText.replace(/:( )+10X Plate T Kit A/i, '');
-
- headerText = headerText.replace(/5'[- ]*GEX/i, 'GEX');
- headerText = headerText.replace(/5[- ]GEX/i, 'GEX');
- headerText = Ext4.String.trim(headerText);
-
- var colData = this.COLUMN_MAP[headerText.toLowerCase()];
- if (colData){
- colNames[colData.name] = idx;
- }
- }, this);
-
- Ext4.Array.forEach(this.COLUMNS, function(colData, idx){
- if (this.IGNORED_COLUMNS.indexOf(colData.name) > -1) {
- return;
- }
-
- if (colData.alwaysShow || colData.allowBlank === false || colNames[colData.name]){
- colData = Ext4.apply({}, colData);
- colData.dataIdx = colNames[colData.name];
-
- colArray.push(colData);
- }
- },this);
-
- return colArray;
- },
-
- parseRows: function(colArray, rows){
- var lastValueByCol = new Array(colArray.length);
- var ret = [];
-
- var doSplitCellsByPool = false;
- Ext4.Array.forEach(rows, function(row, rowIdx){
- var data = {
- objectId: LABKEY.Utils.generateUUID()
- };
-
- Ext4.Array.forEach(colArray, function(col, colIdx){
- var cell = Ext4.isDefined(col.dataIdx) ? row[col.dataIdx] : '';
- if (cell){
- if (col.transform && this.transforms[col.transform]){
- cell = this.transforms[col.transform](cell, this, data);
- }
-
- data[col.name] = cell;
- lastValueByCol[colIdx] = cell;
- }
- else if (col.allowRowSpan && lastValueByCol[colIdx]){
- data[col.name] = lastValueByCol[colIdx];
- }
- else {
- //allow transform even if value is null
- if (col.transform && this.transforms[col.transform]){
- cell = this.transforms[col.transform](cell, this, data);
- }
-
- data[col.name] = cell;
-
- if (!cell && col.name === 'cells' && lastValueByCol[colIdx]) {
- doSplitCellsByPool = true;
- }
- }
- }, this);
-
- ret.push(data);
- }, this);
-
- //split cells across rows
- if (doSplitCellsByPool) {
- var cellCountMap = {};
- Ext4.Array.forEach(ret, function(data) {
- if (data.plateId) {
- cellCountMap[data.plateId] = cellCountMap[data.plateId] || [];
- cellCountMap[data.plateId].push(data.cells);
- }
- }, this);
-
- Ext4.Array.forEach(Ext4.Object.getKeys(cellCountMap), function(plateId) {
- var arr = cellCountMap[plateId];
- var size = arr.length;
- arr = Ext4.Array.remove(arr, null);
- arr = Ext4.Array.remove(arr, '');
- if (arr.length === 1) {
- cellCountMap[plateId] = arr[0] / size;
- }
- else {
- delete cellCountMap[plateId];
- }
- }, this);
-
- Ext4.Array.forEach(ret, function(data) {
- if (data.plateId && cellCountMap[data.plateId]) {
- data.cells = cellCountMap[data.plateId];
- }
- }, this);
- }
-
- return ret;
- },
-
- groupForImport: function(colArray, parsedRows){
- var ret = {
- stimRows: [],
- sortRows: [],
- cDNARows: [],
- readsetRows: []
- };
-
- var errorsMsgs = [];
-
- //stims:
- var stimMap = {};
- var stimIdxs = {};
- var stimIdx = 0;
- Ext4.Array.forEach(parsedRows, function(row){
- var key = this.getStimKey(row);
- if (!stimMap[key]){
- var guid = LABKEY.Utils.generateUUID();
- stimIdx++;
-
- stimMap[key] = guid;
- stimIdxs[key] = stimIdx;
- LDK.Assert.assertNotEmpty('Expected non-null workbook', row.workbook);
- ret.stimRows.push({
- rowId: row.stimId || null,
- animalId: row.animalId,
- date: row.sampleDate,
- stim: row.stim,
- effector: row.effector,
- tissue: row.tissue,
- treatment: row.treatment || 'None',
- objectId: guid,
- workbook: row.workbook
- });
- }
-
- row.stim_num = row.stim_num || stimIdxs[key];
- }, this);
-
- //sorts:
- var sortMap = {};
- Ext4.Array.forEach(parsedRows, function(row){
- LDK.Assert.assertNotEmpty('Expected non-null workbook', row.workbook);
- var stimGUID = stimMap[this.getStimKey(row)];
- var key = this.getSortKey(row);
- if (!sortMap[key]){
- var guid = LABKEY.Utils.generateUUID();
- sortMap[key] = guid;
- ret.sortRows.push({
- rowId: row.sortId || null,
- stimGUID: stimGUID,
- population: row.population,
- replicate: row.replicate,
- cells: row.cells,
- well: row.well || 'Pool',
- hto: row.hto,
- buffer: row.buffer,
- objectId: guid,
- workbook: row.workbook
- });
- }
- }, this);
-
- //cDNA/readsets: group by pool
- var poolMap = {};
- Ext4.Array.forEach(parsedRows, function(row) {
- poolMap[row.plateId] = poolMap[row.plateId] || [];
- poolMap[row.plateId].push(row);
- }, this);
-
- Ext4.Object.each(poolMap, function(poolName, rowArr){
- var readsetGUIDs = {};
-
- var requireHTO = this.down('#requireHTO').getValue();
- var hashingType = this.down('#hashingType').getValue();
- var libraryType = null;
- if (hashingType === 'CD298'){
- libraryType = 'CD298 Hashing';
- }
- else if (hashingType === 'MultiSeq'){
- libraryType = 'MultiSeq';
- }
-
- var rs = this.processReadsetForGroup(poolName, rowArr, ret.readsetRows, 'hto', 'HTO', 'Cell Hashing', libraryType);
- if (Ext4.isString(rs)) {
- readsetGUIDs.hashingReadsetGUID = rs;
- }
- else if (requireHTO){
- errorsMsgs.push('Missing HTO library');
- errorsMsgs = errorsMsgs.concat(rs);
- return false;
- }
-
- var requireCITE = this.down('#requireCITE').getValue();
- var rs = this.processReadsetForGroup(poolName, rowArr, ret.readsetRows, 'citeseq', 'CITE', 'CITE-Seq', null);
- if (Ext4.isString(rs)) {
- readsetGUIDs.citeseqReadsetGUID = rs;
- }
- else if (requireCITE){
- errorsMsgs.push('Missing CITE-Seq library');
- errorsMsgs = errorsMsgs.concat(rs);
- return false;
- }
-
- var requireGEX = this.down('#requireGEX').getValue();
- rs = this.processReadsetForGroup(poolName, rowArr, ret.readsetRows, 'gex', 'GEX', 'RNA-seq, Single Cell', '10x 5\' GEX');
- if (Ext4.isString(rs)) {
- readsetGUIDs.readsetGUID = rs;
- }
- else if (requireGEX){
- errorsMsgs.push('Missing GEX library');
- errorsMsgs = errorsMsgs.concat(rs);
- return false;
- }
-
- var requireTCR = this.down('#requireTCR').getValue();
- rs = this.processReadsetForGroup(poolName, rowArr, ret.readsetRows, 'tcr', 'TCR', 'RNA-seq, Single Cell', '10x 5\' VDJ (Rhesus A/B/G)');
- if (Ext4.isString(rs)) {
- readsetGUIDs.enrichedReadsetGUID = rs;
- }
- else if (requireTCR){
- errorsMsgs.push('Missing TCR library');
- errorsMsgs = errorsMsgs.concat(rs);
- return false;
- }
-
- Ext4.Array.forEach(rowArr, function(row) {
- var sortKey = this.getSortKey(row);
-
- LDK.Assert.assertNotEmpty('Expected non-null workbook', row.workbook);
- var cDNA = Ext4.apply({
- sortGUID: sortMap[sortKey],
- chemistry: null,
- plateId: row.plateId,
- well: row.well || 'Pool',
- citeseqpanel: row.citeseqpanel,
- workbook: row.workbook
- }, readsetGUIDs);
-
- ret.cDNARows.push(cDNA);
- }, this);
- }, this);
-
- if (errorsMsgs.length) {
- errorsMsgs = Ext4.unique(errorsMsgs);
- Ext4.Msg.alert('Error', errorsMsgs.join(' '));
- return null;
- }
-
- return ret;
- },
-
- processReadsetForGroup: function(poolName, rowArr, readsetRows, prefix, type, application, librarytype){
- var idxValues = this.getUniqueValues(rowArr, prefix + '_library_index');
- var conc = this.getUniqueValues(rowArr, prefix + '_library_conc');
- var fragment = this.getUniqueValues(rowArr, prefix + '_library_fragment');
- var workbook = this.getUniqueValues(rowArr, 'workbook');
- if (workbook.length > 1) {
- return ['Error', 'Pool ' + poolName + ' uses more workbook ' + workbook.join(';')];
- }
- workbook = workbook.length === 1 ? workbook[0] : null;
-
- var subjectid = this.getUniqueValues(rowArr, 'animalId');
- subjectid = subjectid.length === 1 ? subjectid[0] : null;
-
- var requireConc = this.down('#requireConc').getValue();
-
- if (idxValues.length === 1){
- if (requireConc && !conc[0]) {
- return ['Pool ' + poolName + ': did not provide concentration for library: ' + type];
- }
-
- var guid = LABKEY.Utils.generateUUID();
- LDK.Assert.assertNotEmpty('Expected non-null workbook', workbook);
- readsetRows.push({
- name: poolName + '-' + type,
- barcode5: idxValues[0],
- concentration: conc[0],
- fragmentSize: fragment[0],
- platform: 'ILLUMINA',
- application: application,
- librarytype: librarytype,
- subjectid: subjectid,
- sampleType: 'mRNA',
- objectId: guid,
- workbook: workbook
- });
-
- return guid;
- }
- else if (idxValues.length > 1) {
- return ['Error', 'Pool ' + poolName + ' uses more than one ' + type + ' index'];
- }
- else if (idxValues.length === 0) {
- var required = this.down('#require' + type).getValue();
- if (required) {
- return ['Error', 'No index found for pool: ' + poolName + ', for library type: ' + type];
- }
- }
- },
-
- getUniqueValues: function(rowArr, colName){
- var ret = [];
- Ext4.Array.forEach(rowArr, function(row){
- if (row[colName])
- ret.push(row[colName]);
- }, this);
-
- return Ext4.unique(ret);
- },
-
- renderPreview: function(colArray, parsedRows, groupedRows){
- var previewArea = this.down('#previewArea');
- previewArea.removeAll();
-
- var columns = [{title: 'Row #'}];
- var colIdxs = [];
- Ext4.Array.forEach(colArray, function(col, idx){
- if (col){
- columns.push({title: col.labels[0], className: 'dt-center'});
- colIdxs.push(idx);
- }
- }, this);
-
- var data = [];
- var missingValues = false;
- var requireHTO = this.down('#requireHTO').getValue() || (this.down('#requireHashTag') && this.down('#requireHashTag').getValue());
- Ext4.Array.forEach(parsedRows, function(row, rowIdx){
- var toAdd = [rowIdx + 1];
- Ext4.Array.forEach(colIdxs, function(colIdx){
- var colDef = colArray[colIdx];
- var propName = colDef.name;
-
- var allowBlank = colDef.allowBlank;
- if (requireHTO && colDef.name === 'hto') {
- allowBlank = false;
- }
-
- if (allowBlank === false && Ext4.isEmpty(row[propName])){
- missingValues = true;
- toAdd.push('MISSING');
- }
- else {
- toAdd.push(row[propName] || 'ND');
- }
-
- }, this);
-
- data.push(toAdd);
- }, this);
-
- var id = '#' + previewArea.getId();
- if ( jQuery.fn.dataTable.isDataTable(id) ) {
- jQuery(id).DataTable().destroy();
- }
-
- jQuery(id).DataTable({
- data: data,
- pageLength: 500,
- dom: 'rt<"bottom"BS><"clear">',
- buttons: missingValues ? [] : [{
- text: 'Submit',
- action: this.onSubmit,
- rowData: {
- colArray: colArray,
- parsedRows: parsedRows,
- groupedRows: groupedRows,
- panel: this
- }
- }],
- columns: columns
- });
-
- previewArea.doLayout();
-
- if (missingValues){
- Ext4.Msg.alert('Error', 'One or more rows is missing data. Any required cells without values are marked MISSING');
- }
- },
-
- onSubmit: function(e, dt, node, config){
- Ext4.Msg.wait('Saving...');
- LABKEY.Ajax.request({
- url: LABKEY.ActionURL.buildURL('tcrdb', 'importTenx', Laboratory.Utils.getQueryContainerPath()),
- method: 'POST',
- jsonData: config.rowData.groupedRows,
- scope: this,
- success: function(){
- Ext4.Msg.hide();
- Ext4.Msg.alert('Success', 'Data Saved', function(){
- window.location = LABKEY.ActionURL.buildURL('query', 'executeQuery.view', Laboratory.Utils.getQueryContainerPath(), {'query.queryName': 'cdnas', schemaName: 'tcrdb', 'query.sort': '-created'})
- }, this);
- },
- failure: LDK.Utils.getErrorCallback()
- });
- },
-
- getStimKey: function(data){
- return [data.stimId, data.animalId, data.stim, data.treatment, data.tissue, (Ext4.isDate(data.sampleDate) ? Ext4.Date.format(data.sampleDate, 'Y-m-d') : data.sampleDate)].join('|');
- },
-
- getSortKey: function(data){
- return [this.getStimKey(data), data.sortId, data.population, data.hto].join('|');
- }
-});
\ No newline at end of file
diff --git a/tcrdb/resources/web/tcrdb/panel/StimPanel.js b/tcrdb/resources/web/tcrdb/panel/StimPanel.js
deleted file mode 100644
index 62e1f2af9..000000000
--- a/tcrdb/resources/web/tcrdb/panel/StimPanel.js
+++ /dev/null
@@ -1,1515 +0,0 @@
-Ext4.define('TCRdb.panel.StimPanel', {
- extend: 'Ext.panel.Panel',
- alias: 'widget.tcrdb-stimpanel',
-
- initComponent: function(){
- Ext4.apply(this, {
- title: null,
- border: false,
- defaults: {
- border: false
- },
- items: [{
- layout: {
- type: 'hbox'
- },
- items: [{
- xtype: 'ldk-integerfield',
- style: 'margin-right: 5px;',
- fieldLabel: 'Current Folder/Workbook',
- labelWidth: 200,
- minValue: 1,
- value: LABKEY.Security.currentContainer.type === 'workbook' ? LABKEY.Security.currentContainer.name : null,
- emptyText: LABKEY.Security.currentContainer.type === 'workbook' ? null : 'Showing All',
- listeners: {
- afterRender: function(field){
- new Ext4.util.KeyNav(field.getEl(), {
- enter : function(e){
- var btn = field.up('panel').down('#goButton');
- btn.handler(btn);
- },
- scope : this
- });
- }
- }
- },{
- xtype: 'button',
- itemId: 'goButton',
- scope: this,
- text: 'Go',
- handler: function(btn){
- var wb = btn.up('panel').down('ldk-integerfield').getValue();
- if (!wb){
- wb = '';
- }
-
- var container = LABKEY.Security.currentContainer.type === 'workbook' ? LABKEY.Security.currentContainer.parentPath + '/' + wb : LABKEY.Security.currentContainer.path + '/' + wb;
- window.location = LABKEY.ActionURL.buildURL('tcrdb', 'stimDashboard', container);
- }
- },{
- xtype: 'button',
- scope: this,
- hidden: !LABKEY.Security.currentUser.canInsert,
- text: 'Create Workbook',
- handler: function(btn){
- Ext4.create('Laboratory.window.WorkbookCreationWindow', {
- abortIfContainerIsWorkbook: false,
- canAddToExistingExperiment: false,
- controller: 'tcrdb',
- action: 'stimDashboard',
- title: 'Create Workbook'
- }).show();
- }
- }]
- },{
- style: 'padding-top: 10px;',
- html: 'This page is designed to help manage samples for the TCR sequencing project. Where possible we try to carry sample information from to step to step; however, each step often generates new info we need to track, and sometimes samples and plates generated at different times are combined for later steps. The basic steps are:
'
- }]
- });
-
- this.callParent(arguments);
-
- Ext4.Msg.wait('Loading...');
- this.loadData();
- },
-
- getFolderSummaryConfig: function(){
-
- },
-
- loadData: function(){
- var multi = new LABKEY.MultiRequest();
- multi.add(LABKEY.Query.selectRows, {
- schemaName: 'laboratory',
- queryName: 'well_layout',
- columns: 'well_96,addressbycolumn_96',
- filterArray: [LABKEY.Filter.create('plate', 1)],
- sort: 'addressbycolumn_96',
- scope: this,
- failure: LDK.Utils.getErrorCallback(),
- success: function(results){
- this.wellNames96 = [];
-
- Ext4.Array.forEach(results.rows, function(r){
- this.wellNames96.push(r.well_96);
- }, this);
- }
- });
-
- multi.add(LABKEY.Query.selectRows, {
- schemaName: 'sequenceanalysis',
- queryName: 'barcodes',
- sort: 'group_name,tag_name',
- scope: this,
- failure: LDK.Utils.getErrorCallback(),
- success: function(results){
- this.barcodeMap = {};
-
- Ext4.Array.forEach(results.rows, function(r){
- this.barcodeMap[r.group_name] = this.barcodeMap[r.group_name] || {};
- this.barcodeMap[r.group_name][r.tag_name] = r.sequence;
- }, this);
- }
- });
-
- multi.add(LABKEY.Query.selectRows, {
- schemaName: 'tcrdb',
- queryName: 'stims',
- columns: 'rowid,tubeNum,animalId,effector,effectors,date,stim,treatment,costim,background,activated,comment,numSorts,status',
- scope: this,
- failure: LDK.Utils.getErrorCallback(),
- success: function(results){
- this.stimRows = results.rows;
- this.stimStats = {
- totalStims: 0,
- lackingSort: 0,
- hasStatus: 0
- };
-
- Ext4.Array.forEach(results.rows, function(r){
- this.stimStats.totalStims++;
- if (!r.numSorts && !r.status){
- this.stimStats.lackingSort++;
- }
-
- if (r.status){
- this.stimStats.hasStatus++;
- }
- }, this);
- }
- });
-
- multi.add(LABKEY.Query.selectRows, {
- schemaName: 'tcrdb',
- queryName: 'sorts',
- columns: 'rowid,stimId,stimId/animalId,stimId/effector,stimId/date,stimId/treatment,population,replicate,cells,plateId,well,well/addressByColumn,numLibraries,maxCellsForPlate,container',
- scope: this,
- failure: LDK.Utils.getErrorCallback(),
- success: function(results){
- this.sortRows = results.rows;
- this.sortStats = {
- totalSorts: 0,
- totalPlates: [],
- lackingLibraries: 0,
- bulkLackingLibraries: 0,
- totalPlatesLackingLibraries: [],
- totalBulkPlatesLackingLibraries: []
- };
-
- Ext4.Array.forEach(results.rows, function(r){
- this.sortStats.totalSorts++;
- if (!r.numLibraries){
- this.sortStats.lackingLibraries++;
-
- if (r.cells > 1) {
- this.sortStats.bulkLackingLibraries++;
- }
-
- if (r.plateId){
- this.sortStats.totalPlatesLackingLibraries.push(r.plateId);
-
- if (r.maxCellsForPlate > 1){
- this.sortStats.totalBulkPlatesLackingLibraries.push(r.plateId);
- }
- }
- }
-
- if (r.plateId){
- this.sortStats.totalPlates.push(r.plateId);
- }
- }, this);
-
- this.sortStats.totalPlates = Ext4.unique(this.sortStats.totalPlates);
- this.sortStats.totalPlatesLackingLibraries = Ext4.unique(this.sortStats.totalPlatesLackingLibraries);
- this.sortStats.totalBulkPlatesLackingLibraries = Ext4.unique(this.sortStats.totalBulkPlatesLackingLibraries);
- }
- });
-
- multi.add(LABKEY.Query.selectRows, {
- schemaName: 'tcrdb',
- queryName: 'cdnas',
- columns: 'rowid,sortId,cells,plateId,well,well/addressByColumn,readsetId,readsetId/totalFiles,enrichedReadsetId,enrichedReadsetId/totalFiles,sortId/stimId,sortId/stimId/animalId,sortId/stimId/effector,sortId/stimId/date,sortId/stimId/treatment,sortId/population,sortId/replicate,sortId/cells,sortId/plateId,sortId/sortId/well,sortId/well/addressByColumn,sortId/stimId/stim',
- sort: 'plateId,well/addressByColumn',
- scope: this,
- failure: LDK.Utils.getErrorCallback(),
- success: function(results){
- this.libraryRows = results.rows;
- this.libraryStats = {
- totalLibraries: 0,
- totalPlates: [],
- lackingAnyReadset: 0,
- totalPlatesLackingAnyReadset: [],
- withReadset: 0,
- totalPlatesWithReadset: [],
- withTCRReadset: 0,
- totalPlatesWithTCRReadset: [],
- lackingBarcodes: 0
-
- };
-
- Ext4.Array.forEach(results.rows, function(r){
- this.libraryStats.totalLibraries++;
- if (r.readsetId){
- this.libraryStats.withReadset++;
- if (r.plateId){
- this.libraryStats.totalPlatesWithReadset.push(r.plateId);
- }
- }
-
- if (r.enrichedReadsetId) {
- this.libraryStats.withTCRReadset++;
- if (r.plateId){
- this.libraryStats.totalPlatesWithTCRReadset.push(r.plateId);
- }
- }
-
- if (!r.enrichedReadsetId && !r.readsetId) {
- this.libraryStats.lackingAnyReadset++;
- if (r.plateId){
- this.libraryStats.totalPlatesLackingAnyReadset.push(r.plateId);
- }
- }
-
- if (r.plateId){
- this.libraryStats.totalPlates.push(r.plateId);
- }
- }, this);
-
- this.libraryStats.totalPlates = Ext4.unique(this.libraryStats.totalPlates);
- this.libraryStats.totalPlatesWithReadset = Ext4.unique(this.libraryStats.totalPlatesWithReadset);
- this.libraryStats.totalPlatesWithTCRReadset = Ext4.unique(this.libraryStats.totalPlatesWithTCRReadset);
- this.libraryStats.totalPlatesLackingAnyReadset = Ext4.unique(this.libraryStats.totalPlatesLackingAnyReadset);
- }
- });
-
- multi.add(LABKEY.Query.selectRows, {
- schemaName: 'sequenceanalysis',
- queryName: 'sequence_readsets',
- columns: 'rowid,name,application,totalFiles',
- scope: this,
- failure: LDK.Utils.getErrorCallback(),
- success: function(results){
- this.readsetRows = results.rows;
- this.readsetStats = {
- totalReadsets: 0,
- lackingData: 0,
- dataImported: 0
- };
-
- Ext4.Array.forEach(results.rows, function(r){
- this.readsetStats.totalReadsets++;
- if (!r.totalFiles){
- this.readsetStats.lackingData++;
- }
- else {
- this.readsetStats.dataImported++;
- }
- }, this);
- }
- });
-
- multi.send(this.onDataLoad, this);
- },
-
- onDataLoad: function(){
- this.add(this.getItemConfig());
-
- Ext4.Msg.hide();
- },
-
- getItemConfig: function(){
- return {
- defaults: {
- border: true,
- style: 'padding-bottom: 10px;',
- bodyStyle: 'padding: 5px;'
- },
- items: [{
- defaults: {
- border: false
- },
- title: 'Step 1: Stims/Blood Draws',
- layout: {
- type: 'table',
- columns: 2,
- tdAttrs: { style: 'padding-right: 10px;' }
- },
- items: [{
- html: 'Total Stims:'
- },{
- html: '' + this.stimStats.totalStims + ' '
- },{
- html: 'Lacking Sorts:'
- },{
- html: '' + this.stimStats.lackingSort + ' '
- },{
- html: 'Non-passing Status:'
- },{
- html: '' + this.stimStats.hasStatus + ' '
- },{
- xtype: 'ldk-linkbutton',
- text: 'Import Stims',
- href: 'javascript:void(0);',
- linkCls: 'labkey-text-link',
- handler: function(btn){
- if (LABKEY.Security.currentContainer.type === 'workbook'){
- Ext4.define('TCRdb.window.StimUploadWindow', {
- extend: 'Ext.window.Window',
- initComponent: function(){
- Ext4.apply(this, {
- title: 'Import Stims',
- items: [{
- xtype: 'labkey-exceluploadpanel',
- bubbleEvents: ['uploadexception', 'uploadcomplete'],
- itemId: 'theForm',
- title: null,
- buttons: null,
- containerPath: Laboratory.Utils.getQueryContainerPath(),
- schemaName: 'tcrdb',
- queryName: 'stims',
- populateTemplates: function(meta){
- Ext4.Msg.hide();
- var toAdd = [];
-
- toAdd.push({
- html: 'Use the button below to download an excel template for uploading stims.',
- border: false,
- style: 'padding-bottom: 10px;',
- width: 700
- });
-
- toAdd.push({
- xtype: 'ldk-integerfield',
- itemId: 'templateRows',
- fieldLabel: 'Total Stims',
- labelWidth: 120,
- value: 10
- });
-
- toAdd.push({
- xtype: 'textfield',
- itemId: 'treatment',
- fieldLabel: 'Treatment',
- labelWidth: 120,
- value: 'TAPI-0'
- });
-
- toAdd.push({
- xtype: 'textfield',
- itemId: 'coStim',
- fieldLabel: 'Co-Stim',
- labelWidth: 120,
- value: 'CD28/CD49d'
- });
-
- toAdd.push({
- xtype: 'textfield',
- itemId: 'effectors',
- fieldLabel: 'Effector',
- labelWidth: 120,
- value: 'PBMC'
- });
-
- toAdd.push({
- xtype: 'ldk-numberfield',
- itemId: 'numEffectors',
- fieldLabel: '# Effectors',
- labelWidth: 120,
- value: 1000000
- });
-
- toAdd.push({
- xtype: 'textfield',
- itemId: 'apc',
- fieldLabel: 'APCs',
- labelWidth: 120,
- value: 'PBMC'
- });
-
- toAdd.push({
- xtype: 'ldk-numberfield',
- itemId: 'numAPC',
- fieldLabel: '# APCs',
- labelWidth: 120,
- value: null
- });
-
- toAdd.push({
- xtype: 'button',
- style: 'margin-bottom: 10px;',
- text: 'Download Template',
- border: true,
- handler: this.generateExcelTemplate
- });
-
- this.down('#templateArea').add(toAdd);
- },
- generateExcelTemplate: function(){
- var win = this.up('window');
- var numRows = win.down('#templateRows').getValue() || 1;
- var effectors = win.down('#effectors').getValue();
- var numEffectors = win.down('#numEffectors').getValue();
- var apc = win.down('#apc').getValue();
- var numAPC = win.down('#numAPC').getValue();
- var treatment = win.down('#treatment').getValue();
- var coStim = win.down('#coStim').getValue();
-
- var data = [];
- data.push(['Tube #', 'Animal/Cell', 'Sample Date', 'Effectors', '# Effectors', 'APCs', '# APCs', 'Treatment', 'Co-stim', 'Peptide/Stim', 'Comment']);
- for (var i=0;i' + this.sortStats.totalSorts + ''
- }, {
- xtype: 'ldk-linkbutton',
- text: '(' + this.sortStats.totalPlates.length + ' plates)',
- href: 'javascript:void(0);',
- handler: this.getPlateCallback(this.sortStats.totalPlates)
- }, {
- html: 'Lacking cDNA Libraries (All):'
- }, {
- html: '' + this.sortStats.lackingLibraries + ' '
- }, {
- xtype: 'ldk-linkbutton',
- text: '(' + this.sortStats.totalPlatesLackingLibraries.length + ' plates)',
- href: 'javascript:void(0);',
- handler: this.getPlateCallback(this.sortStats.totalPlatesLackingLibraries)
- }, {
- html: 'Lacking cDNA Libraries (Bulk):'
- }, {
- html: '' + this.sortStats.bulkLackingLibraries + ' '
- }, {
- xtype: 'ldk-linkbutton',
- text: '(' + this.sortStats.totalBulkPlatesLackingLibraries.length + ' plates)',
- href: 'javascript:void(0);',
- handler: this.getPlateCallback(this.sortStats.totalBulkPlatesLackingLibraries)
- }]
- },{
- xtype: 'ldk-linkbutton',
- text: 'Import Sort Data For Stims',
- href: 'javascript:void(0);',
- linkCls: 'labkey-text-link',
- handler: function (btn) {
- if (LABKEY.Security.currentContainer.type === 'workbook') {
- Ext4.define('TCRdb.window.SortUploadWindow', {
- extend: 'Ext.window.Window',
- initComponent: function () {
- Ext4.apply(this, {
- title: 'Import Sorts for Stims',
- items: [{
- xtype: 'labkey-exceluploadpanel',
- bubbleEvents: ['uploadexception', 'uploadcomplete'],
- itemId: 'theForm',
- title: null,
- buttons: null,
- containerPath: Laboratory.Utils.getQueryContainerPath(),
- schemaName: 'tcrdb',
- queryName: 'sorts',
- populateTemplates: function (meta) {
- Ext4.Msg.hide();
- var toAdd = [];
-
- toAdd.push({
- html: 'Use the button below to download an excel template pre-populated with data from the sorts imported into this workbook.',
- border: false,
- style: 'padding-bottom: 10px;',
- width: 700
- });
-
- toAdd.push({
- xtype: 'textfield',
- itemId: 'buffer',
- fieldLabel: 'Sort Buffer',
- labelWidth: 120,
- value: 'Takara Buffer'
- });
-
- toAdd.push({
- xtype: 'checkbox',
- itemId: 'skipWithData',
- fieldLabel: 'Skip Stims With Sorts Imported',
- labelWidth: 120,
- helpPopup: 'If checked, stims with sort records already importd will be skipped',
- checked: true
- });
-
- toAdd.push({
- xtype: 'ldk-integerfield',
- itemId: 'templateRows',
- fieldLabel: 'Rows Per Stim',
- labelWidth: 120,
- helpPopup: 'For each stim, the template will include this many rows',
- value: 2
- });
-
- toAdd.push({
- xtype: 'button',
- style: 'margin-bottom: 10px;',
- text: 'Download Template',
- border: true,
- handler: this.generateExcelTemplate
- });
-
- this.down('#templateArea').add(toAdd);
- },
- generateExcelTemplate: function () {
- var win = this.up('window');
- var rowsPer = win.down('#templateRows').getValue() || 1;
- var skipWithData = win.down('#skipWithData').getValue();
- var buffer = win.down('#buffer').getValue();
-
- var data = [];
- data.push(['TubeNum', 'StimId', 'AnimalId', 'SampleDate', 'Peptide/Stim', 'Treatment', 'Buffer', 'Population', 'Replicate', 'Cells', 'PlateId', 'Well', 'Comment']);
- Ext4.Array.forEach(win.stimRows, function (r) {
- if (skipWithData && r.numSorts) {
- return;
- }
-
- for (var i = 0; i < rowsPer; i++) {
- data.push([r.tubeNum, r.rowid, r.animalId, r.date, r.stim, r.treatment, buffer, null, null, null, null, null, null, null]);
- }
- }, this);
-
- LABKEY.Utils.convertToExcel({
- fileName: 'SortImport_' + Ext4.Date.format(new Date(), 'Y-m-d H_i_s') + '.xls',
- sheets: [{
- name: 'Sorts',
- data: data
- }]
- });
- },
- listeners: {
- uploadcomplete: function (panel, response) {
- Ext4.Msg.alert('Success', 'Upload Complete!', function (btn) {
- this.up('window').close();
- location.reload();
- }, this);
- }
- }
- }]
- });
-
- this.callParent();
- },
- buttons: [{
- text: 'Upload',
- width: 50,
- handler: function (btn) {
- var form = btn.up('window').down('#theForm');
- form.formSubmit.call(form, btn);
- },
- scope: this,
- formBind: true
- }, {
- text: 'Close',
- width: 50,
- handler: function (btn) {
- btn.up('window').close();
- }
- }]
- });
-
- Ext4.create('TCRdb.window.SortUploadWindow', {
- stimRows: this.up('tcrdb-stimpanel').stimRows
- }).show();
- }
- else {
- Ext4.Msg.alert('Error', 'This is only allowed when in a specific workbook. Please enter the workbook into the box at the top of the page and hit \'Go\'');
- }
- }
- }]
- },{
- title: 'Step 3: cDNA Synthesis / Library Prep',
- defaults: {
- border: false
- },
- items: [{
- layout: {
- type: 'table',
- columns: 3,
- tdAttrs: { style: 'padding-right: 10px;' }
- },
- defaults: {
- border: false
- },
- items: [{
- html: 'Total cDNA Libraries:'
- },{
- html: '' + this.libraryStats.totalLibraries + ' '
- },{
- xtype: 'ldk-linkbutton',
- text: '(' + this.libraryStats.totalPlates.length + ' plates)',
- href: 'javascript:void(0);',
- handler: this.getPlateCallback(this.libraryStats.totalPlates)
- },{
- html: 'Lacking Any Readset:'
- },{
- html: '' + this.libraryStats.lackingAnyReadset + ' '
- },{
- xtype: 'ldk-linkbutton',
- text: '(' + this.libraryStats.totalPlatesLackingAnyReadset.length + ' plates)',
- href: 'javascript:void(0);',
- handler: this.getPlateCallback(this.libraryStats.totalPlatesLackingAnyReadset)
- },{
- html: 'With Whole Transcriptome Readset:'
- },{
- html: '' + this.libraryStats.withReadset + ' '
- },{
- xtype: 'ldk-linkbutton',
- text: '(' + this.libraryStats.totalPlatesWithReadset.length + ' plates)',
- href: 'javascript:void(0);',
- handler: this.getPlateCallback(this.libraryStats.totalPlatesWithReadset)
- },{
- html: 'With TCR Enriched Readset:'
- },{
- html: '' + this.libraryStats.withTCRReadset + ' '
- },{
- xtype: 'ldk-linkbutton',
- text: '(' + this.libraryStats.totalPlatesWithTCRReadset.length + ' plates)',
- href: 'javascript:void(0);',
- handler: this.getPlateCallback(this.libraryStats.totalPlatesWithTCRReadset)
- }]
- },{
- xtype: 'ldk-linkbutton',
- text: 'Create cDNA Libraries From Sorts',
- href: 'javascript:void(0);',
- scope: this,
- linkCls: 'labkey-text-link',
- handler: function(){
- if (LABKEY.Security.currentContainer.type === 'workbook'){
- Ext4.define('TCRdb.window.cDNAUploadWindow', {
- extend: 'Ext.window.Window',
- initComponent: function(){
- Ext4.apply(this, {
- title: 'Create cDNA Libraries From Sorts',
- items: [{
- xtype: 'labkey-exceluploadpanel',
- bubbleEvents: ['uploadexception', 'uploadcomplete'],
- itemId: 'theForm',
- title: null,
- buttons: null,
- containerPath: Laboratory.Utils.getQueryContainerPath(),
- schemaName: 'tcrdb',
- queryName: 'cdnas',
- populateTemplates: function(meta){
- Ext4.Msg.hide();
- var toAdd = [];
-
- toAdd.push({
- html: 'Use the button below to download an excel template pre-populated with data from the selected plate IDs.',
- border: false,
- style: 'padding-bottom: 10px;',
- width: 700
- });
-
- toAdd.push({
- xtype: 'textfield',
- itemId: 'destPlate',
- fieldLabel: 'Destination Plate ID',
- labelWidth: 160
- });
-
- toAdd.push({
- xtype: 'ldk-simplecombo',
- itemId: 'chemistry',
- fieldLabel: 'Chemistry',
- labelWidth: 160,
- storeValues: ['SMART-Seq2', 'Takara SMART-Seq HT', '10x GEX/VDJ'],
- value: 'SMART-Seq2'
- });
-
- toAdd.push({
- xtype: 'textarea',
- itemId: 'plates',
- fieldLabel: 'Source Plates',
- labelWidth: 160,
- //width: 200,
- height: 100
- });
-
- var win = this.up('window');
- toAdd.push({
- xtype: 'ldk-linkbutton',
- itemId: 'showIds',
- style: 'margin-left: 165px;',
- text: 'Show Plate IDs',
- scope: this,
- handler: win.getPlateCallback(win.sortStats.totalPlatesLackingLibraries, 'plates'),
- linkCls: 'labkey-text-link'
- });
-
- toAdd.push({
- xtype: 'button',
- style: 'margin-bottom: 10px;',
- text: 'Download Template',
- border: true,
- handler: this.generateExcelTemplate
- });
-
- toAdd.push({
- xtype: 'checkbox',
- itemId: 'keepWell',
- checked: true,
- fieldLabel: 'Keep Original Well',
- labelWidth: 160
- });
-
- this.down('#templateArea').add(toAdd);
- },
-
- generateExcelTemplate: function(btn) {
- var win = btn.up('window');
- var chemistry = win.down('#chemistry').getValue();
- var destPlate = win.down('#destPlate').getValue();
- var keepWell = win.down('#keepWell').getValue();
-
- if (!destPlate) {
- Ext4.Msg.alert('Error', 'Must provide destination plate IDs');
- return;
- }
-
- var plates = Ext4.String.trim(btn.up('window').down('textarea').getValue());
- if (!plates) {
- Ext4.Msg.alert('Error', 'Must provide source plate IDs');
- return;
- }
-
- plates = plates.replace(/[\r\n]+/g, '\n');
- plates = plates.replace(/[\n]+/g, '\n');
- plates = Ext4.String.trim(plates);
- if (plates){
- plates = plates.split('\n');
- }
-
- Ext4.Msg.wait('Loading...');
- LABKEY.Query.selectRows({
- containerPath: Laboratory.Utils.getQueryContainerPath(),
- schemaName: 'tcrdb',
- queryName: 'sorts',
- sort: 'well/addressByColumn',
- columns: 'rowid,stimId,stimId/animalId,stimId/effector,stimId/stim,stimId/date,stimId/treatment,population,replicate,cells,plateId,well,well/addressByColumn,numLibraries',
- scope: win,
- filterArray: [LABKEY.Filter.create('plateId', plates.join(';'), LABKEY.Filter.Types.IN)],
- failure: LDK.Utils.getErrorCallback(),
- success: function (results) {
- Ext4.Msg.hide();
-
- if (!results || !results.rows || !results.rows.length) {
- Ext4.Msg.alert('Error', 'No sorts found for the selected plates');
- return;
- }
-
- var data = [];
- data.push(['Source Plate', 'Source Well', 'SortId', 'Plate Id', 'Well', 'Name', 'Chemistry', 'Comments']);
- var wellIdx = 0;
- var wellsUsed = {};
- var errors = [];
- Ext4.Array.forEach(plates, function (sourcePlateId) {
- Ext4.Array.forEach(results.rows, function (r) {
- if (r.plateId !== sourcePlateId){
- return;
- }
-
- var name = TCRdb.panel.StimPanel.getNameFromSort(r);
- var targetWell = keepWell ? r.well : this.wellNames96[wellIdx];
- if (wellsUsed[targetWell]){
- errors.push('Duplicate well: ' + targetWell);
- }
- wellsUsed[targetWell] = true;
- data.push([r.plateId, r.well, r.rowid, destPlate, targetWell, name, chemistry, null]);
- wellIdx++;
- }, this);
- }, this);
-
- for (var i=0;i'), function(){
- LABKEY.Utils.convertToExcel({
- fileName: 'cDNAImport_' + Ext4.Date.format(new Date(), 'Y-m-d H_i_s') + '.xls',
- sheets: [{
- name: 'cDNA Libraries',
- data: data
- }]
- });
- }, this);
- }
- else {
- LABKEY.Utils.convertToExcel({
- fileName: 'cDNAImport_' + Ext4.Date.format(new Date(), 'Y-m-d H_i_s') + '.xls',
- sheets: [{
- name: 'cDNA Libraries',
- data: data
- }]
- });
- }
- }
- });
- },
- listeners: {
- uploadcomplete: function(panel, response){
- Ext4.Msg.alert('Success', 'Upload Complete!', function(btn){
- this.up('window').close();
- location.reload();
- }, this);
- }
- }
- }]
- });
-
- this.callParent();
- },
- getWellSort: function(wellNames96){
- return function(a, b){
- var idx1 = wellNames96.indexOf(a[4]);
- var idx2 = wellNames96.indexOf(b[4]);
-
- return idx1 - idx2;
- }
- },
- buttons: [{
- text: 'Upload',
- width: 50,
- handler: function(btn){
- var form = btn.up('window').down('#theForm');
- form.formSubmit.call(form, btn);
- },
- scope: this,
- formBind: true
- },{
- text: 'Close',
- width: 50,
- handler: function(btn){
- btn.up('window').close();
- }
- }]
- });
-
- Ext4.create('TCRdb.window.cDNAUploadWindow', {
- wellNames96: this.wellNames96,
- sortStats: this.sortStats,
- getPlateCallback: this.getPlateCallback
- }).show();
- }
- else {
- Ext4.Msg.alert('Error', 'This is only allowed when in a specific workbook. Please enter the workbook into the box at the top of the page and hit \'Go\'');
- }
- }
- },{
- xtype: 'ldk-linkbutton',
- text: 'Download Library Prep Template (box)',
- linkCls: 'labkey-text-link',
- href: 'https://ohsu.box.com/s/6kncrzm4ba9mxjput12v8u500tjlsip7',
- linkTarget: '_blank'
- },{
- xtype: 'ldk-linkbutton',
- text: 'Download TCR Enrichment Library Prep Template (box)',
- linkCls: 'labkey-text-link',
- href: 'https://ohsu.box.com/s/js55a347q5mioqxwowe1dk3prkvn4d29',
- linkTarget: '_blank'
- },{
- xtype: 'ldk-linkbutton',
- text: 'Download Names To Use In Protocols',
- href: 'javascript:void(0);',
- linkCls: 'labkey-text-link',
- handler: function(){
- if (LABKEY.Security.currentContainer.type === 'workbook'){
- Ext4.Msg.wait('Loading...');
- LABKEY.Query.selectRows({
- containerPath: Laboratory.Utils.getQueryContainerPath(),
- schemaName: 'tcrdb',
- queryName: 'cdnas',
- sort: 'well/addressByColumn',
- scope: this,
- failure: LDK.Utils.getErrorCallback(),
- success: function(results){
- Ext4.Msg.hide();
- if (!results || !results.rows || !results.rows.length){
- Ext4.Msg.alert('Error', 'No cDNA libraries found');
- return;
- }
-
- var rows = [];
- rows.push(['Well', 'Name'].join('\t'));
- Ext4.Array.forEach(results.rows, function(r){
- var name = TCRdb.panel.StimPanel.getNameFromCDNAs(r);
- rows.push([r.well, name].join('\t'));
- }, this);
-
- Ext4.create('Ext.window.Window', {
- bodyStyle: 'padding: 5px;',
- items: [{
- html: 'Please use the following as names for the sorts in the folder',
- border: false,
- style: 'padding-bottom: 10px;'
- },{
- xtype: 'textarea',
- width: 500,
- height: 200,
- value: rows.join('\n')
- }],
- buttons: [{
- text: 'Close',
- handler: function(btn){
- btn.up('window').close();
- }
- }]
- }).show();
- }
- });
- }
- else {
- Ext4.Msg.alert('Error', 'This is only allowed when in a specific workbook. Please enter the workbook into the box at the top of the page and hit \'Go\'');
- }
- }
- }]
- },{
- title: 'Step 4: Create Readsets / Template for Sequencing',
- defaults: {
- border: false
- },
- items: [{
- layout: {
- type: 'table',
- columns: 2,
- tdAttrs: { style: 'padding-right: 10px;' }
- },
- defaults: {
- border: false
- },
- items: [{
- html: 'Total Readsets:'
- },{
- html: '' + this.readsetStats.totalReadsets + ' '
- },{
- html: 'Data Not Imported:'
- },{
- html: '' + this.readsetStats.lackingData + ' '
- }]
- },{
- xtype: 'ldk-linkbutton',
- text: 'Create Readsets From cDNA Libraries',
- href: 'javascript:void(0);',
- linkCls: 'labkey-text-link',
- handler: function(){
- if (LABKEY.Security.currentContainer.type === 'workbook'){
- Ext4.define('TCRdb.window.ReadsetUploadWindow', {
- extend: 'Ext.window.Window',
- initComponent: function(){
- Ext4.apply(this, {
- title: 'Create Readsets From cDNAs',
- bodyStyle: 'padding: 5px;',
- items: [{
- html: 'Use the button below to download an excel template pre-populated with data from the sorts imported into this workbook.',
- border: false,
- style: 'padding-bottom: 10px;',
- width: 700
- },{
- xtype: 'textarea',
- itemId: 'plateIds',
- fieldLabel: 'Plate Id(s)',
- labelWidth: 120,
- height: 100
- },{
- xtype: 'ldk-linkbutton',
- itemId: 'showIds',
- text: 'Show Plate IDs',
- style: 'margin-left: 125px;',
- scope: this,
- handler: this.getPlateCallback(this.libraryStats.totalPlatesLackingAnyReadset, 'plateIds'),
- linkCls: 'labkey-text-link'
- },{
- xtype: 'ldk-simplecombo',
- itemId: 'application',
- fieldLabel: 'Application',
- labelWidth: 120,
- storeValues: ['Whole Transcriptome RNA-Seq', 'TCR Enrichment', '10x GEX Only', '10x GEX/TCR'],
- forceSelection: true,
- multiSelect: true
- },{
- xtype: 'labkey-combo',
- itemId: 'chemistry',
- fieldLabel: 'Chemistry',
- labelWidth: 120,
- store: {
- type: 'labkey-store',
- schemaName: 'sequenceanalysis',
- queryName: 'sequence_chemistries',
- autoLoad: true
- },
- displayField: 'chemistry',
- valueField: 'chemistry',
- value: 'Illumina HiSeq3000',
- forceSelection: true
- },{
- xtype: 'checkbox',
- itemId: 'includeImported',
- fieldLabel: 'Include Those With Existing Readsets'
- },{
- xtype: 'button',
- style: 'margin-bottom: 10px;',
- text: 'Download Template',
- border: true,
- handler: this.generateExcelTemplate
- },{
- xtype: 'textarea',
- height: 350,
- width: 700,
- itemId: 'template'
- }]
- });
-
- this.callParent();
- },
- generateExcelTemplate: function(){
- var win = this.up('window');
-
- //'Whole Transcriptome RNA-Seq', 'TCR Enrichment', 10x GEX Only, 10x GEX/TCR
- var types = win.down('#application').getValue();
- var chemistry = win.down('#chemistry').getValue();
- var includeImported = win.down('#includeImported').getValue();
- var plates = Ext4.String.trim(win.down('textarea').getValue());
- if (!plates) {
- Ext4.Msg.alert('Error', 'Must provide source plate IDs');
- return;
- }
-
- plates = plates.replace(/[\r\n]+/g, '\n');
- plates = plates.replace(/[\n]+/g, '\n');
- plates = Ext4.String.trim(plates);
- if (plates){
- plates = plates.split('\n');
- }
-
- if (!types || !types.length){
- Ext4.Msg.alert('Error', 'Must choose the application(s)');
- return;
- }
-
- var applications = [];
- Ext4.Array.forEach(types, function(type) {
- switch (type) {
- case 'Whole Transcriptome RNA-Seq':
- applications.push('RNA-seq');
- break;
- case 'TCR Enrichment':
- applications.push('RNA-seq + Enrichment');
- break;
- case '10x GEX Only':
- applications.push('10x GEX');
- break;
- case '10x GEX/TCR':
- applications.push('10x GEX');
- applications.push('10x VDJ');
- }
- }, this);
- applications = Ext4.unique(applications);
-
- var data = [];
- data.push(['LibraryId', 'PlateId', 'Source Well', 'Name', 'Subject Id', 'Sample Date', '5-Barcode', '3-Barcode', 'Sample Type', 'Sequencing Platform', 'Application', 'Chemistry', 'Library Type', 'Comments']);
- Ext4.Array.forEach(win.libraryRows, function(r){
- Ext4.Array.forEach(applications, function(application){
- if (plates.indexOf(r.plateId) > -1) {
- if (includeImported || (['RNA-seq + Enrichment', '10x VDJ'].indexOf(application) > -1 && !r.enrichedReadsetId) || (['RNA-seq', '10x GEX'].indexOf(application) > -1 && !r.readsetId)) {
- var applicationValue = application;
- if (application === 'RNA-seq' && r.cells === 1) {
- applicationValue = 'RNA-seq, Single Cell';
- }
- else if (['10x GEX', '10x VDJ'].indexOf(application) > -1){
- applicationValue = 'RNA-seq, Single Cell';
- }
-
- var libraryType = null;
- switch (application){
- case 'RNA-seq':
- libraryType = 'SMART-Seq2';
- break;
- case '10x VDJ':
- libraryType = '10x 5\' VDJ (Rhesus A/B/G)';
- break;
- case '10x GEX':
- libraryType = '10x 5\' GEX';
- }
-
- var name = TCRdb.panel.StimPanel.getNameFromCDNAs(r);
- data.push([r.rowid, r.plateId, r.well, name, r['sortId/stimId/animalId'], r['sortId/stimId/date'], null, null, 'mRNA', 'ILLUMINA', applicationValue, chemistry, libraryType, null]);
- }
- }
- }, this);
- }, this);
-
- if (data.length === 1){
- Ext4.Msg.alert('Error', 'No matching rows found');
- return;
- }
-
- LABKEY.Utils.convertToExcel({
- fileName: 'ReadsetImport_' + Ext4.Date.format(new Date(), 'Y-m-d H_i_s') + '.xls',
- sheets: [{
- name: 'Readsets',
- data: data
- }]
- });
- },
- buttons: [{
- text: 'Upload',
- width: 50,
- handler: function(btn){
- btn.up('window').processUpload();
- },
- scope: this,
- formBind: true
- },{
- text: 'Close',
- width: 50,
- handler: function(btn){
- btn.up('window').close();
- }
- }],
- processUpload: function(){
- var text = this.down('#template').getValue();
- if (!text){
- Ext4.Msg.alert('Error', 'No rows provided');
- return;
- }
- text = LDK.Utils.CSVToArray(Ext4.String.trim(text), '\t');
-
- var header = text.shift();
- var headerToField = {
- Name: 'name',
- 'Subject Id': 'subjectid',
- 'Sample Date': 'sampledate',
- '5-Barcode': 'barcode5',
- '3-Barcode': 'barcode3',
- 'Sample Type': 'sampletype',
- 'Sequencing Platform': 'platform',
- 'Application': 'application',
- 'Chemistry': 'chemsitry',
- 'Library Type': 'libraryType',
- 'Comments': 'comments',
- 'LibraryId': 'libraryId'
-
- };
-
- var readsetToInsert = [];
- var cDNAsToUpdate = {};
- var errorMsgs = [];
-
- Ext4.Array.forEach(text, function (row, rowIdx) {
- var r = {};
- for (var headerName in headerToField){
- var idx = header.indexOf(headerName);
- if (idx !== -1 && row.length > idx){
- r[headerToField[headerName]] = row[idx];
- }
- }
-
- var hasBarcodes;
- switch (r.libraryType){
- case '10x 5\' GEX':
- case '10x 5\' VDJ (Rhesus A/B/G)':
- hasBarcodes = !!r.barcode5 && !r.barcode3;
- if (!hasBarcodes) {errorMsgs.push('10x data must have the 5\' barcode but not 3\'')};
- break;
- default:
- hasBarcodes = !!r.barcode5 && !!r.barcode3;
- }
-
- if (!r.name || !r.application || !hasBarcodes || !r.libraryId){
- errorMsgs.push('Every row must have name, application and proper barcodes');
- return;
- }
-
- //TODO: set container to match sort
- // if (row.libraryId && containerMap[row.libraryId]){
- // row.container = containerMap[row.libraryId];
- // }
- //
- // if (!row.container){
- // //TODO
- // }
-
- if (['10x 5\' GEX', '10x 5\' VDJ (Rhesus A/B/G)'].indexOf(r.libraryType) > -1 ) {
- r.barcode5 = r.barcode5.toUpperCase();
- if (!r.barcode5.match(/^SI-GA-/)) {
- if (r.barcode5.length > 3) {
- errorMsgs.push('Every row must have name, application and proper barcodes');
- }
- else {
- r.barcode5 = 'SI-GA-' + r.barcode5;
- }
- }
- }
-
- readsetToInsert.push(r);
-
- cDNAsToUpdate[r.libraryId] = cDNAsToUpdate[r.libraryId] || {};
- cDNAsToUpdate[r.libraryId].container = row.container;
- if ('rna-seq' === r.application.toLowerCase()){
- cDNAsToUpdate[r.libraryId].readsetIdx = rowIdx;
- }
- else if ('rna-seq, single cell' === r.application.toLowerCase() && r.libraryType === '10x 5\' GEX'){
- cDNAsToUpdate[r.libraryId].readsetIdx = rowIdx;
- }
- else if ('rna-seq + enrichment' === r.application.toLowerCase()){
- cDNAsToUpdate[r.libraryId].enrichedReadsetIdx = rowIdx;
- }
- else if ('rna-seq, single cell' === r.application.toLowerCase() && r.libraryType === '10x 5\' VDJ (Rhesus A/B/G)'){
- cDNAsToUpdate[r.libraryId].enrichedReadsetIdx = rowIdx;
- }
- else {
- errorMsgs.push('Unknown application/libraryType: ' + r.application + ' / ' + r.libraryType);
- }
- }, this);
-
- if (errorMsgs.length){
- errorMsgs = Ext4.unique(errorMsgs);
- Ext4.Msg.alert('Error', errorMsgs.join(' '));
- return;
- }
-
- Ext4.Msg.wait('Saving...');
- LABKEY.Query.insertRows({
- //containerPath: Laboratory.Utils.getQueryContainerPath(),
- schemaName: 'sequenceanalysis',
- queryName: 'sequence_readsets',
- rows: readsetToInsert,
- scope: this,
- failure: LDK.Utils.getErrorCallback(),
- success: function (results) {
- var toUpdate = [];
- for (var libraryId in cDNAsToUpdate){
- //TODO: add container
- var r = {rowid: libraryId};
- if (Ext4.isDefined(cDNAsToUpdate[libraryId].readsetIdx)){
- r.readsetId = results.rows[cDNAsToUpdate[libraryId].readsetIdx].rowId
- }
-
- if (Ext4.isDefined(cDNAsToUpdate[libraryId].enrichedReadsetIdx)){
- r.enrichedReadsetId = results.rows[cDNAsToUpdate[libraryId].enrichedReadsetIdx].rowId
- }
-
- if (Ext4.isDefined(cDNAsToUpdate[libraryId].container)){
- r.container = cDNAsToUpdate[libraryId].container;
- }
-
- if (r.readsetId || r.enrichedReadsetId){
- toUpdate.push(r);
- }
- }
-
- if (toUpdate.length){
- LABKEY.Query.updateRows({
- //containerPath: Laboratory.Utils.getQueryContainerPath(),
- schemaName: 'tcrdb',
- queryName: 'cdnas',
- rows: toUpdate,
- scope: this,
- failure: LDK.Utils.getErrorCallback(),
- success: function (results) {
- Ext4.Msg.hide();
-
- Ext4.Msg.alert('Success', 'Rows saved', function(){
- window.location.reload();
- });
- }
- });
- }
- else {
- Ext4.Msg.hide();
- Ext4.Msg.alert('Error', 'There were no readsets to update');
- }
- }
- });
- }
- });
-
- Ext4.create('TCRdb.window.ReadsetUploadWindow', {
- libraryRows: this.up('tcrdb-stimpanel').libraryRows,
- libraryStats: this.up('tcrdb-stimpanel').libraryStats,
- getPlateCallback: this.up('tcrdb-stimpanel').getPlateCallback
- }).show();
- }
- else {
- Ext4.Msg.alert('Error', 'This is only allowed when in a specific workbook. Please enter the workbook into the box at the top of the page and hit \'Go\'');
- }
- }
- },{
- xtype: 'ldk-linkbutton',
- text: 'Download Blank MPSSR Template (box)',
- linkCls: 'labkey-text-link',
- href: 'https://ohsu.box.com/s/awhkmncp3gphs60inlu0mnts1yd22z25'
- },{
- xtype: 'ldk-linkbutton',
- text: 'Request Runs From MPSSR (iLABS)',
- linkCls: 'labkey-text-link',
- href: 'https://ohsu.corefacilities.org/account/pending/ohsu'
- },{
- xtype: 'ldk-linkbutton',
- text: 'MedGenome Information',
- linkCls: 'labkey-text-link',
- href: 'https://prime-seq.ohsu.edu/wiki/Internal/Bimber/page.view?name=tcrSequenceShipping'
- },{
- xtype: 'ldk-linkbutton',
- text: 'Shipment List',
- linkCls: 'labkey-text-link',
- href: LABKEY.ActionURL.buildURL('query', 'executeQuery', Laboratory.Utils.getQueryContainerPath(), {schemaName: 'lists', queryName: 'MedGenomeShipments'})
- }]
- },{
- title: 'Plate Summary',
- defaults: {
- border: false
- },
- items: [{
- xtype: 'ldk-linkbutton',
- text: 'View Summary of Sorts By Plate (this workbook)',
- linkCls: 'labkey-text-link',
- hidden: LABKEY.Security.currentContainer.type != 'workbook',
- href: LABKEY.ActionURL.buildURL('query', 'executeQuery', null, {schemaName: 'tcrdb', queryName: 'sortStatusByPlate', 'query.isComplete~eq': false})
- },{
- xtype: 'ldk-linkbutton',
- text: 'View Summary of Sorts By Plate (entire folder)',
- linkCls: 'labkey-text-link',
- href: LABKEY.ActionURL.buildURL('query', 'executeQuery', Laboratory.Utils.getQueryContainerPath(), {schemaName: 'tcrdb', queryName: 'sortStatusByPlate', 'query.isComplete~eq': false})
- },{
- xtype: 'ldk-linkbutton',
- text: 'View Summary of Sorts By Animal/Plate (this workbook)',
- linkCls: 'labkey-text-link',
- hidden: LABKEY.Security.currentContainer.type !== 'workbook',
- href: LABKEY.ActionURL.buildURL('query', 'executeQuery', null, {schemaName: 'tcrdb', queryName: 'sortStatusByPlateAndSample'})
- },{
- xtype: 'ldk-linkbutton',
- text: 'View Summary of Sorts By Animal/Plate (entire folder)',
- linkCls: 'labkey-text-link',
- href: LABKEY.ActionURL.buildURL('query', 'executeQuery', Laboratory.Utils.getQueryContainerPath(), {schemaName: 'tcrdb', queryName: 'sortStatusByPlateAndSample'})
- }]
- }]
- }
- },
-
- getPlateCallback: function(plateIds, fieldId){
- return function(f){
- var target;
- if (fieldId){
- target = f.up('window').down('#' + fieldId);
- }
-
- var items = [];
- Ext4.Array.forEach(plateIds, function(id){
- var listener = target ? {
- scope: this,
- afterrender: function(panel){
- panel.mon(panel.getEl(), 'click', function(){
- target.setValue(target.getValue() + (target.getValue() ? '\n' : '') + id);
- }, this);
- }
- } : null;
- items.push({
- html: id,
- bodyStyle: target ? 'text-decoration: underline;cursor: pointer;' : null,
- border: false,
- listeners: listener
- });
- }, this);
-
- if (plateIds.length === 0){
- items.push({html: 'There are no plates in this folder', border: false});
- }
-
- Ext4.create('Ext.window.Window', {
- modal: true,
- title: 'Plate IDs',
- maxHeight: '400',
- autoScroll: true,
- width: 300,
- bodyStyle: 'padding: 5px;',
- items: [{
- xtype: 'container',
- items: items
- }],
- buttons: [{
- text: 'Close',
- handler: function(btn){
- btn.up('window').close();
- }
- }]
- }).show();
- }
- },
-
- statics: {
- getNameFromSort: function(r){
- return [
- r['plateId'],
- r['well'],
- r['stimId/animalId'],
- r['stimId/stim'],
- r['stimId/treatment'],
- r.population + (r.replicate ? '_' + r.replicate : '')
- ].join('_').replace(/ /g, '-');
- },
-
- getNameFromCDNAs: function(r){
- return [
- //NOTE: preferentially retain the original sort plate name, in case of combined cDNA plates
- r['sortId/plateId'] || r['plateId'],
- r['well'],
- r['sortId/stimId/animalId'],
- r['sortId/stimId/stim'],
- r['sortId/stimId/treatment'],
- r['sortId/population'] + (r['sortId/cells'] === 1 ? '_Clone' : '') + (r['sortId/replicate'] ? '_' + r['sortId/replicate'] : '')
- ].join('_').replace(/ /g, '-').replace(/\(/g, '').replace(/\)/g, '').replace(/\+/g, 'Pos');
- }
- }
-});
\ No newline at end of file
diff --git a/tcrdb/resources/web/tcrdb/panel/cDNAImportPanel.js b/tcrdb/resources/web/tcrdb/panel/cDNAImportPanel.js
deleted file mode 100644
index 28c8ef451..000000000
--- a/tcrdb/resources/web/tcrdb/panel/cDNAImportPanel.js
+++ /dev/null
@@ -1,378 +0,0 @@
-Ext4.define('TCRdb.panel.cDNAImportPanel', {
- extend: 'TCRdb.panel.PoolImportPanel',
-
- IGNORED_COLUMNS: ['animalId', 'sampleDate', 'stimId', 'population', 'sortId', 'hto', 'cells', 'stim', 'effector', 'tissue', 'stim_num'],
-
- initComponent: function () {
- this.COLUMN_MAP = {};
- Ext4.Array.forEach(this.COLUMNS, function (col) {
- if (this.IGNORED_COLUMNS.indexOf(col.name) > -1) {
- return;
- }
-
- //Do not allow rowspan for this type of import
- col.allowRowSpan = false;
-
- this.COLUMN_MAP[col.name.toLowerCase()] = col;
- Ext4.Array.forEach(col.labels, function (alias) {
- this.COLUMN_MAP[alias.toLowerCase()] = col;
- }, this);
- }, this);
-
- Ext4.apply(this, {
- title: null,
- border: false,
- defaults: {
- border: false
- },
- items: this.getPanelItems()
- });
-
- this.callParent(arguments);
- },
-
- getPanelItems: function(){
- return [{
- layout: {
- type: 'hbox'
- },
- items: [{
- xtype: 'ldk-integerfield',
- style: 'margin-right: 5px;',
- fieldLabel: 'Current Folder/Workbook',
- labelWidth: 200,
- minValue: 1,
- value: LABKEY.Security.currentContainer.type === 'workbook' ? LABKEY.Security.currentContainer.name : null,
- emptyText: LABKEY.Security.currentContainer.type === 'workbook' ? null : 'Showing All',
- listeners: {
- afterRender: function (field) {
- new Ext4.util.KeyNav(field.getEl(), {
- enter: function (e) {
- var btn = field.up('panel').down('#goButton');
- btn.handler(btn);
- },
- scope: this
- });
- }
- }
- }, {
- xtype: 'button',
- itemId: 'goButton',
- scope: this,
- text: 'Go',
- handler: function (btn) {
- var wb = btn.up('panel').down('ldk-integerfield').getValue();
- if (!wb) {
- wb = '';
- }
-
- var container = LABKEY.Security.currentContainer.type === 'workbook' ? LABKEY.Security.currentContainer.parentPath + '/' + wb : LABKEY.Security.currentContainer.path + '/' + wb;
- window.location = LABKEY.ActionURL.buildURL('tcrdb', 'poolImport', container);
- }
- }, {
- xtype: 'button',
- scope: this,
- hidden: !LABKEY.Security.currentUser.canInsert,
- text: 'Create Workbook',
- handler: function (btn) {
- Ext4.create('Laboratory.window.WorkbookCreationWindow', {
- abortIfContainerIsWorkbook: false,
- canAddToExistingExperiment: false,
- controller: 'tcrdb',
- action: 'poolImport',
- title: 'Create Workbook'
- }).show();
- }
- }]
- }, {
- style: 'padding-top: 10px;',
- html: 'This page is designed to help import the readset/index information for 10x libraries, after the stim/sort data has already been imported.'
- }, {
- layout: 'hbox',
- items: [{
- xtype: 'button',
- text: 'Download Template',
- border: true,
- scope: this,
- href: LABKEY.ActionURL.getContextPath() + '/tcrdb/exampleData/ImportReadsetTemplate.xlsx'
- }]
- }, {
- xtype: 'textfield',
- style: 'margin-top: 20px;',
- fieldLabel: 'Expt Number',
- itemId: 'exptNum',
- value: LABKEY.Security.currentContainer.type === 'workbook' ? LABKEY.Security.currentContainer.name : null
- }, {
- xtype: 'checkbox',
- fieldLabel: 'Require GEX Library',
- itemId: 'requireGEX',
- checked: true
- }, {
- xtype: 'checkbox',
- fieldLabel: 'Require TCR Library',
- itemId: 'requireTCR',
- checked: true
- }, {
- xtype: 'checkbox',
- fieldLabel: 'Require HTO Library',
- itemId: 'requireHTO',
- checked: true
- },{
- xtype: 'checkbox',
- fieldLabel: 'Require Cite-Seq Library',
- itemId: 'requireCITE',
- checked: false
- }, {
- xtype: 'checkbox',
- fieldLabel: 'Require Library Concentrations',
- itemId: 'requireConc',
- checked: true
- },{
- xtype: 'ldk-simplecombo',
- fieldLabel: '10x GEX/TCR Barcode Series',
- itemId: 'barcodeSeries',
- forceSelection: true,
- storeValues: ['SI-GA'],
- value: 'SI-GA'
- },{
- xtype: 'ldk-simplecombo',
- fieldLabel: '10x Cite-Seq Barcode Series',
- itemId: 'citeseqBarcodeSeries',
- forceSelection: true,
- storeValues: ['SI-NA'],
- value: 'SI-NA'
- },{
- xtype: 'ldk-simplecombo',
- fieldLabel: 'Hashing Type',
- itemId: 'hashingType',
- forceSelection: true,
- storeValues: ['CD298', 'MultiSeq'],
- value: 'MultiSeq'
- }, {
- xtype: 'textarea',
- fieldLabel: 'Paste Data Below',
- labelAlign: 'top',
- itemId: 'data',
- width: 1000,
- height: 300
- }, {
- xtype: 'button',
- text: 'Preview',
- border: true,
- scope: this,
- handler: this.onPreview
- }, {
- style: 'margin-top: 20px;margin-bottom: 10px;',
- itemId: 'previewArea',
- autoEl: 'table',
- cls: 'stripe hover'
- }];
- },
-
- onPreview: function (btn) {
- var text = this.down('#data').getValue();
- if (!text) {
- Ext4.Msg.alert('Error', 'Must provide the table of data');
- return;
- }
-
- this.EXPERIMENT = this.down('#exptNum').getValue();
-
- text = Ext4.String.trim(text);
-
- var rows = LDK.Utils.CSVToArray(text, '\t');
- var colArray = this.parseHeader(rows.shift());
- var parsedRows = this.parseRows(colArray, rows);
-
- var groupedRows = this.groupForImport(colArray, parsedRows);
- if (!groupedRows) {
- console.log('No rows after grouping');
- return;
- }
-
- var workbooks = [];
- var hadError = false;
- Ext4.Array.forEach(parsedRows, function(row){
- if (!row.workbook) {
- hadError = true;
- }
- else {
- workbooks.push(row.workbook);
- }
- }, this);
-
- if (hadError) {
- Ext4.Msg.alert('Error', 'One or more rows missing a workbook ID');
- return;
- }
-
- Ext4.Msg.wait('Loading workbooks');
- LABKEY.Query.selectRows({
- containerPath: Laboratory.Utils.getQueryContainerPath(),
- schemaName: 'core',
- queryName: 'workbooks',
- columns: 'Name,EntityId',
- filterArray: [LABKEY.Filter.create('Name', Ext4.unique(workbooks).join(';'), LABKEY.Filter.Types.IN)],
- scope: this,
- success: function(results) {
- Ext4.Msg.hide();
-
- var workbookMap = {};
- Ext4.Array.forEach(results.rows, function(r){
- workbookMap[r.Name] = r.EntityId;
- }, this);
-
- Ext4.Array.forEach(groupedRows.cDNARows, function(r){
- LDK.Assert.assertNotEmpty('Unable to find workbook in map: ' + r.workbook, workbookMap[r.workbook]);
- r.container = workbookMap[r.workbook];
- }, this);
-
- Ext4.Array.forEach(groupedRows.readsetRows, function(r){
- LDK.Assert.assertNotEmpty('Unable to find workbook in map: ' + r.workbook, workbookMap[r.workbook]);
- r.container = workbookMap[r.workbook];
- }, this);
-
- this.onWorkbookQueryLoad(colArray, parsedRows, groupedRows);
- },
- failure: LDK.Utils.getErrorCallback()
- });
- },
-
- onWorkbookQueryLoad: function(colArray, parsedRows, groupedRows) {
- var plateIDs = [];
- var hadError = false;
- Ext4.Array.forEach(parsedRows, function(row){
- if (!row.plateId) {
- hadError = true;
- }
- else {
- plateIDs.push(row.plateId);
- }
- }, this);
-
- if (hadError) {
- Ext4.Msg.alert('Error', 'One or more rows missing plate ID');
- return;
- }
-
- plateIDs = Ext4.unique(plateIDs);
-
- Ext4.Msg.wait('Looking for matching cDNA');
- LABKEY.Query.selectRows({
- containerPath: Laboratory.Utils.getQueryContainerPath(),
- schemaName: 'tcrdb',
- queryName: 'cdnas',
- columns: 'rowid,plateid,readsetid,enrichedreadsetid,hashingreadsetid,citeseqreadsetid,sortid/population,sortid,sortid/stimid,citeseqpanel',
- filterArray: [LABKEY.Filter.create('plateId', plateIDs.join(';'), LABKEY.Filter.Types.IN)],
- scope: this,
- success: function(results) {
- Ext4.Msg.hide();
-
- if (!results.rows || !results.rows.length) {
- Ext4.Msg.alert('Error', 'No matching rows found');
- return;
- }
-
- var plateToCDNAMap = {};
- Ext4.Array.forEach(results.rows, function(row) {
- plateToCDNAMap[row.plateId] = plateToCDNAMap[row.plateId] || [];
- plateToCDNAMap[row.plateId].push(row.rowid);
- }, this);
-
- var missing = [];
- Ext4.Array.forEach(plateIDs, function (r) {
- if (!plateToCDNAMap[r]) {
- missing.push(r);
- }
- }, this);
-
- if (missing.length) {
- Ext4.Msg.alert('Error', 'No cDNA records found for plates: ' + missing.join(', '));
- return;
- }
-
- Ext4.Array.forEach(groupedRows.cDNARows, function(r){
- r.rowIds = plateToCDNAMap[r.plateId];
- }, this);
-
- this.renderPreview(colArray, parsedRows, groupedRows);
- },
- failure: LDK.Utils.getErrorCallback()
- });
- },
-
- onSubmit: function (e, dt, node, config) {
- Ext4.Msg.wait('Saving...');
-
- var data = config.rowData.groupedRows;
-
- LABKEY.Query.insertRows({
- containerPath: Laboratory.Utils.getQueryContainerPath(),
- schemaName: 'sequenceanalysis',
- queryName: 'sequence_readsets',
- rows: data.readsetRows,
- success: function(results){
- var readsetMap = {};
- Ext4.Array.forEach(results.rows, function(row){
- readsetMap[row.name] = row.rowId;
- }, this);
-
- var toUpdate = [];
- Ext4.Array.forEach(data.cDNARows, function(row){
- var baseRow = {};
-
- var gexReadsetId = readsetMap[row.plateId + '-GEX'];
- if (gexReadsetId) {
- baseRow.readsetId = gexReadsetId;
- }
-
- var tcrReadsetId = readsetMap[row.plateId + '-TCR'];
- if (tcrReadsetId) {
- baseRow.enrichedReadsetId = tcrReadsetId;
- }
-
- var htoReadsetId = readsetMap[row.plateId + '-HTO'];
- if (htoReadsetId) {
- baseRow.hashingReadsetId = htoReadsetId;
- }
-
- var citeseqReadsetId = readsetMap[row.plateId + '-CITE'];
- if (citeseqReadsetId) {
- baseRow.citeseqReadsetId = citeseqReadsetId;
- }
-
- baseRow.container = row.container;
-
- if (row.rowIds) {
- Ext4.Array.forEach(row.rowIds, function(r){
- var toAdd = Ext4.apply({
- rowId: r
- }, baseRow);
-
- toUpdate.push(toAdd);
- }, this);
- }
- }, this);
-
- if (toUpdate.length) {
- LABKEY.Query.updateRows({
- containerPath: Laboratory.Utils.getQueryContainerPath(),
- schemaName: 'tcrdb',
- queryName: 'cdnas',
- rows: toUpdate,
- success: function (results) {
- Ext4.Msg.hide();
- Ext4.Msg.alert('Success', 'Data Saved', function(){
- window.location = LABKEY.ActionURL.buildURL('query', 'executeQuery.view', Laboratory.Utils.getQueryContainerPath(), {'query.queryName': 'cdnas', schemaName: 'tcrdb', 'query.sort': '-created'});
- }, this);
- },
- failure: LDK.Utils.getErrorCallback(),
- scope: this
- });
- }
- },
- failure: LDK.Utils.getErrorCallback(),
- scope: this
- });
- }
-});
\ No newline at end of file
diff --git a/tcrdb/src/org/labkey/tcrdb/ImportHelper.java b/tcrdb/src/org/labkey/tcrdb/ImportHelper.java
deleted file mode 100644
index 4ac075926..000000000
--- a/tcrdb/src/org/labkey/tcrdb/ImportHelper.java
+++ /dev/null
@@ -1,110 +0,0 @@
-package org.labkey.tcrdb;
-
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-import org.labkey.api.data.Container;
-import org.labkey.api.data.ContainerManager;
-import org.labkey.api.data.TableInfo;
-import org.labkey.api.data.TableSelector;
-import org.labkey.api.query.FieldKey;
-import org.labkey.api.query.QueryService;
-import org.labkey.api.query.UserSchema;
-import org.labkey.api.security.User;
-import org.labkey.api.security.UserManager;
-import org.labkey.api.util.MemTracker;
-import org.labkey.api.util.PageFlowUtil;
-
-import java.util.HashMap;
-import java.util.Map;
-
-public class ImportHelper
-{
- private Container _container;
- private User _user;
- private TableInfo _table;
- private TableInfo _sortTable;
-
- private static final Logger _log = LogManager.getLogger(ImportHelper.class);
-
- private Map _userSchemaMap = new HashMap<>();
-
- private ImportHelper(String containerId, int userId, String queryName)
- {
- String schemaName = "tcrdb";
-
- _container = ContainerManager.getForId(containerId);
- if (_container == null)
- throw new IllegalArgumentException("Unknown container: " + containerId);
-
- _container = _container.isWorkbook() ? _container.getParent() : _container;
-
- _user = UserManager.getUser(userId);
- if (_user == null)
- throw new IllegalArgumentException("Unknown user: " + userId);
-
- UserSchema us = getUserSchema(schemaName);
- if (us == null)
- throw new IllegalArgumentException("Unknown schema: " + schemaName);
-
- _table = us.getTable(queryName);
- if (_table == null)
- throw new IllegalArgumentException("Unknown table: " + schemaName + "." + queryName);
-
- _sortTable = us.getTable(TCRdbSchema.TABLE_SORTS);
- if (_sortTable == null)
- throw new IllegalArgumentException("Unknown table: " + schemaName + "." + queryName);
-
- MemTracker.getInstance().put(this);
- }
-
- public static ImportHelper create(String containerId, int userId, String queryName)
- {
- return new ImportHelper(containerId, userId, queryName);
- }
-
- private UserSchema getUserSchema(String name)
- {
- if (_userSchemaMap.containsKey(name))
- return _userSchemaMap.get(name);
-
- UserSchema us = QueryService.get().getUserSchema(_user, _container, name);
- _userSchemaMap.put(name, us);
-
- return us;
- }
-
- public Map getInitialWells()
- {
- TableSelector ts = new TableSelector(_table, PageFlowUtil.set("plateId", "well", "rowid"));
- final Map ret = new HashMap<>();
- ts.forEachResults(rs -> {
- String key = (rs.getString(FieldKey.fromString("plateId")) + "<>" + rs.getString(FieldKey.fromString("well"))).toUpperCase();
- ret.put(key, rs.getInt(FieldKey.fromString("rowId")));
- });
-
- return ret;
- }
-
- private Map sortToContainer = null;
-
- public String getContainerForSort(int sortId)
- {
- if (sortToContainer == null)
- {
- sortToContainer = new HashMap<>();
- new TableSelector(_sortTable, PageFlowUtil.set("rowId", "container")).forEachResults(rs -> {
- sortToContainer.put(rs.getInt(FieldKey.fromString("rowId")), rs.getString(FieldKey.fromString("container")));
- });
- }
-
- if (sortToContainer != null && sortToContainer.containsKey(sortId))
- {
- return sortToContainer.get(sortId);
- }
-
- String containerId = new TableSelector(_sortTable, PageFlowUtil.set("container")).getObject(sortId, String.class);
- sortToContainer.put(sortId, containerId);
-
- return containerId;
- }
-}
diff --git a/tcrdb/src/org/labkey/tcrdb/TCRdbBulkImportNavItem.java b/tcrdb/src/org/labkey/tcrdb/TCRdbBulkImportNavItem.java
deleted file mode 100644
index 4203f5097..000000000
--- a/tcrdb/src/org/labkey/tcrdb/TCRdbBulkImportNavItem.java
+++ /dev/null
@@ -1,53 +0,0 @@
-package org.labkey.tcrdb;
-
-import org.labkey.api.data.Container;
-import org.labkey.api.laboratory.AbstractImportingNavItem;
-import org.labkey.api.laboratory.DataProvider;
-import org.labkey.api.laboratory.LaboratoryService;
-import org.labkey.api.module.ModuleLoader;
-import org.labkey.api.query.DetailsURL;
-import org.labkey.api.security.User;
-import org.labkey.api.view.ActionURL;
-
-public class TCRdbBulkImportNavItem extends AbstractImportingNavItem
-{
- public static final String NAME = "TCR/10x Import";
-
- private String _url;
-
- public TCRdbBulkImportNavItem(DataProvider provider, String label, LaboratoryService.NavItemCategory itemType, String reportCategory, String url)
- {
- super(provider, NAME, label, itemType, (reportCategory == null ? "TCRdb" : reportCategory));
- _url = url;
- }
-
- @Override
- public ActionURL getImportUrl(Container c, User u)
- {
- return DetailsURL.fromString(_url).getActionURL();
- }
-
- @Override
- public ActionURL getSearchUrl(Container c, User u)
- {
- return null;
- }
-
- @Override
- public ActionURL getBrowseUrl(Container c, User u)
- {
- return null;
- }
-
- @Override
- public boolean isImportIntoWorkbooks(Container c, User u)
- {
- return true;
- }
-
- @Override
- public boolean getDefaultVisibility(Container c, User u)
- {
- return getTargetContainer(c).getActiveModules().contains(ModuleLoader.getInstance().getModule(TCRdbModule.NAME));
- }
-}
diff --git a/tcrdb/src/org/labkey/tcrdb/TCRdbController.java b/tcrdb/src/org/labkey/tcrdb/TCRdbController.java
index 59b0acc6b..7647b1d63 100644
--- a/tcrdb/src/org/labkey/tcrdb/TCRdbController.java
+++ b/tcrdb/src/org/labkey/tcrdb/TCRdbController.java
@@ -22,32 +22,19 @@
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.jetbrains.annotations.NotNull;
-import org.json.JSONArray;
-import org.labkey.api.action.ApiSimpleResponse;
-import org.labkey.api.action.ApiUsageException;
import org.labkey.api.action.ConfirmAction;
import org.labkey.api.action.ExportAction;
-import org.labkey.api.action.MutatingApiAction;
-import org.labkey.api.action.ReadOnlyApiAction;
-import org.labkey.api.action.SimpleApiJsonForm;
import org.labkey.api.action.SimpleViewAction;
import org.labkey.api.action.SpringActionController;
-import org.labkey.api.collections.CaseInsensitiveHashMap;
import org.labkey.api.data.ColumnInfo;
import org.labkey.api.data.CompareType;
import org.labkey.api.data.Container;
-import org.labkey.api.data.ContainerManager;
-import org.labkey.api.data.ContainerType;
-import org.labkey.api.data.DbScope;
-import org.labkey.api.data.Selector;
import org.labkey.api.data.SimpleFilter;
-import org.labkey.api.data.StopIteratingException;
import org.labkey.api.data.TableInfo;
import org.labkey.api.data.TableSelector;
import org.labkey.api.exp.api.ExpData;
import org.labkey.api.exp.api.ExperimentService;
import org.labkey.api.pipeline.PipelineJobException;
-import org.labkey.api.query.BatchValidationException;
import org.labkey.api.query.FieldKey;
import org.labkey.api.query.QueryAction;
import org.labkey.api.query.QueryService;
@@ -56,7 +43,6 @@
import org.labkey.api.security.IgnoresTermsOfUse;
import org.labkey.api.security.RequiresPermission;
import org.labkey.api.security.permissions.AdminPermission;
-import org.labkey.api.security.permissions.InsertPermission;
import org.labkey.api.security.permissions.ReadPermission;
import org.labkey.api.sequenceanalysis.RefNtSequenceModel;
import org.labkey.api.sequenceanalysis.SequenceAnalysisService;
@@ -78,7 +64,6 @@
import java.io.File;
import java.io.FileInputStream;
import java.io.StringWriter;
-import java.sql.SQLException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
@@ -136,44 +121,38 @@ public ModelAndView getView(ExportAlignmentsForm form, BindException errors) thr
TableInfo ti = us.getTable("data");
final Map> VDJMap = new HashMap<>();
- List rowIds = new ArrayList<>();
- rowIds.addAll(Arrays.asList(form.getAssayRowIds()));
+ List rowIds = new ArrayList<>(Arrays.asList(form.getAssayRowIds()));
TableSelector ts = new TableSelector(ti, new SimpleFilter(FieldKey.fromString("rowid"), rowIds, CompareType.IN), null);
final StringWriter writer = new StringWriter();
- ts.forEach(new Selector.ForEachBlock()
- {
- @Override
- public void exec(AssayRecord r) throws SQLException, StopIteratingException
+ ts.forEach(AssayRecord.class, r -> {
+ if (r.getVdjFile() == null)
{
- if (r.getVdjFile() == null)
- {
- writer.write("ERROR: Row lacks VDJCA file: " + r.getRowId() + "\n");
- return;
- }
-
- ExpData d = ExperimentService.get().getExpData(r.getVdjFile());
- if (d == null)
- {
- writer.write("ERROR: Unable to find VDJCA file for row: " + r.getRowId() + ", ExpData: " + r.getVdjFile() + "\n");
- return;
- }
+ writer.write("ERROR: Row lacks VDJCA file: " + r.getRowId() + "\n");
+ return;
+ }
- if (!d.getFile().exists())
- {
- writer.write("ERROR: Unable to find VDJCA file for row: " + r.getRowId() + ", file does not exist: " + d.getFile().getPath() + "\n");
- return;
- }
+ ExpData d = ExperimentService.get().getExpData(r.getVdjFile());
+ if (d == null)
+ {
+ writer.write("ERROR: Unable to find VDJCA file for row: " + r.getRowId() + ", ExpData: " + r.getVdjFile() + "\n");
+ return;
+ }
- if (!VDJMap.containsKey(d.getFile()))
- {
- VDJMap.put(d.getFile(), new ArrayList<>());
- }
+ if (!d.getFile().exists())
+ {
+ writer.write("ERROR: Unable to find VDJCA file for row: " + r.getRowId() + ", file does not exist: " + d.getFile().getPath() + "\n");
+ return;
+ }
- VDJMap.get(d.getFile()).add(r);
+ if (!VDJMap.containsKey(d.getFile()))
+ {
+ VDJMap.put(d.getFile(), new ArrayList<>());
}
- }, AssayRecord.class);
+
+ VDJMap.get(d.getFile()).add(r);
+ });
if (VDJMap.isEmpty())
{
@@ -973,239 +952,6 @@ public void setQueryName(String queryName)
}
}
- @RequiresPermission(InsertPermission.class)
- public static class ImportTenXAction extends MutatingApiAction
- {
- @Override
- public Object execute(SimpleApiJsonForm form, BindException errors) throws Exception
- {
- List> stimRows = parseRows(form, "stimRows", getContainer());
- List> sortRows = parseRows(form, "sortRows", getContainer());
- List> readsetRows = parseRows(form, "readsetRows", getContainer());
- List> cDNARows = parseRows(form, "cDNARows", getContainer());
-
- UserSchema tcrdb = QueryService.get().getUserSchema(getUser(), getContainer(), TCRdbSchema.NAME);
- UserSchema sequenceAnalysis = QueryService.get().getUserSchema(getUser(), getContainer(), "sequenceanalysis");
-
- try (DbScope.Transaction transaction = DbScope.getLabKeyScope().ensureTransaction())
- {
- BatchValidationException bve = new BatchValidationException();
-
- Map stimMap = new HashMap<>();
- final List> stimRowsToInsert = new ArrayList<>();
- stimRows.forEach(r -> {
- if (r.get("objectId") == null)
- {
- throw new ApiUsageException("Missing objectId for stim row");
- }
-
- if (r.get("rowId") != null && StringUtils.trimToNull(r.get("rowId").toString()) != null)
- {
- stimMap.put((String) r.get("objectId"), (Integer) r.get("rowId"));
- }
- else
- {
- stimRowsToInsert.add(r);
- }
- });
-
-
- List> insertedStimRows = tcrdb.getTable(TCRdbSchema.TABLE_STIMS, null).getUpdateService().insertRows(getUser(), getContainer(), stimRowsToInsert, bve, null, new HashMap<>());
- if (bve.hasErrors())
- {
- throw bve;
- }
-
- insertedStimRows.forEach(r -> {
- if (r.get("rowId") == null)
- {
- throw new ApiUsageException("Missing rowId for inserted stim row");
- }
-
- stimMap.put((String) r.get("objectId"), (Integer) r.get("rowId"));
- });
-
- Map sortMap = new HashMap<>();
- final List> sortRowsToInsert = new ArrayList<>();
- sortRows.forEach(r -> {
- if (stimMap.get(r.get("stimGUID")) == null)
- {
- throw new ApiUsageException("Unable to find stimId for row");
- }
-
- if (r.get("rowId") != null && StringUtils.trimToNull(r.get("rowId").toString()) != null)
- {
- sortMap.put((String) r.get("objectId"), (Integer) r.get("rowId"));
- }
- else
- {
- r.put("stimId", stimMap.get(r.get("stimGUID")));
- sortRowsToInsert.add(r);
- }
- });
-
- sortRows = tcrdb.getTable(TCRdbSchema.TABLE_SORTS, null).getUpdateService().insertRows(getUser(), getContainer(), sortRowsToInsert, bve, null, new HashMap<>());
- if (bve.hasErrors())
- {
- throw bve;
- }
-
- sortRows.forEach(r -> {
- if (r.get("objectId") == null)
- {
- throw new ApiUsageException("Missing objectId for sort row");
- }
-
- sortMap.put((String) r.get("objectId"), (Integer) r.get("rowId"));
- });
-
- readsetRows = sequenceAnalysis.getTable("sequence_readsets", null).getUpdateService().insertRows(getUser(), getContainer(), readsetRows, bve, null, new HashMap<>());
- if (bve.hasErrors())
- {
- throw bve;
- }
-
- Map readsetMap = new HashMap<>();
- readsetRows.forEach(r -> {
- if (r.get("objectId") == null)
- {
- throw new ApiUsageException("Missing objectId for readset row");
- }
-
- readsetMap.put((String)r.get("objectId"), (Integer)r.get("rowId"));
- });
-
- cDNARows.forEach(r -> {
- if (sortMap.get(r.get("sortGUID")) == null)
- {
- throw new ApiUsageException("Unable to find sortId for row");
- }
- r.put("sortId", sortMap.get((String)r.get("sortGUID")));
- });
- cDNARows.forEach(r -> r.put("readsetId", readsetMap.get((String)r.get("readsetGUID"))));
- cDNARows.forEach(r -> r.put("hashingReadsetId", readsetMap.get((String)r.get("hashingReadsetGUID"))));
- cDNARows.forEach(r -> r.put("enrichedReadsetId", readsetMap.get((String)r.get("enrichedReadsetGUID"))));
- cDNARows.forEach(r -> r.put("citeseqReadsetId", readsetMap.get((String)r.get("citeseqReadsetGUID"))));
- tcrdb.getTable(TCRdbSchema.TABLE_CDNAS, null).getUpdateService().insertRows(getUser(), getContainer(), cDNARows, bve, null, new HashMap<>());
- if (bve.hasErrors())
- {
- throw bve;
- }
-
- transaction.commit();
- }
-
- return new ApiSimpleResponse("success", true);
- }
- }
-
- private static List> parseRows(SimpleApiJsonForm form, String propName, Container container) throws ApiUsageException
- {
- if (!form.getJsonObject().containsKey(propName))
- {
- throw new ApiUsageException("Missing property: " + propName);
- }
-
- JSONArray arr = form.getJsonObject().getJSONArray(propName);
-
- List> ret = new ArrayList<>();
- Arrays.stream(arr.toJSONObjectArray()).forEach(m -> {
- Map map = new CaseInsensitiveHashMap<>();
- map.putAll(m);
-
- if (map.containsKey("workbook"))
- {
- Container parent = container.getContainerFor(ContainerType.DataType.folderManagement);
- Container workbook = ContainerManager.getForPath(parent.getPath() + "/" + map.get("workbook").toString());
- if (workbook == null)
- {
- throw new IllegalArgumentException("Unable to identify matching workbook for: " + map.get("workbook"));
- }
-
- map.put("container", workbook == null ? null : workbook.getId());
- }
- ret.add(map);
- });
-
- return ret;
- }
-
- @RequiresPermission(ReadPermission.class)
- public static class GetMatchingStimsAction extends ReadOnlyApiAction
- {
- final List FIELDS = Arrays.asList("animalId", "date", "stim", "treatment", "tissue");
-
- @Override
- public Object execute(SimpleApiJsonForm form, BindException errors) throws Exception
- {
- ApiSimpleResponse resp = new ApiSimpleResponse();
-
- List> stimRows = parseRows(form, "stimRows", getContainer());
-
- UserSchema us = QueryService.get().getUserSchema(getUser(), getContainer(), TCRdbSchema.NAME);
- if (us == null)
- {
- throw new ApiUsageException("Unable to find schema: " + TCRdbSchema.NAME);
- }
-
- TableInfo ti = us.getTable(TCRdbSchema.TABLE_STIMS, null);
- TableInfo tiSort = us.getTable(TCRdbSchema.TABLE_SORTS, null);
-
- List retErrors = new ArrayList<>();
- Map stimRowMap = new HashMap<>();
- Map sortRowMap = new HashMap<>();
- stimRows.forEach(r -> {
- List keys = new ArrayList<>();
- SimpleFilter filter = new SimpleFilter();
- FIELDS.forEach(f -> {
- if (r.get(f) != null)
- {
- filter.addCondition(FieldKey.fromString(f), r.get(f), ("date".equals(f) ? CompareType.DATE_EQUAL : CompareType.EQUAL));
- keys.add(r.get(f));
- }
- });
-
- if (!filter.isEmpty())
- {
- TableSelector ts = new TableSelector(ti, PageFlowUtil.set("rowId"), filter, null);
- long count = ts.getRowCount();
- if (count == 1)
- {
- int rowId = ts.getObject(Integer.class);
- stimRowMap.put(r.get("objectId"), rowId);
-
- if (r.get("population") != null)
- {
- SimpleFilter sortFilter = new SimpleFilter(FieldKey.fromString("stimId"), rowId);
- sortFilter.addCondition(FieldKey.fromString("population"), r.get("population"));
- TableSelector tsSort = new TableSelector(tiSort, PageFlowUtil.set("rowId"), sortFilter, null);
- long countSort = tsSort.getRowCount();
- if (countSort == 1)
- {
- int sortRowId = tsSort.getObject(Integer.class);
- sortRowMap.put(r.get("objectId"), sortRowId);
- }
- else if (countSort > 1)
- {
- retErrors.add("More than one matching sort found: " + StringUtils.join(keys, "|") + "|" + r.get("population"));
- }
- }
- }
- else if (count > 1 && filter.getClauses().size() == FIELDS.size())
- {
- retErrors.add("More than one matching stim found: " + StringUtils.join(keys, "|"));
- }
- }
- });
-
- resp.put("stimMap", stimRowMap);
- resp.put("sortMap", sortRowMap);
- resp.put("recordErrors", retErrors);
-
- return resp;
- }
- }
-
@RequiresPermission(AdminPermission.class)
public class CreateGenomeFromMixcrAction extends ConfirmAction
{
@@ -1247,7 +993,7 @@ public void validateCommand(CreateGenomeFromMixcrForm form, Errors errors)
@Override
public URLHelper getSuccessURL(CreateGenomeFromMixcrForm form)
{
- return QueryService.get().urlFor(getUser(), getContainer(), QueryAction.executeQuery, TCRdbSchema.NAME, TCRdbSchema.TABLE_LIBRARIES);
+ return QueryService.get().urlFor(getUser(), getContainer(), QueryAction.executeQuery, TCRdbSchema.NAME, TCRdbSchema.TABLE_MIXCR_LIBRARIES);
}
}
diff --git a/tcrdb/src/org/labkey/tcrdb/TCRdbImportNavItem.java b/tcrdb/src/org/labkey/tcrdb/TCRdbImportNavItem.java
deleted file mode 100644
index dc74a4fbe..000000000
--- a/tcrdb/src/org/labkey/tcrdb/TCRdbImportNavItem.java
+++ /dev/null
@@ -1,50 +0,0 @@
-package org.labkey.tcrdb;
-
-import org.labkey.api.data.Container;
-import org.labkey.api.laboratory.AbstractImportingNavItem;
-import org.labkey.api.laboratory.DataProvider;
-import org.labkey.api.laboratory.LaboratoryService;
-import org.labkey.api.module.ModuleLoader;
-import org.labkey.api.query.DetailsURL;
-import org.labkey.api.security.User;
-import org.labkey.api.view.ActionURL;
-
-public class TCRdbImportNavItem extends AbstractImportingNavItem
-{
- public static final String NAME = "TCR Sorts/Stims";
-
- public TCRdbImportNavItem(DataProvider provider, String label, LaboratoryService.NavItemCategory itemType, String reportCategory)
- {
- super(provider, NAME, label, itemType, (reportCategory == null ? "TCRdb" : reportCategory));
- }
-
- @Override
- public ActionURL getImportUrl(Container c, User u)
- {
- return DetailsURL.fromString("tcrdb/stimDashboard.view").getActionURL();
- }
-
- @Override
- public ActionURL getSearchUrl(Container c, User u)
- {
- return null;
- }
-
- @Override
- public ActionURL getBrowseUrl(Container c, User u)
- {
- return DetailsURL.fromString("tcrdb/stimDashboard.view").getActionURL();
- }
-
- @Override
- public boolean isImportIntoWorkbooks(Container c, User u)
- {
- return true;
- }
-
- @Override
- public boolean getDefaultVisibility(Container c, User u)
- {
- return getTargetContainer(c).getActiveModules().contains(ModuleLoader.getInstance().getModule(TCRdbModule.NAME));
- }
-}
diff --git a/tcrdb/src/org/labkey/tcrdb/TCRdbManager.java b/tcrdb/src/org/labkey/tcrdb/TCRdbManager.java
index 79b3e06ac..efc318e7e 100644
--- a/tcrdb/src/org/labkey/tcrdb/TCRdbManager.java
+++ b/tcrdb/src/org/labkey/tcrdb/TCRdbManager.java
@@ -69,7 +69,7 @@ public static TCRdbManager get()
public void createGenomeFromMixcrDb(int mixcrRowId, User u, Container c) throws Exception
{
- MixcrLibrary lib = new TableSelector(TCRdbSchema.getInstance().getSchema().getTable(TCRdbSchema.TABLE_LIBRARIES)).getObject(mixcrRowId, MixcrLibrary.class);
+ MixcrLibrary lib = new TableSelector(TCRdbSchema.getInstance().getSchema().getTable(TCRdbSchema.TABLE_MIXCR_LIBRARIES)).getObject(mixcrRowId, MixcrLibrary.class);
if (lib == null)
{
throw new IllegalArgumentException("Unable to find MiXCR library: " + mixcrRowId);
@@ -308,7 +308,7 @@ public void onCreate(Container c, User u, Logger log, int genomeId)
{
if (_mixcrId != null)
{
- TableInfo ti = QueryService.get().getUserSchema(u, c, TCRdbSchema.NAME).getTable(TCRdbSchema.TABLE_LIBRARIES);
+ TableInfo ti = QueryService.get().getUserSchema(u, c, TCRdbSchema.NAME).getTable(TCRdbSchema.TABLE_MIXCR_LIBRARIES);
List> rows = new ArrayList<>();
List> oldKeys = new ArrayList<>();
Map row = new CaseInsensitiveHashMap<>();
diff --git a/tcrdb/src/org/labkey/tcrdb/TCRdbModule.java b/tcrdb/src/org/labkey/tcrdb/TCRdbModule.java
index 0b1796b56..6d5f552f6 100644
--- a/tcrdb/src/org/labkey/tcrdb/TCRdbModule.java
+++ b/tcrdb/src/org/labkey/tcrdb/TCRdbModule.java
@@ -27,13 +27,8 @@
import org.labkey.api.module.ModuleContext;
import org.labkey.api.sequenceanalysis.SequenceAnalysisService;
import org.labkey.api.sequenceanalysis.pipeline.SequencePipelineService;
-import org.labkey.tcrdb.pipeline.CellRangerCellHashingHandler;
-import org.labkey.tcrdb.pipeline.CellRangerSeuratHandler;
import org.labkey.tcrdb.pipeline.CellRangerVDJCellHashingHandler;
-import org.labkey.tcrdb.pipeline.CellRangerVDJWrapper;
import org.labkey.tcrdb.pipeline.MiXCRAnalysis;
-import org.labkey.tcrdb.pipeline.SeuratCellHashingHandler;
-import org.labkey.tcrdb.pipeline.SeuratCiteSeqHandler;
import java.util.Collection;
import java.util.Collections;
@@ -51,7 +46,7 @@ public String getName()
@Override
public Double getSchemaVersion()
{
- return 15.51;
+ return 15.52;
}
@Override
@@ -74,17 +69,13 @@ protected void doStartupAfterSpringConfig(ModuleContext moduleContext)
LaboratoryService.get().registerDataProvider(new TCRdbProvider(this));
SequenceAnalysisService.get().registerDataProvider(new TCRdbProvider(this));
- LaboratoryService.get().registerTableCustomizer(this, TCRdbTableCustomizer.class, "sequenceanalysis", "sequence_readsets");
- LaboratoryService.get().registerTableCustomizer(this, TCRdbTableCustomizer.class, "sequenceanalysis", "sequence_analyses");
- LaboratoryService.get().registerTableCustomizer(this, TCRdbTableCustomizer.class, TCRdbSchema.NAME, TCRdbSchema.TABLE_STIMS);
- LaboratoryService.get().registerTableCustomizer(this, TCRdbTableCustomizer.class, TCRdbSchema.NAME, TCRdbSchema.TABLE_SORTS);
- LaboratoryService.get().registerTableCustomizer(this, TCRdbTableCustomizer.class, TCRdbSchema.NAME, TCRdbSchema.TABLE_CDNAS);
+ LaboratoryService.get().registerTableCustomizer(this, TCRdbTableCustomizer.class, TCRdbSchema.SEQUENCE_ANALYSIS, "sequence_readsets");
+ LaboratoryService.get().registerTableCustomizer(this, TCRdbTableCustomizer.class, TCRdbSchema.SEQUENCE_ANALYSIS, "sequence_analyses");
LaboratoryService.get().registerTableCustomizer(this, TCRdbTableCustomizer.class, TCRdbSchema.NAME, TCRdbSchema.TABLE_CLONES);
+ LaboratoryService.get().registerTableCustomizer(this, TCRdbTableCustomizer.class, TCRdbSchema.SINGLE_CELL, TCRdbSchema.TABLE_CDNAS);
- LDKService.get().registerQueryButton(new ChangeStatusButton(), "tcrdb", "stims");
+ LDKService.get().registerQueryButton(new ChangeStatusButton(), TCRdbSchema.SINGLE_CELL, "samples");
- LDKService.get().registerQueryButton(new ShowBulkEditButton(this, TCRdbSchema.NAME, TCRdbSchema.TABLE_CDNAS), TCRdbSchema.NAME, TCRdbSchema.TABLE_CDNAS);
- LDKService.get().registerQueryButton(new ShowBulkEditButton(this, TCRdbSchema.NAME, TCRdbSchema.TABLE_SORTS), TCRdbSchema.NAME, TCRdbSchema.TABLE_SORTS);
LDKService.get().registerQueryButton(new ShowBulkEditButton(this, TCRdbSchema.NAME, TCRdbSchema.TABLE_CLONES), TCRdbSchema.NAME, TCRdbSchema.TABLE_CLONES);
//register resources
@@ -118,13 +109,7 @@ public PipelineStartup()
else
{
SequencePipelineService.get().registerPipelineStep(new MiXCRAnalysis.Provider());
- SequencePipelineService.get().registerPipelineStep(new CellRangerVDJWrapper.VDJProvider());
-
- SequenceAnalysisService.get().registerFileHandler(new CellRangerCellHashingHandler());
SequenceAnalysisService.get().registerFileHandler(new CellRangerVDJCellHashingHandler());
- SequenceAnalysisService.get().registerFileHandler(new SeuratCellHashingHandler());
- SequenceAnalysisService.get().registerFileHandler(new SeuratCiteSeqHandler());
- SequenceAnalysisService.get().registerFileHandler(new CellRangerSeuratHandler());
_hasRegistered = true;
}
diff --git a/tcrdb/src/org/labkey/tcrdb/TCRdbProvider.java b/tcrdb/src/org/labkey/tcrdb/TCRdbProvider.java
index 1e3f0d31c..0359e45a7 100644
--- a/tcrdb/src/org/labkey/tcrdb/TCRdbProvider.java
+++ b/tcrdb/src/org/labkey/tcrdb/TCRdbProvider.java
@@ -3,20 +3,13 @@
import org.json.JSONObject;
import org.labkey.api.data.Container;
import org.labkey.api.data.ContainerManager;
-import org.labkey.api.laboratory.DetailsUrlWithoutLabelNavItem;
import org.labkey.api.laboratory.LaboratoryService;
import org.labkey.api.laboratory.NavItem;
import org.labkey.api.laboratory.QueryCountNavItem;
import org.labkey.api.laboratory.QueryImportNavItem;
-import org.labkey.api.laboratory.QueryTabbedReportItem;
-import org.labkey.api.laboratory.SimpleSettingsItem;
import org.labkey.api.laboratory.SummaryNavItem;
-import org.labkey.api.laboratory.TabbedReportItem;
import org.labkey.api.ldk.table.QueryCache;
import org.labkey.api.module.Module;
-import org.labkey.api.module.ModuleLoader;
-import org.labkey.api.query.DetailsURL;
-import org.labkey.api.query.FieldKey;
import org.labkey.api.security.User;
import org.labkey.api.sequenceanalysis.AbstractSequenceDataProvider;
import org.labkey.api.view.ActionURL;
@@ -60,15 +53,9 @@ public ActionURL getInstructionsUrl(Container c, User u)
}
@Override
- public List getMiscItems(Container c, User u)
+ public List getSubjectIdSummary(Container c, User u, String subjectId)
{
- List items = new ArrayList<>();
- if (c.getActiveModules().contains(ModuleLoader.getInstance().getModule(TCRdbModule.class)))
- {
- items.add(new DetailsUrlWithoutLabelNavItem(this, "Export 10x Library Information", DetailsURL.fromString("tcrdb/libraryExport.view"), LaboratoryService.NavItemCategory.misc, NAME));
- }
-
- return items;
+ return Collections.emptyList();
}
@Override
@@ -81,27 +68,7 @@ public List getDataNavItems(Container c, User u)
return Collections.emptyList();
}
- TCRdbImportNavItem item = new TCRdbImportNavItem(this, "TCR Stims/Sorts (SMART-seq)", LaboratoryService.NavItemCategory.data, NAME);
- item.setQueryCache(cache);
- items.add(item);
-
- TCRdbBulkImportNavItem item2 = new TCRdbBulkImportNavItem(this, "TCR/10x Import 1: Stims/cDNA", LaboratoryService.NavItemCategory.data, NAME, "tcrdb/poolImport.view");
- item2.setQueryCache(cache);
- items.add(item2);
-
- TCRdbBulkImportNavItem item3 = new TCRdbBulkImportNavItem(this, "TCR/10x Import 2: Libraries/Readsets", LaboratoryService.NavItemCategory.data, NAME, "tcrdb/cDNAImport.view");
- item3.setQueryCache(cache);
- items.add(item3);
-
items.add(new QueryImportNavItem(this, TCRdbSchema.NAME, TCRdbSchema.TABLE_CLONES, "TCR Clones", LaboratoryService.NavItemCategory.data, NAME, cache));
- items.add(new QueryImportNavItem(this, TCRdbSchema.NAME, TCRdbSchema.TABLE_CDNAS, "TCR cDNA Libraries", LaboratoryService.NavItemCategory.data, NAME, cache){
- @Override
- public ActionURL getImportUrl(Container c, User u)
- {
- return null;
- }
- });
-
return Collections.unmodifiableList(items);
}
@@ -117,11 +84,7 @@ public List getSettingsItems(Container c, User u)
List items = new ArrayList<>();
if (ContainerManager.getSharedContainer().equals(c))
{
- items.add(new QueryImportNavItem(this, ContainerManager.getSharedContainer(), TCRdbSchema.NAME, TCRdbSchema.TABLE_LIBRARIES, LaboratoryService.NavItemCategory.settings, "MiXCR Libraries", NAME));
- }
- else
- {
- items.add(new SimpleSettingsItem(this, TCRdbSchema.NAME, "peptides", NAME, "Peptides/Stims"));
+ items.add(new QueryImportNavItem(this, ContainerManager.getSharedContainer(), TCRdbSchema.NAME, TCRdbSchema.TABLE_MIXCR_LIBRARIES, LaboratoryService.NavItemCategory.settings, "MiXCR Libraries", NAME));
}
return items;
@@ -148,56 +111,6 @@ public Module getOwningModule()
@Override
public List getSummary(Container c, User u)
{
- List items = new ArrayList<>();
-
- items.add(new QueryCountNavItem(this, TCRdbSchema.NAME, "stims", LaboratoryService.NavItemCategory.data, LaboratoryService.NavItemCategory.data.name(), "TCR Stims"));
- items.add(new QueryCountNavItem(this, TCRdbSchema.NAME, "sorts", LaboratoryService.NavItemCategory.data, LaboratoryService.NavItemCategory.data.name(), "TCR Sorts"));
- items.add(new QueryCountNavItem(this, TCRdbSchema.NAME, "cdnas", LaboratoryService.NavItemCategory.data, LaboratoryService.NavItemCategory.data.name(), "TCR cDNA Libraries"));
- items.add(new QueryCountNavItem(this, TCRdbSchema.NAME, "clones", LaboratoryService.NavItemCategory.data, LaboratoryService.NavItemCategory.data.name(), "TCR Clones"));
-
- return Collections.unmodifiableList(items);
- }
-
- @Override
- public List getSubjectIdSummary(Container c, User u, String subjectId)
- {
- return Collections.emptyList();
- }
-
- @Override
- public List getTabbedReportItems(Container c, User u)
- {
- if (!c.getActiveModules().contains(getOwningModule()))
- {
- return Collections.emptyList();
- }
-
- List items = new ArrayList<>();
-
- NavItem owner = getDataNavItems(c, u).get(0);
- String category = NAME;
- QueryCache cache = new QueryCache();
-
- TabbedReportItem stims = new QueryTabbedReportItem(cache, this, TCRdbSchema.NAME, TCRdbSchema.TABLE_STIMS, "TCR Stims/Blood Draws", category);
- stims.setOwnerKey(owner.getPropertyManagerKey());
- items.add(stims);
-
- TabbedReportItem sorts = new QueryTabbedReportItem(cache, this, TCRdbSchema.NAME, TCRdbSchema.TABLE_SORTS, "TCR Sorts", category);
- sorts.setSubjectIdFieldKey(FieldKey.fromString("stimId/animalId"));
- sorts.setSampleDateFieldKey(FieldKey.fromString("stimId/date"));
- sorts.setAllProjectsFieldKey(FieldKey.fromString("stimId/allProjectsPivot"));
- sorts.setOverlappingProjectsFieldKey(FieldKey.fromString("stimId/overlappingProjectsPivot"));
- sorts.setOwnerKey(owner.getPropertyManagerKey());
- items.add(sorts);
-
- TabbedReportItem cdnas = new QueryTabbedReportItem(cache, this, TCRdbSchema.NAME, TCRdbSchema.TABLE_CDNAS, "TCR cDNA Libraries", category);
- cdnas.setSubjectIdFieldKey(FieldKey.fromString("sortId/stimId/animalId"));
- cdnas.setSampleDateFieldKey(FieldKey.fromString("sortId/stimId/date"));
- cdnas.setAllProjectsFieldKey(FieldKey.fromString("sortId/stimId/allProjectsPivot"));
- cdnas.setOverlappingProjectsFieldKey(FieldKey.fromString("sortId/stimId/overlappingProjectsPivot"));
- cdnas.setOwnerKey(owner.getPropertyManagerKey());
- items.add(cdnas);
-
- return items;
+ return Collections.singletonList(new QueryCountNavItem(this, TCRdbSchema.NAME, TCRdbSchema.TABLE_CLONES, LaboratoryService.NavItemCategory.data, LaboratoryService.NavItemCategory.data.name(), "TCR Clones"));
}
}
diff --git a/tcrdb/src/org/labkey/tcrdb/TCRdbSchema.java b/tcrdb/src/org/labkey/tcrdb/TCRdbSchema.java
index 0336e338c..32a0ca98f 100644
--- a/tcrdb/src/org/labkey/tcrdb/TCRdbSchema.java
+++ b/tcrdb/src/org/labkey/tcrdb/TCRdbSchema.java
@@ -24,16 +24,13 @@ public class TCRdbSchema
{
private static final TCRdbSchema _instance = new TCRdbSchema();
public static final String NAME = "tcrdb";
- public static final String SEQUENCE_ANALYSIS = "sequenceanalysis";
- public static final String TABLE_LIBRARIES = "mixcr_libraries";
- public static final String TABLE_SORTS = "sorts";
- public static final String TABLE_STIMS = "stims";
- public static final String TABLE_CDNAS = "cdnas";
+ public static final String TABLE_MIXCR_LIBRARIES = "mixcr_libraries";
public static final String TABLE_CLONES = "clones";
- public static final String TABLE_CITE_SEQ_ANTIBODIES = "citeseq_antibodies";
- public static final String TABLE_CITE_SEQ_PANELS = "citeseq_panels";
- public static final String TABLE_PROCESSING = "plate_processing";
+
+ public static final String SEQUENCE_ANALYSIS = "sequenceanalysis";
+ public static final String SINGLE_CELL = "singlecell";
+ public static final String TABLE_CDNAS = "cdna_libraries";
public static TCRdbSchema getInstance()
{
diff --git a/tcrdb/src/org/labkey/tcrdb/TCRdbTableCustomizer.java b/tcrdb/src/org/labkey/tcrdb/TCRdbTableCustomizer.java
index 9c7059a83..cf1a8e14e 100644
--- a/tcrdb/src/org/labkey/tcrdb/TCRdbTableCustomizer.java
+++ b/tcrdb/src/org/labkey/tcrdb/TCRdbTableCustomizer.java
@@ -19,9 +19,7 @@
import org.labkey.api.query.DetailsURL;
import org.labkey.api.query.ExprColumn;
import org.labkey.api.query.FieldKey;
-import org.labkey.api.query.QueryForeignKey;
import org.labkey.api.query.QueryService;
-import org.labkey.api.query.UserSchema;
import java.util.Arrays;
import java.util.List;
@@ -34,27 +32,19 @@ public void customize(TableInfo table)
if (table instanceof AbstractTableInfo)
{
AbstractTableInfo ti = (AbstractTableInfo) table;
- if (matches(ti, "sequenceanalysis", "sequence_analyses"))
+ if (matches(ti, TCRdbSchema.SEQUENCE_ANALYSIS, "sequence_analyses"))
{
addAssayFieldsToAnalyses(ti);
}
- else if (matches(ti, "sequenceanalysis", "sequence_readsets"))
+ else if (matches(ti, TCRdbSchema.SEQUENCE_ANALYSIS, "sequence_readsets"))
{
customizeReadsets(ti);
}
- else if (matches(ti, "tcrdb", "stims"))
- {
- customizeStims(ti);
- }
- else if (matches(ti, "tcrdb", "sorts"))
- {
- customizeSorts(ti);
- }
- else if (matches(ti, "tcrdb", "cdnas"))
+ else if (matches(ti, TCRdbSchema.SINGLE_CELL, TCRdbSchema.TABLE_CDNAS))
{
customizeCdnas(ti);
}
- else if (matches(ti, "tcrdb", "clones"))
+ else if (matches(ti, TCRdbSchema.NAME, TCRdbSchema.TABLE_CLONES))
{
customizeClones(ti);
}
@@ -67,130 +57,12 @@ else if (ti instanceof AssayResultTable)
private void customizeCdnas(AbstractTableInfo ti)
{
- String name = "hasReadsetWithData";
- if (ti.getColumn(name) == null)
- {
- SQLFragment sql = new SQLFragment("CASE " +
- " WHEN (select count(*) as expr FROM sequenceanalysis.sequence_readsets r JOIN sequenceanalysis.readdata d ON (r.rowid = d.readset) WHERE r.rowid = " + ExprColumn.STR_TABLE_ALIAS + ".readsetId) > 0 THEN " + ti.getSqlDialect().getBooleanTRUE() +
- " WHEN (select count(*) as expr FROM sequenceanalysis.sequence_readsets r JOIN sequenceanalysis.readdata d ON (r.rowid = d.readset) WHERE r.rowid = " + ExprColumn.STR_TABLE_ALIAS + ".enrichedReadsetId) > 0 THEN " + ti.getSqlDialect().getBooleanTRUE() +
- " ELSE " + ti.getSqlDialect().getBooleanFALSE() + " END");
-
- ExprColumn newCol = new ExprColumn(ti, name, sql, JdbcType.BOOLEAN, ti.getColumn("readsetId"), ti.getColumn("enrichedReadsetId"));
- newCol.setLabel("Has Any Readset With Data?");
- ti.addColumn(newCol);
- }
-
- String name2 = "allReadsetsHaveData";
- if (ti.getColumn(name2) == null)
- {
- SQLFragment sql = new SQLFragment("CASE " +
- " WHEN (" + ExprColumn.STR_TABLE_ALIAS + ".readsetId IS NOT NULL AND (select count(*) as expr FROM sequenceanalysis.sequence_readsets r JOIN sequenceanalysis.readdata d ON (r.rowid = d.readset) WHERE r.rowid = " + ExprColumn.STR_TABLE_ALIAS + ".readsetId) = 0) THEN " + ti.getSqlDialect().getBooleanFALSE() +
- " WHEN (" + ExprColumn.STR_TABLE_ALIAS + ".enrichedReadsetId IS NOT NULL AND (select count(*) as expr FROM sequenceanalysis.sequence_readsets r JOIN sequenceanalysis.readdata d ON (r.rowid = d.readset) WHERE r.rowid = " + ExprColumn.STR_TABLE_ALIAS + ".enrichedReadsetId) = 0) THEN " + ti.getSqlDialect().getBooleanFALSE() +
- " ELSE " + ti.getSqlDialect().getBooleanTRUE() + " END");
-
- ExprColumn newCol = new ExprColumn(ti, name2, sql, JdbcType.BOOLEAN, ti.getColumn("readsetId"), ti.getColumn("enrichedReadsetId"));
- newCol.setLabel("All Readsets Have Data?");
- ti.addColumn(newCol);
- }
-
addAssayFieldsToCDnas(ti);
-
- LDKService.get().applyNaturalSort(ti, "plateId");
- }
-
- private void customizeSorts(AbstractTableInfo ti)
- {
- LDKService.get().applyNaturalSort(ti, "plateId");
- LDKService.get().applyNaturalSort(ti, "hto");
-
- String name = "numLibraries";
- if (ti.getColumn(name) == null)
- {
- DetailsURL details = DetailsURL.fromString("/query/executeQuery.view?schemaName=tcrdb&query.queryName=cdnas&query.sortId~eq=${rowid}", (ti.getUserSchema().getContainer().isWorkbook() ? ti.getUserSchema().getContainer().getParent() : ti.getUserSchema().getContainer()));
-
- SQLFragment sql = new SQLFragment("(select count(*) as expr FROM " + TCRdbSchema.NAME + "." + TCRdbSchema.TABLE_CDNAS + " s WHERE s.sortId = " + ExprColumn.STR_TABLE_ALIAS + ".rowid)");
- ExprColumn newCol = new ExprColumn(ti, name, sql, JdbcType.INTEGER, ti.getColumn("rowid"));
- newCol.setLabel("# cDNA Libraries");
- newCol.setURL(details);
- ti.addColumn(newCol);
- }
-
- name = "maxCellsForPlate";
- if (ti.getColumn(name) == null)
- {
- SQLFragment sql = new SQLFragment("(select count(*) as expr FROM " + TCRdbSchema.NAME + "." + TCRdbSchema.TABLE_SORTS + " s WHERE s.plateId = " + ExprColumn.STR_TABLE_ALIAS + ".plateId AND s.container = " + ExprColumn.STR_TABLE_ALIAS + ".container)");
- ExprColumn newCol = new ExprColumn(ti, name, sql, JdbcType.INTEGER, ti.getColumn("plateId"), ti.getColumn("container"));
- newCol.setLabel("Max Cells/Well In Plate");
- ti.addColumn(newCol);
- }
-
- name = "processingRequested";
- if (ti.getColumn(name) == null)
- {
- SQLFragment sql = new SQLFragment("(select ").append(ti.getSqlDialect().getGroupConcat(new SQLFragment("p.type"), true, true)).append(new SQLFragment(" as expr FROM " + TCRdbSchema.NAME + "." + TCRdbSchema.TABLE_PROCESSING + " p WHERE p.plateId = " + ExprColumn.STR_TABLE_ALIAS + ".plateId AND p.container = " + ExprColumn.STR_TABLE_ALIAS + ".container)"));
- ExprColumn newCol = new ExprColumn(ti, name, sql, JdbcType.VARCHAR, ti.getColumn("plateId"), ti.getColumn("container"));
- newCol.setLabel("Processing Requested");
- ti.addColumn(newCol);
- }
- }
-
- private void customizeStims(AbstractTableInfo ti)
- {
- String name = "numSorts";
- if (ti.getColumn(name) == null)
- {
- DetailsURL details = DetailsURL.fromString("/query/executeQuery.view?schemaName=tcrdb&query.queryName=sorts&query.stimId~eq=${rowid}", (ti.getUserSchema().getContainer().isWorkbook() ? ti.getUserSchema().getContainer().getParent() : ti.getUserSchema().getContainer()));
-
- SQLFragment sql = new SQLFragment("(select count(*) as expr FROM " + TCRdbSchema.NAME + "." + TCRdbSchema.TABLE_SORTS + " s WHERE s.stimId = " + ExprColumn.STR_TABLE_ALIAS + ".rowid)");
- ExprColumn newCol = new ExprColumn(ti, "numSorts", sql, JdbcType.INTEGER, ti.getColumn("rowid"));
- newCol.setLabel("# Sorts");
- newCol.setURL(details);
- ti.addColumn(newCol);
- }
-
- name = "numLibraries";
- if (ti.getColumn(name) == null)
- {
- DetailsURL details = DetailsURL.fromString("/query/executeQuery.view?schemaName=tcrdb&query.queryName=cdnas&query.sortId/stimId~eq=${rowid}", (ti.getUserSchema().getContainer().isWorkbook() ? ti.getUserSchema().getContainer().getParent() : ti.getUserSchema().getContainer()));
-
- SQLFragment sql = new SQLFragment("(select count(c.rowid) as expr FROM " + TCRdbSchema.NAME + "." + TCRdbSchema.TABLE_SORTS + " so JOIN " + TCRdbSchema.NAME + "." + TCRdbSchema.TABLE_CDNAS + " c ON (so.rowid = c.sortId) WHERE so.stimId = " + ExprColumn.STR_TABLE_ALIAS + ".rowid)");
- ExprColumn newCol = new ExprColumn(ti, "numLibraries", sql, JdbcType.INTEGER, ti.getColumn("rowid"));
- newCol.setLabel("# cDNA Libraries");
- newCol.setURL(details);
- ti.addColumn(newCol);
- }
}
private void customizeReadsets(AbstractTableInfo ti)
{
addAssayFieldsToTable(ti, "analysisId/readset", "LEFT JOIN sequenceanalysis.sequence_analyses a2 ON (a.analysisId = a2.rowId) WHERE a2.readset = " + ExprColumn.STR_TABLE_ALIAS + ".rowid", "rowid");
-
- String name = "numTCRLibraries";
- if (ti.getColumn(name) == null)
- {
- SQLFragment sql = new SQLFragment("(select count(*) as expr FROM " + TCRdbSchema.NAME + "." + TCRdbSchema.TABLE_CDNAS + " c WHERE c.readsetId = " + ExprColumn.STR_TABLE_ALIAS + ".rowid OR c.enrichedReadsetId = " + ExprColumn.STR_TABLE_ALIAS + ".rowid)");
- ExprColumn newCol = new ExprColumn(ti, name, sql, JdbcType.INTEGER, ti.getColumn("rowid"));
- newCol.setLabel("# TCR Libraries");
- ti.addColumn(newCol);
- }
-
- String cDNA = "cDNA";
- if (ti.getColumn(cDNA) == null)
- {
- SQLFragment sql = new SQLFragment("(CASE" +
- " WHEN ((select count(*) as expr FROM " + TCRdbSchema.NAME + "." + TCRdbSchema.TABLE_CDNAS + " c WHERE c.readsetId = " + ExprColumn.STR_TABLE_ALIAS + ".rowid OR c.enrichedReadsetId = " + ExprColumn.STR_TABLE_ALIAS + ".rowid) > 0) " +
- " THEN (select max(c.rowid) FROM " + TCRdbSchema.NAME + "." + TCRdbSchema.TABLE_CDNAS + " c WHERE c.readsetId = " + ExprColumn.STR_TABLE_ALIAS + ".rowid OR c.enrichedReadsetId = " + ExprColumn.STR_TABLE_ALIAS + ".rowid) " +
- " ELSE null " +
- "END)");
- ExprColumn newCol = new ExprColumn(ti, cDNA, sql, JdbcType.INTEGER, ti.getColumn("rowid"));
- newCol.setLabel("cDNA Library");
- UserSchema us = QueryService.get().getUserSchema(ti.getUserSchema().getUser(), (ti.getUserSchema().getContainer().isWorkbook() ? ti.getUserSchema().getContainer().getParent() : ti.getUserSchema().getContainer()), TCRdbSchema.NAME);
- newCol.setFk(QueryForeignKey.from(us, ti.getContainerFilter())
- .table(TCRdbSchema.TABLE_CDNAS)
- .key("rowid")
- .display("rowid"));
- ti.addColumn(newCol);
- }
}
private void addAssayFieldsToAnalyses(AbstractTableInfo ti)
diff --git a/tcrdb/src/org/labkey/tcrdb/TCRdbUserSchema.java b/tcrdb/src/org/labkey/tcrdb/TCRdbUserSchema.java
index 402e11a85..188fcc112 100644
--- a/tcrdb/src/org/labkey/tcrdb/TCRdbUserSchema.java
+++ b/tcrdb/src/org/labkey/tcrdb/TCRdbUserSchema.java
@@ -6,7 +6,6 @@
import org.labkey.api.data.ContainerFilter;
import org.labkey.api.data.DbSchema;
import org.labkey.api.data.TableInfo;
-import org.labkey.api.ldk.table.ContainerScopedTable;
import org.labkey.api.ldk.table.SharedDataTable;
import org.labkey.api.module.Module;
import org.labkey.api.query.DefaultSchema;
@@ -42,15 +41,11 @@ public QuerySchema createSchema(final DefaultSchema schema, Module module)
@Nullable
protected TableInfo createWrappedTable(String name, @NotNull TableInfo sourceTable, ContainerFilter cf)
{
- if (TCRdbSchema.TABLE_LIBRARIES.equalsIgnoreCase(name))
+ if (TCRdbSchema.TABLE_MIXCR_LIBRARIES.equalsIgnoreCase(name))
{
// TODO: assert cf is null or not default?
return new SharedDataTable<>(this, sourceTable).init();
}
- else if (TCRdbSchema.TABLE_CITE_SEQ_ANTIBODIES.equalsIgnoreCase(name))
- {
- return new ContainerScopedTable<>(this, sourceTable, cf, "antibodyName").init();
- }
return super.createWrappedTable(name, sourceTable, cf);
}
diff --git a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerCellHashingHandler.java b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerCellHashingHandler.java
deleted file mode 100644
index 00be83f8c..000000000
--- a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerCellHashingHandler.java
+++ /dev/null
@@ -1,361 +0,0 @@
-package org.labkey.tcrdb.pipeline;
-
-import au.com.bytecode.opencsv.CSVReader;
-import au.com.bytecode.opencsv.CSVWriter;
-import htsjdk.samtools.util.IOUtil;
-import org.json.JSONObject;
-import org.labkey.api.module.ModuleLoader;
-import org.labkey.api.pipeline.PipelineJob;
-import org.labkey.api.pipeline.PipelineJobException;
-import org.labkey.api.pipeline.RecordedAction;
-import org.labkey.api.reader.Readers;
-import org.labkey.api.sequenceanalysis.SequenceOutputFile;
-import org.labkey.api.sequenceanalysis.model.Readset;
-import org.labkey.api.sequenceanalysis.pipeline.AbstractParameterizedOutputHandler;
-import org.labkey.api.sequenceanalysis.pipeline.DefaultPipelineStepOutput;
-import org.labkey.api.sequenceanalysis.pipeline.PipelineStepOutput;
-import org.labkey.api.sequenceanalysis.pipeline.SequenceAnalysisJobSupport;
-import org.labkey.api.sequenceanalysis.pipeline.SequenceOutputHandler;
-import org.labkey.api.sequenceanalysis.pipeline.SequencePipelineService;
-import org.labkey.api.sequenceanalysis.pipeline.ToolParameterDescriptor;
-import org.labkey.api.util.FileType;
-import org.labkey.api.util.FileUtil;
-import org.labkey.api.util.PageFlowUtil;
-import org.labkey.api.writer.PrintWriters;
-import org.labkey.tcrdb.TCRdbModule;
-
-import java.io.File;
-import java.io.FileFilter;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashSet;
-import java.util.LinkedHashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-public class CellRangerCellHashingHandler extends AbstractParameterizedOutputHandler
-{
- private FileType _fileType = new FileType("cloupe", false);
- public static String CATEGORY = "10x GEX Cell Hashing Calls";
-
- public CellRangerCellHashingHandler()
- {
- super(ModuleLoader.getInstance().getModule(TCRdbModule.class), "CellRanger GEX/Cell Hashing", "This will run CiteSeqCount/MultiSeqClassifier to generate a sample-to-cellbarcode TSV based on the filtered barcodes from CellRanger.", new LinkedHashSet<>(PageFlowUtil.set("sequenceanalysis/field/CellRangerAggrTextarea.js")), getDefaultParams());
- }
-
- private static List getDefaultParams()
- {
- List ret = new ArrayList<>(getDefaultHashingParams(true));
- ret.add(
- ToolParameterDescriptor.create("useOutputFileContainer", "Submit to Source File Workbook", "If checked, each job will be submitted to the same workbook as the input file, as opposed to submitting all jobs to the same workbook. This is primarily useful if submitting a large batch of files to process separately. This only applies if 'Run Separately' is selected.", "checkbox", new JSONObject(){{
- put("checked", true);
- }}, true)
- );
-
- return ret;
- }
-
- public static List getDefaultHashingParams(boolean includeExcludeFailedcDNA)
- {
- List ret = new ArrayList<>(Arrays.asList(
- ToolParameterDescriptor.create("scanEditDistances", "Scan Edit Distances", "If checked, CITE-seq-count will be run using edit distances from 0-3 and the iteration with the highest singlets will be used.", "checkbox", new JSONObject(){{
- put("checked", false);
- }}, false),
- ToolParameterDescriptor.create("editDistance", "Edit Distance", null, "ldk-integerfield", null, 3),
- ToolParameterDescriptor.create("minCountPerCell", "Min Reads/Cell", null, "ldk-integerfield", null, 5),
- ToolParameterDescriptor.create("useSeurat", "Use Seurat Calling", "If checked, the seurat HTO calling algorithm will be used.", "checkbox", null, true),
- ToolParameterDescriptor.create("useMultiSeq", "Use MultiSeq Calling", "If checked, the MultiSeq HTO calling algorithm will be used.", "checkbox", null, true)
- ));
-
- if (includeExcludeFailedcDNA)
- {
- ret.add(ToolParameterDescriptor.create("excludeFailedcDNA", "Exclude Failed cDNA", "If selected, cDNAs with non-blank status fields will be omitted", "checkbox", null, true));
- }
-
- return ret;
- }
-
- @Override
- public boolean canProcess(SequenceOutputFile o)
- {
- return o.getFile() != null && _fileType.isType(o.getFile());
- }
-
- @Override
- public boolean doRunRemote()
- {
- return true;
- }
-
- @Override
- public boolean doRunLocal()
- {
- return false;
- }
-
- @Override
- public SequenceOutputProcessor getProcessor()
- {
- return new CellRangerCellHashingHandler.Processor();
- }
-
- @Override
- public boolean doSplitJobs()
- {
- return true;
- }
-
- @Override
- public boolean requiresSingleGenome()
- {
- return false;
- }
-
- public class Processor implements SequenceOutputHandler.SequenceOutputProcessor
- {
- @Override
- public void init(PipelineJob job, SequenceAnalysisJobSupport support, List inputFiles, JSONObject params, File outputDir, List actions, List outputsToCreate) throws UnsupportedOperationException, PipelineJobException
- {
- new CellRangerVDJUtils(job.getLogger(), outputDir).prepareHashingAndCiteSeqFilesIfNeeded(job, support, "readsetId", params.optBoolean("excludeFailedcDNA", true), true, false);
- }
-
- @Override
- public void processFilesOnWebserver(PipelineJob job, SequenceAnalysisJobSupport support, List inputFiles, JSONObject params, File outputDir, List actions, List outputsToCreate) throws UnsupportedOperationException, PipelineJobException
- {
-
- }
-
- @Override
- public void processFilesRemote(List inputFiles, SequenceOutputHandler.JobContext ctx) throws UnsupportedOperationException, PipelineJobException
- {
- RecordedAction action = new RecordedAction(getName());
- Map readsetToHashing = CellRangerVDJUtils.getCachedHashingReadsetMap(ctx.getSequenceSupport());
- ctx.getLogger().debug("total cached readset to hashing pairs: " + readsetToHashing.size());
-
- for (SequenceOutputFile so : inputFiles)
- {
- ctx.getLogger().info("processing file: " + so.getName());
-
- //find TSV:
- File perCellTsv;
- File barcodeDir = null;
- for (String dirName : Arrays.asList("filtered_gene_bc_matrices", "filtered_feature_bc_matrix"))
- {
- File f = new File(so.getFile().getParentFile(), dirName);
- if (f.exists())
- {
- barcodeDir = f;
- break;
- }
- }
-
- if (barcodeDir == null)
- {
- //this might be a re-analysis loupe directory. in this case, use the tsne projection.csv as the whitelist:
- File dir = new File(so.getFile().getParentFile(), "analysis");
- dir = new File(dir, "tsne");
- dir = new File(dir, "2_components");
- if (!dir.exists())
- {
- throw new PipelineJobException("Unable to find barcode or analysis directory: " + dir.getPath());
- }
-
- perCellTsv = new File(dir, "projection.csv");
- }
- //cellranger 2 format
- else if ("filtered_gene_bc_matrices".equals(barcodeDir.getName()))
- {
- File[] children = barcodeDir.listFiles(new FileFilter()
- {
- @Override
- public boolean accept(File pathname)
- {
- return pathname.isDirectory();
- }
- });
-
- if (children == null || children.length != 1)
- {
- throw new PipelineJobException("Expected to find a single subfolder under: " + barcodeDir.getPath());
- }
-
- perCellTsv = new File(children[0], "barcodes.tsv");
- }
- else
- {
- perCellTsv = new File(barcodeDir, "barcodes.tsv.gz");
- }
-
- if (!perCellTsv.exists())
- {
- throw new PipelineJobException("Unable to find file: " + perCellTsv.getPath());
- }
-
- Readset rs = ctx.getSequenceSupport().getCachedReadset(so.getReadset());
- if (rs == null)
- {
- throw new PipelineJobException("Unable to find readset for outputfile: " + so.getRowid());
- }
- else if (rs.getReadsetId() == null)
- {
- throw new PipelineJobException("Readset lacks a rowId for outputfile: " + so.getRowid());
- }
-
- Readset htoReadset = ctx.getSequenceSupport().getCachedReadset(readsetToHashing.get(rs.getReadsetId()));
- if (htoReadset == null)
- {
- throw new PipelineJobException("Unable to find Hashing/Cite-seq readset for GEX readset: " + rs.getReadsetId());
- }
-
- processBarcodeFile(ctx, perCellTsv, rs, htoReadset, so.getLibrary_id(), action, getClientCommandArgs(ctx.getParams()), true, CATEGORY);
- }
-
- ctx.addActions(action);
- }
-
- @Override
- public void complete(PipelineJob job, List inputs, List outputsCreated, SequenceAnalysisJobSupport support) throws PipelineJobException
- {
- for (SequenceOutputFile so : outputsCreated)
- {
- if (so.getCategory().equals(CATEGORY))
- {
- CellRangerVDJCellHashingHandler.processMetrics(so, job, true);
- }
- }
- }
- }
-
- public static File processBarcodeFile(SequenceOutputHandler.JobContext ctx, File perCellTsv, Readset rs, Readset htoOrCiteReadset, int genomeId, RecordedAction action, List commandArgs, boolean writeLoupe, String category) throws PipelineJobException
- {
- return processBarcodeFile(ctx, perCellTsv, rs, htoOrCiteReadset, genomeId, action, commandArgs, writeLoupe, category, true);
- }
-
- public static File processBarcodeFile(SequenceOutputHandler.JobContext ctx, File perCellTsv, Readset rs, Readset htoOrCiteReadset, int genomeId, RecordedAction action, List commandArgs, boolean writeLoupe, String category, boolean generateHtoCalls) throws PipelineJobException
- {
- CellRangerVDJUtils utils = new CellRangerVDJUtils(ctx.getLogger(), ctx.getSourceDirectory());
- return processBarcodeFile(ctx, perCellTsv, rs, htoOrCiteReadset, genomeId, action, commandArgs, writeLoupe, category, true, utils.getValidHashingBarcodeFile(), generateHtoCalls);
- }
-
- public static File processBarcodeFile(SequenceOutputHandler.JobContext ctx, File perCellTsv, Readset rs, Readset htoOrCiteReadset, int genomeId, RecordedAction action, List commandArgs, boolean writeLoupe, String category, boolean createOutputFiles, File htoBarcodeWhitelist, boolean generateHtoCalls) throws PipelineJobException
- {
- ctx.getLogger().debug("inspecting file: " + perCellTsv.getPath());
-
- CellRangerVDJUtils utils = new CellRangerVDJUtils(ctx.getLogger(), ctx.getSourceDirectory());
-
- //prepare whitelist of cell indexes
- File cellBarcodeWhitelist = utils.getValidCellIndexFile();
- Set uniqueBarcodes = new HashSet<>();
- ctx.getLogger().debug("writing cell barcodes");
- try (CSVWriter writer = new CSVWriter(PrintWriters.getPrintWriter(cellBarcodeWhitelist), ',', CSVWriter.NO_QUOTE_CHARACTER);CSVReader reader = new CSVReader(IOUtil.openFileForBufferedUtf8Reading(perCellTsv), '\t'))
- {
- int rowIdx = 0;
- String[] row;
- while ((row = reader.readNext()) != null)
- {
- //skip header
- rowIdx++;
- if (rowIdx > 1)
- {
- String barcode = row[0];
-
- //NOTE: 10x appends "-1" to barcodes
- if (barcode.contains("-"))
- {
- barcode = barcode.split("-")[0];
- }
-
- //This format is written out by the seurat pipeline
- if (barcode.contains("_"))
- {
- barcode = barcode.split("_")[1];
- }
-
- if (!uniqueBarcodes.contains(barcode))
- {
- writer.writeNext(new String[]{barcode});
- uniqueBarcodes.add(barcode);
- }
- }
- }
-
- ctx.getLogger().debug("rows inspected: " + (rowIdx - 1));
- ctx.getLogger().debug("unique cell barcodes: " + uniqueBarcodes.size());
- ctx.getFileManager().addIntermediateFile(cellBarcodeWhitelist);
- }
- catch (IOException e)
- {
- throw new PipelineJobException(e);
- }
-
- //prepare whitelist of barcodes, based on cDNA records
- if (!htoBarcodeWhitelist.exists())
- {
- throw new PipelineJobException("Unable to find file: " + htoBarcodeWhitelist.getPath());
- }
- ctx.getFileManager().addIntermediateFile(htoBarcodeWhitelist);
-
- //run CiteSeqCount.
- List extraParams = new ArrayList<>();
- extraParams.addAll(commandArgs);
-
- boolean scanEditDistances = ctx.getParams().optBoolean("scanEditDistances", false);
- int editDistance = ctx.getParams().optInt("editDistance", 3);
- int minCountPerCell = ctx.getParams().optInt("minCountPerCell", 3);
- boolean useSeurat = ctx.getParams().optBoolean("useSeurat", true);
- boolean useMultiSeq = ctx.getParams().optBoolean("useMultiSeq", true);
-
- PipelineStepOutput output = new DefaultPipelineStepOutput();
- String basename = FileUtil.makeLegalName(rs.getName());
- File cellToHto = SequencePipelineService.get().runCiteSeqCount(output, category, htoOrCiteReadset, htoBarcodeWhitelist, cellBarcodeWhitelist, ctx.getWorkingDirectory(), basename, ctx.getLogger(), extraParams, false, minCountPerCell, ctx.getSourceDirectory(), editDistance, scanEditDistances, rs, genomeId, generateHtoCalls, createOutputFiles, useSeurat, useMultiSeq);
- ctx.getFileManager().addStepOutputs(action, output);
-
- ctx.getFileManager().addOutput(action, category, cellToHto);
- File html = new File(cellToHto.getParentFile(), FileUtil.getBaseName(cellToHto.getName()) + ".html");
- if (html.exists())
- {
- ctx.getFileManager().addOutput(action, "Cell Hashing Report", html);
- }
-
- File citeSeqCountUnknownOutput = new File(cellToHto.getParentFile(), "citeSeqUnknownBarcodes.txt");
- ctx.getFileManager().addOutput(action,"CiteSeqCount Unknown Barcodes", citeSeqCountUnknownOutput);
-
- if (writeLoupe)
- {
- File forLoupe = new File(ctx.getSourceDirectory(), rs.getName() + "-CiteSeqCalls.csv");
- try (CSVReader reader = new CSVReader(Readers.getReader(cellToHto), '\t'); CSVWriter writer = new CSVWriter(PrintWriters.getPrintWriter(forLoupe), ',', CSVWriter.NO_QUOTE_CHARACTER))
- {
- String[] line;
- int idx = 0;
- while ((line = reader.readNext()) != null)
- {
- idx++;
-
- if (idx > 1)
- {
- line[0] = line[0] + "-1";
- }
-
- writer.writeNext(new String[]{line[0], line[1]});
- }
- }
- catch (IOException e)
- {
- throw new PipelineJobException(e);
- }
-
- if (createOutputFiles)
- {
- ctx.getFileManager().addSequenceOutput(forLoupe, rs.getName() + ": Cell Hashing Calls", "10x GEX Cell Hashing Calls (Loupe)", rs.getReadsetId(), null, genomeId, null);
- }
- else
- {
- ctx.getLogger().debug("Output file creation will be skipped");
- }
- }
-
- return cellToHto;
- }
-}
\ No newline at end of file
diff --git a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerSeuratHandler.java b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerSeuratHandler.java
deleted file mode 100644
index 7d1fd5930..000000000
--- a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerSeuratHandler.java
+++ /dev/null
@@ -1,1054 +0,0 @@
-package org.labkey.tcrdb.pipeline;
-
-import au.com.bytecode.opencsv.CSVReader;
-import au.com.bytecode.opencsv.CSVWriter;
-import htsjdk.samtools.util.IOUtil;
-import org.apache.commons.io.FileUtils;
-import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.logging.log4j.Logger;
-import org.apache.commons.lang3.math.NumberUtils;
-import org.json.JSONObject;
-import org.labkey.api.data.CompareType;
-import org.labkey.api.data.DbSchema;
-import org.labkey.api.data.DbSchemaType;
-import org.labkey.api.data.SimpleFilter;
-import org.labkey.api.data.Table;
-import org.labkey.api.data.TableInfo;
-import org.labkey.api.data.TableSelector;
-import org.labkey.api.exp.api.ExpData;
-import org.labkey.api.exp.api.ExpRun;
-import org.labkey.api.exp.api.ExperimentService;
-import org.labkey.api.module.ModuleLoader;
-import org.labkey.api.pipeline.PipelineJob;
-import org.labkey.api.pipeline.PipelineJobException;
-import org.labkey.api.pipeline.PipelineService;
-import org.labkey.api.pipeline.PipelineStatusFile;
-import org.labkey.api.pipeline.RecordedAction;
-import org.labkey.api.query.FieldKey;
-import org.labkey.api.reader.Readers;
-import org.labkey.api.sequenceanalysis.SequenceAnalysisService;
-import org.labkey.api.sequenceanalysis.SequenceOutputFile;
-import org.labkey.api.sequenceanalysis.model.Readset;
-import org.labkey.api.sequenceanalysis.pipeline.AbstractParameterizedOutputHandler;
-import org.labkey.api.sequenceanalysis.pipeline.SequenceAnalysisJobSupport;
-import org.labkey.api.sequenceanalysis.pipeline.SequenceOutputHandler;
-import org.labkey.api.sequenceanalysis.pipeline.SequencePipelineService;
-import org.labkey.api.sequenceanalysis.pipeline.ToolParameterDescriptor;
-import org.labkey.api.sequenceanalysis.run.SimpleScriptWrapper;
-import org.labkey.api.util.FileType;
-import org.labkey.api.util.FileUtil;
-import org.labkey.api.util.PageFlowUtil;
-import org.labkey.api.writer.PrintWriters;
-import org.labkey.tcrdb.TCRdbModule;
-
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.IOException;
-import java.io.PrintWriter;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.LinkedHashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-public class CellRangerSeuratHandler extends AbstractParameterizedOutputHandler
-{
- private FileType _fileType = new FileType("cloupe", false);
- public static final String SEURAT_MAX_THREADS = "seuratMaxThreads";
-
- public CellRangerSeuratHandler()
- {
- super(ModuleLoader.getInstance().getModule(TCRdbModule.class), "Run Seurat", "This will run a standard seurat-based pipeline on the selected 10x/cellranger data and save the resulting Seurat object as an rds file for external use.", new LinkedHashSet<>(PageFlowUtil.set("sequenceanalysis/field/GenomeFileSelectorField.js")), getDefaultParams());
- }
-
- private static List getDefaultParams()
- {
- List ret = new ArrayList<>(Arrays.asList(
- ToolParameterDescriptor.create("projectName", "Output Name", "This will be used as the final sample/file name. If blank, the readset name will be used. The latter cannot be used when merging multiple inputs.", "textfield", new JSONObject(){{
-
- }}, null),
- ToolParameterDescriptor.create("doSplitJobs", "Run Separately", "If checked, each input dataset will be run separately. Otherwise they will be merged", "checkbox", new JSONObject(){{
- put("checked", true);
- }}, false),
- ToolParameterDescriptor.create("skipProcessing", "Skip Processing", "If checked, the initial merge and EmptyDrops processing will be run, but PCA, DimRux, etc. will be skipped. The primary use of this is to created a merged seurat object for manual downstream processing", "checkbox", new JSONObject(){{
- put("checked", false);
- }}, false),
- ToolParameterDescriptor.create("useOutputFileContainer", "Submit to Source File Workbook", "If checked, each job will be submitted to the same workbook as the input file, as opposed to submitting all jobs to the same workbook. This is primarily useful if submitting a large batch of files to process separately. This only applies if 'Run Separately' is selected.", "checkbox", new JSONObject(){{
- put("checked", true);
- }}, false),
- ToolParameterDescriptor.create("dimsToUse", "PCs To Use", "If non-blank, this is the number of PCs that seurat will use for dim reduction steps.", "ldk-integerfield", new JSONObject(){{
-
- }}, null),
- ToolParameterDescriptor.create("minDimsToUse", "Minimum PCs To Use", "If non-blank, the pipeline will attempt to infer the number of PCs to use for dim reduction, but will not use fewer than this value.", "ldk-integerfield", new JSONObject(){{
-
- }}, 12),
- ToolParameterDescriptor.create("doCellFilter", "Perform Cell Filtering", "If selected, cells will be filtered on pct.mito and number of unique genes.", "checkbox", new JSONObject(){{
- put("checked", true);
- }}, true),
- ToolParameterDescriptor.create("doCellCycle", "Perform Cell Cycle Correction", "If selected, the pipeline will attempt to correct for cell cycle.", "checkbox", new JSONObject(){{
- put("checked", true);
- }}, true),
- ToolParameterDescriptor.create("useSCTransform", "Use SCTransform", "If selected, the pipeline will use the newer SCtransform method instead of the standard Seurat pipeline.", "checkbox", new JSONObject(){{
- put("checked", false);
- }}, false),
- ToolParameterDescriptor.create("runSingleR", "Run SingleR", "If selected, SingleR will be run after Seurat processing.", "checkbox", new JSONObject(){{
- put("checked", true);
- }}, true),
- ToolParameterDescriptor.create("mergeMethod", "Merge Method", "This determines whether any batch correction will be applied when merging datasets.", "ldk-simplecombo", new JSONObject(){{
- put("storeValues", "simple;cca");
- }}, "simple"),
- ToolParameterDescriptor.create(SEURAT_MAX_THREADS, "Seurat Max Threads", "Because seurat can behave badly with multiple threads, this allows a separate cap to be used from the main job. This will allow CITE-Seq-Count and other tools to run with more threads.", "ldk-integerfield", null, 1),
- ToolParameterDescriptor.createExpDataParam("gtfFile", "Gene File", "This is the ID of a GTF file containing genes from this genome.", "sequenceanalysis-genomefileselectorfield", new JSONObject()
- {{
- put("extensions", Arrays.asList("gtf"));
- put("width", 400);
- put("allowBlank", false);
- }}, null)
- ));
-
- ret.addAll(CellRangerCellHashingHandler.getDefaultHashingParams(false));
-
- return ret;
- }
-
- @Override
- public boolean canProcess(SequenceOutputFile o)
- {
- return o.getFile() != null && _fileType.isType(o.getFile());
- }
-
- @Override
- public List validateParameters(List outputFiles, JSONObject params)
- {
- if (!params.optBoolean("doSplitJobs", false) && StringUtils.trimToNull(params.optString("projectName")) == null && outputFiles.size() > 1)
- {
- return Collections.singletonList("Must provide the output name when merging multiple inputs");
- }
-
- return null;
- }
-
- @Override
- public boolean doRunRemote()
- {
- return true;
- }
-
- @Override
- public boolean doRunLocal()
- {
- return false;
- }
-
- @Override
- public boolean requiresSingleGenome()
- {
- return false;
- }
-
- @Override
- public SequenceOutputProcessor getProcessor()
- {
- return new CellRangerSeuratHandler.Processor();
- }
-
- @Override
- public boolean doSplitJobs()
- {
- return false;
- }
-
- public class Processor implements SequenceOutputProcessor
- {
- private static final String GTF_FILE_ID = "gtfFileIf";
-
- @Override
- public void init(PipelineJob job, SequenceAnalysisJobSupport support, List inputFiles, JSONObject params, File outputDir, List actions, List outputsToCreate) throws UnsupportedOperationException, PipelineJobException
- {
- for (SequenceOutputFile so : inputFiles)
- {
- if (so.getReadset() != null)
- {
- support.cacheReadset(so.getReadset(), job.getUser());
- }
- else
- {
- job.getLogger().error("Output file lacks a readset and will be skipped: " + so.getRowid());
- }
- }
-
- new CellRangerVDJUtils(job.getLogger(), outputDir).prepareHashingAndCiteSeqFilesIfNeeded(job, support,"readsetId", params.optBoolean("excludeFailedcDNA", true), false, false);
-
- Set gtfIds = new HashSet<>();
- for (SequenceOutputFile so : inputFiles)
- {
- ExpData gtf = null;
- ExpRun run = ExperimentService.get().getExpRun(so.getRunId());
- if (run != null)
- {
- List extends ExpData> gtfDatas = run.getInputDatas("GTF File", null);
- if (!gtfDatas.isEmpty())
- {
- gtf = gtfDatas.get(0);
- }
- else
- {
- //Because existing runs didnt explicitly track GTF as an input, try to infer:
- PipelineStatusFile sf = PipelineService.get().getStatusFile(run.getJobId());
- if (sf != null)
- {
- File log = new File(sf.getFilePath());
- File paramFile = new File(log.getParentFile(), "sequenceAnalysis.json");
- if (paramFile.exists())
- {
- try (BufferedReader reader = Readers.getReader(paramFile))
- {
- List lines = IOUtils.readLines(reader);
-
- JSONObject json = lines.isEmpty() ? new JSONObject() : new JSONObject(StringUtils.join(lines, '\n'));
- Integer expData = json.optInt("alignment.CellRanger.gtfFile", -1);
- if (expData == -1)
- {
-
- }
-
- gtf = ExperimentService.get().getExpData(expData);
- }
- catch (IOException e)
- {
- throw new PipelineJobException(e);
- }
- }
- }
- }
- }
-
- if (gtf == null)
- {
- throw new PipelineJobException("Unable to find GTF for output: " + so.getRowid());
- }
-
- gtfIds.add(gtf.getRowId());
- }
-
- if (gtfIds.size() != 1)
- {
- throw new PipelineJobException("All inputs must use the same GTF file, found: " + StringUtils.join(gtfIds, ","));
- }
-
- support.cacheExpData(ExperimentService.get().getExpData(gtfIds.iterator().next()));
- support.cacheObject(GTF_FILE_ID, gtfIds.iterator().next());
- }
-
- @Override
- public void processFilesOnWebserver(PipelineJob job, SequenceAnalysisJobSupport support, List inputFiles, JSONObject params, File outputDir, List actions, List outputsToCreate) throws UnsupportedOperationException, PipelineJobException
- {
-
- }
-
- @Override
- public void processFilesRemote(List inputFiles, JobContext ctx) throws UnsupportedOperationException, PipelineJobException
- {
- RecordedAction action = new RecordedAction(getName());
- ctx.addActions(action);
-
- int gtfId = ctx.getSequenceSupport().getCachedObject(GTF_FILE_ID, Integer.class);
- File gtfFile = ctx.getSequenceSupport().getCachedData(gtfId);
- if (!gtfFile.exists())
- {
- throw new PipelineJobException("Unable to find GTF file: " + gtfFile.getPath());
- }
- ctx.getFileManager().addInput(action, "GTF File", gtfFile);
-
- Set rsNames = new HashSet<>();
- for (SequenceOutputFile so : inputFiles)
- {
- ctx.getFileManager().addInput(action, "CellRanger Loupe", so.getFile());
- if (so.getReadset() != null)
- {
- rsNames.add(ctx.getSequenceSupport().getCachedReadset(so.getReadset()).getName());
- }
- }
-
- String outPrefix = StringUtils.trimToNull(ctx.getParams().getString("projectName"));
- if (outPrefix == null)
- {
- if (rsNames.size() == 1)
- {
- outPrefix = rsNames.iterator().next();
- }
- else
- {
- throw new PipelineJobException("Must provide the output prefix when merging more than one output file");
- }
- }
- outPrefix = FileUtil.makeLegalName(outPrefix);
-
- File seuratObj = new File(ctx.getWorkingDirectory(), outPrefix + ".seurat.rds");
- File doneFile = new File(seuratObj.getPath() + ".done");
- boolean seuratHasRun = doneFile.exists();
- if (seuratHasRun)
- {
- ctx.getLogger().info("Seurat has already run, will not repeat");
- }
-
- Map dataMap = new HashMap<>();
-
- File pr = ctx.getFolderPipeRoot().getRootPath().getParentFile(); //drop the @files or @pipeline
- for (SequenceOutputFile so : inputFiles)
- {
- //start with seurat 3
- File subDir = new File(so.getFile().getParentFile(), "raw_feature_bc_matrix");
- if (!subDir.exists())
- {
- //try 2
- subDir = new File(so.getFile().getParentFile(), "raw_gene_bc_matrices");
- if (subDir.exists())
- {
- //now infer subdir:
- for (File f : subDir.listFiles())
- {
- if (f.isDirectory())
- {
- subDir = f;
- break;
- }
- }
- }
- }
-
- if (!subDir.exists())
- {
- throw new PipelineJobException("Unable to find raw data for input: " + so.getFile().getPath());
- }
-
- try
- {
- String subDirRel = FileUtil.relativize(pr, subDir, true);
- ctx.getLogger().debug("pipe root: " + pr.getPath());
- ctx.getLogger().debug("file path: " + subDir.getPath());
- ctx.getLogger().debug("relative path: " + subDirRel);
-
- //Copy raw data directory locally to avoid docker permission issues
- String dirName = so.getRowid() + "_RawData";
- File copyDir = new File(ctx.getWorkingDirectory(), dirName);
- if (!seuratHasRun)
- {
- if (copyDir.exists())
- {
- ctx.getLogger().debug("Deleting directory: " + copyDir.getPath());
- FileUtils.deleteDirectory(copyDir);
- }
-
- ctx.getLogger().debug("Copying raw data directory: " + subDir.getPath());
- ctx.getLogger().debug("To: " + copyDir.getPath());
- FileUtils.copyDirectory(subDir, copyDir);
- }
- ctx.getFileManager().addIntermediateFile(copyDir);
-
- dataMap.put(so, dirName);
- }
- catch (IOException e)
- {
- throw new PipelineJobException(e);
- }
- }
-
- File rmdScript = new File(SequenceAnalysisService.get().getScriptPath(TCRdbModule.NAME, "external/scRNAseq/Seurat3.rmd"));
- if (!rmdScript.exists())
- {
- throw new PipelineJobException("Unable to find script: " + rmdScript.getPath());
- }
-
- File wrapperScript = new File(SequenceAnalysisService.get().getScriptPath(TCRdbModule.NAME, "external/scRNAseq/seuratWrapper.sh"));
- if (!wrapperScript.exists())
- {
- throw new PipelineJobException("Unable to find script: " + wrapperScript.getPath());
- }
-
- File tmpScript = new File(ctx.getWorkingDirectory(), "script.R");
- File outHtml = new File(ctx.getWorkingDirectory(), outPrefix + ".html");
- boolean skipProcessing = ctx.getParams().optBoolean("skipProcessing", false);
-
- try (PrintWriter writer = PrintWriters.getPrintWriter(tmpScript))
- {
- File scriptCopy = new File(ctx.getWorkingDirectory(), rmdScript.getName());
- if (scriptCopy.exists())
- {
- scriptCopy.delete();
- }
-
- IOUtil.copyFile(rmdScript, scriptCopy);
- rmdScript = scriptCopy;
- ctx.getFileManager().addIntermediateFile(rmdScript);
-
- scriptCopy = new File(ctx.getWorkingDirectory(), wrapperScript.getName());
- if (scriptCopy.exists())
- {
- scriptCopy.delete();
- }
-
- IOUtil.copyFile(wrapperScript, scriptCopy);
- ctx.getFileManager().addIntermediateFile(scriptCopy);
-
- writer.println("outPrefix <- '" + outPrefix + "'");
- writer.println("resolutionToUse <- 0.6");
- for (String v : new String[]{"dimsToUse", "minDimsToUse"})
- {
- String val = StringUtils.trimToNull(ctx.getParams().optString(v));
- val = val == null ? "NULL" : val;
-
- writer.println(v + " <- " + val);
- }
-
- //GTF file:
- File gtfCopy = new File(ctx.getWorkingDirectory(), gtfId + ".gtf");
- if (gtfCopy.exists())
- {
- gtfCopy.delete();
- }
- IOUtil.copyFile(gtfFile, gtfCopy);
- ctx.getFileManager().addIntermediateFile(gtfCopy);
-
- writer.println("gtfFile <- '" + gtfCopy.getName() + "'");
-
- String mergeMethod = StringUtils.trimToNull(ctx.getParams().optString("mergeMethod"));
- mergeMethod = mergeMethod == null ? "NULL" : "'" + mergeMethod + "'";
- writer.println("mergeMethod <- " + mergeMethod);
-
- boolean doCellFilter = ctx.getParams().optBoolean("doCellFilter", true);
- writer.println("doCellFilter <- " + String.valueOf(doCellFilter).toUpperCase());
-
- writer.println("skipProcessing <- " + String.valueOf(skipProcessing).toUpperCase());
-
- boolean runSingleR = ctx.getParams().optBoolean("runSingleR", true);
- writer.println("runSingleR <- " + String.valueOf(runSingleR).toUpperCase());
-
- boolean doCellCycle = ctx.getParams().optBoolean("doCellCycle", true);
- writer.println("doCellCycle <- " + String.valueOf(doCellCycle).toUpperCase());
-
- boolean useSCTransform = ctx.getParams().optBoolean("useSCTransform", false);
- writer.println("useSCTransform <- " + String.valueOf(useSCTransform).toUpperCase());
-
- writer.println("data <- list(");
- String delim = "";
- for (SequenceOutputFile so : dataMap.keySet())
- {
- writer.println("\t" + delim + "'" + so.getRowid() + "'='" + dataMap.get(so) + "'");
- delim = ",";
- }
- writer.println(")");
- writer.println();
- writer.println();
- writer.println("setwd('/work')");
-
- writer.println("rmarkdown::render('" + rmdScript.getName() + "', clean=TRUE, output_file='" + outHtml.getName() + "')");
- }
- catch (IOException e)
- {
- throw new PipelineJobException(e);
- }
-
- if (!seuratHasRun)
- {
- SimpleScriptWrapper wrapper = new SimpleScriptWrapper(ctx.getLogger());
- wrapper.setWorkingDir(ctx.getWorkingDirectory());
-
- Integer maxThreads = SequencePipelineService.get().getMaxThreads(ctx.getLogger());
- if (maxThreads != null)
- {
- if (ctx.getParams().get(SEURAT_MAX_THREADS) != null)
- {
- maxThreads = Math.min(ctx.getParams().getInt(SEURAT_MAX_THREADS), maxThreads);
- wrapper.addToEnvironment("SEQUENCEANALYSIS_MAX_THREADS", maxThreads.toString());
- }
- }
-
- wrapper.execute(Arrays.asList("/bin/bash", wrapperScript.getName(), pr.getPath()));
-
- try
- {
- FileUtils.touch(doneFile);
- ctx.getFileManager().addIntermediateFile(doneFile);
- }
- catch (IOException e)
- {
- throw new PipelineJobException(e);
- }
- }
-
- if (!seuratObj.exists())
- {
- throw new PipelineJobException("Unable to find expected file: " + seuratObj.getPath());
- }
-
- String dimsToUse = StringUtils.trimToNull(ctx.getParams().optString("dimsToUse"));
- String minDimsToUse = StringUtils.trimToNull(ctx.getParams().optString("minDimsToUse"));
- String mergeMethod = StringUtils.trimToNull(ctx.getParams().optString("mergeMethod"));
-
- String description = StringUtils.join(new String[]{
- "Correct Cell Cycle: " + ctx.getParams().optBoolean("doCellCycle", true),
- "Perform Cell Filtering: " + ctx.getParams().optBoolean("doCellFilter", true),
- "Min. Dims To Use: " + (minDimsToUse == null ? "NA" : minDimsToUse),
- "Dims To Use: " + (dimsToUse == null ? "automatic" : dimsToUse),
- "Use SCTransform: " + ctx.getParams().optBoolean("useSCTransform", false),
- "Merge method: " + mergeMethod
- }, "\n");
-
- if (skipProcessing)
- {
- ctx.getFileManager().addSequenceOutput(seuratObj, "Seurat Raw Counts: " + outPrefix, "Seurat Unprocessed Data", (inputFiles.size() == 1 ? inputFiles.iterator().next().getReadset() : null), null, getGenomeId(inputFiles), "Unprocessed Data");
- }
- else
- {
- ctx.getFileManager().addSequenceOutput(seuratObj, "Seurat Object: " + outPrefix, "Seurat Data", (inputFiles.size() == 1 ? inputFiles.iterator().next().getReadset() : null), null, getGenomeId(inputFiles), description);
- }
-
- ctx.getFileManager().addOutput(action, "Seurat Object", seuratObj);
-
- if (!outHtml.exists())
- {
- throw new PipelineJobException("Unable to find summary report");
- }
- ctx.getFileManager().addOutput(action, "Seurat Report", outHtml);
-
- if (skipProcessing)
- {
- ctx.getFileManager().addSequenceOutput(outHtml, "Seurat Report: " + outPrefix, "Seurat Report", (inputFiles.size() == 1 ? inputFiles.iterator().next().getReadset() : null), null, getGenomeId(inputFiles), "Unprocessed Data");
- }
- else
- {
- ctx.getFileManager().addSequenceOutput(outHtml, "Seurat Report: " + outPrefix, "Seurat Report", (inputFiles.size() == 1 ? inputFiles.iterator().next().getReadset() : null), null, getGenomeId(inputFiles), description);
- }
-
- File seuratObjRaw = new File(ctx.getWorkingDirectory(), outPrefix + ".rawData.rds");
- if (seuratObjRaw.exists())
- {
- ctx.getFileManager().addIntermediateFile(seuratObjRaw);
- }
-
- CellRangerVDJUtils utils = new CellRangerVDJUtils(ctx.getLogger(), ctx.getSourceDirectory());
- if (utils.useCellHashing(ctx.getSequenceSupport()))
- {
- runCellHashing(ctx, inputFiles, seuratObj, action, utils);
- }
- else
- {
- ctx.getLogger().info("Cell hashing was not used");
- }
-
- if (utils.useCiteSeq(ctx.getSequenceSupport(), inputFiles))
- {
- runCiteSeq(ctx, inputFiles, seuratObj, action, outPrefix);
- }
- else
- {
- ctx.getLogger().info("CITE-seq was not used");
- }
- }
-
- private File getAllCellBarcodesFile(File seuratObj) throws PipelineJobException
- {
- File allCellBarcodes = new File(seuratObj.getParentFile(), seuratObj.getName().replaceAll("seurat.rds", "cellBarcodes.csv"));
- if (!allCellBarcodes.exists())
- {
- throw new PipelineJobException("Unable to find expected cell barcodes file. This might indicate the seurat object was created with an older version of the pipeline. Expected: " + allCellBarcodes.getPath());
- }
-
- return allCellBarcodes;
- }
-
- private void runCiteSeq(JobContext ctx, List inputFiles, File seuratObj, RecordedAction action, String outPrefix) throws PipelineJobException
- {
- ctx.getLogger().info("Adding CITE-seq");
-
- Map citeSeqData = new HashMap<>();
- Map markerMetadata = new HashMap<>();
- File allCellBarcodes = getAllCellBarcodesFile(seuratObj);
-
- for (SequenceOutputFile so : inputFiles)
- {
- //This is the loupe file at this point
- String barcodePrefix = so.getRowid().toString();
- Readset rs = ctx.getSequenceSupport().getCachedReadset(so.getReadset());
- if (rs == null)
- {
- throw new PipelineJobException("Unable to find readset for outputfile: " + so.getRowid());
- }
- else if (rs.getReadsetId() == null)
- {
- throw new PipelineJobException("Readset lacks a rowId for outputfile: " + so.getRowid());
- }
-
- File barcodes = subsetBarcodes(allCellBarcodes, barcodePrefix);
- ctx.getFileManager().addIntermediateFile(barcodes);
-
- // write readset-specific HTO list
- Integer citeseqReadsetId = CellRangerVDJUtils.getCachedCiteSeqReadsetMap(ctx.getSequenceSupport()).get(rs.getReadsetId());
- if (citeseqReadsetId == null)
- {
- ctx.getLogger().info("No cite-seq readset for: " + rs.getReadsetId() + ", this probably indicates either hashing is not used or the hashing data is not available.");
- continue;
- }
-
- Readset citeseqReadset = ctx.getSequenceSupport().getCachedReadset(citeseqReadsetId);
- if (citeseqReadset == null)
- {
- throw new PipelineJobException("Unable to find Cite-seq readset for GEX readset: " + rs.getReadsetId());
- }
-
- File perReadsetAdts = CellRangerVDJUtils.getValidCiteSeqBarcodeFile(ctx.getSourceDirectory(), rs.getReadsetId());
- long adtsForReadset = !perReadsetAdts.exists() ? 0 : SequencePipelineService.get().getLineCount(perReadsetAdts) - 1;
-
- if (adtsForReadset > 0)
- {
- ctx.getLogger().info("Total ADTs for readset: " + adtsForReadset);
- File countMatrix = CellRangerCellHashingHandler.processBarcodeFile(ctx, barcodes, rs, citeseqReadset, so.getLibrary_id(), action, getClientCommandArgs(ctx.getParams()), false, SeuratCiteSeqHandler.CATEGORY, true, perReadsetAdts, false);
- citeSeqData.put(barcodePrefix, countMatrix.getParentFile());
- File perReadsetAdtMetadata = CellRangerVDJUtils.getValidCiteSeqBarcodeMetadataFile(ctx.getSourceDirectory(), rs.getReadsetId());
- markerMetadata.put(barcodePrefix, perReadsetAdtMetadata);
- }
- else
- {
- ctx.getLogger().info("No ADTs found for readset: " + rs.getReadsetId());
- }
- }
-
- if (!citeSeqData.isEmpty())
- {
- ctx.getLogger().info("Storing cite-seq data in seurat object");
- File outHtml = appendCiteSeqToSeurat(ctx, seuratObj, citeSeqData, markerMetadata);
-
- ctx.getFileManager().addSequenceOutput(outHtml, "CITE-Seq Report: " + outPrefix, "CITE-Seq Report", (inputFiles.size() == 1 ? inputFiles.iterator().next().getReadset() : null), null, getGenomeId(inputFiles), null);
- }
- else
- {
- ctx.getLogger().info("CITE-seq was not used. Will not append to seurat");
- }
- }
-
- private void runCellHashing(JobContext ctx, List inputFiles, File seuratObj, RecordedAction action, CellRangerVDJUtils utils) throws PipelineJobException
- {
- ctx.getLogger().info("Adding cell hashing");
-
- Map finalCalls = new HashMap<>();
- File allCellBarcodes = getAllCellBarcodesFile(seuratObj);
-
- for (SequenceOutputFile so : inputFiles)
- {
- //This is the loupe file at this point
- String barcodePrefix = so.getRowid().toString();
- Readset rs = ctx.getSequenceSupport().getCachedReadset(so.getReadset());
- if (rs == null)
- {
- throw new PipelineJobException("Unable to find readset for outputfile: " + so.getRowid());
- }
- else if (rs.getReadsetId() == null)
- {
- throw new PipelineJobException("Readset lacks a rowId for outputfile: " + so.getRowid());
- }
-
- File barcodes = subsetBarcodes(allCellBarcodes, barcodePrefix);
- ctx.getFileManager().addIntermediateFile(barcodes);
-
- // write readset-specific HTO list
- Integer hashingReadsetId = CellRangerVDJUtils.getCachedHashingReadsetMap(ctx.getSequenceSupport()).get(rs.getReadsetId());
- if (hashingReadsetId == null)
- {
- ctx.getLogger().info("No hashing readset for: " + rs.getReadsetId() + ", this probably indicates either hashing is not used or the hashing data is not available.");
- continue;
- }
-
- Readset htoReadset = ctx.getSequenceSupport().getCachedReadset(hashingReadsetId);
- if (htoReadset == null)
- {
- throw new PipelineJobException("Unable to find hashing readset for GEX readset: " + rs.getReadsetId());
- }
-
- File perReadsetHtos = new File(allCellBarcodes.getParentFile(), "allowableHtos." + barcodePrefix + ".txt");
- int htosForReadset = 0;
- try (CSVReader reader = new CSVReader(Readers.getReader(utils.getCDNAInfoFile()), '\t'); CSVWriter bcWriter = new CSVWriter(PrintWriters.getPrintWriter(perReadsetHtos), ',', CSVWriter.NO_QUOTE_CHARACTER))
- {
- String[] line;
- while ((line = reader.readNext()) != null)
- {
- if (hashingReadsetId.toString().equals(line[5]))
- {
- htosForReadset++;
- bcWriter.writeNext(new String[]{line[8], line[7]});
- }
- }
- }
- catch (IOException e)
- {
- throw new PipelineJobException(e);
- }
-
- if (htosForReadset > 0)
- {
- ctx.getLogger().info("Total HTOs for readset: " + htosForReadset);
- finalCalls.put(barcodePrefix, CellRangerCellHashingHandler.processBarcodeFile(ctx, barcodes, rs, htoReadset, so.getLibrary_id(), action, getClientCommandArgs(ctx.getParams()), false, SeuratCellHashingHandler.CATEGORY, true, perReadsetHtos, true));
- }
- else
- {
- ctx.getLogger().info("No HTOs found for readset");
- }
- }
-
- if (!finalCalls.isEmpty())
- {
- ctx.getLogger().info("Storing cell hashing calls in seurat object");
- appendHashingCallsToSeurat(ctx, seuratObj, finalCalls);
- }
- else
- {
- ctx.getLogger().info("Cell hashing was not used. will not append to seurat");
- }
- }
-
- private File subsetBarcodes(File allCellBarcodes, String barcodePrefix) throws PipelineJobException
- {
- //Subset barcodes by dataset:
- File barcodes = new File(allCellBarcodes.getParentFile(), "cellBarcodeWhitelist." + barcodePrefix + ".txt");
- try (CSVReader reader = new CSVReader(Readers.getReader(allCellBarcodes), '\t'); CSVWriter writer = new CSVWriter(PrintWriters.getPrintWriter(barcodes), '\t', CSVWriter.NO_QUOTE_CHARACTER))
- {
- String[] line;
- while ((line = reader.readNext()) != null)
- {
- String barcode = line[0];
- if (barcode.startsWith(barcodePrefix + "_"))
- {
- barcode = barcode.split("_")[1];
- writer.writeNext(new String[]{barcode});
- }
- }
- }
- catch (IOException e)
- {
- throw new PipelineJobException(e);
- }
-
- return barcodes;
- }
-
- private File appendCiteSeqToSeurat(JobContext ctx, File seuratObj, Map citeseqData, Map perReadsetAdtMap) throws PipelineJobException
- {
- File rScript = new File(seuratObj.getParentFile(), "appendCiteSeq.Rmd");
- File bashScript = new File(seuratObj.getParentFile(), "runDockerForCiteSeq.sh");
-
- File localRoot = seuratObj.getParentFile();
-
- File outputHtml = new File(localRoot, FileUtil.getBaseName(seuratObj) + ".citeseq.html");
-
- Set toDelete = new HashSet<>();
- try (PrintWriter rWriter = PrintWriters.getPrintWriter(rScript); PrintWriter bashWriter = PrintWriters.getPrintWriter(bashScript))
- {
- rWriter.println("---");
- rWriter.println(" title: 'CITE-seq'");
- rWriter.println("---");
-
- rWriter.println("```{r setup}");
- rWriter.println("library(OOSAP)");
- rWriter.println("```");
- rWriter.println("");
-
- rWriter.println("```{r citeseq}");
- rWriter.println("seuratObj <- readRDS('" + seuratObj.getName() + "')");
- rWriter.println("initialCells <- ncol(seuratObj)");
- rWriter.println("citeSeq <- list(");
- int idx = 0;
- for (String barcodePrefix : citeseqData.keySet()) {
- idx++;
- String localCopy = ensureLocalCopy(localRoot, toDelete, citeseqData.get(barcodePrefix), ctx.getLogger());
- rWriter.println("'" + barcodePrefix + "' = '" + localCopy + "'" + (idx < citeseqData.size() ? "," : ""));
- }
- rWriter.println(")");
- rWriter.println("");
-
- rWriter.println("perReadsetAdtMap <- list(");
- idx = 0;
- for (String barcodePrefix : perReadsetAdtMap.keySet()) {
- idx++;
- String localCopy = ensureLocalCopy(localRoot, toDelete, perReadsetAdtMap.get(barcodePrefix), ctx.getLogger());
- rWriter.println("'" + barcodePrefix + "' = '" + localCopy + "'" + (idx < citeseqData.size() ? "," : ""));
- }
- rWriter.println(")");
- rWriter.println("");
-
- rWriter.println("for (barcodePrefix in names(citeSeq)) {");
- rWriter.println(" seuratObj <- OOSAP:::AppendCiteSeq(seuratObj = seuratObj, countMatrixDir = citeSeq[[barcodePrefix]], barcodePrefix = barcodePrefix, featureLabelTable = perReadsetAdtMap[[barcodePrefix]])");
- rWriter.println("}");
- rWriter.println("if (ncol(seuratObj) != initialCells) { stop('Cell count not equal after appending cite-seq calls!') }");
- rWriter.println("saveRDS(seuratObj, file = '" + seuratObj.getName() + "')");
- rWriter.println("```");
-
- rWriter.println("```{r Plot}");
- rWriter.println("OOSAP:::.PlotCiteSeqCountData(seuratObj)");
- rWriter.println("```");
-
- rWriter.println("```{r SessionInfo}");
- rWriter.println("sessionInfo()");
- rWriter.println("```");
-
-
- bashWriter.println("#!/bin/bash");
- bashWriter.println("set -e");
- bashWriter.println("set -x");
- bashWriter.println("DOCKER=/opt/acc/sbin/exadocker");
- bashWriter.println("WD=`pwd`");
- bashWriter.println("HOME=`echo ~/`");
-
- Integer maxRam = SequencePipelineService.get().getMaxRam();
- String ramOpts = "";
- if (maxRam != null)
- {
- ramOpts = " --memory=" +maxRam +"g ";
- }
-
- bashWriter.println("sudo $DOCKER pull bimberlab/oosap");
- bashWriter.println("sudo $DOCKER run --rm=true " + ramOpts + "-v \"${WD}:/work\" -v \"${HOME}:/homeDir\" -u $UID -e USERID=$UID -w /work -e HOME=/homeDir bimberlab/oosap Rscript -e \"" + "rmarkdown::render('" + rScript.getName() + "', output_file = '" + outputHtml.getName() + "')\"");
-
- }
- catch (IOException e)
- {
- throw new PipelineJobException(e);
- }
-
- try
- {
- SimpleScriptWrapper wrapper = new SimpleScriptWrapper(ctx.getLogger());
- wrapper.setWorkingDir(seuratObj.getParentFile());
- wrapper.execute(Arrays.asList("/bin/bash", bashScript.getName()));
-
- for (File f : toDelete)
- {
- ctx.getLogger().debug("deleting local copy: " + f.getPath());
- if (f.isDirectory())
- {
- FileUtils.deleteDirectory(f);
- }
- else
- {
- f.delete();
- }
- }
- }
- catch (IOException e)
- {
- throw new PipelineJobException(e);
- }
-
- return outputHtml;
- }
-
- private String ensureLocalCopy(File localRoot, Set toDelete, File toCopy, Logger log) throws PipelineJobException
- {
- log.info("copying file locally: " + toCopy.getPath());
-
- if (toCopy.getPath().startsWith(localRoot.getPath()))
- {
- return FileUtil.relativePath(localRoot.getPath(), toCopy.getPath());
- }
-
- try
- {
- File localCopy;
- if (toCopy.isDirectory())
- {
- localCopy = new File(localRoot, toCopy.getName());
-
- File umiDir = new File(toCopy, "umi_count");
- if (!umiDir.exists())
- {
- throw new PipelineJobException("Missing umi_count dir: " + umiDir.getPath());
- }
-
- if (localCopy.exists())
- {
- log.info("local copy exists, skipping: " + localCopy.getPath());
- }
- else
- {
- FileUtils.copyDirectory(umiDir, localCopy);
- }
- }
- else
- {
- localCopy = new File(localRoot, toCopy.getName());
-
- if (localCopy.exists())
- {
- log.info("local copy exists, skipping: " + localCopy.getPath());
- }
- else
- {
- FileUtils.copyFile(toCopy, localCopy);
- }
- }
-
- log.debug("destination: " + localCopy.getPath());
- toDelete.add(localCopy);
-
- return FileUtil.relativePath(localRoot.getPath(), localCopy.getPath());
- }
- catch (IOException e)
- {
- throw new PipelineJobException(e);
- }
- }
-
- private void appendHashingCallsToSeurat(JobContext ctx, File seuratObj, Map finalCalls) throws PipelineJobException
- {
- File rScript = new File(seuratObj.getParentFile(), "appendHashing.R");
- File bashScript = new File(seuratObj.getParentFile(), "runDockerForHashing.sh");
-
- try (PrintWriter rWriter = PrintWriters.getPrintWriter(rScript); PrintWriter bashWriter = PrintWriters.getPrintWriter(bashScript))
- {
- rWriter.println("library(OOSAP)");
- rWriter.println("seuratObj <- readRDS('" + seuratObj.getName() + "')");
- rWriter.println("initialCells <- ncol(seuratObj)");
- rWriter.println("callsFiles <- list(");
- int idx = 0;
- for (String barcodePrefix : finalCalls.keySet())
- {
- idx++;
- rWriter.println("'" + barcodePrefix + "' = '" + finalCalls.get(barcodePrefix).getName() + "'" + (idx < finalCalls.size() ? "," : ""));
- }
-
- rWriter.println(")");
- rWriter.println("");
- rWriter.println("for (barcodePrefix in names(callsFiles)) {");
- rWriter.println(" seuratObj <- OOSAP:::AppendCellHashing(seuratObj = seuratObj, barcodeCallFile = callsFiles[[barcodePrefix]], barcodePrefix = barcodePrefix)");
- rWriter.println("}");
- rWriter.println("if (ncol(seuratObj) != initialCells) { stop('Cell count not equal after appending cell hashing calls!') }");
- rWriter.println("saveRDS(seuratObj, file = '" + seuratObj.getName() + "')");
-
- bashWriter.println("#!/bin/bash");
- bashWriter.println("set -e");
- bashWriter.println("set -x");
- bashWriter.println("DOCKER=/opt/acc/sbin/exadocker");
- bashWriter.println("WD=`pwd`");
- bashWriter.println("HOME=`echo ~/`");
-
- Integer maxRam = SequencePipelineService.get().getMaxRam();
- String ramOpts = "";
- if (maxRam != null)
- {
- ramOpts = " --memory=" + maxRam + "g ";
- }
-
- bashWriter.println("sudo $DOCKER pull bimberlab/oosap");
- bashWriter.println("sudo $DOCKER run --rm=true " + ramOpts + "-v \"${WD}:/work\" -v \"${HOME}:/homeDir\" -u $UID -e USERID=$UID -w /work -e HOME=/homeDir bimberlab/oosap Rscript --vanilla " + rScript.getName());
- }
- catch (IOException e)
- {
- throw new PipelineJobException(e);
- }
-
- SimpleScriptWrapper wrapper = new SimpleScriptWrapper(ctx.getLogger());
- wrapper.setWorkingDir(seuratObj.getParentFile());
- wrapper.execute(Arrays.asList("/bin/bash", bashScript.getName()));
- }
-
- @Override
- public void complete(PipelineJob job, List inputs, List outputsCreated, SequenceAnalysisJobSupport support) throws PipelineJobException
- {
- for (SequenceOutputFile so : outputsCreated)
- {
- if (so.getFile() != null && so.getFile().getPath().endsWith(".seurat.rds"))
- {
- File metrics = new File(so.getFile().getPath().replaceAll(".seurat.rds", ".summary.txt"));
- if (metrics.exists())
- {
- processMetricsFile(job, metrics, so);
- }
- else
- {
- job.getLogger().warn("Unable to find metrics file: " + metrics.getPath());
- }
- }
- else if (so.getFile() != null && so.getFile().getPath().endsWith(".calls.txt"))
- {
- File metrics = new File(so.getFile().getPath().replaceAll(".calls.txt", ".metrics.txt"));
- if (metrics.exists())
- {
- processMetricsFile(job, metrics, so);
- }
- else
- {
- job.getLogger().warn("Unable to find metrics file: " + metrics.getPath());
- }
- }
- }
- }
- }
-
- private void processMetricsFile(PipelineJob job, File metrics, SequenceOutputFile so) throws PipelineJobException
- {
- job.getLogger().info("Loading metrics");
- TableInfo ti = DbSchema.get("sequenceanalysis", DbSchemaType.Module).getTable("quality_metrics");
-
- //NOTE: if this job errored and restarted, we may have duplicate records:
- SimpleFilter filter = new SimpleFilter(FieldKey.fromString("readset"), so.getReadset());
- filter.addCondition(FieldKey.fromString("analysis_id"), so.getAnalysis_id(), CompareType.EQUAL);
- filter.addCondition(FieldKey.fromString("dataid"), so.getDataId(), CompareType.EQUAL);
- filter.addCondition(FieldKey.fromString("container"), job.getContainer().getId(), CompareType.EQUAL);
- TableSelector ts = new TableSelector(ti, PageFlowUtil.set("rowid"), filter, null);
- if (ts.exists())
- {
- job.getLogger().info("Deleting existing QC metrics (probably from prior restarted job)");
- ts.getArrayList(Integer.class).forEach(rowid -> {
- Table.delete(ti, rowid);
- });
- }
-
- int total = 0;
- try (CSVReader reader = new CSVReader(Readers.getReader(metrics), '\t'))
- {
- String[] line;
- while ((line = reader.readNext()) != null)
- {
- if ("Category".equals(line[0]))
- {
- continue;
- }
-
- Map r = new HashMap<>();
- r.put("category", line[0]);
- r.put("metricname", line[1]);
-
- String fieldName = NumberUtils.isCreatable(line[2]) ? "metricvalue" : "qualvalue";
- r.put(fieldName, line[2]);
- r.put("analysis_id", so.getAnalysis_id());
- r.put("dataid", so.getDataId());
- r.put("readset", so.getReadset());
- r.put("container", job.getContainer());
- r.put("createdby", job.getUser().getUserId());
-
- Table.insert(job.getUser(), ti, r);
- total++;
- }
- }
- catch (IOException e)
- {
- throw new PipelineJobException(e);
- }
-
- job.getLogger().info("total metrics: " + total);
- }
-
- private Integer getGenomeId(List inputFiles)
- {
- Set genomeIds = new HashSet<>();
- inputFiles.forEach(x -> {
- genomeIds.add(x.getLibrary_id());
- });
-
- return genomeIds.size() == 1 ? genomeIds.iterator().next() : null;
- }
-}
diff --git a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java
index 19a3fdb61..cab9554ae 100644
--- a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java
+++ b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java
@@ -1,23 +1,14 @@
package org.labkey.tcrdb.pipeline;
import au.com.bytecode.opencsv.CSVReader;
-import org.apache.commons.beanutils.ConversionException;
+import au.com.bytecode.opencsv.CSVWriter;
import org.apache.commons.lang3.StringUtils;
-import org.apache.commons.lang3.math.NumberUtils;
import org.json.JSONObject;
-import org.labkey.api.data.CompareType;
import org.labkey.api.data.ConvertHelper;
-import org.labkey.api.data.DbSchema;
-import org.labkey.api.data.DbSchemaType;
-import org.labkey.api.data.SimpleFilter;
-import org.labkey.api.data.Table;
-import org.labkey.api.data.TableInfo;
-import org.labkey.api.data.TableSelector;
import org.labkey.api.module.ModuleLoader;
import org.labkey.api.pipeline.PipelineJob;
import org.labkey.api.pipeline.PipelineJobException;
import org.labkey.api.pipeline.RecordedAction;
-import org.labkey.api.query.FieldKey;
import org.labkey.api.reader.Readers;
import org.labkey.api.sequenceanalysis.SequenceOutputFile;
import org.labkey.api.sequenceanalysis.model.AnalysisModel;
@@ -27,31 +18,38 @@
import org.labkey.api.sequenceanalysis.pipeline.SequenceAnalysisJobSupport;
import org.labkey.api.sequenceanalysis.pipeline.SequenceOutputHandler;
import org.labkey.api.sequenceanalysis.pipeline.ToolParameterDescriptor;
+import org.labkey.api.singlecell.CellHashingService;
import org.labkey.api.util.FileType;
+import org.labkey.api.util.FileUtil;
import org.labkey.api.util.PageFlowUtil;
+import org.labkey.api.writer.PrintWriters;
import org.labkey.tcrdb.TCRdbModule;
import java.io.File;
import java.io.IOException;
-import java.text.DecimalFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
-
-import static org.labkey.tcrdb.pipeline.CellRangerVDJWrapper.DELETE_EXISTING_ASSAY_DATA;
-import static org.labkey.tcrdb.pipeline.CellRangerVDJWrapper.TARGET_ASSAY;
+import java.util.Set;
public class CellRangerVDJCellHashingHandler extends AbstractParameterizedOutputHandler
{
- private FileType _fileType = new FileType("vloupe", false);
+ private FileType _vloupeFileType = new FileType("vloupe", false);
+ private FileType _htmlFileType = new FileType("html", false);
+
public static final String CATEGORY = "Cell Hashing Calls (VDJ)";
+ public static final String TARGET_ASSAY = "targetAssay";
+ public static final String DELETE_EXISTING_ASSAY_DATA = "deleteExistingAssayData";
+ public static final String USE_GEX_BARCODES = "useGexBarcodes";
+
public CellRangerVDJCellHashingHandler()
{
- super(ModuleLoader.getInstance().getModule(TCRdbModule.class), "CellRanger VDJ Import", "This will either directly import data (if cell hashing is not used), or run CiteSeqCount/MultiSeqClassifier to generate a sample-to-cellbarcode TSV based on the filtered barcodes from CellRanger VDJ and then import.", new LinkedHashSet<>(PageFlowUtil.set("tcrdb/field/AssaySelectorField.js")), getDefaultParams());
+ super(ModuleLoader.getInstance().getModule(TCRdbModule.class), "CellRanger VDJ Import", "This will either directly import data (if cell hashing is not used), or run cellhashR on the hashing count matrix to generate a sample-to-cellbarcode TSV based on the filtered barcodes from CellRanger VDJ and then import.", new LinkedHashSet<>(PageFlowUtil.set("tcrdb/field/AssaySelectorField.js")), getDefaultParams());
}
private static List getDefaultParams()
@@ -63,10 +61,13 @@ private static List getDefaultParams()
}}, true),
ToolParameterDescriptor.create("useOutputFileContainer", "Submit to Source File Workbook", "If checked, each job will be submitted to the same workbook as the input file, as opposed to submitting all jobs to the same workbook. This is primarily useful if submitting a large batch of files to process separately. This only applies if 'Run Separately' is selected.", "checkbox", new JSONObject(){{
put("checked", true);
+ }}, false),
+ ToolParameterDescriptor.create(USE_GEX_BARCODES, "Use GEX and TCR Cell Barcodes", "If checked, the cell barcode whitelist used for cell hashing will be the union of TCR and GEX cell barcodes. If T-cells are a rare component of total cells, this might enhance the effectiveness of the callers by providing more positive signal.", "checkbox", new JSONObject(){{
+ put("checked", true);
}}, false)
));
- ret.addAll(CellRangerCellHashingHandler.getDefaultHashingParams(true));
+ ret.addAll(CellHashingService.get().getHashingCallingParams());
return ret;
}
@@ -74,7 +75,7 @@ private static List getDefaultParams()
@Override
public boolean canProcess(SequenceOutputFile o)
{
- return o.getFile() != null && _fileType.isType(o.getFile());
+ return o.getFile() != null && (_vloupeFileType.isType(o.getFile()) || (_htmlFileType.isType(o.getFile()) && "10x Run Summary".equals(o.getCategory()) && o.getName().contains("VDJ Summary")));
}
@Override
@@ -110,12 +111,17 @@ public boolean requiresSingleGenome()
public class Processor implements SequenceOutputHandler.SequenceOutputProcessor
{
@Override
- public void init(PipelineJob job, SequenceAnalysisJobSupport support, List inputFiles, JSONObject params, File outputDir, List actions, List outputsToCreate) throws UnsupportedOperationException, PipelineJobException
+ public void init(JobContext ctx, List inputFiles, List actions, List outputsToCreate) throws UnsupportedOperationException, PipelineJobException
{
- CellRangerVDJUtils utils = new CellRangerVDJUtils(job.getLogger(), outputDir);
-
//NOTE: this is the pathway to import assay data, whether hashing is used or not
- utils.prepareHashingAndCiteSeqFilesIfNeeded(job, support, "enrichedReadsetId", params.optBoolean("excludeFailedcDNA", true), false, false);
+ CellHashingService.get().prepareHashingIfNeeded(ctx.getOutputDir(), ctx.getJob(), ctx.getSequenceSupport(), "tcrReadsetId", false);
+
+ if (ctx.getParams().optBoolean(USE_GEX_BARCODES, false))
+ {
+ ctx.getJob().getLogger().info("The union of TCR and GEX cell barcodes will be used for calling");
+ Map vLoupeIdToGexBarcodeDir = new HashMap<>();
+
+ }
}
@Override
@@ -131,11 +137,10 @@ public void complete(PipelineJob job, List inputFiles, List<
{
if (CATEGORY.equals(so.getCategory()))
{
- processMetrics(so, job, true);
+ CellHashingService.get().processMetrics(so, job, true);
}
}
- CellRangerVDJUtils utils = new CellRangerVDJUtils(job.getLogger(), job.getLogFile().getParentFile());
if (StringUtils.trimToNull(job.getParameters().get(TARGET_ASSAY)) == null)
{
job.getLogger().info("No assay selected, will not import");
@@ -157,7 +162,7 @@ public void complete(PipelineJob job, List inputFiles, List<
for (SequenceOutputFile so : inputFiles)
{
AnalysisModel model = support.getCachedAnalysis(so.getAnalysis_id());
- utils.importAssayData(job, model, so.getFile().getParentFile(), assayId, null, deleteExistingData);
+ new CellRangerVDJUtils(job.getLogger()).importAssayData(job, model, so.getFile(), job.getLogFile().getParentFile(), assayId, null, deleteExistingData);
}
}
}
@@ -165,15 +170,13 @@ public void complete(PipelineJob job, List inputFiles, List<
@Override
public void processFilesRemote(List inputFiles, JobContext ctx) throws UnsupportedOperationException, PipelineJobException
{
- CellRangerVDJUtils utils = new CellRangerVDJUtils(ctx.getLogger(), ctx.getSourceDirectory());
RecordedAction action = new RecordedAction(getName());
-
for (SequenceOutputFile so : inputFiles)
{
ctx.getLogger().info("processing file: " + so.getName());
//find TSV:
- File perCellTsv = utils.getPerCellCsv(so.getFile().getParentFile());
+ File perCellTsv = CellRangerVDJUtils.getPerCellCsv(so.getFile().getParentFile());
if (!perCellTsv.exists())
{
throw new PipelineJobException("Unable to find file: " + perCellTsv.getPath());
@@ -197,140 +200,111 @@ else if (rs.getReadsetId() == null)
private void processVloupeFile(JobContext ctx, File perCellTsv, Readset rs, RecordedAction action, Integer genomeId) throws PipelineJobException
{
- CellRangerVDJUtils utils = new CellRangerVDJUtils(ctx.getLogger(), ctx.getSourceDirectory());
-
- List extraParams = new ArrayList<>();
- extraParams.addAll(getClientCommandArgs(ctx.getParams()));
-
- //prepare whitelist of cell indexes
AlignmentOutputImpl output = new AlignmentOutputImpl();
- boolean scanEditDistances = ctx.getParams().optBoolean("scanEditDistances", false);
- boolean useSeurat = ctx.getParams().optBoolean("useSeurat", true);
- boolean useMultiSeq = ctx.getParams().optBoolean("useMultiSeq", true);
- int minCountPerCell = ctx.getParams().optInt("minCountPerCell", 3);
- int editDistance = ctx.getParams().optInt("editDistance", 2);
-
- File cellToHto = utils.runRemoteVdjCellHashingTasks(output, CATEGORY, perCellTsv, rs, ctx.getSequenceSupport(), extraParams, ctx.getWorkingDirectory(), ctx.getSourceDirectory(), editDistance, scanEditDistances, genomeId, minCountPerCell, useSeurat, useMultiSeq);
- if (utils.useCellHashing(ctx.getSequenceSupport()) && cellToHto == null)
+
+ Set htosPerReadset = CellHashingService.get().getHtosForParentReadset(rs.getReadsetId(), ctx.getSourceDirectory(), ctx.getSequenceSupport());
+ if (htosPerReadset.size() > 1)
{
- throw new PipelineJobException("Missing cell to HTO file");
+ ctx.getLogger().info("Total HTOs for readset: " + htosPerReadset.size());
- }
+ //TODO: allow union of GEX and TCR cell barcodes for whitelist!
- ctx.getFileManager().addStepOutputs(action, output);
+ CellHashingService.CellHashingParameters parameters = CellHashingService.CellHashingParameters.createFromJson(CellHashingService.BARCODE_TYPE.hashing, ctx.getSourceDirectory(), ctx.getParams(), null, rs);
+ parameters.genomeId = genomeId;
+ parameters.outputCategory = CATEGORY;
+ parameters.basename = FileUtil.makeLegalName(rs.getName());
+ parameters.allowableHtoBarcodes = htosPerReadset;
+ parameters.cellBarcodeWhitelistFile = createCellbarcodeWhitelist(ctx, perCellTsv, true);
+ File existingCountMatrixUmiDir = CellHashingService.get().getExistingFeatureBarcodeCountDir(rs, CellHashingService.BARCODE_TYPE.hashing, ctx.getSequenceSupport());
- }
- }
+ File cellToHto = CellHashingService.get().generateHashingCallsForRawMatrix(rs, output, ctx, parameters, existingCountMatrixUmiDir);
+ if (CellHashingService.get().usesCellHashing(ctx.getSequenceSupport(), ctx.getSourceDirectory()) && cellToHto == null)
+ {
+ throw new PipelineJobException("Missing cell to HTO file");
+ }
- private static File getMetricsFile(File callFile)
- {
- return new File(callFile.getPath().replaceAll(".calls.txt", ".metrics.txt"));
- }
+ action.addOutput(cellToHto, CellRangerVDJUtils.TCR_HASHING_CALLS, false);
+ ctx.getFileManager().addStepOutputs(action, output);
+ }
+ else if (htosPerReadset.size() == 1)
+ {
+ ctx.getLogger().info("Only single HTO used for lane, skipping cell hashing calling");
+ }
+ else
+ {
+ ctx.getLogger().info("No HTOs found for readset");
+ }
+ }
- public static void processMetrics(SequenceOutputFile so, PipelineJob job, boolean updateDescription) throws PipelineJobException
- {
- if (so.getFile() != null)
+ private File createCellbarcodeWhitelist(JobContext ctx, File perCellTsv, boolean allowCellsLackingCDR3) throws PipelineJobException
{
- Map valueMap = new HashMap<>();
-
- File metrics = getMetricsFile(so.getFile());
- if (metrics.exists())
+ //prepare whitelist of cell indexes based on TCR calls:
+ File cellBarcodeWhitelist = new File(ctx.getSourceDirectory(), "validCellIndexes.csv");
+ Set uniqueBarcodes = new HashSet<>();
+ Set uniqueBarcodesIncludingNoCDR3 = new HashSet<>();
+ ctx.getLogger().debug("writing cell barcodes, using file: " + perCellTsv.getPath());
+ ctx.getLogger().debug("allow cells lacking CDR3: " + allowCellsLackingCDR3);
+
+ int totalBarcodeWritten = 0;
+ try (CSVWriter writer = new CSVWriter(PrintWriters.getPrintWriter(cellBarcodeWhitelist), ',', CSVWriter.NO_QUOTE_CHARACTER); CSVReader reader = new CSVReader(Readers.getReader(perCellTsv), ','))
{
- job.getLogger().info("Loading metrics");
- int total = 0;
- TableInfo ti = DbSchema.get("sequenceanalysis", DbSchemaType.Module).getTable("quality_metrics");
-
- //NOTE: if this job errored and restarted, we may have duplicate records:
- SimpleFilter filter = new SimpleFilter(FieldKey.fromString("readset"), so.getReadset());
- filter.addCondition(FieldKey.fromString("analysis_id"), so.getAnalysis_id(), CompareType.EQUAL);
- filter.addCondition(FieldKey.fromString("dataid"), so.getDataId(), CompareType.EQUAL);
- filter.addCondition(FieldKey.fromString("container"), job.getContainer().getId(), CompareType.EQUAL);
- TableSelector ts = new TableSelector(ti, PageFlowUtil.set("rowid"), filter, null);
- if (ts.exists())
+ int rowIdx = 0;
+ int noCallRows = 0;
+ int nonCell = 0;
+ String[] row;
+ while ((row = reader.readNext()) != null)
{
- job.getLogger().info("Deleting existing QC metrics (probably from prior restarted job)");
- ts.getArrayList(Integer.class).forEach(rowid -> {
- Table.delete(ti, rowid);
- });
- }
-
- try (CSVReader reader = new CSVReader(Readers.getReader(metrics), '\t'))
- {
- String[] line;
- while ((line = reader.readNext()) != null)
+ //skip header
+ rowIdx++;
+ if (rowIdx > 1)
{
- if ("Category".equals(line[0]))
+ if ("False".equalsIgnoreCase(row[1]))
{
+ nonCell++;
continue;
}
- Map r = new HashMap<>();
- r.put("category", line[0]);
- r.put("metricname", line[1]);
-
- //NOTE: R saves NaN as NA. This is fixed in the R code, but add this check here to let existing jobs import
- String value = line[2];
- if ("NA".equals(value))
+ //NOTE: allow these to pass for cell-hashing under some conditions
+ boolean hasCDR3 = !"None".equals(row[12]);
+ if (!hasCDR3)
{
- value = "0";
+ noCallRows++;
}
- String fieldName = NumberUtils.isCreatable(value) ? "metricvalue" : "qualvalue";
- r.put(fieldName, value);
-
- r.put("analysis_id", so.getAnalysis_id());
- r.put("dataid", so.getDataId());
- r.put("readset", so.getReadset());
- r.put("container", job.getContainer());
- r.put("createdby", job.getUser().getUserId());
-
- Table.insert(job.getUser(), ti, r);
- total++;
-
- valueMap.put(line[1], value);
- }
-
- job.getLogger().info("total metrics: " + total);
-
- if (updateDescription)
- {
- job.getLogger().debug("Updating description");
- StringBuilder description = new StringBuilder();
- if (StringUtils.trimToNull(so.getDescription()) != null)
+ //NOTE: 10x appends "-1" to barcodes
+ String barcode = row[0].split("-")[0];
+ if ((allowCellsLackingCDR3 || hasCDR3) && !uniqueBarcodes.contains(barcode))
{
- description.append(StringUtils.trimToNull(so.getDescription()));
+ writer.writeNext(new String[]{barcode});
+ uniqueBarcodes.add(barcode);
+ totalBarcodeWritten++;
}
- String delim = description.length() > 0 ? "\n" : "";
-
- DecimalFormat fmt = new DecimalFormat("##.##%");
- for (String metricName : Arrays.asList("InputBarcodes", "TotalCalled", "TotalCounts", "TotalSinglet", "FractionOfInputCalled", "FractionOfInputSinglet", "FractionOfInputDoublet", "FractionOfInputDiscordant", "FractionCalledNotInInput", "SeuratNonNegative", "MultiSeqNonNegative", "UniqueHtos", "UnknownTagMatchingKnown"))
- {
- if (valueMap.get(metricName) != null)
- {
- Double d = null;
- if (metricName.startsWith("Fraction"))
- {
- try
- {
- d = ConvertHelper.convert(valueMap.get(metricName), Double.class);
- }
- catch (ConversionException | IllegalArgumentException e)
- {
- job.getLogger().error("Unable to convert to double: " + valueMap.get(metricName));
- throw e;
- }
- }
-
- description.append(delim).append(metricName).append(": ").append(d == null ? valueMap.get(metricName) : fmt.format(d));
- delim = ",\n";
- }
- }
+ uniqueBarcodesIncludingNoCDR3.add(barcode);
+ }
+ }
- so.setDescription(description.toString());
+ ctx.getLogger().debug("rows inspected: " + (rowIdx - 1));
+ ctx.getLogger().debug("rows without CDR3: " + noCallRows);
+ ctx.getLogger().debug("rows not called as cells: " + nonCell);
+ ctx.getLogger().debug("unique cell barcodes (with CDR3): " + uniqueBarcodes.size());
+ ctx.getLogger().debug("unique cell barcodes (including no CDR3): " + uniqueBarcodesIncludingNoCDR3.size());
+ ctx.getFileManager().addIntermediateFile(cellBarcodeWhitelist);
+ }
+ catch (IOException e)
+ {
+ throw new PipelineJobException(e);
+ }
- TableInfo tableOutputs = DbSchema.get("sequenceanalysis", DbSchemaType.Module).getTable("outputfiles");
- Table.update(job.getUser(), tableOutputs, so, so.getRowid());
+ if (uniqueBarcodes.size() < 500 && uniqueBarcodesIncludingNoCDR3.size() > uniqueBarcodes.size())
+ {
+ ctx.getLogger().info("Total cell barcodes with CDR3s is low, so cell hashing will be performing using an input that includes valid cells that lacked CDR3 data.");
+ try (CSVWriter writer = new CSVWriter(PrintWriters.getPrintWriter(cellBarcodeWhitelist), ',', CSVWriter.NO_QUOTE_CHARACTER))
+ {
+ for (String barcode : uniqueBarcodesIncludingNoCDR3)
+ {
+ writer.writeNext(new String[]{barcode});
+ totalBarcodeWritten++;
}
}
catch (IOException e)
@@ -338,14 +312,15 @@ public static void processMetrics(SequenceOutputFile so, PipelineJob job, boolea
throw new PipelineJobException(e);
}
}
- else
+
+ if (totalBarcodeWritten == 0)
{
- job.getLogger().warn("Unable to find metrics file: " + metrics.getPath());
+ throw new PipelineJobException("No valid cell barcodes found!");
}
- }
- else
- {
- job.getLogger().warn("Unable to update metrics, file id is null: " + so.getName());
+
+ //TODO: consider looking up GEX data?
+
+ return cellBarcodeWhitelist;
}
}
}
\ No newline at end of file
diff --git a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJUtils.java b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJUtils.java
index a71cf5bac..109117429 100644
--- a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJUtils.java
+++ b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJUtils.java
@@ -1,7 +1,6 @@
package org.labkey.tcrdb.pipeline;
import au.com.bytecode.opencsv.CSVReader;
-import au.com.bytecode.opencsv.CSVWriter;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.Logger;
import org.jetbrains.annotations.Nullable;
@@ -10,8 +9,6 @@
import org.labkey.api.assay.AssayProvider;
import org.labkey.api.assay.AssayService;
import org.labkey.api.collections.CaseInsensitiveHashMap;
-import org.labkey.api.data.ColumnInfo;
-import org.labkey.api.data.CompareType;
import org.labkey.api.data.Container;
import org.labkey.api.data.SimpleFilter;
import org.labkey.api.data.TableInfo;
@@ -29,24 +26,19 @@
import org.labkey.api.query.InvalidKeyException;
import org.labkey.api.query.QueryService;
import org.labkey.api.query.QueryUpdateServiceException;
-import org.labkey.api.query.UserSchema;
import org.labkey.api.query.ValidationException;
import org.labkey.api.reader.FastaDataLoader;
import org.labkey.api.reader.FastaLoader;
import org.labkey.api.reader.Readers;
import org.labkey.api.security.User;
-import org.labkey.api.sequenceanalysis.SequenceAnalysisService;
-import org.labkey.api.sequenceanalysis.SequenceOutputFile;
import org.labkey.api.sequenceanalysis.model.AnalysisModel;
-import org.labkey.api.sequenceanalysis.model.Readset;
-import org.labkey.api.sequenceanalysis.pipeline.PipelineStepOutput;
-import org.labkey.api.sequenceanalysis.pipeline.SequenceAnalysisJobSupport;
import org.labkey.api.sequenceanalysis.pipeline.SequencePipelineService;
+import org.labkey.api.singlecell.CellHashingService;
+import org.labkey.api.singlecell.model.CDNA_Library;
import org.labkey.api.util.FileUtil;
import org.labkey.api.util.PageFlowUtil;
import org.labkey.api.view.ViewBackgroundInfo;
import org.labkey.api.view.ViewContext;
-import org.labkey.api.writer.PrintWriters;
import org.labkey.tcrdb.TCRdbSchema;
import java.io.File;
@@ -56,420 +48,27 @@
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
-import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
public class CellRangerVDJUtils
{
- private Logger _log;
- private File _sourceDir;
+ public static final String TCR_HASHING_CALLS = "Cell Hashing TCR Calls";
- public static final String READSET_TO_HASHING_MAP = "readsetToHashingMap";
- public static final String READSET_TO_CITESEQ_MAP = "readsetToCiteSeqMap";
- private static final String HASHING_CALLS = "Cell Hashing TCR Calls";
+ private Logger _log;
- public CellRangerVDJUtils(Logger log, File sourceDir)
+ public CellRangerVDJUtils(Logger log)
{
_log = log;
- _sourceDir = sourceDir;
}
- public void prepareHashingAndCiteSeqFilesIfNeeded(PipelineJob job, SequenceAnalysisJobSupport support, String filterField, final boolean skipFailedCdna, boolean failIfNoHashing, boolean failIfNoCiteSeq) throws PipelineJobException
+ public void importAssayData(PipelineJob job, AnalysisModel model, File vLoupeFile, File outDir, Integer assayId, @Nullable Integer runId, boolean deleteExisting) throws PipelineJobException
{
- Container target = job.getContainer().isWorkbook() ? job.getContainer().getParent() : job.getContainer();
- UserSchema tcr = QueryService.get().getUserSchema(job.getUser(), target, TCRdbSchema.NAME);
- TableInfo cDNAs = tcr.getTable(TCRdbSchema.TABLE_CDNAS, null);
-
- _log.debug("preparing cDNA and cell hashing files");
-
- SequenceAnalysisService.get().writeAllCellHashingBarcodes(_sourceDir, job.getUser(), job.getContainer());
- SequenceAnalysisService.get().writeAllCiteSeqBarcodes(_sourceDir, job.getUser(), job.getContainer());
-
- Map colMap = QueryService.get().getColumns(cDNAs, PageFlowUtil.set(
- FieldKey.fromString("rowid"),
- FieldKey.fromString("sortId/stimId/animalId"),
- FieldKey.fromString("sortId/stimId/stim"),
- FieldKey.fromString("sortId/population"),
- FieldKey.fromString("sortId/hto"),
- FieldKey.fromString("sortId/hto/sequence"),
- FieldKey.fromString("hashingReadsetId"),
- FieldKey.fromString("hashingReadsetId/totalFiles"),
- FieldKey.fromString("citeseqReadsetId"),
- FieldKey.fromString("citeseqReadsetId/totalFiles"),
- FieldKey.fromString("citeseqPanel"),
- FieldKey.fromString("status"))
- );
-
- File output = getCDNAInfoFile();
- File barcodeOutput = getValidHashingBarcodeFile();
- HashMap readsetToHashingMap = new HashMap<>();
- HashMap readsetToCiteSeqMap = new HashMap<>();
- HashMap> gexToPanels = new HashMap<>();
-
- try (CSVWriter writer = new CSVWriter(PrintWriters.getPrintWriter(output), '\t', CSVWriter.NO_QUOTE_CHARACTER); CSVWriter bcWriter = new CSVWriter(PrintWriters.getPrintWriter(barcodeOutput), ',', CSVWriter.NO_QUOTE_CHARACTER))
- {
- writer.writeNext(new String[]{"ReadsetId", "CDNA_ID", "AnimalId", "Stim", "Population", "HashingReadsetId", "HasHashingReads", "HTO_Name", "HTO_Seq", "CiteSeqReadsetId", "HasCiteSeqReads", "CiteSeqPanel"});
- List cachedReadsets = support.getCachedReadsets();
- Set distinctHTOs = new HashSet<>();
- Set hashingStatus = new HashSet<>();
- Set citeseqStatus = new HashSet<>();
- AtomicInteger totalWritten = new AtomicInteger(0);
- for (Readset rs : cachedReadsets)
- {
- AtomicBoolean hasError = new AtomicBoolean(false);
- //find cDNA records using this readset
- new TableSelector(cDNAs, colMap.values(), new SimpleFilter(FieldKey.fromString(filterField), rs.getRowId()), null).forEachResults(results -> {
- if (skipFailedCdna && results.getObject(FieldKey.fromString("status")) != null)
- {
- _log.info("skipping cDNA with non-null status: " + results.getString(FieldKey.fromString("rowid")));
- return;
- }
-
- writer.writeNext(new String[]{
- String.valueOf(rs.getRowId()),
- results.getString(FieldKey.fromString("rowid")),
- results.getString(FieldKey.fromString("sortId/stimId/animalId")),
- results.getString(FieldKey.fromString("sortId/stimId/stim")),
- results.getString(FieldKey.fromString("sortId/population")),
- String.valueOf(results.getObject(FieldKey.fromString("hashingReadsetId")) == null ? "" : results.getInt(FieldKey.fromString("hashingReadsetId"))),
- String.valueOf(results.getObject(FieldKey.fromString("hashingReadsetId/totalFiles")) != null && results.getInt(FieldKey.fromString("hashingReadsetId/totalFiles")) > 0),
- results.getString(FieldKey.fromString("sortId/hto")),
- results.getString(FieldKey.fromString("sortId/hto/sequence")),
- String.valueOf(results.getObject(FieldKey.fromString("citeseqReadsetId")) == null ? "" : results.getInt(FieldKey.fromString("citeseqReadsetId"))),
- String.valueOf(results.getObject(FieldKey.fromString("citeseqReadsetId/totalFiles")) != null && results.getInt(FieldKey.fromString("citeseqReadsetId/totalFiles")) > 0),
- results.getString(FieldKey.fromString("citeseqPanel"))
- });
- totalWritten.getAndIncrement();
-
- boolean useCellHashing = results.getObject(FieldKey.fromString("sortId/hto")) != null;
- hashingStatus.add(useCellHashing);
- if (useCellHashing)
- {
- if (results.getObject(FieldKey.fromString("hashingReadsetId")) == null)
- {
- job.getLogger().error("cDNA specifies HTO, but does not list a hashing readset: " + results.getString(FieldKey.fromString("rowid")));
- hasError.set(true);
- }
- else
- {
- readsetToHashingMap.put(rs.getReadsetId(), results.getInt(FieldKey.fromString("hashingReadsetId")));
-
- String hto = results.getString(FieldKey.fromString("sortId/hto")) + "<>" + results.getString(FieldKey.fromString("sortId/hto/sequence"));
- if (!distinctHTOs.contains(hto) && !StringUtils.isEmpty(results.getString(FieldKey.fromString("sortId/hto/sequence"))))
- {
- distinctHTOs.add(hto);
- bcWriter.writeNext(new String[]{results.getString(FieldKey.fromString("sortId/hto/sequence")), results.getString(FieldKey.fromString("sortId/hto"))});
- }
-
- if (results.getObject(FieldKey.fromString("sortId/hto/sequence")) == null)
- {
- job.getLogger().error("Unable to find sequence for HTO: " + results.getString(FieldKey.fromString("sortId/hto")));
- hasError.set(true);
- }
- }
- }
-
- boolean useCiteSeq = results.getObject(FieldKey.fromString("citeseqPanel")) != null;
- citeseqStatus.add(useCiteSeq);
- if (useCiteSeq)
- {
- if (results.getObject(FieldKey.fromString("citeseqReadsetId")) == null)
- {
- job.getLogger().error("cDNA specifies cite-seq readset but does not list panel: " + results.getString(FieldKey.fromString("rowid")));
- hasError.set(true);
- }
- else
- {
- Set panels = gexToPanels.getOrDefault(rs.getRowId(), new HashSet<>());
- panels.add(results.getString(FieldKey.fromString("citeseqPanel")));
- gexToPanels.put(rs.getRowId(), panels);
-
- readsetToCiteSeqMap.put(rs.getReadsetId(), results.getInt(FieldKey.fromString("citeseqReadsetId")));
- }
- }
- });
-
- if (hasError.get())
- {
- throw new PipelineJobException("No cell hashing readset or HTO found for one or more cDNAs. see the file: " + output.getName());
- }
-
- if (hashingStatus.size() > 1)
- {
- _log.info("The selected readsets/cDNA records use a mixture of cell hashing and non-hashing.");
- }
-
- //NOTE: hashingStatus.isEmpty() indicates there are no cDNA records associated with the data
- }
-
- // if distinct HTOs is 1, no point in running hashing. note: presence of hashing readsets is a trigger downstream
- if (distinctHTOs.size() > 1)
- {
- readsetToHashingMap.forEach((readsetId, hashingReadsetId) -> support.cacheReadset(hashingReadsetId, job.getUser()));
- }
- else if (distinctHTOs.size() == 1)
- {
- job.getLogger().info("There is only a single HTO in this pool, will not use hashing");
- }
-
- if (totalWritten.get() == 0)
- {
- throw new PipelineJobException("No matching cDNA records found");
- }
-
- boolean useCellHashing = hashingStatus.isEmpty() ? false : hashingStatus.size() > 1 ? true : hashingStatus.iterator().next();
- if (useCellHashing && distinctHTOs.isEmpty())
- {
- throw new PipelineJobException("Cell hashing was selected, but no HTOs were found");
- }
- else
- {
- _log.info("distinct HTOs: " + distinctHTOs.size());
- }
-
- support.cacheObject(READSET_TO_HASHING_MAP, readsetToHashingMap);
- support.cacheObject(READSET_TO_CITESEQ_MAP, readsetToCiteSeqMap);
- readsetToCiteSeqMap.forEach((readsetId, citeseqReadsetId) -> support.cacheReadset(citeseqReadsetId, job.getUser()));
- }
- catch (IOException e)
- {
- throw new PipelineJobException(e);
- }
-
- writeCiteSeqBarcodes(job, gexToPanels, _sourceDir);
-
- if (failIfNoHashing && readsetToHashingMap.isEmpty())
- {
- throw new PipelineJobException("Readsets do not use cell hashing");
- }
-
- if (failIfNoCiteSeq && readsetToCiteSeqMap.isEmpty())
- {
- throw new PipelineJobException("Readsets do not use CITE-seq");
- }
- }
-
- public static File getValidCiteSeqBarcodeFile(File sourceDir, int gexReadsetId)
- {
- return new File(sourceDir, "validADTS." + gexReadsetId + ".csv");
- }
-
- public static File getValidCiteSeqBarcodeMetadataFile(File sourceDir, int gexReadsetId)
- {
- return new File(sourceDir, "validADTS." + gexReadsetId + ".metadata.txt");
- }
-
- private void writeCiteSeqBarcodes(PipelineJob job, Map> gexToPanels, File outputDir) throws PipelineJobException
- {
- Container target = job.getContainer().isWorkbook() ? job.getContainer().getParent() : job.getContainer();
- UserSchema tcr = QueryService.get().getUserSchema(job.getUser(), target, TCRdbSchema.NAME);
- TableInfo panels = tcr.getTable(TCRdbSchema.TABLE_CITE_SEQ_PANELS, null);
-
- Map barcodeColMap = QueryService.get().getColumns(panels, PageFlowUtil.set(
- FieldKey.fromString("antibody"),
- FieldKey.fromString("antibody/markerName"),
- FieldKey.fromString("antibody/markerLabel"),
- FieldKey.fromString("markerLabel"),
- FieldKey.fromString("antibody/adaptersequence")
- ));
-
- for (int gexReadsetId : gexToPanels.keySet())
- {
- job.getLogger().info("Writing all unique ADTs for readset: " + gexReadsetId);
- File barcodeOutput = getValidCiteSeqBarcodeFile(outputDir, gexReadsetId);
- File metadataOutput = getValidCiteSeqBarcodeMetadataFile(outputDir, gexReadsetId);
- try (CSVWriter writer = new CSVWriter(PrintWriters.getPrintWriter(barcodeOutput), ',', CSVWriter.NO_QUOTE_CHARACTER);CSVWriter metaWriter = new CSVWriter(PrintWriters.getPrintWriter(metadataOutput), '\t', CSVWriter.NO_QUOTE_CHARACTER))
- {
- metaWriter.writeNext(new String[]{"tagname", "sequence", "markername", "markerlabel"});
- AtomicInteger barcodeCount = new AtomicInteger();
- Set found = new HashSet<>();
- new TableSelector(panels, barcodeColMap.values(), new SimpleFilter(FieldKey.fromString("name"), gexToPanels.get(gexReadsetId), CompareType.IN), new org.labkey.api.data.Sort("antibody")).forEachResults(results -> {
- if (found.contains(results.getString(FieldKey.fromString("antibody/adaptersequence"))))
- {
- return;
- }
-
- found.add(results.getString(FieldKey.fromString("antibody/adaptersequence")));
- barcodeCount.getAndIncrement();
-
- writer.writeNext(new String[]{results.getString(FieldKey.fromString("antibody/adaptersequence")), results.getString(FieldKey.fromString("antibody"))});
-
- //allow aliasing based on DB
- String label = StringUtils.trimToNull(results.getString(FieldKey.fromString("markerLabel"))) == null ? results.getString(FieldKey.fromString("antibody/markerLabel")) : results.getString(FieldKey.fromString("markerLabel"));
- String name = StringUtils.trimToNull(results.getString(FieldKey.fromString("markerLabel"))) != null ? results.getString(FieldKey.fromString("markerLabel")) :
- StringUtils.trimToNull(results.getString(FieldKey.fromString("antibody/markerName"))) != null ? results.getString(FieldKey.fromString("antibody/markerName")) : results.getString(FieldKey.fromString("antibody"));
- metaWriter.writeNext(new String[]{results.getString(FieldKey.fromString("antibody")), results.getString(FieldKey.fromString("antibody/adaptersequence")), name, label});
- });
-
- job.getLogger().info("Total CITE-seq barcodes written: " + barcodeCount.get());
- }
- catch (IOException e)
- {
- throw new PipelineJobException(e);
- }
- }
- }
-
- public File getCDNAInfoFile()
- {
- return getCDNAInfoFile(_sourceDir);
- }
+ File cellRangerOutDir = vLoupeFile.getParentFile();
- public static File getCDNAInfoFile(File sourceDir)
- {
- return new File(sourceDir, "cDNAInfo.txt");
- }
-
- public File getValidHashingBarcodeFile()
- {
- return getValidHashingBarcodeFile(_sourceDir);
- }
-
- public static File getValidHashingBarcodeFile(File sourceDir)
- {
- return new File(sourceDir, "validHashingBarcodes.csv");
- }
-
- public File getValidCellIndexFile()
- {
- return new File(_sourceDir, "validCellIndexes.csv");
- }
-
- public File getPerCellCsv(File outDir)
- {
- return new File(outDir, "all_contig_annotations.csv");
- }
-
- public File runRemoteVdjCellHashingTasks(PipelineStepOutput output, String outputCategory, File perCellTsv, Readset rs, SequenceAnalysisJobSupport support, List extraParams, File workingDir, File sourceDir, Integer editDistance, boolean scanEditDistances, Integer genomeId, Integer minCountPerCell, boolean useSeurat, boolean useMultiSeq) throws PipelineJobException
- {
- Map readsetToHashing = getCachedHashingReadsetMap(support);
- if (readsetToHashing.isEmpty())
- {
- _log.info("No cached hashing readsets, skipping");
- return null;
- }
-
- //prepare whitelist of barcodes, based on cDNA records
- File htoBarcodeWhitelist = getValidHashingBarcodeFile();
- if (!htoBarcodeWhitelist.exists())
- {
- throw new PipelineJobException("Unable to find file: " + htoBarcodeWhitelist.getPath());
- }
-
- long lineCount = SequencePipelineService.get().getLineCount(htoBarcodeWhitelist);
- if (lineCount == 1)
- {
- _log.info("Only one HTO is used, will not use hashing");
- return null;
- }
-
- _log.debug("total cached readset/hashing readset pairs: " + readsetToHashing.size());
- _log.debug("unique HTOs: " + lineCount);
-
- //prepare whitelist of cell indexes
- File cellBarcodeWhitelist = getValidCellIndexFile();
- Set uniqueBarcodes = new HashSet<>();
- Set uniqueBarcodesIncludingNoCDR3 = new HashSet<>();
- _log.debug("writing cell barcodes");
- try (CSVWriter writer = new CSVWriter(PrintWriters.getPrintWriter(cellBarcodeWhitelist), ',', CSVWriter.NO_QUOTE_CHARACTER); CSVReader reader = new CSVReader(Readers.getReader(perCellTsv), ','))
- {
- int rowIdx = 0;
- int noCallRows = 0;
- int nonCell = 0;
- String[] row;
- while ((row = reader.readNext()) != null)
- {
- //skip header
- rowIdx++;
- if (rowIdx > 1)
- {
- if ("False".equalsIgnoreCase(row[1]))
- {
- nonCell++;
- continue;
- }
-
- //NOTE: allow these to pass for cell-hashing under some conditions
- boolean hasCDR3 = !"None".equals(row[12]);
- if (!hasCDR3)
- {
- noCallRows++;
- }
-
- //NOTE: 10x appends "-1" to barcodes
- String barcode = row[0].split("-")[0];
- if (hasCDR3 && !uniqueBarcodes.contains(barcode))
- {
- writer.writeNext(new String[]{barcode});
- uniqueBarcodes.add(barcode);
- }
-
- uniqueBarcodesIncludingNoCDR3.add(barcode);
- }
- }
-
- _log.debug("rows inspected: " + (rowIdx - 1));
- _log.debug("rows without CDR3: " + noCallRows);
- _log.debug("rows not called as cells: " + nonCell);
- _log.debug("unique cell barcodes (with CDR3): " + uniqueBarcodes.size());
- _log.debug("unique cell barcodes (including no CDR3): " + uniqueBarcodesIncludingNoCDR3.size());
- output.addIntermediateFile(cellBarcodeWhitelist);
- }
- catch (IOException e)
- {
- throw new PipelineJobException(e);
- }
-
- if (uniqueBarcodes.size() < 500 && uniqueBarcodesIncludingNoCDR3.size() > uniqueBarcodes.size())
- {
- _log.info("Total cell barcodes with CDR3s is low, so cell hashing will be performing using an input that includes valid cells that lacked CDR3 data.");
- try (CSVWriter writer = new CSVWriter(PrintWriters.getPrintWriter(cellBarcodeWhitelist), ',', CSVWriter.NO_QUOTE_CHARACTER))
- {
- for (String barcode : uniqueBarcodesIncludingNoCDR3)
- {
- writer.writeNext(new String[]{barcode});
- }
- }
- catch (IOException e)
- {
- throw new PipelineJobException(e);
- }
- }
-
- Readset htoReadset = support.getCachedReadset(readsetToHashing.get(rs.getReadsetId()));
- if (htoReadset == null)
- {
- throw new PipelineJobException("Unable to find HTO readset for readset: " + rs.getRowId());
- }
-
- //run CiteSeqCount. this will use Multiseq to make calls per cell
- String basename = FileUtil.makeLegalName(rs.getName());
- File hashtagCalls = SequencePipelineService.get().runCiteSeqCount(output, outputCategory, htoReadset, htoBarcodeWhitelist, cellBarcodeWhitelist, workingDir, basename, _log, extraParams, false, minCountPerCell, sourceDir, editDistance, scanEditDistances, rs, genomeId, true, true, useSeurat, useMultiSeq);
- if (!hashtagCalls.exists())
- {
- throw new PipelineJobException("Unable to find expected file: " + hashtagCalls.getPath());
- }
- output.addOutput(hashtagCalls, HASHING_CALLS);
-
- File html = new File(hashtagCalls.getParentFile(), FileUtil.getBaseName(FileUtil.getBaseName(hashtagCalls.getName())) + ".html");
- if (!html.exists())
- {
- throw new PipelineJobException("Unable to find HTML file: " + html.getPath());
- }
-
- output.addOutput(html, "Cell Hashing TCR Report");
-
- return hashtagCalls;
- }
-
- public void importAssayData(PipelineJob job, AnalysisModel model, File outDir, Integer assayId, @Nullable Integer runId, boolean deleteExisting) throws PipelineJobException
- {
if (assayId == null)
{
_log.info("No assay selected, will not import");
@@ -482,32 +81,29 @@ public void importAssayData(PipelineJob job, AnalysisModel model, File outDir, I
throw new PipelineJobException("Unable to find protocol: " + assayId);
}
- File allCsv = getPerCellCsv(outDir);
+ File allCsv = getPerCellCsv(cellRangerOutDir);
if (!allCsv.exists())
{
_log.warn("unable to find consensus contigs: " + allCsv .getPath());
return;
}
- File consensusCsv = new File(outDir, "consensus_annotations.csv");
+ File consensusCsv = new File(cellRangerOutDir, "consensus_annotations.csv");
if (!consensusCsv .exists())
{
- _log.warn("unable to find consensus contigs: " + consensusCsv .getPath());
- return;
+ throw new PipelineJobException("unable to find consensus contigs: " + consensusCsv .getPath());
}
- File consensusFasta = new File(outDir, "consensus.fasta");
+ File consensusFasta = new File(cellRangerOutDir, "consensus.fasta");
if (!consensusFasta.exists())
{
- _log.warn("unable to find FASTA: " + consensusFasta.getPath());
- return;
+ throw new PipelineJobException("unable to find FASTA: " + consensusFasta.getPath());
}
- File allFasta = new File(outDir, "all_contig.fasta");
+ File allFasta = new File(cellRangerOutDir, "all_contig.fasta");
if (!allFasta.exists())
{
- _log.warn("unable to find FASTA: " + allFasta.getPath());
- return;
+ throw new PipelineJobException("unable to find FASTA: " + allFasta.getPath());
}
_log.info("loading results into assay: " + assayId);
@@ -521,9 +117,9 @@ public void importAssayData(PipelineJob job, AnalysisModel model, File outDir, I
job.getLogger().debug("Using supplied runId: " + runId);
}
- File cDNAFile = getCDNAInfoFile();
- Map htoNameToCDNAMap = new HashMap<>();
- Map cDNAMap = new HashMap<>();
+ File cDNAFile = CellHashingService.get().getCDNAInfoFile(outDir);
+ Map htoNameToCDNAMap = new HashMap<>();
+ Map cDNAMap = new HashMap<>();
if (cDNAFile.exists())
{
try (CSVReader reader = new CSVReader(Readers.getReader(cDNAFile), '\t'))
@@ -539,7 +135,7 @@ public void importAssayData(PipelineJob job, AnalysisModel model, File outDir, I
String htoName = StringUtils.trimToNull(line[7]);
- CDNA cdna = CDNA.getRowId(Integer.parseInt(line[1]));
+ CDNA_Library cdna = CellHashingService.get().getLibraryById(Integer.parseInt(line[1]));
cDNAMap.put(Integer.parseInt(line[1]), cdna);
if (htoName != null)
{
@@ -600,15 +196,28 @@ public void importAssayData(PipelineJob job, AnalysisModel model, File outDir, I
int doublet = 0;
int discordant = 0;
int negative = 0;
+
+ int consensusIdx = -1;
while ((line = reader.readNext()) != null)
{
+ if (line.length < 3)
+ {
+ throw new PipelineJobException("Line too short");
+ }
+
//header
- if ("CellBarcode".equals(line[0]))
+ if ("cellbarcode".equalsIgnoreCase(line[0]))
{
+ consensusIdx = Arrays.asList(line).indexOf("consensuscall");
continue;
}
- String hto = line[1];
+ if (consensusIdx == -1)
+ {
+ throw new PipelineJobException("consensuscall column not found");
+ }
+
+ String hto = line[consensusIdx];
if ("Doublet".equals(hto))
{
doublet++;
@@ -627,7 +236,7 @@ else if ("Negative".equals(hto))
continue;
}
- CDNA cDNA = htoNameToCDNAMap.get(hto);
+ CDNA_Library cDNA = htoNameToCDNAMap.get(hto);
if (cDNA == null)
{
_log.warn("Unable to find cDNA record for hto: " + hto);
@@ -696,10 +305,15 @@ else if ("Negative".equals(hto))
continue;
}
- if ("None".equals(line[9]))
+ String cGene = removeNone(line[9]);
+ if (cGene == null)
{
- noCGene++;
- continue;
+ // Only discard these if chain type doesnt match between JGene and VGene.
+ if (!line[8].substring(0, 3).equals(line[6].substring(0,3)))
+ {
+ noCGene++;
+ continue;
+ }
}
if ("False".equals(line[10]))
@@ -747,9 +361,9 @@ else if (discordantBarcodes.contains(barcode))
//NOTE: chimeras with a TRDV / TRAJ / TRAC are relatively common. categorize as TRA for reporting ease
String locus = line[5];
- if (locus.equals("Multi") && removeNone(line[9]) != null && removeNone(line[8]) != null && removeNone(line[6]) != null)
+ if (locus.equals("Multi") && cGene != null && removeNone(line[8]) != null && removeNone(line[6]) != null)
{
- if (removeNone(line[9]).contains("TRAC") && removeNone(line[8]).contains("TRAJ") && removeNone(line[6]).contains("TRDV"))
+ if (cGene.contains("TRAC") && removeNone(line[8]).contains("TRAJ") && removeNone(line[6]).contains("TRDV"))
{
locus = "TRA";
multiChainConverted++;
@@ -757,7 +371,7 @@ else if (discordantBarcodes.contains(barcode))
}
// Aggregate by: cDNA_ID, cdr3, chain, raw_clonotype_id, sequenceContigName, vHit, dHit, jHit, cHit, cdr3_nt
- String key = StringUtils.join(new String[]{cDNA.toString(), line[12], locus, clonotypeId, sequenceContigName, removeNone(line[6]), removeNone(line[7]), removeNone(line[8]), removeNone(line[9]), removeNone(line[13])}, "<>");
+ String key = StringUtils.join(new String[]{cDNA.toString(), line[12], locus, clonotypeId, sequenceContigName, removeNone(line[6]), removeNone(line[7]), removeNone(line[8]), cGene, removeNone(line[13])}, "<>");
AssayModel am;
if (!rows.containsKey(key))
{
@@ -865,7 +479,7 @@ private AssayModel createForRow(String[] line, String sequenceContigName, Intege
private File getCellToHtoFile(ExpRun run) throws PipelineJobException
{
- List extends ExpData> datas = run.getInputDatas(HASHING_CALLS, ExpProtocol.ApplicationType.ExperimentRunOutput);
+ List extends ExpData> datas = run.getInputDatas(TCR_HASHING_CALLS, ExpProtocol.ApplicationType.ExperimentRunOutput);
if (datas.isEmpty())
{
throw new PipelineJobException("Unable to find hashing calls output");
@@ -901,9 +515,9 @@ private static class AssayModel
private String sequenceContigName;
}
- private Map processRow(AssayModel assayModel, AnalysisModel model, Map cDNAMap, Integer runId, Map> totalCellsBySample, Map sequenceMap) throws PipelineJobException
+ private Map processRow(AssayModel assayModel, AnalysisModel model, Map cDNAMap, Integer runId, Map> totalCellsBySample, Map sequenceMap) throws PipelineJobException
{
- CDNA cDNARecord = cDNAMap.get(assayModel.cdna);
+ CDNA_Library cDNARecord = cDNAMap.get(assayModel.cdna);
if (cDNARecord == null)
{
throw new PipelineJobException("Unable to find cDNA for ID: " + assayModel.cdna);
@@ -912,8 +526,8 @@ private Map processRow(AssayModel assayModel, AnalysisModel mode
Map row = new CaseInsensitiveHashMap<>();
row.put("sampleName", cDNARecord.getAssaySampleName());
- row.put("subjectId", cDNARecord.getSortRecord().getStimRecord().getAnimalId());
- row.put("sampleDate", cDNARecord.getSortRecord().getStimRecord().getDate());
+ row.put("subjectId", cDNARecord.getSortRecord().getSampleRecord().getSubjectId());
+ row.put("sampleDate", cDNARecord.getSortRecord().getSampleRecord().getSampledate());
row.put("cDNA", assayModel.cdna);
row.put("alignmentId", model.getAlignmentFile());
@@ -963,7 +577,7 @@ private void saveRun(PipelineJob job, ExpProtocol protocol, AnalysisModel model,
if (model.getLibraryId() != null)
{
- TableSelector ts = new TableSelector(TCRdbSchema.getInstance().getSchema().getTable(TCRdbSchema.TABLE_LIBRARIES), PageFlowUtil.set("rowid"), new SimpleFilter(FieldKey.fromString("libraryId"), model.getLibraryId()), null);
+ TableSelector ts = new TableSelector(TCRdbSchema.getInstance().getSchema().getTable(TCRdbSchema.TABLE_MIXCR_LIBRARIES), PageFlowUtil.set("rowid"), new SimpleFilter(FieldKey.fromString("libraryId"), model.getLibraryId()), null);
if (ts.exists())
{
int mixcrId = ts.getObject(Integer.class);
@@ -1052,307 +666,8 @@ public static void deleteExistingData(AssayProvider ap, ExpProtocol protocol, Co
}
}
- public static Map getCachedHashingReadsetMap(SequenceAnalysisJobSupport support) throws PipelineJobException
- {
- return support.getCachedObject(CellRangerVDJUtils.READSET_TO_HASHING_MAP, PipelineJob.createObjectMapper().getTypeFactory().constructParametricType(Map.class, Integer.class, Integer.class));
- }
-
- public static Map getCachedCiteSeqReadsetMap(SequenceAnalysisJobSupport support) throws PipelineJobException
- {
- return support.getCachedObject(CellRangerVDJUtils.READSET_TO_CITESEQ_MAP, PipelineJob.createObjectMapper().getTypeFactory().constructParametricType(Map.class, Integer.class, Integer.class));
- }
-
- //NOTE: if readset ID is null, this will be interpreted as any readset using hashing
- public boolean useCellHashing(SequenceAnalysisJobSupport support) throws PipelineJobException
- {
- Map gexToHashingMap = getCachedHashingReadsetMap(support);
- if (gexToHashingMap == null || gexToHashingMap.isEmpty())
- return false;
-
- File htoBarcodeWhitelist = getValidHashingBarcodeFile();
- if (!htoBarcodeWhitelist.exists())
- {
- throw new PipelineJobException("Unable to find file: " + htoBarcodeWhitelist.getPath());
- }
-
- return SequencePipelineService.get().getLineCount(htoBarcodeWhitelist) > 1;
- }
-
- //NOTE: if readset ID is null, this will be interpreted as any readset using hashing
- public boolean useCiteSeq(SequenceAnalysisJobSupport support, List inputFiles) throws PipelineJobException
+ public static File getPerCellCsv(File cellRangerOutDir)
{
- Map gexToCiteMap = getCachedCiteSeqReadsetMap(support);
- if (gexToCiteMap == null || gexToCiteMap.isEmpty())
- return false;
-
- for (SequenceOutputFile so : inputFiles)
- {
- if (gexToCiteMap.containsKey(so.getReadset()))
- {
- return true;
- }
- }
-
- return false;
- }
-
- public static class CDNA
- {
- private int _rowId;
- private Integer _sortId;
- private String _chemistry;
- private Double _concentration;
- private String _plateId;
- private String _well;
-
- private Integer _readsetId;
- private Integer _enrichedReadsetId;
- private Integer _hashingReadsetId;
- private String _container;
-
- private Sort _sortRecord;
-
- public int getRowId()
- {
- return _rowId;
- }
-
- public void setRowId(int rowId)
- {
- _rowId = rowId;
- }
-
- public Integer getSortId()
- {
- return _sortId;
- }
-
- public void setSortId(Integer sortId)
- {
- _sortId = sortId;
- }
-
- public String getChemistry()
- {
- return _chemistry;
- }
-
- public void setChemistry(String chemistry)
- {
- _chemistry = chemistry;
- }
-
- public Double getConcentration()
- {
- return _concentration;
- }
-
- public void setConcentration(Double concentration)
- {
- _concentration = concentration;
- }
-
- public String getPlateId()
- {
- return _plateId;
- }
-
- public void setPlateId(String plateId)
- {
- _plateId = plateId;
- }
-
- public String getWell()
- {
- return _well;
- }
-
- public void setWell(String well)
- {
- _well = well;
- }
-
- public Integer getReadsetId()
- {
- return _readsetId;
- }
-
- public void setReadsetId(Integer readsetId)
- {
- _readsetId = readsetId;
- }
-
- public Integer getEnrichedReadsetId()
- {
- return _enrichedReadsetId;
- }
-
- public void setEnrichedReadsetId(Integer enrichedReadsetId)
- {
- _enrichedReadsetId = enrichedReadsetId;
- }
-
- public Integer getHashingReadsetId()
- {
- return _hashingReadsetId;
- }
-
- public void setHashingReadsetId(Integer hashingReadsetId)
- {
- _hashingReadsetId = hashingReadsetId;
- }
-
- public String getContainer()
- {
- return _container;
- }
-
- public void setContainer(String container)
- {
- _container = container;
- }
-
- public Sort getSortRecord()
- {
- if (_sortRecord == null)
- {
- _sortRecord = Sort.getRowId(_sortId);
- }
-
- return _sortRecord;
- }
-
- public String getAssaySampleName()
- {
- return getPlateId() + "_" + getWell() + "_" + getSortRecord().getStimRecord().getAnimalId() + "_" + getSortRecord().getStimRecord().getStim() + "_" + getSortRecord().getPopulation() + (getSortRecord().getHto() == null ? "" : "_" + getSortRecord().getHto());
- }
-
- public static CDNA getRowId(int rowId)
- {
- return new TableSelector(TCRdbSchema.getInstance().getSchema().getTable(TCRdbSchema.TABLE_CDNAS)).getObject(rowId, CDNA.class);
- }
- }
-
- public static class Sort
- {
- private int _rowId;
- private Integer _stimId;
- private String _population;
- private String _hto;
-
- private Stim _stimRecord;
-
- public Stim getStimRecord()
- {
- if (_stimRecord == null)
- {
- _stimRecord = Stim.getRowId(_stimId);
- }
-
- return _stimRecord;
- }
-
- public int getRowId()
- {
- return _rowId;
- }
-
- public void setRowId(int rowId)
- {
- _rowId = rowId;
- }
-
- public Integer getStimId()
- {
- return _stimId;
- }
-
- public void setStimId(Integer stimId)
- {
- _stimId = stimId;
- }
-
- public String getPopulation()
- {
- return _population;
- }
-
- public void setPopulation(String population)
- {
- _population = population;
- }
-
- public String getHto()
- {
- return _hto;
- }
-
- public void setHto(String hto)
- {
- _hto = hto;
- }
-
- public void setStimRecord(Stim stimRecord)
- {
- _stimRecord = stimRecord;
- }
-
- public static Sort getRowId(int rowId)
- {
- return new TableSelector(TCRdbSchema.getInstance().getSchema().getTable(TCRdbSchema.TABLE_SORTS)).getObject(rowId, Sort.class);
- }
- }
-
- public static class Stim
- {
- private int _rowId;
- private String _animalId;
- private String _stim;
- private Date _date;
-
- public int getRowId()
- {
- return _rowId;
- }
-
- public void setRowId(int rowId)
- {
- _rowId = rowId;
- }
-
- public String getAnimalId()
- {
- return _animalId;
- }
-
- public void setAnimalId(String animalId)
- {
- _animalId = animalId;
- }
-
- public String getStim()
- {
- return _stim;
- }
-
- public void setStim(String stim)
- {
- _stim = stim;
- }
-
- public Date getDate()
- {
- return _date;
- }
-
- public void setDate(Date date)
- {
- _date = date;
- }
-
- public static Stim getRowId(int rowId)
- {
- return new TableSelector(TCRdbSchema.getInstance().getSchema().getTable(TCRdbSchema.TABLE_STIMS)).getObject(rowId, Stim.class);
- }
+ return new File(cellRangerOutDir, "all_contig_annotations.csv");
}
}
diff --git a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJWrapper.java b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJWrapper.java
deleted file mode 100644
index 1fc0aaf0d..000000000
--- a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJWrapper.java
+++ /dev/null
@@ -1,693 +0,0 @@
-package org.labkey.tcrdb.pipeline;
-
-import au.com.bytecode.opencsv.CSVReader;
-import org.apache.commons.io.FileUtils;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.logging.log4j.Logger;
-import org.jetbrains.annotations.Nullable;
-import org.json.JSONObject;
-import org.labkey.api.collections.CaseInsensitiveHashMap;
-import org.labkey.api.data.CompareType;
-import org.labkey.api.data.ConvertHelper;
-import org.labkey.api.data.DbSchema;
-import org.labkey.api.data.DbSchemaType;
-import org.labkey.api.data.SimpleFilter;
-import org.labkey.api.data.Table;
-import org.labkey.api.data.TableInfo;
-import org.labkey.api.data.TableSelector;
-import org.labkey.api.pipeline.PipelineJob;
-import org.labkey.api.pipeline.PipelineJobException;
-import org.labkey.api.query.FieldKey;
-import org.labkey.api.query.QueryService;
-import org.labkey.api.query.UserSchema;
-import org.labkey.api.reader.Readers;
-import org.labkey.api.sequenceanalysis.RefNtSequenceModel;
-import org.labkey.api.sequenceanalysis.model.AnalysisModel;
-import org.labkey.api.sequenceanalysis.model.ReadData;
-import org.labkey.api.sequenceanalysis.model.Readset;
-import org.labkey.api.sequenceanalysis.pipeline.AbstractAlignmentStepProvider;
-import org.labkey.api.sequenceanalysis.pipeline.AlignerIndexUtil;
-import org.labkey.api.sequenceanalysis.pipeline.AlignmentOutputImpl;
-import org.labkey.api.sequenceanalysis.pipeline.AlignmentStep;
-import org.labkey.api.sequenceanalysis.pipeline.AlignmentStepProvider;
-import org.labkey.api.sequenceanalysis.pipeline.CommandLineParam;
-import org.labkey.api.sequenceanalysis.pipeline.IndexOutputImpl;
-import org.labkey.api.sequenceanalysis.pipeline.PipelineContext;
-import org.labkey.api.sequenceanalysis.pipeline.ReferenceGenome;
-import org.labkey.api.sequenceanalysis.pipeline.SequenceAnalysisJobSupport;
-import org.labkey.api.sequenceanalysis.pipeline.SequencePipelineService;
-import org.labkey.api.sequenceanalysis.pipeline.ToolParameterDescriptor;
-import org.labkey.api.sequenceanalysis.run.AbstractAlignmentPipelineStep;
-import org.labkey.api.sequenceanalysis.run.AbstractCommandWrapper;
-import org.labkey.api.sequenceanalysis.run.SimpleScriptWrapper;
-import org.labkey.api.util.FileUtil;
-import org.labkey.api.util.PageFlowUtil;
-import org.labkey.api.writer.PrintWriters;
-
-import java.io.File;
-import java.io.IOException;
-import java.io.PrintWriter;
-import java.nio.file.Files;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Date;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-public class CellRangerVDJWrapper extends AbstractCommandWrapper
-{
- public CellRangerVDJWrapper(@Nullable Logger logger)
- {
- super(logger);
- }
-
- public static final String TARGET_ASSAY = "targetAssay";
- public static final String DELETE_EXISTING_ASSAY_DATA = "deleteExistingAssayData";
-
- public static class VDJProvider extends AbstractAlignmentStepProvider
- {
- public VDJProvider()
- {
- super("CellRanger VDJ", "Cell Ranger is an alignment/analysis pipeline specific to 10x genomic data, and this can only be used on fastqs generated by 10x.", Arrays.asList(
- //--sample
-
- ToolParameterDescriptor.create("id", "Run ID Suffix", "If provided, this will be appended to the ID of this run (readset name will be first).", "textfield", new JSONObject(){{
- put("allowBlank", true);
- }}, null),
- ToolParameterDescriptor.createCommandLineParam(CommandLineParam.create("--force-cells"), "force-cells", "Force Cells", "Force pipeline to use this number of cells, bypassing the cell detection algorithm. Use this if the number of cells estimated by Cell Ranger is not consistent with the barcode rank plot.", "ldk-integerfield", new JSONObject(){{
- put("minValue", 0);
- }}, null),
- ToolParameterDescriptor.create(TARGET_ASSAY, "Target Assay", "Results will be loaded into this assay. If no assay is selected, a table will be created with nothing in the DB.", "tcr-assayselectorfield", new JSONObject(){{
- put("autoSelectAssay", false);
- }}, null),
- ToolParameterDescriptor.create(DELETE_EXISTING_ASSAY_DATA, "Delete Any Existing Assay Data", "If selected, prior to importing assay data, and existing assay runs in the target container from this readset will be deleted.", "checkbox", new JSONObject(){{
- put("checked", true);
- }}, true),
- ToolParameterDescriptor.create("excludeFailedcDNA", "Exclude Failed cDNA", "If selected, cDNAs with non-blank status fields will be omitted", "checkbox", null, true)
-
- ), PageFlowUtil.set("tcrdb/field/AssaySelectorField.js"), "https://support.10xgenomics.com/single-cell-gene-expression/software/pipelines/latest/what-is-cell-ranger", true, false, false, ALIGNMENT_MODE.MERGE_THEN_ALIGN);
- }
-
- @Override
- public boolean shouldRunIdxstats()
- {
- return false;
- }
-
- public String getName()
- {
- return "CellRanger-VDJ";
- }
-
- public String getDescription()
- {
- return null;
- }
-
- public AlignmentStep create(PipelineContext context)
- {
- return new CellRangerVDJAlignmentStep(this, context, new CellRangerVDJWrapper(context.getLogger()));
- }
- }
-
- public static class CellRangerVDJAlignmentStep extends AbstractAlignmentPipelineStep implements AlignmentStep
- {
- private CellRangerVDJUtils _utils = null;
-
- private CellRangerVDJUtils getUtils()
- {
- if (_utils == null)
- {
- _utils = new CellRangerVDJUtils(getPipelineCtx().getLogger(), getPipelineCtx().getSourceDirectory());
- }
-
- return _utils;
- }
-
- public CellRangerVDJAlignmentStep(AlignmentStepProvider provider, PipelineContext ctx, CellRangerVDJWrapper wrapper)
- {
- super(provider, ctx, wrapper);
- }
-
- @Override
- public boolean supportsMetrics()
- {
- return false;
- }
-
- @Override
- public void init(SequenceAnalysisJobSupport support) throws PipelineJobException
- {
- ReferenceGenome referenceGenome = support.getCachedGenomes().iterator().next();
- boolean hasCachedIndex = AlignerIndexUtil.hasCachedIndex(this.getPipelineCtx(), getIndexCachedDirName(getPipelineCtx().getJob()), referenceGenome);
- if (!hasCachedIndex)
- {
- getPipelineCtx().getLogger().info("Creating FASTA for CellRanger VDJ Index for genome: " + referenceGenome.getName());
- File fasta = getGenomeFasta();
- try (PrintWriter writer = PrintWriters.getPrintWriter(fasta))
- {
- final AtomicInteger i = new AtomicInteger(0);
- UserSchema us = QueryService.get().getUserSchema(getPipelineCtx().getJob().getUser(), getPipelineCtx().getJob().getContainer(), "sequenceanalysis");
- List seqIds = new TableSelector(us.getTable("reference_library_members", null), PageFlowUtil.set("ref_nt_id"), new SimpleFilter(FieldKey.fromString("library_id"), referenceGenome.getGenomeId()), null).getArrayList(Integer.class);
- new TableSelector(us.getTable("ref_nt_sequences", null), new SimpleFilter(FieldKey.fromString("rowid"), seqIds, CompareType.IN), null).forEach(nt -> {
-
- if (nt.getLocus() == null)
- {
- throw new IllegalArgumentException("Locus was empty for NT with ID: " + nt.getRowid());
- }
-
- //NOTE: this allows dual TRA/TRD segments
- String[] loci = nt.getLocus().split("/");
- for (String locus : loci)
- {
- i.getAndIncrement(); //cant use sequenceId since sequences might be represented multiple times across loci
-
- String seq = nt.getSequence();
-
- //example: >1|TRAV41*01 TRAV41|TRAV41|L-REGION+V-REGION|TR|TRA|None|None
- StringBuilder header = new StringBuilder();
- header.append(">").append(i.get()).append("|").append(nt.getName()).append(" ").append(nt.getLineage()).append("|").append(nt.getLineage()).append("|");
- //translate into V_Region
- String type;
- if (nt.getLineage().contains("J"))
- {
- type = "J-REGION";
- }
- else if (nt.getLineage().contains("V"))
- {
- if (seq.length() < 300)
- {
- getPipelineCtx().getLogger().info("Using V-REGION due to short length: " + nt.getName() + " / " + nt.getSeqLength());
- type = "V-REGION";
- }
- else
- {
- type = "L-REGION+V-REGION";
- }
- }
- else if (nt.getLineage().contains("C"))
- {
- type = "C-REGION";
- }
- else if (nt.getLineage().contains("D"))
- {
- type = "D-REGION";
- }
- else
- {
- throw new RuntimeException("Unknown lineage: " + nt.getLineage());
- }
-
- header.append(type).append("|TR|").append(locus).append("|None|None");
-
- writer.write(header + "\n");
- writer.write(seq + "\n");
- }
- nt.clearCachedSequence();
- }, RefNtSequenceModel.class);
- }
- catch (IllegalArgumentException | IOException e)
- {
- throw new PipelineJobException(e);
- }
- }
-
- boolean excludeFailedcDNA = getProvider().getParameterByName("excludeFailedcDNA").extractValue(getPipelineCtx().getJob(), getProvider(), getStepIdx(), Boolean.class, true);
- getUtils().prepareHashingAndCiteSeqFilesIfNeeded(getPipelineCtx().getJob(), getPipelineCtx().getSequenceSupport(), "enrichedReadsetId", excludeFailedcDNA, false, false);
- }
-
- private File getGenomeFasta()
- {
- return new File(getPipelineCtx().getSourceDirectory(), "cellRangerVDJ.fasta");
- }
-
- @Override
- public String getIndexCachedDirName(PipelineJob job)
- {
- return getProvider().getName();
- }
-
- @Override
- public AlignmentStep.IndexOutput createIndex(ReferenceGenome referenceGenome, File outputDir) throws PipelineJobException
- {
- IndexOutputImpl output = new IndexOutputImpl(referenceGenome);
-
- File indexDir = new File(outputDir, getIndexCachedDirName(getPipelineCtx().getJob()));
- boolean hasCachedIndex = AlignerIndexUtil.hasCachedIndex(this.getPipelineCtx(), getIndexCachedDirName(getPipelineCtx().getJob()), referenceGenome);
- if (!hasCachedIndex)
- {
- getPipelineCtx().getLogger().info("Creating CellRanger VDJ Index");
- getPipelineCtx().getLogger().info("using file: " + getGenomeFasta().getPath());
- output.addIntermediateFile(getGenomeFasta());
-
- //remove if directory exists
- if (indexDir.exists())
- {
- try
- {
- FileUtils.deleteDirectory(indexDir);
- }
- catch (IOException e)
- {
- throw new PipelineJobException(e);
- }
- }
-
- output.addInput(getGenomeFasta(), "Input FASTA");
-
- List args = new ArrayList<>();
- args.add(getWrapper().getExe().getPath());
- args.add("mkvdjref");
- args.add("--seqs=" + getGenomeFasta().getPath());
- args.add("--genome=" + indexDir.getName());
-
- getWrapper().setWorkingDir(indexDir.getParentFile());
- getWrapper().execute(args);
-
- output.appendOutputs(referenceGenome.getWorkingFastaFile(), indexDir);
-
- //recache if not already
- AlignerIndexUtil.saveCachedIndex(hasCachedIndex, getPipelineCtx(), indexDir, getIndexCachedDirName(getPipelineCtx().getJob()), referenceGenome);
-
- }
-
- return output;
- }
-
- @Override
- public AlignmentStep.AlignmentOutput performAlignment(Readset rs, File inputFastq1, @Nullable File inputFastq2, File outputDirectory, ReferenceGenome referenceGenome, String basename, String readGroupId, @Nullable String platformUnit) throws PipelineJobException
- {
- AlignmentOutputImpl output = new AlignmentOutputImpl();
-
- List args = new ArrayList<>();
- args.add(getWrapper().getExe().getPath());
- args.add("vdj");
-
- String idParam = StringUtils.trimToNull(getProvider().getParameterByName("id").extractValue(getPipelineCtx().getJob(), getProvider(), getStepIdx(), String.class));
- String id = FileUtil.makeLegalName(rs.getName()) + (idParam == null ? "" : "-" + idParam);
- id = id.replaceAll("[^a-zA-z0-9_\\-]", "_");
- args.add("--id=" + id);
-
- File indexDir = AlignerIndexUtil.getIndexDir(referenceGenome, getIndexCachedDirName(getPipelineCtx().getJob()));
- args.add("--reference=" + indexDir.getPath());
-
- args.addAll(getClientCommandArgs("="));
-
- Integer maxThreads = SequencePipelineService.get().getMaxThreads(getPipelineCtx().getLogger());
- if (maxThreads != null)
- {
- args.add("--localcores=" + maxThreads.toString());
- }
-
- Integer maxRam = SequencePipelineService.get().getMaxRam();
- if (maxRam != null)
- {
- args.add("--localmem=" + maxRam.toString());
- }
-
- File localFqDir = new File(outputDirectory, "localFq");
- output.addIntermediateFile(localFqDir);
- Set sampleNames = prepareFastqSymlinks(rs, localFqDir);
- args.add("--fastqs=" + localFqDir.getPath());
-
- getPipelineCtx().getLogger().debug("Sample names: [" + StringUtils.join(sampleNames, ",") + "]");
- if (sampleNames.size() > 1)
- {
- args.add("--sample=" + StringUtils.join(sampleNames, ","));
- }
-
- getWrapper().setWorkingDir(outputDirectory);
-
- //Note: we can safely assume only this server is working on these files, so if the _lock file exists, it was from a previous failed job.
- File lockFile = new File(outputDirectory, id + "/_lock");
- if (lockFile.exists())
- {
- getPipelineCtx().getLogger().info("Lock file exists, deleting: " + lockFile.getPath());
- lockFile.delete();
- }
-
- getWrapper().execute(args);
-
- File outdir = new File(outputDirectory, id);
- outdir = new File(outdir, "outs");
-
- File bam = new File(outdir, "all_contig.bam");
- if (!bam.exists())
- {
- throw new PipelineJobException("Unable to find file: " + bam.getPath());
- }
- output.setBAM(bam);
-
- //NOTE: run these before cleanup in case of failure
- Integer assayId = getProvider().getParameterByName(TARGET_ASSAY).extractValue(getPipelineCtx().getJob(), getProvider(), getStepIdx(), Integer.class);
- if (assayId != null)
- {
- boolean scanEditDistances = getProvider().getParameterByName("scanEditDistances").extractValue(getPipelineCtx().getJob(), getProvider(), getStepIdx(), Boolean.class, false);
- int editDistance = getProvider().getParameterByName("editDistance").extractValue(getPipelineCtx().getJob(), getProvider(), getStepIdx(), Integer.class, 2);
- int minCountPerCell = getProvider().getParameterByName("minCountPerCell").extractValue(getPipelineCtx().getJob(), getProvider(), getStepIdx(), Integer.class, 3);
- boolean useSeurat = getProvider().getParameterByName("useSeurat").extractValue(getPipelineCtx().getJob(), getProvider(), getStepIdx(), Boolean.class, true);
- boolean useMultiSeq = getProvider().getParameterByName("useMultiSeq").extractValue(getPipelineCtx().getJob(), getProvider(), getStepIdx(), Boolean.class, true);
-
- getUtils().runRemoteVdjCellHashingTasks(output, CellRangerVDJCellHashingHandler.CATEGORY, getUtils().getPerCellCsv(output.getBAM().getParentFile()), rs, getPipelineCtx().getSequenceSupport(), null, getPipelineCtx().getWorkingDirectory(), getPipelineCtx().getSourceDirectory(), editDistance, scanEditDistances, referenceGenome.getGenomeId(), minCountPerCell, useSeurat, useMultiSeq);
- }
- else
- {
- getPipelineCtx().getLogger().debug("No target assay selected, skipping cell hashing steps");
- }
-
- //now do cleanup/rename:
- try
- {
- String prefix = FileUtil.makeLegalName(rs.getName() + "_");
- File outputHtml = new File(outdir, "web_summary.html");
- if (!outputHtml.exists())
- {
- throw new PipelineJobException("Unable to find file: " + outputHtml.getPath());
- }
-
- File outputHtmlRename = new File(outdir, prefix + outputHtml.getName());
- if (outputHtmlRename.exists())
- {
- outputHtmlRename.delete();
- }
- FileUtils.moveFile(outputHtml, outputHtmlRename);
-
- output.addSequenceOutput(outputHtmlRename, rs.getName() + " 10x VDJ Summary", "10x Run Summary", rs.getRowId(), null, referenceGenome.getGenomeId(), null);
-
- File outputVloupe = new File(outdir, "vloupe.vloupe");
- if (!outputVloupe.exists())
- {
- throw new PipelineJobException("Unable to find file: " + outputVloupe.getPath());
- }
-
- File outputVloupeRename = new File(outdir, prefix + outputVloupe.getName());
- if (outputVloupeRename.exists())
- {
- outputVloupeRename.delete();
- }
- FileUtils.moveFile(outputVloupe, outputVloupeRename);
- output.addSequenceOutput(outputVloupeRename, rs.getName() + " 10x VLoupe", "10x VLoupe", rs.getRowId(), null, referenceGenome.getGenomeId(), null);
- }
- catch (IOException e)
- {
- throw new PipelineJobException(e);
- }
-
- //NOTE: this folder has many unnecessary files and symlinks that get corrupted when we rename the main outputs
- File directory = new File(outdir.getParentFile(), "SC_VDJ_ASSEMBLER_CS");
- if (directory.exists())
- {
- //NOTE: this will have lots of symlinks, including corrupted ones, which java handles badly
- new SimpleScriptWrapper(getPipelineCtx().getLogger()).execute(Arrays.asList("rm", "-Rf", directory.getPath()));
- }
- else
- {
- getPipelineCtx().getLogger().warn("Unable to find folder: " + directory.getPath());
- }
-
- deleteSymlinks(localFqDir);
-
- return output;
- }
-
- @Override
- public boolean doAddReadGroups()
- {
- return false;
- }
-
- @Override
- public boolean doSortIndexBam()
- {
- return false;
- }
-
- @Override
- public boolean alwaysCopyIndexToWorkingDir()
- {
- return false;
- }
-
- @Override
- public boolean supportsGzipFastqs()
- {
- return true;
- }
-
- private String getSymlinkFileName(String fileName, boolean doRename, String sampleName, int idx, boolean isReversed)
- {
- //NOTE: cellranger is very picky about file name formatting
- if (doRename)
- {
- sampleName = FileUtil.makeLegalName(sampleName.replaceAll("_", "-")).replaceAll(" ", "-").replaceAll("\\.", "-");;
- return sampleName + "_S1_L001_R" + (isReversed ? "2" : "1") + "_" + StringUtils.leftPad(String.valueOf(idx), 3, "0") + ".fastq.gz";
- }
- else
- {
- Matcher m = FILE_PATTERN.matcher(fileName);
- if (m.matches())
- {
- if (!StringUtils.isEmpty(m.group(7)))
- {
- return m.group(1).replaceAll("_", "-") + StringUtils.trimToEmpty(m.group(2)) + "_L" + StringUtils.trimToEmpty(m.group(3)) + "_" + StringUtils.trimToEmpty(m.group(4)) + StringUtils.trimToEmpty(m.group(5)) + StringUtils.trimToEmpty(m.group(6)) + ".fastq.gz";
- }
- else if (m.group(1).contains("_"))
- {
- getPipelineCtx().getLogger().info("replacing underscores in file/sample name");
- return m.group(1).replaceAll("_", "-") + StringUtils.trimToEmpty(m.group(2)) + "_L" + StringUtils.trimToEmpty(m.group(3)) + "_" + StringUtils.trimToEmpty(m.group(4)) + StringUtils.trimToEmpty(m.group(5)) + StringUtils.trimToEmpty(m.group(6)) + ".fastq.gz";
- }
- else
- {
- getPipelineCtx().getLogger().info("no additional characters found");
- }
- }
- else
- {
- getPipelineCtx().getLogger().warn("filename does not match Illumina formatting: " + fileName);
- }
- }
-
- return FileUtil.makeLegalName(fileName);
- }
-
- public Set prepareFastqSymlinks(Readset rs, File localFqDir) throws PipelineJobException
- {
- Set ret = new HashSet<>();
- if (!localFqDir.exists())
- {
- localFqDir.mkdirs();
- }
-
- String[] files = localFqDir.list();
- if (files != null && files.length > 0)
- {
- deleteSymlinks(localFqDir);
- }
-
- int idx = 0;
- boolean doRename = true; //cellranger is too picky - simply rename files all the time
- for (ReadData rd : rs.getReadData())
- {
- idx++;
- try
- {
- File target1 = new File(localFqDir, getSymlinkFileName(rd.getFile1().getName(), doRename, rs.getName(), idx, false));
- getPipelineCtx().getLogger().debug("file: " + rd.getFile1().getPath());
- getPipelineCtx().getLogger().debug("target: " + target1.getPath());
- if (target1.exists())
- {
- getPipelineCtx().getLogger().debug("deleting existing symlink: " + target1.getName());
- Files.delete(target1.toPath());
- }
-
- Files.createSymbolicLink(target1.toPath(), rd.getFile1().toPath());
- ret.add(getSampleName(target1.getName()));
-
- if (rd.getFile2() != null)
- {
- File target2 = new File(localFqDir, getSymlinkFileName(rd.getFile2().getName(), doRename, rs.getName(), idx, true));
- getPipelineCtx().getLogger().debug("file: " + rd.getFile2().getPath());
- getPipelineCtx().getLogger().debug("target: " + target2.getPath());
- if (target2.exists())
- {
- getPipelineCtx().getLogger().debug("deleting existing symlink: " + target2.getName());
- Files.delete(target2.toPath());
- }
- Files.createSymbolicLink(target2.toPath(), rd.getFile2().toPath());
- ret.add(getSampleName(target2.getName()));
- }
- }
- catch (IOException e)
- {
- throw new PipelineJobException(e);
- }
- }
-
- return ret;
- }
-
- public void deleteSymlinks(File localFqDir) throws PipelineJobException
- {
- for (File fq : localFqDir.listFiles())
- {
- try
- {
- getPipelineCtx().getLogger().debug("deleting symlink: " + fq.getName());
- Files.delete(fq.toPath());
- }
- catch (IOException e)
- {
- throw new PipelineJobException(e);
- }
- }
- }
-
- public void addMetrics(AnalysisModel model) throws PipelineJobException
- {
- getPipelineCtx().getLogger().debug("adding 10x metrics");
-
- File metrics = new File(model.getAlignmentFileObject().getParentFile(), "metrics_summary.csv");
- if (metrics.exists())
- {
- try (CSVReader reader = new CSVReader(Readers.getReader(metrics)))
- {
- String[] line;
- String[] header = null;
- String[] metricValues = null;
-
- int i = 0;
- while ((line = reader.readNext()) != null)
- {
- if (i == 0)
- {
- header = line;
- }
- else
- {
- metricValues = line;
- break;
- }
-
- i++;
- }
-
- int totalAdded = 0;
- TableInfo ti = DbSchema.get("sequenceanalysis", DbSchemaType.Module).getTable("quality_metrics");
-
- //NOTE: if this job errored and restarted, we may have duplicate records:
- SimpleFilter filter = new SimpleFilter(FieldKey.fromString("readset"), model.getReadset());
- filter.addCondition(FieldKey.fromString("analysis_id"), model.getRowId(), CompareType.EQUAL);
- filter.addCondition(FieldKey.fromString("dataid"), model.getAlignmentFile(), CompareType.EQUAL);
- filter.addCondition(FieldKey.fromString("category"), "Cell Ranger VDJ", CompareType.EQUAL);
- filter.addCondition(FieldKey.fromString("container"), getPipelineCtx().getJob().getContainer().getId(), CompareType.EQUAL);
- TableSelector ts = new TableSelector(ti, PageFlowUtil.set("rowid"), filter, null);
- if (ts.exists())
- {
- getPipelineCtx().getLogger().info("Deleting existing QC metrics (probably from prior restarted job)");
- ts.getArrayList(Integer.class).forEach(rowid -> {
- Table.delete(ti, rowid);
- });
- }
-
- for (int j = 0; j < header.length; j++)
- {
- Map toInsert = new CaseInsensitiveHashMap<>();
- toInsert.put("container", getPipelineCtx().getJob().getContainer().getId());
- toInsert.put("createdby", getPipelineCtx().getJob().getUser().getUserId());
- toInsert.put("created", new Date());
- toInsert.put("readset", model.getReadset());
- toInsert.put("analysis_id", model.getRowId());
- toInsert.put("dataid", model.getAlignmentFile());
-
- toInsert.put("category", "Cell Ranger VDJ");
- toInsert.put("metricname", header[j]);
-
- metricValues[j] = metricValues[j].replaceAll(",", "");
- Object val = metricValues[j];
- if (metricValues[j].contains("%"))
- {
- metricValues[j] = metricValues[j].replaceAll("%", "");
- Double d = ConvertHelper.convert(metricValues[j], Double.class);
- d = d / 100.0;
- val = d;
- }
-
- toInsert.put("metricvalue", val);
-
- Table.insert(getPipelineCtx().getJob().getUser(), ti, toInsert);
- totalAdded++;
- }
-
- getPipelineCtx().getLogger().info("total metrics added: " + totalAdded);
- }
- catch (IOException e)
- {
- throw new PipelineJobException(e);
- }
- }
- else
- {
- getPipelineCtx().getLogger().warn("unable to find metrics file: " + metrics.getPath());
- }
- }
-
- public void complete(SequenceAnalysisJobSupport support, AnalysisModel model) throws PipelineJobException
- {
- addMetrics(model);
-
- File bam = model.getAlignmentData().getFile();
- if (bam.exists())
- {
- Integer assayId = getProvider().getParameterByName(TARGET_ASSAY).extractValue(getPipelineCtx().getJob(), getProvider(), getStepIdx(), Integer.class);
- Boolean deleteExisting = getProvider().getParameterByName(DELETE_EXISTING_ASSAY_DATA).extractValue(getPipelineCtx().getJob(), getProvider(), getStepIdx(), Boolean.class, false);
- getUtils().importAssayData(getPipelineCtx().getJob(), model, bam.getParentFile(), assayId, null, deleteExisting);
- }
- else
- {
- getPipelineCtx().getLogger().warn("BAM not found, expected: " + bam.getPath());
- }
- }
-
- private static Pattern FILE_PATTERN = Pattern.compile("^(.+?)(_S[0-9]+){0,1}_L(.+?)_(R){0,1}([0-9])(_[0-9]+){0,1}(.*?)(\\.f(ast){0,1}q)(\\.gz)?$");
- private static Pattern SAMPLE_PATTERN = Pattern.compile("^(.+)_S[0-9]+(.*)$");
-
- private String getSampleName(String fn)
- {
- Matcher matcher = FILE_PATTERN.matcher(fn);
- if (matcher.matches())
- {
- String ret = matcher.group(1);
- Matcher matcher2 = SAMPLE_PATTERN.matcher(ret);
- if (matcher2.matches())
- {
- ret = matcher2.group(1);
- }
- else
- {
- getPipelineCtx().getLogger().debug("_S not found in sample: [" + ret + "]");
- }
-
- ret = ret.replaceAll("_", "-");
-
- return ret;
- }
- else
- {
- getPipelineCtx().getLogger().debug("file does not match illumina pattern: [" + fn + "]");
- }
-
- throw new IllegalArgumentException("Unable to infer Illumina sample name: " + fn);
- }
- }
-
- protected File getExe()
- {
- //NOTE: cellranger 4 doesnt work w/ custom libraries currently. update to CR4 when fixed
- return SequencePipelineService.get().getExeForPackage("CELLRANGERPATH", "cellranger-31");
- }
-}
diff --git a/tcrdb/src/org/labkey/tcrdb/pipeline/MiXCRAnalysis.java b/tcrdb/src/org/labkey/tcrdb/pipeline/MiXCRAnalysis.java
index 09b89b5c8..a9738b517 100644
--- a/tcrdb/src/org/labkey/tcrdb/pipeline/MiXCRAnalysis.java
+++ b/tcrdb/src/org/labkey/tcrdb/pipeline/MiXCRAnalysis.java
@@ -13,6 +13,7 @@
import org.labkey.api.assay.AssayService;
import org.labkey.api.collections.CaseInsensitiveHashMap;
import org.labkey.api.data.CompareType;
+import org.labkey.api.data.Container;
import org.labkey.api.data.SimpleFilter;
import org.labkey.api.data.TableInfo;
import org.labkey.api.data.TableSelector;
@@ -25,6 +26,7 @@
import org.labkey.api.module.ModuleLoader;
import org.labkey.api.pipeline.PipelineJobException;
import org.labkey.api.query.FieldKey;
+import org.labkey.api.query.QueryService;
import org.labkey.api.query.ValidationException;
import org.labkey.api.reader.Readers;
import org.labkey.api.resource.FileResource;
@@ -76,8 +78,6 @@
import java.util.TreeSet;
import java.util.zip.GZIPInputStream;
-import static org.labkey.tcrdb.pipeline.CellRangerVDJWrapper.DELETE_EXISTING_ASSAY_DATA;
-
/**
* Created by bimber on 5/10/2016.
@@ -126,7 +126,7 @@ public Provider()
{{
put("value", "ALL");
}}, true),
- ToolParameterDescriptor.create(DELETE_EXISTING_ASSAY_DATA, "Delete Any Existing Assay Data", "If selected, prior to importing assay data, and existing assay runs in the target container from this readset will be deleted.", "checkbox", new JSONObject(){{
+ ToolParameterDescriptor.create(CellRangerVDJCellHashingHandler.DELETE_EXISTING_ASSAY_DATA, "Delete Any Existing Assay Data", "If selected, prior to importing assay data, and existing assay runs in the target container from this readset will be deleted.", "checkbox", new JSONObject(){{
put("checked", true);
}}, true),
ToolParameterDescriptor.create(FLAG_MISSENSE, "Flag Missense CDR3", "If checked, if a sample has duplicate CDR3 clones from the same locus, and and one of these is missense, that clone will be flagged and excluded from many reports.", "checkbox", new JSONObject()
@@ -1215,6 +1215,12 @@ private void inspectForOrphanAlignment(Readset rs, String[] line, Set
}
}
+ private TableInfo getCdnaTable()
+ {
+ Container target = getPipelineCtx().getJob().getContainer().isWorkbook() ? getPipelineCtx().getJob().getContainer().getParent() : getPipelineCtx().getJob().getContainer();
+ return QueryService.get().getUserSchema(getPipelineCtx().getJob().getUser(), target, TCRdbSchema.SINGLE_CELL).getTable(TCRdbSchema.TABLE_CDNAS);
+ }
+
private void parseCloneOutput(Map runMap, File table, AnalysisModel model, File inputBam) throws PipelineJobException
{
Integer runId = SequencePipelineService.get().getExpRunIdForJob(getPipelineCtx().getJob());
@@ -1223,6 +1229,7 @@ private void parseCloneOutput(Map runMap, File table, AnalysisM
List extends ExpData> cloneDatas = run.getInputDatas(CLONES_FILE, ExpProtocol.ApplicationType.ExperimentRunOutput);
List extends ExpData> vdjDatas = run.getInputDatas(FINAL_VDJ_FILE, ExpProtocol.ApplicationType.ExperimentRunOutput);
+ TableInfo cDNATable = getCdnaTable();
try (CSVReader reader = new CSVReader(Readers.getReader(table), '\t'))
{
int lineNo = 0;
@@ -1235,7 +1242,7 @@ private void parseCloneOutput(Map runMap, File table, AnalysisM
continue;
}
- Map row = getBaseRow(model, runId);
+ Map row = getBaseRow(model, runId, cDNATable);
if (line.length != (FIELDS.size() + TOTAL_EXPORTED_FIELDS_NOT_IN_DB)) //this includes one additional field appended to the end
{
@@ -1323,7 +1330,7 @@ private void parseCloneOutput(Map runMap, File table, AnalysisM
}
}
- private Map getBaseRow(AnalysisModel model, Integer runId) throws PipelineJobException
+ private Map getBaseRow(AnalysisModel model, Integer runId, TableInfo cDNATable) throws PipelineJobException
{
Map row = new CaseInsensitiveHashMap<>();
if (model.getReadset() != null)
@@ -1356,13 +1363,12 @@ private Map getBaseRow(AnalysisModel model, Integer runId) throw
row.put("analysisId", model.getRowId());
//attempt to locate cDNA:
- TableInfo cDNATable = TCRdbSchema.getInstance().getSchema().getTable(TCRdbSchema.TABLE_CDNAS);
if (model.getReadset() != null)
{
SimpleFilter filter = new SimpleFilter();
filter.addClause(new SimpleFilter.OrClause(
new CompareType.CompareClause(FieldKey.fromString("readsetId"), CompareType.EQUAL, model.getReadset()),
- new CompareType.CompareClause(FieldKey.fromString("enrichedReadsetId"), CompareType.EQUAL, model.getReadset())
+ new CompareType.CompareClause(FieldKey.fromString("tcrReadsetId"), CompareType.EQUAL, model.getReadset())
));
TableSelector ts1 = new TableSelector(cDNATable, PageFlowUtil.set("rowId"), filter, null);
@@ -1592,7 +1598,8 @@ private void importRun(RunData rd, File outDir, AnalysisModel model, ExpProtocol
if (rd.rows.isEmpty())
{
//NOTE: we need to add a placeholder row since assay import will die w/ a run-only import:
- Map row = getBaseRow(model, runId);
+ TableInfo cDNATable = getCdnaTable();
+ Map row = getBaseRow(model, runId, cDNATable);
row.put("species", rd.species);
row.put("libraryId", rd.libraryId);
row.put("locus", "None");
@@ -1602,7 +1609,7 @@ private void importRun(RunData rd, File outDir, AnalysisModel model, ExpProtocol
getPipelineCtx().getLogger().debug("saving assay file to: " + assayTmp.getPath());
AssayProvider ap = AssayService.get().getProvider(protocol);
- boolean deleteExistingAssayData = getProvider().getParameterByName(DELETE_EXISTING_ASSAY_DATA).extractValue(getPipelineCtx().getJob(), getProvider(), getStepIdx(), Boolean.class, false);
+ boolean deleteExistingAssayData = getProvider().getParameterByName(CellRangerVDJCellHashingHandler.DELETE_EXISTING_ASSAY_DATA).extractValue(getPipelineCtx().getJob(), getProvider(), getStepIdx(), Boolean.class, false);
if (deleteExistingAssayData)
{
if (model.getReadset() == null)
diff --git a/tcrdb/src/org/labkey/tcrdb/pipeline/SeuratCellHashingHandler.java b/tcrdb/src/org/labkey/tcrdb/pipeline/SeuratCellHashingHandler.java
deleted file mode 100644
index 8e526dfed..000000000
--- a/tcrdb/src/org/labkey/tcrdb/pipeline/SeuratCellHashingHandler.java
+++ /dev/null
@@ -1,153 +0,0 @@
-package org.labkey.tcrdb.pipeline;
-
-import org.json.JSONObject;
-import org.labkey.api.module.ModuleLoader;
-import org.labkey.api.pipeline.PipelineJob;
-import org.labkey.api.pipeline.PipelineJobException;
-import org.labkey.api.pipeline.RecordedAction;
-import org.labkey.api.sequenceanalysis.SequenceOutputFile;
-import org.labkey.api.sequenceanalysis.model.Readset;
-import org.labkey.api.sequenceanalysis.pipeline.AbstractParameterizedOutputHandler;
-import org.labkey.api.sequenceanalysis.pipeline.SequenceAnalysisJobSupport;
-import org.labkey.api.sequenceanalysis.pipeline.SequenceOutputHandler;
-import org.labkey.api.sequenceanalysis.pipeline.ToolParameterDescriptor;
-import org.labkey.api.util.FileType;
-import org.labkey.tcrdb.TCRdbModule;
-
-import java.io.File;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-
-public class SeuratCellHashingHandler extends AbstractParameterizedOutputHandler
-{
- private FileType _fileType = new FileType(".seurat.rds", false);
- public static final String CATEGORY = "Seurat Cell Hashing Calls";
-
- public SeuratCellHashingHandler()
- {
- super(ModuleLoader.getInstance().getModule(TCRdbModule.class), "Seurat GEX/Cell Hashing", "This will run CiteSeqCount/MultiSeqClassifier to generate a sample-to-cellbarcode TSV based on the cell barcodes present in the saved Seurat object.", null, getDefaultParams());
- }
-
- private static List getDefaultParams()
- {
- List ret = new ArrayList<>();
- ret.add(ToolParameterDescriptor.create("useOutputFileContainer", "Submit to Source File Workbook", "If checked, each job will be submitted to the same workbook as the input file, as opposed to submitting all jobs to the same workbook. This is primarily useful if submitting a large batch of files to process separately..", "checkbox", new JSONObject()
- {{
- put("checked", true);
- }}, false));
-
- ret.addAll(CellRangerCellHashingHandler.getDefaultHashingParams(true));
-
- return ret;
- }
-
- @Override
- public boolean canProcess(SequenceOutputFile o)
- {
- return o.getFile() != null && _fileType.isType(o.getFile());
- }
-
- @Override
- public boolean doRunRemote()
- {
- return true;
- }
-
- @Override
- public boolean doRunLocal()
- {
- return false;
- }
-
- @Override
- public SequenceOutputProcessor getProcessor()
- {
- return new Processor();
- }
-
- @Override
- public boolean doSplitJobs()
- {
- return true;
- }
-
- @Override
- public boolean requiresSingleGenome()
- {
- return false;
- }
-
- public class Processor implements SequenceOutputHandler.SequenceOutputProcessor
- {
- @Override
- public void init(PipelineJob job, SequenceAnalysisJobSupport support, List inputFiles, JSONObject params, File outputDir, List actions, List outputsToCreate) throws UnsupportedOperationException, PipelineJobException
- {
- new CellRangerVDJUtils(job.getLogger(), outputDir).prepareHashingAndCiteSeqFilesIfNeeded(job, support, "readsetId", params.optBoolean("excludeFailedcDNA", true), true, false);
- }
-
- @Override
- public void processFilesOnWebserver(PipelineJob job, SequenceAnalysisJobSupport support, List inputFiles, JSONObject params, File outputDir, List actions, List outputsToCreate) throws UnsupportedOperationException, PipelineJobException
- {
-
- }
-
- @Override
- public void processFilesRemote(List inputFiles, SequenceOutputHandler.JobContext ctx) throws UnsupportedOperationException, PipelineJobException
- {
- RecordedAction action = new RecordedAction(getName());
- Map readsetToHashing = CellRangerVDJUtils.getCachedHashingReadsetMap(ctx.getSequenceSupport());
- ctx.getLogger().debug("total cached readset to hashing pairs: " + readsetToHashing.size());
-
- for (SequenceOutputFile so : inputFiles)
- {
- ctx.getLogger().info("processing file: " + so.getName());
-
- File barcodes = getBarcodesFromSeurat(so.getFile());
-
- Readset rs = ctx.getSequenceSupport().getCachedReadset(so.getReadset());
- if (rs == null)
- {
- throw new PipelineJobException("Unable to find readset for outputfile: " + so.getRowid());
- }
- else if (rs.getReadsetId() == null)
- {
- throw new PipelineJobException("Readset lacks a rowId for outputfile: " + so.getRowid());
- }
-
- Readset htoReadset = ctx.getSequenceSupport().getCachedReadset(readsetToHashing.get(rs.getReadsetId()));
- if (htoReadset == null)
- {
- throw new PipelineJobException("Unable to find Hashing/Cite-seq readset for GEX readset: " + rs.getReadsetId());
- }
-
- CellRangerCellHashingHandler.processBarcodeFile(ctx, barcodes, rs, htoReadset, so.getLibrary_id(), action, getClientCommandArgs(ctx.getParams()), false, CATEGORY);
- }
-
- ctx.addActions(action);
- }
-
- @Override
- public void complete(PipelineJob job, List inputs, List outputsCreated, SequenceAnalysisJobSupport support) throws PipelineJobException
- {
- for (SequenceOutputFile so : outputsCreated)
- {
- if (so.getCategory().equals(CATEGORY))
- {
- CellRangerVDJCellHashingHandler.processMetrics(so, job, true);
- }
- }
- }
- }
-
- public static File getBarcodesFromSeurat(File seuratObj) throws PipelineJobException
- {
- File barcodes = new File(seuratObj.getParentFile(), seuratObj.getName().replaceAll("seurat.rds", "cellBarcodes.csv"));
- if (!barcodes.exists())
- {
- throw new PipelineJobException("Unable to find expected cell barcodes file. This might indicate the seurat object was created with an older version of the pipeline. Expected: " + barcodes.getPath());
- }
-
- return barcodes;
- }
-}
diff --git a/tcrdb/src/org/labkey/tcrdb/pipeline/SeuratCiteSeqHandler.java b/tcrdb/src/org/labkey/tcrdb/pipeline/SeuratCiteSeqHandler.java
deleted file mode 100644
index 853a68a0b..000000000
--- a/tcrdb/src/org/labkey/tcrdb/pipeline/SeuratCiteSeqHandler.java
+++ /dev/null
@@ -1,131 +0,0 @@
-package org.labkey.tcrdb.pipeline;
-
-import org.json.JSONObject;
-import org.labkey.api.module.ModuleLoader;
-import org.labkey.api.pipeline.PipelineJob;
-import org.labkey.api.pipeline.PipelineJobException;
-import org.labkey.api.pipeline.RecordedAction;
-import org.labkey.api.sequenceanalysis.SequenceOutputFile;
-import org.labkey.api.sequenceanalysis.model.Readset;
-import org.labkey.api.sequenceanalysis.pipeline.AbstractParameterizedOutputHandler;
-import org.labkey.api.sequenceanalysis.pipeline.SequenceAnalysisJobSupport;
-import org.labkey.api.sequenceanalysis.pipeline.SequenceOutputHandler;
-import org.labkey.api.sequenceanalysis.pipeline.ToolParameterDescriptor;
-import org.labkey.api.util.FileType;
-import org.labkey.tcrdb.TCRdbModule;
-
-import java.io.File;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Map;
-
-public class SeuratCiteSeqHandler extends AbstractParameterizedOutputHandler
-{
- protected FileType _fileType = new FileType(".seurat.rds", false);
- public static final String CATEGORY = "Seurat CITE-Seq Count Matrix";
-
- public SeuratCiteSeqHandler()
- {
- super(ModuleLoader.getInstance().getModule(TCRdbModule.class), "Seurat GEX/CITE-seq Counts", "This will run CiteSeqCount to generate a sample-to-cellbarcode TSV based on the cell barcodes present in the saved Seurat object.", null, Arrays.asList(
- ToolParameterDescriptor.create("editDistance", "Edit Distance", null, "ldk-integerfield", null, 3),
- ToolParameterDescriptor.create("excludeFailedcDNA", "Exclude Failed cDNA", "If selected, cDNAs with non-blank status fields will be omitted", "checkbox", null, true),
- ToolParameterDescriptor.create("minCountPerCell", "Min Reads/Cell (Cell Hashing)", null, "ldk-integerfield", null, 5),
- ToolParameterDescriptor.create("useOutputFileContainer", "Submit to Source File Workbook", "If checked, each job will be submitted to the same workbook as the input file, as opposed to submitting all jobs to the same workbook. This is primarily useful if submitting a large batch of files to process separately..", "checkbox", new JSONObject()
- {{
- put("checked", true);
- }}, false)
- ));
- }
-
- @Override
- public boolean canProcess(SequenceOutputFile o)
- {
- return o.getFile() != null && _fileType.isType(o.getFile());
- }
-
- @Override
- public boolean doRunRemote()
- {
- return true;
- }
-
- @Override
- public boolean doRunLocal()
- {
- return false;
- }
-
- @Override
- public SequenceOutputProcessor getProcessor()
- {
- return new Processor();
- }
-
- @Override
- public boolean doSplitJobs()
- {
- return true;
- }
-
- @Override
- public boolean requiresSingleGenome()
- {
- return false;
- }
-
- public class Processor implements SequenceOutputHandler.SequenceOutputProcessor
- {
- @Override
- public void init(PipelineJob job, SequenceAnalysisJobSupport support, List inputFiles, JSONObject params, File outputDir, List actions, List outputsToCreate) throws UnsupportedOperationException, PipelineJobException
- {
- new CellRangerVDJUtils(job.getLogger(), outputDir).prepareHashingAndCiteSeqFilesIfNeeded(job, support,"readsetId", params.optBoolean("excludeFailedcDNA", true), false, true);
- }
-
- @Override
- public void processFilesOnWebserver(PipelineJob job, SequenceAnalysisJobSupport support, List inputFiles, JSONObject params, File outputDir, List actions, List outputsToCreate) throws UnsupportedOperationException, PipelineJobException
- {
-
- }
-
- @Override
- public void processFilesRemote(List inputFiles, SequenceOutputHandler.JobContext ctx) throws UnsupportedOperationException, PipelineJobException
- {
- RecordedAction action = new RecordedAction(getName());
-
- Map readsetToCiteSeq = CellRangerVDJUtils.getCachedCiteSeqReadsetMap(ctx.getSequenceSupport());
- ctx.getLogger().debug("total cached readset to GEX/citeseq pairs: " + readsetToCiteSeq.size());
-
- for (SequenceOutputFile so : inputFiles)
- {
- ctx.getLogger().info("processing file: " + so.getName());
-
- File barcodes = SeuratCellHashingHandler.getBarcodesFromSeurat(so.getFile());
-
- Readset rs = ctx.getSequenceSupport().getCachedReadset(so.getReadset());
- if (rs == null)
- {
- throw new PipelineJobException("Unable to find readset for outputfile: " + so.getRowid());
- }
- else if (rs.getReadsetId() == null)
- {
- throw new PipelineJobException("Readset lacks a rowId for outputfile: " + so.getRowid());
- }
-
- Readset citeseqReadset = ctx.getSequenceSupport().getCachedReadset(readsetToCiteSeq.get(rs.getReadsetId()));
- if (citeseqReadset == null)
- {
- throw new PipelineJobException("Unable to find Cite-seq readset for GEX readset: " + rs.getReadsetId());
- }
-
- File adtWhitelist = CellRangerVDJUtils.getValidCiteSeqBarcodeFile(ctx.getSourceDirectory(), so.getReadset());
- File citeSeqMatrix = CellRangerCellHashingHandler.processBarcodeFile(ctx, barcodes, rs, citeseqReadset, so.getLibrary_id(), action, getClientCommandArgs(ctx.getParams()), false, CATEGORY, true, adtWhitelist, false);
- if (!citeSeqMatrix.exists())
- {
- throw new PipelineJobException("Unable to find expected file: " + citeSeqMatrix.getPath());
- }
- }
-
- ctx.addActions(action);
- }
- }
-}
diff --git a/variantdb/module.properties b/variantdb/module.properties
index 251fad2e6..b3faf5d00 100644
--- a/variantdb/module.properties
+++ b/variantdb/module.properties
@@ -1,3 +1,2 @@
ModuleClass: org.labkey.variantdb.VariantDBModule
-ConsolidateScripts: false
ManageVersion: false
diff --git a/variantdb/resources/credits/dependencies.txt b/variantdb/resources/credits/dependencies.txt
new file mode 100644
index 000000000..4e8ead70b
--- /dev/null
+++ b/variantdb/resources/credits/dependencies.txt
@@ -0,0 +1,3 @@
+# direct external dependencies for project :server:modules:BimberLabKeyModules:variantdb
+commons-net-3.5.jar
+commons-math3-3.6.1.jar
diff --git a/variantdb/src/org/labkey/variantdb/VariantDBManager.java b/variantdb/src/org/labkey/variantdb/VariantDBManager.java
index 984551cb2..8a5445e20 100644
--- a/variantdb/src/org/labkey/variantdb/VariantDBManager.java
+++ b/variantdb/src/org/labkey/variantdb/VariantDBManager.java
@@ -149,71 +149,66 @@ public void exec(ResultSet rs) throws SQLException
final Pair matches = Pair.of(0, 0);
TableSelector variantTs = new TableSelector(VariantDBSchema.getInstance().getSchema().getTable(VariantDBSchema.TABLE_VARIANTS), variantFilter, null);
- variantTs.forEach(new Selector.ForEachBlock()
- {
- @Override
- public void exec(Variant v) throws SQLException
+ variantTs.forEach(Variant.class, v -> {
+ String name = resolveSequenceName(v.getSequenceId());
+ if (name != null)
{
- String name = resolveSequenceName(v.getSequenceId());
- if (name != null)
- {
- matches.second++;
+ matches.second++;
- //only delete once
- deletePs.setString(1, v.getObjectid());
- deletePs.addBatch();
+ //only delete once
+ deletePs.setString(1, v.getObjectid());
+ deletePs.addBatch();
- v.setSequenceName(name);
- for (Integer targetId : liftOverMap.keySet())
+ v.setSequenceName(name);
+ for (Integer targetId : liftOverMap.keySet())
+ {
+ LiftedVariant lv = VariantDBManager.get().liftOverVariant(liftOverMap.get(targetId), v, chainFileMap.get(targetId));
+ if (lv.successfulLiftover())
{
- LiftedVariant lv = VariantDBManager.get().liftOverVariant(liftOverMap.get(targetId), v, chainFileMap.get(targetId));
- if (lv.successfulLiftover())
- {
- matches.first++;
- }
-
- //variantid, sequenceid, startPosition, endPosition, reference, allele, referenceVariantId, referenceAlleleId, batchId, chainFile, created, createdBy, modified, modifiedBy
- insertPs.setString(1, v.getObjectid());
- if (lv.successfulLiftover())
- {
- insertPs.setInt(2, lv.getSequenceId());
- insertPs.setInt(3, lv.getStartPosition());
- insertPs.setInt(4, lv.getEndPosition());
- }
- else
- {
- insertPs.setInt(2, -1);
- insertPs.setInt(3, 0);
- insertPs.setInt(4, 0);
- }
- insertPs.setString(5, null);
- insertPs.setString(6, null);
-
- insertPs.setString(7, v.getReferenceVariantId());
- insertPs.setString(8, v.getReferenceAlleleId());
- insertPs.setString(9, batchId);
- insertPs.setInt(10, lv.getChainFile());
- insertPs.setDate(11, new Date(System.currentTimeMillis()));
- insertPs.setInt(12, u.getUserId());
- insertPs.setDate(13, new Date(System.currentTimeMillis()));
- insertPs.setInt(14, u.getUserId());
-
- insertPs.addBatch();
+ matches.first++;
}
- if (matches.second % batchSize == 0)
+ //variantid, sequenceid, startPosition, endPosition, reference, allele, referenceVariantId, referenceAlleleId, batchId, chainFile, created, createdBy, modified, modifiedBy
+ insertPs.setString(1, v.getObjectid());
+ if (lv.successfulLiftover())
+ {
+ insertPs.setInt(2, lv.getSequenceId());
+ insertPs.setInt(3, lv.getStartPosition());
+ insertPs.setInt(4, lv.getEndPosition());
+ }
+ else
{
- log.info("processed: " + matches.second + " variants");
- deletePs.executeBatch();
- insertPs.executeBatch();
+ insertPs.setInt(2, -1);
+ insertPs.setInt(3, 0);
+ insertPs.setInt(4, 0);
}
+ insertPs.setString(5, null);
+ insertPs.setString(6, null);
+
+ insertPs.setString(7, v.getReferenceVariantId());
+ insertPs.setString(8, v.getReferenceAlleleId());
+ insertPs.setString(9, batchId);
+ insertPs.setInt(10, lv.getChainFile());
+ insertPs.setDate(11, new Date(System.currentTimeMillis()));
+ insertPs.setInt(12, u.getUserId());
+ insertPs.setDate(13, new Date(System.currentTimeMillis()));
+ insertPs.setInt(14, u.getUserId());
+
+ insertPs.addBatch();
}
- else
+
+ if (matches.second % batchSize == 0)
{
- log.error("unable to resolve sequenceId: " + v.getSequenceId());
+ log.info("processed: " + matches.second + " variants");
+ deletePs.executeBatch();
+ insertPs.executeBatch();
}
}
- }, Variant.class);
+ else
+ {
+ log.error("unable to resolve sequenceId: " + v.getSequenceId());
+ }
+ });
//execute any remaining commands
log.info("processed: " + matches.second + " variants");
diff --git a/variantdb/src/org/labkey/variantdb/analysis/GBSAnalysisHandler.java b/variantdb/src/org/labkey/variantdb/analysis/GBSAnalysisHandler.java
index 773f99e05..1356bec46 100644
--- a/variantdb/src/org/labkey/variantdb/analysis/GBSAnalysisHandler.java
+++ b/variantdb/src/org/labkey/variantdb/analysis/GBSAnalysisHandler.java
@@ -80,16 +80,16 @@ public SequenceOutputProcessor getProcessor()
public class Processor implements SequenceOutputProcessor
{
@Override
- public void init(PipelineJob job, SequenceAnalysisJobSupport support, List inputFiles, JSONObject params, File outputDir, List actions, List outputsToCreate) throws UnsupportedOperationException, PipelineJobException
+ public void init(JobContext ctx, List inputFiles, List actions, List outputsToCreate) throws UnsupportedOperationException, PipelineJobException
{
for (ToolParameterDescriptor pd : getParameters())
{
- if (params.containsKey(pd.getName()) && !StringUtils.isEmpty(params.getString(pd.getName())))
+ if (ctx.getParams().containsKey(pd.getName()) && !StringUtils.isEmpty(ctx.getParams().getString(pd.getName())))
{
- ExpData d = ExperimentService.get().getExpData(params.getInt(pd.getName()));
+ ExpData d = ExperimentService.get().getExpData(ctx.getParams().getInt(pd.getName()));
if (d != null)
{
- support.cacheExpData(d);
+ ctx.getSequenceSupport().cacheExpData(d);
}
}
}
@@ -98,7 +98,7 @@ public void init(PipelineJob job, SequenceAnalysisJobSupport support, List inputFiles, JSONObject params, File outputDir, List actions, List outputsToCreate) throws UnsupportedOperationException, PipelineJobException
+ public void init(JobContext ctx, List inputFiles, List actions, List outputsToCreate) throws UnsupportedOperationException, PipelineJobException
{
//find genome
Set ids = new HashSet<>();
@@ -166,7 +166,7 @@ public void init(PipelineJob job, SequenceAnalysisJobSupport support, List pedigreeRecords = generatePedigree(job, params);
+ List pedigreeRecords = generatePedigree(ctx.getJob(), ctx.getParams());
- File gatkPed = new File(job.getJobSupport(FileAnalysisJobSupport.class).getAnalysisDirectory(), "gatkPed.ped");
- File morganPed = new File(job.getJobSupport(FileAnalysisJobSupport.class).getAnalysisDirectory(), "morgan.ped");
+ File gatkPed = new File(ctx.getJob().getJobSupport(FileAnalysisJobSupport.class).getAnalysisDirectory(), "gatkPed.ped");
+ File morganPed = new File(ctx.getSourceDirectory(), "morgan.ped");
try (PrintWriter gatkWriter = PrintWriters.getPrintWriter(gatkPed); PrintWriter morganWriter = PrintWriters.getPrintWriter(morganPed))
{
morganWriter.write("input pedigree size " + pedigreeRecords.size() + '\n');
@@ -204,39 +204,39 @@ public void init(PipelineJob job, SequenceAnalysisJobSupport support, List inputFiles, JSONObject params, File outputDir, List actions, List outputsToCreate) throws UnsupportedOperationException, PipelineJobException
+ public void init(JobContext ctx, List inputFiles, List actions, List outputsToCreate) throws UnsupportedOperationException, PipelineJobException
{
for (ToolParameterDescriptor pd : getParameters())
{
- if (params.containsKey(pd.getName()) && !StringUtils.isEmpty(params.getString(pd.getName())))
+ if (ctx.getParams().containsKey(pd.getName()) && !StringUtils.isEmpty(ctx.getParams().getString(pd.getName())))
{
- ExpData d = ExperimentService.get().getExpData(params.getInt(pd.getName()));
+ ExpData d = ExperimentService.get().getExpData(ctx.getParams().getInt(pd.getName()));
if (d != null)
{
- support.cacheExpData(d);
+ ctx.getSequenceSupport().cacheExpData(d);
}
}
}
@@ -95,7 +95,7 @@ public void init(PipelineJob job, SequenceAnalysisJobSupport support, List