diff --git a/mGAP/resources/folderTypes/mGAP.folderType.xml b/mGAP/resources/folderTypes/mGAP.folderType.xml index bee390d74..bf7576130 100644 --- a/mGAP/resources/folderTypes/mGAP.folderType.xml +++ b/mGAP/resources/folderTypes/mGAP.folderType.xml @@ -84,6 +84,10 @@ mGAP Variant Releases body + + mGAP Release Notes + body + mGAP Gene Search right diff --git a/mGAP/resources/views/contact.html b/mGAP/resources/views/contact.html index 1b110d8cc..e1f2f6b53 100644 --- a/mGAP/resources/views/contact.html +++ b/mGAP/resources/views/contact.html @@ -49,7 +49,7 @@ success: function(response){ console.log(response); - Ext4.Msg.alert('Success', 'An account has been requested. You should receive a reply shortly.', function(){ + Ext4.Msg.alert('Success', 'Your request has been sent. You should receive a reply shortly.', function(){ window.location = LABKEY.ActionURL.getContextPath() + '/'; }); }, diff --git a/mGAP/resources/views/releaseNotes.html b/mGAP/resources/views/releaseNotes.html new file mode 100644 index 000000000..110244e37 --- /dev/null +++ b/mGAP/resources/views/releaseNotes.html @@ -0,0 +1,12 @@ +

Release 2.0:

+ + +

Future Plans:

+ \ No newline at end of file diff --git a/mGAP/resources/views/releaseNotes.view.xml b/mGAP/resources/views/releaseNotes.view.xml new file mode 100644 index 000000000..c9a2b3e33 --- /dev/null +++ b/mGAP/resources/views/releaseNotes.view.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/mGAP/resources/views/releaseNotes.webpart.xml b/mGAP/resources/views/releaseNotes.webpart.xml new file mode 100644 index 000000000..f7f58d800 --- /dev/null +++ b/mGAP/resources/views/releaseNotes.webpart.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/mGAP/resources/views/variants.html b/mGAP/resources/views/variants.html index 3d66a6667..f59586d79 100644 --- a/mGAP/resources/views/variants.html +++ b/mGAP/resources/views/variants.html @@ -9,7 +9,7 @@ title: 'Variant Catalog Releases', schemaName: 'mgap', queryName: 'variantCatalogReleases', - maxRows: 50, + maxRows: 3, showRecordSelectors: false, showDetailsColumn: false, buttonBar: {position: 'none', includeStandardButtons: false, items: []} diff --git a/mGAP/src/org/labkey/mgap/pipeline/mGapReleaseGenerator.java b/mGAP/src/org/labkey/mgap/pipeline/mGapReleaseGenerator.java index 848a0a09d..a3fd61c17 100644 --- a/mGAP/src/org/labkey/mgap/pipeline/mGapReleaseGenerator.java +++ b/mGAP/src/org/labkey/mgap/pipeline/mGapReleaseGenerator.java @@ -953,7 +953,7 @@ private void inspectAndSummarizeVcf(JobContext ctx, File vcfInput, GeneToNameTra File interestingVariantTable = getVariantTableName(ctx, vcfInput); try (VCFFileReader reader = new VCFFileReader(vcfInput); CloseableIterator it = reader.iterator(); CSVWriter writer = new CSVWriter(PrintWriters.getPrintWriter(interestingVariantTable), '\t', CSVWriter.NO_QUOTE_CHARACTER)) { - writer.writeNext(new String[]{"Chromosome", "Position", "Reference", "Allele", "Source", "Reason", "Description", "Overlapping Gene(s)", "OMIM Entries", "OMIM Phenotypes", "AF", "CADD_PH"}); + writer.writeNext(new String[]{"Chromosome", "Position", "Reference", "Allele", "Source", "Reason", "Description", "Overlapping Gene(s)", "OMIM Entries", "OMIM Phenotypes", "AF", "Identifier", "CADD_PH"}); while (it.hasNext()) { Set> queuedLines = new LinkedHashSet<>(); @@ -1113,7 +1113,7 @@ private void inspectAndSummarizeVcf(JobContext ctx, File vcfInput, GeneToNameTra try { String allele = clnAlleles.get(i); - maybeWriteVariantLine(queuedLines, vc, allele, "ClinVar", diseaseSplit.get(j), description, overlappingGenes, omims, omimds, ctx.getLogger(), "ClinVar:" + clnAlleleIds.get(j)); + maybeWriteVariantLine(queuedLines, vc, allele, "ClinVar", diseaseSplit.get(j), description, overlappingGenes, omims, omimds, ctx.getLogger(), "ClinVar:" + clnAlleleIds.get(i)); } catch (IndexOutOfBoundsException e) diff --git a/mcc/resources/etls/snprc.xml b/mcc/resources/etls/snprc.xml index 127947d46..4a959b289 100644 --- a/mcc/resources/etls/snprc.xml +++ b/mcc/resources/etls/snprc.xml @@ -1,14 +1,121 @@ SNPRC_Data + + SNPRC Clinical/Demographics Data - + Copy to target - - + + + AnimalId + date + gender + geographic_origin + birth + death + species + objectid + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Copy to target + + + AnimalId + date + weight + objectid + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/mcc/resources/etls/wnprc.xml b/mcc/resources/etls/wnprc.xml index e5f1ee6d1..f8efab248 100644 --- a/mcc/resources/etls/wnprc.xml +++ b/mcc/resources/etls/wnprc.xml @@ -3,17 +3,108 @@ WNPRC_Data WNPRC Clinical/Demographics Data - + Copy to target - - + + + Id + date + gender + geographic_origin + birth + death + species + objectid + + + + + + + + + + Copy to target + + + Id + date + parent + relationship + method + objectid + + + + + + + + + + + + Copy to target + + + Id + date + gender + species + geographic_origin + dam + sire + objectid + + + + + + + + + + + + Copy to target + + + Id + date + weight + objectid + + + + + + + + + + + Copy to target + + + Id + date + cause + objectid + + + + + + + + - + - - + + diff --git a/mcc/resources/module.xml b/mcc/resources/module.xml new file mode 100644 index 000000000..b07899cb8 --- /dev/null +++ b/mcc/resources/module.xml @@ -0,0 +1,21 @@ + + + + false + This is the path to the container holding the primary MCC Study. Use of slashes is very important - it should be in the format '/myProject/mcc' + + ADMIN + + + + false + This is a comma separated list of LabKey user names of users that should be notified by email when requests are submitted through MCC. + + ADMIN + + + + + + + diff --git a/mcc/resources/queries/study/animalGroupMembership.js b/mcc/resources/queries/study/animalGroupMembership.js new file mode 100644 index 000000000..c8510ccb2 --- /dev/null +++ b/mcc/resources/queries/study/animalGroupMembership.js @@ -0,0 +1,14 @@ +/* + * Copyright (c) 2011-2014 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0 + */ + +require("ehr/triggers").initScript(this); + +function onInit(event, helper){ + helper.setScriptOptions({ + allowFutureDates: true, + removeTimeFromDate: true + }); +} \ No newline at end of file diff --git a/mcc/resources/queries/study/deaths.js b/mcc/resources/queries/study/deaths.js new file mode 100644 index 000000000..9dda9861c --- /dev/null +++ b/mcc/resources/queries/study/deaths.js @@ -0,0 +1,97 @@ +/* + * Copyright (c) 2018-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0 + */ + +require("ehr/triggers").initScript(this); +EHR.Server.Utils = require("ehr/utils").EHR.Server.Utils; + +var demographicsUpdates = []; +var validIds = []; + +function onInit(event, helper){ + helper.setScriptOptions({ + requiresStatusRecalc: true + }); + + helper.decodeExtraContextProperty('deathsInTransaction'); + + // Cache valid Ids for check on each row + LABKEY.Query.selectRows({ + requiredVersion: 9.1, + schemaName: 'study', + queryName: 'demographics', + columns: ['Id'], + scope: this, + success: function (results) { + if (!results || !results.rows || results.rows.length < 1) + return; + + for(var i=0; i 0) { + console.log('updating demographics death date for ' + demographicsUpdates.length + " animals"); + helper.getJavaHelper().updateDemographicsRecord(demographicsUpdates); + } + + var deaths = helper.getDeaths(); + if (deaths){ + var ids = []; + for (var id in deaths){ + ids.push(id); + } + + if (!helper.isETL()) { + console.log('sending death notification'); + helper.getJavaHelper().sendDeathNotification(ids); + } + } +} \ No newline at end of file diff --git a/mcc/resources/queries/study/demographics.js b/mcc/resources/queries/study/demographics.js new file mode 100644 index 000000000..992138d43 --- /dev/null +++ b/mcc/resources/queries/study/demographics.js @@ -0,0 +1,23 @@ +/* + * Copyright (c) 2010-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0 + */ + +require("ehr/triggers").initScript(this); + +function onInit(event, helper){ + helper.setScriptOptions({ + allowAnyId: true, + requiresStatusRecalc: false, + allowDatesInDistantPast: true + }); +} + +function onUpsert(helper, scriptErrors, row, oldRow){ + //NOTE: this should be getting set by the birth, death, arrival & departure tables + //ALSO: it should be rare to insert directly into this table. usually this record will be created by inserting into either birth or arrival + if (!row.calculated_status && !helper.isETL()){ + row.calculated_status = helper.getJavaHelper().getCalculatedStatusValue(row.Id); + } +} \ No newline at end of file diff --git a/mcc/resources/queries/study/encounters.js b/mcc/resources/queries/study/encounters.js new file mode 100644 index 000000000..fbb17ad72 --- /dev/null +++ b/mcc/resources/queries/study/encounters.js @@ -0,0 +1,13 @@ +/* + * Copyright (c) 2018-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0 + */ + +require("ehr/triggers").initScript(this); + +function onUpsert(helper, scriptErrors, row, oldRow){ + if (!helper.isETL() && row.date && !row.requestdate){ + row.requestdate = row.date; + } +} \ No newline at end of file diff --git a/mcc/resources/queries/study/flags.js b/mcc/resources/queries/study/flags.js new file mode 100644 index 000000000..454371e4c --- /dev/null +++ b/mcc/resources/queries/study/flags.js @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2012-2018 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0 + */ + +require("ehr/triggers").initScript(this); + +function onInit(event, helper){ + helper.setScriptOptions({ + allowFutureDates: true, + removeTimeFromDate: true, + removeTimeFromEndDate: true + }); +} + +function onUpsert(helper, scriptErrors, row, oldRow){ + //if the animal is not at the center, automatically set the enddate + if (!helper.isETL() && row.Id && !row.enddate){ + EHR.Server.Utils.findDemographics({ + participant: row.Id, + helper: helper, + scope: this, + callback: function(data){ + if (!data) + return; + + if (data && data.calculated_status && data.calculated_status != 'Alive'){ + row.enddate = data.death || data.departure; + } + } + }); + + } + + if (!helper.isETL() && row.Id && row.date && row.flag){ + var active = helper.getJavaHelper().getOverlappingFlags(row.Id, row.flag, row.objectid || null, row.date); + if (active > 0){ + EHR.Server.Utils.addError(scriptErrors, 'flag', 'There are already ' + active + ' active flag(s) of the same type spanning this date.', 'INFO'); + } + } +} + +function onAfterInsert(helper, errors, row){ + //if this category enforces only a single active flag at once, enforce it + //note: if this flag has a future date, preemptively set enddate on flags, since isActive should handle this + if (!helper.isETL() && row.Id && row.flag && !row.enddate && row.date){ + helper.getJavaHelper().ensureSingleFlagCategoryActive(row.Id, row.flag, row.objectId, row.date); + } +} \ No newline at end of file diff --git a/mcc/resources/queries/study/labwork.js b/mcc/resources/queries/study/labwork.js new file mode 100644 index 000000000..d4075c56c --- /dev/null +++ b/mcc/resources/queries/study/labwork.js @@ -0,0 +1,8 @@ +/* + * Copyright (c) 2018-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0 + */ + +require("ehr/triggers").initScript(this); + diff --git a/mcc/resources/queries/study/labworkResults.js b/mcc/resources/queries/study/labworkResults.js new file mode 100644 index 000000000..2004fb6cf --- /dev/null +++ b/mcc/resources/queries/study/labworkResults.js @@ -0,0 +1,14 @@ +/* + * Copyright (c) 2012-2018 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0 + */ + +require("ehr/triggers").initScript(this); + +function onInit(event, helper){ + helper.setScriptOptions({ + removeTimeFromDate: false, + allowDatesInDistantPast: true + }); +} \ No newline at end of file diff --git a/mcc/resources/queries/study/medicationAdministration.js b/mcc/resources/queries/study/medicationAdministration.js new file mode 100644 index 000000000..d4075c56c --- /dev/null +++ b/mcc/resources/queries/study/medicationAdministration.js @@ -0,0 +1,8 @@ +/* + * Copyright (c) 2018-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0 + */ + +require("ehr/triggers").initScript(this); + diff --git a/mcc/resources/queries/study/medicationOrders.js b/mcc/resources/queries/study/medicationOrders.js new file mode 100644 index 000000000..d4075c56c --- /dev/null +++ b/mcc/resources/queries/study/medicationOrders.js @@ -0,0 +1,8 @@ +/* + * Copyright (c) 2018-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0 + */ + +require("ehr/triggers").initScript(this); + diff --git a/mcc/resources/queries/study/parentage.js b/mcc/resources/queries/study/parentage.js new file mode 100644 index 000000000..0cea7a1c5 --- /dev/null +++ b/mcc/resources/queries/study/parentage.js @@ -0,0 +1,13 @@ +/* + * Copyright (c) 2013 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0 + */ + +require("ehr/triggers").initScript(this); + +function onInit(event, helper){ + helper.setScriptOptions({ + lookupValidationFields: ['relationship', 'method'] + }); +} diff --git a/mcc/resources/queries/study/samples.js b/mcc/resources/queries/study/samples.js new file mode 100644 index 000000000..64a117a3e --- /dev/null +++ b/mcc/resources/queries/study/samples.js @@ -0,0 +1,7 @@ +/* + * Copyright (c) 2011-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0 + */ + +require("ehr/triggers").initScript(this); \ No newline at end of file diff --git a/mcc/resources/queries/study/weight.js b/mcc/resources/queries/study/weight.js new file mode 100644 index 000000000..a515311e1 --- /dev/null +++ b/mcc/resources/queries/study/weight.js @@ -0,0 +1,89 @@ +/* + * Copyright (c) 2010-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0 + */ + +require("ehr/triggers").initScript(this); + +function onInit(event, helper){ + helper.setScriptOptions({ + allowAnyId: true, + allowDeadIds: true, + skipIdFormatCheck: true + }); + + helper.registerRowProcessor(function(helper, row){ + if (!row) + return; + + if (!row.Id || !row.weight){ + return; + } + + var weightInTransaction = helper.getProperty('weightInTransaction'); + weightInTransaction = weightInTransaction || {}; + weightInTransaction[row.Id] = weightInTransaction[row.Id] || []; + + var shouldAdd = true; + if (row.objectid){ + LABKEY.ExtAdapter.each(weightInTransaction[row.Id], function(r){ + if (r.objectid === row.objectid){ + if (r.weight !== row.weight){ + r.weight = row.weight; + } + else { + shouldAdd = false; + return false; + } + } + }, this); + } + + if (shouldAdd){ + weightInTransaction[row.Id].push({ + objectid: row.objectid, + date: row.date, + qcstate: row.QCState, + weight: row.weight + }); + } + + helper.setProperty('weightInTransaction', weightInTransaction); + }); +} + +function onUpsert(helper, scriptErrors, row, oldRow){ + if (!row.weight){ + EHR.Server.Utils.addError(scriptErrors, 'weight', 'This field is required', 'WARN'); + } + + // warn if more than 10% different from last weight + // the highest error this can produce is WARN. therefore skip this check if we would ignore it anyway in order to save the overhead. + // this would normally occur when finalizing a form + if (!helper.isETL() && row.Id && row.weight && EHR.Server.Utils.shouldIncludeError('WARN', helper.getErrorThreshold(), helper)){ + EHR.Server.Utils.findDemographics({ + participant: row.Id, + helper: helper, + scope: this, + callback: function(data){ + if (!data) + return; + + if (data.mostRecentWeight && (row.weight <= data.mostRecentWeight * 0.9)){ + EHR.Server.Utils.addError(scriptErrors, 'weight', 'Weight drop of >10%. Last weight ' + data.mostRecentWeight + ' kg', 'INFO'); + } + else if (data.mostRecentWeight && (row.weight >= data.mostRecentWeight / 0.9)){ + EHR.Server.Utils.addError(scriptErrors, 'weight', 'Weight gain of >10%. Last weight ' + data.mostRecentWeight + ' kg', 'INFO'); + } + + if (data && data.species){ + var msg = helper.getJavaHelper().verifyWeightRange(row.id, row.weight, data.species); + if (msg != null){ + EHR.Server.Utils.addError(scriptErrors, 'weight', msg, 'WARN'); + } + } + } + }); + } +} \ No newline at end of file diff --git a/mcc/resources/queries/wnprcSource/birth.sql b/mcc/resources/queries/wnprcSource/birth.sql new file mode 100644 index 000000000..06808b9af --- /dev/null +++ b/mcc/resources/queries/wnprcSource/birth.sql @@ -0,0 +1,6 @@ +SELECT + +Id, date, gender, species, geographic_origin, dam, sire, objectid, modified + +FROM "/WNPRC/EHR/".study.birth +WHERE species = 'Marmoset'; \ No newline at end of file diff --git a/mcc/resources/queries/wnprcSource/deaths.sql b/mcc/resources/queries/wnprcSource/deaths.sql new file mode 100644 index 000000000..94c301bb7 --- /dev/null +++ b/mcc/resources/queries/wnprcSource/deaths.sql @@ -0,0 +1,8 @@ +SELECT + + Id, date, + cause, + objectid, modified + +FROM "/WNPRC/EHR/".study.weight +WHERE Id.demographics.species = 'Marmoset'; \ No newline at end of file diff --git a/mcc/resources/queries/wnprcSource/demographics.sql b/mcc/resources/queries/wnprcSource/demographics.sql new file mode 100644 index 000000000..7a860fef3 --- /dev/null +++ b/mcc/resources/queries/wnprcSource/demographics.sql @@ -0,0 +1,6 @@ +SELECT + +Id, date, gender, geographic_origin, birth, death, species, objectid, modified + +FROM "/WNPRC/EHR/".study.demographics +WHERE species = 'Marmoset'; \ No newline at end of file diff --git a/mcc/resources/queries/wnprcSource/parentage.sql b/mcc/resources/queries/wnprcSource/parentage.sql new file mode 100644 index 000000000..a8fe2e80c --- /dev/null +++ b/mcc/resources/queries/wnprcSource/parentage.sql @@ -0,0 +1,27 @@ +SELECT + + Id, + date, + sire as parent, + 'Sire' as relationship, + 'Observed' as method, + cast(objectid as varchar) || '-Sire' as objectid, + modified + +FROM "/WNPRC/EHR/".study.demographics +WHERE species = 'Marmoset' and sire is not null + +UNION ALL + +SELECT + + Id, + date, + sire as parent, + 'Dam' as relationship, + 'Observed' as method, + cast(objectid as varchar) || '-Dam' as objectid, + modified + +FROM "/WNPRC/EHR/".study.demographics +WHERE species = 'Marmoset' and dam is not null \ No newline at end of file diff --git a/mcc/resources/queries/wnprcSource/weight.sql b/mcc/resources/queries/wnprcSource/weight.sql new file mode 100644 index 000000000..db7bce107 --- /dev/null +++ b/mcc/resources/queries/wnprcSource/weight.sql @@ -0,0 +1,8 @@ +SELECT + + Id, date, + weight, + objectid, modified + +FROM "/WNPRC/EHR/".study.weight +WHERE Id.demographics.species = 'Marmoset'; \ No newline at end of file diff --git a/mcc/resources/referenceStudy/datasets/datasets_manifest.xml b/mcc/resources/referenceStudy/datasets/datasets_manifest.xml index 3600f8261..47d75b91c 100644 --- a/mcc/resources/referenceStudy/datasets/datasets_manifest.xml +++ b/mcc/resources/referenceStudy/datasets/datasets_manifest.xml @@ -18,7 +18,10 @@ - + + + + @@ -30,26 +33,17 @@ - + - - - - - - - - - diff --git a/mcc/resources/referenceStudy/datasets/datasets_metadata.xml b/mcc/resources/referenceStudy/datasets/datasets_metadata.xml index 8f19b36f9..480c6a89d 100644 --- a/mcc/resources/referenceStudy/datasets/datasets_metadata.xml +++ b/mcc/resources/referenceStudy/datasets/datasets_metadata.xml @@ -135,33 +135,12 @@ varchar - - varchar - - - varchar - varchar - - varchar - - - varchar - - - timestamp - - - integer - varchar - - timestamp - varchar urn:ehr.labkey.org/#ObjectId @@ -207,36 +186,6 @@ urn:ehr.labkey.org/#ObjectId true - - varchar - urn:ehr.labkey.org/#VetReview - - - timestamp - urn:ehr.labkey.org/#VetReviewDate - - - varchar - - - varchar - - - varchar - - - varchar - - - timestamp - urn:ehr.labkey.org/#DateRequested - - - varchar - - - varchar - integer @@ -264,6 +213,9 @@ integer urn:ehr.labkey.org/#Project + + varchar + varchar @@ -272,35 +224,6 @@ urn:ehr.labkey.org/#ObjectId true - - varchar - urn:ehr.labkey.org/#VetReview - - - timestamp - urn:ehr.labkey.org/#VetReviewDate - - - varchar - - - varchar - - - varchar - - - varchar - - - varchar - - - varchar - - - varchar - timestamp urn:ehr.labkey.org/#EndDate @@ -308,16 +231,10 @@ timestamp - - varchar - - - varchar - Clinical Remarks - +
varchar @@ -389,12 +306,6 @@ varchar - - timestamp - - - varchar - Medication Administration
@@ -503,7 +414,7 @@ Weight - +
varchar @@ -524,33 +435,15 @@ varchar - - varchar - integer urn:ehr.labkey.org/#Project - - varchar - entityid urn:ehr.labkey.org/#ObjectId true - - varchar - - - varchar - - - varchar - - - varchar - varchar @@ -566,16 +459,13 @@ varchar - - varchar - timestamp - Clinpath Runs + Labwork
- +
varchar @@ -630,132 +520,9 @@ varchar - Hematology Results + Lab Results
- - - - varchar - http://cpas.labkey.com/Study#ParticipantId - - ptid - - - - timestamp - http://cpas.labkey.com/Study#VisitDate - http://cpas.labkey.com/Study#VisitDate - - - varchar - - - entityid - urn:ehr.labkey.org/#ObjectId - true - - - varchar - - - integer - urn:ehr.labkey.org/#Project - - - timestamp - urn:ehr.labkey.org/#EndDate - - - double - - - varchar - - - varchar - - - varchar - - - varchar - - - Parasitology Results -
- - - - varchar - http://cpas.labkey.com/Study#ParticipantId - - ptid - - - - timestamp - http://cpas.labkey.com/Study#VisitDate - http://cpas.labkey.com/Study#VisitDate - - - varchar - - - double - - - varchar - - - double - - - double - - - double - - - varchar - - - varchar - - - entityid - urn:ehr.labkey.org/#ObjectId - true - - - integer - urn:ehr.labkey.org/#Project - - - timestamp - urn:ehr.labkey.org/#EndDate - - - varchar - - - double - - - varchar - - - double - - - double - - - varchar - - - Urinalysis Results -
- +
varchar @@ -791,134 +558,9 @@ boolean - - varchar - - - varchar - - - varchar - - - varchar - - - varchar - - - varchar - - - varchar - - - integer - - - integer - - - varchar - - - varchar - - - varchar - - - timestamp - Arrival
- - - - varchar - http://cpas.labkey.com/Study#ParticipantId - - ptid - - - - timestamp - http://cpas.labkey.com/Study#VisitDate - http://cpas.labkey.com/Study#VisitDate - - - timestamp - urn:ehr.labkey.org/#EndDate - - - integer - urn:ehr.labkey.org/#Project - - - entityid - urn:ehr.labkey.org/#ObjectId - true - - - timestamp - - - integer - - - integer - - - integer - - - varchar - - - timestamp - - - timestamp - - - varchar - - - Assignment -
- - - - varchar - http://cpas.labkey.com/Study#ParticipantId - - ptid - - - - timestamp - http://cpas.labkey.com/Study#VisitDate - http://cpas.labkey.com/Study#VisitDate - - - timestamp - urn:ehr.labkey.org/#EndDate - - - integer - - - entityid - urn:ehr.labkey.org/#ObjectId - true - - - varchar - - - Animal Group Members -
@@ -936,9 +578,6 @@ varchar - - varchar - varchar urn:ehr.labkey.org/#ObjectId @@ -951,21 +590,6 @@ timestamp urn:ehr.labkey.org/#EndDate - - varchar - - - varchar - - - varchar - - - varchar - - - boolean - Deaths
@@ -1013,151 +637,10 @@ varchar - - Demographics - - - - + varchar - http://cpas.labkey.com/Study#ParticipantId - - ptid - - - - timestamp - http://cpas.labkey.com/Study#VisitDate - http://cpas.labkey.com/Study#VisitDate - - - varchar - - - varchar - - - entityid - urn:ehr.labkey.org/#ObjectId - true - - - integer - urn:ehr.labkey.org/#Project - - - timestamp - urn:ehr.labkey.org/#EndDate - - - Departure -
- - - - varchar - http://cpas.labkey.com/Study#ParticipantId - - ptid - - - - timestamp - http://cpas.labkey.com/Study#VisitDate - http://cpas.labkey.com/Study#VisitDate - - - timestamp - urn:ehr.labkey.org/#EndDate - - - varchar - - - varchar - - - integer - - - integer - - - integer - - - entityid - urn:ehr.labkey.org/#ObjectId - true - - - varchar - - - integer - urn:ehr.labkey.org/#Project - Housing -
- - - - varchar - http://cpas.labkey.com/Study#ParticipantId - - ptid - - - - timestamp - http://cpas.labkey.com/Study#VisitDate - http://cpas.labkey.com/Study#VisitDate - - - varchar - - - varchar - - - entityid - urn:ehr.labkey.org/#ObjectId - true - - - integer - urn:ehr.labkey.org/#Project - - - timestamp - urn:ehr.labkey.org/#EndDate - - - - - - - - - varchar - - - varchar - - - varchar - - - - - - double - - - boolean - - - Tissue Samples + Demographics
diff --git a/mcc/resources/schemas/dbscripts/postgresql/mcc-20.000-20.001.sql b/mcc/resources/schemas/dbscripts/postgresql/mcc-20.000-20.001.sql new file mode 100644 index 000000000..567bfef27 --- /dev/null +++ b/mcc/resources/schemas/dbscripts/postgresql/mcc-20.000-20.001.sql @@ -0,0 +1,18 @@ +CREATE TABLE mcc.userRequests ( + rowid serial, + email varchar(1000), + firstName varchar(1000), + lastName varchar(1000), + title varchar(1000), + institution varchar(1000), + reason varchar(4000), + userid userid, + + container entityid, + created timestamp, + createdby userid, + modified timestamp, + modifiedby userid, + + CONSTRAINT PK_userRequests PRIMARY KEY (rowid) +); \ No newline at end of file diff --git a/mcc/resources/schemas/dbscripts/sqlserver/mcc-20.000-20.001.sql b/mcc/resources/schemas/dbscripts/sqlserver/mcc-20.000-20.001.sql new file mode 100644 index 000000000..de27af5ef --- /dev/null +++ b/mcc/resources/schemas/dbscripts/sqlserver/mcc-20.000-20.001.sql @@ -0,0 +1,18 @@ +CREATE TABLE mcc.userRequests ( + rowid int identity(1,1), + email varchar(1000), + firstName varchar(1000), + lastName varchar(1000), + title varchar(1000), + institution varchar(1000), + reason varchar(4000), + userid userid, + + container entityid, + created datetime, + createdby userid, + modified datetime, + modifiedby userid, + + CONSTRAINT PK_userRequests PRIMARY KEY (rowid) +); \ No newline at end of file diff --git a/mcc/resources/schemas/mcc.xml b/mcc/resources/schemas/mcc.xml index 2bba6c71d..e69524513 100644 --- a/mcc/resources/schemas/mcc.xml +++ b/mcc/resources/schemas/mcc.xml @@ -17,4 +17,92 @@ --> \ No newline at end of file + xmlns="http://labkey.org/data/xml" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"> + + + + + + + + rowid + Requests For Logins + DETAILED + + + true + false + false + false + false + Request Id + + + Email + false + + + First Name + false + + + Last Name + false + + + Title + false + + + Institution + false + + + Reason For Request + false + + + false + + core + Users + UserId + + + + true + + + true + + + false + false + false + true + true + + + true + + + false + false + false + true + true + + + + ldk.context + /mcc/Security.js + + MCC.Security.approveUserRequests(dataRegionName); + + + +
+ + \ No newline at end of file diff --git a/mcc/resources/views/about.html b/mcc/resources/views/about.html new file mode 100644 index 000000000..436f532b2 --- /dev/null +++ b/mcc/resources/views/about.html @@ -0,0 +1,3 @@ +MCC is supported by NIH U24 xxxxxxx. +

+Please remember to cite this funding source in all publications that make use of MCC data. \ No newline at end of file diff --git a/mcc/resources/views/about.view.xml b/mcc/resources/views/about.view.xml new file mode 100644 index 000000000..48ca9ced8 --- /dev/null +++ b/mcc/resources/views/about.view.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/mcc/resources/views/about.webpart.xml b/mcc/resources/views/about.webpart.xml new file mode 100644 index 000000000..f40c189c7 --- /dev/null +++ b/mcc/resources/views/about.webpart.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/mcc/resources/views/contact.html b/mcc/resources/views/contact.html new file mode 100644 index 000000000..714f95170 --- /dev/null +++ b/mcc/resources/views/contact.html @@ -0,0 +1,75 @@ + \ No newline at end of file diff --git a/mcc/resources/views/contact.view.xml b/mcc/resources/views/contact.view.xml new file mode 100644 index 000000000..0fc3bfa31 --- /dev/null +++ b/mcc/resources/views/contact.view.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/mcc/resources/views/dashboard.html b/mcc/resources/views/dashboard.html new file mode 100644 index 000000000..014a425a7 --- /dev/null +++ b/mcc/resources/views/dashboard.html @@ -0,0 +1,9 @@ + \ No newline at end of file diff --git a/mcc/resources/views/dashboard.view.xml b/mcc/resources/views/dashboard.view.xml new file mode 100644 index 000000000..252242cd6 --- /dev/null +++ b/mcc/resources/views/dashboard.view.xml @@ -0,0 +1,9 @@ + + + + + + + + + \ No newline at end of file diff --git a/mcc/resources/views/dashboard.webpart.xml b/mcc/resources/views/dashboard.webpart.xml new file mode 100644 index 000000000..4e56bf8c1 --- /dev/null +++ b/mcc/resources/views/dashboard.webpart.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/mcc/resources/views/helpMenu.html b/mcc/resources/views/helpMenu.html new file mode 100644 index 000000000..932f848cc --- /dev/null +++ b/mcc/resources/views/helpMenu.html @@ -0,0 +1 @@ +We'd love to hear your feedback! Click here to send a help or feature request, or email mcc@ohsu.edu for any questions. \ No newline at end of file diff --git a/mcc/resources/views/helpMenu.view.xml b/mcc/resources/views/helpMenu.view.xml new file mode 100644 index 000000000..3960a69c1 --- /dev/null +++ b/mcc/resources/views/helpMenu.view.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/mcc/resources/views/helpMenu.webpart.xml b/mcc/resources/views/helpMenu.webpart.xml new file mode 100644 index 000000000..3f4624d6a --- /dev/null +++ b/mcc/resources/views/helpMenu.webpart.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/mcc/resources/views/login.html b/mcc/resources/views/login.html new file mode 100644 index 000000000..71bcbf7d3 --- /dev/null +++ b/mcc/resources/views/login.html @@ -0,0 +1,40 @@ + +
+
Sign In
+
+
+ + + + + Remember my email address + + +
+ + + + or + Request an account + +
+ + + +
+
+ \ No newline at end of file diff --git a/mcc/resources/views/login.view.xml b/mcc/resources/views/login.view.xml new file mode 100644 index 000000000..7ce47abe2 --- /dev/null +++ b/mcc/resources/views/login.view.xml @@ -0,0 +1,9 @@ + + + + + + + + + \ No newline at end of file diff --git a/mcc/resources/views/overview.html b/mcc/resources/views/overview.html new file mode 100644 index 000000000..f03b886bc --- /dev/null +++ b/mcc/resources/views/overview.html @@ -0,0 +1 @@ +This will hold the overview information for MCC. \ No newline at end of file diff --git a/mcc/resources/views/overview.view.xml b/mcc/resources/views/overview.view.xml new file mode 100644 index 000000000..f9e80a68a --- /dev/null +++ b/mcc/resources/views/overview.view.xml @@ -0,0 +1,10 @@ + + + + + + + + + + \ No newline at end of file diff --git a/mcc/resources/views/overview.webpart.xml b/mcc/resources/views/overview.webpart.xml new file mode 100644 index 000000000..c81f56bb6 --- /dev/null +++ b/mcc/resources/views/overview.webpart.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/mcc/resources/views/requestLogin.html b/mcc/resources/views/requestLogin.html new file mode 100644 index 000000000..6c3b1fb7f --- /dev/null +++ b/mcc/resources/views/requestLogin.html @@ -0,0 +1,117 @@ + \ No newline at end of file diff --git a/mcc/resources/views/requestLogin.view.xml b/mcc/resources/views/requestLogin.view.xml new file mode 100644 index 000000000..13c485f9f --- /dev/null +++ b/mcc/resources/views/requestLogin.view.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/mcc/resources/web/mcc/Security.js b/mcc/resources/web/mcc/Security.js new file mode 100644 index 000000000..5ab385833 --- /dev/null +++ b/mcc/resources/web/mcc/Security.js @@ -0,0 +1,49 @@ +Ext4.namespace('MCC.Security'); + +MCC.Security = new function(){ + return { + approveUserRequests: function(dataRegionName){ + var dr = LABKEY.DataRegions[dataRegionName]; + if (!dr){ + alert('Unable to find DataRegion with name: ' + dataRegionName); + return; + } + + var rowIds = dr.getChecked(); + if (!rowIds.length){ + alert('Must select one or more rows'); + return; + } + + Ext4.Msg.confirm('Approve Requests', 'You are able to approve ' + rowIds.length + ' user requests. Continue?', function(val){ + if (val === 'yes'){ + Ext4.Msg.wait('Loading...'); + LABKEY.Ajax.request({ + method: 'POST', + url: LABKEY.ActionURL.buildURL('mcc', 'approveUserRequests'), + params: { + requestIds: rowIds + }, + success: function(){ + Ext4.Msg.hide(); + Ext4.Msg.alert('Success', 'Requests approved!', function(){ + //note: drop view, so we see the newly added user(s) + LABKEY.DataRegions[dataRegionName].changeView(null); + }); + }, + failure: LDK.Utils.getErrorCallback({ + showAlertOnError: false, + scope: this, + callback: function(responseObj){ + if (responseObj.errorMsg){ + Ext4.Msg.alert('Error', responseObj.errorMsg); + } + } + }) + }); + + } + }, this); + } + } +}; \ No newline at end of file diff --git a/mcc/resources/web/mcc/dashboard.js b/mcc/resources/web/mcc/dashboard.js new file mode 100644 index 000000000..4c94d7e87 --- /dev/null +++ b/mcc/resources/web/mcc/dashboard.js @@ -0,0 +1,18 @@ +var MCC = {}; + +MCC.Dashboard = new function() { + return { + loadData: function () { + LABKEY.Query.selectRows({ + schemaName: 'study', + queryName: 'demographics', + columns: 'Id,birth,death,gender,species,Id/age/AgeFriendly', + success: function(results) { + console.log(results.rows); + }, + error: LDK.Utils.getErrorCallback(), + scope: this + }); + } + } +}; diff --git a/mcc/src/org/labkey/mcc/MccController.java b/mcc/src/org/labkey/mcc/MccController.java index 0967188da..ea84cef04 100644 --- a/mcc/src/org/labkey/mcc/MccController.java +++ b/mcc/src/org/labkey/mcc/MccController.java @@ -16,22 +16,508 @@ package org.labkey.mcc; -import org.labkey.api.action.SimpleViewAction; +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.labkey.api.action.ApiSimpleResponse; +import org.labkey.api.action.MutatingApiAction; import org.labkey.api.action.SpringActionController; +import org.labkey.api.data.Container; +import org.labkey.api.data.CoreSchema; +import org.labkey.api.data.DbScope; +import org.labkey.api.data.SimpleFilter; +import org.labkey.api.data.Table; +import org.labkey.api.data.TableInfo; +import org.labkey.api.data.TableSelector; +import org.labkey.api.module.AllowedDuringUpgrade; +import org.labkey.api.query.DetailsURL; +import org.labkey.api.query.FieldKey; +import org.labkey.api.security.IgnoresTermsOfUse; +import org.labkey.api.security.MutableSecurityPolicy; +import org.labkey.api.security.RequiresNoPermission; import org.labkey.api.security.RequiresPermission; +import org.labkey.api.security.SecurityManager; +import org.labkey.api.security.SecurityPolicyManager; +import org.labkey.api.security.User; +import org.labkey.api.security.UserManager; +import org.labkey.api.security.ValidEmail; +import org.labkey.api.security.permissions.AdminPermission; import org.labkey.api.security.permissions.ReadPermission; -import org.labkey.api.view.JspView; -import org.labkey.api.view.NavTree; +import org.labkey.api.security.roles.ReaderRole; +import org.labkey.api.settings.AppProps; +import org.labkey.api.settings.LookAndFeelProperties; +import org.labkey.api.util.ConfigurationException; +import org.labkey.api.util.ExceptionUtil; +import org.labkey.api.util.MailHelper; +import org.labkey.api.util.PageFlowUtil; import org.springframework.validation.BindException; -import org.springframework.web.servlet.ModelAndView; +import org.springframework.validation.Errors; + +import javax.mail.Address; +import javax.mail.Message; +import javax.mail.internet.InternetAddress; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; public class MccController extends SpringActionController { private static final DefaultActionResolver _actionResolver = new DefaultActionResolver(MccController.class); public static final String NAME = "mcc"; + private static final Logger _log = LogManager.getLogger(MccController.class); + public MccController() { setActionResolver(_actionResolver); } + + @RequiresNoPermission + @IgnoresTermsOfUse + @AllowedDuringUpgrade + public class RequestUserAction extends MutatingApiAction + { + @Override + public void validateForm(RequestUserForm form, Errors errors) + { + Container mccContainer = MccManager.get().getMCCContainer(); + if (mccContainer == null) + { + errors.reject(ERROR_MSG, "The MCC project has not been set on this server. This is an administrator error."); + return; + } + + if (StringUtils.isEmpty(form.getEmail()) || StringUtils.isEmpty(form.getEmailConfirmation())) + { + errors.reject(ERROR_REQUIRED, "No email address provided"); + } + else if (StringUtils.isEmpty(form.getFirstName()) || StringUtils.isEmpty(form.getLastName()) || StringUtils.isEmpty(form.getTitle()) || StringUtils.isEmpty(form.getInstitution()) || StringUtils.isEmpty(form.getReason())) + { + errors.reject(ERROR_REQUIRED, "You must provide your first and last name, title, institution, and reason for requesting access"); + } + else + { + try + { + ValidEmail email = new ValidEmail(form.getEmail()); + if (!form.getEmail().equals(form.getEmailConfirmation())) + { + errors.reject(ERROR_MSG, "The email addresses you have entered do not match. Please verify your email addresses below."); + } + + TableInfo ti = MccSchema.getInstance().getSchema().getTable(MccSchema.TABLE_USER_REQUESTS); + + //first check if this email exists: + SimpleFilter filter = new SimpleFilter(FieldKey.fromString("email"), form.getEmail()); + filter.addCondition(FieldKey.fromString("container"), mccContainer.getId()); + if (new TableSelector(ti, filter, null).exists()) + { + errors.reject(ERROR_MSG, "A login has already been requested for this email. You should receive a reply shortly from the site administrator."); + } + } + catch (ValidEmail.InvalidEmailException e) + { + errors.reject(ERROR_MSG, "Your email address is not valid. Please verify your email address below."); + } + } + } + + @Override + public Object execute(RequestUserForm form, BindException errors) throws Exception + { + ApiSimpleResponse response = new ApiSimpleResponse(); + + try + { + TableInfo ti = MccSchema.getInstance().getSchema().getTable(MccSchema.TABLE_USER_REQUESTS); + Map row = new HashMap<>(); + row.put("email", form.getEmail()); + row.put("firstName", form.getFirstName()); + row.put("lastName", form.getLastName()); + row.put("title", form.getTitle()); + row.put("institution", form.getInstitution()); + row.put("reason", form.getReason()); + row.put("container", MccManager.get().getMCCContainer().getId()); + + Table.insert(UserManager.getGuestUser(), ti, row); + + Set users = MccManager.get().getNotificationUsers(); + if (users != null && !users.isEmpty()) + { + try + { + Set
emails = new HashSet<>(); + for (User u : users) + { + emails.add(new InternetAddress(u.getEmail())); + } + + MailHelper.MultipartMessage mail = MailHelper.createMultipartMessage(); + Container c = MccManager.get().getMCCContainer(); + if (c == null) + { + c = getContainer(); + _log.warn("MCC container was not set, using: " + c.getPath()); + } + + DetailsURL url = DetailsURL.fromString("/query/executeQuery.view?schemaName=mcc&query.queryName=userRequests&query.viewName=Pending Requests", c); + mail.setEncodedHtmlContent("A user requested an account on MCC. Click here to view/approve this request"); + mail.setFrom(getReplyEmail(getContainer())); + mail.setSubject("MCC Account Request"); + mail.addRecipients(Message.RecipientType.TO, emails.toArray(new Address[0])); + + MailHelper.send(mail, getUser(), c); + } + catch (Exception e) + { + ExceptionUtil.logExceptionToMothership(null, e); + } + } + + + } + catch (ConfigurationException e) + { + errors.reject(ERROR_MSG, "There was a problem sending the registration email. Please contact your administrator."); + _log.error("Error adding self registered user", e); + } + + response.put("success", !errors.hasErrors()); + if (!errors.hasErrors()) + response.put("email", form.getEmail()); + + return response; + } + } + + public static class RequestUserForm extends Object + { + private String email; + private String emailConfirmation; + private String firstName; + private String lastName; + private String title; + private String institution; + private String reason; + + public void setEmail(String email) + { + this.email = email; + } + + public String getEmail() + { + return this.email; + } + + public void setEmailConfirmation(String email) + { + this.emailConfirmation = email; + } + + public String getEmailConfirmation() + { + return this.emailConfirmation; + } + + public String getFirstName() + { + return firstName; + } + + public void setFirstName(String firstName) + { + this.firstName = firstName; + } + + public String getLastName() + { + return lastName; + } + + public void setLastName(String lastName) + { + this.lastName = lastName; + } + + public String getTitle() + { + return title; + } + + public void setTitle(String title) + { + this.title = title; + } + + public String getInstitution() + { + return institution; + } + + public void setInstitution(String institution) + { + this.institution = institution; + } + + public String getReason() + { + return reason; + } + + public void setReason(String reason) + { + this.reason = reason; + } + } + + @RequiresPermission(AdminPermission.class) + public class ApproveUserRequestsAction extends MutatingApiAction + { + @Override + public void validateForm(ApproveUserRequestsForm form, Errors errors) + { + Container mccContainer = MccManager.get().getMCCContainer(); + if (mccContainer == null) + { + errors.reject(ERROR_MSG, "The MCC project has not been set on this server. This is an administrator error."); + return; + } + + if (form.getRequestIds() == null || form.getRequestIds().length == 0) + { + errors.reject(ERROR_MSG, "No request IDs provided"); + } + + TableInfo ti = MccSchema.getInstance().getSchema().getTable(MccSchema.TABLE_USER_REQUESTS); + for (int requestId : form.getRequestIds()) + { + TableSelector ts = new TableSelector(ti, PageFlowUtil.set("userId"), new SimpleFilter(FieldKey.fromString("rowId"), requestId), null); + if (!ts.exists()) + { + errors.reject(ERROR_MSG, "No request found for request ID: " + requestId); + break; + } + } + } + + @Override + public Object execute(ApproveUserRequestsForm form, BindException errors) throws Exception + { + ApiSimpleResponse response = new ApiSimpleResponse(); + MutableSecurityPolicy policy = new MutableSecurityPolicy(MccManager.get().getMCCContainer().getPolicy()); + List newUserStatusList = new ArrayList<>(); + List existingUsersGivenAccess = new ArrayList<>(); + try (DbScope.Transaction transaction = CoreSchema.getInstance().getScope().ensureTransaction()) + { + TableInfo ti = MccSchema.getInstance().getSchema().getTable(MccSchema.TABLE_USER_REQUESTS); + for (int requestId : form.getRequestIds()) + { + TableSelector ts = new TableSelector(ti, new SimpleFilter(FieldKey.fromString("rowId"), requestId), null); + Map map = ts.getMap(requestId); + + User u; + if (map.get("userId") != null) + { + Integer userId = (Integer)map.get("userId"); + u = UserManager.getUser(userId); + existingUsersGivenAccess.add(u); + } + else + { + ValidEmail ve = new ValidEmail((String)map.get("email")); + u = UserManager.getUser(ve); + if (u != null) + { + existingUsersGivenAccess.add(u); + } + else + { + SecurityManager.NewUserStatus st = SecurityManager.addUser(ve, getUser()); + u = st.getUser(); + u.setFirstName((String)map.get("firstName")); + u.setLastName((String)map.get("lastName")); + UserManager.updateUser(getUser(), u); + + if (st.isLdapEmail()) + { + existingUsersGivenAccess.add(st.getUser()); + } + else + { + newUserStatusList.add(st); + } + } + } + + Map row = new HashMap<>(); + row.put("rowId", requestId); + row.put("userId", u.getUserId()); + Table.update(getUser(), ti, row, requestId); + + if (!policy.hasPermission(u, ReadPermission.class)) + { + policy.addRoleAssignment(u, ReaderRole.class); + } + else + { + _log.info("user already has read permission on MCC container: " + u.getDisplayName(getUser())); + } + } + + SecurityPolicyManager.savePolicy(policy); + + transaction.commit(); + } + + //send emails: + for (SecurityManager.NewUserStatus st : newUserStatusList) + { + SecurityManager.sendRegistrationEmail(getViewContext(), st.getEmail(), null, st, null); + } + + for (User u : existingUsersGivenAccess) + { + Container mccContainer = MccManager.get().getMCCContainer(); + boolean isLDAP = SecurityManager.isLdapEmail(new ValidEmail(u.getEmail())); + + MailHelper.MultipartMessage mail = MailHelper.createMultipartMessage(); + mail.setEncodedHtmlContent("Your account request has been approved for MCC! " + "Click here to access the site." + (isLDAP ? " Use your normal OHSU email/password to login." : "")); + mail.setFrom(getReplyEmail(getContainer())); + mail.setSubject("MCC Account Request"); + mail.addRecipients(Message.RecipientType.TO, u.getEmail()); + + MailHelper.send(mail, getUser(), getContainer()); + } + + response.put("success", !errors.hasErrors()); + + return response; + } + } + + private String getReplyEmail(Container c) + { + LookAndFeelProperties lfp = LookAndFeelProperties.getInstance(getContainer()); + String email = lfp.getSystemEmailAddress(); + if (email == null) + { + return AppProps.getInstance().getAdministratorContactEmail(true); + } + + return email; + } + + public static class ApproveUserRequestsForm + { + private int[] requestIds; + + public int[] getRequestIds() + { + return requestIds; + } + + public void setRequestIds(int[] requestIds) + { + this.requestIds = requestIds; + } + } + + @RequiresNoPermission + @IgnoresTermsOfUse + @AllowedDuringUpgrade + public class RequestHelpAction extends MutatingApiAction + { + @Override + public void validateForm(RequestHelpForm form, Errors errors) + { + Container mccContainer = MccManager.get().getMCCContainer(); + if (mccContainer == null) + { + errors.reject(ERROR_MSG, "The MCC project has not been set on this server. This is an administrator error."); + return; + } + + if (StringUtils.isEmpty(form.getEmail()) || StringUtils.isEmpty(form.getComment())) + { + errors.reject(ERROR_REQUIRED, "Must provide both an email address and question/comment"); + } + else + { + try + { + new ValidEmail(form.getEmail()); + } + catch (ValidEmail.InvalidEmailException e) + { + errors.reject(ERROR_MSG, "Your email address is not valid. Please verify your email address below."); + } + } + } + + @Override + public Object execute(RequestHelpForm form, BindException errors) throws Exception + { + Set users = MccManager.get().getNotificationUsers(); + if (users != null && !users.isEmpty()) + { + try + { + Set
emails = new HashSet<>(); + for (User u : users) + { + emails.add(new InternetAddress(u.getEmail())); + } + + MailHelper.MultipartMessage mail = MailHelper.createMultipartMessage(); + mail.setEncodedHtmlContent("A support request was submitted from MCC by: " + form.getEmail() + "

Message:
" + form.getComment()); + mail.setFrom(form.getEmail()); + mail.setSubject("MCC Help Request"); + mail.addRecipients(Message.RecipientType.TO, emails.toArray(new Address[0])); + + MailHelper.send(mail, getUser(), getContainer()); + } + catch (Exception e) + { + ExceptionUtil.logExceptionToMothership(null, e); + } + } + else + { + _log.error("A help request was received by MCC, but the admin emails have not been configured. The request from: " + form.getEmail()); + _log.error(form.getComment()); + } + + return new ApiSimpleResponse("success", true); + } + } + + public static class RequestHelpForm + { + private String _email; + private String _comment; + + public String getEmail() + { + return _email; + } + + public void setEmail(String email) + { + _email = email; + } + + public String getComment() + { + return _comment; + } + + public void setComment(String comment) + { + _comment = comment; + } + } } diff --git a/mcc/src/org/labkey/mcc/MccManager.java b/mcc/src/org/labkey/mcc/MccManager.java index d6417b79d..d0f657de9 100644 --- a/mcc/src/org/labkey/mcc/MccManager.java +++ b/mcc/src/org/labkey/mcc/MccManager.java @@ -16,8 +16,28 @@ package org.labkey.mcc; +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerManager; +import org.labkey.api.module.Module; +import org.labkey.api.module.ModuleLoader; +import org.labkey.api.module.ModuleProperty; +import org.labkey.api.security.User; +import org.labkey.api.security.UserManager; +import org.labkey.api.security.ValidEmail; + +import java.util.HashSet; +import java.util.Set; + public class MccManager { + private static final Logger _log = LogManager.getLogger(MccManager.class); + + public static final String ContainerPropName = "MCCContainer"; + public static final String NotifyPropName = "MCCContactUsers"; + private static final MccManager _instance = new MccManager(); private MccManager() @@ -29,4 +49,54 @@ public static MccManager get() { return _instance; } + + public Container getMCCContainer() + { + Module m = ModuleLoader.getInstance().getModule(MccModule.NAME); + ModuleProperty mp = m.getModuleProperties().get(MccManager.ContainerPropName); + String path = mp.getEffectiveValue(ContainerManager.getRoot()); + if (path == null) + return null; + + return ContainerManager.getForPath(path); + } + + public Set getNotificationUsers() + { + Module m = ModuleLoader.getInstance().getModule(MccModule.NAME); + ModuleProperty mp = m.getModuleProperties().get(MccManager.NotifyPropName); + String userNames = mp.getEffectiveValue(ContainerManager.getRoot()); + userNames = StringUtils.trimToNull(userNames); + if (userNames == null) + return null; + + Set ret = new HashSet<>(); + for (String username : userNames.split(",")) + { + User u = UserManager.getUserByDisplayName(username); + if (u == null) + { + try + { + u = UserManager.getUser(new ValidEmail(username)); + } + catch (ValidEmail.InvalidEmailException e) + { + //ignore + } + } + + if (u == null) + { + _log.error("Unknown user registered for MCC notifcations: " + username); + } + + if (u != null) + { + ret.add(u); + } + } + + return ret; + } } \ No newline at end of file diff --git a/mcc/src/org/labkey/mcc/MccModule.java b/mcc/src/org/labkey/mcc/MccModule.java index 03182cb68..ea28010a7 100644 --- a/mcc/src/org/labkey/mcc/MccModule.java +++ b/mcc/src/org/labkey/mcc/MccModule.java @@ -41,7 +41,7 @@ public String getName() @Override public @Nullable Double getSchemaVersion() { - return 20.000; + return 20.001; } @Override @@ -86,19 +86,5 @@ public Set getSchemaNames() private void registerEHRResources() { EHRService.get().registerModule(this); - //EHRService.get().registerTableCustomizer(this, ONPRC_EHRCustomizer.class); - - //Resource r = getModuleResource("/scripts/mcc/mcc_triggers.js"); - //assert r != null; - //EHRService.get().registerTriggerScript(this, r); - - //EHRService.get().registerClientDependency(ClientDependency.supplierFromPath("Ext4"), this); - //EHRService.get().registerClientDependency(ClientDependency.supplierFromPath("onprc_ehr/panel/BloodSummaryPanel.js"), this); - - //EHRService.get().registerReportLink(EHRService.REPORT_LINK_TYPE.housing, "List Single Housed Animals", this, DetailsURL.fromString("/query/executeQuery.view?schemaName=study&query.queryName=demographicsPaired&query.viewName=Single Housed"), "Commonly Used Queries"); - //EHRService.get().registerReportLink(EHRService.REPORT_LINK_TYPE.moreReports, "Clinical Snapshot Printable Report", this, DetailsURL.fromString("/onprc_ehr/SnapshotPrintableReport.view"), "Clinical"); - - //EHRService.get().registerDemographicsProvider(new ActiveCasesDemographicsProvider(this)); - //EHRService.get().registerHistoryDataSource(new DefaultSustainedReleaseDatasource(this)); } } \ No newline at end of file diff --git a/mcc/src/org/labkey/mcc/MccSchema.java b/mcc/src/org/labkey/mcc/MccSchema.java index d072b94a9..be31ef6f1 100644 --- a/mcc/src/org/labkey/mcc/MccSchema.java +++ b/mcc/src/org/labkey/mcc/MccSchema.java @@ -25,6 +25,8 @@ public class MccSchema private static final MccSchema _instance = new MccSchema(); public static final String NAME = "mcc"; + public static final String TABLE_USER_REQUESTS = "userRequests"; + public static MccSchema getInstance() { return _instance; diff --git a/mcc/src/org/labkey/mcc/query/UserRequestCustomizer.java b/mcc/src/org/labkey/mcc/query/UserRequestCustomizer.java new file mode 100644 index 000000000..7f789d464 --- /dev/null +++ b/mcc/src/org/labkey/mcc/query/UserRequestCustomizer.java @@ -0,0 +1,44 @@ +package org.labkey.mcc.query; + +import org.labkey.api.data.AbstractTableInfo; +import org.labkey.api.data.JdbcType; +import org.labkey.api.data.SQLFragment; +import org.labkey.api.data.TableCustomizer; +import org.labkey.api.data.TableInfo; +import org.labkey.api.ldk.LDKService; +import org.labkey.api.query.ExprColumn; + +public class UserRequestCustomizer implements TableCustomizer +{ + @Override + public void customize(TableInfo tableInfo) + { + LDKService.get().getDefaultTableCustomizer().customize(tableInfo); + + if (tableInfo instanceof AbstractTableInfo) + { + addUserCol((AbstractTableInfo)tableInfo); + } + } + + public void addUserCol(AbstractTableInfo ti) + { + String colName = "hasAccess"; + if (ti.getColumn(colName) != null) + { + return; + } + + ExprColumn col = new ExprColumn(ti, colName, new SQLFragment("(CASE WHEN (exists (" + + "select u.rowid from mcc.userrequests u " + + "left join core.RoleAssignments ra " + + "on (u.userid = ra.UserId AND u.container = ra.ResourceId) " + + "WHERE ra.Role = 'org.labkey.api.security.roles.ReaderRole' AND u.rowid = " + ExprColumn.STR_TABLE_ALIAS + ".rowid " + + ")) THEN " + ti.getSqlDialect().getBooleanTRUE() + " ELSE " + ti.getSqlDialect().getBooleanFALSE() + " END)"), JdbcType.BOOLEAN, ti.getColumn("userId")); + col.setLabel("Has MCC Access?"); + col.setReadOnly(true); + col.setIsUnselectable(true); + col.setUserEditable(false); + ti.addColumn(col); + } +} diff --git a/primeseq/resources/views/geneticsMenu.html b/primeseq/resources/views/geneticsMenu.html index 3ff3df451..5177cd413 100644 --- a/primeseq/resources/views/geneticsMenu.html +++ b/primeseq/resources/views/geneticsMenu.html @@ -44,7 +44,7 @@ 'data-qtip="You do not have permission to view this page"', 'style="width: 300px;height: auto;" class="thumb-wrap thumb-wrap-side">', '', - '{title:htmlEncode}', + '{title:htmlEncode}', '', '', '', @@ -88,15 +88,15 @@ items: [{ title: 'Public Resources', itemId: 'public' - },{ - itemId: 'collaborations', - title: 'Collaborations' },{ title: 'Labs', itemId: 'labs' },{ itemId: 'internal', title: 'Internal Projects' + },{ + itemId: 'collaborations', + title: 'Bimber Lab Collaborations' }] }); diff --git a/primeseq/src/org/labkey/primeseq/PrimeseqController.java b/primeseq/src/org/labkey/primeseq/PrimeseqController.java index bc680d82c..de1835eab 100644 --- a/primeseq/src/org/labkey/primeseq/PrimeseqController.java +++ b/primeseq/src/org/labkey/primeseq/PrimeseqController.java @@ -30,6 +30,8 @@ import org.labkey.api.data.ContainerType; import org.labkey.api.module.Module; import org.labkey.api.module.ModuleLoader; +import org.labkey.api.pipeline.PipeRoot; +import org.labkey.api.pipeline.PipelineService; import org.labkey.api.pipeline.PipelineUrls; import org.labkey.api.security.RequiresPermission; import org.labkey.api.security.RequiresSiteAdmin; @@ -39,6 +41,7 @@ import org.labkey.api.util.URLHelper; import org.labkey.api.view.ActionURL; import org.labkey.api.view.HtmlView; +import org.labkey.primeseq.pipeline.MhcMigrationPipelineJob; import org.springframework.validation.BindException; import org.springframework.validation.Errors; import org.springframework.web.servlet.ModelAndView; @@ -68,7 +71,7 @@ public ApiResponse execute(Object form, BindException errors) { Map resultProperties = new HashMap<>(); - resultProperties.put("collaborations", getSection("/Public/Collaborations")); + resultProperties.put("collaborations", getSection("/Labs/Bimber/Collaborations")); resultProperties.put("internal", getSection("/Internal")); resultProperties.put("labs", getSection("/Labs")); @@ -125,12 +128,6 @@ private List getSection(String path) { for (Container c : mainContainer.getChildren()) { - //NOTE: unlike EHR, omit children if the current user cannot read them - if (!c.hasPermission(getUser(), ReadPermission.class)) - { - continue; - } - JSONObject json = new JSONObject(); json.put("name", c.getName()); json.put("title", c.getTitle()); @@ -203,4 +200,48 @@ public URLHelper getSuccessURL(Object o) } } + @RequiresSiteAdmin + public class SyncMhcAction extends ConfirmAction + { + @Override + public ModelAndView getConfirmView(Object o, BindException errors) throws Exception + { + setTitle("Sync MHC Data from PRIMe"); + + return new HtmlView(HtmlString.of("This will attempt to sync MHC typing data from PRIMe to the current folder, creating all sequence records and workbooks. Do you want to continue?")); + } + + @Override + public boolean handlePost(Object o, BindException errors) throws Exception + { + try + { + PipeRoot pipelineRoot = PipelineService.get().findPipelineRoot(getContainer()); + MhcMigrationPipelineJob job = new MhcMigrationPipelineJob(getContainer(), getUser(), getViewContext().getActionURL(), pipelineRoot, "PRIMe", "ONPRC/Core Facilities/Genetics Core/MHC_Typing/"); + PipelineService.get().queueJob(job); + } + catch (Exception e) + { + _log.error(e); + errors.reject(ERROR_MSG, e.getMessage()); + return false; + + } + + return true; + } + + @Override + public void validateCommand(Object o, Errors errors) + { + + } + + @NotNull + @Override + public URLHelper getSuccessURL(Object o) + { + return PageFlowUtil.urlProvider(PipelineUrls.class).urlBegin(getContainer()); + } + } } \ No newline at end of file diff --git a/primeseq/src/org/labkey/primeseq/PrimeseqModule.java b/primeseq/src/org/labkey/primeseq/PrimeseqModule.java index bc65a9262..bd36014d1 100644 --- a/primeseq/src/org/labkey/primeseq/PrimeseqModule.java +++ b/primeseq/src/org/labkey/primeseq/PrimeseqModule.java @@ -23,6 +23,7 @@ import org.labkey.api.data.Container; import org.labkey.api.ldk.ExtendedSimpleModule; import org.labkey.api.module.ModuleContext; +import org.labkey.api.pipeline.PipelineService; import org.labkey.api.sequenceanalysis.SequenceAnalysisService; import org.labkey.api.sequenceanalysis.pipeline.SequencePipelineService; import org.labkey.api.util.PageFlowUtil; @@ -36,6 +37,7 @@ import org.labkey.primeseq.pipeline.BlastPipelineJobResourceAllocator; import org.labkey.primeseq.pipeline.ClusterMaintenanceTask; import org.labkey.primeseq.pipeline.ExacloudResourceSettings; +import org.labkey.primeseq.pipeline.MhcMigrationPipelineJob; import org.labkey.primeseq.pipeline.SequenceJobResourceAllocator; import java.util.Collection; @@ -75,6 +77,8 @@ protected void doStartupAfterSpringConfig(ModuleContext moduleContext) ClusterService.get().registerResourceAllocator(new BlastPipelineJobResourceAllocator.Factory()); ClusterService.get().registerResourceAllocator(new SequenceJobResourceAllocator.Factory()); + PipelineService.get().registerPipelineProvider(new MhcMigrationPipelineJob.Provider(this)); + //register resources new PipelineStartup(); diff --git a/primeseq/src/org/labkey/primeseq/pipeline/MhcMigrationPipelineJob.java b/primeseq/src/org/labkey/primeseq/pipeline/MhcMigrationPipelineJob.java new file mode 100644 index 000000000..dfd668352 --- /dev/null +++ b/primeseq/src/org/labkey/primeseq/pipeline/MhcMigrationPipelineJob.java @@ -0,0 +1,1133 @@ +package org.labkey.primeseq.pipeline; + +import org.apache.commons.io.FileUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.labkey.api.collections.CaseInsensitiveHashMap; +import org.labkey.api.data.CompareType; +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerManager; +import org.labkey.api.data.DbSchema; +import org.labkey.api.data.DbSchemaType; +import org.labkey.api.data.DbScope; +import org.labkey.api.data.SimpleFilter; +import org.labkey.api.data.Sort; +import org.labkey.api.data.Table; +import org.labkey.api.data.TableInfo; +import org.labkey.api.data.TableSelector; +import org.labkey.api.data.WorkbookContainerType; +import org.labkey.api.di.DataIntegrationService; +import org.labkey.api.exp.api.DataType; +import org.labkey.api.exp.api.ExpData; +import org.labkey.api.exp.api.ExpRun; +import org.labkey.api.exp.api.ExperimentService; +import org.labkey.api.files.FileUrls; +import org.labkey.api.module.Module; +import org.labkey.api.pipeline.AbstractTaskFactory; +import org.labkey.api.pipeline.AbstractTaskFactorySettings; +import org.labkey.api.pipeline.PipeRoot; +import org.labkey.api.pipeline.PipelineDirectory; +import org.labkey.api.pipeline.PipelineJob; +import org.labkey.api.pipeline.PipelineJobException; +import org.labkey.api.pipeline.PipelineJobService; +import org.labkey.api.pipeline.PipelineProvider; +import org.labkey.api.pipeline.PipelineService; +import org.labkey.api.pipeline.PipelineStatusFile; +import org.labkey.api.pipeline.RecordedActionSet; +import org.labkey.api.pipeline.TaskId; +import org.labkey.api.pipeline.TaskPipeline; +import org.labkey.api.query.BatchValidationException; +import org.labkey.api.query.FieldKey; +import org.labkey.api.query.QueryService; +import org.labkey.api.query.UserSchema; +import org.labkey.api.security.User; +import org.labkey.api.sequenceanalysis.SequenceAnalysisService; +import org.labkey.api.sequenceanalysis.model.Readset; +import org.labkey.api.util.FileType; +import org.labkey.api.util.FileUtil; +import org.labkey.api.util.PageFlowUtil; +import org.labkey.api.view.ActionURL; +import org.labkey.api.view.ViewBackgroundInfo; +import org.labkey.api.view.ViewContext; +import org.labkey.remoteapi.CommandException; +import org.labkey.remoteapi.Connection; +import org.labkey.remoteapi.query.Filter; +import org.labkey.remoteapi.query.SelectRowsCommand; +import org.labkey.remoteapi.query.SelectRowsResponse; + +import java.io.File; +import java.io.IOException; +import java.net.URI; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; + +public class MhcMigrationPipelineJob extends PipelineJob +{ + private static final Logger _log = LogManager.getLogger(MhcMigrationPipelineJob.class); + + private String remoteServerFolder; + private String remoteConnectionName; + + private Container targetContainer; + + public static class Provider extends PipelineProvider + { + public static final String NAME = "mhcMigrationPipeline"; + + public Provider(Module owningModule) + { + super(NAME, owningModule); + } + + @Override + public void updateFileProperties(ViewContext context, PipeRoot pr, PipelineDirectory directory, boolean includeAll) + { + + } + } + + // Default constructor for serialization + protected MhcMigrationPipelineJob() + { + } + + public MhcMigrationPipelineJob(Container c, User u, ActionURL url, PipeRoot pipeRoot, String remoteConnectionName, String remoteServerFolder) + { + super(Provider.NAME, new ViewBackgroundInfo(c, u, url), pipeRoot); + + this.targetContainer = c; + this.remoteConnectionName = remoteConnectionName; + this.remoteServerFolder = remoteServerFolder; + + File subdir = new File(pipeRoot.getRootPath(), Provider.NAME); + if (!subdir.exists()) + { + subdir.mkdirs(); + } + + setLogFile(new File(subdir, FileUtil.makeFileNameWithTimestamp("mhcMigration", "log"))); + + } + + @Override + public ActionURL getStatusHref() + { + return PageFlowUtil.urlProvider(FileUrls.class).urlBegin(getContainer()); + } + + @Override + public String getDescription() + { + return "Find Orphan Sequence Files"; + } + + @Override + public TaskPipeline getTaskPipeline() + { + return PipelineJobService.get().getTaskPipeline(new TaskId(MhcMigrationPipelineJob.class)); + } + + public static class Task extends PipelineJob.Task + { + protected Task(Factory factory, PipelineJob job) + { + super(factory, job); + } + + public static class Factory extends AbstractTaskFactory + { + public Factory() + { + super(Task.class); + } + + @Override + public List getInputTypes() + { + return Collections.emptyList(); + } + + @Override + public String getStatusName() + { + return PipelineJob.TaskStatus.running.toString(); + } + + @Override + public List getProtocolActionNames() + { + return Arrays.asList("Migrate MHC Data"); + } + + @Override + public PipelineJob.Task createTask(PipelineJob job) + { + return new Task(this, job); + } + + @Override + public boolean isJobComplete(PipelineJob job) + { + return false; + } + } + + private MhcMigrationPipelineJob getPipelineJob() + { + return (MhcMigrationPipelineJob)getJob(); + } + + private Connection getConnection() + { + DataIntegrationService.RemoteConnection rc = DataIntegrationService.get().getRemoteConnection(getPipelineJob().remoteConnectionName, getPipelineJob().targetContainer, _log); + + return(rc.connection); + } + + @Override + public RecordedActionSet run() throws PipelineJobException + { + try (DbScope.Transaction transaction = DbScope.getLabKeyScope().ensureTransaction()) + { + createWorkbooks(); + + createLibraries(); + createLibraryMembers(); + + createReadsets(); + transaction.commitAndKeepConnection(); + + createReaddata(); + + createAnalyses(); + createOutputFiles(); + + //TODO: + //samples + //alignment_summary + //alignment_summary_junction + //quality_metrics + //subjects + //WaNPRC + + //sequenceanalysis.haplotypes + //sequenceanalysis.haplotype_types + //sequenceanalysis.haplotype_sequences + + //Create assay runs, including data and haplotypes + + transaction.commit(); + } + + return new RecordedActionSet(); + } + + private void replaceEntireTable(String schema, String query, List columns, String workbookColName, boolean truncateExisting) throws Exception + { + SelectRowsCommand sr = new SelectRowsCommand(schema, query); + sr.setColumns(columns); + SelectRowsResponse srr = sr.execute(getConnection(), getPipelineJob().remoteServerFolder); + + List> toInsert = new ArrayList<>(); + srr.getRowset().forEach(r -> { + Map row = new CaseInsensitiveHashMap<>(); + srr.getColumnModel().forEach(col -> { + String colName = (String) col.get("Name"); + Object val = r.getValue(colName); + if ("readset".equals(colName) || "readsetid".equals(colName)) + { + if (!readsetMap.containsKey((int) val)) + { + throw new IllegalStateException("Unable to find readset: " + val); + } + + val = readsetMap.get((int) val); + } + else if ("library_id".equals(colName)) + { + if (!libraryMap.containsKey((int) val)) + { + throw new IllegalStateException("Unable to find library: " + val); + } + + val = libraryMap.get((int) val); + + } + else if ("ref_nt_id".equals(colName)) + { + if (!sequenceMap.containsKey((int) val)) + { + throw new IllegalStateException("Unable to find sequence: " + val); + } + + val = sequenceMap.get((int) val); + } + else if ("analysis_id".equals(colName)) + { + if (!analysisMap.containsKey((int) val)) + { + throw new IllegalStateException("Unable to find analysis: " + val); + } + + val = analysisMap.get((int) val); + } + + row.put(colName, val); + }); + + if (workbookColName != null) + { + Object workbookId = r.getValue(workbookColName); + if (workbookId != null) + { + row.put("container", workbookMap.get(Integer.parseInt(String.valueOf(workbookId))).getId()); + } + } + + toInsert.add(row); + }); + + + } + + //All of these map remote Id to local Id + private final Map workbookMap = new HashMap<>(); + private final Map readsetMap = new HashMap<>(); + private final Map readdataMap = new HashMap<>(); + private final Map analysisMap = new HashMap<>(); + private final Map libraryMap = new HashMap<>(); + private final Map outputFileMap = new HashMap<>(); + private final Map sequenceMap = new HashMap<>(); + private final Map runIdMap = new HashMap<>(); + private final Map jobIdMap = new HashMap<>(); + + private void createLibraryMembers() + { + _log.info("Creating library members"); + + final UserSchema us = QueryService.get().getUserSchema(getJob().getUser(), getPipelineJob().targetContainer, "sequenceanalysis"); + final TableInfo ti = us.getTable("reference_library_members"); + final TableInfo refNtTable = us.getTable("ref_nt_sequences"); + + try + { + SelectRowsCommand sr = new SelectRowsCommand("sequenceanalysis", "reference_library_members"); + sr.setColumns(Arrays.asList("rowid", "library_id", "ref_nt_id", "ref_nt_id/name", "ref_nt_id/seqLength", "workbook/workbookId")); + + SelectRowsResponse srr = sr.execute(getConnection(), getPipelineJob().remoteServerFolder); + + srr.getRowset().forEach(rd -> { + int remoteId = Integer.parseInt(String.valueOf(rd.getValue("rowid"))); + int seqLength = Integer.parseInt(String.valueOf(rd.getValue("ref_nt_id/seqLength"))); + + int remoteSeqId = Integer.parseInt(String.valueOf(rd.getValue("ref_nt_id"))); + String name = String.valueOf(rd.getValue("ref_nt_id/name")); + int localSeqId = getOrCreateSequence(remoteSeqId, name, seqLength, refNtTable); + + int remoteLibraryId = Integer.parseInt(String.valueOf(rd.getValue("library_id"))); + Integer localLibraryId = libraryMap.get(remoteLibraryId); + if (localLibraryId == null) + { + throw new IllegalStateException("Unable to find library id: " + remoteLibraryId); + } + + SimpleFilter filter = new SimpleFilter(FieldKey.fromString("library_id"), localLibraryId); + filter.addCondition(FieldKey.fromString("ref_nt_id"), localSeqId); + + if (new TableSelector(ti, PageFlowUtil.set("rowid"), filter, null).exists()) + { + //Already exists: + return; + } + + Map toCreate = new CaseInsensitiveHashMap<>(); + toCreate.put("library_id", localLibraryId); + toCreate.put("ref_nt_id", localSeqId); + + try + { + BatchValidationException bve = new BatchValidationException(); + List> created = ti.getUpdateService().insertRows(getJob().getUser(), getPipelineJob().targetContainer, Arrays.asList(toCreate), bve, null, null); + if (bve.hasErrors()) + { + throw new RuntimeException(bve); + } + } + catch (Exception e) + { + _log.error(e.getMessage(), e); + throw new RuntimeException(e); + } + }); + } + catch (Exception e) + { + _log.error(e.getMessage(), e); + throw new RuntimeException(e); + } + } + + private int getOrCreateSequence(int remoteSeqId, String name, int seqLength, TableInfo refNtTable) + { + if (sequenceMap.containsKey(remoteSeqId)) + { + return sequenceMap.get(remoteSeqId); + } + else + { + SimpleFilter filter = new SimpleFilter(FieldKey.fromString("name"), name); + filter.addCondition(FieldKey.fromString("datedisabled"), null, CompareType.ISBLANK); + TableSelector ts = new TableSelector(refNtTable, PageFlowUtil.set("rowid", "seqLength"), filter, new Sort("rowid")); + if (ts.exists()) + { + if (ts.getRowCount() > 1) + { + _log.info("Duplicate ref name: " + name); + } + + AtomicInteger localId = new AtomicInteger(-1); + ts.forEachResults(rs -> { + if (rs.getInt(FieldKey.fromString("seqLength")) < seqLength) + { + _log.warn("length doesnt match for " + name + ", expected: " + seqLength); + return; + } + + localId.set(rs.getInt(FieldKey.fromString("rowid"))); + }); + + if (localId.get() != -1) + { + sequenceMap.put(remoteSeqId, localId.get()); + return localId.get(); + } + } + + //TODO: Create sequence? + //throw new IllegalStateException("Expected sequence to exist: " + name); + _log.error("Sequence missing: " + name); + return -1; + } + } + + public String getParent(String path) + { + final char separatorChar = '/'; + + int index = path.lastIndexOf(separatorChar); + + return path.substring(0, index); + } + + private void createLibraries() + { + _log.info("Creating libraries"); + try + { + final TableInfo libraryTable = QueryService.get().getUserSchema(getJob().getUser(), getPipelineJob().targetContainer, "sequenceanalysis").getTable("reference_libraries"); + + SelectRowsCommand sr = new SelectRowsCommand("sequenceanalysis", "reference_libraries"); + sr.setColumns(Arrays.asList("rowid", "name", "description", "fasta_file", "datedisabled", "assemblyId", "fasta_file/DataFileUrl", "workbook/workbookId")); + + SelectRowsResponse srr = sr.execute(getConnection(), getPipelineJob().remoteServerFolder); + + srr.getRowset().forEach(rd -> { + int remoteId = Integer.parseInt(String.valueOf(rd.getValue("rowid"))); + + Integer remoteWorkbook = rd.getValue("workbook/workbookId") == null ? null : Integer.parseInt(String.valueOf(rd.getValue("workbook/workbookId"))); + Container targetContainer = remoteWorkbook == null ? getPipelineJob().targetContainer : workbookMap.get(remoteWorkbook); + + SimpleFilter filter = new SimpleFilter(FieldKey.fromString("name"), rd.getValue("name")); + TableSelector ts = new TableSelector(libraryTable, PageFlowUtil.set("rowid"), filter, null); + if (ts.exists()) + { + libraryMap.put(remoteId, ts.getObject(Integer.class)); + } + else + { + Map toCreate = new CaseInsensitiveHashMap<>(); + toCreate.put("name", rd.getValue("name")); + toCreate.put("description", rd.getValue("description")); + toCreate.put("datedisabled", rd.getValue("datedisabled")); + toCreate.put("assemblyId", rd.getValue("assemblyId")); + try + { + String remoteJobRoot = getParent(URI.create(String.valueOf(rd.getValue("fasta_file/DatafileUrl"))).getPath()); + URI localJobRoot = PipelineService.get().getPipelineRootSetting(targetContainer).getRootPath().toURI(); + URI localFasta = translateURI(String.valueOf(rd.getValue("fasta_file/DatafileUrl")), remoteJobRoot, localJobRoot.getPath()); + toCreate.put("fasta_file", getOrCreateExpData(localFasta, targetContainer)); + + //Ensure parent folder exists: + File localJobRootFile = new File(localFasta).getParentFile(); + if (!localJobRootFile.getParentFile().exists()) + { + localJobRootFile.getParentFile().mkdirs(); + } + + _log.info(remoteJobRoot); + _log.info(localJobRoot.getPath()); + File remoteJobRootFile = new File(remoteJobRoot); + if (remoteJobRootFile.exists()) + { + FileUtils.copyDirectory(remoteJobRootFile, localJobRootFile); + } + + BatchValidationException bve = new BatchValidationException(); + List> created = libraryTable.getUpdateService().insertRows(getJob().getUser(), getPipelineJob().targetContainer, Arrays.asList(toCreate), bve, null, null); + if (bve.hasErrors()) + { + throw new RuntimeException(bve); + } + + libraryMap.put(remoteId, Integer.parseInt(String.valueOf(created.get(0).get("rowid")))); + } + catch (Exception e) + { + throw new RuntimeException(e); + } + } + }); + } + catch (Exception e) + { + _log.error(e.getMessage(), e); + throw new RuntimeException(e); + } + } + + private void createOutputFiles() + { + _log.info("Creating outputfiles"); + try + { + final TableInfo outputTable = QueryService.get().getUserSchema(getJob().getUser(), getPipelineJob().targetContainer, "sequenceanalysis").getTable("outputfiles"); + + SelectRowsCommand sr = new SelectRowsCommand("sequenceanalysis", "outputfiles"); + sr.setColumns(Arrays.asList("rowid", "name", "description", "dataid", "library_id", "readset", "analysis_id", "category", "sra_accession", "dataid/DataFileUrl", "runid/jobid", "runid/Name", "workbook/workbookId", "runid/JobId", "runid/Name", "runid/JobId/FilePath")); + + SelectRowsResponse srr = sr.execute(getConnection(), getPipelineJob().remoteServerFolder); + + srr.getRowset().forEach(rd -> { + int remoteId = Integer.parseInt(String.valueOf(rd.getValue("rowid"))); + int remoteReadset = Integer.parseInt(String.valueOf(rd.getValue("readset"))); + Integer localReadset = readsetMap.get(remoteReadset); + if (localReadset == null) + { + throw new IllegalArgumentException("Unable to find readset for remote id: " + remoteReadset); + } + + int remoteLibrary = Integer.parseInt(String.valueOf(rd.getValue("library_id"))); + Integer localLibrary = libraryMap.get(remoteLibrary); + if (localLibrary == null) + { + throw new IllegalArgumentException("Unable to find genome for remote id: " + remoteLibrary); + } + + int remoteAnalysis = Integer.parseInt(String.valueOf(rd.getValue("analysis_id"))); + Integer localAnalysis = analysisMap.get(remoteAnalysis); + if (localAnalysis == null) + { + throw new IllegalArgumentException("Unable to find analysis for remote id: " + remoteAnalysis); + } + + Readset rs = SequenceAnalysisService.get().getReadset(localReadset, getJob().getUser()); + Container targetWorkbook = ContainerManager.getForId(rs.getContainer()); + + SimpleFilter filter = new SimpleFilter(FieldKey.fromString("readset"), rs.getRowId()); + filter.addCondition(FieldKey.fromString("name"), rd.getValue("name")); + filter.addCondition(FieldKey.fromString("category"), rd.getValue("category")); + filter.addCondition(FieldKey.fromString("analysis_id"), localAnalysis); + filter.addCondition(FieldKey.fromString("container"), targetWorkbook.getId(), CompareType.EQUAL); + + TableSelector tsOutputFiles = new TableSelector(outputTable, PageFlowUtil.set("rowid"), filter, null); + if (tsOutputFiles.exists()) + { + outputFileMap.put(remoteId, tsOutputFiles.getObject(Integer.class)); + } + else + { + Map toCreate = new CaseInsensitiveHashMap<>(); + toCreate.put("readset", rs.getRowId()); + toCreate.put("analysis_id", localAnalysis); + toCreate.put("description", rd.getValue("description")); + toCreate.put("sra_accession", rd.getValue("sra_accession")); + toCreate.put("library_id", localLibrary); + toCreate.put("name", rd.getValue("name")); + toCreate.put("category", rd.getValue("category")); + + try + { + int remoteJobId = Integer.parseInt(String.valueOf(rd.getValue("runid/JobId"))); + int jobId = getOrCreateJob(remoteJobId, targetWorkbook); + PipelineStatusFile sf = PipelineService.get().getStatusFile(jobId); + + String localJobRoot = getParent(sf.getFilePath()); + String remoteJobRoot = getParent(URI.create(String.valueOf(rd.getValue("runid/JobId/FilePath")).replaceAll(" ", "_")).getPath()); + + URI newFileAlignment = translateURI(String.valueOf(rd.getValue("dataid/DatafileUrl")), remoteJobRoot, localJobRoot); + toCreate.put("dataid", getOrCreateExpData(newFileAlignment, targetWorkbook)); + + //Create run: + if (rd.getValue("runid") != null && rd.getValue("runid/JobId") != null) + { + int runId = createExpRun(Integer.parseInt(String.valueOf(rd.getValue("runid"))), targetWorkbook, String.valueOf(rd.getValue("runid/Name")), jobId); + toCreate.put("runid", runId); + } + else + { + _log.error("output missing runid: " + remoteId); + } + + BatchValidationException bve = new BatchValidationException(); + List> created = outputTable.getUpdateService().insertRows(getJob().getUser(), getPipelineJob().targetContainer, Arrays.asList(toCreate), bve, null, null); + if (bve.hasErrors()) + { + throw new RuntimeException(bve); + } + + outputFileMap.put(remoteId, Integer.parseInt(String.valueOf(created.get(0).get("rowid")))); + } + catch (Exception e) + { + throw new RuntimeException(e); + } + } + }); + } + catch (Exception e) + { + _log.error(e.getMessage(), e); + throw new RuntimeException(e); + } + } + + private void createAnalyses() + { + _log.info("Creating analyses"); + try + { + final TableInfo analysisTable = QueryService.get().getUserSchema(getJob().getUser(), getPipelineJob().targetContainer, "sequenceanalysis").getTable("sequence_analyses"); + + SelectRowsCommand sr = new SelectRowsCommand("sequenceanalysis", "sequence_analyses"); + sr.setColumns(Arrays.asList("rowid", "type", "description", "synopsis", "runid", "readset", "alignmentfile", "reference_library", "library_id", "sra_accession", "alignmentfile/DataFileUrl", "alignmentfile/Name", "reference_library", "reference_library/DataFileUrl", "runid/jobid", "runid/Name", "workbook/workbookId", "runid/JobId", "runid/Name", "runid/JobId/FilePath", "runid/JobId/Description")); + + SelectRowsResponse srr = sr.execute(getConnection(), getPipelineJob().remoteServerFolder); + + srr.getRowset().forEach(rd -> { + int remoteId = Integer.parseInt(String.valueOf(rd.getValue("rowid"))); + if (rd.getValue("readset") == null) + { + _log.warn("analysis lacks readset, skipping: " + remoteId); + return; + } + + int remoteReadset = Integer.parseInt(String.valueOf(rd.getValue("readset"))); + Integer localReadset = readsetMap.get(remoteReadset); + if (localReadset == null) + { + throw new IllegalArgumentException("Unable to find readset for remote id: " + remoteReadset); + } + + Integer localLibrary = null; + if (rd.getValue("library_id") != null) + { + int remoteLibrary = Integer.parseInt(String.valueOf(rd.getValue("library_id"))); + localLibrary = libraryMap.get(remoteLibrary); + if (localLibrary == null) + { + throw new IllegalArgumentException("Unable to find genome for remote id: " + remoteLibrary); + } + } + + Readset rs = SequenceAnalysisService.get().getReadset(localReadset, getJob().getUser()); + Container targetWorkbook = ContainerManager.getForId(rs.getContainer()); + + SimpleFilter filter = new SimpleFilter(FieldKey.fromString("readset"), rs.getRowId()); + filter.addCondition(FieldKey.fromString("runid/JobId/Description"), rd.getValue("runid/JobId/Description")); + filter.addCondition(FieldKey.fromString("container"), targetWorkbook.getId(), CompareType.EQUAL); + + TableSelector tsAnalyses = new TableSelector(analysisTable, PageFlowUtil.set("rowid"), filter, null); + if (tsAnalyses.exists()) + { + analysisMap.put(remoteId, tsAnalyses.getObject(Integer.class)); + } + else + { + Map toCreate = new CaseInsensitiveHashMap<>(); + + toCreate.put("readset", rs.getRowId()); + toCreate.put("synopsis", rd.getValue("synopsis")); + toCreate.put("centerName", rd.getValue("centerName")); + toCreate.put("type", rd.getValue("type")); + toCreate.put("description", rd.getValue("description")); + toCreate.put("sra_accession", rd.getValue("sra_accession")); + if (localLibrary != null) + { + toCreate.put("library_id", localLibrary); + } + + try + { + if (rd.getValue("runid/JobId") == null) + { + _log.info("skipping analysis without runid: " + remoteId); + return; + } + + int remoteJobId = Integer.parseInt(String.valueOf(rd.getValue("runid/JobId"))); + int jobId = getOrCreateJob(remoteJobId, targetWorkbook); + PipelineStatusFile sf = PipelineService.get().getStatusFile(jobId); + + String localJobRoot = getParent(sf.getFilePath()); + String remoteJobRoot = getParent(URI.create(String.valueOf(rd.getValue("runid/JobId/FilePath")).replaceAll(" ", "_")).getPath()); + + URI newFileAlignment = translateURI(String.valueOf(rd.getValue("alignmentfile/DatafileUrl")), remoteJobRoot, localJobRoot); + toCreate.put("alignmentfile", getOrCreateExpData(newFileAlignment, targetWorkbook)); + + if (rd.getValue("reference_library") != null) + { + URI newFile2 = translateURI(String.valueOf(rd.getValue("reference_library/DatafileUrl")), remoteJobRoot, localJobRoot); + toCreate.put("reference_library", getOrCreateExpData(newFile2, targetWorkbook)); + } + + //Create run: + if (rd.getValue("runid") != null && rd.getValue("runid/JobId") != null) + { + int runId = createExpRun(Integer.parseInt(String.valueOf(rd.getValue("runid"))), targetWorkbook, String.valueOf(rd.getValue("runid/Name")), jobId); + toCreate.put("runid", runId); + } + else + { + _log.error("analysis missing runid: " + remoteId); + } + + BatchValidationException bve = new BatchValidationException(); + List> created = analysisTable.getUpdateService().insertRows(getJob().getUser(), getPipelineJob().targetContainer, Arrays.asList(toCreate), bve, null, null); + if (bve.hasErrors()) + { + throw new RuntimeException(bve); + } + + analysisMap.put(remoteId, Integer.parseInt(String.valueOf(created.get(0).get("rowid")))); + } + catch (Exception e) + { + throw new RuntimeException(e); + } + } + }); + } + catch (Exception e) + { + _log.error(e.getMessage(), e); + throw new RuntimeException(e); + } + } + + private void createReaddata() + { + _log.info("Creating read data"); + try + { + final TableInfo readdataTable = QueryService.get().getUserSchema(getJob().getUser(), getPipelineJob().targetContainer, "sequenceanalysis").getTable("readdata"); + + SelectRowsCommand sr = new SelectRowsCommand("sequenceanalysis", "readdata"); + sr.setColumns(Arrays.asList("rowid", "readset", "platformUnit", "centerName", "date", "fileid1", "fileid1/DataFileUrl", "fileid2", "fileid2/DataFileUrl", "fileid1/Name", "description", "sra_accession", "runid", "runid/jobid", "runid/Name", "readset/workbook/workbookId", "runid/JobId", "runid/Name", "runid/JobId/FilePath", "runid/JobId/Description")); + + SelectRowsResponse srr = sr.execute(getConnection(), getPipelineJob().remoteServerFolder); + + srr.getRowset().forEach(rd -> { + int remoteId = Integer.parseInt(String.valueOf(rd.getValue("rowid"))); + int remoteReadset = Integer.parseInt(String.valueOf(rd.getValue("readset"))); + Integer localReadset = readsetMap.get(remoteReadset); + if (localReadset == null) + { + throw new IllegalArgumentException("Unable to find readset for remote id: " + remoteReadset); + } + + Readset rs = SequenceAnalysisService.get().getReadset(localReadset, getJob().getUser()); + Container targetWorkbook = ContainerManager.getForId(rs.getContainer()); + + SimpleFilter rdFilter = new SimpleFilter(FieldKey.fromString("readset"), rs.getRowId()); + rdFilter.addCondition(FieldKey.fromString("runid/JobId/Description"), rd.getValue("runid/JobId/Description")); + rdFilter.addCondition(FieldKey.fromString("fileid1/Name"), rd.getValue("fileid1/Name")); + rdFilter.addCondition(FieldKey.fromString("container"), targetWorkbook.getId(), CompareType.EQUAL); + + if (rd.getValue("platformUnit") != null) + { + rdFilter.addCondition(FieldKey.fromString("platformUnit"), rd.getValue("platformUnit")); + } + + TableSelector tsReaddata = new TableSelector(readdataTable, PageFlowUtil.set("rowid"), rdFilter, null); + if (tsReaddata.exists()) + { + readdataMap.put(remoteId, tsReaddata.getObject(Integer.class)); + } + else + { + Map toCreate = new CaseInsensitiveHashMap<>(); + toCreate.put("readset", rs.getRowId()); + toCreate.put("platformUnit", rd.getValue("platformUnit")); + toCreate.put("centerName", rd.getValue("centerName")); + toCreate.put("date", rd.getValue("date")); + toCreate.put("description", rd.getValue("description")); + toCreate.put("sra_accession", rd.getValue("sra_accession")); + try + { + if (rd.getValue("runid/JobId") != null) + { + int remoteJobId = Integer.parseInt(String.valueOf(rd.getValue("runid/JobId"))); + int jobId = getOrCreateJob(remoteJobId, targetWorkbook); + PipelineStatusFile sf = PipelineService.get().getStatusFile(jobId); + + String localJobRoot = getParent(sf.getFilePath()); + String remoteJobRoot = getParent(URI.create(String.valueOf(rd.getValue("runid/JobId/FilePath")).replaceAll(" ", "_")).getPath()); + + if (rd.getValue("fileid1/DataFileUrl") != null) + { + URI newFile1 = translateURI(String.valueOf(rd.getValue("fileid1/DataFileUrl")), remoteJobRoot, localJobRoot); + toCreate.put("fileid1", getOrCreateExpData(newFile1, targetWorkbook)); + } + + if (rd.getValue("fileid2/DataFileUrl") != null) + { + URI newFile2 = translateURI(String.valueOf(rd.getValue("fileid2/DatafileUrl")), remoteJobRoot, localJobRoot); + toCreate.put("fileid2", getOrCreateExpData(newFile2, targetWorkbook)); + } + } + else + { + _log.error("readddata missing jobid: " + remoteId); + } + + //Create run: + if (rd.getValue("runid") != null && rd.getValue("runid/JobId") != null) + { + int remoteJobId = Integer.parseInt(String.valueOf(rd.getValue("runid/JobId"))); + int jobId = getOrCreateJob(remoteJobId, targetWorkbook); + int runId = createExpRun(Integer.parseInt(String.valueOf(rd.getValue("runid"))), targetWorkbook, String.valueOf(rd.getValue("runid/Name")), jobId); + toCreate.put("runid", runId); + } + else + { + _log.error("readddata missing runid: " + remoteId); + } + + BatchValidationException bve = new BatchValidationException(); + List> created = readdataTable.getUpdateService().insertRows(getJob().getUser(), getPipelineJob().targetContainer, Arrays.asList(toCreate), bve, null, null); + if (bve.hasErrors()) + { + throw new RuntimeException(bve); + } + + readdataMap.put(remoteId, Integer.parseInt(String.valueOf(created.get(0).get("rowid")))); + } + catch (Exception e) + { + throw new RuntimeException(e); + } + } + }); + } + catch (Exception e) + { + _log.error(e.getMessage(), e); + throw new RuntimeException(e); + } + } + + private int getOrCreateExpData(URI file, Container workbook) + { + ExpData ret = ExperimentService.get().getExpDataByURL(new File(file), workbook); + if (ret == null) + { + ret = ExperimentService.get().createData(workbook, new DataType("Data")); + ret.setDataFileURI(file); + ret.save(getJob().getUser()); + } + + return ret.getRowId(); + } + + private void createReadsets() + { + _log.info("Creating readsets"); + try + { + final UserSchema us = QueryService.get().getUserSchema(getJob().getUser(), getPipelineJob().targetContainer, "sequenceanalysis"); + final TableInfo readsetTable = us.getTable("sequence_readsets"); + + SelectRowsCommand sr = new SelectRowsCommand("sequenceanalysis", "sequence_readsets"); + sr.setColumns(Arrays.asList("rowid", "name", "platform", "application", "librarytype", "chemistry", "comments", "status", "subjectid", "subjectdate", "sampletype", "sampleid", "barcode5", "barcode3", "runid", "runid/jobid", "runid/Name", "workbook/workbookId", "runid/JobId", "runid/Name", "runid/JobId/FilePath")); + + SelectRowsResponse srr = sr.execute(getConnection(), getPipelineJob().remoteServerFolder); + + srr.getRowset().forEach(rs -> { + int remoteId = Integer.parseInt(String.valueOf(rs.getValue("rowid"))); + int sourceWorkbook = Integer.parseInt(String.valueOf(rs.getValue("workbook/workbookId"))); + Container targetWorkbook = workbookMap.get(sourceWorkbook); + if (targetWorkbook == null) + { + throw new IllegalArgumentException("Unable to find local workbook for source: " + sourceWorkbook); + } + + SimpleFilter rsFilter = new SimpleFilter(FieldKey.fromString("name"), rs.getValue("name")); + rsFilter.addCondition(FieldKey.fromString("container"), targetWorkbook.getId(), CompareType.EQUAL); + if (rs.getValue("subjectid") != null) + { + rsFilter.addCondition(FieldKey.fromString("subjectid"), rs.getValue("subjectid"), CompareType.EQUAL); + } + + TableSelector tsReadset = new TableSelector(readsetTable, PageFlowUtil.set("rowid"), rsFilter, null); + if (tsReadset.exists()) + { + readsetMap.put(remoteId, tsReadset.getObject(Integer.class)); + } + else + { + Map toCreate = new CaseInsensitiveHashMap<>(); + toCreate.put("name", rs.getValue("name")); + toCreate.put("platform", rs.getValue("platform")); + toCreate.put("application", rs.getValue("application")); + toCreate.put("barcode5", rs.getValue("barcode5")); + toCreate.put("barcode3", rs.getValue("barcode3")); + toCreate.put("subjectid", rs.getValue("subjectid")); + + toCreate.put("sampleid", rs.getValue("sampleid")); + toCreate.put("sampledate", rs.getValue("sampledate")); + toCreate.put("librarytype", rs.getValue("librarytype")); + toCreate.put("sampletype", rs.getValue("sampletype")); + toCreate.put("chemistry", rs.getValue("chemistry")); + toCreate.put("comments", rs.getValue("comments")); + toCreate.put("status", rs.getValue("status")); + + toCreate.put("container", targetWorkbook.getId()); + + try + { + //Create run: + if (rs.getValue("runid") != null && rs.getValue("runid/JobId") != null) + { + int remoteJobId = Integer.parseInt(String.valueOf(rs.getValue("runid/JobId"))); + int jobId = getOrCreateJob(remoteJobId, targetWorkbook); + int runid = createExpRun(Integer.parseInt(String.valueOf(rs.getValue("runid"))), targetWorkbook, String.valueOf(rs.getValue("runid/Name")), jobId); + toCreate.put("runid", runid); + } + else + { + _log.error("readset missing run id: " + remoteId); + } + + BatchValidationException bve = new BatchValidationException(); + List> created = readsetTable.getUpdateService().insertRows(getJob().getUser(), getPipelineJob().targetContainer, Arrays.asList(toCreate), bve, null, null); + if (bve.hasErrors()) + { + throw new RuntimeException(bve); + } + + readsetMap.put(remoteId, Integer.parseInt(String.valueOf(created.get(0).get("rowid")))); + } + catch (Exception e) + { + throw new RuntimeException(e); + } + } + }); + } + catch (Exception e) + { + _log.error(e.getMessage(), e); + throw new RuntimeException(e); + } + } + + private int getOrCreateJob(int remoteJobId, Container targetWorkbook) + { + if (jobIdMap.containsKey(remoteJobId)) + { + return jobIdMap.get(remoteJobId); + } + + TableInfo ti = DbSchema.get("pipeline", DbSchemaType.Module).getTable("StatusFiles"); + + try + { + SelectRowsCommand sr = new SelectRowsCommand("pipeline", "job"); + sr.addFilter(new Filter("rowid", remoteJobId, Filter.Operator.EQUAL)); + sr.setColumns(Arrays.asList("RowId", "Info", "FilePath", "Email", "Description", "DataUrl", "Job", "Provider", "HadError", "ActiveTaskId")); + + SelectRowsResponse srr = sr.execute(getConnection(), getPipelineJob().remoteServerFolder); + + File fr = PipelineService.get().getPipelineRootSetting(targetWorkbook).getRootPath(); + + AtomicInteger ret = new AtomicInteger(); + srr.getRowset().forEach(pj -> { + String filepath = String.valueOf(pj.getValue("FilePath")); + if (!filepath.contains("@files")) + { + //This appears to be an error in PRIMe's data: + if (filepath.contains("illuminaImport")) + { + filepath = filepath.replace("illuminaImport", "@files/illuminaImport"); + } + else if (filepath.contains("sequenceAnalysis")) + { + filepath = filepath.replace("sequenceAnalysis", "@files/sequenceAnalysis"); + } + else + { + _log.error("Unexpected filepath: " + pj.getValue("FilePath")); + } + } + + File remoteDir = new File(URI.create(filepath.replaceAll(" ", "_")).getPath()); + File localDir = new File(fr, filepath.split("@files")[1]); + + //Check for existing row: + TableSelector ts = new TableSelector(ti, PageFlowUtil.set("RowId"), new SimpleFilter(FieldKey.fromString("Job"), pj.getValue("Job")), null); + if (ts.exists()) + { + ret.set(ts.getObject(Integer.class)); + } + else + { + Map toCreate = new CaseInsensitiveHashMap<>(); + toCreate.put("Info", pj.getValue("Info")); + toCreate.put("FilePath", localDir.getPath()); + toCreate.put("Email", pj.getValue("Email")); + toCreate.put("Description", pj.getValue("Description")); + toCreate.put("DataUrl", pj.getValue("DataUrl")); + toCreate.put("Job", pj.getValue("Job")); + toCreate.put("Provider", pj.getValue("Provider")); + toCreate.put("HadError", pj.getValue("HadError")); + toCreate.put("ActiveTaskId", pj.getValue("ActiveTaskId")); + toCreate.put("Container", targetWorkbook.getId()); + + toCreate = Table.insert(getJob().getUser(), ti, toCreate); + + ret.set((int) toCreate.get("RowId")); + } + + if (localDir.exists()) + { + _log.info("Directory exists, will not re-copy: " + localDir.getPath()); + return; + } + + try + { + _log.info(remoteDir.getPath()); + _log.info(localDir.getPath()); + + if (!localDir.getParentFile().exists()) + { + localDir.getParentFile().mkdirs(); + } + + if (remoteDir.exists()) + { + FileUtils.copyDirectory(remoteDir, localDir); + } + else + { + _log.error("source folder not found: " + remoteDir.getPath()); + } + } + catch (Exception e) + { + throw new RuntimeException(e); + } + }); + + jobIdMap.put(remoteJobId, ret.get()); + + return ret.get(); + } + catch (Exception e) + { + _log.error(e.getMessage(), e); + throw new RuntimeException(e); + } + } + + private int createExpRun(int remoteId, Container c, String name, int localJobId) throws Exception + { + if (runIdMap.containsKey(remoteId)) + { + return runIdMap.get(remoteId); + } + else + { + ExpRun ret = ExperimentService.get().createRunForProvenanceRecording(c, getJob().getUser(), new RecordedActionSet(), name, localJobId); + runIdMap.put(remoteId, ret.getRowId()); + + return ret.getRowId(); + } + } + + private void createWorkbooks() + { + _log.info("Creating workbooks"); + try + { + TableInfo containers = QueryService.get().getUserSchema(getJob().getUser(), getPipelineJob().targetContainer, "core").getTable("containers"); + + SelectRowsCommand sr = new SelectRowsCommand("core", "workbooks"); + sr.setColumns(Arrays.asList("Name", "Title", "Description")); + SelectRowsResponse srr = sr.execute(getConnection(), getPipelineJob().remoteServerFolder); + + srr.getRowset().forEach(wb -> { + String localTitle = (String) wb.getValue("Title"); + + TableSelector ts = new TableSelector(containers, PageFlowUtil.set("RowId"), new SimpleFilter(FieldKey.fromString("Title"), localTitle), null); + if (ts.exists()) + { + Container workbook = ContainerManager.getForRowId(ts.getObject(Integer.class)); + workbookMap.put(Integer.parseInt(String.valueOf(wb.getValue("Name"))), workbook); + } + else + { + String description = String.valueOf(wb.getValue("Description")); + if (description != null) + { + description = description + ". "; + } + else + { + description = ""; + } + + description = description + "Originally PRIMe workbook: " + wb.getValue("Name"); + + Container workbook = ContainerManager.createContainer(getPipelineJob().targetContainer, null, localTitle, description, WorkbookContainerType.NAME, getJob().getUser()); + workbookMap.put(Integer.parseInt(String.valueOf(wb.getValue("Name"))), workbook); + } + }); + } + catch (CommandException | IOException e) + { + throw new RuntimeException(e); + } + } + + private URI translateURI(String databaseURI, String remoteFolderRoot, String localFolderRoot) + { + databaseURI = databaseURI.replace("\\", "/"); + remoteFolderRoot = remoteFolderRoot.replace("\\", "/").split("@files")[0]; + localFolderRoot = localFolderRoot.replace("\\", "/").split("@files")[0]; + if (localFolderRoot.startsWith("C:")) + { + localFolderRoot = localFolderRoot.replaceAll("^C:", ""); + } + + databaseURI = databaseURI.replace(remoteFolderRoot, localFolderRoot); + + return URI.create(databaseURI); + } + } +} diff --git a/primeseq/webapp/WEB-INF/primeseqContext.xml b/primeseq/webapp/WEB-INF/primeseqContext.xml index 411bb99a8..b91d3dfb3 100644 --- a/primeseq/webapp/WEB-INF/primeseqContext.xml +++ b/primeseq/webapp/WEB-INF/primeseqContext.xml @@ -3,6 +3,28 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-2.0.xsd"> + + + + + + + + + + + + + + + org.labkey.primeseq.pipeline.MhcMigrationPipelineJob.Task + + + + + + + diff --git a/tcrdb/resources/assay/TCRdb/queries/Data/cDNA Info.qview.xml b/tcrdb/resources/assay/TCRdb/queries/Data/cDNA Info.qview.xml index d8518ac12..2b3104ee9 100644 --- a/tcrdb/resources/assay/TCRdb/queries/Data/cDNA Info.qview.xml +++ b/tcrdb/resources/assay/TCRdb/queries/Data/cDNA Info.qview.xml @@ -2,9 +2,9 @@ - - - + + + diff --git a/tcrdb/resources/queries/singlecell/cdna_libraries/Assay Info.qview.xml b/tcrdb/resources/queries/singlecell/cdna_libraries/Assay Info.qview.xml index 8a69ed8a5..109d35ad5 100644 --- a/tcrdb/resources/queries/singlecell/cdna_libraries/Assay Info.qview.xml +++ b/tcrdb/resources/queries/singlecell/cdna_libraries/Assay Info.qview.xml @@ -2,10 +2,10 @@ - - - - + + + + diff --git a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java index 71d1c1820..26b1e01fa 100644 --- a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java +++ b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java @@ -38,7 +38,9 @@ public class CellRangerVDJCellHashingHandler extends AbstractParameterizedOutputHandler { - private FileType _fileType = new FileType("vloupe", false); + private FileType _vloupeFileType = new FileType("vloupe", false); + private FileType _htmlFileType = new FileType("html", false); + public static final String CATEGORY = "Cell Hashing Calls (VDJ)"; public static final String TARGET_ASSAY = "targetAssay"; @@ -47,7 +49,7 @@ public class CellRangerVDJCellHashingHandler extends AbstractParameterizedOutput public CellRangerVDJCellHashingHandler() { - super(ModuleLoader.getInstance().getModule(TCRdbModule.class), "CellRanger VDJ Import", "This will either directly import data (if cell hashing is not used), or run CiteSeqCount/MultiSeqClassifier to generate a sample-to-cellbarcode TSV based on the filtered barcodes from CellRanger VDJ and then import.", new LinkedHashSet<>(PageFlowUtil.set("tcrdb/field/AssaySelectorField.js")), getDefaultParams()); + super(ModuleLoader.getInstance().getModule(TCRdbModule.class), "CellRanger VDJ Import", "This will either directly import data (if cell hashing is not used), or run cellhashR on the hashing count matrix to generate a sample-to-cellbarcode TSV based on the filtered barcodes from CellRanger VDJ and then import.", new LinkedHashSet<>(PageFlowUtil.set("tcrdb/field/AssaySelectorField.js")), getDefaultParams()); } private static List getDefaultParams() @@ -65,7 +67,7 @@ private static List getDefaultParams() }}, false) )); - ret.addAll(CellHashingService.get().getDefaultHashingParams(true)); + ret.addAll(CellHashingService.get().getHashingCallingParams()); return ret; } @@ -73,7 +75,7 @@ private static List getDefaultParams() @Override public boolean canProcess(SequenceOutputFile o) { - return o.getFile() != null && _fileType.isType(o.getFile()); + return o.getFile() != null && (_vloupeFileType.isType(o.getFile()) || (_htmlFileType.isType(o.getFile()) && "10x Run Summary".equals(o.getCategory()) && o.getName().contains("VDJ Summary"))); } @Override @@ -112,7 +114,7 @@ public class Processor implements SequenceOutputHandler.SequenceOutputProcessor public void init(JobContext ctx, List inputFiles, List actions, List outputsToCreate) throws UnsupportedOperationException, PipelineJobException { //NOTE: this is the pathway to import assay data, whether hashing is used or not - CellHashingService.get().prepareHashingAndCiteSeqFilesIfNeeded(ctx.getOutputDir(), ctx.getJob(), ctx.getSequenceSupport(), "tcrReadsetId", ctx.getParams().optBoolean("excludeFailedcDNA", true), false, false); + CellHashingService.get().prepareHashingAndCiteSeqFilesIfNeeded(ctx.getOutputDir(), ctx.getJob(), ctx.getSequenceSupport(), "tcrReadsetId", false, false); if (ctx.getParams().optBoolean(USE_GEX_BARCODES, false)) { @@ -160,7 +162,7 @@ public void complete(PipelineJob job, List inputFiles, List< for (SequenceOutputFile so : inputFiles) { AnalysisModel model = support.getCachedAnalysis(so.getAnalysis_id()); - new CellRangerVDJUtils(job.getLogger()).importAssayData(job, model, job.getLogFile().getParentFile(), assayId, null, deleteExistingData); + new CellRangerVDJUtils(job.getLogger()).importAssayData(job, model, so.getFile(), job.getLogFile().getParentFile(), assayId, null, deleteExistingData); } } } @@ -200,27 +202,28 @@ private void processVloupeFile(JobContext ctx, File perCellTsv, Readset rs, Reco { AlignmentOutputImpl output = new AlignmentOutputImpl(); - List htosPerReadset = CellHashingService.get().getHtosForParentReadset(rs.getReadsetId(), ctx.getSourceDirectory(), ctx.getSequenceSupport()); + Set htosPerReadset = CellHashingService.get().getHtosForParentReadset(rs.getReadsetId(), ctx.getSourceDirectory(), ctx.getSequenceSupport()); if (htosPerReadset.size() > 1) { ctx.getLogger().info("Total HTOs for readset: " + htosPerReadset.size()); //TODO: allow union of GEX and TCR cell barcodes for whitelist! - CellHashingService.CellHashingParameters parameters = CellHashingService.CellHashingParameters.createFromJson(CellHashingService.BARCODE_TYPE.hashing, ctx.getSourceDirectory(), ctx.getParams(), null, rs, null); - parameters.cellBarcodeWhitelistFile = createCellbarcodeWhitelist(ctx, perCellTsv, true); + CellHashingService.CellHashingParameters parameters = CellHashingService.CellHashingParameters.createFromJson(CellHashingService.BARCODE_TYPE.hashing, ctx.getSourceDirectory(), ctx.getParams(), null, rs); parameters.genomeId = genomeId; parameters.outputCategory = CATEGORY; parameters.basename = FileUtil.makeLegalName(rs.getName()); - parameters.allowableHtoOrCiteseqBarcodes = htosPerReadset; + parameters.allowableHtoBarcodes = htosPerReadset; + parameters.cellBarcodeWhitelistFile = createCellbarcodeWhitelist(ctx, perCellTsv, true); + File existingCountMatrixUmiDir = CellHashingService.get().getExistingFeatureBarcodeCountDir(rs, CellHashingService.BARCODE_TYPE.hashing, ctx.getSequenceSupport()); - File cellToHto = CellHashingService.get().processCellHashingOrCiteSeqForParent(rs, output, ctx, parameters); + File cellToHto = CellHashingService.get().generateHashingCallsForRawMatrix(rs, output, ctx, parameters, existingCountMatrixUmiDir); if (CellHashingService.get().usesCellHashing(ctx.getSequenceSupport(), ctx.getSourceDirectory()) && cellToHto == null) { throw new PipelineJobException("Missing cell to HTO file"); - } + action.addOutput(cellToHto, CellRangerVDJUtils.TCR_HASHING_CALLS, false); ctx.getFileManager().addStepOutputs(action, output); } else if (htosPerReadset.size() == 1) @@ -241,6 +244,8 @@ private File createCellbarcodeWhitelist(JobContext ctx, File perCellTsv, boolean Set uniqueBarcodesIncludingNoCDR3 = new HashSet<>(); ctx.getLogger().debug("writing cell barcodes, using file: " + perCellTsv.getPath()); ctx.getLogger().debug("allow cells lacking CDR3: " + allowCellsLackingCDR3); + + int totalBarcodeWritten = 0; try (CSVWriter writer = new CSVWriter(PrintWriters.getPrintWriter(cellBarcodeWhitelist), ',', CSVWriter.NO_QUOTE_CHARACTER); CSVReader reader = new CSVReader(Readers.getReader(perCellTsv), ',')) { int rowIdx = 0; @@ -272,6 +277,7 @@ private File createCellbarcodeWhitelist(JobContext ctx, File perCellTsv, boolean { writer.writeNext(new String[]{barcode}); uniqueBarcodes.add(barcode); + totalBarcodeWritten++; } uniqueBarcodesIncludingNoCDR3.add(barcode); @@ -298,6 +304,7 @@ private File createCellbarcodeWhitelist(JobContext ctx, File perCellTsv, boolean for (String barcode : uniqueBarcodesIncludingNoCDR3) { writer.writeNext(new String[]{barcode}); + totalBarcodeWritten++; } } catch (IOException e) @@ -306,6 +313,11 @@ private File createCellbarcodeWhitelist(JobContext ctx, File perCellTsv, boolean } } + if (totalBarcodeWritten == 0) + { + throw new PipelineJobException("No valid cell barcodes found!"); + } + //TODO: consider looking up GEX data? return cellBarcodeWhitelist; diff --git a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJUtils.java b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJUtils.java index b6b9640c3..b52974def 100644 --- a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJUtils.java +++ b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJUtils.java @@ -56,6 +56,8 @@ public class CellRangerVDJUtils { + public static final String TCR_HASHING_CALLS = "Cell Hashing TCR Calls"; + private Logger _log; public CellRangerVDJUtils(Logger log) @@ -63,8 +65,10 @@ public CellRangerVDJUtils(Logger log) _log = log; } - public void importAssayData(PipelineJob job, AnalysisModel model, File outDir, Integer assayId, @Nullable Integer runId, boolean deleteExisting) throws PipelineJobException + public void importAssayData(PipelineJob job, AnalysisModel model, File vLoupeFile, File outDir, Integer assayId, @Nullable Integer runId, boolean deleteExisting) throws PipelineJobException { + File cellRangerOutDir = vLoupeFile.getParentFile(); + if (assayId == null) { _log.info("No assay selected, will not import"); @@ -77,32 +81,29 @@ public void importAssayData(PipelineJob job, AnalysisModel model, File outDir, I throw new PipelineJobException("Unable to find protocol: " + assayId); } - File allCsv = getPerCellCsv(outDir); + File allCsv = getPerCellCsv(cellRangerOutDir); if (!allCsv.exists()) { _log.warn("unable to find consensus contigs: " + allCsv .getPath()); return; } - File consensusCsv = new File(outDir, "consensus_annotations.csv"); + File consensusCsv = new File(cellRangerOutDir, "consensus_annotations.csv"); if (!consensusCsv .exists()) { - _log.warn("unable to find consensus contigs: " + consensusCsv .getPath()); - return; + throw new PipelineJobException("unable to find consensus contigs: " + consensusCsv .getPath()); } - File consensusFasta = new File(outDir, "consensus.fasta"); + File consensusFasta = new File(cellRangerOutDir, "consensus.fasta"); if (!consensusFasta.exists()) { - _log.warn("unable to find FASTA: " + consensusFasta.getPath()); - return; + throw new PipelineJobException("unable to find FASTA: " + consensusFasta.getPath()); } - File allFasta = new File(outDir, "all_contig.fasta"); + File allFasta = new File(cellRangerOutDir, "all_contig.fasta"); if (!allFasta.exists()) { - _log.warn("unable to find FASTA: " + allFasta.getPath()); - return; + throw new PipelineJobException("unable to find FASTA: " + allFasta.getPath()); } _log.info("loading results into assay: " + assayId); @@ -291,10 +292,15 @@ else if ("Negative".equals(hto)) continue; } - if ("None".equals(line[9])) + String cGene = removeNone(line[9]); + if (cGene == null) { - noCGene++; - continue; + // Only discard these if chain type doesnt match between JGene and VGene. + if (!line[8].substring(0, 3).equals(line[6].substring(0,3))) + { + noCGene++; + continue; + } } if ("False".equals(line[10])) @@ -342,9 +348,9 @@ else if (discordantBarcodes.contains(barcode)) //NOTE: chimeras with a TRDV / TRAJ / TRAC are relatively common. categorize as TRA for reporting ease String locus = line[5]; - if (locus.equals("Multi") && removeNone(line[9]) != null && removeNone(line[8]) != null && removeNone(line[6]) != null) + if (locus.equals("Multi") && cGene != null && removeNone(line[8]) != null && removeNone(line[6]) != null) { - if (removeNone(line[9]).contains("TRAC") && removeNone(line[8]).contains("TRAJ") && removeNone(line[6]).contains("TRDV")) + if (cGene.contains("TRAC") && removeNone(line[8]).contains("TRAJ") && removeNone(line[6]).contains("TRDV")) { locus = "TRA"; multiChainConverted++; @@ -352,7 +358,7 @@ else if (discordantBarcodes.contains(barcode)) } // Aggregate by: cDNA_ID, cdr3, chain, raw_clonotype_id, sequenceContigName, vHit, dHit, jHit, cHit, cdr3_nt - String key = StringUtils.join(new String[]{cDNA.toString(), line[12], locus, clonotypeId, sequenceContigName, removeNone(line[6]), removeNone(line[7]), removeNone(line[8]), removeNone(line[9]), removeNone(line[13])}, "<>"); + String key = StringUtils.join(new String[]{cDNA.toString(), line[12], locus, clonotypeId, sequenceContigName, removeNone(line[6]), removeNone(line[7]), removeNone(line[8]), cGene, removeNone(line[13])}, "<>"); AssayModel am; if (!rows.containsKey(key)) { @@ -460,7 +466,7 @@ private AssayModel createForRow(String[] line, String sequenceContigName, Intege private File getCellToHtoFile(ExpRun run) throws PipelineJobException { - List datas = run.getInputDatas(CellHashingService.HASHING_CALLS, ExpProtocol.ApplicationType.ExperimentRunOutput); + List datas = run.getInputDatas(TCR_HASHING_CALLS, ExpProtocol.ApplicationType.ExperimentRunOutput); if (datas.isEmpty()) { throw new PipelineJobException("Unable to find hashing calls output"); @@ -647,8 +653,8 @@ public static void deleteExistingData(AssayProvider ap, ExpProtocol protocol, Co } } - public static File getPerCellCsv(File outDir) + public static File getPerCellCsv(File cellRangerOutDir) { - return new File(outDir, "all_contig_annotations.csv"); + return new File(cellRangerOutDir, "all_contig_annotations.csv"); } }