Allow studies to prevent new timepoint creation on data import.

api/schemas/study.xsd

@@ -469,6 +469,13 @@
  </xs:documentation>
  </xs:annotation>
  </xs:attribute>
+ <xs:attribute name="failForUndefinedTimepoints" type="xs:boolean">
+ <xs:annotation>
+ <xs:documentation>
+ Indicates whether new timepoints can be created automatically when data is imported into the study.
+ </xs:documentation>
+ </xs:annotation>
+ </xs:attribute>
  <xs:anyAttribute processContents="lax"/>
  </xs:complexType>
 	</xs:element>

api/src/org/labkey/api/study/Study.java

@@ -152,6 +152,7 @@ public interface Study extends StudyEntity
  boolean isAllowReqLocClinic();
  boolean isAllowReqLocSal();
  boolean isAllowReqLocEndpoint();
+ boolean isFailForUndefinedTimepoints();
 
  // "is" prefix doesn't work with "Boolean", use get
  /**

pipeline/src/org/labkey/pipeline/startPipelineImport.jsp

@@ -93,7 +93,7 @@ Ext4.onReady(function()
  isApplyToMultipleFolders: <%=bean.isApplyToMultipleFolders()%>,
  isFailForUndefinedVisits: <%=bean.isFailForUndefinedVisits()%>,
  isCloudRoot: <%=bean.isCloudRoot()%>, // Remove as part of Issue #43835
- showFailForUndefinedVisits: <%=timepointType == null || timepointType == TimepointType.VISIT%>
+ showFailForUndefinedVisits: <%=(timepointType == null || timepointType == TimepointType.VISIT) && ((study == null) || !study.isFailForUndefinedTimepoints())%>
  });
  },
  failure: function(response)

specimen/src/org/labkey/specimen/SpecimenModule.java

@@ -38,6 +38,7 @@
 import org.labkey.api.query.FieldKey;
 import org.labkey.api.query.QueryParam;
 import org.labkey.api.query.QueryView;
+import org.labkey.api.query.ValidationException;
 import org.labkey.api.security.User;
 import org.labkey.api.security.roles.RoleManager;
 import org.labkey.api.specimen.SpecimenMigrationService;
@@ -189,7 +190,7 @@ public ActionURL getUpdateSpecimenQueryRowURL(Container c, String schemaName, Ta
  }
 
  @Override
- public void importSpecimenArchive(@Nullable Path inputFile, PipelineJob job, SimpleStudyImportContext ctx, boolean merge, boolean syncParticipantVisit) throws PipelineJobException
+ public void importSpecimenArchive(@Nullable Path inputFile, PipelineJob job, SimpleStudyImportContext ctx, boolean merge, boolean syncParticipantVisit) throws PipelineJobException, ValidationException
  {
  AbstractSpecimenTask.doImport(inputFile, job, ctx, merge, syncParticipantVisit);
  }

specimen/src/org/labkey/specimen/actions/ShowUploadSpecimensAction.java

@@ -25,6 +25,7 @@
 import org.labkey.api.data.RuntimeSQLException;
 import org.labkey.api.pipeline.PipelineUrls;
 import org.labkey.api.portal.ProjectUrls;
+import org.labkey.api.query.ValidationException;
 import org.labkey.api.reader.ColumnDescriptor;
 import org.labkey.api.reader.TabLoader;
 import org.labkey.api.security.RequiresPermission;
@@ -262,7 +263,7 @@ else if (study.getTimepointType() == TimepointType.VISIT && null == row.get(Simp
  if (!errors.hasErrors())
  importer.process(specimenRows, form.isMerge());
  }
- catch (IllegalStateException e)
+ catch (IllegalStateException | ValidationException e)
  {
  errors.reject(SpringActionController.ERROR_MSG, e.getMessage());
  }

specimen/src/org/labkey/specimen/importer/AbstractSpecimenTask.java

@@ -25,6 +25,7 @@
 import org.labkey.api.pipeline.PipelineJobException;
 import org.labkey.api.pipeline.RecordedActionSet;
 import org.labkey.api.pipeline.TaskFactory;
+import org.labkey.api.query.ValidationException;
 import org.labkey.api.specimen.importer.SpecimenImporter;
 import org.labkey.api.study.SpecimenService;
 import org.labkey.api.study.SpecimenTransform;
@@ -98,7 +99,7 @@ protected SimpleStudyImportContext getImportContext(PipelineJob job)
  }
 
  public static void doImport(@Nullable Path inputFile, PipelineJob job, SimpleStudyImportContext ctx, boolean merge,
- boolean syncParticipantVisit) throws PipelineJobException
+ boolean syncParticipantVisit) throws PipelineJobException, ValidationException
  {
  doImport(inputFile, job, ctx, merge, syncParticipantVisit, new DefaultImportHelper());
  }
@@ -115,7 +116,7 @@ private static void doPostTransform(SpecimenTransform transformer, Path inputFil
  }
 
  public static void doImport(@Nullable Path inputFile, PipelineJob job, SimpleStudyImportContext ctx, boolean merge,
- boolean syncParticipantVisit, ImportHelper importHelper) throws PipelineJobException
+ boolean syncParticipantVisit, ImportHelper importHelper) throws PipelineJobException, ValidationException
  {
  // do nothing if we've specified data types and specimen is not one of them
  if (!ctx.isDataTypeSelected(SpecimenArchiveDataTypes.SPECIMENS))
@@ -172,7 +173,11 @@ public static void doImport(@Nullable Path inputFile, PipelineJob job, SimpleStu
  // Since changing specimens in this study will impact specimens in ancillary studies dependent on this study,
  // we need to force a participant/visit refresh in those study containers (if any):
  for (Study dependentStudy : StudyService.get().getAncillaryStudies(ctx.getContainer()))
- VisitService.get().updateParticipantVisits(dependentStudy, ctx.getUser());
+ {
+ ValidationException errors = VisitService.get().updateParticipantVisits(dependentStudy, ctx.getUser());
+ if (errors.hasErrors())
+ throw errors;
+ }
  }
  }
 

study/api-src/org/labkey/api/specimen/SpecimenMigrationService.java

@@ -5,6 +5,7 @@
 import org.labkey.api.data.TableInfo;
 import org.labkey.api.pipeline.PipelineJob;
 import org.labkey.api.pipeline.PipelineJobException;
+import org.labkey.api.query.ValidationException;
 import org.labkey.api.security.User;
 import org.labkey.api.services.ServiceRegistry;
 import org.labkey.api.study.importer.SimpleStudyImportContext;
@@ -36,7 +37,7 @@ static void setInstance(SpecimenMigrationService impl)
  ActionURL getUpdateSpecimenQueryRowURL(Container c, String schemaName, TableInfo table);
 
  void importSpecimenArchive(@Nullable Path inputFile, PipelineJob job, SimpleStudyImportContext ctx, boolean merge,
- boolean syncParticipantVisit) throws PipelineJobException;
+ boolean syncParticipantVisit) throws PipelineJobException, ValidationException;
 
  void clearRequestCaches(Container c);
  void clearGroupedValuesForColumn(Container container);

study/api-src/org/labkey/api/specimen/importer/SpecimenImporter.java

@@ -256,7 +256,7 @@ public SpecimenImporter(Container container, User user)
  SpecimenSchema.get().getTableInfoSpecimenPrimaryType(getContainer()).getSelectName(), SpecimenColumns.PRIMARYTYPE_COLUMNS);
  }
 
- private void resyncStudy(boolean syncParticipantVisit)
+ private void resyncStudy(boolean syncParticipantVisit, boolean failForUndefinedVisits) throws ValidationException
  {
  TableInfo tableParticipant = SpecimenSchema.get().getTableInfoParticipant();
  TableInfo tableSpecimen = getTableInfoSpecimen();
@@ -271,7 +271,9 @@ private void resyncStudy(boolean syncParticipantVisit)
  {
  Study study = StudyService.get().getStudy(getContainer());
  info("Updating study-wide subject/visit information...");
- VisitService.get().updateParticipantVisitsWithCohortUpdate(study, getUser(), _logger);
+ ValidationException errors = VisitService.get().updateParticipantVisitsWithCohortUpdate(study, getUser(), failForUndefinedVisits, _logger);
+ if (errors.hasErrors())
+ throw errors;
  info("Subject/visit update complete.");
  }
 
@@ -302,14 +304,16 @@ public void process(VirtualFile specimensDir, boolean merge, SimpleStudyImportCo
  throws IOException, ValidationException
  {
  Map<SpecimenTableType, SpecimenImportFile> sifMap = populateFileMap(specimensDir, new HashMap<>());
- process(sifMap, merge, ctx.getLogger(), job, syncParticipantVisit, false, ctx.isFailForUndefinedVisits());
+ Study study = StudyService.get().getStudy(getContainer());
+ process(sifMap, merge, ctx.getLogger(), job, syncParticipantVisit, false, ctx.isFailForUndefinedVisits() || study.isFailForUndefinedTimepoints());
  }
 
  protected void process(Map<SpecimenTableType, SpecimenImportFile> sifMap, boolean merge, Logger logger, @Nullable PipelineJob job,
  boolean syncParticipantVisit, boolean editingSpecimens)
  throws IOException, ValidationException
  {
- process(sifMap, merge, logger, job, syncParticipantVisit, editingSpecimens, false);
+ Study study = StudyService.get().getStudy(getContainer());
+ process(sifMap, merge, logger, job, syncParticipantVisit, editingSpecimens, study.isFailForUndefinedTimepoints());
  }
 
  private void process(Map<SpecimenTableType, SpecimenImportFile> sifMap, boolean merge, Logger logger, @Nullable PipelineJob job,
@@ -367,10 +371,6 @@ private void process(Map<SpecimenTableType, SpecimenImportFile> sifMap, boolean
  SpecimenImportFile specimenFile = sifMap.get(_specimensTableType);
  SpecimenLoadInfo loadInfo = populateTempSpecimensTable(specimenFile, merge);
 
- Study study = StudyService.get().getStudy(getContainer());
- if (loadInfo.getRowCount() > 0 && failForUndefinedVisits && study.getTimepointType() == TimepointType.VISIT)
- checkForUndefinedVisits(loadInfo, study);
-
  // NOTE: if no rows were loaded in the temp table, don't remove existing materials/specimens/vials/events.
  if (loadInfo.getRowCount() > 0)
  populateSpecimenTables(loadInfo, merge);
@@ -390,7 +390,7 @@ private void process(Map<SpecimenTableType, SpecimenImportFile> sifMap, boolean
 
  setStatus(GENERAL_JOB_STATUS_MSG + " (update study)");
  _iTimer.setPhase(ImportPhases.ResyncStudy);
- resyncStudy(syncParticipantVisit);
+ resyncStudy(syncParticipantVisit, failForUndefinedVisits);
 
  ensureNotCanceled();
  _iTimer.setPhase(ImportPhases.SetLastSpecimenLoad);
@@ -452,29 +452,6 @@ private SpecimenLoadInfo populateTempSpecimensTable(SpecimenImportFile file, boo
  return new SpecimenLoadInfo(getUser(), getContainer(), DbSchema.getTemp(), columns, rowCount, tempTablesHolder.getTempTableInfo());
  }
 
- private void checkForUndefinedVisits(SpecimenLoadInfo info, Study study) throws ValidationException
- {
- SQLFragment sql = new SQLFragment()
- .append("SELECT DISTINCT VisitValue FROM ")
- .append(info.getTempTableName())
- .append(" tt ")
- .append("\nLEFT JOIN study.Visit v")
- .append("\nON tt.VisitValue >= v.SequenceNumMin AND tt.VisitValue <=v.SequenceNumMax AND v.Container = ?")
- .append("\nWHERE tt.VisitValue IS NOT NULL AND v.RowId IS NULL");
-
- // shared visit container
- Study visitStudy = StudyService.get().getStudyForVisits(study);
- sql.add(visitStudy.getContainer().getId());
-
- SqlSelector selector = new SqlSelector(SpecimenSchema.get().getSchema(), sql);
- List<Double> undefinedVisits = selector.getArrayList(Double.class);
- if (!undefinedVisits.isEmpty())
- {
- Collections.sort(undefinedVisits);
- throw new ValidationException("The following undefined visits exist in the specimen data: " + StringUtils.join(undefinedVisits, ", "));
- }
- }
-
  private void populateSpecimenTables(SpecimenLoadInfo info, boolean merge) throws ValidationException
  {
  setStatus(GENERAL_JOB_STATUS_MSG + " (populate tables)");

study/api-src/org/labkey/api/study/model/VisitService.java

@@ -2,6 +2,7 @@
 
 import org.apache.logging.log4j.Logger;
 import org.jetbrains.annotations.Nullable;
+import org.labkey.api.query.ValidationException;
 import org.labkey.api.security.User;
 import org.labkey.api.services.ServiceRegistry;
 import org.labkey.api.study.Study;
@@ -27,10 +28,10 @@ static void setInstance(VisitService impl)
 
  List<? extends Visit> getVisits(Study study, Visit.Order order);
 
-void updateParticipantVisitsWithCohortUpdate(Study study, User user, @Nullable Logger logger);
+ValidationException updateParticipantVisitsWithCohortUpdate(Study study, User user, boolean failForUndefinedVisits, @Nullable Logger logger);
 
  /**
  * Updates this study's participant, visit, and participant visit tables. Also updates automatic cohort assignments.
  */
-void updateParticipantVisits(Study study, User user);
+ValidationException updateParticipantVisits(Study study, User user);
 }

study/resources/schemas/dbscripts/postgresql/study-23.001-23.002.sql

@@ -0,0 +1 @@
+ALTER TABLE study.study ADD COLUMN FailForUndefinedTimepoints BOOLEAN NOT NULL DEFAULT FALSE;

study/resources/schemas/dbscripts/sqlserver/study-23.001-23.002.sql

@@ -0,0 +1 @@
+ALTER TABLE study.study ADD FailForUndefinedTimepoints BIT NOT NULL DEFAULT 0;

study/resources/schemas/study.xml

@@ -159,6 +159,9 @@
  <column columnName="ValidateQueriesAfterImport">
  <description>Flag to perform query validation after import.</description>
  </column>
+ <column columnName="FailForUndefinedTimepoints">
+ <description>Flag to prevent the creation of new timepoints during data import.</description>
+ </column>
  </columns>
  </table>
  <table tableName="Site" tableDbType="NOT_IN_DB">

study/src/org/labkey/study/StudyModule.java

@@ -209,7 +209,7 @@ public String getName()
  @Override
  public Double getSchemaVersion()
  {
- return 23.001;
+ return 23.002;
  }
 
  @Override

study/src/org/labkey/study/VisitServiceImpl.java

@@ -1,7 +1,9 @@
 package org.labkey.study;
 
 import org.apache.logging.log4j.Logger;
+import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
+import org.labkey.api.query.ValidationException;
 import org.labkey.api.security.User;
 import org.labkey.api.study.Study;
 import org.labkey.api.study.Visit;
@@ -20,14 +22,14 @@ public List<? extends Visit> getVisits(Study study, Visit.Order order)
  }
 
  @Override
- public void updateParticipantVisitsWithCohortUpdate(Study study, User user, @Nullable Logger logger)
+ public @NotNull ValidationException updateParticipantVisitsWithCohortUpdate(Study study, User user, boolean failForUndefinedVisits, @Nullable Logger logger)
  {
- StudyManager.getInstance().getVisitManager(study).updateParticipantVisitsWithCohortUpdate(user, logger);
+return StudyManager.getInstance().getVisitManager(study).updateParticipantVisitsWithCohortUpdate(user, failForUndefinedVisits, logger);
  }
 
  @Override
- public void updateParticipantVisits(Study study, User user)
+ public @NotNull ValidationException updateParticipantVisits(Study study, User user)
  {
- StudyManager.getInstance().getVisitManager(study).updateParticipantVisits(user, Collections.emptySet());
+return StudyManager.getInstance().getVisitManager(study).updateParticipantVisits(user, Collections.emptySet());
  }
 }

study/src/org/labkey/study/assay/StudyPublishManager.java

@@ -456,16 +456,21 @@ else if (!datasetPublishSourceId.equals(publishSource.second) || dataset.getPubl
  logPublishEvent(publishSource.first, source, dataMaps, user, sourceContainer, targetContainer, dataset);
  }
 
+ //Make sure that the study is updated with the correct timepoints.
+ ValidationException validationErrors = StudyManager.getInstance().getVisitManager(targetStudy).updateParticipantVisits(user, Collections.singleton(dataset));
+ if (validationErrors.hasErrors())
+ {
+ errors.add(validationErrors.getMessage());
+ return null;
+ }
+
  transaction.commit();
  }
  catch (ChangePropertyDescriptorException | IOException e)
  {
  throw UnexpectedException.wrap(e);
  }
 
- //Make sure that the study is updated with the correct timepoints.
- StudyManager.getInstance().getVisitManager(targetStudy).updateParticipantVisits(user, Collections.singleton(dataset));
-
  return PageFlowUtil.urlProvider(StudyUrls.class).getDatasetURL(targetContainer, dataset.getRowId());
  }
 
@@ -978,12 +983,17 @@ public Pair<List<String>, UploadLog> importDatasetTSV(User user, StudyImpl study
  defaultQCState = DataStateManager.getInstance().getStateForRowId(study.getContainer(), defaultQCStateId.intValue());
  lsids = StudyManager.getInstance().importDatasetData(user, dsd, dl, columnMap, errors, DatasetDefinition.CheckForDuplicates.sourceOnly,
  defaultQCState, insertOption, null, importLookupByAlternateKey, auditBehaviorType);
+
+ if (!errors.hasErrors())
+ {
+ ValidationException validationException = StudyManager.getInstance().getVisitManager(study).updateParticipantVisits(user, Collections.singleton(dsd));
+ if (validationException.hasErrors())
+ errors.addRowError(validationException);
+ }
+
  if (!errors.hasErrors())
  transaction.commit();
  }
-
- if (!errors.hasErrors())
- StudyManager.getInstance().getVisitManager(study).updateParticipantVisits(user, Collections.singleton(dsd));
  }
  catch (IOException x)
  {
@@ -1272,6 +1282,9 @@ public void autoLinkSamples(ExpSampleType sampleType, List<Map<FieldKey, Object>
  ExpMaterialTable.Column.RowId.toString(),
  publishErrors
  );
+
+ if (!publishErrors.isEmpty())
+ publishErrors.forEach(LOG::error);
  }
  else
  {