1185: Support incremental update of materialized samples view

experiment/resources/schemas/dbscripts/postgresql/exp-26.006-26.007.sql

@@ -0,0 +1,8 @@
+/*
+ * Copyright (c) 2026 LabKey Corporation
+ *
+ * Licensed under the Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0
+ */
+-- For samples, incremental materialized-view updates filter exp.material by (CpasType, Modified) to find rows changed
+-- since modification began. This index allows for the query to avoid a full table scan.
+CREATE INDEX IX_Material_CpasType_Modified ON exp.material (CpasType, Modified);

experiment/resources/schemas/dbscripts/sqlserver/exp-26.006-26.007.sql

@@ -0,0 +1,8 @@
+/*
+ * Copyright (c) 2026 LabKey Corporation
+ *
+ * Licensed under the Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0
+ */
+-- For samples, incremental materialized-view updates filter exp.material by (CpasType, Modified) to find rows changed
+-- since modification began. This index allows for the query to avoid a full table scan.
+CREATE INDEX IX_Material_CpasType_Modified ON exp.Material (CpasType, Modified);

experiment/src/org/labkey/experiment/ExperimentModule.java

@@ -123,6 +123,7 @@
 import org.labkey.experiment.api.ExpDataClassType;
 import org.labkey.experiment.api.ExpDataImpl;
 import org.labkey.experiment.api.ExpDataTableImpl;
+import org.labkey.experiment.api.ExpMaterialTableImpl;
 import org.labkey.experiment.api.ExpMaterialImpl;
 import org.labkey.experiment.api.ExpProtocolImpl;
 import org.labkey.experiment.api.ExpSampleTypeImpl;
@@ -207,7 +208,7 @@ public String getName()
     @Override
     public Double getSchemaVersion()
     {
-        return 26.006;
+        return 26.007;
     }
 
     @Nullable
@@ -1119,6 +1120,7 @@ public Collection<String> getSummary(Container c)
             DomainImpl.TestCase.class,
             DomainPropertyImpl.TestCase.class,
             ExpDataTableImpl.TestCase.class,
+            ExpMaterialTableImpl.IncrementalUpdateTestCase.class,
             ExperimentServiceImpl.AuditDomainUriTest.class,
             ExperimentServiceImpl.LineageQueryTestCase.class,
             ExperimentServiceImpl.ParseInputOutputAliasTestCase.class,

experiment/src/org/labkey/experiment/api/ExpSampleTypeImpl.java

@@ -75,6 +75,7 @@
 import org.labkey.experiment.controllers.exp.ExperimentController;
 
 import java.io.IOException;
+import java.sql.Timestamp;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
@@ -865,9 +866,9 @@ public ExpProtocol[] getProtocols(User user)
         return ret;
     }
 
-    public void onSamplesChanged(User user, List<Material> materials, SampleTypeServiceImpl.SampleChangeType reason)
+    public void onSamplesChanged(User user, List<Material> materials, SampleTypeServiceImpl.SampleChangeType reason, @Nullable Timestamp changedSince)
     {
-        SampleTypeServiceImpl.get().refreshSampleTypeMaterializedView(this, reason);
+        SampleTypeServiceImpl.get().refreshSampleTypeMaterializedView(this, reason, changedSince);
 
         ExpProtocol[] protocols = getProtocols(user);
         if (protocols.length != 0)
@@ -892,7 +893,6 @@ public void onSamplesChanged(User user, List<Material> materials, SampleTypeServ
         }
     }
 
-
     @Override
     public void setContainer(Container container)
     {

experiment/src/org/labkey/experiment/api/SampleTypeServiceImpl.java

@@ -135,6 +135,7 @@
 import java.sql.Connection;
 import java.sql.ResultSet;
 import java.sql.SQLException;
+import java.sql.Timestamp;
 import java.time.LocalDateTime;
 import java.time.ZoneId;
 import java.time.format.DateTimeFormatter;
@@ -1205,7 +1206,7 @@ public ValidationException updateSampleType(GWTDomain<? extends GWTPropertyDescr
                 if (hasNameChange)
                     ExperimentService.get().addObjectLegacyName(st.getObjectId(), ExperimentServiceImpl.getNamespacePrefix(ExpSampleType.class), oldSampleTypeName, user);
 
-                transaction.addCommitTask(() -> SampleTypeServiceImpl.get().indexSampleType(st, SearchService.get().defaultTask().getQueue(container, SearchService.PRIORITY.modified)), POSTCOMMIT);
+                transaction.addCommitTask(() -> indexSampleType(st, SearchService.get().defaultTask().getQueue(container, SearchService.PRIORITY.modified)), POSTCOMMIT);
                 transaction.commit();
                 refreshSampleTypeMaterializedView(st, SampleChangeType.schema);
             }
@@ -1971,6 +1972,7 @@ public Map<String, Integer> moveSamples(Collection<? extends ExpMaterial> sample
         updateCounts.put("sampleAuditEvents", 0);
         Map<Long, List<FileFieldRenameData>> fileMovesBySampleId = new LongHashMap<>();
         ExperimentService expService = ExperimentService.get();
+        Timestamp changedSince = SampleTypeUpdateServiceDI.captureChangedSince();
 
         try (DbScope.Transaction transaction = ensureTransaction())
         {
@@ -1981,7 +1983,7 @@ public Map<String, Integer> moveSamples(Collection<? extends ExpMaterial> sample
                 AbstractQueryUpdateService.addTransactionAuditEvent(transaction, user, auditEvent);
             }
 
-            for (Map.Entry<ExpSampleType, List<ExpMaterial>> entry: sampleTypesMap.entrySet())
+            for (Map.Entry<ExpSampleType, List<ExpMaterial>> entry : sampleTypesMap.entrySet())
             {
                 ExpSampleType sampleType = entry.getKey();
                 SamplesSchema schema = new SamplesSchema(user, sampleType.getContainer());
@@ -2055,10 +2057,10 @@ public Map<String, Integer> moveSamples(Collection<? extends ExpMaterial> sample
                 for (ExpSampleType sampleType : sampleTypesMap.keySet())
                 {
                     // force refresh of materialized view
-                    SampleTypeServiceImpl.get().refreshSampleTypeMaterializedView(sampleType, SampleChangeType.update);
+                    refreshSampleTypeMaterializedView(sampleType, SampleChangeType.update, changedSince);
                     // update search index for moved samples via indexSampleType() helper, it filters for samples to index
                     // based on the modified date
-                    SampleTypeServiceImpl.get().indexSampleType(sampleType, SearchService.get().defaultTask().getQueue(sampleType.getContainer(), SearchService.PRIORITY.modified));
+                    indexSampleType(sampleType, SearchService.get().defaultTask().getQueue(sampleType.getContainer(), SearchService.PRIORITY.modified));
                 }
             }, DbScope.CommitTaskOption.IMMEDIATE, POSTCOMMIT, POSTROLLBACK);
 
@@ -2399,13 +2401,22 @@ public long getCurrentCount(NameGenerator.EntityCounter counterType, Container c
         return getProjectSampleCount(container, counterType == NameGenerator.EntityCounter.rootSampleCount);
     }
 
-    public enum SampleChangeType { insert, update, delete, rollup /* aliquot count */, schema }
+    public enum SampleChangeType { insert, update, merge, delete, rollup /* aliquot count */, schema }
 
     public void refreshSampleTypeMaterializedView(@NotNull ExpSampleType st, SampleChangeType reason)
     {
-        ExpMaterialTableImpl.refreshMaterializedView(st.getLSID(), reason);
+        refreshSampleTypeMaterializedView(st, reason, null);
     }
 
+    /**
+     * @param changedSince a database-clock watermark captured before the update's writes, at or after which the changed
+     *                     samples were modified (only meaningful for update); null means the caller could not capture a
+     *                     watermark, forcing a full re-sync on the next read.
+     */
+    public void refreshSampleTypeMaterializedView(@NotNull ExpSampleType st, SampleChangeType reason, @Nullable Timestamp changedSince)
+    {
+        ExpMaterialTableImpl.refreshMaterializedView(st.getLSID(), reason, changedSince);
+    }
 
     public static class TestCase extends Assert
     {

experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java

@@ -46,6 +46,7 @@
 import org.labkey.api.data.RemapCache;
 import org.labkey.api.data.RuntimeSQLException;
 import org.labkey.api.data.SimpleFilter;
+import org.labkey.api.data.SqlSelector;
 import org.labkey.api.data.TableInfo;
 import org.labkey.api.data.TableSelector;
 import org.labkey.api.data.UpdateableTableInfo;
@@ -116,6 +117,7 @@
 
 import java.io.IOException;
 import java.sql.SQLException;
+import java.sql.Timestamp;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
@@ -149,6 +151,7 @@
 import static org.labkey.api.util.IntegerUtils.asLong;
 import static org.labkey.experiment.ExpDataIterators.incrementCounts;
 import static org.labkey.experiment.api.SampleTypeServiceImpl.SampleChangeType.insert;
+import static org.labkey.experiment.api.SampleTypeServiceImpl.SampleChangeType.merge;
 import static org.labkey.experiment.api.SampleTypeServiceImpl.SampleChangeType.rollup;
 import static org.labkey.experiment.api.SampleTypeServiceImpl.SampleChangeType.update;
 
@@ -466,12 +469,16 @@ public int loadRows(User user, Container container, DataIteratorBuilder rows, Da
 
         context.putConfigParameter(ExperimentService.QueryOptions.GetSampleRecomputeCol, true);
         ArrayList<Map<String, Object>> outputRows = new ArrayList<>();
+        InsertOption insertOption = context.getInsertOption();
+        Timestamp changedSince = insertOption.allowUpdate ? captureChangedSince() : null;
+
         int ret = super.loadRows(user, container, rows, outputRows, context, extraScriptContext);
         if (ret > 0 && !context.getErrors().hasErrors() && _sampleType != null)
         {
-            boolean isMediaUpdate = _sampleType.isMedia() && context.getInsertOption().updateOnly;
-            onSamplesChanged(!isMediaUpdate ? outputRows : null, context.getConfigParameters(), container, context.getInsertOption().allowUpdate ? update : insert);
-            audit(context.getInsertOption().auditAction);
+            boolean isMediaUpdate = _sampleType.isMedia() && insertOption.updateOnly;
+            SampleTypeServiceImpl.SampleChangeType reason = insertOption.updateOnly ? update : insertOption.allowUpdate ? merge : insert;
+            onSamplesChanged(!isMediaUpdate ? outputRows : null, context.getConfigParameters(), container, reason, changedSince);
+            audit(insertOption.auditAction);
         }
         return ret;
     }
@@ -480,10 +487,11 @@ public int loadRows(User user, Container container, DataIteratorBuilder rows, Da
     public int mergeRows(User user, Container container, DataIteratorBuilder rows, BatchValidationException errors, @Nullable Map<Enum, Object> configParameters, Map<String, Object> extraScriptContext)
     {
         assert _sampleType != null : "SampleType required for insert/update, but not required for read/delete";
+        Timestamp changedSince = captureChangedSince();
         int ret = _importRowsUsingDIB(user, container, rows, null, getDataIteratorContext(errors, InsertOption.MERGE, configParameters), extraScriptContext);
         if (ret > 0 && !errors.hasErrors())
         {
-            onSamplesChanged(null, configParameters, container, update); // mergeRows not really used, skip wiring recalc
+            onSamplesChanged(null, configParameters, container, merge, changedSince); // mergeRows not really used, skip wiring recalc
             audit(QueryService.AuditAction.MERGE);
         }
         return ret;
@@ -510,7 +518,7 @@ public List<Map<String, Object>> insertRows(User user, Container container, List
 
         if (results != null && !results.isEmpty() && !errors.hasErrors())
         {
-            onSamplesChanged(results, configParameters, container, SampleTypeServiceImpl.SampleChangeType.insert);
+            onSamplesChanged(results, configParameters, container, insert);
             audit(QueryService.AuditAction.INSERT);
         }
         return results;
@@ -553,6 +561,7 @@ public List<Map<String, Object>> updateRows(
         List<Map<String, Object>> results;
         Map<Enum, Object> finalConfigParameters = configParameters == null ? new HashMap<>() : configParameters;
         recordDataIteratorUsed(finalConfigParameters);
+        Timestamp changedSince = captureChangedSince();
 
         try
         {
@@ -567,7 +576,7 @@ public List<Map<String, Object>> updateRows(
 
         if (results != null && !results.isEmpty() && !errors.hasErrors())
         {
-            onSamplesChanged(!_sampleType.isMedia() ? results : null, configParameters, container, update);
+            onSamplesChanged(!_sampleType.isMedia() ? results : null, configParameters, container, update, changedSince);
             audit(QueryService.AuditAction.UPDATE);
         }
 
@@ -1139,6 +1148,11 @@ protected Map<String, Object> getRow(User user, Container container, Map<String,
     }
 
     private void onSamplesChanged(List<Map<String, Object>> results, Map<Enum, Object> params, Container container, SampleTypeServiceImpl.SampleChangeType reason)
+    {
+        onSamplesChanged(results, params, container, reason, null);
+    }
+
+    private void onSamplesChanged(List<Map<String, Object>> results, Map<Enum, Object> params, Container container, SampleTypeServiceImpl.SampleChangeType reason, @Nullable Timestamp changedSince)
     {
         var tx = getSchema().getDbSchema().getScope().getCurrentTransaction();
         Pair<Set<Long>, Set<String>> parentKeys = getSampleParentsForRecalc(results);
@@ -1163,7 +1177,7 @@ private void onSamplesChanged(List<Map<String, Object>> results, Map<Enum, Objec
                 boolean finalUseBackgroundRecalc = useBackgroundRecalc;
                 boolean finalSkipRecalc = skipRecalc;
                 tx.addCommitTask(() -> {
-                    fireSamplesChanged(reason);
+                    fireSamplesChanged(reason, changedSince);
                     if (finalUseBackgroundRecalc && !finalSkipRecalc)
                         handleRecalc(parentKeys.first, parentKeys.second, true, container);
                 }, DbScope.CommitTaskOption.POSTCOMMIT);
@@ -1173,7 +1187,7 @@ private void onSamplesChanged(List<Map<String, Object>> results, Map<Enum, Objec
         }
         else
         {
-            fireSamplesChanged(reason);
+            fireSamplesChanged(reason, changedSince);
         }
     }
 
@@ -1205,10 +1219,15 @@ private void handleRecalc(Set<Long> rootRowIds, Set<String> parentNames, boolean
         }
     }
 
-    private void fireSamplesChanged(SampleTypeServiceImpl.SampleChangeType reason)
+    private void fireSamplesChanged(SampleTypeServiceImpl.SampleChangeType reason, @Nullable Timestamp changedSince)
     {
         if (_sampleType != null)
-            _sampleType.onSamplesChanged(getUser(), null, reason);
+            _sampleType.onSamplesChanged(getUser(), null, reason, changedSince);
+    }
+
+    static @Nullable Timestamp captureChangedSince()
+    {
+        return new SqlSelector(DbScope.getLabKeyScope(), "SELECT CURRENT_TIMESTAMP").getObject(Timestamp.class);
     }
 
     void audit(QueryService.AuditAction auditAction)