Skip to content

DM-55993: Move option to scale template variance#469

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
isullivan wants to merge 2 commits into
base:main
Choose a base branch
from
Open

DM-55993: Move option to scale template variance #469

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
385e543
Move option to scale template variance to getTemplate.py
isullivan Aug 31, 2026
ffb7738
Add unit tests of template variance scaling
isullivan Aug 31, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 21 additions & 1 deletion python/lsst/ip/diffim/getTemplate.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -34,7 +34,7 @@

from lsst.skymap import BaseSkyMap
from lsst.ip.diffim.dcrModel import DcrModel
from lsst.meas.algorithms import CoaddPsf, CoaddPsfConfig, SubtractBackgroundTask
from lsst.meas.algorithms import CoaddPsf, CoaddPsfConfig, SubtractBackgroundTask, ScaleVarianceTask
from lsst.utils.timer import timeMethod

__all__ = [
Expand DownExpand Up@@ -120,6 +120,15 @@ class GetTemplateConfig(
doc="Minimum fraction of unmasked pixels needed to set the"
" HIGH_VARIANCE mask plane.",
)
doScaleVariance = pexConfig.Field(
dtype=bool,
default=True,
doc="Scale variance of the template image?"
)
scaleVariance = pexConfig.ConfigurableField(
target=ScaleVarianceTask,
doc="Subtask to rescale the variance of the template to the statistically expected level."
)

def setDefaults(self):
# Use a smaller cache: per SeparableKernel.computeCache, this should
Expand DownExpand Up@@ -152,6 +161,8 @@ class GetTemplateTask(pipeBase.PipelineTask):

def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
if self.config.doScaleVariance:
self.makeSubtask("scaleVariance")
self.warper = afwMath.Warper.fromConfig(self.config.warp)
self.schema = afwTable.ExposureTable.makeMinimalSchema()
self.schema.addField(
Expand DownExpand Up@@ -381,6 +392,14 @@ def run(self, *, coaddExposureHandles, bbox, wcs, dataIds, physical_filter, visi
if count == 0:
raise pipeBase.NoWorkFound("No valid pixels in warped template.")

if self.config.doScaleVariance:
# Scale the variance of the template image before subtraction, if
# needed. Note that the science variance is scaled
# independently in ``AlardLuptonSubtractTask``.
varianceFactor = self.scaleVariance.run(template.maskedImage)
self.log.info("Template variance scaling factor: %.2f", varianceFactor)
self.metadata["scaleTemplateVarianceFactor"] = varianceFactor

# Make a single catalog containing all the inputs that were accepted.
catalog = afwTable.ExposureCatalog(self.schema)
catalog.reserve(sum([len(c) for c in catalogs]))
Expand All@@ -394,6 +413,7 @@ def run(self, *, coaddExposureHandles, bbox, wcs, dataIds, physical_filter, visi
template.setFilter(afwImage.FilterLabel(band, physical_filter))
template.setPhotoCalib(photoCalib)
template.setPsf(self._makePsf(template, catalog, wcs))

@khernerkhernerAug 31, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Was this blank line intentional?

@isullivanisullivanAug 31, 2026

Copy link
Copy Markdown
ContributorAuthor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It was an artifact of adding lines there originally, then moving them earlier. But, I liked the visual space so I left it in. I can remove it if you prefer!

# Record the input coadd patches as the template's coadd inputs.
coaddInputs = afwImage.CoaddInputs(afwTable.ExposureTable.makeMinimalSchema(), self.schema)
coaddInputs.ccds.extend(catalog, deep=True)
Expand Down
13 changes: 6 additions & 7 deletions python/lsst/ip/diffim/subtractImages.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -207,7 +207,8 @@ class AlardLuptonSubtractBaseConfig(lsst.pex.config.Config):
doScaleVariance = lsst.pex.config.Field(
dtype=bool,
default=True,
doc="Scale variance of the image difference?"
doc="Scale variance of the science image? Note that the template variance is NOT scaled"
" here. The template variance may be scaled independently in ``GetTemplateTask``."
)
scaleVariance = lsst.pex.config.ConfigurableField(
target=ScaleVarianceTask,
Expand DownExpand Up@@ -1044,13 +1045,11 @@ def _prepareInputs(self, template, science, visitSummary=None):
self.metadata["templateCoveragePercent"] = 100*templateCoverageFraction

if self.config.doScaleVariance:
# Scale the variance of the template and science images before
# convolution, subtraction, or decorrelation so that they have the
# correct ratio.
templateVarFactor = self.scaleVariance.run(template.maskedImage)
# Scale the variance of the science image before
# convolution, subtraction, or decorrelation so that it has the
# correct ratio. Note that the template variance is scaled
# independently in ``GetTemplateTask``.
sciVarFactor = self.scaleVariance.run(science.maskedImage)
self.log.info("Template variance scaling factor: %.2f", templateVarFactor)
self.metadata["scaleTemplateVarianceFactor"] = templateVarFactor
self.log.info("Science variance scaling factor: %.2f", sciVarFactor)
self.metadata["scaleScienceVarianceFactor"] = sciVarFactor

Expand Down
88 changes: 88 additions & 0 deletions tests/test_getTemplate.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -337,6 +337,94 @@ def testNanInputs(self, box=None, nInput=None):
# in the template are closer to the original anymore.
self.assertTrue(np.isfinite(result.template.image.array).all())

def _scaleInputVariance(self, tract, factor):
"""Return fresh handles for one tract's patches, with their variance
planes multiplied by ``factor``.

Parameters
----------
tract : `int`
Id of the tract whose patches should be copied.
factor : `float`
Factor to multiply the input variance planes by.

Returns
-------
handles : `list` [`lsst.pipe.base.InMemoryDatasetHandle`]
Handles to the modified patches.
"""
handles = []
for handle in self.patches[tract]:
# ``copy=True`` on the original handles means this is a copy, so
# the patches shared with the other tests are left untouched.
patch = handle.get()
patch.variance.array *= factor
handles.append(pipeBase.InMemoryDatasetHandle(patch,
storageClass="ExposureF",
copy=True,
dataId=handle.dataId))
return handles

def testScaleVariance(self):
"""Test that the template variance plane is rescaled to match the
empirical pixel noise, and that the factor used is recorded in the
task metadata.
"""
scaleFactor = 1.345
box = lsst.geom.Box2I(lsst.geom.Point2I(0, 0), lsst.geom.Point2I(180, 180))

def _configureAndRunTask(doScaleVariance, varianceScale=1.):
"""Build a template from tract 0, optionally rescaling the input
variance planes by ``varianceScale`` first.
"""
config = lsst.ip.diffim.GetTemplateTask.ConfigClass()
config.doScaleVariance = doScaleVariance
task = lsst.ip.diffim.GetTemplateTask(config=config)
# Task modifies the input bbox, so pass a copy.
result = task.run(coaddExposureHandles={0: self._scaleInputVariance(0, varianceScale)},
bbox=lsst.geom.Box2I(box),
wcs=self.exposure.wcs,
dataIds={0: self.dataIds[0]},
physical_filter="a_test")
return task, result.template

# With scaling disabled the subtask is never constructed, and nothing
# is recorded in the metadata.
taskOff, templateOff = _configureAndRunTask(False)
self.assertFalse(hasattr(taskOff, "scaleVariance"))
self.assertNotIn("scaleTemplateVarianceFactor", taskOff.metadata)

# Both warps -- lanczos5 in ``_makePatches`` and lanczos3 in the
# task -- correlate the noise. The variance plane tracks only the
# per-pixel diagonal, which the second warp leaves too low, so
# ``scaleVariance`` measures a factor well above 1 even though the
# input variance planes are correct.
#
taskOn, templateOn = _configureAndRunTask(True)
factor = taskOn.metadata["scaleTemplateVarianceFactor"]
# TODO DM-55879: this value is pinned on purpose. The lanczos warping
# kernels introduce small correlations that artificially suppress the
# image pixel stddev and inflate the variance scaling factor. This
# should be changed to 1.0 after DM-55879 is merged.
self.assertFloatsAlmostEqual(factor, 1.1465, atol=0.01,
msg="Measured template variance scaling changed; see the"
" comment above if the correlation correction landed.")
# The only difference from the unscaled template is the constant
# factor applied to the variance plane.
self.assertFloatsAlmostEqual(templateOn.variance.array,
templateOff.variance.array*factor, rtol=1e-5)
# Tolerance here is float32 round-off: repeated runs of the task are
# not bitwise identical.
self.assertImagesAlmostEqual(templateOn.image, templateOff.image, rtol=1e-5, atol=1e-5)

# If the input variance planes under-estimate the noise by a known
# factor, the measured factor grows by that amount and the same
# output variance plane is recovered.
taskLow, templateLow = _configureAndRunTask(True, varianceScale=1/scaleFactor)
self.assertFloatsAlmostEqual(taskLow.metadata["scaleTemplateVarianceFactor"],
factor*scaleFactor, rtol=1e-5)
self.assertImagesAlmostEqual(templateLow.variance, templateOn.variance, rtol=1e-5)


def setup_module(module):
lsst.utils.tests.init()
Expand Down
26 changes: 9 additions & 17 deletions tests/test_subtractTask.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -349,7 +349,6 @@ def _run_and_check_images(statsCtrl, statsCtrlDetect, scienceNoiseLevel, templat
templateBorderSize=20, doApplyCalibration=True)
task = self._setup_subtraction(mode="convolveScience")
output = task.run(template, science, sources)
self.assertFloatsAlmostEqual(task.metadata["scaleTemplateVarianceFactor"], 1., atol=.05)
self.assertFloatsAlmostEqual(task.metadata["scaleScienceVarianceFactor"], 1., atol=.05)
# Mean of difference image should be close to zero.
nGoodPix = np.sum(np.isfinite(output.difference.image.array))
Expand DownExpand Up@@ -385,7 +384,6 @@ def _run_and_check_images(statsCtrl, statsCtrlDetect, scienceNoiseLevel, templat
templateBorderSize=20, doApplyCalibration=True)
task = self._setup_subtraction()
output = task.run(template, science, sources)
self.assertFloatsAlmostEqual(task.metadata["scaleTemplateVarianceFactor"], 1., atol=.05)
self.assertFloatsAlmostEqual(task.metadata["scaleScienceVarianceFactor"], 1., atol=.05)
# There should be no NaNs in the image if we convolve the template with a buffer
self.assertTrue(np.all(np.isfinite(output.difference.image.array)))
Expand DownExpand Up@@ -632,8 +630,6 @@ def _run_and_check_images(science, template, sources, statsCtrl,
)
output = task.run(template.clone(), science.clone(), sources)
if doScaleVariance:
self.assertFloatsAlmostEqual(task.metadata["scaleTemplateVarianceFactor"],
scaleFactor, atol=0.05)
self.assertFloatsAlmostEqual(task.metadata["scaleScienceVarianceFactor"],
scaleFactor, atol=0.05)

Expand All@@ -646,7 +642,8 @@ def _run_and_check_images(science, template, sources, statsCtrl,
statsCtrl)

if doScaleVariance:
templateNoise *= scaleFactor
# Only the science variance is scaled here. The template
# variance is scaled independently in ``GetTemplateTask``.
scienceNoise *= scaleFactor
varMean = computeRobustStatistics(output.difference.variance, output.difference.mask, statsCtrl)
self.assertFloatsAlmostEqual(varMean, scienceNoise + templateNoise, rtol=0.1)
Expand All@@ -666,8 +663,7 @@ def _run_and_check_images(science, template, sources, statsCtrl,
doDecorrelation=False, doScaleVariance=False)

# Verify that the variance plane of the difference image is correct
# when the template variance plane is incorrect
template.variance.array /= scaleFactor
# when the input science variance plane is incorrect
science.variance.array /= scaleFactor
_run_and_check_images(science, template, sources, statsCtrl,
doDecorrelation=True, doScaleVariance=True, scaleFactor=scaleFactor)
Expand DownExpand Up@@ -700,8 +696,6 @@ def _run_and_check_images(science, template, sources, statsCtrl,
)
output = task.run(template.clone(), science.clone(), sources)
if doScaleVariance:
self.assertFloatsAlmostEqual(task.metadata["scaleTemplateVarianceFactor"],
scaleFactor, atol=0.05)
self.assertFloatsAlmostEqual(task.metadata["scaleScienceVarianceFactor"],
scaleFactor, atol=0.05)

Expand All@@ -714,7 +708,8 @@ def _run_and_check_images(science, template, sources, statsCtrl,
statsCtrl)

if doScaleVariance:
templateNoise *= scaleFactor
# Only the science variance is scaled here. The template
# variance is scaled independently in ``GetTemplateTask``.
scienceNoise *= scaleFactor

varMean = computeRobustStatistics(output.difference.variance, output.difference.mask, statsCtrl)
Expand All@@ -735,9 +730,8 @@ def _run_and_check_images(science, template, sources, statsCtrl,
doDecorrelation=False, doScaleVariance=False)

# Verify that the variance plane of the difference image is correct
# when the template and science variance planes are incorrect
# when the input science variance plane is incorrect
science.variance.array /= scaleFactor
template.variance.array /= scaleFactor
_run_and_check_images(science, template, sources, statsCtrl,
doDecorrelation=True, doScaleVariance=True, scaleFactor=scaleFactor)
_run_and_check_images(science, template, sources, statsCtrl,
Expand DownExpand Up@@ -1259,8 +1253,6 @@ def _run_and_check_images(science, template, sources, statsCtrl,
)
output = task.run(template.clone(), science.clone(), sources)
if doScaleVariance:
self.assertFloatsAlmostEqual(task.metadata["scaleTemplateVarianceFactor"],
scaleFactor, atol=0.05)
self.assertFloatsAlmostEqual(task.metadata["scaleScienceVarianceFactor"],
scaleFactor, atol=0.05)

Expand All@@ -1278,7 +1270,8 @@ def _run_and_check_images(science, template, sources, statsCtrl,
output.matchedTemplate.mask,
statsCtrl)
if doScaleVariance:
templateNoise *= scaleFactor
# Only the science variance is scaled here. The template
# variance is scaled independently in ``GetTemplateTask``.
scienceNoise *= scaleFactor
varMean = computeRobustStatistics(output.scoreExposure.variance,
output.scoreExposure.mask,
Expand All@@ -1302,8 +1295,7 @@ def _run_and_check_images(science, template, sources, statsCtrl,
doDecorrelation=False, doScaleVariance=False)

# Verify that the variance plane of the Score image is correct
# when the template variance plane is incorrect
template.variance.array /= scaleFactor
# when the input science variance plane is incorrect
science.variance.array /= scaleFactor
_run_and_check_images(science, template, sources, statsCtrl,
doDecorrelation=True, doScaleVariance=True, scaleFactor=scaleFactor)
Expand Down
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { // Add copy buttons to all 
 blocks
(function() {
function addCopyButtons() {
document.querySelectorAll('pre code').forEach(function(codeBlock) {
if (codeBlock.parentElement.hasAttribute('data-copy-added')) return;
codeBlock.parentElement.setAttribute('data-copy-added', 'true');
var btn = document.createElement('button');
btn.textContent = 'Copy';
btn.style.cssText = 'position:absolute;top:4px;right:4px;padding:2px 8px;font-size:11px;background:#4ecdc4;border:none;border-radius:4px;color:#1a1a2e;cursor:pointer;opacity:0.7;transition:opacity 0.2s;';
btn.onmouseover = function() { this.style.opacity = '1'; };
btn.onmouseout = function() { this.style.opacity = '0.7'; };
btn.onclick = function() {
navigator.clipboard.writeText(codeBlock.textContent).then(function() {
btn.textContent = 'Copied!';
setTimeout(function() { btn.textContent = 'Copy'; }, 1500);
});
};
codeBlock.parentElement.style.position = 'relative';
codeBlock.parentElement.appendChild(btn);
});
}
addCopyButtons();
// Re-run on dynamic content
var observer = new MutationObserver(addCopyButtons);
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:Add Copy Buttons to Code Blocks]', __e); }
})();
(function(){
try {
var __m = "github.com";
var __re = new RegExp('^' + "github\\.com" + '
DM-55993: Move option to scale template variance by isullivan · Pull Request #469 · lsst/ip_diffim · GitHub
Skip to content
DM-55993: Move option to scale template variance#469

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Open

isullivan
wants to merge
2
commits into

base:main
Choose a base branch

from

 Open


DM-55993: Move option to scale template variance
#469
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


2 commits


Select commit
Hold shift + click to select a range
385e543

Move option to scale template variance to getTemplate.py


isullivan Aug 31, 2026
ffb7738

Add unit tests of template variance scaling


isullivan Aug 31, 2026

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


22 changes: 21 additions & 1 deletion
python/lsst/ip/diffim/getTemplate.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -34,7 +34,7 @@

from lsst.skymap import BaseSkyMap
from lsst.ip.diffim.dcrModel import DcrModel
from lsst.meas.algorithms import CoaddPsf, CoaddPsfConfig, SubtractBackgroundTask
from lsst.meas.algorithms import CoaddPsf, CoaddPsfConfig, SubtractBackgroundTask, ScaleVarianceTask
from lsst.utils.timer import timeMethod

__all__ = [
Expand DownExpand Up@@ -120,6 +120,15 @@ class GetTemplateConfig(
 doc="Minimum fraction of unmasked pixels needed to set the"
 " HIGH_VARIANCE mask plane.",
 )
 doScaleVariance = pexConfig.Field(
 dtype=bool,
 default=True,
 doc="Scale variance of the template image?"
 )
 scaleVariance = pexConfig.ConfigurableField(
 target=ScaleVarianceTask,
 doc="Subtask to rescale the variance of the template to the statistically expected level."
 )

 def setDefaults(self):
 # Use a smaller cache: per SeparableKernel.computeCache, this should
Expand DownExpand Up@@ -152,6 +161,8 @@ class GetTemplateTask(pipeBase.PipelineTask):

 def __init__(self, *args, **kwargs):
 super().__init__(*args, **kwargs)
 if self.config.doScaleVariance:
 self.makeSubtask("scaleVariance")
 self.warper = afwMath.Warper.fromConfig(self.config.warp)
 self.schema = afwTable.ExposureTable.makeMinimalSchema()
 self.schema.addField(
Expand DownExpand Up@@ -381,6 +392,14 @@ def run(self, *, coaddExposureHandles, bbox, wcs, dataIds, physical_filter, visi
 if count == 0:
 raise pipeBase.NoWorkFound("No valid pixels in warped template.")

 if self.config.doScaleVariance:
 # Scale the variance of the template image before subtraction, if
 # needed. Note that the science variance is scaled
 # independently in ``AlardLuptonSubtractTask``.
 varianceFactor = self.scaleVariance.run(template.maskedImage)
 self.log.info("Template variance scaling factor: %.2f", varianceFactor)
 self.metadata["scaleTemplateVarianceFactor"] = varianceFactor

 # Make a single catalog containing all the inputs that were accepted.
 catalog = afwTable.ExposureCatalog(self.schema)
 catalog.reserve(sum([len(c) for c in catalogs]))
Expand All@@ -394,6 +413,7 @@ def run(self, *, coaddExposureHandles, bbox, wcs, dataIds, physical_filter, visi
 template.setFilter(afwImage.FilterLabel(band, physical_filter))
 template.setPhotoCalib(photoCalib)
 template.setPsf(self._makePsf(template, catalog, wcs))

@khernerkhernerAug 31, 2026

Copy link


Copy Markdown

Contributor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Was this blank line intentional?
@isullivanisullivanAug 31, 2026

Copy link


Copy Markdown

ContributorAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It was an artifact of adding lines there originally, then moving them earlier. But, I liked the visual space so I left it in. I can remove it if you prefer!
 # Record the input coadd patches as the template's coadd inputs.
 coaddInputs = afwImage.CoaddInputs(afwTable.ExposureTable.makeMinimalSchema(), self.schema)
 coaddInputs.ccds.extend(catalog, deep=True)
Expand Down

13 changes: 6 additions & 7 deletions
python/lsst/ip/diffim/subtractImages.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -207,7 +207,8 @@ class AlardLuptonSubtractBaseConfig(lsst.pex.config.Config):
 doScaleVariance = lsst.pex.config.Field(
 dtype=bool,
 default=True,
 doc="Scale variance of the image difference?"
 doc="Scale variance of the science image? Note that the template variance is NOT scaled"
 " here. The template variance may be scaled independently in ``GetTemplateTask``."
 )
 scaleVariance = lsst.pex.config.ConfigurableField(
 target=ScaleVarianceTask,
Expand DownExpand Up@@ -1044,13 +1045,11 @@ def _prepareInputs(self, template, science, visitSummary=None):
 self.metadata["templateCoveragePercent"] = 100*templateCoverageFraction

 if self.config.doScaleVariance:
 # Scale the variance of the template and science images before
 # convolution, subtraction, or decorrelation so that they have the
 # correct ratio.
templateVarFactor = self.scaleVariance.run(template.maskedImage)
 # Scale the variance of the science image before
 # convolution, subtraction, or decorrelation so that it has the
 # correct ratio. Note that the template variance is scaled
# independently in ``GetTemplateTask``.
 sciVarFactor = self.scaleVariance.run(science.maskedImage)
 self.log.info("Template variance scaling factor: %.2f", templateVarFactor)
 self.metadata["scaleTemplateVarianceFactor"] = templateVarFactor
 self.log.info("Science variance scaling factor: %.2f", sciVarFactor)
 self.metadata["scaleScienceVarianceFactor"] = sciVarFactor

Expand Down

88 changes: 88 additions & 0 deletions
tests/test_getTemplate.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -337,6 +337,94 @@ def testNanInputs(self, box=None, nInput=None):
 # in the template are closer to the original anymore.
 self.assertTrue(np.isfinite(result.template.image.array).all())

 def _scaleInputVariance(self, tract, factor):
 """Return fresh handles for one tract's patches, with their variance
 planes multiplied by ``factor``.

 Parameters
 ----------
 tract : `int`
 Id of the tract whose patches should be copied.
 factor : `float`
 Factor to multiply the input variance planes by.

 Returns
 -------
 handles : `list` [`lsst.pipe.base.InMemoryDatasetHandle`]
 Handles to the modified patches.
 """
 handles = []
 for handle in self.patches[tract]:
 # ``copy=True`` on the original handles means this is a copy, so
 # the patches shared with the other tests are left untouched.
 patch = handle.get()
 patch.variance.array *= factor
 handles.append(pipeBase.InMemoryDatasetHandle(patch,
 storageClass="ExposureF",
 copy=True,
 dataId=handle.dataId))
 return handles

 def testScaleVariance(self):
 """Test that the template variance plane is rescaled to match the
 empirical pixel noise, and that the factor used is recorded in the
 task metadata.
 """
 scaleFactor = 1.345
 box = lsst.geom.Box2I(lsst.geom.Point2I(0, 0), lsst.geom.Point2I(180, 180))

 def _configureAndRunTask(doScaleVariance, varianceScale=1.):
 """Build a template from tract 0, optionally rescaling the input
 variance planes by ``varianceScale`` first.
 """
 config = lsst.ip.diffim.GetTemplateTask.ConfigClass()
 config.doScaleVariance = doScaleVariance
 task = lsst.ip.diffim.GetTemplateTask(config=config)
 # Task modifies the input bbox, so pass a copy.
 result = task.run(coaddExposureHandles={0: self._scaleInputVariance(0, varianceScale)},
 bbox=lsst.geom.Box2I(box),
 wcs=self.exposure.wcs,
 dataIds={0: self.dataIds[0]},
 physical_filter="a_test")
 return task, result.template

 # With scaling disabled the subtask is never constructed, and nothing
 # is recorded in the metadata.
 taskOff, templateOff = _configureAndRunTask(False)
 self.assertFalse(hasattr(taskOff, "scaleVariance"))
 self.assertNotIn("scaleTemplateVarianceFactor", taskOff.metadata)

 # Both warps -- lanczos5 in ``_makePatches`` and lanczos3 in the
 # task -- correlate the noise. The variance plane tracks only the
 # per-pixel diagonal, which the second warp leaves too low, so
 # ``scaleVariance`` measures a factor well above 1 even though the
 # input variance planes are correct.
 #
 taskOn, templateOn = _configureAndRunTask(True)
 factor = taskOn.metadata["scaleTemplateVarianceFactor"]
 # TODO DM-55879: this value is pinned on purpose. The lanczos warping
 # kernels introduce small correlations that artificially suppress the
 # image pixel stddev and inflate the variance scaling factor. This
 # should be changed to 1.0 after DM-55879 is merged.
 self.assertFloatsAlmostEqual(factor, 1.1465, atol=0.01,
 msg="Measured template variance scaling changed; see the"
 " comment above if the correlation correction landed.")
 # The only difference from the unscaled template is the constant
 # factor applied to the variance plane.
 self.assertFloatsAlmostEqual(templateOn.variance.array,
 templateOff.variance.array*factor, rtol=1e-5)
 # Tolerance here is float32 round-off: repeated runs of the task are
 # not bitwise identical.
 self.assertImagesAlmostEqual(templateOn.image, templateOff.image, rtol=1e-5, atol=1e-5)

 # If the input variance planes under-estimate the noise by a known
 # factor, the measured factor grows by that amount and the same
 # output variance plane is recovered.
 taskLow, templateLow = _configureAndRunTask(True, varianceScale=1/scaleFactor)
 self.assertFloatsAlmostEqual(taskLow.metadata["scaleTemplateVarianceFactor"],
 factor*scaleFactor, rtol=1e-5)
 self.assertImagesAlmostEqual(templateLow.variance, templateOn.variance, rtol=1e-5)


def setup_module(module):
 lsst.utils.tests.init()
Expand Down

26 changes: 9 additions & 17 deletions
tests/test_subtractTask.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -349,7 +349,6 @@ def _run_and_check_images(statsCtrl, statsCtrlDetect, scienceNoiseLevel, templat
 templateBorderSize=20, doApplyCalibration=True)
 task = self._setup_subtraction(mode="convolveScience")
 output = task.run(template, science, sources)
 self.assertFloatsAlmostEqual(task.metadata["scaleTemplateVarianceFactor"], 1., atol=.05)
 self.assertFloatsAlmostEqual(task.metadata["scaleScienceVarianceFactor"], 1., atol=.05)
 # Mean of difference image should be close to zero.
 nGoodPix = np.sum(np.isfinite(output.difference.image.array))
Expand DownExpand Up@@ -385,7 +384,6 @@ def _run_and_check_images(statsCtrl, statsCtrlDetect, scienceNoiseLevel, templat
 templateBorderSize=20, doApplyCalibration=True)
 task = self._setup_subtraction()
 output = task.run(template, science, sources)
 self.assertFloatsAlmostEqual(task.metadata["scaleTemplateVarianceFactor"], 1., atol=.05)
 self.assertFloatsAlmostEqual(task.metadata["scaleScienceVarianceFactor"], 1., atol=.05)
 # There should be no NaNs in the image if we convolve the template with a buffer
 self.assertTrue(np.all(np.isfinite(output.difference.image.array)))
Expand DownExpand Up@@ -632,8 +630,6 @@ def _run_and_check_images(science, template, sources, statsCtrl,
 )
 output = task.run(template.clone(), science.clone(), sources)
 if doScaleVariance:
 self.assertFloatsAlmostEqual(task.metadata["scaleTemplateVarianceFactor"],
 scaleFactor, atol=0.05)
 self.assertFloatsAlmostEqual(task.metadata["scaleScienceVarianceFactor"],
 scaleFactor, atol=0.05)

Expand All@@ -646,7 +642,8 @@ def _run_and_check_images(science, template, sources, statsCtrl,
 statsCtrl)

 if doScaleVariance:
 templateNoise *= scaleFactor
 # Only the science variance is scaled here. The template
 # variance is scaled independently in ``GetTemplateTask``.
 scienceNoise *= scaleFactor
 varMean = computeRobustStatistics(output.difference.variance, output.difference.mask, statsCtrl)
 self.assertFloatsAlmostEqual(varMean, scienceNoise + templateNoise, rtol=0.1)
Expand All@@ -666,8 +663,7 @@ def _run_and_check_images(science, template, sources, statsCtrl,
 doDecorrelation=False, doScaleVariance=False)

 # Verify that the variance plane of the difference image is correct
 # when the template variance plane is incorrect
 template.variance.array /= scaleFactor
 # when the input science variance plane is incorrect
 science.variance.array /= scaleFactor
 _run_and_check_images(science, template, sources, statsCtrl,
 doDecorrelation=True, doScaleVariance=True, scaleFactor=scaleFactor)
Expand DownExpand Up@@ -700,8 +696,6 @@ def _run_and_check_images(science, template, sources, statsCtrl,
 )
 output = task.run(template.clone(), science.clone(), sources)
 if doScaleVariance:
 self.assertFloatsAlmostEqual(task.metadata["scaleTemplateVarianceFactor"],
 scaleFactor, atol=0.05)
 self.assertFloatsAlmostEqual(task.metadata["scaleScienceVarianceFactor"],
 scaleFactor, atol=0.05)

Expand All@@ -714,7 +708,8 @@ def _run_and_check_images(science, template, sources, statsCtrl,
 statsCtrl)

 if doScaleVariance:
 templateNoise *= scaleFactor
 # Only the science variance is scaled here. The template
 # variance is scaled independently in ``GetTemplateTask``.
 scienceNoise *= scaleFactor

 varMean = computeRobustStatistics(output.difference.variance, output.difference.mask, statsCtrl)
Expand All@@ -735,9 +730,8 @@ def _run_and_check_images(science, template, sources, statsCtrl,
 doDecorrelation=False, doScaleVariance=False)

 # Verify that the variance plane of the difference image is correct
 # when the template and science variance planes are incorrect
 # when the input science variance plane is incorrect
 science.variance.array /= scaleFactor
 template.variance.array /= scaleFactor
 _run_and_check_images(science, template, sources, statsCtrl,
 doDecorrelation=True, doScaleVariance=True, scaleFactor=scaleFactor)
 _run_and_check_images(science, template, sources, statsCtrl,
Expand DownExpand Up@@ -1259,8 +1253,6 @@ def _run_and_check_images(science, template, sources, statsCtrl,
 )
 output = task.run(template.clone(), science.clone(), sources)
 if doScaleVariance:
 self.assertFloatsAlmostEqual(task.metadata["scaleTemplateVarianceFactor"],
 scaleFactor, atol=0.05)
 self.assertFloatsAlmostEqual(task.metadata["scaleScienceVarianceFactor"],
 scaleFactor, atol=0.05)

Expand All@@ -1278,7 +1270,8 @@ def _run_and_check_images(science, template, sources, statsCtrl,
 output.matchedTemplate.mask,
 statsCtrl)
 if doScaleVariance:
 templateNoise *= scaleFactor
 # Only the science variance is scaled here. The template
 # variance is scaled independently in ``GetTemplateTask``.
 scienceNoise *= scaleFactor
 varMean = computeRobustStatistics(output.scoreExposure.variance,
 output.scoreExposure.mask,
Expand All@@ -1302,8 +1295,7 @@ def _run_and_check_images(science, template, sources, statsCtrl,
 doDecorrelation=False, doScaleVariance=False)

 # Verify that the variance plane of the Score image is correct
 # when the template variance plane is incorrect
 template.variance.array /= scaleFactor
 # when the input science variance plane is incorrect
 science.variance.array /= scaleFactor
 _run_and_check_images(science, template, sources, statsCtrl,
 doDecorrelation=True, doScaleVariance=True, scaleFactor=scaleFactor)
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Force GitHub README to respect dark mode
(function() {
var style = document.createElement('style');
style.textContent = '
.markdown-body {
color-scheme: dark light;
}
.markdown-body pre { background: #161b22 !important; }
.markdown-body code { background: rgba(110, 118, 129, 0.4) !important; }
.markdown-body table th, .markdown-body table td { border-color: #30363d !important; }
.markdown-body img { background: #0d1117; }
.markdown-body blockquote { border-left-color: #8b949e; }
.markdown-body hr { border-color: #30363d; }
';
document.head.appendChild(style);
})();
}
} catch(__e) { console.warn('[Userscript:GitHub Dark Mode README Fix]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '
DM-55993: Move option to scale template variance by isullivan · Pull Request #469 · lsst/ip_diffim · GitHub
Skip to content
DM-55993: Move option to scale template variance#469

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Open

isullivan
wants to merge
2
commits into

base:main
Choose a base branch

from

 Open


DM-55993: Move option to scale template variance
#469
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


2 commits


Select commit
Hold shift + click to select a range
385e543

Move option to scale template variance to getTemplate.py


isullivan Aug 31, 2026
ffb7738

Add unit tests of template variance scaling


isullivan Aug 31, 2026

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


22 changes: 21 additions & 1 deletion
python/lsst/ip/diffim/getTemplate.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -34,7 +34,7 @@

from lsst.skymap import BaseSkyMap
from lsst.ip.diffim.dcrModel import DcrModel
from lsst.meas.algorithms import CoaddPsf, CoaddPsfConfig, SubtractBackgroundTask
from lsst.meas.algorithms import CoaddPsf, CoaddPsfConfig, SubtractBackgroundTask, ScaleVarianceTask
from lsst.utils.timer import timeMethod

__all__ = [
Expand DownExpand Up@@ -120,6 +120,15 @@ class GetTemplateConfig(
 doc="Minimum fraction of unmasked pixels needed to set the"
 " HIGH_VARIANCE mask plane.",
 )
 doScaleVariance = pexConfig.Field(
 dtype=bool,
 default=True,
 doc="Scale variance of the template image?"
 )
 scaleVariance = pexConfig.ConfigurableField(
 target=ScaleVarianceTask,
 doc="Subtask to rescale the variance of the template to the statistically expected level."
 )

 def setDefaults(self):
 # Use a smaller cache: per SeparableKernel.computeCache, this should
Expand DownExpand Up@@ -152,6 +161,8 @@ class GetTemplateTask(pipeBase.PipelineTask):

 def __init__(self, *args, **kwargs):
 super().__init__(*args, **kwargs)
 if self.config.doScaleVariance:
 self.makeSubtask("scaleVariance")
 self.warper = afwMath.Warper.fromConfig(self.config.warp)
 self.schema = afwTable.ExposureTable.makeMinimalSchema()
 self.schema.addField(
Expand DownExpand Up@@ -381,6 +392,14 @@ def run(self, *, coaddExposureHandles, bbox, wcs, dataIds, physical_filter, visi
 if count == 0:
 raise pipeBase.NoWorkFound("No valid pixels in warped template.")

 if self.config.doScaleVariance:
 # Scale the variance of the template image before subtraction, if
 # needed. Note that the science variance is scaled
 # independently in ``AlardLuptonSubtractTask``.
 varianceFactor = self.scaleVariance.run(template.maskedImage)
 self.log.info("Template variance scaling factor: %.2f", varianceFactor)
 self.metadata["scaleTemplateVarianceFactor"] = varianceFactor

 # Make a single catalog containing all the inputs that were accepted.
 catalog = afwTable.ExposureCatalog(self.schema)
 catalog.reserve(sum([len(c) for c in catalogs]))
Expand All@@ -394,6 +413,7 @@ def run(self, *, coaddExposureHandles, bbox, wcs, dataIds, physical_filter, visi
 template.setFilter(afwImage.FilterLabel(band, physical_filter))
 template.setPhotoCalib(photoCalib)
 template.setPsf(self._makePsf(template, catalog, wcs))

@khernerkhernerAug 31, 2026

Copy link


Copy Markdown

Contributor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Was this blank line intentional?
@isullivanisullivanAug 31, 2026

Copy link


Copy Markdown

ContributorAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It was an artifact of adding lines there originally, then moving them earlier. But, I liked the visual space so I left it in. I can remove it if you prefer!
 # Record the input coadd patches as the template's coadd inputs.
 coaddInputs = afwImage.CoaddInputs(afwTable.ExposureTable.makeMinimalSchema(), self.schema)
 coaddInputs.ccds.extend(catalog, deep=True)
Expand Down

13 changes: 6 additions & 7 deletions
python/lsst/ip/diffim/subtractImages.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -207,7 +207,8 @@ class AlardLuptonSubtractBaseConfig(lsst.pex.config.Config):
 doScaleVariance = lsst.pex.config.Field(
 dtype=bool,
 default=True,
 doc="Scale variance of the image difference?"
 doc="Scale variance of the science image? Note that the template variance is NOT scaled"
 " here. The template variance may be scaled independently in ``GetTemplateTask``."
 )
 scaleVariance = lsst.pex.config.ConfigurableField(
 target=ScaleVarianceTask,
Expand DownExpand Up@@ -1044,13 +1045,11 @@ def _prepareInputs(self, template, science, visitSummary=None):
 self.metadata["templateCoveragePercent"] = 100*templateCoverageFraction

 if self.config.doScaleVariance:
 # Scale the variance of the template and science images before
 # convolution, subtraction, or decorrelation so that they have the
 # correct ratio.
templateVarFactor = self.scaleVariance.run(template.maskedImage)
 # Scale the variance of the science image before
 # convolution, subtraction, or decorrelation so that it has the
 # correct ratio. Note that the template variance is scaled
# independently in ``GetTemplateTask``.
 sciVarFactor = self.scaleVariance.run(science.maskedImage)
 self.log.info("Template variance scaling factor: %.2f", templateVarFactor)
 self.metadata["scaleTemplateVarianceFactor"] = templateVarFactor
 self.log.info("Science variance scaling factor: %.2f", sciVarFactor)
 self.metadata["scaleScienceVarianceFactor"] = sciVarFactor

Expand Down

88 changes: 88 additions & 0 deletions
tests/test_getTemplate.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -337,6 +337,94 @@ def testNanInputs(self, box=None, nInput=None):
 # in the template are closer to the original anymore.
 self.assertTrue(np.isfinite(result.template.image.array).all())

 def _scaleInputVariance(self, tract, factor):
 """Return fresh handles for one tract's patches, with their variance
 planes multiplied by ``factor``.

 Parameters
 ----------
 tract : `int`
 Id of the tract whose patches should be copied.
 factor : `float`
 Factor to multiply the input variance planes by.

 Returns
 -------
 handles : `list` [`lsst.pipe.base.InMemoryDatasetHandle`]
 Handles to the modified patches.
 """
 handles = []
 for handle in self.patches[tract]:
 # ``copy=True`` on the original handles means this is a copy, so
 # the patches shared with the other tests are left untouched.
 patch = handle.get()
 patch.variance.array *= factor
 handles.append(pipeBase.InMemoryDatasetHandle(patch,
 storageClass="ExposureF",
 copy=True,
 dataId=handle.dataId))
 return handles

 def testScaleVariance(self):
 """Test that the template variance plane is rescaled to match the
 empirical pixel noise, and that the factor used is recorded in the
 task metadata.
 """
 scaleFactor = 1.345
 box = lsst.geom.Box2I(lsst.geom.Point2I(0, 0), lsst.geom.Point2I(180, 180))

 def _configureAndRunTask(doScaleVariance, varianceScale=1.):
 """Build a template from tract 0, optionally rescaling the input
 variance planes by ``varianceScale`` first.
 """
 config = lsst.ip.diffim.GetTemplateTask.ConfigClass()
 config.doScaleVariance = doScaleVariance
 task = lsst.ip.diffim.GetTemplateTask(config=config)
 # Task modifies the input bbox, so pass a copy.
 result = task.run(coaddExposureHandles={0: self._scaleInputVariance(0, varianceScale)},
 bbox=lsst.geom.Box2I(box),
 wcs=self.exposure.wcs,
 dataIds={0: self.dataIds[0]},
 physical_filter="a_test")
 return task, result.template

 # With scaling disabled the subtask is never constructed, and nothing
 # is recorded in the metadata.
 taskOff, templateOff = _configureAndRunTask(False)
 self.assertFalse(hasattr(taskOff, "scaleVariance"))
 self.assertNotIn("scaleTemplateVarianceFactor", taskOff.metadata)

 # Both warps -- lanczos5 in ``_makePatches`` and lanczos3 in the
 # task -- correlate the noise. The variance plane tracks only the
 # per-pixel diagonal, which the second warp leaves too low, so
 # ``scaleVariance`` measures a factor well above 1 even though the
 # input variance planes are correct.
 #
 taskOn, templateOn = _configureAndRunTask(True)
 factor = taskOn.metadata["scaleTemplateVarianceFactor"]
 # TODO DM-55879: this value is pinned on purpose. The lanczos warping
 # kernels introduce small correlations that artificially suppress the
 # image pixel stddev and inflate the variance scaling factor. This
 # should be changed to 1.0 after DM-55879 is merged.
 self.assertFloatsAlmostEqual(factor, 1.1465, atol=0.01,
 msg="Measured template variance scaling changed; see the"
 " comment above if the correlation correction landed.")
 # The only difference from the unscaled template is the constant
 # factor applied to the variance plane.
 self.assertFloatsAlmostEqual(templateOn.variance.array,
 templateOff.variance.array*factor, rtol=1e-5)
 # Tolerance here is float32 round-off: repeated runs of the task are
 # not bitwise identical.
 self.assertImagesAlmostEqual(templateOn.image, templateOff.image, rtol=1e-5, atol=1e-5)

 # If the input variance planes under-estimate the noise by a known
 # factor, the measured factor grows by that amount and the same
 # output variance plane is recovered.
 taskLow, templateLow = _configureAndRunTask(True, varianceScale=1/scaleFactor)
 self.assertFloatsAlmostEqual(taskLow.metadata["scaleTemplateVarianceFactor"],
 factor*scaleFactor, rtol=1e-5)
 self.assertImagesAlmostEqual(templateLow.variance, templateOn.variance, rtol=1e-5)


def setup_module(module):
 lsst.utils.tests.init()
Expand Down

26 changes: 9 additions & 17 deletions
tests/test_subtractTask.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -349,7 +349,6 @@ def _run_and_check_images(statsCtrl, statsCtrlDetect, scienceNoiseLevel, templat
 templateBorderSize=20, doApplyCalibration=True)
 task = self._setup_subtraction(mode="convolveScience")
 output = task.run(template, science, sources)
 self.assertFloatsAlmostEqual(task.metadata["scaleTemplateVarianceFactor"], 1., atol=.05)
 self.assertFloatsAlmostEqual(task.metadata["scaleScienceVarianceFactor"], 1., atol=.05)
 # Mean of difference image should be close to zero.
 nGoodPix = np.sum(np.isfinite(output.difference.image.array))
Expand DownExpand Up@@ -385,7 +384,6 @@ def _run_and_check_images(statsCtrl, statsCtrlDetect, scienceNoiseLevel, templat
 templateBorderSize=20, doApplyCalibration=True)
 task = self._setup_subtraction()
 output = task.run(template, science, sources)
 self.assertFloatsAlmostEqual(task.metadata["scaleTemplateVarianceFactor"], 1., atol=.05)
 self.assertFloatsAlmostEqual(task.metadata["scaleScienceVarianceFactor"], 1., atol=.05)
 # There should be no NaNs in the image if we convolve the template with a buffer
 self.assertTrue(np.all(np.isfinite(output.difference.image.array)))
Expand DownExpand Up@@ -632,8 +630,6 @@ def _run_and_check_images(science, template, sources, statsCtrl,
 )
 output = task.run(template.clone(), science.clone(), sources)
 if doScaleVariance:
 self.assertFloatsAlmostEqual(task.metadata["scaleTemplateVarianceFactor"],
 scaleFactor, atol=0.05)
 self.assertFloatsAlmostEqual(task.metadata["scaleScienceVarianceFactor"],
 scaleFactor, atol=0.05)

Expand All@@ -646,7 +642,8 @@ def _run_and_check_images(science, template, sources, statsCtrl,
 statsCtrl)

 if doScaleVariance:
 templateNoise *= scaleFactor
 # Only the science variance is scaled here. The template
 # variance is scaled independently in ``GetTemplateTask``.
 scienceNoise *= scaleFactor
 varMean = computeRobustStatistics(output.difference.variance, output.difference.mask, statsCtrl)
 self.assertFloatsAlmostEqual(varMean, scienceNoise + templateNoise, rtol=0.1)
Expand All@@ -666,8 +663,7 @@ def _run_and_check_images(science, template, sources, statsCtrl,
 doDecorrelation=False, doScaleVariance=False)

 # Verify that the variance plane of the difference image is correct
 # when the template variance plane is incorrect
 template.variance.array /= scaleFactor
 # when the input science variance plane is incorrect
 science.variance.array /= scaleFactor
 _run_and_check_images(science, template, sources, statsCtrl,
 doDecorrelation=True, doScaleVariance=True, scaleFactor=scaleFactor)
Expand DownExpand Up@@ -700,8 +696,6 @@ def _run_and_check_images(science, template, sources, statsCtrl,
 )
 output = task.run(template.clone(), science.clone(), sources)
 if doScaleVariance:
 self.assertFloatsAlmostEqual(task.metadata["scaleTemplateVarianceFactor"],
 scaleFactor, atol=0.05)
 self.assertFloatsAlmostEqual(task.metadata["scaleScienceVarianceFactor"],
 scaleFactor, atol=0.05)

Expand All@@ -714,7 +708,8 @@ def _run_and_check_images(science, template, sources, statsCtrl,
 statsCtrl)

 if doScaleVariance:
 templateNoise *= scaleFactor
 # Only the science variance is scaled here. The template
 # variance is scaled independently in ``GetTemplateTask``.
 scienceNoise *= scaleFactor

 varMean = computeRobustStatistics(output.difference.variance, output.difference.mask, statsCtrl)
Expand All@@ -735,9 +730,8 @@ def _run_and_check_images(science, template, sources, statsCtrl,
 doDecorrelation=False, doScaleVariance=False)

 # Verify that the variance plane of the difference image is correct
 # when the template and science variance planes are incorrect
 # when the input science variance plane is incorrect
 science.variance.array /= scaleFactor
 template.variance.array /= scaleFactor
 _run_and_check_images(science, template, sources, statsCtrl,
 doDecorrelation=True, doScaleVariance=True, scaleFactor=scaleFactor)
 _run_and_check_images(science, template, sources, statsCtrl,
Expand DownExpand Up@@ -1259,8 +1253,6 @@ def _run_and_check_images(science, template, sources, statsCtrl,
 )
 output = task.run(template.clone(), science.clone(), sources)
 if doScaleVariance:
 self.assertFloatsAlmostEqual(task.metadata["scaleTemplateVarianceFactor"],
 scaleFactor, atol=0.05)
 self.assertFloatsAlmostEqual(task.metadata["scaleScienceVarianceFactor"],
 scaleFactor, atol=0.05)

Expand All@@ -1278,7 +1270,8 @@ def _run_and_check_images(science, template, sources, statsCtrl,
 output.matchedTemplate.mask,
 statsCtrl)
 if doScaleVariance:
 templateNoise *= scaleFactor
 # Only the science variance is scaled here. The template
 # variance is scaled independently in ``GetTemplateTask``.
 scienceNoise *= scaleFactor
 varMean = computeRobustStatistics(output.scoreExposure.variance,
 output.scoreExposure.mask,
Expand All@@ -1302,8 +1295,7 @@ def _run_and_check_images(science, template, sources, statsCtrl,
 doDecorrelation=False, doScaleVariance=False)

 # Verify that the variance plane of the Score image is correct
 # when the template variance plane is incorrect
 template.variance.array /= scaleFactor
 # when the input science variance plane is incorrect
 science.variance.array /= scaleFactor
 _run_and_check_images(science, template, sources, statsCtrl,
 doDecorrelation=True, doScaleVariance=True, scaleFactor=scaleFactor)
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Highlight search terms from Google/DuckDuckGo/Bing referrer
(function() {
var ref = document.referrer;
var terms = [];
if (ref.includes('google.com') || ref.includes('duckduckgo.com') || ref.includes('bing.com')) {
var url = new URL(ref);
var q = url.searchParams.get('q') || url.searchParams.get('p');
if (q) {
terms = q.split(/\s+/).filter(function(t) { return t.length > 2; });
}
}
if (terms.length === 0) return;
var style = document.createElement('style');
style.textContent = '.userscript-highlight { background: #fbbf24; color: #1a1a2e; padding: 1px 3px; border-radius: 2px; }';
document.head.appendChild(style);
function highlight(node) {
if (node.nodeType === 3) { // text node
var text = node.textContent;
var found = false;
terms.forEach(function(term) {
var regex = new RegExp('(' + term.replace(/[.*+?^${}()|[\]\\]/g, '\\') + ')', 'gi');
if (regex.test(text)) {
found = true;
var frag = document.createDocumentFragment();
var parts = text.split(regex);
parts.forEach(function(part, i) {
if (i % 2 === 0) {
frag.appendChild(document.createTextNode(part));
} else {
var span = document.createElement('span');
span.className = 'userscript-highlight';
span.textContent = part;
frag.appendChild(span);
}
});
node.parentNode.replaceChild(frag, node);
}
});
} else if (node.nodeType === 1 && node.childNodes) { // element
var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT'];
if (!skipTags.includes(node.tagName)) {
Array.from(node.childNodes).forEach(highlight);
}
}
}
highlight(document.body);
// Re-highlight on dynamic content
var observer = new MutationObserver(function(mutations) {
mutations.forEach(function(m) {
m.addedNodes.forEach(function(node) {
if (node.nodeType === 1 || node.nodeType === 3) highlight(node);
});
});
});
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '
DM-55993: Move option to scale template variance by isullivan · Pull Request #469 · lsst/ip_diffim · GitHub
Skip to content
DM-55993: Move option to scale template variance#469

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Open

isullivan
wants to merge
2
commits into

base:main
Choose a base branch

from

 Open


DM-55993: Move option to scale template variance
#469
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


2 commits


Select commit
Hold shift + click to select a range
385e543

Move option to scale template variance to getTemplate.py


isullivan Aug 31, 2026
ffb7738

Add unit tests of template variance scaling


isullivan Aug 31, 2026

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


22 changes: 21 additions & 1 deletion
python/lsst/ip/diffim/getTemplate.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -34,7 +34,7 @@

from lsst.skymap import BaseSkyMap
from lsst.ip.diffim.dcrModel import DcrModel
from lsst.meas.algorithms import CoaddPsf, CoaddPsfConfig, SubtractBackgroundTask
from lsst.meas.algorithms import CoaddPsf, CoaddPsfConfig, SubtractBackgroundTask, ScaleVarianceTask
from lsst.utils.timer import timeMethod

__all__ = [
Expand DownExpand Up@@ -120,6 +120,15 @@ class GetTemplateConfig(
 doc="Minimum fraction of unmasked pixels needed to set the"
 " HIGH_VARIANCE mask plane.",
 )
 doScaleVariance = pexConfig.Field(
 dtype=bool,
 default=True,
 doc="Scale variance of the template image?"
 )
 scaleVariance = pexConfig.ConfigurableField(
 target=ScaleVarianceTask,
 doc="Subtask to rescale the variance of the template to the statistically expected level."
 )

 def setDefaults(self):
 # Use a smaller cache: per SeparableKernel.computeCache, this should
Expand DownExpand Up@@ -152,6 +161,8 @@ class GetTemplateTask(pipeBase.PipelineTask):

 def __init__(self, *args, **kwargs):
 super().__init__(*args, **kwargs)
 if self.config.doScaleVariance:
 self.makeSubtask("scaleVariance")
 self.warper = afwMath.Warper.fromConfig(self.config.warp)
 self.schema = afwTable.ExposureTable.makeMinimalSchema()
 self.schema.addField(
Expand DownExpand Up@@ -381,6 +392,14 @@ def run(self, *, coaddExposureHandles, bbox, wcs, dataIds, physical_filter, visi
 if count == 0:
 raise pipeBase.NoWorkFound("No valid pixels in warped template.")

 if self.config.doScaleVariance:
 # Scale the variance of the template image before subtraction, if
 # needed. Note that the science variance is scaled
 # independently in ``AlardLuptonSubtractTask``.
 varianceFactor = self.scaleVariance.run(template.maskedImage)
 self.log.info("Template variance scaling factor: %.2f", varianceFactor)
 self.metadata["scaleTemplateVarianceFactor"] = varianceFactor

 # Make a single catalog containing all the inputs that were accepted.
 catalog = afwTable.ExposureCatalog(self.schema)
 catalog.reserve(sum([len(c) for c in catalogs]))
Expand All@@ -394,6 +413,7 @@ def run(self, *, coaddExposureHandles, bbox, wcs, dataIds, physical_filter, visi
 template.setFilter(afwImage.FilterLabel(band, physical_filter))
 template.setPhotoCalib(photoCalib)
 template.setPsf(self._makePsf(template, catalog, wcs))

@khernerkhernerAug 31, 2026

Copy link


Copy Markdown

Contributor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Was this blank line intentional?
@isullivanisullivanAug 31, 2026

Copy link


Copy Markdown

ContributorAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It was an artifact of adding lines there originally, then moving them earlier. But, I liked the visual space so I left it in. I can remove it if you prefer!
 # Record the input coadd patches as the template's coadd inputs.
 coaddInputs = afwImage.CoaddInputs(afwTable.ExposureTable.makeMinimalSchema(), self.schema)
 coaddInputs.ccds.extend(catalog, deep=True)
Expand Down

13 changes: 6 additions & 7 deletions
python/lsst/ip/diffim/subtractImages.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -207,7 +207,8 @@ class AlardLuptonSubtractBaseConfig(lsst.pex.config.Config):
 doScaleVariance = lsst.pex.config.Field(
 dtype=bool,
 default=True,
 doc="Scale variance of the image difference?"
 doc="Scale variance of the science image? Note that the template variance is NOT scaled"
 " here. The template variance may be scaled independently in ``GetTemplateTask``."
 )
 scaleVariance = lsst.pex.config.ConfigurableField(
 target=ScaleVarianceTask,
Expand DownExpand Up@@ -1044,13 +1045,11 @@ def _prepareInputs(self, template, science, visitSummary=None):
 self.metadata["templateCoveragePercent"] = 100*templateCoverageFraction

 if self.config.doScaleVariance:
 # Scale the variance of the template and science images before
 # convolution, subtraction, or decorrelation so that they have the
 # correct ratio.
templateVarFactor = self.scaleVariance.run(template.maskedImage)
 # Scale the variance of the science image before
 # convolution, subtraction, or decorrelation so that it has the
 # correct ratio. Note that the template variance is scaled
# independently in ``GetTemplateTask``.
 sciVarFactor = self.scaleVariance.run(science.maskedImage)
 self.log.info("Template variance scaling factor: %.2f", templateVarFactor)
 self.metadata["scaleTemplateVarianceFactor"] = templateVarFactor
 self.log.info("Science variance scaling factor: %.2f", sciVarFactor)
 self.metadata["scaleScienceVarianceFactor"] = sciVarFactor

Expand Down

88 changes: 88 additions & 0 deletions
tests/test_getTemplate.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -337,6 +337,94 @@ def testNanInputs(self, box=None, nInput=None):
 # in the template are closer to the original anymore.
 self.assertTrue(np.isfinite(result.template.image.array).all())

 def _scaleInputVariance(self, tract, factor):
 """Return fresh handles for one tract's patches, with their variance
 planes multiplied by ``factor``.

 Parameters
 ----------
 tract : `int`
 Id of the tract whose patches should be copied.
 factor : `float`
 Factor to multiply the input variance planes by.

 Returns
 -------
 handles : `list` [`lsst.pipe.base.InMemoryDatasetHandle`]
 Handles to the modified patches.
 """
 handles = []
 for handle in self.patches[tract]:
 # ``copy=True`` on the original handles means this is a copy, so
 # the patches shared with the other tests are left untouched.
 patch = handle.get()
 patch.variance.array *= factor
 handles.append(pipeBase.InMemoryDatasetHandle(patch,
 storageClass="ExposureF",
 copy=True,
 dataId=handle.dataId))
 return handles

 def testScaleVariance(self):
 """Test that the template variance plane is rescaled to match the
 empirical pixel noise, and that the factor used is recorded in the
 task metadata.
 """
 scaleFactor = 1.345
 box = lsst.geom.Box2I(lsst.geom.Point2I(0, 0), lsst.geom.Point2I(180, 180))

 def _configureAndRunTask(doScaleVariance, varianceScale=1.):
 """Build a template from tract 0, optionally rescaling the input
 variance planes by ``varianceScale`` first.
 """
 config = lsst.ip.diffim.GetTemplateTask.ConfigClass()
 config.doScaleVariance = doScaleVariance
 task = lsst.ip.diffim.GetTemplateTask(config=config)
 # Task modifies the input bbox, so pass a copy.
 result = task.run(coaddExposureHandles={0: self._scaleInputVariance(0, varianceScale)},
 bbox=lsst.geom.Box2I(box),
 wcs=self.exposure.wcs,
 dataIds={0: self.dataIds[0]},
 physical_filter="a_test")
 return task, result.template

 # With scaling disabled the subtask is never constructed, and nothing
 # is recorded in the metadata.
 taskOff, templateOff = _configureAndRunTask(False)
 self.assertFalse(hasattr(taskOff, "scaleVariance"))
 self.assertNotIn("scaleTemplateVarianceFactor", taskOff.metadata)

 # Both warps -- lanczos5 in ``_makePatches`` and lanczos3 in the
 # task -- correlate the noise. The variance plane tracks only the
 # per-pixel diagonal, which the second warp leaves too low, so
 # ``scaleVariance`` measures a factor well above 1 even though the
 # input variance planes are correct.
 #
 taskOn, templateOn = _configureAndRunTask(True)
 factor = taskOn.metadata["scaleTemplateVarianceFactor"]
 # TODO DM-55879: this value is pinned on purpose. The lanczos warping
 # kernels introduce small correlations that artificially suppress the
 # image pixel stddev and inflate the variance scaling factor. This
 # should be changed to 1.0 after DM-55879 is merged.
 self.assertFloatsAlmostEqual(factor, 1.1465, atol=0.01,
 msg="Measured template variance scaling changed; see the"
 " comment above if the correlation correction landed.")
 # The only difference from the unscaled template is the constant
 # factor applied to the variance plane.
 self.assertFloatsAlmostEqual(templateOn.variance.array,
 templateOff.variance.array*factor, rtol=1e-5)
 # Tolerance here is float32 round-off: repeated runs of the task are
 # not bitwise identical.
 self.assertImagesAlmostEqual(templateOn.image, templateOff.image, rtol=1e-5, atol=1e-5)

 # If the input variance planes under-estimate the noise by a known
 # factor, the measured factor grows by that amount and the same
 # output variance plane is recovered.
 taskLow, templateLow = _configureAndRunTask(True, varianceScale=1/scaleFactor)
 self.assertFloatsAlmostEqual(taskLow.metadata["scaleTemplateVarianceFactor"],
 factor*scaleFactor, rtol=1e-5)
 self.assertImagesAlmostEqual(templateLow.variance, templateOn.variance, rtol=1e-5)


def setup_module(module):
 lsst.utils.tests.init()
Expand Down

26 changes: 9 additions & 17 deletions
tests/test_subtractTask.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -349,7 +349,6 @@ def _run_and_check_images(statsCtrl, statsCtrlDetect, scienceNoiseLevel, templat
 templateBorderSize=20, doApplyCalibration=True)
 task = self._setup_subtraction(mode="convolveScience")
 output = task.run(template, science, sources)
 self.assertFloatsAlmostEqual(task.metadata["scaleTemplateVarianceFactor"], 1., atol=.05)
 self.assertFloatsAlmostEqual(task.metadata["scaleScienceVarianceFactor"], 1., atol=.05)
 # Mean of difference image should be close to zero.
 nGoodPix = np.sum(np.isfinite(output.difference.image.array))
Expand DownExpand Up@@ -385,7 +384,6 @@ def _run_and_check_images(statsCtrl, statsCtrlDetect, scienceNoiseLevel, templat
 templateBorderSize=20, doApplyCalibration=True)
 task = self._setup_subtraction()
 output = task.run(template, science, sources)
 self.assertFloatsAlmostEqual(task.metadata["scaleTemplateVarianceFactor"], 1., atol=.05)
 self.assertFloatsAlmostEqual(task.metadata["scaleScienceVarianceFactor"], 1., atol=.05)
 # There should be no NaNs in the image if we convolve the template with a buffer
 self.assertTrue(np.all(np.isfinite(output.difference.image.array)))
Expand DownExpand Up@@ -632,8 +630,6 @@ def _run_and_check_images(science, template, sources, statsCtrl,
 )
 output = task.run(template.clone(), science.clone(), sources)
 if doScaleVariance:
 self.assertFloatsAlmostEqual(task.metadata["scaleTemplateVarianceFactor"],
 scaleFactor, atol=0.05)
 self.assertFloatsAlmostEqual(task.metadata["scaleScienceVarianceFactor"],
 scaleFactor, atol=0.05)

Expand All@@ -646,7 +642,8 @@ def _run_and_check_images(science, template, sources, statsCtrl,
 statsCtrl)

 if doScaleVariance:
 templateNoise *= scaleFactor
 # Only the science variance is scaled here. The template
 # variance is scaled independently in ``GetTemplateTask``.
 scienceNoise *= scaleFactor
 varMean = computeRobustStatistics(output.difference.variance, output.difference.mask, statsCtrl)
 self.assertFloatsAlmostEqual(varMean, scienceNoise + templateNoise, rtol=0.1)
Expand All@@ -666,8 +663,7 @@ def _run_and_check_images(science, template, sources, statsCtrl,
 doDecorrelation=False, doScaleVariance=False)

 # Verify that the variance plane of the difference image is correct
 # when the template variance plane is incorrect
 template.variance.array /= scaleFactor
 # when the input science variance plane is incorrect
 science.variance.array /= scaleFactor
 _run_and_check_images(science, template, sources, statsCtrl,
 doDecorrelation=True, doScaleVariance=True, scaleFactor=scaleFactor)
Expand DownExpand Up@@ -700,8 +696,6 @@ def _run_and_check_images(science, template, sources, statsCtrl,
 )
 output = task.run(template.clone(), science.clone(), sources)
 if doScaleVariance:
 self.assertFloatsAlmostEqual(task.metadata["scaleTemplateVarianceFactor"],
 scaleFactor, atol=0.05)
 self.assertFloatsAlmostEqual(task.metadata["scaleScienceVarianceFactor"],
 scaleFactor, atol=0.05)

Expand All@@ -714,7 +708,8 @@ def _run_and_check_images(science, template, sources, statsCtrl,
 statsCtrl)

 if doScaleVariance:
 templateNoise *= scaleFactor
 # Only the science variance is scaled here. The template
 # variance is scaled independently in ``GetTemplateTask``.
 scienceNoise *= scaleFactor

 varMean = computeRobustStatistics(output.difference.variance, output.difference.mask, statsCtrl)
Expand All@@ -735,9 +730,8 @@ def _run_and_check_images(science, template, sources, statsCtrl,
 doDecorrelation=False, doScaleVariance=False)

 # Verify that the variance plane of the difference image is correct
 # when the template and science variance planes are incorrect
 # when the input science variance plane is incorrect
 science.variance.array /= scaleFactor
 template.variance.array /= scaleFactor
 _run_and_check_images(science, template, sources, statsCtrl,
 doDecorrelation=True, doScaleVariance=True, scaleFactor=scaleFactor)
 _run_and_check_images(science, template, sources, statsCtrl,
Expand DownExpand Up@@ -1259,8 +1253,6 @@ def _run_and_check_images(science, template, sources, statsCtrl,
 )
 output = task.run(template.clone(), science.clone(), sources)
 if doScaleVariance:
 self.assertFloatsAlmostEqual(task.metadata["scaleTemplateVarianceFactor"],
 scaleFactor, atol=0.05)
 self.assertFloatsAlmostEqual(task.metadata["scaleScienceVarianceFactor"],
 scaleFactor, atol=0.05)

Expand All@@ -1278,7 +1270,8 @@ def _run_and_check_images(science, template, sources, statsCtrl,
 output.matchedTemplate.mask,
 statsCtrl)
 if doScaleVariance:
 templateNoise *= scaleFactor
 # Only the science variance is scaled here. The template
 # variance is scaled independently in ``GetTemplateTask``.
 scienceNoise *= scaleFactor
 varMean = computeRobustStatistics(output.scoreExposure.variance,
 output.scoreExposure.mask,
Expand All@@ -1302,8 +1295,7 @@ def _run_and_check_images(science, template, sources, statsCtrl,
 doDecorrelation=False, doScaleVariance=False)

 # Verify that the variance plane of the Score image is correct
 # when the template variance plane is incorrect
 template.variance.array /= scaleFactor
 # when the input science variance plane is incorrect
 science.variance.array /= scaleFactor
 _run_and_check_images(science, template, sources, statsCtrl,
 doDecorrelation=True, doScaleVariance=True, scaleFactor=scaleFactor)
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Strip utm_, fbclid, gclid, etc. from all links on page
(function() {
var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',
'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid',
'ref', 'ref_src', 'source', 'medium', 'campaign'];
function cleanUrl(url) {
try {
var u = new URL(url, window.location.origin);
var changed = false;
trackingParams.forEach(function(p) {
if (u.searchParams.has(p)) {
u.searchParams.delete(p);
changed = true;
}
});
return changed ? u.toString() : url;
} catch (e) {
return url;
}
}
function cleanLinks() {
document.querySelectorAll('a[href]').forEach(function(a) {
var clean = cleanUrl(a.href);
if (clean !== a.href) a.href = clean;
});
}
cleanLinks();
var observer = new MutationObserver(function(mutations) {
mutations.forEach(function(m) {
m.addedNodes.forEach(function(node) {
if (node.nodeType === 1) {
if (node.tagName === 'A') cleanLinks();
node.querySelectorAll('a[href]').forEach(function(a) {
var clean = cleanUrl(a.href);
if (clean !== a.href) a.href = clean;
});
}
});
});
});
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); }
})();
(function(){
try {
var __m = "youtube.com";
var __re = new RegExp('^' + "youtube\\.com" + '
DM-55993: Move option to scale template variance by isullivan · Pull Request #469 · lsst/ip_diffim · GitHub
Skip to content
DM-55993: Move option to scale template variance#469

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Open

isullivan
wants to merge
2
commits into

base:main
Choose a base branch

from

 Open


DM-55993: Move option to scale template variance
#469
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


2 commits


Select commit
Hold shift + click to select a range
385e543

Move option to scale template variance to getTemplate.py


isullivan Aug 31, 2026
ffb7738

Add unit tests of template variance scaling


isullivan Aug 31, 2026

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


22 changes: 21 additions & 1 deletion
python/lsst/ip/diffim/getTemplate.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -34,7 +34,7 @@

from lsst.skymap import BaseSkyMap
from lsst.ip.diffim.dcrModel import DcrModel
from lsst.meas.algorithms import CoaddPsf, CoaddPsfConfig, SubtractBackgroundTask
from lsst.meas.algorithms import CoaddPsf, CoaddPsfConfig, SubtractBackgroundTask, ScaleVarianceTask
from lsst.utils.timer import timeMethod

__all__ = [
Expand DownExpand Up@@ -120,6 +120,15 @@ class GetTemplateConfig(
 doc="Minimum fraction of unmasked pixels needed to set the"
 " HIGH_VARIANCE mask plane.",
 )
 doScaleVariance = pexConfig.Field(
 dtype=bool,
 default=True,
 doc="Scale variance of the template image?"
 )
 scaleVariance = pexConfig.ConfigurableField(
 target=ScaleVarianceTask,
 doc="Subtask to rescale the variance of the template to the statistically expected level."
 )

 def setDefaults(self):
 # Use a smaller cache: per SeparableKernel.computeCache, this should
Expand DownExpand Up@@ -152,6 +161,8 @@ class GetTemplateTask(pipeBase.PipelineTask):

 def __init__(self, *args, **kwargs):
 super().__init__(*args, **kwargs)
 if self.config.doScaleVariance:
 self.makeSubtask("scaleVariance")
 self.warper = afwMath.Warper.fromConfig(self.config.warp)
 self.schema = afwTable.ExposureTable.makeMinimalSchema()
 self.schema.addField(
Expand DownExpand Up@@ -381,6 +392,14 @@ def run(self, *, coaddExposureHandles, bbox, wcs, dataIds, physical_filter, visi
 if count == 0:
 raise pipeBase.NoWorkFound("No valid pixels in warped template.")

 if self.config.doScaleVariance:
 # Scale the variance of the template image before subtraction, if
 # needed. Note that the science variance is scaled
 # independently in ``AlardLuptonSubtractTask``.
 varianceFactor = self.scaleVariance.run(template.maskedImage)
 self.log.info("Template variance scaling factor: %.2f", varianceFactor)
 self.metadata["scaleTemplateVarianceFactor"] = varianceFactor

 # Make a single catalog containing all the inputs that were accepted.
 catalog = afwTable.ExposureCatalog(self.schema)
 catalog.reserve(sum([len(c) for c in catalogs]))
Expand All@@ -394,6 +413,7 @@ def run(self, *, coaddExposureHandles, bbox, wcs, dataIds, physical_filter, visi
 template.setFilter(afwImage.FilterLabel(band, physical_filter))
 template.setPhotoCalib(photoCalib)
 template.setPsf(self._makePsf(template, catalog, wcs))

@khernerkhernerAug 31, 2026

Copy link


Copy Markdown

Contributor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Was this blank line intentional?
@isullivanisullivanAug 31, 2026

Copy link


Copy Markdown

ContributorAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It was an artifact of adding lines there originally, then moving them earlier. But, I liked the visual space so I left it in. I can remove it if you prefer!
 # Record the input coadd patches as the template's coadd inputs.
 coaddInputs = afwImage.CoaddInputs(afwTable.ExposureTable.makeMinimalSchema(), self.schema)
 coaddInputs.ccds.extend(catalog, deep=True)
Expand Down

13 changes: 6 additions & 7 deletions
python/lsst/ip/diffim/subtractImages.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -207,7 +207,8 @@ class AlardLuptonSubtractBaseConfig(lsst.pex.config.Config):
 doScaleVariance = lsst.pex.config.Field(
 dtype=bool,
 default=True,
 doc="Scale variance of the image difference?"
 doc="Scale variance of the science image? Note that the template variance is NOT scaled"
 " here. The template variance may be scaled independently in ``GetTemplateTask``."
 )
 scaleVariance = lsst.pex.config.ConfigurableField(
 target=ScaleVarianceTask,
Expand DownExpand Up@@ -1044,13 +1045,11 @@ def _prepareInputs(self, template, science, visitSummary=None):
 self.metadata["templateCoveragePercent"] = 100*templateCoverageFraction

 if self.config.doScaleVariance:
 # Scale the variance of the template and science images before
 # convolution, subtraction, or decorrelation so that they have the
 # correct ratio.
templateVarFactor = self.scaleVariance.run(template.maskedImage)
 # Scale the variance of the science image before
 # convolution, subtraction, or decorrelation so that it has the
 # correct ratio. Note that the template variance is scaled
# independently in ``GetTemplateTask``.
 sciVarFactor = self.scaleVariance.run(science.maskedImage)
 self.log.info("Template variance scaling factor: %.2f", templateVarFactor)
 self.metadata["scaleTemplateVarianceFactor"] = templateVarFactor
 self.log.info("Science variance scaling factor: %.2f", sciVarFactor)
 self.metadata["scaleScienceVarianceFactor"] = sciVarFactor

Expand Down

88 changes: 88 additions & 0 deletions
tests/test_getTemplate.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -337,6 +337,94 @@ def testNanInputs(self, box=None, nInput=None):
 # in the template are closer to the original anymore.
 self.assertTrue(np.isfinite(result.template.image.array).all())

 def _scaleInputVariance(self, tract, factor):
 """Return fresh handles for one tract's patches, with their variance
 planes multiplied by ``factor``.

 Parameters
 ----------
 tract : `int`
 Id of the tract whose patches should be copied.
 factor : `float`
 Factor to multiply the input variance planes by.

 Returns
 -------
 handles : `list` [`lsst.pipe.base.InMemoryDatasetHandle`]
 Handles to the modified patches.
 """
 handles = []
 for handle in self.patches[tract]:
 # ``copy=True`` on the original handles means this is a copy, so
 # the patches shared with the other tests are left untouched.
 patch = handle.get()
 patch.variance.array *= factor
 handles.append(pipeBase.InMemoryDatasetHandle(patch,
 storageClass="ExposureF",
 copy=True,
 dataId=handle.dataId))
 return handles

 def testScaleVariance(self):
 """Test that the template variance plane is rescaled to match the
 empirical pixel noise, and that the factor used is recorded in the
 task metadata.
 """
 scaleFactor = 1.345
 box = lsst.geom.Box2I(lsst.geom.Point2I(0, 0), lsst.geom.Point2I(180, 180))

 def _configureAndRunTask(doScaleVariance, varianceScale=1.):
 """Build a template from tract 0, optionally rescaling the input
 variance planes by ``varianceScale`` first.
 """
 config = lsst.ip.diffim.GetTemplateTask.ConfigClass()
 config.doScaleVariance = doScaleVariance
 task = lsst.ip.diffim.GetTemplateTask(config=config)
 # Task modifies the input bbox, so pass a copy.
 result = task.run(coaddExposureHandles={0: self._scaleInputVariance(0, varianceScale)},
 bbox=lsst.geom.Box2I(box),
 wcs=self.exposure.wcs,
 dataIds={0: self.dataIds[0]},
 physical_filter="a_test")
 return task, result.template

 # With scaling disabled the subtask is never constructed, and nothing
 # is recorded in the metadata.
 taskOff, templateOff = _configureAndRunTask(False)
 self.assertFalse(hasattr(taskOff, "scaleVariance"))
 self.assertNotIn("scaleTemplateVarianceFactor", taskOff.metadata)

 # Both warps -- lanczos5 in ``_makePatches`` and lanczos3 in the
 # task -- correlate the noise. The variance plane tracks only the
 # per-pixel diagonal, which the second warp leaves too low, so
 # ``scaleVariance`` measures a factor well above 1 even though the
 # input variance planes are correct.
 #
 taskOn, templateOn = _configureAndRunTask(True)
 factor = taskOn.metadata["scaleTemplateVarianceFactor"]
 # TODO DM-55879: this value is pinned on purpose. The lanczos warping
 # kernels introduce small correlations that artificially suppress the
 # image pixel stddev and inflate the variance scaling factor. This
 # should be changed to 1.0 after DM-55879 is merged.
 self.assertFloatsAlmostEqual(factor, 1.1465, atol=0.01,
 msg="Measured template variance scaling changed; see the"
 " comment above if the correlation correction landed.")
 # The only difference from the unscaled template is the constant
 # factor applied to the variance plane.
 self.assertFloatsAlmostEqual(templateOn.variance.array,
 templateOff.variance.array*factor, rtol=1e-5)
 # Tolerance here is float32 round-off: repeated runs of the task are
 # not bitwise identical.
 self.assertImagesAlmostEqual(templateOn.image, templateOff.image, rtol=1e-5, atol=1e-5)

 # If the input variance planes under-estimate the noise by a known
 # factor, the measured factor grows by that amount and the same
 # output variance plane is recovered.
 taskLow, templateLow = _configureAndRunTask(True, varianceScale=1/scaleFactor)
 self.assertFloatsAlmostEqual(taskLow.metadata["scaleTemplateVarianceFactor"],
 factor*scaleFactor, rtol=1e-5)
 self.assertImagesAlmostEqual(templateLow.variance, templateOn.variance, rtol=1e-5)


def setup_module(module):
 lsst.utils.tests.init()
Expand Down

26 changes: 9 additions & 17 deletions
tests/test_subtractTask.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -349,7 +349,6 @@ def _run_and_check_images(statsCtrl, statsCtrlDetect, scienceNoiseLevel, templat
 templateBorderSize=20, doApplyCalibration=True)
 task = self._setup_subtraction(mode="convolveScience")
 output = task.run(template, science, sources)
 self.assertFloatsAlmostEqual(task.metadata["scaleTemplateVarianceFactor"], 1., atol=.05)
 self.assertFloatsAlmostEqual(task.metadata["scaleScienceVarianceFactor"], 1., atol=.05)
 # Mean of difference image should be close to zero.
 nGoodPix = np.sum(np.isfinite(output.difference.image.array))
Expand DownExpand Up@@ -385,7 +384,6 @@ def _run_and_check_images(statsCtrl, statsCtrlDetect, scienceNoiseLevel, templat
 templateBorderSize=20, doApplyCalibration=True)
 task = self._setup_subtraction()
 output = task.run(template, science, sources)
 self.assertFloatsAlmostEqual(task.metadata["scaleTemplateVarianceFactor"], 1., atol=.05)
 self.assertFloatsAlmostEqual(task.metadata["scaleScienceVarianceFactor"], 1., atol=.05)
 # There should be no NaNs in the image if we convolve the template with a buffer
 self.assertTrue(np.all(np.isfinite(output.difference.image.array)))
Expand DownExpand Up@@ -632,8 +630,6 @@ def _run_and_check_images(science, template, sources, statsCtrl,
 )
 output = task.run(template.clone(), science.clone(), sources)
 if doScaleVariance:
 self.assertFloatsAlmostEqual(task.metadata["scaleTemplateVarianceFactor"],
 scaleFactor, atol=0.05)
 self.assertFloatsAlmostEqual(task.metadata["scaleScienceVarianceFactor"],
 scaleFactor, atol=0.05)

Expand All@@ -646,7 +642,8 @@ def _run_and_check_images(science, template, sources, statsCtrl,
 statsCtrl)

 if doScaleVariance:
 templateNoise *= scaleFactor
 # Only the science variance is scaled here. The template
 # variance is scaled independently in ``GetTemplateTask``.
 scienceNoise *= scaleFactor
 varMean = computeRobustStatistics(output.difference.variance, output.difference.mask, statsCtrl)
 self.assertFloatsAlmostEqual(varMean, scienceNoise + templateNoise, rtol=0.1)
Expand All@@ -666,8 +663,7 @@ def _run_and_check_images(science, template, sources, statsCtrl,
 doDecorrelation=False, doScaleVariance=False)

 # Verify that the variance plane of the difference image is correct
 # when the template variance plane is incorrect
 template.variance.array /= scaleFactor
 # when the input science variance plane is incorrect
 science.variance.array /= scaleFactor
 _run_and_check_images(science, template, sources, statsCtrl,
 doDecorrelation=True, doScaleVariance=True, scaleFactor=scaleFactor)
Expand DownExpand Up@@ -700,8 +696,6 @@ def _run_and_check_images(science, template, sources, statsCtrl,
 )
 output = task.run(template.clone(), science.clone(), sources)
 if doScaleVariance:
 self.assertFloatsAlmostEqual(task.metadata["scaleTemplateVarianceFactor"],
 scaleFactor, atol=0.05)
 self.assertFloatsAlmostEqual(task.metadata["scaleScienceVarianceFactor"],
 scaleFactor, atol=0.05)

Expand All@@ -714,7 +708,8 @@ def _run_and_check_images(science, template, sources, statsCtrl,
 statsCtrl)

 if doScaleVariance:
 templateNoise *= scaleFactor
 # Only the science variance is scaled here. The template
 # variance is scaled independently in ``GetTemplateTask``.
 scienceNoise *= scaleFactor

 varMean = computeRobustStatistics(output.difference.variance, output.difference.mask, statsCtrl)
Expand All@@ -735,9 +730,8 @@ def _run_and_check_images(science, template, sources, statsCtrl,
 doDecorrelation=False, doScaleVariance=False)

 # Verify that the variance plane of the difference image is correct
 # when the template and science variance planes are incorrect
 # when the input science variance plane is incorrect
 science.variance.array /= scaleFactor
 template.variance.array /= scaleFactor
 _run_and_check_images(science, template, sources, statsCtrl,
 doDecorrelation=True, doScaleVariance=True, scaleFactor=scaleFactor)
 _run_and_check_images(science, template, sources, statsCtrl,
Expand DownExpand Up@@ -1259,8 +1253,6 @@ def _run_and_check_images(science, template, sources, statsCtrl,
 )
 output = task.run(template.clone(), science.clone(), sources)
 if doScaleVariance:
 self.assertFloatsAlmostEqual(task.metadata["scaleTemplateVarianceFactor"],
 scaleFactor, atol=0.05)
 self.assertFloatsAlmostEqual(task.metadata["scaleScienceVarianceFactor"],
 scaleFactor, atol=0.05)

Expand All@@ -1278,7 +1270,8 @@ def _run_and_check_images(science, template, sources, statsCtrl,
 output.matchedTemplate.mask,
 statsCtrl)
 if doScaleVariance:
 templateNoise *= scaleFactor
 # Only the science variance is scaled here. The template
 # variance is scaled independently in ``GetTemplateTask``.
 scienceNoise *= scaleFactor
 varMean = computeRobustStatistics(output.scoreExposure.variance,
 output.scoreExposure.mask,
Expand All@@ -1302,8 +1295,7 @@ def _run_and_check_images(science, template, sources, statsCtrl,
 doDecorrelation=False, doScaleVariance=False)

 # Verify that the variance plane of the Score image is correct
 # when the template variance plane is incorrect
 template.variance.array /= scaleFactor
 # when the input science variance plane is incorrect
 science.variance.array /= scaleFactor
 _run_and_check_images(science, template, sources, statsCtrl,
 doDecorrelation=True, doScaleVariance=True, scaleFactor=scaleFactor)
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Auto-enable theater mode on YouTube
(function() {
function tryTheater() {
var btn = document.querySelector('button[aria-label="Theater mode"], ytd-player #player button[title="Theater mode"]');
if (btn && !btn.classList.contains('activated')) {
btn.click();
}
}
// Try immediately
tryTheater();
// Try after navigation (SPA)
var lastUrl = location.href;
setInterval(function() {
if (location.href !== lastUrl) {
lastUrl = location.href;
setTimeout(tryTheater, 500);
}
}, 1000);
// Also try on player load
var observer = new MutationObserver(tryTheater);
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '
DM-55993: Move option to scale template variance by isullivan · Pull Request #469 · lsst/ip_diffim · GitHub
Skip to content
DM-55993: Move option to scale template variance#469

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Open

isullivan
wants to merge
2
commits into

base:main
Choose a base branch

from

 Open


DM-55993: Move option to scale template variance
#469
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


2 commits


Select commit
Hold shift + click to select a range
385e543

Move option to scale template variance to getTemplate.py


isullivan Aug 31, 2026
ffb7738

Add unit tests of template variance scaling


isullivan Aug 31, 2026

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


22 changes: 21 additions & 1 deletion
python/lsst/ip/diffim/getTemplate.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -34,7 +34,7 @@

from lsst.skymap import BaseSkyMap
from lsst.ip.diffim.dcrModel import DcrModel
from lsst.meas.algorithms import CoaddPsf, CoaddPsfConfig, SubtractBackgroundTask
from lsst.meas.algorithms import CoaddPsf, CoaddPsfConfig, SubtractBackgroundTask, ScaleVarianceTask
from lsst.utils.timer import timeMethod

__all__ = [
Expand DownExpand Up@@ -120,6 +120,15 @@ class GetTemplateConfig(
 doc="Minimum fraction of unmasked pixels needed to set the"
 " HIGH_VARIANCE mask plane.",
 )
 doScaleVariance = pexConfig.Field(
 dtype=bool,
 default=True,
 doc="Scale variance of the template image?"
 )
 scaleVariance = pexConfig.ConfigurableField(
 target=ScaleVarianceTask,
 doc="Subtask to rescale the variance of the template to the statistically expected level."
 )

 def setDefaults(self):
 # Use a smaller cache: per SeparableKernel.computeCache, this should
Expand DownExpand Up@@ -152,6 +161,8 @@ class GetTemplateTask(pipeBase.PipelineTask):

 def __init__(self, *args, **kwargs):
 super().__init__(*args, **kwargs)
 if self.config.doScaleVariance:
 self.makeSubtask("scaleVariance")
 self.warper = afwMath.Warper.fromConfig(self.config.warp)
 self.schema = afwTable.ExposureTable.makeMinimalSchema()
 self.schema.addField(
Expand DownExpand Up@@ -381,6 +392,14 @@ def run(self, *, coaddExposureHandles, bbox, wcs, dataIds, physical_filter, visi
 if count == 0:
 raise pipeBase.NoWorkFound("No valid pixels in warped template.")

 if self.config.doScaleVariance:
 # Scale the variance of the template image before subtraction, if
 # needed. Note that the science variance is scaled
 # independently in ``AlardLuptonSubtractTask``.
 varianceFactor = self.scaleVariance.run(template.maskedImage)
 self.log.info("Template variance scaling factor: %.2f", varianceFactor)
 self.metadata["scaleTemplateVarianceFactor"] = varianceFactor

 # Make a single catalog containing all the inputs that were accepted.
 catalog = afwTable.ExposureCatalog(self.schema)
 catalog.reserve(sum([len(c) for c in catalogs]))
Expand All@@ -394,6 +413,7 @@ def run(self, *, coaddExposureHandles, bbox, wcs, dataIds, physical_filter, visi
 template.setFilter(afwImage.FilterLabel(band, physical_filter))
 template.setPhotoCalib(photoCalib)
 template.setPsf(self._makePsf(template, catalog, wcs))

@khernerkhernerAug 31, 2026

Copy link


Copy Markdown

Contributor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Was this blank line intentional?
@isullivanisullivanAug 31, 2026

Copy link


Copy Markdown

ContributorAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It was an artifact of adding lines there originally, then moving them earlier. But, I liked the visual space so I left it in. I can remove it if you prefer!
 # Record the input coadd patches as the template's coadd inputs.
 coaddInputs = afwImage.CoaddInputs(afwTable.ExposureTable.makeMinimalSchema(), self.schema)
 coaddInputs.ccds.extend(catalog, deep=True)
Expand Down

13 changes: 6 additions & 7 deletions
python/lsst/ip/diffim/subtractImages.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -207,7 +207,8 @@ class AlardLuptonSubtractBaseConfig(lsst.pex.config.Config):
 doScaleVariance = lsst.pex.config.Field(
 dtype=bool,
 default=True,
 doc="Scale variance of the image difference?"
 doc="Scale variance of the science image? Note that the template variance is NOT scaled"
 " here. The template variance may be scaled independently in ``GetTemplateTask``."
 )
 scaleVariance = lsst.pex.config.ConfigurableField(
 target=ScaleVarianceTask,
Expand DownExpand Up@@ -1044,13 +1045,11 @@ def _prepareInputs(self, template, science, visitSummary=None):
 self.metadata["templateCoveragePercent"] = 100*templateCoverageFraction

 if self.config.doScaleVariance:
 # Scale the variance of the template and science images before
 # convolution, subtraction, or decorrelation so that they have the
 # correct ratio.
templateVarFactor = self.scaleVariance.run(template.maskedImage)
 # Scale the variance of the science image before
 # convolution, subtraction, or decorrelation so that it has the
 # correct ratio. Note that the template variance is scaled
# independently in ``GetTemplateTask``.
 sciVarFactor = self.scaleVariance.run(science.maskedImage)
 self.log.info("Template variance scaling factor: %.2f", templateVarFactor)
 self.metadata["scaleTemplateVarianceFactor"] = templateVarFactor
 self.log.info("Science variance scaling factor: %.2f", sciVarFactor)
 self.metadata["scaleScienceVarianceFactor"] = sciVarFactor

Expand Down

88 changes: 88 additions & 0 deletions
tests/test_getTemplate.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -337,6 +337,94 @@ def testNanInputs(self, box=None, nInput=None):
 # in the template are closer to the original anymore.
 self.assertTrue(np.isfinite(result.template.image.array).all())

 def _scaleInputVariance(self, tract, factor):
 """Return fresh handles for one tract's patches, with their variance
 planes multiplied by ``factor``.

 Parameters
 ----------
 tract : `int`
 Id of the tract whose patches should be copied.
 factor : `float`
 Factor to multiply the input variance planes by.

 Returns
 -------
 handles : `list` [`lsst.pipe.base.InMemoryDatasetHandle`]
 Handles to the modified patches.
 """
 handles = []
 for handle in self.patches[tract]:
 # ``copy=True`` on the original handles means this is a copy, so
 # the patches shared with the other tests are left untouched.
 patch = handle.get()
 patch.variance.array *= factor
 handles.append(pipeBase.InMemoryDatasetHandle(patch,
 storageClass="ExposureF",
 copy=True,
 dataId=handle.dataId))
 return handles

 def testScaleVariance(self):
 """Test that the template variance plane is rescaled to match the
 empirical pixel noise, and that the factor used is recorded in the
 task metadata.
 """
 scaleFactor = 1.345
 box = lsst.geom.Box2I(lsst.geom.Point2I(0, 0), lsst.geom.Point2I(180, 180))

 def _configureAndRunTask(doScaleVariance, varianceScale=1.):
 """Build a template from tract 0, optionally rescaling the input
 variance planes by ``varianceScale`` first.
 """
 config = lsst.ip.diffim.GetTemplateTask.ConfigClass()
 config.doScaleVariance = doScaleVariance
 task = lsst.ip.diffim.GetTemplateTask(config=config)
 # Task modifies the input bbox, so pass a copy.
 result = task.run(coaddExposureHandles={0: self._scaleInputVariance(0, varianceScale)},
 bbox=lsst.geom.Box2I(box),
 wcs=self.exposure.wcs,
 dataIds={0: self.dataIds[0]},
 physical_filter="a_test")
 return task, result.template

 # With scaling disabled the subtask is never constructed, and nothing
 # is recorded in the metadata.
 taskOff, templateOff = _configureAndRunTask(False)
 self.assertFalse(hasattr(taskOff, "scaleVariance"))
 self.assertNotIn("scaleTemplateVarianceFactor", taskOff.metadata)

 # Both warps -- lanczos5 in ``_makePatches`` and lanczos3 in the
 # task -- correlate the noise. The variance plane tracks only the
 # per-pixel diagonal, which the second warp leaves too low, so
 # ``scaleVariance`` measures a factor well above 1 even though the
 # input variance planes are correct.
 #
 taskOn, templateOn = _configureAndRunTask(True)
 factor = taskOn.metadata["scaleTemplateVarianceFactor"]
 # TODO DM-55879: this value is pinned on purpose. The lanczos warping
 # kernels introduce small correlations that artificially suppress the
 # image pixel stddev and inflate the variance scaling factor. This
 # should be changed to 1.0 after DM-55879 is merged.
 self.assertFloatsAlmostEqual(factor, 1.1465, atol=0.01,
 msg="Measured template variance scaling changed; see the"
 " comment above if the correlation correction landed.")
 # The only difference from the unscaled template is the constant
 # factor applied to the variance plane.
 self.assertFloatsAlmostEqual(templateOn.variance.array,
 templateOff.variance.array*factor, rtol=1e-5)
 # Tolerance here is float32 round-off: repeated runs of the task are
 # not bitwise identical.
 self.assertImagesAlmostEqual(templateOn.image, templateOff.image, rtol=1e-5, atol=1e-5)

 # If the input variance planes under-estimate the noise by a known
 # factor, the measured factor grows by that amount and the same
 # output variance plane is recovered.
 taskLow, templateLow = _configureAndRunTask(True, varianceScale=1/scaleFactor)
 self.assertFloatsAlmostEqual(taskLow.metadata["scaleTemplateVarianceFactor"],
 factor*scaleFactor, rtol=1e-5)
 self.assertImagesAlmostEqual(templateLow.variance, templateOn.variance, rtol=1e-5)


def setup_module(module):
 lsst.utils.tests.init()
Expand Down

26 changes: 9 additions & 17 deletions
tests/test_subtractTask.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -349,7 +349,6 @@ def _run_and_check_images(statsCtrl, statsCtrlDetect, scienceNoiseLevel, templat
 templateBorderSize=20, doApplyCalibration=True)
 task = self._setup_subtraction(mode="convolveScience")
 output = task.run(template, science, sources)
 self.assertFloatsAlmostEqual(task.metadata["scaleTemplateVarianceFactor"], 1., atol=.05)
 self.assertFloatsAlmostEqual(task.metadata["scaleScienceVarianceFactor"], 1., atol=.05)
 # Mean of difference image should be close to zero.
 nGoodPix = np.sum(np.isfinite(output.difference.image.array))
Expand DownExpand Up@@ -385,7 +384,6 @@ def _run_and_check_images(statsCtrl, statsCtrlDetect, scienceNoiseLevel, templat
 templateBorderSize=20, doApplyCalibration=True)
 task = self._setup_subtraction()
 output = task.run(template, science, sources)
 self.assertFloatsAlmostEqual(task.metadata["scaleTemplateVarianceFactor"], 1., atol=.05)
 self.assertFloatsAlmostEqual(task.metadata["scaleScienceVarianceFactor"], 1., atol=.05)
 # There should be no NaNs in the image if we convolve the template with a buffer
 self.assertTrue(np.all(np.isfinite(output.difference.image.array)))
Expand DownExpand Up@@ -632,8 +630,6 @@ def _run_and_check_images(science, template, sources, statsCtrl,
 )
 output = task.run(template.clone(), science.clone(), sources)
 if doScaleVariance:
 self.assertFloatsAlmostEqual(task.metadata["scaleTemplateVarianceFactor"],
 scaleFactor, atol=0.05)
 self.assertFloatsAlmostEqual(task.metadata["scaleScienceVarianceFactor"],
 scaleFactor, atol=0.05)

Expand All@@ -646,7 +642,8 @@ def _run_and_check_images(science, template, sources, statsCtrl,
 statsCtrl)

 if doScaleVariance:
 templateNoise *= scaleFactor
 # Only the science variance is scaled here. The template
 # variance is scaled independently in ``GetTemplateTask``.
 scienceNoise *= scaleFactor
 varMean = computeRobustStatistics(output.difference.variance, output.difference.mask, statsCtrl)
 self.assertFloatsAlmostEqual(varMean, scienceNoise + templateNoise, rtol=0.1)
Expand All@@ -666,8 +663,7 @@ def _run_and_check_images(science, template, sources, statsCtrl,
 doDecorrelation=False, doScaleVariance=False)

 # Verify that the variance plane of the difference image is correct
 # when the template variance plane is incorrect
 template.variance.array /= scaleFactor
 # when the input science variance plane is incorrect
 science.variance.array /= scaleFactor
 _run_and_check_images(science, template, sources, statsCtrl,
 doDecorrelation=True, doScaleVariance=True, scaleFactor=scaleFactor)
Expand DownExpand Up@@ -700,8 +696,6 @@ def _run_and_check_images(science, template, sources, statsCtrl,
 )
 output = task.run(template.clone(), science.clone(), sources)
 if doScaleVariance:
 self.assertFloatsAlmostEqual(task.metadata["scaleTemplateVarianceFactor"],
 scaleFactor, atol=0.05)
 self.assertFloatsAlmostEqual(task.metadata["scaleScienceVarianceFactor"],
 scaleFactor, atol=0.05)

Expand All@@ -714,7 +708,8 @@ def _run_and_check_images(science, template, sources, statsCtrl,
 statsCtrl)

 if doScaleVariance:
 templateNoise *= scaleFactor
 # Only the science variance is scaled here. The template
 # variance is scaled independently in ``GetTemplateTask``.
 scienceNoise *= scaleFactor

 varMean = computeRobustStatistics(output.difference.variance, output.difference.mask, statsCtrl)
Expand All@@ -735,9 +730,8 @@ def _run_and_check_images(science, template, sources, statsCtrl,
 doDecorrelation=False, doScaleVariance=False)

 # Verify that the variance plane of the difference image is correct
 # when the template and science variance planes are incorrect
 # when the input science variance plane is incorrect
 science.variance.array /= scaleFactor
 template.variance.array /= scaleFactor
 _run_and_check_images(science, template, sources, statsCtrl,
 doDecorrelation=True, doScaleVariance=True, scaleFactor=scaleFactor)
 _run_and_check_images(science, template, sources, statsCtrl,
Expand DownExpand Up@@ -1259,8 +1253,6 @@ def _run_and_check_images(science, template, sources, statsCtrl,
 )
 output = task.run(template.clone(), science.clone(), sources)
 if doScaleVariance:
 self.assertFloatsAlmostEqual(task.metadata["scaleTemplateVarianceFactor"],
 scaleFactor, atol=0.05)
 self.assertFloatsAlmostEqual(task.metadata["scaleScienceVarianceFactor"],
 scaleFactor, atol=0.05)

Expand All@@ -1278,7 +1270,8 @@ def _run_and_check_images(science, template, sources, statsCtrl,
 output.matchedTemplate.mask,
 statsCtrl)
 if doScaleVariance:
 templateNoise *= scaleFactor
 # Only the science variance is scaled here. The template
 # variance is scaled independently in ``GetTemplateTask``.
 scienceNoise *= scaleFactor
 varMean = computeRobustStatistics(output.scoreExposure.variance,
 output.scoreExposure.mask,
Expand All@@ -1302,8 +1295,7 @@ def _run_and_check_images(science, template, sources, statsCtrl,
 doDecorrelation=False, doScaleVariance=False)

 # Verify that the variance plane of the Score image is correct
 # when the template variance plane is incorrect
 template.variance.array /= scaleFactor
 # when the input science variance plane is incorrect
 science.variance.array /= scaleFactor
 _run_and_check_images(science, template, sources, statsCtrl,
 doDecorrelation=True, doScaleVariance=True, scaleFactor=scaleFactor)
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Remove or un-stick sticky/fixed headers that block content
(function() {
function unstick() {
document.querySelectorAll('header, nav, [role="banner"], .header, .navbar, .sticky, .fixed-top, [style*="position: fixed"], [style*="position:sticky"]').forEach(function(el) {
if (el.style.position === 'fixed' || el.style.position === 'sticky' || getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') {
el.style.position = 'static';
el.style.top = 'auto';
el.style.zIndex = 'auto';
}
});
}
unstick();
var observer = new MutationObserver(unstick);
observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] });
})();
}
} catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '
DM-55993: Move option to scale template variance by isullivan · Pull Request #469 · lsst/ip_diffim · GitHub
Skip to content
DM-55993: Move option to scale template variance#469

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Open

isullivan
wants to merge
2
commits into

base:main
Choose a base branch

from

 Open


DM-55993: Move option to scale template variance
#469
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


2 commits


Select commit
Hold shift + click to select a range
385e543

Move option to scale template variance to getTemplate.py


isullivan Aug 31, 2026
ffb7738

Add unit tests of template variance scaling


isullivan Aug 31, 2026

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


22 changes: 21 additions & 1 deletion
python/lsst/ip/diffim/getTemplate.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -34,7 +34,7 @@

from lsst.skymap import BaseSkyMap
from lsst.ip.diffim.dcrModel import DcrModel
from lsst.meas.algorithms import CoaddPsf, CoaddPsfConfig, SubtractBackgroundTask
from lsst.meas.algorithms import CoaddPsf, CoaddPsfConfig, SubtractBackgroundTask, ScaleVarianceTask
from lsst.utils.timer import timeMethod

__all__ = [
Expand DownExpand Up@@ -120,6 +120,15 @@ class GetTemplateConfig(
 doc="Minimum fraction of unmasked pixels needed to set the"
 " HIGH_VARIANCE mask plane.",
 )
 doScaleVariance = pexConfig.Field(
 dtype=bool,
 default=True,
 doc="Scale variance of the template image?"
 )
 scaleVariance = pexConfig.ConfigurableField(
 target=ScaleVarianceTask,
 doc="Subtask to rescale the variance of the template to the statistically expected level."
 )

 def setDefaults(self):
 # Use a smaller cache: per SeparableKernel.computeCache, this should
Expand DownExpand Up@@ -152,6 +161,8 @@ class GetTemplateTask(pipeBase.PipelineTask):

 def __init__(self, *args, **kwargs):
 super().__init__(*args, **kwargs)
 if self.config.doScaleVariance:
 self.makeSubtask("scaleVariance")
 self.warper = afwMath.Warper.fromConfig(self.config.warp)
 self.schema = afwTable.ExposureTable.makeMinimalSchema()
 self.schema.addField(
Expand DownExpand Up@@ -381,6 +392,14 @@ def run(self, *, coaddExposureHandles, bbox, wcs, dataIds, physical_filter, visi
 if count == 0:
 raise pipeBase.NoWorkFound("No valid pixels in warped template.")

 if self.config.doScaleVariance:
 # Scale the variance of the template image before subtraction, if
 # needed. Note that the science variance is scaled
 # independently in ``AlardLuptonSubtractTask``.
 varianceFactor = self.scaleVariance.run(template.maskedImage)
 self.log.info("Template variance scaling factor: %.2f", varianceFactor)
 self.metadata["scaleTemplateVarianceFactor"] = varianceFactor

 # Make a single catalog containing all the inputs that were accepted.
 catalog = afwTable.ExposureCatalog(self.schema)
 catalog.reserve(sum([len(c) for c in catalogs]))
Expand All@@ -394,6 +413,7 @@ def run(self, *, coaddExposureHandles, bbox, wcs, dataIds, physical_filter, visi
 template.setFilter(afwImage.FilterLabel(band, physical_filter))
 template.setPhotoCalib(photoCalib)
 template.setPsf(self._makePsf(template, catalog, wcs))

@khernerkhernerAug 31, 2026

Copy link


Copy Markdown

Contributor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Was this blank line intentional?
@isullivanisullivanAug 31, 2026

Copy link


Copy Markdown

ContributorAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It was an artifact of adding lines there originally, then moving them earlier. But, I liked the visual space so I left it in. I can remove it if you prefer!
 # Record the input coadd patches as the template's coadd inputs.
 coaddInputs = afwImage.CoaddInputs(afwTable.ExposureTable.makeMinimalSchema(), self.schema)
 coaddInputs.ccds.extend(catalog, deep=True)
Expand Down

13 changes: 6 additions & 7 deletions
python/lsst/ip/diffim/subtractImages.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -207,7 +207,8 @@ class AlardLuptonSubtractBaseConfig(lsst.pex.config.Config):
 doScaleVariance = lsst.pex.config.Field(
 dtype=bool,
 default=True,
 doc="Scale variance of the image difference?"
 doc="Scale variance of the science image? Note that the template variance is NOT scaled"
 " here. The template variance may be scaled independently in ``GetTemplateTask``."
 )
 scaleVariance = lsst.pex.config.ConfigurableField(
 target=ScaleVarianceTask,
Expand DownExpand Up@@ -1044,13 +1045,11 @@ def _prepareInputs(self, template, science, visitSummary=None):
 self.metadata["templateCoveragePercent"] = 100*templateCoverageFraction

 if self.config.doScaleVariance:
 # Scale the variance of the template and science images before
 # convolution, subtraction, or decorrelation so that they have the
 # correct ratio.
templateVarFactor = self.scaleVariance.run(template.maskedImage)
 # Scale the variance of the science image before
 # convolution, subtraction, or decorrelation so that it has the
 # correct ratio. Note that the template variance is scaled
# independently in ``GetTemplateTask``.
 sciVarFactor = self.scaleVariance.run(science.maskedImage)
 self.log.info("Template variance scaling factor: %.2f", templateVarFactor)
 self.metadata["scaleTemplateVarianceFactor"] = templateVarFactor
 self.log.info("Science variance scaling factor: %.2f", sciVarFactor)
 self.metadata["scaleScienceVarianceFactor"] = sciVarFactor

Expand Down

88 changes: 88 additions & 0 deletions
tests/test_getTemplate.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -337,6 +337,94 @@ def testNanInputs(self, box=None, nInput=None):
 # in the template are closer to the original anymore.
 self.assertTrue(np.isfinite(result.template.image.array).all())

 def _scaleInputVariance(self, tract, factor):
 """Return fresh handles for one tract's patches, with their variance
 planes multiplied by ``factor``.

 Parameters
 ----------
 tract : `int`
 Id of the tract whose patches should be copied.
 factor : `float`
 Factor to multiply the input variance planes by.

 Returns
 -------
 handles : `list` [`lsst.pipe.base.InMemoryDatasetHandle`]
 Handles to the modified patches.
 """
 handles = []
 for handle in self.patches[tract]:
 # ``copy=True`` on the original handles means this is a copy, so
 # the patches shared with the other tests are left untouched.
 patch = handle.get()
 patch.variance.array *= factor
 handles.append(pipeBase.InMemoryDatasetHandle(patch,
 storageClass="ExposureF",
 copy=True,
 dataId=handle.dataId))
 return handles

 def testScaleVariance(self):
 """Test that the template variance plane is rescaled to match the
 empirical pixel noise, and that the factor used is recorded in the
 task metadata.
 """
 scaleFactor = 1.345
 box = lsst.geom.Box2I(lsst.geom.Point2I(0, 0), lsst.geom.Point2I(180, 180))

 def _configureAndRunTask(doScaleVariance, varianceScale=1.):
 """Build a template from tract 0, optionally rescaling the input
 variance planes by ``varianceScale`` first.
 """
 config = lsst.ip.diffim.GetTemplateTask.ConfigClass()
 config.doScaleVariance = doScaleVariance
 task = lsst.ip.diffim.GetTemplateTask(config=config)
 # Task modifies the input bbox, so pass a copy.
 result = task.run(coaddExposureHandles={0: self._scaleInputVariance(0, varianceScale)},
 bbox=lsst.geom.Box2I(box),
 wcs=self.exposure.wcs,
 dataIds={0: self.dataIds[0]},
 physical_filter="a_test")
 return task, result.template

 # With scaling disabled the subtask is never constructed, and nothing
 # is recorded in the metadata.
 taskOff, templateOff = _configureAndRunTask(False)
 self.assertFalse(hasattr(taskOff, "scaleVariance"))
 self.assertNotIn("scaleTemplateVarianceFactor", taskOff.metadata)

 # Both warps -- lanczos5 in ``_makePatches`` and lanczos3 in the
 # task -- correlate the noise. The variance plane tracks only the
 # per-pixel diagonal, which the second warp leaves too low, so
 # ``scaleVariance`` measures a factor well above 1 even though the
 # input variance planes are correct.
 #
 taskOn, templateOn = _configureAndRunTask(True)
 factor = taskOn.metadata["scaleTemplateVarianceFactor"]
 # TODO DM-55879: this value is pinned on purpose. The lanczos warping
 # kernels introduce small correlations that artificially suppress the
 # image pixel stddev and inflate the variance scaling factor. This
 # should be changed to 1.0 after DM-55879 is merged.
 self.assertFloatsAlmostEqual(factor, 1.1465, atol=0.01,
 msg="Measured template variance scaling changed; see the"
 " comment above if the correlation correction landed.")
 # The only difference from the unscaled template is the constant
 # factor applied to the variance plane.
 self.assertFloatsAlmostEqual(templateOn.variance.array,
 templateOff.variance.array*factor, rtol=1e-5)
 # Tolerance here is float32 round-off: repeated runs of the task are
 # not bitwise identical.
 self.assertImagesAlmostEqual(templateOn.image, templateOff.image, rtol=1e-5, atol=1e-5)

 # If the input variance planes under-estimate the noise by a known
 # factor, the measured factor grows by that amount and the same
 # output variance plane is recovered.
 taskLow, templateLow = _configureAndRunTask(True, varianceScale=1/scaleFactor)
 self.assertFloatsAlmostEqual(taskLow.metadata["scaleTemplateVarianceFactor"],
 factor*scaleFactor, rtol=1e-5)
 self.assertImagesAlmostEqual(templateLow.variance, templateOn.variance, rtol=1e-5)


def setup_module(module):
 lsst.utils.tests.init()
Expand Down

26 changes: 9 additions & 17 deletions
tests/test_subtractTask.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -349,7 +349,6 @@ def _run_and_check_images(statsCtrl, statsCtrlDetect, scienceNoiseLevel, templat
 templateBorderSize=20, doApplyCalibration=True)
 task = self._setup_subtraction(mode="convolveScience")
 output = task.run(template, science, sources)
 self.assertFloatsAlmostEqual(task.metadata["scaleTemplateVarianceFactor"], 1., atol=.05)
 self.assertFloatsAlmostEqual(task.metadata["scaleScienceVarianceFactor"], 1., atol=.05)
 # Mean of difference image should be close to zero.
 nGoodPix = np.sum(np.isfinite(output.difference.image.array))
Expand DownExpand Up@@ -385,7 +384,6 @@ def _run_and_check_images(statsCtrl, statsCtrlDetect, scienceNoiseLevel, templat
 templateBorderSize=20, doApplyCalibration=True)
 task = self._setup_subtraction()
 output = task.run(template, science, sources)
 self.assertFloatsAlmostEqual(task.metadata["scaleTemplateVarianceFactor"], 1., atol=.05)
 self.assertFloatsAlmostEqual(task.metadata["scaleScienceVarianceFactor"], 1., atol=.05)
 # There should be no NaNs in the image if we convolve the template with a buffer
 self.assertTrue(np.all(np.isfinite(output.difference.image.array)))
Expand DownExpand Up@@ -632,8 +630,6 @@ def _run_and_check_images(science, template, sources, statsCtrl,
 )
 output = task.run(template.clone(), science.clone(), sources)
 if doScaleVariance:
 self.assertFloatsAlmostEqual(task.metadata["scaleTemplateVarianceFactor"],
 scaleFactor, atol=0.05)
 self.assertFloatsAlmostEqual(task.metadata["scaleScienceVarianceFactor"],
 scaleFactor, atol=0.05)

Expand All@@ -646,7 +642,8 @@ def _run_and_check_images(science, template, sources, statsCtrl,
 statsCtrl)

 if doScaleVariance:
 templateNoise *= scaleFactor
 # Only the science variance is scaled here. The template
 # variance is scaled independently in ``GetTemplateTask``.
 scienceNoise *= scaleFactor
 varMean = computeRobustStatistics(output.difference.variance, output.difference.mask, statsCtrl)
 self.assertFloatsAlmostEqual(varMean, scienceNoise + templateNoise, rtol=0.1)
Expand All@@ -666,8 +663,7 @@ def _run_and_check_images(science, template, sources, statsCtrl,
 doDecorrelation=False, doScaleVariance=False)

 # Verify that the variance plane of the difference image is correct
 # when the template variance plane is incorrect
 template.variance.array /= scaleFactor
 # when the input science variance plane is incorrect
 science.variance.array /= scaleFactor
 _run_and_check_images(science, template, sources, statsCtrl,
 doDecorrelation=True, doScaleVariance=True, scaleFactor=scaleFactor)
Expand DownExpand Up@@ -700,8 +696,6 @@ def _run_and_check_images(science, template, sources, statsCtrl,
 )
 output = task.run(template.clone(), science.clone(), sources)
 if doScaleVariance:
 self.assertFloatsAlmostEqual(task.metadata["scaleTemplateVarianceFactor"],
 scaleFactor, atol=0.05)
 self.assertFloatsAlmostEqual(task.metadata["scaleScienceVarianceFactor"],
 scaleFactor, atol=0.05)

Expand All@@ -714,7 +708,8 @@ def _run_and_check_images(science, template, sources, statsCtrl,
 statsCtrl)

 if doScaleVariance:
 templateNoise *= scaleFactor
 # Only the science variance is scaled here. The template
 # variance is scaled independently in ``GetTemplateTask``.
 scienceNoise *= scaleFactor

 varMean = computeRobustStatistics(output.difference.variance, output.difference.mask, statsCtrl)
Expand All@@ -735,9 +730,8 @@ def _run_and_check_images(science, template, sources, statsCtrl,
 doDecorrelation=False, doScaleVariance=False)

 # Verify that the variance plane of the difference image is correct
 # when the template and science variance planes are incorrect
 # when the input science variance plane is incorrect
 science.variance.array /= scaleFactor
 template.variance.array /= scaleFactor
 _run_and_check_images(science, template, sources, statsCtrl,
 doDecorrelation=True, doScaleVariance=True, scaleFactor=scaleFactor)
 _run_and_check_images(science, template, sources, statsCtrl,
Expand DownExpand Up@@ -1259,8 +1253,6 @@ def _run_and_check_images(science, template, sources, statsCtrl,
 )
 output = task.run(template.clone(), science.clone(), sources)
 if doScaleVariance:
 self.assertFloatsAlmostEqual(task.metadata["scaleTemplateVarianceFactor"],
 scaleFactor, atol=0.05)
 self.assertFloatsAlmostEqual(task.metadata["scaleScienceVarianceFactor"],
 scaleFactor, atol=0.05)

Expand All@@ -1278,7 +1270,8 @@ def _run_and_check_images(science, template, sources, statsCtrl,
 output.matchedTemplate.mask,
 statsCtrl)
 if doScaleVariance:
 templateNoise *= scaleFactor
 # Only the science variance is scaled here. The template
 # variance is scaled independently in ``GetTemplateTask``.
 scienceNoise *= scaleFactor
 varMean = computeRobustStatistics(output.scoreExposure.variance,
 output.scoreExposure.mask,
Expand All@@ -1302,8 +1295,7 @@ def _run_and_check_images(science, template, sources, statsCtrl,
 doDecorrelation=False, doScaleVariance=False)

 # Verify that the variance plane of the Score image is correct
 # when the template variance plane is incorrect
 template.variance.array /= scaleFactor
 # when the input science variance plane is incorrect
 science.variance.array /= scaleFactor
 _run_and_check_images(science, template, sources, statsCtrl,
 doDecorrelation=True, doScaleVariance=True, scaleFactor=scaleFactor)
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Universal Dark Mode - works on any site
(function() {
var enabled = true;
function applyDarkMode() {
if (!enabled) return;
// Create style element if it doesn't exist
var style = document.getElementById('universal-dark-mode-style');
if (!style) {
style = document.createElement('style');
style.id = 'universal-dark-mode-style';
document.head.appendChild(style);
}
// Dark mode CSS - inverts colors but preserves images/video
style.textContent = '
/* Invert everything except media */
html {
filter: invert(1) hue-rotate(180deg) !important;
background: #1a1a2e !important;
}
/* Restore images, videos, iframes, canvas */
img, video, iframe, canvas, svg, picture, [style*="background-image"] {
filter: invert(1) hue-rotate(180deg) !important;
}
/* Preserve specific elements that should not be inverted */
.no-dark-mode, .no-dark-mode *,
[data-theme="light"], [data-theme="light"],
.ace_editor, .ace_editor *,
.CodeMirror, .CodeMirror *,
.monaco-editor, .monaco-editor *,
.markdown-body pre, .markdown-body pre *,
.highlight, .highlight *,
pre code, pre code * {
filter: none !important;
}
/* Fix common UI elements */
.modal, .popup, .dropdown-menu, .tooltip, .popover {
filter: invert(1) hue-rotate(180deg) !important;
background: #2d2d44 !important;
border-color: #444 !important;
}
/* Scrollbars */
::-webkit-scrollbar { background: #1a1a2e !important; }
::-webkit-scrollbar-thumb { background: #444 !important; }
::-webkit-scrollbar-thumb:hover { background: #555 !important; }
/* Selection */
::selection { background: #4ecdc4 !important; color: #1a1a2e !important; }
::-moz-selection { background: #4ecdc4 !important; color: #1a1a2e !important; }
';
}
function removeDarkMode() {
var style = document.getElementById('universal-dark-mode-style');
if (style) style.remove();
}
// Toggle with Alt+Shift+D
document.addEventListener('keydown', function(e) {
if (e.altKey && e.shiftKey && e.key === 'D') {
e.preventDefault();
enabled = !enabled;
if (enabled) {
applyDarkMode();
console.log('[Universal Dark Mode] Enabled');
} else {
removeDarkMode();
console.log('[Universal Dark Mode] Disabled');
}
}
});
// Apply on load
applyDarkMode();
// Re-apply on dynamic content
var observer = new MutationObserver(function(mutations) {
if (enabled && !document.getElementById('universal-dark-mode-style')) {
applyDarkMode();
}
});
observer.observe(document.head, { childList: true });
console.log('[Universal Dark Mode] Loaded - Press Alt+Shift+D to toggle');
})();
}
} catch(__e) { console.warn('[Userscript:Universal Dark Mode]', __e); }
})();
})();
DM-55993: Move option to scale template variance by isullivan · Pull Request #469 · lsst/ip_diffim · GitHub
Skip to content
DM-55993: Move option to scale template variance#469

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Open

isullivan
wants to merge
2
commits into

base:main
Choose a base branch

from

 Open


DM-55993: Move option to scale template variance
#469
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


2 commits


Select commit
Hold shift + click to select a range
385e543

Move option to scale template variance to getTemplate.py


isullivan Aug 31, 2026
ffb7738

Add unit tests of template variance scaling


isullivan Aug 31, 2026

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


22 changes: 21 additions & 1 deletion
python/lsst/ip/diffim/getTemplate.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -34,7 +34,7 @@

from lsst.skymap import BaseSkyMap
from lsst.ip.diffim.dcrModel import DcrModel
from lsst.meas.algorithms import CoaddPsf, CoaddPsfConfig, SubtractBackgroundTask
from lsst.meas.algorithms import CoaddPsf, CoaddPsfConfig, SubtractBackgroundTask, ScaleVarianceTask
from lsst.utils.timer import timeMethod

__all__ = [
Expand DownExpand Up@@ -120,6 +120,15 @@ class GetTemplateConfig(
 doc="Minimum fraction of unmasked pixels needed to set the"
 " HIGH_VARIANCE mask plane.",
 )
 doScaleVariance = pexConfig.Field(
 dtype=bool,
 default=True,
 doc="Scale variance of the template image?"
 )
 scaleVariance = pexConfig.ConfigurableField(
 target=ScaleVarianceTask,
 doc="Subtask to rescale the variance of the template to the statistically expected level."
 )

 def setDefaults(self):
 # Use a smaller cache: per SeparableKernel.computeCache, this should
Expand DownExpand Up@@ -152,6 +161,8 @@ class GetTemplateTask(pipeBase.PipelineTask):

 def __init__(self, *args, **kwargs):
 super().__init__(*args, **kwargs)
 if self.config.doScaleVariance:
 self.makeSubtask("scaleVariance")
 self.warper = afwMath.Warper.fromConfig(self.config.warp)
 self.schema = afwTable.ExposureTable.makeMinimalSchema()
 self.schema.addField(
Expand DownExpand Up@@ -381,6 +392,14 @@ def run(self, *, coaddExposureHandles, bbox, wcs, dataIds, physical_filter, visi
 if count == 0:
 raise pipeBase.NoWorkFound("No valid pixels in warped template.")

 if self.config.doScaleVariance:
 # Scale the variance of the template image before subtraction, if
 # needed. Note that the science variance is scaled
 # independently in ``AlardLuptonSubtractTask``.
 varianceFactor = self.scaleVariance.run(template.maskedImage)
 self.log.info("Template variance scaling factor: %.2f", varianceFactor)
 self.metadata["scaleTemplateVarianceFactor"] = varianceFactor

 # Make a single catalog containing all the inputs that were accepted.
 catalog = afwTable.ExposureCatalog(self.schema)
 catalog.reserve(sum([len(c) for c in catalogs]))
Expand All@@ -394,6 +413,7 @@ def run(self, *, coaddExposureHandles, bbox, wcs, dataIds, physical_filter, visi
 template.setFilter(afwImage.FilterLabel(band, physical_filter))
 template.setPhotoCalib(photoCalib)
 template.setPsf(self._makePsf(template, catalog, wcs))

@khernerkhernerAug 31, 2026

Copy link


Copy Markdown

Contributor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Was this blank line intentional?
@isullivanisullivanAug 31, 2026

Copy link


Copy Markdown

ContributorAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It was an artifact of adding lines there originally, then moving them earlier. But, I liked the visual space so I left it in. I can remove it if you prefer!
 # Record the input coadd patches as the template's coadd inputs.
 coaddInputs = afwImage.CoaddInputs(afwTable.ExposureTable.makeMinimalSchema(), self.schema)
 coaddInputs.ccds.extend(catalog, deep=True)
Expand Down

13 changes: 6 additions & 7 deletions
python/lsst/ip/diffim/subtractImages.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -207,7 +207,8 @@ class AlardLuptonSubtractBaseConfig(lsst.pex.config.Config):
 doScaleVariance = lsst.pex.config.Field(
 dtype=bool,
 default=True,
 doc="Scale variance of the image difference?"
 doc="Scale variance of the science image? Note that the template variance is NOT scaled"
 " here. The template variance may be scaled independently in ``GetTemplateTask``."
 )
 scaleVariance = lsst.pex.config.ConfigurableField(
 target=ScaleVarianceTask,
Expand DownExpand Up@@ -1044,13 +1045,11 @@ def _prepareInputs(self, template, science, visitSummary=None):
 self.metadata["templateCoveragePercent"] = 100*templateCoverageFraction

 if self.config.doScaleVariance:
 # Scale the variance of the template and science images before
 # convolution, subtraction, or decorrelation so that they have the
 # correct ratio.
templateVarFactor = self.scaleVariance.run(template.maskedImage)
 # Scale the variance of the science image before
 # convolution, subtraction, or decorrelation so that it has the
 # correct ratio. Note that the template variance is scaled
# independently in ``GetTemplateTask``.
 sciVarFactor = self.scaleVariance.run(science.maskedImage)
 self.log.info("Template variance scaling factor: %.2f", templateVarFactor)
 self.metadata["scaleTemplateVarianceFactor"] = templateVarFactor
 self.log.info("Science variance scaling factor: %.2f", sciVarFactor)
 self.metadata["scaleScienceVarianceFactor"] = sciVarFactor

Expand Down

88 changes: 88 additions & 0 deletions
tests/test_getTemplate.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -337,6 +337,94 @@ def testNanInputs(self, box=None, nInput=None):
 # in the template are closer to the original anymore.
 self.assertTrue(np.isfinite(result.template.image.array).all())

 def _scaleInputVariance(self, tract, factor):
 """Return fresh handles for one tract's patches, with their variance
 planes multiplied by ``factor``.

 Parameters
 ----------
 tract : `int`
 Id of the tract whose patches should be copied.
 factor : `float`
 Factor to multiply the input variance planes by.

 Returns
 -------
 handles : `list` [`lsst.pipe.base.InMemoryDatasetHandle`]
 Handles to the modified patches.
 """
 handles = []
 for handle in self.patches[tract]:
 # ``copy=True`` on the original handles means this is a copy, so
 # the patches shared with the other tests are left untouched.
 patch = handle.get()
 patch.variance.array *= factor
 handles.append(pipeBase.InMemoryDatasetHandle(patch,
 storageClass="ExposureF",
 copy=True,
 dataId=handle.dataId))
 return handles

 def testScaleVariance(self):
 """Test that the template variance plane is rescaled to match the
 empirical pixel noise, and that the factor used is recorded in the
 task metadata.
 """
 scaleFactor = 1.345
 box = lsst.geom.Box2I(lsst.geom.Point2I(0, 0), lsst.geom.Point2I(180, 180))

 def _configureAndRunTask(doScaleVariance, varianceScale=1.):
 """Build a template from tract 0, optionally rescaling the input
 variance planes by ``varianceScale`` first.
 """
 config = lsst.ip.diffim.GetTemplateTask.ConfigClass()
 config.doScaleVariance = doScaleVariance
 task = lsst.ip.diffim.GetTemplateTask(config=config)
 # Task modifies the input bbox, so pass a copy.
 result = task.run(coaddExposureHandles={0: self._scaleInputVariance(0, varianceScale)},
 bbox=lsst.geom.Box2I(box),
 wcs=self.exposure.wcs,
 dataIds={0: self.dataIds[0]},
 physical_filter="a_test")
 return task, result.template

 # With scaling disabled the subtask is never constructed, and nothing
 # is recorded in the metadata.
 taskOff, templateOff = _configureAndRunTask(False)
 self.assertFalse(hasattr(taskOff, "scaleVariance"))
 self.assertNotIn("scaleTemplateVarianceFactor", taskOff.metadata)

 # Both warps -- lanczos5 in ``_makePatches`` and lanczos3 in the
 # task -- correlate the noise. The variance plane tracks only the
 # per-pixel diagonal, which the second warp leaves too low, so
 # ``scaleVariance`` measures a factor well above 1 even though the
 # input variance planes are correct.
 #
 taskOn, templateOn = _configureAndRunTask(True)
 factor = taskOn.metadata["scaleTemplateVarianceFactor"]
 # TODO DM-55879: this value is pinned on purpose. The lanczos warping
 # kernels introduce small correlations that artificially suppress the
 # image pixel stddev and inflate the variance scaling factor. This
 # should be changed to 1.0 after DM-55879 is merged.
 self.assertFloatsAlmostEqual(factor, 1.1465, atol=0.01,
 msg="Measured template variance scaling changed; see the"
 " comment above if the correlation correction landed.")
 # The only difference from the unscaled template is the constant
 # factor applied to the variance plane.
 self.assertFloatsAlmostEqual(templateOn.variance.array,
 templateOff.variance.array*factor, rtol=1e-5)
 # Tolerance here is float32 round-off: repeated runs of the task are
 # not bitwise identical.
 self.assertImagesAlmostEqual(templateOn.image, templateOff.image, rtol=1e-5, atol=1e-5)

 # If the input variance planes under-estimate the noise by a known
 # factor, the measured factor grows by that amount and the same
 # output variance plane is recovered.
 taskLow, templateLow = _configureAndRunTask(True, varianceScale=1/scaleFactor)
 self.assertFloatsAlmostEqual(taskLow.metadata["scaleTemplateVarianceFactor"],
 factor*scaleFactor, rtol=1e-5)
 self.assertImagesAlmostEqual(templateLow.variance, templateOn.variance, rtol=1e-5)


def setup_module(module):
 lsst.utils.tests.init()
Expand Down

26 changes: 9 additions & 17 deletions
tests/test_subtractTask.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -349,7 +349,6 @@ def _run_and_check_images(statsCtrl, statsCtrlDetect, scienceNoiseLevel, templat
 templateBorderSize=20, doApplyCalibration=True)
 task = self._setup_subtraction(mode="convolveScience")
 output = task.run(template, science, sources)
 self.assertFloatsAlmostEqual(task.metadata["scaleTemplateVarianceFactor"], 1., atol=.05)
 self.assertFloatsAlmostEqual(task.metadata["scaleScienceVarianceFactor"], 1., atol=.05)
 # Mean of difference image should be close to zero.
 nGoodPix = np.sum(np.isfinite(output.difference.image.array))
Expand DownExpand Up@@ -385,7 +384,6 @@ def _run_and_check_images(statsCtrl, statsCtrlDetect, scienceNoiseLevel, templat
 templateBorderSize=20, doApplyCalibration=True)
 task = self._setup_subtraction()
 output = task.run(template, science, sources)
 self.assertFloatsAlmostEqual(task.metadata["scaleTemplateVarianceFactor"], 1., atol=.05)
 self.assertFloatsAlmostEqual(task.metadata["scaleScienceVarianceFactor"], 1., atol=.05)
 # There should be no NaNs in the image if we convolve the template with a buffer
 self.assertTrue(np.all(np.isfinite(output.difference.image.array)))
Expand DownExpand Up@@ -632,8 +630,6 @@ def _run_and_check_images(science, template, sources, statsCtrl,
 )
 output = task.run(template.clone(), science.clone(), sources)
 if doScaleVariance:
 self.assertFloatsAlmostEqual(task.metadata["scaleTemplateVarianceFactor"],
 scaleFactor, atol=0.05)
 self.assertFloatsAlmostEqual(task.metadata["scaleScienceVarianceFactor"],
 scaleFactor, atol=0.05)

Expand All@@ -646,7 +642,8 @@ def _run_and_check_images(science, template, sources, statsCtrl,
 statsCtrl)

 if doScaleVariance:
 templateNoise *= scaleFactor
 # Only the science variance is scaled here. The template
 # variance is scaled independently in ``GetTemplateTask``.
 scienceNoise *= scaleFactor
 varMean = computeRobustStatistics(output.difference.variance, output.difference.mask, statsCtrl)
 self.assertFloatsAlmostEqual(varMean, scienceNoise + templateNoise, rtol=0.1)
Expand All@@ -666,8 +663,7 @@ def _run_and_check_images(science, template, sources, statsCtrl,
 doDecorrelation=False, doScaleVariance=False)

 # Verify that the variance plane of the difference image is correct
 # when the template variance plane is incorrect
 template.variance.array /= scaleFactor
 # when the input science variance plane is incorrect
 science.variance.array /= scaleFactor
 _run_and_check_images(science, template, sources, statsCtrl,
 doDecorrelation=True, doScaleVariance=True, scaleFactor=scaleFactor)
Expand DownExpand Up@@ -700,8 +696,6 @@ def _run_and_check_images(science, template, sources, statsCtrl,
 )
 output = task.run(template.clone(), science.clone(), sources)
 if doScaleVariance:
 self.assertFloatsAlmostEqual(task.metadata["scaleTemplateVarianceFactor"],
 scaleFactor, atol=0.05)
 self.assertFloatsAlmostEqual(task.metadata["scaleScienceVarianceFactor"],
 scaleFactor, atol=0.05)

Expand All@@ -714,7 +708,8 @@ def _run_and_check_images(science, template, sources, statsCtrl,
 statsCtrl)

 if doScaleVariance:
 templateNoise *= scaleFactor
 # Only the science variance is scaled here. The template
 # variance is scaled independently in ``GetTemplateTask``.
 scienceNoise *= scaleFactor

 varMean = computeRobustStatistics(output.difference.variance, output.difference.mask, statsCtrl)
Expand All@@ -735,9 +730,8 @@ def _run_and_check_images(science, template, sources, statsCtrl,
 doDecorrelation=False, doScaleVariance=False)

 # Verify that the variance plane of the difference image is correct
 # when the template and science variance planes are incorrect
 # when the input science variance plane is incorrect
 science.variance.array /= scaleFactor
 template.variance.array /= scaleFactor
 _run_and_check_images(science, template, sources, statsCtrl,
 doDecorrelation=True, doScaleVariance=True, scaleFactor=scaleFactor)
 _run_and_check_images(science, template, sources, statsCtrl,
Expand DownExpand Up@@ -1259,8 +1253,6 @@ def _run_and_check_images(science, template, sources, statsCtrl,
 )
 output = task.run(template.clone(), science.clone(), sources)
 if doScaleVariance:
 self.assertFloatsAlmostEqual(task.metadata["scaleTemplateVarianceFactor"],
 scaleFactor, atol=0.05)
 self.assertFloatsAlmostEqual(task.metadata["scaleScienceVarianceFactor"],
 scaleFactor, atol=0.05)

Expand All@@ -1278,7 +1270,8 @@ def _run_and_check_images(science, template, sources, statsCtrl,
 output.matchedTemplate.mask,
 statsCtrl)
 if doScaleVariance:
 templateNoise *= scaleFactor
 # Only the science variance is scaled here. The template
 # variance is scaled independently in ``GetTemplateTask``.
 scienceNoise *= scaleFactor
 varMean = computeRobustStatistics(output.scoreExposure.variance,
 output.scoreExposure.mask,
Expand All@@ -1302,8 +1295,7 @@ def _run_and_check_images(science, template, sources, statsCtrl,
 doDecorrelation=False, doScaleVariance=False)

 # Verify that the variance plane of the Score image is correct
 # when the template variance plane is incorrect
 template.variance.array /= scaleFactor
 # when the input science variance plane is incorrect
 science.variance.array /= scaleFactor
 _run_and_check_images(science, template, sources, statsCtrl,
 doDecorrelation=True, doScaleVariance=True, scaleFactor=scaleFactor)
Expand Down
Loading