Merge pull request #136 from lsst/tickets/DM-38974

DM-38974: Move photometric repeatability metrics from faro to analysis_tools

Author: jeffcarlin

.pre-commit-config.yaml

@@ -13,7 +13,7 @@ repos:
         # supported by your project here, or alternatively use
         # pre-commit's default_language_version, see
         # https://pre-commit.com/#top_level-default_language_version
-        language_version: "python3.10"
+        language_version: "python3.11"
   - repo: https://github.com/pycqa/isort
     rev: 5.12.0
     hooks:

pipelines/matchedVisitQualityCore.yaml

@@ -23,3 +23,53 @@ tasks:
       atools.stellarAstrometricRepeatability3.process.calculateActions.rms.threshAD: 30
       python: |
         from lsst.analysis.tools.atools import *
+  analyzeMatchedVisitExtended:
+    class: lsst.analysis.tools.tasks.AssociatedSourcesTractAnalysisTask
+    config:
+      connections.outputName: matchedVisitExtended
+      atools.modelPhotRepStarSn5to10: StellarPhotometricRepeatability
+      atools.modelPhotRepStarSn5to10.fluxType: gaussianFlux
+      atools.modelPhotRepStarSn5to10.process.filterActions.perGroupStdevFiltered.selectors.sn.minimum: 5
+      atools.modelPhotRepStarSn5to10.process.filterActions.perGroupStdevFiltered.selectors.sn.maximum: 10
+      atools.modelPhotRepStarSn5to10.produce.plot: NoPlot
+
+      atools.modelPhotRepStarSn10to20: StellarPhotometricRepeatability
+      atools.modelPhotRepStarSn10to20.fluxType: gaussianFlux
+      atools.modelPhotRepStarSn10to20.process.filterActions.perGroupStdevFiltered.selectors.sn.minimum: 10
+      atools.modelPhotRepStarSn10to20.process.filterActions.perGroupStdevFiltered.selectors.sn.maximum: 20
+      atools.modelPhotRepStarSn10to20.produce.plot: NoPlot
+
+      atools.modelPhotRepStarSn20to40: StellarPhotometricRepeatability
+      atools.modelPhotRepStarSn20to40.fluxType: gaussianFlux
+      atools.modelPhotRepStarSn20to40.process.filterActions.perGroupStdevFiltered.selectors.sn.minimum: 20
+      atools.modelPhotRepStarSn20to40.process.filterActions.perGroupStdevFiltered.selectors.sn.maximum: 40
+      atools.modelPhotRepStarSn20to40.produce.plot: NoPlot
+
+      atools.modelPhotRepStarSn40to80: StellarPhotometricRepeatability
+      atools.modelPhotRepStarSn40to80.fluxType: gaussianFlux
+      atools.modelPhotRepStarSn40to80.process.filterActions.perGroupStdevFiltered.selectors.sn.minimum: 40
+      atools.modelPhotRepStarSn40to80.process.filterActions.perGroupStdevFiltered.selectors.sn.maximum: 80
+      atools.modelPhotRepStarSn40to80.produce.plot: NoPlot
+
+      atools.psfPhotRepStarSn5to10: StellarPhotometricRepeatability
+      atools.psfPhotRepStarSn5to10.process.filterActions.perGroupStdevFiltered.selectors.sn.minimum: 5
+      atools.psfPhotRepStarSn5to10.process.filterActions.perGroupStdevFiltered.selectors.sn.maximum: 10
+      atools.psfPhotRepStarSn5to10.produce.plot: NoPlot
+
+      atools.psfPhotRepStarSn10to20: StellarPhotometricRepeatability
+      atools.psfPhotRepStarSn10to20.process.filterActions.perGroupStdevFiltered.selectors.sn.minimum: 10
+      atools.psfPhotRepStarSn10to20.process.filterActions.perGroupStdevFiltered.selectors.sn.maximum: 20
+      atools.psfPhotRepStarSn10to20.produce.plot: NoPlot
+
+      atools.psfPhotRepStarSn20to40: StellarPhotometricRepeatability
+      atools.psfPhotRepStarSn20to40.process.filterActions.perGroupStdevFiltered.selectors.sn.minimum: 20
+      atools.psfPhotRepStarSn20to40.process.filterActions.perGroupStdevFiltered.selectors.sn.maximum: 40
+      atools.psfPhotRepStarSn20to40.produce.plot: NoPlot
+
+      atools.psfPhotRepStarSn40to80: StellarPhotometricRepeatability
+      atools.psfPhotRepStarSn40to80.process.filterActions.perGroupStdevFiltered.selectors.sn.minimum: 40
+      atools.psfPhotRepStarSn40to80.process.filterActions.perGroupStdevFiltered.selectors.sn.maximum: 80
+      atools.psfPhotRepStarSn40to80.produce.plot: NoPlot
+      python: |
+        from lsst.analysis.tools.atools import *
+        from lsst.analysis.tools.interfaces import *

python/lsst/analysis/tools/actions/vector/calcBinnedStats.py

@@ -105,7 +105,7 @@ def __call__(self, data: KeyedData, **kwargs) -> KeyedData:
         for name, value in action(data, **kwargs).items():
             results[getattr(self, f"name_{name}").format(**kwargs_format)] = value
 
-        values = cast(Vector, data[self.selector_range.key][mask])  # type: ignore
+        values = cast(Vector, data[self.selector_range.vectorKey][mask])  # type: ignore
         valid = np.sum(np.isfinite(values)) > 0
 
         results[self.name_select_maximum.format(**kwargs_format)] = cast(

python/lsst/analysis/tools/actions/vector/selectors.py

@@ -187,12 +187,12 @@ def setDefaults(self):
 class RangeSelector(VectorAction):
     """Selects rows within a range, inclusive of min/exclusive of max."""
 
-    key = Field[str](doc="Key to select from data")
+    vectorKey = Field[str](doc="Key to select from data")
     maximum = Field[float](doc="The maximum value", default=np.Inf)
     minimum = Field[float](doc="The minimum value", default=np.nextafter(-np.Inf, 0.0))
 
     def getInputSchema(self) -> KeyedDataSchema:
-        yield self.key, Vector
+        yield self.vectorKey, Vector
 
     def __call__(self, data: KeyedData, **kwargs) -> Vector:
         """Return a mask of rows with values within the specified range.
@@ -206,10 +206,10 @@ def __call__(self, data: KeyedData, **kwargs) -> Vector:
         result : `Vector`
             A mask of the rows with values within the specified range.
         """
-        values = cast(Vector, data[self.key])
+        values = cast(Vector, data[self.vectorKey])
         mask = (values >= self.minimum) & (values < self.maximum)
 
-        return np.array(mask)
+        return cast(Vector, mask)
 
 
 class SnSelector(SelectorBase):

python/lsst/analysis/tools/atools/astrometricRepeatability.py

@@ -231,7 +231,9 @@ def setDefaults(self):
 
         # Select only sources with magnitude between 17 and 21.5
         self.process.filterActions.coord_ra = DownselectVector(vectorKey="coord_ra")
-        self.process.filterActions.coord_ra.selector = RangeSelector(key="mags", minimum=17, maximum=21.5)
+        self.process.filterActions.coord_ra.selector = RangeSelector(
+            vectorKey="mags", minimum=17, maximum=21.5
+        )
         self.process.filterActions.coord_dec = DownselectVector(
             vectorKey="coord_dec", selector=self.process.filterActions.coord_ra.selector
         )

python/lsst/analysis/tools/atools/diffMatched.py

@@ -146,7 +146,7 @@ def configureMetrics(
             for minimum in range(self._mag_low_min, self._mag_low_max + 1):
                 action = getattr(self.process.calculateActions, f"{name}{minimum}")
                 action.selector_range = RangeSelector(
-                    key=x_key,
+                    vectorKey=x_key,
                     minimum=minimum,
                     maximum=minimum + self._mag_interval,
                 )
@@ -180,7 +180,7 @@ def setDefaults(self):
                     CalcBinnedStatsAction(
                         key_vector=key,
                         selector_range=RangeSelector(
-                            key=key,
+                            vectorKey=key,
                             minimum=minimum,
                             maximum=minimum + self._mag_interval,
                         ),

python/lsst/analysis/tools/atools/photometricRepeatability.py

@@ -36,6 +36,7 @@
     LoadVector,
     MultiCriteriaDownselectVector,
     PerGroupStatistic,
+    RangeSelector,
     ResidualWithPerGroupStatistic,
     SnSelector,
     ThresholdSelector,
@@ -93,10 +94,9 @@ def setDefaults(self):
             op="ge",
             threshold=3,
         )
-        self.process.filterActions.perGroupStdevFiltered.selectors.sn = ThresholdSelector(
+        self.process.filterActions.perGroupStdevFiltered.selectors.sn = RangeSelector(
             vectorKey="perGroupSn",
-            op="ge",
-            threshold=200,
+            minimum=200,
         )
         self.process.filterActions.perGroupStdevFiltered.selectors.extendedness = ThresholdSelector(
             vectorKey="perGroupExtendedness",

tests/test_actions.py

@@ -145,7 +145,7 @@ def _checkSchema(self, action, truth):
     # VectorActions with their own files
 
     def testCalcBinnedStats(self):
-        selector = RangeSelector(key="r_vector", minimum=0, maximum=self.size)
+        selector = RangeSelector(vectorKey="r_vector", minimum=0, maximum=self.size)
         prefix = "a_"
         stats = CalcBinnedStatsAction(name_prefix=prefix, selector_range=selector, key_vector="r_vector")
         result = stats(self.data)
@@ -489,7 +489,7 @@ def testCoaddPlotFlagSelector(self):
         np.testing.assert_array_equal(result, truth)
 
     def testRangeSelector(self):
-        selector = RangeSelector(key="r_psfFlux", minimum=np.nextafter(20, 30), maximum=50)
+        selector = RangeSelector(vectorKey="r_psfFlux", minimum=np.nextafter(20, 30), maximum=50)
         self._checkSchema(selector, ["r_psfFlux"])
         result = self.data["r_psfFlux"][selector(self.data)]
         truth = [30, 40]