Skip to content

Framework to support the same metric limits as in the Databricks layer #498

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
operdeck merged 10 commits into from
Jan 6, 2026
Merged

Framework to support the same metric limits as in the Databricks layer #498

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
90b96a2
Framework to support the same metric limits as in the Databricks layer
operdeck Jan 3, 2026
1083bca
Consolidate CDH_Guidelines into MetricLimits with CSV-based thresholds
operdeck Jan 3, 2026
155b1fd
Supports string to bool mapping, brings itable and gt more in line
operdeck Jan 4, 2026
926a762
Number formatting consistent between both versions
operdeck Jan 4, 2026
c5522b6
Number formatting consistent between both versions
operdeck Jan 4, 2026
0787a51
Extended the number formatting, supporting even version strings and d…
operdeck Jan 4, 2026
840959f
Moved version and cloud checks out of PDS tools, should be in DSI layer
operdeck Jan 5, 2026
ebe767a
Introduced versatile number formatting class
operdeck Jan 5, 2026
330888d
Generalized handling of boolean metric limits a bit
operdeck Jan 6, 2026
416f741
Cleanup prep for merge
operdeck Jan 6, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .pre-commit-config.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ repos:
hooks:
- id: trailing-whitespace
- id: end-of-file-fixer
exclude: '\.csv$'
- id: check-yaml
- id: check-added-large-files
exclude: 'uv.lock'
Expand Down
3 changes: 2 additions & 1 deletion pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ include = ['pdstools*']
exclude = ['/tests/*', '*.dist-info', '/docs/*']

[tool.setuptools.package-data]
"*" = ['*.qmd', '*/pega-report-overrides.css', 'py.typed', '*.sql', 'GlobalExplanations/**/*']
"*" = ['*.qmd', '*/pega-report-overrides.css', 'py.typed', '*.sql', 'GlobalExplanations/**/*', 'resources/*.csv']

[project]
name = "pdstools"
Expand Down Expand Up @@ -72,6 +72,7 @@ tests = [
'openpyxl',
'pdstools[all]',
'coverage',
'itables',
]

[project.urls]
Expand Down
9 changes: 3 additions & 6 deletions python/pdstools/adm/ADMDatamart.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@
from .ADMTrees import AGB
from .Aggregates import Aggregates
from .BinAggregator import BinAggregator
from .CDH_Guidelines import CDHGuidelines
from .Plots import Plots
from .Reports import Reports

Expand Down Expand Up @@ -98,7 +97,6 @@ class ADMDatamart:
aggregates: Aggregates
agb: AGB
generate: Reports
cdh_guidelines: CDHGuidelines
bin_aggregator: BinAggregator
first_action_dates: Optional[pl.LazyFrame]

Expand All @@ -122,9 +120,6 @@ def __init__(
self.aggregates = Aggregates(datamart=self)
self.agb = AGB(datamart=self)
self.generate = Reports(datamart=self)
self.cdh_guidelines = (
CDHGuidelines()
) # not sure if this should be part of the ADM DM

model_data_validated = self._validate_model_data(
model_df, extract_pyname_keys=extract_pyname_keys
Expand All @@ -143,7 +138,9 @@ def __init__(
)
self.bin_aggregator = BinAggregator(dm=self)

def _get_first_action_dates(self, df: Optional[pl.LazyFrame]) -> Optional[pl.LazyFrame]:
def _get_first_action_dates(
self, df: Optional[pl.LazyFrame]
) -> Optional[pl.LazyFrame]:
if df is None:
return df
return (
Expand Down
17 changes: 9 additions & 8 deletions python/pdstools/adm/Aggregates.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,11 @@
import polars.selectors as cs

from ..utils import cdh_utils
from ..utils.metric_limits import (
get_standard_NBAD_channels,
is_standard_NBAD_configuration,
)
from ..utils.types import QUERY
from .CDH_Guidelines import CDHGuidelines

if TYPE_CHECKING: # pragma: no cover
from .ADMDatamart import ADMDatamart
Expand All @@ -16,7 +19,6 @@
class Aggregates:
def __init__(self, datamart: "ADMDatamart"):
self.datamart = datamart
self.cdh_guidelines = CDHGuidelines()

def last(
self,
Expand Down Expand Up @@ -312,13 +314,14 @@ def name_normalizer(x):
grouping += ["Period"]

if by_channel:
standard_channels = get_standard_NBAD_channels()
channelGroupMapping = (
pl.concat(
[
pl.DataFrame(
{
"Channel": self.cdh_guidelines.standard_channels,
"ChannelGroup": self.cdh_guidelines.standard_channels,
"Channel": standard_channels,
"ChannelGroup": standard_channels,
}
),
pl.DataFrame(
Expand Down Expand Up @@ -536,9 +539,7 @@ def _summarize_model_usage(
debug: bool,
) -> pl.LazyFrame:
result = model_data.group_by(grouping).agg(
self.cdh_guidelines.is_standard_configuration()
.any(ignore_nulls=False)
.alias("usesNBAD"),
is_standard_NBAD_configuration().any(ignore_nulls=False).alias("usesNBAD"),
(pl.col("ModelTechnique") == "GradientBoost")
.any(ignore_nulls=False)
.alias("usesAGB"),
Expand Down Expand Up @@ -757,7 +758,7 @@ def summary_by_configuration(self) -> pl.LazyFrame:
self.last(table="model_data")
.group_by(group_by_cols)
.agg(
self.cdh_guidelines.is_standard_configuration()
is_standard_NBAD_configuration()
.any(ignore_nulls=False)
.alias("usesNBAD"),
(pl.col("ModelTechnique") == "GradientBoost")
Expand Down
217 changes: 0 additions & 217 deletions python/pdstools/adm/CDH_Guidelines.py
View file
Open in desktop

This file was deleted.

Loading
Loading