Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions docs/_quarto.yml
Original file line number Diff line number Diff line change
Expand Up @@ -157,6 +157,8 @@ quartodoc:
- name: Validate.col_vals_outside
- name: Validate.col_vals_in_set
- name: Validate.col_vals_not_in_set
- name: Validate.col_vals_increasing
- name: Validate.col_vals_decreasing
- name: Validate.col_vals_null
- name: Validate.col_vals_not_null
- name: Validate.col_vals_regex
Expand Down
28 changes: 28 additions & 0 deletions pointblank/_constants.py
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,8 @@
"col_vals_not_in_set": "not_in_set",
"col_vals_regex": "regex",
"col_vals_within_spec": "within_spec",
"col_vals_increasing": "increasing",
"col_vals_decreasing": "decreasing",
"col_vals_null": "null",
"col_vals_not_null": "not_null",
"col_vals_expr": "expr",
Expand Down Expand Up @@ -81,6 +83,8 @@
"col_vals_not_in_set",
"col_vals_regex",
"col_vals_within_spec",
"col_vals_increasing",
"col_vals_decreasing",
"col_vals_null",
"col_vals_not_null",
"col_vals_expr",
Expand Down Expand Up @@ -310,6 +314,30 @@
<polygon id="line_white" fill="#FFFFFF" transform="translate(34.661982, 31.674853) rotate(-320.000000) translate(-34.661982, -31.674853) " points="34.1619821 8.063154 35.1619821 8.063154 35.1619821 55.2865512 34.1619821 55.2865512"></polygon>
</g>
</g>
</svg>""",
"col_vals_increasing": """<?xml version="1.0" encoding="UTF-8"?>
<svg width="67px" height="67px" viewBox="0 0 67 67" version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
<title>col_vals_increasing</title>
<g id="All-Icons" stroke="none" stroke-width="1" fill="none" fill-rule="evenodd">
<g id="col_vals_increasing" transform="translate(0.000000, 0.103448)">
<path d="M56.712234,1 C59.1975153,1 61.4475153,2.00735931 63.076195,3.63603897 C64.7048747,5.26471863 65.712234,7.51471863 65.712234,10 L65.712234,10 L65.712234,65 L10.712234,65 C8.22695259,65 5.97695259,63.9926407 4.34827294,62.363961 C2.71959328,60.7352814 1.71223397,58.4852814 1.71223397,56 L1.71223397,56 L1.71223397,10 C1.71223397,7.51471863 2.71959328,5.26471863 4.34827294,3.63603897 C5.97695259,2.00735931 8.22695259,1 10.712234,1 L10.712234,1 Z" id="rectangle" stroke="#000000" stroke-width="2" fill="#FFFFFF"></path>
<g id="increasing" transform="translate(13.712234, 13.000000)" fill="#000000" fill-rule="nonzero">
<path d="M31.2,0 L31.2,1.6 L37.25,1.6 L27.2,11.65 L24.575,9.025 L24,8.475 L23.425,9.025 L14.4,18.05 L11.775,15.425 L11.2,14.875 L10.625,15.425 L0.225,25.825 L1.375,26.975 L11.2,17.15 L13.825,19.775 L14.4,20.325 L14.975,19.775 L24,10.75 L26.625,13.375 L27.2,13.925 L27.775,13.375 L38.4,2.75 L38.4,8.8 L40,8.8 L40,0 L31.2,0 Z M33.6,11.2 L33.6,40 L35.2,40 L35.2,11.2 L33.6,11.2 Z M38.4,12 L38.4,40 L40,40 L40,12 L38.4,12 Z M24,16 L24,40 L25.6,40 L25.6,16 L24,16 Z M28.8,16 L28.8,40 L30.4,40 L30.4,16 L28.8,16 Z M19.2,19.2 L19.2,40 L20.8,40 L20.8,19.2 L19.2,19.2 Z M9.6,22.4 L9.6,40 L11.2,40 L11.2,22.4 L9.6,22.4 Z M14.4,24 L14.4,40 L16,40 L16,24 L14.4,24 Z M4.8,27.2 L4.8,40 L6.4,40 L6.4,27.2 L4.8,27.2 Z M0,30.4 L0,40 L1.6,40 L1.6,30.4 L0,30.4 Z" id="Shape"></path>
</g>
</g>
</g>
</svg>""",
"col_vals_decreasing": """<?xml version="1.0" encoding="UTF-8"?>
<svg width="67px" height="67px" viewBox="0 0 67 67" version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
<title>col_vals_decreasing</title>
<g id="All-Icons" stroke="none" stroke-width="1" fill="none" fill-rule="evenodd">
<g id="col_vals_decreasing" transform="translate(0.000000, 0.586207)">
<path d="M56.712234,1 C59.1975153,1 61.4475153,2.00735931 63.076195,3.63603897 C64.7048747,5.26471863 65.712234,7.51471863 65.712234,10 L65.712234,10 L65.712234,65 L10.712234,65 C8.22695259,65 5.97695259,63.9926407 4.34827294,62.363961 C2.71959328,60.7352814 1.71223397,58.4852814 1.71223397,56 L1.71223397,56 L1.71223397,10 C1.71223397,7.51471863 2.71959328,5.26471863 4.34827294,3.63603897 C5.97695259,2.00735931 8.22695259,1 10.712234,1 L10.712234,1 Z" id="rectangle" stroke="#000000" stroke-width="2" fill="#FFFFFF"></path>
<g id="decreasing" transform="translate(13.712234, 12.500000)" fill="#000000" fill-rule="nonzero">
<path d="M1.375,0.225 L0.225,1.375 L12.225,13.375 L12.8,13.925 L13.375,13.375 L16,10.75 L25.025,19.775 L25.6,20.325 L26.175,19.775 L28.8,17.15 L37.25,25.6 L32,25.6 L32,27.2 L40,27.2 L40,18.4 L38.4,18.4 L38.4,24.45 L29.375,15.425 L28.8,14.875 L28.225,15.425 L25.6,18.05 L16.575,9.025 L16,8.475 L15.425,9.025 L12.8,11.65 L1.375,0.225 Z M0,6.4 L0,40 L1.6,40 L1.6,6.4 L0,6.4 Z M4.8,11.2 L4.8,40 L6.4,40 L6.4,11.2 L4.8,11.2 Z M9.6,16 L9.6,40 L11.2,40 L11.2,16 L9.6,16 Z M14.4,16 L14.4,40 L16,40 L16,16 L14.4,16 Z M19.2,19.2 L19.2,40 L20.8,40 L20.8,19.2 L19.2,19.2 Z M28.8,22.4 L28.8,40 L30.4,40 L30.4,22.4 L28.8,22.4 Z M24,24 L24,40 L25.6,40 L25.6,24 L24,24 Z M33.6,30.4 L33.6,40 L35.2,40 L35.2,30.4 L33.6,30.4 Z M38.4,30.4 L38.4,40 L40,40 L40,30.4 L38.4,30.4 Z" id="Shape"></path>
</g>
</g>
</g>
</svg>""",
"col_vals_null": """<?xml version="1.0" encoding="UTF-8"?>
<svg width="67px" height="67px" viewBox="0 0 67 67" version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
Expand Down
116 changes: 116 additions & 0 deletions pointblank/_interrogation.py
Original file line number Diff line number Diff line change
Expand Up @@ -2197,6 +2197,122 @@ def interrogate_not_null(tbl: FrameT, column: str) -> FrameT:
return result_tbl.to_native()


def interrogate_increasing(
tbl: FrameT, column: str, allow_stationary: bool, decreasing_tol: float, na_pass: bool
) -> FrameT:
"""
Increasing interrogation.

Checks whether column values are increasing row by row.

Parameters
----------
tbl
The table to interrogate.
column
The column to check.
allow_stationary
Whether to allow consecutive equal values (stationary phases).
decreasing_tol
Optional tolerance for negative movement (decreasing values).
na_pass
Whether NA/null values should be considered as passing.

Returns
-------
FrameT
The table with a `pb_is_good_` column indicating pass/fail for each row.
"""
nw_tbl = nw.from_native(tbl)

# Create a lagged difference column
result_tbl = nw_tbl.with_columns(pb_lagged_difference_=nw.col(column) - nw.col(column).shift(1))

# Build the condition based on allow_stationary and decreasing_tol
if allow_stationary or decreasing_tol != 0:
# Allow stationary (diff >= 0) or within tolerance
threshold = -abs(decreasing_tol) if decreasing_tol != 0 else 0
good_condition = nw.col("pb_lagged_difference_") >= threshold
else:
# Strictly increasing (diff > 0)
good_condition = nw.col("pb_lagged_difference_") > 0

# Apply the validation logic
# The logic is:
# 1. If lagged_diff is null AND current value is NOT null -> pass (first row or after NA)
# 2. If current value is null -> apply na_pass
# 3. Otherwise -> apply the good_condition
result_tbl = result_tbl.with_columns(
pb_is_good_=nw.when(nw.col("pb_lagged_difference_").is_null() & ~nw.col(column).is_null())
.then(nw.lit(True)) # First row or row after NA (can't validate)
.otherwise(
nw.when(nw.col(column).is_null())
.then(nw.lit(na_pass)) # Handle NA values in current row
.otherwise(good_condition)
)
)

return result_tbl.drop("pb_lagged_difference_").to_native()


def interrogate_decreasing(
tbl: FrameT, column: str, allow_stationary: bool, increasing_tol: float, na_pass: bool
) -> FrameT:
"""
Decreasing interrogation.

Checks whether column values are decreasing row by row.

Parameters
----------
tbl
The table to interrogate.
column
The column to check.
allow_stationary
Whether to allow consecutive equal values (stationary phases).
increasing_tol
Optional tolerance for positive movement (increasing values).
na_pass
Whether NA/null values should be considered as passing.

Returns
-------
FrameT
The table with a `pb_is_good_` column indicating pass/fail for each row.
"""
nw_tbl = nw.from_native(tbl)

# Create a lagged difference column
result_tbl = nw_tbl.with_columns(pb_lagged_difference_=nw.col(column) - nw.col(column).shift(1))

# Build the condition based on allow_stationary and increasing_tol
if allow_stationary or increasing_tol != 0:
# Allow stationary (diff <= 0) or within tolerance
threshold = abs(increasing_tol) if increasing_tol != 0 else 0
good_condition = nw.col("pb_lagged_difference_") <= threshold
else:
# Strictly decreasing (diff < 0)
good_condition = nw.col("pb_lagged_difference_") < 0

# Apply the validation logic
# The logic is:
# 1. If lagged_diff is null AND current value is NOT null -> pass (first row or after NA)
# 2. If current value is null -> apply na_pass
# 3. Otherwise -> apply the good_condition
result_tbl = result_tbl.with_columns(
pb_is_good_=nw.when(nw.col("pb_lagged_difference_").is_null() & ~nw.col(column).is_null())
.then(nw.lit(True)) # First row or row after NA (can't validate)
.otherwise(
nw.when(nw.col(column).is_null())
.then(nw.lit(na_pass)) # Handle NA values in current row
.otherwise(good_condition)
)
)

return result_tbl.drop("pb_lagged_difference_").to_native()


def _interrogate_comparison_base(
tbl: FrameT, column: str, compare: any, na_pass: bool, operator: str
) -> FrameT:
Expand Down
3 changes: 3 additions & 0 deletions pointblank/_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -670,9 +670,12 @@ def _get_api_text() -> str:
"Validate.col_vals_outside",
"Validate.col_vals_in_set",
"Validate.col_vals_not_in_set",
"Validate.col_vals_increasing",
"Validate.col_vals_decreasing",
"Validate.col_vals_null",
"Validate.col_vals_not_null",
"Validate.col_vals_regex",
"Validate.col_vals_within_spec",
"Validate.col_vals_expr",
"Validate.rows_distinct",
"Validate.rows_complete",
Expand Down
Loading
Loading