Adding front_coding window function

Author: Yomguithereal

CHANGELOG.md

@@ -1,5 +1,11 @@
 # Changelog
 
+## 0.57.0 (provisional)
+
+*Features*
+
+* Adding `front_coding` window function.
+
 ## 0.56.0
 
 *Features*

docs/moonblade/window.md

@@ -6,6 +6,7 @@
 - **cumsum**(*\<expr\>*) -> `number`: Returns the cumulative sum of the numbers yielded by given expression.
 - **dense_rank**(*\<expr\>*) -> `number`: Returns the dense rank (there will be no gaps, but ties remain possible for a same rank) of numbers yielded by given expression. This requires buffering whole file or group!
 - **frac**(*\<expr\>*, *decimals?*) -> `number`: Returns the fraction represented by numbers yielded by given expression over the total sum of them. This requires buffering whole file or group!
+- **front_coding**(*\<expr\>*) -> `string`: Compress string returned by given expression using incremental encoding (useful to compress sorted data).<br>See https://en.wikipedia.org/wiki/Incremental_encoding
 - **lag**(*\<expr\>*, *steps?*, *\<expr\>?*) -> `any`: Returns a value yielded by given expression, lagged by n steps or 1 step by default. Can take a second expression after the number of steps to return a default value for rows that come before first lagged value.
 - **lead**(*\<expr\>*, *steps?*, *\<expr\>?*) -> `any`: Returns a value yielded by given expression, leading by n steps or 1 step by default. Can take a second expression after the number of steps to return a default value for rows that come after last lead value.
 - **ntile**(*k*, *\<expr\>*) -> `number`: Splits numbers yielded by given expression into `k` nearly equal-sized consecutive groups. This requires buffering whole file or group!

src/moonblade/agg/window.rs

@@ -159,6 +159,7 @@ impl Ranking {
 enum ConcreteWindowAggregation {
     Lead(ConcreteExpr, usize, ConcreteExpr),
     Lag(ConcreteExpr, usize, ConcreteExpr),
+    FrontCoding(ConcreteExpr, Option<String>),
     RowNumber(usize),
     RowIndex(usize),
     CumulativeSum(ConcreteExpr, Sum),
@@ -389,6 +390,43 @@ impl ConcreteWindowAggregation {
 
                 Ok(value)
             }
+            Self::FrontCoding(expr, last_string_opt) => {
+                let value = eval_expression(expr, Some(index), record, headers_index)?;
+                let string = value
+                    .try_as_str()
+                    .map_err(|err| err.anonymous())?
+                    .into_owned();
+
+                fn first_mismatch_index(a: &str, b: &str) -> usize {
+                    for (i, (ca, cb)) in a.chars().zip(b.chars()).enumerate() {
+                        if i == b.len() || ca != cb {
+                            return i;
+                        }
+                    }
+
+                    return a.len();
+                }
+
+                // TODO: try_into_string
+                match last_string_opt {
+                    None => {
+                        let result = DynamicValue::from("0 ".to_string() + &string);
+
+                        *last_string_opt = Some(string);
+
+                        Ok(result)
+                    }
+                    Some(last_string) => {
+                        let i = first_mismatch_index(&string, &last_string);
+
+                        let result = DynamicValue::from(format!("{} ", i) + &string[i..]);
+
+                        *last_string_opt = Some(string);
+
+                        Ok(result)
+                    }
+                }
+            }
             Self::RowNumber(counter) => {
                 *counter += 1;
                 Ok(DynamicValue::from(*counter))
@@ -489,6 +527,9 @@ impl ConcreteWindowAggregation {
                 welford.clear();
             }
             Self::Lag(_, _, _) | Self::Lead(_, _, _) => (),
+            Self::FrontCoding(_, string) => {
+                *string = None;
+            }
             Self::Frac(_, sum, _) => {
                 sum.clear();
             }
@@ -508,9 +549,8 @@ fn get_function(name: &str) -> Option<FunctionArguments> {
         "row_number" | "row_index" => FunctionArguments::nullary(),
         "frac" => FunctionArguments::with_range(1..=2),
         "lag" | "lead" => FunctionArguments::with_range(1..=3),
-        "cumsum" | "cummin" | "cummax" | "dense_rank" | "rank" | "cume_dist" | "percent_rank" => {
-            FunctionArguments::unary()
-        }
+        "cumsum" | "cummin" | "cummax" | "dense_rank" | "rank" | "cume_dist" | "percent_rank"
+        | "front_coding" => FunctionArguments::unary(),
         "rolling_sum" | "rolling_mean" | "rolling_avg" | "rolling_var" | "rolling_stddev"
         | "ntile" => FunctionArguments::binary(),
         _ => return None,
@@ -605,7 +645,7 @@ fn concretize_window_aggregations(
 
                 concrete_aggs.push((agg.agg_name, concrete_agg));
             }
-            "cumsum" | "cummin" | "cummax" => {
+            "cumsum" | "cummin" | "cummax" | "front_coding" => {
                 let expr = concretize_expression(agg.args.pop().unwrap(), headers, None)?;
 
                 concrete_aggs.push((
@@ -614,6 +654,7 @@ fn concretize_window_aggregations(
                         "cumsum" => ConcreteWindowAggregation::CumulativeSum(expr, Sum::new()),
                         "cummin" => ConcreteWindowAggregation::CumulativeMin(expr, None),
                         "cummax" => ConcreteWindowAggregation::CumulativeMax(expr, None),
+                        "front_coding" => ConcreteWindowAggregation::FrontCoding(expr, None),
                         _ => unreachable!(),
                     },
                 ))

src/moonblade/doc/window.json

@@ -35,6 +35,12 @@
     "returns": "number",
     "help": "Returns the fraction represented by numbers yielded by given expression over the total sum of them. This requires buffering whole file or group!"
   },
+  {
+    "name": "front_coding",
+    "arguments": ["<expr>"],
+    "returns": "string",
+    "help": "Compress string returned by given expression using incremental encoding (useful to compress sorted data).\nSee https://en.wikipedia.org/wiki/Incremental_encoding"
+  },
   {
     "name": "lag",
     "arguments": ["<expr>", "steps?", "<expr>?"],