Merge pull request #120 from opencultureconsulting:119-transform-line-breaks-n-in-expression-lead-to-parsing-abort

transform: allow \n in expression

Author: felixlohmeier

help/README.md

@@ -1,4 +1,4 @@
-# orcli 0.2.0
+# orcli 0.2.1
 
 ## command help screens
 

orcli

@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-# This script was generated by bashly 1.1.1 (https://bashly.dannyb.co)
+# This script was generated by bashly 1.1.2 (https://bashly.dannyb.co)
 # Modifying it manually is not recommended
 
 # :wrapper.bash3_bouncer
@@ -2231,7 +2231,8 @@ orcli_transform_command() {
       fi
       for line in "${jsonlines[@]}"; do
           # parse one line/operation into array
-          declare -A array="($(echo "$line" | jq -r 'to_entries | map("[\(.key)]=" + @sh "\(.value|tostring)") | .[]'))"
+          filter='[to_entries[]|"["+(.key|@sh)+"]="+(.value|tostring|@sh)]|"("+join(" ")+")"'
+          declare -A array=$(jq --join-output "${filter}" <<< "$line")
           if [[ ! ${array[op]} ]]; then
               error "parsing ${files[$i]} failed!"
           fi
@@ -2259,6 +2260,8 @@ orcli_transform_command() {
           unset "array[engineConfig]"
           # drop description
           unset "array[description]"
+          # remove line breaks in expression
+          array[expression]="${array[expression]//$'\n'/}"
           # prepare curl options
           mapfile -t curloptions < <(for K in "${!array[@]}"; do
               echo "--data"
@@ -4305,7 +4308,7 @@ orcli_export_jsonl_parse_requirements() {
   [[ -n ${args['--encoding']:-} ]] || args['--encoding']="UTF-8"
 
   # :command.whitelist_filter
-  if [[ ${args['--mode']} ]] && [[ ! ${args['--mode']} =~ ^(rows|records)$ ]]; then
+  if [[ ${args['--mode']:-} ]] && [[ ! ${args['--mode']:-} =~ ^(rows|records)$ ]]; then
     printf "%s\n" "--mode must be one of: rows, records" >&2
     exit 1
   fi
@@ -4454,7 +4457,7 @@ orcli_export_tsv_parse_requirements() {
   [[ -n ${args['--encoding']:-} ]] || args['--encoding']="UTF-8"
 
   # :command.whitelist_filter
-  if [[ ${args['--mode']} ]] && [[ ! ${args['--mode']} =~ ^(rows|records)$ ]]; then
+  if [[ ${args['--mode']:-} ]] && [[ ! ${args['--mode']:-} =~ ^(rows|records)$ ]]; then
     printf "%s\n" "--mode must be one of: rows, records" >&2
     exit 1
   fi
@@ -4638,7 +4641,7 @@ orcli_export_template_parse_requirements() {
   [[ -n ${args['--encoding']:-} ]] || args['--encoding']="UTF-8"
 
   # :command.whitelist_filter
-  if [[ ${args['--mode']} ]] && [[ ! ${args['--mode']} =~ ^(rows|records)$ ]]; then
+  if [[ ${args['--mode']:-} ]] && [[ ! ${args['--mode']:-} =~ ^(rows|records)$ ]]; then
     printf "%s\n" "--mode must be one of: rows, records" >&2
     exit 1
   fi
@@ -4747,7 +4750,7 @@ orcli_run_parse_requirements() {
 
 # :command.initialize
 initialize() {
-  version="0.2.0"
+  version="0.2.1"
   long_usage=''
   set -e
 

src/bashly.yml

@@ -1,6 +1,6 @@
 name: orcli
 help: OpenRefine command-line interface written in Bash
-version: 0.2.0
+version: 0.2.1
 footer: https://github.com/opencultureconsulting/orcli
 
 dependencies:

src/transform_command.sh

@@ -55,7 +55,8 @@ for i in "${!files[@]}"; do
     fi
     for line in "${jsonlines[@]}"; do
         # parse one line/operation into array
-        declare -A array="($(echo "$line" | jq -r 'to_entries | map("[\(.key)]=" + @sh "\(.value|tostring)") | .[]'))"
+        filter='[to_entries[]|"["+(.key|@sh)+"]="+(.value|tostring|@sh)]|"("+join(" ")+")"'
+        declare -A array=$(jq --join-output "${filter}" <<< "$line")
         if [[ ! ${array[op]} ]]; then
             error "parsing ${files[$i]} failed!"
         fi
@@ -83,6 +84,8 @@ for i in "${!files[@]}"; do
         unset "array[engineConfig]"
         # drop description
         unset "array[description]"
+        # remove line breaks in expression
+        array[expression]="${array[expression]//$'\n'/}"
         # prepare curl options
         mapfile -t curloptions < <(for K in "${!array[@]}"; do
             echo "--data"

tests/data/duplicates-history.json

@@ -0,0 +1,70 @@
+[
+    {
+      "op": "core/row-reorder",
+      "description": "Reorder rows",
+      "mode": "record-based",
+      "sorting": {
+        "criteria": [
+          {
+            "errorPosition": 1,
+            "caseSensitive": false,
+            "valueType": "string",
+            "column": "email",
+            "blankPosition": 2,
+            "reverse": false
+          }
+        ]
+      }
+    },
+    {
+      "op": "core/column-addition",
+      "description": "Create column count at index 1 based on column email using expression grel:facetCount(value, \"value\", \"email\")",
+      "engineConfig": {
+        "mode": "row-based",
+        "facets": []
+      },
+      "newColumnName": "count",
+      "columnInsertIndex": 1,
+      "baseColumnName": "email",
+      "expression": "grel:facetCount(\nvalue,\n \"value\",\n \"email\"\n)",
+      "onError": "set-to-blank"
+    },
+    {
+      "op": "core/blank-down",
+      "description": "Blank down cells in column email",
+      "engineConfig": {
+        "mode": "row-based",
+        "facets": []
+      },
+      "columnName": "email"
+    },
+    {
+      "op": "core/row-removal",
+      "description": "Remove rows",
+      "engineConfig": {
+        "mode": "row-based",
+        "facets": [
+          {
+            "omitError": false,
+            "expression": "isBlank(value)",
+            "selectBlank": false,
+            "selection": [
+              {
+                "v": {
+                  "v": true,
+                  "l": "true"
+                }
+              }
+            ],
+            "selectError": false,
+            "invert": false,
+            "name": "email",
+            "omitBlank": false,
+            "type": "list",
+            "columnName": "email"
+          }
+        ]
+      }
+    }
+  ]
+  

tests/transform.sh

@@ -9,10 +9,13 @@ trap '{ rm -rf "${tmpdir}"; }' 0 2 3 15
 # assertion
 cp data/duplicates-transformed.tsv "${tmpdir}/${t}.assert"
 
+# transform
+cp data/duplicates-history.json "${tmpdir}/${t}.history"
+
 # action
 cd "${tmpdir}" || exit 1
 orcli import csv "https://git.io/fj5hF" --projectName "duplicates"
-orcli transform "duplicates" "https://git.io/fj5ju"
+orcli transform "duplicates" "${t}.history"
 orcli export tsv "duplicates" --output "${t}.output"
 
 # test