Flatten change array

Author: kamicut

pipeline/parser.js

@@ -21,10 +21,11 @@ fs.readFile(fileName, "utf8", (err, data) => {
       result[changesetId].forEach((element) => {
         const change = changesetParser.elementParser(element);
 
-        // do some validation here
-        console.log(JSON.stringify(change));
+        change.forEach((feature) => {
+          // do some validation here
+          console.log(JSON.stringify(feature));
+        });
       });
     });
-    console.log(JSON.stringify(featureCollection));
   });
 });

pipeline/process.sh

@@ -48,11 +48,12 @@ seq $start_num $end_num | parallel fetchOscFunction
 
 # Parse .osc files and collect into a single geojson using parser.js and jq
 echo 'Parsing files'
-for (( i=start_num; i<end_num; i++ )); do
-    echo $i
-    file_path="/tmp/${i}.osc"
-    node parser.js $file_path > "/tmp/${start_date}-${start_hour}-${i}.geojsonld"
-done
+parallel -j +0 "
+    echo {};
+    file_path=\"/tmp/{}.osc\";
+    node parser.js \$file_path > \"/tmp/${start_date}-${start_hour}-{}.geojsonld\"
+" ::: $(seq $start_num $((end_num-1)))
+
 echo 'Done parsing files'
 
 echo "Combining files"