More options to normalize_json; Add is_subset_of(json1, json2) (#21)

* normalize json

* test interface

* is subset

* is_subset_of

* ready to PR

* fix

* fix

Author: district10

Makefile

@@ -105,7 +105,9 @@ python_sdist:
 	# tar -tvf dist/geobuf-*.tar.gz
 python_test: pytest
 
-cli_test:
+cli_test: cli_test1 cli_test2 cli_test3 cli_test4
+
+cli_test1:
 	python3 -m pybind11_geobuf
 	python3 -m pybind11_geobuf --help
 	python3 -m pybind11_geobuf json2geobuf data/sample1.json build/sample1.pbf
@@ -120,6 +122,21 @@ cli_test2:
 	python3 -m pybind11_geobuf round_trip data/sample2.json -o build/test/cxx_py --json2pb_use_python=False --pb2json_use_python=True
 	python3 -m pybind11_geobuf round_trip data/sample2.json -o build/test/py_cxx --json2pb_use_python=True --pb2json_use_python=False
 
+cli_test3:
+	python3 -m pybind11_geobuf normalize_json data/feature_collection.json build/fc.json && cat build/fc.json | grep '"double": 3.142,'
+	python3 -m pybind11_geobuf normalize_json data/feature_collection.json build/fc.json --round_geojson_non_geometry=None && cat build/fc.json | grep '"double": 3.141592653'
+	python3 -m pybind11_geobuf normalize_json data/feature_collection.json build/fc.json --round_geojson_geometry=None && cat build/fc.json | grep 3.3333333333333
+	python3 -m pybind11_geobuf normalize_json data/feature_collection.json build/fc.json --round_geojson_geometry=3,3,2 && cat build/fc.json | grep '120.285,'
+	python3 -m pybind11_geobuf normalize_json data/feature_collection.json build/fc.json --round_geojson_geometry=0,0,0 && cat build/fc.json | grep '120,'
+	python3 -m pybind11_geobuf normalize_json data/feature_collection.json build/fc.json --round_geojson_geometry=0,0,0 --denoise_double_0=False && cat build/fc.json | grep '120.0,'
+	python3 -m pybind11_geobuf normalize_json data/feature_collection.json build/fc.json --round_geojson_geometry=8,8,-1 && cat build/fc.json | wc -l | grep 49
+	python3 -m pybind11_geobuf normalize_json data/feature_collection.json build/fc.json --round_geojson_geometry=8,8,-1 --strip_geometry_z_0=False && cat build/fc.json | wc -l | grep 53
+
+cli_test4:
+	python3 -m pybind11_geobuf is_subset_of data/feature_collection.json data/feature_collection.json
+
+.PHONY: cli_test cli_test1 cli_test2 cli_test3
+
 # conda create -y -n py36 python=3.6
 # conda create -y -n py37 python=3.7
 # conda create -y -n py38 python=3.8

docs/about/release-notes.md

@@ -10,6 +10,11 @@ To upgrade `pybind11-geobuf` to the latest version, use pip:
 pip install -U pybind11-geobuf
 ```
 
+## Version 0.1.4 (2023-04-15)
+
+*   More options to normalize_json
+*   Add `is_subset_of(json1, json2)`
+
 ## Version 0.1.3 (2023-04-11)
 
 *   Fix round geojson-non-geometry

pybind11_geobuf/__main__.py

@@ -1,9 +1,11 @@
 import os
+from typing import Tuple
 
 from loguru import logger
 
 from pybind11_geobuf import rapidjson  # noqa
 from pybind11_geobuf import Decoder, Encoder  # noqa
+from pybind11_geobuf import is_subset_of as is_subset_of_impl  # noqa
 from pybind11_geobuf import normalize_json as normalize_json_impl  # noqa
 from pybind11_geobuf import pbf_decode as pbf_decode_impl  # noqa
 
@@ -90,6 +92,10 @@ def normalize_json(
     sort_keys: bool = True,
     precision: int = -1,
     only_xy: bool = False,
+    denoise_double_0: bool = True,
+    strip_geometry_z_0: bool = True,
+    round_geojson_non_geometry: int = 3,
+    round_geojson_geometry: Tuple[int, int, int] = (8, 8, 3),
 ):
     logger.info(
         f"normalize_json {input_path} ({__filesize(input_path):,} bytes)"
@@ -116,6 +122,10 @@ def normalize_json(
             output_path,
             indent=indent,
             sort_keys=sort_keys,
+            denoise_double_0=denoise_double_0,
+            strip_geometry_z_0=strip_geometry_z_0,
+            round_geojson_non_geometry=round_geojson_non_geometry,
+            round_geojson_geometry=round_geojson_geometry,
         ), f"failed to normalize json to {output_path}"
     logger.info(f"wrote to {output_path} ({__filesize(output_path):,} bytes)")
 
@@ -155,12 +165,6 @@ def round_trip(
     json2pb_use_python: bool = False,
     pb2json_use_python: bool = False,
 ):
-    """
-    _0.json
-    _1.pbf
-    _2.pbf.txt
-    _3.json
-    """
     assert path.endswith((".json", ".geojson")) and os.path.isfile(path)
     path = os.path.abspath(path)
     output_dir = os.path.abspath(output_dir or os.path.dirname(path))
@@ -200,13 +204,18 @@ def round_trip(
     logger.info(f"wrote to {opath}")
 
 
+def is_subset_of(path1: str, path2: str):
+    assert is_subset_of_impl(path1, path2)
+
+
 if __name__ == "__main__":
     import fire
 
     fire.core.Display = lambda lines, out: print(*lines, file=out)
     fire.Fire(
         {
             "geobuf2json": geobuf2json,
+            "is_subset_of": is_subset_of,
             "json2geobuf": json2geobuf,
             "normalize_geobuf": normalize_geobuf,
             "normalize_json": normalize_json,

setup.py

@@ -127,7 +127,7 @@ def build_extension(self, ext):
 # logic and declaration, and simpler if you include description/version in a file.
 setup(
     name="pybind11_geobuf",
-    version="0.1.3",
+    version="0.1.4",
     author="tzx",
     author_email="dvorak4tzx@gmail.com",
     url="https://geobuf-cpp.readthedocs.io",

src/geobuf/rapidjson_helpers.hpp

@@ -12,6 +12,7 @@
 #include "rapidjson/stringbuffer.h"
 #include <fstream>
 #include <iostream>
+#include <set>
 
 namespace cubao
 {
@@ -512,6 +513,54 @@ inline RapidjsonValue to_rapidjson(const mapbox::geojson::value &json)
 //     return mapbox::geojson::convert<mapbox::geojson::value>(json);
 // }
 
+inline bool is_subset_of(const RapidjsonValue &a, const RapidjsonValue &b)
+{
+    if (a.IsArray()) {
+        if (!b.IsArray() || a.Size() != b.Size()) {
+            return false;
+        }
+        auto aArr = a.GetArray();
+        auto bArr = b.GetArray();
+        for (int i = 0, N = a.Size(); i < N; ++i) {
+            if (!is_subset_of(aArr[i], bArr[i])) {
+                return false;
+            }
+        }
+        return true;
+    }
+    if (!a.IsObject()) {
+        if (b.IsObject()) {
+            return false;
+        }
+        return a == b;
+    }
+    if (!b.IsObject()) {
+        return false;
+    }
+    auto aObj = a.GetObject();
+    auto bObj = b.GetObject();
+    std::set<std::string> aKeys, bKeys;
+    std::for_each(aObj.MemberBegin(), aObj.MemberEnd(), [&](auto &kv) {
+        aKeys.insert(
+            std::string(kv.name.GetString(), kv.name.GetStringLength()));
+    });
+    std::for_each(bObj.MemberBegin(), bObj.MemberEnd(), [&](auto &kv) {
+        bKeys.insert(
+            std::string(kv.name.GetString(), kv.name.GetStringLength()));
+    });
+    if (!std::includes(bKeys.begin(), bKeys.end(), //
+                       aKeys.begin(), aKeys.end())) {
+        return false;
+    }
+    for (const auto &k : aKeys) {
+        if (!is_subset_of(aObj.FindMember(k.c_str())->value,
+                          bObj.FindMember(k.c_str())->value)) {
+            return false;
+        }
+    }
+    return true;
+}
+
 } // namespace cubao
 
 #endif

src/main.cpp

@@ -79,6 +79,15 @@ PYBIND11_MODULE(_pybind11_geobuf, m)
             "round_geojson_geometry"_a = std::array<int, 3>{8, 8, 3},
             rvp::reference_internal);
 
+    m.def(
+        "is_subset_of",
+        [](const std::string &path1, const std::string &path2) {
+            auto json1 = mapbox::geobuf::load_json(path1);
+            auto json2 = mapbox::geobuf::load_json(path2);
+            return cubao::is_subset_of(json1, json2);
+        },
+        "path1"_a, "path2"_a);
+
     m.def(
         "str2json2str",
         [](const std::string &json_string, //

src/pybind11_rapidjson.cpp

@@ -109,6 +109,10 @@ void bind_rapidjson(py::module &m)
                     return values;
                 },
                 rvp::reference_internal)
+            //
+            .def("is_subset_of", [](const RapidjsonValue &self, const RapidjsonValue &other) -> bool {
+                return is_subset_of(self, other);
+            }, "other"_a)
             // load/dump file
             .def(
                 "load",