handle nan inf; normalize non geojson json (#22)

* dump json example succ

* locate inf nan

* export bad json

* round non geojson

* release

---------

Co-authored-by: TANG ZHIXIONG <zhixiong.tang@momenta.ai>

Author: district10

docs/about/release-notes.md

@@ -10,6 +10,11 @@ To upgrade `pybind11-geobuf` to the latest version, use pip:
 pip install -U pybind11-geobuf
 ```
 
+## Version 0.1.5 (2023-06-02)
+
+*   Add `round_non_geojson` to `normalize_json`
+*   Handle NaN, Inf in json, add `locate_nan_inf`
+
 ## Version 0.1.4 (2023-04-15)
 
 *   More options to normalize_json

pybind11_geobuf/__main__.py

@@ -1,5 +1,5 @@
 import os
-from typing import Tuple
+from typing import Optional, Tuple
 
 from loguru import logger
 
@@ -94,8 +94,9 @@ def normalize_json(
     only_xy: bool = False,
     denoise_double_0: bool = True,
     strip_geometry_z_0: bool = True,
-    round_geojson_non_geometry: int = 3,
-    round_geojson_geometry: Tuple[int, int, int] = (8, 8, 3),
+    round_non_geojson: Optional[int] = 3,
+    round_geojson_non_geometry: Optional[int] = 3,
+    round_geojson_geometry: Optional[Tuple[int, int, int]] = (8, 8, 3),
 ):
     logger.info(
         f"normalize_json {input_path} ({__filesize(input_path):,} bytes)"
@@ -124,6 +125,7 @@ def normalize_json(
             sort_keys=sort_keys,
             denoise_double_0=denoise_double_0,
             strip_geometry_z_0=strip_geometry_z_0,
+            round_non_geojson=round_non_geojson,
             round_geojson_non_geometry=round_geojson_non_geometry,
             round_geojson_geometry=round_geojson_geometry,
         ), f"failed to normalize json to {output_path}"

setup.py

@@ -127,7 +127,7 @@ def build_extension(self, ext):
 # logic and declaration, and simpler if you include description/version in a file.
 setup(
     name="pybind11_geobuf",
-    version="0.1.4",
+    version="0.1.5",
     author="tzx",
     author_email="dvorak4tzx@gmail.com",
     url="https://geobuf-cpp.readthedocs.io",

src/geobuf/geobuf.cpp

@@ -75,15 +75,16 @@ bool dump_json(FILE *fp, const RapidjsonValue &json, bool indent,
     using namespace rapidjson;
     char writeBuffer[65536];
     FileWriteStream os(fp, writeBuffer, sizeof(writeBuffer));
+    bool succ = false;
     if (indent) {
         PrettyWriter<FileWriteStream> writer(os);
-        json.Accept(writer);
+        succ = json.Accept(writer);
     } else {
         Writer<FileWriteStream> writer(os);
-        json.Accept(writer);
+        succ = json.Accept(writer);
     }
     fclose(fp);
-    return true;
+    return succ;
 }
 
 bool dump_json(const std::string &path, const RapidjsonValue &json, bool indent,

src/geobuf/rapidjson_helpers.hpp

@@ -69,6 +69,37 @@ inline void sort_keys_inplace(RapidjsonValue &json)
     }
 }
 
+inline std::optional<std::string> locate_nan_inf(const RapidjsonValue &json,
+                                                 const std::string &path = "")
+{
+    if (json.IsObject()) {
+        for (auto &kv : json.GetObject()) {
+            auto p = locate_nan_inf(kv.value);
+            if (p) {
+                return path + "[\"" +
+                       std::string(kv.name.GetString(),
+                                   kv.name.GetStringLength()) +
+                       "\"]" + *p;
+            }
+        }
+    } else if (json.IsArray()) {
+        int index = -1;
+        for (auto &m : json.GetArray()) {
+            ++index;
+            auto p = locate_nan_inf(m);
+            if (p) {
+                return path + "[" + std::to_string(index) + "]" + *p;
+            }
+        }
+    } else if (json.IsDouble()) {
+        double d = json.GetDouble();
+        if (std::isnan(d) || std::isinf(d)) {
+            return path;
+        }
+    }
+    return {};
+}
+
 inline void round_rapidjson(RapidjsonValue &json, double scale, int depth = 1,
                             const std::vector<std::string> &skip_keys = {})
 {
@@ -97,6 +128,30 @@ inline void round_rapidjson(RapidjsonValue &json, double scale, int depth = 1,
     }
 }
 
+inline void round_non_geojson(RapidjsonValue &json, double scale)
+{
+    if (json.IsObject()) {
+        auto itr = json.FindMember("type");
+        if (itr != json.MemberEnd() && itr->value.IsString()) {
+            const auto type = std::string(itr->value.GetString(),
+                                          itr->value.GetStringLength());
+            if (                             //
+                type == "FeatureCollection"  //
+                || type == "Feature"         //
+                || type == "Point"           //
+                || type == "MultiPoint"      //
+                || type == "LineString"      //
+                || type == "MultiLineString" //
+                || type == "Polygon"         //
+                || type == "MultiPolygon"    //
+                || type == "GeometryCollection") {
+                return;
+            }
+        }
+    }
+    round_rapidjson(json, scale, INT_MAX);
+}
+
 inline void round_geojson_non_geometry(RapidjsonValue &json, double scale)
 {
     if (!json.IsObject()) {
@@ -284,8 +339,9 @@ normalize_json(RapidjsonValue &json,                              //
                bool sort_keys = true,                             //
                std::optional<int> round_geojson_non_geometry = 3, //
                const std::optional<std::array<int, 3>> &round_geojson_geometry =
-                   std::array<int, 3>{8, 8, 3}, //
-               bool denoise_double_0 = true,    //
+                   std::array<int, 3>{8, 8, 3},          //
+               std::optional<int> round_non_geojson = 3, //
+               bool denoise_double_0 = true,             //
                bool strip_geometry_z_0 = true)
 {
     if (sort_keys) {
@@ -301,6 +357,10 @@ normalize_json(RapidjsonValue &json,                              //
                                              std::pow(10.0, precision[1]),
                                              std::pow(10.0, precision[2])});
     }
+    if (round_non_geojson) {
+        double scale = std::pow(10.0, *round_non_geojson);
+        cubao::round_non_geojson(json, scale);
+    }
     if (strip_geometry_z_0) {
         cubao::strip_geometry_z_0(json);
     }
@@ -341,24 +401,25 @@ inline bool dump_json(const std::string &path, const RapidjsonValue &json,
     }
     using namespace rapidjson;
     char writeBuffer[65536];
+    bool succ = false;
     FileWriteStream os(fp, writeBuffer, sizeof(writeBuffer));
     if (indent) {
         PrettyWriter<FileWriteStream> writer(os);
         if (sort_keys) {
-            cubao::sort_keys(json).Accept(writer);
+            succ = cubao::sort_keys(json).Accept(writer);
         } else {
-            json.Accept(writer);
+            succ = json.Accept(writer);
         }
     } else {
         Writer<FileWriteStream> writer(os);
         if (sort_keys) {
-            cubao::sort_keys(json).Accept(writer);
+            succ = cubao::sort_keys(json).Accept(writer);
         } else {
-            json.Accept(writer);
+            succ = json.Accept(writer);
         }
     }
     fclose(fp);
-    return true;
+    return succ;
 }
 
 inline RapidjsonValue loads(const std::string &json)
@@ -508,11 +569,6 @@ inline RapidjsonValue to_rapidjson(const mapbox::geojson::value &json)
     return to_rapidjson(json, allocator);
 }
 
-// inline mapbox::geojson::value to_geojson_value(const RapidjsonValue &json)
-// {
-//     return mapbox::geojson::convert<mapbox::geojson::value>(json);
-// }
-
 inline bool is_subset_of(const RapidjsonValue &a, const RapidjsonValue &b)
 {
     if (a.IsArray()) {

src/main.cpp

@@ -36,36 +36,40 @@ PYBIND11_MODULE(_pybind11_geobuf, m)
          "normalize_json",
          [](const std::string &input, const std::string &output, bool indent,
             bool sort_keys, bool denoise_double_0, bool strip_geometry_z_0,
+            std::optional<int> round_non_geojson,
             std::optional<int> round_geojson_non_geometry,
             const std::optional<std::array<int, 3>> &round_geojson_geometry) {
              auto json = mapbox::geobuf::load_json(input);
              cubao::normalize_json(json,                       //
                                    sort_keys,                  //
                                    round_geojson_non_geometry, //
                                    round_geojson_geometry,     //
+                                   round_non_geojson,          //
                                    denoise_double_0,           //
                                    strip_geometry_z_0);
              return mapbox::geobuf::dump_json(output, json, indent);
          },
-         "input_path"_a, "output_path"_a, //
-         py::kw_only(),                   //
-         "indent"_a = true,               //
-         "sort_keys"_a = true,            //
-         "denoise_double_0"_a = true,     //
-         "strip_geometry_z_0"_a = true,   //
-         "round_geojson_non_geometry"_a = 3,
+         "input_path"_a, "output_path"_a,    //
+         py::kw_only(),                      //
+         "indent"_a = true,                  //
+         "sort_keys"_a = true,               //
+         "denoise_double_0"_a = true,        //
+         "strip_geometry_z_0"_a = true,      //
+         "round_non_geojson"_a = 3,          //
+         "round_geojson_non_geometry"_a = 3, //
          "round_geojson_geometry"_a = std::array<int, 3>{8, 8, 3})
         .def(
             "normalize_json",
             [](RapidjsonValue &json, bool sort_keys, bool denoise_double_0,
-               bool strip_geometry_z_0,
+               bool strip_geometry_z_0, std::optional<int> round_non_geojson,
                std::optional<int> round_geojson_non_geometry,
                const std::optional<std::array<int, 3>> &round_geojson_geometry)
                 -> RapidjsonValue & {
                 cubao::normalize_json(json,                       //
                                       sort_keys,                  //
                                       round_geojson_non_geometry, //
                                       round_geojson_geometry,     //
+                                      round_non_geojson,          //
                                       denoise_double_0,           //
                                       strip_geometry_z_0);
                 return json;
@@ -75,6 +79,7 @@ PYBIND11_MODULE(_pybind11_geobuf, m)
             "sort_keys"_a = true,          //
             "denoise_double_0"_a = true,   //
             "strip_geometry_z_0"_a = true, //
+            "round_non_geojson"_a = 3,     //
             "round_geojson_non_geometry"_a = 3,
             "round_geojson_geometry"_a = std::array<int, 3>{8, 8, 3},
             rvp::reference_internal);

src/pybind11_rapidjson.cpp

@@ -149,6 +149,10 @@ void bind_rapidjson(py::module &m)
                 sort_keys_inplace(self);
                 return self;
             }, rvp::reference_internal)
+            // locate_nan_inf
+            .def("locate_nan_inf", [](const RapidjsonValue &self) -> std::optional<std::string> {
+                return locate_nan_inf(self);
+            })
             .def("round", [](RapidjsonValue &self, double precision, int depth, //
                 const std::vector<std::string> &skip_keys) -> RapidjsonValue & {
                     round_rapidjson(self, std::pow(10, precision), depth, skip_keys);

tests/test_geobuf.py

@@ -1,8 +1,11 @@
 import base64
+import contextlib
 import json
 import os
 import pickle
+import shutil
 import sys
+import tempfile
 from copy import deepcopy
 
 import numpy as np
@@ -1652,6 +1655,106 @@ def pytest_main(dir: str, *, test_file: str = None):
     )
 
 
+@contextlib.contextmanager
+def context_tempdir():
+    temp_dir = tempfile.mkdtemp()
+    try:
+        yield temp_dir
+    finally:
+        if os.path.isdir(temp_dir):
+            shutil.rmtree(temp_dir)
+
+
+def test_rapidjson_dump_nan():
+    j = rapidjson(
+        {
+            "key": "value",
+            "pi": 3.14,
+            "nan": np.nan,
+        }
+    )
+    assert j.dumps() == '{"key":"value","pi":3.14,"nan":'
+    assert j.locate_nan_inf() == '["nan"]'
+    del j["nan"]
+    assert j.locate_nan_inf() is None
+    j["inf"] = np.inf
+    assert j.locate_nan_inf() == '["inf"]'
+
+    assert rapidjson(np.inf).locate_nan_inf() == ""
+    assert rapidjson(np.nan).locate_nan_inf() == ""
+    assert rapidjson("json").locate_nan_inf() is None
+
+    jj = rapidjson({"root": {}})
+    jj["root"]["child"] = j
+    assert jj.locate_nan_inf() == '["root"]["child"]["inf"]'
+
+    with context_tempdir() as dir:
+        j = rapidjson({"root": {}})
+        path = f"{dir}/okay.json"
+        assert j.dump(path)
+        with open(path) as f:
+            text = f.read()
+            assert text == '{"root":{}}'
+
+    with context_tempdir() as dir:
+        path = f"{dir}/fail.json"
+        assert not jj.dump(path)
+
+
+def test_rapidjson_normalize_non_geojson():
+    j = normalize_json(rapidjson({"value": 3.14156}))
+    assert j.dumps() == '{"value":3.142}'
+
+    j = normalize_json(rapidjson({"value": 3.14156}), round_non_geojson=None)
+    assert j.dumps() == '{"value":3.14156}'
+
+    j = normalize_json(
+        rapidjson(
+            {
+                "type": "Point",
+                "coordinates": [1.23456, 7.890123, 4.567],
+                "value": 3.123456,
+            }
+        ).sort_keys(),
+        round_geojson_geometry=[3, 2, 1],
+    )
+    assert (
+        j.dumps()
+        == '{"coordinates":[1.235,7.89,4.6],"type":"Point","value":3.123}'  # noqa
+    )
+
+    j = normalize_json(
+        rapidjson(
+            {
+                "type": "Point",
+                "coordinates": [1.23456, 7.890123, 4.567],
+                "value": 3.123456,
+            }
+        ).sort_keys(),
+        round_geojson_geometry=None,
+    )
+    assert (
+        j.dumps()
+        == '{"coordinates":[1.23456,7.890123,4.567],"type":"Point","value":3.123}'  # noqa
+    )
+
+    j = normalize_json(
+        rapidjson(
+            {
+                "type": "Point",
+                "coordinates": [1.23456, 7.890123, 4.567],
+                "value": 3.123456,
+            }
+        ).sort_keys(),
+        round_geojson_geometry=[1, 1, 1],
+        round_geojson_non_geometry=None,
+    )
+    assert (
+        j.dumps()
+        == '{"coordinates":[1.2,7.9,4.6],"type":"Point","value":3.123456}'  # noqa
+    )
+
+
 if __name__ == "__main__":
     np.set_printoptions(suppress=True)
     pwd = os.path.abspath(os.path.dirname(__file__))