sort geobuf keys, rounding just like js, python (#16)

* sort keys

* not ready

* export geobuf with ordered keys!

* only_xy=false

* dim

* remove normalize

* add release notes

* use round

* add test

Author: district10

docs/about/release-notes.md

@@ -10,6 +10,11 @@ To upgrade `pybind11-geobuf` to the latest version, use pip:
 pip install -U pybind11-geobuf
 ```
 
+## Version 0.0.9 (2023-03-30)
+
+*   Export to geobuf with only xy (no z), `only_xy=False`
+*   Geobuf property keys are sorted now!
+
 ## Version 0.0.8 (2023-03-28)
 
 *   Integrate geobuf into geojson!

pybind11_geobuf/__main__.py

@@ -38,12 +38,13 @@ def json2geobuf(
     output_path: str,
     *,
     precision: int = 8,
+    only_xy: bool = False,
 ):
     logger.info(
         f"geobuf encoding {input_path} ({__filesize(input_path):,} bytes)..."
     )  # noqa
     os.makedirs(os.path.dirname(os.path.abspath(output_path)), exist_ok=True)
-    encoder = Encoder(max_precision=int(10**precision))
+    encoder = Encoder(max_precision=int(10**precision), only_xy=only_xy)
     assert encoder.encode(
         geojson=input_path,
         geobuf=output_path,
@@ -57,6 +58,7 @@ def normalize_geobuf(
     *,
     sort_keys: bool = True,
     precision: int = -1,
+    only_xy: bool = False,
 ):
     logger.info(
         f"normalize_geobuf {input_path} ({__filesize(input_path):,} bytes)"
@@ -70,7 +72,7 @@ def normalize_geobuf(
         logger.info(f"auto precision from geobuf: {precision}")
     else:
         logger.info(f"user precision: {precision}")
-    encoder = Encoder(max_precision=int(10**precision))
+    encoder = Encoder(max_precision=int(10**precision), only_xy=only_xy)
     encoded = encoder.encode(json)
     logger.info(f"encoded #bytes: {len(encoded):,}")
     output_path = output_path or input_path
@@ -87,6 +89,7 @@ def normalize_json(
     indent: bool = True,
     sort_keys: bool = True,
     precision: int = -1,
+    only_xy: bool = False,
 ):
     logger.info(
         f"normalize_json {input_path} ({__filesize(input_path):,} bytes)"
@@ -97,7 +100,7 @@ def normalize_json(
         logger.info(
             f"convert to geobuf (precision: {precision}), then back to geojson"
         )  # noqa
-        encoder = Encoder(max_precision=int(10**precision))
+        encoder = Encoder(max_precision=int(10**precision), only_xy=only_xy)
         geojson = rapidjson().load(input_path)
         assert geojson.IsObject(), f"invalid geojson: {input_path}"
         encoded = encoder.encode(geojson)

setup.py

@@ -127,7 +127,7 @@ def build_extension(self, ext):
 # logic and declaration, and simpler if you include description/version in a file.
 setup(
     name="pybind11_geobuf",
-    version="0.0.8",
+    version="0.0.9",
     author="tzx",
     author_email="dvorak4tzx@gmail.com",
     url="https://geobuf-cpp.readthedocs.io",

src/geobuf/geobuf.cpp

@@ -4,6 +4,7 @@
 #include <array>
 #include <mapbox/geojson_impl.hpp>
 #include <mapbox/geojson_value_impl.hpp>
+#include <set>
 
 #include "rapidjson/error/en.h"
 #include "rapidjson/filereadstream.h"
@@ -212,25 +213,27 @@ std::string dump(const mapbox::geojson::geojson &geojson, //
 
 std::string Encoder::encode(const mapbox::geojson::geojson &geojson)
 {
+    std::string data;
+    Encoder::Pbf pbf{data};
+
     dim = MAPBOX_GEOBUF_DEFAULT_DIM;
     e = 1;
     keys.clear();
     analyze(geojson);
-
-    std::vector<std::pair<const std::string *, uint32_t>> keys_vec;
-    keys_vec.reserve(keys.size());
-    for (auto &pair : keys) {
-        keys_vec.emplace_back(&pair.first, pair.second);
+    if (onlyXY) {
+        dim = 2u;
     }
-    std::sort(keys_vec.begin(), keys_vec.end(),
-              [](const auto &kv1, const auto &kv2) {
-                  return kv1.second < kv2.second;
-              });
 
-    std::string data;
-    Encoder::Pbf pbf{data};
-    for (auto &kv : keys_vec) {
-        pbf.add_string(1, *kv.first);
+    {
+        auto kk = std::set<std::string>();
+        for (auto &pair : keys) {
+            kk.insert(pair.first);
+        }
+        int idx = -1;
+        for (auto &k : kk) {
+            pbf.add_string(1, k);
+            keys[k] = ++idx;
+        }
     }
     if (dim != MAPBOX_GEOBUF_DEFAULT_DIM) {
         pbf.add_uint32(2, dim);
@@ -346,15 +349,23 @@ void Encoder::analyzePoints(const PointsType &points)
     }
 }
 
+inline double ROUND(double v, double s)
+{
+    return std::floor(v * s + 0.5) / s;
+    // return std::round(v * s) / s;
+}
+
 void Encoder::analyzePoint(const mapbox::geojson::point &point)
 {
-    dim = std::max(point.z == 0 ? dimXY : dimXYZ, dim);
+    if (!onlyXY) {
+        dim = std::max(point.z == 0 ? dimXY : dimXYZ, dim);
+    }
     if (e >= maxPrecision) {
         return;
     }
     const double *ptr = &point.x;
     for (int i = 0; i < dim; ++i) {
-        while (std::round(ptr[i] * e) / e != ptr[i] && e < maxPrecision) {
+        while (ROUND(ptr[i], e) != ptr[i] && e < maxPrecision) {
             e *= 10;
         }
     }
@@ -502,7 +513,7 @@ void Encoder::writePoint(const mapbox::geojson::point &point, Encoder::Pbf &pbf)
     coords.reserve(dim);
     const double *ptr = &point.x;
     for (int i = 0; i < dim; ++i) {
-        coords.push_back(static_cast<int64_t>(std::round(ptr[i] * e)));
+        coords.push_back(static_cast<int64_t>(std::floor(ptr[i] * e + 0.5)));
     }
     pbf.add_packed_sint64(3, coords.begin(), coords.end());
 }
@@ -570,7 +581,8 @@ void Encoder::populateLine(std::vector<int64_t> &coords, //
     for (int i = 0; i < len; ++i) {
         const double *ptr = &line[i].x;
         for (int j = 0; j < dim; ++j) {
-            auto n = static_cast<int64_t>(std::round(ptr[j] * e)) - sum[j];
+            auto n =
+                static_cast<int64_t>(std::floor(ptr[j] * e + 0.5)) - sum[j];
             coords.push_back(n);
             sum[j] += n;
         }

src/geobuf/geobuf.hpp

@@ -80,8 +80,9 @@ struct Encoder
 {
     using Pbf = protozero::pbf_writer;
     Encoder(uint32_t maxPrecision = std::pow(10,
-                                             MAPBOX_GEOBUF_DEFAULT_PRECISION))
-        : maxPrecision(maxPrecision)
+                                             MAPBOX_GEOBUF_DEFAULT_PRECISION),
+            bool onlyXY = false)
+        : maxPrecision(maxPrecision), onlyXY(onlyXY)
     {
     }
     std::string encode(const mapbox::geojson::geojson &geojson);
@@ -138,6 +139,7 @@ struct Encoder
                       bool closed);
 
     const uint32_t maxPrecision;
+    const bool onlyXY;
     uint32_t dim = MAPBOX_GEOBUF_DEFAULT_DIM;
     uint32_t e = 1;
     std::unordered_map<std::string, std::uint32_t> keys;
@@ -153,6 +155,8 @@ struct Decoder
     bool decode(const std::string &input_path, const std::string &output_path,
                 bool indent = false, bool sort_keys = false);
     int precision() const { return std::log10(e); }
+    int __dim() const { return dim; }
+    std::vector<std::string> __keys() const { return keys; }
 
   private:
     mapbox::geojson::feature_collection readFeatureCollection(Pbf &pbf);

src/geobuf/geojson_helpers.hpp

@@ -451,10 +451,8 @@ inline double round_coords(double value, double scale)
     // also note that, javascript Math.round differs from C++ round
     // e.g. std::round(-0.5) => -1.0
     //      Math.round(-0.5) => -0.0
-    return std::round(value * scale) / scale;
-    // TODO, all should use Math.round!!!
-    // Math.round equivalent:
-    //  std::floor(value * scale + 0.5) / scale
+    return std::floor(value * scale + 0.5) / scale; // like in JS
+    // return std::round(value * scale) / scale;
 }
 
 inline void round_coords(mapbox::geojson::point &xyz,

src/main.cpp

@@ -92,9 +92,10 @@ PYBIND11_MODULE(_pybind11_geobuf, m)
         "indent"_a = "");
 
     py::class_<Encoder>(m, "Encoder", py::module_local()) //
-        .def(py::init<uint32_t>(),                        //
+        .def(py::init<uint32_t, bool>(),                  //
              py::kw_only(),
-             "max_precision"_a = std::pow(10, MAPBOX_GEOBUF_DEFAULT_PRECISION))
+             "max_precision"_a = std::pow(10, MAPBOX_GEOBUF_DEFAULT_PRECISION),
+             "only_xy"_a = false)
         //
         .def(
             "encode",
@@ -149,6 +150,7 @@ PYBIND11_MODULE(_pybind11_geobuf, m)
         .def(py::init<>())
         //
         .def("precision", &Decoder::precision)
+        .def("dim", &Decoder::__dim)
         .def(
             "decode",
             [](Decoder &self, const std::string &geobuf, bool indent,
@@ -188,6 +190,7 @@ PYBIND11_MODULE(_pybind11_geobuf, m)
             "geojson"_a,        //
             "indent"_a = false, //
             "sort_keys"_a = false)
+        .def("keys", &Decoder::__keys)
         //
         ;