remove necessity of --extra-attributes, use style only

Author: Nakaner

db-copy.hpp

@@ -290,6 +290,41 @@ class db_copy_mgr_t
         m_current->buffer += "\",";
     }
 
+    /**
+     * Add a key/value pair to a hash column without escaping.
+     *
+     * Key and value must be strings and will NOT be appropriately escaped.
+     * A separator for the next pair is added at the end.
+     */
+    void add_hash_elem_noescape(char const *k, char const *v)
+    {
+        m_current->buffer += '"';
+        m_current->buffer += k;
+        m_current->buffer += "\"=>\"";
+        m_current->buffer += v;
+        m_current->buffer += "\",";
+    }
+
+    /**
+     * Add a key (unescaped) and a numeric value to a hash column.
+     *
+     * Key must be string and come from a safe source because it will NOT be
+     * escaped! The value should be convertible using std::to_string.
+     * A separator for the next pair is added at the end.
+     *
+     * This method is suitable to insert safe input, e.g. numeric OSM metadata
+     * (eg. uid) but not unsafe input like user names.
+     */
+    template <typename T>
+    void add_hstore_num_noescape(char const *k, T const value)
+    {
+        m_current->buffer += '"';
+        m_current->buffer += k;
+        m_current->buffer += "\"=>\"";
+        m_current->buffer += std::to_string(value);
+        m_current->buffer += "\",";
+    }
+
     /**
      * Close a hash previously started with new_hash().
      *

gazetteer-style.cpp

@@ -42,7 +42,6 @@ void gazetteer_style_t::clear()
     m_main.clear();
     m_names.clear();
     m_extra.clear();
-    m_metadata.clear();
     m_address.clear();
     m_operator = nullptr;
     m_admin_level = MAX_ADMINLEVEL;
@@ -73,11 +72,6 @@ void gazetteer_style_t::load_style(std::string const &filename)
     }
 }
 
-void gazetteer_style_t::set_metadata(const bool enabled)
-{
-    m_metadata_enabled = enabled;
-}
-
 gazetteer_style_t::flag_t gazetteer_style_t::parse_flags(std::string const &str)
 {
     flag_t out = 0;
@@ -134,6 +128,25 @@ gazetteer_style_t::flag_t gazetteer_style_t::parse_flags(std::string const &str)
     return out;
 }
 
+bool gazetteer_style_t::add_metadata_style_entry(std::string const &key,
+                                                 std::string const &value)
+{
+    if (key == "osm_version") {
+        m_metadata_fields.set_version(true);
+    } else if (key == "osm_timestamp") {
+        m_metadata_fields.set_timestamp(true);
+    } else if (key == "osm_changeset") {
+        m_metadata_fields.set_changeset(true);
+    } else if (key == "osm_uid") {
+        m_metadata_fields.set_uid(true);
+    } else if (key == "osm_user") {
+        m_metadata_fields.set_user(true);
+    } else {
+        return false;
+    }
+    return true;
+}
+
 void gazetteer_style_t::add_style_entry(std::string const &key,
                                         std::string const &value,
                                         gazetteer_style_t::flag_t flags)
@@ -177,6 +190,13 @@ void gazetteer_style_t::add_style_entry(std::string const &key,
         }
     }
 
+    if (add_metadata_style_entry(key, value)) {
+        if (!value.empty()) {
+            throw std::runtime_error("Style error. Rules for OSM metadata "
+                                     "attributes must have an empty value.\n");
+        }
+        return;
+    }
     if (value.empty()) {
         m_matcher.emplace_back(key, flags, matcher_t::MT_KEY);
     } else {
@@ -355,45 +375,6 @@ void gazetteer_style_t::process_tags(osmium::OSMObject const &o)
     } else if (postcode_fallback && postcode) {
         m_main.emplace_back("place", "postcode", SF_MAIN | SF_MAIN_FALLBACK);
     }
-
-    // add metadata fields as tags if enabled
-    if (m_metadata_enabled) {
-        if (o.version()) {
-            add_metadata_field_num<osmium::object_version_type>("osm_version", o.version());
-        }
-        if (o.uid()) {
-            add_metadata_field_num<osmium::user_id_type>("osm_uid", o.uid());
-        }
-
-        if (o.user() && *(o.user()) != '\0') {
-            std::string username = o.user();
-            add_metadata_field("osm_user", std::move(username));
-        }
-
-        if (o.changeset()) {
-            add_metadata_field_num<osmium::changeset_id_type>("osm_changeset", o.changeset());
-        }
-
-        if (o.timestamp()) {
-            add_metadata_field("osm_timestamp", std::move(o.timestamp().to_iso()));
-        }
-    }
-}
-
-void gazetteer_style_t::add_metadata_field(const std::string&& field, const std::string&& value) {
-    // We have to work with std::string, not char* because metadata fields converted to char*
-    // would require an allocation on heap and a cleanup at the end.
-    flag_t flag = find_flag(field.c_str(), value.c_str());
-    if (flag & SF_EXTRA) {
-        m_metadata.emplace_back(std::move(field), std::move(value));
-    }
-}
-
-template <typename T>
-void gazetteer_style_t::add_metadata_field_num(const std::string&& field, const T value) {
-    // This method is not linked to from outside this class. Therefore, it can stay in the source file.
-    std::string value_str = std::to_string(value);
-    add_metadata_field(std::move(field), std::move(value_str));
 }
 
 void gazetteer_style_t::copy_out(osmium::OSMObject const &o,
@@ -501,15 +482,30 @@ bool gazetteer_style_t::copy_out_maintag(pmaintag_t const &tag,
         buffer.finish_hash();
     }
     // extra tags
-    if (m_extra.empty() && m_metadata.empty()) {
+    if (m_extra.empty() && m_metadata_fields.none()) {
         buffer.add_null_column();
     } else {
         buffer.new_hash();
         for (auto const &entry : m_extra) {
             buffer.add_hash_elem(entry.first, entry.second);
         }
-        for (auto const &entry : m_metadata) {
-            buffer.add_hash_elem(entry.first, entry.second);
+        if (m_metadata_fields.version() && o.version()) {
+            buffer.add_hstore_num_noescape<osmium::object_version_type>(
+                "osm_version", o.version());
+        }
+        if (m_metadata_fields.uid() && o.uid()) {
+            buffer.add_hstore_num_noescape<osmium::user_id_type>("osm_uid", o.uid());
+        }
+        if (m_metadata_fields.user() && o.user() && *(o.user()) != '\0') {
+            buffer.add_hash_elem("osm_user", o.user());
+        }
+        if (m_metadata_fields.changeset() && o.changeset()) {
+            buffer.add_hstore_num_noescape<osmium::changeset_id_type>(
+                "osm_changeset", o.changeset());
+        }
+        if (m_metadata_fields.timestamp() && o.timestamp()) {
+            std::string timestamp = o.timestamp().to_iso();
+            buffer.add_hash_elem_noescape("osm_timestamp", timestamp.c_str());
         }
         buffer.finish_hash();
     }

gazetteer-style.hpp

@@ -7,14 +7,14 @@
 #include <vector>
 
 #include <boost/format.hpp>
+#include <osmium/osm/metadata_options.hpp>
 
 #include "db-copy.hpp"
 
 class gazetteer_style_t
 {
     using flag_t = uint16_t;
     using ptag_t = std::pair<char const *, char const *>;
-    using ptag_str_t = std::pair<const std::string, const std::string>;
     using pmaintag_t = std::tuple<char const *, char const *, flag_t>;
 
     enum style_flags
@@ -67,17 +67,14 @@ class gazetteer_style_t
 
     bool has_data() const { return !m_main.empty(); }
 
-    void set_metadata(const bool enabled);
-
 private:
+    bool add_metadata_style_entry(std::string const &key,
+                                  std::string const &value);
     void add_style_entry(std::string const &key, std::string const &value,
                          flag_t flags);
     flag_t parse_flags(std::string const &str);
     flag_t find_flag(char const *k, char const *v) const;
-    void add_metadata_field(const std::string&& field, const std::string&& value);
 
-    template <typename T>
-    void add_metadata_field_num(const std::string&& field, const T value);
     bool copy_out_maintag(pmaintag_t const &tag, osmium::OSMObject const &o,
                           std::string const &geom, db_copy_mgr_t &buffer);
     void clear();
@@ -95,8 +92,6 @@ class gazetteer_style_t
     std::vector<ptag_t> m_names;
     /// extratags to include
     std::vector<ptag_t> m_extra;
-    /// metadata fields to include
-    std::vector<ptag_str_t> m_metadata;
     /// addresstags to include
     std::vector<ptag_t> m_address;
     /// value of operator tag
@@ -106,8 +101,8 @@ class gazetteer_style_t
     /// True if there is an actual name to the object (not a ref).
     bool m_is_named;
 
-    /// enable writing of metadata as tags (osm_version, osm_timestamp, osm_uid, osm_user, osm_changeset)
-    bool m_metadata_enabled {false};
+    /// which metadata fields of the OSM objects should be written to the output
+    osmium::metadata_options m_metadata_fields{"none"};
 
     boost::format m_single_fmt{"%1%\t"};
 };

output-gazetteer.hpp

@@ -38,7 +38,6 @@ class output_gazetteer_t : public output_t
       osmium_buffer(PLACE_BUFFER_SIZE, osmium::memory::Buffer::auto_grow::yes)
     {
         m_style.load_style(options.style);
-        m_style.set_metadata(options.extra_attributes);
     }
 
     virtual ~output_gazetteer_t();