Add vector_layers JSON support

systemed · systemed · commit 9202e38d7d43 · 2018-02-17T23:24:51.000Z
diff --git a/include/osm_object.h b/include/osm_object.h
@@ -9,6 +9,9 @@
 #include "geomtypes.h"
 #include "osm_store.h"
 #include "output_object.h"
+#include "rapidjson/document.h"
+#include "rapidjson/writer.h"
+#include "rapidjson/stringbuffer.h"
 
 // Protobuf
 #include "osmformat.pb.h"
@@ -22,6 +25,7 @@ struct LayerDef {
 	double simplifyLevel;
 	double simplifyLength;
 	double simplifyRatio;
+	std::map<std::string, uint> attributeMap;
 };
 
 /*
@@ -220,6 +224,11 @@ class OSMObject {
 	void Attribute(const std::string &key, const std::string &val);
 	void AttributeNumeric(const std::string &key, const float val);
 	void AttributeBoolean(const std::string &key, const bool val);
+
+	// ----	vector_layers metadata entry
+
+	void setVectorLayerMetadata(const uint_least8_t layer, const std::string &key, const uint type);
+	std::string serialiseLayerJSON();
 };
 
 #endif //_OSM_OBJECT_H
diff --git a/include/read_shp.h b/include/read_shp.h
@@ -7,6 +7,7 @@
 #include <map>
 #include "geomtypes.h"
 #include "output_object.h"
+#include "osm_object.h"
 
 // Shapelib
 #include "shapefil.h"
@@ -24,13 +25,15 @@ void addToTileIndexPolyline(OutputObject &oo, std::map< uint, std::vector<Output
 void addShapefileAttributes(DBFHandle &dbf, OutputObject &oo, int recordNum, std::unordered_map<int,std::string> &columnMap, std::unordered_map<int,int> &columnTypeMap);
 
 /// Read shapefile, and create OutputObjects for all objects within the specified bounding box
-void readShapefile(std::string filename, 
+void readShapefile(std::string filename,
                    std::vector<std::string> &columns,
-                   Box &clippingBox, 
-                   std::map< uint, std::vector<OutputObject> > &tileIndex, 
-                   std::vector<Geometry> &cachedGeometries, std::map< uint, std::string > &cachedGeometryNames,
+                   Box &clippingBox,
+                   std::map< uint, std::vector<OutputObject> > &tileIndex,
+                   std::vector<Geometry> &cachedGeometries,
+                   OSMObject &osmObject,
                    uint baseZoom, uint layerNum, std::string &layerName,
-                   bool isIndexed, std::map<std::string,RTree> &indices, std::string &indexName);
+                   bool isIndexed,
+                   std::string &indexName);
 
 #endif //_READ_SHP_H
 
diff --git a/src/osm_object.cpp b/src/osm_object.cpp
@@ -1,5 +1,6 @@
 #include "osm_object.h"
 using namespace std;
+using namespace rapidjson;
 
 // ----	initialization routines
 
@@ -15,7 +16,7 @@ OSMObject::OSMObject(kaguya::State *luaPtr, map< string, RTree> *idxPtr, vector<
 // Define a layer (as read from the .json file)
 uint OSMObject::addLayer(string name, uint minzoom, uint maxzoom,
 		uint simplifyBelow, double simplifyLevel, double simplifyLength, double simplifyRatio, string writeTo) {
-	LayerDef layer = { name, minzoom, maxzoom, simplifyBelow, simplifyLevel, simplifyLength, simplifyRatio };
+	LayerDef layer = { name, minzoom, maxzoom, simplifyBelow, simplifyLevel, simplifyLength, simplifyRatio, std::map<std::string,uint>() };
 	layers.push_back(layer);
 	uint layerNum = layers.size()-1;
 	layerMap[name] = layerNum;
@@ -256,19 +257,59 @@ void OSMObject::Attribute(const string &key, const string &val) {
 	vector_tile::Tile_Value v;
 	v.set_string_value(val);
 	outputs[outputs.size()-1].addAttribute(key, v);
+	setVectorLayerMetadata(outputs[outputs.size()-1].layer, key, 0);
 }
 
 void OSMObject::AttributeNumeric(const string &key, const float val) {
 	if (outputs.size()==0) { cerr << "Can't add Attribute " << key << " if no Layer set" << endl; return; }
 	vector_tile::Tile_Value v;
 	v.set_float_value(val);
 	outputs[outputs.size()-1].addAttribute(key, v);
+	setVectorLayerMetadata(outputs[outputs.size()-1].layer, key, 1);
 }
 
 void OSMObject::AttributeBoolean(const string &key, const bool val) {
 	if (outputs.size()==0) { cerr << "Can't add Attribute " << key << " if no Layer set" << endl; return; }
 	vector_tile::Tile_Value v;
 	v.set_bool_value(val);
 	outputs[outputs.size()-1].addAttribute(key, v);
+	setVectorLayerMetadata(outputs[outputs.size()-1].layer, key, 2);
 }
 
+// Record attribute name/type for vector_layers table
+void OSMObject::setVectorLayerMetadata(const uint_least8_t layer, const string &key, const uint type) {
+	layers[layer].attributeMap[key] = type;
+}
+std::string OSMObject::serialiseLayerJSON() {
+	Document document;
+	document.SetObject();
+	Document::AllocatorType& allocator = document.GetAllocator();
+
+	Value layerArray(kArrayType);
+	for (auto it = layers.begin(); it != layers.end(); ++it) {
+		Value fieldObj(kObjectType);
+		for (auto jt = it->attributeMap.begin(); jt != it->attributeMap.end(); ++jt) {
+			Value k(jt->first.c_str(), allocator);
+			switch (jt->second) {
+				case 0: fieldObj.AddMember(k, "String" , allocator); break;
+				case 1:	fieldObj.AddMember(k, "Number" , allocator); break;
+				case 2:	fieldObj.AddMember(k, "Boolean", allocator); break;
+			}
+		}
+		Value layerObj(kObjectType);
+		Value name(it->name.c_str(), allocator);
+		layerObj.AddMember("id",      name,        allocator);
+		layerObj.AddMember("fields",  fieldObj,    allocator);
+		layerObj.AddMember("minzoom", it->minzoom, allocator);
+		layerObj.AddMember("maxzoom", it->maxzoom, allocator);
+		layerArray.PushBack(layerObj, allocator);
+	}
+
+	document.AddMember("vector_layers", layerArray, allocator);
+
+	StringBuffer buffer;
+	Writer<StringBuffer> writer(buffer);
+	document.Accept(writer);
+	string json(buffer.GetString(), buffer.GetSize());
+	return json;
+}
diff --git a/src/read_shp.cpp b/src/read_shp.cpp
@@ -64,15 +64,26 @@ void addToTileIndexPolyline(OutputObject &oo, map< uint, vector<OutputObject> >
 }
 
 // Read requested attributes from a shapefile, and encode into an OutputObject
-void addShapefileAttributes(DBFHandle &dbf, OutputObject &oo, int recordNum, unordered_map<int,string> &columnMap, unordered_map<int,int> &columnTypeMap) {
+void addShapefileAttributes(
+		DBFHandle &dbf,
+		OutputObject &oo,
+		int recordNum, unordered_map<int,string> &columnMap, unordered_map<int,int> &columnTypeMap,
+		OSMObject &osmObject) {
+
 	for (auto it : columnMap) {
 		int pos = it.first;
 		string key = it.second;
 		vector_tile::Tile_Value v;
 		switch (columnTypeMap[pos]) {
-			case 1:  v.set_int_value(DBFReadIntegerAttribute(dbf, recordNum, pos)); break;
-			case 2:  v.set_double_value(DBFReadDoubleAttribute(dbf, recordNum, pos)); break;
-			default: v.set_string_value(DBFReadStringAttribute(dbf, recordNum, pos)); break;
+			case 1:  v.set_int_value(DBFReadIntegerAttribute(dbf, recordNum, pos));
+			         osmObject.setVectorLayerMetadata(oo.layer, key, 1);
+			         break;
+			case 2:  v.set_double_value(DBFReadDoubleAttribute(dbf, recordNum, pos));
+			         osmObject.setVectorLayerMetadata(oo.layer, key, 1);
+			         break;
+			default: v.set_string_value(DBFReadStringAttribute(dbf, recordNum, pos));
+			         osmObject.setVectorLayerMetadata(oo.layer, key, 3);
+			         break;
 		}
 		oo.addAttribute(key, v);
 	}
@@ -84,9 +95,11 @@ void readShapefile(string filename,
                    vector<string> &columns,
                    Box &clippingBox, 
                    map< uint, vector<OutputObject> > &tileIndex, 
-                   vector<Geometry> &cachedGeometries, map< uint, string > &cachedGeometryNames,
+                   vector<Geometry> &cachedGeometries,
+				   OSMObject &osmObject,
                    uint baseZoom, uint layerNum, string &layerName,
-                   bool isIndexed, map<string,RTree> &indices, string &indexName) {
+                   bool isIndexed,
+				   string &indexName) {
 
 	// open shapefile
 	SHPHandle shp = SHPOpen(filename.c_str(), "rb");
@@ -124,12 +137,12 @@ void readShapefile(string filename,
 				uint tiley = latp2tiley(p.y(), baseZoom);
 				cachedGeometries.push_back(p);
 				OutputObject oo(CACHED_POINT, layerNum, cachedGeometries.size()-1);
-				addShapefileAttributes(dbf,oo,i,columnMap,columnTypeMap);
+				addShapefileAttributes(dbf,oo,i,columnMap,columnTypeMap,osmObject);
 				tileIndex[tilex*65536+tiley].push_back(oo);
 				if (isIndexed) {
 					uint id = cachedGeometries.size()-1;
-					geom::envelope(p, box); indices[layerName].insert(std::make_pair(box, id));
-					if (indexField>-1) { cachedGeometryNames[id]=DBFReadStringAttribute(dbf, i, indexField); }
+					geom::envelope(p, box); osmObject.indices->at(layerName).insert(std::make_pair(box, id));
+					if (indexField>-1) { osmObject.cachedGeometryNames->operator[](id)=DBFReadStringAttribute(dbf, i, indexField); }
 				}
 			}
 
@@ -146,12 +159,12 @@ void readShapefile(string filename,
 				for (MultiLinestring::const_iterator it = out.begin(); it != out.end(); ++it) {
 					cachedGeometries.push_back(*it);
 					OutputObject oo(CACHED_LINESTRING, layerNum, cachedGeometries.size()-1);
-					addShapefileAttributes(dbf,oo,i,columnMap,columnTypeMap);
+					addShapefileAttributes(dbf,oo,i,columnMap,columnTypeMap,osmObject);
 					addToTileIndexPolyline(oo, tileIndex, baseZoom, *it);
 					if (isIndexed) {
 						uint id = cachedGeometries.size()-1;
-						geom::envelope(*it, box); indices[layerName].insert(std::make_pair(box, id));
-						if (indexField>-1) { cachedGeometryNames[id]=DBFReadStringAttribute(dbf, i, indexField); }
+						geom::envelope(*it, box); osmObject.indices->at(layerName).insert(std::make_pair(box, id));
+						if (indexField>-1) { osmObject.cachedGeometryNames->operator[](id)=DBFReadStringAttribute(dbf, i, indexField); }
 					}
 				}
 			}
@@ -212,15 +225,15 @@ void readShapefile(string filename,
 				// create OutputObject
 				cachedGeometries.push_back(out);
 				OutputObject oo(CACHED_POLYGON, layerNum, cachedGeometries.size()-1);
-				addShapefileAttributes(dbf,oo,i,columnMap,columnTypeMap);
+				addShapefileAttributes(dbf,oo,i,columnMap,columnTypeMap,osmObject);
 				// add to tile index
 				geom::model::box<Point> box;
 				geom::envelope(out, box);
 				addToTileIndexByBbox(oo, tileIndex, baseZoom, box.min_corner().get<0>(), box.min_corner().get<1>(), box.max_corner().get<0>(), box.max_corner().get<1>());
 				if (isIndexed) {
 					uint id = cachedGeometries.size()-1;
-					indices[layerName].insert(std::make_pair(box, id));
-					if (indexField>-1) { cachedGeometryNames[id]=DBFReadStringAttribute(dbf, i, indexField); }
+					osmObject.indices->at(layerName).insert(std::make_pair(box, id));
+					if (indexField>-1) { osmObject.cachedGeometryNames->operator[](id)=DBFReadStringAttribute(dbf, i, indexField); }
 				}
 			}
 
diff --git a/src/shared_data.cpp b/src/shared_data.cpp
@@ -108,7 +108,10 @@ class SharedData
 				}
 				string indexName = it->value.HasMember("index_column") ? it->value["index_column"].GetString() : "";
 				readShapefile(it->value["source"].GetString(), sourceColumns, clippingBox, tileIndex,
-				              cachedGeometries, *osmObject.cachedGeometryNames, baseZoom, layerNum, layerName, indexed, *osmObject.indices, indexName);
+				              cachedGeometries,
+				              osmObject,
+				              baseZoom, layerNum, layerName, indexed,
+				              indexName);
 			}
 		}
 	}
diff --git a/src/tilemaker.cpp b/src/tilemaker.cpp
@@ -433,19 +433,6 @@ int main(int argc, char* argv[]) {
 		sharedData.mbtiles.writeMetadata("bounds",bounds.str());
 		sharedData.mbtiles.writeMetadata("minzoom",to_string(sharedData.startZoom));
 		sharedData.mbtiles.writeMetadata("maxzoom",to_string(sharedData.endZoom));
-		if (jsonConfig["settings"].HasMember("metadata")) {
-			const rapidjson::Value &md = jsonConfig["settings"]["metadata"];
-			for(rapidjson::Value::ConstMemberIterator it=md.MemberBegin(); it != md.MemberEnd(); ++it) {
-				if (it->value.IsString()) {
-					sharedData.mbtiles.writeMetadata(it->name.GetString(), it->value.GetString());
-				} else {
-					rapidjson::StringBuffer strbuf;
-					rapidjson::Writer<rapidjson::StringBuffer> writer(strbuf);
-					it->value.Accept(writer);
-					sharedData.mbtiles.writeMetadata(it->name.GetString(), strbuf.GetString());
-				}
-			}
-		}
 	}
 
 	// ----	Read all PBFs
@@ -804,10 +791,32 @@ int main(int argc, char* argv[]) {
 		sharedData.tileIndexForZoom = nullptr;
 	}
 
-	cout << endl << "Filled the tileset with good things at " << sharedData.outputFile << endl;
-	if (sqlite) { sharedData.mbtiles.close(); }
+	// ----	Close tileset
+
+	if (sqlite) {
+		// Write mbtiles 1.3+ json object
+		sharedData.mbtiles.writeMetadata("json", sharedData.osmObject.serialiseLayerJSON());
+
+		// Write user-defined metadata
+		if (jsonConfig["settings"].HasMember("metadata")) {
+			const rapidjson::Value &md = jsonConfig["settings"]["metadata"];
+			for(rapidjson::Value::ConstMemberIterator it=md.MemberBegin(); it != md.MemberEnd(); ++it) {
+				if (it->value.IsString()) {
+					sharedData.mbtiles.writeMetadata(it->name.GetString(), it->value.GetString());
+				} else {
+					rapidjson::StringBuffer strbuf;
+					rapidjson::Writer<rapidjson::StringBuffer> writer(strbuf);
+					it->value.Accept(writer);
+					sharedData.mbtiles.writeMetadata(it->name.GetString(), strbuf.GetString());
+				}
+			}
+		}
+		sharedData.mbtiles.close();
+	}
 	google::protobuf::ShutdownProtobufLibrary();
 
-	// ---- Call exit_function of Lua logic
+	// Call exit_function of Lua logic
 	luaState("if exit_function~=nil then exit_function() end");
+
+	cout << endl << "Filled the tileset with good things at " << sharedData.outputFile << endl;
 }

Original file line number	Diff line number	Diff line change
`@@ -108,7 +108,10 @@ class SharedData`
`108`	`108`	`}`
`109`	`109`	`string indexName = it->value.HasMember("index_column") ? it->value["index_column"].GetString() : "";`
`110`	`110`	`readShapefile(it->value["source"].GetString(), sourceColumns, clippingBox, tileIndex,`
`111`		`- cachedGeometries, osmObject.cachedGeometryNames, baseZoom, layerNum, layerName, indexed, osmObject.indices, indexName);`
	`111`	`+ cachedGeometries,`
	`112`	`+ osmObject,`
	`113`	`+ baseZoom, layerNum, layerName, indexed,`
	`114`	`+ indexName);`
`112`	`115`	`}`
`113`	`116`	`}`
`114`	`117`	`}`