Skip to content

Commit b9cf333

Browse files
benoitdm-oslandiabenoitdm-oslandia
committed
feat(CREATE): implements AddTableFeature in catalog_db
1 parent 1aeb5c1 commit b9cf333

File tree

7 files changed

+338
-23
lines changed

7 files changed

+338
-23
lines changed

‎internal/api/api.go‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -82,9 +82,9 @@ const (
8282
ErrMsgFunctionNotFound = "Function not found: %v"
8383
ErrMsgFunctionAccess = "Unable to access Function: %v"
8484
ErrMsgInvalidParameterValue = "Invalid value for parameter %v: %v"
85-
ErrMsgInvalidQuery = "Invalid query parameters"
85+
ErrMsgInvalidQuery = "Invalid query parameters"
8686
ErrMsgDataReadError = "Unable to read data from: %v"
87-
ErrMsgDataWriteError = "Unable to write data to: %v"
87+
ErrMsgDataWriteError = "Unable to write data to: %v"
8888
ErrMsgNoDataRead = "No data read from: %v"
8989
ErrMsgRequestTimeout = "Maximum time exceeded. Request cancelled."
9090
)
@@ -645,6 +645,7 @@ func PathItem(name string, fid string) string {
645645

646646
var Db2OpenapiFormatMap = map[string]string{
647647
"int": "integer",
648+
"int4": "integer",
648649
"long": "int64",
649650
"text": "string",
650651
}

‎internal/data/catalog_db.go‎

Lines changed: 68 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -155,8 +155,8 @@ func (cat *catalogDB) Tables() ([]*Table, error) {
155155
}
156156

157157
func (cat *catalogDB) TableReload(name string) {
158-
tbl, ok := cat.tableMap[name]
159-
if !ok {
158+
tbl, err := cat.TableByName(name)
159+
if err != nil {
160160
return
161161
}
162162
// load extent (which may change over time
@@ -196,7 +196,11 @@ func (cat *catalogDB) TableByName(name string) (*Table, error) {
196196
cat.refreshTables(false)
197197
tbl, ok := cat.tableMap[name]
198198
if !ok {
199-
return nil, nil
199+
tbl, ok := cat.tableMap["public."+name]
200+
if !ok {
201+
return nil, nil
202+
}
203+
return tbl, nil
200204
}
201205
return tbl, nil
202206
}
@@ -237,7 +241,60 @@ func (cat *catalogDB) TableFeature(ctx context.Context, name string, id string,
237241
}
238242

239243
func (cat *catalogDB) AddTableFeature(ctx context.Context, tableName string, jsonData []byte) (int64, error) {
240-
panic("catalogDB::AddTableFeature unimplemented")
244+
var schemaObject geojsonFeatureData
245+
err := json.Unmarshal(jsonData, &schemaObject)
246+
if err != nil {
247+
return -9999, err
248+
}
249+
var columnStr string
250+
var placementStr string
251+
var values []interface{}
252+
253+
tbl, err := cat.TableByName(tableName)
254+
if err != nil {
255+
return -9999, err
256+
}
257+
var i = 0
258+
for c, t := range tbl.DbTypes {
259+
if c == tbl.IDColumn {
260+
continue // ignore id column
261+
}
262+
263+
i++
264+
columnStr += c
265+
placementStr += fmt.Sprintf("$%d", i)
266+
if t.Type == "int4" {
267+
values = append(values, int(schemaObject.Props[c].(float64)))
268+
} else {
269+
values = append(values, schemaObject.Props[c])
270+
}
271+
272+
if i < len(tbl.Columns)-1 {
273+
columnStr += ", "
274+
placementStr += ", "
275+
}
276+
277+
}
278+
279+
i++
280+
columnStr += ", " + tbl.GeometryColumn
281+
placementStr += fmt.Sprintf(", ST_GeomFromGeoJSON($%d)", i)
282+
geomJson, _ := schemaObject.Geom.MarshalJSON()
283+
values = append(values, geomJson)
284+
285+
sqlStatement := fmt.Sprintf(`
286+
INSERT INTO %s (%s)
287+
VALUES (%s)
288+
RETURNING %s`,
289+
tbl.ID, columnStr, placementStr, tbl.IDColumn)
290+
291+
var id int64 = -1
292+
err = cat.dbconn.QueryRow(ctx, sqlStatement, values...).Scan(&id)
293+
if err != nil {
294+
return -9999, err
295+
}
296+
297+
return id, nil
241298
}
242299

243300
func (cat *catalogDB) refreshTables(force bool) {
@@ -391,7 +448,6 @@ func readFeatures(ctx context.Context, db *pgxpool.Pool, sql string, idColIndex
391448
return readFeaturesWithArgs(ctx, db, sql, nil, idColIndex, propCols)
392449
}
393450

394-
//nolint:unused
395451
func readFeaturesWithArgs(ctx context.Context, db *pgxpool.Pool, sql string, args []interface{}, idColIndex int, propCols []string) ([]string, error) {
396452
start := time.Now()
397453
rows, err := db.Query(ctx, sql, args...)
@@ -447,7 +503,7 @@ func scanFeature(rows pgx.Rows, idColIndex int, propNames []string) string {
447503
id = fmt.Sprintf("%v", vals[idColIndex+propOffset])
448504
}
449505

450-
props := extractProperties(vals, propOffset, propNames)
506+
props := extractProperties(vals, idColIndex, propOffset, propNames)
451507

452508
//--- geom value is expected to be a GeoJSON string or geojson object
453509
//--- convert NULL to an empty string
@@ -462,9 +518,12 @@ func scanFeature(rows pgx.Rows, idColIndex int, propNames []string) string {
462518
}
463519
}
464520

465-
func extractProperties(vals []interface{}, propOffset int, propNames []string) map[string]interface{} {
521+
func extractProperties(vals []interface{}, idColIndex int, propOffset int, propNames []string) map[string]interface{} {
466522
props := make(map[string]interface{})
467523
for i, name := range propNames {
524+
if i == idColIndex {
525+
continue
526+
}
468527
// offset vals index by 2 to skip geom, id
469528
props[name] = toJSONValue(vals[i+propOffset])
470529
//fmt.Printf("%v: %v\n", name, val)
@@ -591,13 +650,15 @@ func makeFeatureJSON(id string, geom string, props map[string]interface{}) strin
591650
return jsonStr
592651
}
593652

653+
// TODO should be exported in catalog.go
594654
type geojsonFeatureData struct {
595655
Type string `json:"type"`
596656
ID string `json:"id,omitempty"`
597657
Geom *geojson.Geometry `json:"geometry"`
598658
Props map[string]interface{} `json:"properties"`
599659
}
600660

661+
// TODO should be exported in catalog.go
601662
func makeGeojsonFeatureJSON(id string, geom geojson.Geometry, props map[string]interface{}) string {
602663
featData := geojsonFeatureData{
603664
Type: "Feature",

‎internal/data/catalog_db_fun.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -282,6 +282,6 @@ func scanDataRow(rows pgx.Rows, hasID bool, propNames []string) map[string]inter
282282
//fmt.Println(vals)
283283

284284
//fmt.Println(geom)
285-
props := extractProperties(vals, 0, propNames)
285+
props := extractProperties(vals, -1, 0, propNames)
286286
return props
287287
}

‎internal/data/catalog_mock.go‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -92,9 +92,9 @@ func newCatalogMock() CatalogMock {
9292
}
9393

9494
tableData := map[string][]*featureMock{}
95-
tableData["mock_a"] = makePointFeatures(layerA.Extent, 3, 3)
96-
tableData["mock_b"] = makePointFeatures(layerB.Extent, 10, 10)
97-
tableData["mock_c"] = makePointFeatures(layerC.Extent, 100, 100)
95+
tableData["mock_a"] = MakePointFeatures(layerA.Extent, 3, 3)
96+
tableData["mock_b"] = MakePointFeatures(layerB.Extent, 10, 10)
97+
tableData["mock_c"] = MakePointFeatures(layerC.Extent, 100, 100)
9898

9999
var tables []*Table
100100
tables = append(tables, layerA)
@@ -306,7 +306,7 @@ func MakeFeatureMockPointAsJSON(id int, x float64, y float64, columns []string)
306306
return feat.toJSON(columns)
307307
}
308308

309-
func makePointFeatures(extent Extent, nx int, ny int) []*featureMock {
309+
func MakePointFeatures(extent Extent, nx int, ny int) []*featureMock {
310310
basex := extent.Minx
311311
basey := extent.Miny
312312
dx := (extent.Maxx - extent.Minx) / float64(nx)

‎internal/service/db_test/handler_db_test.go‎

Lines changed: 149 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,149 @@
1+
package db_test
2+
3+
import (
4+
"context"
5+
"encoding/json"
6+
"fmt"
7+
"io/ioutil"
8+
"net/http"
9+
"os"
10+
"strconv"
11+
"strings"
12+
"testing"
13+
14+
"github.com/CrunchyData/pg_featureserv/internal/api"
15+
"github.com/CrunchyData/pg_featureserv/internal/data"
16+
"github.com/CrunchyData/pg_featureserv/internal/service"
17+
"github.com/CrunchyData/pg_featureserv/util"
18+
"github.com/jackc/pgx/v4/pgxpool"
19+
"github.com/paulmach/orb/geojson"
20+
)
21+
22+
var hTest util.HttpTesting
23+
var db *pgxpool.Pool
24+
var cat data.Catalog
25+
26+
// extracted from catalog_db.go
27+
// TODO should be imported from catalog.go
28+
type geojsonFeatureData struct {
29+
Type string `json:"type"`
30+
ID string `json:"id,omitempty"`
31+
Geom *geojson.Geometry `json:"geometry"`
32+
Props map[string]interface{} `json:"properties"`
33+
}
34+
35+
func TestMain(m *testing.M) {
36+
db = util.CreateTestDb()
37+
defer util.CloseTestDb(db)
38+
39+
cat = data.CatDBInstance()
40+
service.SetCatalogInstance(cat)
41+
42+
hTest = util.MakeHttpTesting("http://test", "/pg_featureserv", service.InitRouter("/pg_featureserv"))
43+
service.Initialize()
44+
45+
os.Exit(m.Run())
46+
}
47+
48+
func TestProperDbInit(t *testing.T) {
49+
tables, _ := cat.Tables()
50+
util.Equals(t, 2, len(tables), "# table in DB")
51+
}
52+
53+
func TestTestPropertiesAllFromDb(t *testing.T) {
54+
/*rr := hTest.DoRequest(t, "/collections/mock_a/items?limit=2")
55+
56+
var v FeatureCollection
57+
errUnMarsh := json.Unmarshal(hTest.ReadBody(rr), &v)
58+
util.Assert(t, errUnMarsh == nil, fmt.Sprintf("%v", errUnMarsh))
59+
60+
// Note that JSON numbers are read as float64
61+
util.Equals(t, 2, len(v.Features), "# features")
62+
util.Equals(t, 4, len(v.Features[0].Props), "feature 1 # properties")
63+
64+
util.Equals(t, "propA", v.Features[0].Props["prop_a"], "feature 1 # property A")
65+
util.Equals(t, 1.0, v.Features[0].Props["prop_b"], "feature 1 # property B")
66+
util.Equals(t, "propC", v.Features[0].Props["prop_c"], "feature 1 # property C")
67+
util.Equals(t, 1.0, v.Features[0].Props["prop_d"], "feature 1 # property D")*/
68+
}
69+
70+
func TestCreateFeatureWithBadGeojsonInputDb(t *testing.T) {
71+
var header = make(http.Header)
72+
header.Add("Content-Type", "application/geo+json")
73+
74+
jsonStr := `[{
75+
"id": 101,
76+
"name": "Test",
77+
"email": "[email protected]"
78+
}, {
79+
"id": 102,
80+
"name": "Sample",
81+
"email": "[email protected]"
82+
}]`
83+
84+
rr := hTest.DoRequestMethodStatus(t, "POST", "/collections/mock_a/items", []byte(jsonStr), header, http.StatusInternalServerError)
85+
86+
util.Equals(t, http.StatusInternalServerError, rr.Code, "Should have failed")
87+
util.Assert(t, strings.Index(rr.Body.String(), fmt.Sprintf(api.ErrMsgCreateFeatureNotConform+"\n", "mock_a")) == 0, "Should have failed with not conform")
88+
}
89+
90+
func TestCreateFeatureDb(t *testing.T) {
91+
var header = make(http.Header)
92+
header.Add("Content-Type", "application/geo+json")
93+
94+
//--- retrieve max feature id before insert
95+
var features []string
96+
params := data.QueryParam{Limit: 100, Offset: 0, Crs: 4326}
97+
features, _ = cat.TableFeatures(context.Background(), "mock_a", &params)
98+
maxIdBefore := len(features)
99+
100+
//--- generate json from new object
101+
tables, _ := cat.Tables()
102+
var cols []string
103+
for _, t := range tables {
104+
if t.ID == "public.mock_a" {
105+
for _, c := range t.Columns {
106+
if c != "id" {
107+
cols = append(cols, c)
108+
}
109+
}
110+
break
111+
}
112+
}
113+
jsonStr := data.MakeFeatureMockPointAsJSON(99, 12, 34, cols)
114+
fmt.Println(jsonStr)
115+
116+
// -- do the request call but we have to force the catalogInstance to db during this operation
117+
rr := hTest.DoPostRequest(t, "/collections/mock_a/items", []byte(jsonStr), header)
118+
119+
loc := rr.Header().Get("Location")
120+
121+
//--- retrieve max feature id after insert
122+
features, _ = cat.TableFeatures(context.Background(), "mock_a", &params)
123+
maxIdAfter := len(features)
124+
125+
util.Assert(t, maxIdAfter > maxIdBefore, "# feature in db")
126+
util.Assert(t, len(loc) > 1, "Header location must not be empty")
127+
util.Equals(t, fmt.Sprintf("http://test/collections/mock_a/items/%d", maxIdAfter), loc,
128+
"Header location must contain valid data")
129+
130+
// check if it can be read
131+
checkItem(t, maxIdAfter)
132+
}
133+
134+
// check if item is available and is not empty
135+
// copy from service/handler_test.go
136+
func checkItem(t *testing.T, id int) {
137+
path := fmt.Sprintf("/collections/mock_a/items/%d", id)
138+
resp := hTest.DoRequest(t, path)
139+
body, _ := ioutil.ReadAll(resp.Body)
140+
141+
var v geojsonFeatureData
142+
errUnMarsh := json.Unmarshal(body, &v)
143+
util.Assert(t, errUnMarsh == nil, fmt.Sprintf("%v", errUnMarsh))
144+
145+
util.Equals(t, "Feature", v.Type, "feature type")
146+
actId, _ := strconv.Atoi(v.ID)
147+
util.Equals(t, id, actId, "feature id")
148+
util.Equals(t, 4, len(v.Props), "# feature props")
149+
}

‎internal/service/handler.go‎

Lines changed: 13 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -435,7 +435,9 @@ func getCreateItemSchema(ctx context.Context, table *data.Table) (openapi3.Schem
435435
requiredTypeKeys := make([]string, 0, len(table.DbTypes))
436436

437437
for k := range table.DbTypes {
438-
requiredTypeKeys = append(requiredTypeKeys, k)
438+
if k != table.IDColumn {
439+
requiredTypeKeys = append(requiredTypeKeys, k)
440+
}
439441
}
440442
sort.Strings(requiredTypeKeys)
441443

@@ -451,14 +453,16 @@ func getCreateItemSchema(ctx context.Context, table *data.Table) (openapi3.Schem
451453
// update properties by their name and type
452454
props.Properties = make(map[string]*openapi3.SchemaRef)
453455
for k, v := range table.DbTypes {
454-
propType := v.Type
455-
if api.Db2OpenapiFormatMap[v.Type] != "" {
456-
propType = api.Db2OpenapiFormatMap[v.Type]
457-
}
458-
props.Properties[k] = &openapi3.SchemaRef{
459-
Value: &openapi3.Schema{
460-
Type: propType,
461-
},
456+
if k != table.IDColumn {
457+
propType := v.Type
458+
if api.Db2OpenapiFormatMap[v.Type] != "" {
459+
propType = api.Db2OpenapiFormatMap[v.Type]
460+
}
461+
props.Properties[k] = &openapi3.SchemaRef{
462+
Value: &openapi3.Schema{
463+
Type: propType,
464+
},
465+
}
462466
}
463467
}
464468

0 commit comments

Comments
 (0)