Skip to content
This repository was archived by the owner on Jan 13, 2026. It is now read-only.

Commit eeed31a

Browse files
rkettelerijrkettelerij
authored
Merge pull request #60 from PDOK/PDOK-17166
Implemented the use of partitions for the search_index table
2 parents b54ea47 + fa5d307 commit eeed31a

File tree

6 files changed

+52
-38
lines changed

6 files changed

+52
-38
lines changed

‎go.mod‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ require (
88
github.com/docker/go-connections v0.5.0
99
github.com/docker/go-units v0.5.0
1010
github.com/elnormous/contenttype v1.0.4
11-
github.com/getkin/kin-openapi v0.128.0
11+
github.com/getkin/kin-openapi v0.131.0
1212
github.com/go-chi/chi/v5 v5.1.0
1313
github.com/go-chi/cors v1.2.1
1414
github.com/go-playground/validator/v10 v10.22.1
@@ -63,7 +63,6 @@ require (
6363
github.com/go-playground/universal-translator v0.18.1 // indirect
6464
github.com/gogo/protobuf v1.3.2 // indirect
6565
github.com/gorilla/mux v1.8.1 // indirect
66-
github.com/invopop/yaml v0.3.1 // indirect
6766
github.com/jackc/pgpassfile v1.0.0 // indirect
6867
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
6968
github.com/jackc/puddle/v2 v2.2.2 // indirect
@@ -80,6 +79,8 @@ require (
8079
github.com/moby/term v0.5.0 // indirect
8180
github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect
8281
github.com/morikuni/aec v1.0.0 // indirect
82+
github.com/oasdiff/yaml v0.0.0-20250309154309-f31be36b4037 // indirect
83+
github.com/oasdiff/yaml3 v0.0.0-20250309153720-d2182401db90 // indirect
8384
github.com/opencontainers/go-digest v1.0.0 // indirect
8485
github.com/opencontainers/image-spec v1.1.0 // indirect
8586
github.com/perimeterx/marshmallow v1.1.5 // indirect

‎go.sum‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -53,8 +53,8 @@ github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2
5353
github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
5454
github.com/gabriel-vasile/mimetype v1.4.4 h1:QjV6pZ7/XZ7ryI2KuyeEDE8wnh7fHP9YnQy+R0LnH8I=
5555
github.com/gabriel-vasile/mimetype v1.4.4/go.mod h1:JwLei5XPtWdGiMFB5Pjle1oEeoSeEuJfJE+TtfvdB/s=
56-
github.com/getkin/kin-openapi v0.128.0 h1:jqq3D9vC9pPq1dGcOCv7yOp1DaEe7c/T1vzcLbITSp4=
57-
github.com/getkin/kin-openapi v0.128.0/go.mod h1:OZrfXzUfGrNbsKj+xmFBx6E5c6yH3At/tAKSc2UszXM=
56+
github.com/getkin/kin-openapi v0.131.0 h1:NO2UeHnFKRYhZ8wg6Nyh5Cq7dHk4suQQr72a4pMrDxE=
57+
github.com/getkin/kin-openapi v0.131.0/go.mod h1:3OlG51PCYNsPByuiMB0t4fjnNlIDnaEDsjiKUV8nL58=
5858
github.com/go-chi/chi/v5 v5.1.0 h1:acVI1TYaD+hhedDJ3r54HyA6sExp3HfXq7QWEEY/xMw=
5959
github.com/go-chi/chi/v5 v5.1.0/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8=
6060
github.com/go-chi/cors v1.2.1 h1:xEC8UT3Rlp2QuWNEr4Fs/c2EAGVKBwy/1vHx3bppil4=
@@ -103,8 +103,6 @@ github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUq
103103
github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg=
104104
github.com/iancoleman/strcase v0.3.0 h1:nTXanmYxhfFAMjZL34Ov6gkzEsSJZ5DbhxWjvSASxEI=
105105
github.com/iancoleman/strcase v0.3.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho=
106-
github.com/invopop/yaml v0.3.1 h1:f0+ZpmhfBSS4MhG+4HYseMdJhoeeopbSKbq5Rpeelso=
107-
github.com/invopop/yaml v0.3.1/go.mod h1:PMOp3nn4/12yEZUFfmOuNHJsZToEEOwoWsT+D81KkeA=
108106
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
109107
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
110108
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
@@ -154,6 +152,10 @@ github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A=
154152
github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc=
155153
github.com/nicksnyder/go-i18n/v2 v2.4.1 h1:zwzjtX4uYyiaU02K5Ia3zSkpJZrByARkRB4V3YPrr0g=
156154
github.com/nicksnyder/go-i18n/v2 v2.4.1/go.mod h1:++Pl70FR6Cki7hdzZRnEEqdc2dJt+SAGotyFg/SvZMk=
155+
github.com/oasdiff/yaml v0.0.0-20250309154309-f31be36b4037 h1:G7ERwszslrBzRxj//JalHPu/3yz+De2J+4aLtSRlHiY=
156+
github.com/oasdiff/yaml v0.0.0-20250309154309-f31be36b4037/go.mod h1:2bpvgLBZEtENV5scfDFEtB/5+1M4hkQhDQrccEJ/qGw=
157+
github.com/oasdiff/yaml3 v0.0.0-20250309153720-d2182401db90 h1:bQx3WeLcUWy+RletIKwUIt4x3t8n2SxavmoclizMb8c=
158+
github.com/oasdiff/yaml3 v0.0.0-20250309153720-d2182401db90/go.mod h1:y5+oSEHCPT/DGrS++Wc/479ERge0zTFxaF8PbGKcg2o=
157159
github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
158160
github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
159161
github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug=

‎internal/etl/etl.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ type Load interface {
3636
Init(index string, srid int, lang language.Tag) error
3737

3838
// Load records into search index
39-
Load(records []t.SearchIndexRecord, index string) (int64, error)
39+
Load(collectionID string, records []t.SearchIndexRecord, index string) (int64, error)
4040

4141
// Optimize once ETL is completed (optionally)
4242
Optimize() error
@@ -105,7 +105,7 @@ func ImportFile(collection config.GeoSpatialCollection, searchIndex string, file
105105
return fmt.Errorf("failed to transform raw records to search index records: %w", err)
106106
}
107107
log.Printf("transform completed, %d source records transformed into %d target records", sourceRecordCount, len(targetRecords))
108-
loaded, err := target.Load(targetRecords, searchIndex)
108+
loaded, err := target.Load(collection.ID, targetRecords, searchIndex)
109109
if err != nil {
110110
return fmt.Errorf("failed loading records into target: %w", err)
111111
}

‎internal/etl/etl_test.go‎

Lines changed: 17 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -182,38 +182,31 @@ func terminateContainer(ctx context.Context, t *testing.T, container testcontain
182182

183183
func insertTestData(ctx context.Context, conn string) error {
184184
db, err := pgx.Connect(ctx, conn)
185+
185186
if err != nil {
186187
return fmt.Errorf("unable to connect to database: %w", err)
187188
}
188189
defer db.Close(ctx)
189190

190-
// TODO: Disabled since partitioning is disabled for now (see)
191-
// // Create required partitions for testData
192-
// //nolint:misspell
193-
// partitions := `
194-
// create table search_index_addres partition of search_index
195-
// for values in ('adres');
196-
// -- partition by list(collection_version);
197-
// create table search_index_weg partition of search_index
198-
// for values in ('weg');
199-
// -- partition by list(collection_version);
200-
// `
201-
//
202-
// _, err = db.Exec(ctx, partitions)
203-
// if err != nil {
204-
// log.Printf("Error creating partitions: %v\n", err)
205-
// }
206-
207-
// language=postgresql
191+
var partitions = [3]string{"addresses", "roads"}
192+
193+
for i := range partitions {
194+
partition := `create table if not exists search_index_` + partitions[i] + ` partition of search_index for values in ('` + partitions[i] + `');`
195+
_, err = db.Exec(ctx, partition)
196+
if err != nil {
197+
log.Printf("Error creating partition: %s Error: %v\n", partitions[i], err)
198+
}
199+
}
200+
208201
testData := `
209202
insert into search_index(feature_id, collection_id, collection_version, display_name, suggest, geometry_type, bbox)
210203
values
211-
('408f5e13', 'adres', 1, 'Daendelsweg 4A, 7315AJ Apeldoorn', 'Daendelsweg 4A, 7315AJ Apeldoorn', 'POINT' , 'POLYGON((-180 -90, -180 90, 180 90, 180 -90, -180 -90))'),
212-
('408f5e13', 'adres', 1, 'Daendelsweg 4A, 7315AJ Apeldoorn', 'Daendelsweg 4A, 7315AJ Apeldoorn', 'POINT' , 'POLYGON((-180 -90, -180 90, 180 90, 180 -90, -180 -90))'),
213-
('408f5e13', 'adres', 1, 'Daendelsweg 4A, 7315AJ Apeldoorn', 'Daendelsweg 4A, 7315AJ Apeldoorn', 'POINT' , 'POLYGON((-180 -90, -180 90, 180 90, 180 -90, -180 -90))'),
214-
('408f5e13', 'adres', 1, 'Daendelsweg 4A, 7315AJ Apeldoorn', 'Daendelsweg 4A, 7315AJ Apeldoorn', 'POINT' , 'POLYGON((-180 -90, -180 90, 180 90, 180 -90, -180 -90))'),
215-
('1e99b620', 'weg' , 2, 'Daendelsweg, 7315AJ Apeldoorn' , 'Daendelsweg 4A, 7315AJ Apeldoorn', 'LINESTRING', 'POLYGON((-180 -90, -180 90, 180 90, 180 -90, -180 -90))'),
216-
('1e99b620', 'weg' , 2, 'Daendelsweg, 7315AJ Apeldoorn' , 'Daendelsweg 4A, 7315AJ Apeldoorn', 'LINESTRING', 'POLYGON((-180 -90, -180 90, 180 90, 180 -90, -180 -90))');
204+
('408f5e13', 'addresses', 1, 'Daendelsweg 4A, 7315AJ Apeldoorn', 'Daendelsweg 4A, 7315AJ Apeldoorn', 'POINT' , 'POLYGON((-180 -90, -180 90, 180 90, 180 -90, -180 -90))'),
205+
('408f5e13', 'addresses', 1, 'Daendelsweg 4A, 7315AJ Apeldoorn', 'Daendelsweg 4A, 7315AJ Apeldoorn', 'POINT' , 'POLYGON((-180 -90, -180 90, 180 90, 180 -90, -180 -90))'),
206+
('408f5e13', 'addresses', 1, 'Daendelsweg 4A, 7315AJ Apeldoorn', 'Daendelsweg 4A, 7315AJ Apeldoorn', 'POINT' , 'POLYGON((-180 -90, -180 90, 180 90, 180 -90, -180 -90))'),
207+
('408f5e13', 'addresses', 1, 'Daendelsweg 4A, 7315AJ Apeldoorn', 'Daendelsweg 4A, 7315AJ Apeldoorn', 'POINT' , 'POLYGON((-180 -90, -180 90, 180 90, 180 -90, -180 -90))'),
208+
('1e99b620', 'roads' , 2, 'Daendelsweg, 7315AJ Apeldoorn' , 'Daendelsweg 4A, 7315AJ Apeldoorn', 'LINESTRING', 'POLYGON((-180 -90, -180 90, 180 90, 180 -90, -180 -90))'),
209+
('1e99b620', 'roads' , 2, 'Daendelsweg, 7315AJ Apeldoorn' , 'Daendelsweg 4A, 7315AJ Apeldoorn', 'LINESTRING', 'POLYGON((-180 -90, -180 90, 180 90, 180 -90, -180 -90))');
217210
`
218211

219212
_, err = db.Exec(ctx, testData)

‎internal/etl/load/postgres.go‎

Lines changed: 24 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,10 @@ func NewPostgres(dbConn string) (*Postgres, error) {
2121
if err != nil {
2222
return nil, fmt.Errorf("unable to connect to database: %w", err)
2323
}
24+
err = createExtensions(ctx, db)
25+
if err != nil {
26+
return nil, err
27+
}
2428
// add support for Go <-> PostGIS conversions
2529
if err := pgxgeom.Register(ctx, db); err != nil {
2630
return nil, err
@@ -32,7 +36,14 @@ func (p *Postgres) Close() {
3236
_ = p.db.Close(p.ctx)
3337
}
3438

35-
func (p *Postgres) Load(records []t.SearchIndexRecord, index string) (int64, error) {
39+
func (p *Postgres) Load(collectionID string, records []t.SearchIndexRecord, index string) (int64, error) {
40+
partition := `create table if not exists search_index_` + collectionID +
41+
` partition of search_index for values in ('` + collectionID + `');`
42+
_, err := p.db.Exec(p.ctx, partition)
43+
if err != nil {
44+
return -1, fmt.Errorf("error creating partition: %s Error: %w", collectionID, err)
45+
}
46+
3647
loaded, err := p.db.CopyFrom(
3748
p.ctx,
3849
pgx.Identifier{index},
@@ -61,7 +72,6 @@ func (p *Postgres) Optimize() error {
6172
// Since not all DDL in Postgres support the "if not exists" syntax we use a bit
6273
// of pl/pgsql to make it idempotent.
6374
func (p *Postgres) Init(index string, srid int, lang language.Tag) error {
64-
6575
geometryType := `
6676
do $$ begin
6777
create type geometry_type as enum ('POINT', 'MULTIPOINT', 'LINESTRING', 'MULTILINESTRING', 'POLYGON', 'MULTIPOLYGON');
@@ -112,7 +122,8 @@ func (p *Postgres) Init(index string, srid int, lang language.Tag) error {
112122
geometry geometry(point, %[2]d) null,
113123
ts tsvector generated always as (to_tsvector('custom_dict', suggest)) stored,
114124
primary key (id, collection_id, collection_version)
115-
) -- partition by list(collection_id);`, index, srid) // TODO partitioning comes later
125+
) partition by list(collection_id);`, index, srid)
126+
116127
_, err = p.db.Exec(p.ctx, searchIndexTable)
117128
if err != nil {
118129
return fmt.Errorf("error creating search index table: %w", err)
@@ -154,3 +165,13 @@ func (p *Postgres) Init(index string, srid int, lang language.Tag) error {
154165

155166
return err
156167
}
168+
169+
func createExtensions(ctx context.Context, db *pgx.Conn) error {
170+
for _, ext := range []string{"postgis", "unaccent"} {
171+
_, err := db.Exec(ctx, `create extension if not exists `+ext+`;`)
172+
if err != nil {
173+
return fmt.Errorf("error creating %s extension: %w", ext, err)
174+
}
175+
}
176+
return nil
177+
}

‎tests/testdata/sql/init-db.sql‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,2 @@
11
DROP DATABASE IF EXISTS test_db;
22
CREATE DATABASE test_db;
3-
\connect test_db;
4-
CREATE EXTENSION IF NOT EXISTS postgis;
5-
CREATE EXTENSION IF NOT EXISTS unaccent;

0 commit comments

Comments
 (0)