fix the fake inprocess database, do various cleanups

srabraham · srabraham · commit bc8e6e7f5653 · 2025-05-30T10:22:50.000-04:00
the main thing needed was to set MaxOpenConns to 1 for the fake DB, because it doesn't support parallel queries: dolthub/go-mysql-server#1306 (comment) the only thing still not working in the fake DB is striking/unstriking of report entries, because of this issue I logged: dolthub/dolt#9268 but a fix is incoming: dolthub/go-mysql-server#3000
diff --git a/Dockerfile b/Dockerfile
@@ -27,7 +27,7 @@ RUN go run ./bin/fetchbuilddeps/fetchbuilddeps.go
 COPY ./ ./
 
 # Build the server
-RUN CGO_ENABLED=0 GOOS=linux go build -tags noinprocessdb -o /app/ranger-ims-go
+RUN CGO_ENABLED=0 GOOS=linux go build -tags nofakedb -o /app/ranger-ims-go
 
 # Allow IMS to bind to privileged port numbers
 RUN setcap "cap_net_bind_service=+ep" /app/ranger-ims-go
@@ -42,6 +42,7 @@ COPY --from=build /app/ranger-ims-go /opt/ims/bin/ims
 # Docker-specific default configuration
 ENV IMS_HOSTNAME="0.0.0.0"
 ENV IMS_PORT="80"
+ENV IMS_DB_STORE_TYPE="MariaDB"
 ENV IMS_DIRECTORY="ClubhouseDB"
 
 # Use a non-root user to run the server
diff --git a/api/integration/auth_test.go b/api/integration/auth_test.go
@@ -125,7 +125,7 @@ func TestGetAuthWithEvent(t *testing.T) {
 	require.NoError(t, resp.Body.Close())
 	resp = apisAdmin.editAccess(ctx, imsjson.EventsAccess{
 		eventName: imsjson.EventAccess{
-			Readers: []imsjson.AccessRule{imsjson.AccessRule{
+			Readers: []imsjson.AccessRule{{
 				Expression: "person:" + userAdminHandle,
 				Validity:   "always",
 			}},
diff --git a/api/integration/main_test.go b/api/integration/main_test.go
@@ -95,7 +95,6 @@ func setup(ctx context.Context, tempDir string) {
 	shared.cfg.AttachmentsStore.Local = conf.LocalAttachments{
 		Dir: tempRoot,
 	}
-	shared.cfg.Store.Type = conf.DBStoreTypeMaria
 	shared.cfg.Store.MariaDB.Database = "ims-" + rand.NonCryptoText()
 	shared.cfg.Store.MariaDB.Username = "rangers-" + rand.NonCryptoText()
 	shared.cfg.Store.MariaDB.Password = "password-" + rand.NonCryptoText()
diff --git a/bin/test_docker b/bin/test_docker
@@ -11,17 +11,24 @@ set -o errexit -o nounset -o pipefail
 cd "$(git rev-parse --show-toplevel)"
 
 exit_code=0
-# `docker compose up` will only rebuild from the local Dockerfile if no image
-# exists for the given name, e.g. ranger-ims-go. To change that behavior, you'd
-# add a --build arg to the `docker compose up`.
-echo "Starting Docker containers. Note this may not pick up any local changes"
+image="ranger-ims-go:${IMAGE_TAG:-latest}"
+docker image inspect "${image}" > /dev/null 2>&1 || exit_code=$?
+if [ "${exit_code}" -ne "0" ]; then
+    echo "No local version of ${image} was found, so docker compose will build it"
+else
+    # If you want the image to be rebuilt, then add "--build" to the "docker compose up"
+    # below. That is not desirable behavior for CI though.
+    echo "A local version of ${image} was found, so docker compose will not rebuild it"
+fi
+
+exit_code=0
 docker compose up --wait-timeout 30 --wait || exit_code=$?
 
 echo "Stopping and deleting those Docker containers"
 docker compose rm -fsv
 
 if [ "${exit_code}" -ne "0" ]; then
-    echo "App failed health check"
+    echo '!!! App failed health check !!!'
     exit ${exit_code}
 else
     echo "App started and passed health check"
diff --git a/cmd/serve.go b/cmd/serve.go
@@ -199,7 +199,7 @@ func mustInitConfig(envFileName string) *conf.IMSConfig {
 		must(err)
 	}
 	if v, ok := lookupEnv("IMS_DEPLOYMENT"); ok {
-		newCfg.Core.Deployment = strings.ToLower(v)
+		newCfg.Core.Deployment = conf.DeploymentType(strings.ToLower(v))
 	}
 	// This should really be called "IMS_REFRESH_TOKEN_LIFETIME". This name of
 	// "IMS_TOKEN_LIFETIME" predates our use of refresh tokens, and what it tried
@@ -246,7 +246,7 @@ func mustInitConfig(envFileName string) *conf.IMSConfig {
 		newCfg.Core.JWTSecret = v
 	}
 	if v, ok := lookupEnv("IMS_DB_STORE_TYPE"); ok {
-		newCfg.Store.Type = conf.DBStoreType(v)
+		newCfg.Store.Type = conf.DBStoreType(strings.ToLower(v))
 	}
 	if v, ok := lookupEnv("IMS_DB_HOST_NAME"); ok {
 		newCfg.Store.MariaDB.HostName = v
diff --git a/cmd/serve_test.go b/cmd/serve_test.go
@@ -67,7 +67,7 @@ func TestMustInitConfig(t *testing.T) {
 
 	assert.Equal(t, "host", cfg.Core.Host)
 	assert.Equal(t, int32(1234), cfg.Core.Port)
-	assert.Equal(t, "dev", cfg.Core.Deployment)
+	assert.Equal(t, conf.DeploymentTypeDev, cfg.Core.Deployment)
 	assert.Equal(t, 1000*time.Second, cfg.Core.RefreshTokenLifetime)
 	assert.Equal(t, 100*time.Second, cfg.Core.AccessTokenLifetime)
 	assert.Equal(t, 3*time.Minute, cfg.Core.CacheControlShort)
diff --git a/conf/imsconfig.go b/conf/imsconfig.go
@@ -50,14 +50,20 @@ func DefaultIMS() *IMSConfig {
 				HostName: "localhost",
 				HostPort: 3306,
 				Database: "ims",
+				// Some arbitrary value. We'll get errors from MariaDB if the server
+				// hits the DB with too many parallel requests.
+				MaxOpenConns: 20,
 			},
-			InProcess: DBStoreMaria{
+			Fake: DBStoreMaria{
 				HostName: "localhost",
 				// HostPort can be left as 0 for automatic port selection on startup
 				HostPort: 0,
 				Database: "ims-db",
 				Username: "ims-db-user",
 				Password: rand.Text(),
+				// This needs to be 1 for the fake DB because of
+				// https://github.com/dolthub/go-mysql-server/issues/1306
+				MaxOpenConns: 1,
 			},
 		},
 		Directory: Directory{
@@ -78,17 +84,37 @@ func DefaultIMS() *IMSConfig {
 // Validate should be called after an IMSConfig has been fully configured.
 func (c *IMSConfig) Validate() error {
 	var errs []error
+
+	// IMS database
 	errs = append(errs, c.Store.Type.Validate())
-	if c.Store.Type == DBStoreTypeNoOp {
+	if c.Store.Type != DBStoreTypeMaria {
 		c.Store.MariaDB = DBStoreMaria{}
 	}
+	if c.Store.Type != DBStoreTypeFake {
+		c.Store.Fake = DBStoreMaria{}
+	}
+
+	// User directory
 	errs = append(errs, c.Directory.Directory.Validate())
 	if c.Directory.Directory != DirectoryTypeTestUsers {
 		c.Directory.TestUsers = nil
 	}
 	if c.Directory.Directory != DirectoryTypeClubhouseDB {
 		c.Directory.ClubhouseDB = ClubhouseDB{}
 	}
+
+	// Deployment
+	errs = append(errs, c.Core.Deployment.Validate())
+	if c.Core.Deployment != DeploymentTypeDev {
+		if c.Directory.Directory != DirectoryTypeClubhouseDB {
+			errs = append(errs, errors.New("non-dev environments must use a ClubhouseDB directory"))
+		}
+		if c.Store.Type != DBStoreTypeMaria {
+			errs = append(errs, errors.New("non-dev environments must use a MariaDB datastore"))
+		}
+	}
+
+	// Attachments store
 	errs = append(errs, c.AttachmentsStore.Type.Validate())
 	if c.AttachmentsStore.Type == AttachmentsStoreLocal {
 		if c.AttachmentsStore.Local.Dir == nil {
@@ -106,9 +132,8 @@ func (c *IMSConfig) Validate() error {
 		}
 		c.AttachmentsStore.Local = LocalAttachments{}
 	}
-	if c.Core.Deployment != "dev" && c.Directory.Directory == DirectoryTypeTestUsers {
-		errs = append(errs, errors.New("do not use TestUsers outside dev! A ClubhouseDB must be provided"))
-	}
+
+	// Assorted other validations
 	if c.Core.AccessTokenLifetime > c.Core.RefreshTokenLifetime {
 		errs = append(errs, errors.New("access token lifetime should not be greater than refresh token lifetime"))
 	}
@@ -137,6 +162,7 @@ type DeploymentType string
 
 type DBStoreType string
 
+// All these consts should have lowercase values to allow case-insensitive matching.
 const (
 	DirectoryTypeClubhouseDB DirectoryType        = "clubhousedb"
 	DirectoryTypeTestUsers   DirectoryType        = "testusers"
@@ -148,12 +174,12 @@ const (
 	DeploymentTypeProduction DeploymentType       = "production"
 	DBStoreTypeMaria         DBStoreType          = "mariadb"
 	DBStoreTypeNoOp          DBStoreType          = "noop"
-	DBStoreTypeInProcess     DBStoreType          = "inprocess"
+	DBStoreTypeFake          DBStoreType          = "fake"
 )
 
 func (d DBStoreType) Validate() error {
 	switch d {
-	case DBStoreTypeMaria, DBStoreTypeNoOp, DBStoreTypeInProcess:
+	case DBStoreTypeMaria, DBStoreTypeNoOp, DBStoreTypeFake:
 		return nil
 	default:
 		return fmt.Errorf("unknown DB store type %v", d)
@@ -195,7 +221,7 @@ type ConfigCore struct {
 	Admins               []string
 	MasterKey            string `redact:"true"`
 	JWTSecret            string `redact:"true"`
-	Deployment           string
+	Deployment           DeploymentType
 
 	// CacheControlShort is the duration we set in various responses' Cache-Control headers
 	// for resources that aren't expected to change often, but still do change (e.g. the list of
@@ -216,17 +242,18 @@ type ConfigCore struct {
 }
 
 type DBStore struct {
-	Type      DBStoreType
-	MariaDB   DBStoreMaria
-	InProcess DBStoreMaria
+	Type    DBStoreType
+	MariaDB DBStoreMaria
+	Fake    DBStoreMaria
 }
 
 type DBStoreMaria struct {
-	HostName string
-	HostPort int32
-	Database string
-	Username string
-	Password string `redact:"true"`
+	HostName     string
+	HostPort     int32
+	Database     string
+	Username     string
+	Password     string `redact:"true"`
+	MaxOpenConns int32
 }
 
 type TestUser struct {
diff --git a/go.mod b/go.mod
@@ -5,9 +5,9 @@ go 1.24.3
 require (
 	github.com/a-h/templ v0.3.865
 	github.com/aws/aws-sdk-go-v2/config v1.29.14
-	github.com/aws/aws-sdk-go-v2/service/s3 v1.79.4
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.80.0
 	github.com/aws/smithy-go v1.22.3
-	github.com/dolthub/go-mysql-server v0.20.0
+	github.com/dolthub/go-mysql-server v0.20.1-0.20250523181431-8c883e3e574d
 	github.com/go-sql-driver/mysql v1.9.2
 	github.com/golang-jwt/jwt/v5 v5.2.2
 	github.com/joho/godotenv v1.5.1
@@ -64,7 +64,7 @@ require (
 	github.com/dolthub/flatbuffers/v23 v23.3.3-dh.2 // indirect
 	github.com/dolthub/go-icu-regex v0.0.0-20250327004329-6799764f2dad // indirect
 	github.com/dolthub/jsonpath v0.0.2-0.20240227200619-19675ab05c71 // indirect
-	github.com/dolthub/vitess v0.0.0-20250512224608-8fb9c6ea092c // indirect
+	github.com/dolthub/vitess v0.0.0-20250523011542-0f6cf9472d1c // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect
 	github.com/ebitengine/purego v0.8.4 // indirect
 	github.com/fatih/color v1.18.0 // indirect
diff --git a/go.sum b/go.sum
@@ -55,8 +55,8 @@ github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15 h1:dM9/92u2
 github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15/go.mod h1:SwFBy2vjtA0vZbjjaFtfN045boopadnoVPhu4Fv66vY=
 github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.15 h1:moLQUoVq91LiqT1nbvzDukyqAlCv89ZmwaHw/ZFlFZg=
 github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.15/go.mod h1:ZH34PJUc8ApjBIfgQCFvkWcUDBtl/WTD+uiYHjd8igA=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.79.4 h1:4yxno6bNHkekkfqG/a1nz/gC2gBwhJSojV1+oTE7K+4=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.79.4/go.mod h1:qbn305Je/IofWBJ4bJz/Q7pDEtnnoInw/dGt71v6rHE=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.80.0 h1:fV4XIU5sn/x8gjRouoJpDVHj+ExJaUk4prYF+eb6qTs=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.80.0/go.mod h1:qbn305Je/IofWBJ4bJz/Q7pDEtnnoInw/dGt71v6rHE=
 github.com/aws/aws-sdk-go-v2/service/sso v1.25.3 h1:1Gw+9ajCV1jogloEv1RRnvfRFia2cL6c9cuKV2Ps+G8=
 github.com/aws/aws-sdk-go-v2/service/sso v1.25.3/go.mod h1:qs4a9T5EMLl/Cajiw2TcbNt2UNo/Hqlyp+GiuG4CFDI=
 github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.1 h1:hXmVKytPfTy5axZ+fYbR5d0cFmC3JvwLm5kM83luako=
@@ -135,12 +135,12 @@ github.com/dolthub/flatbuffers/v23 v23.3.3-dh.2 h1:u3PMzfF8RkKd3lB9pZ2bfn0qEG+1G
 github.com/dolthub/flatbuffers/v23 v23.3.3-dh.2/go.mod h1:mIEZOHnFx4ZMQeawhw9rhsj+0zwQj7adVsnBX7t+eKY=
 github.com/dolthub/go-icu-regex v0.0.0-20250327004329-6799764f2dad h1:66ZPawHszNu37VPQckdhX1BPPVzREsGgNxQeefnlm3g=
 github.com/dolthub/go-icu-regex v0.0.0-20250327004329-6799764f2dad/go.mod h1:ylU4XjUpsMcvl/BKeRRMXSH7e7WBrPXdSLvnRJYrxEA=
-github.com/dolthub/go-mysql-server v0.20.0 h1:oB1WXD5TwdjhdyJDbF6VgVxyEbCevDRok9yEXefpoyI=
-github.com/dolthub/go-mysql-server v0.20.0/go.mod h1:5ZdrW0fHZbz+8CngT9gksqSX4H3y+7v1pns7tJCEpu0=
+github.com/dolthub/go-mysql-server v0.20.1-0.20250523181431-8c883e3e574d h1:MyV/Er1lqEtFiBGMwiXe0yUHUXNffJMPpZGQ3IGxc5k=
+github.com/dolthub/go-mysql-server v0.20.1-0.20250523181431-8c883e3e574d/go.mod h1:Zn9XK5KLYwPbyMpwfeUP+TgnhlgyID2vXf1WcF0M6Fk=
 github.com/dolthub/jsonpath v0.0.2-0.20240227200619-19675ab05c71 h1:bMGS25NWAGTEtT5tOBsCuCrlYnLRKpbJVJkDbrTRhwQ=
 github.com/dolthub/jsonpath v0.0.2-0.20240227200619-19675ab05c71/go.mod h1:2/2zjLQ/JOOSbbSboojeg+cAwcRV0fDLzIiWch/lhqI=
-github.com/dolthub/vitess v0.0.0-20250512224608-8fb9c6ea092c h1:imdag6PPCHAO2rZNsFoQoR4I/vIVTmO/czoOl5rUnbk=
-github.com/dolthub/vitess v0.0.0-20250512224608-8fb9c6ea092c/go.mod h1:1gQZs/byeHLMSul3Lvl3MzioMtOW1je79QYGyi2fd70=
+github.com/dolthub/vitess v0.0.0-20250523011542-0f6cf9472d1c h1:23KvsBtJk2GmHpXwQ/RkwIkdNpWL8tWdHRCiidhnaUA=
+github.com/dolthub/vitess v0.0.0-20250523011542-0f6cf9472d1c/go.mod h1:1gQZs/byeHLMSul3Lvl3MzioMtOW1je79QYGyi2fd70=
 github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
 github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
 github.com/ebitengine/purego v0.8.4 h1:CF7LEKg5FFOsASUj0+QwaXf8Ht6TlFxg09+S9wz0omw=
diff --git a/store/fakeimsdb/fakeimsdb.go b/store/fakeimsdb/fakeimsdb.go
@@ -14,9 +14,9 @@
 // limitations under the License.
 //
 
-//go:build !noinprocessdb
+//go:build !nofakedb
 
-package inprocessdb
+package fakeimsdb
 
 import (
 	"context"
diff --git a/store/fakeimsdb/fakeimsdb_nobuild.go b/store/fakeimsdb/fakeimsdb_nobuild.go
@@ -14,18 +14,18 @@
 // limitations under the License.
 //
 
-//go:build noinprocessdb
+//go:build nofakedb
 
-package inprocessdb
+package fakeimsdb
 
 import "context"
 
 func Start(
 	ctx context.Context, dbName, hostname string, port int32, username, password string,
 ) (actualPort int, err error) {
-	panic("the in-memory DB was not compiled into the program")
+	panic("the in-process fake DB was not compiled into the program")
 }
 
 func SeedData() string {
-	panic("the in-memory DB was not compiled into the program")
+	panic("the in-process fake DB was not compiled into the program")
 }
diff --git a/store/fakeimsdb/seed.sql b/store/fakeimsdb/seed.sql
diff --git a/store/integration/fakedb_test.go b/store/integration/fakedb_test.go
@@ -0,0 +1,51 @@
+//
+// See the file COPYRIGHT for copyright information.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package integration_test
+
+import (
+	"github.com/burningmantech/ranger-ims-go/conf"
+	"github.com/burningmantech/ranger-ims-go/store"
+	"github.com/stretchr/testify/require"
+	"testing"
+)
+
+func TestMigrateFakeDB(t *testing.T) {
+	t.Parallel()
+	ctx := t.Context()
+
+	db, err := store.SqlDB(ctx,
+		conf.DBStore{
+			Type: conf.DBStoreTypeFake,
+			Fake: conf.DefaultIMS().Store.Fake,
+		},
+		true,
+	)
+	require.NoError(t, err)
+	defer shut(db)
+
+	// Check that the schema version has been bumped high enough (15 as of 2025-05-31)
+	r := db.QueryRowContext(ctx, "select VERSION from SCHEMA_INFO")
+	var version int64
+	require.NoError(t, r.Scan(&version))
+	require.GreaterOrEqual(t, version, int64(15))
+
+	// Check that the data seeding worked (from seed.sql)
+	r = db.QueryRowContext(ctx, "select NAME from INCIDENT_TYPE where ID = 3")
+	var name string
+	require.NoError(t, r.Scan(&name))
+	require.Equal(t, "Sound Complaint", name)
+}
diff --git a/store/integration/migrate_test.go b/store/integration/migrate_test.go
@@ -149,16 +149,18 @@ func newUnmigratedDB(t *testing.T, ctx context.Context, database, username, pass
 	port, err := ctr.MappedPort(ctx, "3306/tcp")
 	require.NoError(t, err)
 	dbHostPort := int32(port.Int())
-	db, err := store.SqlDB(ctx, conf.DBStore{
-		Type: conf.DBStoreTypeMaria,
-		MariaDB: conf.DBStoreMaria{
-			HostName: "",
-			HostPort: dbHostPort,
-			Database: database,
-			Username: username,
-			Password: password,
+	db, err := store.SqlDB(ctx,
+		conf.DBStore{
+			Type: conf.DBStoreTypeMaria,
+			MariaDB: conf.DBStoreMaria{
+				HostName: "",
+				HostPort: dbHostPort,
+				Database: database,
+				Username: username,
+				Password: password,
+			},
 		},
-	}, false)
+		false)
 	require.NoError(t, err)
 	return ctr, db
 }
diff --git a/store/store.go b/store/store.go
diff --git a/web/mux.go b/web/mux.go

Original file line number	Diff line number	Diff line change
`@@ -95,7 +95,6 @@ func setup(ctx context.Context, tempDir string) {`
`95`	`95`	`shared.cfg.AttachmentsStore.Local = conf.LocalAttachments{`
`96`	`96`	`Dir: tempRoot,`
`97`	`97`	`}`
`98`		`- shared.cfg.Store.Type = conf.DBStoreTypeMaria`
`99`	`98`	`shared.cfg.Store.MariaDB.Database = "ims-" + rand.NonCryptoText()`
`100`	`99`	`shared.cfg.Store.MariaDB.Username = "rangers-" + rand.NonCryptoText()`
`101`	`100`	`shared.cfg.Store.MariaDB.Password = "password-" + rand.NonCryptoText()`
Original file line number	Diff line number	Diff line change
`@@ -199,7 +199,7 @@ func mustInitConfig(envFileName string) *conf.IMSConfig {`
`199`	`199`	`must(err)`
`200`	`200`	`}`
`201`	`201`	`if v, ok := lookupEnv("IMS_DEPLOYMENT"); ok {`
`202`		`- newCfg.Core.Deployment = strings.ToLower(v)`
	`202`	`+ newCfg.Core.Deployment = conf.DeploymentType(strings.ToLower(v))`
`203`	`203`	`}`
`204`	`204`	`// This should really be called "IMS_REFRESH_TOKEN_LIFETIME". This name of`
`205`	`205`	`// "IMS_TOKEN_LIFETIME" predates our use of refresh tokens, and what it tried`
`@@ -246,7 +246,7 @@ func mustInitConfig(envFileName string) *conf.IMSConfig {`
`246`	`246`	`newCfg.Core.JWTSecret = v`
`247`	`247`	`}`
`248`	`248`	`if v, ok := lookupEnv("IMS_DB_STORE_TYPE"); ok {`
`249`		`- newCfg.Store.Type = conf.DBStoreType(v)`
	`249`	`+ newCfg.Store.Type = conf.DBStoreType(strings.ToLower(v))`
`250`	`250`	`}`
`251`	`251`	`if v, ok := lookupEnv("IMS_DB_HOST_NAME"); ok {`
`252`	`252`	`newCfg.Store.MariaDB.HostName = v`