herd-db-upgrade (#657)

* herd-db-upgrade

* delay with jitter

* simpler, no-txn solution as that causes performance issues

* better breaks

* idempotent

* make gen ?

* fiximports

* manual fix

Author: snadrus

Makefile

@@ -275,6 +275,10 @@ gensimple: api-gen go-generate cfgdoc-gen docsgen marketgen docsgen-cli
 	go mod tidy
 .PHONY: gen
 
+fiximports:
+	$(GOCC) run ./scripts/fiximports
+.PHONY: fiximports
+
 forest-test: GOFLAGS+=-tags=forest
 forest-test: buildall
 

harmony/harmonydb/harmonydb.go

@@ -268,6 +268,8 @@ retry:
 //go:embed sql
 var fs embed.FS
 
+var ITestUpgradeFunc func(*pgxpool.Pool, string, string)
+
 func (db *DB) upgrade() error {
 	// Does the version table exist? if not, make it.
 	// NOTE: This cannot change except via the next sql file.
@@ -337,6 +339,10 @@ func (db *DB) upgrade() error {
 			return xerrors.New(msg) // makes devs lives easier by placing message at the end.
 		}
 
+		if ITestUpgradeFunc != nil {
+			ITestUpgradeFunc(db.pgx, name, megaSql)
+		}
+
 		// Mark Completed.
 		_, err = db.Exec(context.Background(), "INSERT INTO base (entry) VALUES ($1)", name[:8])
 		if err != nil {

harmony/harmonydb/sql/20230706-itest_scratch.sql

@@ -1,4 +1,4 @@
-CREATE TABLE itest_scratch (
+CREATE TABLE IF NOT EXISTS itest_scratch (
     id SERIAL PRIMARY KEY,
 	content TEXT,
     some_int INTEGER,

harmony/harmonydb/sql/20230712-sector_index.sql

@@ -1,4 +1,4 @@
-create table sector_location
+create table if not exists sector_location
 (
     miner_id         bigint    not null,
     sector_num       bigint    not null,
@@ -19,7 +19,7 @@ alter table sector_location
 alter table sector_location
     alter column read_refs set default 0;
 
-create table storage_path
+create table if not exists storage_path
 (
     "storage_id"  varchar not null
         constraint "storage_path_pkey"

harmony/harmonydb/sql/20230719-harmony.sql

@@ -1,6 +1,6 @@
 /* For HarmonyTask base implementation. */
 
-CREATE TABLE harmony_machines (
+CREATE TABLE IF NOT EXISTS harmony_machines (
     id SERIAL PRIMARY KEY NOT NULL,
     last_contact TIMESTAMP NOT NULL DEFAULT current_timestamp,
     host_and_port varchar(300) NOT NULL, 
@@ -9,7 +9,7 @@ CREATE TABLE harmony_machines (
     gpu FLOAT NOT NULL
 );
 
-CREATE TABLE harmony_task (
+CREATE TABLE IF NOT EXISTS harmony_task (
     id SERIAL PRIMARY KEY NOT NULL,
     initiated_by INTEGER,     
     update_time TIMESTAMP NOT NULL DEFAULT current_timestamp,
@@ -28,7 +28,7 @@ COMMENT ON COLUMN harmony_task.name IS 'The name of the task type.';
 COMMENT ON COLUMN harmony_task.owner_id IS 'may be null if between owners or not yet taken';
 COMMENT ON COLUMN harmony_task.update_time IS 'When it was last modified. not a heartbeat';
 
-CREATE TABLE harmony_task_history (
+CREATE TABLE IF NOT EXISTS harmony_task_history (
     id SERIAL PRIMARY KEY NOT NULL,  
     task_id INTEGER NOT NULL, 
     name VARCHAR(16) NOT NULL,
@@ -40,14 +40,14 @@ CREATE TABLE harmony_task_history (
 );
 COMMENT ON COLUMN harmony_task_history.result IS 'Use to detemine if this was a successful run.';
 
-CREATE TABLE harmony_task_follow (
+CREATE TABLE IF NOT EXISTS harmony_task_follow (
     id SERIAL PRIMARY KEY NOT NULL,  
     owner_id INTEGER NOT NULL REFERENCES harmony_machines (id) ON DELETE CASCADE,
     to_type VARCHAR(16) NOT NULL,
     from_type VARCHAR(16) NOT NULL
 );
 
-CREATE TABLE harmony_task_impl (
+CREATE TABLE IF NOT EXISTS harmony_task_impl (
     id SERIAL PRIMARY KEY NOT NULL,  
     owner_id INTEGER NOT NULL REFERENCES harmony_machines (id) ON DELETE CASCADE,
     name VARCHAR(16) NOT NULL

harmony/harmonydb/sql/20230823-wdpost.sql

@@ -1,4 +1,4 @@
-create table wdpost_partition_tasks
+create table if not exists wdpost_partition_tasks
 (
     task_id              bigint not null
         constraint wdpost_partition_tasks_pk
@@ -17,7 +17,7 @@ comment on column wdpost_partition_tasks.proving_period_start is 'proving period
 comment on column wdpost_partition_tasks.deadline_index is 'deadline index within the proving period';
 comment on column wdpost_partition_tasks.partition_index is 'partition index within the deadline';
 
-create table wdpost_proofs
+create table if not exists wdpost_proofs
 (
     sp_id                bigint not null,
     proving_period_start bigint not null,
@@ -34,7 +34,7 @@ create table wdpost_proofs
         unique (sp_id, proving_period_start, deadline, partition)
 );
 
-create table wdpost_recovery_tasks
+create table if not exists wdpost_recovery_tasks
 (
     task_id              bigint not null
         constraint wdpost_recovery_tasks_pk

harmony/harmonydb/sql/20230919-config.sql

@@ -1,4 +1,4 @@
-CREATE TABLE harmony_config (
+CREATE TABLE IF NOT EXISTS harmony_config (
     id SERIAL PRIMARY KEY NOT NULL,
     title VARCHAR(300) UNIQUE NOT NULL,
     config TEXT NOT NULL

harmony/harmonydb/sql/20231103-chain_sends.sql

@@ -1,4 +1,4 @@
-create table message_sends
+create table if not exists message_sends
 (
     from_key     text   not null,
     to_addr      text   not null,
@@ -37,14 +37,14 @@ comment on column message_sends.send_time is 'time when the send task was execut
 comment on column message_sends.send_success is 'whether this message was broadcasted to the network already, null if not yet attempted, true if successful, false if failed';
 comment on column message_sends.send_error is 'error message if send_success is false';
 
-create unique index message_sends_success_index
+create unique index if not exists message_sends_success_index
     on message_sends (from_key, nonce)
     where send_success is not false;
 
 comment on index message_sends_success_index is
 'message_sends_success_index enforces sender/nonce uniqueness, it is a conditional index that only indexes rows where send_success is not false. This allows us to have multiple rows with the same sender/nonce, as long as only one of them was successfully broadcasted (true) to the network or is in the process of being broadcasted (null).';
 
-create table message_send_locks
+create table if not exists message_send_locks
 (
     from_key text   not null,
     task_id  bigint not null,

harmony/harmonydb/sql/20231110-mining_tasks.sql

@@ -1,4 +1,4 @@
-create table mining_tasks
+create table if not exists mining_tasks
 (
     task_id bigint not null
         constraint mining_tasks_pk
@@ -20,7 +20,7 @@ create table mining_tasks
         unique (sp_id, epoch)
 );
 
-create table mining_base_block
+create table if not exists mining_base_block
 (
     id        bigserial not null
         constraint mining_base_block_pk

harmony/harmonydb/sql/20231113-harmony_taskhistory_oops.sql

@@ -1 +1 @@
-ALTER TABLE harmony_task_history ADD COLUMN completed_by_host_and_port varchar(300) NOT NULL;
+ALTER TABLE harmony_task_history ADD COLUMN IF NOT EXISTS completed_by_host_and_port varchar(300) NOT NULL;