Merge pull request #78 from dolthub/daylon/init-changes

Changed default database creation behavior

Author: Hydrocharged

.github/workflows/ci-staticcheck.yaml

@@ -22,6 +22,5 @@ jobs:
         working-directory: ./postgres/parser
         shell: bash
       - name: Run check
-        run: |
-          go install honnef.co/go/tools/cmd/staticcheck@2023.1.6
-          staticcheck ./...
+        run: ./run_staticcheck.sh
+        working-directory: ./scripts

CONTRIBUTING.md

@@ -27,6 +27,11 @@ An alternative is to use [BashSupport Pro](https://plugins.jetbrains.com/plugin/
 Additionally, our [Bats](https://github.com/bats-core/bats-core) tests assume that you have a `doltgresql` (not `doltgres`) binary on your PATH.
 For Windows users, this means that the binary should _not_ end with the `.exe` file extension.
 Remember to recompile the executable on your PATH whenever you want to re-test any [Bats](https://github.com/bats-core/bats-core) tests.
+9. **Change the data directory**: This is optional but recommended.
+By default, we create databases within the `~/doltgres/databases` directory.
+For developmental purposes, you may want to change this behavior. You have two options:
+   1. Set the `DOLTGRES_DATA_DIR` environment variable to a different directory. A value of `.` causes DoltgreSQL to use the current directory as the data directory, so you can have multiple data directories simply by running the program in different directories. This behavior is more consistent with [Dolt's](https://github.com/dolthub/dolt) behavior. This is the recommended option for development.
+   2. Specify the directory in the `--data-dir` argument. This overrides the environment variable if it is present. 
 
 ### Note for Windows Users
 

README.md

@@ -35,46 +35,38 @@ If you are interested in using Doltgres now or in the future, please:
 * [Try Doltgres](#getting-started)
 * Create [issues](https://github.com/dolthub/doltgresql/issues) if you find bugs
 * Create [issues](https://github.com/dolthub/doltgresql/issues) for missing functionality you want
-* Contribute Code for features you want (see [Building From Source](#building-from-source))
-
-Contribution Guide coming soon.
+* Contribute code for features you want (see the [Contribution Guide](https://github.com/dolthub/doltgresql/blob/main/CONTRIBUTING.md))
 
 # Getting Started
 
 1. Download the latest release of `doltgres`
 
 2. Put `doltgres` on your `PATH`
 
-3. Navigate to a directory you want your database data stored (ie. `~/doltgresql`).
-```bash
-$ mkdir ~/doltgresql
-$ cd ~/doltgresql
-```
-
-4. Run `doltgres`. This will create a `doltgres` user and a `doltgres` database.
+3. Run `doltgres`. This will create a `doltgres` user and a `doltgres` database in `~/doltgres/databases` (add the `--data-dir` argument or change the `DOLTGRES_DATA_DIR` environment variable to use a different directory).
 ```bash
 $ doltgres
 Successfully initialized dolt data repository.
 Starting server with Config HP="localhost:5432"|T="28800000"|R="false"|L="info"|S="/tmp/mysql.sock"
 ```
 
-5. Make sure you have Postgres version 15 or higher installed. I used Homebrew to install Postgres on my Mac.
+4. Make sure you have Postgres version 15 or higher installed. I used Homebrew to install Postgres on my Mac.
 This requires I manually add `/opt/homebrew/opt/postgresql@15/bin` to my path. On Postgres version 14 or lower,
-`\` commands (ie. `\d`, `\l`) do not work with Doltgres. 
+`\` commands (ie. `\d`, `\l`) do not yet work with Doltgres. 
 ```
 export PATH="/opt/homebrew/opt/postgresql@15/bin:$PATH"
 ```
 
-6. Open a new terminal. Connect with the following command: `psql -h localhost -U doltgres`. This will connect to the `doltgres` database with the `doltgres` user.
+5. Open a new terminal. Connect with the following command: `psql -h localhost -U doltgres`. This will connect to the `doltgres` database with the `doltgres` user.
 ```bash
-$ psql -h 127.0.0.1 -U doltgres                  
+$ psql -h 127.0.0.1 -U doltgres
 psql (15.4 (Homebrew), server 15.0)
 Type "help" for help.
 
 doltgres=>
 ```
 
-7. Create a `getting_started` database. Create the `getting_started` example tables.
+6. Create a `getting_started` database. Create the `getting_started` example tables.
 ```sql
 doltgres=> create database getting_started;
 --
@@ -117,7 +109,7 @@ getting_started=> \d
 (3 rows)
 ```
 
-8. Make a Dolt Commit.
+7. Make a Dolt Commit.
 ```sql
 getting_started=> select * from dolt_status;
    table_name    | staged |  status   
@@ -147,7 +139,7 @@ getting_started=> call dolt_commit('-m', 'Created initial schema');
 (1 row)
 ```
 
-9. View the Dolt log.
+8. View the Dolt log.
 ```
 getting_started=> select * from dolt_log;
            commit_hash            | committer |       email        |        date         |          message           
@@ -157,17 +149,12 @@ getting_started=> select * from dolt_log;
 (2 rows)
 ```
 
-10. Continue with [Dolt Getting Started](https://docs.dolthub.com/introduction/getting-started/database#insert-some-data) 
+9. Continue with [Dolt Getting Started](https://docs.dolthub.com/introduction/getting-started/database#insert-some-data) 
 to test out more Doltgres versioning functionality.
 
 # Building From Source
 
-Due to the rapid pace of development at this early stage, building from source will guarantee that you're always working
-with the latest improvement and features.
-
-1. Clone the repository to your local drive
-2. Run `./postgres/parser/build.sh` to generate the parser
-3. Run `go build .` in the root directory
+Please follow the [Contributor's Guide](https://github.com/dolthub/doltgresql/blob/main/CONTRIBUTING.md#getting-set-up) to learn how to build from source.
 
 # Limitations
 

main.go

@@ -18,6 +18,7 @@ import (
 	"context"
 	"fmt"
 	"os"
+	"path/filepath"
 	"strconv"
 	"strings"
 
@@ -58,11 +59,13 @@ func init() {
 	}
 }
 
-const chdirFlag = "--chdir"
-const stdInFlag = "--stdin"
-const stdOutFlag = "--stdout"
-const stdErrFlag = "--stderr"
-const stdOutAndErrFlag = "--out-and-err"
+const (
+	chdirFlag        = "--chdir"
+	stdInFlag        = "--stdin"
+	stdOutFlag       = "--stdout"
+	stdErrFlag       = "--stderr"
+	stdOutAndErrFlag = "--out-and-err"
+)
 
 func main() {
 	ctx := context.Background()
@@ -80,7 +83,12 @@ func main() {
 
 	warnIfMaxFilesTooLow()
 
-	fs := filesys.LocalFS
+	var fs filesys.Filesys
+	fs, args, err = loadFileSystem(args)
+	if err != nil {
+		cli.PrintErrln(err.Error())
+		os.Exit(1)
+	}
 	dEnv := env.Load(ctx, env.GetCurrentUserHomeDir, fs, doltdb.LocalDirDoltDB, server.Version)
 
 	globalConfig, _ := dEnv.Config.GetConfig(env.GlobalConfig)
@@ -118,6 +126,107 @@ func main() {
 	os.Exit(exitCode)
 }
 
+// loadFileSystem loads the file system from a combination of the given arguments and environment variables.
+func loadFileSystem(args []string) (fs filesys.Filesys, outArgs []string, err error) {
+	// We can't use the argument parser yet since it relies on the environment, so we'll handle the data directory
+	// argument here. This will also remove it from the args, so that the Dolt layer doesn't try to move the directory
+	// again (in the case of relative paths).
+	var dataDir string
+	var hasDataDirArgument bool
+	for i, arg := range args {
+		arg = strings.ToLower(arg)
+		if arg == "--data-dir" {
+			if len(args) <= i+1 {
+				return nil, args, fmt.Errorf("--data-dir is missing the directory")
+			}
+			dataDir = args[i+1]
+			hasDataDirArgument = true
+			// os.Args is read from Dolt, so we have to update it, else we'll apply the move twice
+			newArgs := make([]string, len(args)-2)
+			copy(newArgs, args[:i])
+			copy(newArgs[i:], args[i+2:])
+			args = newArgs
+			os.Args = os.Args[:len(args)+1]
+			copy(os.Args[1:], args)
+			break
+		} else if strings.HasPrefix(arg, "--data-dir=") {
+			dataDir = arg[11:]
+			hasDataDirArgument = true
+			// os.Args is read from Dolt, so we have to update it, else we'll apply the move twice
+			newArgs := make([]string, len(args)-1)
+			copy(newArgs, args[:i])
+			copy(newArgs[i:], args[i+1:])
+			args = newArgs
+			os.Args = os.Args[:len(args)+1]
+			copy(os.Args[1:], args)
+		}
+	}
+
+	if len(args) >= 1 && args[0] == "init" {
+		// If "init" is passed as the command, then we use the current directory
+		fs = filesys.LocalFS
+	} else if hasDataDirArgument {
+		// If the --data-dir argument was used, then we'll use the directory that it points to
+		fileInfo, err := os.Stat(dataDir)
+		if os.IsNotExist(err) {
+			if err = os.MkdirAll(dataDir, 0755); err != nil {
+				return nil, args, err
+			}
+		} else if err != nil {
+			return nil, args, err
+		} else if !fileInfo.IsDir() {
+			return nil, args, fmt.Errorf("Attempted to use the directory `%s` as the DoltgreSQL database directory, "+
+				"however the preceding is a file and not a directory. Please change the `--data-dir` argument so "+
+				"that it points to a directory.", dataDir)
+		}
+		fs, err = filesys.LocalFilesysWithWorkingDir(dataDir)
+		if err != nil {
+			return nil, args, err
+		}
+	} else {
+		// We should use the directory as pointed to by "DOLTGRES_DATA_DIR", if has been set, otherwise we'll use the default
+		var dbDir string
+		if envDir := os.Getenv(server.DOLTGRES_DATA_DIR); len(envDir) > 0 {
+			dbDir = envDir
+			fileInfo, err := os.Stat(dbDir)
+			if os.IsNotExist(err) {
+				if err = os.MkdirAll(dbDir, 0755); err != nil {
+					return nil, args, err
+				}
+			} else if err != nil {
+				return nil, args, err
+			} else if !fileInfo.IsDir() {
+				return nil, args, fmt.Errorf("Attempted to use the directory `%s` as the DoltgreSQL database directory, "+
+					"however the preceding is a file and not a directory. Please change the environment variable `%s` so "+
+					"that it points to a directory.", dbDir, server.DOLTGRES_DATA_DIR)
+			}
+		} else {
+			homeDir, err := env.GetCurrentUserHomeDir()
+			if err != nil {
+				return nil, args, err
+			}
+			dbDir = filepath.Join(homeDir, server.DOLTGRES_DATA_DIR_DEFAULT)
+			fileInfo, err := os.Stat(dbDir)
+			if os.IsNotExist(err) {
+				if err = os.MkdirAll(dbDir, 0755); err != nil {
+					return nil, args, err
+				}
+			} else if err != nil {
+				return nil, args, err
+			} else if !fileInfo.IsDir() {
+				return nil, args, fmt.Errorf("Attempted to use the directory `%s` as the DoltgreSQL database directory, "+
+					"however the preceding is a file and not a directory. Please change the environment variable `%s` so "+
+					"that it points to a directory.", dbDir, server.DOLTGRES_DATA_DIR)
+			}
+		}
+		fs, err = filesys.LocalFilesysWithWorkingDir(dbDir)
+		if err != nil {
+			return nil, args, err
+		}
+	}
+	return fs, args, nil
+}
+
 func configureCliCtx(subcommand string, apr *argparser.ArgParseResults, fs filesys.Filesys, dEnv *env.DoltEnv, ctx context.Context) (cli.CliContext, error) {
 	dataDir, hasDataDir := apr.GetValue(commands.DataDirFlag)
 	if hasDataDir {
@@ -136,8 +245,9 @@ func configureCliCtx(subcommand string, apr *argparser.ArgParseResults, fs files
 	}
 
 	if dEnv.HasDoltDataDir() {
-		return nil, fmt.Errorf("Cannot start a server within a directory containing a Dolt or Doltgres database." +
-			"To use the current directory as a database, start the server from the parent directory.")
+		cwd, _ := dEnv.FS.Abs(".")
+		return nil, fmt.Errorf("Cannot start a server within a directory containing a Dolt or Doltgres database. "+
+			"To use the current directory (%s) as a database, start the server from the parent directory.", cwd)
 	}
 
 	err := reconfigIfTempFileMoveFails(dEnv)

scripts/run_staticcheck.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+# Set the working directory to the directory of the script's location
+cd "$(cd -P -- "$(dirname -- "$0")" && pwd -P)"
+cd ..
+
+go run honnef.co/go/tools/cmd/staticcheck@2023.1.6 ./...

server/server.go

@@ -38,6 +38,11 @@ import (
 
 const (
 	Version = "0.2.0"
+
+	// DOLTGRES_DATA_DIR is an environment variable that defines the location of DoltgreSQL databases
+	DOLTGRES_DATA_DIR = "DOLTGRES_DATA_DIR"
+	// DOLTGRES_DATA_DIR_DEFAULT is the portion to append to the user's home directory if DOLTGRES_DATA_DIR has not been specified
+	DOLTGRES_DATA_DIR_DEFAULT = "doltgres/databases"
 )
 
 var sqlServerDocs = cli.CommandDocumentationContent{
@@ -52,7 +57,7 @@ var sqlServerDocs = cli.CommandDocumentationContent{
 		indentLines(sqlserver.ServerConfigAsYAMLConfig(sqlserver.DefaultServerConfig()).String()) + "\n\n" + `
 SUPPORTED CONFIG FILE FIELDS:
 
-{{.EmphasisLeft}}data_dir{{.EmphasisRight}}: A directory where the server will load dolt databases to serve, and create new ones. Defaults to the current directory.
+{{.EmphasisLeft}}data_dir{{.EmphasisRight}}: A directory where the server will load dolt databases to serve, and create new ones. Defaults to the DOLTGRES_DATA_DIR environment variable, or {{.EmphasisLeft}}~/doltgres/databases{{.EmphasisRight}}.
 
 {{.EmphasisLeft}}cfg_dir{{.EmphasisRight}}: A directory where the server will load and store non-database configuration data, such as permission information. Defaults {{.EmphasisLeft}}$data_dir/.doltcfg{{.EmphasisRight}}.
 
@@ -152,8 +157,9 @@ func runServer(ctx context.Context, args []string, dEnv *env.DoltEnv) (*svcs.Con
 	}
 
 	if dEnv.HasDoltDataDir() {
-		return nil, fmt.Errorf("Cannot start a server within a directory containing a Dolt or Doltgres database." +
-			"To use the current directory as a database, start the server from the parent directory.")
+		cwd, _ := dEnv.FS.Abs(".")
+		return nil, fmt.Errorf("Cannot start a server within a directory containing a Dolt or Doltgres database. "+
+			"To use the current directory (%s) as a database, start the server from the parent directory.", cwd)
 	}
 
 	defer tempfiles.MovableTempFileProvider.Clean()
@@ -206,11 +212,12 @@ func runServer(ctx context.Context, args []string, dEnv *env.DoltEnv) (*svcs.Con
 				return nil, fmt.Errorf("failed to initialize doltgres database")
 			}
 		} else if !isDirectory {
+			workingDir, _ := dEnv.FS.Abs(".")
 			// The else branch means that there's a Doltgres item, so we need to error if it's a file since we
 			// enforce the creation of a Doltgres database/directory, which would create a name conflict with the file
-			return nil, fmt.Errorf("Attempted to create the default `doltgres` database, but a file with" +
-				" the same name was found. Please run the doltgres command in a directory that does not contain a" +
-				" file with the name doltgres")
+			return nil, fmt.Errorf("Attempted to create the default `doltgres` database at `%s`, but a file with "+
+				"the same name was found. Either remove the file, change the directory using the `--data-dir` argument, "+
+				"or change the environment variable `%s` so that it points to a different directory.", workingDir, DOLTGRES_DATA_DIR)
 		}
 	}
 

testing/bats/setup/common.bash

@@ -56,6 +56,7 @@ setup_common() {
     # multiple tests can be run in parallel on the same machine
     mkdir "dolt-repo-$$"
     cd "dolt-repo-$$"
+    nativevar DOLTGRES_DATA_DIR "$(pwd)" /p
 
     mkdir "postgres"
     cd "postgres"

testing/bats/setup/query-server-common.bash

@@ -29,13 +29,14 @@ wait_for_connection() {
 start_sql_server() {
     DEFAULT_DB="$1"
     DEFAULT_DB="${DEFAULT_DB:=postgres}"
+    nativevar DEFAULT_DB "$DEFAULT_DB" /w
     logFile="$2"
     PORT=$( definePORT )
     if [[ $logFile ]]
     then
-        doltgresql --host 0.0.0.0 --port=$PORT --user "${SQL_USER:-postgres}" > $logFile 2>&1 &
+        doltgresql --host 0.0.0.0 --port=$PORT --data-dir=. --user "${SQL_USER:-postgres}" > $logFile 2>&1 &
     else
-        doltgresql --host 0.0.0.0 --port=$PORT --user "${SQL_USER:-postgres}" &
+        doltgresql --host 0.0.0.0 --port=$PORT --data-dir=. --user "${SQL_USER:-postgres}" &
     fi
     SERVER_PID=$!
     wait_for_connection $PORT 7500
@@ -46,6 +47,8 @@ start_sql_server() {
 # this func)
 start_sql_server_with_args() {
     DEFAULT_DB=""
+    nativevar DEFAULT_DB "$DEFAULT_DB" /w
+    nativevar DOLTGRES_DATA_DIR "$(pwd)" /p
     PORT=$( definePORT )
     doltgresql "$@" --port=$PORT &
     SERVER_PID=$!
@@ -55,6 +58,8 @@ start_sql_server_with_args() {
 start_sql_server_with_config() {
     DEFAULT_DB="$1"
     DEFAULT_DB="${DEFAULT_DB:=postgres}"
+    nativevar DEFAULT_DB "$DEFAULT_DB" /w
+    nativevar DOLTGRES_DATA_DIR "$(pwd)" /p
     PORT=$( definePORT )
     echo "
 log_level: debug
@@ -79,6 +84,8 @@ behavior:
 start_sql_multi_user_server() {
     DEFAULT_DB="$1"
     DEFAULT_DB="${DEFAULT_DB:=postgres}"
+    nativevar DEFAULT_DB "$DEFAULT_DB" /w
+    nativevar DOLTGRES_DATA_DIR "$(pwd)" /p
     PORT=$( definePORT )
     echo "
 log_level: debug
@@ -102,6 +109,7 @@ behavior:
 start_multi_db_server() {
     DEFAULT_DB="$1"
     DEFAULT_DB="${DEFAULT_DB:=postgres}"
+    nativevar DEFAULT_DB "$DEFAULT_DB" /w
     PORT=$( definePORT )
     doltgresql --host 0.0.0.0 --port=$PORT --user postgres --data-dir ./ &
     SERVER_PID=$!

testing/bats/setup/windows-compat.bash

@@ -17,7 +17,7 @@ if [ -d "$WINDOWS_BASE_DIR"/Windows/System32 ]  || [ "$IS_WINDOWS" == true ]; th
     }
     nativevar() {
         eval export "$1"="$2"
-        export WSLENV="$1$3"
+        export WSLENV="$WSLENV:$1$3"
     }
     skiponwindows() {
         skip "$1"