Introduce 'create-init' command to load a new database from schema/data (#303)

## Usage and product changes

We introduce a new command `database create-init`, which 

1) create the new database
2) loads a provided schema file (from URL, or from local file)
3) loads a provided data file (from URL or from local file)

Command format:
```
database create-init <db> <schema file> <data file> <[optional] schema file sha256 (hex or sha256:hex)> <[optional] data file sha256 (hex or sha256:hex)>
```

Usage example to load bookstore example from `typedb-examples`:
```
>> database create-init bookstore https://github.com/typedb/typedb-examples/releases/download/3.5.0/bookstore-schema.tql https://github.com/typedb/typedb-examples/releases/download/3.5.0/bookstore-data.tql
```

You can also optionally provide sha256 checksums to verify your files
are correct (these come from the Github releases page):
```
>> database create-init bookstore https://github.com/typedb/typedb-examples/releases/download/3.5.0/bookstore-schema.tql https://github.com/typedb/typedb-examples/releases/download/3.5.0/bookstore-data.tql sha256:b2de488d9f64ccdfdba016029c7932be69ec5d35d18977c85bb12ad4cc97e95f sha256:828806afe1ce939d0ee87d6ae89598f5d7f967155c7757263b151673480bcad1
```

We've also upgraded the `source` command in a transaction to allow
reading from a remote URL, not just local files:
```
bookstore::write >> source https://github.com/typedb/typedb-examples/releases/download/3.5.0/bookstore-data.tql 
```

This can also optionally take a sha256 (the `sha256:` prefix optional):
```
bookstore::write >> source https://github.com/typedb/typedb-examples/releases/download/3.5.0/bookstore-data.tql 828806afe1ce939d0ee87d6ae89598f5d7f967155c7757263b151673480bcad1
```

These commands can also receive local in relative or absolute path
formats.

## Implementation

Chain together database creation, and schema+data file sourcing. 

Extend the existing `source` behaviour to allow retriving the file over
the network from a remote URL.

Refactor the help menu to make it handle the long content more
effectively.

Author: flyingsilverfin

BUILD

@@ -33,6 +33,8 @@ rust_binary(
         "@crates//:rustyline",
         "@crates//:sentry",
         "@crates//:serde_json",
+        "@crates//:sha2",
+        "@crates//:ureq",
         "@crates//:tokio",
     ],
     compile_data = ["//:VERSION"],

Cargo.lock

@@ -23,6 +23,31 @@ features = {}
 		version = "1.45.0"
 		default-features = false
 
+	[dependencies.glob]
+		features = []
+		version = "0.3.2"
+		default-features = false
+
+	[dependencies.futures]
+		features = ["alloc", "async-await", "default", "executor", "futures-executor", "std", "thread-pool"]
+		version = "0.3.31"
+		default-features = false
+
+	[dependencies.rpassword]
+		features = []
+		version = "7.4.0"
+		default-features = false
+
+	[dependencies.home]
+		features = []
+		version = "0.5.11"
+		default-features = false
+
+	[dependencies.sha2]
+		features = ["default", "std"]
+		version = "0.10.9"
+		default-features = false
+
 	[dependencies.rustyline]
 		features = ["custom-bindings", "fd-lock", "radix_trie", "with-file-history"]
 		version = "15.0.0"
@@ -33,11 +58,6 @@ features = {}
 		version = "4.5.38"
 		default-features = false
 
-	[dependencies.glob]
-		features = []
-		version = "0.3.2"
-		default-features = false
-
 	[dependencies.typeql]
 		features = []
 		git = "https://github.com/typedb/typeql"
@@ -50,31 +70,21 @@ features = {}
 		tag = "3.4.4"
 		default-features = false
 
-	[dependencies.futures]
-		features = ["alloc", "async-await", "default", "executor", "futures-executor", "std", "thread-pool"]
-		version = "0.3.31"
-		default-features = false
-
 	[dependencies.serde_json]
 		features = ["alloc", "default", "indexmap", "preserve_order", "raw_value", "std"]
 		version = "1.0.140"
 		default-features = false
 
-	[dependencies.rpassword]
-		features = []
-		version = "7.4.0"
-		default-features = false
-
-	[dependencies.home]
-		features = []
-		version = "0.5.11"
-		default-features = false
-
 	[dependencies.sentry]
 		features = ["backtrace", "contexts", "httpdate", "panic", "sentry-backtrace", "sentry-contexts", "sentry-panic", "ureq"]
 		version = "0.36.0"
 		default-features = false
 
+	[dependencies.ureq]
+		features = ["default", "gzip", "tls"]
+		version = "2.12.1"
+		default-features = false
+
 [[test]]
 	path = "tests/assembly/test_assembly.rs"
 	name = "test-assembly-native"

Cargo.toml

@@ -30,10 +30,10 @@ platform-specific distribution.
 
 ## Command line arguments
 
-You can provide several command arguments when running console in the terminal.
+You can provide several command arguments when running console in the terminal, of which the typically-used ones are:
 
 - `--username=<username>` : TypeDB server username to log in with (mandatory).
-- `--address=<address>` : TypeDB server address to which the console will connect to (mandatory). Ensure it is prefixed with `https://` when using TLS.
+- `--address=<address>` : TypeDB server address to which the console will connect to (mandatory)
 - `--script=<path>` : Run commands in the script file in non-interactive mode.
 - `--command=<command1> --command=<command2> ...` : Run commands in non-interactive mode.
 - `-V, --version` : Print version information and exit.
@@ -54,8 +54,6 @@ Alternatively, you may securely connect by managing your own certificates for bo
 and provide your certificate to the console with:
 `--tls-root-ca=<path>`
 
-See documentation at https://typedb.com/docs/manual/configure/encryption for further details.
-
 ## Console commands
 
 TypeDB Console provides two levels of interaction: server-level commands and transaction-level commands. 

README.md

@@ -28,10 +28,10 @@ use crate::{
     cli::{Args, ADDRESS_VALUE_NAME, USERNAME_VALUE_NAME},
     completions::{database_name_completer_fn, file_completer},
     operations::{
-        database_create, database_delete, database_export, database_import, database_list, database_schema,
-        transaction_close, transaction_commit, transaction_query, transaction_read, transaction_rollback,
-        transaction_schema, transaction_source, transaction_write, user_create, user_delete, user_list,
-        user_update_password,
+        database_create, database_create_init, database_delete, database_export, database_import, database_list,
+        database_schema, transaction_close, transaction_commit, transaction_query, transaction_read,
+        transaction_rollback, transaction_schema, transaction_source, transaction_write, user_create, user_delete,
+        user_list, user_update_password,
     },
     repl::{
         command::{get_word, parse_one_query, CommandInput, CommandLeaf, Subcommand},
@@ -318,7 +318,8 @@ fn execute_commands(
                 match command.execute(context, arguments) {
                     Ok(_) => &input[next_command_index..],
                     Err(err) => {
-                        let message = format!("Error executing command: '{}'\n{}", command_string.trim(), err);
+                        let message =
+                            format!("**Error executing command**\n{}\n--> Error\n{}", command_string.trim(), err);
                         println_error!("{}", message);
                         return Err(CommandError { message });
                     }
@@ -340,43 +341,54 @@ fn entry_repl(driver: Arc<TypeDBDriver>, runtime: BackgroundRuntime) -> Repl<Con
         .add(CommandLeaf::new_with_input(
             "create",
             "Create a new database with the given name.",
-            CommandInput::new("db", get_word, None, None),
+            CommandInput::new_required("db", get_word, None),
             database_create,
         ))
+        .add(CommandLeaf::new_with_inputs(
+            "create-init",
+            "Create a new database with the given name and load schema and data from files. Files may be HTTP-hosted files, or absolute or relative paths. File contents are treated identically to 'transaction source' commands run explicitly, and may contain multiple queries separated by 'end;' markers. File sha256 sums maybe provided.",
+            vec![
+                CommandInput::new_required("db", get_word, None),
+                CommandInput::new_required("schema file", get_word, None),
+                CommandInput::new_required("data file", get_word, None),
+                CommandInput::new_optional("schema file sha256 (hex or sha256:hex)", get_word, None),
+                CommandInput::new_optional("data file sha256 (hex or sha256:hex)", get_word, None),
+            ],
+            database_create_init,
+        ))
         .add(CommandLeaf::new_with_input(
             "delete",
             "Delete the database with the given name.",
-            CommandInput::new("db", get_word, None, Some(database_name_completer_fn(driver.clone(), runtime.clone()))),
+            CommandInput::new_required("db", get_word, Some(database_name_completer_fn(driver.clone(), runtime.clone()))),
             database_delete,
         ))
         .add(CommandLeaf::new_with_input(
             "schema",
             "Retrieve the TypeQL representation of a database's schema.",
-            CommandInput::new("db", get_word, None, Some(database_name_completer_fn(driver.clone(), runtime.clone()))),
+            CommandInput::new_required("db", get_word, Some(database_name_completer_fn(driver.clone(), runtime.clone()))),
             database_schema,
         ))
         .add(CommandLeaf::new_with_inputs(
             "import",
-            "Create a database with the given name based on another previously exported database.",
+            "Create a database with the given name based on another previously exported database. File paths must be absolute or relative paths on the local machine.",
             vec![
-                CommandInput::new("db", get_word, None, None),
-                CommandInput::new("schema file path", get_word, None, None),
-                CommandInput::new("data file path", get_word, None, None),
+                CommandInput::new_required("db", get_word, None),
+                CommandInput::new_required("schema file path", get_word, None),
+                CommandInput::new_required("data file path", get_word, None),
             ],
             database_import,
         ))
         .add(CommandLeaf::new_with_inputs(
             "export",
             "Export a database into a schema definition and a data files.",
             vec![
-                CommandInput::new(
+                CommandInput::new_required(
                     "db",
                     get_word,
-                    None,
                     Some(database_name_completer_fn(driver.clone(), runtime.clone())),
                 ),
-                CommandInput::new("schema file path", get_word, None, None),
-                CommandInput::new("data file path", get_word, None, None),
+                CommandInput::new_required("schema file path", get_word, None),
+                CommandInput::new_required("data file path", get_word, None),
             ],
             database_export,
         ));
@@ -387,23 +399,23 @@ fn entry_repl(driver: Arc<TypeDBDriver>, runtime: BackgroundRuntime) -> Repl<Con
             "create",
             "Create new user.",
             vec![
-                CommandInput::new("name", get_word, None, None),
-                CommandInput::new("password", get_word, Some(get_word), None),
+                CommandInput::new_required("name", get_word, None),
+                CommandInput::new_hidden("password", get_word, get_word, None),
             ],
             user_create,
         ))
         .add(CommandLeaf::new_with_input(
             "delete",
             "Delete existing user.",
-            CommandInput::new("name", get_word, None, None),
+            CommandInput::new_required("name", get_word, None),
             user_delete,
         ))
         .add(CommandLeaf::new_with_inputs(
             "update-password",
             "Set existing user's password.",
             vec![
-                CommandInput::new("name", get_word, None, None),
-                CommandInput::new("new password", get_word, Some(get_word), None),
+                CommandInput::new_required("name", get_word, None),
+                CommandInput::new_hidden("new password", get_word, get_word, None),
             ],
             user_update_password,
         ));
@@ -412,19 +424,31 @@ fn entry_repl(driver: Arc<TypeDBDriver>, runtime: BackgroundRuntime) -> Repl<Con
         .add(CommandLeaf::new_with_input(
             "read",
             "Open read transaction.",
-            CommandInput::new("db", get_word, None, Some(database_name_completer_fn(driver.clone(), runtime.clone()))),
+            CommandInput::new_required(
+                "db",
+                get_word,
+                Some(database_name_completer_fn(driver.clone(), runtime.clone())),
+            ),
             transaction_read,
         ))
         .add(CommandLeaf::new_with_input(
             "write",
             "Open write transaction.",
-            CommandInput::new("db", get_word, None, Some(database_name_completer_fn(driver.clone(), runtime.clone()))),
+            CommandInput::new_required(
+                "db",
+                get_word,
+                Some(database_name_completer_fn(driver.clone(), runtime.clone())),
+            ),
             transaction_write,
         ))
         .add(CommandLeaf::new_with_input(
             "schema",
             "Open schema transaction.",
-            CommandInput::new("db", get_word, None, Some(database_name_completer_fn(driver.clone(), runtime.clone()))),
+            CommandInput::new_required(
+                "db",
+                get_word,
+                Some(database_name_completer_fn(driver.clone(), runtime.clone())),
+            ),
             transaction_schema,
         ));
 
@@ -457,17 +481,20 @@ fn transaction_repl(database: &str, transaction_type: TransactionType) -> Repl<C
             "Close the current transaction.",
             transaction_close,
         ))
-        .add(CommandLeaf::new_with_input(
+        .add(CommandLeaf::new_with_inputs(
             "source",
-            "Synchronously execute a file containing a sequence of TypeQL queries with full validation. Queries can be explicitly ended with 'end;' if required. Path may be absolute or relative to the invoking script (if there is one) otherwise relative to the current working directory.",
-            CommandInput::new("file", get_word, None, Some(Box::new(file_completer))),
+            "Synchronously execute a file containing a sequence of TypeQL queries with full validation. Queries can be explicitly separated with 'end;' if required. May be a HTTP-hosted file, an absolute path, or a relative path. Relative paths are relative to the invoking script (if there is one) or else a path relative to the current working directory. A sha256 may be optionally provided.",
+            vec![
+                CommandInput::new_required("file", get_word, Some(Box::new(file_completer))),
+                CommandInput::new_optional("file sha256 (hex or sha256:hex)", get_word, None),
+            ],
             transaction_source,
         ))
         // default: no token
         .add(CommandLeaf::new_with_input(
             "",
             "Execute query string.",
-            CommandInput::new("query", parse_one_query, None, None),
+            CommandInput::new_required("query", parse_one_query, None),
             transaction_query,
         ));
     repl