Merge remote-tracking branch 'origin/main' into contrib

Author: macneale4

.github/workflows/bump-dependency.yaml

@@ -173,6 +173,27 @@ jobs:
 
               if (pull.keepAlive) process.exit(0);
 
+              const checkSuiteRes = await github.rest.checks.listSuitesForRef({
+                owner,
+                repo,
+                ref: pull.headRef,
+              });
+
+              if (checkSuiteRes.data) {
+                for (const suite of checkSuiteRes.data.check_suites) {
+                  console.log("suite id:", suite.id);
+                  console.log("suite app slug:", suite.app.slug);
+                  console.log("suite status:", suite.status);
+                  console.log("suite conclusion:", suite.conclusion);
+                  if (suite.app.slug === "github-actions") {
+                     if (suite.status !== "completed" || suite.conclusion !== "success") {
+                       console.log(`Leaving pr open due to status:${suite.status} conclusion${suite.conclusion}`);
+                       process.exit(0);
+                     }
+                  }
+                }
+              }
+
               console.log(`Closing open pr ${pull.number}`);
               await github.rest.issues.createComment({
                 issue_number: pull.number,

.github/workflows/cd-release-pgo.yaml

@@ -170,6 +170,26 @@ jobs:
           asset_path: go/out/install.sh
           asset_name: install.sh
           asset_content_type: text/plain
+      - name: Upload Linux AMD64 RPM
+        id: upload-linux-amd64-rpm
+        uses: dolthub/upload-release-asset@v2
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ steps.create_release.outputs.upload_url }}
+          asset_path: go/out/dolt-${{ needs.format-version.outputs.version }}-1.x86_64.rpm
+          asset_name: dolt-${{ needs.format-version.outputs.version }}-1.x86_64.rpm
+          asset_content_type: application/zip
+      - name: Upload Linux ARM64 RPM
+        id: upload-linux-arm64-rpm
+        uses: dolthub/upload-release-asset@v2
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ steps.create_release.outputs.upload_url }}
+          asset_path: go/out/dolt-${{ needs.format-version.outputs.version }}-1.aarch64.rpm
+          asset_name: dolt-${{ needs.format-version.outputs.version }}-1.aarch64.rpm
+          asset_content_type: application/zip
 
   create-windows-msi:
     needs: [format-version, create-pgo-release]

.github/workflows/ci-bats-macos.yaml

@@ -89,9 +89,9 @@ jobs:
       - name: Install Maven
         working-directory: ./.ci_bin
         run: |
-          curl -LO https://dlcdn.apache.org/maven/maven-3/3.9.11/binaries/apache-maven-3.9.11-bin.tar.gz
-          tar -xf apache-maven-3.9.11-bin.tar.gz
-          echo "$(pwd)/apache-maven-3.9.11/bin" >> $GITHUB_PATH
+          curl -LO https://dlcdn.apache.org/maven/maven-3/3.9.11/binaries/apache-maven-3.9.12-bin.tar.gz
+          tar -xf apache-maven-3.9.12-bin.tar.gz
+          echo "$(pwd)/apache-maven-3.9.12/bin" >> $GITHUB_PATH
       - name: Install Hadoop
         working-directory: ./.ci_bin
         run: |

.github/workflows/ci-bats-unix-remote.yaml

@@ -78,9 +78,9 @@ jobs:
         if: ${{ env.use_credentials != 'true' }}
         working-directory: ./.ci_bin
         run: |
-          curl -LO https://dlcdn.apache.org/maven/maven-3/3.9.11/binaries/apache-maven-3.9.11-bin.tar.gz
-          tar -xf apache-maven-3.9.11-bin.tar.gz
-          echo "$(pwd)/apache-maven-3.9.11/bin" >> $GITHUB_PATH
+          curl -LO https://dlcdn.apache.org/maven/maven-3/3.9.12/binaries/apache-maven-3.9.12-bin.tar.gz
+          tar -xf apache-maven-3.9.12-bin.tar.gz
+          echo "$(pwd)/apache-maven-3.9.12/bin" >> $GITHUB_PATH
       - name: Install Hadoop
         if: ${{ env.use_credentials != 'true' }}
         working-directory: ./.ci_bin

.github/workflows/ci-bats-unix.yaml

@@ -94,9 +94,9 @@ jobs:
       - name: Install Maven
         working-directory: ./.ci_bin
         run: |
-          curl -LO https://dlcdn.apache.org/maven/maven-3/3.9.11/binaries/apache-maven-3.9.11-bin.tar.gz
-          tar -xf apache-maven-3.9.11-bin.tar.gz
-          echo "$(pwd)/apache-maven-3.9.11/bin" >> $GITHUB_PATH
+          curl -LO https://dlcdn.apache.org/maven/maven-3/3.9.12/binaries/apache-maven-3.9.12-bin.tar.gz
+          tar -xf apache-maven-3.9.12-bin.tar.gz
+          echo "$(pwd)/apache-maven-3.9.12/bin" >> $GITHUB_PATH
       - name: Install Hadoop
         working-directory: ./.ci_bin
         run: |

go/cmd/dolt/cli/arg_parser_helpers.go

@@ -46,15 +46,6 @@ func ParseAuthor(authorStr string) (string, string, error) {
 	return name, email, nil
 }
 
-const (
-	SyncBackupId        = "sync"
-	SyncBackupUrlId     = "sync-url"
-	RestoreBackupId     = "restore"
-	AddBackupId         = "add"
-	RemoveBackupId      = "remove"
-	RemoveBackupShortId = "rm"
-)
-
 var branchForceFlagDesc = "Reset {{.LessThan}}branchname{{.GreaterThan}} to {{.LessThan}}startpoint{{.GreaterThan}}, even if {{.LessThan}}branchname{{.GreaterThan}} exists already. Without {{.EmphasisLeft}}-f{{.EmphasisRight}}, {{.EmphasisLeft}}dolt branch{{.EmphasisRight}} refuses to change an existing branch. In combination with {{.EmphasisLeft}}-d{{.EmphasisRight}} (or {{.EmphasisLeft}}--delete{{.EmphasisRight}}), allow deleting the branch irrespective of its merged status. In combination with -m (or {{.EmphasisLeft}}--move{{.EmphasisRight}}), allow renaming the branch even if the new branch name already exists, the same applies for {{.EmphasisLeft}}-c{{.EmphasisRight}} (or {{.EmphasisLeft}}--copy{{.EmphasisRight}})."
 
 // CreateCommitArgParser creates the argparser shared dolt commit cli and DOLT_COMMIT.
@@ -97,6 +88,7 @@ func CreateMergeArgParser() *argparser.ArgParser {
 		return errors.New("Error: Dolt does not support merging from multiple commits. You probably meant to checkout one and then merge from the other.")
 	}
 	ap.SupportsFlag(NoFFParam, "", "Create a merge commit even when the merge resolves as a fast-forward.")
+	ap.SupportsFlag(FFOnlyParam, "", "Refuse to merge unless the current HEAD is already up to date or the merge can be resolved as a fast-forward.")
 	ap.SupportsFlag(SquashParam, "", "Merge changes to the working set without updating the commit history")
 	ap.SupportsString(MessageArg, "m", "msg", "Use the given {{.LessThan}}msg{{.GreaterThan}} as the commit message.")
 	ap.SupportsFlag(AbortParam, "", "Abort the in-progress merge and return the working set to the state before the merge started.")
@@ -227,6 +219,7 @@ func CreatePullArgParser() *argparser.ArgParser {
 	ap.ArgListHelp = append(ap.ArgListHelp, [2]string{"remoteBranch", "The name of a branch on the specified remote to be merged into the current working set."})
 	ap.SupportsFlag(SquashParam, "", "Merge changes to the working set without updating the commit history")
 	ap.SupportsFlag(NoFFParam, "", "Create a merge commit even when the merge resolves as a fast-forward.")
+	ap.SupportsFlag(FFOnlyParam, "", "Refuse to merge unless the current HEAD is already up to date or the merge can be resolved as a fast-forward.")
 	ap.SupportsFlag(ForceFlag, "f", "Update from the remote HEAD even if there are errors.")
 	ap.SupportsFlag(CommitFlag, "", "Perform the merge and commit the result. This is the default option, but can be overridden with the --no-commit flag. Note that this option does not affect fast-forward merges, which don't create a new merge commit, and if any merge conflicts or constraint violations are detected, no commit will be attempted.")
 	ap.SupportsFlag(NoCommitFlag, "", "Perform the merge and stop just before creating a merge commit. Note this will not prevent a fast-forward merge; use the --no-ff arg together with the --no-commit arg to prevent both fast-forwards and merge commits.")
@@ -320,6 +313,7 @@ func CreateDiffArgParser(isTableFunction bool) *argparser.ArgParser {
 		ap.SupportsString(FormatFlag, "r", "result output format", "How to format diff output. Valid values are tabular, sql, json. Defaults to tabular.")
 		ap.SupportsString(WhereParam, "", "column", "filters columns based on values in the diff.  See {{.EmphasisLeft}}dolt diff --help{{.EmphasisRight}} for details.")
 		ap.SupportsInt(LimitParam, "", "record_count", "limits to the first N diffs.")
+		ap.SupportsString(FilterParam, "", "diff_type", "filters results based on the type of change (added, modified, renamed, dropped). 'removed' is accepted as an alias for 'dropped'.")
 		ap.SupportsFlag(StagedFlag, "", "Show only the staged data changes.")
 		ap.SupportsFlag(CachedFlag, "c", "Synonym for --staged")
 		ap.SupportsFlag(MergeBase, "", "Uses merge base of the first commit and second commit (or HEAD if not supplied) as the first commit")
@@ -391,36 +385,21 @@ func CreateGlobalArgParser(name string) *argparser.ArgParser {
 	return ap
 }
 
-var awsParams = []string{dbfactory.AWSRegionParam, dbfactory.AWSCredsTypeParam, dbfactory.AWSCredsFileParam, dbfactory.AWSCredsProfile}
+var AwsParams = []string{dbfactory.AWSRegionParam, dbfactory.AWSCredsTypeParam, dbfactory.AWSCredsFileParam, dbfactory.AWSCredsProfile}
 var ossParams = []string{dbfactory.OSSCredsFileParam, dbfactory.OSSCredsProfile}
 
-func ProcessBackupArgs(apr *argparser.ArgParseResults, scheme, backupUrl string) (map[string]string, error) {
-	params := map[string]string{}
-
-	var err error
-	switch scheme {
-	case dbfactory.AWSScheme:
-		err = AddAWSParams(backupUrl, apr, params)
-	case dbfactory.OSSScheme:
-		err = AddOSSParams(backupUrl, apr, params)
-	default:
-		err = VerifyNoAwsParams(apr)
-	}
-	return params, err
-}
-
 func AddAWSParams(remoteUrl string, apr *argparser.ArgParseResults, params map[string]string) error {
 	isAWS := strings.HasPrefix(remoteUrl, "aws")
 
 	if !isAWS {
-		for _, p := range awsParams {
+		for _, p := range AwsParams {
 			if _, ok := apr.GetValue(p); ok {
 				return fmt.Errorf("%s param is only valid for aws cloud remotes in the format aws://dynamo-table:s3-bucket/database", p)
 			}
 		}
 	}
 
-	for _, p := range awsParams {
+	for _, p := range AwsParams {
 		if val, ok := apr.GetValue(p); ok {
 			params[p] = val
 		}
@@ -450,7 +429,7 @@ func AddOSSParams(remoteUrl string, apr *argparser.ArgParseResults, params map[s
 }
 
 func VerifyNoAwsParams(apr *argparser.ArgParseResults) error {
-	if awsParams := apr.GetValues(awsParams...); len(awsParams) > 0 {
+	if awsParams := apr.GetValues(AwsParams...); len(awsParams) > 0 {
 		awsParamKeys := make([]string, 0, len(awsParams))
 		for k := range awsParams {
 			awsParamKeys = append(awsParamKeys, k)

go/cmd/dolt/cli/flags.go

@@ -53,6 +53,7 @@ const (
 	NoCommitFlag         = "no-commit"
 	NoEditFlag           = "no-edit"
 	NoFFParam            = "no-ff"
+	FFOnlyParam          = "ff-only"
 	NoPrettyFlag         = "no-pretty"
 	NoTLSFlag            = "no-tls"
 	NoJsonMergeFlag      = "dont-merge-json"
@@ -99,6 +100,7 @@ const (
 	SummaryFlag  = "summary"
 	WhereParam   = "where"
 	LimitParam   = "limit"
+	FilterParam  = "filter"
 	MergeBase    = "merge-base"
 	DiffMode     = "diff-mode"
 	ReverseFlag  = "reverse"

go/cmd/dolt/commands/admin/journal_inspect.go

@@ -18,6 +18,7 @@ import (
 	"context"
 	"os"
 	"path/filepath"
+	"time"
 
 	"github.com/sirupsen/logrus"
 
@@ -47,7 +48,12 @@ func (cmd JournalInspectCmd) Docs() *cli.CommandDocumentation {
 		ShortDesc: "Inspect a Dolt journal file and display information about it",
 		LongDesc: `This tool is intented for debugging Dolt journal files. Since it is intended to debug potentially
 corrupted files, it is best run from a location which doesn't attempt to load databases. Ie, go to /tmp, and run
-dolt admin journal-inspect /path/to/journal/file`,
+dolt admin journal-inspect /path/to/journal/file
+
+When using the --filter-roots or --filter-chunks options, a new journal file will be created next to the original 
+file with the .filtered extension. This new journal will be identical to the original except that it will not contain 
+records of the specified type with any of the specified hashes. Multiple hashes can be provided as a comma-separated 
+list. The two filter options can be used together to filter both root and chunk records.`,
 		Synopsis: []string{
 			"<journal-path>",
 		},
@@ -61,10 +67,12 @@ func (cmd JournalInspectCmd) ArgParser() *argparser.ArgParser {
 	ap.SupportsFlag("verbose", "v", "Display verbose output during inspection (same as -r -c")
 	ap.SupportsFlag("crc-scan", "", "Scan invalid sections for valid CRCs. SLOW.")
 	ap.SupportsFlag("snappy-scan", "", "Scan invalid sections for snappy tags and content headers.")
+	ap.SupportsString("filter-roots", "", "hashcode1,hashcode2,...", "Create filtered copy of journal excluding the specified root hashes (comma-separated)")
+	ap.SupportsString("filter-chunks", "", "hashcode1,hashcode2,...", "Create filtered copy of journal excluding the specified chunk hashes (comma-separated)")
 	return ap
 }
 
-func (cmd JournalInspectCmd) Exec(_ context.Context, commandStr string, args []string, dEnv *env.DoltEnv, cliCtx cli.CliContext) int {
+func (cmd JournalInspectCmd) Exec(_ context.Context, commandStr string, args []string, _ *env.DoltEnv, _ cli.CliContext) int {
 	ap := cmd.ArgParser()
 	usage, _ := cli.HelpAndUsagePrinters(cli.CommandDocsForCommandString(commandStr, cli.CommandDocumentationContent{}, ap))
 	apr := cli.ParseArgsOrDie(ap, args, usage)
@@ -93,6 +101,8 @@ func (cmd JournalInspectCmd) Exec(_ context.Context, commandStr string, args []s
 
 	crcScan := apr.Contains("crc-scan")
 	snappScan := apr.Contains("snappy-scan")
+	filterRootsStr := apr.GetValueOrDefault("filter-roots", "")
+	filterChunksStr := apr.GetValueOrDefault("filter-chunks", "")
 
 	if _, err := os.Stat(journalPath); os.IsNotExist(err) {
 		cli.PrintErrln("Error: Journal file does not exist:", journalPath)
@@ -105,6 +115,31 @@ func (cmd JournalInspectCmd) Exec(_ context.Context, commandStr string, args []s
 		return 1
 	}
 
+	// Handle filter mode
+	if filterRootsStr != "" || filterChunksStr != "" {
+		result, exitCode := nbs.JournalFilter(absPath, filterRootsStr, filterChunksStr)
+		if exitCode != 0 {
+			return exitCode
+		}
+
+		// Only print shell commands if a filtered file was actually created
+		if result.OutputPath != "" && result.FilteredRecords > 0 {
+			// Print shell commands to replace the journal file
+			now := time.Now()
+			dateString := now.Format("2006_01_02_150405")
+
+			cli.Println("")
+			cli.Printf("Filtered file: %s\n", result.OutputPath)
+			cli.Println("")
+			cli.Println("To replace the original journal file, run these commands:")
+			cli.Printf("cp %s %s_saved_%s\n", result.OriginalPath, result.OriginalPath, dateString)
+			cli.Printf("mv %s %s\n", result.OutputPath, result.OriginalPath)
+			cli.Printf("rm %s\n", filepath.Join(filepath.Dir(result.OriginalPath), "journal.idx"))
+		}
+
+		return 0
+	}
+
 	// JournalInspect returns an exit code. It's entire purpose it to print errors, after all.
 	return nbs.JournalInspect(absPath, seeRoots, seeChunks, crcScan, snappScan)
 }

go/cmd/dolt/commands/assist.go

@@ -36,6 +36,7 @@ import (
 	"github.com/dolthub/dolt/go/libraries/doltcore/dconfig"
 	"github.com/dolthub/dolt/go/libraries/doltcore/doltdb"
 	"github.com/dolthub/dolt/go/libraries/doltcore/env"
+	"github.com/dolthub/dolt/go/libraries/doltcore/sqle/overrides"
 	"github.com/dolthub/dolt/go/libraries/utils/argparser"
 )
 
@@ -581,6 +582,7 @@ func jsonMessage(role string, content string) ([]byte, error) {
 }
 
 func getCreateTableStatements(ctx *sql.Context, sqlEngine *engine.SqlEngine, dEnv *env.DoltEnv) (string, error) {
+	formatter := overrides.SchemaFormatterFromContext(ctx)
 	sb := strings.Builder{}
 
 	root, err := dEnv.WorkingRoot(ctx)
@@ -590,7 +592,7 @@ func getCreateTableStatements(ctx *sql.Context, sqlEngine *engine.SqlEngine, dEn
 
 	tables, err := root.GetTableNames(ctx, doltdb.DefaultSchemaName, true)
 	for _, table := range tables {
-		_, iter, _, err := sqlEngine.Query(ctx, fmt.Sprintf("SHOW CREATE TABLE %s", sql.QuoteIdentifier(table)))
+		_, iter, _, err := sqlEngine.Query(ctx, fmt.Sprintf("SHOW CREATE TABLE %s", formatter.QuoteIdentifier(table)))
 		if err != nil {
 			return "", err
 		}