Add full clone setting for prebuilds (#20177)

* proto update

Co-authored-by: Filip Troníček <[email protected]>
Co-authored-by: Gero Posmyk-Leinemann <[email protected]>

* dashboard + server

---------

Co-authored-by: Filip Troníček <[email protected]>
Co-authored-by: Gero Posmyk-Leinemann <[email protected]>

Author: 3 people

components/content-service-api/go/initializer.pb.go

@@ -61,6 +61,9 @@ message GitInitializer {
 
     // config specifies the Git configuration for this workspace
     GitConfig config = 6;
+
+    // full_clone determines if the entire repository should be cloned, instead of with `--depth=1`
+    bool full_clone = 7;
 }
 
 // CloneTargetMode is the target state in which we want to leave a GitWorkspace

components/content-service-api/initializer.proto

@@ -13,6 +13,7 @@ import (
 	"os/exec"
 	"path/filepath"
 	"strings"
+	"time"
 
 	"github.com/opentracing/opentracing-go"
 	"golang.org/x/xerrors"
@@ -95,6 +96,9 @@ type Client struct {
 
 	// if true will run git command as gitpod user (should be executed as root that has access to sudo in this case)
 	RunAsGitpodUser bool
+
+	// FullClone indicates whether we should do a full checkout or a shallow clone
+	FullClone bool
 }
 
 // Status describes the status of a Git repo/working copy akin to "git status"
@@ -338,8 +342,18 @@ func (c *Client) Clone(ctx context.Context) (err error) {
 		log.WithError(err).Error("cannot create clone location")
 	}
 
+	now := time.Now()
+
+	defer func() {
+		log.WithField("duration", time.Since(now).String()).WithField("FullClone", c.FullClone).Info("clone repository took")
+	}()
+
 	args := []string{"--depth=1", "--shallow-submodules", c.RemoteURI}
 
+	if c.FullClone {
+		args = []string{c.RemoteURI}
+	}
+
 	for key, value := range c.Config {
 		args = append(args, "--config")
 		args = append(args, strings.TrimSpace(key)+"="+strings.TrimSpace(value))

components/content-service-api/typescript/src/initializer_pb.d.ts

@@ -10,6 +10,7 @@ import (
 	"fmt"
 	"os"
 	"os/exec"
+	"strconv"
 	"strings"
 	"time"
 
@@ -199,6 +200,20 @@ func (ws *GitInitializer) Run(ctx context.Context, mappings []archive.IDMapping)
 	return
 }
 
+func (ws *GitInitializer) isShallowRepository(ctx context.Context) bool {
+	out, err := ws.GitWithOutput(ctx, nil, "rev-parse", "--is-shallow-repository")
+	if err != nil {
+		log.WithError(err).Error("unexpected error checking if git repository is shallow")
+		return true
+	}
+	isShallow, err := strconv.ParseBool(strings.TrimSpace(string(out)))
+	if err != nil {
+		log.WithError(err).WithField("input", string(out)).Error("unexpected error parsing bool")
+		return true
+	}
+	return isShallow
+}
+
 // realizeCloneTarget ensures the clone target is checked out
 func (ws *GitInitializer) realizeCloneTarget(ctx context.Context) (err error) {
 	//nolint:ineffassign
@@ -235,8 +250,13 @@ func (ws *GitInitializer) realizeCloneTarget(ctx context.Context) (err error) {
 		//
 		// We don't recurse submodules because callers realizeCloneTarget() are expected to update submodules explicitly,
 		// and deal with any error appropriately (i.e. emit a warning rather than fail).
-		if err := ws.Git(ctx, "fetch", "--depth=1", "origin", "--recurse-submodules=no", ws.CloneTarget); err != nil {
-			log.WithError(err).WithField("remoteURI", ws.RemoteURI).WithField("branch", ws.CloneTarget).Error("Cannot fetch remote branch")
+		fetchArgs := []string{"--depth=1", "origin", "--recurse-submodules=no", ws.CloneTarget}
+		isShallow := ws.isShallowRepository(ctx)
+		if !isShallow {
+			fetchArgs = []string{"origin", "--recurse-submodules=no", ws.CloneTarget}
+		}
+		if err := ws.Git(ctx, "fetch", fetchArgs...); err != nil {
+			log.WithError(err).WithField("isShallow", isShallow).WithField("remoteURI", ws.RemoteURI).WithField("branch", ws.CloneTarget).Error("Cannot fetch remote branch")
 			return err
 		}
 

components/content-service-api/typescript/src/initializer_pb.js

@@ -304,6 +304,7 @@ func newGitInitializer(ctx context.Context, loc string, req *csapi.GitInitialize
 			AuthMethod:        authMethod,
 			AuthProvider:      authProvider,
 			RunAsGitpodUser:   forceGitpodUser,
+			FullClone:         req.FullClone,
 		},
 		TargetMode:  targetMode,
 		CloneTarget: req.CloneTaget,

components/content-service/pkg/git/git.go

@@ -24,6 +24,7 @@ const featureFlags = {
     dataops: false,
     showBrowserExtensionPromotion: false,
     enable_experimental_jbtb: false,
+    enabled_configuration_prebuild_full_clone: false,
 };
 
 type FeatureFlags = typeof featureFlags;

components/content-service/pkg/initializer/git.go

@@ -26,6 +26,8 @@ import { useUserLoader } from "../../../hooks/use-user-loader";
 import { useUpdateCurrentUserMutation } from "../../../data/current-user/update-mutation";
 import { trackEvent } from "../../../Analytics";
 import dayjs from "dayjs";
+import { SwitchInputField } from "@podkit/switch/Switch";
+import { useFeatureFlag } from "../../../data/featureflag-query";
 
 const DEFAULT_PREBUILD_COMMIT_INTERVAL = 20;
 
@@ -40,6 +42,7 @@ export const PrebuildSettingsForm: FC<Props> = ({ configuration }) => {
 
     const { user } = useUserLoader();
     const { mutate: updateUser } = useUpdateCurrentUserMutation();
+    const isEnabledPrebuildFullClone = useFeatureFlag("enabled_configuration_prebuild_full_clone");
 
     const updateConfiguration = useConfigurationMutation();
 
@@ -55,6 +58,9 @@ export const PrebuildSettingsForm: FC<Props> = ({ configuration }) => {
     const [workspaceClass, setWorkspaceClass] = useState<string>(
         configuration.prebuildSettings?.workspaceClass || DEFAULT_WS_CLASS,
     );
+    const [fullClone, setFullClone] = useState<boolean>(
+        configuration.prebuildSettings?.cloneSettings?.fullClone ?? false,
+    );
 
     const [isTriggerNotificationOpen, setIsTriggerNotificationOpen] = useState(true);
 
@@ -72,6 +78,9 @@ export const PrebuildSettingsForm: FC<Props> = ({ configuration }) => {
                     branchStrategy,
                     branchMatchingPattern,
                     workspaceClass,
+                    cloneSettings: {
+                        fullClone,
+                    },
                 },
             };
 
@@ -90,6 +99,7 @@ export const PrebuildSettingsForm: FC<Props> = ({ configuration }) => {
             toast,
             updateConfiguration,
             workspaceClass,
+            fullClone,
         ],
     );
 
@@ -200,6 +210,20 @@ export const PrebuildSettingsForm: FC<Props> = ({ configuration }) => {
                     />
                 )}
 
+                {isEnabledPrebuildFullClone && (
+                    <InputField
+                        label="Clone repositories in full"
+                        hint="Make prebuilds fully clone the repository, maintaining the entire git history for use in prebuilds and workspaces."
+                    >
+                        <SwitchInputField
+                            id="prebuild-full-clone-enabled"
+                            checked={fullClone}
+                            onCheckedChange={setFullClone}
+                            label=""
+                        />
+                    </InputField>
+                )}
+
                 <Heading3 className="mt-8">Machine type</Heading3>
                 <Subheading>Choose the workspace machine type for your prebuilds.</Subheading>
 

components/content-service/pkg/initializer/initializer.go

@@ -32,6 +32,9 @@ export interface ProjectSettings {
 export namespace PrebuildSettings {
     export type BranchStrategy = "default-branch" | "all-branches" | "matched-branches";
     export type TriggerStrategy = "activity-based" | "webhook-based";
+    export interface CloneSettings {
+        fullClone?: boolean;
+    }
 }
 
 export interface PrebuildSettings {
@@ -61,6 +64,8 @@ export interface PrebuildSettings {
      * The activation strategy for prebuilds. Defaults to "webhook-based"
      */
     triggerStrategy?: PrebuildSettings.TriggerStrategy;
+
+    cloneSettings?: PrebuildSettings.CloneSettings;
 }
 
 export interface Project {