feat: v2 charts with backwards compatibility (#174)

Author: bernielomax

internal/cmd/images/manifest_cmd.go

@@ -45,11 +45,29 @@ func (c *ManifestCmd) Run(ctx context.Context, newSvcMgrClients service.ManagerC
 func (c *ManifestCmd) findAirbyteImages(ctx context.Context, helmClient goHelm.Client) ([]string, error) {
 	valuesYaml, err := helm.BuildAirbyteValues(ctx, helm.ValuesOpts{
 		ValuesFile: c.Values,
-	})
+	}, c.ChartVersion)
 	if err != nil {
 		return nil, err
 	}
 
-	airbyteChartLoc := helm.LocateLatestAirbyteChart(c.ChartVersion, c.Chart)
-	return helm.FindImagesFromChart(helmClient, valuesYaml, airbyteChartLoc, c.ChartVersion)
+	// Determine and set defaults for chart flags.
+	err = c.setDefaultChartFlags(helmClient)
+	if err != nil {
+		return nil, fmt.Errorf("failed to set chart flag defaults: %w", err)
+	}
+
+	return helm.FindImagesFromChart(helmClient, valuesYaml, c.Chart, c.ChartVersion)
+}
+
+func (c *ManifestCmd) setDefaultChartFlags(helmClient goHelm.Client) error {
+	resolver := helm.NewChartResolver(helmClient)
+	resolvedChart, resolvedVersion, err := resolver.ResolveChartReference(c.Chart, c.ChartVersion)
+	if err != nil {
+		return fmt.Errorf("failed to resolve chart flags: %w", err)
+	}
+
+	c.Chart = resolvedChart
+	c.ChartVersion = resolvedVersion
+
+	return nil
 }

internal/cmd/local/install.go

@@ -10,6 +10,7 @@ import (
 	"github.com/airbytehq/abctl/internal/service"
 	"github.com/airbytehq/abctl/internal/telemetry"
 	"github.com/airbytehq/abctl/internal/trace"
+	goHelm "github.com/mittwald/go-helm-client"
 	"github.com/pterm/pterm"
 	"go.opentelemetry.io/otel/attribute"
 )
@@ -103,6 +104,18 @@ func (i *InstallCmd) Run(ctx context.Context, provider k8s.Provider, newSvcMgrCl
 			pterm.Success.Printfln("Cluster '%s' created", provider.ClusterName)
 		}
 
+		// Load the required service manager clients.
+		k8sClient, helmClient, err := newSvcMgrClients(provider.Kubeconfig, provider.Context)
+		if err != nil {
+			return err
+		}
+
+		// Determine and set defaults for chart flags.
+		err = i.setDefaultChartFlags(helmClient)
+		if err != nil {
+			return fmt.Errorf("failed to set chart defaults: %w", err)
+		}
+
 		// Overrides Helm chart images.
 		overrideImages := []string{}
 
@@ -115,15 +128,6 @@ func (i *InstallCmd) Run(ctx context.Context, provider k8s.Provider, newSvcMgrCl
 			overrideImages = append(overrideImages, "airbyte/db:"+helm.Psql17AirbyteTag)
 		}
 
-		// Load the required service manager clients.
-		// TODO(bernielomax): The Helm client will eventually be dependency-injected
-		// into the build values function to support querying the Helm chart for
-		// version compatibility operations.
-		k8sClient, helmClient, err := newSvcMgrClients(provider.Kubeconfig, provider.Context)
-		if err != nil {
-			return err
-		}
-
 		svcMgr, err := service.NewManager(provider,
 			service.WithK8sClient(k8sClient),
 			service.WithHelmClient(helmClient),
@@ -186,7 +190,7 @@ func (i *InstallCmd) installOpts(ctx context.Context, user string) (*service.Ins
 
 	opts := &service.InstallOpts{
 		HelmChartVersion: i.ChartVersion,
-		AirbyteChartLoc:  helm.LocateLatestAirbyteChart(i.ChartVersion, i.Chart),
+		AirbyteChartLoc:  i.Chart,
 		Secrets:          i.Secret,
 		Hosts:            i.Host,
 		LocalStorage:     !supportMinio,
@@ -216,11 +220,26 @@ func (i *InstallCmd) installOpts(ctx context.Context, user string) (*service.Ins
 		valuesOpts.TelemetryUser = user
 	}
 
-	valuesYAML, err := helm.BuildAirbyteValues(ctx, valuesOpts)
+	// Build the values.yaml file for the Airbyte chart.
+	valuesYAML, err := helm.BuildAirbyteValues(ctx, valuesOpts, i.ChartVersion)
 	if err != nil {
 		return nil, err
 	}
+
 	opts.HelmValuesYaml = valuesYAML
 
 	return opts, nil
 }
+
+func (i *InstallCmd) setDefaultChartFlags(helmClient goHelm.Client) error {
+	resolver := helm.NewChartResolver(helmClient)
+	resolvedChart, resolvedVersion, err := resolver.ResolveChartReference(i.Chart, i.ChartVersion)
+	if err != nil {
+		return fmt.Errorf("failed to resolve chart flags: %w", err)
+	}
+
+	i.Chart = resolvedChart
+	i.ChartVersion = resolvedVersion
+
+	return nil
+}

internal/cmd/local/local_test.go

@@ -10,12 +10,12 @@ import (
 
 	"github.com/airbytehq/abctl/internal/abctl"
 	"github.com/airbytehq/abctl/internal/k8s"
-	"github.com/airbytehq/abctl/internal/service"
-	"github.com/alecthomas/kong"
-
 	"github.com/airbytehq/abctl/internal/paths"
+	"github.com/airbytehq/abctl/internal/service"
 	"github.com/airbytehq/abctl/internal/telemetry"
+	"github.com/alecthomas/kong"
 	"github.com/google/go-cmp/cmp"
+	goHelm "github.com/mittwald/go-helm-client"
 )
 
 func TestCheckAirbyteDir(t *testing.T) {
@@ -119,27 +119,39 @@ func TestValues_FileDoesntExist(t *testing.T) {
 
 func TestValues_BadYaml(t *testing.T) {
 	cmd := InstallCmd{Values: "./testdata/invalid.values.yaml"}
-	err := cmd.Run(context.Background(), k8s.TestProvider, nil, telemetry.NoopClient{})
+	// Does not need actual clients for tests.
+	testFactory := func(kubeConfig, kubeContext string) (k8s.Client, goHelm.Client, error) {
+		return nil, nil, nil
+	}
+	err := cmd.Run(context.Background(), k8s.TestProvider, testFactory, telemetry.NoopClient{})
 	if err == nil {
 		t.Fatal("expected error")
 	}
 
-	if !strings.HasPrefix(err.Error(), "failed to unmarshal file") {
+	if !strings.HasPrefix(err.Error(), "failed to build values yaml for v2 chart: failed to unmarshal file") {
 		t.Errorf("unexpected error: %v", err)
 	}
 }
 
 func TestInvalidHostFlag_IpAddr(t *testing.T) {
 	cmd := InstallCmd{Host: []string{"ok", "1.2.3.4"}}
-	err := cmd.Run(context.Background(), k8s.TestProvider, nil, telemetry.NoopClient{})
+	// Does not need actual clients for tests.
+	testFactory := func(kubeConfig, kubeContext string) (k8s.Client, goHelm.Client, error) {
+		return nil, nil, nil
+	}
+	err := cmd.Run(context.Background(), k8s.TestProvider, testFactory, telemetry.NoopClient{})
 	if !errors.Is(err, abctl.ErrIpAddressForHostFlag) {
 		t.Errorf("expected ErrIpAddressForHostFlag but got %v", err)
 	}
 }
 
 func TestInvalidHostFlag_IpAddrWithPort(t *testing.T) {
 	cmd := InstallCmd{Host: []string{"ok", "1.2.3.4:8000"}}
-	err := cmd.Run(context.Background(), k8s.TestProvider, nil, telemetry.NoopClient{})
+	// Does not need actual clients for tests.
+	testFactory := func(kubeConfig, kubeContext string) (k8s.Client, goHelm.Client, error) {
+		return nil, nil, nil
+	}
+	err := cmd.Run(context.Background(), k8s.TestProvider, testFactory, telemetry.NoopClient{})
 	if !errors.Is(err, abctl.ErrInvalidHostFlag) {
 		t.Errorf("expected ErrInvalidHostFlag but got %v", err)
 	}

internal/common/const.go

@@ -8,7 +8,8 @@ const (
 	AirbyteNamespace         = "airbyte-abctl"
 	AirbyteKubeContext       = "kind-airbyte-abctl"
 	AirbyteRepoName          = "airbyte"
-	AirbyteRepoURL           = "https://airbytehq.github.io/helm-charts"
+	AirbyteRepoURLv1         = "https://airbytehq.github.io/helm-charts"
+	AirbyteRepoURLv2         = "https://airbytehq.github.io/charts"
 	NginxChartName           = "nginx/ingress-nginx"
 	NginxChartRelease        = "ingress-nginx"
 	NginxNamespace           = "ingress-nginx"

internal/helm/airbyte_values.go

@@ -9,6 +9,7 @@ import (
 	"go.opentelemetry.io/otel/trace"
 )
 
+// ValuesOpts contains configuration options for building Airbyte Helm values.
 type ValuesOpts struct {
 	ValuesFile      string
 	LowResourceMode bool
@@ -21,10 +22,31 @@ type ValuesOpts struct {
 }
 
 const (
+	// Psql17AirbyteTag is the image tag for PostgreSQL 17 compatibility
 	Psql17AirbyteTag = "1.7.0-17"
 )
 
-func BuildAirbyteValues(ctx context.Context, opts ValuesOpts) (string, error) {
+// BuildAirbyteValues generates a values yaml string for the Airbyte Helm chart based on the chart version.
+// It delegates to BuildAirbyteValuesV1 for v1 charts and BuildAirbyteValuesV2 for v2+ charts.
+func BuildAirbyteValues(ctx context.Context, valuesOpts ValuesOpts, chartVersion string) (string, error) {
+	if ChartIsV2Plus(chartVersion) {
+		valuesYAML, err := buildAirbyteValuesV2(ctx, valuesOpts)
+		if err != nil {
+			return "", fmt.Errorf("failed to build values yaml for v2 chart: %w", err)
+		}
+		return valuesYAML, nil
+	} else {
+		valuesYAML, err := buildAirbyteValuesV1(ctx, valuesOpts)
+		if err != nil {
+			return "", fmt.Errorf("failed to build values yaml for v1 chart: %w", err)
+		}
+		return valuesYAML, nil
+	}
+}
+
+// buildAirbyteValuesV1 generates values string for v1 Airbyte Helm charts.
+// It applies configuration options and merges them with user-provided values file.
+func buildAirbyteValuesV1(ctx context.Context, opts ValuesOpts) (string, error) {
 	span := trace.SpanFromContext(ctx)
 
 	vals := []string{
@@ -89,6 +111,74 @@ func BuildAirbyteValues(ctx context.Context, opts ValuesOpts) (string, error) {
 	return mergeValuesWithValuesYAML(vals, fileVals)
 }
 
+// buildAirbyteValuesV2 generates values string for v2+ Airbyte Helm charts.
+// It applies configuration options and merges them with user-provided values file.
+func buildAirbyteValuesV2(ctx context.Context, opts ValuesOpts) (string, error) {
+	span := trace.SpanFromContext(ctx)
+
+	vals := []string{
+		"server.env_vars.WEBAPP_URL=http://airbyte-abctl-airbyte-server-svc:80",
+		"global.env_vars.AIRBYTE_INSTALLATION_ID=" + opts.TelemetryUser,
+		"global.jobs.resources.limits.cpu=3",
+		"global.jobs.resources.limits.memory=4Gi",
+		"airbyte-bootloader.env_vars.PLATFORM_LOG_FORMAT=json",
+	}
+
+	if opts.LocalStorage {
+		vals = append(vals, "global.storage.type=local")
+	}
+
+	if opts.EnablePsql17 {
+		vals = append(vals, "postgresql.image.tag="+Psql17AirbyteTag)
+	}
+
+	span.SetAttributes(
+		attribute.Bool("low-resource-mode", opts.LowResourceMode),
+		attribute.Bool("insecure-cookies", opts.InsecureCookies),
+		attribute.Bool("image-pull-secret", opts.ImagePullSecret != ""),
+		attribute.Bool("local-storage", opts.LocalStorage),
+	)
+
+	if !opts.DisableAuth {
+		vals = append(vals, "global.auth.enabled=true")
+	}
+
+	if opts.LowResourceMode {
+		vals = append(vals,
+			"server.env_vars.JOB_RESOURCE_VARIANT_OVERRIDE=lowresource",
+			"global.jobs.resources.requests.cpu=0",
+			"global.jobs.resources.requests.memory=0",
+
+			"connectorBuilderServer.enabled=false",
+
+			"workloadLauncher.env_vars.CHECK_JOB_MAIN_CONTAINER_CPU_REQUEST=0",
+			"workloadLauncher.env_vars.CHECK_JOB_MAIN_CONTAINER_MEMORY_REQUEST=0",
+			"workloadLauncher.env_vars.DISCOVER_JOB_MAIN_CONTAINER_CPU_REQUEST=0",
+			"workloadLauncher.env_vars.DISCOVER_JOB_MAIN_CONTAINER_MEMORY_REQUEST=0",
+			"workloadLauncher.env_vars.SPEC_JOB_MAIN_CONTAINER_CPU_REQUEST=0",
+			"workloadLauncher.env_vars.SPEC_JOB_MAIN_CONTAINER_MEMORY_REQUEST=0",
+			"workloadLauncher.env_vars.SIDECAR_MAIN_CONTAINER_CPU_REQUEST=0",
+			"workloadLauncher.env_vars.SIDECAR_MAIN_CONTAINER_MEMORY_REQUEST=0",
+		)
+	}
+
+	if opts.ImagePullSecret != "" {
+		vals = append(vals, fmt.Sprintf("global.imagePullSecrets[0].name=%s", opts.ImagePullSecret))
+	}
+
+	if opts.InsecureCookies {
+		// Boolean is a string value in the v1 Helm chart.
+		vals = append(vals, `global.auth.cookieSecureSetting="false"`)
+	}
+
+	fileVals, err := maps.FromYAMLFile(opts.ValuesFile)
+	if err != nil {
+		return "", err
+	}
+
+	return mergeValuesWithValuesYAML(vals, fileVals)
+}
+
 // mergeValuesWithValuesYAML ensures that the values defined within this code have a lower
 // priority than any values defined in a values.yaml file.
 // By default, the helm-client we're using reversed this priority, putting the values