New collector based tests

Dockerfile

@@ -14,15 +14,16 @@
 
 FROM golang:1.23 AS gobuild
 WORKDIR /src
+ARG BUILD_TAGS
 
 # cache deps before copying source so that we don't re-download as much
 COPY go.mod go.sum ./
 RUN go mod download
 
 COPY . .
-RUN CGO_ENABLED=0 go test -tags=e2e -c ./e2etestrunner -o opentelemetry-operations-e2e-testing.test
+RUN set -x; if [ "$BUILD_TAGS" = "e2ecollector" ]; then BUILD_DIRECTORY="e2etestrunner-collector"; else BUILD_TAGS="e2e"; BUILD_DIRECTORY="e2etestrunner"; fi; CGO_ENABLED=0 go test -timeout 3600s -tags=$BUILD_TAGS -c "./$BUILD_DIRECTORY" -o opentelemetry-operations-e2e-testing.test
 
 FROM hashicorp/terraform:light as tfbuild
 COPY tf /src/tf
 WORKDIR /src/tf
 ENV TF_PLUGIN_CACHE_DIR=/src/tf/terraform-plugin-cache

cloudbuild-collector.yaml

@@ -0,0 +1,35 @@
+# Copyright 2021 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Builds a docker image, tagging it as _DOCKER_TAG (substitution must be
+# provided) and latest.
+
+steps:
+- name: docker
+  env:
+  - DOCKER_BUILDKIT=1
+  args:
+  - build
+  - --tag=${_TEST_RUNNER_IMAGE_NAME}:${_DOCKER_TAG}
+  - --tag=${_TEST_RUNNER_IMAGE_NAME}:latest
+  - --build-arg=BUILDKIT_INLINE_CACHE=1
+  - --build-arg=BUILD_TAGS=e2ecollector
+  - --cache-from=${_TEST_RUNNER_IMAGE_NAME}:latest
+  - .
+
+images: ["${_TEST_RUNNER_IMAGE_NAME}"]
+options:
+  dynamic_substitutions: true
+substitutions:
+  _TEST_RUNNER_IMAGE_NAME: us-central1-docker.pkg.dev/${PROJECT_ID}/e2e-testing/opentelemetry-operations-e2e-testing

cmd/testmatrix/main.go

@@ -25,6 +25,7 @@ import (
 	"bufio"
 	"context"
 	"fmt"
+	"github.com/GoogleCloudPlatform/opentelemetry-operations-e2e-testing/util"
 	"html/template"
 	"log"
 	"os"
@@ -33,7 +34,6 @@ import (
 	"strings"
 
 	"cloud.google.com/go/storage"
-	"github.com/GoogleCloudPlatform/opentelemetry-operations-e2e-testing/e2etestrunner"
 	"github.com/alexflint/go-arg"
 	"golang.org/x/sync/errgroup"
 	"google.golang.org/api/cloudbuild/v1"
@@ -88,7 +88,7 @@ This will fetch recent Cloud Build logs to automatically update the statuses in
 )
 
 type Args struct {
-	e2etestrunner.CmdWithProjectId
+	util.CmdWithProjectId
 }
 
 type status string
@@ -106,7 +106,7 @@ var (
 )
 
 func main() {
-	args := e2etestrunner.Args{}
+	args := util.Args{}
 	arg.MustParse(&args)
 
 	ctx := context.Background()

e2etestrunner-collector/main_test.go

@@ -0,0 +1,86 @@
+//go:build e2ecollector
+
+package e2etestrunner_collector
+
+import (
+	"context"
+	"flag"
+	"github.com/GoogleCloudPlatform/opentelemetry-operations-e2e-testing/util"
+	"github.com/GoogleCloudPlatform/opentelemetry-operations-e2e-testing/util/setuptf"
+	"log"
+	"math/rand"
+	"os"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/alexflint/go-arg"
+)
+
+var (
+	args         util.Args
+	resourceType string
+)
+
+func TestMain(m *testing.M) {
+	rand.New(rand.NewSource(time.Now().UnixNano()))
+	p := arg.MustParse(&args)
+	if p.Subcommand() == nil {
+		p.Fail("missing command")
+	}
+	// Need a logger just for TestMain() before testing.T is available
+	logger := log.New(os.Stdout, "TestMain: ", log.LstdFlags|log.Lshortfile)
+	ctx := context.Background()
+
+	// Handle special case of just creating persistent resources
+	if args.ApplyPersistent != nil {
+		err := setuptf.ApplyPersistentCollector(ctx, args.ProjectID, args.ApplyPersistent.AutoApprove, logger)
+		if err != nil {
+			logger.Panic(err)
+		}
+		return
+	}
+
+	// hacky but works
+	os.Args = append([]string{os.Args[0]}, strings.Fields(args.GoTestFlags)...)
+	flag.Parse()
+
+	// handle any complex defaults
+	if args.TestRunID == "" {
+		hex, err := util.RandomHex(6)
+		if err != nil {
+			logger.Fatalf("error generating random hex string: %v\n", err)
+		}
+		args.TestRunID = hex
+	}
+
+	var setupFunc util.SetupCollectorFunc
+	switch {
+	case args.GceCollector != nil:
+		setupFunc = SetupGceCollector
+		resourceType = "gce_instance"
+	case args.GkeCollector != nil:
+		setupFunc = SetupGkeCollector
+		resourceType = "k8s_container"
+	case args.GkeOperatorCollector != nil:
+		setupFunc = SetupGkeOperatorCollector
+		resourceType = "k8s_container"
+	case args.CloudRunCollector != nil:
+		setupFunc = SetupCloudRunCollector
+		resourceType = "generic_task"
+	}
+	cleanup, err := setupFunc(ctx, &args, logger)
+
+	defer cleanup()
+	if err != nil {
+		logger.Panic(err)
+	}
+
+	// wait for instrumented test server to be healthy
+	logger.Printf("Waiting for health check on (will timeout after %v)\n", args.HealthCheckTimeout)
+	time.Sleep(args.HealthCheckTimeout)
+	// Run tests
+	logger.Print(util.BeginOutputArt)
+	m.Run()
+	logger.Print(util.EndOutputArt)
+}

e2etestrunner-collector/setupgcecollector.go

@@ -0,0 +1,49 @@
+// Copyright 2021 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package e2etestrunner_collector
+
+import (
+	"context"
+	"github.com/GoogleCloudPlatform/opentelemetry-operations-e2e-testing/util"
+	"github.com/GoogleCloudPlatform/opentelemetry-operations-e2e-testing/util/setuptf"
+	"log"
+)
+
+const gceCollectorTfDir string = "tf/gce-collector"
+
+// SetupGceCollector Set up the collector to run in GCE container. Creates a new
+// GCE VM resources, and runs the specified container image. The
+// returned cleanup function tears down the VM.
+func SetupGceCollector(
+	ctx context.Context,
+	args *util.Args,
+	logger *log.Logger,
+) (util.Cleanup, error) {
+	_, cleanupTf, err := setuptf.SetupTf(
+		ctx,
+		args.ProjectID,
+		args.TestRunID,
+		gceCollectorTfDir,
+		map[string]string{
+			"image": args.GceCollector.Image,
+		},
+		logger,
+	)
+	if err != nil {
+		return cleanupTf, err
+	}
+
+	return cleanupTf, err
+}

e2etestrunner-collector/setupgcloudruncollector.go

@@ -0,0 +1,49 @@
+// Copyright 2021 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package e2etestrunner_collector
+
+import (
+	"context"
+	"github.com/GoogleCloudPlatform/opentelemetry-operations-e2e-testing/util"
+	"github.com/GoogleCloudPlatform/opentelemetry-operations-e2e-testing/util/setuptf"
+	"log"
+)
+
+const cloudRunCollectorTfDir string = "tf/cloud-run-collector"
+
+// SetupCloudRunCollector sets up the collector to run in Cloud Run.
+// Creates a new service and runs the specified container image as a revision.
+// The returned cleanup function tears down everything.
+func SetupCloudRunCollector(
+	ctx context.Context,
+	args *util.Args,
+	logger *log.Logger,
+) (util.Cleanup, error) {
+	_, cleanupTf, err := setuptf.SetupTf(
+		ctx,
+		args.ProjectID,
+		args.TestRunID,
+		cloudRunCollectorTfDir,
+		map[string]string{
+			"image": args.CloudRunCollector.Image,
+		},
+		logger,
+	)
+	if err != nil {
+		return cleanupTf, err
+	}
+
+	return cleanupTf, err
+}

e2etestrunner-collector/setupgkecollector.go

@@ -0,0 +1,35 @@
+package e2etestrunner_collector
+
+import (
+	"context"
+	"github.com/GoogleCloudPlatform/opentelemetry-operations-e2e-testing/util"
+	"github.com/GoogleCloudPlatform/opentelemetry-operations-e2e-testing/util/setuptf"
+	"log"
+)
+
+const gkeCollectorTfDir string = "tf/gke-collector"
+
+// SetupGkeCollector Set up the collector to run in GKE.
+// Creates a new pod and runs the specified container image in a pod.
+// The returned cleanup function tears down the whole cluster.
+func SetupGkeCollector(
+	ctx context.Context,
+	args *util.Args,
+	logger *log.Logger,
+) (util.Cleanup, error) {
+	_, cleanupTf, err := setuptf.SetupTf(
+		ctx,
+		args.ProjectID,
+		args.TestRunID,
+		gkeCollectorTfDir,
+		map[string]string{
+			"image": args.GkeCollector.Image,
+		},
+		logger,
+	)
+	if err != nil {
+		return cleanupTf, err
+	}
+
+	return cleanupTf, err
+}

e2etestrunner-collector/setupgkeoperatorcollector.go

@@ -0,0 +1,35 @@
+package e2etestrunner_collector
+
+import (
+	"context"
+	"github.com/GoogleCloudPlatform/opentelemetry-operations-e2e-testing/util"
+	"github.com/GoogleCloudPlatform/opentelemetry-operations-e2e-testing/util/setuptf"
+	"log"
+)
+
+const gkeOperatorCollectorTfDir string = "tf/gke-operator-collector"
+
+// SetupGkeOperatorCollector Set up the collector to run in GKE.
+// Creates a new pod and runs the specified container image in a pod.
+// The returned cleanup function tears down the whole cluster.
+func SetupGkeOperatorCollector(
+	ctx context.Context,
+	args *util.Args,
+	logger *log.Logger,
+) (util.Cleanup, error) {
+	_, cleanupTf, err := setuptf.SetupTf(
+		ctx,
+		args.ProjectID,
+		args.TestRunID,
+		gkeOperatorCollectorTfDir,
+		map[string]string{
+			"image": args.GkeOperatorCollector.Image,
+		},
+		logger,
+	)
+	if err != nil {
+		return cleanupTf, err
+	}
+
+	return cleanupTf, err
+}