Various improvements (#88)

* Update K6 scripts to call all APIs
* Remove SQL scripts and add new K6 scripts to populate database
(VehicleInventoryService was overwriting SQL scripts on startup).
* Update READMEs
* Init and populate DB with some basic data
* Update copyrights

Testing:

```
OverheadTests > runAllTestConfigurations() > all STANDARD_OUT
    ----------------------------------------------------------
     Run at Thu Feb 29 23:45:51 UTC 2024
     all : Compares all DistroConfigs
     5 users, 5000 iterations
    ----------------------------------------------------------
    DistroConfig        :              noneapp_signals_disabledapp_signals_no_tracesapp_signals_traces
    Run duration        :          00:10:04         00:10:02         00:10:02         00:10:02
    Avg. CPU (user) %   :               0.0              0.0              0.0              0.0
    Max. CPU (user) %   :               0.0              0.0              0.0              0.0
    Avg. mch tot cpu %  :               0.0              0.0              0.0              0.0
    Startup time (ms)   :              4011             5014             5016             5013
    Total allocated MB  :              0.00             0.00             0.00             0.00
    Thread switch rate  :               0.0              0.0              0.0              0.0
    GC time (ms)        :                 0                0                0                0
    GC pause time (ms)  :                 0                0                0                0
    Req. mean (ms)      :            104.11           105.04           106.39           104.71
    Req. p95 (ms)       :            389.89           389.92           389.93           389.90
    Iter. mean (ms)     :           1458.85          1471.86          1490.91          1467.22
    Iter. p95 (ms)      :           1520.01          1540.09          1629.93          1540.02
    Net read avg (bps)  :              0.00             0.00             0.00             0.00
    Net write avg (bps) :              0.00             0.00             0.00             0.00
    Peak threads        :                 0                0                0                0

Gradle Test Executor 41 finished executing tests.

> Task :test
Finished generating test XML results (0.696 secs) into: /workplace/thp/python-sdk/aws-otel-python-instrumentation/performance-tests/build/test-results/test
Generating HTML test report...
Finished generating test html results (0.73 secs) into: /workplace/thp/python-sdk/aws-otel-python-instrumentation/performance-tests/build/reports/tests/test

Deprecated Gradle features were used in this build, making it incompatible with Gradle 9.0.

You can use '--warning-mode all' to show the individual deprecation warnings and determine if they come from your own scripts or plugins.

For more on this, please refer to https://docs.gradle.org/8.6/userguide/command_line_interface.html#sec:command_line_warnings in the Gradle documentation.

BUILD SUCCESSFUL in 41m 9s
3 actionable tasks: 1 executed, 2 up-to-date
```

By submitting this pull request, I confirm that you can use, modify,
copy, and redistribute this contribution, under the terms of your
choice.

Author: thpierce

aws-opentelemetry-distro/src/amazon/opentelemetry/distro/_instrumentation_patch.py

@@ -1,5 +1,6 @@
 # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 # SPDX-License-Identifier: Apache-2.0
+# Modifications Copyright The OpenTelemetry Authors. Licensed under the Apache License 2.0 License.
 import importlib
 
 from opentelemetry.instrumentation.botocore.extensions import _KNOWN_EXTENSIONS
@@ -76,6 +77,7 @@ def patch_extract_attributes(self, attributes: _AttributeMapT):
     _SqsExtension.extract_attributes = patch_extract_attributes
 
 
+# The OpenTelemetry Authors code
 def _lazy_load(module, cls):
     """Clone of upstream opentelemetry.instrumentation.botocore.extensions.lazy_load
 
@@ -89,6 +91,9 @@ def loader():
     return loader
 
 
+# END The OpenTelemetry Authors code
+
+
 class _S3Extension(_AwsSdkExtension):
     def extract_attributes(self, attributes: _AttributeMapT):
         bucket_name = self._call_context.params.get("Bucket")

aws-opentelemetry-distro/src/amazon/opentelemetry/distro/aws_opentelemetry_configurator.py

@@ -1,5 +1,6 @@
 # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 # SPDX-License-Identifier: Apache-2.0
+# Modifications Copyright The OpenTelemetry Authors. Licensed under the Apache License 2.0 License.
 import os
 from logging import Logger, getLogger
 from typing import ClassVar, Dict, Type
@@ -84,6 +85,9 @@ def _configure(self, **kwargs):
         _initialize_components()
 
 
+# The OpenTelemetry Authors code
+# Long term, we wish to contribute this to upstream to improve initialization customizability and reduce dependency on
+# internal logic.
 def _initialize_components():
     trace_exporters, metric_exporters, log_exporters = _import_exporters(
         _get_exporter_names("traces"),
@@ -146,6 +150,9 @@ def _init_tracing(
     set_tracer_provider(trace_provider)
 
 
+# END The OpenTelemetry Authors code
+
+
 def _exclude_urls_for_instrumentations():
     urls_to_exclude_instr = "SamplingTargets,GetSamplingRules"
     requests_excluded_urls = os.environ.pop("OTEL_PYTHON_REQUESTS_EXCLUDED_URLS", "")

performance-tests/README.md

@@ -8,41 +8,35 @@
 - [Setup and Usage](#setup-and-usage)
 - [Visualization](#visualization)
 
-This directory will contain tools and utilities
-that help us to measure the performance overhead introduced by
-the distro and to measure how this overhead changes over time.
+This directory will contain tools and utilities that help us to measure the performance overhead introduced by the distro and to measure how this overhead changes over time.
 
-The overhead tests here should be considered a "macro" benchmark. They serve to measure high-level
-overhead as perceived by the operator of a "typical" application. Tests are performed on a Java 11
-distribution from [Eclipse Temurin](https://projects.eclipse.org/projects/adoptium.temurin).
+The overhead tests here should be considered a "macro" benchmark. They serve to measure high-level overhead as perceived by the operator of a "typical" application. Tests are performed on Python 3.10.
 
 ## Process
 
-There is one dynamic test here called [OverheadTests](https://github.com/open-telemetry/opentelemetry-java-instrumentation/blob/main/benchmark-overhead/src/test/java/io/opentelemetry/OverheadTests.java).
-The `@TestFactory` method creates a test pass for each of the [defined configurations](https://github.com/open-telemetry/opentelemetry-java-instrumentation/blob/main/benchmark-overhead/src/test/java/io/opentelemetry/config/Configs.java).
-Before the tests run, a single collector instance is started. Each test pass has one or more distroConfigs and those are tested in series.
-For each distro defined in a configuration, the test runner (using [testcontainers](https://www.testcontainers.org/)) will:
+There is one dynamic test here called OverheadTests. The `@TestFactory` method creates a test pass for each of the defined configurations.  Before the tests run, a single collector instance is started. Each test pass has one or more distroConfigs and those are tested in series. For each distro defined in a configuration, the test runner (using [testcontainers](https://www.testcontainers.org/)) will:
 
 1. create a fresh postgres instance and populate it with initial data.
-2. create a fresh instance of [spring-petclinic-rest](https://github.com/spring-petclinic/spring-petclinic-rest) instrumented with the specified distroConfig
-3. measure the time until the petclinic app is marked "healthy" and then write it to a file.
-4. if configured, perform a warmup phase. During the warmup phase, a bit of traffic is generated in order to get the application into a steady state (primarily helping facilitate jit compilations). Currently, we use a 30 second warmup time.
-5. start a JFR recording by running `jcmd` inside the petclinic container
-6. run the [k6 test script](https://github.com/open-telemetry/opentelemetry-java-instrumentation/blob/main/benchmark-overhead/k6/basic.js) with the configured number of iterations through the file and the configured number of concurrent virtual users (VUs).
-7. after k6 completes, petclinic is shut down
-8. after petclinic is shut down, postgres is shut down
+2. create a fresh instance of vehicle inventory service instrumented with the specified distroConfig, and image service (not currently instrumented)
+3. measure the time until the app is marked "healthy" and then write it to a file.
+4. if configured, perform a warmup phase. During the warmup phase, a bit of traffic is generated in order to get the application into a steady state (primarily helping facilitate jit compilations).
+5. start a profiling recording by running a script that relies on psutils inside the application container
+6. run a k6 test script with the configured number of iterations through the file and the configured number of concurrent virtual users (VUs).
+7. after k6 completes, application is shut down
+8. after application is shut down, postgres is shut down
 
 And this repeats for every distro configured in each test configuration.
 
 After all the tests are complete, the results are collected and committed back to the `/results` subdirectory as csv and summary text files.
 
 ## What do we measure?
 
-For each test pass, we record the following metrics in order to compare distroConfigs and determine
-relative overhead.
+For each test pass, we record the following metrics in order to compare distroConfigs and determine relative overhead.
+
+// WIP: This list will change once we finalize the profiling script.
 
 | metric name              | units  | description                                                                  |
-| ------------------------ | ------ | ---------------------------------------------------------------------------- |
+|--------------------------| ------ |------------------------------------------------------------------------------|
 | Startup time             | ms     | How long it takes for the spring app to report "healthy"                     |
 | Total allocated mem      | bytes  | Across the life of the application                                           |
 | Heap (min)               | bytes  | Smallest observed heap size                                                  |
@@ -56,10 +50,10 @@ relative overhead.
 | Peak threads             | #      | Highest number of running threads in the VM, including distroConfig threads  |
 | Network read mean        | bits/s | Average network read rate                                                    |
 | Network write mean       | bits/s | Average network write rate                                                   |
-| Average JVM user CPU     | %      | Average observed user CPU (range 0.0-1.0)                                    |
-| Max JVM user CPU         | %      | Max observed user CPU used (range 0.0-1.0)                                   |
+| Average user CPU         | %      | Average observed user CPU (range 0.0-1.0)                                    |
+| Max user CPU             | %      | Max observed user CPU used (range 0.0-1.0)                                   |
 | Average machine tot. CPU | %      | Average percentage of machine CPU used (range 0.0-1.0)                       |
-| Total GC pause nanos     | ns     | JVM time spent paused due to GC                                              |
+| Total GC pause nanos     | ns     |  time spent paused due to GC                                                 |
 | Run duration ms          | ms     | Duration of the test run, in ms                                              |
 
 ## Config
@@ -74,41 +68,25 @@ Each config contains the following:
 - totalIterations - the number of passes to make through the k6 test script
 - warmupSeconds - how long to wait before starting conducting measurements
 
-Currently, we test:
-
-- no distro versus latest released distro
-- no distro versus latest snapshot
-- latest release vs. latest snapshot
-
 Additional configurations can be created by submitting a PR against the `Configs` class.
 
 ### DistroConfigs
 
-An distroConfig is defined in code as a name, description, optional URL, and optional additional
-arguments to be passed to the JVM (not including `-javaagent:`). New distroConfigs may be defined
-by creating new instances of the `Distro` class. The `AgentResolver` is used to download
-the relevant distroConfig jar for an `Distro` definition.
-
-## Automation
-
-The tests are run nightly via github actions. The results are collected and appended to
-a csv file, which is committed back to the repo in the `/results` subdirectory.
+An distroConfig is defined in code as a name, description, flag for instrumentation. and optional additional arguments to be passed to the application container. New distroConfigs may be defined by creating new instances of the `Distro` class. The `AgentResolver` is used to download the relevant distroConfig jar for an `Distro` definition.
 
 ## Setup and Usage
 
-The tests require docker to be running. Simply run `OverheadTests` in your IDE.
+Pre-requirements:
+* Have `docker` installed and running - verify by running the `docker` command.
+* Export AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN, and S3_BUCKET environment variables.
 
-Alternatively, you can run the tests from
-the command line with gradle:
-
-```
-cd benchmark-overhead
+Steps:
+* From `aws-otel-python-instrumentation` dir, execute:
+```sh
+./scripts/build_and_install_distro.sh
+./scripts/set-up-performance-tests.sh
+cd performance-tests
 ./gradlew test
-
 ```
 
-## Visualization
-
-None yet. Help wanted! Our goal is to have the results and a rich UI running in the
-`gh-pages` branch similar to [earlier tools](https://breedx-splk.github.io/iguanodon/web/).
-Please help us make this happen.
+The last step can be run or you can run from IDE (after setting environment variables appropriately).

performance-tests/k6/basic.js

@@ -0,0 +1,49 @@
+import http from "k6/http";
+import {check} from "k6";
+
+const baseUri = 'http://vehicle-service:8001/vehicle-inventory/';
+// Fake ID to trigger 400's.
+const badId = 1000;
+
+export default function() {
+    /**
+     * Calls all Vehicle Inventory Service APIs.
+     *
+     * Note that there are no modifications to the database - all POSTs and DELETEs will fail.
+     * This ensures the database does not grow over time. Data is initially added in setUp.js.
+     */
+
+    invokeApi("oops", "GET", 404)
+    invokeApi("", "GET", 200)
+    invokeApi("", "POST", 400)
+    invokeApi("1", "GET", 200)
+    invokeApi(`${badId}`, "DELETE", 404)
+    invokeApi("make/Toyota", "GET", 200)
+    invokeApi("1/image", "GET", 200)
+    invokeApi("image/toy_rav_24.png", "GET", 200)
+    invokeApi("image/", "POST", 404)
+    invokeApi("history/", "GET", 200)
+    invokeApi("history/", "POST", 400)
+    invokeApi("history/1", "GET", 200)
+    invokeApi(`history/${badId}`, "DELETE", 404)
+    invokeApi("history/1/vehicle", "GET", 200)
+
+    function invokeApi(path, method, status) {
+        const url = `${baseUri}${path}`;
+        let response;
+        switch(method) {
+            case "GET":
+                response = http.get(url);
+                break;
+            case "POST":
+                response = http.post(url, JSON.stringify({"badKey": "badValue"}));
+                break;
+            case "DELETE":
+                response = http.del(url);
+                break;
+        }
+        check(response, {
+            [`${method} ${path} response ${status}`] : (response) => response.status === status
+        });
+    }
+};

performance-tests/k6/performanceTest.js

@@ -0,0 +1,29 @@
+import http from "k6/http";
+import {check} from "k6";
+
+const baseUri = 'http://vehicle-service:8001/vehicle-inventory/';
+
+export default function() {
+    postVehicle({'id': 1,'make': 'Toyota', 'model': 'Rav4', 'year': 2024, 'image_name': 'toy_rav_24.png'});
+    postVehicle({'id': 2,'make': 'Toyota', 'model': 'Rav4', 'year': 2023, 'image_name': 'toy_rav_23.png'});
+    postVehicle({'id': 3,'make': 'Honda', 'model': 'Odyssey', 'year': 2022, 'image_name': 'hon_ody_22.png'});
+    postVehicle({'id': 4,'make': 'Honda', 'model': 'Odyssey', 'year': 2024, 'image_name': 'hon_ody_24.png'});
+
+    postVehicleHistory({'id': 1, 'purchase_date': '2024-01-29', 'purchase_price': 55999, 'vehicle_id': 1});
+    postVehicleHistory({'id': 2, 'purchase_date': '2024-02-29', 'purchase_price': 54999, 'vehicle_id': 1});
+    postVehicleHistory({'id': 3, 'purchase_date': '2023-12-14', 'purchase_price': 13999, 'vehicle_id': 3});
+
+    function postVehicle(body) {
+        const response = http.post(baseUri, JSON.stringify(body));
+        check(response, {
+            [`POST vehicle response 200`] : (response) => response.status === 200
+        });
+    }
+
+    function postVehicleHistory(body) {
+        const response = http.post(`${baseUri}history/`, JSON.stringify(body));
+        check(response, {
+            [`POST vehicle history response 200`] : (response) => response.status === 200
+        });
+    }
+};

performance-tests/k6/setUp.js

@@ -1,6 +1,7 @@
 /*
  * Copyright The OpenTelemetry Authors
  * SPDX-License-Identifier: Apache-2.0
+ * Modifications Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
  */
 
 package io.opentelemetry;
@@ -96,6 +97,8 @@ void runAppOnce(TestConfig config, DistroConfig distroConfig) throws Exception {
     vehicleInventoryService.start();
     writeStartupTimeFile(distroConfig, start);
 
+    populateDatabase();
+
     if (config.getWarmupSeconds() > 0) {
       // doWarmupPhase(config, vehicleInventoryService);
     }
@@ -167,6 +170,16 @@ private void doWarmupPhase(TestConfig testConfig, GenericContainer<?> vehicleInv
     System.out.println("Warmup complete.");
   }
 
+  private void populateDatabase() {
+    GenericContainer<?> k6 =
+        new GenericContainer<>(DockerImageName.parse("loadimpact/k6"))
+            .withNetwork(NETWORK)
+            .withCopyFileToContainer(MountableFile.forHostPath("./k6"), "/app")
+            .withCommand("run", "/app/setUp.js")
+            .withStartupCheckStrategy(new OneShotStartupCheckStrategy());
+    k6.start();
+  }
+
   private void writeStartupTimeFile(DistroConfig distroConfig, long start) throws IOException {
     long delta = System.currentTimeMillis() - start;
     Path startupPath = namingConventions.local.startupDurationFile(distroConfig);

performance-tests/src/test/java/io/opentelemetry/OverheadTests.java

@@ -1,6 +1,7 @@
 /*
  * Copyright The OpenTelemetry Authors
  * SPDX-License-Identifier: Apache-2.0
+ * Modifications Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
  */
 
 package io.opentelemetry.config;

performance-tests/src/test/java/io/opentelemetry/config/Configs.java

@@ -1,6 +1,7 @@
 /*
  * Copyright The OpenTelemetry Authors
  * SPDX-License-Identifier: Apache-2.0
+ * Modifications Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
  */
 
 package io.opentelemetry.config;

performance-tests/src/test/java/io/opentelemetry/config/TestConfig.java

@@ -1,6 +1,7 @@
 /*
  * Copyright The OpenTelemetry Authors
  * SPDX-License-Identifier: Apache-2.0
+ * Modifications Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
  */
 
 package io.opentelemetry.containers;