Merge remote-tracking branch 'origin/master' into fetch/values-lookup

Author: romseygeek

.ci/bwcVersions

@@ -28,7 +28,7 @@ BWC_VERSION:
   - "7.10.0"
   - "7.10.1"
   - "7.10.2"
-  - "7.10.3"
   - "7.11.0"
+  - "7.11.1"
   - "7.12.0"
   - "8.0.0"

CONTRIBUTING.md

@@ -1,7 +1,7 @@
 Contributing to elasticsearch
 =============================
 
-Elasticsearch is an open source project and we love to receive contributions from our community — you! There are many ways to contribute, from writing tutorials or blog posts, improving the documentation, submitting bug reports and feature requests or writing code which can be incorporated into Elasticsearch itself.
+Elasticsearch is a free and open project and we love to receive contributions from our community — you! There are many ways to contribute, from writing tutorials or blog posts, improving the documentation, submitting bug reports and feature requests or writing code which can be incorporated into Elasticsearch itself.
 
 If you want to be rewarded for your contributions, sign up for the [Elastic Contributor Program](https://www.elastic.co/community/contributor). Each time you
 make a valid contribution, you’ll earn points that increase your chances of winning prizes and being recognized as a top contributor.

TESTING.asciidoc

@@ -518,7 +518,7 @@ For example to run a specific tests from the x-pack rolling upgrade from 7.7.0:
 
 Tests are ran for versions that are not yet released but with which the current version will be compatible with.
 These are automatically checked out and built from source.
-See link:./buildSrc/src/main/java/org/elasticsearch/gradle/VersionCollection.java[VersionCollection]
+See link:./buildSrc/src/main/java/org/elasticsearch/gradle/BwcVersions.java[BwcVersions]
 and link:./distribution/bwc/build.gradle[distribution/bwc/build.gradle]
 for more information.
 

build.gradle

@@ -103,6 +103,26 @@ subprojects {
   }
 }
 
+/**
+ * This is a convenient method for declaring test artifact dependencies provided by the internal
+ * test artifact plugin. It replaces basically the longer dependency notation with explicit capability
+ * declaration like this:
+ *
+ * testImplementation(project(xpackModule('repositories-metering-api'))) {
+ *    capabilities {
+ *         requireCapability("org.elasticsearch.gradle:repositories-metering-api-test-artifacts")
+ *    }
+ * }
+ *
+ * */
+ext.testArtifact = { p ->
+  def projectDependency = p.dependencies.create(p)
+  projectDependency.capabilities {
+    requireCapabilities("org.elasticsearch.gradle:${projectDependency.name}-test-artifacts")
+  };
+}
+
+
 tasks.register("updateCIBwcVersions") {
   doLast {
     File yml = file(".ci/bwcVersions")

buildSrc/build.gradle

@@ -32,6 +32,11 @@ if (project == rootProject) {
 Properties props = VersionPropertiesLoader.loadBuildSrcVersion(project.file('version.properties'))
 version = props.getProperty("elasticsearch")
 
+gradlePlugin {
+  // We already configure publication and we don't need or want the one that comes
+  // with the java-gradle-plugin
+  automatedPublishing = false
+}
 def generateVersionProperties = tasks.register("generateVersionProperties", WriteProperties) {
   outputFile = "${buildDir}/version.properties"
   comment = 'Generated version properties'
@@ -240,16 +245,6 @@ if (project != rootProject) {
     maxParallelForks = providers.systemProperty('tests.jvms').forUseAtConfigurationTime().getOrElse(org.elasticsearch.gradle.info.BuildParams.defaultParallel.toString()) as Integer
   }
 
-  /*
-   * We already configure publication and we don't need or want this one that
-   * comes from the java-gradle-plugin.
-   */
-  afterEvaluate {
-    tasks.named("generatePomFileForPluginMavenPublication").configure {
-      enabled = false
-    }
-  }
-
   publishing.publications.named("nebula").configure {
     suppressPomMetadataWarningsFor("testFixturesApiElements")
     suppressPomMetadataWarningsFor("testFixturesRuntimeElements")

buildSrc/src/integTest/groovy/org/elasticsearch/gradle/YamlRestCompatTestPluginFuncTest.groovy

@@ -8,13 +8,24 @@
 
 package org.elasticsearch.gradle
 
+import com.fasterxml.jackson.databind.ObjectMapper
+import com.fasterxml.jackson.databind.ObjectReader
+import com.fasterxml.jackson.databind.ObjectWriter
+import com.fasterxml.jackson.databind.node.ObjectNode
+import com.fasterxml.jackson.dataformat.yaml.YAMLFactory
 import org.elasticsearch.gradle.fixtures.AbstractRestResourcesFuncTest
 import org.elasticsearch.gradle.internal.rest.compat.YamlRestCompatTestPlugin
 import org.gradle.testkit.runner.TaskOutcome
 
 class YamlRestCompatTestPluginFuncTest extends AbstractRestResourcesFuncTest {
 
-    def intermediateDir = YamlRestCompatTestPlugin.TEST_INTERMEDIATE_DIR_NAME;
+    private static final String intermediateDir = YamlRestCompatTestPlugin.TEST_INTERMEDIATE_DIR_NAME
+    private static final String transformTask  = ":" + YamlRestCompatTestPlugin.TRANSFORM_TASK_NAME
+    private static final YAMLFactory YAML_FACTORY = new YAMLFactory()
+    private static final ObjectMapper MAPPER = new ObjectMapper(YAML_FACTORY)
+    private static final ObjectReader READER = MAPPER.readerFor(ObjectNode.class)
+    private static final ObjectWriter WRITER = MAPPER.writerFor(ObjectNode.class)
+
 
     def "yamlRestCompatTest does nothing when there are no tests"() {
         given:
@@ -39,7 +50,7 @@ class YamlRestCompatTestPluginFuncTest extends AbstractRestResourcesFuncTest {
         result.task(':yamlRestCompatTest').outcome == TaskOutcome.NO_SOURCE
         result.task(':copyRestCompatApiTask').outcome == TaskOutcome.NO_SOURCE
         result.task(':copyRestCompatTestTask').outcome == TaskOutcome.NO_SOURCE
-        result.task(':transformCompatTests').outcome == TaskOutcome.NO_SOURCE
+        result.task(transformTask).outcome == TaskOutcome.NO_SOURCE
     }
 
     def "yamlRestCompatTest executes and copies api and transforms tests from :bwc:minor"() {
@@ -91,7 +102,7 @@ class YamlRestCompatTestPluginFuncTest extends AbstractRestResourcesFuncTest {
         result.task(':yamlRestCompatTest').outcome == TaskOutcome.SKIPPED
         result.task(':copyRestCompatApiTask').outcome == TaskOutcome.SUCCESS
         result.task(':copyRestCompatTestTask').outcome == TaskOutcome.SUCCESS
-        result.task(':transformCompatTests').outcome == TaskOutcome.SUCCESS
+        result.task(transformTask).outcome == TaskOutcome.SUCCESS
 
         file("/build/resources/yamlRestCompatTest/rest-api-spec/api/" + api).exists()
         file("/build/resources/yamlRestCompatTest/rest-api-spec/test/" + test).exists()
@@ -119,7 +130,7 @@ class YamlRestCompatTestPluginFuncTest extends AbstractRestResourcesFuncTest {
         result.task(':yamlRestCompatTest').outcome == TaskOutcome.SKIPPED
         result.task(':copyRestCompatApiTask').outcome == TaskOutcome.UP_TO_DATE
         result.task(':copyRestCompatTestTask').outcome == TaskOutcome.UP_TO_DATE
-        result.task(':transformCompatTests').outcome == TaskOutcome.UP_TO_DATE
+        result.task(transformTask).outcome == TaskOutcome.UP_TO_DATE
     }
 
     def "yamlRestCompatTest is wired into check and checkRestCompat"() {
@@ -148,7 +159,7 @@ class YamlRestCompatTestPluginFuncTest extends AbstractRestResourcesFuncTest {
         result.task(':yamlRestCompatTest').outcome == TaskOutcome.NO_SOURCE
         result.task(':copyRestCompatApiTask').outcome == TaskOutcome.NO_SOURCE
         result.task(':copyRestCompatTestTask').outcome == TaskOutcome.NO_SOURCE
-        result.task(':transformCompatTests').outcome == TaskOutcome.NO_SOURCE
+        result.task(transformTask).outcome == TaskOutcome.NO_SOURCE
 
         when:
         buildFile << """
@@ -162,7 +173,140 @@ class YamlRestCompatTestPluginFuncTest extends AbstractRestResourcesFuncTest {
         result.task(':yamlRestCompatTest').outcome == TaskOutcome.SKIPPED
         result.task(':copyRestCompatApiTask').outcome == TaskOutcome.SKIPPED
         result.task(':copyRestCompatTestTask').outcome == TaskOutcome.SKIPPED
-        result.task(':transformCompatTests').outcome == TaskOutcome.SKIPPED
+        result.task(transformTask).outcome == TaskOutcome.SKIPPED
+    }
+
+    def "transform task executes and works as configured"() {
+        given:
+        internalBuild()
+
+        addSubProject(":distribution:bwc:minor") << """
+        configurations { checkout }
+        artifacts {
+            checkout(new File(projectDir, "checkoutDir"))
+        }
+        """
+
+        buildFile << """
+            apply plugin: 'elasticsearch.yaml-rest-compat-test'
+
+            // avoids a dependency problem in this test, the distribution in use here is inconsequential to the test
+            import org.elasticsearch.gradle.testclusters.TestDistribution;
+            testClusters {
+              yamlRestCompatTest.setTestDistribution(TestDistribution.INTEG_TEST)
+            }
+
+            dependencies {
+               yamlRestTestImplementation "junit:junit:4.12"
+            }
+            tasks.named("transformV7RestTests").configure({ task ->
+              task.replaceMatch("_type", "_doc")
+              task.replaceMatch("_source.values", ["z", "x", "y"], "one")
+              task.removeMatch("_source.blah")
+              task.removeMatch("_source.junk", "two")
+              task.addMatch("_source.added", [name: 'jake', likes: 'cheese'], "one")
+            })
+            // can't actually spin up test cluster from this test
+           tasks.withType(Test).configureEach{ enabled = false }
+        """
+
+        setupRestResources([], [])
+
+        file("distribution/bwc/minor/checkoutDir/src/yamlRestTest/resources/rest-api-spec/test/test.yml" ) << """
+        "one":
+          - do:
+              get:
+                index: test
+                id: 1
+          - match: { _source.values: ["foo"] }
+          - match: { _type: "_foo" }
+          - match: { _source.blah: 1234 }
+          - match: { _source.junk: true }
+        ---
+        "two":
+          - do:
+              get:
+                index: test
+                id: 1
+          - match: { _source.values: ["foo"] }
+          - match: { _type: "_foo" }
+          - match: { _source.blah: 1234 }
+          - match: { _source.junk: true }
+
+        """.stripIndent()
+        when:
+        def result = gradleRunner("yamlRestCompatTest").build()
+
+        then:
+
+        result.task(transformTask).outcome == TaskOutcome.SUCCESS
+
 
+        file("/build/resources/yamlRestCompatTest/rest-api-spec/test/test.yml" ).exists()
+        List<ObjectNode> actual = READER.readValues(file("/build/resources/yamlRestCompatTest/rest-api-spec/test/test.yml")).readAll()
+        List<ObjectNode> expectedAll = READER.readValues(
+        """
+        ---
+        setup:
+        - skip:
+            features: "headers"
+        ---
+        one:
+        - do:
+            get:
+              index: "test"
+              id: 1
+            headers:
+              Content-Type: "application/vnd.elasticsearch+json;compatible-with=7"
+              Accept: "application/vnd.elasticsearch+json;compatible-with=7"
+        - match:
+            _source.values:
+            - "z"
+            - "x"
+            - "y"
+        - match:
+            _type: "_doc"
+        - match: {}
+        - match:
+            _source.junk: true
+        - match:
+            _source.added:
+              name: "jake"
+              likes: "cheese"
+        ---
+        two:
+        - do:
+            get:
+              index: "test"
+              id: 1
+            headers:
+              Content-Type: "application/vnd.elasticsearch+json;compatible-with=7"
+              Accept: "application/vnd.elasticsearch+json;compatible-with=7"
+        - match:
+            _source.values:
+            - "foo"
+        - match:
+            _type: "_doc"
+        - match: {}
+        - match: {}
+        """.stripIndent()).readAll()
+
+        expectedAll.eachWithIndex{ ObjectNode expected, int i ->
+           assert expected == actual.get(i)
+        }
+
+        when:
+        result = gradleRunner(transformTask).build()
+
+        then:
+        result.task(transformTask).outcome == TaskOutcome.UP_TO_DATE
+
+        when:
+        buildFile.write(buildFile.text.replace("blah", "baz"))
+        result = gradleRunner(transformTask).build()
+
+        then:
+        result.task(transformTask).outcome == TaskOutcome.SUCCESS
     }
+
 }

buildSrc/src/main/java/org/elasticsearch/gradle/internal/InternalTestArtifactPlugin.java

@@ -0,0 +1,65 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.gradle.internal;
+
+import org.gradle.api.Plugin;
+import org.gradle.api.Project;
+import org.gradle.api.artifacts.Configuration;
+import org.gradle.api.artifacts.Dependency;
+import org.gradle.api.artifacts.dsl.DependencyHandler;
+import org.gradle.api.plugins.BasePluginConvention;
+import org.gradle.api.plugins.JavaPlugin;
+import org.gradle.api.plugins.JavaPluginExtension;
+import org.gradle.api.provider.ProviderFactory;
+import org.gradle.api.tasks.SourceSetContainer;
+import org.gradle.jvm.tasks.Jar;
+
+import javax.inject.Inject;
+
+/**
+ * Ideally, this plugin is intended to be temporary and in the long run we want to move
+ * forward to port our test fixtures to use the gradle test fixtures plugin.
+ * */
+public class InternalTestArtifactPlugin implements Plugin<Project> {
+
+    private final ProviderFactory providerFactory;
+
+    @Inject
+    public InternalTestArtifactPlugin(ProviderFactory providers) {
+        this.providerFactory = providers;
+    }
+
+    @Override
+    public void apply(Project project) {
+        JavaPluginExtension javaPluginExtension = project.getExtensions().getByType(JavaPluginExtension.class);
+        javaPluginExtension.registerFeature("testArtifacts", featureSpec -> {
+            featureSpec.usingSourceSet(project.getExtensions().getByType(SourceSetContainer.class).getByName("test"));
+            featureSpec.capability("org.elasticsearch.gradle", project.getName() + "-test-artifacts", "1.0");
+            // This feature is only used internally in the
+            // elasticsearch build so we do not need any publication.
+            featureSpec.disablePublication();
+        });
+        Configuration testApiElements = project.getConfigurations().getByName("testApiElements");
+        testApiElements.extendsFrom(project.getConfigurations().getByName("testCompileClasspath"));
+        DependencyHandler dependencies = project.getDependencies();
+        project.getPlugins().withType(JavaPlugin.class, javaPlugin -> {
+            Dependency projectDependency = dependencies.create(project);
+            dependencies.add("testApiElements", projectDependency);
+            dependencies.add("testRuntimeElements", projectDependency);
+        });
+        // PolicyUtil doesn't handle classifier notation well probably.
+        // Instead of fixing PoliceUtil we stick to the pattern of changing
+        // the basename here to indicate its a test artifacts jar.
+        BasePluginConvention convention = (BasePluginConvention) project.getConvention().getPlugins().get("base");
+        project.getTasks().named("testJar", Jar.class).configure(jar -> {
+            jar.getArchiveBaseName().convention(providerFactory.provider(() -> convention.getArchivesBaseName() + "-test-artifacts"));
+            jar.getArchiveClassifier().set("");
+        });
+    }
+}