Merge pull request #318 from zvezdan/fix-abi-container

zvezdan · web-flow · commit d39e50a30e1a · 2019-10-07T13:18:38.000-07:00
Fix empty ABI container issue.
diff --git a/pygradle-plugin/src/integTest/groovy/com/linkedin/gradle/python/plugin/ParallelWheelsIntegrationTest.groovy b/pygradle-plugin/src/integTest/groovy/com/linkedin/gradle/python/plugin/ParallelWheelsIntegrationTest.groovy
@@ -152,6 +152,7 @@ class ParallelWheelsIntegrationTest extends Specification {
 
         then:
         result.task(':foo:pytest').outcome == TaskOutcome.SUCCESS  //using pytest since it will always require deps
-        result.task(':foo:parallelWheels').outcome == TaskOutcome.SUCCESS //the task isn't part of the graph
+        def taskOutcome = result.task(':foo:parallelWheels').outcome
+        taskOutcome == TaskOutcome.SUCCESS || taskOutcome == TaskOutcome.UP_TO_DATE
     }
 }
diff --git a/pygradle-plugin/src/integTest/groovy/com/linkedin/gradle/python/plugin/PythonPluginIntegrationTest.groovy b/pygradle-plugin/src/integTest/groovy/com/linkedin/gradle/python/plugin/PythonPluginIntegrationTest.groovy
@@ -60,6 +60,7 @@ class PythonPluginIntegrationTest extends Specification {
         result.task(':foo:installPythonRequirements').outcome == TaskOutcome.SUCCESS
         result.task(':foo:installTestRequirements').outcome == TaskOutcome.SUCCESS
         result.task(':foo:createVirtualEnvironment').outcome == TaskOutcome.SUCCESS
+        result.task(':foo:getProbedTags').outcome == TaskOutcome.SUCCESS
         result.task(':foo:installProject').outcome == TaskOutcome.SUCCESS
         result.task(':foo:pytest').outcome == TaskOutcome.SUCCESS
         result.task(':foo:check').outcome == TaskOutcome.SUCCESS
@@ -122,6 +123,7 @@ class PythonPluginIntegrationTest extends Specification {
         result.task(':foo:installPythonRequirements').outcome == TaskOutcome.SUCCESS
         result.task(':foo:installTestRequirements').outcome == TaskOutcome.SUCCESS
         result.task(':foo:createVirtualEnvironment').outcome == TaskOutcome.SUCCESS
+        result.task(':foo:getProbedTags').outcome == TaskOutcome.SUCCESS
         result.task(':foo:installProject').outcome == TaskOutcome.SUCCESS
         result.task(':foo:pytest').outcome == TaskOutcome.SUCCESS
         result.task(':foo:check').outcome == TaskOutcome.SUCCESS
diff --git a/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/plugin/PythonPlugin.java b/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/plugin/PythonPlugin.java
@@ -21,6 +21,7 @@
 import com.linkedin.gradle.python.plugin.internal.ValidationPlugin;
 import com.linkedin.gradle.python.tasks.CleanSaveVenvTask;
 import com.linkedin.gradle.python.tasks.GenerateSetupPyTask;
+import com.linkedin.gradle.python.tasks.GetProbedTagsTask;
 import com.linkedin.gradle.python.tasks.InstallVirtualEnvironmentTask;
 import com.linkedin.gradle.python.tasks.PinRequirementsTask;
 import com.linkedin.gradle.python.tasks.provides.ProvidesVenv;
@@ -49,6 +50,7 @@
 import static com.linkedin.gradle.python.util.StandardTextValues.CONFIGURATION_VENV;
 import static com.linkedin.gradle.python.util.StandardTextValues.CONFIGURATION_WHEEL;
 import static com.linkedin.gradle.python.util.StandardTextValues.TASK_CLEAN_SAVE_VENV;
+import static com.linkedin.gradle.python.util.StandardTextValues.TASK_GET_PROBED_TAGS;
 import static com.linkedin.gradle.python.util.StandardTextValues.TASK_PIN_REQUIREMENTS;
 import static com.linkedin.gradle.python.util.StandardTextValues.TASK_SETUP_LINKS;
 import static com.linkedin.gradle.python.util.StandardTextValues.TASK_SETUP_PY_WRITER;
@@ -103,12 +105,18 @@ public void apply(final Project project) {
             task.dependsOn(pinRequirementsTask);
             task.setPythonDetails(settings.getDetails());
         });
+        // Ensure that ABI container is populated when virtual environment creation task is up-to-date.
+        project.getTasks().create(TASK_GET_PROBED_TAGS.getValue(), GetProbedTagsTask.class, task -> {
+            task.dependsOn(TASK_VENV_CREATE.getValue());
+            task.setPythonDetails(settings.getDetails());
+        });
 
         /*
          * Creates a link so users can activate into the virtual environment.
          */
         project.getTasks().create(TASK_SETUP_LINKS.getValue(), task -> {
             task.dependsOn(project.getTasks().getByName(TASK_VENV_CREATE.getValue()));
+            task.dependsOn(project.getTasks().getByName(TASK_GET_PROBED_TAGS.getValue()));
             task.getOutputs().file(settings.getDetails().getActivateLink());
 
             task.doLast(it -> {
diff --git a/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/tasks/GetProbedTagsTask.java b/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/tasks/GetProbedTagsTask.java
@@ -0,0 +1,56 @@
+/*
+ * Copyright 2016 LinkedIn Corp.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.linkedin.gradle.python.tasks;
+
+import com.linkedin.gradle.python.extension.PythonDetails;
+import com.linkedin.gradle.python.tasks.action.ProbeVenvInfoAction;
+import com.linkedin.gradle.python.tasks.provides.ProvidesVenv;
+import com.linkedin.gradle.python.wheel.EditablePythonAbiContainer;
+import org.gradle.api.DefaultTask;
+import org.gradle.api.tasks.TaskAction;
+
+
+/**
+ * Get supported wheel tags from previously probed virtual environment.
+ *
+ * <p>When virtual environment creation task is up-to-date,
+ * then ABI container does not get populated, remains empty, and
+ * no cached wheels can match any tags. This task ensures that
+ * the container does get populated, regardless of virtual environment
+ * creation task outcome.</p>
+ */
+public class GetProbedTagsTask extends DefaultTask implements ProvidesVenv {
+    private PythonDetails pythonDetails;
+    private EditablePythonAbiContainer editablePythonAbiContainer;
+
+    @TaskAction
+    public void getProbedTags() {
+        ProbeVenvInfoAction.getProbedTags(getProject(), pythonDetails, editablePythonAbiContainer);
+    }
+
+    @Override
+    public void setEditablePythonAbiContainer(EditablePythonAbiContainer editablePythonAbiContainer) {
+        this.editablePythonAbiContainer = editablePythonAbiContainer;
+    }
+
+    public PythonDetails getPythonDetails() {
+        return pythonDetails;
+    }
+
+    public void setPythonDetails(PythonDetails pythonDetails) {
+        this.pythonDetails = pythonDetails;
+    }
+}
diff --git a/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/tasks/action/ProbeVenvInfoAction.java b/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/tasks/action/ProbeVenvInfoAction.java
@@ -33,8 +33,9 @@
 import java.io.InputStream;
 import java.io.OutputStream;
 
-class ProbeVenvInfoAction {
+public class ProbeVenvInfoAction {
 
+    private static final String PROBE_DIR_NAME = "probe-venv";
     private static final Logger logger = Logging.getLogger(ProbeVenvInfoAction.class);
 
     private ProbeVenvInfoAction() {
@@ -50,6 +51,25 @@ static void probeVenv(Project project, PythonDetails pythonDetails,
         }
     }
 
+    /**
+     * Populate the ABI container with supported wheel tags from probed environment.
+     *
+     * @param project current project
+     * @param pythonDetails current python details
+     * @param editablePythonAbiContainer the ABI container object to populate
+     */
+    public static void getProbedTags(Project project,
+                                    PythonDetails pythonDetails,
+                                    EditablePythonAbiContainer editablePythonAbiContainer) {
+        File probeDir = new File(project.getBuildDir(), PROBE_DIR_NAME);
+        File supportedAbiFormatsFile = getSupportedAbiFormatsFile(probeDir, pythonDetails);
+        try {
+            getSavedTags(pythonDetails, editablePythonAbiContainer, supportedAbiFormatsFile);
+        } catch (IOException e) {
+            logger.info("Unable to probe venv for supported wheel tags. Ignoring the error. May slow the build.");
+        }
+    }
+
     private static void doProbe(Project project, PythonDetails pythonDetails,
                           EditablePythonAbiContainer editablePythonAbiContainer) throws IOException {
         InputStream wheelApiResource = ProbeVenvInfoAction.class.getClassLoader()
@@ -58,7 +78,7 @@ private static void doProbe(Project project, PythonDetails pythonDetails,
         byte[] buffer = new byte[wheelApiResource.available()];
         wheelApiResource.read(buffer);
 
-        File probeDir = new File(project.getBuildDir(), "probe-venv");
+        File probeDir = new File(project.getBuildDir(), PROBE_DIR_NAME);
         probeDir.mkdirs();
 
         OutputStream outStream = new FileOutputStream(getPythonFileForSupportedWheels(probeDir));
@@ -71,7 +91,20 @@ private static void doProbe(Project project, PythonDetails pythonDetails,
             execSpec.args(supportedAbiFormatsFile.getAbsolutePath());
         });
 
+        /*
+         * The code for this function was originally here.
+         * Still making this call to benefit AbstractPythonInfrastructureDefaultTask,
+         * although it's not necessary for InstallVirtualEnvironmentTask because
+         * GetProbedTagsTask will get the tags.
+         */
+        getSavedTags(pythonDetails, editablePythonAbiContainer, supportedAbiFormatsFile);
+    }
+
+    private static void getSavedTags(PythonDetails pythonDetails,
+                                     EditablePythonAbiContainer editablePythonAbiContainer,
+                                     File supportedAbiFormatsFile) throws IOException {
         JsonArray array = Json.parse(new FileReader(supportedAbiFormatsFile)).asArray();
+
         for (JsonValue jsonValue : array) {
             JsonObject entry = jsonValue.asObject();
             String pythonTag = entry.get("pythonTag").asString();
diff --git a/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/util/StandardTextValues.java b/pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/util/StandardTextValues.java
@@ -48,6 +48,7 @@ public enum StandardTextValues {
     TASK_PYTEST("pytest"),
     TASK_SETUP_LINKS("installLinks"),
     TASK_VENV_CREATE("createVirtualEnvironment"),
+    TASK_GET_PROBED_TAGS("getProbedTags"),
     TASK_PIN_REQUIREMENTS("pinRequirements"),
     TASK_SETUP_PY_WRITER("generateSetupPy"),
     DOCUMENTATION_GROUP("documentation"),

Original file line number	Diff line number	Diff line change
`@@ -152,6 +152,7 @@ class ParallelWheelsIntegrationTest extends Specification {`
`152`	`152`
`153`	`153`	`then:`
`154`	`154`	`result.task(':foo:pytest').outcome == TaskOutcome.SUCCESS //using pytest since it will always require deps`
`155`		`- result.task(':foo:parallelWheels').outcome == TaskOutcome.SUCCESS //the task isn't part of the graph`
	`155`	`+ def taskOutcome = result.task(':foo:parallelWheels').outcome`
	`156`	`+ taskOutcome == TaskOutcome.SUCCESS \|\| taskOutcome == TaskOutcome.UP_TO_DATE`
`156`	`157`	`}`
`157`	`158`	`}`