Merge pull request #221 from warsaw/zipapp-refactor

Refactor to support generic Python "zipapps"

Author: Ethan Hall

pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/extension/PexExtension.java

@@ -21,36 +21,56 @@
 import java.io.File;
 
 
-public class PexExtension {
-
-    private File pexCache;
-    private boolean fatPex = OperatingSystem.current().isWindows(); //Defaulting to fat pex's on windows
+public class PexExtension implements ZipappExtension {
+    private File cache;
+    // Default to fat zipapps on Windows, since our wrappers are fairly POSIX specific.
+    private boolean isFat = OperatingSystem.current().isWindows();
     private boolean pythonWrapper = true;
 
     public PexExtension(Project project) {
-        pexCache = new File(project.getBuildDir(), "pex-cache");
+        this.cache = new File(project.getBuildDir(), "pex-cache");
     }
 
     public File getPexCache() {
-        return pexCache;
+        return cache;
     }
 
     public void setPexCache(File pexCache) {
-        this.pexCache = pexCache;
+        cache = pexCache;
     }
 
+    // These are kept for API backward compatibility.
+
     /**
      * @return when <code>true</code>, then skinny pex's will be used.
      */
+    @Deprecated
     public boolean isFatPex() {
-        return fatPex;
+        return isFat();
     }
 
     /**
      * @param fatPex when <code>true</code>, wrappers will be made all pointing to a single pex file.
      */
+    @Deprecated
     public void setFatPex(boolean fatPex) {
-        this.fatPex = fatPex;
+        isFat = fatPex;
+     }
+
+    // Use these properties instead.
+
+    /**
+     * @return when <code>true</code>, then skinny pex's will be used.
+     */
+    public boolean isFat() {
+        return isFat;
+    }
+
+    /**
+     * @param fat when <code>true</code>, wrappers will be made all pointing to a single pex file.
+     */
+    public void setIsFat(boolean isFat) {
+        this.isFat = isFat;
     }
 
     /**
@@ -65,4 +85,12 @@ public boolean isPythonWrapper() {
     public void setPythonWrapper(boolean pythonWrapper) {
         this.pythonWrapper = pythonWrapper;
     }
+
+    public File getCache() {
+        return cache;
+    }
+
+    public void setCache(File cache) {
+        this.cache = cache;
+    }
 }

…thon/util/internal/pex/PexGenerator.java …le/python/extension/ZipappExtension.javapygradle-plugin/src/main/groovy/com/linkedin/gradle/python/util/internal/pex/PexGenerator.java renamed to pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/extension/ZipappExtension.java pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/util/internal/pex/PexGenerator.java renamed to pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/extension/ZipappExtension.java

@@ -13,14 +13,17 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package com.linkedin.gradle.python.util.internal.pex;
+package com.linkedin.gradle.python.extension;
 
-public interface PexGenerator {
 
+public interface ZipappExtension {
     /**
-     * When called will generate entry point files for a pex.
-     *
-     * @throws Exception when failing to build entry point
+     * @return when <code>true</code>, then skinny pex's will be used.
      */
-    void buildEntryPoints() throws Exception;
+    public boolean isFat();
+
+    /**
+     * @param fat when <code>true</code>, wrappers will be made all pointing to a single pex file.
+     */
+    public void setIsFat(boolean isFat);
 }

pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/tasks/BuildPexTask.java

@@ -22,8 +22,8 @@
 import com.linkedin.gradle.python.util.ExtensionUtils;
 import com.linkedin.gradle.python.util.internal.pex.FatPexGenerator;
 import com.linkedin.gradle.python.util.internal.pex.ThinPexGenerator;
-import com.linkedin.gradle.python.util.pex.DefaultEntryPointTemplateProvider;
-import com.linkedin.gradle.python.util.pex.EntryPointTemplateProvider;
+import com.linkedin.gradle.python.util.pex.DefaultPexEntryPointTemplateProvider;
+import com.linkedin.gradle.python.util.zipapp.EntryPointTemplateProvider;
 import org.apache.commons.io.FileUtils;
 import org.gradle.api.DefaultTask;
 import org.gradle.api.Project;
@@ -58,7 +58,7 @@
 public class BuildPexTask extends DefaultTask implements FailureReasonProvider {
 
     private Map<String, String> additionalProperties;
-    private EntryPointTemplateProvider templateProvider = new DefaultEntryPointTemplateProvider();
+    private EntryPointTemplateProvider templateProvider = new DefaultPexEntryPointTemplateProvider();
     private TeeOutputContainer container = new TeeOutputContainer(System.out, System.err);
     private List<String> pexOptions = new ArrayList<>();
 
@@ -87,7 +87,7 @@ public void buildPex() throws Exception {
 
         deployableExtension.getDeployableBuildDir().mkdirs();
 
-        if (pexExtension.isFatPex()) {
+        if (pexExtension.isFat()) {
             new FatPexGenerator(project, pexOptions).buildEntryPoints();
         } else {
             new ThinPexGenerator(project, pexOptions, templateProvider, additionalProperties).buildEntryPoints();

pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/tasks/BuildWebAppTask.java

@@ -67,7 +67,7 @@ public void buildWebapp() throws IOException, ClassNotFoundException {
         Project project = getProject();
         PexExtension pexExtension = ExtensionUtils.getPythonComponentExtension(project, PexExtension.class);
 
-        if (pexExtension.isFatPex()) {
+        if (pexExtension.isFat()) {
             new FatPexGenerator(project, pexOptions).buildEntryPoint(
                 PexFileUtil.createFatPexFilename(executable.getName()), entryPoint, null);
         } else {

pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/util/internal/pex/FatPexGenerator.java

@@ -15,18 +15,22 @@
  */
 package com.linkedin.gradle.python.util.internal.pex;
 
+import com.linkedin.gradle.python.PythonExtension;
 import com.linkedin.gradle.python.util.EntryPointHelpers;
 import com.linkedin.gradle.python.util.PexFileUtil;
+import com.linkedin.gradle.python.util.internal.zipapp.ZipappGenerator;
 import com.linkedin.gradle.python.util.pip.PipFreezeAction;
 import org.gradle.api.Project;
 import org.gradle.api.logging.Logger;
 import org.gradle.api.logging.Logging;
 import org.gradle.process.ExecResult;
 
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
-public class FatPexGenerator implements PexGenerator {
+
+public class FatPexGenerator implements ZipappGenerator {
 
     private static final Logger logger = Logging.getLogger(FatPexGenerator.class);
 
@@ -38,6 +42,12 @@ public FatPexGenerator(Project project, List<String> pexOptions) {
         this.pexOptions = pexOptions;
     }
 
+    @Override
+    public Map<String, String> buildSubstitutions(PythonExtension extension, String entry) {
+        // Not used for fat pexes.
+        return new HashMap<>();
+    }
+
     @Override
     public void buildEntryPoints() {
         Map<String, String> dependencies = new PipFreezeAction(project).getDependencies();

pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/util/internal/pex/ThinPexGenerator.java

@@ -17,53 +17,54 @@
 
 import com.linkedin.gradle.python.PythonExtension;
 import com.linkedin.gradle.python.extension.DeployableExtension;
+import com.linkedin.gradle.python.extension.PexExtension;
 import com.linkedin.gradle.python.util.EntryPointHelpers;
 import com.linkedin.gradle.python.util.ExtensionUtils;
 import com.linkedin.gradle.python.util.PexFileUtil;
 import com.linkedin.gradle.python.util.entrypoint.EntryPointWriter;
-import com.linkedin.gradle.python.util.pex.EntryPointTemplateProvider;
+import com.linkedin.gradle.python.util.internal.zipapp.DefaultTemplateProviderOptions;
+import com.linkedin.gradle.python.util.internal.zipapp.ThinZipappGenerator;
 import com.linkedin.gradle.python.util.pip.PipFreezeAction;
+import com.linkedin.gradle.python.util.zipapp.EntryPointTemplateProvider;
 import org.gradle.api.Project;
-import org.gradle.api.logging.Logger;
 import org.gradle.api.logging.Logging;
 import org.gradle.process.ExecResult;
 
 import java.io.File;
-import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
 
-public class ThinPexGenerator implements PexGenerator {
-
-    private static final Logger logger = Logging.getLogger(ThinPexGenerator.class);
-
-    private final Project project;
-    private final List<String> pexOptions;
-    private final EntryPointTemplateProvider templateProvider;
-    private final Map<String, String> extraProperties;
+public class ThinPexGenerator extends ThinZipappGenerator {
 
     public ThinPexGenerator(
         Project project,
         List<String> pexOptions,
         EntryPointTemplateProvider templateProvider,
         Map<String, String> extraProperties) {
-        this.project = project;
-        this.pexOptions = pexOptions;
-        this.templateProvider = templateProvider;
-        this.extraProperties = extraProperties == null ? new HashMap<>() : extraProperties;
+
+        super(project, pexOptions, templateProvider, extraProperties);
+        logger = Logging.getLogger(ThinPexGenerator.class);
+    }
+
+    @Override
+    public Map<String, String> buildSubstitutions(PythonExtension extension, String entry) {
+        Map<String, String> substitutions = super.buildSubstitutions(extension, entry);
+        substitutions.put("realPex", PexFileUtil.createThinPexFilename(project.getName()));
+        return substitutions;
     }
 
     @Override
     public void buildEntryPoints() throws Exception {
         PythonExtension extension = ExtensionUtils.getPythonExtension(project);
+        PexExtension pexExtension = ExtensionUtils.getPythonComponentExtension(extension, PexExtension.class);
         DeployableExtension deployableExtension = ExtensionUtils.getPythonComponentExtension(
             extension, DeployableExtension.class);
 
         Map<String, String> dependencies = new PipFreezeAction(project).getDependencies();
 
         PexExecSpecAction action = PexExecSpecAction.withOutEntryPoint(
-            project, project.getName(), pexOptions, dependencies);
+            project, project.getName(), options, dependencies);
 
         ExecResult exec = project.exec(action);
         new PexExecOutputParser(action, exec).validatePexBuildSuccessfully();
@@ -73,17 +74,13 @@ public void buildEntryPoints() throws Exception {
             String[] split = it.split("=");
             String name = split[0].trim();
             String entry = split[1].trim();
-
-            Map<String, String> propertyMap = new HashMap<>();
-            propertyMap.putAll(extraProperties);
-            propertyMap.put("realPex", PexFileUtil.createThinPexFilename(project.getName()));
-            propertyMap.put("entryPoint", entry);
-            propertyMap.put("pythonExecutable", extension.getDetails().getSystemPythonInterpreter().getAbsolutePath());
-            propertyMap.put("toolName", project.getName());
+            Map<String, String> substitutions = buildSubstitutions(extension, entry);
 
             DefaultTemplateProviderOptions providerOptions = new DefaultTemplateProviderOptions(project, extension, entry);
-            new EntryPointWriter(project, templateProvider.retrieveTemplate(providerOptions))
-                .writeEntryPoint(new File(deployableExtension.getDeployableBinDir(), name), propertyMap);
+            new EntryPointWriter(
+                project,
+                templateProvider.retrieveTemplate(providerOptions, pexExtension.isPythonWrapper()))
+                .writeEntryPoint(new File(deployableExtension.getDeployableBinDir(), name), substitutions);
         }
     }
 }

…/pex/DefaultTemplateProviderOptions.java …papp/DefaultTemplateProviderOptions.javapygradle-plugin/src/main/groovy/com/linkedin/gradle/python/util/internal/pex/DefaultTemplateProviderOptions.java renamed to pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/util/internal/zipapp/DefaultTemplateProviderOptions.java pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/util/internal/pex/DefaultTemplateProviderOptions.java renamed to pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/util/internal/zipapp/DefaultTemplateProviderOptions.java

@@ -13,10 +13,10 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package com.linkedin.gradle.python.util.internal.pex;
+package com.linkedin.gradle.python.util.internal.zipapp;
 
 import com.linkedin.gradle.python.PythonExtension;
-import com.linkedin.gradle.python.util.pex.TemplateProviderOptions;
+import com.linkedin.gradle.python.util.zipapp.TemplateProviderOptions;
 import org.gradle.api.Project;
 
 /**

pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/util/internal/zipapp/ThinZipappGenerator.java

@@ -0,0 +1,65 @@
+/*
+ * Copyright 2016 LinkedIn Corp.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.linkedin.gradle.python.util.internal.zipapp;
+
+import com.linkedin.gradle.python.PythonExtension;
+import com.linkedin.gradle.python.util.zipapp.EntryPointTemplateProvider;
+import org.gradle.api.Project;
+import org.gradle.api.logging.Logger;
+import org.gradle.api.logging.Logging;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+
+public class ThinZipappGenerator implements ZipappGenerator {
+
+    protected static Logger logger = Logging.getLogger(ThinZipappGenerator.class);
+
+    protected final Project project;
+    protected final List<String> options;
+    protected final EntryPointTemplateProvider templateProvider;
+    protected final Map<String, String> extraProperties;
+
+    public ThinZipappGenerator(
+        Project project,
+        List<String> options,
+        EntryPointTemplateProvider templateProvider,
+        Map<String, String> extraProperties) {
+
+        this.project = project;
+        this.options = options;
+        this.templateProvider = templateProvider;
+        this.extraProperties = extraProperties == null ? new HashMap<>() : extraProperties;
+    }
+
+    @Override
+    public Map<String, String> buildSubstitutions(PythonExtension extension, String entry) {
+            Map<String, String> substitutions = new HashMap<>();
+            substitutions.putAll(extraProperties);
+            substitutions.put("entryPoint", entry);
+            substitutions.put("pythonExecutable", extension.getDetails().getSystemPythonInterpreter().getAbsolutePath());
+            substitutions.put("toolName", project.getName());
+            return substitutions;
+    }
+
+    @Override
+    public void buildEntryPoints() throws Exception {
+        // Generic zipapps don't build anything, so subclasses should override
+        // this to build their entry point specific artifacts.
+    };
+}

pygradle-plugin/src/main/groovy/com/linkedin/gradle/python/util/internal/zipapp/ZipappGenerator.java

@@ -0,0 +1,36 @@
+/*
+ * Copyright 2016 LinkedIn Corp.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.linkedin.gradle.python.util.internal.zipapp;
+
+import com.linkedin.gradle.python.PythonExtension;
+import java.util.Map;
+
+
+public interface ZipappGenerator {
+
+    /**
+     * When called, create the substitution map for the template generation.
+     */
+    Map<String, String> buildSubstitutions(PythonExtension extension, String entry);
+
+    /**
+     * When called will generate entry point files for a zipapp.
+     *
+     * @throws Exception when failing to build entry point
+     */
+    void buildEntryPoints() throws Exception;
+}