Merge pull request #265 from wttech/file-output-speedy

File output / FS

Author: krystian-panek-vmltech

core/src/main/java/dev/vml/es/acm/core/code/ExecutionHistory.java

@@ -91,7 +91,12 @@ private void saveFileOutput(FileOutput output, Resource entry) {
                             "Output '%s' cannot be flushed before saving to the execution history!", output.getName()),
                     e);
         }
-        file.saveFile(output.getMimeType(), output.getInputStream());
+        try (InputStream inputStream = output.getInputStream()) {
+            file.saveFile(output.getMimeType(), inputStream);
+        } catch (IOException e) {
+            throw new AcmException(
+                    String.format("Output '%s' cannot be saved to the execution history!", output.getName()), e);
+        }
     }
 
     public InputStream readOutputByName(Execution execution, String name) {

core/src/main/java/dev/vml/es/acm/core/code/FileOutput.java

@@ -1,22 +1,28 @@
 package dev.vml.es.acm.core.code;
 
 import com.fasterxml.jackson.annotation.JsonIgnore;
-import dev.vml.es.acm.core.gui.SpaSettings;
-import dev.vml.es.acm.core.osgi.OsgiContext;
-import dev.vml.es.acm.core.repo.RepoChunks;
+import dev.vml.es.acm.core.AcmException;
 import java.io.Closeable;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
 import java.io.Flushable;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
 import java.io.PrintStream;
+import org.apache.commons.io.FileUtils;
 import org.apache.commons.lang3.StringUtils;
-import org.apache.sling.api.resource.ResourceResolverFactory;
 
 public class FileOutput extends Output implements Flushable, Closeable {
 
+    private static final String TEMP_DIR = "acm/execution/file";
+
+    @JsonIgnore
+    private transient File tempFile;
+
     @JsonIgnore
-    private transient RepoChunks repoChunks;
+    private transient FileOutputStream fileOutputStream;
 
     @JsonIgnore
     private transient PrintStream printStream;
@@ -59,28 +65,47 @@ public void setMimeType(String mimeType) {
     }
 
     @JsonIgnore
-    private RepoChunks getRepoChunks() {
-        if (repoChunks == null) {
-            OsgiContext osgi = executionContext.getCodeContext().getOsgiContext();
-            SpaSettings spaSettings = osgi.getService(SpaSettings.class);
-            ResourceResolverFactory resolverFactory = osgi.getService(ResourceResolverFactory.class);
-            String chunkFolderPath = String.format(
-                    "%s/outputs/%s",
-                    ExecutionContext.varPath(executionContext.getId()), StringUtils.replace(getName(), "/", "-"));
-            repoChunks =
-                    new RepoChunks(resolverFactory, chunkFolderPath, spaSettings.getExecutionFileOutputChunkSize());
+    private File getTempFile() {
+        if (tempFile == null) {
+            File tmpDir = FileUtils.getTempDirectory();
+            String contextPrefix = StringUtils.replace(executionContext.getId(), "/", "-");
+            String sanitizedName = StringUtils.replace(getName(), "/", "-");
+            String fileName = String.format("%s_%s.out", contextPrefix, sanitizedName);
+            File result = new File(new File(tmpDir, TEMP_DIR), fileName);
+            File parentDir = result.getParentFile();
+            if (parentDir != null && !parentDir.exists()) {
+                try {
+                    FileUtils.forceMkdir(parentDir);
+                } catch (IOException e) {
+                    throw new AcmException(
+                            String.format("Cannot create temp directory for output '%s'!", getName()), e);
+                }
+            }
+            this.tempFile = result;
         }
-        return repoChunks;
+        return tempFile;
     }
 
     @JsonIgnore
     public OutputStream getOutputStream() {
-        return getRepoChunks().getOutputStream();
+        if (fileOutputStream == null) {
+            try {
+                fileOutputStream = new FileOutputStream(getTempFile());
+            } catch (IOException e) {
+                throw new AcmException(
+                        String.format("Cannot create output stream for file output '%s'!", getName()), e);
+            }
+        }
+        return fileOutputStream;
     }
 
     @JsonIgnore
     public InputStream getInputStream() {
-        return getRepoChunks().getInputStream();
+        try {
+            return new FileInputStream(getTempFile());
+        } catch (IOException e) {
+            throw new AcmException(String.format("Cannot read file output '%s'!", getName()), e);
+        }
     }
 
     @JsonIgnore
@@ -96,11 +121,38 @@ public void flush() throws IOException {
         if (printStream != null) {
             printStream.flush();
         }
-        getRepoChunks().flush();
+        if (fileOutputStream != null) {
+            fileOutputStream.flush();
+        }
     }
 
     @Override
     public void close() throws IOException {
-        getRepoChunks().close();
+        IOException exception = null;
+        try {
+            if (printStream != null) {
+                printStream.close();
+            }
+            if (fileOutputStream != null) {
+                fileOutputStream.close();
+            }
+        } catch (IOException e) {
+            exception = e;
+        } finally {
+            try {
+                if (tempFile != null && tempFile.exists()) {
+                    FileUtils.forceDelete(tempFile);
+                }
+            } catch (IOException e) {
+                if (exception != null) {
+                    exception.addSuppressed(e);
+                } else {
+                    exception = e;
+                }
+            }
+        }
+        if (exception != null) {
+            throw exception;
+        }
     }
 }

ui.content.example/src/main/content/jcr_root/conf/acm/settings/script/manual/example/ACME-203_output-csv.groovy

@@ -0,0 +1,59 @@
+import java.time.LocalDate
+import java.util.Random
+
+boolean canRun() {
+  return conditions.always()
+}
+
+void describeRun() {
+  inputs.integerNumber("count") { label = "Users to generate"; min = 1; value = 5000000 }
+  inputs.text("firstNames") { label = "First names"; description = "One first name per line"; value = "John\nJane\nJack\nAlice\nBob"}
+  inputs.text("lastNames") { label = "Last names"; description = "One last name per line"; value = "Doe\nSmith\nBrown\nJohnson\nWhite" }
+}
+
+void doRun() {
+  log.info "Users CSV report generation started"
+
+  int count = inputs.value("count")
+  def firstNames = (inputs.value("firstNames")).readLines().findAll { it.trim() }
+  def lastNames  = (inputs.value("lastNames")).readLines().findAll { it.trim() }
+
+  def random = new Random()
+  def now = LocalDate.now()
+  def hundredYearsAgo = now.minusYears(100)
+
+  def report = outputs.file("report") {
+    label = "Report"
+    description = "Users report generated as CSV file"
+    downloadName = "report.csv"
+  }
+
+  // CSV header
+  report.out.println("Name,Surname,Birth Date")
+
+  // Generate users
+  (1..count).each { i ->
+    context.checkAborted()
+    
+    def name = firstNames[random.nextInt(firstNames.size())]
+    def surname = lastNames[random.nextInt(lastNames.size())]
+    def birthDate = randomDateBetween(hundredYearsAgo, now)
+
+    report.out.println("${name},${surname},${birthDate}")
+
+    if (i % 100 == 0) log.info("Generated ${i} users...")
+  }
+
+  outputs.text("summary") {
+    value = "Processed ${count} user(s)"
+  }
+
+  log.info "Users CSV report generation ended successfully"
+}
+
+LocalDate randomDateBetween(LocalDate start, LocalDate end) {
+  long startDay = start.toEpochDay()
+  long endDay = end.toEpochDay()
+  long randomDay = startDay + new Random().nextInt((int)(endDay - startDay + 1))
+  return LocalDate.ofEpochDay(randomDay)
+}

…t/manual/example/ACME-203_outputs.groovy …anual/example/ACME-203_output-xls.groovyui.content.example/src/main/content/jcr_root/conf/acm/settings/script/manual/example/ACME-203_outputs.groovy renamed to ui.content.example/src/main/content/jcr_root/conf/acm/settings/script/manual/example/ACME-203_output-xls.groovy ui.content.example/src/main/content/jcr_root/conf/acm/settings/script/manual/example/ACME-203_outputs.groovy renamed to ui.content.example/src/main/content/jcr_root/conf/acm/settings/script/manual/example/ACME-203_output-xls.groovy

@@ -10,7 +10,7 @@ boolean canRun() {
 }
 
 void describeRun() {
-  inputs.integerNumber("count") { label = "Users to generate"; min = 1; value = 1000 }
+  inputs.integerNumber("count") { label = "Users to generate"; min = 1; value = 100000 }
   inputs.text("firstNames") { label = "First names"; description = "One first name per line"; value = "John\nJane\nJack\nAlice\nBob"}
   inputs.text("lastNames") { label = "Last names"; description = "One last name per line"; value = "Doe\nSmith\nBrown\nJohnson\nWhite" }
 }
@@ -38,6 +38,8 @@ void doRun() {
   dateStyle.setDataFormat(helper.createDataFormat().getFormat("yyyy-mm-dd"))
 
   (1..count).each { i ->
+    context.checkAborted()
+
     def name = firstNames[random.nextInt(firstNames.size())]
     def surname = lastNames[random.nextInt(lastNames.size())]
     def birthDate = randomDateBetween(hundredYearsAgo, now)