Final fixes.

Author: aartiPl

build.gradle.kts

@@ -18,6 +18,7 @@ dependencies {
     compile("com.offbytwo:docopt:0.6.0.20150202")
 
     implementation("org.jetbrains.kotlin:kotlin-scripting-common:$kotlinVersion")
+    implementation("org.jetbrains.kotlin:kotlin-reflect:$kotlinVersion")
     implementation("org.jetbrains.kotlin:kotlin-scripting-jvm:$kotlinVersion")
     implementation("org.jetbrains.kotlin:kotlin-scripting-dependencies:$kotlinVersion")
     implementation("org.jetbrains.kotlin:kotlin-scripting-dependencies-maven:$kotlinVersion")

src/main/kotlin/kscript/app/AppHelpers.kt

@@ -2,6 +2,7 @@ package kscript.app
 
 import kscript.app.ShellUtils.requireInPath
 import java.io.*
+import java.net.URI
 import java.net.URL
 import java.nio.file.Files
 import java.nio.file.Paths
@@ -135,6 +136,10 @@ fun createTmpScript(scriptText: String, extension: String = "kts"): File {
     }
 }
 
+fun isFile(uri: URI) = uri.scheme.startsWith("file")
+fun isUrl(uri: URI) = uri.scheme.startsWith("http") || uri.scheme.startsWith("https")
+
+fun isUrl(scriptResource: String) = scriptResource.startsWith("http://") || scriptResource.startsWith("https://")
 
 fun fetchFromURL(scriptURL: String): File {
     val urlHash = md5(scriptURL)
@@ -157,6 +162,24 @@ fun fetchFromURL(scriptURL: String): File {
     return urlCache
 }
 
+fun fetchFromURI(scriptURI: URI): List<String> {
+    if (isFile(scriptURI)) {
+        return scriptURI.toURL().readText().lines()
+    }
+
+    val urlHash = md5(scriptURI.toString())
+    val urlCache = File(KSCRIPT_CACHE_DIR, "/url_cache_${urlHash}")
+
+    if (urlCache.exists()) {
+        return urlCache.readText().lines()
+    }
+
+    val urlContent = scriptURI.toURL().readText()
+    urlCache.writeText(urlContent)
+
+    return urlContent.lines()
+}
+
 
 fun md5(byteProvider: () -> ByteArray): String {
     // from https://stackoverflow.com/questions/304268/getting-a-files-md5-checksum-in-java

src/main/kotlin/kscript/app/Kscript.kt

@@ -8,6 +8,7 @@ import java.io.File
 import java.io.FileInputStream
 import java.io.InputStreamReader
 import java.lang.IllegalArgumentException
+import java.net.URI
 import java.net.URL
 import java.net.UnknownHostException
 import java.util.*
@@ -92,31 +93,38 @@ fun main(args: Array<String>) {
     val loggingEnabled = !docopt.getBoolean("silent")
 
 
-    // create cache dir if it does not yet exist
-    if (!KSCRIPT_CACHE_DIR.isDirectory) {
-        KSCRIPT_CACHE_DIR.mkdir()
-    }
-
     // optionally clear up the jar cache
     if (docopt.getBoolean("clear-cache")) {
         info("Cleaning up cache...")
-        KSCRIPT_CACHE_DIR.listFiles().forEach { it.delete() }
+        KSCRIPT_CACHE_DIR.listFiles()?.forEach { it.delete() }
         quit(0)
     }
 
+    // create cache dir if it does not yet exist
+    if (!KSCRIPT_CACHE_DIR.isDirectory) {
+        KSCRIPT_CACHE_DIR.mkdir()
+    }
+
     // Resolve the script resource argument into an actual file
     val scriptResource = docopt.getString("script")
 
-    val enableSupportApi = docopt.getBoolean("text")
-    val rawScript = prepareScript(scriptResource)
+    val rawUri = prepareScript(scriptResource)
 
     if (docopt.getBoolean("add-bootstrap-header")) {
+        errorIf(!isFile(rawUri)) {
+            "Can not add bootstrap header to URL resources: $rawUri"
+        }
+
+        val rawScript = File(rawUri)
+
         errorIf(!rawScript.canWrite()) {
             "Script file not writable: $rawScript"
         }
+
         errorIf(rawScript.parentFile == SCRIPT_TEMP_DIR) {
             "Temporary script file detected: $rawScript, created from $scriptResource"
         }
+
         val scriptLines = rawScript.readLines().dropWhile {
             it.startsWith("#!/") && it != "#!/bin/bash"
         }
@@ -138,23 +146,25 @@ fun main(args: Array<String>) {
         quit(0)
     }
 
+    val enableSupportApi = docopt.getBoolean("text")
+
     // post process script (text-processing mode, custom dsl preamble, resolve includes)
     // and finally resolve all includes (see https://github.com/holgerbrandl/kscript/issues/34)
-    val (scriptFile, includeURLs) = resolveIncludes(resolvePreambles(rawScript, enableSupportApi))
+    val (scriptFile, includeURLs) = resolveIncludes(resolvePreambles(rawUri, enableSupportApi))
 
     val script = Script(scriptFile)
 
     // Find all //DEPS directives and concatenate their values
-    val dependencies = (script.collectDependencies() + Script(rawScript).collectDependencies()).distinct()
-    val customRepos = (script.collectRepos() + Script(rawScript).collectRepos()).distinct()
+    val dependencies = script.collectDependencies().distinct()
+    val customRepos = script.collectRepos().distinct()
 
     // Extract kotlin arguments
     val kotlinOpts = script.collectRuntimeOptions()
     val compilerOpts = script.collectCompilerOptions()
 
     //  Create temporary dev environment
     if (docopt.getBoolean("idea")) {
-        println(launchIdeaWithKscriptlet(rawScript, userArgs, dependencies, customRepos, includeURLs, compilerOpts))
+        println(launchIdeaWithKscriptlet(scriptFile, userArgs, dependencies, customRepos, includeURLs, compilerOpts))
         exitProcess(0)
     }
 
@@ -319,63 +329,64 @@ private fun versionCheck() {
 }
 
 
-fun prepareScript(scriptResource: String):File {
-    var scriptFile: File?
+fun prepareScript(scriptResource: String): URI {
+    if (isUrl(scriptResource)) {
+        return URI(scriptResource)
+    }
+
+    var scriptUri: URI?
 
     // map script argument to script file
-    scriptFile = with(File(scriptResource)) {
+    scriptUri = with(File(scriptResource)) {
         if (!canRead()) {
             // not a file so let's keep the script-file undefined here
             null
         } else if (listOf("kts", "kt").contains(extension)) {
-            this
+            this.toURI()
         } else {
             // if we can "just" read from script resource create tmp file
             // i.e. script input is process substitution file handle
             // not FileInputStream(this).bufferedReader().use{ readText()} does not work nor does this.readText
-            createTmpScript(FileInputStream(this).bufferedReader().readText())
+            createTmpScript(FileInputStream(this).bufferedReader().readText()).toURI()
         }
     }
 
     // support stdin
     if (scriptResource == "-" || scriptResource == "/dev/stdin") {
         val scriptText = generateSequence() { readLine() }.joinToString("\n").trim()
-        scriptFile = createTmpScript(scriptText)
+        scriptUri = createTmpScript(scriptText).toURI()
     }
 
-
-    // Support URLs as script files
-    if (scriptResource.startsWith("http://") || scriptResource.startsWith("https://")) {
-        scriptFile = fetchFromURL(scriptResource)
-    }
-
-
     // Support for support process substitution and direct script arguments
-    if (scriptFile == null && !scriptResource.endsWith(".kts") && !scriptResource.endsWith(".kt")) {
+    if (scriptUri == null && !scriptResource.endsWith(".kts") && !scriptResource.endsWith(".kt")) {
         val scriptText = if (File(scriptResource).canRead()) {
             File(scriptResource).readText().trim()
         } else {
             // the last resort is to assume the input to be a kotlin program
             scriptResource.trim()
         }
 
-        scriptFile = createTmpScript(scriptText)
+        scriptUri = createTmpScript(scriptText).toURI()
     }
 
     // just proceed if the script file is a regular file at this point
-    errorIf(scriptFile == null || !scriptFile.canRead()) {
+    errorIf(scriptUri == null) {
         "Could not read script argument '$scriptResource'"
     }
 
     // note script file must be not null at this point
 
-    return scriptFile!!
+    return scriptUri!!
 }
 
 
-private fun resolvePreambles(rawScript: File, enableSupportApi: Boolean): File {
+private fun resolvePreambles(rawUri: URI, enableSupportApi: Boolean): URI {
+    if (!isFile(rawUri)) {
+        return rawUri
+    }
+
     // include preamble for custom interpreters (see https://github.com/holgerbrandl/kscript/issues/67)
-    var scriptFile = rawScript
+    var scriptFile = File(rawUri)
 
     System.getenv("CUSTOM_KSCRIPT_PREAMBLE")?.let { interpPreamble ->
         //        rawScript = Script(rawScript!!).prependWith(interpPreamble).createTmpScript()
@@ -404,5 +415,5 @@ private fun resolvePreambles(rawScript: File, enableSupportApi: Boolean): File {
         scriptFile = Script(scriptFile).prependWith("//INCLUDE ${preambleFile.absolutePath}").createTmpScript()
     }
 
-    return scriptFile
+    return scriptFile.toURI()
 }

src/main/kotlin/kscript/app/ResolveIncludes.kt

@@ -16,17 +16,27 @@ const val IMPORT_STATEMENT_PREFIX = "import " // todo make more solid by using o
 data class IncludeResult(val scriptFile: File, val includes: List<URL> = emptyList())
 
 /** Resolve include declarations in a script file. Resolved script will be put into another temporary script */
-fun resolveIncludes(file: File): IncludeResult {
+fun resolveIncludes(uri: URI): IncludeResult {
+    val scriptText = uri.toURL().readText()
+
+    val urlExtension = when {
+        uri.toString().endsWith(".kt") -> "kt"
+        uri.toString().endsWith(".kts") -> "kts"
+        else -> if (scriptText.contains("fun main")) {
+            "kt"
+        } else {
+            "kts"
+        }
+    }
+
     val includes = mutableListOf<URI>()
-    //TODO: in case initial file is a remote file, we shouldn't allow local references, so initial 'true' here is a wrong assumption
-    //TODO: first resolve redirects: https://stackoverflow.com/questions/2659000/java-how-to-find-the-redirected-url-of-a-url
-    val lines = resolve(true, file.toURI(), includes)
-    val script = Script(lines, file.extension)
+    val lines = resolve(isFile(uri), uri, includes)
+    val script = Script(lines, urlExtension)
 
     return IncludeResult(script.consolidateStructure().createTmpScript(), includes.map { it.toURL() })
 }
 
-private fun resolve(allowLocalReferences: Boolean, scriptUri: URI, includes: MutableList<URI>): List<String> {
+private fun resolve(allowFileReferences: Boolean, scriptUri: URI, includes: MutableList<URI>): List<String> {
     val lines = readLines(scriptUri)
     val scriptPath = scriptUri.resolve(".")
     val result = mutableListOf<String>()
@@ -36,7 +46,7 @@ private fun resolve(allowLocalReferences: Boolean, scriptUri: URI, includes: Mut
             val include = extractTarget(line)
             val includeUri = resolveUri(scriptPath, include)
 
-            if (!allowLocalReferences && isLocal(includeUri)) {
+            if (!allowFileReferences && isFile(includeUri)) {
                 errorMsg("References to local filesystem from remote scripts are not allowed.\nIn script: $scriptUri; Reference: $includeUri")
                 quit(1)
             }
@@ -49,7 +59,7 @@ private fun resolve(allowLocalReferences: Boolean, scriptUri: URI, includes: Mut
 
             includes.add(includeUri)
 
-            val resolvedLines = resolve(allowLocalReferences && isLocal(includeUri), includeUri, includes)
+            val resolvedLines = resolve(allowFileReferences && isFile(includeUri), includeUri, includes)
             result.addAll(resolvedLines)
             continue
         }
@@ -62,7 +72,7 @@ private fun resolve(allowLocalReferences: Boolean, scriptUri: URI, includes: Mut
 
 private fun readLines(uri: URI): List<String> {
     try {
-        return uri.toURL().readText().lines()
+        return fetchFromURI(uri)
     } catch (e: Exception) {
         errorMsg("Failed to resolve include with URI: '${uri}'")
         System.err.println(e.message?.lines()!!.map { it.prependIndent("[kscript] [ERROR] ") })
@@ -80,8 +90,6 @@ private fun resolveUri(scriptPath: URI, include: String): URI {
     return result.normalize()
 }
 
-internal fun isLocal(uri: URI) = uri.scheme.startsWith("file")
-
 internal fun isIncludeDirective(line: String) = line.startsWith("//INCLUDE") || line.startsWith(INCLUDE_ANNOT_PREFIX)
 
 internal fun extractTarget(incDirective: String) = when {
@@ -108,6 +116,6 @@ private const val INCLUDE_ANNOT_PREFIX = "@file:Include("
 object ResolveIncludes {
     @JvmStatic
     fun main(args: Array<String>) {
-        System.err.println(resolveIncludes(File(args[0])).scriptFile.readText())
+        System.err.println(resolveIncludes(File(args[0]).toURI()).scriptFile.readText())
     }
 }

src/main/kotlin/kscript/app/Script.kt

@@ -7,7 +7,7 @@ data class Script(val lines: List<String>, val extension: String = "kts") : Iter
 
     constructor(scriptFile: File) : this(scriptFile.readLines(), scriptFile.extension)
 
-    /** Returns a the namespace/package of the script (if declared). */
+    /** Returns a namespace/package of the script (if declared). */
     val pckg by lazy {
         lines.find { it.startsWith("package ") }?.split("[ ]+".toRegex())?.get(1)?.run { this + "." }
     }

src/test/kotlin/Tests.kt

@@ -196,7 +196,7 @@ class Tests {
         val file = File("test/resources/consolidate_includes/template.kts")
         val expected = File("test/resources/consolidate_includes/expected.kts")
 
-        val result = resolveIncludes(file)
+        val result = resolveIncludes(file.toURI())
 
         result.scriptFile.readText() shouldBe (expected.readText())
     }
@@ -207,22 +207,22 @@ class Tests {
         val file = File("test/resources/includes/include_variations.kts")
         val expected = File("test/resources/includes/expected_variations.kts")
 
-        val result = resolveIncludes(file)
+        val result = resolveIncludes(file.toURI())
 
         result.scriptFile.readText() shouldBe (expected.readText())
     }
 
     @Test
     fun test_include_detection() {
-        val result = resolveIncludes(File("test/resources/includes/include_variations.kts"))
+        val result = resolveIncludes(File("test/resources/includes/include_variations.kts").toURI())
 
         result.includes.filter { it.protocol == "file" }.map { File(it.toURI()).name } shouldBe List(7) { "include_${it + 1}.kt" }
         result.includes.filter { it.protocol != "file" }.size shouldBe 2
     }
 
     @Test
     fun `test include detection - should not include dependency twice`() {
-      val result = resolveIncludes(File("test/resources/includes/dup_include/dup_include.kts"))
+      val result = resolveIncludes(File("test/resources/includes/dup_include/dup_include.kts").toURI())
 
       result.includes.map { File(it.toURI()).name } shouldBe listOf(
             "dup_include_1.kt",

test/test_suite.sh

@@ -41,9 +41,9 @@ echo
 echo "Starting $SUITE test suite... Compiling... Please wait..."
 
 # exit code of `true` is expected to be 0 (see https://github.com/lehmannro/assert.sh)
-cd "$PROJECT_DIR" || exit
+cd "$PROJECT_DIR"
 assert_raises "./gradlew build"
-cd - || exit
+cd -
 
 assert_end "$SUITE"