jextract: generate one output swift file per input file

Rather than generating an output file per TYPE which we did before.

The per-type mode cannot work correctly because SwiftPM expects to know
all the output Swift files. So we'd have to parse sources and determine
up front what the outputs will be -- doable, but problematic -- instead,
we now generate files based on input files, and map which type goes into
which output file.

This makes it also easier to find where thunks are -- they are in the
same named file as the original type or func was declared in.

This may have some edge case problems still.

Resolves #365

Author: ktoso

Plugins/JExtractSwiftPlugin/JExtractSwiftPlugin.swift

@@ -93,6 +93,7 @@ struct JExtractSwiftBuildToolPlugin: SwiftJavaPluginProtocol, BuildToolPlugin {
       $0.pathExtension == "swift"
     }
 
+    // Output Swift files are just Java filename based converted to Swift files one-to-one
     var outputSwiftFiles: [URL] = swiftFiles.compactMap { sourceFileURL in
       guard sourceFileURL.isFileURL else {
         return nil as URL?
@@ -102,7 +103,7 @@ struct JExtractSwiftBuildToolPlugin: SwiftJavaPluginProtocol, BuildToolPlugin {
       guard sourceFilePath.starts(with: sourceDir) else {
         fatalError("Could not get relative path for source file \(sourceFilePath)")
       }
-      var outputURL = outputSwiftDirectory
+      let outputURL = outputSwiftDirectory
         .appending(path: String(sourceFilePath.dropFirst(sourceDir.count).dropLast(sourceFileURL.lastPathComponent.count + 1)))
 
       let inputFileName = sourceFileURL.deletingPathExtension().lastPathComponent
@@ -116,11 +117,12 @@ struct JExtractSwiftBuildToolPlugin: SwiftJavaPluginProtocol, BuildToolPlugin {
 
     // If the module uses 'Data' type, the thunk file is emitted as if 'Data' is declared
     // in that module. Declare the thunk file as the output.
-    // FIXME: Make this conditional.
     outputSwiftFiles += [
-      outputSwiftDirectory.appending(path: "Data+SwiftJava.swift")
+      outputSwiftDirectory.appending(path: "Foundation+SwiftJava.swift")
     ]
 
+    print("[swift-java-plugin] Output swift files:\n - \(outputSwiftFiles.map({$0.absoluteString}).joined(separator: "\n - "))")
+
     return [
       .buildCommand(
         displayName: "Generate Java wrappers for Swift types",

Samples/SwiftJavaExtractFFMSampleApp/Sources/MySwiftLibrary/MultiplePublicTypes.swift

@@ -0,0 +1,26 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2024 Apple Inc. and the Swift.org project authors
+// Licensed under Apache License v2.0
+//
+// See LICENSE.txt for license information
+// See CONTRIBUTORS.txt for the list of Swift.org project authors
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+//===----------------------------------------------------------------------===//
+
+// This file exists to exercise the swiftpm plugin generating separate output Java files 
+// for the public types; because Java public types must be in a file with the same name as the type.
+
+public struct PublicTypeOne {
+  public init() {}
+  public func test() {}
+}
+
+public struct PublicTypeTwo { 
+  public init() {}
+  public func test() {}
+}

Sources/JExtractSwiftLib/CodePrinter.swift

@@ -216,14 +216,28 @@ extension CodePrinter {
 
   /// - Returns: the output path of the generated file, if any (i.e. not in accumulate in memory mode)
   package mutating func writeContents(
-    outputDirectory: String,
+    outputDirectory _outputDirectory: String,
     javaPackagePath: String?,
-    filename: String
+    filename _filename: String
   ) throws -> URL? {
+
+    // We handle 'filename' that has a path, since that simplifies passing paths from root output directory enourmously.
+    // This just moves the directory parts into the output directory part in order for us to create the sub-directories.
+    let outputDirectory: String
+    let filename: String
+    if _filename.contains(PATH_SEPARATOR) {
+      let parts = _filename.split(separator: PATH_SEPARATOR)
+      outputDirectory = _outputDirectory.appending(PATH_SEPARATOR).appending(parts.dropLast().joined(separator: PATH_SEPARATOR))
+      filename = "\(parts.last!)"
+    } else {
+      outputDirectory = _outputDirectory
+      filename = _filename
+    }
+
     guard self.mode != .accumulateAll else {
       // if we're accumulating everything, we don't want to finalize/flush any contents
       // let's mark that this is where a write would have happened though:
-      print("// ^^^^ Contents of: \(outputDirectory)/\(filename)")
+      print("// ^^^^ Contents of: \(outputDirectory)\(PATH_SEPARATOR)\(filename)")
       return nil
     }
 
@@ -233,7 +247,7 @@ extension CodePrinter {
         "// ==== ---------------------------------------------------------------------------------------------------"
       )
       if let javaPackagePath {
-        print("// \(javaPackagePath)/\(filename)")
+        print("// \(javaPackagePath)\(PATH_SEPARATOR)\(filename)")
       } else {
         print("// \(filename)")
       }
@@ -242,9 +256,15 @@ extension CodePrinter {
     }
 
     let targetDirectory = [outputDirectory, javaPackagePath].compactMap { $0 }.joined(separator: PATH_SEPARATOR)
-    log.trace("Prepare target directory: \(targetDirectory)")
-    try FileManager.default.createDirectory(
-      atPath: targetDirectory, withIntermediateDirectories: true)
+    log.debug("Prepare target directory: '\(targetDirectory)' for file \(filename.bold)")
+    do {
+      try FileManager.default.createDirectory(
+        atPath: targetDirectory, withIntermediateDirectories: true)
+    } catch {
+      // log and throw since it can be confusing what the reason for failing the write was otherwise
+      log.warning("Failed to create directory: \(targetDirectory)")
+      throw error
+    }
 
     let outputPath = Foundation.URL(fileURLWithPath: targetDirectory).appendingPathComponent(filename)
     try contents.write(

Sources/JExtractSwiftLib/FFM/FFMSwift2JavaGenerator+SwiftThunkPrinting.swift

@@ -22,8 +22,19 @@ extension FFMSwift2JavaGenerator {
   }
 
   package func writeSwiftExpectedEmptySources() throws {
+    let pendingFileCount = self.expectedOutputSwiftFiles.count
+    guard pendingFileCount > 0 else {
+      return // no need to write any empty files, yay
+    }
+    
+    print("[swift-java] Write empty [\(self.expectedOutputSwiftFiles.count)] 'expected' files in: \(swiftOutputDirectory)/")
+    
     for expectedFileName in self.expectedOutputSwiftFiles {
-      log.trace("Write empty file: \(expectedFileName) ...")
+      log.debug("Write SwiftPM-'expected' empty file: \(expectedFileName.bold)")
+
+      if expectedFileName.contains("MultiplePublicTypes") {
+        log.error("MultiplePublicTypes MUST NOT BE EMPTY!") // FIXME: xxx
+      }
 
       var printer = CodePrinter()
       printer.print("// Empty file generated on purpose")
@@ -46,32 +57,49 @@ extension FFMSwift2JavaGenerator {
         outputDirectory: self.swiftOutputDirectory,
         javaPackagePath: nil,
         filename: moduleFilename) {
-        print("[swift-java] Generated: \(moduleFilenameBase.bold).swift (at \(outputFile))")
+        log.info("Generated: \(moduleFilenameBase.bold).swift (at \(outputFile.absoluteString))")
         self.expectedOutputSwiftFiles.remove(moduleFilename)
       }
     } catch {
       log.warning("Failed to write to Swift thunks: \(moduleFilename)")
     }
 
     // === All types
-    // FIXME: write them all into the same file they were declared from +SwiftJava
-    for (_, ty) in self.analysis.importedTypes.sorted(by: { (lhs, rhs) in lhs.key < rhs.key }) {
-      let fileNameBase = "\(ty.swiftNominal.qualifiedName)+SwiftJava"
-      let filename = "\(fileNameBase).swift"
-      log.debug("Printing contents: \(filename)")
+    // We have to write all types to their corresponding output file that matches the file they were declared in,
+    // because otherwise SwiftPM plugins will not pick up files apropriately -- we expect 1 output +SwiftJava.swift file for every input.
+    for group: (key: String, value: [Dictionary<String, ImportedNominalType>.Element]) in Dictionary(grouping: self.analysis.importedTypes, by: { $0.value.sourceFilePath }) {
+      log.warning("Writing types in file group: \(group.key): \(group.value.map(\.key))")
+
+      let importedTypesForThisFile = group.value
+        .map(\.value)
+        .sorted(by: { $0.qualifiedName < $1.qualifiedName })
+
+      let inputFileName = "\(group.key)".split(separator: "/").last ?? "__Unknown.swift"
+      let filename = "\(inputFileName)".replacing(".swift", with: "+SwiftJava.swift")
+
+      for ty in importedTypesForThisFile {
+        log.info("Printing Swift thunks for type: \(ty.qualifiedName.bold)")
+        printer.printSeparator("Thunks for \(ty.qualifiedName)")
+
+        do {
+          try printSwiftThunkSources(&printer, ty: ty)
+        } catch {
+          log.warning("Failed to print to Swift thunks for type'\(ty.qualifiedName)' to '\(filename)', error: \(error)")
+        }
+        
+      }
 
+      log.warning("Write Swift thunks file: \(filename.bold)")
       do {
-        try printSwiftThunkSources(&printer, ty: ty)
-
         if let outputFile = try printer.writeContents(
           outputDirectory: self.swiftOutputDirectory,
           javaPackagePath: nil,
           filename: filename) {
-          print("[swift-java] Generated: \(fileNameBase.bold).swift (at \(outputFile))")
+          log.info("Done writing Swift thunks to: \(outputFile.absoluteString)")
           self.expectedOutputSwiftFiles.remove(filename)
         }
       } catch {
-        log.warning("Failed to write to Swift thunks: \(filename)")
+        log.warning("Failed to write to Swift thunks: \(filename), error: \(error)")
       }
     }
   }

Sources/JExtractSwiftLib/FFM/FFMSwift2JavaGenerator.swift

@@ -60,7 +60,7 @@ package class FFMSwift2JavaGenerator: Swift2JavaGenerator {
     // If we are forced to write empty files, construct the expected outputs
     if translator.config.writeEmptyFiles ?? false {
       self.expectedOutputSwiftFiles = Set(translator.inputs.compactMap { (input) -> String? in
-        guard let filePathPart = input.filePath.split(separator: "/\(translator.swiftModuleName)/").last else {
+        guard let filePathPart = input.path.split(separator: "/\(translator.swiftModuleName)/").last else {
           return nil
         }
 
@@ -77,16 +77,12 @@ package class FFMSwift2JavaGenerator: Swift2JavaGenerator {
 
   func generate() throws {
     try writeSwiftThunkSources()
-    print("[swift-java] Generated Swift sources (module: '\(self.swiftModuleName)') in: \(swiftOutputDirectory)/")
+    log.info("Generated Swift sources (module: '\(self.swiftModuleName)') in: \(swiftOutputDirectory)/")
 
     try writeExportedJavaSources()
-    print("[swift-java] Generated Java sources (package: '\(javaPackage)') in: \(javaOutputDirectory)/")
+    log.info("Generated Java sources (package: '\(javaPackage)') in: \(javaOutputDirectory)/")
 
-    let pendingFileCount = self.expectedOutputSwiftFiles.count
-    if pendingFileCount > 0 {
-      print("[swift-java] Write empty [\(pendingFileCount)] 'expected' files in: \(swiftOutputDirectory)/")
-      try writeSwiftExpectedEmptySources()
-    }
+    try writeSwiftExpectedEmptySources()
   }
 }
 
@@ -134,7 +130,7 @@ extension FFMSwift2JavaGenerator {
         javaPackagePath: javaPackagePath,
         filename: filename
       ) {
-        print("[swift-java] Generated: \(ty.swiftNominal.name.bold).java (at \(outputFile))")
+        log.info("Generated: \((ty.swiftNominal.name.bold + ".java").bold) (at \(outputFile.absoluteString))")
       }
     }
 
@@ -148,7 +144,7 @@ extension FFMSwift2JavaGenerator {
         javaPackagePath: javaPackagePath,
         filename: filename)
       {
-        print("[swift-java] Generated: \(self.swiftModuleName).java (at \(outputFile))")
+        log.info("Generated: \((self.swiftModuleName + ".java").bold) (at \(outputFile.absoluteString))")
       }
     }
   }

Sources/JExtractSwiftLib/ImportedDecls.swift

@@ -27,9 +27,15 @@ package enum SwiftAPIKind {
 
 /// Describes a Swift nominal type (e.g., a class, struct, enum) that has been
 /// imported and is being translated into Java.
-package class ImportedNominalType: ImportedDecl {
+package final class ImportedNominalType: ImportedDecl {
   let swiftNominal: SwiftNominalTypeDeclaration
 
+  // The short path from module root to the file in which this nominal was originally declared.
+  // E.g. for `Sources/Example/My/Types.swift` it would be `My/Types.swift`.
+  package var sourceFilePath: String {
+    self.swiftNominal.sourceFilePath
+  }
+
   package var initializers: [ImportedFunc] = []
   package var methods: [ImportedFunc] = []
   package var variables: [ImportedFunc] = []

Sources/JExtractSwiftLib/JNI/JNISwift2JavaGenerator.swift

@@ -67,7 +67,7 @@ package class JNISwift2JavaGenerator: Swift2JavaGenerator {
      // If we are forced to write empty files, construct the expected outputs
     if translator.config.writeEmptyFiles ?? false {
       self.expectedOutputSwiftFiles = Set(translator.inputs.compactMap { (input) -> String? in
-        guard let filePathPart = input.filePath.split(separator: "/\(translator.swiftModuleName)/").last else {
+        guard let filePathPart = input.path.split(separator: "/\(translator.swiftModuleName)/").last else {
           return nil
         }
 

Sources/JExtractSwiftLib/Swift2JavaTranslator.swift

@@ -32,12 +32,7 @@ public final class Swift2JavaTranslator {
 
   // ==== Input
 
-  struct Input {
-    let filePath: String
-    let syntax: SourceFileSyntax
-  }
-
-  var inputs: [Input] = []
+  var inputs: [SwiftJavaInputFile] = []
 
   /// A list of used Swift class names that live in dependencies, e.g. `JavaInteger`
   package var dependenciesClasses: [String] = []
@@ -85,15 +80,12 @@ extension Swift2JavaTranslator {
   package func add(filePath: String, text: String) {
     log.trace("Adding: \(filePath)")
     let sourceFileSyntax = Parser.parse(source: text)
-    self.inputs.append(Input(filePath: filePath, syntax: sourceFileSyntax))
+    self.inputs.append(SwiftJavaInputFile(syntax: sourceFileSyntax, path: filePath))
   }
 
   /// Convenient method for analyzing single file.
-  package func analyze(
-    file: String,
-    text: String
-  ) throws {
-    self.add(filePath: file, text: text)
+  package func analyze(filePath: String, text: String) throws {
+    self.add(filePath: filePath, text: text)
     try self.analyze()
   }
 
@@ -104,16 +96,16 @@ extension Swift2JavaTranslator {
     let visitor = Swift2JavaVisitor(translator: self)
 
     for input in self.inputs {
-      log.trace("Analyzing \(input.filePath)")
-      visitor.visit(sourceFile: input.syntax)
+      log.trace("Analyzing \(input.path)")
+      visitor.visit(inputFile: input)
     }
 
     // If any API uses 'Foundation.Data', import 'Data' as if it's declared in
     // this module.
     if let dataDecl = self.symbolTable[.data] {
       let dataProtocolDecl = self.symbolTable[.dataProtocol]!
       if self.isUsing(where: { $0 == dataDecl || $0 == dataProtocolDecl }) {
-        visitor.visit(nominalDecl: dataDecl.syntax!.asNominal!, in: nil)
+        visitor.visit(nominalDecl: dataDecl.syntax!.asNominal!, in: nil, sourceFilePath: "Foundation/FAKE_FOUNDATION_DATA.swift")
       }
     }
   }
@@ -123,7 +115,7 @@ extension Swift2JavaTranslator {
 
     let symbolTable = SwiftSymbolTable.setup(
       moduleName: self.swiftModuleName,
-      inputs.map({ $0.syntax }) + [dependenciesSource],
+      inputs + [dependenciesSource],
       log: self.log
     )
     self.lookupContext = SwiftTypeLookupContext(symbolTable: symbolTable)
@@ -184,13 +176,14 @@ extension Swift2JavaTranslator {
   }
 
   /// Returns a source file that contains all the available dependency classes.
-  private func buildDependencyClassesSourceFile() -> SourceFileSyntax {
+  private func buildDependencyClassesSourceFile() -> SwiftJavaInputFile {
     let contents = self.dependenciesClasses.map {
       "public class \($0) {}"
     }
     .joined(separator: "\n")
 
-    return SourceFileSyntax(stringLiteral: contents)
+    let syntax = SourceFileSyntax(stringLiteral: contents)
+    return SwiftJavaInputFile(syntax: syntax, path: "FakeDependencyClassesSourceFile.swift")
   }
 }