Fixing the native build so it can find the tsl headers. (#534)

Author: Craigacp

tensorflow-core/tensorflow-core-generator/src/main/java/org/tensorflow/generator/op/OpGenerator.java

@@ -37,6 +37,7 @@
 import java.util.Map;
 import java.util.Scanner;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;
 import org.bytedeco.javacpp.BytePointer;
 import org.springframework.core.io.support.PathMatchingResourcePatternResolver;
 import org.tensorflow.internal.c_api.TF_ApiDefMap;
@@ -72,7 +73,7 @@ public final class OpGenerator {
 
   private static final String DEFAULT_OP_DEF_FILE = "org/tensorflow/ops.pbtxt";
 
-  private static final Scanner USER_PROMPT = new Scanner(System.in);
+  private static final Scanner USER_PROMPT = new Scanner(System.in, StandardCharsets.UTF_8);
 
   /**
    * Args should be {@code <outputDir> <opDefFile> [base_package]}.
@@ -195,7 +196,8 @@ private static String arg(String[] args, int idx) {
   private static OpList readOpList(String filename, InputStream protoInput) {
     try {
       if (filename.endsWith(".pbtxt")) {
-        return TextFormat.parse(new String(protoInput.readAllBytes()), OpList.class);
+        return TextFormat.parse(
+            new String(protoInput.readAllBytes(), StandardCharsets.UTF_8), OpList.class);
       }
       return OpList.parseFrom(protoInput);
 
@@ -286,9 +288,8 @@ private void mergeBaseApiDefs(TF_ApiDefMap apiDefMap, TF_Status status) {
   }
 
   private void mergeApiDefs(TF_ApiDefMap apiDefMap, TF_Status status) {
-    try {
-      Files.walk(apiDefsPath)
-          .filter(p -> p.toString().endsWith(".pbtxt"))
+    try (Stream<Path> s = Files.walk(apiDefsPath)) {
+      s.filter(p -> p.toString().endsWith(".pbtxt"))
           .forEach(
               p -> {
                 try {
@@ -366,7 +367,7 @@ private void createApiDef(OpDef opDef, File apiDefFile) throws IOException {
     if (!apiDefFile.exists() && !apiDefFile.createNewFile()) {
       System.err.println("Cannot create API definition file \"" + apiDefFile.getPath() + "\"");
     }
-    try (var apiDefWriter = new FileWriter(apiDefFile)) {
+    try (var apiDefWriter = new FileWriter(apiDefFile, StandardCharsets.UTF_8)) {
       var apiDefs = ApiDefs.newBuilder();
       apiDefs.addOp(apiDef.build());
       apiDefWriter.write(TextFormat.printer().printToString(apiDefs.build()));

tensorflow-core/tensorflow-core-generator/src/main/java/org/tensorflow/generator/op/javadoc/JavaDocNodeRendererContext.java

@@ -9,6 +9,8 @@
 public interface JavaDocNodeRendererContext {
 
   /**
+   * Encode a URL into a String.
+   *
    * @param url to be encoded
    * @return an encoded URL (depending on the configuration)
    */
@@ -26,11 +28,15 @@ public interface JavaDocNodeRendererContext {
   Map<String, String> extendAttributes(Node node, String tagName, Map<String, String> attributes);
 
   /**
+   * Gets the HTML writer.
+   *
    * @return the HTML writer to use
    */
   JavaDocWriter getWriter();
 
   /**
+   * The HTML for a line break.
+   *
    * @return HTML that should be rendered for a soft line break
    */
   String getSoftbreak();
@@ -45,17 +51,23 @@ public interface JavaDocNodeRendererContext {
   void render(Node node);
 
   /**
+   * Should HTML be escaped?
+   *
    * @return whether HTML blocks and tags should be escaped or not
    */
   boolean shouldEscapeHtml();
 
   /**
+   * Should URLs be sanitized?
+   *
    * @return true if the {@link UrlSanitizer} should be used.
    * @since 0.14.0
    */
   boolean shouldSanitizeUrls();
 
   /**
+   * Gets the URL sanitizer.
+   *
    * @return Sanitizer to use for securing {@link Link} href and {@link Image} src if {@link
    *     #shouldSanitizeUrls()} is true.
    * @since 0.14.0

tensorflow-core/tensorflow-core-native/pom.xml

@@ -318,6 +318,7 @@
             <includePath>${native.source.directory}/org/tensorflow/internal/c_api/</includePath>
             <!-- additional include paths in case of a full native build -->
             <includePath>${project.basedir}/bazel-${project.artifactId}/external/org_tensorflow/</includePath>
+            <includePath>${project.basedir}/bazel-${project.artifactId}/external/local_tsl/</includePath>
             <includePath>${project.basedir}/bazel-bin/external/org_tensorflow/</includePath>
             <includePath>${project.basedir}/bazel-${project.artifactId}/external/com_google_absl/</includePath>
             <includePath>${project.basedir}/bazel-${project.artifactId}/external/eigen_archive/</includePath>

tensorflow-core/tensorflow-core-native/src/gen/java/org/tensorflow/internal/c_api/global/tensorflow.java

@@ -11,7 +11,7 @@
 public class tensorflow extends org.tensorflow.internal.c_api.presets.tensorflow {
     static { Loader.load(); }
 
-// Parsed from tensorflow/tsl/platform/ctstring_internal.h
+// Parsed from tsl/platform/ctstring_internal.h
 
 /* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
 
@@ -165,7 +165,7 @@ public static native void TF_TString_Copy(TF_TString dst, String src,
 // #endif  // TENSORFLOW_TSL_PLATFORM_CTSTRING_INTERNAL_H_
 
 
-// Parsed from tensorflow/tsl/platform/ctstring.h
+// Parsed from tsl/platform/ctstring.h
 
 /* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
 
@@ -274,7 +274,7 @@ public static native void TF_TString_Copy(TF_TString dst, String src,
 // #endif  // TENSORFLOW_TSL_PLATFORM_CTSTRING_H_
 
 
-// Parsed from tensorflow/tsl/c/tsl_status.h
+// Parsed from tsl/c/tsl_status.h
 
 /* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
 

tensorflow-core/tensorflow-core-native/src/main/java/org/tensorflow/internal/c_api/presets/tensorflow.java

@@ -36,9 +36,11 @@
           value = {"linux", "macosx", "windows"},
           compiler = "cpp17",
           include = {
-            "tensorflow/tsl/platform/ctstring_internal.h",
-            "tensorflow/tsl/platform/ctstring.h",
-            "tensorflow/tsl/c/tsl_status.h",
+            // TSL headers are in different places in a bazel build and the downloaded whl
+            // The lower part is still the same, so multiple roots are set in the pom file.
+            "tsl/platform/ctstring_internal.h",
+            "tsl/platform/ctstring.h",
+            "tsl/c/tsl_status.h",
             "tensorflow/c/c_api_macros.h",
             "tensorflow/c/tf_datatype.h",
             "tensorflow/c/tf_status.h",