Attach protobuf generated classes to TF Core API

.github/workflows/ci.yml

@@ -25,7 +25,7 @@ jobs:
           tar xzf $HOME/apache-maven-3.6.3-bin.tar.gz -C /opt/
           ln -sf /opt/apache-maven-3.6.3/bin/mvn /usr/bin/mvn
           echo Downloading Bazel
-          curl -L https://github.com/bazelbuild/bazel/releases/download/0.29.1/bazel-0.29.1-installer-linux-x86_64.sh -o bazel.sh --retry 10
+          curl -L https://github.com/bazelbuild/bazel/releases/download/2.0.0/bazel-2.0.0-installer-linux-x86_64.sh -o bazel.sh --retry 10
           bash bazel.sh
       - name: Checkout repository
         uses: actions/checkout@v1
@@ -55,7 +55,7 @@ jobs:
         run: |
           python3 -m pip install six
           echo Downloading Bazel
-          curl -L https://github.com/bazelbuild/bazel/releases/download/0.29.1/bazel-0.29.1-installer-darwin-x86_64.sh -o bazel.sh --retry 10
+          curl -L https://github.com/bazelbuild/bazel/releases/download/2.0.0/bazel-2.0.0-installer-darwin-x86_64.sh -o bazel.sh --retry 10
           bash bazel.sh
       - name: Checkout repository
         uses: actions/checkout@v1
@@ -88,7 +88,7 @@ jobs:
           bash.exe -lc "find 'C:/Program Files (x86)/Microsoft Visual Studio/2019/Enterprise/VC/' -iname '14.1*' -exec rm -Rf {} \;"
           echo Downloading Bazel
           mkdir C:\bazel
-          curl.exe -L https://github.com/bazelbuild/bazel/releases/download/0.29.1/bazel-0.29.1-windows-x86_64.exe -o C:/bazel/bazel.exe --retry 10
+          curl.exe -L https://github.com/bazelbuild/bazel/releases/download/2.0.0/bazel-2.0.0-windows-x86_64.exe -o C:/bazel/bazel.exe --retry 10
           echo Installing CUDA
           curl.exe -L http://developer.download.nvidia.com/compute/cuda/10.1/Prod/local_installers/cuda_10.1.243_426.00_windows.exe -o cuda.exe
           curl.exe -L https://developer.download.nvidia.com/compute/redist/cudnn/v7.6.4/cudnn-10.1-windows7-x64-v7.6.4.38.zip -o cudnn.zip

tensorflow-core/tensorflow-core-api/BUILD

@@ -1,4 +1,5 @@
 load("@org_tensorflow//tensorflow:tensorflow.bzl", "tf_copts", "tf_cc_binary")
+load("@rules_java//java:defs.bzl", "java_proto_library")
 load(":tensorflow-java.bzl", "tf_java_op_gen_srcjar")
 
 tf_java_op_gen_srcjar(
@@ -57,3 +58,28 @@ filegroup(
     name = "java_api_def",
     srcs = glob(["src/bazel/api_def/*"])
 )
+
+tf_cc_binary(
+    name = "java_api_import",
+    srcs = [
+        "src/bazel/api_def/import/api_import.cc",
+    ],
+    copts = tf_copts() + ["-Wno-unused-result"],
+    linkopts = select({
+        "@org_tensorflow//tensorflow:windows": [],
+        "//conditions:default": ["-lm"],
+    }),
+    linkstatic = 1,
+    deps = [
+        "@org_tensorflow//tensorflow/core:framework",
+        "@org_tensorflow//tensorflow/core:lib",
+        "@org_tensorflow//tensorflow/core:ops",
+        "@org_tensorflow//tensorflow/core:op_gen_lib",
+        "@org_tensorflow//tensorflow/tools/api/lib:api_objects_proto_cc"
+    ],
+)
+
+java_proto_library(
+    name = "java_proto_gen_sources",
+    deps = ["@org_tensorflow//tensorflow/core:protos_all"]
+)

tensorflow-core/tensorflow-core-api/WORKSPACE

@@ -1,26 +1,24 @@
 workspace(name = "tensorflow_core_api")
 
-#local_repository(
-#    name = "tensorflow",
-#    path = "/Users/klessard/Documents/Projects/MachineLearning/Sources/tensorflow",
-#)
-
 load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
 
+# TensorFlow archive
+# Note: Make sure to synchronize Maven dependencies inherited from TensorFlow binaries when updating
+# the version of this archive (e.g. google protobuf)
 http_archive(
     name = "org_tensorflow",
     patches = [
         ":tensorflow-windows.patch", # https://github.com/tensorflow/tensorflow/issues/25213
-        ":tensorflow-api-def.patch",
-        ":tensorflow-grpc-fix-for-gettid.patch" # https://github.com/clearlinux-pkgs/tensorflow/blob/master/Add-grpc-fix-for-gettid.patch
+        ":tensorflow-proto.patch",
+        ":tensorflow-api-def.patch"
     ],
     patch_args = ["-p1"],
+    patch_cmds = ["grep -rl 'java_package' tensorflow/core | xargs sed -i.bak 's/^\(.* java_package = \"org\.tensorflow\.\)\(.*\"\)/\\1proto.\\2'/"],
     urls = [
-       "https://mirror.bazel.build/github.com/tensorflow/tensorflow/archive/v2.1.0.tar.gz",
-       "https://github.com/tensorflow/tensorflow/archive/v2.1.0.tar.gz",
+       "https://github.com/tensorflow/tensorflow/archive/v2.2.0-rc2.tar.gz",
     ],
-    sha256 = "638e541a4981f52c69da4a311815f1e7989bf1d67a41d204511966e1daed14f7",
-    strip_prefix = "tensorflow-2.1.0"
+    sha256 = "3e0cffc98ad3767dd16a09b7f25a99961e21d65cfcad67ed3a9b1a01318c7e94",
+    strip_prefix = "tensorflow-2.2.0-rc2"
 )
 
 # START: Upstream TensorFlow dependencies
@@ -35,17 +33,13 @@ http_archive(
         "https://github.com/bazelbuild/rules_closure/archive/308b05b2419edb5c8ee0471b67a40403df940149.tar.gz",  # 2019-06-13
     ],
 )
-http_archive(
-    name = "bazel_skylib",
-    sha256 = "2ef429f5d7ce7111263289644d233707dba35e39696377ebab8b0bc701f7818e",
-    urls = ["https://github.com/bazelbuild/bazel-skylib/releases/download/0.8.0/bazel-skylib.0.8.0.tar.gz"],
-)
 # END: Upstream TensorFlow dependencies
-load("@org_tensorflow//tensorflow:workspace.bzl", "tf_repositories")
-tf_repositories()
 
-load("@io_bazel_rules_closure//closure:defs.bzl", "closure_repositories")
-closure_repositories()
+load("@org_tensorflow//tensorflow:workspace.bzl", "tf_workspace")
+tf_workspace()
+
+load("@com_github_grpc_grpc//bazel:grpc_deps.bzl", "grpc_deps")
+grpc_deps()
 
-load("@org_tensorflow//tensorflow:workspace.bzl", "tf_bind")
-tf_bind()
+load("@upb//bazel:repository_defs.bzl", "bazel_version_repository")
+bazel_version_repository(name = "bazel_version")

tensorflow-core/tensorflow-core-api/build.sh

@@ -30,28 +30,40 @@ if [[ "${EXTENSION:-}" == *gpu* ]]; then
 fi
 
 # Build C API of TensorFlow itself including a target to generate ops for Java
-bazel build $BUILD_FLAGS --python_path="$PYTHON_BIN_PATH" --config=monolithic --output_filter=DONT_MATCH_ANYTHING --verbose_failures \
-    @org_tensorflow//tensorflow:tensorflow @org_tensorflow//tensorflow/tools/lib_package:jnilicenses_generate :java_op_gen_sources
-ls -l bazel-bin/external/org_tensorflow/tensorflow/
+bazel build $BUILD_FLAGS --experimental_repo_remote_exec --python_path="$PYTHON_BIN_PATH" --config=monolithic --output_filter=DONT_MATCH_ANYTHING --verbose_failures \
+    @org_tensorflow//tensorflow:tensorflow \
+    @org_tensorflow//tensorflow/tools/lib_package:jnilicenses_generate \
+    :java_op_gen_sources \
+    :java_api_import \
+    :java_proto_gen_sources
+
+export BAZEL_BIN=$(pwd -P)/bazel-bin
+export TENSORFLOW_BIN=$BAZEL_BIN/external/org_tensorflow/tensorflow
 
 # Normalize some paths with symbolic links
-TENSORFLOW_SO=(bazel-bin/external/org_tensorflow/tensorflow/libtensorflow.so.?.?.?)
+TENSORFLOW_SO=($TENSORFLOW_BIN/libtensorflow.so.?.?.?)
 if [[ -f $TENSORFLOW_SO ]]; then
-    ln -sf $(basename $TENSORFLOW_SO) bazel-bin/external/org_tensorflow/tensorflow/libtensorflow.so
-    ln -sf $(basename $TENSORFLOW_SO) bazel-bin/external/org_tensorflow/tensorflow/libtensorflow.so.2
+    ln -sf $(basename $TENSORFLOW_SO) $TENSORFLOW_BIN/libtensorflow.so
+    ln -sf $(basename $TENSORFLOW_SO) $TENSORFLOW_BIN/libtensorflow.so.2
 fi
-TENSORFLOW_DYLIB=(bazel-bin/external/org_tensorflow/tensorflow/libtensorflow.?.?.?.dylib)
+TENSORFLOW_DYLIB=($TENSORFLOW_BIN/tensorflow/libtensorflow.?.?.?.dylib)
 if [[ -f $TENSORFLOW_DYLIB ]]; then
-    ln -sf $(basename $TENSORFLOW_DYLIB) bazel-bin/external/org_tensorflow/tensorflow/libtensorflow.dylib
-    ln -sf $(basename $TENSORFLOW_DYLIB) bazel-bin/external/org_tensorflow/tensorflow/libtensorflow.2.dylib
+    ln -sf $(basename $TENSORFLOW_DYLIB) $TENSORFLOW_BIN/libtensorflow.dylib
+    ln -sf $(basename $TENSORFLOW_DYLIB) $TENSORFLOW_BIN/libtensorflow.2.dylib
 fi
-TENSORFLOW_LIBS=(bazel-bin/external/org_tensorflow/tensorflow/tensorflow.dll.if.lib bazel-bin/external/org_tensorflow/tensorflow/libtensorflow.dll.ifso)
+TENSORFLOW_LIBS=($TENSORFLOW_BIN/tensorflow.dll.if.lib $TENSORFLOW_BIN/libtensorflow.dll.ifso)
 for TENSORFLOW_LIB in ${TENSORFLOW_LIBS[@]}; do
     if [[ -f $TENSORFLOW_LIB ]]; then
-        ln -sf $(basename $TENSORFLOW_LIB) bazel-bin/external/org_tensorflow/tensorflow/tensorflow.lib
+        ln -sf $(basename $TENSORFLOW_LIB) $TENSORFLOW_BIN/tensorflow.lib
     fi
 done
 
-# Copy only main generated Java source files for ops
 mkdir -p src/gen/java/
-cp -r bazel-genfiles/ops/src/* src/gen/java/
+cd src/gen/java
+
+# Copy only generated Java source files for ops
+cp -r $BAZEL_BIN/ops/src/* .
+
+# Copy generated Java protos from source jars
+find $TENSORFLOW_BIN/core -name \*-speed-src.jar -exec jar xf {} \;
+rm -rf META-INF

tensorflow-core/tensorflow-core-api/external/tensorflow-api-def.patch

@@ -1,7 +1,6 @@
-diff --git a/tensorflow/core/api_def/BUILD b/tensorflow/core/api_def/BUILD
-index f96645ad9f..63f94d3b87 100644
---- a/tensorflow/core/api_def/BUILD
-+++ b/tensorflow/core/api_def/BUILD
+diff --ruN a/tensorflow/core/api_def/BUILD b/tensorflow/core/api_def/BUILD
+--- a/tensorflow/core/api_def/BUILD 2020-03-26 18:19:19.000000000 -0400
++++ b/tensorflow/core/api_def/BUILD	2020-04-01 22:50:37.000000000 -0400
 @@ -28,7 +28,7 @@ package(
  filegroup(
      name = "base_api_def",
@@ -11,3 +10,14 @@ index f96645ad9f..63f94d3b87 100644
  )
 
  filegroup(
+diff -ruN a/tensorflow/tools/api/lib/BUILD b/tensorflow/tools/api/lib/BUILD
+--- a/tensorflow/tools/api/lib/BUILD	2020-03-26 18:19:19.000000000 -0400
++++ b/tensorflow/tools/api/lib/BUILD	2020-04-01 22:50:37.000000000 -0400
+@@ -13,6 +13,7 @@
+ tf_proto_library(
+     name = "api_objects_proto",
+     srcs = ["api_objects.proto"],
++    visibility = ["//visibility:public"],
+ )
+
+ py_library(