Building TF.Lite benchmark tool with TF.text ops support #50924
Comments
UsharaniPagadala
added
comp:lite
ymodak
added
stat:awaiting tensorflower
|
Hi Tian / Thai, Wondering if you have an example of using building TFLite with tf.text flex ops? Thanks, |
|
For the first attempt, you should modify If you want a smaller binary size, You might try to define a new custom library as: https://github.com/tensorflow/tflite-support/blob/master/tensorflow_lite_support/custom_ops/BUILD#L10 then use it to replace the |
|
Hey @thaink, @wangtz, thanks for the quick response! I hadn't seen the flex benchmark tool, was just adding the flex delegate to the base I've tried building that tool this morning, but I'm seeing an Additional change for `benchmark_model_plus_flex`
diff --git a/tensorflow/lite/tools/benchmark/BUILD b/tensorflow/lite/tools/benchmark/BUILD
index 0793db4a417..91d7b4dfa15 100644
--- a/tensorflow/lite/tools/benchmark/BUILD
+++ b/tensorflow/lite/tools/benchmark/BUILD
@@ -91,6 +91,7 @@ tf_cc_binary(
"//tensorflow/lite/delegates/flex:delegate",
"//tensorflow/lite/testing:init_tensorflow",
"//tensorflow/lite/tools:logging",
+ "@org_tensorflow_text//tensorflow_text:ops_lib",
],
)Build error
bazel build --config=monolithic -c opt tensorflow/lite/tools/benchmark:benchmark_model_plus_flex --verbose_failures Any other ideas on what may be wrong here? I can build a working benchmark tool, even with flex ops, so I think the external dependencies are available for use. It's just that they aren't able to be found when the target file is in the |
|
@broken Is possible to depend on |
|
Yes; this is what we do with model server. I have to admit that I'm inexperienced building tf text as flex ops, but the tflite_cc_shared_object in our build does and is working. From the error message, it looks like the build is using the normal TF deps rather than the mobile target & portable tflite deps. Would adding --config=android fix this? FYI; the tflite-support library has a few TF Lite version of TF Text ops which includes a mobile version of Sentencepiece. This will be more performant. We're currently working on getting these in the main tf text repo and expanding the offering of native tf lite/text ops. |
|
@broken @thaink Thanks for the quick feedback!! I've been continuing to work on this based on your suggestions. (I wasn't able to get it working yet with that tflite-support library, but it looks really cool -- will try again to use it in the future.) In case others are interested, I was able to finally get builds working by applying the diff I've included here. Diff
diff --git a/tensorflow/lite/c/BUILD b/tensorflow/lite/c/BUILD
index 3119cf30557..3b7ff060518 100644
--- a/tensorflow/lite/c/BUILD
+++ b/tensorflow/lite/c/BUILD
@@ -38,6 +38,8 @@ tflite_cc_shared_object(
":c_api_experimental",
":exported_symbols.lds",
":version_script.lds",
+ "//tensorflow/lite/delegates/flex:delegate",
+ "@org_tensorflow_text//tensorflow_text:ops_lib",
],
)
diff --git a/tensorflow/lite/tools/benchmark/BUILD b/tensorflow/lite/tools/benchmark/BUILD
index 815efb776e9..c100bad4223 100644
--- a/tensorflow/lite/tools/benchmark/BUILD
+++ b/tensorflow/lite/tools/benchmark/BUILD
@@ -88,6 +88,7 @@ tf_cc_binary(
"//tensorflow/lite/delegates/flex:delegate",
"//tensorflow/lite/testing:init_tensorflow",
"//tensorflow/lite/tools:logging",
+ "@org_tensorflow_text//tensorflow_text:ops_lib"
],
)
diff --git a/tensorflow/workspace0.bzl b/tensorflow/workspace0.bzl
index 22374bc1297..abf1eb96677 100644
--- a/tensorflow/workspace0.bzl
+++ b/tensorflow/workspace0.bzl
@@ -102,6 +102,38 @@ def workspace():
],
)
+ # TF.text repo
+ http_archive(
+ name = "org_tensorflow_text",
+ sha256 = "d82856dc04c04bbce347f72d0ad7df59bb6b26b7030f96f25f036cfe8a138312",
+ strip_prefix = "text-2.5.0",
+ urls = [
+ "https://github.com/tensorflow/text/archive/v2.5.0.zip",
+ ],
+ patches = ["@//third_party/tf_text:tf_text_delegate.patch"],
+ patch_args = ["-p1"],
+ )
+
+ # TF.text dependencies
+ http_archive(
+ name = "com_google_sentencepiece",
+ strip_prefix = "sentencepiece-1.0.0",
+ sha256 = "c05901f30a1d0ed64cbcf40eba08e48894e1b0e985777217b7c9036cac631346",
+ urls = [
+ "https://github.com/google/sentencepiece/archive/1.0.0.zip",
+ ],
+ )
+
+ http_archive(
+ name = "com_google_glog",
+ sha256 = "1ee310e5d0a19b9d584a855000434bb724aa744745d5b8ab1855c85bff8a8e21",
+ strip_prefix = "glog-028d37889a1e80e8a07da1b8945ac706259e5fd8",
+ urls = [
+ "https://mirror.bazel.build/github.com/google/glog/archive/028d37889a1e80e8a07da1b8945ac706259e5fd8.tar.gz",
+ "https://github.com/google/glog/archive/028d37889a1e80e8a07da1b8945ac706259e5fd8.tar.gz",
+ ],
+ )
+
bazel_toolchains_repositories()
# Use `swift_rules_dependencies` to fetch the toolchains. With the
diff --git a/third_party/icu/BUILD.bazel b/third_party/icu/BUILD.bazel
index 14cadffc841..691585ce5ba 100644
--- a/third_party/icu/BUILD.bazel
+++ b/third_party/icu/BUILD.bazel
@@ -19,6 +19,16 @@ cc_library(
],
)
+alias(
+ name = "nfkc",
+ actual = ":common",
+)
+
+alias(
+ name = "nfkc_cf",
+ actual = ":common",
+)
+
cc_library(
name = "common",
hdrs = glob(["icu4c/source/common/unicode/*.h"]),
diff --git a/third_party/tf_text/BUILD b/third_party/tf_text/BUILD
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/third_party/tf_text/tf_text_delegate.patch b/third_party/tf_text/tf_text_delegate.patch
new file mode 100644
index 00000000000..b0cba5225d5
--- /dev/null
+++ b/third_party/tf_text/tf_text_delegate.patch
@@ -0,0 +1,30 @@
+diff --git a/tensorflow_text/tftext.bzl b/tensorflow_text/tftext.bzl
+index 7f703b3..888bea4 100644
+--- a/tensorflow_text/tftext.bzl
++++ b/tensorflow_text/tftext.bzl
+@@ -93,8 +93,8 @@ def tf_deps(deps = []):
+ "@org_tensorflow//tensorflow/core:portable_tensorflow_lib_lite",
+ ],
+ "//conditions:default": [
+- "@local_config_tf//:libtensorflow_framework",
+- "@local_config_tf//:tf_header_lib",
++ "@org_tensorflow//tensorflow/core:framework",
++ "@org_tensorflow//tensorflow/core:lib",
+ ] + deps + oss_deps,
+ })
+
+diff --git a/third_party/sentencepiece/processor.patch b/third_party/sentencepiece/processor.patch
+index 5fa1b84..427c104 100644
+--- a/third_party/sentencepiece/processor.patch
++++ b/third_party/sentencepiece/processor.patch
+@@ -22,8 +22,8 @@ index b4298d2..7ce779f 100644
+ + "@org_tensorflow//tensorflow/core:tflite_portable_logging",
+ + ],
+ + "//conditions:default": [
+-+ "@local_config_tf//:libtensorflow_framework",
+-+ "@local_config_tf//:tf_header_lib",
+++ "@org_tensorflow//tensorflow/core:framework",
+++ "@org_tensorflow//tensorflow/core:lib",
+ + "@com_google_absl//absl/functional:function_ref",
+ + "@com_google_absl//absl/strings:cord",
+ + ],Here are also a few other error messages that I saw along the way but didn't post above, in case it helps someone else arrive here: Error message
ERROR: Op type not registered 'SentencepieceOp' in binary running on ip-10-22-3-51. Make sure the Op and Kernel are registered in the binary running in this process. Note that if you are loading a saved graph which used ops from tf.contrib, accessing (e.g.) tf.contrib.resampler should be done before importing the graph, as contrib ops are lazily registered when the module is first accessed. INFO: TfLiteFlexDelegate delegate: 0 nodes delegated out of 11 nodes with 0 partitions. INFO: TfLiteFlexDelegate delegate: 0 nodes delegated out of 3 nodes with 0 partitions. INFO: TfLiteFlexDelegate delegate: 0 nodes delegated out of 17 nodes with 0 partitions. ERROR: Delegate kernel was not initialized Error message
/home/ubuntu/.cache/bazel/_bazel_ubuntu/..../external/org_tensorflow_text/tensorflow_text/BUILD:1162:19: no such package '@local_config_tf//': The repository '@local_config_tf' could not be resolved and referenced by '@org_tensorflow_text//tensorflow_text:wordpiece_tokenizer_cc' Error message
ERROR: /home/ubuntu/src/tensorflow/tensorflow/lite/delegates/flex/BUILD:73:23: error loading package '@org_tensorflow_text//tensorflow_text': Every .bzl file must have a corresponding package, but '@org_tensorflow_text//tensorflow/lite:build_def.bzl' does not have one. Please create a BUILD file in the same or any parent directory. Note that this BUILD file does not need to do anything except exist. and referenced by '//tensorflow/lite/delegates/flex:delegate' Error message
ERROR: Analysis of target '//tensorflow_text:tensorflowlite_c' failed; build aborted: no such package 'tensorflow': BUILD file not found in any of the following directories. Add a BUILD file to a directory to mark it as a package. |
|
I'm glad you got it working, and thanks for such a detailed closing comment for the people of the future. |
|
@thisisjoshstone Thanks for such a detailed issue description and closure comment. I was not looking for tflite but just integrating TF.text ops in a shared library but this helped me along the way. Usually no one has this detailed issue description, I will be yanking your way of doing it going forward. |
Does anyone have an example of re-building a TF.Lite binary with support for tensorflow-text operations?
I have a TF.Lite model which includes USE-multilingual, which uses several Sentencepiece ops from tensorflow-text. After reading tensorflow/hub#463 and a few others, I was able to successfully covert the model, and it returns results during inference as expected when using the TF.Lite Interpreter.
I'd like to run the TF.Lite native benchmark tool against this model as well, but doing so requires building the tool from source with tensorflow-text ops. Thus far, I haven't been able to build a working version of the tool for my model. A few things I've tried are listed below, any help would be much appreciated. Thanks!
Attempt 1: Include tensorflow-text as an external repository and target tensorflow_text:ops_lib
My first attempt was to build
tensorflow_text:ops_libdirectly into the tool from source, which seems to be the more Bazel-appropriate way to tackle the problem. The diff below shows my most recent attempt to build the benchmark tool in this manner.Diff
This build fails with a number of different dependency path/access errors, depending on Bazel build order. Typically, it either can't find an
Eigenheader or aProtobufheader when starting from tensorflow-text source (this is what led me to attempt changing some of the rule visibilities, but with no luck). I've included a couple of these error messages for reference.Example 1
ERROR: /home/ubuntu/.cache/bazel/bazel_ubuntu/b813f517b143a3c1665dd902035fe00f/external/org_tensorflow_text/tensorflow_text/core/kernels/BUILD:355:23: C++ compilation of rule '@org_tensorflow_text//tensorflow_text/core/kernels:sentencepiece_kernels' failed (Exit 1): gcc failed: error executing command
(cd /home/ubuntu/.cache/bazel/bazel_ubuntu/b813f517b143a3c1665dd902035fe00f/execroot/org_tensorflow &&
exec env -
CUDA_TOOLKIT_PATH=/usr/local/cuda-10.0
LD_LIBRARY_PATH=/usr/local/cuda/lib:/usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64:/opt/amazon/efa/lib:/opt/amazon/openmpi/lib:/usr/local/lib:/usr/lib:
PATH=/home/ubuntu/.cache/bazelisk/downloads/bazelbuild/bazel-3.7.2-linux-x86_64/bin:/usr/local/cuda:/home/ubuntu/src/tensorflow/.venv/bin:/opt/amazon/openmpi/bin/:/opt/amazon/efa/bin/:/home/ubuntu/anaconda3/condabin:/home/ubuntu/.dl_binaries/bin:/usr/local/cuda/bin:/opt/aws/neuron/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin
PWD=/proc/self/cwd
PYTHON_BIN_PATH=/home/ubuntu/src/tensorflow/.venv/bin/python3
PYTHON_LIB_PATH=/home/ubuntu/src/tensorflow/.venv/lib/python3.6/site-packages
TF2_BEHAVIOR=1
/usr/bin/gcc -U_FORTIFY_SOURCE -fstack-protector -Wall -Wunused-but-set-parameter -Wno-free-nonheap-object -fno-omit-frame-pointer -g0 -O2 '-D_FORTIFY_SOURCE=1' -DNDEBUG -ffunction-sections -fdata-sections '-std=c++0x' -MD -MF bazel-out/k8-opt/bin/external/org_tensorflow_text/tensorflow_text/core/kernels/objs/sentencepiece_kernels/sentencepiece_kernels.d '-frandom-seed=bazel-out/k8-opt/bin/external/org_tensorflow_text/tensorflow_text/core/kernels/objs/sentencepiece_kernels/sentencepiece_kernels.o' -iquote external/org_tensorflow_text -iquote bazel-out/k8-opt/bin/external/org_tensorflow_text -iquote external/com_google_absl -iquote bazel-out/k8-opt/bin/external/com_google_absl -iquote external/com_google_sentencepiece -iquote bazel-out/k8-opt/bin/external/com_google_sentencepiece -iquote external/com_google_protobuf -iquote bazel-out/k8-opt/bin/external/com_google_protobuf -iquote external/zlib -iquote bazel-out/k8-opt/bin/external/zlib -iquote external/com_github_gflags_gflags -iquote bazel-out/k8-opt/bin/external/com_github_gflags_gflags -iquote external/com_google_glog -iquote bazel-out/k8-opt/bin/external/com_google_glog -iquote external/com_google_googletest -iquote bazel-out/k8-opt/bin/external/com_google_googletest -iquote . -iquote bazel-out/k8-opt/bin -Ibazel-out/k8-opt/bin/external/com_google_glog/virtual_includes/glog -isystem external/com_google_protobuf/src -isystem bazel-out/k8-opt/bin/external/com_google_protobuf/src -isystem external/zlib -isystem bazel-out/k8-opt/bin/external/zlib -isystem external/com_github_gflags_gflags/include -isystem bazel-out/k8-opt/bin/external/com_github_gflags_gflags/include -isystem external/com_google_googletest/googlemock -isystem bazel-out/k8-opt/bin/external/com_google_googletest/googlemock -isystem external/com_google_googletest/googlemock/include -isystem bazel-out/k8-opt/bin/external/com_google_googletest/googlemock/include -isystem external/com_google_googletest/googletest -isystem bazel-out/k8-opt/bin/external/com_google_googletest/googletest -isystem external/com_google_googletest/googletest/include -isystem bazel-out/k8-opt/bin/external/com_google_googletest/googletest/include -w -DAUTOLOAD_DYNAMIC_KERNELS '-std=c++14' -fno-canonical-system-headers -Wno-builtin-macro-redefined '-D__DATE="redacted"' '-D__TIMESTAMP="redacted"' '-D__TIME_="redacted"' -c external/org_tensorflow_text/tensorflow_text/core/kernels/sentencepiece_kernels.cc -o bazel-out/k8-opt/bin/external/org_tensorflow_text/tensorflow_text/core/kernels/_objs/sentencepiece_kernels/sentencepiece_kernels.o)
Execution platform: @local_execution_config_platform//:platform
In file included from ./tensorflow/core/framework/bounds_check.h:21,
from external/org_tensorflow_text/tensorflow_text/core/kernels/sentencepiece_kernels.cc:25:
./third_party/eigen3/Eigen/Core:1:10: fatal error: Eigen/Core: No such file or directory
#include "Eigen/Core"
^~~~~~~~~~~~
compilation terminated.
Example 2
ERROR: /home/ubuntu/.cache/bazel/bazel_ubuntu/b813f517b143a3c1665dd902035fe00f/external/org_tensorflow_text/tensorflow_text/BUILD:1162:19: C++ compilation of rule '@org_tensorflow_text//tensorflow_text:wordpiece_tokenizer_cc' failed (Exit 1): gcc failed: error executing command
(cd /home/ubuntu/.cache/bazel/bazel_ubuntu/b813f517b143a3c1665dd902035fe00f/execroot/org_tensorflow &&
exec env -
LD_LIBRARY_PATH=/usr/local/cuda/lib:/usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64:/opt/amazon/efa/lib:/opt/amazon/openmpi/lib:/usr/local/lib:/usr/lib:
PATH=/home/ubuntu/.cache/bazelisk/downloads/bazelbuild/bazel-3.7.2-linux-x86_64/bin:/home/ubuntu/src/tensorflow/.venv/bin:/opt/amazon/openmpi/bin/:/opt/amazon/efa/bin/:/home/ubuntu/anaconda3/condabin:/home/ubuntu/.dl_binaries/bin:/usr/local/cuda/bin:/opt/aws/neuron/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin
PWD=/proc/self/cwd
PYTHON_BIN_PATH=/home/ubuntu/src/tensorflow/.venv/bin/python
PYTHON_LIB_PATH=/home/ubuntu/src/tensorflow/.venv/lib/python3.6/site-packages
TF2_BEHAVIOR=1
/usr/bin/gcc -U_FORTIFY_SOURCE -fstack-protector -Wall -Wunused-but-set-parameter -Wno-free-nonheap-object -fno-omit-frame-pointer -g0 -O2 '-D_FORTIFY_SOURCE=1' -DNDEBUG -ffunction-sections -fdata-sections '-std=c++0x' -MD -MF bazel-out/k8-opt/bin/external/org_tensorflow_text/tensorflow_text/objs/wordpiece_tokenizer_cc/wordpiece_op.pic.d '-frandom-seed=bazel-out/k8-opt/bin/external/org_tensorflow_text/tensorflow_text/objs/wordpiece_tokenizer_cc/wordpiece_op.pic.o' -fPIC -iquote external/org_tensorflow_text -iquote bazel-out/k8-opt/bin/external/org_tensorflow_text -iquote external/icu -iquote bazel-out/k8-opt/bin/external/icu -iquote . -iquote bazel-out/k8-opt/bin -iquote external/com_google_absl -iquote bazel-out/k8-opt/bin/external/com_google_absl -isystem external/icu/icu4c/source/common -isystem bazel-out/k8-opt/bin/external/icu/icu4c/source/common -w -DAUTOLOAD_DYNAMIC_KERNELS '-std=c++14' -pthread -fno-canonical-system-headers -Wno-builtin-macro-redefined '-D__DATE="redacted"' '-D__TIMESTAMP="redacted"' '-D__TIME__="redacted"' -c external/org_tensorflow_text/tensorflow_text/core/ops/wordpiece_op.cc -o bazel-out/k8-opt/bin/external/org_tensorflow_text/tensorflow_text/_objs/wordpiece_tokenizer_cc/wordpiece_op.pic.o)
Execution platform: @local_execution_config_platform//:platform
In file included from ./tensorflow/core/framework/op_def_builder.h:24,
from ./tensorflow/core/framework/op.h:23,
from external/org_tensorflow_text/tensorflow_text/core/ops/wordpiece_op.cc:15:
bazel-out/k8-opt/bin/tensorflow/core/framework/op_def.pb.h:10:10: fatal error: google/protobuf/port_def.inc: No such file or directory
#include <google/protobuf/port_def.inc>
^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
compilation terminated.
Attempt 2: Compile tensorflow-text ops separately and link library
I was able to compile tensorflow-text with
linkstatic=1andlinkshared=True, hoping this would give me the best chance at a usable library (based on thelinkshareddescription in the Bazelcc_binarydocs). However, when I try to link this intensorflow/lite/tools/benchmark:benchmark_model, the build completes, but gives the following runtime error:When I try to link that library as well (which doesn't make much sense to include in a separate TF build, admittedly), I get the following runtime crash:
System information
The text was updated successfully, but these errors were encountered: