Revert "Two new classes:"

This reverts commit 44af5a4.

Author: broken

WORKSPACE

@@ -94,20 +94,6 @@ http_archive(
     ],
 )
 
-# NOTE: according to
-# https://docs.bazel.build/versions/master/external.html#transitive-dependencies
-# we should list the transitive dependencies of @org_tensorflow_hub in this
-# WORKSPACE file.  Still, all of them are already listed by tf_workspace() which
-# is called later in this file.
-http_archive(
-    name = "org_tensorflow_hub",
-    strip_prefix = "hub-0.8.0",
-    sha256 = "968af30c448d51c36501b68df2c916fb4a61007db3240adc9248fa3a9be2da6f",
-    urls = [
-        "https://github.com/tensorflow/hub/archive/v0.8.0.zip"
-    ],
-)
-
 http_archive(
     name = "org_tensorflow",
     strip_prefix = "tensorflow-2.2.0",

oss_scripts/model_server/save_models.py

@@ -115,30 +115,10 @@ def __call__(self, x):
     (sentencepiece, _, _) = sp_tokenizer.tokenize_with_offsets(sentencepiece)
     sentencepiece_size = sp_tokenizer.vocab_size()
     sentencepiece_id = sp_tokenizer.id_to_string(1)
-    # Split merge tokenizer
+    # Split merge tokenizer - not in this version
     sm_tokenizer = text.SplitMergeTokenizer()
     split_merge = sm_tokenizer.tokenize(b'IloveFlume!',
                                         [0, 0, 1, 1, 1, 0, 1, 1, 1, 1, 0])
-    # Split merge from logits tokenizer
-    smfl_tokenizer = text.SplitMergeFromLogitsTokenizer()
-    split_merge_from_logits = smfl_tokenizer.tokenize(
-        b'IloveFlume!',
-        # One pair of logits for each Unicode character from the text.  Each
-        # pair indicates a "split" action if the first component is greater than
-        # the second one, and a "merge" otherwise.
-        [
-            [2.7, -0.3],  # I: split
-            [4.1, 0.82],  # l: split
-            [-2.3, 4.3],  # o: merge
-            [3.1, 12.2],  # v: merge
-            [-3.0, 4.7],  # e: merge
-            [2.7, -0.7],  # F: split
-            [0.7, 15.0],  # l: merge
-            [1.6, 23.0],  # u: merge
-            [2.1, 11.0],  # m: merge
-            [0.0, 20.0],  # e: merge
-            [18.0, 0.7],  # !: split
-        ])
     # Confirm TF unicode_script op that requires ICU works
     tf_unicode_script = tf.strings.unicode_script(
         [ord('a'), 0x0411, 0x82b8, ord(',')])
@@ -174,7 +154,6 @@ def assert_check(tensor):
     sentencepiece_id_assert = assert_check(sentencepiece_id)
     sentencepiece_size_assert = assert_check(sentencepiece_size)
     split_merge_assert = assert_check(split_merge)
-    split_merge_from_logits_assert = assert_check(split_merge_from_logits)
     tf_unicode_script_assert = assert_check(tf_unicode_script)
     unicode_script_assert = assert_check(unicode_script.to_tensor())
     whitespace_assert = assert_check(whitespace.to_tensor())
@@ -191,7 +170,6 @@ def assert_check(tensor):
                                   sentencepiece_id_assert,
                                   sentencepiece_size_assert,
                                   split_merge_assert,
-                                  split_merge_from_logits_assert,
                                   tf_unicode_script_assert,
                                   unicode_script_assert,
                                   whitespace_assert,
@@ -224,3 +202,4 @@ def assert_check(tensor):
         continue
       os.remove(dst_file)
     shutil.move(src_file, dst_dir)
+

tensorflow_text/BUILD

@@ -32,7 +32,6 @@ cc_library(
         ":regex_split_ops_cc",
         ":sentence_breaking_ops_cc",
         ":sentencepiece_tokenizer_cc",
-        ":split_merge_from_logits_tokenizer_cc",
         ":split_merge_tokenizer_cc",
         ":text_similarity_metric_ops_cc",
         ":unicode_script_tokenizer_cc",
@@ -57,7 +56,6 @@ py_library(
         ":bert_tokenizer",
         ":create_feature_bitmask_op",
         ":greedy_constrained_sequence_op",
-        ":hub_module_tokenizer",
         ":mst_ops",
         ":ngrams_op",
         ":normalize_ops",
@@ -67,7 +65,6 @@ py_library(
         ":sentence_breaking_ops",
         ":sentencepiece_tokenizer",
         ":sliding_window_op",
-        ":split_merge_from_logits_tokenizer",
         ":split_merge_tokenizer",
         ":string_ops",
         ":text_similarity_metric_ops",
@@ -700,65 +697,6 @@ py_test(
     ],
 )
 
-py_library(
-    name = "hub_module_tokenizer",
-    srcs = ["python/ops/hub_module_tokenizer.py"],
-    deps = [
-        ":tokenization",
-        "@org_tensorflow_hub//tensorflow_hub",
-        # python:array_ops tensorflow dep,
-        # python/ops/ragged:ragged_tensor tensorflow dep,
-    ],
-)
-
-py_test(
-    name = "hub_module_tokenizer_test",
-    size = "large",
-    srcs = ["python/ops/hub_module_tokenizer_test.py"],
-    data = [
-        ":python/ops/test_data/segmenter_hub_module",
-    ],
-    srcs_version = "PY2AND3",
-    deps = [
-        ":ops",
-        # python:client_testlib tensorflow dep,
-        # python:framework_ops tensorflow dep,
-        # python:framework_test_lib tensorflow dep,
-        # python:lookup_ops tensorflow dep,
-        # python:variables tensorflow dep,
-        # python/ops/ragged:ragged_factory_ops tensorflow dep,
-    ],
-)
-
-py_tf_text_library(
-    name = "split_merge_from_logits_tokenizer",
-    srcs = ["python/ops/split_merge_from_logits_tokenizer.py"],
-    cc_op_defs = ["core/ops/tokenizer_from_logits_op.cc"],
-    cc_op_kernels = [
-        "//tensorflow_text/core/kernels:tokenizer_from_logits_kernel",
-    ],
-    deps = [
-        ":tokenization",
-        # python:dtypes tensorflow dep,
-        # python:framework_ops tensorflow dep,
-        # python/ops/ragged tensorflow dep,
-    ],
-)
-
-py_test(
-    name = "split_merge_from_logits_tokenizer_test",
-    size = "small",
-    srcs = ["python/ops/split_merge_from_logits_tokenizer_test.py"],
-    srcs_version = "PY2AND3",
-    deps = [
-        ":split_merge_from_logits_tokenizer",
-        # python:client_testlib tensorflow dep,
-        # python:errors tensorflow dep,
-        # python:framework_test_lib tensorflow dep,
-        # python/ops/ragged:ragged_factory_ops tensorflow dep,
-    ],
-)
-
 py_library(
     name = "unicode_char_tokenizer",
     srcs = ["python/ops/unicode_char_tokenizer.py"],

tensorflow_text/__init__.py

@@ -51,9 +51,7 @@
     "UnicodeScriptTokenizer",
     "viterbi_constrained_sequence",
     "WhitespaceTokenizer",
-    "HubModuleTokenizer",
     "SplitMergeTokenizer",
-    "SplitMergeFromLogitsTokenizer",
     "wordshape",
     "WordShape",
     "WordpieceTokenizer",

tensorflow_text/core/kernels/BUILD

@@ -433,18 +433,6 @@ tf_kernel_library(
     ],
 )
 
-tf_kernel_library(
-    name = "tokenizer_from_logits_kernel",
-    srcs = ["tokenizer_from_logits_kernel.cc"],
-    deps = OSS_DEPS + [
-        "@icu//:common",
-        # absl/strings dep
-        "@com_google_absl//absl/base:core_headers",
-        # tf:framework tensorflow dep,
-        # tf:lib tensorflow dep,
-    ],
-)
-
 cc_library(
     name = "wordpiece_tokenizer",
     srcs = ["wordpiece_tokenizer.cc"],