Skip to content

Commit c933215

Browse files
fengwuyaocopybara-github
fengwuyao
authored and
copybara-github
committed
Add script to deploy and run semantic similarity sample on Android.
LiteRT-PiperOrigin-RevId: 820457336
1 parent 39b5e32 commit c933215

File tree

10 files changed

+302
-57
lines changed

10 files changed

+302
-57
lines changed

‎BUILD‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
# Copyright 2025 The Google AI Edge Authors. All Rights Reserved.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.

‎BUILD.darts_clone‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
# Copyright 2025 The Google AI Edge Authors. All Rights Reserved.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
cc_library(
16+
name = "darts_clone",
17+
hdrs = ["include/darts.h"],
18+
visibility = ["//visibility:public"],
19+
)

‎BUILD.sentencepiece‎

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
# Copyright 2025 The Google AI Edge Authors. All Rights Reserved.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
package(default_visibility = ["//visibility:public"])
16+
17+
proto_library(
18+
name = "sentencepiece_proto",
19+
srcs = ["sentencepiece.proto"],
20+
)
21+
22+
cc_proto_library(
23+
name = "sentencepiece_cc_proto",
24+
deps = [":sentencepiece_proto"],
25+
)
26+
27+
proto_library(
28+
name = "sentencepiece_model_proto",
29+
srcs = ["sentencepiece_model.proto"],
30+
)
31+
32+
cc_proto_library(
33+
name = "sentencepiece_model_cc_proto",
34+
deps = [":sentencepiece_model_proto"],
35+
)
36+
37+
cc_library(
38+
name = "sentencepiece_processor",
39+
srcs = [
40+
"bpe_model.cc",
41+
"char_model.cc",
42+
"filesystem.cc",
43+
"model_factory.cc",
44+
"model_interface.cc",
45+
"normalizer.cc",
46+
"sentencepiece_processor.cc",
47+
"unigram_model.cc",
48+
"util.cc",
49+
"word_model.cc",
50+
],
51+
hdrs = [
52+
"bpe_model.h",
53+
"char_model.h",
54+
"common.h",
55+
"config.h",
56+
"filesystem.h",
57+
"freelist.h",
58+
"model_factory.h",
59+
"model_interface.h",
60+
"normalizer.h",
61+
"sentencepiece_processor.h",
62+
"sentencepiece_trainer.h",
63+
"trainer_interface.h",
64+
"unigram_model.h",
65+
"util.h",
66+
"word_model.h",
67+
],
68+
copts = [
69+
"-DENABLE_NFKC_COMPILE",
70+
"-DSENTENCEPIECE_PG3_BUILD",
71+
],
72+
deps = [
73+
":sentencepiece_cc_proto",
74+
":sentencepiece_model_cc_proto",
75+
"@com_google_absl//absl/base:core_headers",
76+
"@com_google_absl//absl/cleanup",
77+
"@com_google_absl//absl/container:flat_hash_map",
78+
"@com_google_absl//absl/container:flat_hash_set",
79+
"@com_google_absl//absl/log",
80+
"@com_google_absl//absl/log:check",
81+
"@com_google_absl//absl/memory",
82+
"@com_google_absl//absl/status",
83+
"@com_google_absl//absl/status:statusor",
84+
"@com_google_absl//absl/strings",
85+
"@com_google_absl//absl/strings:str_format",
86+
"@darts_clone",
87+
],
88+
)

‎PATCH.sentencepiece‎

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
# Copyright 2025 The Google AI Edge Authors. All Rights Reserved.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
--- sentencepiece_processor.h 2025-04-27 15:47:21.057722872 -0700
16+
+++ sentencepiece_processor.h 2025-04-27 15:50:12.601047364 -0700
17+
@@ -22,6 +22,8 @@
18+
#include <utility>
19+
#include <vector>
20+
21+
+#include "absl/status/status.h"
22+
+
23+
#ifndef SWIG
24+
namespace absl {
25+
using std::string_view;
26+
@@ -30,50 +32,8 @@
27+
28+
namespace sentencepiece {
29+
namespace util {
30+
-
31+
-enum class StatusCode : int {
32+
- kOk = 0,
33+
- kCancelled = 1,
34+
- kUnknown = 2,
35+
- kInvalidArgument = 3,
36+
- kDeadlineExceeded = 4,
37+
- kNotFound = 5,
38+
- kAlreadyExists = 6,
39+
- kPermissionDenied = 7,
40+
- kResourceExhausted = 8,
41+
- kFailedPrecondition = 9,
42+
- kAborted = 10,
43+
- kOutOfRange = 11,
44+
- kUnimplemented = 12,
45+
- kInternal = 13,
46+
- kUnavailable = 14,
47+
- kDataLoss = 15,
48+
- kUnauthenticated = 16,
49+
-};
50+
-
51+
-class Status {
52+
- public:
53+
- Status();
54+
- ~Status();
55+
- Status(StatusCode code, absl::string_view error_message);
56+
- Status(const Status &s);
57+
- void operator=(const Status &s);
58+
- bool operator==(const Status &s) const;
59+
- bool operator!=(const Status &s) const;
60+
- inline bool ok() const { return rep_ == nullptr; }
61+
-
62+
- void set_error_message(const char *str);
63+
- const char *error_message() const;
64+
- const char *message() const { return error_message(); }
65+
- StatusCode code() const;
66+
- std::string ToString() const;
67+
-
68+
- void IgnoreError();
69+
-
70+
- private:
71+
- struct Rep;
72+
- std::unique_ptr<Rep> rep_;
73+
-};
74+
+using StatusCode = absl::StatusCode;
75+
+using Status = absl::Status;
76+
} // namespace util
77+
78+
// SentencePieceProcessor:
79+
80+
--- common.h 2025-04-27 15:55:22.642953123 -0700
81+
+++ common.h 2025-04-27 16:02:22.377904469 -0700
82+
@@ -94,7 +94,7 @@
83+
~Die() {
84+
std::cerr << std::endl;
85+
if (die_) {
86+
- Abort();
87+
+ exit(-1);
88+
}
89+
}
90+
int operator&(std::ostream &) { return 0; }

‎WORKSPACE‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -212,6 +212,33 @@ http_archive(
212212
url = "https://github.com/bazelbuild/rules_kotlin/releases/download/v2.1.3/rules_kotlin-v2.1.3.tar.gz",
213213
)
214214

215+
# Sentencepiece
216+
http_archive(
217+
name = "sentencepiece",
218+
build_file = "@//:BUILD.sentencepiece",
219+
patch_cmds = [
220+
# Empty config.h seems enough.
221+
"touch config.h",
222+
# Replace third_party/absl/ with absl/ in *.h and *.cc files.
223+
"sed -i -e 's|#include \"third_party/absl/|#include \"absl/|g' *.h *.cc",
224+
# Replace third_party/darts_clone/ with include/ in *.h and *.cc files.
225+
"sed -i -e 's|#include \"third_party/darts_clone/|#include \"include/|g' *.h *.cc",
226+
],
227+
patches = ["@//:PATCH.sentencepiece"],
228+
sha256 = "9970f0a0afee1648890293321665e5b2efa04eaec9f1671fcf8048f456f5bb86",
229+
strip_prefix = "sentencepiece-0.2.0/src",
230+
url = "https://github.com/google/sentencepiece/archive/refs/tags/v0.2.0.tar.gz",
231+
)
232+
233+
# Darts Clone
234+
http_archive(
235+
name = "darts_clone",
236+
build_file = "@//:BUILD.darts_clone",
237+
sha256 = "4a562824ec2fbb0ef7bd0058d9f73300173d20757b33bb69baa7e50349f65820",
238+
strip_prefix = "darts-clone-e40ce4627526985a7767444b6ed6893ab6ff8983",
239+
url = "https://github.com/s-yata/darts-clone/archive/e40ce4627526985a7767444b6ed6893ab6ff8983.tar.gz",
240+
)
241+
215242
load("@rules_kotlin//kotlin:repositories.bzl", "kotlin_repositories")
216243

217244
kotlin_repositories() # if you want the default. Otherwise see custom kotlinc distribution below

‎litert/opensource_only.files‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,3 +9,4 @@ litert/compiler/BUILD:
99
litert/python/aot/core/tflxx_util.py:
1010
litert/samples/async_segmentation/README.md:
1111
litert/samples/semantic_similarity/README.md:
12+
litert/samples/semantic_similarity/deploy_and_run_android.sh:

‎litert/samples/semantic_similarity/BUILD‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -23,13 +23,13 @@ cc_binary(
2323
name = "tokenizer_client",
2424
srcs = ["tokenizer_client.cc"],
2525
deps = [
26-
"//base",
26+
# copybara:uncomment "//base",
2727
"@com_google_absl//absl/flags:flag",
2828
"@com_google_absl//absl/flags:parse",
2929
"@com_google_absl//absl/log:absl_log",
3030
"@com_google_absl//absl/status",
3131
"@com_google_absl//absl/strings",
32-
"@com_google_sentencepiece//:sentencepiece_processor",
32+
"@sentencepiece//:sentencepiece_processor",
3333
],
3434
)
3535

@@ -38,9 +38,7 @@ cc_binary(
3838
srcs = ["main.cc"],
3939
# copybara:uncomment data = ["//litert/runtime/accelerators/gpu:ml_drift_cl_accelerator_so"],
4040
deps = [
41-
"//base",
42-
"//litert/cc:litert_api_with_dynamic_runtime",
43-
"//litert/cc/options:litert_gpu_options",
41+
# copybara:uncomment "//base",
4442
"@com_google_absl//absl/flags:flag",
4543
"@com_google_absl//absl/flags:parse",
4644
"@com_google_absl//absl/log",
@@ -51,20 +49,22 @@ cc_binary(
5149
"@com_google_absl//absl/status:statusor",
5250
"@com_google_absl//absl/strings",
5351
"@com_google_absl//absl/types:span",
54-
"@com_google_sentencepiece//:sentencepiece_processor",
52+
"//litert/cc:litert_api_with_dynamic_runtime",
53+
"//litert/cc/options:litert_gpu_options",
54+
"@sentencepiece//:sentencepiece_processor",
5555
],
5656
)
5757

5858
cc_binary(
5959
name = "embedding_client",
6060
srcs = ["embedding_client.cc"],
6161
deps = [
62-
"//base",
63-
"//litert/cc:litert_api_with_dynamic_runtime",
62+
# copybara:uncomment "//base",
6463
"@com_google_absl//absl/flags:flag",
6564
"@com_google_absl//absl/flags:parse",
6665
"@com_google_absl//absl/log:absl_log",
6766
"@com_google_absl//absl/strings",
6867
"@com_google_absl//absl/types:span",
68+
"//litert/cc:litert_api_with_dynamic_runtime",
6969
],
7070
)

0 commit comments

Comments
 (0)