Rename third_party folder to Avoid Copybara g3 Errors (#148)

* rename third_party

* fix ut

* fix ut

Author: jyj0w0

Makefile

@@ -51,4 +51,4 @@ unit-tests:
 	coverage run -m unittest -v
 
 check-test-coverage:
-	coverage report -m --omit="jetstream/core/proto/*,jetstream/engine/tokenizer_pb2.py,jetstream/third_party/*" --fail-under=96
+	coverage report -m --omit="jetstream/core/proto/*,jetstream/engine/tokenizer_pb2.py,jetstream/external_tokenizers/*" --fail-under=96

benchmarks/benchmark_serving.py

@@ -73,7 +73,7 @@
 from jetstream.core.proto import jetstream_pb2
 from jetstream.core.proto import jetstream_pb2_grpc
 from jetstream.engine.token_utils import load_vocab
-from jetstream.third_party.llama3 import llama3_tokenizer
+from jetstream.external_tokenizers.llama3 import llama3_tokenizer
 import numpy as np
 from tqdm.asyncio import tqdm  # pytype: disable=pyi-error
 import pandas

jetstream/engine/token_utils.py

@@ -28,7 +28,7 @@
 from jetstream.engine import mock_utils
 from jetstream.engine import tokenizer_api
 from jetstream.engine import tokenizer_pb2
-from jetstream.third_party.llama3 import llama3_tokenizer
+from jetstream.external_tokenizers.llama3 import llama3_tokenizer
 
 # ResultToken class to store tokens ids.
 ResultTokens = Any

jetstream/third_party/__init__.py jetstream/external_tokenizers/__init__.pyjetstream/third_party/__init__.py renamed to jetstream/external_tokenizers/__init__.py

@@ -55,7 +55,7 @@ def decode(self, t: int) -> str:
 class TokenUtilsTest(unittest.TestCase):
 
   def setup_sentencepiece(self):
-    self.tokenizer_path = "third_party/llama2/tokenizer.model"
+    self.tokenizer_path = "external_tokenizers/llama2/tokenizer.model"
     current_dir = os.path.dirname(__file__)
     self.tokenizer_path = os.path.join(current_dir, self.tokenizer_path)
     print(f"model_path: {self.tokenizer_path}")
@@ -66,7 +66,7 @@ def setup_sentencepiece(self):
     self.jt_tokenizer = JetStreamTokenizer(self.tokenizer_path)
 
   def setup_tiktoken(self):
-    self.tokenizer_path = "third_party/llama3/tokenizer.model"
+    self.tokenizer_path = "external_tokenizers/llama3/tokenizer.model"
     current_dir = os.path.dirname(__file__)
     self.tokenizer_path = os.path.join(current_dir, self.tokenizer_path)
     print(f"model_path: {self.tokenizer_path}")

jetstream/third_party/llama3/__init__.py …m/external_tokenizers/llama3/__init__.pyjetstream/third_party/llama3/__init__.py renamed to jetstream/external_tokenizers/llama3/__init__.py jetstream/third_party/llama3/__init__.py renamed to jetstream/external_tokenizers/llama3/__init__.py

@@ -9,7 +9,7 @@
 [MAIN]
 
 # Files or directories to be skipped. They should be base names, not paths.
-ignore=third_party
+ignore=external_tokenizers
 
 # Files or directories matching the regex patterns are skipped. The regex
 # matches against base names, not paths.