c10::string_view -> std::string_view in

Summary: Public message about pytorch/text/torchtext/csrc/vocab.h here

Reviewed By: ezyang

Differential Revision: D65543740

fbshipit-source-id: a504a36197b3f4b6cbf5840313bfcc2011347a5c

Author: r-barnes

torchtext/csrc/vocab.cpp

@@ -12,7 +12,7 @@ Vocab::Vocab(StringList tokens, const std::optional<int64_t>& default_index)
     : stoi_(MAX_VOCAB_SIZE, -1), default_index_{default_index} {
   for (auto& token : tokens) {
     // throw error if duplicate token is found
-    auto id = _find(c10::string_view{token});
+    auto id = _find(std::string_view{token});
     TORCH_CHECK(
         stoi_[id] == -1, "Duplicate token found in tokens list: " + token);
 
@@ -26,15 +26,15 @@ int64_t Vocab::__len__() const {
   return itos_.size();
 }
 
-bool Vocab::__contains__(const c10::string_view& token) const {
+bool Vocab::__contains__(const std::string_view& token) const {
   int64_t id = _find(token);
   if (stoi_[id] != -1) {
     return true;
   }
   return false;
 }
 
-int64_t Vocab::__getitem__(const c10::string_view& token) const {
+int64_t Vocab::__getitem__(const std::string_view& token) const {
   int64_t id = _find(token);
   if (stoi_[id] != -1)
     return stoi_[id];
@@ -59,7 +59,7 @@ std::optional<int64_t> Vocab::get_default_index() const {
 
 void Vocab::append_token(std::string token) {
   // throw error if token already exist in vocab
-  auto id = _find(c10::string_view{token});
+  auto id = _find(std::string_view{token});
   TORCH_CHECK(
       stoi_[id] == -1,
       "Token " + token + " already exists in the Vocab with index: " +
@@ -80,10 +80,10 @@ void Vocab::insert_token(std::string token, const int64_t& index) {
 
   // need to offset all tokens greater than or equal index by 1
   for (size_t i = index; i < __len__(); i++) {
-    stoi_[_find(c10::string_view{itos_[i]})] = i + 1;
+    stoi_[_find(std::string_view{itos_[i]})] = i + 1;
   }
 
-  stoi_[_find(c10::string_view{token})] = index;
+  stoi_[_find(std::string_view{token})] = index;
   itos_.insert(itos_.begin() + index, std::move(token));
 }
 
@@ -115,7 +115,7 @@ StringList Vocab::lookup_tokens(const std::vector<int64_t>& indices) {
 }
 
 std::vector<int64_t> Vocab::lookup_indices(
-    const std::vector<c10::string_view>& tokens) {
+    const std::vector<std::string_view>& tokens) {
   std::vector<int64_t> indices(tokens.size());
   for (size_t i = 0; i < tokens.size(); i++) {
     indices[i] = __getitem__(tokens[i]);
@@ -127,7 +127,7 @@ std::unordered_map<std::string, int64_t> Vocab::get_stoi() const {
   std::unordered_map<std::string, int64_t> stoi;
   // construct tokens and index list
   for (const auto& item : itos_) {
-    stoi[item] = __getitem__(c10::string_view{item});
+    stoi[item] = __getitem__(std::string_view{item});
   }
   return stoi;
 }

torchtext/csrc/vocab.h

@@ -46,8 +46,8 @@ struct Vocab : torch::CustomClassHolder {
       StringList tokens,
       const std::optional<int64_t>& default_index);
   TORCHTEXT_API int64_t __len__() const;
-  TORCHTEXT_API int64_t __getitem__(const c10::string_view& token) const;
-  TORCHTEXT_API bool __contains__(const c10::string_view& token) const;
+  TORCHTEXT_API int64_t __getitem__(const std::string_view& token) const;
+  TORCHTEXT_API bool __contains__(const std::string_view& token) const;
   TORCHTEXT_API void set_default_index(std::optional<int64_t> index);
   TORCHTEXT_API std::optional<int64_t> get_default_index() const;
   TORCHTEXT_API void insert_token(std::string token, const int64_t& index);
@@ -56,12 +56,12 @@ struct Vocab : torch::CustomClassHolder {
   TORCHTEXT_API std::vector<std::string> lookup_tokens(
       const std::vector<int64_t>& indices);
   std::vector<int64_t> lookup_indices(
-      const std::vector<c10::string_view>& tokens);
+      const std::vector<std::string_view>& tokens);
   TORCHTEXT_API std::unordered_map<std::string, int64_t> get_stoi() const;
   TORCHTEXT_API std::vector<std::string> get_itos() const;
 
  protected:
-  uint32_t _hash(const c10::string_view& str) const {
+  uint32_t _hash(const std::string_view& str) const {
     uint32_t h = 2166136261;
     for (size_t i = 0; i < str.size(); i++) {
       h = h ^ uint32_t(uint8_t(str[i]));
@@ -70,7 +70,7 @@ struct Vocab : torch::CustomClassHolder {
     return h;
   }
 
-  uint32_t _find(const c10::string_view& w) const {
+  uint32_t _find(const std::string_view& w) const {
     uint32_t stoi_size = stoi_.size();
     uint32_t id = _hash(w) % stoi_size;
     while (stoi_[id] != -1 && itos_[stoi_[id]] != w) {
@@ -80,7 +80,7 @@ struct Vocab : torch::CustomClassHolder {
   }
 
   void _add(std::string w) {
-    uint32_t h = _find(c10::string_view{w.data(), w.size()});
+    uint32_t h = _find(std::string_view{w.data(), w.size()});
     if (stoi_[h] == -1) {
       itos_.emplace_back(std::move(w));
       stoi_[h] = itos_.size() - 1;