Update proto with tokenize API enhancements

And remove unused generate.proto file

(cherry picked from commit 5f9705816863e741d1a43b9e1b6e00b259d1c58a)

Author: njhill

proto/generation.proto

@@ -204,7 +204,11 @@ message TokenInfo {
 message BatchedTokenizeRequest {
   string model_id = 1;
   repeated TokenizeRequest requests = 2;
-  bool return_tokens = 3; //TBD
+  bool return_tokens = 3;
+  bool return_offsets = 4;
+
+  // Zero means don't truncate.
+  uint32 truncate_input_tokens = 5;
 }
 
 message BatchedTokenizeResponse {
@@ -216,10 +220,17 @@ message TokenizeRequest {
 }
 
 message TokenizeResponse {
+  message Offset {
+    uint32 start = 1;
+    uint32 end = 2;
+  }
+
   uint32 token_count = 1;
-  repeated string tokens = 2; // if include_tokens = true
 
-  // We'll possibly add more later
+  // if return_tokens = true
+  repeated string tokens = 2;
+  // if return_tokens = true
+  repeated Offset offsets = 3;
 }