Fix version mismatch between kubernetes and minikube (#2986)

Author: wjsi

.github/workflows/install-minikube.sh

@@ -5,24 +5,10 @@ export CHANGE_MINIKUBE_NONE_USER=true
 sudo apt-get -q update || true
 sudo apt-get install -yq conntrack jq
 
-get_latest_release() {
-  curl --silent "https://api.github.com/repos/$1/releases" |
-    jq -c '[.[] | select(.prerelease == false)][0].tag_name' |
-    sed -E 's/.*"([^"]+)".*/\1/'
-}
-
-K8S_VERSION=$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)
-if [[ "$K8S_VERSION" == *"alpha"* ]] || [[ "$K8S_VERSION" == *"beta"* ]] || [[ "$K8S_VERSION" == *"rc"* ]]; then
-  K8S_VERSION=$(get_latest_release "kubernetes/kubernetes")
-fi
-
-curl -Lo kubectl https://storage.googleapis.com/kubernetes-release/release/$K8S_VERSION/bin/linux/amd64/kubectl && \
-  chmod +x kubectl && sudo mv kubectl /usr/local/bin/
-
 curl -Lo minikube https://storage.googleapis.com/minikube/releases/latest/minikube-linux-amd64 && \
   chmod +x minikube && sudo mv minikube /usr/local/bin/
 
-sudo minikube start --vm-driver=none --kubernetes-version=$K8S_VERSION
+sudo minikube start --vm-driver=none
 export KUBECONFIG=$HOME/.kube/config
 sudo cp -R /root/.kube /root/.minikube $HOME/
 sudo chown -R $(id -u):$(id -g) $HOME/.kube $HOME/.minikube
@@ -32,6 +18,10 @@ mv tmp $KUBECONFIG
 
 minikube update-context
 
+K8S_VERSION=$(minikube kubectl -- version --client --output='json' | jq -r '.clientVersion.gitVersion')
+curl -Lo kubectl https://storage.googleapis.com/kubernetes-release/release/$K8S_VERSION/bin/linux/amd64/kubectl && \
+  chmod +x kubectl && sudo mv kubectl /usr/local/bin/
+
 JSONPATH='{range .items[*]}{@.metadata.name}:{range @.status.conditions[*]}{@.type}={@.status};{end}{end}'
 until kubectl get nodes -o jsonpath="$JSONPATH" 2>&1 | grep -q "Ready=True"; do
   sleep 1

benchmarks/asv_bench/benchmarks/tokenize.py

@@ -0,0 +1,73 @@
+# Copyright 1999-2022 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import numpy as np
+import pandas as pd
+
+from mars.core import OutputType
+from mars.core.operand import Operand
+from mars.serialization.serializables import (
+    Int64Field,
+    Float64Field,
+    ListField,
+    DataTypeField,
+    SeriesField,
+    NDArrayField,
+    StringField,
+    FieldTypes,
+)
+from mars.tensor.operands import TensorOperandMixin
+from mars.utils import tokenize
+
+
+class MockOperand(Operand, TensorOperandMixin):
+    _op_code_ = 102345
+
+    str_field = StringField("str_field")
+    int_field = Int64Field("int_field")
+    float_field = Float64Field("float_field")
+    dtype_field = DataTypeField("dtype_field")
+    series_field = SeriesField("series_field")
+    ndarray_field = NDArrayField("ndarray_field")
+    int_list_field = ListField("int_list_field", field_type=FieldTypes.int64)
+    float_list_field = ListField("float_list_field", field_type=FieldTypes.float64)
+    str_list_field = ListField("str_list_field", field_type=FieldTypes.string)
+
+
+class TokenizeOperandSuite:
+    def setup(self):
+        chunks = []
+        for idx in range(1000):
+            op = MockOperand(
+                str_field="abcd" * 1024,
+                int_field=idx,
+                float_field=float(idx) * 1.42,
+                dtype_field=np.dtype("<M8"),
+                series_field=pd.Series([np.dtype(int)] * 1024, name="dtype"),
+                ndarray_field=np.random.rand(1000),
+                int_list_field=np.random.randint(0, 1000, size=(1000,)).tolist(),
+                float_list_field=np.random.rand(1000).tolist(),
+                str_list_field=[str(i * 2.8571) for i in range(100)],
+            )
+            chunks.append(op.new_chunk([], output_type=OutputType.tensor))
+        self.test_data = chunks
+
+    def time_tokenize(self):
+        tokenize(self.test_data)
+
+
+if __name__ == "__main__":
+    suite = TokenizeOperandSuite()
+    suite.setup()
+    suite.time_tokenize()

mars/_utils.pyx

@@ -152,13 +152,14 @@ def tokenize_int(*args, **kwargs):
 cdef class Tokenizer(TypeDispatcher):
     def __call__(self, object obj, *args, **kwargs):
         try:
-            return super().__call__(obj, *args, **kwargs)
+            return self.get_handler(type(obj))(obj, *args, **kwargs)
         except KeyError:
             if hasattr(obj, '__mars_tokenize__') and not isinstance(obj, type):
                 if len(args) == 0 and len(kwargs) == 0:
                     return obj.__mars_tokenize__()
                 else:
-                    return super().__call__(obj.__mars_tokenize__(), *args, **kwargs)
+                    obj = obj.__mars_tokenize__()
+                    return self.get_handler(type(obj))(obj, *args, **kwargs)
             if callable(obj):
                 if PDTick is not None and not isinstance(obj, PDTick):
                     return tokenize_function(obj)
@@ -176,14 +177,20 @@ cdef inline list iterative_tokenize(object ob):
     while dq_pos < len(dq):
         x = dq[dq_pos]
         dq_pos += 1
-        if isinstance(x, (list, tuple)):
+        if type(x) in _primitive_types:
+            h_list.append(x)
+        elif isinstance(x, (list, tuple)):
             dq.extend(x)
         elif isinstance(x, set):
             dq.extend(sorted(x))
         elif isinstance(x, dict):
             dq.extend(sorted(x.items()))
         else:
             h_list.append(tokenize_handler(x))
+
+        if dq_pos >= 64 and len(dq) < dq_pos * 2:  # pragma: no cover
+            dq = dq[dq_pos:]
+            dq_pos = 0
     return h_list
 
 
@@ -202,20 +209,20 @@ cdef inline tuple tokenize_numpy(ob):
                 ob.shape, ob.strides, offset)
     if ob.dtype.hasobject:
         try:
-            data = mmh_hash_bytes('-'.join(ob.flat).encode('utf-8', errors='surrogatepass')).hex()
+            data = mmh_hash_bytes('-'.join(ob.flat).encode('utf-8', errors='surrogatepass'))
         except UnicodeDecodeError:
-            data = mmh_hash_bytes(b'-'.join([to_binary(x) for x in ob.flat])).hex()
+            data = mmh_hash_bytes(b'-'.join([to_binary(x) for x in ob.flat]))
         except TypeError:
             try:
-                data = mmh_hash_bytes(pickle.dumps(ob, pickle.HIGHEST_PROTOCOL)).hex()
+                data = mmh_hash_bytes(pickle.dumps(ob, pickle.HIGHEST_PROTOCOL))
             except:
                 # nothing can do, generate uuid
                 data = uuid.uuid4().hex
     else:
         try:
-            data = mmh_hash_bytes(ob.ravel().view('i1').data).hex()
+            data = mmh_hash_bytes(ob.ravel().view('i1').data)
         except (BufferError, AttributeError, ValueError):
-            data = mmh_hash_bytes(ob.copy().ravel().view('i1').data).hex()
+            data = mmh_hash_bytes(ob.copy().ravel().view('i1').data)
     return data, ob.dtype, ob.shape, ob.strides
 
 
@@ -332,20 +339,19 @@ def tokenize_cudf(ob):
 
 cdef Tokenizer tokenize_handler = Tokenizer()
 
-base_types = (int, float, str, unicode, bytes, complex,
-              type(None), type, slice, date, datetime, timedelta)
-for t in base_types:
+cdef set _primitive_types = {
+    int, float, str, unicode, bytes, complex, type(None), type, slice, date, datetime, timedelta
+}
+for t in _primitive_types:
     tokenize_handler.register(t, lambda ob: ob)
 
 for t in (np.dtype, np.generic):
-    tokenize_handler.register(t, lambda ob: repr(ob))
+    tokenize_handler.register(t, lambda ob: ob)
 
 for t in (list, tuple, dict, set):
     tokenize_handler.register(t, iterative_tokenize)
 
 tokenize_handler.register(np.ndarray, tokenize_numpy)
-tokenize_handler.register(dict, lambda ob: iterative_tokenize(sorted(ob.items())))
-tokenize_handler.register(set, lambda ob: iterative_tokenize(sorted(ob)))
 tokenize_handler.register(np.random.RandomState, lambda ob: iterative_tokenize(ob.get_state()))
 tokenize_handler.register(memoryview, lambda ob: mmh3_hash_from_buffer(ob))
 tokenize_handler.register(Enum, tokenize_enum)