Merge pull request #107 from iamneha/verify-scripts

k8s-ci-robot · web-flow · commit 8497dfb53782 · 2019-01-22T18:49:26.000-08:00
Verify-boilerplate script
diff --git a/.travis.yml b/.travis.yml
@@ -35,4 +35,5 @@ install:
 
 script:
   - ./run_tox.sh tox
+  - ./hack/verify-boilerplate.sh
 
diff --git a/config/__init__.py b/config/__init__.py
@@ -1,3 +1,5 @@
+#!/usr/bin/env python
+
 # Copyright 2016 The Kubernetes Authors.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/config/config_exception.py b/config/config_exception.py
@@ -1,3 +1,5 @@
+#!/usr/bin/env python
+
 # Copyright 2016 The Kubernetes Authors.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/config/dateutil.py b/config/dateutil.py
@@ -1,3 +1,5 @@
+#!/usr/bin/env python
+
 # Copyright 2017 The Kubernetes Authors.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/config/dateutil_test.py b/config/dateutil_test.py
@@ -1,3 +1,5 @@
+#!/usr/bin/env python
+
 # Copyright 2016 The Kubernetes Authors.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/config/exec_provider.py b/config/exec_provider.py
@@ -1,3 +1,5 @@
+#!/usr/bin/env python
+
 # Copyright 2018 The Kubernetes Authors.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/config/exec_provider_test.py b/config/exec_provider_test.py
@@ -1,3 +1,5 @@
+#!/usr/bin/env python
+
 # Copyright 2018 The Kubernetes Authors.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/config/incluster_config.py b/config/incluster_config.py
@@ -1,3 +1,5 @@
+#!/usr/bin/env python
+
 # Copyright 2016 The Kubernetes Authors.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/config/incluster_config_test.py b/config/incluster_config_test.py
@@ -1,3 +1,5 @@
+#!/usr/bin/env python
+
 # Copyright 2016 The Kubernetes Authors.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/config/kube_config.py b/config/kube_config.py
@@ -1,3 +1,5 @@
+#!/usr/bin/env python
+
 # Copyright 2018 The Kubernetes Authors.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/config/kube_config_test.py b/config/kube_config_test.py
@@ -1,3 +1,5 @@
+#!/usr/bin/env python
+
 # Copyright 2018 The Kubernetes Authors.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/hack/boilerplate/boilerplate.py b/hack/boilerplate/boilerplate.py
@@ -0,0 +1,197 @@
+#!/usr/bin/env python
+
+# Copyright 2018 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import print_function
+
+import argparse
+import datetime
+import difflib
+import glob
+import os
+import re
+import sys
+
+parser = argparse.ArgumentParser()
+parser.add_argument(
+    "filenames",
+    help="list of files to check, all files if unspecified",
+    nargs='*')
+
+rootdir = os.path.dirname(__file__) + "/../../"
+rootdir = os.path.abspath(rootdir)
+parser.add_argument(
+    "--rootdir", default=rootdir, help="root directory to examine")
+
+default_boilerplate_dir = os.path.join(rootdir, "hack/boilerplate")
+parser.add_argument(
+    "--boilerplate-dir", default=default_boilerplate_dir)
+
+parser.add_argument(
+    "-v", "--verbose",
+    help="give verbose output regarding why a file does not pass",
+    action="store_true")
+
+args = parser.parse_args()
+
+verbose_out = sys.stderr if args.verbose else open("/dev/null", "w")
+
+
+def get_refs():
+    refs = {}
+
+    for path in glob.glob(os.path.join(args.boilerplate_dir, "boilerplate.*.txt")):
+        extension = os.path.basename(path).split(".")[1]
+
+        ref_file = open(path, 'r')
+        ref = ref_file.read().splitlines()
+        ref_file.close()
+        refs[extension] = ref
+
+    return refs
+
+
+def file_passes(filename, refs, regexs):
+    try:
+        f = open(filename, 'r')
+    except Exception as exc:
+        print("Unable to open %s: %s" % (filename, exc), file=verbose_out)
+        return False
+
+    data = f.read()
+    f.close()
+
+    basename = os.path.basename(filename)
+    extension = file_extension(filename)
+
+    if extension != "":
+        ref = refs[extension]
+    else:
+        ref = refs[basename]
+
+    # remove extra content from the top of files
+    if extension == "sh":
+        p = regexs["shebang"]
+        (data, found) = p.subn("", data, 1)
+
+    data = data.splitlines()
+
+    # if our test file is smaller than the reference it surely fails!
+    if len(ref) > len(data):
+        print('File %s smaller than reference (%d < %d)' %
+              (filename, len(data), len(ref)),
+              file=verbose_out)
+        return False
+
+    # trim our file to the same number of lines as the reference file
+    data = data[:len(ref)]
+
+    p = regexs["year"]
+    for d in data:
+        if p.search(d):
+            print('File %s has the YEAR field, but missing the year of date' %
+                  filename, file=verbose_out)
+            return False
+
+    # Replace all occurrences of the regex "2014|2015|2016|2017|2018" with "YEAR"
+    p = regexs["date"]
+    for i, d in enumerate(data):
+        (data[i], found) = p.subn('YEAR', d)
+        if found != 0:
+            break
+
+    # if we don't match the reference at this point, fail
+    if ref != data:
+        print("Header in %s does not match reference, diff:" %
+              filename, file=verbose_out)
+        if args.verbose:
+            print(file=verbose_out)
+            for line in difflib.unified_diff(ref, data, 'reference', filename, lineterm=''):
+                print(line, file=verbose_out)
+            print(file=verbose_out)
+        return False
+
+    return True
+
+
+def file_extension(filename):
+    return os.path.splitext(filename)[1].split(".")[-1].lower()
+
+
+# list all the files contain 'DO NOT EDIT', but are not generated
+skipped_ungenerated_files = ['hack/boilerplate/boilerplate.py']
+
+
+def normalize_files(files):
+    newfiles = []
+    for pathname in files:
+        newfiles.append(pathname)
+    for i, pathname in enumerate(newfiles):
+        if not os.path.isabs(pathname):
+            newfiles[i] = os.path.join(args.rootdir, pathname)
+    return newfiles
+
+
+def get_files(extensions):
+    files = []
+    if len(args.filenames) > 0:
+        files = args.filenames
+    else:
+        for root, dirs, walkfiles in os.walk(args.rootdir):
+            for name in walkfiles:
+                pathname = os.path.join(root, name)
+                files.append(pathname)
+
+    files = normalize_files(files)
+    outfiles = []
+    for pathname in files:
+        basename = os.path.basename(pathname)
+        extension = file_extension(pathname)
+        if extension in extensions or basename in extensions:
+            outfiles.append(pathname)
+    return outfiles
+
+
+def get_dates():
+    years = datetime.datetime.now().year
+    return '(%s)' % '|'.join((str(year) for year in range(2014, years+1)))
+
+
+def get_regexs():
+    regexs = {}
+    # Search for "YEAR" which exists in the boilerplate, but shouldn't in the real thing
+    regexs["year"] = re.compile('YEAR')
+    # get_dates return 2014, 2015, 2016, 2017, or 2018 until the current year as a regex like: "(2014|2015|2016|2017|2018)";
+    # company holder names can be anything
+    regexs["date"] = re.compile(get_dates())
+    # strip #!.* from shell scripts
+    regexs["shebang"] = re.compile(r"^(#!.*\n)\n*", re.MULTILINE)
+    return regexs
+
+
+def main():
+    regexs = get_regexs()
+    refs = get_refs()
+    filenames = get_files(refs.keys())
+
+    for filename in filenames:
+        if not file_passes(filename, refs, regexs):
+            print(filename, file=sys.stdout)
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())
diff --git a/hack/boilerplate/boilerplate.py.txt b/hack/boilerplate/boilerplate.py.txt
@@ -0,0 +1,15 @@
+#!/usr/bin/env python
+
+# Copyright YEAR The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/hack/boilerplate/boilerplate.sh.txt b/hack/boilerplate/boilerplate.sh.txt
@@ -0,0 +1,13 @@
+# Copyright YEAR The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/hack/verify-boilerplate.sh b/hack/verify-boilerplate.sh
@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+
+# Copyright 2018 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+KUBE_ROOT=$(dirname "${BASH_SOURCE}")/..
+
+boilerDir="${KUBE_ROOT}/hack/boilerplate"
+boiler="${boilerDir}/boilerplate.py"
+
+files_need_boilerplate=($(${boiler} "$@"))
+
+# Run boilerplate check
+if [[ ${#files_need_boilerplate[@]} -gt 0 ]]; then
+  for file in "${files_need_boilerplate[@]}"; do
+    echo "Boilerplate header is wrong for: ${file}" >&2
+  done
+
+  exit 1
+fi
diff --git a/run_tox.sh b/run_tox.sh
@@ -11,7 +11,7 @@
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and 
+# See the License for the specific language governing permissions and
 # limitations under the License.
 
 set -o errexit
@@ -51,4 +51,3 @@ git status
 echo "Running tox from the main repo on $TOXENV environment"
 # Run the user-provided command.
 "${@}"
-
diff --git a/stream/__init__.py b/stream/__init__.py
@@ -1,3 +1,5 @@
+#!/usr/bin/env python
+
 # Copyright 2017 The Kubernetes Authors.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/stream/stream.py b/stream/stream.py
@@ -1,14 +1,18 @@
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
+#!/usr/bin/env python
+
+# Copyright 2018 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
 #
-#      http://www.apache.org/licenses/LICENSE-2.0
+#     http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 
 from . import ws_client
 
diff --git a/stream/ws_client.py b/stream/ws_client.py
diff --git a/stream/ws_client_test.py b/stream/ws_client_test.py
diff --git a/watch/__init__.py b/watch/__init__.py
diff --git a/watch/watch.py b/watch/watch.py
diff --git a/watch/watch_test.py b/watch/watch_test.py

Original file line number	Diff line number	Diff line change
`@@ -35,4 +35,5 @@ install:`
`35`	`35`
`36`	`36`	`script:`
`37`	`37`	`- ./run_tox.sh tox`
	`38`	`+ - ./hack/verify-boilerplate.sh`
`38`	`39`
Original file line number	Diff line number	Diff line change
`@@ -1,3 +1,5 @@`
	`1`	`+#!/usr/bin/env python`
	`2`	`+`
`1`	`3`	`# Copyright 2016 The Kubernetes Authors.`
`2`	`4`	`#`
`3`	`5`	`# Licensed under the Apache License, Version 2.0 (the "License");`
Original file line number	Diff line number	Diff line change
`@@ -1,3 +1,5 @@`
	`1`	`+#!/usr/bin/env python`
	`2`	`+`
`1`	`3`	`# Copyright 2017 The Kubernetes Authors.`
`2`	`4`	`#`
`3`	`5`	`# Licensed under the Apache License, Version 2.0 (the "License");`
Original file line number	Diff line number	Diff line change
`@@ -1,3 +1,5 @@`
	`1`	`+#!/usr/bin/env python`
	`2`	`+`
`1`	`3`	`# Copyright 2018 The Kubernetes Authors.`
`2`	`4`	`#`
`3`	`5`	`# Licensed under the Apache License, Version 2.0 (the "License");`