Merge pull request #1924 from openvinotoolkit/release

Merge OpenVINO toolkit 2021.2 content into master

Author: Roman Donchenko

.gitattributes

@@ -1,7 +1,15 @@
 * text=auto
 
+/demos/thirdparty/** -whitespace
+
+# In patches, a context line is prefixed by a space, and if it's empty,
+# that space becomes a trailing space. That's a valid use for trailing spaces,
+# so allow it.
+*.patch whitespace=-blank-at-eol
+
 .git* -omz.package
 .editorconfig -omz.package
+.yamllint -omz.package
 /ci/** -omz.package
 /demos/build_demos.sh omz.package=l,m
 /demos/build_demos_msvc.bat omz.package=w

.yamllint

@@ -0,0 +1,40 @@
+yaml-files:
+  - '*.yaml'
+  - '*.yml'
+  - '.yamllint'
+
+# The tools/accuracy_checker/ ignores are there because those files
+# are all symlinks, and:
+# a) it's pointless to check symlinks if we check the real files too;
+# b) on Windows, Git turns links into regular files containing the target name,
+#    which yamllint complains about.
+
+ignore: |
+  demos/thirdparty/
+  tools/accuracy_checker/.yamllint
+  tools/accuracy_checker/configs/
+
+rules:
+  line-length: disable
+  braces: enable
+  brackets: enable
+  colons: disable
+  commas: enable
+  comments:
+    level: warning
+  comments-indentation:
+    level: warning
+  document-end: disable
+  document-start: disable
+  empty-lines: enable
+  empty-values: enable
+  hyphens: enable
+  indentation: enable
+  key-duplicates: enable
+  key-ordering: disable
+  new-line-at-end-of-file: enable
+  new-lines: disable
+  octal-values: disable
+  quoted-strings: disable
+  trailing-spaces: enable
+  truthy: disable

README.md

@@ -1,5 +1,5 @@
 # [OpenVINO™ Toolkit](https://01.org/openvinotoolkit) - Open Model Zoo repository
-[![Stable release](https://img.shields.io/badge/version-2021.1-green.svg)](https://github.com/opencv/open_model_zoo/releases/tag/2021.1)
+[![Stable release](https://img.shields.io/badge/version-2021.2-green.svg)](https://github.com/openvinotoolkit/open_model_zoo/releases/tag/2021.2)
 [![Gitter chat](https://badges.gitter.im/gitterHQ/gitter.png)](https://gitter.im/open_model_zoo/community)
 [![Apache License Version 2.0](https://img.shields.io/badge/license-Apache_2.0-green.svg)](LICENSE)
 
@@ -31,7 +31,7 @@ Open Model Zoo is licensed under [Apache License Version 2.0](LICENSE).
 We welcome community contributions to the Open Model Zoo repository. If you have an idea how to improve the product, please share it with us doing the following steps:
 * Make sure you can build the product and run all the demos with your patch.
 * In case of a larger feature, provide a relevant demo.
-* Submit a pull request at https://github.com/opencv/open_model_zoo/pulls
+* Submit a pull request at https://github.com/openvinotoolkit/open_model_zoo/pulls
 
 You can find additional information about model contribution [here](CONTRIBUTING.md).
 
@@ -42,7 +42,7 @@ Open Model Zoo is licensed under Apache License, Version 2.0. By contributing to
 ## Support
 Please report questions, issues and suggestions using:
 * [\#open_model_zoo](https://stackoverflow.com/search?q=%23open_model_zoo) tag on StackOverflow*
-* [GitHub* Issues](https://github.com/opencv/open_model_zoo/issues)
+* [GitHub* Issues](https://github.com/openvinotoolkit/open_model_zoo/issues)
 * [Forum](https://software.intel.com/en-us/forums/intel-distribution-of-openvino-toolkit)
 * [Gitter](https://gitter.im/open_model_zoo/community)
 

ci/check-basics.py

@@ -0,0 +1,145 @@
+#!/usr/bin/env python3
+
+# Copyright (c) 2020 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+This script performs various checks on the source code that can be run quickly.
+The idea is that these are the kind of checks that can be run on every pull
+request without substantially impacting build time.
+"""
+
+import re
+import subprocess
+import sys
+
+from pathlib import Path
+
+
+OMZ_ROOT = Path(__file__).resolve().parents[1]
+
+RE_SHEBANG_LINE = re.compile(r'\#! \s* (\S+) (?: \s+ (\S+))? \s*', re.VERBOSE)
+
+def main():
+    all_passed = True
+
+    def complain(message):
+        nonlocal all_passed
+        all_passed = False
+        print(message, file=sys.stderr)
+
+    # get the Git magic empty tree hash
+    empty_tree_hash = subprocess.run(
+        ['git', 'hash-object', '-t', 'tree', '--stdin'],
+        stdin=subprocess.DEVNULL,
+        stdout=subprocess.PIPE,
+        check=True,
+        universal_newlines=True,
+        cwd=OMZ_ROOT,
+    ).stdout.strip()
+
+    if subprocess.run(['git', '--no-pager', 'diff', '--check', empty_tree_hash, '--'],
+            cwd=OMZ_ROOT).returncode != 0:
+        all_passed = False
+
+    raw_diff_lines = subprocess.run(
+        ['git', 'diff', '--raw', '-z', empty_tree_hash, '--'],
+        check=True,
+        stdout=subprocess.PIPE,
+        cwd=OMZ_ROOT,
+    ).stdout.decode()[:-1].split('\0')
+
+    numstat_lines = subprocess.run(
+        ['git', 'diff', '--numstat', '-z', empty_tree_hash, '--'],
+        check=True,
+        stdout=subprocess.PIPE,
+        cwd=OMZ_ROOT,
+    ).stdout.decode()[:-1].split('\0')
+
+    for raw_diff, path, numstat_line in zip(*[iter(raw_diff_lines)] * 2, numstat_lines):
+        num_added_lines, num_removed_lines, numstat_path = numstat_line.split('\t', 2)
+        assert path == numstat_path, "git diff --raw and --numstat list different files"
+
+        # Overly long paths cause problems in internal infrastructure due to Windows's
+        # 259 character path length limit. Cap the repo path lengths at a known safe value.
+        PATH_LENGTH_LIMIT = 135
+        if len(path) > PATH_LENGTH_LIMIT:
+            complain(f"{path}: path length ({len(path)}) over limit ({PATH_LENGTH_LIMIT})")
+
+        mode = raw_diff.split()[1]
+
+        absolute_path = OMZ_ROOT / path
+
+        if path.startswith('tools/accuracy_checker/configs/') and path.endswith('.yml'):
+            if mode == '120000':
+                try:
+                    if absolute_path.is_symlink():
+                        real_path = absolute_path.resolve(strict=True)
+                    else:
+                        with open(absolute_path, 'r', newline='') as file:
+                            link_target = file.read()
+                        real_path = (absolute_path.parent / link_target).resolve(strict=True)
+                except FileNotFoundError:
+                    complain(f"{path}: should be a symbolic link to existing accuracy-check.yml from models directory")
+                else:
+                    model_name = absolute_path.stem
+                    if real_path.name != 'accuracy-check.yml' or real_path.parent.name != model_name:
+                        complain(f"{path}: should be a symbolic link to accuracy-check.yml from {model_name} model "
+                                 "directory")
+            else:
+                complain(f"{path}: isn't a symbolic link but it should be a symbolic link to accuracy-check.yml "
+                         "from models directory")
+
+        if mode not in {'100644', '100755'}: # not a regular or executable file
+            continue
+
+        if num_added_lines == '-': # binary file
+            continue
+
+        if path.startswith('demos/thirdparty/'):
+            continue
+
+        with open(absolute_path, encoding='UTF-8') as f:
+            lines = list(f)
+
+        if lines and not lines[-1].endswith('\n'):
+            complain(f"{path}:{len(lines)}: last line doesn't end with a newline character")
+
+        has_shebang = lines and lines[0].startswith('#!')
+        is_executable = mode == '100755'
+
+        if is_executable and not has_shebang:
+            complain(f"{path}: is executable, but doesn't have a shebang line")
+
+        if has_shebang:
+            if not is_executable:
+                complain(f"{path}: has a shebang line, but isn't executable")
+
+            shebang_program, shebang_args = \
+                RE_SHEBANG_LINE.fullmatch(lines[0].rstrip('\n')).groups()
+
+            # Python 2 is EOL and OpenVINO doesn't support it anymore. If someone
+            # uses `#!/usr/bin/python` or something similar, it's likely a mistake.
+            if shebang_program.endswith('/python') or (
+                    shebang_program.endswith('/env') and shebang_args == 'python'):
+                complain(f"{path}:1: use 'python3', not 'python'")
+
+    if subprocess.run([sys.executable, '-m', 'yamllint', '-s', '.'], cwd=OMZ_ROOT).returncode != 0:
+        all_passed = False
+
+    sys.exit(0 if all_passed else 1)
+
+
+if __name__ == '__main__':
+    main()

ci/check-release-readiness.py

@@ -45,9 +45,10 @@ def complain(format, *args):
 
     for models_lst_path in OMZ_ROOT.glob('demos/**/models.lst'):
         with models_lst_path.open() as models_lst:
-            for line in models_lst:
+            for line_num, line in enumerate(models_lst):
                 if RE_TODO_COMMENT.search(line):
-                    complain('{} has a TODO comment', models_lst_path.relative_to(OMZ_ROOT))
+                    complain('{}:{}: line contains TODO comment',
+                        models_lst_path.relative_to(OMZ_ROOT), line_num + 1)
 
     sys.exit(0 if all_passed else 1)
 

ci/get-jobs-for-changes.py

@@ -33,6 +33,8 @@
 
 from pathlib import Path, PurePosixPath
 
+OMZ_ROOT = Path(__file__).resolve().parents[1]
+
 RE_ATTRIB_NAME = re.compile(r"omz\.ci\.job-for-change\.(.+)")
 
 def group_by_n(iterable, n):
@@ -44,24 +46,34 @@ def main():
     args = parser.parse_args()
 
     git_diff_output = subprocess.check_output(
-        ["git", "diff", "--name-only", "--no-renames", "-z", args.base_commit + "...HEAD"])
+        ["git", "diff", "--name-only", "--no-renames", "-z", args.base_commit + "...HEAD"],
+        cwd=OMZ_ROOT)
     changed_files = list(map(PurePosixPath, git_diff_output.decode()[:-1].split("\0")))
 
     models_dir = PurePosixPath("models")
 
     jobs = {}
 
     for changed_file in changed_files:
-        if models_dir in changed_file.parents and changed_file.name == "model.yml":
-            if Path(changed_file).exists(): # it might've been deleted in the branch
-                jobs.setdefault("models", []).append(changed_file.parent.name)
-            else:
-                # make sure no models.lst files reference the deleted model
-                jobs["models_lst"] = True
+        if models_dir in changed_file.parents:
+            if changed_file.name == "model.yml":
+                if (OMZ_ROOT / changed_file).exists(): # it might've been deleted in the branch
+                    jobs.setdefault("models", set()).add(changed_file.parent.name)
+                else:
+                    # make sure no models.lst files reference the deleted model
+                    jobs["models_lst"] = True
+
+            if changed_file.suffix == ".py":
+                for parent in changed_file.parents:
+                    if (OMZ_ROOT / parent / "model.yml").exists():
+                        jobs.setdefault("models", set()).add(parent.name)
+
+    if "models" in jobs:
+        jobs["models"] = sorted(jobs["models"]) # JSON can't work with a set
 
     git_check_attr_output = subprocess.run(
         ["git", "check-attr", "--stdin", "-z", "--all"],
-        input=git_diff_output, stdout=subprocess.PIPE, check=True).stdout
+        input=git_diff_output, stdout=subprocess.PIPE, check=True, cwd=OMZ_ROOT).stdout
 
     for path, attribute, value in group_by_n(git_check_attr_output.decode()[:-1].split("\0"), 3):
         attribute_match = RE_ATTRIB_NAME.fullmatch(attribute)

ci/requirements-ac-test.txt

@@ -1,42 +1,48 @@
 # use update-requirements.py to update this file
 
 atomicwrites==1.4.0       # via -r tools/accuracy_checker/requirements-test.in
-attrs==19.3.0             # via pytest
+attrs==20.2.0             # via pytest
+certifi==2020.6.20        # via matplotlib
+click==7.1.2              # via nltk
 cycler==0.10.0            # via matplotlib
 decorator==4.4.2          # via networkx
 editdistance==0.5.3       # via -r tools/accuracy_checker/requirements.in
+fast-ctc-decode==0.2.5    # via -r tools/accuracy_checker/requirements.in
 imageio==2.9.0            # via scikit-image
-importlib-metadata==1.7.0  # via pluggy, pytest
-joblib==0.14.1            # via scikit-learn
-kiwisolver==1.1.0         # via matplotlib
-matplotlib==3.0.3         # via scikit-image
-more-itertools==8.4.0     # via pytest
-networkx==2.4             # via scikit-image
-nibabel==3.0.2            # via -r tools/accuracy_checker/requirements.in
-numpy==1.17.5             # via -r tools/accuracy_checker/requirements-core.in, imageio, matplotlib, nibabel, pywavelets, scikit-learn, scipy
-packaging==20.4           # via pytest
-pathlib2==2.3.5           # via pytest
-pillow==7.2.0             # via -r tools/accuracy_checker/requirements.in, imageio, scikit-image
+importlib-metadata==2.0.0  # via pluggy, pytest
+joblib==0.17.0            # via nltk, scikit-learn
+kiwisolver==1.2.0         # via matplotlib
+matplotlib==3.3.2         # via scikit-image
+more-itertools==8.5.0     # via pytest
+networkx==2.5             # via scikit-image
+nibabel==3.1.1            # via -r tools/accuracy_checker/requirements.in
+nltk==3.5                 # via -r tools/accuracy_checker/requirements.in
+numpy==1.17.5             # via -r tools/accuracy_checker/requirements-core.in, imageio, matplotlib, nibabel, parasail, pywavelets, scikit-image, scikit-learn, scipy, tifffile
+packaging==20.4           # via nibabel, pytest
+parasail==1.2             # via -r tools/accuracy_checker/requirements.in
+pillow==8.0.0             # via -r tools/accuracy_checker/requirements-core.in, imageio, matplotlib, scikit-image
 pluggy==0.13.1            # via pytest
-py-cpuinfo==4.0.0         # via -r tools/accuracy_checker/requirements.in
+py-cpuinfo==7.0.0         # via -r tools/accuracy_checker/requirements.in
 py==1.9.0                 # via pytest
 pydicom==2.0.0            # via -r tools/accuracy_checker/requirements.in
 pyparsing==2.4.7          # via matplotlib, packaging
+pypi-kenlm==0.1.20190403  # via -r tools/accuracy_checker/requirements.in
 pytest-mock==2.0.0        # via -r tools/accuracy_checker/requirements-test.in
 pytest==5.4.3             # via -r tools/accuracy_checker/requirements-test.in, pytest-mock
 python-dateutil==2.8.1    # via matplotlib
 pywavelets==1.1.1         # via scikit-image
 pyyaml==5.3.1             # via -r tools/accuracy_checker/requirements-core.in
-scikit-image==0.15.0      # via -r tools/accuracy_checker/requirements.in
-scikit-learn==0.22.2.post1  # via -r tools/accuracy_checker/requirements.in
-scipy==1.4.1              # via -r tools/accuracy_checker/requirements.in, scikit-image, scikit-learn
+regex==2020.10.15         # via nltk
+scikit-image==0.17.2      # via -r tools/accuracy_checker/requirements.in
+scikit-learn==0.23.2      # via -r tools/accuracy_checker/requirements.in
+scipy==1.5.2              # via -r tools/accuracy_checker/requirements.in, scikit-image, scikit-learn
 sentencepiece==0.1.91     # via -r tools/accuracy_checker/requirements.in
-shapely==1.7.0            # via -r tools/accuracy_checker/requirements.in
-six==1.15.0               # via packaging, pathlib2, python-dateutil
-tokenizers==0.8.0         # via -r tools/accuracy_checker/requirements.in
-tqdm==4.47.0              # via -r tools/accuracy_checker/requirements.in
+shapely==1.7.1            # via -r tools/accuracy_checker/requirements.in
+six==1.15.0               # via packaging, python-dateutil
+threadpoolctl==2.1.0      # via scikit-learn
+tifffile==2020.9.3        # via scikit-image
+tokenizers==0.9.1         # via -r tools/accuracy_checker/requirements.in
+tqdm==4.50.2              # via -r tools/accuracy_checker/requirements.in, nltk
 wcwidth==0.2.5            # via pytest
-zipp==1.2.0               # via importlib-metadata
-
-# The following packages are considered to be unsafe in a requirements file:
-# setuptools
+wheel==0.35.1             # via parasail
+zipp==3.3.0               # via importlib-metadata