Merge pull request #2557 from openvinotoolkit/release

Merge OpenVINO toolkit 2021.4 content into master

Author: Wovchena

.flake8

@@ -3,10 +3,12 @@ exclude=
     # generic excludes
     .git,__pycache__,
     # generated files
-    demos/speech_recognition_demo/python/ctcdecode-numpy/ctcdecode_numpy/impl.py,
+    demos/speech_recognition_deepspeech_demo/python/ctcdecode-numpy/ctcdecode_numpy/impl.py,
     models/public/mozilla-deepspeech-0.8.2/mds_convert_utils/memmapped_file_system_pb2.py,
 
 select=
+    # flake8-comprehensions issues
+    C4,
     # indentation problems
     E10,E11,
     # closing bracket does not match indentation of opening brackets line

.gitattributes

@@ -17,6 +17,15 @@
 /demos/tests/** -omz.package
 /tools/downloader/tests/** -omz.package
 
+* omz.package.component=dev
+/LICENSE omz.package.component=tools
+/data/dataset_definitions.yml omz.package.component=tools
+/models/**/*.py omz.package.component=tools
+/models/**/*.yml omz.package.component=tools
+/tools/**/* omz.package.component=tools
+
+/ci/dependencies.yml omz.ci.job-for-change.ac omz.ci.job-for-change.demos omz.ci.job-for-change.models
+
 /ci/requirements-ac.txt omz.ci.job-for-change.ac
 /ci/requirements-ac-test.txt omz.ci.job-for-change.ac
 /ci/requirements-conversion.txt omz.ci.job-for-change.demos omz.ci.job-for-change.models
@@ -25,11 +34,9 @@
 /ci/requirements-quantization.txt omz.ci.job-for-change.downloader
 
 /demos/** omz.ci.job-for-change.demos
-/demos/**/*.md -omz.ci.job-for-change.demos
-
-/models/** omz.ci.job-for-change.documentation
-/models/**/*.yml -omz.ci.job-for-change.documentation
 
 /tools/accuracy_checker/** omz.ci.job-for-change.ac
+/tools/accuracy_checker/configs/*.yml -omz.ci.job-for-change.ac
 /tools/downloader/** omz.ci.job-for-change.downloader
-/tools/**/*.md -omz.ci.job-for-change.ac -omz.ci.job-for-change.downloader
+
+*.md omz.ci.job-for-change.documentation -omz.ci.job-for-change.ac -omz.ci.job-for-change.demos -omz.ci.job-for-change.downloader

CONTRIBUTING.md

@@ -44,7 +44,7 @@ Place your files as shown in the table below:
 File | Destination
 ---|---
 configuration file | `models/public/<model_name>/model.yml`
-documentation file | `models/public/<model_name>/<model_name>.md`
+documentation file | `models/public/<model_name>/README.md`
 validation configuration file|`models/public/<model_name>/accuracy-check.yml`
 demo|`demos/<demo_name>`<br>or<br>`demos/python_demos/<demo_name>`
 
@@ -76,7 +76,7 @@ Description of the model. Must match with the description from the model [docume
 
 **`task_type`**
 
-[Model task type](tools/downloader/README.md#model-information-dumper-usage). If there is no task type of your model, add a new one to the list `KNOWN_TASK_TYPES` of the [tools/downloader/common.py](tools/downloader/common.py) file.
+[Model task type](tools/downloader/README.md#model-information-dumper-usage). If there is no task type of your model, add a new one to the list `KNOWN_TASK_TYPES` of the [`open_model_zoo.model_tools._common`](tools/downloader/src/open_model_zoo/model_tools/_common.py) module.
 
 **`files`**
 
@@ -193,7 +193,7 @@ Demos are required to support the following keys:
 
  -  `-i "<input>"`: Required. An input to process. The input can usually be a single image, a folder of images or anything that OpenCV's `VideoCapture` can process.
  -  `-m "<path>"`: Required. Path to an .xml file with a trained model. If the demo uses several models at the same time, use other keys prefixed with `-m_`.
- -  `-d "<device>"`: Optional. Specifies a target device to infer on. CPU, GPU, FPGA, HDDL or MYRIAD is acceptable. Default must be CPU. If the demo uses several models at the same time, use keys prefixed with `d_` (just like keys `m_*` above) to specify device for each model.
+ -  `-d "<device>"`: Optional. Specifies a target device to infer on. CPU, GPU, HDDL or MYRIAD is acceptable. Default must be CPU. If the demo uses several models at the same time, use keys prefixed with `d_` (just like keys `m_*` above) to specify device for each model.
  -  `-no_show`: Optional. Do not visualize inference results.
 
 > **TIP**: For Python, it is preferable to use `--` instead of `-` for long keys.
@@ -204,7 +204,7 @@ Add `README.md` file, which describes demo usage. Update [demos' README.md](demo
 
 ## Accuracy Validation
 
-Accuracy validation can be performed by the [Accuracy Checker](./tools/accuracy_checker) tool. This tool can use either IE to run a converted model, or an original framework to run an original model. Accuracy Checker supports lots of datasets, metrics and preprocessing options, which simplifies validation if a task is supported by the tool. You only need to create a configuration file that contains necessary parameters for accuracy validation (specify a dataset and annotation, pre- and post-processing parameters, accuracy metrics to compute and so on) of converted model. For details, refer to [Testing new models](./tools/accuracy_checker#testing-new-models).
+Accuracy validation can be performed by the [Accuracy Checker](./tools/accuracy_checker/README.md) tool. This tool can use either IE to run a converted model, or an original framework to run an original model. Accuracy Checker supports lots of datasets, metrics and preprocessing options, which simplifies validation if a task is supported by the tool. You only need to create a configuration file that contains necessary parameters for accuracy validation (specify a dataset and annotation, pre- and post-processing parameters, accuracy metrics to compute and so on) of converted model. For details, refer to [Testing new models](./tools/accuracy_checker/README.md#testing-new-models).
 
 If a model uses a dataset which is not supported by the Accuracy Checker, you also must provide the license and the link to it and mention it in the PR description.
 
@@ -232,7 +232,7 @@ models:
 
 ## Documentation
 
-Documentation is a very important part of model contribution as it helps to better understand the possible usage of the model. Documentation must be named in accordance with the name of the model.
+Documentation is a very important part of model contribution as it helps to better understand the possible usage of the model. It must be located in a `README.md` file in the model subdirectory.
 The documentation should contain:
 * description of a model
 	* main purpose
@@ -248,7 +248,7 @@ The documentation should contain:
 * detailed description of input and output for original and converted models
 * the model's licensing terms
 
-Learn the detailed structure and headers naming convention from any model documentation (for example, [alexnet](./models/public/alexnet/alexnet.md)).
+Learn the detailed structure and headers naming convention from any model documentation (for example, [alexnet](./models/public/alexnet/README.md)).
 
 ## Legal Information
 

README.md

@@ -1,34 +1,38 @@
 # [OpenVINO™ Toolkit](https://01.org/openvinotoolkit) - Open Model Zoo repository
-[![Stable release](https://img.shields.io/badge/version-2021.3-green.svg)](https://github.com/openvinotoolkit/open_model_zoo/releases/tag/2021.3)
+[![Stable release](https://img.shields.io/badge/version-2021.4-green.svg)](https://github.com/openvinotoolkit/open_model_zoo/releases/tag/2021.4)
 [![Gitter chat](https://badges.gitter.im/gitterHQ/gitter.png)](https://gitter.im/open_model_zoo/community)
 [![Apache License Version 2.0](https://img.shields.io/badge/license-Apache_2.0-green.svg)](LICENSE)
 
 This repository includes optimized deep learning models and a set of demos to expedite development of high-performance deep learning inference applications. Use these free pre-trained models instead of training your own models to speed-up the development and production deployment process.
 
+Intel is committed to the respect of human rights and avoiding complicity in human rights abuses, a policy reflected in the [Intel Global Human Rights Principles](https://www.intel.com/content/www/us/en/policy/policy-human-rights.html). Accordingly, by accessing the Intel material on this platform you agree that you will not use the material in a product or application that causes or contributes to a violation of an internationally recognized human right.
+
 ## Repository Components:
-* [Pre-Trained Models](models/intel/index.md)
-* [Public Models Description](models/public/index.md)
+* [Intel Pre-Trained Models](models/intel/index.md)
+* [Public Pre-Trained Models](models/public/index.md)
 * [Model Downloader](tools/downloader/README.md) and other automation tools
 * [Demos](demos/README.md) that demonstrate models usage with Deep Learning Deployment Toolkit
 * [Accuracy Checker](tools/accuracy_checker/README.md) tool for models accuracy validation
 
 ## License
 Open Model Zoo is licensed under [Apache License Version 2.0](LICENSE).
 
-## Documentation
+## Online Documentation
 * [OpenVINO™ Release Notes](https://software.intel.com/en-us/articles/OpenVINO-RelNotes)
-* [Pre-Trained Models](https://software.intel.com/en-us/openvino-toolkit/documentation/pretrained-models)
-* [Demos and samples](https://software.intel.com/en-us/articles/OpenVINO-IE-Samples)
+* [Pre-Trained Models](https://docs.openvinotoolkit.org/latest/trained_models.html)
+* [Demos and Samples](https://docs.openvinotoolkit.org/latest/omz_demos.html)
 
-## Other usage examples
+## Other Usage Examples
 * [Open Visual Cloud](https://01.org/openvisualcloud)
   * [Tutorial: Build and Run the AD Insertion Sample on public cloud or local machine](https://01.org/openvisualcloud/documents/tutorial-build-and-run-ad-insertion-sample-public-cloud-or-local-machine)
   * [GitHub Repo for Ad Insertion Sample](https://github.com/OpenVisualCloud/Ad-Insertion-Sample)
 * [OpenVINO for Smart City](https://github.com/incluit/OpenVino-For-SmartCity)
 * [OpenVINO Driver Behavior](https://github.com/incluit/OpenVino-Driver-Behaviour)
+* [OpenVINO Python\* Notebooks](https://github.com/openvinotoolkit/openvino_notebooks/blob/main/README.md)
 
 ## How to Contribute
 We welcome community contributions to the Open Model Zoo repository. If you have an idea how to improve the product, please share it with us doing the following steps:
+
 * Make sure you can build the product and run all the demos with your patch.
 * In case of a larger feature, provide a relevant demo.
 * Submit a pull request at https://github.com/openvinotoolkit/open_model_zoo/pulls

ci/check-basics.py

@@ -49,7 +49,7 @@ def complain(message):
         cwd=OMZ_ROOT,
     ).stdout.strip()
 
-    print('running text checks...', flush=True)
+    print('running miscellaneous checks...', flush=True)
 
     if subprocess.run(['git', '--no-pager', 'diff', '--check', empty_tree_hash, '--'],
             cwd=OMZ_ROOT).returncode != 0:
@@ -103,6 +103,10 @@ def complain(message):
                 complain(f"{path}: isn't a symbolic link but it should be a symbolic link to accuracy-check.yml "
                          "from models directory")
 
+        if path.startswith('models/') and '/description/' in path:
+            complain(f"{path}: the model documentation convention has changed;"
+                " put the text in README.md and the images under /assets/")
+
         if mode not in {'100644', '100755'}: # not a regular or executable file
             continue
 
@@ -145,6 +149,10 @@ def complain(message):
     if subprocess.run([sys.executable, '-m', 'flake8', '--config=.flake8'], cwd=OMZ_ROOT).returncode != 0:
         all_passed = False
 
+    print('running documentation checks...', flush=True)
+    if subprocess.run([sys.executable, '--', str(OMZ_ROOT / 'ci/check-documentation.py')]).returncode != 0:
+        all_passed = False
+
     sys.exit(0 if all_passed else 1)
 
 

ci/check-documentation.py

@@ -0,0 +1,157 @@
+#!/usr/bin/env python3
+
+"""
+This script is like check-basics.py, but specific to the documentation.
+It's split off into a separate script, so that it can be easily run on its own.
+"""
+
+import re
+import sys
+import urllib.parse
+import urllib.request
+
+from pathlib import Path
+
+OMZ_ROOT = Path(__file__).resolve().parents[1]
+
+sys.path.append(str(OMZ_ROOT / 'ci/lib'))
+
+import omzdocs
+
+HTML_FRAGMENT_RE = re.compile(r'</?([^>\s]+)', re.IGNORECASE)
+
+# taken from https://www.doxygen.nl/manual/htmlcmds.html
+ALLOWED_HTML_ELEMENTS = frozenset([
+    'a', 'b', 'blockquote', 'br', 'caption', 'center', 'code', 'dd', 'del',
+    'dfn', 'div', 'dl', 'dt', 'em', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6',
+    'hr', 'i', 'img', 'ins', 'kbd', 'li', 'ol', 'p', 'pre', 's', 'small',
+    'span', 'strike', 'strong', 'sub', 'sup', 'table', 'td', 'th', 'tr',
+    'tt', 'u', 'ul', 'var',
+])
+
+def find_md_files():
+    thirdparty_dir = OMZ_ROOT / 'demos' / 'thirdparty'
+
+    for path in OMZ_ROOT.glob('**/*.md'):
+        if thirdparty_dir in path.parents: continue
+        yield path
+
+def main():
+    all_passed = True
+
+    index_file_paths = (
+        OMZ_ROOT / 'models/intel/index.md',
+        OMZ_ROOT / 'models/public/index.md',
+        OMZ_ROOT / 'demos/README.md',
+    )
+
+    all_md_files = tuple(find_md_files())
+
+    def complain(message):
+        nonlocal all_passed
+        all_passed = False
+        print(message, file=sys.stderr)
+
+    index_child_md_links = {}
+    for index_file_path in index_file_paths:
+        if not index_file_path.exists():
+            complain(f'{index_file_path}: file not found')
+            continue
+
+        required_md_links = []
+        for md_file in all_md_files:
+            if md_file.name == "README.md" and md_file.parent != index_file_path.parent:
+                try:
+                    md_rel_path = md_file.relative_to(index_file_path.parent)
+                except ValueError:
+                    continue
+
+                md_intermediate_parents = list(md_rel_path.parents)[1:-1] # removed root and first parent dirs
+
+                if not any((index_file_path.parent / parent_dir / 'README.md').exists()
+                        for parent_dir in md_intermediate_parents):
+                    required_md_links.append(md_file)
+
+        index_child_md_links[index_file_path] = sorted(required_md_links)
+
+    omz_reference_prefix = '<omz_dir>/'
+
+    for md_path in sorted(all_md_files):
+        referenced_md_files = set()
+
+        md_path_rel = md_path.relative_to(OMZ_ROOT)
+
+        doc_page = omzdocs.DocumentationPage(md_path.read_text(encoding='UTF-8'))
+
+        # check local link validity
+
+        for url in sorted([ref.url for ref in doc_page.external_references()]):
+            try:
+                components = urllib.parse.urlparse(url)
+            except ValueError:
+                complain(f'{md_path_rel}: invalid URL reference {url!r}')
+                continue
+
+            if components.scheme: # non-local URLs
+                continue
+
+            if components.netloc or components.path.startswith('/'):
+                complain(f'{md_path_rel}: non-relative local URL reference "{url}"')
+                continue
+
+            if not components.path: # self-link
+                continue
+
+            target_path = (md_path.parent / urllib.request.url2pathname(components.path)).resolve()
+
+            if OMZ_ROOT not in target_path.parents:
+                complain(f'{md_path_rel}: URL reference "{url}" points outside the OMZ directory')
+                continue
+
+            if not target_path.is_file():
+                complain(f'{md_path_rel}: URL reference "{url}" target'
+                    ' does not exist or is not a file')
+                continue
+
+            if md_path in index_child_md_links:
+                referenced_md_files.add(target_path)
+
+        # check <omz_dir> reference validity
+
+        for code_span in doc_page.code_spans():
+            if code_span.startswith(omz_reference_prefix):
+                target_path_rel = Path(code_span[len(omz_reference_prefix):])
+                target_path = OMZ_ROOT / target_path_rel
+
+                if ".." in target_path_rel.parts:
+                    complain(f'{md_path_rel}: OMZ reference "{code_span}"'
+                        ' contains a ".." component.')
+                    continue
+
+                if not target_path.exists():
+                    complain(f'{md_path_rel}: OMZ reference "{code_span}" target'
+                        ' does not exist')
+
+        # check for existence of links to README.md files of models and demos
+
+        if md_path in index_child_md_links:
+            for md_file in index_child_md_links[md_path]:
+                if md_file not in referenced_md_files:
+                    complain(f"{md_path_rel}: {md_file.relative_to(OMZ_ROOT)} is not referenced")
+
+        # check for HTML fragments that are unsupported by Doxygen
+
+        for html_fragment in doc_page.html_fragments():
+            match = HTML_FRAGMENT_RE.match(html_fragment)
+            if not match:
+                complain(f'{md_path_rel}: cannot parse HTML fragment {html_fragment!r}')
+                continue
+
+            if match.group(1).lower() not in ALLOWED_HTML_ELEMENTS:
+                complain(f'{md_path_rel}: unknown/disallowed HTML element in {html_fragment!r}')
+                continue
+
+    sys.exit(0 if all_passed else 1)
+
+if __name__ == '__main__':
+    main()

ci/check-release-readiness.py

@@ -32,17 +32,23 @@ def complain(format, *args):
         all_passed = False
 
     for model_dir in OMZ_ROOT.glob('models/*/*/'):
-        # searching recursively, because this could be a composite model
-        has_config = bool(list(model_dir.glob('**/model.yml')))
+        has_model_yml = (model_dir / 'model.yml').is_file()
+        has_composite_model_yml = (model_dir / 'composite-model.yml').is_file()
 
-        has_doc = bool(list(model_dir.glob('**/*.md')))
+        has_config = has_model_yml or has_composite_model_yml
+
+        has_doc = (model_dir / 'README.md').is_file()
 
         if has_config and not has_doc:
             complain('model {} has no documentation', model_dir.name)
 
         if has_doc and not has_config:
             complain('model {} has no config file', model_dir.name)
 
+        if has_composite_model_yml:
+            if not list(model_dir.glob('*/model.yml')):
+                complain('composite model {} has no components', model_dir.name)
+
     for models_lst_path in OMZ_ROOT.glob('demos/**/models.lst'):
         with models_lst_path.open() as models_lst:
             for line_num, line in enumerate(models_lst):

ci/dependencies.yml

@@ -0,0 +1,2 @@
+openvino_linux: '2021.4.563'
+openvino_windows: '2021.4.562'

ci/documentation_updater/documentation_updater.py

@@ -56,12 +56,12 @@ def parse():
     return args
 
 
-def collect_readme(directory, ignored_files):
-    files = {file.stem: file for file in directory.glob('**/*.md') if file.name not in ignored_files}
+def collect_readme(directory):
+    files = {file.parent.name: file for file in directory.glob('**/README.md')}
     logging.info('Collected {} description files'.format(len(files)))
     if not files:
-        logging.error("No markdown file found in {}. Exceptions - {}. Ensure, that you set right directory."
-                      .format(directory, ignored_files))
+        logging.error("No markdown file found in {}. Ensure, that you set right directory."
+                      .format(directory))
         exit(1)
     return files
 
@@ -174,8 +174,7 @@ def main():
     args = parse()
     logging.basicConfig(level=getattr(logging, args.log_level.upper()), format='%(levelname)s: %(message)s')
 
-    ignored_files = ('index.md',)
-    descriptions = collect_descriptions(collect_readme(args.model_dir, ignored_files))
+    descriptions = collect_descriptions(collect_readme(args.model_dir))
     models = get_models_from_configs(args.model_dir)
     update_model_configs(models, descriptions, args.mode)