merge from master

Author: rscohn2

.github/workflows/main.yml

@@ -1,20 +1,28 @@
 name: CI
 
-on: [push]
+on: [push, pull_request]
 
 jobs:
   build:
-
     runs-on: ubuntu-latest
-
     container: rscohn2/oneapi-spec:latest
     steps:
     - uses: actions/checkout@v2
-    - name: build in container
+    - name: Build
       env:
         AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID}}
         AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY}}
       run: |
           python3 scripts/oneapi.py spec-venv
           . spec-venv/bin/activate
           python scripts/oneapi.py --branch $GITHUB_REF ci
+    - name: Archive site
+      uses: actions/upload-artifact@v1
+      with:
+          name: oneapi-spec-artifacts
+          path: site.zip
+    - name: Archive spellcheck
+      uses: actions/upload-artifact@v1
+      with:
+          name: spell check
+          path: build/spelling/output.txt

.gitignore

@@ -1,5 +1,6 @@
 doxygen
 build
+_build
 *~
 
 /docker/requirements.txt
@@ -10,5 +11,6 @@ build
 /.vscode
 /site
 /tarballs
+/site.zip
 
 __pycache__

.gitlab-ci-onetbb.yml

@@ -1,18 +1,55 @@
-build_tbb_spec:
+build_oneTBB_html:
   stage: build
-  tags: [python-sphinx]
+  tags: [docker_linux]
+  image: tr_ubuntu18.04:oneapi-spec
   script:
-    - sphinx-build -b html source/elements/oneTBB/source onetbb_spec
+    - python3 -m pip install -r requirements.txt
+    - python3 scripts/oneapi.py html source/elements/oneTBB
   artifacts:
-    paths: [onetbb_spec]
+    paths: [source/elements/oneTBB/build/html]
+    expire_in: 3d
+
+build_oneTBB_pdf:
+  stage: build
+  tags: [docker_linux]
+  image: tr_ubuntu18.04:oneapi-spec
+  script:
+    - python3 -m pip install -r requirements.txt
+    - python3 scripts/oneapi.py latexpdf source/elements/oneTBB
+  artifacts:
+    paths: [source/elements/oneTBB/build/latex/*.pdf]
+    expire_in: 3d
+
+build_oneDPL_html:
+  stage: build
+  tags: [docker_linux]
+  image: tr_ubuntu18.04:oneapi-spec
+  script:
+    - python3 -m pip install -r requirements.txt
+    - python3 scripts/oneapi.py html source/elements/oneDPL
+  artifacts:
+    paths: [source/elements/oneDPL/build/html]
+    expire_in: 3d
+
+build_oneDPL_pdf:
+  stage: build
+  tags: [docker_linux]
+  image: tr_ubuntu18.04:oneapi-spec
+  script:
+    - python3 -m pip install -r requirements.txt
+    - python3 scripts/oneapi.py latexpdf source/elements/oneDPL
+  artifacts:
+    paths: [source/elements/oneDPL/build/latex/*.pdf]
     expire_in: 3d
 
 pages:
   stage: deploy
   tags: [linux]
-  needs: [build_tbb_spec]
+  needs: [build_oneTBB_html, build_oneDPL_html]
   only: [develop]
   script:
-  - mv onetbb_spec public
+  - mkdir -p public
+  - mv source/elements/oneTBB/build/html public/oneTBB
+  - mv source/elements/oneDPL/build/html public/oneDPL
   artifacts:
     paths: [public]

README.rst

@@ -5,16 +5,16 @@ oneAPI Specifications
 .. image:: https://github.com/oneapi-src/oneapi-spec/workflows/CI/badge.svg
    :target: https://github.com/oneapi-src/oneapi-spec/actions?query=workflow%3ACI
 
-This repo contains the sources for the `oneAPI Specification`_
+This repo contains the sources for the `oneAPI Specification`_.
 
 The document is built with `Sphinx`_ using a theme provided by `Read
-the Docs`_
+the Docs`_.
 
 ---------------------------------
-Editing with Github Web Interface
+Editing with GitHub Web Interface
 ---------------------------------
 
-The simplest and quickest way to edit is directly in the github web
+The simplest and quickest way to edit is directly in the GitHub web
 interface. It has an editor, previewer, and lets you commit
 changes. You won't need to install any local tools. The previewer
 knows how to render RST, but not the sphinx directives so it will not
@@ -32,11 +32,11 @@ to do the same task manually.
 Setup
 -----
 
-Install python 3 and doxygen (>= 1.8.17).  To install on **Ubuntu**::
+Install Python 3 and doxygen (>= 1.8.17).  To install on **Ubuntu**::
 
    sudo scripts/install.sh
 
-Create and activate a python virtual environment with all required tools::
+Create and activate a Python virtual environment with all required tools::
 
   python scripts/oneapi.py spec-venv
   source spec-venv/bin/activate
@@ -45,12 +45,12 @@ To install directly with pip::
 
   pip install -r requirements.txt
 
-On windows::
+On Windows::
 
   python scripts\oneapi.py spec-venv
   spec-venv\Scripts\activate
   
-MKL, DAL, and Level Zero are temporarily in separate private repos. If you have access to the repos you can clone them::
+MKL and Level Zero are temporarily in separate private repos. If you have access to the repos, you can clone them::
 
   python scripts/oneapi.py clones
 
@@ -61,8 +61,8 @@ To build the html document::
 
   python scripts/oneapi.py html
 
-This will not work on windows because we are using symbolic links for
-the elements that are in separate repos. However, windows can build
+This will not work on Windows because we are using symbolic links for
+the elements that are in separate repos. However, Windows can build
 individual specs for individual elements.
 
 The document is organized as a book with chapters. Each element of
@@ -91,7 +91,7 @@ examples. We may not want to try to correct them.
 Editing Tools
 -------------
 
-For simple edits to individual files, you can use the github web
+For simple edits to individual files, you can use the GitHub web
 interface.
 
 **Emacs** has a built-in ReST mode. It does some syntax highlighting and
@@ -125,12 +125,12 @@ You can run a docker container with::
 CI
 --
 
-We use Github actions. See .github/workflows/main.yml
+We use GitHub actions. See `<.github/workflows/main.yml>`_
 
 On every commit, the CI system builds and publishes the document to
 the staging server. To see the URL, look at the end of the log for the
 build step in the CI system. The staging server is an s3 bucket, and
-the access keys are managed as github action secrets. PR's based on
+the access keys are managed as GitHub action secrets. PR's based on
 forks do not have access to the keys and will not publish on the
 staging server.
 

requirements.txt

@@ -8,3 +8,8 @@ gitpython
 Sphinx>=2.4.0
 sphinx-notfound-page
 sphinx-rtd-theme
+sphinx-prompt
+sphinx_substitution_extensions
+sphinx_tabs
+sphinxcontrib-spelling
+graphviz

roadmap.rst

@@ -149,10 +149,6 @@ Date       Milestone
   - Add introduction, execution, primitive lifecycle, and programming model
     sections
 
-- oneCCL
-
-  - Add initial version of API for scale-up
-
 - Level Zero
 
   - Switched to RST as format, using oneapi-spec as upstream repo
@@ -217,8 +213,8 @@ Date       Milestone
 
 - oneCCL
 
-  - Update page with API for scale-up
-  - Update page with GPU programming model
+  - Add page with API for multi-GPU
+  - Add page with multi-GPU programming model
   - Extend page with collective operations
 
 - Level Zero

scripts/cleanup-mkl.py

@@ -0,0 +1,22 @@
+from os import walk
+from os.path import join
+import re
+
+r = re.compile(r'\W+:name: (return-values|include-files|note|note-1|note-2|note-3|syntax|description|input-parameters|output-parameters|example)')
+
+for root, dirs, files in walk("."):
+    for file in files:
+        if not file.endswith(".rst"):
+            continue
+        p = join(root, file)
+        print(p)
+        in_lines = open(p).readlines()
+        l2 = in_lines[2]
+        dup_label = '.. _%s:\n' % l2.rstrip()
+        print('dup_label:', dup_label)
+        with open(p, 'w') as fout:
+            for line in in_lines:
+                if r.match(line) or line == dup_label:
+                    print('omitting:', line)
+                else:
+                    fout.write(line)

scripts/install.sh

@@ -46,6 +46,9 @@ make install
 set -e
 popd
 apt-get install -y \
+                    enchant \
+                    graphviz \
+                    imagemagick \
                     latexmk \
                     texlive-latex-base \
                     texlive-fonts-recommended \

scripts/oneapi.py

@@ -42,14 +42,14 @@
 import subprocess
 import tarfile
 import venv
+from zipfile import ZipFile
 
 sys.path.insert(0, os.path.abspath(join('source','conf')))
 import common_conf
 
 args = None
 
 
-sphinx_opts    = '-q'
 sphinx_build   = 'sphinx-build'
 source_dir     = 'source'
 build_dir      = 'build'
@@ -116,13 +116,14 @@ def makedirs(path):
     os.makedirs(path)
 
 def sphinx(root, target):
-    os.environ['LATEXMKOPTS'] = '--silent'
-    os.environ['LATEXOPTS'] = '-interaction=nonstopmode -halt-on-error'
+    if not args.verbose:
+        os.environ['LATEXMKOPTS'] = '--silent'
+        os.environ['LATEXOPTS'] = '-interaction=nonstopmode -halt-on-error'
     shell('%s -M %s %s %s %s' % (sphinx_build,
                                  target,
                                  join(root,source_dir),
                                  join(root,build_dir),
-                                 sphinx_opts))
+                                 '' if args.verbose else '-q'))
 
 def get_env(var):
     return os.environ[var] if var in os.environ else ''
@@ -151,11 +152,15 @@ def dockerpush(root, target=None):
 def dockerrun(root, target=None):
     root_only(root)
     shell('docker run --rm -it'
+          ' -e http_proxy=%s'
+          ' -e https_proxy=%s'
+          ' -e no_proxy=%s'
           ' --user %s:%s'
           ' --volume=%s:/build'
           ' --workdir=/build'
           ' rscohn2/oneapi-spec'
-          % (os.getuid(), os.getgid(), os.getcwd()))
+          % (get_env('http_proxy'), get_env('https_proxy'), get_env('no_proxy'),
+             os.getuid(), os.getgid(), os.getcwd()))
 
 @action
 def clean(root, target=None):
@@ -203,9 +208,16 @@ def build(root, target):
 @action
 def ci_publish(root, target=None):
     root_only(root)
+    with ZipFile('site.zip', 'w') as site_zip:
+        for r, dirs, files in os.walk('site', topdown=True):
+            # Exclude DAL API because it is 1.7G
+            if os.path.basename(r) == 'oneDAL':
+                dirs = remove_elements(dirs, ['api', '_sources'])
+            for file in files:
+                site_zip.write(join(r, file))
     if not args.branch:
         exit('Error: --branch <branchname> is required')
-    if 'AWS_SECRET_ACCESS_KEY' in os.environ:
+    if 'AWS_SECRET_ACCESS_KEY' in os.environ and os.environ['AWS_SECRET_ACCESS_KEY'] != '':
         shell('aws s3 sync --only-show-errors --delete site s3://%s/exclude/ci/branches/%s' % (staging_host, args.branch))
         log('published at http://staging.spec.oneapi.com.s3-website-us-west-2.amazonaws.com/exclude/ci/branches/%s/'
             % (args.branch))
@@ -311,15 +323,24 @@ def purge(root, target=None):
     root_only(root)
     for (r,dirs,files) in os.walk('site', topdown=True):
         r = r.replace('site/','')
-        dirs = remove_elements(dirs,['oneDAL', 'oneL0', 'oneMKL'])
+        dirs = remove_elements(dirs,['oneL0'])
         for file in files:
             print('http://spec.oneapi.com/%s/%s' % (r, file))
 
+@action
+def sort_words(root, target=None):
+    with open(join('source', 'spelling_wordlist.txt')) as fin:
+        lines = fin.readlines()
+    with open(join('source', 'spelling_wordlist.txt'), 'w') as fout:
+        for l in sorted(list(set(lines))):
+            fout.write(l)
+        
 @action
 def ci(root, target=None):
     root_only(root)
     get_tarballs(root)
     site(root)
+    build('.', 'spelling')
     if args.branch == 'publish' or args.branch == 'refs/heads/publish':
         stage_publish(root)
     else:
@@ -336,10 +357,12 @@ def ci(root, target=None):
             'dockerrun': dockerrun,
             'html': build,
             'latexpdf': build,
+            'spelling': build,
             'prep': prep,
             'prod-publish': prod_publish,
             'purge': purge,
             'site': site,
+            'sort-words': sort_words,
             'spec-venv': spec_venv,
             'stage-publish': stage_publish}
 
@@ -352,16 +375,15 @@ def ci(root, target=None):
         'oneDPL',
         'oneDNN']
 
-tarballs = ['oneMKL',
-            'oneL0',
-            'oneDAL']
+tarballs = ['oneL0']
 
 def main():
     global args
     parser = argparse.ArgumentParser(description='Build oneapi spec.')
-    parser.add_argument('action',choices=commands.keys())
+    parser.add_argument('action',choices=commands.keys(), default='html', nargs='?')
     parser.add_argument('root', nargs='?', default='.')
     parser.add_argument('--branch')
+    parser.add_argument('--verbose', action='store_true')
     parser.add_argument('--dry-run', action='store_true')
     args = parser.parse_args()