25.14_ext Release (#98)

Co-authored-by: RahulJaikumar <kelly.deng@intel.com>

Author: sashwin04

CMakeLists.txt

@@ -33,56 +33,62 @@ else()
     endif()
     set(PLATFORM "${platform}")
 endif()
+include(official-release OPTIONAL)
+execute_process(COMMAND bash -c "GIT_SSH_COMMAND='ssh -o BatchMode=yes' GIT_ASKPASS=echo git fetch --tags > /dev/null;GIT_ASKPASS=echo git tag | sort -V" TIMEOUT 10 OUTPUT_VARIABLE official_releases RESULT_VARIABLE status OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_QUIET)
+if(NOT (status EQUAL 0))
+    set(official_releases "")
+endif()
 
-include(registry OPTIONAL)
-execute_process(COMMAND bash -c "GIT_SSH_COMMAND='ssh -o BatchMode=yes' GIT_ASKPASS=echo git fetch --tags > /dev/null;GIT_ASKPASS=echo git tag | sort -V" TIMEOUT 10 OUTPUT_VARIABLE tmp RESULT_VARIABLE status OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_QUIET)
-if (RELEASE MATCHES "^:?v?[0-9]+[.][0-9]+[.]*[0-9]*$")
+if (RELEASE MATCHES "^:?v?[0-9][0-9]*[.][0-9][0-9]*")
     string(REPLACE ":" "" release1 "${RELEASE}")
-    if ((status EQUAL 0) AND (tmp MATCHES "${release1}"))
+    string(REPLACE "." "[.]" release1 "${release1}")
+    if (official_releases MATCHES "${release1}")
         if((NOT DEFINED REGISTRY) OR (NOT REGISTRY))
             set(REGISTRY "${DEFAULT_REGISTRY}")
         endif()
     endif()
 endif()
 
-string(REGEX REPLACE ".*\n" "" tmp "${tmp}")
-string(REGEX REPLACE "(v?[0-9]*[.][0-9]*)[.].*" "\\1" mtmp "${tmp}")
-if (tmp AND (status EQUAL 0))
-    if ((NOT RELEASE STREQUAL ":${tmp}") AND (NOT RELEASE STREQUAL "${tmp}"))
+if(NOT (EXISTS "${PROJECT_SOURCE_DIR}/.hybrid_release"))
+    string(REGEX REPLACE ".*\n" "" tmp "${official_releases}")
+    string(REGEX REPLACE "(v?[0-9]*[.][0-9]*)[.].*" "\\1" mtmp "${tmp}")
+    if (tmp)
+        if ((NOT RELEASE STREQUAL ":${tmp}") AND (NOT RELEASE STREQUAL "${tmp}"))
+            message("")
+            message("*************************************************************")
+            message("")
+            if (tmp STREQUAL mtmp)
+                message("${green}INFO:${reset} Detected major release ${mtmp}.")
+            else()
+                message("${green}INFO:${reset} Detected major release ${mtmp} and minor releases up to ${tmp}.")
+                message("${green}INFO:${reset} Minor releases do not cover all workloads. See workload README.")
+            endif()
+            message("${green}INFO:${reset} Switch to the latest release as follows:")
+            string(REPLACE "v" "" btmp "${tmp}")
+            execute_process(COMMAND bash -c "GIT_SSH_COMMAND='ssh -o BatchMode=yes' GIT_ASKPASS=echo git branch --no-color -a --contains tags/${tmp} | grep -E 'remotes/.*/${btmp}$' | tail -n1 | tr -d ' '" TIMEOUT 5 OUTPUT_VARIABLE branch_id OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_QUIET)
+            if(branch_id MATCHES "remotes")
+                string(REPLACE "remotes/" "" branch_id "${branch_id}")
+            else()
+                set(branch_id "tags/${tmp}")
+            endif()
+            message("${green}INFO:${reset} git checkout tags/${tmp} # ${branch_id} with hotfix" )
+            message("${green}INFO:${reset} cmake -DRELEASE=${tmp} ..")
+            message("")
+            message("*************************************************************")
+            message("")
+        endif()
+    else()
         message("")
-        message("*************************************************************")
+        message("****************************************************************************")
         message("")
-        if (tmp STREQUAL mtmp)
-            message("${green}INFO:${reset} Detected major release ${mtmp}.")
-        else()
-            message("${green}INFO:${reset} Detected major release ${mtmp} and minor releases up to ${tmp}.")
-            message("${green}INFO:${reset} Minor releases do not cover all workloads. See workload README.")
-        endif()
-        message("${green}INFO:${reset} Switch to the latest release as follows:")
-        string(REPLACE "v" "" btmp "${tmp}")
-        execute_process(COMMAND bash -c "GIT_SSH_COMMAND='ssh -o BatchMode=yes' GIT_ASKPASS=echo git branch --no-color -a --contains tags/${tmp} | grep -E 'remotes/.*/${btmp}$' | tail -n1 | tr -d ' '" TIMEOUT 5 OUTPUT_VARIABLE branch_id OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_QUIET)
-        if(branch_id MATCHES "remotes")
-            string(REPLACE "remotes/" "" branch_id "${branch_id}")
-        else()
-            set(branch_id "tags/${tmp}")
-        endif()
-        message("${green}INFO:${reset} git checkout tags/${tmp} # ${branch_id} with hotfix" )
-        message("${green}INFO:${reset} cmake -DRELEASE=${tmp} ..")
+        message("${red}WARNING:${reset} Failed to detect any official release.")
+        message("  Switch to any release as follows:")
         message("")
-        message("*************************************************************")
+        message("git checkout <VERSION>")
+        message("cmake -DREGISTRY=${DEFAULT_REGISTRY} -DRELEASE=v<VERSION> ..")
         message("")
+        message("****************************************************************************")
     endif()
-else()
-    message("")
-    message("****************************************************************************")
-    message("")
-    message("${red}WARNING:${reset} Failed to detect any official release.")
-    message("  Switch to any release as follows:")
-    message("")
-    message("git checkout <VERSION>")
-    message("cmake -DREGISTRY=${DEFAULT_REGISTRY} -DRELEASE=v<VERSION> ..")
-    message("")
-    message("****************************************************************************")
 endif()
 
 if (NOT ${REGISTRY} MATCHES "/$")
@@ -172,7 +178,7 @@ if(COMMAND detect_backend_warnings)
     detect_backend_warnings()
 endif()
 
-if (RELEASE MATCHES "^:v[0-9]+[.][0-9]+[.]*[0-9]*$")
+if ((RELEASE MATCHES "^:v[0-9]+[.][0-9]+[.]*[0-9]*$") AND (NOT (EXISTS "${PROJECT_SOURCE_DIR}/.hybrid_release")))
     string(REPLACE ":" "" release1 "${RELEASE}")
     if((NOT REGISTRY STREQUAL "${DEFAULT_REGISTRY}") AND (NOT REGISTRY STREQUAL "${DEFAULT_REGISTRY}/") AND DEFAULT_REGISTRY)
         message("${red}WARNING:${reset} Issue 'cmake -DREGISTRY=${DEFAULT_REGISTRY} ..' to use official ${release1} images")

CONTRIBUTING.md

@@ -1,57 +1,29 @@
-# Contributing
+## Welcome to the Service Framework Workload Repository Contributing Guide
 
-### License
+### Evaluate Workloads
 
-<PROJECT NAME> is licensed under the terms in [LICENSE]<link to license file in repo>. By contributing to the project, you agree to the license and copyright terms therein and release your contribution under these terms.
+Follow the [README](README.md#prerequisite) instructions to setup [local](doc/user-guide/preparing-infrastructure/setup-docker.md), [remote](doc/user-guide/preparing-infrastructure/setup-cumulus.md), or [Cloud](doc/user-guide/preparing-infrastructure/setup-cumulus.md) systems to evaluate any [supported workloads](worklod/README.md#list-of-workloads). 
 
-### Sign your work
+You can choose to build the workloads and evaluate the workload execution with `ctest`, which manage the workload test cases. You can run any subset of the test cases or all of them. 
 
-Please use the sign-off line at the end of the patch. Your signature certifies that you wrote the patch or otherwise have the right to pass it on as an open-source patch. The rules are pretty simple: if you can certify
-the below (from [developercertificate.org](http://developercertificate.org/)):
+### Submit Issues
 
-```
-Developer Certificate of Origin
-Version 1.1
+If you spot a problem with the repository, submit an issue at the **github issues**.  
 
-Copyright (C) 2004, 2006 The Linux Foundation and its contributors.
-660 York Street, Suite 102,
-San Francisco, CA 94110 USA
+### Contribute to Workload Development
 
-Everyone is permitted to copy and distribute verbatim copies of this
-license document, but changing it is not allowed.
+Here is a list of references you can follow for workload development:
+- A workload consists of a few critical pieces of scripts or manifests, documented in [Workload Elements](doc/developer-guide/component-design/workload.md):
+  - [`CMakeLists.txt`](doc/developer-guide/component-design/cmakelists.md)  
+  - [`build.sh`](doc/developer-guide/component-design/build.md)  
+  - [`Dockerfiles`](doc/developer-guide/component-design/dockerfile.md)  
+  - [`cluster-config.yaml.m4`](doc/developer-guide/component-design/cluster-config.md)  
+  - [`kubernetes-config.yaml.m4`](doc/developer-guide/component-design/kubernetes-config.md)  
+  - [`validate.sh`](doc/developer-guide/component-design/validate.md)  
+  - [`kpi.sh`](doc/developer-guide/component-design/kpi.md)  
+- The best way to start a new workload development is by copying the [dummy](workload/dummy) workload and then modifying it to your needs. 
 
-Developer's Certificate of Origin 1.1
+### Submit Contributions
 
-By making a contribution to this project, I certify that:
+Thanks for your contribution to the Service Framework workload repository. Whether you plan to modify an existing workload or to create a new workload, please fork the SF workload repository to your own private work place. Make modifications there. Then submit a merge request. The branches of the main repository are reserved for release-related activities.  
 
-(a) The contribution was created in whole or in part by me and I
-    have the right to submit it under the open source license
-    indicated in the file; or
-
-(b) The contribution is based upon previous work that, to the best
-    of my knowledge, is covered under an appropriate open source
-    license and I have the right under that license to submit that
-    work with modifications, whether created in whole or in part
-    by me, under the same open source license (unless I am
-    permitted to submit under a different license), as indicated
-    in the file; or
-
-(c) The contribution was provided directly to me by some other
-    person who certified (a), (b) or (c) and I have not modified
-    it.
-
-(d) I understand and agree that this project and the contribution
-    are public and that a record of the contribution (including all
-    personal information I submit with it, including my sign-off) is
-    maintained indefinitely and may be redistributed consistent with
-    this project or the open source license(s) involved.
-```
-
-Then you just add a line to every git commit message:
-
-    Signed-off-by: Joe Smith <joe.smith@email.com>
-
-Use your real name (sorry, no pseudonyms or anonymous contributions.)
-
-If you set your `user.name` and `user.email` git configs, you can sign your
-commit automatically with `git commit -s`.

README.md

@@ -4,15 +4,13 @@
 
 ### Introduction
 
-Welcome to the  **Workload Services Framework** repository. The repository contains a set of workloads optimized for Intel(R) Xeon(R) platforms. You can find the list of supported workloads in the [workload](workload) directory.  
+This is the **Workload Services Framework** repository. The repository contains a set of workloads optimized for Intel(R) Xeon(R) platforms. See the list of supported workloads under the [workload](workload) directory.  
 
 ### Prerequisite
 
-Before you begin, ensure the following:
-
-- Sync your system date and time. This is required for any credential authorization.  
+- Sync your system date/time. This is required by any credential authorization.  
 - If you are behind a corporate firewall, please setup `http_proxy`, `https_proxy` and `no_proxy` in `/etc/environment`, and source the settings into the current shell environment.  
-- Run the [`setup-dev.sh`](doc/user-guide/preparing-infrastructure/setup-wsf.md#setup-devsh) script to setup the development host for workload development and evaluation. Refer to [Cloud and On-Premises Setup](doc/user-guide/preparing-infrastructure/setup-wsf.md) for additional SUT setup. SUT stands for `System Under Test`, or `workload test machines`.   
+- Run the [`setup-dev.sh`](doc/user-guide/preparing-infrastructure/setup-wsf.md#setup-devsh) script to setup the development host for workload development and evaluation. See [Cloud and On-Premises Setup](doc/user-guide/preparing-infrastructure/setup-wsf.md) for additional SUT setup. SUT stands for System Under Test, or workload test machines.   
 
 ### Evaluate Workload
 
@@ -36,9 +34,6 @@ The WSF supports multiple validation backends. By default, the [terraform](doc/u
 
 ### Build Workload
 
-To build a workload, use the following commands:
-
-
 ```
 mkdir -p build
 cd build
@@ -50,16 +45,14 @@ make
 
 > TIP: You can specify `BENCHMARK` to limit the repository scope to the specified workload. The build and test operations on all other workloads are disabled. See [Build Options](doc/user-guide/executing-workload/cmake.md) for details.  
 
-Alternatively, you can use:
-
 ```
 cd build
 cmake -DBENCHMARK=ResNet-50
 make
 ./ctest.sh -N
 ```
 
-### Additional Resources
+### See Also
 
 - [Build Options](doc/user-guide/executing-workload/cmake.md)   
 - [Test Options](doc/user-guide/executing-workload/ctest.md)   

Security.md

@@ -0,0 +1,6 @@
+# Security Policy
+Intel is committed to rapidly addressing security vulnerabilities affecting our customers and providing clear guidance on the solution, impact, severity and mitigation. 
+
+## Reporting a Vulnerability
+Please report any security vulnerabilities in this project [utilizing the guidelines here](https://www.intel.com/content/www/us/en/security-center/vulnerability-handling-guidelines.html).
+

doc/developer-guide/component-design/docker-config.md

@@ -23,7 +23,13 @@ worker-0:
 where
 - The top level keys are the SUT hostnames. The number of the SUT hosts must match what is specified in `cluster-config.yaml`. The SUT hosts are named against their SUT workgroup. For example, for the workers, the SUT hosts are named as `worker-0`, `worker-1`, etc. For clients, the SUT hosts are named as `client-0`, `client-1`, etc.  
 - The value of each SUT host is a list of containers to be scheduled on the SUT host. The list order is not enforced.  
-- Each container is described as a dictionary of `image` (the full docker image name), `options` (the docker run command line arguments, as a string or a list), `command` (optional, will overwrite commands inside container if defined) and `export-logs` (whether logs should be collected on the container).   
+- Each container is described as a dictionary of
+  - `image`: Specify the full docker image name
+  - `options`: Specify the docker run command line arguments, as a string or a list.
+  - `command`: Optional. Specify any startup command. This will overwrite whatever is defined in the docker image.
+  - `export-logs` or `service-logs`: Optional. Specify whether logs should be collected on the container.
+
+> The script will first collect logs on containers whose `export-logs` is true, which also signals that the workload execution is completed. Then collect logs on containers whose `service-logs` is `true`. `export-logs` and `service-logs` are exclusive options and can not both be true.
 
 # Test Time Considerations
 

doc/developer-guide/component-design/kubernetes-config.md

@@ -43,49 +43,9 @@ spec:
 
 ### About `imagePullPolicy`
 
-To ensure that the validation runs always on the latest code, it is recommended to use `imagePullPolicy: Always`. However, this requires to use a private docker registry. In a local development, `imagePullPolicy: IfNotPresent` is desired. 
+To ensure that the validation runs always on the latest code, it is recommended to use `imagePullPolicy: Always`.
 
-If you use the `.m4`, `.j2`, or helm template, the variable `IMAGEPULLPOLICY` is defined to be either `IfNotPresent` or `Always`.  
-
-```m4
-...
-    spec:
-      containers:
-      - name: database
-        image: IMAGENAME(wordpress5mt-defn(`DATABASE'))
-        imagePullPolicy: IMAGEPULLPOLICY
-...
-```
-
-If you use the `.j2` template, use the following conditions:
-
-```jinja
-...
-spec:
-...
-    spec:
-      containers:
-      - name: database
-        image: "{{ REGISTRY }}wordpress5mt-{{ DATABASE }}{{ IMAGESUFFIX }}{{ RELEASE }}"
-        imagePullPolicy: "{{ IMAGEPULLPOLICY }}"
-...
-```
-
-If you use helm charts, use the following conditions:
-
-```jinja
-...
-spec:
-...
-    spec:
-      containers:
-      - name: database
-        image: "{{ REGISTRY }}wordpress5mt-{{ DATABASE }}{{ IMAGESUFFIX }}{{ RELEASE }}"
-        imagePullPolicy: "{{ IMAGEPULLPOLICY }}"
-...
-```
-
-Not all docker images are built equally. Some are less frequently updated and less sensitive to performance. Thus it is preferrable to use `imagePullPolicy: IfNotPresent` in all cases.   
+Not all docker images are built equally. Some are less frequently updated and less sensitive to performance. Thus it is preferrable to use `imagePullPolicy: IfNotPresent`.
 
 ### About `podAntiAffinity`
 

doc/user-guide/collecting-results/list-kpi.md

@@ -16,14 +16,10 @@ Execute this command in the workload folder to fetch KPIs:
 Usage: [options] [logs-directory]
 --primary             List only the primary KPI.  
 --all                 List all KPIs.  
---outlier <n>         Remove outliers beyond N-stdev.  
 --params              List workload configurations.  
 --sutinfo             List sutinfo information.   
---format list|xls-list|xls-ai|xls-inst|xls-table  
+--format list|xls-list
                       Specify the output format.
---var[1-9] <value>    Specify the spread sheet variables.   
---filter _(real|throughput)
-                      Specify a trim filter to shorten spreadsheet name.  
 --file <filename>     Specify the spread sheet filename. 
 --uri                 Show the WSF portal URI if present.   
 --intel_publish       Publish to the WSF dashboard. 
@@ -32,26 +28,3 @@ Usage: [options] [logs-directory]
 --tags <tags>         Set the publisher tags.  
 --recent              List KPIs for recent testcases.  
 ```
-
-The `xls-ai` option writes the KPI data in the `kpi-report.xls` spread sheet as follows:
-
-![image-ss-ai][image-ss-ai]
-    
-where `--var1=batch_size` `--var2=cores_per_instance` `--var3='*Throughput'` `--var4=Throughput_`.
-
-The `xls-inst` option writes the KPI data in the `kpi-report.xls` spread sheet as follows:
-
-![image-ss-inst][image-ss-inst]
-    
-The `xls-table` option writes the KPI data in the `kpi-report.xls` spread sheet as follows:
-
-![image-ss-table][image-ss-table]
-    
-where `--var1=scale`, `--var2=sleep_time`. Optionally, you can specify `--var3` and `--var4` variables for multiple tables in the same spreadsheet.
-
-[validate.sh]: ../../developer-guide/component-design/validate.md
-[Cloud SUT Reuse]: #cloud-sut-reuse
-
-[image-ss-ai]: ../../image/ss-ai.png
-[image-ss-inst]: ../../image/ss-inst.png
-[image-ss-table]: ../../image/ss-table.png

doc/user-guide/preparing-infrastructure/setup-wsf.md

@@ -102,8 +102,8 @@ where, if Kubernetes is used, the SUT hosts are assumed to form a Kubernetes clu
 
 Use the following setup steps:
 - Run the [`setup-dev.sh`][setup-dev.sh-self] script to setup the dev host. 
-- Completely optional in this setup, run the [`setup-reg.sh`][setup-reg.sh-self] script (on the dev host), if you plan to setup a local docker registry for building workloads and storing the built images.   
-- Depending on the workload types, you can run either the [`setup-sut-native.sh`][setup-sut-native.sh-self], [`setup-sut-docker.sh`][setup-sut-docker.sh-self] script or the [`setup-sut-k8s.sh`][setup-sut-k8s.sh-self] script to setup the SUT hosts. The native setup can run any baremetal native workloads. The docker setup can run most of the single-node containerized workloads (docker or docker compose). The Kubernetes setup can run all containerized workloads (not tied to any Cloud services) on premises. 
+- Completely optional in this setup, run the [`setup-reg.sh`][setup-reg.sh-self] script (**on the dev host**), if you plan to setup a local docker registry for building workloads and storing the built images.
+- Depending on the workload types, you can run either the [`setup-sut-native.sh`][setup-sut-native.sh-self], [`setup-sut-docker.sh`][setup-sut-docker.sh-self] script or the [`setup-sut-k8s.sh`][setup-sut-k8s.sh-self] script (**on the dev host**) to setup the SUT hosts. The native setup can run any baremetal native workloads. The docker setup can run most of the single-node containerized workloads (docker or docker compose). The Kubernetes setup can run all containerized workloads (not tied to any Cloud services) on premises.
 
 ---