Update DPF-Post module (0.2.0) regarding the DPF-Core upgrade (0.3.0) (#21)

* First part of the unit tests have been updated and works now.

* More tests are now succeding.

* All tests are succeeding.

* op_average removed (not necessary anymore because ElectricField have been fixed)

* Tests added (compare result_objects values to dpf.core operators outputs)

* Enhancement of tests using dpf comparison operator

* Enhancement of unit testing.

* Fix plotting with labels helpers

* Templates and script to build doc added to the .ci

* Permission set up for Linux build

* ansys-dpf-post relies on ansys-dpf-core 0.3.0 only

* .ci for DPF-Post (DPF-Core 0.3.0 have been published)

* Add pytest-reruns to test_requirements and fix the docs/_build folder for documentation

* Don't start server before testing anymore

* Linux chmod added

* Add session fixture to close server at the end of the tests

* Remove check server log step for Windows .ci

* .ci of Documentation enabled

* Reset azure-pipelines.yml and fix ci-build.yml

* Remove ci-build.yml (doc is now generated under .ci/azure-pipelines.yml)

* Template for kill-server-linux step

* Add template for kill-server-linux

* Remove the copying example files step

* Splitting test that were plotting (specific plotting test have been added)

* Use of np.isclose

Co-authored-by: Lea Paradis <[email protected]>

Author: anslpa

.ci/azure-pipelines.yml

@@ -26,76 +26,58 @@ jobs:
     python.version: '3.8'
     DISPLAY: ':99.0'
     PYANSYS_OFF_SCREEN: True
-    DPF_PORT: 32772
-    DPF_START_SERVER: False
   pool:
     vmImage: 'windows-2019'
 
   steps:
-    - powershell: |
-        powershell .ci/install_opengl.ps1
-        .ci/setup_headless_display.sh
-        pip install -r .ci/requirements_test_xvfb.txt
-        python .ci/display_test.py
-      displayName: Install and start a virtual framebuffer
-
-    - task: UsePythonVersion@0
-      inputs:
-        versionSpec: $(python.version)
-        addToPath: true
-
-    - task: PipAuthenticate@1
+    - template: templates\prepare-environment-windows.yml   
+    
+    - task: PublishBuildArtifacts@1
+      displayName: 'WHEEL: publish artifacts'
       inputs:
-        artifactFeeds: 'pyansys'
-        onlyAddExtraIndex: true
-
-    - script: |
-        pip install -r requirements_build.txt
-        python setup.py bdist_wheel
-        pip install --find-links=dist ansys_dpf_post
-        cd tests
-        python -c "from ansys.dpf import post; print(post.Report(gpu=False))"
-      displayName: Install ansys-dpf-post
-
-    - task: UniversalPackages@0
-      inputs:
-        command: 'download'
-        downloadDirectory: '$(System.DefaultWorkingDirectory)'
-        feedsToUse: 'internal'
-        vstsFeed: '705e121a-9631-49f5-8aaf-c7142856f923'
-        vstsFeedPackage: 'f913c1d3-1fe4-404c-8c28-15a234e56803'
-        vstsPackageVersion: '21.1.4'
-
-    - script: |
-        @echo on
-        dir $(System.DefaultWorkingDirectory)
-        set THISDIR=$(System.DefaultWorkingDirectory)
-        set PATH=%THISDIR%\server\v211\tp\IntelMKL\2020.0.166\winx64\;%THISDIR%\server\v211\tp\hdf5\1.8.14\winx64\;%THISDIR%\server\v211\tp\CFFSDK\lib\winx64;%THISDIR%\res_files\;%PATH%
-        cd %THISDIR%\server\v211\aisol\bin\winx64
-        START /B Ans.Dpf.Grpc.exe --address 127.0.0.1 --port %DPF_PORT% > log.txt 2>&1
-        python -c "from ansys.dpf import core; core.connect_to_server(port=$(DPF_PORT)); print('Python Connected')"
-      displayName: Start DPF Server
+        PathtoPublish: '$(System.DefaultWorkingDirectory)\dist'
+        ArtifactName: 'ansys_dpf_post_wheel'
+      enabled: true
 
     - script: |
         pip install -r requirements_test.txt
-        cd tests
-        pytest -v --junitxml=junit/test-results.xml --cov ansys.dpf.post --cov-report=xml
+        set THISDIR=$(System.DefaultWorkingDirectory)
+        cd tests=$(System.DefaultWorkingDirectory)
+        set AWP_ROOT212=%THISDIR%\server\v212
+        pytest -v --junitxml=junit/test-results.xml --cov ansys.dpf.post --cov-report=xml --reruns 3
+        echo %PATH%
+        
       displayName: Test Post API
-
-    - script: |
-        type $(System.DefaultWorkingDirectory)\server\v211\aisol\bin\winx64\log.txt
-      displayName:  'Show DPF Server Logs'
+      
+    - task: PublishTestResults@2
+      inputs:
+        testResultsFormat: 'JUnit'
+        testResultsFiles: 'junit/test-results.xml' 
+        testRunTitle: 'windowsTests'
+        publishRunAttachments: true      
       condition: always()
+      
+    - script: |
+        pip install twine
+        python setup.py sdist
+        twine upload --skip-existing dist/*
+      displayName: 'Upload to PyPi'
+      condition: contains(variables['Build.SourceBranch'], 'refs/tags/')
+      env:
+        TWINE_USERNAME: __token__
+        TWINE_PASSWORD: $(PYPI_TOKEN)
+        TWINE_REPOSITORY_URL: "https://upload.pypi.org/legacy/"   
+         
+    - template: templates\kill-servers-windows.yml 
+    
 
 - job: Linux
   variables:
     python.version: '3.7'  # due to VTK 8.1.2 requirement for docbuild
     DISPLAY: ':99.0'
     PYANSYS_OFF_SCREEN: True
-    DPF_PORT: 32772
-    DPF_START_SERVER: False
-    DPF_IMAGE: docker.pkg.github.com/pyansys/dpf-core/dpf:v2021.1
-    DPF_DOCKER: True
+    TEMP: $(System.DefaultWorkingDirectory)/temp
+    
   pool:
     vmImage: 'ubuntu-20.04'
   steps:
@@ -104,6 +86,11 @@ jobs:
         versionSpec: '$(python.version)'
       displayName: 'Use Python $(python.version)'
 
+    - task: PipAuthenticate@1
+      inputs:
+        artifactFeeds: 'pyansys'
+        onlyAddExtraIndex: true
+
     - script: |
         ls -lh .ci
         chmod +x .ci/setup_headless_display.sh
@@ -115,43 +102,107 @@ jobs:
     - script: |
         pip install -r requirements_build.txt
         python setup.py bdist_wheel
-        pip install --find-links=dist ansys_dpf_post
+        export WHEELNAME=`ls dist/*.whl`
+        echo ${WHEELNAME}
+        pip install ${WHEELNAME}
         cd tests
         python -c "from ansys.dpf import post; print(post.Report())"
       displayName: Install ansys-dpf-post
 
-    # this step is necessary as we're running in docker and the
-    # examples directory must be local to the docker enviornment
-    - script: |
-       EXAMPLES_DIR=$(python -c "from ansys.dpf.core import examples; import os; print(os.path.dirname(examples.__file__))")
-       mkdir -p ansys/dpf/core
-       cp -r $EXAMPLES_DIR ansys/dpf/core
-      displayName: Copy ansys.dpf.core examples to local directory
-
+    - task: UniversalPackages@0
+      inputs:
+        command: 'download'
+        downloadDirectory: '$(System.DefaultWorkingDirectory)'
+        feedsToUse: 'internal'
+        vstsFeed: '705e121a-9631-49f5-8aaf-c7142856f923'
+        vstsFeedPackage: 'dpf-linux' #TODO: update hash of packages
+        vstsPackageVersion: '21.2.5'
+        
+    - template: templates\kill-servers-linux.yml 
+      
     - script: |
-        set -ex
-        echo $(PAT) | docker login -u $(GH_USERNAME) --password-stdin docker.pkg.github.com
-        docker pull $(DPF_IMAGE)
-        touch log.txt
-        docker run --restart always --name dpf -v `pwd`:/dpf -v /tmp:/dpf/_cache -p $(DPF_PORT):50054 $(DPF_IMAGE) > log.txt &
-        grep -q 'server started on ip' <(timeout 60 tail -f log.txt)
-        python -c "from ansys.dpf import core as dpf; dpf.connect_to_server(port=$(DPF_PORT)); print('Python Connected')"
-      displayName: Pull, launch, and validate DPF service
+        env
+      displayName: Display env
 
     - script: |
         pip install -r requirements_test.txt
-        pip install pytest-azurepipelines
+        pip install pytest-azurepipelines        
+        export AWP_ROOT212=${SYSTEM_DEFAULTWORKINGDIRECTORY}/server/v212
+        chmod 755 ${AWP_ROOT212}/aisol/bin/linx64/Ans.Dpf.Grpc.sh
+        chmod 755 ${AWP_ROOT212}/aisol/bin/linx64/Ans.Dpf.Grpc.exe   
         cd tests
-        pytest -v --junitxml=junit/test-results.xml --cov ansys.dpf.post --cov-report=xml --cov-report=html
+        pytest -v --junitxml=junit/test-results.xml --cov ansys.dpf.post --cov-report=xml  --reruns 3      
+        export PATH=`pwd`
+        echo ${PATH} 
       displayName: Test Post API
+      
+    - task: PublishTestResults@2
+      inputs:
+        testResultsFormat: 'JUnit'
+        testResultsFiles: 'junit/test-results.xml' 
+        testRunTitle: 'linuxTests'
+        publishRunAttachments: true
+        searchFolder: 'tests/'
+      condition: always()
+      
+      
+    - template: templates\kill-servers-linux.yml 
+      
+- job: DocumentationWindows
+  variables:
+    python.version: '3.8'
+    DISPLAY: ':99.0'
+    PYANSYS_OFF_SCREEN: True
+    DPF_PORT: 32772
+    GH_DOC_BRANCH: 'gh-pages'
+  pool:
+    vmImage: 'windows-2019'
 
+  steps:  
+    - template: templates\prepare-environment-windows.yml    
+   
     - script: |
-        pip install twine
-        python setup.py sdist
-        twine upload --skip-existing dist/*
-      displayName: 'Upload to PyPi'
+          pip install -r requirements_docs.txt
+          cd .ci
+          dir 
+          set THISDIR=$(System.DefaultWorkingDirectory)
+          set AWP_ROOT212=%THISDIR%\server\v212
+          ECHO %AWP_ROOT212%
+          build_doc.bat
+      displayName: Build Documentation
+        
+    - task: ArchiveFiles@2
+      inputs:
+        rootFolderOrFile: '$(System.DefaultWorkingDirectory)\docs\_build' 
+        includeRootFolder: false 
+        archiveType: 'zip'
+        archiveFile: '$(System.DefaultWorkingDirectory)\docs\archive\doc-ansys-dpf-post.zip'
+        replaceExistingArchive: true 
+      displayName: 'DOCUMENTATION: zip artifacts'
+    
+    - task: PublishBuildArtifacts@1
+      displayName: 'DOCUMENTATION: publish artifacts'
+      inputs:
+        PathtoPublish: '$(System.DefaultWorkingDirectory)\docs\archive'
+        ArtifactName: doc-ansys-dpf-post
+      enabled: true      
+        
+    - powershell: |
+        git init
+        git checkout -b $(GH_DOC_BRANCH)
+        git config --global user.name "pyansys-ci-bot"
+        git config --global user.email "$(GH_EMAIL)"
+        New-Item -ItemType file .nojekyll
+        git add .
+        git commit -m "Documentation generated by $(Build.DefinitionName)"
+      displayName: "Init git and add docs"
+      workingDirectory: docs\_build\html
+
+    - script: |
+        git remote add origin https://$(GH_PAT)@github.com/pyansys/DPF-Post-docs
+        git push -u origin $(GH_DOC_BRANCH) --force
+      displayName: "Publish GitHub Pages merge commit"
+      workingDirectory: docs\_build\html
       condition: contains(variables['Build.SourceBranch'], 'refs/tags/')
-      env:
-        TWINE_USERNAME: __token__
-        TWINE_PASSWORD: $(PYPI_TOKEN)
-        TWINE_REPOSITORY_URL: "https://upload.pypi.org/legacy/"
+          
+    - template: templates\kill-servers-windows.yml   

.ci/build_doc.bat

@@ -0,0 +1,8 @@
+ECHO %AWP_ROOT212%
+set SPHINX_APIDOC_OPTIONS=inherited-members
+call sphinx-apidoc -o ../docs/source/api ../ansys -f --implicit-namespaces --separate  --no-headings
+pushd .
+cd ../docs/
+call make clean
+call make html
+popd

.ci/templates/kill-servers-linux.yml

@@ -0,0 +1,10 @@
+steps:  
+- script : |
+    echo $0
+    if pgrep -x "Ans.Dpf.Grpc" > /dev/null
+    then
+        pkill -f Ans.Dpf.Grpc.exe
+    fi
+  displayName: 'Kill all servers'
+  condition: always()
+  continueOnError: true

.ci/templates/kill-servers-windows.yml

@@ -0,0 +1,8 @@
+steps:  
+    - script : |
+        tasklist /FI "IMAGENAME eq Ans.Dpf.Grpc.exe" 2>NUL | find /I /N "Ans.Dpf.Grpc.exe">NUL
+        ECHO %ERRORLEVEL%
+        if "%ERRORLEVEL%"=="0"(taskkill /f /im Ans.Dpf.Grpc.exe)
+      displayName: 'Kill all servers'
+      condition: always()
+      continueOnError: true

.ci/templates/prepare-environment-windows.yml

@@ -0,0 +1,44 @@
+steps:  
+    - powershell: |
+        Set-StrictMode -Version Latest
+        $ErrorActionPreference = "Stop"
+        $PSDefaultParameterValues['*:ErrorAction']='Stop'
+        git clone --depth 1 git://github.com/pyvista/gl-ci-helpers.git
+        powershell gl-ci-helpers/appveyor/install_opengl.ps1
+      displayName: 'Install OpenGL'
+
+    - powershell: |
+        .ci/setup_headless_display.sh
+        pip install -r .ci/requirements_test_xvfb.txt
+        python .ci/display_test.py
+      displayName: Install test offscreen rendering
+      
+    - task: UsePythonVersion@0
+      inputs:
+        versionSpec: $(python.version)
+        addToPath: true
+
+    - task: PipAuthenticate@1
+      inputs:
+        artifactFeeds: 'pyansys'
+        onlyAddExtraIndex: true
+        
+    - script: |
+        pip install -r requirements_build.txt
+        python setup.py bdist_wheel
+        FOR /F %%a in ('dir /s/b dist\*.whl') do SET WHEELPATH=%%a
+        ECHO %WHEELPATH% 
+        pip install %WHEELPATH%
+        cd tests
+        python -c "from ansys.dpf import post; print(post.Report(gpu=False))"           
+      
+      displayName: Install ansys-dpf-post
+
+    - task: UniversalPackages@0
+      inputs:
+        command: 'download'
+        downloadDirectory: '$(System.DefaultWorkingDirectory)'
+        feedsToUse: 'internal'
+        vstsFeed: '705e121a-9631-49f5-8aaf-c7142856f923'
+        vstsFeedPackage: 'dpf-windows'
+        vstsPackageVersion: '21.2.3'