Merge pull request #1494 from hyperledger/develop

Hyperledger Iroha v1.0.0_beta-3

Author: bakhtin

.jenkinsci/artifacts.groovy

@@ -11,17 +11,30 @@ def uploadArtifacts(filePaths, uploadPath, artifactServers=['artifact.soramitsu.
   }
   def shaSumBinary = 'sha256sum'
   def md5SumBinary = 'md5sum'
+  def gpgKeyBinary = 'gpg --armor --detach-sign --no-tty --batch --yes --passphrase-fd 0'
   if (agentType == 'Darwin') {
     shaSumBinary = 'shasum -a 256'
     md5SumBinary = 'md5 -r'
+    gpgKeyBinary = 'GPG_TTY=\$(tty) gpg --pinentry-mode loopback --armor --detach-sign --no-tty --batch --yes --passphrase-fd 0'
   }
   sh "> \$(pwd)/batch.txt"
-  filePathsConverted.each {
-    sh "echo put ${it} $uploadPath >> \$(pwd)/batch.txt;"
-    sh "$shaSumBinary ${it} | cut -d' ' -f1 > \$(pwd)/\$(basename ${it}).sha256"
-    sh "$md5SumBinary ${it} | cut -d' ' -f1 > \$(pwd)/\$(basename ${it}).md5"
-    sh "echo put \$(pwd)/\$(basename ${it}).sha256 $uploadPath >> \$(pwd)/batch.txt;"
-    sh "echo put \$(pwd)/\$(basename ${it}).md5 $uploadPath >> \$(pwd)/batch.txt;"
+
+  withCredentials([file(credentialsId: 'ci_gpg_privkey', variable: 'CI_GPG_PRIVKEY'), string(credentialsId: 'ci_gpg_masterkey', variable: 'CI_GPG_MASTERKEY')]) {
+    if (!agentType.contains('MSYS_NT')) {
+      sh "gpg --yes --batch --no-tty --import ${CI_GPG_PRIVKEY} || true"
+    }
+    filePathsConverted.each {
+      sh "echo put ${it} $uploadPath >> \$(pwd)/batch.txt;"
+      sh "$shaSumBinary ${it} | cut -d' ' -f1 > \$(pwd)/\$(basename ${it}).sha256"
+      sh "$md5SumBinary ${it} | cut -d' ' -f1 > \$(pwd)/\$(basename ${it}).md5"
+      // TODO @bakhtin 30.05.18 IR-1384. Make gpg command options and paths compatible with Windows OS.
+      if (!agentType.contains('MSYS_NT')) {
+        sh "echo \"${CI_GPG_MASTERKEY}\" | $gpgKeyBinary -o \$(pwd)/\$(basename ${it}).asc ${it}"
+        sh "echo put \$(pwd)/\$(basename ${it}).asc $uploadPath >> \$(pwd)/batch.txt;"
+      }
+      sh "echo put \$(pwd)/\$(basename ${it}).sha256 $uploadPath >> \$(pwd)/batch.txt;"
+      sh "echo put \$(pwd)/\$(basename ${it}).md5 $uploadPath >> \$(pwd)/batch.txt;"
+    }
   }
   // mkdirs recursively
   uploadPath = uploadPath.split('/')
@@ -35,10 +48,11 @@ def uploadArtifacts(filePaths, uploadPath, artifactServers=['artifact.soramitsu.
   sshagent(['jenkins-artifact']) {
     sh "ssh-agent"
     artifactServers.each {
-      sh "sftp -b \$(pwd)/mkdirs.txt jenkins@${it} || true"
-      sh "sftp -b \$(pwd)/batch.txt jenkins@${it}"
+      sh "sftp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -b \$(pwd)/mkdirs.txt jenkins@${it} || true"
+      sh "sftp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -b \$(pwd)/batch.txt jenkins@${it}"
     }
   }
 }
 
 return this
+

.jenkinsci/bindings.groovy

@@ -1,57 +1,116 @@
 #!/usr/bin/env groovy
 
-def doJavaBindings(buildType=Release) {
+def doJavaBindings(os, buildType=Release) {
   def currentPath = sh(script: "pwd", returnStdout: true).trim()
   def commit = env.GIT_COMMIT
-  def artifactsPath = sprintf('%1$s/java-bindings-%2$s-%3$s-%4$s.zip', 
-    [currentPath, buildType, sh(script: 'date "+%Y%m%d"', returnStdout: true).trim(), commit.substring(0,6)])
+  def artifactsPath = sprintf('%1$s/java-bindings-%2$s-%3$s-%4$s-%5$s.zip',
+    [currentPath, buildType, os, sh(script: 'date "+%Y%m%d"', returnStdout: true).trim(), commit.substring(0,6)])
+  def cmakeOptions = ""
+  if (os == 'windows') {
+    sh "mkdir -p /tmp/${env.GIT_COMMIT}/bindings-artifact"
+    cmakeOptions = '-DCMAKE_TOOLCHAIN_FILE=/c/Users/Administrator/Downloads/vcpkg-master/vcpkg-master/scripts/buildsystems/vcpkg.cmake -G "NMake Makefiles"'
+  }
+  if (os == 'linux') {
+    // do not use preinstalled libed25519
+    sh "rm -rf /usr/local/include/ed25519*; unlink /usr/local/lib/libed25519.so; rm -f /usr/local/lib/libed25519.so.1.2.2"
+  }
   sh """
     cmake \
-      -H. \
+      -Hshared_model \
       -Bbuild \
       -DCMAKE_BUILD_TYPE=$buildType \
-      -DSWIG_JAVA=ON
+      -DSWIG_JAVA=ON \
+      ${cmakeOptions}
   """
-  sh "cd build; make -j${params.PARALLELISM} irohajava"
-  sh "zip -j $artifactsPath build/shared_model/bindings/*.java build/shared_model/bindings/libirohajava.so"
-  sh "cp $artifactsPath /tmp/bindings-artifact"
+  def parallelismParam = (os == 'windows') ? '' : "-j${params.PARALLELISM}"
+  sh "cmake --build build --target irohajava -- ${parallelismParam}"
+  // TODO 29.05.18 @bakhtin Java tests never finishes on Windows Server 2016. IR-1380
+  sh "zip -j $artifactsPath build/bindings/*.java build/bindings/*.dll build/bindings/libirohajava.so"
+  if (os == 'windows') {
+    sh "cp $artifactsPath /tmp/${env.GIT_COMMIT}/bindings-artifact"
+  }
+  else {
+    sh "cp $artifactsPath /tmp/bindings-artifact"
+  }
   return artifactsPath
 }
 
-def doPythonBindings(buildType=Release) {
+def doPythonBindings(os, buildType=Release) {
   def currentPath = sh(script: "pwd", returnStdout: true).trim()
   def commit = env.GIT_COMMIT
   def supportPython2 = "OFF"
-  def artifactsPath = sprintf('%1$s/python-bindings-%2$s-%3$s-%4$s-%5$s.zip', 
-    [currentPath, env.PBVersion, buildType, sh(script: 'date "+%Y%m%d"', returnStdout: true).trim(), commit.substring(0,6)])
-  // do not use preinstalled libed25519
-  sh "rm -rf /usr/local/include/ed25519*; unlink /usr/local/lib/libed25519.so; rm -f /usr/local/lib/libed25519.so.1.2.2"
+  def artifactsPath = sprintf('%1$s/python-bindings-%2$s-%3$s-%4$s-%5$s-%6$s.zip',
+    [currentPath, env.PBVersion, buildType, os, sh(script: 'date "+%Y%m%d"', returnStdout: true).trim(), commit.substring(0,6)])
+  def cmakeOptions = ""
+  if (os == 'windows') {
+    sh "mkdir -p /tmp/${env.GIT_COMMIT}/bindings-artifact"
+    cmakeOptions = '-DCMAKE_TOOLCHAIN_FILE=/c/Users/Administrator/Downloads/vcpkg-master/vcpkg-master/scripts/buildsystems/vcpkg.cmake -G "NMake Makefiles"'
+  }
+  if (os == 'linux') {
+    // do not use preinstalled libed25519
+    sh "rm -rf /usr/local/include/ed25519*; unlink /usr/local/lib/libed25519.so; rm -f /usr/local/lib/libed25519.so.1.2.2"
+  }
   if (env.PBVersion == "python2") { supportPython2 = "ON" }
   sh """
     cmake \
-      -H. \
+      -Hshared_model \
       -Bbuild \
       -DCMAKE_BUILD_TYPE=$buildType \
       -DSWIG_PYTHON=ON \
-      -DSUPPORT_PYTHON2=$supportPython2
+      -DSUPPORT_PYTHON2=$supportPython2 \
+      ${cmakeOptions}
   """
+  def parallelismParam = (os == 'windows') ? '' : "-j${params.PARALLELISM}"
+  sh "cmake --build build --target irohapy -- ${parallelismParam}"
   sh "cmake --build build --target python_tests"
-  sh "cd build; make -j${params.PARALLELISM} irohapy"
-  sh "protoc --proto_path=schema --python_out=build/shared_model/bindings block.proto primitive.proto commands.proto queries.proto responses.proto endpoint.proto"
-  sh "${env.PBVersion} -m grpc_tools.protoc --proto_path=schema --python_out=build/shared_model/bindings --grpc_python_out=build/shared_model/bindings endpoint.proto yac.proto ordering.proto loader.proto"
-  sh "zip -j $artifactsPath build/shared_model/bindings/*.py build/shared_model/bindings/*.so"
-  sh "cp $artifactsPath /tmp/bindings-artifact"
+  sh "cd build; ctest -R python --output-on-failure"
+  if (os == 'linux') {
+    sh """
+      protoc --proto_path=schema \
+        --python_out=build/bindings \
+        block.proto primitive.proto commands.proto queries.proto responses.proto endpoint.proto
+    """
+    sh """
+      ${env.PBVersion} -m grpc_tools.protoc --proto_path=schema --python_out=build/bindings \
+        --grpc_python_out=build/bindings endpoint.proto yac.proto ordering.proto loader.proto
+    """
+  }
+  else if (os == 'windows') {
+    sh """
+      protoc --proto_path=schema \
+        --proto_path=/c/Users/Administrator/Downloads/vcpkg-master/vcpkg-master/buildtrees/protobuf/src/protobuf-3.5.1-win32/include \
+        --python_out=build/bindings \
+        block.proto primitive.proto commands.proto queries.proto responses.proto endpoint.proto
+    """
+    sh """
+      ${env.PBVersion} -m grpc_tools.protoc \
+        --proto_path=/c/Users/Administrator/Downloads/vcpkg-master/vcpkg-master/buildtrees/protobuf/src/protobuf-3.5.1-win32/include \
+        --proto_path=schema --python_out=build/bindings --grpc_python_out=build/bindings \
+        endpoint.proto yac.proto ordering.proto loader.proto
+    """
+  }
+  sh """
+    zip -j $artifactsPath build/bindings/*.py build/bindings/*.dll build/bindings/*.so \
+      build/bindings/*.py build/bindings/*.pyd build/bindings/*.lib build/bindings/*.dll \
+      build/bindings/*.exp build/bindings/*.manifest
+    """
+  if (os == 'windows') {
+    sh "cp $artifactsPath /tmp/${env.GIT_COMMIT}/bindings-artifact"
+  }
+  else {
+    sh "cp $artifactsPath /tmp/bindings-artifact"
+  }
   return artifactsPath
 }
 
 def doAndroidBindings(abiVersion) {
   def currentPath = sh(script: "pwd", returnStdout: true).trim()
   def commit = env.GIT_COMMIT
-  def artifactsPath = sprintf('%1$s/android-bindings-%2$s-%3$s-%4$s-%5$s-%6$s.zip', 
+  def artifactsPath = sprintf('%1$s/android-bindings-%2$s-%3$s-%4$s-%5$s-%6$s.zip',
     [currentPath, "\$PLATFORM", abiVersion, "\$BUILD_TYPE_A", sh(script: 'date "+%Y%m%d"', returnStdout: true).trim(), commit.substring(0,6)])
   sh """
     (cd /iroha; git init; git remote add origin https://github.com/hyperledger/iroha.git; \
-    git fetch --depth 1 origin develop; git checkout -t origin/develop)
+    git fetch origin ${GIT_COMMIT}; git checkout FETCH_HEAD)
   """
   sh """
     . /entrypoint.sh; \
@@ -63,7 +122,7 @@ def doAndroidBindings(abiVersion) {
     sed -i.bak "s~find_program(protoc_EXECUTABLE protoc~set(protoc_EXECUTABLE \"/protobuf/host_build/protoc\"~" /iroha/cmake/Modules/Findprotobuf.cmake; \
     cmake -H/iroha/shared_model -B/iroha/shared_model/build -DCMAKE_SYSTEM_NAME=Android -DCMAKE_SYSTEM_VERSION=$abiVersion -DCMAKE_ANDROID_ARCH_ABI=\$PLATFORM \
       -DANDROID_NDK=\$NDK_PATH -DCMAKE_ANDROID_STL_TYPE=c++_static -DCMAKE_BUILD_TYPE=\$BUILD_TYPE_A -DTESTING=OFF \
-      -DSHARED_MODEL_DISABLE_COMPATIBILITY=ON -DSWIG_JAVA=ON -DCMAKE_PREFIX_PATH=\$DEPS_DIR
+      -DSWIG_JAVA=ON -DCMAKE_PREFIX_PATH=\$DEPS_DIR
     """
   sh "cmake --build /iroha/shared_model/build --target irohajava -- -j${params.PARALLELISM}"
   sh "zip -j $artifactsPath /iroha/shared_model/build/bindings/*.java /iroha/shared_model/build/bindings/libirohajava.so"

.jenkinsci/cancel-nightly-except-develop.groovy

@@ -9,7 +9,7 @@ def doDebugBuild(coverageEnabled=false) {
   def previousCommit = pCommit.previousCommitOrCurrent()
   // params are always null unless job is started
   // this is the case for the FIRST build only.
-  // So just set this to same value as default. 
+  // So just set this to same value as default.
   // This is a known bug. See https://issues.jenkins-ci.org/browse/JENKINS-41929
   if (!parallelism) {
     parallelism = 4
@@ -25,9 +25,9 @@ def doDebugBuild(coverageEnabled=false) {
                                            ['PARALLELISM': parallelism])
 
   if (GIT_LOCAL_BRANCH == 'develop' && manifest.manifestSupportEnabled()) {
-    manifest.manifestCreate("${DOCKER_REGISTRY_BASENAME}:develop-build", 
-      ["${DOCKER_REGISTRY_BASENAME}:x86_64-develop-build", 
-       "${DOCKER_REGISTRY_BASENAME}:armv7l-develop-build", 
+    manifest.manifestCreate("${DOCKER_REGISTRY_BASENAME}:develop-build",
+      ["${DOCKER_REGISTRY_BASENAME}:x86_64-develop-build",
+       "${DOCKER_REGISTRY_BASENAME}:armv7l-develop-build",
        "${DOCKER_REGISTRY_BASENAME}:aarch64-develop-build"])
     manifest.manifestAnnotate("${DOCKER_REGISTRY_BASENAME}:develop-build",
       [
@@ -70,7 +70,7 @@ def doDebugBuild(coverageEnabled=false) {
         ccache --show-stats
         ccache --zero-stats
         ccache --max-size=5G
-      """  
+      """
       sh """
         cmake \
           -DTESTING=ON \
@@ -85,7 +85,7 @@ def doDebugBuild(coverageEnabled=false) {
       if ( coverageEnabled ) {
         sh "cmake --build build --target coverage.init.info"
       }
-      def testExitCode = sh(script: 'CTEST_OUTPUT_ON_FAILURE=1 cmake --build build --target test', returnStatus: true)
+      def testExitCode = sh(script: """cd build && ctest --output-on-failure""", returnStatus: true)
       if (testExitCode != 0) {
         currentBuild.result = "UNSTABLE"
       }

.jenkinsci/debug-build.groovy

@@ -3,7 +3,6 @@
 def doDockerCleanup() {
 
     sh """
-      docker rm -f $IROHA_POSTGRES_HOST || true
       # Check whether the image is the last-standing man
       # i.e., no other tags exist for this image
       docker rmi \$(docker images --no-trunc --format '{{.Repository}}:{{.Tag}}\\t{{.ID}}' | grep \$(docker images --no-trunc --format '{{.ID}}' ${iC.id}) | head -n -1 | cut -f 1) || true

.jenkinsci/docker-cleanup.groovy

@@ -1,7 +1,7 @@
 #!/usr/bin/env groovy
 
 def doDoxygen() {
-    
+
     sh """
         doxygen Doxyfile
         #rsync docs/doxygen

.jenkinsci/doxygen.groovy

@@ -10,10 +10,8 @@ def linuxPostStep() {
       }
     }
     finally {
-      if (env.BUILD_TYPE == 'Debug') {
-        def cleanup = load ".jenkinsci/docker-cleanup.groovy"
-        cleanup.doDockerCleanup()
-      }
+      def cleanup = load ".jenkinsci/docker-cleanup.groovy"
+      cleanup.doDockerCleanup()
       cleanWs()
     }
   }

.jenkinsci/linux-post-step.groovy

@@ -19,7 +19,7 @@ def doReleaseBuild(coverageEnabled=false) {
       -DPACKAGE_TGZ=ON \
       -DCMAKE_BUILD_TYPE=Release \
       -DIROHA_VERSION=${env.IROHA_VERSION}
-    
+
     cmake --build build --target package -- -j${params.PARALLELISM}
     mv ./build/iroha-${env.IROHA_VERSION}-*.tar.gz ./build/iroha.tar.gz
     ccache --show-stats