Merge pull request #182 from lrytz/appveyor

AppVeyor

Author: lrytz

.travis.yml

@@ -1,15 +1,29 @@
 # we need rpmbuild but it's unlikely to be whitelisted, according to
 # https://github.com/travis-ci/apt-package-whitelist/pull/1700
 sudo: required
-before_install:
+
+install:
   - sudo apt-get -qq update
   - sudo apt-get install -y rpm
 
 language: scala
 jdk: oraclejdk8
 
+env:
+  global:
+    # PRIV_KEY_SECRET
+    - secure: "NlnFqZs4mvCi63GqCdUNDsx9BoiyrjgoV0cUaKlhVYp49/qAMlEaKf6JABWy/oCHoWsTyysyLEtZLsWfjAyE4+FUHSzngNUHVFBbtyudA6aKQ7jMsqkcMM/K0h+ymnhHSv8bugc8jXKFgmfYk4YTi5dZgv8CAOSQFAFUaxVdYTY="
+    # APPVEYOR_TOKEN
+    - secure: "aHM0UWkbFN0bXQja2f9lUcSgqHzXYkYqqV8/nP5MTLKJG8BpuaLYxI4gHfkPmxSm9/TTniA3zUVN//lJTvhco5J5Dtxfo5DeCLWsjQcQxrsgAFD97FpMpGbtFAViZuiv7SdPGvzXQY6AwvteBpxOday4T224aH5OjZJ0DxySx8Q="
+    # TRAVIS_TOKEN
+    - secure: "K6G3SeMRMgC8JUZADyd4RKvQpgHOZb8/8kk3AlDtRWZmrBPkZpTrtcuQLYXRiEAO/beQH956i3+uDeFryF079YTJZKtdqLTxwXS53gVurOv9VVbZFubu4HzXWMWeCIi9Np7S0eB1qc4NIKB+T5pbmIfEFGKThBGK179uwf2lqQ8="
+    # AWS_ACCESS_KEY_ID
+    - secure: "BrsLFLXLdJ3aesJGuoVXFdKluYl7QYBYjn5+ttoep2nc6/E+L3UDaR//ztxwex4a9yFi5q2jCs911FzVsk78ODamr5w8FBlOuu06RoVx642+U/agG7yUdAtvEzH2KVTBNjY0oYHZ+OlVEkeYgnkrsbfki+ujPWEnxkakWyrGtJw="
+    # AWS_SECRET_ACCESS_KEY
+    - secure: "dv0pR9uqnLyKVHu1L1nvl5TVpo3bOxh4icLNybwFyCFQd9XiM/RU8Vhx3svZqqnAG+GiCDSiCaBQkTLUS5u9e+9eVw3cUj5meNr9EW673f8D6H8Tr433jlvu54CynD9DsBjhNo/xIrECOKTq+0wu480OLSjOkuNoclG2dSm4Dis="
+
 script:
-- sbt -Dproject.version=2.12.0 s3Upload::mappings
+  - admin/build.sh
 
 cache:
   directories:

admin/build.ps1

@@ -0,0 +1,41 @@
+$ErrorActionPreference = "Stop"
+
+Function checkExit() {
+  if (-not $?) {
+    echo "Last command failed."
+    Exit 1
+  }
+}
+
+# oh boy. i don't (want to) fully understand, but executing commands and redirecting is difficult.
+#  - https://mnaoumov.wordpress.com/2015/01/11/execution-of-external-commands-in-powershell-done-right/
+#  - https://stackoverflow.com/a/35980675/248998
+# letting cmd do the redirect avoids confusing powershell.
+& cmd /c 'java -version' '2>&1'
+checkExit
+
+Function ensureVersion() {
+  $verPat="^[0-9]+\.[0-9]+\.[0-9]+(-[A-Za-z0-9-]+)?$"
+  if ($env:version -notmatch $verPat) {
+    echo "Not a valid Scala version: '$env:version'"
+    Exit 1
+  }
+}
+
+if ($env:APPVEYOR_FORCED_BUILD -eq 'true') {
+  ensureVersion
+  if ($env:mode -eq 'release') {
+    echo "Running a release for $env:version"
+    $repositoriesFile="$env:APPVEYOR_BUILD_FOLDER\conf\repositories"
+    & cmd /c "sbt ""-Dsbt.override.build.repos=true"" ""-Dsbt.repository.config=$repositoriesFile"" ""-Dproject.version=$env:version"" ""show fullResolvers"" clean update s3Upload" '2>&1'
+    checkExit
+  } else {
+    echo "Unknown mode: '$env:mode'"
+    Exit 1
+  }
+} else {
+  # By default, test building the packages (but don't uplaod)
+  # Need to redirect stderr, otherwise any error output (like jvm warning) fails the build (ErrorActionPreference)
+  & cmd /c 'sbt "-Dproject.version=2.12.4" "show s3Upload::mappings"' '2>&1'
+  checkExit
+}

admin/build.sh

@@ -0,0 +1,124 @@
+#!/bin/bash
+
+set -ex
+
+# Triggering jobs for a release:
+#  - Open https://travis-ci.org/scala/scala-dist
+#  - On the right: "More options" - "Trigger build"
+#  - Chose the branch and enter a title for the build in the commit message filed
+#  - Add a `before_install` custom config (see below) to set the `mode` and `version` env vars.
+#    Using an `env: global: ...` section does not work because that overrides the default `env`
+#    from .travis.yml. There's no way to specify the "merge mode" (*) from the web UI, that only
+#    works in the REST API. We use `before_install` and assume it's not used otherwise.
+#    (*) https://docs.travis-ci.com/user/triggering-builds/#Customizing-the-build-configuration
+#  - Available modes:
+#    - `release` to build native packages and upload them to S3
+#    - `archive` to copy archives to chara (for scala-lang.org)
+#    - `update-api` to update the scaladoc api symlinks
+#    In all of the above modes, the `version` needs to be specified.
+#
+# before_install: export version=2.12.N mode=release
+
+
+# Encryping files (if you need to encrypt a new file but no longer have the secret, create a new
+# secret and re-encrypt all files):
+#
+# 1. Generate secret
+#    cat /dev/urandom | head -c 10000 | openssl sha1 > ./secret
+# 2. Save the secret on travis
+#    travis encrypt "PRIV_KEY_SECRET=$(cat ./secret)"
+# 3. Encrypt the file
+#    openssl aes-256-cbc -pass "file:./secret" -in jenkins_lightbend_chara -out jenkins_lightbend_chara.enc
+# 4. Decrypt the file
+#    openssl aes-256-cbc -d -pass "pass:$PRIV_KEY_SECRET" -in admin/files/jenkins_lightbend_chara.enc > ~/.ssh/jenkins_lightbend_chara 2>/dev/null
+
+
+function ensureVersion() {
+  local verPat="[0-9]+\.[0-9]+\.[0-9]+(-[A-Za-z0-9-]+)?"
+  [[ "$version" =~ $verPat ]] || {
+    echo "Not a valid Scala version: '$version'"
+    exit 1
+  }
+}
+
+function decrypt() {
+  # Even though we're running bash with -x, travis hides the private key from the log
+  openssl aes-256-cbc -d -pass "pass:$PRIV_KEY_SECRET" -in $1 > $2 2>/dev/null
+}
+
+function setupSSH() {
+  mkdir -p ~/.ssh
+  cp admin/files/ssh_config ~/.ssh/config
+  echo 'chara.epfl.ch,128.178.154.107 ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBBLER9rqy0iCfz24+8BBBObh7FXXQJCoLLE9UuyQHNFUU4SS5FSzNjEoKXwTj8nqNy+8l0rOkj3KG8p2cLxsqjY=' >> ~/.ssh/known_hosts
+  decrypt admin/files/jenkins_lightbend_chara.enc ~/.ssh/jenkins_lightbend_chara
+  chmod 700 ~/.ssh && chmod 600 ~/.ssh/*
+}
+
+curlOut="curlOut.txt"
+
+function checkStatus() {
+  cat $curlOut
+  rm -f $curlOut
+  [[ "$1" == "$2" ]] || {
+    echo "Failed to start smoketest job"
+    exit 1
+  }
+}
+
+function triggerMsiRelease() {
+  local jsonTemplate='{ "accountName": "scala", "projectSlug": "scala-dist", "branch": "%s", "commitId": "%s", "environmentVariables": { "mode": "%s", "version": "%s" } }'
+  local json=$(printf "$jsonTemplate" "$TRAVIS_BRANCH" "$TRAVIS_COMMIT" "$mode" "$version")
+
+  local curlStatus=$(curl \
+    -s -o $curlOut -w "%{http_code}" \
+    -H "Authorization: Bearer $APPVEYOR_TOKEN" \
+    -H "Content-Type: application/json" \
+    -d "$json" \
+    https://ci.appveyor.com/api/builds)
+
+  checkStatus $curlStatus "200"
+}
+
+function triggerSmoketest() {
+  local jsonTemplate='{ "request": { "branch": "%s", "message": "Smoketest %s", "config": { "before_install": "export version=%s" } } }'
+  local json=$(printf "$jsonTemplate" "$TRAVIS_BRANCH" "$version" "$version")
+
+  local curlStatus=$(curl \
+    -s -o $curlOut -w "%{http_code}" \
+    -H "Travis-API-Version: 3" \
+    -H "Authorization: token $TRAVIS_TOKEN" \
+    -H "Content-Type: application/json" \
+    -d "$json" \
+    https://api.travis-ci.org/repo/scala%2Fscala-dist-smoketest/requests)
+
+  checkStatus $curlStatus "202"
+}
+
+if [[ "$TRAVIS_EVENT_TYPE" == "api" ]]; then
+  ensureVersion
+  if [[ "$mode" == "archives" ]]; then
+    echo "Running 'archives' for $version"
+    setupSSH
+    . scripts/jobs/release/website/archives
+  elif [[ "$mode" == "update-api" ]]; then
+    echo "Running 'update-api' for $version"
+    setupSSH
+    . scripts/jobs/release/website/update-api
+  elif [[ "$mode" == "release" ]]; then
+    echo "Running a release for $version"
+    triggerMsiRelease
+    repositoriesFile="$TRAVIS_BUILD_DIR/conf/repositories"
+    # The log is too long for the travis UI, so remove ANSI codes to have a clean raw version
+    sbt -Dsbt.log.noformat=true \
+      -Dsbt.override.build.repos=true -Dsbt.repository.config="$repositoriesFile" \
+      -Dproject.version=$version \
+      "show fullResolvers" clean update s3Upload
+    triggerSmoketest
+  else
+    echo "Unknown build mode: '$mode'"
+    exit 1
+  fi
+else
+  # By default, test building the packages (but don't uplaod)
+  sbt -Dproject.version=2.12.4 "show s3Upload::mappings"
+fi

admin/files/jenkins_lightbend_chara.enc

@@ -0,0 +1,4 @@
+Host chara
+  HostName chara.epfl.ch
+  IdentityFile /home/travis/.ssh/jenkins_lightbend_chara
+  User scalatest

admin/files/ssh_config

@@ -0,0 +1,28 @@
+version: '{build}'
+
+branches:
+  only:
+    - 2.11.x
+    - 2.12.x
+    - 2.13.x
+
+image: Visual Studio 2015
+
+environment:
+  JAVA_HOME: C:\Program Files\Java\jdk1.8.0
+  AWS_ACCESS_KEY_ID:
+    secure: X1Ix1soRBDMtfbi8IFNPOggDP2XquhW+uKcJ+XC0kiM=
+  AWS_SECRET_ACCESS_KEY:
+    secure: woXwpM2+P3uZ1+LFM+SbJEKLfi7Kax5PvqJSN62nn1PQrJgEzVUAIoRGoXBx6X72
+
+install:
+  - cmd: choco install sbt -ia "INSTALLDIR=""C:\sbt"""
+  - cmd: SET PATH=C:\sbt\bin;%JAVA_HOME%\bin;%PATH%
+
+build_script:
+  - ps: .\admin\build.ps1
+
+cache:
+  - C:\sbt\
+  - C:\Users\appveyor\.m2
+  - C:\Users\appveyor\.ivy2

appveyor.yml

@@ -22,19 +22,17 @@ upload := {
   import com.amazonaws.{ClientConfiguration, Protocol}
   import com.amazonaws.auth.DefaultAWSCredentialsProviderChain
   import com.amazonaws.services.s3.AmazonS3ClientBuilder
-  import com.amazonaws.services.s3.model.{CannedAccessControlList, PutObjectRequest}
+  import com.amazonaws.services.s3.model.PutObjectRequest
   import com.amazonaws.regions.Regions
 
-  // the publishing job runs on an EC2 instance that has access to the S3 bucket via its IAM instance profile
-  val client = AmazonS3ClientBuilder.standard.withRegion(Regions.US_EAST_1).build 
+  // The standard client picks credentials from AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY env vars
+  val client = AmazonS3ClientBuilder.standard.withRegion(Regions.US_EAST_1).build
 
   val log = streams.value.log
 
   (mappings in upload).value map { case (file, key) =>
     log.info("Uploading "+ file.getAbsolutePath() +" as "+ key)
-
-    // since the s3 bucket is in a separate account from where the EC2 CI instances are, must explicitly set acl as public-read
-    client.putObject(new PutObjectRequest("downloads.typesafe.com", key, file).withCannedAcl(CannedAccessControlList.PublicRead))
+    client.putObject(new PutObjectRequest("downloads.typesafe.com", key, file))
   }
 }
 
@@ -48,4 +46,4 @@ enablePlugins(UniversalPlugin, RpmPlugin, JDebPackaging, WindowsPlugin)
 
 // resolvers += "local" at "file:///e:/.m2/repository"
 // resolvers += Resolver.mavenLocal
-// to test, run e.g., stage, or windows:packageBin, show s3-upload::mappings
+// to test, run e.g., stage, or windows:packageBin, show s3Upload::mappings

build.sbt

@@ -1,6 +1,4 @@
 #!/bin/bash -ex
-# need to re-declare it as an array, not sure how to do that directly in jenkins
-declare -a sshCharaArgs="$sshCharaArgs"
 
 url="https://downloads.lightbend.com/scala/$version"
 
@@ -10,24 +8,23 @@ else archivesDir="~linuxsoft/archives/scala"
 fi
 
 # make this build restartable (if interrupted, partial files may remain, and resuming isn't very reliable)
-[[ -n $version ]] && ssh "${sshCharaArgs[@]}" "rm -f $archivesDir/scala-*$version* && rm -rf $archivesDir/api/$version/ ||:"
+[[ -n $version ]] && ssh chara "rm -f $archivesDir/scala-*$version* && rm -rf $archivesDir/api/$version/ ||:"
 
 cacheBuster="$(date +%s | tail -c2 | head -c1)"
-ssh "${sshCharaArgs[@]}" "cd $archivesDir && wget -nv -O scala-$version.msi       '$url/scala-$version.msi?$cacheBuster'"
-ssh "${sshCharaArgs[@]}" "cd $archivesDir && wget -nv -O scala-$version.zip       '$url/scala-$version.zip?$cacheBuster'"
-ssh "${sshCharaArgs[@]}" "cd $archivesDir && wget -nv -O scala-$version.tgz       '$url/scala-$version.tgz?$cacheBuster'"
-ssh "${sshCharaArgs[@]}" "cd $archivesDir && wget -nv -O scala-$version.deb       '$url/scala-$version.deb?$cacheBuster'"
-ssh "${sshCharaArgs[@]}" "cd $archivesDir && wget -nv -O scala-$version.rpm       '$url/scala-$version.rpm?$cacheBuster'"
-ssh "${sshCharaArgs[@]}" "cd $archivesDir && wget -nv -O scala-docs-$version.zip  '$url/scala-docs-$version.zip?$cacheBuster'"
-ssh "${sshCharaArgs[@]}" "cd $archivesDir && wget -nv -O scala-docs-$version.tgz  '$url/scala-docs-$version.tgz?$cacheBuster'"
-ssh "${sshCharaArgs[@]}" "cd $archivesDir && wget -nv -O scala-docs-$version.txz  '$url/scala-docs-$version.txz?$cacheBuster'"
+ssh chara "cd $archivesDir && wget -nv -O scala-$version.msi       '$url/scala-$version.msi?$cacheBuster'"
+ssh chara "cd $archivesDir && wget -nv -O scala-$version.zip       '$url/scala-$version.zip?$cacheBuster'"
+ssh chara "cd $archivesDir && wget -nv -O scala-$version.tgz       '$url/scala-$version.tgz?$cacheBuster'"
+ssh chara "cd $archivesDir && wget -nv -O scala-$version.deb       '$url/scala-$version.deb?$cacheBuster'"
+ssh chara "cd $archivesDir && wget -nv -O scala-$version.rpm       '$url/scala-$version.rpm?$cacheBuster'"
+ssh chara "cd $archivesDir && wget -nv -O scala-docs-$version.zip  '$url/scala-docs-$version.zip?$cacheBuster'"
+ssh chara "cd $archivesDir && wget -nv -O scala-docs-$version.tgz  '$url/scala-docs-$version.tgz?$cacheBuster'"
+ssh chara "cd $archivesDir && wget -nv -O scala-docs-$version.txz  '$url/scala-docs-$version.txz?$cacheBuster'"
 
 echo "Expanding scala-library API docs for $version to api/$version (with subdirectories for scala-compiler and scala-reflect)."
-ssh "${sshCharaArgs[@]}" "cd $archivesDir && mkdir -p api/$version && tar -xvz --strip-component 2 -f scala-docs-$version.tgz -C api/$version && mv api/$version/scala-library/* api/$version/" # tar on jenkins does not support the -s option
+ssh chara "cd $archivesDir && mkdir -p api/$version && tar -xvz --strip-component 2 -f scala-docs-$version.tgz -C api/$version && mv api/$version/scala-library/* api/$version/" # tar on jenkins does not support the -s option
 
 echo "Expanding the Scaladoc for other bundled modules"
 modules="scala-parser-combinators scala-swing scala-xml"
 for m in $modules; do 
-ssh "${sshCharaArgs[@]}" "(set -x; cd $archivesDir && mkdir -p api/$version/$m && unzip \$(find api/$version/jars -name ${m}_'*javadoc.jar' -o -name ${m}-${version}-javadoc.jar) -d api/$version/$m;)"
+ssh chara "(set -x; cd $archivesDir && mkdir -p api/$version/$m && unzip \$(find api/$version/jars -name ${m}_'*javadoc.jar' -o -name ${m}-${version}-javadoc.jar) -d api/$version/$m;)"
 done
-

scripts/jobs/release/website/archives

@@ -1,6 +1,4 @@
 #!/bin/bash -ex
-# need to re-declare it as an array, not sure how to do that directly in jenkins
-declare -a sshCharaArgs="$sshCharaArgs"
 
 if [[ "$version" =~ ^.*-(bin|pre)-[0-9a-f]+$ ]]
 then archivesDir="~linuxsoft/archives/scala/nightly/2.12.x"
@@ -9,8 +7,7 @@ fi
 
 echo "Symlinking $archivesDir/api/$version to $archivesDir/api/2.12.x."
 
-ssh "${sshCharaArgs[@]}" "cd $archivesDir/api/ ; [[ -d $version ]] && ln -sfn $version 2.12.x"
+ssh chara "cd $archivesDir/api/ ; [[ -d $version ]] && ln -sfn $version 2.12.x"
 
 # needs to run once on new major release:
-# ssh "${sshCharaArgs[@]}" "cd $archivesDir/api/ ; [[ -d $version ]] && ln -sfn 2.12.x current"
-
+# ssh chara "cd $archivesDir/api/ ; [[ -d $version ]] && ln -sfn 2.12.x current"