Docker pull testers (#1518)

* docker pull testers

* Tag what you pull

* Focus on wac tests

* Run wac tests interactively

* Fix test runner script

Author: michielbdejong

.travis.yml

@@ -29,7 +29,7 @@ script:
   - npm install -g solid-server-*.tgz
   # Run the Solid test-suite
   - npm install
-  - sh test/surface/run-solid-test-suite.sh
+  - bash test/surface/run-solid-test-suite.sh
 
 after_success:
   - snyk monitor

test/surface/run-solid-test-suite.sh

@@ -1,45 +1,60 @@
 #!/bin/bash
 set -e
 
-docker network create testnet
-docker build -t server test/surface/docker/server
-docker build -t cookie test/surface/docker/cookie
-
-docker build -t webid-provider test/surface/docker/webid-provider
-docker build -t solid-crud test/surface/docker/solid-crud
-docker build -t web-access-control test/surface/docker/web-access-control
-
-docker run -d --env-file test/surface/server-env.list --name server --network=testnet -v `pwd`:/travis -w /node-solid-server server /travis/bin/solid-test start --config-file /node-solid-server/config.json
-docker run -d --env-file test/surface/thirdparty-env.list --name thirdparty --network=testnet -v `pwd`/test/surface:/surface server /node-solid-server/bin/solid-test start --config-file /surface/thirdparty-config.json
-
-until docker run --rm --network=testnet webid-provider curl -kI https://server 2> /dev/null > /dev/null
-do
-  echo Waiting for server to start, this can take up to a minute ...
-  docker ps -a
-  docker logs server
-  sleep 1
-done
-
-until docker run --rm --network=testnet webid-provider curl -kI https://thirdparty 2> /dev/null > /dev/null
-do
-  echo Server started, waiting for thirdparty to start too ...
-  docker ps -a
-  docker logs thirdparty
-  sleep 1
-done
-
-docker ps -a
-docker logs server
-
-export COOKIE_ALICE="`docker run --rm -e SERVER_ROOT="https://server" --network=testnet cookie`"
-export COOKIE_BOB="`docker run --rm -e SERVER_ROOT="https://thirdparty" --network=testnet cookie`"
-
-docker run --rm --network=testnet --env COOKIE="$COOKIE_ALICE" --env-file test/surface/webid-provider-tests-env.list webid-provider
-docker run --rm --network=testnet --env COOKIE="$COOKIE_ALICE" --env-file test/surface/solid-crud-tests-env.list solid-crud
-docker run --rm --network=testnet --env COOKIE_ALICE="$COOKIE_ALICE" --env COOKIE_BOB="$COOKIE_BOB" --env-file test/surface/web-access-control-tests-env.list web-access-control
-
-docker stop server
-docker rm server
-docker stop thirdparty
-docker rm thirdparty
-docker network remove testnet
+function setup {
+  docker network create testnet
+  docker build -t server test/surface/docker/server
+  docker build -t cookie test/surface/docker/cookie
+  docker run -d --env-file test/surface/server-env.list --name server --network=testnet -v `pwd`:/travis -w /node-solid-server server /travis/bin/solid-test start --config-file /node-solid-server/config.json
+  docker run -d --env-file test/surface/thirdparty-env.list --name thirdparty --network=testnet -v `pwd`/test/surface:/surface server /node-solid-server/bin/solid-test start --config-file /surface/thirdparty-config.json
+}
+function teardown {
+  docker stop `docker ps --filter network=testnet -q`
+  docker rm `docker ps --filter network=testnet -qa`
+  docker network remove testnet
+}
+
+function waitForNss {
+  until docker run --rm --network=testnet solidtestsuite/webid-provider-tests curl -kI https://$1 2> /dev/null > /dev/null
+  do
+    echo Waiting for $1 to start, this can take up to a minute ...
+    docker ps -a
+    docker logs $1
+    sleep 1
+  done
+
+  docker logs $1
+  echo Getting cookie for $1...
+  export COOKIE_$1="`docker run --cap-add=SYS_ADMIN --network=testnet --env-file test/surface/$1-env.list cookie`"
+}
+
+function runTests {
+  docker pull solidtestsuite/$1:$2
+  
+  echo "Running $1 against server with cookie $COOKIE_server"
+  docker run --rm --network=testnet \
+    --env COOKIE="$COOKIE_server" \
+    --env COOKIE_ALICE="$COOKIE_server" \
+    --env COOKIE_BOB="$COOKIE_thirdparty" \
+    --env-file test/surface/$1-env.list solidtestsuite/$1:$2
+}
+
+# ...
+teardown || true
+setup
+waitForNss server
+runTests webid-provider-tests latest
+runTests solid-crud-tests nss-skips
+waitForNss thirdparty
+runTests web-access-control-tests nss-skips
+teardown
+
+# To debug, e.g. running web-access-control-tests jest interactively,
+# comment out `teardown` and uncomment this instead:
+# env
+# docker run -it --network=testnet \
+#     --env COOKIE="$COOKIE_server" \
+#     --env COOKIE_ALICE="$COOKIE_server" \
+#     --env COOKIE_BOB="$COOKIE_thirdparty" \
+#     --env-file test/surface/$1-env.list
+#   solidtestsuite/web-access-control-tests:latest /bin/bash

test/surface/thirdparty-env.list

@@ -2,4 +2,4 @@ ALICE_WEBID=https://thirdparty/profile/card#me
 SERVER_ROOT=https://thirdparty
 USERNAME=alice
 PASSWORD=123
-ACL_CACHE_TIME=5chc
+ACL_CACHE_TIME=5