diff --git a/.gitignore b/.gitignore index 38f7598ff..3d349bbc5 100644 --- a/.gitignore +++ b/.gitignore @@ -15,7 +15,7 @@ doc/ *.jar coverage tags -riak-* +riak* *.png .rebar/ .local_dialyzer_plt diff --git a/src/rebar_riak_test_plugin.erl b/.plugins/rebar_riak_test_plugin.erl similarity index 100% rename from src/rebar_riak_test_plugin.erl rename to .plugins/rebar_riak_test_plugin.erl diff --git a/INSTALL b/INSTALL index 142b5e950..b985bda49 100644 --- a/INSTALL +++ b/INSTALL @@ -25,7 +25,8 @@ {rt_deps, ["/usr/local/basho/evan/riak2.0/deps"]}, %% should be really long to allow full bitcasks to %% come up - {rt_max_wait_time, 600000000000000}, + {rt_max_receive_wait_time, 600000000000000}, + {test_timeout, 600000000000000}, {basho_bench, "/usr/local/basho/evan/basho_bench/"}, {basho_bench_statedir, "/tmp/bb_seqstate/"}, {rt_retry_delay, 500}, @@ -39,6 +40,6 @@ at least right now. this will problably change as the configuration stuff gets sorted out. some of these may not be necessary. - rt_max_wait_time is (could maybe be set to infinity? maybe by the + rt_max_receive_wait_time is (could maybe be set to infinity? maybe by the harness?), perf_* and basho_bench* are also critical. rt_deps should maybe be dynamic? diff --git a/Makefile b/Makefile index f20c9306a..d08a44f5b 100644 --- a/Makefile +++ b/Makefile @@ -1,11 +1,6 @@ .PHONY: deps -APPS = kernel stdlib sasl erts ssl tools os_mon runtime_tools crypto inets \ - xmerl webtool eunit syntax_tools compiler hipe mnesia public_key \ - observer wx gs -PLT = $(HOME)/.riak-test_dialyzer_plt - -all: deps compile +all: deps compile testcases ./rebar skip_deps=true escriptize SMOKE_TEST=1 ./rebar skip_deps=true escriptize @@ -18,7 +13,7 @@ docsclean: compile: deps ./rebar compile -clean: +clean: clean_testcases @./rebar clean distclean: clean @@ -28,6 +23,12 @@ quickbuild: ./rebar skip_deps=true compile ./rebar escriptize +testcases: + @(cd search-corpus; tar fx spam.0.1.tar.gz) + +clean_testcases: + @rm -rf search-corpus/spam.0/ + ################## # Dialyzer targets ################## diff --git a/README.md b/README.md index 527b43549..293444c18 100644 --- a/README.md +++ b/README.md @@ -8,16 +8,16 @@ in Erlang, and can interact with the cluster using distributed Erlang. ### How does it work? -`riak_test` runs tests in a sandbox, typically `$HOME/rt/riak`. The sanbox +`riak_test` runs tests in a sandbox, typically `$HOME/rt/riak`. The sandbox uses `git` to reset back to a clean state after tests are run. The contents of `$HOME/rt/riak` might look something like this: ``` $ ls $HOME/rt/riak -current riak-1.2.1 riak-1.3.2 riak-1.4.10 +riak-head riak-1.4.12 riak-2.0.5 riak-2.1.1 ``` -Inside each of these directories is a `dev` folder, typically +Inside each of these directories is a series `dev[0-9]+` directories, typically created with your normal `make [stage]devrel`. So how does this sandbox get populated to begin with? @@ -31,13 +31,13 @@ help you get both `~/test-releases` and `$HOME/rt/riak` all set up. A full tutorial for using them exists further down in this README. There is one folder in `$HOME/rt/riak` that does not come from -`~/test-releases`: `current`. The `current` folder can refer +`~/test-releases`: `head`. The `head` folder can refer to any version of Riak, but is typically used for something like the `master` branch, a feature branch, or a release candidate. -The `$HOME/rt/riak/current` dev release gets populated from a devrel of Riak +The `$HOME/rt/riak/riak-head` dev release gets populated from a stagedevrel of Riak that can come from anywhere, but is usually your 'normal' git checkout -of Riak. The `bin/rtdev-current.sh` can be run from within that folder -to copy `dev/` into `$HOME/rt/riak/current`. +of Riak. The `bin/rtdev-install.sh` can be run from within that folder +to copy `dev/` into `$HOME/rt/riak/riak-head`. Once you have everything set up (again, instructions for this are below), you'll want to run and write tests. This repository also holds code for @@ -53,37 +53,46 @@ previous versions of Riak. Together, we'll get your test environment up and running. Scripts to help in this process are located in the `bin` directory of this project. +### Prerequisites + +In order to successfully build Erlang and Riak there are several dependencies +which need to be fulfilled. Details can be found at +[Installing Erlang](http://docs.basho.com/riak/latest/ops/building/installing/erlang/). + +Essentially these packages need to be on your system for a successful build: +- autoconf +- gcc/g++ +- curses +- JDK +- make +- openssl (a very current one) + + ### rtdev-all.sh -This script is for the lazy. It performs all of the setup steps described -in the other scripts, including installing the current "master" branch from -Github into "current". The releases will be built in your current working +This script is for a complete installation. It performs all of the setup steps +including installing the current "master" branch from +Github into "riak-head". The releases will be built in your current working directory, so create an empty one in a place you'd like to store these builds for posterity, so that you don't have to rebuild them if your installation path (`$HOME/rt/riak` by the way this script installs it) gets into a bad state. -If you do want to restore your `$HOME/rt/riak` folder to factory condition, see -`rtdev-setup-releases.sh` and if you want to change the current riak under -test, see `rtdev-current.sh`. - -### rtdev-build-releases.sh - -The first one that we want to look at is `rtdev-build-releases.sh`. If -left unchanged, this script is going to do the following: +This script is going to do the following: 1. Download the source for the past three major Riak versions (e.g. - 1.3.2, 1.4.10 and 2.0.0) + 1.4.12, 2.0.5 and 2.1.1) +1. Any additional, test-specific versions required to run all tests (e.g. `kv679_mixed` needs 2.0.2 and 2.0.4) 1. Build the proper version of Erlang that release was built with, - using kerl (which it will also download) + using **kerl** (which it will also download) 1. Build those releases of Riak. You'll want to run this script from an empty directory. Also, you might be -thinking that you already have all the required versions of erlang. Great! You +thinking that you already have all the required versions of Erlang. Great! You should set and export the following environment variables prior to running this and other `riak_test` scripts: -Here, kerl is configured to use "$HOME/.kerl/installs" as the installation +Here, **kerl** is configured to use "$HOME/.kerl/installs" as the installation directory for erlang builds. ```bash @@ -92,12 +101,8 @@ export R16B02="$HOME/.kerl/installs/erlang-R16B02" export CURRENT_OTP="$R16B02" ``` -**Kerlveat**: If you want kerl to build erlangs with serious 64-bit -Macintosh action, you'll need a `~/.kerlrc` file that looks like this: - -``` -KERL_CONFIGURE_OPTIONS="--disable-hipe --enable-smp-support --enable-threads --enable-kernel-poll --enable-darwin-64bit --without-odbc" -``` +If you have your own versions of Erlang, just set the above environment +variables before running `rtdev-all.sh`. The script will check that all these paths exist. If even one of them is missing, it will prompt you to install kerl, even if you already @@ -105,45 +110,47 @@ have kerl. If you say no, the script quits. If you say yes, or all of your erlang paths check out, then go get a cup of coffee, you'll be building for a little while. -### rtdev-setup-releases.sh - -The `rtdev-setup-releases.sh` will get the releases you just built -into a local git repository. Run this script from the -same directory that you just built all of your releases into. -By default this script initializes the repository into `$HOME/rt/riak` but -you can override [`$RT_DEST_DIR`](https://github.com/basho/riak_test/blob/master/bin/rtdev-setup-releases.sh#L11). +To use `riak_ee` instead of `riak` set [`$RT_USE_EE`](https://github.com/basho/riak_test/blob/master/bin/rtdev-all.sh#L46) +to any non-empty string. -**Note**: There is a bug in 1.3.x `leveldb` which does not properly resolve +**Historical Note**: There is a bug in 1.3.x `leveldb` which does not properly resolve the location of `pthread.h` when building on Macintosh OS X 10.9, aka -Mavericks. This has been fixed in subsequent releases, but for now a fix -is to manually add `#include ` to the top of +Mavericks, and 10.10 (Yosemite). This has been fixed in subsequent releases, +but for now a fix is to manually add `#include ` to the top of `deps/eleveldb/c_src/leveldb/include/leveldb/env.h`. Also the version -of `meck` needs to be updated, too. This is handled autmatically by +of `meck` needs to be updated, too. This is handled automatically by the script. -### rtdev-current.sh - -`rtdev-current.sh` is where it gets interesting. You need to run that -from the Riak source folder you're wanting to test as the current -version of Riak. Also, make sure that you've already run `make devrel` -or `make stagedevrel` before you run `rtdev-current.sh`. Like setting up -releases you can override [`$RT_DEST_DIR`](https://github.com/basho/riak_test/blob/master/bin/rtdev-current.sh#L6) -so all your riak builds are in one place. Also you can override the tag -of the current version pulled by setting [`$RT_CURRENT_TAG`](https://github.com/basho/riak_test/blob/master/bin/rtdev-current.sh#L7) -to a release number, e.g. `2.0.0`. It will automatically be prefixed with -the repo name, e.g. `riak_ee-2.0.0`. To use `riak_ee` instead of `riak` set [`$RT_USE_EE`](https://github.com/basho/riak_test/blob/master/bin/rtdev-setup-releases.sh#L23) -to any non-empty string. +### rtdev-install.sh -#### reset-current-env.sh +`rtdev-install.sh` will check the releases you just built +into a local git repository. Run this script from the +same directory in which you just built all of your releases. +By default this script initializes the repository into `$HOME/rt/riak` but +you can override [`$RT_DEST_DIR`](https://github.com/basho/riak_test/blob/master/bin/rtdev-install.sh#L24). + +Also, make sure that you've already run `make devrel` +or `make stagedevrel` before you run `rtdev-install.sh`. Like setting up +releases you can override [`$RT_VERSION`](https://github.com/basho/riak_test/blob/master/bin/rtdev-install.sh#L28) +so all your Riak builds are in one place. + +### rtdev-migrate.sh + +`rtdev-migrate.sh` will convert existing devrels installed in `$RT_DEST_DIR` +from the legacy format to the new format. It also will reset the local +Git repo. It is only necessary to run this script once. WORK IN PROGRESS. + + +### reset-current-env.sh -`reset-current-env.sh` resets test environments setup using `rtdev-current.sh` +`reset-current-env.sh` resets test environments setup using `rtdev-install.sh` using the following process: 1. Delete the current stagedevrel/devrel environment 1. `make stagedevrel` for the Riak release being tested (current default is 2.0, overidden with the `-v` flag). When the `-c` option is specified, `make devclean` will be executed before rebuilding. - 1. Execute `rtdev-current.sh` for the Riak release being tested + 1. Execute `rtdev-install.sh` for the Riak release being tested 1. Rebuild the current riak_test branch. When the `-c` option is specified, 'make clean' will be executed before rebuilding. @@ -169,43 +176,145 @@ to tell riak_test about them. The method of choice is to create a {giddyup_host, "localhost:5000"}, {giddyup_user, "user"}, {giddyup_password, "password"}, - {rt_max_wait_time, 600000}, + {giddyup_platform, "osx-64"}, + {test_timeout, 1800000}, + {rt_max_receive_wait_time, 600000}, {rt_retry_delay, 1000}, {rt_harness, rtdev}, {rt_scratch_dir, "/tmp/riak_test_scratch"}, {basho_bench, "/home/you/basho/basho_bench"}, {spam_dir, "/home/you/basho/riak_test/search-corpus/spam.0"}, - {platform, "osx-64"} + {load_workers, 3}, + {test_paths, ["/home/you/basho/riak_test/ebin"]}, + {lager_console_level, debug}, + {lager_level, debug}, + {conn_fail_time, 60000}, + {offset, 2}, + {workers, 5} ]}. {rtdev, [ - {rt_project, "riak"}, - {rtdev_path, [{root, "/home/you/rt/riak"}, - {current, "/home/you/rt/riak/current"}, - {previous, "/home/you/rt/riak/riak-1.4.10"}, - {legacy, "/home/you/rt/riak/riak-1.3.2"} - ]} -]}. + {yz_dir, ["/home/you/basho/riak_ee/deps/yokozuna"]}, + {test_paths, ["/home/you/basho/riak_ee/yokozuna/riak_test/ebin"]}, + {root_path, "/home/you/basho/rt/riak"}, + {versions, [ + {'latest_1.4_ee', {riak_ee, "1.4.12"}}, + {'latest_2.0_ee', {riak_ee, "2.0.5"}}, + {'latest_2.1_ee', {riak_ee, "2.1.1"}}, + {default, 'latest_2.1_ee'}, + {previous, 'latest_2.0_ee'}, + {legacy, 'latest_1.4_ee'} + ]}, + {upgrade_paths, [ + {full, ['latest_1.4_ee', 'latest_2.0_ee', 'latest_2.1_ee']}, + {previous, ['latest_2.0_ee', 'latest_2.1_ee']} + {legacy, ['latest_1.4_ee', 'latest_2.1_ee']} + ]} +]} ``` The `default` section of the config file will be overridden by the config name you specify. For example, running the command below will use an -`rt_retry_delay` of 500 and an `rt_max_wait_time` of 180000. If your +`rt_retry_delay` of 500 and an `rt_max_receive_wait_time` of 180000. If your defaults contain every option you need, you can run riak_test without the `-c` argument. +** Note **: You *need* to have a built version of +[basho_bench](http://www.github.com/basho/basho_bench) setup and added to +your config file before running riak_test. + Some configuration parameters: - -#### rt_default_config + +#### basho_bench +Path to local installation of `basho_bench`; used by performance testing. + +#### giddyup_host +This is a hostname and port number for communicating with Basho's internal build +reporting tool, Giddyup. + +#### giddyup_password +String used as a password when communicating with Giddyup. + +#### giddyup_user +String used to identify the user when communicating with Giddyup. + +#### giddyup_platform +String identifying the current testing platform when reporting to Giddyup. +Current values include `centos-5-64`, `centos-6-64`, `fedora-17-64`, +`freebsd-9-64`, `osx-64`, `solaris-10u9-64`, `ubuntu-1004-64`, `ubuntu-1204-64` + +#### spam_dir +Name of a `tar` file containing ancient SPAM e-mail used as a data load +for a few tests. + +#### rt_harness +Which testing harness should be used. Current valid values include +- `rtdev` - Local devrel +- `rtssh` - Remote host +- `rtperf` - Performance testing + +#### rt_max_receive_wait_time +Number of milliseconds allowed for a `receive` operation to complete. + +#### rt_scratch_dir +Path to scratch directory used by `riak_test`. It's used for downloading +external files and for cleaning out the `data` directory between runs. + +#### rt_retry_delay +Number of milliseconds between attempts to send a message. + +#### test_timeout +Number of milliseconds allowed for a single test to run. + +#### rtdev +This is the devrel configuration section. + +##### root_path +Path to the top of the installed devrel instances. + +##### default +If specific versions of Riak are not specified, this one is tested. +The default value is `head` which is typically the head of `develop`. + +##### previous +Previous version of Riak EE, if not specified defaults to `2.0.5`. +Will be removed after tests have been ported. + +##### legacy +Ancient version of Riak EE, if not specified defaults to `1.4.12`. +Will be removed after tests have been ported. + +##### rt_default_config Default configuration parameters that will be used for nodes deployed by riak_test. Tests can override these. - ```erlang {rtdev, [ { rt_default_config, [ {riak_core, [ {ring_creation_size, 16} ]} ] } ]} ``` + +##### upgrade_paths +Definitions of lists of versions for programatic upgrades. The names `legacy` and `previous` support tests which have not yet been updated to the current framework. + +##### test_paths +List of directorys to search for tests. Currently used to help support Yokozuna tests. + +##### yz_dir +Directory to root of the built Yokozuna tree. This will go away once Yokozuna tests join the main herd under `riak`. + +##### lager_console_level and lager_level +This are [lager](https://github.com/basho/lager)-specific settings. By default they are set to `info`. + +##### load_workers +Number of concurrent processes used to load the system in the `loaded_upgrade` test. + +##### conn_fail_time +A magic value to override the default timeout in `replication2_ssl`. + +##### offset/workers +Values used to reproducably shuffle the order in which tests are executed. + #### Coverage You can generate a coverage report for a test run through [Erlang Cover](http://www.erlang.org/doc/apps/tools/cover_chapter.html). Coverage information for all **current** code run on any Riak node started by any of the tests in the run will be output as HTML in the coverage directory. @@ -251,6 +360,7 @@ This is an example test result JSON message posted to a webhook: "giddyup_url": "http://giddyup.basho.com/test_results/53" } ``` Notice that the giddyup URL is not the page for the test result, but a resource from which you can GET information about the test in JSON. + ### Running riak_test for the first time Run a test! After running `make` from the root of your `riak_test` diff --git a/bin/reset-current-env.sh b/bin/reset-current-env.sh index 354fa40e3..e19d3027c 100755 --- a/bin/reset-current-env.sh +++ b/bin/reset-current-env.sh @@ -22,7 +22,7 @@ VERSION="2.0" NUM_NODES=5 usage() { - echo "Resets the current riak_test environment by rebuilding riak and riak_test using rtdev-current.sh" + echo "Resets the current riak_test environment by rebuilding riak and riak_test using rtdev-install.sh" echo " -c: Perform a devclean on the riak and clean on riak_test projects (default: $FULL_CLEAN)" echo " -n: Number of nodes on which to test (default: $NUM_NODES)" echo " -v: The Riak version to test. The Riak home is calculated as $RT_HOME/riak- (default: $VERSION)" @@ -35,8 +35,7 @@ while getopts chn:v: opt; do ;; v) VERSION=$OPTARG ;; - n) echo "parsing num nodes" - NUM_NODES=$OPTARG + n) NUM_NODES=$OPTARG ;; h) usage exit 0 @@ -46,14 +45,14 @@ done shift $(($OPTIND-1)) -RIAK_HOME=$RT_HOME/riak-$VERSION +RIAK_HOME=$RT_HOME/$VERSION if ! [[ -d $RIAK_HOME || -h $RIAK_HOME ]]; then echo "Riak home $RIAK_HOME does not exist." exit 1 fi -echo "Reseting the riak_test environment using RIAK_HOME=$RIAK_HOME, RT_HOME=$RT_HOME, NUM_NODES=$NUM_NODES, VERSION=$VERSION, and FULL_CLEAN=$FULL_CLEAN" +echo "Resetting the riak_test environment using RIAK_HOME=$RIAK_HOME, RT_HOME=$RT_HOME, NUM_NODES=$NUM_NODES, VERSION=$VERSION, and FULL_CLEAN=$FULL_CLEAN" cd $RIAK_HOME rm -rf current @@ -64,7 +63,6 @@ if [ "$FULL_CLEAN" = true ] ; then make distclean fi - echo "Removing previous stagedevrel instance from $RIAK_HOME and rebuilding ..." make devclean @@ -72,7 +70,7 @@ make devclean echo "Building Riak stagedevrel with $NUM_NODES nodes in $RIAK_HOME ..." make stagedevrel DEVNODES=$NUM_NODES -$RT_HOME/bin/rtdev-current.sh +$RT_HOME/bin/rtdev-install.sh cd $RT_HOME diff --git a/bin/rtdev-all.sh b/bin/rtdev-all.sh index 32d9595b7..3368c304b 100755 --- a/bin/rtdev-all.sh +++ b/bin/rtdev-all.sh @@ -1,12 +1,225 @@ #!/usr/bin/env bash +# +# Bootstrap an entire riak_test tree +# +# Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved. +# +# This file is provided to you under the Apache License, +# Version 2.0 (the "License"); you may not use this file +# except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# You need to use this script once to build a set of stagedevrels for prior +# releases of Riak (for mixed version / upgrade testing). You should +# create a directory and then run this script from within that directory. +# I have ~/test-releases that I created once, and then re-use for testing. +# + +# Different versions of Riak were released using different Erlang versions, +# make sure to build with the appropriate version. + +# This is based on my usage of having multiple Erlang versions in different +# directories. If using kerl or whatever, modify to use kerl's activate logic. +# Or, alternatively, just substitute the paths to the kerl install paths as +# that should work too. + +# Set these values for non-default behavior +# Path to the Erlang R15B01 Installation +: ${R15B01:=$HOME/erlang/R15B01} +# Path to the Erlang R15B01 Installation +: ${R16B02:=$HOME/erlang/R16B02} +# Current version of Erlang (for "head" version) +: ${CURRENT_OTP:=$R16B02} +# Label of the "current" version +: ${DEFAULT_VERSION:="riak-head"} +# By default the Open Source version of Riak will be used, but for internal +# testing you can override this variable to use `riak_ee` instead +: ${RT_USE_EE:=""} ORIGDIR=`pwd` pushd `dirname $0` > /dev/null SCRIPT_DIR=`pwd` popd > /dev/null -: ${CURRENT_OTP:=$HOME/erlang-R16B02} -: ${RT_CURRENT_TAG:=""} +GITURL_RIAK="git://github.com/basho/riak" +GITURL_RIAK_EE="git@github.com:basho/riak_ee" + +# Determine if Erlang has already been built +checkbuild() +{ + ERLROOT=$1 + + if [ ! -d $ERLROOT ]; then + echo -n " - $ERLROOT cannot be found, install with kerl? [Y|n]: " + read ans + if [[ $ans == n || $ans == N ]]; then + echo + echo " [ERROR] Can't build $ERLROOT without kerl, aborting!" + exit 1 + else + if [ ! -x kerl ]; then + echo " - Fetching kerl." + if [ ! `which curl` ]; then + echo "You need 'curl' to be able to run this script, exiting" + exit 1 + fi + curl -O https://raw.githubusercontent.com/spawngrid/kerl/master/kerl; chmod a+x kerl + fi + fi + fi +} + +# Build and install Erlangs +kerl() +{ + RELEASE=$1 + BUILDNAME=$2 + + export CFLAGS="-g -O2" + export LDFLAGS="-g" + if [ -n "`uname -r | grep el6`" ]; then + export CFLAGS="-g -DOPENSSL_NO_EC=1" + fi + if [ "`uname`" == "Darwin" ]; then + export CFLAGS="-g -O0" + export KERL_CONFIGURE_OPTIONS="--disable-hipe --enable-smp-support --enable-threads --enable-kernel-poll --without-odbc --enable-darwin-64bit" + else + export KERL_CONFIGURE_OPTIONS="--disable-hipe --enable-smp-support --enable-threads --without-odbc --enable-m64-build" + fi + echo " - Building Erlang $RELEASE (this could take a while)" + # Use the patched version of Erlang for R16B02 builds + if [ "$RELEASE" == "R15B01" ]; then + ./kerl build git git://github.com/basho/otp.git basho_OTP_R15B01p $BUILDNAME + elif [ "$RELEASE" == "R16B02" ]; then + ./kerl build git git://github.com/basho/otp.git r16 $BUILDNAME + else + ./kerl build $RELEASE $BUILDNAME + fi + RES=$? + if [ "$RES" -ne 0 ]; then + echo "[ERROR] Kerl build $BUILDNAME failed" + exit 1 + fi + + echo " - Installing $RELEASE into $HOME/$BUILDNAME" + ./kerl install $BUILDNAME $HOME/$BUILDNAME + RES=$? + if [ "$RES" -ne 0 ]; then + echo "[ERROR] Kerl install $BUILDNAME failed" + exit 1 + fi +} + +# Build stagedevrels for testing +build() +{ + SRCDIR=$1 + ERLROOT=$2 + if [ -z "$RT_USE_EE" ]; then + GITURL=$GITURL_RIAK + else + GITURL=$GITURL_RIAK_EE + fi + + echo "Building $SRCDIR:" + + checkbuild $ERLROOT + if [ ! -d $ERLROOT ]; then + BUILDNAME=`basename $ERLROOT` + RELEASE=`echo $BUILDNAME | awk -F- '{ print $2 }'` + kerl $RELEASE $BUILDNAME + fi + + GITRES=1 + echo " - Cloning $GITURL" + rm -rf $SRCDIR + git clone $GITURL $SRCDIR + GITRES=$? + if [ $GITRES -eq 0 -a -n "$SRCDIR" ]; then + cd $SRCDIR + git checkout $SRCDIR + GITRES=$? + cd .. + fi + RUN="env PATH=$ERLROOT/bin:$ERLROOT/lib/erlang/bin:$PATH \ + C_INCLUDE_PATH=$ERLROOT/usr/include \ + LD_LIBRARY_PATH=$ERLROOT/usr/lib" + fix_riak_1_3 $SRCDIR "$RUN" + + echo " - Building stagedevrel in $SRCDIR (this could take a while)" + cd $SRCDIR + + # For non-tagged builds (i.e. head), use make deps. Otherwise, use + # make locked-deps for tagged builds ... + if [ -n "`echo ${SRCDIR} | grep head`" ]; then + make deps + else + $RUN make locked-deps + fi + + $RUN make all stagedevrel + RES=$? + if [ "$RES" -ne 0 ]; then + echo "[ERROR] make stagedevrel failed" + exit 1 + fi + echo " - $SRCDIR built." + $SCRIPT_DIR/rtdev-install.sh + cd .. +} + +# Riak 1.3 has a few artifacts which need to be updated in order to build +# properly +fix_riak_1_3() +{ + SRCDIR=$1 + RUN="$2" + + if [ "`echo $SRCDIR | cut -d- -f2 | cut -d. -f1-2`" != "1.3" ]; then + return 0 + fi + + echo "- Patching Riak 1.3.x" + cd $SRCDIR + if [ "$SRCDIR" == "riak-1.3.2" ]; then + cat < + #include + #include ++#include + #include "leveldb/perf_count.h" + #include "leveldb/status.h" +EOF + cd ../../../../../../.. +} if [ -n "$DEBUG_RTDEV" ]; then echo "= Configuration =================================================" @@ -34,22 +247,19 @@ echo echo "= Building Riak Releases ========================================" echo -source $SCRIPT_DIR/rtdev-build-releases.sh - -echo "= Installing Riak Releases ======================================" echo -source $SCRIPT_DIR/rtdev-setup-releases.sh - -echo -echo "= Building and Installing Riak from Git =========================" -echo - -cd $ORIGDIR -build "current" $CURRENT_OTP $RT_CURRENT_TAG -echo -cd current -source $SCRIPT_DIR/rtdev-current.sh +if [ -z "$RT_USE_EE" ]; then + build "riak-1.3.2" $R15B01 + build "riak-1.4.12" $R15B01 +else + build "riak_ee-1.3.4" $R15B01 + build "riak_ee-1.4.12" $R15B01 + if [ "${DEFAULT_VERSION}" == "riak-head" ]; then + DEFAULT_VERSION="riak_ee-head" + fi + echo "Default version: $DEFAULT_VERSION" +fi +build $DEFAULT_VERSION $R16B02 -cd $ORIGDIR echo echo "= Build complete! ===============================================" diff --git a/bin/rtdev-build-releases.sh b/bin/rtdev-build-releases.sh deleted file mode 100755 index e170a31d1..000000000 --- a/bin/rtdev-build-releases.sh +++ /dev/null @@ -1,176 +0,0 @@ -#!/usr/bin/env bash - -# You need to use this script once to build a set of stagedevrels for prior -# releases of Riak (for mixed version / upgrade testing). You should -# create a directory and then run this script from within that directory. -# I have ~/test-releases that I created once, and then re-use for testing. -# -# See rtdev-setup-releases.sh as an example of setting up mixed version layout -# for testing. - -# Different versions of Riak were released using different Erlang versions, -# make sure to build with the appropriate version. - -# This is based on my usage of having multiple Erlang versions in different -# directories. If using kerl or whatever, modify to use kerl's activate logic. -# Or, alternatively, just substitute the paths to the kerl install paths as -# that should work too. - -: ${R15B01:=$HOME/erlang-R15B01} -: ${R16B02:=$HOME/erlang-R16B02} - -# By default the Open Source version of Riak will be used, but for internal -# testing you can override this variable to use `riak_ee` instead -: ${RT_USE_EE:=""} -GITURL_RIAK="git://github.com/basho/riak" -GITURL_RIAK_EE="git@github.com:basho/riak_ee" - - -checkbuild() -{ - ERLROOT=$1 - - if [ ! -d $ERLROOT ]; then - echo -n " - $ERLROOT cannot be found, install with kerl? [Y|n]: " - read ans - if [[ $ans == n || $ans == N ]]; then - echo - echo " [ERROR] Can't build $ERLROOT without kerl, aborting!" - exit 1 - else - if [ ! -x kerl ]; then - echo " - Fetching kerl." - if [ ! `which curl` ]; then - echo "You need 'curl' to be able to run this script, exiting" - exit 1 - fi - curl -O https://raw.github.com/spawngrid/kerl/master/kerl > /dev/null 2>&1; chmod a+x kerl - fi - fi - fi -} - -kerl() -{ - RELEASE=$1 - BUILDNAME=$2 - - echo " - Building Erlang $RELEASE (this could take a while)" - ./kerl build $RELEASE $BUILDNAME > /dev/null 2>&1 - RES=$? - if [ "$RES" -ne 0 ]; then - echo "[ERROR] Kerl build $BUILDNAME failed" - exit 1 - fi - - echo " - Installing $RELEASE into $HOME/$BUILDNAME" - ./kerl install $BUILDNAME $HOME/$BUILDNAME > /dev/null 2>&1 - RES=$? - if [ "$RES" -ne 0 ]; then - echo "[ERROR] Kerl install $BUILDNAME failed" - exit 1 - fi -} - -build() -{ - SRCDIR=$1 - ERLROOT=$2 - TAG="$3" - if [ -z "$RT_USE_EE" ]; then - GITURL=$GITURL_RIAK - GITTAG=riak-$TAG - else - GITURL=$GITURL_RIAK_EE - GITTAG=riak_ee-$TAG - fi - - echo "Building $SRCDIR:" - - checkbuild $ERLROOT - if [ ! -d $ERLROOT ]; then - BUILDNAME=`basename $ERLROOT` - RELEASE=`echo $BUILDNAME | awk -F- '{ print $2 }'` - kerl $RELEASE $BUILDNAME - fi - - GITRES=1 - echo " - Cloning $GITURL" - rm -rf $SRCDIR - git clone $GITURL $SRCDIR - GITRES=$? - if [ $GITRES -eq 0 -a -n "$TAG" ]; then - cd $SRCDIR - git checkout $GITTAG - GITRES=$? - cd .. - fi - - RUN="env PATH=$ERLROOT/bin:$ERLROOT/lib/erlang/bin:$PATH \ - C_INCLUDE_PATH=$ERLROOT/usr/include \ - LD_LIBRARY_PATH=$ERLROOT/usr/lib" - fix_riak_1_3 $SRCDIR $TAG "$RUN" - - echo " - Building stagedevrel in $SRCDIR (this could take a while)" - cd $SRCDIR - - $RUN make all stagedevrel - RES=$? - if [ "$RES" -ne 0 ]; then - echo "[ERROR] make stagedevrel failed" - exit 1 - fi - cd .. - echo " - $SRCDIR built." -} - -# Riak 1.3 has a few artifacts which need to be updated in order to build -# properly -fix_riak_1_3() -{ - SRCDIR=$1 - TAG="$2" - RUN="$3" - - if [ "`echo $TAG | cut -d . -f1-2`" != "1.3" ]; then - return 0 - fi - - echo "- Patching Riak 1.3.x" - cd $SRCDIR - cat < - #include - #include -+#include - #include "leveldb/perf_count.h" - #include "leveldb/status.h" -EOF - cd ../../../../../../.. -} - -build "riak-1.4.10" $R15B01 http://s3.amazonaws.com/downloads.basho.com/riak/1.4/1.4.10/riak-1.4.10.tar.gz -echo -if [ -z "$RT_USE_EE" ]; then - build "riak-1.3.2" $R15B01 1.3.2 -else - build "riak-1.3.4" $R15B01 1.3.4 -fi -echo diff --git a/bin/rtdev-current.sh b/bin/rtdev-current.sh deleted file mode 100755 index 2b5a3f150..000000000 --- a/bin/rtdev-current.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/usr/bin/env bash - -# bail out if things go south -set -e - -: ${RT_DEST_DIR:="$HOME/rt/riak"} -# If RT_CURRENT_TAG is specified, it will use that version number -# otherwise the last annotated tag will be used -: ${RT_CURRENT_TAG:=""} - -echo "Making $(pwd) the current release:" -cwd=$(pwd) -echo -n " - Determining version: " -if [ -n "$RT_CURRENT_TAG" ]; then - VERSION=$RT_CURRENT_TAG -elif [ -f $cwd/dependency_manifest.git ]; then - VERSION=`cat $cwd/dependency_manifest.git | awk '/^-/ { print $NF }'` -else - VERSION="$(git describe --tags)-$(git branch | awk '/\*/ {print $2}')" -fi -echo $VERSION -cd $RT_DEST_DIR -echo " - Resetting existing $RT_DEST_DIR" -export GIT_WORK_TREE="$RT_DEST_DIR" -git reset HEAD --hard > /dev/null -git clean -fd > /dev/null -echo " - Removing and recreating $RT_DEST_DIR/current" -rm -rf $RT_DEST_DIR/current -mkdir $RT_DEST_DIR/current -cd $cwd -echo " - Copying devrel to $RT_DEST_DIR/current" -cp -p -P -R dev $RT_DEST_DIR/current -echo " - Writing $RT_DEST_DIR/current/VERSION" -echo -n $VERSION > $RT_DEST_DIR/current/VERSION -cd $RT_DEST_DIR -echo " - Reinitializing git state" -git add . -git commit -a -m "riak_test init" --amend > /dev/null diff --git a/bin/rtdev-install.sh b/bin/rtdev-install.sh new file mode 100755 index 000000000..fb5ad5a56 --- /dev/null +++ b/bin/rtdev-install.sh @@ -0,0 +1,112 @@ +#!/usr/bin/env bash +# +# Install a devrel or stagedevrel version of Riak for riak_test +# +# Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved. +# +# This file is provided to you under the Apache License, +# Version 2.0 (the "License"); you may not use this file +# except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# just bail out if things go south +set -e + +: ${RT_DEST_DIR:="$HOME/rt/riak"} + +cwd=$(pwd) +echo -n " - Determining version: " +if [ -z "${RT_VERSION+xxx}" ] || ([ -z "$RT_VERSION" ] && [ "${RT_VERSION+xxx}" = "xxx" ]); then + if [ -f $cwd/dependency_manifest.git ]; then + # For packaged distributions + VERSION=`cat $cwd/dependency_manifest.git | awk '/^-/ { print $NF }'` + else + echo "Making $(pwd) a tagged release:" + TAGS=`git describe --tags` + CURRENT=`git rev-parse HEAD` + HEAD=`git show-ref | grep HEAD | cut -f1 -d" "` + PRODUCT="" + if [ -n "`echo ${TAGS} | grep riak_ee`" ]; then + # For riak_ee + PRODUCT="riak_ee" + VERSION=`echo ${TAGS} | awk '{sub(/riak_ee-/,"",$0);print}'` + elif [ -n "`echo ${TAGS} | grep riak_cs`" ]; then + # For riak_cs + PRODUCT="riak_cs" + VERSION=`echo ${TAGS} | awk '{sub(/riak_cs-/,"",$0);print}'` + else + # For open source riak + PRODUCT="riak" + RT_VERSION=`echo ${TAGS} | awk '{sub(/riak-/,"",$0);print}'` + fi + # If we are on the tip, call the version "head" + if [ "${CURRENT}" == "${HEAD}" ]; then + VERSION="head" + fi + fi + RT_VERSION="${PRODUCT}-${VERSION}" +fi +echo "Version: ${RT_VERSION}" + +# Create the RT_DEST_DIR directory if it does not yet exist +if [ ! -d $RT_DEST_DIR ]; then + mkdir -p $RT_DEST_DIR +fi + +# Reinitialize the Git repo if it already exists, +# including removing untracked files +cd $RT_DEST_DIR +if [ -d ".git" ]; then + echo " - Resetting existing $RT_DEST_DIR" + git reset HEAD --hard > /dev/null 2>&1 + git clean -fd > /dev/null 2>&1 +fi + +RT_VERSION_DIR=$RT_DEST_DIR/$RT_VERSION +echo " - Removing and recreating $RT_VERSION_DIR" +rm -rf $RT_VERSION_DIR +mkdir $RT_VERSION_DIR +cd $cwd +echo " - Copying devrel to $RT_VERSION_DIR" +if [ ! -d "dev" ]; then + echo "You need to run \"make devrel\" or \"make stagedevrel\" first" + exit 1 +fi +cd dev + +# Clone the existing dev directory into RT_DEST_DIR +for i in `ls`; do cp -p -P -R $i $RT_DEST_DIR/$RT_VERSION/; done + + VERSION_FILE=$RT_VERSION_DIR/VERSION + echo " - Writing $VERSION_FILE" + echo -n $RT_VERSION > $VERSION_FILE + + cd $RT_DEST_DIR + if [ -d ".git" ]; then + echo " - Reinitializing git state" + git add --ignore-removal -f . + git commit -a -m "riak_test init" --amend > /dev/null 2>&1 + else + git init + cat > .gitignore < /dev/null + echo " - Successfully completed initial git commit of $RT_DEST_DIR" + fi diff --git a/bin/rtdev-migrate.sh b/bin/rtdev-migrate.sh new file mode 100644 index 000000000..d160091a2 --- /dev/null +++ b/bin/rtdev-migrate.sh @@ -0,0 +1,69 @@ +#!/usr/bin/env bash +# +# Bootstrap an entire riak_test tree +# +# Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved. +# +# This file is provided to you under the Apache License, +# Version 2.0 (the "License"); you may not use this file +# except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# just bail out if things go south +set -e + +: ${RT_DEST_DIR:="$HOME/rt/riak"} +cd ${RT_DEST_DIR} + +# New "standard" for current version of Riak +if [ -d current ]; then + echo "Renaming current to head" + mv -f current riak-head +fi + +# Chop off the "riak-" prefix +for name in riak-*; do + if [ -d "${name}" ]; then + NEW_NAME=`echo ${name} | cut -d- -f2` + echo "Renaming ${name} to ${NEW_NAME}" + mv ${name} ${NEW_NAME} + fi +done + +# Remove the intermediate "dev" directory +for version in `ls -1`; do + if [ -d "${version}/dev" ]; then + echo "Removing the dev directory from ${version}" + cd ${version}/dev + mv dev* .. + cd .. + rmdir dev + cd .. + fi +done + +# Set up local Git repo +if [ -d ".git" ]; then + echo " - Reinitializing git state" + git add -f --ignore-removal . + git commit -a -m "riak_test init" --amend > /dev/null 2>&1 +else + git init + + ## Some versions of git and/or OS require these fields + git config user.name "Riak Test" + git config user.email "dev@basho.com" + + git add --ignore-removal . + git commit -a -m "riak_test init" > /dev/null + echo " - Successfully completed initial git commit of $RT_DEST_DIR" +fi diff --git a/bin/rtdev-setup-releases.sh b/bin/rtdev-setup-releases.sh deleted file mode 100755 index a692e5a21..000000000 --- a/bin/rtdev-setup-releases.sh +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env bash - -# bail out if things go south -set -e - -# Creates a mixed-version directory structure for running riak_test -# using rtdev-mixed.config settings. Should be run inside a directory -# that contains devrels for prior Riak releases. Easy way to create this -# is to use the rtdev-build-releases.sh script - -: ${RT_DEST_DIR:="$HOME/rt/riak"} - -echo "Setting up releases from $(pwd):" -echo " - Creating $RT_DEST_DIR" - -rm -rf $RT_DEST_DIR -mkdir -p $RT_DEST_DIR - -count=$(ls */dev 2> /dev/null | wc -l) -if [ "$count" -ne "0" ] -then - for rel in */dev; do - vsn=$(dirname "$rel") - echo " - Initializing $RT_DEST_DIR/$vsn" - mkdir -p "$RT_DEST_DIR/$vsn" - cp -p -P -R "$rel" "$RT_DEST_DIR/$vsn" - done -else - # This is useful when only testing with 'current' - # The repo still needs to be initialized for current - # and we don't want to bomb out if */dev doesn't exist - touch $RT_DEST_DIR/.current_init - echo "No devdirs found. Not copying any releases." -fi - -cd $RT_DEST_DIR -git init - -## Some versions of git and/or OS require these fields -git config user.name "Riak Test" -git config user.email "dev@basho.com" - -git add . -git commit -a -m "riak_test init" > /dev/null -echo " - Successfully completed initial git commit of $RT_DEST_DIR" diff --git a/doc/overview.edoc b/doc/overview.edoc index f40863b1b..f0d4b9796 100644 --- a/doc/overview.edoc +++ b/doc/overview.edoc @@ -145,7 +145,10 @@ Erlang term based config files. The references `rtdev.config' file above: {rt_deps, ["/Users/jtuple/basho/riak/deps"]}. %% Maximum time in milliseconds for wait_until to wait -{rt_max_wait_time, 180000}. +{rt_max_receive_wait_time, 180000}. + +%% Maximum time in milliseconds allowed for a single test +{test_timeout, 1800000}. %% Delay between each retry in wait_until {rt_retry_delay, 500}. diff --git a/examples/riak_test.config b/examples/riak_test.config index d5d950894..cfa7f82ae 100644 --- a/examples/riak_test.config +++ b/examples/riak_test.config @@ -22,7 +22,7 @@ %% builder. Typically this is in the format %% "NAME-VERSION-ARCHITECTURE". See GiddyUp for valid platform %% names. - {platform, "osx-64"}, + {giddyup_platform, "osx-64"}, %% riak_test includes various wait_for_X functions that will %% repeatedly test for specific conditions until they are @@ -58,18 +58,31 @@ %% The path to a corpus of spam emails to be used when testing %% Riak Search. This is typically expanded from the tarball %% included in riak_test. - {spam_dir, "/Users/dparfitt/riak_test/search-corpus/spam.0"}, + {spam_dir, "/Users/dparfitt/riak_test/search-corpus"}, %% The number of workers-per-node to spawn when executing the %% `loaded_upgrade' test. If unspecified, this will default to %% `10'. For older/slower machines, use a lower number to avoid %% unexpected node crashes. - {load_workers, 10}, + {load_workers, 3}, %% lager_level defaults to info, which is should mean %% "relevant test output". debug level output is for helping %% test writers. - {lager_level, info} + {lager_level, info}, + + %% lager_console_level defaults to info, which is should mean + %% "relevant test output" to the console. debug level output is for helping + %% test writers. + {lager_console_level, debug}, + + %% Number of processes to run during testing + {workers, 5}, + {offset, 2}, + + %% Continue to run subsequent test cases instead of halting immediately. + %% False by default + {continue_on_fail, false} ]}. %% =============================================================== @@ -84,28 +97,35 @@ %% The name of the project/product, used when fetching the test %% suite and reporting. {rt_project, "riak"}, - + %% Paths to the locations of various versions of the project. This %% is only valid for the `rtdev' harness. - {rtdev_path, [ - %% This is the root of the built `rtdev' repository, - %% used for manipulating the repo with git. All - %% versions should be inside this directory. - {root, "/Users/dparfitt/rt/riak"}, - - %% The path to the `current' version, which is used - %% exclusively except during upgrade tests. - {current, "/Users/dparfitt/rt/riak/current"}, - - %% The path to the most immediately previous version - %% of the project, which is used when doing upgrade - %% tests. - {previous, "/Users/dparfitt/rt/riak/riak-1.2.1"}, - - %% The path to the version before `previous', which - %% is used when doing upgrade tests. - {legacy, "/Users/dparfitt/rt/riak/riak-1.1.4"} - ]} + {root_path, "/Users/hazen/dev/rt/riak"}, + {versions, [ + %% The path to the version before `previous', which + %% is used when doing upgrade tests. + %% Versions are tuple of product and version number + {'latest_1.4_ee', {riak, "1.4.12"}}, + %% The path to the most immediately previous version + %% of the project, which is used when doing upgrade + %% tests. + {'latest_2.0_ee', {riak, "2.0.5"}}, + %% The path to the `current' version, which is used + %% exclusively except during upgrade tests. + {'latest_2.1_ee', {riak, "2.1.1"}}, + {'previous_2.1_ee', {riak, "2.1.0"}}, + %% Backwards-compatible aliases to versions + {default, 'latest_2.1_ee'}, + {previous, 'latest_2.0_ee'}, + {legacy, 'latest_1.4_ee'} + ]}, + {upgrade_paths, [ + %% Lists of aliases to possible upgrade versions + {full, ['latest_1.4_ee', 'latest_2.0_ee', 'latest_2.1_ee']}, + {legacy, ['latest_1.4_ee', 'latest_2.1_ee']}, + {previous, ['latest_2.0_ee', 'latest_2.1_ee']}, + {minor, ['previous_2.1_ee', 'latest_2.1_ee']} + ]} ]}. %% Sample project for Riak EDS ("EE"). @@ -129,11 +149,22 @@ {repl_upgrade_order, "forwards"}, %% [See rtdev.rtdev_path above] - {rtdev_path, [{root, "/Users/dparfitt/rt/riak_ee"}, - {current, "/Users/dparfitt/rt/riak_ee/current"}, - {previous, "/Users/dparfitt/rt/riak_ee/riak_ee-1.2.1"}, - {legacy, "/Users/dparfitt/rt/riak_ee/riak_ee-1.1.4"} - ]} + {root_path, "/Users/hazen/dev/rt/riak"}, + {versions, [ + {'latest_1.4_ee', {riak_ee, "1.4.12"}}, + {'latest_2.0_ee', {riak_ee, "2.0.5"}}, + {'latest_2.1_ee', {riak_ee, "2.1.1"}}, + {'previous_2.1_ee', {riak_ee, "2.1.0"}}, + {default, 'latest_2.1_ee'}, + {previous, 'latest_2.0_ee'}, + {legacy, 'latest_1.4_ee'} + ]}, + {upgrade_paths, [ + {full, ['latest_1.4_ee', 'latest_2.0_ee', 'latest_2.1_ee']}, + {previous, ['latest_2.0_ee', 'latest_2.1_ee']}, + {legacy, ['latest_1.4_ee', 'latest_2.1_ee']}, + {minor, ['previous_2.1_ee', 'latest_2.1_ee']} + ]} ]}. %% Sample project to demonstrate use of local configs for the diff --git a/examples/riak_test.config.perf b/examples/riak_test.config.perf index f3b75f64f..e24cdd526 100644 --- a/examples/riak_test.config.perf +++ b/examples/riak_test.config.perf @@ -2,7 +2,8 @@ {rt_deps, ["/mnt/riak_ee/deps"]}, %% should be really long to allow full bitcasks to %% come up - {rt_max_wait_time, 600000000000000}, + {rt_max_receive_wait_time, 600000000000000}, + {test_timeout, 600000000000000}, {basho_bench, "/mnt/basho_bench"}, {basho_bench_escript, "/usr/local/erlang-r16b02/bin/escript"}, {basho_bench_statedir, "/tmp/bb_seqstate/"}, @@ -11,8 +12,6 @@ {load_intercepts, false}, {perf_builds, "/mnt/perf_builds"}, {perf_loadgens, ["bench101.aws"]}, - {rtdev_path, [{root, "/mnt/rt/riak_ee"}, - {current, "/mnt/rt/riak_ee/riak-ee-2.0.0rc1"}, - {previous, "/mnt/rt/riak_ee/riak-ee-1.4.8"}, - {legacy, "/mnt/rt/riak_ee/riak-ee-1.3.4"}]} + {root_path, "/mnt/rt/riak"}, + {default_version, head} ]}. diff --git a/include/rt.hrl b/include/rt.hrl deleted file mode 100644 index 78de7e028..000000000 --- a/include/rt.hrl +++ /dev/null @@ -1,25 +0,0 @@ -%% ------------------------------------------------------------------- -%% -%% Copyright (c) 2013 Basho Technologies, Inc. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%% ------------------------------------------------------------------- - --record(rt_webhook, { - name :: string(), - url :: string(), - headers=[] :: [{atom(), string()}] - }). diff --git a/rebar b/rebar index f38f1451a..a8cbf1b7c 100755 Binary files a/rebar and b/rebar differ diff --git a/rebar.config b/rebar.config index 2ecbeec34..1ebe176d6 100644 --- a/rebar.config +++ b/rebar.config @@ -6,7 +6,7 @@ %%{edoc_opts, [{layout, my_layout}, {file_suffix, ".xml"}, {pretty_printer, erl_pp}]}. {erl_opts, [{src_dirs, [src, intercepts, perf]}, warnings_as_errors, {parse_transform, lager_transform}]}. -{erl_first_files, ["src/rt_intercept_pt.erl"]}. +{erl_first_files, ["src/rt_driver.erl", "src/rt_intercept_pt.erl", "src/rt_host.erl"]}. {eunit_opts, [verbose]}. @@ -18,12 +18,14 @@ {riakc, ".*", {git, "git://github.com/basho/riak-erlang-client", {branch, "master"}}}, {riakhttpc, ".*", {git, "git://github.com/basho/riak-erlang-http-client", {branch, "master"}}}, {kvc, "1.3.0", {git, "https://github.com/etrepum/kvc", {tag, "v1.3.0"}}}, - {druuid, ".*", {git, "git://github.com/kellymclaughlin/druuid.git", {tag, "0.2"}}} + {druuid, ".*", {git, "git://github.com/kellymclaughlin/druuid.git", {tag, "0.2"}}}, + {clique, ".*", {git, "git://github.com/basho/clique", {branch, "develop"}}}, + {exec, ".*", {git, "https://github.com/saleyn/erlexec.git", "6c311527bdafc87f3491e4e60b29e7a1b0f33c6e"}} ]}. {escript_incl_apps, [goldrush, lager, getopt, riakhttpc, riakc, ibrowse, mochiweb, kvc]}. {escript_emu_args, "%%! -escript main riak_test_escript +K true +P 10000 -env ERL_MAX_PORTS 10000\n"}. -{plugin_dir, "src"}. +{plugin_dir, ".plugins"}. {plugins, [rebar_riak_test_plugin]}. {riak_test, [ {test_paths, ["tests", "perf"]}, diff --git a/regression_test_wrapper.sh b/regression_test_wrapper.sh new file mode 100755 index 000000000..224ccad04 --- /dev/null +++ b/regression_test_wrapper.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +# Bail out on error ... +#set -e + +if [ -z $1 ]; then + echo "An r_t configuration is required as the first parameter" + exit 1 +fi + +REPL_TEST_CASES="replication,replication_object_reformat,replication2,repl_fs_stat_caching,replication2_pg:test_12_pg_mode_repl12,replication2_pg:test_12_pg_mode_repl12_ssl,replication2_pg:test_12_pg_mode_repl_mixed,replication2_pg:test_12_pg_mode_repl_mixed_ssl,replication2_pg:test_basic_pg_mode_mixed,replication2_pg:test_basic_pg_mode_mixed_ssl,replication2_pg:test_basic_pg_mode_repl13,replication2_pg:test_bidirectional_pg,replication2_pg:test_bidirectional_pg_ssl,replication2_pg:test_mixed_pg,replication2_pg:test_mixed_pg_ssl,replication2_pg:test_multiple_sink_pg,replication2_pg:test_multiple_sink_pg_ssl,replication2_pg:test_pg_proxy,replication2_pg:test_pg_proxy_ssl" +ALL_BACKEND_TEST_CASES="always_pass_test,verify_riak_stats,verify_down,verify_staged_clustering,verify_leave,partition_repair,verify_build_cluster,riak_control_authentication,always_fail_test,basic_command_line,jmx_verify,verify_aae,verify_claimant,verify_object_limits,ensemble_interleave,ensemble_byzantine,gh_riak_core_155,pb_security,verify_search,verify_handoff,verify_capabilities,verify_handoff_mixed,verify_bitcask_tombstone2_upgrade,${REPL_TEST_CASES}" + +BITCASK_BACKEND_TEST_CASES="$ALL_BACKEND_TEST_CASES,loaded_upgrade" +ELEVELDB_BACKEND_TEST_CASES="verify_2i_aae,loaded_upgrade" +MEMORY_BACKEND_TEST_CASES="verify_2i_aae,verify_membackend" + +ROOT_RESULTS_DIR="results/regression" +RESULTS=`date +"%y%m%d%H%M%s"` +RESULTS_DIR="$ROOT_RESULTS_DIR/$RESULTS" +mkdir -p $RESULTS_DIR + +RESULTS_SYMLINK=$ROOT_RESULTS_DIR/current +rm -f $RESULTS_SYMLINK +ln -s $RESULTS $RESULTS_SYMLINK + +RT_OPTS="-v --continue -c $1" + +echo "Running bitcask regression tests using the following test cases: $BITCASK_BACKEND_TEST_CASES" +./riak_test $RT_OPTS -t $BITCASK_BACKEND_TEST_CASES -o $RESULTS_DIR &> $RESULTS_DIR/bitcask_results.log + +echo "Running leveldb regression tests using the following test cases: $ELEVELDB_BACKEND_TEST_CASES" +./riak_test $RT_OPTS -t $ELEVELDB_BACKEND_TEST_CASES -b eleveldb -o $RESULTS_DIR &> $RESULTS_DIR/leveldb_results.log + +echo "Running memory regression tests using the following test cases: $MEMORY_BACKEND_TEST_CASES" +./riak_test $RT_OPTS -t $MEMORY_BACKEND_TEST_CASES -b memory -o $RESULTS_DIR &> $RESULTS_DIR/memory_results.log + +echo "Results of the test run written to $RESULTS_DIR" diff --git a/riak b/riak deleted file mode 120000 index a5208ef76..000000000 --- a/riak +++ /dev/null @@ -1 +0,0 @@ -riak-2.0 \ No newline at end of file diff --git a/src/giddyup.erl b/src/giddyup.erl index 1902d0f8a..ec8ee4273 100644 --- a/src/giddyup.erl +++ b/src/giddyup.erl @@ -1,6 +1,6 @@ -%% ------------------------------------------------------------------- +%%------------------------------------------------------------------- %% -%% Copyright (c) 2013 Basho Technologies, Inc. +%% Copyright (c) 2015 Basho Technologies, Inc. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -17,50 +17,289 @@ %% under the License. %% %% ------------------------------------------------------------------- +%% @author Brett Hazen +%% @copyright (C) 2015, Basho Technologies +%% @doc +%% Communicate with the GiddyUp web service. Pulls jobs to do and +%% report results back up to GiddyUp. +%% @end +%% Created : 20. Apr 2015 10:39 AM +%%------------------------------------------------------------------- -module(giddyup). +-author("Brett Hazen"). + +-behaviour(gen_server). + +%% API +-export([start_link/7, + get_test_plans/0, + post_result/2, + post_artifact/3]). --export([get_suite/1, post_result/1, post_artifact/2]). -define(STREAM_CHUNK_SIZE, 8192). --include("rt.hrl"). --spec get_suite(string()) -> [{atom(), term()}]. -get_suite(Platform) -> - Schema = get_schema(Platform), - Name = kvc:path('project.name', Schema), - lager:info("Retrieved Project: ~s", [Name]), +-record(rt_webhook, { + name :: string(), + url :: string(), + headers=[] :: [{atom(), string()}] +}). + +%% gen_server callbacks +-export([init/1, + stop/0, + handle_call/3, + handle_cast/2, + handle_info/2, + terminate/2, + code_change/3]). + +-define(SERVER, ?MODULE). + +-record(state, { + platform :: string(), + default_product :: string(), + default_version_number :: string(), + default_version :: string(), + giddyup_host :: string(), + giddyup_user :: string(), + giddyup_password :: string(), + %% A dictionary of Base URLs to store artifacts + artifact_base :: dict() +}). + +%%%=================================================================== +%%% API +%%%=================================================================== + +%%-------------------------------------------------------------------- +%% @doc +%% Starts the server +%% +%% @end +%%-------------------------------------------------------------------- +-spec(start_link(string(), string(), string(), string(), string(), string(), string()) -> + {ok, Pid :: pid()} | ignore | {error, Reason :: term()}). +start_link(Platform, Product, VersionNumber, Version, GiddyUpHost, GiddyUpUser, GiddyUpPassword) -> + gen_server:start_link({local, ?SERVER}, ?MODULE, [Platform, Product, VersionNumber, Version, GiddyUpHost, GiddyUpUser, GiddyUpPassword], []). + +%%-------------------------------------------------------------------- +%% @doc +%% Stops the server +%% +%% @end +%%-------------------------------------------------------------------- +-spec stop() -> ok. +stop() -> + gen_server:call(?MODULE, stop, infinity). + +%%-------------------------------------------------------------------- +%% @doc +%% Get the entire test suite for the specified platform +%% +%% @end +%%-------------------------------------------------------------------- +get_test_plans() -> + %% TODO: Is this a good timeout? + gen_server:call(?MODULE, get_test_plans, 60000). + +%%-------------------------------------------------------------------- +%% @doc +%% Send a test result back up to GiddyUp +%% @end +%%-------------------------------------------------------------------- + +-spec post_result(rt_test_plan:test_plan(), pass | {fail, string()}) -> ok. +post_result(TestPlan, TestResult) -> + gen_server:cast(?MODULE, {post_result, TestPlan, TestResult}). + +%%-------------------------------------------------------------------- +%% @doc +%% Send a test log file back up to GiddyUp +%% @end +%%-------------------------------------------------------------------- + +-spec(post_artifact(string(), string(), string()) -> ok | error). +post_artifact(TestPlan, Label, Filename) -> + gen_server:cast(?MODULE, {post_artifact, TestPlan, Label, Filename}). + +%%%=================================================================== +%%% gen_server callbacks +%%%=================================================================== + +%%-------------------------------------------------------------------- +%% @private +%% @doc +%% Initializes the GiddyUp server +%% +%% @spec init(Args) -> {ok, State} | +%% {ok, State, Timeout} | +%% ignore | +%% {stop, Reason} +%% @end +%%-------------------------------------------------------------------- +-spec(init(Args :: term()) -> + {ok, State :: #state{}} | {ok, State :: #state{}, timeout() | hibernate} | + {stop, Reason :: term()} | ignore). +init([Platform, Product, VersionNumber, Version, GiddyUpHost, GiddyUpUser, GiddyUpPassword]) -> + load_and_start(ibrowse), + {ok, #state{platform=Platform, + default_version=Version, + default_version_number=VersionNumber, + default_product=Product, + giddyup_host=GiddyUpHost, + giddyup_user=GiddyUpUser, + giddyup_password=GiddyUpPassword, + artifact_base = dict:new()}}. + +%%-------------------------------------------------------------------- +%% @private +%% @doc +%% Handling call messages +%% +%% @end +%%-------------------------------------------------------------------- +-spec(handle_call(Request :: term(), From :: {pid(), Tag :: term()}, + State :: #state{}) -> + {reply, Reply :: term(), NewState :: #state{}} | + {reply, Reply :: term(), NewState :: #state{}, timeout() | hibernate} | + {noreply, NewState :: #state{}} | + {noreply, NewState :: #state{}, timeout() | hibernate} | + {stop, Reason :: term(), Reply :: term(), NewState :: #state{}} | + {stop, Reason :: term(), NewState :: #state{}}). +handle_call(get_test_plans, _From, State) -> + TestPlans = fetch_all_test_plans(State#state.platform, State#state.default_product, State#state.default_version_number, State#state.giddyup_host), + {reply, TestPlans, State}; +handle_call(_Request, _From, State) -> + {reply, ok, State}. + +%%-------------------------------------------------------------------- +%% @private +%% @doc +%% Handling cast messages +%% +%% @end +%%-------------------------------------------------------------------- +-spec(handle_cast(Request :: term(), State :: #state{}) -> + {noreply, NewState :: #state{}} | + {noreply, NewState :: #state{}, timeout() | hibernate} | + {stop, Reason :: term(), NewState :: #state{}}). +handle_cast({post_result, TestPlan, TestResult}, State) -> + {ok, Location} = post_result(TestPlan, TestResult, State#state.giddyup_host, State#state.giddyup_user, State#state.giddyup_password), + Dictionary = State#state.artifact_base, + %% Store the Base URL in a dictionary keyed off the test name + {noreply, State#state{artifact_base=dict:store(rt_test_plan:get_name(TestPlan), Location, Dictionary)}}; +handle_cast({post_artifact, TestPlan, Label, Filename}, State) -> + BaseURL = dict:fetch(rt_test_plan:get_name(TestPlan), State#state.artifact_base), + post_artifact(BaseURL, Label, Filename, State#state.giddyup_user, State#state.giddyup_password), + {noreply, State}; +handle_cast(_Request, State) -> + {noreply, State}. + +%%-------------------------------------------------------------------- +%% @private +%% @doc +%% Handling all non call/cast messages +%% +%% @spec handle_info(Info, State) -> {noreply, State} | +%% {noreply, State, Timeout} | +%% {stop, Reason, State} +%% @end +%%-------------------------------------------------------------------- +-spec(handle_info(Info :: timeout() | term(), State :: #state{}) -> + {noreply, NewState :: #state{}} | + {noreply, NewState :: #state{}, timeout() | hibernate} | + {stop, Reason :: term(), NewState :: #state{}}). +handle_info(_Info, State) -> + {noreply, State}. + +%%-------------------------------------------------------------------- +%% @private +%% @doc +%% This function is called by a gen_server when it is about to +%% terminate. It should be the opposite of Module:init/1 and do any +%% necessary cleaning up. When it returns, the gen_server terminates +%% with Reason. The return value is ignored. +%% +%% @spec terminate(Reason, State) -> void() +%% @end +%%-------------------------------------------------------------------- +-spec(terminate(Reason :: (normal | shutdown | {shutdown, term()} | term()), + State :: #state{}) -> term()). +terminate(_Reason, _State) -> + ok. + +%%-------------------------------------------------------------------- +%% @private +%% @doc +%% Convert process state when code is changed +%% +%% @spec code_change(OldVsn, State, Extra) -> {ok, NewState} +%% @end +%%-------------------------------------------------------------------- +-spec(code_change(OldVsn :: term() | {down, term()}, State :: #state{}, + Extra :: term()) -> + {ok, NewState :: #state{}} | {error, Reason :: term()}). +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +%%%=================================================================== +%%% Internal functions +%%%=================================================================== + +%%-------------------------------------------------------------------- +%% @private +%% @doc +%% Get the entire test suite from Giddyup in an `rt_test_plan' list +%% @end +%%-------------------------------------------------------------------- + +-spec fetch_all_test_plans(string(), string(), string(), string()) -> [rt_test_plan:test_plan()]. +fetch_all_test_plans(Platform, Product, VersionNumber, Host) -> + %% Make sure ibrowse is up and running + rt:check_ibrowse(), + Schema = get_schema(Platform, Product, VersionNumber, Host), + Project = kvc:path('project.name', Schema), + lager:info("Retrieved Project: ~s", [Project]), Tests = kvc:path('project.tests', Schema), TestProps = fun(Test) -> - [ - {id, kvc:path(id, Test)}, - {backend, - case kvc:path('tags.backend', Test) of - [] -> undefined; - X -> binary_to_atom(X, utf8) - end}, - {platform, list_to_binary(Platform)}, - {version, rt:get_version()}, - {project, Name} - ] ++ - case kvc:path('tags.upgrade_version', Test) of - [] -> []; - UpgradeVsn -> [{upgrade_version, binary_to_atom(UpgradeVsn, utf8)}] - end ++ - case kvc:path('tags.multi_config', Test) of - [] -> []; - MultiConfig -> [{multi_config, binary_to_atom(MultiConfig, utf8)}] - end + Id = kvc:path(id, Test), + Module = binary_to_atom(kvc:path(name, Test), utf8), + Plan0 = rt_test_plan:new([{id, Id}, {module, Module}, {project, Project}, {platform, Platform}, {version, VersionNumber}]), + {ok, Plan1} = case kvc:path('tags.backend', Test) of + %% Bitcask is the default version + [] -> rt_test_plan:set(backend, bitcask, Plan0); + Backend -> rt_test_plan:set(backend, binary_to_atom(Backend, utf8), Plan0) + end, + {ok, Plan2} = case kvc:path('tags.upgrade_version', Test) of + [] -> {ok, Plan1}; + UpgradeVsn -> + rt_test_plan:set(upgrade_path, binary_to_atom(UpgradeVsn, utf8), Plan1) + end, + %% TODO: Remove? No tests currently use this multi_config setting + %% Plan3 = case kvc:path('tags.multi_config', Test) of + %% [] -> Plan1; + %% MultiConfig -> rt_test_plan:set(multi_config, binary_to_atom(MultiConfig, utf8), Plan2) + %%end, + lager:debug("Giddyup Module ~p using TestPlan ~p", [Module, Plan2]), + Plan2 end, - [ { binary_to_atom(kvc:path(name, Test), utf8), TestProps(Test) } || Test <- Tests]. + [ TestProps(Test) || Test <- Tests]. -get_schema(Platform) -> - get_schema(Platform, 3). +%%-------------------------------------------------------------------- +%% @private +%% @doc +%% Get the GiddyUp Schema in JSON format (decoded to Erlang terms) +%% Retry up to 3 times. +%% @end +%%-------------------------------------------------------------------- -get_schema(Platform, Retries) -> - Host = rt_config:get(giddyup_host), - Project = rt_config:get(rt_project), - Version = rt:get_version(), - URL = lists:flatten(io_lib:format("http://~s/projects/~s?platform=~s&version=~s", [Host, Project, Platform, Version])), +-spec(get_schema(atom(), string(), string(), string()) -> term()). +get_schema(Platform, Product, VersionNumber, Host) -> + get_schema(Platform, Product, VersionNumber, Host, 3). + +get_schema(Platform, Product, VersionNumber, Host, Retries) -> + URL = lists:flatten(io_lib:format("http://~s/projects/~s?platform=~s&version=~s", [Host, Product, Platform, VersionNumber])), lager:info("giddyup url: ~s", [URL]), rt:check_ibrowse(), @@ -73,16 +312,23 @@ get_schema(Platform, Retries) -> lager:warning("GiddyUp GET failed: ~p", [Error]), lager:warning("GiddyUp trying ~p more times", [Retries]), timer:sleep(60000), - get_schema(Platform, Retries - 1) + get_schema(Platform, Product, VersionNumber, Host, Retries - 1) end. --spec post_result([{atom(), term()}]) -> {ok, string()} | error. -post_result(TestResult) -> - Host = rt_config:get(giddyup_host), +%%-------------------------------------------------------------------- +%% @private +%% @doc +%% Send a test result back up to GiddyUp +%% @end +%%-------------------------------------------------------------------- + +-spec post_result(rt_test_plan:test_plan(), pass | {fail, string()}, string(), string(), string()) -> {ok, string()} | error. +post_result(TestPlan, TestResult, Host, User, Password) -> URL = "http://" ++ Host ++ "/test_results", lager:info("giddyup url: ~s", [URL]), rt:check_ibrowse(), - case rt:post_result(TestResult, #rt_webhook{name="GiddyUp", url=URL, headers=[basic_auth()]}) of + BasicAuth = {basic_auth, {User, Password}}, + case post_result(TestPlan, TestResult, #rt_webhook{name="GiddyUp", url=URL, headers=[BasicAuth]}) of {ok, RC, Headers} -> {_, Location} = lists:keyfind("Location", 1, Headers), lager:info("Test Result successfully POSTed to GiddyUp! ResponseCode: ~s, URL: ~s", [RC, Location]), @@ -91,46 +337,102 @@ post_result(TestResult) -> error end. -post_artifact(TRURL, {FName, Body}) -> +-spec(post_result(rt_test_plan:test_plan(), pass | {fail, string()}, term()) -> {ok, integer(), [string()]} | error). +post_result(TestPlan, TestResult, #rt_webhook{url=URL, headers=HookHeaders, name=Name}) -> + Status = case TestResult of + pass -> pass; + _ -> fail + end, + GiddyupResult = [ + {test, rt_test_plan:get_module(TestPlan)}, + {status, Status}, + {backend, rt_test_plan:get(backend, TestPlan)}, + {id, rt_test_plan:get(id, TestPlan)}, + {platform, rt_test_plan:get(platform, TestPlan)}, + {version, rt_test_plan:get(version, TestPlan)}, + {project, rt_test_plan:get(project, TestPlan)} + ], + try ibrowse:send_req(URL, + [{"Content-Type", "application/json"}], + post, + mochijson2:encode(GiddyupResult), + [{content_type, "application/json"}] ++ HookHeaders, + 300000) of %% 5 minute timeout + + {ok, RC=[$2|_], Headers, _Body} -> + {ok, RC, Headers}; + {ok, ResponseCode, Headers, Body} -> + lager:info("Test Result did not generate the expected 2XX HTTP response code."), + lager:debug("Post"), + lager:debug("Response Code: ~p", [ResponseCode]), + lager:debug("Headers: ~p", [Headers]), + lager:debug("Body: ~p", [Body]), + error; + X -> + lager:warning("Some error POSTing test result: ~p", [X]), + error + catch + Class:Reason -> + lager:error("Error reporting to ~s. ~p:~p", [Name, Class, Reason]), + lager:error("Payload: ~p", [GiddyupResult]), + error + end. + +%%-------------------------------------------------------------------- +%% @private +%% @doc +%% Send a test log file back up to GiddyUp +%% @end +%%-------------------------------------------------------------------- + +-spec(post_artifact(string(), string(), string(), string(), string()) -> ok | error). +post_artifact(BaseURL, Label, Filename, User, Password) -> %% First compute the path of where to post the artifact - URL = artifact_url(TRURL, FName), - ReqBody = make_req_body(Body), - CType = guess_ctype(FName), + URL = artifact_url(BaseURL, Label), + {ok, BodyIoDevice} = file:open(Filename, [read, binary]), + ReqBody = make_req_body(BodyIoDevice), + ok = file:close(BodyIoDevice), + CType = guess_ctype(Label), + BasicAuth = {basic_auth, {User, Password}}, + %% Send request try ibrowse:send_req(URL, [{"Content-Type", CType}], - post, - ReqBody, - [{content_type, CType}, basic_auth()], - 300000) of + post, + ReqBody, + [{content_type, CType}, BasicAuth], + 300000) of {ok, [$2|_], Headers, _Body} -> {_, Location} = lists:keyfind("Location", 1, Headers), - lager:info("Successfully uploaded test artifact ~s to GiddyUp! URL: ~s", [FName, Location]), + lager:info("Successfully uploaded test artifact ~s to GiddyUp! URL: ~s", [Label, Location]), ok; {ok, RC, Headers, Body} -> - lager:info("Test artifact ~s failed to upload!", [FName]), + lager:info("Test artifact ~s failed to upload!", [Label]), lager:debug("Status: ~p~nHeaders: ~p~nBody: ~s~n", [RC, Headers, Body]), error; X -> lager:error("Error uploading ~s to giddyup. ~p~n" - "URL: ~p~nRequest Body: ~p~nContent Type: ~p~n", - [FName, X, URL, ReqBody, CType]), + "URL: ~p~nRequest Body: ~p~nContent Type: ~p~n", + [Label, X, URL, ReqBody, CType]), error catch Throws -> lager:error("Error uploading ~s to giddyup. ~p~n" - "URL: ~p~nRequest Body: ~p~nContent Type: ~p~n", - [FName, Throws, URL, ReqBody, CType]) + "URL: ~p~nRequest Body: ~p~nContent Type: ~p~n", + [Label, Throws, URL, ReqBody, CType]) end. -basic_auth() -> - {basic_auth, {rt_config:get(giddyup_user), rt_config:get(giddyup_password)}}. - +%%-------------------------------------------------------------------- +%% @private +%% @doc %% Given a URI parsed by http_uri, reconstitute it. -generate({_Scheme, _UserInfo, _Host, _Port, _Path, _Query}=URI) -> - generate(URI, http_uri:scheme_defaults()). +%% @end +%%-------------------------------------------------------------------- + +generate_uri({_Scheme, _UserInfo, _Host, _Port, _Path, _Query}=URI) -> + generate_uri(URI, http_uri:scheme_defaults()). -generate({Scheme, UserInfo, Host, Port, Path, Query}, SchemeDefaults) -> +generate_uri({Scheme, UserInfo, Host, Port, Path, Query}, SchemeDefaults) -> {Scheme, DefaultPort} = lists:keyfind(Scheme, 1, SchemeDefaults), lists:flatten([ [ atom_to_list(Scheme), "://" ], @@ -140,26 +442,45 @@ generate({Scheme, UserInfo, Host, Port, Path, Query}, SchemeDefaults) -> Path, Query ]). +%%-------------------------------------------------------------------- +%% @private +%% @doc %% Given the test result URL, constructs the appropriate URL for the artifact. -artifact_url(TRURL, FName) -> - {ok, {Scheme, UserInfo, Host, Port, Path, Query}} = http_uri:parse(TRURL), +%% @end +%%-------------------------------------------------------------------- + +artifact_url(BaseURL, FName) -> + {ok, {Scheme, UserInfo, Host, Port, Path, Query}} = http_uri:parse(BaseURL), ArtifactPath = filename:join([Path, "artifacts", FName]), - generate({Scheme, UserInfo, Host, Port, ArtifactPath, Query}). + generate_uri({Scheme, UserInfo, Host, Port, ArtifactPath, Query}). +%%-------------------------------------------------------------------- +%% @private +%% @doc %% ibrowse support streaming request bodies, so in the case where we %% have a Port/File to read from, we should stream it. +%% @end +%%-------------------------------------------------------------------- + make_req_body(Body) when is_port(Body); is_pid(Body) -> read_fully(Body); -make_req_body(Body) when is_list(Body); - is_binary(Body) -> +make_req_body(Body) when is_list(Body); is_binary(Body) -> Body. +%%-------------------------------------------------------------------- +%% @private +%% @doc %% Read the file/port fully until eof. This is a workaround for the %% fact that ibrowse doesn't seem to send file streams correctly, or %% giddyup dislikes them. (shrug) +%% @end +%%-------------------------------------------------------------------- + +-spec(read_fully(string() | port()) -> binary()). read_fully(File) -> read_fully(File, <<>>). +-spec(read_fully(string() | port(), binary()) -> binary()). read_fully(File, Data0) -> case file:read(File, ?STREAM_CHUNK_SIZE) of {ok, Data} -> @@ -168,10 +489,19 @@ read_fully(File, Data0) -> Data0 end. +%%-------------------------------------------------------------------- +%% @private +%% @doc %% Guesses the MIME type of the file being uploaded. +%% @end +%%-------------------------------------------------------------------- + +-spec(guess_ctype(string()) -> string()). guess_ctype(FName) -> case string:tokens(filename:basename(FName), ".") of [_, "log"|_] -> "text/plain"; %% console.log, erlang.log.5, etc + [_, "conf"|_] -> "text/plain"; %% riak.conf + [_, "config"|_] -> "text/plain"; %% advanced.config, etc ["erl_crash", "dump"] -> "text/plain"; %% An erl_crash.dump file [_, Else] -> case mochiweb_mime:from_extension(Else) of @@ -180,3 +510,14 @@ guess_ctype(FName) -> end; _ -> "binary/octet-stream" end. + +%%-------------------------------------------------------------------- +%% @private +%% @doc +%% Verify that an application is running +%% @end +%%-------------------------------------------------------------------- + +load_and_start(Application) -> + application:load(Application), + application:start(Application). \ No newline at end of file diff --git a/src/node_manager.erl b/src/node_manager.erl new file mode 100644 index 000000000..59e3e21b9 --- /dev/null +++ b/src/node_manager.erl @@ -0,0 +1,251 @@ +-module(node_manager). + +-behavior(gen_server). + +%% API +-export([start_link/3, + reserve_nodes/3, + deploy_nodes/5, + upgrade_nodes/5, + return_nodes/1, + status/0, + stop/0]). + +%% gen_server callbacks +-export([init/1, + handle_call/3, + handle_cast/2, + handle_info/2, + terminate/2, + code_change/3]). + +-record(state, {nodes :: [string()], + node_map :: [{string(), node()}], + nodes_available :: [string()], + nodes_deployed=[] :: [string()], + deployed_versions=[] :: [{string(), string()}], + version_map :: [{string(), [string()]}]}). + +%%%=================================================================== +%%% API +%%%=================================================================== + +start_link(Nodes, NodeMap, VersionMap) -> + Args = [Nodes, NodeMap, VersionMap], + gen_server:start_link({local, ?MODULE}, ?MODULE, Args, []). + +-spec reserve_nodes(pos_integer(), [string()], function()) -> ok. +reserve_nodes(NodeCount, Versions, NotifyFun) -> + lager:debug("reserve_nodes(~p, ~p, ~p)", [NodeCount, Versions, NotifyFun]), + gen_server:cast(?MODULE, {reserve_nodes, NodeCount, Versions, NotifyFun}). + +-spec deploy_nodes([string()], string(), term(), list(atom()), function()) -> ok. +deploy_nodes(Nodes, Version, Config, Services, NotifyFun) -> + gen_server:cast(?MODULE, {deploy_nodes, Nodes, Version, Config, Services, NotifyFun}). + +-spec upgrade_nodes([string()], string(), string(), term(), function()) -> ok. +upgrade_nodes(Nodes, CurrentVersion, NewVersion, Config, NotifyFun) -> + gen_server:cast(?MODULE, {deploy_nodes, Nodes, CurrentVersion, NewVersion, Config, NotifyFun}). + +-spec return_nodes([string()]) -> ok. +return_nodes(Nodes) -> + gen_server:cast(?MODULE, {return_nodes, Nodes}). + +-spec status() -> [{atom(), list()}]. +status() -> + gen_server:call(?MODULE, status, infinity). + +-spec stop() -> ok. +stop() -> + gen_server:call(?MODULE, stop, infinity). + +%%%=================================================================== +%%% gen_server callbacks +%%%=================================================================== + +init([Nodes, NodeMap, VersionMap]) -> + SortedNodes = lists:sort(Nodes), + {ok, #state{nodes=SortedNodes, + node_map=NodeMap, + nodes_available=SortedNodes, + version_map=VersionMap}}. + +handle_call(status, _From, State) -> + Status = [{nodes, State#state.nodes}, + {nodes_available, State#state.nodes_available}, + {version_map, State#state.version_map}], + {reply, Status, State}; +handle_call(stop, _From, State) -> + {stop, normal, ok, State}. + +handle_cast({reserve_nodes, Count, Versions, NotifyFun}, State) -> + lager:debug("Handling cast to reserve ~p nodes with versions ~p.", [Count, Versions]), + {Result, UpdState} = + reserve(Count, Versions, State), + NotifyFun(Result), + {noreply, UpdState}; +handle_cast({deploy_nodes, Nodes, Version, Config, Services, NotifyFun}, State) -> + Result = deploy(Nodes, State#state.node_map, Version, Config, Services), + DeployedVersions = State#state.deployed_versions, + UpdDeployedVersions = update_deployed_versions(deploy, Nodes, Version, DeployedVersions), + NotifyFun({nodes_deployed, Result}), + {noreply, State#state{deployed_versions=UpdDeployedVersions}}; +handle_cast({upgrade_nodes, Nodes, CurrentVersion, NewVersion, Config, NotifyFun}, State) -> + Result = upgrade(Nodes, CurrentVersion, NewVersion, Config), + DeployedVersions = State#state.deployed_versions, + UpdDeployedVersions = update_deployed_versions(upgrade, Nodes, NewVersion, DeployedVersions), + NotifyFun({nodes_upgraded, Result}), + {noreply, State#state{deployed_versions=UpdDeployedVersions}}; +handle_cast({return_nodes, Nodes}, State) -> + %% Stop nodes and clean data dirs so they are ready for next use. + NodesAvailable = State#state.nodes_available, + DeployedVersions = State#state.deployed_versions, + NodeMap = State#state.node_map, + stop_and_clean(Nodes, NodeMap, DeployedVersions, false), + NodesNowAvailable = lists:merge(lists:sort(Nodes), NodesAvailable), + + UpdDeployedVersions = update_deployed_versions(return, Nodes, undefined, DeployedVersions), + {noreply, State#state{nodes_available=NodesNowAvailable, + deployed_versions=UpdDeployedVersions}}; +handle_cast(_Msg, State) -> + {noreply, State}. + +handle_info(_Info, State) -> + {noreply, State}. + +terminate(_Reason, _State) -> + %% Stop and reset all deployed nodes + %%stop(State#state.nodes_deployed, + %% State#state.node_map, + %% State#state.deployed_versions), + ok. + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +%%%=================================================================== +%%% Internal functions +%%%=================================================================== + +% stop(NodeIds, NodeMap, DeployedVersions) -> +% [begin +% case version_deployed(NodeId, DeployedVersions) of +% undefined -> +% ok; +% Version -> +% rt_node:stop_and_wait(NodeId, +% rt_node:node_name(NodeId, NodeMap), +% Version) +% end +% end || NodeId <- NodeIds]. + + +stop_and_clean(NodeIds, NodeMap, DeployedVersions, Wait) -> + [begin + case version_deployed(NodeId, DeployedVersions) of + undefined -> + ok; + Version -> + rt_node:stop_and_wait(NodeId, + rt_node:node_name(NodeId, NodeMap), + Version), + wait_for_cleaner(rt_node:clean_data_dir(NodeId, Version), Wait) + end + end || NodeId <- NodeIds]. + +wait_for_cleaner(Pid, true) -> + WaitFun = + fun() -> + not is_process_alive(Pid) + end, + rt:wait_until(WaitFun); +wait_for_cleaner(_, false) -> + ok. + +reserve(Count, _Versions, State) when Count == 0 -> + lager:debug("Reserving no nodes ..."), + {{nodes, [], State#state.node_map}, State}; +reserve(Count, _Versions, State=#state{nodes_available=NodesAvailable}) + when Count > length(NodesAvailable) -> + {not_enough_nodes, State}; +reserve(Count, Versions, State=#state{nodes_available=NodesAvailable, + nodes_deployed=NodesDeployed, + version_map=VersionMap}) + when Count =:= length(NodesAvailable) -> + case versions_available(Count, Versions, VersionMap) of + true -> + UpdNodesDeployed = lists:sort(NodesDeployed ++ NodesAvailable), + Result = {nodes, NodesAvailable, State#state.node_map}, + {Result, State#state{nodes_available=[], + nodes_deployed=UpdNodesDeployed}}; + false -> + {insufficient_versions_available, State} + end; +reserve(Count, Versions, State=#state{nodes_available=NodesAvailable, + nodes_deployed=NodesDeployed, + version_map=VersionMap}) -> + case versions_available(Count, Versions, VersionMap) of + true -> + {Reserved, UpdNodesAvailable} = lists:split(Count, NodesAvailable), + UpdNodesDeployed = lists:sort(NodesDeployed ++ Reserved), + UpdState = State#state{nodes_available=UpdNodesAvailable, + nodes_deployed=UpdNodesDeployed}, + Result = {nodes, Reserved, UpdState#state.node_map}, + {Result, UpdState}; + false -> + {insufficient_versions_available, State} + end. + +versions_available(Count, Versions, VersionMap) -> + lists:all(version_available_fun(Count, VersionMap), Versions). + +version_available_fun(Count, VersionMap) -> + fun(Version) -> + case lists:keyfind(Version, 1, VersionMap) of + {Version, VersionNodes} when length(VersionNodes) >= Count -> + true; + {Version, _} -> + false; + false -> + false + end + end. + +deploy([], _NodeMap, _Version, _Config, []) -> + []; +deploy(Nodes, NodeMap, Version, Config, Services) -> + rt_harness_util:deploy_nodes(Nodes, NodeMap, Version, Config, Services). + +upgrade(Nodes, CurrentVersion, NewVersion, Config) -> + [rt_node:upgrade(Node, CurrentVersion, NewVersion, Config) || + Node <- Nodes]. + +update_deployed_versions(deploy, Nodes, Version, DeployedVersions) -> + {_, UpdDeployedVersions} = lists:foldl(fun add_deployed_version/2, + {Version, DeployedVersions}, + Nodes), + UpdDeployedVersions; +update_deployed_versions(upgrade, Nodes, Version, DeployedVersions) -> + {_, UpdDeployedVersions} = lists:foldl(fun replace_deployed_version/2, + {Version, DeployedVersions}, + Nodes), + UpdDeployedVersions; +update_deployed_versions(return, Nodes, _, DeployedVersions) -> + lists:foldl(fun remove_deployed_version/2, DeployedVersions, Nodes). + +add_deployed_version(Node, {Version, DeployedVersions}) -> + {Version, [{Node, Version} | DeployedVersions]}. + +replace_deployed_version(Node, {Version, DeployedVersions}) -> + {Version, lists:keyreplace(Node, 1, DeployedVersions, {Node, Version})}. + +remove_deployed_version(Node, DeployedVersions) -> + lists:keydelete(Node, 1, DeployedVersions). + +version_deployed(Node, DeployedVersions) -> + case lists:keyfind(Node, 1, DeployedVersions) of + {Node, Version} -> + Version; + false -> + undefined + end. diff --git a/src/riak_test.app.src b/src/riak_test.app.src index 86c5cc8d3..d1cb3bcc2 100644 --- a/src/riak_test.app.src +++ b/src/riak_test.app.src @@ -11,7 +11,8 @@ {env, [ {platform, undefined}, {rt_scratch_dir, "/tmp/riak_test_scratch"}, - {rt_max_wait_time, 180000}, + {rt_max_receive_wait_time, 600000}, + {test_timeout, 1800000}, {rt_retry_delay, 500}, {rt_harness, rtdev}, {java, [{fat_be_url, "http://riak-java-client.s3.amazonaws.com/riak-client-1.4.2-jar-with-dependencies-and-tests.jar"}, diff --git a/src/riak_test.erl b/src/riak_test.erl index 9c73ce592..edd8fbe78 100644 --- a/src/riak_test.erl +++ b/src/riak_test.erl @@ -1,3 +1,4 @@ +-module(riak_test). %% ------------------------------------------------------------------- %% %% Copyright (c) 2013 Basho Technologies, Inc. @@ -18,7 +19,7 @@ %% %% ------------------------------------------------------------------- --module(riak_test). %% Define the riak_test behavior +%% -callback confirm(rt_properties:properties()) -> pass | fail. -callback confirm() -> pass | fail. diff --git a/src/riak_test_escript.erl b/src/riak_test_escript.erl index a34da59f8..182a6d6fa 100644 --- a/src/riak_test_escript.erl +++ b/src/riak_test_escript.erl @@ -20,411 +20,461 @@ %% @private -module(riak_test_escript). --include("rt.hrl"). +%% TODO: Temporary build workaround, remove!! +-compile(export_all). -export([main/1]). --export([add_deps/1]). -add_deps(Path) -> - {ok, Deps} = file:list_dir(Path), - [code:add_path(lists:append([Path, "/", Dep, "/ebin"])) || Dep <- Deps], - ok. +-ifdef(TEST). +-include_lib("eunit/include/eunit.hrl"). +-endif. + +main(Args) -> + %% TODO Should we use clique? -jsb + %% Parse command line arguments ... + {ParsedArgs, _NonOptionArgs} = parse_args(Args), + load_initial_config(ParsedArgs), + + %% Configure logging ... + OutDir = proplists:get_value(outdir, ParsedArgs, "log"), + ensure_dir(OutDir), + lager_setup(OutDir), + ok = erlang_setup(ParsedArgs), + + %% Do we use GiddyUp for this run? + Platform = report_platform(ParsedArgs), + UseGiddyUp = case Platform of + undefined -> false; + _ -> true + end, + start_giddyup(Platform), + {Tests, NonTests} = generate_test_lists(UseGiddyUp, ParsedArgs), + + ok = prepare_tests(Tests, NonTests), + Results = execute(Tests, OutDir, ParsedArgs), + finalize(Results, ParsedArgs), + stop_giddyup(UseGiddyUp). + +% @doc Validate the command-line options +parse_args(Args) -> + validate_args(getopt:parse(cli_options(), Args)). + +validate_args({ok, {[], _}}) -> + print_help(); +validate_args({ok, {ParsedArgs, NonOptionArgs}}) -> + case proplists:is_defined(help, ParsedArgs) of + true -> + print_help(); + _ -> + {ParsedArgs, NonOptionArgs} + end; +validate_args(_) -> + print_help(). -cli_options() -> %% Option Name, Short Code, Long Code, Argument Spec, Help Message +cli_options() -> [ - {help, $h, "help", undefined, "Print this usage page"}, - {config, $c, "conf", string, "specifies the project configuration"}, - {tests, $t, "tests", string, "specifies which tests to run"}, - {suites, $s, "suites", string, "which suites to run"}, - {dir, $d, "dir", string, "run all tests in the specified directory"}, - {skip, $x, "skip", string, "list of tests to skip in a directory"}, - {verbose, $v, "verbose", undefined, "verbose output"}, - {outdir, $o, "outdir", string, "output directory"}, - {backend, $b, "backend", atom, "backend to test [memory | bitcask | eleveldb]"}, - {upgrade_version, $u, "upgrade", atom, "which version to upgrade from [ previous | legacy ]"}, - {keep, undefined, "keep", boolean, "do not teardown cluster"}, - {report, $r, "report", string, "you're reporting an official test run, provide platform info (e.g. ubuntu-1204-64)\nUse 'config' if you want to pull from ~/.riak_test.config"}, - {file, $F, "file", string, "use the specified file instead of ~/.riak_test.config"} + {help, $h, "help", undefined, "Print this usage page"}, + {config, $c, "conf", string, "specifies the project configuration"}, + {tests, $t, "tests", string, "specifies which tests to run"}, + {dir, $d, "dir", string, "run all tests in the specified directory"}, + {skip, $x, "skip", string, "list of tests to skip in a directory"}, + {verbose, $v, "verbose", undefined, "verbose output"}, + {outdir, $o, "outdir", string, "output directory"}, + {backend, $b, "backend", atom, "backend to test [memory | bitcask | eleveldb | multi]"}, + {upgrade_path, $u, "upgrade", atom, "which user-defined upgrade path to use [ e.g. previous | legacy ]"}, + {keep, undefined, "keep", boolean, "do not teardown cluster"}, + {continue_on_fail, $n, "continue", boolean, "continues executing tests on failure"}, + {report, $r, "report", string, "you're reporting an official test run, provide platform info (e.g. ubuntu-1404-64)\nUse 'config' if you want to pull from ~/.riak_test.config"}, + {file, $F, "file", string, "use the specified file instead of ~/.riak_test.config"} ]. print_help() -> - getopt:usage(cli_options(), - escript:script_name()), + getopt:usage(cli_options(), escript:script_name()), halt(0). -run_help([]) -> true; -run_help(ParsedArgs) -> - lists:member(help, ParsedArgs). - -main(Args) -> - case filelib:is_dir("./ebin") of - true -> - code:add_patha("./ebin"); - _ -> - meh - end, - - register(riak_test, self()), - {ParsedArgs, HarnessArgs} = case getopt:parse(cli_options(), Args) of - {ok, {P, H}} -> {P, H}; - _ -> print_help() - end, - - case run_help(ParsedArgs) of - true -> print_help(); - _ -> ok - end, - - %% ibrowse - application:load(ibrowse), - application:start(ibrowse), - %% Start Lager - application:load(lager), +report_platform(ParsedArgs) -> + case proplists:get_value(report, ParsedArgs, undefined) of + undefined -> + undefined; + "config" -> + rt_config:get(giddyup_platform); + R -> + R + end. - Config = proplists:get_value(config, ParsedArgs), - ConfigFile = proplists:get_value(file, ParsedArgs), +%% @doc Print help string if it's specified, otherwise parse the arguments +generate_test_lists(UseGiddyUp, ParsedArgs) -> + %% Have to load the `riak_test' config prior to assembling the + %% test metadata + + TestData = compose_test_data(ParsedArgs), + CmdLineBackends = rt_util:convert_to_atom_list(proplists:get_value(backend, ParsedArgs)), + Backends = determine_backends(CmdLineBackends, UseGiddyUp), + UpgradeList = rt_util:convert_to_atom_list(proplists:get_value(upgrade_path, ParsedArgs)), + {Tests, NonTests} = wrap_test_in_test_plan(UseGiddyUp, Backends, UpgradeList, TestData), + Offset = rt_config:get(offset, undefined), + Workers = rt_config:get(workers, undefined), + shuffle_tests(Tests, NonTests, Offset, Workers). + +%% @doc Which backends should be tested? +%% Use the command-line specified backend, otherwise default to bitcask +%% If running under GiddyUp, then default to ALL backends +%% First argument is a list of command-line backends and second is whether or not in GiddyUp mode +-spec(determine_backends(atom(), boolean()) -> list()). +determine_backends(undefined, true) -> + [memory, bitcask, eleveldb, multi]; +determine_backends(undefined, _) -> + [bitcask]; +determine_backends(Backends, _) -> + Backends. + +%% @doc Set values in the configuration with values specified on the command line +maybe_override_setting(Argument, Value, Arguments) -> + maybe_override_setting(proplists:is_defined(Argument, Arguments), Argument, + Value, Arguments). + +maybe_override_setting(true, Argument, Value, Arguments) -> + rt_config:set(Argument, proplists:get_value(Argument, Arguments, Value)); +maybe_override_setting(false, _Argument, _Value, _Arguments) -> + ok. +load_initial_config(ParsedArgs) -> %% Loads application defaults application:load(riak_test), %% Loads from ~/.riak_test.config - rt_config:load(Config, ConfigFile), - - %% Sets up extra paths earlier so that tests can be loadable - %% without needing the -d flag. - code:add_paths(rt_config:get(test_paths, [])), - - %% Ensure existance of scratch_dir - case file:make_dir(rt_config:get(rt_scratch_dir)) of - ok -> great; - {error, eexist} -> great; - {ErrorType, ErrorReason} -> lager:error("Could not create scratch dir, {~p, ~p}", [ErrorType, ErrorReason]) - end, - - %% Fileoutput - Outdir = proplists:get_value(outdir, ParsedArgs), - ConsoleLagerLevel = case Outdir of - undefined -> rt_config:get(lager_level, info); + rt_config:load(proplists:get_value(config, ParsedArgs), + proplists:get_value(file, ParsedArgs)), + + %% Override any command-line settings in config + maybe_override_setting(continue_on_fail, true, ParsedArgs). + +%% @doc Shuffle the order in which tests are scheduled +shuffle_tests([], _, _, _) -> + io:format("ERROR: No tests are scheduled to run~n"), + lager:error("No tests are scheduled to run"), + halt(1); +shuffle_tests(Tests, NonTests, undefined, _) -> + {Tests, NonTests}; +shuffle_tests(Tests, NonTests, _, undefined) -> + {Tests, NonTests}; +shuffle_tests(Tests, NonTests, Offset, Workers) -> + TestCount = length(Tests), + %% Avoid dividing by zero, computers hate that + Denominator = case Workers rem (TestCount+1) of + 0 -> 1; + D -> D + end, + ActualOffset = ((TestCount div Denominator) * Offset) rem (TestCount+1), + {TestA, TestB} = lists:split(ActualOffset, Tests), + lager:info("Offsetting ~b tests by ~b (~b workers, ~b offset)", + [TestCount, ActualOffset, Workers, Offset]), + {TestB ++ TestA, NonTests}. + +prepare_tests(Tests, NonTests) -> + [lager:notice("Test to run: ~p", [rt_test_plan:get_name(Test)]) || Test <- Tests], + case NonTests of + [] -> + ok; _ -> - filelib:ensure_dir(Outdir), - notice + [lager:notice("Test not to run: ~p", [rt_test_plan:get_name(Test)]) || Test <- NonTests] end, + test_setup(). + +execute(TestPlans, OutDir, ParsedArgs) -> + %% TODO: Clean up upgrade paths. Living in test plans at the moment. + %% UpgradeList = upgrade_list( + %% proplists:get_value(upgrade_path, ParsedArgs)), + + {ok, Executor} = riak_test_executor:start_link(TestPlans, + OutDir, + report_platform(ParsedArgs), + undefined, + self()), + wait_for_results(Executor, [], length(TestPlans), 0). + + +%% TODO: Use `TestCount' and `Completed' to display progress output +wait_for_results(Executor, TestResults, TestCount, Completed) -> + receive + {_Executor, {test_result, Result}} -> + wait_for_results(Executor, [Result | TestResults], TestCount, Completed+1); + {_Executor, done} -> + rt_cover:stop(), + TestResults; + _ -> + wait_for_results(Executor, TestResults, TestCount, Completed) + end. - application:set_env(lager, handlers, [{lager_console_backend, ConsoleLagerLevel}, - {lager_file_backend, [{file, "log/test.log"}, - {level, ConsoleLagerLevel}]}]), - lager:start(), +finalize(TestResults, Args) -> + %% TODO: Fixup coverage reporting + %% [rt_cover:maybe_import_coverage(proplists:get_value(coverdata, R)) || + %% R <- TestResults], + %% CoverDir = rt_config:get(cover_output, "coverage"), + %% Coverage = rt_cover:maybe_write_coverage(all, CoverDir), + %% Verbose = proplists:is_defined(verbose, Args), - %% Report - Report = case proplists:get_value(report, ParsedArgs, undefined) of - undefined -> undefined; - "config" -> rt_config:get(platform, undefined); - R -> R - end, + Teardown = not proplists:get_value(keep, Args, false), + maybe_teardown(Teardown, TestResults), + ok. + +test_setup() -> + %% Prepare the test harness + {NodeIds, NodeMap, VersionMap} = rt_harness:setup(), - Verbose = proplists:is_defined(verbose, ParsedArgs), + %% Start the node manager + _ = node_manager:start_link(NodeIds, NodeMap, VersionMap), - Suites = proplists:get_all_values(suites, ParsedArgs), - case Suites of - [] -> ok; - _ -> io:format("Suites are not currently supported.") + %% Ensure existence of scratch_dir + case ensure_dir(rt_config:get(rt_scratch_dir) ++ "/test.file") of + ok -> + great; + {error, ErrorReason} -> + lager:error("Could not create scratch dir, ~p", + [ErrorReason]) end, + ok. - CommandLineTests = parse_command_line_tests(ParsedArgs), - Tests0 = which_tests_to_run(Report, CommandLineTests), +erlang_setup(_ParsedArgs) -> + register(riak_test, self()), + maybe_add_code_path("./ebin"), - case Tests0 of - [] -> - lager:warning("No tests are scheduled to run"), - init:stop(1); - _ -> keep_on_keepin_on - end, + %% Sets up extra paths earlier so that tests can be loadable + %% without needing the -d flag. + code:add_paths(rt_config:get(test_paths, [])), - Tests = case {rt_config:get(offset, undefined), rt_config:get(workers, undefined)} of - {undefined, undefined} -> - Tests0; - {undefined, _} -> - Tests0; - {_, undefined} -> - Tests0; - {Offset, Workers} -> - TestCount = length(Tests0), - %% Avoid dividing by zero, computers hate that - Denominator = case Workers rem (TestCount+1) of - 0 -> 1; - D -> D - end, - ActualOffset = ((TestCount div Denominator) * Offset) rem (TestCount+1), - {TestA, TestB} = lists:split(ActualOffset, Tests0), - lager:info("Offsetting ~b tests by ~b (~b workers, ~b" - " offset)", [TestCount, ActualOffset, Workers, - Offset]), - TestB ++ TestA - end, - - io:format("Tests to run: ~p~n", [Tests]), %% Two hard-coded deps... - add_deps(rt:get_deps()), - add_deps("deps"), + rt_util:add_deps(rt:get_deps()), + rt_util:add_deps("deps"), - [add_deps(Dep) || Dep <- rt_config:get(rt_deps, [])], - ENode = rt_config:get(rt_nodename, 'riak_test@127.0.0.1'), - Cookie = rt_config:get(rt_cookie, riak), - CoverDir = rt_config:get(cover_output, "coverage"), + [rt_util:add_deps(Dep) || Dep <- rt_config:get(rt_deps, [])], [] = os:cmd("epmd -daemon"), - net_kernel:start([ENode]), - erlang:set_cookie(node(), Cookie), + net_kernel:start([rt_config:get(rt_nodename, 'riak_test@127.0.0.1')]), + erlang:set_cookie(node(), rt_config:get(rt_cookie, riak)), + ok. - TestResults = lists:filter(fun results_filter/1, [ run_test(Test, Outdir, TestMetaData, Report, HarnessArgs, length(Tests)) || {Test, TestMetaData} <- Tests]), - [rt_cover:maybe_import_coverage(proplists:get_value(coverdata, R)) || R <- TestResults], - Coverage = rt_cover:maybe_write_coverage(all, CoverDir), +maybe_add_code_path(Path) -> + maybe_add_code_path(Path, filelib:is_dir(Path)). - Teardown = not proplists:get_value(keep, ParsedArgs, false), - maybe_teardown(Teardown, TestResults, Coverage, Verbose), - ok. +maybe_add_code_path(Path, true) -> + code:add_patha(Path); +maybe_add_code_path(_, false) -> + meh. + +ensure_dir(undefined) -> + ok; +ensure_dir(Dir) -> + filelib:ensure_dir(Dir). + +lager_setup(OutputDir) -> + set_lager_env(OutputDir, + rt_config:get(lager_console_level, notice), + rt_config:get(lager_file_level, info)), + lager:start(). + +set_lager_env(OutputDir, ConsoleLevel, FileLevel) -> + application:load(lager), + HandlerConfig = [{lager_console_backend, ConsoleLevel}, + {lager_file_backend, [{file, filename:join(OutputDir, "test.log")}, + {level, FileLevel}]}], + application:set_env(lager, handlers, HandlerConfig). -maybe_teardown(false, TestResults, Coverage, Verbose) -> - print_summary(TestResults, Coverage, Verbose), +maybe_teardown(false, _TestResults) -> lager:info("Keeping cluster running as requested"); -maybe_teardown(true, TestResults, Coverage, Verbose) -> - case {length(TestResults), proplists:get_value(status, hd(TestResults))} of - {1, fail} -> - print_summary(TestResults, Coverage, Verbose), - so_kill_riak_maybe(); - _ -> - lager:info("Multiple tests run or no failure"), - rt:teardown(), - print_summary(TestResults, Coverage, Verbose) - end, +maybe_teardown(Keep, TestResults) when is_list(TestResults) andalso + erlang:length(TestResults) == 1 -> + maybe_teardown(Keep, hd(TestResults)); +maybe_teardown(true, {_, {fail, _}, _}) -> + so_kill_riak_maybe(), + ok; +maybe_teardown(true, _TestResults) -> + lager:info("Multiple tests run or no failure"), + rt_cluster:teardown(), ok. -parse_command_line_tests(ParsedArgs) -> - Backends = case proplists:get_all_values(backend, ParsedArgs) of - [] -> [undefined]; - Other -> Other - end, - Upgrades = case proplists:get_all_values(upgrade_version, ParsedArgs) of - [] -> [undefined]; - UpgradeList -> UpgradeList - end, +-spec comma_tokenizer(string(), [string()]) -> [string()]. +comma_tokenizer(S, Acc) -> + string:tokens(S, ", ") ++ Acc. + +compose_test_data(ParsedArgs) -> + RawTestList = proplists:get_all_values(tests, ParsedArgs), + RawGroupList = proplists:get_all_values(groups, ParsedArgs), + TestList = lists:foldl(fun comma_tokenizer/2, [], RawTestList), + GroupList = lists:foldl(fun comma_tokenizer/2, [], RawGroupList), + %% Parse Command Line Tests {CodePaths, SpecificTests} = lists:foldl(fun extract_test_names/2, {[], []}, - proplists:get_all_values(tests, ParsedArgs)), + TestList), + [code:add_patha(CodePath) || CodePath <- CodePaths, CodePath /= "."], - Dirs = proplists:get_all_values(dir, ParsedArgs), + + Dirs = get_test_dirs(ParsedArgs, default_test_dir(GroupList)), SkipTests = string:tokens(proplists:get_value(skip, ParsedArgs, []), [$,]), - DirTests = lists:append([load_tests_in_dir(Dir, SkipTests) || Dir <- Dirs]), - lists:foldl(fun(Test, Tests) -> - [{ - list_to_atom(Test), - [ - {id, -1}, - {platform, <<"local">>}, - {version, rt:get_version()}, - {project, list_to_binary(rt_config:get(rt_project, "undefined"))} - ] ++ - [ {backend, Backend} || Backend =/= undefined ] ++ - [ {upgrade_version, Upgrade} || Upgrade =/= undefined ]} - || Backend <- Backends, - Upgrade <- Upgrades ] ++ Tests - end, [], lists:usort(DirTests ++ SpecificTests)). + DirTests = lists:append([load_tests_in_dir(Dir, GroupList, SkipTests) || Dir <- Dirs]), + lists:usort(DirTests ++ SpecificTests). + +-spec default_test_dir([string()]) -> [string()]. +%% @doc If any groups have been specified then we want to check in the +%% local test directory by default; otherwise, the default behavior is +%% that no directory is used to pull tests from. +default_test_dir([]) -> + []; +default_test_dir(_) -> + ["./ebin"]. + +-spec get_test_dirs(term(), [string()]) -> [string()]. +get_test_dirs(ParsedArgs, DefaultDirs) -> + case proplists:get_all_values(dir, ParsedArgs) of + [] -> + DefaultDirs; + Dirs -> + Dirs + end. extract_test_names(Test, {CodePaths, TestNames}) -> {[filename:dirname(Test) | CodePaths], - [filename:rootname(filename:basename(Test)) | TestNames]}. - -which_tests_to_run(undefined, CommandLineTests) -> - {Tests, NonTests} = - lists:partition(fun is_runnable_test/1, CommandLineTests), - lager:info("These modules are not runnable tests: ~p", - [[NTMod || {NTMod, _} <- NonTests]]), - Tests; -which_tests_to_run(Platform, []) -> giddyup:get_suite(Platform); -which_tests_to_run(Platform, CommandLineTests) -> - Suite = filter_zip_suite(Platform, CommandLineTests), - {Tests, NonTests} = - lists:partition(fun is_runnable_test/1, - lists:foldr(fun filter_merge_tests/2, [], Suite)), - - lager:info("These modules are not runnable tests: ~p", - [[NTMod || {NTMod, _} <- NonTests]]), - Tests. - -filter_zip_suite(Platform, CommandLineTests) -> - [ {SModule, SMeta, CMeta} || {SModule, SMeta} <- giddyup:get_suite(Platform), - {CModule, CMeta} <- CommandLineTests, - SModule =:= CModule]. - -filter_merge_tests({Module, SMeta, CMeta}, Tests) -> - case filter_merge_meta(SMeta, CMeta, [backend, upgrade_version]) of - false -> - Tests; - Meta -> - [{Module, Meta}|Tests] - end. - -filter_merge_meta(SMeta, _CMeta, []) -> - SMeta; -filter_merge_meta(SMeta, CMeta, [Field|Rest]) -> - case {kvc:value(Field, SMeta, undefined), kvc:value(Field, CMeta, undefined)} of - {X, X} -> - filter_merge_meta(SMeta, CMeta, Rest); - {_, undefined} -> - filter_merge_meta(SMeta, CMeta, Rest); - {undefined, X} -> - filter_merge_meta(lists:keystore(Field, 1, SMeta, {Field, X}), CMeta, Rest); + [list_to_atom(filename:rootname(filename:basename(Test))) | TestNames]}. + +%% @doc Determine which tests to run based on command-line argument +%% If the platform is defined, consult GiddyUp, otherwise just shovel +%% the whole thing into the Planner +-spec(load_up_test_planner(boolean(), [string()], undefined | [string()], list()) -> list()). +load_up_test_planner(true, Backends, _UpgradePaths, CommandLineTests) -> + rt_planner:load_from_giddyup(Backends, CommandLineTests); +load_up_test_planner(_, Backends, UpgradePaths, CommandLineTests) -> + [rt_planner:add_test_plan(Name, undefined, Backends, UpgradePaths, undefined) || Name <- CommandLineTests]. + +%% @doc Push all of the test into the Planner for now and wrap them in an `rt_test_plan' +%% TODO: Let the Planner do the work, not the riak_test_executor +-spec(wrap_test_in_test_plan(boolean(), [atom()], undefined | [atom()], [atom()]) -> {list(), list()}). +wrap_test_in_test_plan(UseGiddyUp, Backends, UpgradeList, CommandLineTests) -> + {ok, _Pid} = rt_planner:start_link(), + load_up_test_planner(UseGiddyUp, Backends, UpgradeList, CommandLineTests), + TestPlans = [rt_planner:fetch_test_plan() || _ <- lists:seq(1, rt_planner:number_of_plans())], + NonRunnableTestPlans = [rt_planner:fetch_test_non_runnable_plan() || _ <- lists:seq(1, rt_planner:number_of_non_runable_plans())], + rt_planner:stop(), + {TestPlans, NonRunnableTestPlans}. + +%% @doc Pull all jobs from the Planner +%% Better than using rt_planner:number_of_plans/0 +-spec(fetch_all_test_plans(list()) -> list()). +fetch_all_test_plans(Acc) -> + Plan = rt_planner:fetch_test_plan(), + case Plan of + empty -> + Acc; _ -> - false + fetch_all_test_plans([Plan|Acc]) end. -%% Check for api compatibility -is_runnable_test({TestModule, _}) -> - {Mod, Fun} = riak_test_runner:function_name(TestModule), - code:ensure_loaded(Mod), - erlang:function_exported(Mod, Fun, 0). - -run_test(Test, Outdir, TestMetaData, Report, HarnessArgs, NumTests) -> - rt_cover:maybe_start(Test), - SingleTestResult = riak_test_runner:confirm(Test, Outdir, TestMetaData, - HarnessArgs), - CoverDir = rt_config:get(cover_output, "coverage"), - case NumTests of - 1 -> keep_them_up; - _ -> rt:teardown() - end, - CoverageFile = rt_cover:maybe_export_coverage(Test, CoverDir, erlang:phash2(TestMetaData)), - case Report of - undefined -> ok; - _ -> - {value, {log, L}, TestResult} = lists:keytake(log, 1, SingleTestResult), - case giddyup:post_result(TestResult) of - error -> woops; - {ok, Base} -> - %% Now push up the artifacts, starting with the test log - giddyup:post_artifact(Base, {"riak_test.log", L}), - [ giddyup:post_artifact(Base, File) || File <- rt:get_node_logs() ], - [giddyup:post_artifact(Base, {filename:basename(CoverageFile) ++ ".gz", - zlib:gzip(element(2,file:read_file(CoverageFile)))}) || CoverageFile /= cover_disabled ], - ResultPlusGiddyUp = TestResult ++ [{giddyup_url, list_to_binary(Base)}], - [ rt:post_result(ResultPlusGiddyUp, WebHook) || WebHook <- get_webhooks() ] - end - end, - rt_cover:stop(), - [{coverdata, CoverageFile} | SingleTestResult]. - -get_webhooks() -> - Hooks = lists:foldl(fun(E, Acc) -> [parse_webhook(E) | Acc] end, - [], - rt_config:get(webhooks, [])), - lists:filter(fun(E) -> E =/= undefined end, Hooks). - -parse_webhook(Props) -> - Url = proplists:get_value(url, Props), - case is_list(Url) of - true -> - #rt_webhook{url= Url, - name=proplists:get_value(name, Props, "Webhook"), - headers=proplists:get_value(headers, Props, [])}; - false -> - lager:error("Invalid configuration for webhook : ~p", Props), - undefined - end. - -print_summary(TestResults, CoverResult, Verbose) -> - io:format("~nTest Results:~n"), - - Results = [ - [ atom_to_list(proplists:get_value(test, SingleTestResult)) ++ "-" ++ - backend_list(proplists:get_value(backend, SingleTestResult)), - proplists:get_value(status, SingleTestResult), - proplists:get_value(reason, SingleTestResult)] - || SingleTestResult <- TestResults], - Width = test_name_width(Results), - - Print = fun(Test, Status, Reason) -> - case {Status, Verbose} of - {fail, true} -> io:format("~s: ~s ~p~n", [string:left(Test, Width), Status, Reason]); - _ -> io:format("~s: ~s~n", [string:left(Test, Width), Status]) - end - end, - [ Print(Test, Status, Reason) || [Test, Status, Reason] <- Results], - - PassCount = length(lists:filter(fun(X) -> proplists:get_value(status, X) =:= pass end, TestResults)), - FailCount = length(lists:filter(fun(X) -> proplists:get_value(status, X) =:= fail end, TestResults)), - io:format("---------------------------------------------~n"), - io:format("~w Tests Failed~n", [FailCount]), - io:format("~w Tests Passed~n", [PassCount]), - Percentage = case PassCount == 0 andalso FailCount == 0 of - true -> 0; - false -> (PassCount / (PassCount + FailCount)) * 100 - end, - io:format("That's ~w% for those keeping score~n", [Percentage]), +get_group_tests(Tests, Groups) -> + lists:filter(fun(Test) -> + Mod = list_to_atom(Test), + Attrs = Mod:module_info(attributes), + match_group_attributes(Attrs, Groups) + end, Tests). - case CoverResult of - cover_disabled -> - ok; - {Coverage, AppCov} -> - io:format("Coverage : ~.1f%~n", [Coverage]), - [io:format(" ~s : ~.1f%~n", [App, Cov]) - || {App, Cov, _} <- AppCov] - end, - ok. - -test_name_width(Results) -> - lists:max([ length(X) || [X | _T] <- Results ]). - -backend_list(Backend) when is_atom(Backend) -> - atom_to_list(Backend); -backend_list(Backends) when is_list(Backends) -> - FoldFun = fun(X, []) -> - atom_to_list(X); - (X, Acc) -> - Acc ++ "," ++ atom_to_list(X) - end, - lists:foldl(FoldFun, [], Backends). - -results_filter(Result) -> - case proplists:get_value(status, Result) of - not_a_runnable_test -> +match_group_attributes(Attributes, Groups) -> + case proplists:get_value(test_type, Attributes) of + undefined -> false; - _ -> - true + TestTypes -> + lists:member(true, + [ TestType == list_to_atom(Group) + || Group <- Groups, TestType <- TestTypes ]) end. -load_tests_in_dir(Dir, SkipTests) -> +load_tests_in_dir(Dir, Groups, SkipTests) -> case filelib:is_dir(Dir) of true -> - code:add_path(Dir), + lists:sort( - lists:foldl(load_tests_folder(SkipTests), + lists:foldl(load_tests_folder(Groups, SkipTests), [], filelib:wildcard("*.beam", Dir))); - _ -> io:format("~s is not a dir!~n", [Dir]) + _ -> + io:format("~s is not a dir!~n", [Dir]) end. -load_tests_folder(SkipTests) -> +load_tests_folder([], SkipTests) -> fun(X, Acc) -> + %% Drop the .beam suffix Test = string:substr(X, 1, length(X) - 5), case lists:member(Test, SkipTests) of true -> Acc; false -> - [Test | Acc] + [list_to_atom(Test) | Acc] + end + end; +load_tests_folder(Groups, SkipTests) -> + fun(X, Acc) -> + %% Drop the .beam suffix + Test = string:substr(X, 1, length(X) - 5), + case group_match(Test, Groups) + andalso not lists:member(Test, SkipTests) of + true -> + [list_to_atom(Test) | Acc]; + false -> + Acc end end. +-spec group_match(string(), [string()]) -> boolean(). +group_match(Test, Groups) -> + Mod = list_to_atom(Test), + Attrs = Mod:module_info(attributes), + match_group_attributes(Attrs, Groups). + so_kill_riak_maybe() -> io:format("~n~nSo, we find ourselves in a tricky situation here. ~n"), io:format("You've run a single test, and it has failed.~n"), io:format("Would you like to leave Riak running in order to debug?~n"), Input = io:get_chars("[Y/n] ", 1), case Input of - "n" -> rt:teardown(); - "N" -> rt:teardown(); + "n" -> + rt_cluster:teardown(); + "N" -> + rt_cluster:teardown(); _ -> io:format("Leaving Riak Up... "), rt:whats_up() end. + +%% @doc Start the GiddyUp reporting service if the report is defined +start_giddyup(undefined) -> + ok; +start_giddyup(Platform) -> + {ok, _Pid} = giddyup:start_link(Platform, + rt_config:get_default_version_product(), + rt_config:get_default_version_number(), + rt_config:get_default_version(), + rt_config:get(giddyup_host), + rt_config:get(giddyup_user), + rt_config:get(giddyup_password)). + +%% @doc Stop the GiddyUp reporting service if the report is defined +stop_giddyup(true) -> + giddyup:stop(); +stop_giddyup(_) -> + ok. + +-ifdef(TEST). +%% Make sure that bitcask is the default backend +default_backend_test() -> + ?assertEqual([bitcask], determine_backends(undefined, false)). + +%% Make sure that GiddyUp supports all backends +default_giddyup_backend_test() -> + ?assertEqual([bitcask, eleveldb, memory, multi], lists:sort(determine_backends(undefined, true))). + +%% Command-line backends should always rule +cmdline_backend_test() -> + ?assertEqual([memory], determine_backends([memory], false)), + ?assertEqual([memory], determine_backends([memory], true)), + ?assertEqual([eleveldb, memory], lists:sort(determine_backends([memory, eleveldb], false))), + ?assertEqual([eleveldb, memory], lists:sort(determine_backends([memory, eleveldb], true))). +-endif. diff --git a/src/riak_test_executor.erl b/src/riak_test_executor.erl new file mode 100644 index 000000000..51e0484bf --- /dev/null +++ b/src/riak_test_executor.erl @@ -0,0 +1,331 @@ +-module(riak_test_executor). + +-behavior(gen_fsm). + +%% API +-export([start_link/5, + send_event/1, + stop/0]). + +%% gen_fsm callbacks +-export([init/1, + gather_properties/2, + gather_properties/3, + request_nodes/2, + request_nodes/3, + launch_test/2, + launch_test/3, + wait_for_completion/2, + wait_for_completion/3, + handle_event/3, + handle_sync_event/4, + handle_info/3, + terminate/3, + code_change/4]). + +-type execution_mode() :: serial | parallel. +-record(state, {pending_tests :: [rt_test_plan:test_plan()], + running_tests=[] :: [rt_test_plan:test_plan()], + waiting_tests=[] :: [rt_test_plan:test_plan()], + upgrade_list :: [string()], + test_properties :: [proplists:proplist()], + runner_pids=[] :: [pid()], + log_dir :: string(), + execution_mode :: execution_mode(), + continue_on_fail :: boolean(), + reporter_pid :: pid()}). + +%%%=================================================================== +%%% API +%%%=================================================================== + +%% @doc Start the test executor +-spec start_link([rt_test_plan:test_plan()], string(), string()|giddyup, [string()], pid()) -> {ok, pid()} | ignore | {error, term()}. +start_link(Tests, LogDir, Platform, UpgradeList, NotifyPid) -> + Args = [Tests, LogDir, Platform, UpgradeList, NotifyPid], + gen_fsm:start_link({local, ?MODULE}, ?MODULE, Args, []). + +send_event(Msg) -> + gen_fsm:send_event(?MODULE, Msg). + +%% @doc Stop the executor +-spec stop() -> ok | {error, term()}. +stop() -> + gen_fsm:sync_send_all_state_event(?MODULE, stop, infinity). + +%%%=================================================================== +%%% gen_fsm callbacks +%%%=================================================================== + +init([Tests, LogDir, Platform, UpgradeList, NotifyPid]) -> + %% TODO Change the default when parallel execution support is implemented -jsb + ExecutionMode = rt_config:get(rt_execution_mode, serial), + + ContinueOnFail = rt_config:get(continue_on_fail), + + UploadToGiddyUp = case Platform of + undefined -> false; + _ -> true + end, + {ok, Reporter} = rt_reporter:start_link(UploadToGiddyUp, LogDir, NotifyPid), + + lager:notice("Starting the Riak Test executor in ~p execution mode", [ExecutionMode]), + State = #state{pending_tests=Tests, + log_dir=LogDir, + upgrade_list=UpgradeList, + execution_mode=ExecutionMode, + continue_on_fail=ContinueOnFail, + reporter_pid=Reporter}, + {ok, gather_properties, State, 0}. + +%% @doc there are no all-state events for this fsm +handle_event(_Event, StateName, State) -> + {next_state, StateName, State}. + +%% @doc Handle synchronous events that should be handled +%% the same regardless of the current state. +-spec handle_sync_event(term(), term(), atom(), #state{}) -> + {reply, term(), atom(), #state{}}. +handle_sync_event(_Event, _From, _StateName, _State) -> + {reply, ok, ok, _State}. + +handle_info(_Info, StateName, State) -> + {next_state, StateName, State}. + +terminate(normal, _StateName, _State) -> + rt_reporter:send_result(done), + rt_reporter:stop(), + ok; +terminate(_Reason, _StateName, _State) -> + ok. + +%% @doc this fsm has no special upgrade process +code_change(_OldVsn, StateName, State, _Extra) -> + {ok, StateName, State}. + +%%% Asynchronous call handling functions for each FSM state + +%% TODO: Modify property gathering to account for `upgrade_path' +%% specified via the command line and replace accordingly in +%% properties record. +gather_properties(timeout, State) -> + OverrideProps = override_props(State), +Properties = test_properties(State#state.pending_tests, OverrideProps), + {next_state, request_nodes, State#state{test_properties=Properties}, 0}; +gather_properties(_Event, _State) -> + {next_state, gather_properties, _State}. + +request_nodes(timeout, State) -> + #state{pending_tests=[NextTest | _], + test_properties=PropertiesList} = State, + %% Find the properties for the next pending test + {NextTest, TestProps} = lists:keyfind(NextTest, 1, PropertiesList), + + ok = maybe_reserve_nodes(NextTest, TestProps), + + {next_state, launch_test, State}; +request_nodes({test_complete, Test, Pid, _Results}, State) -> + #state{pending_tests=Pending, + waiting_tests=Waiting, + running_tests=Running, + runner_pids=Pids, + execution_mode=ExecutionMode}= State, + UpdState = State#state{running_tests=lists:delete(Test, Running), + runner_pids=lists:delete(Pid, Pids), + pending_tests=Pending++Waiting, + waiting_tests=[], + execution_mode=ExecutionMode}, + {next_state, request_nodes, UpdState}; +request_nodes(_Event, _State) -> + {next_state, request_nodes, _State}. + +launch_test(insufficient_versions_available, State) -> + lager:debug("riak_test_executor:launch_test insufficient_versions_available"), + #state{pending_tests=[HeadPending | RestPending], + execution_mode=ExecutionMode} = State, + rt_reporter:send_result({test_result, {HeadPending, {skipped, insufficient_versions}, 0}}), + UpdState = State#state{pending_tests=RestPending, + execution_mode=ExecutionMode}, + launch_test_transition(UpdState); +launch_test(not_enough_nodes, State) -> + %% Move head of pending to waiting and try next test if there is + %% one left in pending. + lager:debug("riak_test_executor:launch_test not_enough_nodes"), + #state{pending_tests=[HeadPending | RestPending], + waiting_tests=Waiting, + execution_mode=ExecutionMode} = State, + rt_reporter:send_result({test_result, {HeadPending, {skipped, not_enough_nodes}, 0}}), + UpdState = State#state{pending_tests=RestPending, + waiting_tests=[HeadPending | Waiting], + execution_mode=ExecutionMode}, + launch_test_transition(UpdState); +launch_test({nodes, Nodes, NodeMap}, State) -> + %% Spawn a test runner for the head of pending. If pending is now + %% empty transition to `wait_for_completion'; otherwise, + %% transition to `request_nodes'. + #state{pending_tests=[NextTestPlan | RestPending], + execution_mode=ExecutionMode, + test_properties=PropertiesList, + runner_pids=Pids, + running_tests=Running, + continue_on_fail=ContinueOnFail, + reporter_pid=ReporterPid, + log_dir=LogDir} = State, + NextTestModule = rt_test_plan:get_module(NextTestPlan), + lager:debug("Executing test ~p in mode ~p", [NextTestModule, ExecutionMode]), + {NextTestPlan, TestProps} = lists:keyfind(NextTestPlan, 1, PropertiesList), + UpdTestProps = rt_properties:set([{node_map, NodeMap}, {node_ids, Nodes}], + TestProps), + {RunnerPids, RunningTests} = run_test(ExecutionMode, NextTestPlan, UpdTestProps, + Pids, Running, ContinueOnFail, ReporterPid, LogDir), + UpdState = State#state{pending_tests=RestPending, + execution_mode=ExecutionMode, + runner_pids=RunnerPids, + running_tests=RunningTests}, + + launch_test_transition(UpdState); +launch_test({test_complete, TestPlan, Pid, _Results}, State) -> + #state{pending_tests=Pending, + waiting_tests=Waiting, + running_tests=Running, + runner_pids=Pids, + execution_mode=ExecutionMode} = State, + UpdState = State#state{running_tests=lists:delete(TestPlan, Running), + runner_pids=lists:delete(Pid, Pids), + pending_tests=Pending++Waiting, + waiting_tests=[], + execution_mode=ExecutionMode}, + {next_state, launch_test, UpdState}; +launch_test(Event, State) -> + lager:error("Unknown event ~p with state ~p.", [Event, State]), + ok. + +maybe_reserve_nodes(NextTestPlan, TestProps) -> + %% TODO: Clean up upgrade resolution. Go either with executor or test plan. + %% VersionsToTest = versions_to_test(TestProps), + VersionsToTest = [rt_config:convert_to_string(rt_test_plan:get(version, NextTestPlan))], + maybe_reserve_nodes(erlang:function_exported(rt_test_plan:get_module(NextTestPlan), confirm, 1), + NextTestPlan, VersionsToTest, TestProps). + +maybe_reserve_nodes(true, NextTest, VersionsToTest, TestProps) -> + NodeCount = rt_properties:get(node_count, TestProps), + + %% Send async request to node manager + lager:notice("Requesting ~p nodes for the next test, ~p", [NodeCount, rt_test_plan:get_name(NextTest)]), + node_manager:reserve_nodes(NodeCount, + VersionsToTest, + reservation_notify_fun()); +maybe_reserve_nodes(false, NextTest, VersionsToTest, _TestProps) -> + lager:warning("~p is an old style test that requires conversion.", [rt_test_plan:get_name(NextTest)]), + node_manager:reserve_nodes(0, VersionsToTest, reservation_notify_fun()), + ok. + +wait_for_completion({test_complete, Test, Pid, Results}, State) -> + lager:debug("Test ~p complete", [rt_test_plan:get_module(Test)]), + #state{pending_tests=Pending, + waiting_tests=Waiting, + running_tests=Running, + runner_pids=Pids, + execution_mode=ExecutionMode} = State, + UpdState = State#state{running_tests=lists:delete(Test, Running), + runner_pids=lists:delete(Pid, Pids), + pending_tests=Pending++Waiting, + waiting_tests=[], + execution_mode=ExecutionMode}, + wait_for_completion_transition(Results, UpdState); +wait_for_completion(_Event, _State) -> + ok. + +%% Synchronous call handling functions for each FSM state + +gather_properties(_Event, _From, _State) -> + ok. + +request_nodes(_Event, _From, _State) -> + ok. + +launch_test(_Event, _From, _State) -> + ok. + +wait_for_completion(_Event, _From, _State) -> + ok. + +%%%=================================================================== +%%% Internal functions +%%%=================================================================== + +wait_for_completion_transition({_Status, _Reason}, State=#state{continue_on_fail=ContinueOnFail}) when ContinueOnFail == false -> + {stop, normal, State}; +wait_for_completion_transition(_Result, State=#state{pending_tests=[], + running_tests=[]}) -> + {stop, normal, State}; +wait_for_completion_transition(_Result, State=#state{pending_tests=[]}) -> + {next_state, wait_for_completion, State}; +wait_for_completion_transition(_Result, State) -> + {next_state, request_nodes, State, 0}. + +launch_test_transition(State=#state{pending_tests=PendingTests, + execution_mode=ExecutionMode}) when PendingTests == [] orelse ExecutionMode == serial -> + PendingModules = [rt_test_plan:get_module(Test) || Test <- PendingTests], + lager:debug("Waiting for completion: execution mode ~p with pending tests ~p", [ExecutionMode, PendingModules]), + {next_state, wait_for_completion, State}; +launch_test_transition(State) -> + {next_state, request_nodes, State, 0}. + +%%launch_test_transition(State) -> +%% {next_state, wait_for_completion, State}. + +reservation_notify_fun() -> + fun(X) -> + ?MODULE:send_event(X) + end. + +test_properties(Tests, OverriddenProps) -> + lists:foldl(test_property_fun(OverriddenProps), [], Tests). + +test_property_fun(OverrideProps) -> + fun(TestPlan, Acc) -> + {PropsMod, PropsFun} = riak_test_runner:function_name(properties, + rt_test_plan:get_module(TestPlan), + 0, + rt_cluster), + Properties = rt_properties:set(OverrideProps, PropsMod:PropsFun()), + [{TestPlan, Properties} | Acc] + end. + +%% An `upgrade_path' specified on the command line overrides the test +%% property setting. If the `rolling_upgrade' property is is `false' +%% then the `start_version' property of the test is the only version +%% tested. +%% versions_to_test(Properties) -> +%% versions_to_test(Properties, rt_properties:get(rolling_upgrade, Properties)). +%% +%% versions_to_test(Properties, true) -> +%% case rt_properties:get(upgrade_path, Properties) of +%% undefined -> +%% versions_to_test(Properties, false); +%% UpgradePath -> +%% [rt_config:convert_to_string(Upgrade) || Upgrade <- UpgradePath] +%% end; +%% versions_to_test(Properties, false) -> +%% InitialVersion = rt_properties:get(start_version, Properties), +%% [rt_config:convert_to_string(InitialVersion)]. + +%% Function to abstract away the details of what properties +%% can be overridden on the command line. +override_props(State) -> + case State#state.upgrade_list of + undefined -> + []; + UpgradeList -> + [{upgrade_path, UpgradeList}] + end. + +-spec run_test(parallel | serial, atom(), proplists:proplist(), [pid()], [rt_test_plan:test_plan()], boolean(), pid(), string()) -> {[pid()], [atom()]}. +run_test(parallel, TestPlan, Properties, RunningPids, RunningTests, ContinueOnFail, ReporterPid, LogDir) -> + Pid = spawn_link(riak_test_runner, start, [TestPlan, Properties, ContinueOnFail, ReporterPid, LogDir]), + {[Pid | RunningPids], [TestPlan | RunningTests]}; +run_test(serial, TestPlan, Properties, RunningPids, RunningTests, ContinueOnFail, ReporterPid, LogDir) -> + riak_test_runner:start(TestPlan, Properties, ContinueOnFail, ReporterPid, LogDir), + {RunningPids, RunningTests}. + diff --git a/src/riak_test_group_leader.erl b/src/riak_test_group_leader.erl index 5a673ee4b..230c5e761 100644 --- a/src/riak_test_group_leader.erl +++ b/src/riak_test_group_leader.erl @@ -107,4 +107,4 @@ io_requests(_, Result) -> %% If we get multiple lines, we'll split them up for lager to maximize the prettiness. log_chars(Chars) -> - [lager:info("~s", [Line]) || Line <- string:tokens(lists:flatten(Chars), "\n")]. \ No newline at end of file + [lager:info("~s", [Line]) || Line <- string:tokens(lists:flatten(Chars), "\n")]. diff --git a/src/riak_test_lager_backend.erl b/src/riak_test_lager_backend.erl index b1644bef8..a925861b0 100644 --- a/src/riak_test_lager_backend.erl +++ b/src/riak_test_lager_backend.erl @@ -183,11 +183,9 @@ log_test_() -> lager:info("Here's a message"), lager:debug("Here's another message"), {ok, Logs} = gen_event:delete_handler(lager_event, riak_test_lager_backend, []), - ?assertEqual(3, length(Logs)), - - ?assertMatch([_, "[debug]", "Lager installed handler riak_test_lager_backend into lager_event"], re:split(lists:nth(1, Logs), " ", [{return, list}, {parts, 3}])), - ?assertMatch([_, "[info]", "Here's a message"], re:split(lists:nth(2, Logs), " ", [{return, list}, {parts, 3}])), - ?assertMatch([_, "[debug]", "Here's another message"], re:split(lists:nth(3, Logs), " ", [{return, list}, {parts, 3}])) + ?assertEqual(2, length(Logs)), + ?assertMatch([_, "[info]", "Here's a message\r\n"], re:split(lists:nth(1, Logs), " ", [{return, list}, {parts, 3}])), + ?assertMatch([_, "[debug]", "Here's another message\r\n"], re:split(lists:nth(2, Logs), " ", [{return, list}, {parts, 3}])) end } diff --git a/src/riak_test_runner.erl b/src/riak_test_runner.erl index eac78838b..8ba63a866 100644 --- a/src/riak_test_runner.erl +++ b/src/riak_test_runner.erl @@ -1,4 +1,4 @@ -%% ------------------------------------------------------------------- +% ------------------------------------------------------------------- %% %% Copyright (c) 2013 Basho Technologies, Inc. %% @@ -18,112 +18,434 @@ %% %% ------------------------------------------------------------------- -%% @doc riak_test_runner runs a riak_test module's run/0 function. +%% @doc riak_test_runner runs a riak_test module's `confirm/0' function. -module(riak_test_runner). --export([confirm/4, metadata/0, metadata/1, function_name/1]). -%% Need to export to use with `spawn_link'. --export([return_to_exit/3]). + +-behavior(gen_fsm). + +%% API +-export([start/5, + send_event/2, + stop/0]). + +-export([function_name/2, + function_name/4]). + +%% gen_fsm callbacks +-export([init/1, + setup/2, + setup/3, + execute/2, + execute/3, + wait_for_completion/2, + wait_for_completion/3, + wait_for_upgrade/2, + wait_for_upgrade/3, + handle_event/3, + handle_sync_event/4, + handle_info/3, + metadata/0, + terminate/3, + code_change/4]). + -include_lib("eunit/include/eunit.hrl"). +-type test_type() :: {new | old}. +-record(state, {test_plan :: rt_test_plan:test_plan(), + test_module :: atom(), + test_type :: test_type(), + properties :: proplists:proplist(), + backend :: atom(), + test_timeout :: integer(), + execution_pid :: pid(), + group_leader :: pid(), + start_time :: erlang:timestamp(), + end_time :: erlang:timestamp(), + setup_modfun :: {atom(), atom()}, + confirm_modfun :: {atom(), atom()}, + backend_check :: atom(), + prereq_check :: atom(), + current_version :: string(), + remaining_versions :: [string()], + test_results :: [term()], + continue_on_fail :: boolean(), + log_dir :: string(), + reporter_pids :: pid()}). + +-deprecated([{metadata,0,next_major_release}]). + +%%%=================================================================== +%%% API +%%%=================================================================== + +%% @doc Start the test runner +start(TestPlan, Properties, ContinueOnFail, ReporterPids, LogDir) -> + Args = [TestPlan, Properties, ContinueOnFail, ReporterPids, LogDir], + gen_fsm:start_link(?MODULE, Args, []). + +send_event(Pid, Msg) -> + gen_fsm:send_event(Pid, Msg). + +%% @doc Stop the executor +-spec stop() -> ok | {error, term()}. +stop() -> + gen_fsm:sync_send_all_state_event(?MODULE, stop, infinity). + -spec(metadata() -> [{atom(), term()}]). %% @doc fetches test metadata from spawned test process metadata() -> - riak_test ! metadata, - receive - {metadata, TestMeta} -> TestMeta - end. + FSMPid = get(test_runner_fsm), + gen_fsm:sync_send_all_state_event(FSMPid, metadata_event, infinity). -metadata(Pid) -> - riak_test ! {metadata, Pid}, - receive - {metadata, TestMeta} -> TestMeta - end. +%%%=================================================================== +%%% gen_fsm callbacks +%%%=================================================================== --spec(confirm(integer(), atom(), [{atom(), term()}], list()) -> [tuple()]). -%% @doc Runs a module's run/0 function after setting up a log capturing backend for lager. -%% It then cleans up that backend and returns the logs as part of the return proplist. -confirm(TestModule, Outdir, TestMetaData, HarnessArgs) -> - start_lager_backend(TestModule, Outdir), - rt:setup_harness(TestModule, HarnessArgs), - BackendExtras = case proplists:get_value(multi_config, TestMetaData) of - undefined -> []; - Value -> [{multi_config, Value}] - end, - Backend = rt:set_backend(proplists:get_value(backend, TestMetaData), BackendExtras), - {Mod, Fun} = function_name(TestModule), - {Status, Reason} = case check_prereqs(Mod) of - true -> - lager:notice("Running Test ~s", [TestModule]), - execute(TestModule, {Mod, Fun}, TestMetaData); - not_present -> - {fail, test_does_not_exist}; - _ -> - {fail, all_prereqs_not_present} +%% @doc Read the storage schedule and go to idle. +%% compose_test_datum(Version, Project, undefined, undefined) -> +init([TestPlan, Properties, ContinueOnFail, ReporterPid, LogDir]) -> + lager:debug("Started riak_test_runnner with pid ~p (continue on fail: ~p)", [self(), ContinueOnFail]), + Project = list_to_binary(rt_config:get(rt_project, "undefined")), + Backend = rt_test_plan:get(backend, TestPlan), + TestModule = rt_test_plan:get_module(TestPlan), + %% Populate metadata for backwards-compatiblity + MetaData0 = [{id, -1}, + {platform, <<"local">>}, + {version, rt:get_version()}, + {backend, Backend}, + {project, Project}], + %% Upgrade Version is legacy for 'previous' or 'legacy' to 'current' upgrade + %% For old tests, keep those labels. Converted tests could use an arbitrary name. + UpgradePath = rt_test_plan:get(upgrade_path, TestPlan), + MetaData = case UpgradePath of + undefined -> MetaData0; + _ -> MetaData0 ++ [{upgrade_version, UpgradePath}] end, - lager:notice("~s Test Run Complete", [TestModule]), - {ok, Logs} = stop_lager_backend(), - Log = unicode:characters_to_binary(Logs), + %% TODO: Remove after all tests ported 2.0 -- workaround to support + %% backend command line argument fo v1 cluster provisioning -jsb + rt_config:set(rt_backend, Backend), + lager:info("Using backend ~p", [Backend]), - RetList = [{test, TestModule}, {status, Status}, {log, Log}, {backend, Backend} | proplists:delete(backend, TestMetaData)], - case Status of - fail -> RetList ++ [{reason, iolist_to_binary(io_lib:format("~p", [Reason]))}]; - _ -> RetList - end. - -start_lager_backend(TestModule, Outdir) -> - case Outdir of - undefined -> ok; - _ -> - gen_event:add_handler(lager_event, lager_file_backend, - {Outdir ++ "/" ++ atom_to_list(TestModule) ++ ".dat_test_output", - rt_config:get(lager_level, info), 10485760, "$D0", 1}), - lager:set_loglevel(lager_file_backend, rt_config:get(lager_level, info)) + {ok, UpdProperties} = + rt_properties:set(metadata, MetaData, Properties), + TestTimeout = rt_config:get(test_timeout, rt_config:get(rt_max_receive_wait_time)), + SetupModFun = function_name(setup, TestModule, 1, rt_cluster), + {ConfirmMod, _} = ConfirmModFun = function_name(confirm, TestModule), + lager:debug("Confirm function -- ~p:~p", [ConfirmMod, ConfirmModFun]), + TestType = case erlang:function_exported(TestModule, confirm, 1) of + true -> new; + false -> old end, - gen_event:add_handler(lager_event, riak_test_lager_backend, [rt_config:get(lager_level, info), false]), - lager:set_loglevel(riak_test_lager_backend, rt_config:get(lager_level, info)). + BackendCheck = check_backend(Backend, + rt_properties:get(valid_backends, Properties)), + PreReqCheck = check_prereqs(ConfirmMod), + State = #state{test_plan=TestPlan, + test_module=TestModule, + test_type=TestType, + properties=UpdProperties, + backend=Backend, + test_timeout=TestTimeout, + setup_modfun=SetupModFun, + confirm_modfun=ConfirmModFun, + backend_check=BackendCheck, + prereq_check=PreReqCheck, + group_leader=group_leader(), + continue_on_fail=ContinueOnFail, + reporter_pids=ReporterPid, + log_dir=LogDir}, + {ok, setup, State, 0}. -stop_lager_backend() -> - gen_event:delete_handler(lager_event, lager_file_backend, []), - gen_event:delete_handler(lager_event, riak_test_lager_backend, []). +%% @doc there are no all-state events for this fsm +handle_event(_Event, StateName, State) -> + {next_state, StateName, State}. + +%% @doc Handle synchronous events that should be handled +%% the same regardless of the current state. +-spec handle_sync_event(term(), term(), atom(), #state{}) -> + {reply, term(), atom(), #state{}}. +handle_sync_event(metadata_event, _From, StateName, State) -> + Properties = State#state.properties, + MetaData = rt_properties:get(metadata, Properties), + {reply, MetaData, StateName, State}; +handle_sync_event(_Event, _From, StateName, _State) -> + {reply, ok, StateName, _State}. + +handle_info(_Info, StateName, State) -> + {next_state, StateName, State}. -%% does some group_leader swapping, in the style of EUnit. -execute(TestModule, {Mod, Fun}, TestMetaData) -> - process_flag(trap_exit, true), - OldGroupLeader = group_leader(), +terminate(_Reason, _StateName, _State) -> + ok. + +%% @doc this fsm has no special upgrade process +code_change(_OldVsn, StateName, State, _Extra) -> + {ok, StateName, State}. + +%% Asynchronous call handling functions for each FSM state + +setup(timeout, State=#state{backend_check=false}) -> + report_cleanup_and_notify({skipped, invalid_backend}, State), + {stop, normal, State}; +setup(timeout, State=#state{prereq_check=false}) -> + report_cleanup_and_notify({fail, prereq_check_failed}, State), + {stop, normal, State}; +setup(timeout, State=#state{test_type=TestType, + test_module=TestModule, + backend=Backend, + properties=Properties}) -> NewGroupLeader = riak_test_group_leader:new_group_leader(self()), group_leader(NewGroupLeader, self()), {0, UName} = rt:cmd("uname -a"), - lager:info("Test Runner `uname -a` : ~s", [UName]), + lager:info("Test Runner: ~s", [UName]), - Pid = spawn_link(?MODULE, return_to_exit, [Mod, Fun, []]), - Ref = case rt_config:get(test_timeout, undefined) of - Timeout when is_integer(Timeout) -> - erlang:send_after(Timeout, self(), test_took_too_long); - _ -> - undefined - end, + {StartVersion, OtherVersions} = test_versions(Properties), - {Status, Reason} = rec_loop(Pid, TestModule, TestMetaData), - case Ref of - undefined -> - ok; - _ -> - erlang:cancel_timer(Ref) - end, - riak_test_group_leader:tidy_up(OldGroupLeader), - case Status of - fail -> - ErrorHeader = "================ " ++ atom_to_list(TestModule) ++ " failure stack trace =====================", - ErrorFooter = [ $= || _X <- lists:seq(1,length(ErrorHeader))], - Error = io_lib:format("~n~s~n~p~n~s~n", [ErrorHeader, Reason, ErrorFooter]), - lager:error(Error); - _ -> meh + case TestType of + new -> + Config = rt_backend:set(Backend, rt_properties:get(config, Properties)), + NodeIds = rt_properties:get(node_ids, Properties), + Services = rt_properties:get(required_services, Properties); + old -> + Config = rt:set_backend(Backend), + NodeIds = [], + Services = [], + lager:warning("Test ~p has not been ported to the new framework.", [TestModule]) end, - {Status, Reason}. -function_name(TestModule) -> + node_manager:deploy_nodes(NodeIds, StartVersion, Config, Services, notify_fun(self())), + lager:info("Waiting for deploy nodes response at ~p", [self()]), + + %% Set the initial value for `current_version' in the properties record + {ok, UpdProperties} = + rt_properties:set(current_version, StartVersion, Properties), + + UpdState = State#state{current_version=StartVersion, + remaining_versions=OtherVersions, + properties=UpdProperties}, + {next_state, execute, UpdState}; +setup(_Event, _State) -> + ok. + +execute({nodes_deployed, _}, State) -> + #state{test_plan=TestPlan, + test_module=TestModule, + test_type=TestType, + properties=Properties, + setup_modfun=SetupModFun, + confirm_modfun=ConfirmModFun, + test_timeout=TestTimeout, + log_dir=OutDir} = State, + lager:notice("Running ~s", [TestModule]), + lager:notice("Properties: ~p", [Properties]), + + StartTime = os:timestamp(), + %% Perform test setup which includes clustering of the nodes if + %% required by the test properties. The cluster information is placed + %% into the properties record and returned by the `setup' function. + start_lager_backend(rt_test_plan:get_name(TestPlan), OutDir), + SetupResult = maybe_setup_test(TestModule, TestType, SetupModFun, Properties), + UpdState = maybe_execute_test(SetupResult, TestModule, TestType, ConfirmModFun, StartTime, State), + + {next_state, wait_for_completion, UpdState, TestTimeout}; +execute(_Event, _State) -> + {next_state, execute, _State}. + +maybe_setup_test(_TestModule, old, _SetupModFun, Properties) -> + {ok, Properties}; +maybe_setup_test(TestModule, new, {SetupMod, SetupFun}, Properties) -> + lager:debug("Setting up test ~p using ~p:~p", [TestModule, SetupMod, SetupFun]), + SetupMod:SetupFun(Properties). + +maybe_execute_test({ok, Properties}, _TestModule, TestType, ConfirmModFun, StartTime, State) -> + Pid = spawn_link(test_fun(TestType, + Properties, + ConfirmModFun, + self())), + State#state{execution_pid=Pid, + test_type=TestType, + properties=Properties, + start_time=StartTime}; +maybe_execute_test(Error, TestModule, TestType, _ConfirmModFun, StartTime, State) -> + lager:error("Setup of test ~p failed due to ~p", [TestModule, Error]), + ?MODULE:send_event(self(), test_result({fail, test_setup_failed})), + State#state{test_type=TestType, + start_time=StartTime}. + + +wait_for_completion(timeout, State=#state{test_module=TestModule, + test_type=TestType, + group_leader=GroupLeader}) -> + %% Test timed out + UpdState = State#state{test_module=TestModule, + test_type=TestType, + group_leader=GroupLeader, + end_time=os:timestamp()}, + report_cleanup_and_notify(timeout, UpdState), + {stop, normal, UpdState}; +wait_for_completion({test_result, {fail, Reason}}, State=#state{test_module=TestModule, + test_type=TestType, + group_leader=GroupLeader, + continue_on_fail=ContinueOnFail, + remaining_versions=[]}) -> + Result = {fail, Reason}, + lager:debug("Test Result ~p = {fail, ~p}", [TestModule, Reason]), + UpdState = State#state{test_module=TestModule, + test_type=TestType, + test_results=Result, + group_leader=GroupLeader, + continue_on_fail=ContinueOnFail, + end_time=os:timestamp()}, + lager:debug("ContinueOnFail: ~p", [ContinueOnFail]), + report_cleanup_and_notify(Result, ContinueOnFail, UpdState), + {stop, normal, UpdState}; +wait_for_completion({test_result, Result}, State=#state{test_module=TestModule, + test_type=TestType, + group_leader=GroupLeader, + remaining_versions=[]}) -> + lager:debug("Test Result ~p = {~p}", [TestModule, Result]), + %% TODO: Format results for aggregate test runs if needed. For + %% upgrade tests with failure return which versions had failure + %% along with reasons. + UpdState = State#state{test_module=TestModule, + test_type=TestType, + group_leader=GroupLeader, + end_time=os:timestamp()}, + report_cleanup_and_notify(Result, UpdState), + {stop, normal, UpdState}; +wait_for_completion({test_result, Result}, State) -> + #state{backend=Backend, + test_module=TestModule, + test_type=TestType, + test_results=TestResults, + current_version=CurrentVersion, + remaining_versions=[NextVersion | RestVersions], + properties=Properties} = State, + lager:debug("Test Result ~p = {~p}", [TestModule, Result]), + Config = rt_backend:set(Backend, rt_properties:get(config, Properties)), + NodeIds = rt_properties:get(node_ids, Properties), + node_manager:upgrade_nodes(NodeIds, + CurrentVersion, + NextVersion, + Config, + notify_fun(self())), + UpdState = State#state{test_results=[Result | TestResults], + test_module=TestModule, + test_type=TestType, + current_version=NextVersion, + remaining_versions=RestVersions}, + {next_state, wait_for_upgrade, UpdState}; +wait_for_completion(_Msg, _State) -> + {next_state, wait_for_completion, _State}. + +wait_for_upgrade(nodes_upgraded, State) -> + #state{properties=Properties, + test_type=TestType, + confirm_modfun=ConfirmModFun, + current_version=CurrentVersion, + test_timeout=TestTimeout} = State, + + %% Update the `current_version' in the properties record + {ok, UpdProperties} = + rt_properties:set(current_version, CurrentVersion, Properties), + + %% TODO: Maybe wait for transfers. Probably should be + %% a call to an exported function in `rt_cluster' + Pid = spawn_link(test_fun(TestType, + UpdProperties, + ConfirmModFun, + self())), + UpdState = State#state{execution_pid=Pid, + test_type=TestType, + properties=UpdProperties}, + {next_state, wait_for_completion, UpdState, TestTimeout}; +wait_for_upgrade(_Event, _State) -> + {next_state, wait_for_upgrade, _State}. + +%% Synchronous call handling functions for each FSM state + +setup(_Event, _From, _State) -> + ok. + +execute(_Event, _From, _State) -> + ok. + +wait_for_completion(_Event, _From, _State) -> + ok. + +wait_for_upgrade(_Event, _From, _State) -> + ok. + +%%%=================================================================== +%%% Internal functions +%%%=================================================================== + +-spec test_fun(test_type(), rt_properties:properties(), {atom(), atom()}, + pid()) -> function(). +test_fun(TestType, Properties, {ConfirmMod, ConfirmFun}, NotifyPid) -> + test_fun(TestType, Properties, ConfirmMod, ConfirmFun, NotifyPid). + +-spec test_fun(test_type(), rt_properties:properties(), atom(), atom(), + pid()) -> function(). +test_fun(new, Properties, ConfirmMod, ConfirmFun, NotifyPid) -> + test_fun(fun() -> ConfirmMod:ConfirmFun(Properties) end, NotifyPid); +test_fun(old, _Properties, ConfirmMod, ConfirmFun, NotifyPid) -> + lager:debug("Building test fun for ~p:~p/0 (defined: ~p)", [ConfirmMod, ConfirmFun, erlang:function_exported(ConfirmMod, ConfirmFun, 0)]), + test_fun(fun() -> ConfirmMod:ConfirmFun() end, NotifyPid). + +-spec test_fun(function(), pid()) -> function(). +test_fun(ConfirmFun, NotifyPid) -> + fun() -> + %% Store the FSM Pid for use in unported tests + put(test_runner_fsm, NotifyPid), + %% Exceptions and their handling sucks, but eunit throws + %% errors `erlang:error' so here we are + try ConfirmFun() of + TestResult -> + ?MODULE:send_event(NotifyPid, test_result(TestResult)) + catch + Error:Reason -> + lager:error("Failed to execute confirm function ~p due to ~p with reason ~p (trace: ~p)", + [ConfirmFun, Error, Reason, erlang:get_stacktrace()]), + TestResult = format_eunit_error(Reason), + ?MODULE:send_event(NotifyPid, test_result(TestResult)) + end + end. + +format_eunit_error({assertion_failed, InfoList}) -> + LineNum = proplists:get_value(line, InfoList), + Expression = proplists:get_value(expression, InfoList), + Value = proplists:get_value(value, InfoList), + ErrorStr = io_lib:format("Assertion ~s is ~p at line ~B", + [Expression, Value, LineNum]), + {fail, ErrorStr}; +format_eunit_error({assertCmd_failed, InfoList}) -> + LineNum = proplists:get_value(line, InfoList), + Command = proplists:get_value(command, InfoList), + Status = proplists:get_value(status, InfoList), + ErrorStr = io_lib:format("Command \"~s\" returned a status of ~B at line ~B", + [Command, Status, LineNum]), + {fail, ErrorStr}; +format_eunit_error({assertMatch_failed, InfoList}) -> + LineNum = proplists:get_value(line, InfoList), + Pattern = proplists:get_value(pattern, InfoList), + Value = proplists:get_value(value, InfoList), + ErrorStr = io_lib:format("Pattern ~s did not match value ~p at line ~B", + [Pattern, Value, LineNum]), + {fail, ErrorStr}; +format_eunit_error({assertEqual_failed, InfoList}) -> + LineNum = proplists:get_value(line, InfoList), + Expression = proplists:get_value(expression, InfoList), + Expected = proplists:get_value(expected, InfoList), + Value = proplists:get_value(value, InfoList), + ErrorStr = io_lib:format("~s = ~p is not equal to expected value ~p at line ~B", + [Expression, Value, Expected, LineNum]), + {fail, ErrorStr}; +format_eunit_error(Other) -> + ErrorStr = io_lib:format("Unknown error encountered: ~p", [Other]), + {fail, ErrorStr}. + +function_name(confirm, TestModule) -> TMString = atom_to_list(TestModule), Tokz = string:tokens(TMString, ":"), case length(Tokz) of @@ -133,46 +455,144 @@ function_name(TestModule) -> {list_to_atom(Module), list_to_atom(Function)} end. -rec_loop(Pid, TestModule, TestMetaData) -> - receive - test_took_too_long -> - exit(Pid, kill), - {fail, test_timed_out}; - metadata -> - Pid ! {metadata, TestMetaData}, - rec_loop(Pid, TestModule, TestMetaData); - {metadata, P} -> - P ! {metadata, TestMetaData}, - rec_loop(Pid, TestModule, TestMetaData); - {'EXIT', Pid, normal} -> {pass, undefined}; - {'EXIT', Pid, Error} -> - lager:warning("~s failed: ~p", [TestModule, Error]), - {fail, Error} +function_name(FunName, TestModule, Arity, Default) when is_atom(TestModule) -> + case erlang:function_exported(TestModule, FunName, Arity) of + true -> + {TestModule, FunName}; + false -> + {Default, FunName} end. +start_lager_backend(TestName, Outdir) -> + LogLevel = rt_config:get(lager_level, info), + case Outdir of + undefined -> ok; + _ -> + gen_event:add_handler(lager_event, lager_file_backend, + {filename:join([Outdir, TestName, "riak_test.log"]), + LogLevel, 10485760, "$D0", 1}), + lager:set_loglevel(lager_file_backend, LogLevel) + end, + gen_event:add_handler(lager_event, riak_test_lager_backend, [LogLevel, false]), + lager:set_loglevel(riak_test_lager_backend, LogLevel). + +stop_lager_backend() -> + gen_event:delete_handler(lager_event, lager_file_backend, []), + gen_event:delete_handler(lager_event, riak_test_lager_backend, []). + %% A return of `fail' must be converted to a non normal exit since %% status is determined by `rec_loop'. %% %% @see rec_loop/3 --spec return_to_exit(module(), atom(), list()) -> ok. -return_to_exit(Mod, Fun, Args) -> - case apply(Mod, Fun, Args) of - pass -> - %% same as exit(normal) - ok; - fail -> - exit(fail) - end. +%% -spec return_to_exit(module(), atom(), list()) -> ok. +%% return_to_exit(Mod, Fun, Args) -> +%% case apply(Mod, Fun, Args) of +%% pass -> +%% %% same as exit(normal) +%% ok; +%% fail -> +%% exit(fail) +%% end. + +-spec check_backend(atom(), all | [atom()]) -> boolean(). +check_backend(_Backend, all) -> + true; +check_backend(Backend, ValidBackends) -> + lists:member(Backend, ValidBackends). +%% Check the prequisites for executing the test check_prereqs(Module) -> - try Module:module_info(attributes) of - Attrs -> - Prereqs = proplists:get_all_values(prereq, Attrs), - P2 = [ {Prereq, rt_local:which(Prereq)} || Prereq <- Prereqs], - lager:info("~s prereqs: ~p", [Module, P2]), - [ lager:warning("~s prereq '~s' not installed.", [Module, P]) || {P, false} <- P2], - lists:all(fun({_, Present}) -> Present end, P2) - catch - _DontCare:_Really -> - not_present + Attrs = Module:module_info(attributes), + Prereqs = proplists:get_all_values(prereq, Attrs), + P2 = [{Prereq, rt_local:which(Prereq)} || Prereq <- Prereqs], + lager:info("~s prereqs: ~p", [Module, P2]), + [lager:warning("~s prereq '~s' not installed.", + [Module, P]) || {P, false} <- P2], + lists:all(fun({_, Present}) -> Present end, P2). + +notify_fun(Pid) -> + fun(X) -> + ?MODULE:send_event(Pid, X) end. + +%% @doc Send the results report, cleanup the nodes and +%% Notify the executor that we are done with the test run +%% @end +report_cleanup_and_notify(Result, State) -> + report_cleanup_and_notify(Result, true, State). + +%% @doc Send the results report, cleanup the nodes (optionally) and +%% Notify the executor that we are done with the test run +%% @end +-spec(report_cleanup_and_notify(tuple(), boolean(), term()) -> ok). +report_cleanup_and_notify(Result, CleanUp, State=#state{test_plan=TestPlan, + start_time=Start, + end_time=End}) -> + Duration = now_diff(End, Start), + ResultMessage = test_result_message(Result), + rt_reporter:send_result(test_result({TestPlan, ResultMessage, Duration})), + maybe_cleanup(CleanUp, State), + {ok, Logs} = stop_lager_backend(), + _Log = unicode:characters_to_binary(Logs), + Notification = {test_complete, TestPlan, self(), ResultMessage}, + riak_test_executor:send_event(Notification). + +maybe_cleanup(true, State) -> + cleanup(State); +maybe_cleanup(false, _State) -> + ok. + +cleanup(#state{group_leader=OldGroupLeader, + test_module=TestModule, + test_type=TestType}) when TestType == old -> + lager:debug("Cleaning up old style test ~p", [TestModule]), + %% Reset the state of the nodes ... + rt_harness:setup(), + + %% Reset the global variables + rt_config:set(rt_nodes, []), + rt_config:set(rt_nodenames, []), + rt_config:set(rt_versions, []), + + riak_test_group_leader:tidy_up(OldGroupLeader); +cleanup(#state{test_module=TestModule, + group_leader=OldGroupLeader, + properties=Properties}) -> + lager:debug("Cleaning up new style test ~p", [TestModule]), + node_manager:return_nodes(rt_properties:get(node_ids, Properties)), + riak_test_group_leader:tidy_up(OldGroupLeader). + +% @doc Convert test result into report message +test_result_message(timeout) -> + {fail, timeout}; +test_result_message(fail) -> + {fail, unknown}; +test_result_message(pass) -> + pass; +test_result_message(FailResult) -> + FailResult. + +test_versions(Properties) -> + StartVersion = rt_properties:get(start_version, Properties), + UpgradePath = rt_properties:get(upgrade_path, Properties), + case UpgradePath of + undefined -> + {StartVersion, []}; + [] -> + {StartVersion, []}; + _ -> + [UpgradeHead | Rest] = UpgradePath, + {UpgradeHead, Rest} + end. + +now_diff(undefined, _) -> + 0; +now_diff(_, undefined) -> + 0; +now_diff(End, Start) -> + timer:now_diff(End, Start). + +%% Simple function to hide the details of the message wrapping +test_result(Result) -> + {test_result, Result}. + diff --git a/src/rt.erl b/src/rt.erl index 8b4db1490..ef395aadd 100644 --- a/src/rt.erl +++ b/src/rt.erl @@ -1,6 +1,6 @@ %% ------------------------------------------------------------------- %% -%% Copyright (c) 2013-2014 Basho Technologies, Inc. +%% Copyright (c) 2013-2015 Basho Technologies, Inc. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -24,7 +24,7 @@ %% Please extend this module with new functions that prove useful between %% multiple independent tests. -module(rt). --include("rt.hrl"). +-deprecated(module). -include_lib("eunit/include/eunit.hrl"). -compile(export_all). @@ -64,9 +64,11 @@ get_deps/0, get_ip/1, get_node_logs/0, + get_node_logs/2, get_replica/5, get_ring/1, get_version/0, + get_version/1, heal/1, http_url/1, https_url/1, @@ -97,15 +99,12 @@ pbc_put_file/4, pbc_really_deleted/3, pmap/2, - post_result/2, product/1, priv_dir/0, - random_sublist/2, remove/2, riak/2, riak_repl/2, rpc_get_env/2, - select_random/1, set_backend/1, set_backend/2, set_conf/2, @@ -171,131 +170,62 @@ -define(HARNESS, (rt_config:get(rt_harness))). priv_dir() -> - LocalPrivDir = "./priv", - %% XXX for some reason, codew:priv_dir returns riak_test/riak_test/priv, - %% which is wrong, so fix it. - DepPrivDir = re:replace(code:priv_dir(riak_test), "riak_test(/riak_test)*", - "riak_test", [{return, list}]), - PrivDir = case {filelib:is_dir(LocalPrivDir), filelib:is_dir(DepPrivDir)} of - {true, _} -> - lager:debug("Local ./priv detected, using that..."), - %% we want an absolute path! - filename:absname(LocalPrivDir); - {false, true} -> - lager:debug("riak_test dependency priv_dir detected, using that..."), - DepPrivDir; - _ -> - ?assertEqual({true, bad_priv_dir}, {false, bad_priv_dir}) - end, - - lager:info("priv dir: ~p -> ~p", [code:priv_dir(riak_test), PrivDir]), - ?assert(filelib:is_dir(PrivDir)), - PrivDir. + rt2:priv_dir(). %% @doc gets riak deps from the appropriate harness -spec get_deps() -> list(). -get_deps() -> ?HARNESS:get_deps(). +get_deps() -> + rt2:get_deps(). %% @doc if String contains Substr, return true. -spec str(string(), string()) -> boolean(). str(String, Substr) -> - case string:str(String, Substr) of - 0 -> false; - _ -> true - end. + rt2:str(String, Substr). -spec set_conf(atom(), [{string(), string()}]) -> ok. set_conf(all, NameValuePairs) -> - ?HARNESS:set_conf(all, NameValuePairs); + rt_harness:set_conf(all, NameValuePairs); set_conf(Node, NameValuePairs) -> - stop(Node), - ?assertEqual(ok, rt:wait_until_unpingable(Node)), - ?HARNESS:set_conf(Node, NameValuePairs), - start(Node). + rt_config:set_conf(Node, NameValuePairs). -spec set_advanced_conf(atom(), [{string(), string()}]) -> ok. set_advanced_conf(all, NameValuePairs) -> - ?HARNESS:set_advanced_conf(all, NameValuePairs); + rt_config:set_advanced_conf(all, NameValuePairs); set_advanced_conf(Node, NameValuePairs) -> - stop(Node), - ?assertEqual(ok, rt:wait_until_unpingable(Node)), - ?HARNESS:set_advanced_conf(Node, NameValuePairs), - start(Node). + rt_config:set_advanced_conf(Node, NameValuePairs). %% @doc Rewrite the given node's app.config file, overriding the varialbes %% in the existing app.config with those in `Config'. update_app_config(all, Config) -> - ?HARNESS:update_app_config(all, Config); + rt_config:update_app_config(all, Config); update_app_config(Node, Config) -> - stop(Node), - ?assertEqual(ok, rt:wait_until_unpingable(Node)), - ?HARNESS:update_app_config(Node, Config), - start(Node). + rt_config:update_app_config(Node, Config). %% @doc Helper that returns first successful application get_env result, %% used when different versions of Riak use different app vars for %% the same setting. -rpc_get_env(_, []) -> - undefined; -rpc_get_env(Node, [{App,Var}|Others]) -> - case rpc:call(Node, application, get_env, [App, Var]) of - {ok, Value} -> - {ok, Value}; - _ -> - rpc_get_env(Node, Others) - end. +rpc_get_env(Node, AppVars) -> + rt2:rpc_get_env(Node, AppVars). -type interface() :: {http, tuple()} | {pb, tuple()}. -type interfaces() :: [interface()]. -type conn_info() :: [{node(), interfaces()}]. -spec connection_info(node() | [node()]) -> interfaces() | conn_info(). -connection_info(Node) when is_atom(Node) -> - {ok, [{PB_IP, PB_Port}]} = get_pb_conn_info(Node), - {ok, [{HTTP_IP, HTTP_Port}]} = get_http_conn_info(Node), - case get_https_conn_info(Node) of - undefined -> - [{http, {HTTP_IP, HTTP_Port}}, {pb, {PB_IP, PB_Port}}]; - {ok, [{HTTPS_IP, HTTPS_Port}]} -> - [{http, {HTTP_IP, HTTP_Port}}, {https, {HTTPS_IP, HTTPS_Port}}, {pb, {PB_IP, PB_Port}}] - end; -connection_info(Nodes) when is_list(Nodes) -> - [ {Node, connection_info(Node)} || Node <- Nodes]. +connection_info(Node) -> + rt2:connection_info(Node). -spec get_pb_conn_info(node()) -> [{inet:ip_address(), pos_integer()}]. get_pb_conn_info(Node) -> - case rpc_get_env(Node, [{riak_api, pb}, - {riak_api, pb_ip}, - {riak_kv, pb_ip}]) of - {ok, [{NewIP, NewPort}|_]} -> - {ok, [{NewIP, NewPort}]}; - {ok, PB_IP} -> - {ok, PB_Port} = rpc_get_env(Node, [{riak_api, pb_port}, - {riak_kv, pb_port}]), - {ok, [{PB_IP, PB_Port}]}; - _ -> - undefined - end. + rt_pb:get_pb_conn_info(Node). -spec get_http_conn_info(node()) -> [{inet:ip_address(), pos_integer()}]. get_http_conn_info(Node) -> - case rpc_get_env(Node, [{riak_api, http}, - {riak_core, http}]) of - {ok, [{IP, Port}|_]} -> - {ok, [{IP, Port}]}; - _ -> - undefined - end. + rt_http:get_http_conn_info(Node). -spec get_https_conn_info(node()) -> [{inet:ip_address(), pos_integer()}]. get_https_conn_info(Node) -> - case rpc_get_env(Node, [{riak_api, https}, - {riak_core, https}]) of - {ok, [{IP, Port}|_]} -> - {ok, [{IP, Port}]}; - _ -> - undefined - end. + rt_http:get_https_conn_info(Node). %% @doc Deploy a set of freshly installed Riak nodes, returning a list of the %% nodes deployed. @@ -303,7 +233,8 @@ get_https_conn_info(Node) -> deploy_nodes(Versions) when is_list(Versions) -> deploy_nodes(Versions, [riak_kv]); deploy_nodes(NumNodes) when is_integer(NumNodes) -> - deploy_nodes([ current || _ <- lists:seq(1, NumNodes)]). + [NodeIds, NodeMap, _] = allocate_nodes(NumNodes, rt_config:get_default_version()), + deploy_nodes(NodeIds, NodeMap, rt_config:get_default_version(), rt_properties:default_config(), [riak_kv]). %% @doc Deploy a set of freshly installed Riak nodes with the given %% `InitialConfig', returning a list of the nodes deployed. @@ -311,16 +242,72 @@ deploy_nodes(NumNodes) when is_integer(NumNodes) -> deploy_nodes(NumNodes, InitialConfig) when is_integer(NumNodes) -> deploy_nodes(NumNodes, InitialConfig, [riak_kv]); deploy_nodes(Versions, Services) -> - NodeConfig = [ version_to_config(Version) || Version <- Versions ], - Nodes = ?HARNESS:deploy_nodes(NodeConfig), + MappedVersions = [map_version_and_config(Vsn) || Vsn <- Versions], + lager:debug("Starting nodes using config and versions ~p", [MappedVersions]), + + Nodes = rt_harness:deploy_nodes(MappedVersions), lager:info("Waiting for services ~p to start on ~p.", [Services, Nodes]), [ ok = wait_for_service(Node, Service) || Node <- Nodes, Service <- Services ], Nodes. deploy_nodes(NumNodes, InitialConfig, Services) when is_integer(NumNodes) -> - NodeConfig = [{current, InitialConfig} || _ <- lists:seq(1,NumNodes)], - deploy_nodes(NodeConfig, Services). + Version = rt_config:get_default_version(), + [NodeIds, NodeMap, _] = allocate_nodes(NumNodes, Version), + + deploy_nodes(NodeIds, NodeMap, Version, InitialConfig, Services). + +deploy_nodes(NodeIds, NodeMap, Version, Config, Services) -> + _ = rt_harness_util:deploy_nodes(NodeIds, NodeMap, Version, Config, Services), + lists:foldl(fun({_, NodeName}, Nodes) -> [NodeName|Nodes] end, + [], + NodeMap). + +map_version_and_config({Vsn, Cfg}) -> + {rt_config:version_to_tag(Vsn), Cfg}; +map_version_and_config(Vsn) -> + {rt_config:version_to_tag(Vsn), rt_properties:default_config()}. + +allocate_nodes(NumNodes, Version) when is_atom(Version) -> + allocate_nodes(NumNodes, atom_to_list(Version)); +allocate_nodes(NumNodes, Version) -> + [_, AvailableNodeMap, VersionMap] = rt_harness:available_resources(), + lager:debug("Available node map ~p and version map ~p.", [AvailableNodeMap, VersionMap]), + + AvailableNodeIds = proplists:get_value(Version, VersionMap), + lager:debug("Availabe node ids ~p for version ~p", [AvailableNodeIds, Version]), + AllocatedNodeIds = lists:sublist(AvailableNodeIds, NumNodes), + lager:debug("Allocated node ids ~p", [AllocatedNodeIds]), + + [AllocatedNodeMap, NodeNames, _] = lists:foldl( + fun(NodeId, [AllocatedNodeMapAcc, NodeNamesAcc, Number]) -> + NodeName = proplists:get_value(NodeId, AvailableNodeMap), + [[{NodeId, NodeName}|AllocatedNodeMapAcc], + orddict:append(NodeId, Number, NodeNamesAcc), + Number + 1] + end, + [[], orddict:new(), 1], + AllocatedNodeIds), + lager:debug("AllocatedNodeMap: ~p", [AllocatedNodeMap]), + [Nodes, Versions] = lists:foldl( + fun({NodeId, NodeName}, [NodesAcc, VersionsAcc]) -> + [orddict:append(NodeName, NodeId, NodesAcc), + orddict:append(NodeName, Version, VersionsAcc)] + end, + [orddict:new(), orddict:new()], + AllocatedNodeMap), + + lager:debug("Allocated node map ~p", [AllocatedNodeMap]), + + rt_config:set(rt_nodes, Nodes), + rt_config:set(rt_nodenames, NodeNames), + rt_config:set(rt_versions, Versions), + + lager:debug("Set rt_nodes: ~p", [ rt_config:get(rt_nodes) ]), + lager:debug("Set rt_nodenames: ~p", [ rt_config:get(rt_nodenames) ]), + lager:debug("Set rt_versions: ~p", [ rt_config:get(rt_versions) ]), + + [AllocatedNodeIds, AllocatedNodeMap, VersionMap]. version_to_config(Config) when is_tuple(Config)-> Config; version_to_config(Version) -> {Version, default}. @@ -328,17 +315,19 @@ version_to_config(Version) -> {Version, default}. deploy_clusters(Settings) -> ClusterConfigs = [case Setting of Configs when is_list(Configs) -> + lager:info("deploy_cluster Configs"), Configs; NumNodes when is_integer(NumNodes) -> - [{current, default} || _ <- lists:seq(1, NumNodes)]; + [{rt_config:get_default_version(), default} || _ <- lists:seq(1, NumNodes)]; {NumNodes, InitialConfig} when is_integer(NumNodes) -> - [{current, InitialConfig} || _ <- lists:seq(1,NumNodes)]; + [{rt_config:get_default_version(), InitialConfig} || _ <- lists:seq(1,NumNodes)]; {NumNodes, Vsn, InitialConfig} when is_integer(NumNodes) -> - [{Vsn, InitialConfig} || _ <- lists:seq(1,NumNodes)] + [{rt_config:version_to_tag(Vsn), InitialConfig} || _ <- lists:seq(1,NumNodes)] end || Setting <- Settings], ?HARNESS:deploy_clusters(ClusterConfigs). build_clusters(Settings) -> + lager:debug("build_clusters ~p", [Settings]), Clusters = deploy_clusters(Settings), [begin join_cluster(Nodes), @@ -348,7 +337,8 @@ build_clusters(Settings) -> %% @doc Start the specified Riak node start(Node) -> - ?HARNESS:start(Node). + %% TODO Determine the best way to implement the current version specification. -jsb + rt_node:start(Node, rt_harness:node_version(Node)). %% @doc Start the specified Riak `Node' and wait for it to be pingable start_and_wait(Node) -> @@ -356,14 +346,13 @@ start_and_wait(Node) -> ?assertEqual(ok, wait_until_pingable(Node)). async_start(Node) -> - spawn(fun() -> start(Node) end). + %% TODO Determine the best way to implement the current version specification. -jsb + rt_node:async_start(Node, rt_harness:node_version(Node)). %% @doc Stop the specified Riak `Node'. stop(Node) -> - lager:info("Stopping riak on ~p", [Node]), - timer:sleep(10000), %% I know, I know! - ?HARNESS:stop(Node). - %%rpc:call(Node, init, stop, []). + %% TODO Determine the best way to implement the current version specification. -jsb + rt_node:stop(Node, rt_harness:node_version(Node)). %% @doc Stop the specified Riak `Node' and wait until it is not pingable stop_and_wait(Node) -> @@ -372,15 +361,18 @@ stop_and_wait(Node) -> %% @doc Upgrade a Riak `Node' to the specified `NewVersion'. upgrade(Node, NewVersion) -> - ?HARNESS:upgrade(Node, NewVersion). + %% GAP: The new API does not provide an analog to this function. -jsb + rt_harness:upgrade(Node, NewVersion). %% @doc Upgrade a Riak `Node' to the specified `NewVersion' and update %% the config based on entries in `Config'. upgrade(Node, NewVersion, Config) -> - ?HARNESS:upgrade(Node, NewVersion, Config). + %% TODO Determine the best way to implement the current version specification. -jsb + rt_node:upgrade(Node, rt_harness:node_version(Node), NewVersion, Config). %% @doc Upgrade a Riak node to a specific version using the alternate %% leave/upgrade/rejoin approach +%% GAP: rt_node does not current provide slow_upgrade(Node, NewVersion, Nodes) -> lager:info("Perform leave/upgrade/join upgrade on ~p", [Node]), lager:info("Leaving ~p", [Node]), @@ -396,181 +388,84 @@ slow_upgrade(Node, NewVersion, Nodes) -> %% @doc Have `Node' send a join request to `PNode' join(Node, PNode) -> - R = rpc:call(Node, riak_core, join, [PNode]), - lager:info("[join] ~p to (~p): ~p", [Node, PNode, R]), - ?assertEqual(ok, R), - ok. + rt_node:join(Node, PNode). %% @doc Have `Node' send a join request to `PNode' staged_join(Node, PNode) -> - R = rpc:call(Node, riak_core, staged_join, [PNode]), - lager:info("[join] ~p to (~p): ~p", [Node, PNode, R]), - ?assertEqual(ok, R), - ok. + rt_node:staged_join(Node, PNode). plan_and_commit(Node) -> - timer:sleep(500), - lager:info("planning and commiting cluster join"), - case rpc:call(Node, riak_core_claimant, plan, []) of - {error, ring_not_ready} -> - lager:info("plan: ring not ready"), - timer:sleep(100), - plan_and_commit(Node); - {ok, _, _} -> - lager:info("plan: done"), - do_commit(Node) - end. + rt_node:plan_and_commit(Node). do_commit(Node) -> - case rpc:call(Node, riak_core_claimant, commit, []) of - {error, plan_changed} -> - lager:info("commit: plan changed"), - timer:sleep(100), - maybe_wait_for_changes(Node), - plan_and_commit(Node); - {error, ring_not_ready} -> - lager:info("commit: ring not ready"), - timer:sleep(100), - maybe_wait_for_changes(Node), - do_commit(Node); - {error,nothing_planned} -> - %% Assume plan actually committed somehow - ok; - ok -> - ok - end. + rt_node:do_commit(Node). maybe_wait_for_changes(Node) -> - Ring = get_ring(Node), - Changes = riak_core_ring:pending_changes(Ring), - Joining = riak_core_ring:members(Ring, [joining]), - lager:info("maybe_wait_for_changes, changes: ~p joining: ~p", - [Changes, Joining]), - if Changes =:= [] -> - ok; - Joining =/= [] -> - ok; - true -> - ok = wait_until_no_pending_changes([Node]) - end. + rt2:maybe_wait_for_changes(Node). %% @doc Have the `Node' leave the cluster leave(Node) -> - R = rpc:call(Node, riak_core, leave, []), - lager:info("[leave] ~p: ~p", [Node, R]), - ?assertEqual(ok, R), - ok. + rt_node:leave(Node). %% @doc Have `Node' remove `OtherNode' from the cluster remove(Node, OtherNode) -> - ?assertEqual(ok, - rpc:call(Node, riak_kv_console, remove, [[atom_to_list(OtherNode)]])). + rt_node:remove(Node, OtherNode). %% @doc Have `Node' mark `OtherNode' as down down(Node, OtherNode) -> - rpc:call(Node, riak_kv_console, down, [[atom_to_list(OtherNode)]]). + rt_node:down(Node, OtherNode). %% @doc partition the `P1' from `P2' nodes %% note: the nodes remained connected to riak_test@local, %% which is how `heal/1' can still work. partition(P1, P2) -> - OldCookie = rpc:call(hd(P1), erlang, get_cookie, []), - NewCookie = list_to_atom(lists:reverse(atom_to_list(OldCookie))), - [true = rpc:call(N, erlang, set_cookie, [N, NewCookie]) || N <- P1], - [[true = rpc:call(N, erlang, disconnect_node, [P2N]) || N <- P1] || P2N <- P2], - wait_until_partitioned(P1, P2), - {NewCookie, OldCookie, P1, P2}. + rt_node:partition(P1, P2). %% @doc heal the partition created by call to `partition/2' %% `OldCookie' is the original shared cookie -heal({_NewCookie, OldCookie, P1, P2}) -> - Cluster = P1 ++ P2, - % set OldCookie on P1 Nodes - [true = rpc:call(N, erlang, set_cookie, [N, OldCookie]) || N <- P1], - wait_until_connected(Cluster), - {_GN, []} = rpc:sbcast(Cluster, riak_core_node_watcher, broadcast), - ok. +heal({NewCookie, OldCookie, P1, P2}) -> + rt_node:heal({NewCookie, OldCookie, P1, P2}). %% @doc Spawn `Cmd' on the machine running the test harness spawn_cmd(Cmd) -> - ?HARNESS:spawn_cmd(Cmd). + rt2:spawn_cmd(Cmd). %% @doc Spawn `Cmd' on the machine running the test harness spawn_cmd(Cmd, Opts) -> - ?HARNESS:spawn_cmd(Cmd, Opts). + rt2:spawn_cmd(Cmd, Opts). %% @doc Wait for a command spawned by `spawn_cmd', returning %% the exit status and result wait_for_cmd(CmdHandle) -> - ?HARNESS:wait_for_cmd(CmdHandle). + rt2:wait_for_cmd(CmdHandle). %% @doc Spawn `Cmd' on the machine running the test harness, returning %% the exit status and result cmd(Cmd) -> - ?HARNESS:cmd(Cmd). + rt2:cmd(Cmd). %% @doc Spawn `Cmd' on the machine running the test harness, returning %% the exit status and result cmd(Cmd, Opts) -> - ?HARNESS:cmd(Cmd, Opts). + rt2:cmd(Cmd, Opts). %% @doc pretty much the same as os:cmd/1 but it will stream the output to lager. %% If you're running a long running command, it will dump the output %% once per second, as to not create the impression that nothing is happening. -spec stream_cmd(string()) -> {integer(), string()}. stream_cmd(Cmd) -> - Port = open_port({spawn, binary_to_list(iolist_to_binary(Cmd))}, [stream, stderr_to_stdout, exit_status]), - stream_cmd_loop(Port, "", "", now()). + rt2:stream_cmd(Cmd). %% @doc same as rt:stream_cmd/1, but with options, like open_port/2 -spec stream_cmd(string(), string()) -> {integer(), string()}. stream_cmd(Cmd, Opts) -> - Port = open_port({spawn, binary_to_list(iolist_to_binary(Cmd))}, [stream, stderr_to_stdout, exit_status] ++ Opts), - stream_cmd_loop(Port, "", "", now()). - -stream_cmd_loop(Port, Buffer, NewLineBuffer, Time={_MegaSecs, Secs, _MicroSecs}) -> - receive - {Port, {data, Data}} -> - {_, Now, _} = now(), - NewNewLineBuffer = case Now > Secs of - true -> - lager:info(NewLineBuffer), - ""; - _ -> - NewLineBuffer - end, - case rt:str(Data, "\n") of - true -> - lager:info(NewNewLineBuffer), - Tokens = string:tokens(Data, "\n"), - [ lager:info(Token) || Token <- Tokens ], - stream_cmd_loop(Port, Buffer ++ NewNewLineBuffer ++ Data, "", Time); - _ -> - stream_cmd_loop(Port, Buffer, NewNewLineBuffer ++ Data, now()) - end; - {Port, {exit_status, Status}} -> - catch port_close(Port), - {Status, Buffer} - after rt:config(rt_max_wait_time) -> - {-1, Buffer} - end. + rt2:stream_cmd(Cmd, Opts). %%%=================================================================== %%% Remote code management %%%=================================================================== -load_modules_on_nodes([], Nodes) - when is_list(Nodes) -> - ok; -load_modules_on_nodes([Module | MoreModules], Nodes) - when is_list(Nodes) -> - case code:get_object_code(Module) of - {Module, Bin, File} -> - {_, []} = rpc:multicall(Nodes, code, load_binary, [Module, File, Bin]); - error -> - error(lists:flatten(io_lib:format("unable to get_object_code(~s)", [Module]))) - end, - load_modules_on_nodes(MoreModules, Nodes). - +load_modules_on_nodes(Modules, Nodes) -> + rt2:load_modules_on_nodes(Modules, Nodes). %%%=================================================================== %%% Status / Wait Functions @@ -578,47 +473,22 @@ load_modules_on_nodes([Module | MoreModules], Nodes) %% @doc Is the `Node' up according to net_adm:ping is_pingable(Node) -> - net_adm:ping(Node) =:= pong. + rt_node:is_pingable(Node). -is_mixed_cluster(Nodes) when is_list(Nodes) -> - %% If the nodes are bad, we don't care what version they are - {Versions, _BadNodes} = rpc:multicall(Nodes, init, script_id, [], rt_config:get(rt_max_wait_time)), - length(lists:usort(Versions)) > 1; -is_mixed_cluster(Node) -> - Nodes = rpc:call(Node, erlang, nodes, []), - is_mixed_cluster(Nodes). +is_mixed_cluster(Nodes) -> + rt2:is_mixed_cluster(Nodes). %% @private is_ready(Node) -> - case rpc:call(Node, riak_core_ring_manager, get_raw_ring, []) of - {ok, Ring} -> - case lists:member(Node, riak_core_ring:ready_members(Ring)) of - true -> true; - false -> {not_ready, Node} - end; - Other -> - Other - end. - -%% @private -is_ring_ready(Node) -> - case rpc:call(Node, riak_core_ring_manager, get_raw_ring, []) of - {ok, Ring} -> - riak_core_ring:ring_ready(Ring); - _ -> - false - end. + rt_node:is_ready(Node). %% @doc Utility function used to construct test predicates. Retries the %% function `Fun' until it returns `true', or until the maximum %% number of retries is reached. The retry limit is based on the -%% provided `rt_max_wait_time' and `rt_retry_delay' parameters in +%% provided `rt_max_receive_wait_time' and `rt_retry_delay' parameters in %% specified `riak_test' config file. wait_until(Fun) when is_function(Fun) -> - MaxTime = rt_config:get(rt_max_wait_time), - Delay = rt_config:get(rt_retry_delay), - Retry = MaxTime div Delay, - wait_until(Fun, Retry, Delay). + rt2:wait_until(Fun). %% @doc Convenience wrapper for wait_until for the myriad functions that %% take a node as single argument. @@ -629,351 +499,113 @@ wait_until(Node, Fun) when is_atom(Node), is_function(Fun) -> %% milliseconds between retries. This is our eventual consistency bread %% and butter wait_until(Fun, Retry, Delay) when Retry > 0 -> - Res = Fun(), - case Res of - true -> - ok; - _ when Retry == 1 -> - {fail, Res}; - _ -> - timer:sleep(Delay), - wait_until(Fun, Retry-1, Delay) - end. + rt2:wait_until(Fun, Retry, Delay). %% @doc Wait until the specified node is considered ready by `riak_core'. %% As of Riak 1.0, a node is ready if it is in the `valid' or `leaving' %% states. A ready node is guaranteed to have current preflist/ownership %% information. wait_until_ready(Node) -> - lager:info("Wait until ~p ready", [Node]), - ?assertEqual(ok, wait_until(Node, fun is_ready/1)), - ok. + rt2:wait_until_ready(Node). %% @doc Wait until status can be read from riak_kv_console wait_until_status_ready(Node) -> - lager:info("Wait until status ready in ~p", [Node]), - ?assertEqual(ok, wait_until(Node, - fun(_) -> - case rpc:call(Node, riak_kv_console, status, [[]]) of - ok -> - true; - Res -> - Res - end - end)). + rt2:wait_until_status_ready(Node). %% @doc Given a list of nodes, wait until all nodes believe there are no %% on-going or pending ownership transfers. -spec wait_until_no_pending_changes([node()]) -> ok | fail. wait_until_no_pending_changes(Nodes) -> - lager:info("Wait until no pending changes on ~p", [Nodes]), - F = fun() -> - rpc:multicall(Nodes, riak_core_vnode_manager, force_handoffs, []), - {Rings, BadNodes} = rpc:multicall(Nodes, riak_core_ring_manager, get_raw_ring, []), - Changes = [ riak_core_ring:pending_changes(Ring) =:= [] || {ok, Ring} <- Rings ], - BadNodes =:= [] andalso length(Changes) =:= length(Nodes) andalso lists:all(fun(T) -> T end, Changes) - end, - ?assertEqual(ok, wait_until(F)), - ok. + rt2:wait_until_no_pending_changes(Nodes). %% @doc Waits until no transfers are in-flight or pending, checked by %% riak_core_status:transfers(). -spec wait_until_transfers_complete([node()]) -> ok | fail. -wait_until_transfers_complete([Node0|_]) -> - lager:info("Wait until transfers complete ~p", [Node0]), - F = fun(Node) -> - {DownNodes, Transfers} = rpc:call(Node, riak_core_status, transfers, []), - DownNodes =:= [] andalso Transfers =:= [] - end, - ?assertEqual(ok, wait_until(Node0, F)), - ok. +wait_until_transfers_complete(Nodes) -> + rt2:wait_until_transfers_complete(Nodes). wait_for_service(Node, Services) when is_list(Services) -> - F = fun(N) -> - case rpc:call(N, riak_core_node_watcher, services, [N]) of - {badrpc, Error} -> - {badrpc, Error}; - CurrServices when is_list(CurrServices) -> - lists:all(fun(Service) -> lists:member(Service, CurrServices) end, Services); - Res -> - Res - end - end, - ?assertEqual(ok, wait_until(Node, F)), - ok; + rt2:wait_for_service(Node, Services); wait_for_service(Node, Service) -> - wait_for_service(Node, [Service]). + rt2:wait_for_service(Node, [Service]). wait_for_cluster_service(Nodes, Service) -> - lager:info("Wait for cluster service ~p in ~p", [Service, Nodes]), - F = fun(N) -> - UpNodes = rpc:call(N, riak_core_node_watcher, nodes, [Service]), - (Nodes -- UpNodes) == [] - end, - [?assertEqual(ok, wait_until(Node, F)) || Node <- Nodes], - ok. + rt2:wait_for_cluster_service(Nodes, Service). %% @doc Given a list of nodes, wait until all nodes are considered ready. %% See {@link wait_until_ready/1} for definition of ready. wait_until_nodes_ready(Nodes) -> - lager:info("Wait until nodes are ready : ~p", [Nodes]), - [?assertEqual(ok, wait_until(Node, fun is_ready/1)) || Node <- Nodes], - ok. + rt_node:wait_until_nodes_ready(Nodes). %% @doc Wait until all nodes in the list `Nodes' believe each other to be %% members of the cluster. wait_until_all_members(Nodes) -> - wait_until_all_members(Nodes, Nodes). + rt2:wait_until_all_members(Nodes, Nodes). %% @doc Wait until all nodes in the list `Nodes' believes all nodes in the %% list `Members' are members of the cluster. wait_until_all_members(Nodes, ExpectedMembers) -> - lager:info("Wait until all members ~p ~p", [Nodes, ExpectedMembers]), - S1 = ordsets:from_list(ExpectedMembers), - F = fun(Node) -> - case members_according_to(Node) of - {badrpc, _} -> - false; - ReportedMembers -> - S2 = ordsets:from_list(ReportedMembers), - ordsets:is_subset(S1, S2) - end - end, - [?assertEqual(ok, wait_until(Node, F)) || Node <- Nodes], - ok. + rt2:wait_until_all_members(Nodes, ExpectedMembers). %% @doc Given a list of nodes, wait until all nodes believe the ring has %% converged (ie. `riak_core_ring:is_ready' returns `true'). wait_until_ring_converged(Nodes) -> - lager:info("Wait until ring converged on ~p", [Nodes]), - [?assertEqual(ok, wait_until(Node, fun is_ring_ready/1)) || Node <- Nodes], - ok. + rt2:wait_until_ring_converged(Nodes). wait_until_legacy_ringready(Node) -> - lager:info("Wait until legacy ring ready on ~p", [Node]), - rt:wait_until(Node, - fun(_) -> - case rpc:call(Node, riak_kv_status, ringready, []) of - {ok, _Nodes} -> - true; - Res -> - Res - end - end). + rt2:wait_until_legacy_ringready(Node). %% @doc wait until each node in Nodes is disterl connected to each. wait_until_connected(Nodes) -> - lager:info("Wait until connected ~p", [Nodes]), - NodeSet = sets:from_list(Nodes), - F = fun(Node) -> - Connected = rpc:call(Node, erlang, nodes, []), - sets:is_subset(NodeSet, sets:from_list(([Node] ++ Connected) -- [node()])) - end, - [?assertEqual(ok, wait_until(Node, F)) || Node <- Nodes], - ok. + rt2:wait_until_connected(Nodes). %% @doc Wait until the specified node is pingable wait_until_pingable(Node) -> - lager:info("Wait until ~p is pingable", [Node]), - F = fun(N) -> - net_adm:ping(N) =:= pong - end, - ?assertEqual(ok, wait_until(Node, F)), - ok. + rt2:wait_until_pingable(Node). %% @doc Wait until the specified node is no longer pingable wait_until_unpingable(Node) -> - lager:info("Wait until ~p is not pingable", [Node]), - _OSPidToKill = rpc:call(Node, os, getpid, []), - F = fun() -> net_adm:ping(Node) =:= pang end, - %% riak stop will kill -9 after 5 mins, so we try to wait at least that - %% amount of time. - Delay = rt_config:get(rt_retry_delay), - Retry = lists:max([360000, rt_config:get(rt_max_wait_time)]) div Delay, - case wait_until(F, Retry, Delay) of - ok -> ok; - _ -> - lager:error("Timed out waiting for node ~p to shutdown", [Node]), - ?assert(node_shutdown_timed_out) - end. - + rt2:wait_until_unpingable(Node). % Waits until a certain registered name pops up on the remote node. wait_until_registered(Node, Name) -> - lager:info("Wait until ~p is up on ~p", [Name, Node]), - - F = fun() -> - Registered = rpc:call(Node, erlang, registered, []), - lists:member(Name, Registered) - end, - case wait_until(F) of - ok -> - ok; - _ -> - lager:info("The server with the name ~p on ~p is not coming up.", - [Name, Node]), - ?assert(registered_name_timed_out) - end. - + rt2:wait_until_registered(Node, Name). %% Waits until the cluster actually detects that it is partitioned. wait_until_partitioned(P1, P2) -> - lager:info("Waiting until partition acknowledged: ~p ~p", [P1, P2]), - [ begin - lager:info("Waiting for ~p to be partitioned from ~p", [Node, P2]), - wait_until(fun() -> is_partitioned(Node, P2) end) - end || Node <- P1 ], - [ begin - lager:info("Waiting for ~p to be partitioned from ~p", [Node, P1]), - wait_until(fun() -> is_partitioned(Node, P1) end) - end || Node <- P2 ]. + rt2:wait_until_partitioned(P1, P2). is_partitioned(Node, Peers) -> - AvailableNodes = rpc:call(Node, riak_core_node_watcher, nodes, [riak_kv]), - lists:all(fun(Peer) -> not lists:member(Peer, AvailableNodes) end, Peers). + rt2:is_partitioned(Node, Peers). % when you just can't wait brutal_kill(Node) -> - rt_cover:maybe_stop_on_node(Node), - lager:info("Killing node ~p", [Node]), - OSPidToKill = rpc:call(Node, os, getpid, []), - %% try a normal kill first, but set a timer to - %% kill -9 after 5 seconds just in case - rpc:cast(Node, timer, apply_after, - [5000, os, cmd, [io_lib:format("kill -9 ~s", [OSPidToKill])]]), - rpc:cast(Node, os, cmd, [io_lib:format("kill -15 ~s", [OSPidToKill])]), - ok. + rt_node:brutal_kill(Node). -capability(Node, all) -> - rpc:call(Node, riak_core_capability, all, []); capability(Node, Capability) -> - rpc:call(Node, riak_core_capability, get, [Capability]). + rt2:capability(Node, Capability). capability(Node, Capability, Default) -> - rpc:call(Node, riak_core_capability, get, [Capability, Default]). + rt2:capability(Node, Capability, Default). wait_until_capability(Node, Capability, Value) -> - rt:wait_until(Node, - fun(_) -> - Cap = capability(Node, Capability), - lager:info("Capability on node ~p is ~p~n",[Node, Cap]), - cap_equal(Value, Cap) - end). + rt2:wait_until_capability(Node, Capability, Value). wait_until_capability(Node, Capability, Value, Default) -> - rt:wait_until(Node, - fun(_) -> - Cap = capability(Node, Capability, Default), - lager:info("Capability on node ~p is ~p~n",[Node, Cap]), - cap_equal(Value, Cap) - end). - -wait_until_capability_contains(Node, Capability, Value) -> - rt:wait_until(Node, - fun(_) -> - Cap = capability(Node, Capability), - lager:info("Capability on node ~p is ~p~n",[Node, Cap]), - cap_subset(Value, Cap) - end). - -cap_equal(Val, Cap) when is_list(Cap) -> - lists:sort(Cap) == lists:sort(Val); -cap_equal(Val, Cap) -> - Val == Cap. + rt2:wait_until_capability(Node, Capability, Value, Default). -cap_subset(Val, Cap) when is_list(Cap) -> - sets:is_subset(sets:from_list(Val), sets:from_list(Cap)). +cap_equal(Val, Cap) -> + rt2:cap_equal(Val, Cap). wait_until_owners_according_to(Node, Nodes) -> - SortedNodes = lists:usort(Nodes), - F = fun(N) -> - owners_according_to(N) =:= SortedNodes - end, - ?assertEqual(ok, wait_until(Node, F)), - ok. + rt_node:wait_until_owners_according_to(Node, Nodes). wait_until_nodes_agree_about_ownership(Nodes) -> - lager:info("Wait until nodes agree about ownership ~p", [Nodes]), - Results = [ wait_until_owners_according_to(Node, Nodes) || Node <- Nodes ], - ?assert(lists:all(fun(X) -> ok =:= X end, Results)). + rt_node:wait_until_nodes_agree_about_ownership(Nodes). %% AAE support wait_until_aae_trees_built(Nodes) -> - lager:info("Wait until AAE builds all partition trees across ~p", [Nodes]), - BuiltFun = fun() -> lists:foldl(aae_tree_built_fun(), true, Nodes) end, - ?assertEqual(ok, wait_until(BuiltFun)), - ok. - -aae_tree_built_fun() -> - fun(Node, _AllBuilt = true) -> - case get_aae_tree_info(Node) of - {ok, TreeInfos} -> - case all_trees_have_build_times(TreeInfos) of - true -> - Partitions = [I || {I, _} <- TreeInfos], - all_aae_trees_built(Node, Partitions); - false -> - some_trees_not_built - end; - Err -> - Err - end; - (_Node, Err) -> - Err - end. - -% It is unlikely but possible to get a tree built time from compute_tree_info -% but an attempt to use the tree returns not_built. This is because the build -% process has finished, but the lock on the tree won't be released until it -% dies and the manager detects it. Yes, this is super freaking paranoid. -all_aae_trees_built(Node, Partitions) -> - %% Notice that the process locking is spawned by the - %% pmap. That's important! as it should die eventually - %% so the lock is released and the test can lock the tree. - IndexBuilts = rt:pmap(index_built_fun(Node), Partitions), - BadOnes = [R || R <- IndexBuilts, R /= true], - case BadOnes of - [] -> - true; - _ -> - BadOnes - end. - -get_aae_tree_info(Node) -> - case rpc:call(Node, riak_kv_entropy_info, compute_tree_info, []) of - {badrpc, _} -> - {error, {badrpc, Node}}; - Info -> - lager:debug("Entropy table on node ~p : ~p", [Node, Info]), - {ok, Info} - end. - -all_trees_have_build_times(Info) -> - not lists:keymember(undefined, 2, Info). - -index_built_fun(Node) -> - fun(Idx) -> - case rpc:call(Node, riak_kv_vnode, - hashtree_pid, [Idx]) of - {ok, TreePid} -> - case rpc:call(Node, riak_kv_index_hashtree, - get_lock, [TreePid, for_riak_test]) of - {badrpc, _} -> - {error, {badrpc, Node}}; - TreeLocked when TreeLocked == ok; - TreeLocked == already_locked -> - true; - Err -> - % Either not_built or some unhandled result, - % in which case update this case please! - {error, {index_not_built, Node, Idx, Err}} - end; - {error, _}=Err -> - Err; - {badrpc, _} -> - {error, {badrpc, Node}} - end - end. + rt_aae:wait_until_aae_trees_built(Nodes). %%%=================================================================== %%% Ring Functions @@ -982,84 +614,47 @@ index_built_fun(Node) -> %% @doc Ensure that the specified node is a singleton node/cluster -- a node %% that owns 100% of the ring. check_singleton_node(Node) -> - lager:info("Check ~p is a singleton", [Node]), - {ok, Ring} = rpc:call(Node, riak_core_ring_manager, get_raw_ring, []), - Owners = lists:usort([Owner || {_Idx, Owner} <- riak_core_ring:all_owners(Ring)]), - ?assertEqual([Node], Owners), - ok. + rt_ring:check_singleton_node(Node). % @doc Get list of partitions owned by node (primary). partitions_for_node(Node) -> - Ring = get_ring(Node), - [Idx || {Idx, Owner} <- riak_core_ring:all_owners(Ring), Owner == Node]. + rt_ring:partitions_for_node(Node). %% @doc Get the raw ring for `Node'. get_ring(Node) -> - {ok, Ring} = rpc:call(Node, riak_core_ring_manager, get_raw_ring, []), - Ring. + rt_ring:get_ring(Node). assert_nodes_agree_about_ownership(Nodes) -> - ?assertEqual(ok, wait_until_ring_converged(Nodes)), - ?assertEqual(ok, wait_until_all_members(Nodes)), - [ ?assertEqual({Node, Nodes}, {Node, owners_according_to(Node)}) || Node <- Nodes]. + rt_ring:assert_nodes_agree_about_ownership(Nodes). %% @doc Return a list of nodes that own partitions according to the ring %% retrieved from the specified node. owners_according_to(Node) -> - case rpc:call(Node, riak_core_ring_manager, get_raw_ring, []) of - {ok, Ring} -> - Owners = [Owner || {_Idx, Owner} <- riak_core_ring:all_owners(Ring)], - lists:usort(Owners); - {badrpc, _}=BadRpc -> - BadRpc - end. + rt_ring:owners_according_to(Node). %% @doc Return a list of cluster members according to the ring retrieved from %% the specified node. members_according_to(Node) -> - case rpc:call(Node, riak_core_ring_manager, get_raw_ring, []) of - {ok, Ring} -> - Members = riak_core_ring:all_members(Ring), - Members; - {badrpc, _}=BadRpc -> - BadRpc - end. + rt_ring:members_according_to(Node). %% @doc Return an appropriate ringsize for the node count passed %% in. 24 is the number of cores on the bigger intel machines, but this %% may be too large for the single-chip machines. nearest_ringsize(Count) -> - nearest_ringsize(Count * 24, 2). + rt_ring:nearest_ringsize(Count). nearest_ringsize(Count, Power) -> - case Count < trunc(Power * 0.9) of - true -> - Power; - false -> - nearest_ringsize(Count, Power * 2) - end. + rt_ring:nearest_ringsize(Count, Power). %% @doc Return the cluster status of `Member' according to the ring %% retrieved from `Node'. status_of_according_to(Member, Node) -> - case rpc:call(Node, riak_core_ring_manager, get_raw_ring, []) of - {ok, Ring} -> - Status = riak_core_ring:member_status(Ring, Member), - Status; - {badrpc, _}=BadRpc -> - BadRpc - end. + rt_ring:status_of_according_to(Member, Node). %% @doc Return a list of nodes that own partitions according to the ring %% retrieved from the specified node. claimant_according_to(Node) -> - case rpc:call(Node, riak_core_ring_manager, get_raw_ring, []) of - {ok, Ring} -> - Claimant = riak_core_ring:claimant(Ring), - Claimant; - {badrpc, _}=BadRpc -> - BadRpc - end. + rt_ring:claimant_according_to(Node). %%%=================================================================== %%% Cluster Utility Functions @@ -1068,9 +663,16 @@ claimant_according_to(Node) -> %% @doc Safely construct a new cluster and return a list of the deployed nodes %% @todo Add -spec and update doc to reflect mult-version changes build_cluster(Versions) when is_list(Versions) -> - build_cluster(length(Versions), Versions, default); + UpdatedVersions = [build_version_config(Vsn) || Vsn <- Versions], + build_cluster(length(Versions), UpdatedVersions, rt_properties:default_config()); build_cluster(NumNodes) -> - build_cluster(NumNodes, default). + build_cluster(NumNodes, rt_properties:default_config()). + +%% @doc Use the default configuration when a configuration is not specified +build_version_config(Vsn) when not is_tuple(Vsn) -> + {rt_config:version_to_tag(Vsn), rt_properties:default_config()}; +build_version_config({Vsn, Cfg}) -> + {rt_config:version_to_tag(Vsn), Cfg}. %% @doc Safely construct a `NumNode' size cluster using %% `InitialConfig'. Return a list of the deployed nodes. @@ -1095,10 +697,6 @@ join_cluster(Nodes) -> %% Ensure each node owns 100% of it's own ring [?assertEqual([Node], owners_according_to(Node)) || Node <- Nodes], - %% Potential fix for BTA-116 and other similar "join before nodes ready" issues. - %% TODO: Investigate if there is an actual race in Riak relating to cluster joins. - [ok = wait_for_service(Node, riak_kv) || Node <- Nodes], - %% Join nodes [Node1|OtherNodes] = Nodes, case OtherNodes of @@ -1134,19 +732,9 @@ product(Node) -> HasRiak -> riak; true -> unknown end. - -try_nodes_ready([Node1 | _Nodes], 0, _SleepMs) -> - lager:info("Nodes not ready after initial plan/commit, retrying"), - plan_and_commit(Node1); + try_nodes_ready(Nodes, N, SleepMs) -> - ReadyNodes = [Node || Node <- Nodes, is_ready(Node) =:= true], - case ReadyNodes of - Nodes -> - ok; - _ -> - timer:sleep(SleepMs), - try_nodes_ready(Nodes, N-1, SleepMs) - end. + rt_cluster:try_nodes_ready(Nodes, N, SleepMs). %% @doc Stop nodes and wipe out their data directories clean_cluster(Nodes) when is_list(Nodes) -> @@ -1163,27 +751,24 @@ clean_data_dir(Nodes, SubDir) when is_list(Nodes) -> %% @doc Shutdown every node, this is for after a test run is complete. teardown() -> - %% stop all connected nodes, 'cause it'll be faster that - %%lager:info("RPC stopping these nodes ~p", [nodes()]), - %%[ rt:stop(Node) || Node <- nodes()], - %% Then do the more exhaustive harness thing, in case something was up - %% but not connected. - ?HARNESS:teardown(). + rt_cluster:teardown(). +%% TODO: Only used in verify_capabalities. Probably should be refactored. versions() -> - ?HARNESS:versions(). + proplists:get_keys(rt_config:get(versions)). + %%%=================================================================== %%% Basic Read/Write Functions %%%=================================================================== systest_write(Node, Size) -> - systest_write(Node, Size, 2). + rt_systest:write(Node, Size). systest_write(Node, Size, W) -> - systest_write(Node, 1, Size, <<"systest">>, W). + rt_systest:write(Node, Size, W). systest_write(Node, Start, End, Bucket, W) -> - systest_write(Node, Start, End, Bucket, W, <<>>). + rt_systest:write(Node, Start, End, Bucket, W). %% @doc Write (End-Start)+1 objects to Node. Objects keys will be %% `Start', `Start+1' ... `End', each encoded as a 32-bit binary @@ -1193,37 +778,20 @@ systest_write(Node, Start, End, Bucket, W) -> %% encountered. If all writes were successful, return value is an %% empty list. Each error has the form `{N :: integer(), Error :: term()}', %% where N is the unencoded key of the object that failed to store. -systest_write(Node, Start, End, Bucket, W, CommonValBin) - when is_binary(CommonValBin) -> - rt:wait_for_service(Node, riak_kv), - {ok, C} = riak:client_connect(Node), - F = fun(N, Acc) -> - Obj = riak_object:new(Bucket, <>, - <>), - try C:put(Obj, W) of - ok -> - Acc; - Other -> - [{N, Other} | Acc] - catch - What:Why -> - [{N, {What, Why}} | Acc] - end - end, - lists:foldl(F, [], lists:seq(Start, End)). +systest_write(Node, Start, End, Bucket, W, CommonValBin) -> + rt_systest:write(Node, Start, End, Bucket, W, CommonValBin). systest_read(Node, Size) -> - systest_read(Node, Size, 2). + rt_systest:read(Node, Size, 2). systest_read(Node, Size, R) -> - systest_read(Node, 1, Size, <<"systest">>, R). + rt_systest:read(Node, 1, Size, <<"systest">>, R). systest_read(Node, Start, End, Bucket, R) -> - systest_read(Node, Start, End, Bucket, R, <<>>). + rt_systest:read(Node, Start, End, Bucket, R, <<>>). -systest_read(Node, Start, End, Bucket, R, CommonValBin) - when is_binary(CommonValBin) -> - systest_read(Node, Start, End, Bucket, R, CommonValBin, false). +systest_read(Node, Start, End, Bucket, R, CommonValBin) -> + rt_systest:read(Node, Start, End, Bucket, R, CommonValBin, false). %% Read and verify the values of objects written with %% `systest_write'. The `SquashSiblings' parameter exists to @@ -1235,149 +803,34 @@ systest_read(Node, Start, End, Bucket, R, CommonValBin) %% writes, but also performs them locally or when a put coordinator %% fails to send an acknowledgment within the timeout window and %% another put request is issued. -systest_read(Node, Start, End, Bucket, R, CommonValBin, SquashSiblings) - when is_binary(CommonValBin) -> - rt:wait_for_service(Node, riak_kv), - {ok, C} = riak:client_connect(Node), - lists:foldl(systest_read_fold_fun(C, Bucket, R, CommonValBin, SquashSiblings), - [], - lists:seq(Start, End)). - -systest_read_fold_fun(C, Bucket, R, CommonValBin, SquashSiblings) -> - fun(N, Acc) -> - GetRes = C:get(Bucket, <>, R), - Val = object_value(GetRes, SquashSiblings), - update_acc(value_matches(Val, N, CommonValBin), Val, N, Acc) - end. - -object_value({error, _}=Error, _) -> - Error; -object_value({ok, Obj}, SquashSiblings) -> - object_value(riak_object:value_count(Obj), Obj, SquashSiblings). - -object_value(1, Obj, _SquashSiblings) -> - riak_object:get_value(Obj); -object_value(_ValueCount, Obj, false) -> - riak_object:get_value(Obj); -object_value(_ValueCount, Obj, true) -> - lager:debug("Siblings detected for ~p:~p", [riak_object:bucket(Obj), riak_object:key(Obj)]), - Contents = riak_object:get_contents(Obj), - case lists:foldl(fun sibling_compare/2, {true, undefined}, Contents) of - {true, {_, _, _, Value}} -> - lager:debug("Siblings determined to be a single value"), - Value; - {false, _} -> - {error, siblings} - end. - -sibling_compare({MetaData, Value}, {true, undefined}) -> - Dot = case dict:find(<<"dot">>, MetaData) of - {ok, DotVal} -> - DotVal; - error -> - {error, no_dot} - end, - VTag = dict:fetch(<<"X-Riak-VTag">>, MetaData), - LastMod = dict:fetch(<<"X-Riak-Last-Modified">>, MetaData), - {true, {element(2, Dot), VTag, LastMod, Value}}; -sibling_compare(_, {false, _}=InvalidMatch) -> - InvalidMatch; -sibling_compare({MetaData, Value}, {true, PreviousElements}) -> - Dot = case dict:find(<<"dot">>, MetaData) of - {ok, DotVal} -> - DotVal; - error -> - {error, no_dot} - end, - VTag = dict:fetch(<<"X-Riak-VTag">>, MetaData), - LastMod = dict:fetch(<<"X-Riak-Last-Modified">>, MetaData), - ComparisonElements = {element(2, Dot), VTag, LastMod, Value}, - {ComparisonElements =:= PreviousElements, ComparisonElements}. - -value_matches(<>, N, CommonValBin) -> - true; -value_matches(_WrongVal, _N, _CommonValBin) -> - false. - -update_acc(true, _, _, Acc) -> - Acc; -update_acc(false, {error, _}=Val, N, Acc) -> - [{N, Val} | Acc]; -update_acc(false, Val, N, Acc) -> - [{N, {wrong_val, Val}} | Acc]. +systest_read(Node, Start, End, Bucket, R, CommonValBin, SquashSiblings) -> + rt_systest:read(Node, Start, End, Bucket, R, CommonValBin, SquashSiblings). % @doc Reads a single replica of a value. This issues a get command directly % to the vnode handling the Nth primary partition of the object's preflist. get_replica(Node, Bucket, Key, I, N) -> - BKey = {Bucket, Key}, - Chash = rpc:call(Node, riak_core_util, chash_key, [BKey]), - Pl = rpc:call(Node, riak_core_apl, get_primary_apl, [Chash, N, riak_kv]), - {{Partition, PNode}, primary} = lists:nth(I, Pl), - Ref = Reqid = make_ref(), - Sender = {raw, Ref, self()}, - rpc:call(PNode, riak_kv_vnode, get, - [{Partition, PNode}, BKey, Ref, Sender]), - receive - {Ref, {r, Result, _, Reqid}} -> - Result; - {Ref, Reply} -> - Reply - after - 60000 -> - lager:error("Replica ~p get for ~p/~p timed out", - [I, Bucket, Key]), - ?assert(false) - end. + rt2:get_replica(Node, Bucket, Key, I, N). %%%=================================================================== %% @doc PBC-based version of {@link systest_write/1} pbc_systest_write(Node, Size) -> - pbc_systest_write(Node, Size, 2). + rt_pb:pbc_systest_write(Node, Size). pbc_systest_write(Node, Size, W) -> - pbc_systest_write(Node, 1, Size, <<"systest">>, W). + rt_pb:pbc_systest_write(Node, Size, W). pbc_systest_write(Node, Start, End, Bucket, W) -> - rt:wait_for_service(Node, riak_kv), - Pid = pbc(Node), - F = fun(N, Acc) -> - Obj = riakc_obj:new(Bucket, <>, <>), - try riakc_pb_socket:put(Pid, Obj, W) of - ok -> - Acc; - Other -> - [{N, Other} | Acc] - catch - What:Why -> - [{N, {What, Why}} | Acc] - end - end, - lists:foldl(F, [], lists:seq(Start, End)). + rt_pb:pbc_systest_write(Node, Start, End, Bucket, W). pbc_systest_read(Node, Size) -> - pbc_systest_read(Node, Size, 2). + rt_pb:pbc_systest_read(Node, Size). pbc_systest_read(Node, Size, R) -> - pbc_systest_read(Node, 1, Size, <<"systest">>, R). + rt_pb:pbc_systest_read(Node, Size, R). pbc_systest_read(Node, Start, End, Bucket, R) -> - rt:wait_for_service(Node, riak_kv), - Pid = pbc(Node), - F = fun(N, Acc) -> - case riakc_pb_socket:get(Pid, Bucket, <>, R) of - {ok, Obj} -> - case riakc_obj:get_value(Obj) of - <> -> - Acc; - WrongVal -> - [{N, {wrong_val, WrongVal}} | Acc] - end; - Other -> - [{N, Other} | Acc] - end - end, - lists:foldl(F, [], lists:seq(Start, End)). + rt_pb:pbc_systest_read(Node, Start, End, Bucket, R). %%%=================================================================== %%% PBC & HTTPC Functions @@ -1386,15 +839,11 @@ pbc_systest_read(Node, Start, End, Bucket, R) -> %% @doc get me a protobuf client process and hold the mayo! -spec pbc(node()) -> pid(). pbc(Node) -> - pbc(Node, [{auto_reconnect, true}]). + rt_pb:pbc(Node). -spec pbc(node(), proplists:proplist()) -> pid(). pbc(Node, Options) -> - rt:wait_for_service(Node, riak_kv), - ConnInfo = proplists:get_value(Node, connection_info([Node])), - {IP, PBPort} = proplists:get_value(pb, ConnInfo), - {ok, Pid} = riakc_pb_socket:start_link(IP, PBPort, Options), - Pid. + rt_pb:pbc(Node, Options). %% @doc does a read via the erlang protobuf client -spec pbc_read(pid(), binary(), binary()) -> binary(). @@ -1403,8 +852,7 @@ pbc_read(Pid, Bucket, Key) -> -spec pbc_read(pid(), binary(), binary(), [any()]) -> binary(). pbc_read(Pid, Bucket, Key, Options) -> - {ok, Value} = riakc_pb_socket:get(Pid, Bucket, Key, Options), - Value. + rt_pb:pbc_read(Pid, Bucket, Key, Options). -spec pbc_read_check(pid(), binary(), binary(), [any()]) -> boolean(). pbc_read_check(Pid, Bucket, Key, Allowed) -> @@ -1412,102 +860,62 @@ pbc_read_check(Pid, Bucket, Key, Allowed) -> -spec pbc_read_check(pid(), binary(), binary(), [any()], [any()]) -> boolean(). pbc_read_check(Pid, Bucket, Key, Allowed, Options) -> - case riakc_pb_socket:get(Pid, Bucket, Key, Options) of - {ok, _} -> - true = lists:member(ok, Allowed); - Other -> - lists:member(Other, Allowed) orelse throw({failed, Other, Allowed}) - end. + rt_pb:pbc_read_check(Pid, Bucket, Key, Allowed, Options). %% @doc does a write via the erlang protobuf client -spec pbc_write(pid(), binary(), binary(), binary()) -> atom(). pbc_write(Pid, Bucket, Key, Value) -> - Object = riakc_obj:new(Bucket, Key, Value), - riakc_pb_socket:put(Pid, Object). + rt_pb:pbc_write(Pid, Bucket, Key, Value). %% @doc does a write via the erlang protobuf client plus content-type -spec pbc_write(pid(), binary(), binary(), binary(), list()) -> atom(). pbc_write(Pid, Bucket, Key, Value, CT) -> - Object = riakc_obj:new(Bucket, Key, Value, CT), - riakc_pb_socket:put(Pid, Object). + rt_pb:pbc_write(Pid, Bucket, Key, Value, CT). %% @doc sets a bucket property/properties via the erlang protobuf client -spec pbc_set_bucket_prop(pid(), binary(), [proplists:property()]) -> atom(). pbc_set_bucket_prop(Pid, Bucket, PropList) -> - riakc_pb_socket:set_bucket(Pid, Bucket, PropList). + rt_pb:pbc_set_bucket_prop(Pid, Bucket, PropList). %% @doc Puts the contents of the given file into the given bucket using the %% filename as a key and assuming a plain text content type. pbc_put_file(Pid, Bucket, Key, Filename) -> - {ok, Contents} = file:read_file(Filename), - riakc_pb_socket:put(Pid, riakc_obj:new(Bucket, Key, Contents, "text/plain")). + rt_pb:pbc_put_file(Pid, Bucket, Key, Filename). %% @doc Puts all files in the given directory into the given bucket using the %% filename as a key and assuming a plain text content type. pbc_put_dir(Pid, Bucket, Dir) -> - lager:info("Putting files from dir ~p into bucket ~p", [Dir, Bucket]), - {ok, Files} = file:list_dir(Dir), - [pbc_put_file(Pid, Bucket, list_to_binary(F), filename:join([Dir, F])) - || F <- Files]. + rt_pb:pbc_put_dir(Pid, Bucket, Dir). %% @doc True if the given keys have been really, really deleted. %% Useful when you care about the keys not being there. Delete simply writes %% tombstones under the given keys, so those are still seen by key folding %% operations. pbc_really_deleted(Pid, Bucket, Keys) -> - StillThere = - fun(K) -> - Res = riakc_pb_socket:get(Pid, Bucket, K, - [{r, 1}, - {notfound_ok, false}, - {basic_quorum, false}, - deletedvclock]), - case Res of - {error, notfound} -> - false; - _ -> - %% Tombstone still around - true - end - end, - [] == lists:filter(StillThere, Keys). + rt_pb:pbc_really_deleted(Pid, Bucket, Keys). %% @doc Returns HTTPS URL information for a list of Nodes -https_url(Nodes) when is_list(Nodes) -> - [begin - {Host, Port} = orddict:fetch(https, Connections), - lists:flatten(io_lib:format("https://~s:~b", [Host, Port])) - end || {_Node, Connections} <- connection_info(Nodes)]; -https_url(Node) -> - hd(https_url([Node])). +https_url(Nodes) -> + rt_http:https_url(Nodes). %% @doc Returns HTTP URL information for a list of Nodes -http_url(Nodes) when is_list(Nodes) -> - [begin - {Host, Port} = orddict:fetch(http, Connections), - lists:flatten(io_lib:format("http://~s:~b", [Host, Port])) - end || {_Node, Connections} <- connection_info(Nodes)]; -http_url(Node) -> - hd(http_url([Node])). +http_url(Nodes) -> + rt_http:http_url(Nodes). %% @doc get me an http client. -spec httpc(node()) -> term(). httpc(Node) -> - rt:wait_for_service(Node, riak_kv), - {ok, [{IP, Port}]} = get_http_conn_info(Node), - rhc:create(IP, Port, "riak", []). + rt_http:httpc(Node). %% @doc does a read via the http erlang client. -spec httpc_read(term(), binary(), binary()) -> binary(). httpc_read(C, Bucket, Key) -> - {_, Value} = rhc:get(C, Bucket, Key), - Value. + rt_http:httpc_read(C, Bucket, Key). %% @doc does a write via the http erlang client. -spec httpc_write(term(), binary(), binary(), binary()) -> atom(). httpc_write(C, Bucket, Key, Value) -> - Object = riakc_obj:new(Bucket, Key, Value), - rhc:put(C, Object). + rt_http:httpc_write(C, Bucket, Key, Value). %%%=================================================================== %%% Command Line Functions @@ -1521,20 +929,18 @@ admin(Node, Args) -> %% The third parameter is a list of options. Valid options are: %% * `return_exit_code' - Return the exit code along with the command output admin(Node, Args, Options) -> - ?HARNESS:admin(Node, Args, Options). + rt_cmd_line:admin(Node, Args, Options). %% @doc Call 'bin/riak' command on `Node' with arguments `Args' riak(Node, Args) -> - ?HARNESS:riak(Node, Args). - + rt_cmd_line:riak(Node, Args). %% @doc Call 'bin/riak-repl' command on `Node' with arguments `Args' riak_repl(Node, Args) -> - ?HARNESS:riak_repl(Node, Args). + rt_cmd_line:riak_repl(Node, Args). search_cmd(Node, Args) -> - {ok, Cwd} = file:get_cwd(), - rpc:call(Node, riak_search_cmd, command, [[Cwd | Args]]). + rt_cmd_line:search_cmd(Node, Args). %% @doc Runs `riak attach' on a specific node, and tests for the expected behavoir. %% Here's an example: ``` @@ -1550,17 +956,17 @@ search_cmd(Node, Args) -> %% expect will process based on the output following the sent data. %% attach(Node, Expected) -> - ?HARNESS:attach(Node, Expected). + rt_cmd_line:attach(Node, Expected). %% @doc Runs 'riak attach-direct' on a specific node %% @see rt:attach/2 attach_direct(Node, Expected) -> - ?HARNESS:attach_direct(Node, Expected). + rt_cmd_line:attach_direct(Node, Expected). %% @doc Runs `riak console' on a specific node %% @see rt:attach/2 console(Node, Expected) -> - ?HARNESS:console(Node, Expected). + rt_cmd_line:console(Node, Expected). %%%=================================================================== %%% Search @@ -1569,9 +975,8 @@ console(Node, Expected) -> %% doc Enable the search KV hook for the given `Bucket'. Any `Node' %% in the cluster may be used as the change is propagated via the %% Ring. -enable_search_hook(Node, Bucket) when is_binary(Bucket) -> - lager:info("Installing search hook for bucket ~p", [Bucket]), - ?assertEqual(ok, rpc:call(Node, riak_search_kv_hook, install, [Bucket])). +enable_search_hook(Node, Bucket) -> + rt2:enable_search_hook(Node, Bucket). %%%=================================================================== %%% Test harness setup, configuration, and internal utilities @@ -1591,6 +996,8 @@ set_backend(Backend) -> -spec set_backend(atom(), [{atom(), term()}]) -> atom()|[atom()]. set_backend(bitcask, _) -> set_backend(riak_kv_bitcask_backend); +set_backend(leveldb, Extras) -> + set_backend(eleveldb, Extras); set_backend(eleveldb, _) -> set_backend(riak_kv_eleveldb_backend); set_backend(memory, _) -> @@ -1646,15 +1053,19 @@ get_backend(AppConfigProplist) -> %% or something like that, it's the version you're upgrading to. -spec get_version() -> binary(). get_version() -> - ?HARNESS:get_version(). + rt2:get_version(). + +-spec get_version(string()) -> binary(). +get_version(Node) -> + rt_harness:get_version(Node). %% @doc outputs some useful information about nodes that are up whats_up() -> - ?HARNESS:whats_up(). + rt2:whats_up(). -spec get_ip(node()) -> string(). get_ip(Node) -> - ?HARNESS:get_ip(Node). + rt2:get_ip(Node). %% @doc Log a message to the console of the specified test nodes. %% Messages are prefixed by the string "---riak_test--- " @@ -1666,81 +1077,30 @@ log_to_nodes(Nodes, Fmt) -> %% Messages are prefixed by the string "---riak_test--- " %% Uses lager:info/2 'LFmt' and 'LArgs' semantics log_to_nodes(Nodes0, LFmt, LArgs) -> - %% This logs to a node's info level, but if riak_test is running - %% at debug level, we want to know when we send this and what - %% we're saying - Nodes = lists:flatten(Nodes0), - lager:debug("log_to_nodes: " ++ LFmt, LArgs), - Module = lager, - Function = log, - Meta = [], - Args = case LArgs of - [] -> [info, Meta, "---riak_test--- " ++ LFmt]; - _ -> [info, Meta, "---riak_test--- " ++ LFmt, LArgs] - end, - [rpc:call(Node, Module, Function, Args) || Node <- lists:flatten(Nodes)]. - -%% @private utility function + rt2:log_to_nodes(Nodes0, LFmt, LArgs). + pmap(F, L) -> - Parent = self(), - lists:foldl( - fun(X, N) -> - spawn_link(fun() -> - Parent ! {pmap, N, F(X)} - end), - N+1 - end, 0, L), - L2 = [receive {pmap, N, R} -> {N,R} end || _ <- L], - {_, L3} = lists:unzip(lists:keysort(1, L2)), - L3. + rt2:pmap(F, L). -%% @private -setup_harness(Test, Args) -> - ?HARNESS:setup_harness(Test, Args). +setup_harness(_Test, _Args) -> + rt_harness:setup(). -%% @doc Downloads any extant log files from the harness's running -%% nodes. +%% @doc Copy all of the nodes' log files to a local dir +-spec(get_node_logs(string(), string()) -> list()). +get_node_logs(LogFile, DestDir) -> + rt2:get_node_logs(LogFile, DestDir). + +%% @doc Open all of the nodes' log files to a port +%% OBSOLETE +-spec(get_node_logs() -> list()). get_node_logs() -> - ?HARNESS:get_node_logs(). + rt2:get_node_logs(). check_ibrowse() -> - try sys:get_status(ibrowse) of - {status, _Pid, {module, gen_server} ,_} -> ok - catch - Throws -> - lager:error("ibrowse error ~p", [Throws]), - lager:error("Restarting ibrowse"), - application:stop(ibrowse), - application:start(ibrowse) - end. + rt2:check_ibrowse(). -post_result(TestResult, #rt_webhook{url=URL, headers=HookHeaders, name=Name}) -> - lager:info("Posting result to ~s ~s", [Name, URL]), - try ibrowse:send_req(URL, - [{"Content-Type", "application/json"}], - post, - mochijson2:encode(TestResult), - [{content_type, "application/json"}] ++ HookHeaders, - 300000) of %% 5 minute timeout - - {ok, RC=[$2|_], Headers, _Body} -> - {ok, RC, Headers}; - {ok, ResponseCode, Headers, Body} -> - lager:info("Test Result did not generate the expected 2XX HTTP response code."), - lager:debug("Post"), - lager:debug("Response Code: ~p", [ResponseCode]), - lager:debug("Headers: ~p", [Headers]), - lager:debug("Body: ~p", [Body]), - error; - X -> - lager:warning("Some error POSTing test result: ~p", [X]), - error - catch - Class:Reason -> - lager:error("Error reporting to ~s. ~p:~p", [Name, Class, Reason]), - lager:error("Payload: ~p", [TestResult]), - error - end. +post_result(TestResult, Webhook) -> + rt2:post_result(TestResult, Webhook). %%%=================================================================== %%% Bucket Types Functions @@ -1748,184 +1108,43 @@ post_result(TestResult, #rt_webhook{url=URL, headers=HookHeaders, name=Name}) -> %% @doc create and immediately activate a bucket type create_and_activate_bucket_type(Node, Type, Props) -> - ok = rpc:call(Node, riak_core_bucket_type, create, [Type, Props]), - wait_until_bucket_type_status(Type, ready, Node), - ok = rpc:call(Node, riak_core_bucket_type, activate, [Type]), - wait_until_bucket_type_status(Type, active, Node). - -wait_until_bucket_type_status(Type, ExpectedStatus, Nodes) when is_list(Nodes) -> - [wait_until_bucket_type_status(Type, ExpectedStatus, Node) || Node <- Nodes]; -wait_until_bucket_type_status(Type, ExpectedStatus, Node) -> - F = fun() -> - ActualStatus = rpc:call(Node, riak_core_bucket_type, status, [Type]), - ExpectedStatus =:= ActualStatus - end, - ?assertEqual(ok, rt:wait_until(F)). + rt_bucket_types:create_and_activate_bucket_type(Node, Type, Props). --spec bucket_type_visible([atom()], binary()|{binary(), binary()}) -> boolean(). -bucket_type_visible(Nodes, Type) -> - MaxTime = rt_config:get(rt_max_wait_time), - IsVisible = fun erlang:is_list/1, - {Res, NodesDown} = rpc:multicall(Nodes, riak_core_bucket_type, get, [Type], MaxTime), - NodesDown == [] andalso lists:all(IsVisible, Res). +wait_until_bucket_type_status(Type, ExpectedStatus, Nodes) -> + rt_bucket_types:wait_until_bucket_type_status(Type, ExpectedStatus, Nodes). wait_until_bucket_type_visible(Nodes, Type) -> - F = fun() -> bucket_type_visible(Nodes, Type) end, - ?assertEqual(ok, rt:wait_until(F)). - --spec see_bucket_props([atom()], binary()|{binary(), binary()}, - proplists:proplist()) -> boolean(). -see_bucket_props(Nodes, Bucket, ExpectProps) -> - MaxTime = rt_config:get(rt_max_wait_time), - IsBad = fun({badrpc, _}) -> true; - ({error, _}) -> true; - (Res) when is_list(Res) -> false - end, - HasProps = fun(ResProps) -> - lists:all(fun(P) -> lists:member(P, ResProps) end, - ExpectProps) - end, - case rpc:multicall(Nodes, riak_core_bucket, get_bucket, [Bucket], MaxTime) of - {Res, []} -> - % No nodes down, check no errors - case lists:any(IsBad, Res) of - true -> - false; - false -> - lists:all(HasProps, Res) - end; - {_, _NodesDown} -> - false - end. + rt_bucket_types:wait_until_bucket_type_visible(Nodes, Type). wait_until_bucket_props(Nodes, Bucket, Props) -> - F = fun() -> - see_bucket_props(Nodes, Bucket, Props) - end, - ?assertEqual(ok, rt:wait_until(F)). + rt_bucket_types:wait_until_bucket_props(Nodes, Bucket, Props). %% @doc Set up in memory log capture to check contents in a test. -setup_log_capture(Nodes) when is_list(Nodes) -> - rt:load_modules_on_nodes([riak_test_lager_backend], Nodes), - [?assertEqual({Node, ok}, - {Node, - rpc:call(Node, - gen_event, - add_handler, - [lager_event, - riak_test_lager_backend, - [info, false]])}) || Node <- Nodes], - [?assertEqual({Node, ok}, - {Node, - rpc:call(Node, - lager, - set_loglevel, - [riak_test_lager_backend, - info])}) || Node <- Nodes]; -setup_log_capture(Node) when not is_list(Node) -> - setup_log_capture([Node]). - +setup_log_capture(Nodes) -> + rt2:setup_log_capture(Nodes). expect_in_log(Node, Pattern) -> - CheckLogFun = fun() -> - Logs = rpc:call(Node, riak_test_lager_backend, get_logs, []), - lager:info("looking for pattern ~s in logs for ~p", - [Pattern, Node]), - case re:run(Logs, Pattern, []) of - {match, _} -> - lager:info("Found match"), - true; - nomatch -> - lager:info("No match"), - false - end - end, - case rt:wait_until(CheckLogFun) of - ok -> - true; - _ -> - false - end. + rt2:expect_in_log(Node, Pattern). %% @doc Wait for Riak Control to start on a single node. %% %% Non-optimal check, because we're blocking for the gen_server to start %% to ensure that the routes have been added by the supervisor. %% -wait_for_control(_Vsn, Node) when is_atom(Node) -> - lager:info("Waiting for riak_control to start on node ~p.", [Node]), - - %% Wait for the gen_server. - rt:wait_until(Node, fun(N) -> - case rpc:call(N, - riak_control_session, - get_version, - []) of - {ok, _} -> - true; - Error -> - lager:info("Error was ~p.", [Error]), - false - end - end), - - lager:info("Waiting for routes to be added to supervisor..."), - - %% Wait for routes to be added by supervisor. - rt:wait_until(Node, fun(N) -> - case rpc:call(N, - webmachine_router, - get_routes, - []) of - {badrpc, Error} -> - lager:info("Error was ~p.", [Error]), - false; - Routes -> - case is_control_gui_route_loaded(Routes) of - false -> - false; - _ -> - true - end - end - end). - -%% @doc Is the riak_control GUI route loaded? -is_control_gui_route_loaded(Routes) -> - lists:keymember(admin_gui, 2, Routes) orelse lists:keymember(riak_control_wm_gui, 2, Routes). +wait_for_control(Vsn, Node) -> + rt2:wait_for_control(Vsn, Node). %% @doc Wait for Riak Control to start on a series of nodes. -wait_for_control(VersionedNodes) when is_list(VersionedNodes) -> - [wait_for_control(Vsn, Node) || {Vsn, Node} <- VersionedNodes]. - --spec select_random([any()]) -> any(). -select_random(List) -> - Length = length(List), - Idx = random:uniform(Length), - lists:nth(Idx, List). - -%% @doc Returns a random element from a given list. --spec random_sublist([any()], integer()) -> [any()]. -random_sublist(List, N) -> - % Properly seeding the process. - <> = crypto:rand_bytes(12), - random:seed({A, B, C}), - % Assign a random value for each element in the list. - List1 = [{random:uniform(), E} || E <- List], - % Sort by the random number. - List2 = lists:sort(List1), - % Take the first N elements. - List3 = lists:sublist(List2, N), - % Remove the random numbers. - [ E || {_,E} <- List3]. +wait_for_control(VersionedNodes) -> + rt2:wait_for_control(VersionedNodes). -ifdef(TEST). verify_product(Applications, ExpectedApplication) -> ?_test(begin meck:new(rpc, [unstick]), - meck:expect(rpc, call, fun([], application, which_applications, []) -> + meck:expect(rpc, call, fun([], application, which_applications, []) -> Applications end), ?assertMatch(ExpectedApplication, product([])), meck:unload(rpc) @@ -1940,5 +1159,5 @@ product_test_() -> verify_product([riak_repl, riak_kv], riak_ee), verify_product([riak_kv], riak), verify_product([kernel], unknown)]}. - + -endif. diff --git a/src/rt2.erl b/src/rt2.erl new file mode 100644 index 000000000..81a01f8b4 --- /dev/null +++ b/src/rt2.erl @@ -0,0 +1,758 @@ +%% ------------------------------------------------------------------- +%% +%% Copyright (c) 2013-2015 Basho Technologies, Inc. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% ------------------------------------------------------------------- + +%% @doc +%% Implements the base `riak_test' API, providing the ability to control +%% nodes in a Riak cluster as well as perform commonly reused operations. +%% Please extend this module with new functions that prove useful between +%% multiple independent tests. +-module(rt2). +-include_lib("eunit/include/eunit.hrl"). + +-compile(export_all). +-export([ + capability/2, + capability/3, + check_ibrowse/0, + cmd/1, + cmd/2, + connection_info/1, + enable_search_hook/2, + expect_in_log/2, + get_deps/0, + get_ip/1, + get_node_logs/2, + get_replica/5, + get_version/0, + is_mixed_cluster/1, + load_modules_on_nodes/2, + log_to_nodes/2, + log_to_nodes/3, + pmap/2, + priv_dir/0, + product/1, + rpc_get_env/2, + setup_harness/2, + setup_log_capture/1, + stream_cmd/1, + stream_cmd/2, + spawn_cmd/1, + spawn_cmd/2, + str/2, + wait_for_cluster_service/2, + wait_for_cmd/1, + wait_for_service/2, + wait_for_control/1, + wait_for_control/2, + wait_until/3, + wait_until/2, + wait_until/1, + wait_until_all_members/1, + wait_until_all_members/2, + wait_until_capability/3, + wait_until_capability/4, + wait_until_connected/1, + wait_until_legacy_ringready/1, + wait_until_no_pending_changes/1, + wait_until_pingable/1, + wait_until_ready/1, + wait_until_registered/2, + wait_until_ring_converged/1, + wait_until_status_ready/1, + wait_until_transfers_complete/1, + wait_until_unpingable/1, + whats_up/0 + ]). + +priv_dir() -> + LocalPrivDir = "./priv", + %% XXX for some reason, codew:priv_dir returns riak_test/riak_test/priv, + %% which is wrong, so fix it. + DepPrivDir = re:replace(code:priv_dir(riak_test), "riak_test(/riak_test)*", + "riak_test", [{return, list}]), + PrivDir = case {filelib:is_dir(LocalPrivDir), filelib:is_dir(DepPrivDir)} of + {true, _} -> + lager:debug("Local ./priv detected, using that..."), + %% we want an absolute path! + filename:absname(LocalPrivDir); + {false, true} -> + lager:debug("riak_test dependency priv_dir detected, using that..."), + DepPrivDir; + _ -> + ?assertEqual({true, bad_priv_dir}, {false, bad_priv_dir}) + end, + + lager:info("priv dir: ~p -> ~p", [code:priv_dir(riak_test), PrivDir]), + ?assert(filelib:is_dir(PrivDir)), + PrivDir. + +%% @doc gets riak deps from the appropriate harness +-spec get_deps() -> list(). +get_deps() -> rt_harness:get_deps(). + +%% @doc if String contains Substr, return true. +-spec str(string(), string()) -> boolean(). +str(String, Substr) -> + case string:str(String, Substr) of + 0 -> false; + _ -> true + end. + +%% @doc Helper that returns first successful application get_env result, +%% used when different versions of Riak use different app vars for +%% the same setting. +rpc_get_env(_, []) -> + undefined; +rpc_get_env(Node, [{App,Var}|Others]) -> + case rpc:call(Node, application, get_env, [App, Var]) of + {ok, Value} -> + {ok, Value}; + _ -> + rpc_get_env(Node, Others) + end. + +-type interface() :: {http, tuple()} | {pb, tuple()}. +-type interfaces() :: [interface()]. +-type conn_info() :: [{node(), interfaces()}]. + +-spec connection_info(node() | [node()]) -> interfaces() | conn_info(). +connection_info(Node) when is_atom(Node) -> + {ok, [{PB_IP, PB_Port}]} = rt_pb:get_pb_conn_info(Node), + {ok, [{HTTP_IP, HTTP_Port}]} = rt_http:get_http_conn_info(Node), + case rt_http:get_https_conn_info(Node) of + undefined -> + [{http, {HTTP_IP, HTTP_Port}}, {pb, {PB_IP, PB_Port}}]; + {ok, [{HTTPS_IP, HTTPS_Port}]} -> + [{http, {HTTP_IP, HTTP_Port}}, {https, {HTTPS_IP, HTTPS_Port}}, {pb, {PB_IP, PB_Port}}] + end; +connection_info(Nodes) when is_list(Nodes) -> + [ {Node, connection_info(Node)} || Node <- Nodes]. + +maybe_wait_for_changes(Node) -> + Ring = rt_ring:get_ring(Node), + Changes = riak_core_ring:pending_changes(Ring), + Joining = riak_core_ring:members(Ring, [joining]), + lager:info("maybe_wait_for_changes, changes: ~p joining: ~p", + [Changes, Joining]), + if Changes =:= [] -> + ok; + Joining =/= [] -> + ok; + true -> + ok = wait_until_no_pending_changes([Node]) + end. + +%% @doc Spawn `Cmd' on the machine running the test harness +spawn_cmd(Cmd) -> + rt_harness:spawn_cmd(Cmd). + +%% @doc Spawn `Cmd' on the machine running the test harness +spawn_cmd(Cmd, Opts) -> + rt_harness:spawn_cmd(Cmd, Opts). + +%% @doc Wait for a command spawned by `spawn_cmd', returning +%% the exit status and result +wait_for_cmd(CmdHandle) -> + rt_harness:wait_for_cmd(CmdHandle). + +%% @doc Spawn `Cmd' on the machine running the test harness, returning +%% the exit status and result +cmd(Cmd) -> + rt_harness:cmd(Cmd). + +%% @doc Spawn `Cmd' on the machine running the test harness, returning +%% the exit status and result +cmd(Cmd, Opts) -> + rt_harness:cmd(Cmd, Opts). + +%% @doc pretty much the same as os:cmd/1 but it will stream the output to lager. +%% If you're running a long running command, it will dump the output +%% once per second, as to not create the impression that nothing is happening. +-spec stream_cmd(string()) -> {integer(), string()}. +stream_cmd(Cmd) -> + Port = open_port({spawn, binary_to_list(iolist_to_binary(Cmd))}, [stream, stderr_to_stdout, exit_status]), + stream_cmd_loop(Port, "", "", now()). + +%% @doc same as rt:stream_cmd/1, but with options, like open_port/2 +-spec stream_cmd(string(), string()) -> {integer(), string()}. +stream_cmd(Cmd, Opts) -> + Port = open_port({spawn, binary_to_list(iolist_to_binary(Cmd))}, [stream, stderr_to_stdout, exit_status] ++ Opts), + stream_cmd_loop(Port, "", "", now()). + +stream_cmd_loop(Port, Buffer, NewLineBuffer, Time={_MegaSecs, Secs, _MicroSecs}) -> + receive + {Port, {data, Data}} -> + {_, Now, _} = now(), + NewNewLineBuffer = case Now > Secs of + true -> + lager:info(NewLineBuffer), + ""; + _ -> + NewLineBuffer + end, + case rt:str(Data, "\n") of + true -> + lager:info(NewNewLineBuffer), + Tokens = string:tokens(Data, "\n"), + [ lager:info(Token) || Token <- Tokens ], + stream_cmd_loop(Port, Buffer ++ NewNewLineBuffer ++ Data, "", Time); + _ -> + stream_cmd_loop(Port, Buffer, NewNewLineBuffer ++ Data, now()) + end; + {Port, {exit_status, Status}} -> + catch port_close(Port), + {Status, Buffer} + after rt_config:get(rt_max_receive_wait_time) -> + {-1, Buffer} + end. + +%%%=================================================================== +%%% Remote code management +%%%=================================================================== +load_modules_on_nodes([], Nodes) when is_list(Nodes) -> + ok; +load_modules_on_nodes([Module | MoreModules], Nodes) when is_list(Nodes) -> + case code:get_object_code(Module) of + {Module, Bin, File} -> + {_, []} = rpc:multicall(Nodes, code, load_binary, [Module, File, Bin]); + error -> + error(lists:flatten(io_lib:format("unable to get_object_code(~s)", [Module]))) + end, + load_modules_on_nodes(MoreModules, Nodes). + + +%%%=================================================================== +%%% Status / Wait Functions +%%%=================================================================== + +is_mixed_cluster(Nodes) when is_list(Nodes) -> + %% If the nodes are bad, we don't care what version they are + {Versions, _BadNodes} = rpc:multicall(Nodes, init, script_id, [], rt_config:get(rt_max_receive_wait_time)), + length(lists:usort(Versions)) > 1; + is_mixed_cluster(Node) -> + Nodes = rpc:call(Node, erlang, nodes, []), + is_mixed_cluster(Nodes). + +%% @private +is_ring_ready(Node) -> + case rpc:call(Node, riak_core_ring_manager, get_raw_ring, []) of + {ok, Ring} -> + riak_core_ring:ring_ready(Ring); + _ -> + false + end. + +-type products() :: riak | riak_ee | riak_cs | unknown. + +-spec product(node()) -> products(). +product(Node) -> + Applications = rpc:call(Node, application, which_applications, []), + + HasRiakCS = proplists:is_defined(riak_cs, Applications), + HasRiakEE = proplists:is_defined(riak_repl, Applications), + HasRiak = proplists:is_defined(riak_kv, Applications), + if HasRiakCS -> + riak_cs; + HasRiakEE -> riak_ee; + HasRiak -> riak; + true -> unknown + end. + +%% @doc Utility function used to construct test predicates. Retries the +%% function `Fun' until it returns `true', or until the maximum +%% number of retries is reached. The retry limit is based on the +%% provided `rt_max_receive_wait_time' and `rt_retry_delay' parameters in +%% specified `riak_test' config file. +wait_until(Fun) when is_function(Fun) -> + MaxTime = rt_config:get(rt_max_receive_wait_time), + Delay = rt_config:get(rt_retry_delay), + Retry = MaxTime div Delay, + wait_until(Fun, Retry, Delay). + +%% @doc Convenience wrapper for wait_until for the myriad functions that +%% take a node as single argument. +wait_until(Node, Fun) when is_atom(Node), is_function(Fun) -> + wait_until(fun() -> Fun(Node) end). + +%% @doc Retry `Fun' until it returns `Retry' times, waiting `Delay' +%% milliseconds between retries. This is our eventual consistency bread +%% and butter +wait_until(Fun, Retry, Delay) when Retry > 0 -> + Res = Fun(), + case Res of + true -> + ok; + _ when Retry == 1 -> + {fail, Res}; + _ -> + timer:sleep(Delay), + wait_until(Fun, Retry-1, Delay) + end. + +%% @doc Wait until the specified node is considered ready by `riak_core'. +%% As of Riak 1.0, a node is ready if it is in the `valid' or `leaving' +%% states. A ready node is guaranteed to have current preflist/ownership +%% information. +wait_until_ready(Node) -> + lager:info("Wait until ~p ready", [Node]), + ?assertEqual(ok, wait_until(Node, fun rt_node:is_ready/1)), + ok. + +%% @doc Wait until status can be read from riak_kv_console +wait_until_status_ready(Node) -> + lager:info("Wait until status ready in ~p", [Node]), + ?assertEqual(ok, wait_until(Node, + fun(_) -> + case rpc:call(Node, riak_kv_console, status, [[]]) of + ok -> + true; + Res -> + Res + end + end)). + +%% @doc Given a list of nodes, wait until all nodes believe there are no +%% on-going or pending ownership transfers. +-spec wait_until_no_pending_changes([node()]) -> ok | fail. +wait_until_no_pending_changes(Nodes) -> + lager:info("Wait until no pending changes on ~p", [Nodes]), + F = fun() -> + rpc:multicall(Nodes, riak_core_vnode_manager, force_handoffs, []), + {Rings, BadNodes} = rpc:multicall(Nodes, riak_core_ring_manager, get_raw_ring, []), + Changes = [ riak_core_ring:pending_changes(Ring) =:= [] || {ok, Ring} <- Rings ], + BadNodes =:= [] andalso length(Changes) =:= length(Nodes) andalso lists:all(fun(T) -> T end, Changes) + end, + ?assertEqual(ok, wait_until(F)), + ok. + +%% @doc Waits until no transfers are in-flight or pending, checked by +%% riak_core_status:transfers(). +-spec wait_until_transfers_complete([node()]) -> ok | fail. +wait_until_transfers_complete([Node0|_]) -> + lager:info("Wait until transfers complete ~p", [Node0]), + F = fun(Node) -> + {DownNodes, Transfers} = rpc:call(Node, riak_core_status, transfers, []), + DownNodes =:= [] andalso Transfers =:= [] + end, + ?assertEqual(ok, wait_until(Node0, F)), + ok. + +wait_for_service(Node, Services) when is_list(Services) -> + F = fun(N) -> + case rpc:call(N, riak_core_node_watcher, services, [N]) of + {badrpc, Error} -> + {badrpc, Error}; + CurrServices when is_list(CurrServices) -> + lists:all(fun(Service) -> lists:member(Service, CurrServices) end, Services); + Res -> + Res + end + end, + ?assertEqual(ok, wait_until(Node, F)), + ok; +wait_for_service(Node, Service) -> + wait_for_service(Node, [Service]). + +wait_for_cluster_service(Nodes, Service) -> + lager:info("Wait for cluster service ~p in ~p", [Service, Nodes]), + F = fun(N) -> + UpNodes = rpc:call(N, riak_core_node_watcher, nodes, [Service]), + (Nodes -- UpNodes) == [] + end, + [?assertEqual(ok, wait_until(Node, F)) || Node <- Nodes], + ok. + +%% @doc Wait until all nodes in the list `Nodes' believe each other to be +%% members of the cluster. +wait_until_all_members(Nodes) -> + wait_until_all_members(Nodes, Nodes). + +%% @doc Wait until all nodes in the list `Nodes' believes all nodes in the +%% list `Members' are members of the cluster. +wait_until_all_members(Nodes, ExpectedMembers) -> + lager:info("Wait until all members ~p ~p", [Nodes, ExpectedMembers]), + S1 = ordsets:from_list(ExpectedMembers), + F = fun(Node) -> + case rt_ring:members_according_to(Node) of + {badrpc, _} -> + false; + ReportedMembers -> + S2 = ordsets:from_list(ReportedMembers), + ordsets:is_subset(S1, S2) + end + end, + [?assertEqual(ok, wait_until(Node, F)) || Node <- Nodes], + ok. + +%% @doc Given a list of nodes, wait until all nodes believe the ring has +%% converged (ie. `riak_core_ring:is_ready' returns `true'). +wait_until_ring_converged(Nodes) -> + lager:info("Wait until ring converged on ~p", [Nodes]), + [?assertEqual(ok, wait_until(Node, fun is_ring_ready/1)) || Node <- Nodes], + ok. + +wait_until_legacy_ringready(Node) -> + lager:info("Wait until legacy ring ready on ~p", [Node]), + rt:wait_until(Node, + fun(_) -> + case rpc:call(Node, riak_kv_status, ringready, []) of + {ok, _Nodes} -> + true; + Res -> + Res + end + end). + +%% @doc wait until each node in Nodes is disterl connected to each. +wait_until_connected(Nodes) -> + lager:info("Wait until connected ~p", [Nodes]), + F = fun(Node) -> + Connected = rpc:call(Node, erlang, nodes, []), + lists:sort(Nodes) == lists:sort([Node]++Connected)--[node()] + end, + [?assertEqual(ok, wait_until(Node, F)) || Node <- Nodes], + ok. + +%% @doc Wait until the specified node is pingable +wait_until_pingable(Node) -> + lager:info("Wait until ~p is pingable", [Node]), + F = fun(N) -> + net_adm:ping(N) =:= pong + end, + ?assertEqual(ok, wait_until(Node, F)), + ok. + +%% @doc Wait until the specified node is no longer pingable +wait_until_unpingable(Node) -> + _OSPidToKill = rpc:call(Node, os, getpid, []), + F = fun() -> net_adm:ping(Node) =:= pang end, + %% riak stop will kill -9 after 5 mins, so we try to wait at least that + %% amount of time. + Delay = rt_config:get(rt_retry_delay), + Retry = lists:max([360000, rt_config:get(rt_max_receive_wait_time)]) div Delay, + lager:info("Wait until ~p is not pingable for ~p seconds with a retry of ~p", + [Node, Delay, Retry]), + case wait_until(F, Retry, Delay) of + ok -> ok; + _ -> + lager:error("Timed out waiting for node ~p to shutdown", [Node]), + ?assert(node_shutdown_timed_out) + end. + +% Waits until a certain registered name pops up on the remote node. +wait_until_registered(Node, Name) -> + lager:info("Wait until ~p is up on ~p", [Name, Node]), + + F = fun() -> + Registered = rpc:call(Node, erlang, registered, []), + lists:member(Name, Registered) + end, + case wait_until(F) of + ok -> + ok; + _ -> + lager:info("The server with the name ~p on ~p is not coming up.", + [Name, Node]), + ?assert(registered_name_timed_out) + end. + +%% Waits until the cluster actually detects that it is partitioned. +wait_until_partitioned(P1, P2) -> + lager:info("Waiting until partition acknowledged: ~p ~p", [P1, P2]), + [ begin + lager:info("Waiting for ~p to be partitioned from ~p", [Node, P2]), + wait_until(fun() -> is_partitioned(Node, P2) end) + end || Node <- P1 ], + [ begin + lager:info("Waiting for ~p to be partitioned from ~p", [Node, P1]), + wait_until(fun() -> is_partitioned(Node, P1) end) + end || Node <- P2 ]. + +is_partitioned(Node, Peers) -> + AvailableNodes = rpc:call(Node, riak_core_node_watcher, nodes, [riak_kv]), + lists:all(fun(Peer) -> not lists:member(Peer, AvailableNodes) end, Peers). + +capability(Node, all) -> + rpc:call(Node, riak_core_capability, all, []); +capability(Node, Capability) -> + rpc:call(Node, riak_core_capability, get, [Capability]). + +capability(Node, Capability, Default) -> + rpc:call(Node, riak_core_capability, get, [Capability, Default]). + +wait_until_capability(Node, Capability, Value) -> + rt:wait_until(Node, + fun(_) -> + cap_equal(Value, capability(Node, Capability)) + end). + +wait_until_capability(Node, Capability, Value, Default) -> + rt:wait_until(Node, + fun(_) -> + Cap = capability(Node, Capability, Default), + io:format("capability is ~p ~p",[Node, Cap]), + cap_equal(Value, Cap) + end). + +cap_equal(Val, Cap) when is_list(Cap) -> + lists:sort(Cap) == lists:sort(Val); +cap_equal(Val, Cap) -> + Val == Cap. + +% @doc Reads a single replica of a value. This issues a get command directly +% to the vnode handling the Nth primary partition of the object's preflist. +get_replica(Node, Bucket, Key, I, N) -> + BKey = {Bucket, Key}, + Chash = rpc:call(Node, riak_core_util, chash_key, [BKey]), + Pl = rpc:call(Node, riak_core_apl, get_primary_apl, [Chash, N, riak_kv]), + {{Partition, PNode}, primary} = lists:nth(I, Pl), + Ref = Reqid = make_ref(), + Sender = {raw, Ref, self()}, + rpc:call(PNode, riak_kv_vnode, get, + [{Partition, PNode}, BKey, Ref, Sender]), + receive + {Ref, {r, Result, _, Reqid}} -> + Result; + {Ref, Reply} -> + Reply + after + 60000 -> + lager:error("Replica ~p get for ~p/~p timed out", + [I, Bucket, Key]), + ?assert(false) + end. + +%%%=================================================================== +%%% Search +%%%=================================================================== + +%% doc Enable the search KV hook for the given `Bucket'. Any `Node' +%% in the cluster may be used as the change is propagated via the +%% Ring. +enable_search_hook(Node, Bucket) when is_binary(Bucket) -> + lager:info("Installing search hook for bucket ~p", [Bucket]), + ?assertEqual(ok, rpc:call(Node, riak_search_kv_hook, install, [Bucket])). + +%% @doc Gets the current version under test. In the case of an upgrade test +%% or something like that, it's the version you're upgrading to. +-spec get_version() -> binary(). +get_version() -> + rt_harness:get_version(). + +%% @doc outputs some useful information about nodes that are up +whats_up() -> + rt_harness:whats_up(). + +-spec get_ip(node()) -> string(). +get_ip(Node) -> + rt_harness:get_ip(Node). + +%% @doc Log a message to the console of the specified test nodes. +%% Messages are prefixed by the string "---riak_test--- " +%% Uses lager:info/1 'Fmt' semantics +log_to_nodes(Nodes, Fmt) -> + log_to_nodes(Nodes, Fmt, []). + +%% @doc Log a message to the console of the specified test nodes. +%% Messages are prefixed by the string "---riak_test--- " +%% Uses lager:info/2 'LFmt' and 'LArgs' semantics +log_to_nodes(Nodes0, LFmt, LArgs) -> + %% This logs to a node's info level, but if riak_test is running + %% at debug level, we want to know when we send this and what + %% we're saying + Nodes = lists:flatten(Nodes0), + lager:debug("log_to_nodes: " ++ LFmt, LArgs), + Module = lager, + Function = log, + Meta = [], + Args = case LArgs of + [] -> [info, Meta, "---riak_test--- " ++ LFmt]; + _ -> [info, Meta, "---riak_test--- " ++ LFmt, LArgs] + end, + [rpc:call(Node, Module, Function, Args) || Node <- lists:flatten(Nodes)]. + +%% @doc Parallel Map: Runs function F for each item in list L, then +%% returns the list of results +-spec pmap(F :: fun(), L :: list()) -> list(). +pmap(F, L) -> + Parent = self(), + lists:foldl( + fun(X, N) -> + spawn_link(fun() -> + Parent ! {pmap, N, F(X)} + end), + N+1 + end, 0, L), + L2 = [receive {pmap, N, R} -> {N,R} end || _ <- L], + {_, L3} = lists:unzip(lists:keysort(1, L2)), + L3. + +%% @private +setup_harness(Test, Args) -> + rt_harness:setup_harness(Test, Args). + +%% @doc Downloads any extant log files from the harness's running +%% nodes. +get_node_logs(LogFile, DestDir) -> + rt_harness:get_node_logs(LogFile, DestDir). + +get_node_logs() -> + rt_harness:get_node_logs(). + +check_ibrowse() -> + try sys:get_status(ibrowse) of + {status, _Pid, {module, gen_server} ,_} -> ok + catch + Throws -> + lager:error("ibrowse error ~p", [Throws]), + lager:error("Restarting ibrowse"), + application:stop(ibrowse), + application:start(ibrowse) + end. + + +%%%=================================================================== +%%% Bucket Types Functions +%%%=================================================================== + +%% TODO: Determine if this can leverage riak_test_runner:start_lager_backend/2 +%% @doc Set up in memory log capture to check contents in a test. +setup_log_capture(Nodes) when is_list(Nodes) -> + rt:load_modules_on_nodes([riak_test_lager_backend], Nodes), + [?assertEqual({Node, ok}, + {Node, + rpc:call(Node, + gen_event, + add_handler, + [lager_event, + riak_test_lager_backend, + [info, false]])}) || Node <- Nodes], + [?assertEqual({Node, ok}, + {Node, + rpc:call(Node, + lager, + set_loglevel, + [riak_test_lager_backend, + info])}) || Node <- Nodes]; +setup_log_capture(Node) when not is_list(Node) -> + setup_log_capture([Node]). + + +expect_in_log(Node, Pattern) -> + CheckLogFun = fun() -> + Logs = rpc:call(Node, riak_test_lager_backend, get_logs, []), + lager:info("looking for pattern ~s in logs for ~p", + [Pattern, Node]), + case re:run(Logs, Pattern, []) of + {match, _} -> + lager:info("Found match"), + true; + nomatch -> + lager:info("No match"), + false + end + end, + case rt:wait_until(CheckLogFun) of + ok -> + true; + _ -> + false + end. + +%% @doc Wait for Riak Control to start on a single node. +%% +%% Non-optimal check, because we're blocking for the gen_server to start +%% to ensure that the routes have been added by the supervisor. +%% +wait_for_control(_Vsn, Node) when is_atom(Node) -> + lager:info("Waiting for riak_control to start on node ~p.", [Node]), + + %% Wait for the gen_server. + rt:wait_until(Node, fun(N) -> + case rpc:call(N, + riak_control_session, + get_version, + []) of + {ok, _} -> + true; + Error -> + lager:info("Error was ~p.", [Error]), + false + end + end), + + lager:info("Waiting for routes to be added to supervisor..."), + + %% Wait for routes to be added by supervisor. + rt:wait_until(Node, fun(N) -> + case rpc:call(N, + webmachine_router, + get_routes, + []) of + {badrpc, Error} -> + lager:info("Error was ~p.", [Error]), + false; + Routes -> + case is_control_gui_route_loaded(Routes) of + false -> + false; + _ -> + true + end + end + end). + +%% @doc Is the riak_control GUI route loaded? +is_control_gui_route_loaded(Routes) -> + lists:keymember(admin_gui, 2, Routes) orelse lists:keymember(riak_control_wm_gui, 2, Routes). + +%% @doc Wait for Riak Control to start on a series of nodes. +wait_for_control(VersionedNodes) when is_list(VersionedNodes) -> + [wait_for_control(Vsn, Node) || {Vsn, Node} <- VersionedNodes]. + +node_id(Node) -> + rt_harness:node_id(Node). + +node_version(Node) -> + rt_harness:node_version(Node). + +%% TODO: Is this the right location for this? +-ifdef(TEST). + +verify_product(Applications, ExpectedApplication) -> + ?_test(begin + meck:new(rpc, [unstick]), + meck:expect(rpc, call, fun([], application, which_applications, []) -> + Applications end), + ?assertMatch(ExpectedApplication, product([])), + meck:unload(rpc) + end). + +product_test_() -> + {foreach, + fun() -> ok end, + [verify_product([riak_cs], riak_cs), + verify_product([riak_repl, riak_kv, riak_cs], riak_cs), + verify_product([riak_repl], riak_ee), + verify_product([riak_repl, riak_kv], riak_ee), + verify_product([riak_kv], riak), + verify_product([kernel], unknown)]}. + +-endif. diff --git a/src/rt_aae.erl b/src/rt_aae.erl new file mode 100644 index 000000000..170221103 --- /dev/null +++ b/src/rt_aae.erl @@ -0,0 +1,101 @@ +%% ------------------------------------------------------------------- +%% +%% Copyright (c) 2014 Basho Technologies, Inc. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% ------------------------------------------------------------------- + +-module(rt_aae). +-include_lib("eunit/include/eunit.hrl"). + +-export([wait_until_aae_trees_built/1]). + +wait_until_aae_trees_built(Nodes) -> + lager:info("Wait until AAE builds all partition trees across ~p", [Nodes]), + BuiltFun = fun() -> lists:foldl(aae_tree_built_fun(), true, Nodes) end, + ?assertEqual(ok, rt:wait_until(BuiltFun)), + ok. + +aae_tree_built_fun() -> + fun(Node, _AllBuilt = true) -> + case get_aae_tree_info(Node) of + {ok, TreeInfos} -> + case all_trees_have_build_times(TreeInfos) of + true -> + Partitions = [I || {I, _} <- TreeInfos], + all_aae_trees_built(Node, Partitions); + false -> + some_trees_not_built + end; + Err -> + Err + end; + (_Node, Err) -> + Err + end. + +% It is unlikely but possible to get a tree built time from compute_tree_info +% but an attempt to use the tree returns not_built. This is because the build +% process has finished, but the lock on the tree won't be released until it +% dies and the manager detects it. Yes, this is super freaking paranoid. +all_aae_trees_built(Node, Partitions) -> + %% Notice that the process locking is spawned by the + %% pmap. That's important! as it should die eventually + %% so the lock is released and the test can lock the tree. + IndexBuilts = rt:pmap(index_built_fun(Node), Partitions), + BadOnes = [R || R <- IndexBuilts, R /= true], + case BadOnes of + [] -> + true; + _ -> + BadOnes + end. + +get_aae_tree_info(Node) -> + case rpc:call(Node, riak_kv_entropy_info, compute_tree_info, []) of + {badrpc, _} -> + {error, {badrpc, Node}}; + Info -> + lager:debug("Entropy table on node ~p : ~p", [Node, Info]), + {ok, Info} + end. + +all_trees_have_build_times(Info) -> + not lists:keymember(undefined, 2, Info). + +index_built_fun(Node) -> + fun(Idx) -> + case rpc:call(Node, riak_kv_vnode, + hashtree_pid, [Idx]) of + {ok, TreePid} -> + case rpc:call(Node, riak_kv_index_hashtree, + get_lock, [TreePid, for_riak_test]) of + {badrpc, _} -> + {error, {badrpc, Node}}; + TreeLocked when TreeLocked == ok; + TreeLocked == already_locked -> + true; + Err -> + % Either not_built or some unhandled result, + % in which case update this case please! + {error, {index_not_built, Node, Idx, Err}} + end; + {error, _}=Err -> + Err; + {badrpc, _} -> + {error, {badrpc, Node}} + end + end. diff --git a/src/rt_backend.erl b/src/rt_backend.erl new file mode 100644 index 000000000..949f26781 --- /dev/null +++ b/src/rt_backend.erl @@ -0,0 +1,139 @@ +%% ------------------------------------------------------------------- +%% +%% Copyright (c) 2013-2014 Basho Technologies, Inc. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% ------------------------------------------------------------------- +-module(rt_backend). +-include_lib("eunit/include/eunit.hrl"). + +-compile(export_all). +-export([set/2]). + +%%%=================================================================== +%%% Test harness setup, configuration, and internal utilities +%%%=================================================================== + +replace_backend(Backend, false) -> + [{storage_backend, Backend}]; +replace_backend(Backend, {riak_kv, KVSection}) -> + lists:keystore(storage_backend, 1, KVSection, {storage_backend, Backend}). + +%% TODO: Probably should abstract this into the rt_config module and +%% make a sensible API to hide the ugliness of dealing with the lists +%% module funs. +%% +%% TODO: Add support for Riak CS backend and arbitrary multi backend +%% configurations +set(eleveldb, Config) -> + UpdKVSection = replace_backend(riak_kv_eleveldb_backend, + lists:keyfind(riak_kv, 1, Config)), + lists:keystore(riak_kv, 1, Config, {riak_kv, UpdKVSection}); +set(memory, Config) -> + UpdKVSection = replace_backend(riak_kv_memory_backend, + lists:keyfind(riak_kv, 1, Config)), + lists:keystore(riak_kv, 1, Config, {riak_kv, UpdKVSection}); +set(multi, Config) -> + UpdKVSection = + replace_backend(riak_kv_multi_backend, + lists:keyfind(riak_kv, 1, Config)) ++ + multi_backend_config(default), + lists:keystore(riak_kv, 1, Config, {riak_kv, UpdKVSection}); +set({multi, indexmix}, Config) -> + UpdKVSection = + replace_backend(riak_kv_multi_backend, + lists:keyfind(riak_kv, 1, Config)) ++ + multi_backend_config(indexmix), + lists:keystore(riak_kv, 1, Config, {riak_kv, UpdKVSection}); +set(_, Config) -> + UpdKVSection = replace_backend(riak_kv_bitcask_backend, + lists:keyfind(riak_kv, 1, Config)), + lists:keystore(riak_kv, 1, Config, {riak_kv, UpdKVSection}). + +multi_backend_config(default) -> + [{multi_backend_default, <<"eleveldb1">>}, + {multi_backend, [{<<"eleveldb1">>, riak_kv_eleveldb_backend, []}, + {<<"memory1">>, riak_kv_memory_backend, []}, + {<<"bitcask1">>, riak_kv_bitcask_backend, []}]}]; +multi_backend_config(indexmix) -> + [{multi_backend_default, <<"eleveldb1">>}, + {multi_backend, [{<<"eleveldb1">>, riak_kv_eleveldb_backend, []}, + {<<"memory1">>, riak_kv_memory_backend, []}]}]. + +%% @doc Sets the backend of ALL nodes that could be available to riak_test. +%% this is not limited to the nodes under test, but any node that +%% riak_test is able to find. It then queries each available node +%% for it's backend, and returns it if they're all equal. If different +%% nodes have different backends, it returns a list of backends. +%% Currently, there is no way to request multiple backends, so the +%% list return type should be considered an error. +-spec set_backend(atom()) -> atom()|[atom()]. +set_backend(Backend) -> + set_backend(Backend, []). + +-spec set_backend(atom(), [{atom(), term()}]) -> atom()|[atom()]. +set_backend(bitcask, _) -> + set_backend(riak_kv_bitcask_backend); +set_backend(eleveldb, _) -> + set_backend(riak_kv_eleveldb_backend); +set_backend(memory, _) -> + set_backend(riak_kv_memory_backend); +set_backend(multi, Extras) -> + set_backend(riak_kv_multi_backend, Extras); +set_backend(Backend, _) when Backend == riak_kv_bitcask_backend; Backend == riak_kv_eleveldb_backend; Backend == riak_kv_memory_backend -> + lager:info("rt_backend:set_backend(~p)", [Backend]), + rt_config:update_app_config(all, [{riak_kv, [{storage_backend, Backend}]}]), + get_backends(); +set_backend(Backend, Extras) when Backend == riak_kv_multi_backend -> + MultiConfig = proplists:get_value(multi_config, Extras, default), + Config = make_multi_backend_config(MultiConfig), + rt_config:update_app_config(all, [{riak_kv, Config}]), + get_backends(); +set_backend(Other, _) -> + lager:warning("rt_backend:set_backend doesn't recognize ~p as a legit backend, using the default.", [Other]), + get_backends(). + +make_multi_backend_config(default) -> + [{storage_backend, riak_kv_multi_backend}, + {multi_backend_default, <<"eleveldb1">>}, + {multi_backend, [{<<"eleveldb1">>, riak_kv_eleveldb_backend, []}, + {<<"memory1">>, riak_kv_memory_backend, []}, + {<<"bitcask1">>, riak_kv_bitcask_backend, []}]}]; +make_multi_backend_config(indexmix) -> + [{storage_backend, riak_kv_multi_backend}, + {multi_backend_default, <<"eleveldb1">>}, + {multi_backend, [{<<"eleveldb1">>, riak_kv_eleveldb_backend, []}, + {<<"memory1">>, riak_kv_memory_backend, []}]}]; +make_multi_backend_config(Other) -> + lager:warning("rt:set_multi_backend doesn't recognize ~p as legit multi-backend config, using default", [Other]), + make_multi_backend_config(default). + +get_backends() -> + Backends = rt_harness:get_backends(), + case Backends of + [riak_kv_bitcask_backend] -> bitcask; + [riak_kv_eleveldb_backend] -> eleveldb; + [riak_kv_memory_backend] -> memory; + [Other] -> Other; + MoreThanOne -> MoreThanOne + end. + +-spec get_backend([proplists:property()]) -> atom() | error. +get_backend(AppConfigProplist) -> + case kvc:path('riak_kv.storage_backend', AppConfigProplist) of + [] -> error; + Backend -> Backend + end. diff --git a/src/rt_bucket_types.erl b/src/rt_bucket_types.erl new file mode 100644 index 000000000..9d07f8d1d --- /dev/null +++ b/src/rt_bucket_types.erl @@ -0,0 +1,102 @@ +%% ------------------------------------------------------------------- +%% +%% Copyright (c) 2014 Basho Technologies, Inc. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% ------------------------------------------------------------------- + +-module(rt_bucket_types). +-include_lib("eunit/include/eunit.hrl"). + +-export([create_and_wait/3, + create_and_activate_bucket_type/3, + wait_until_bucket_type_visible/2, + wait_until_bucket_type_status/3, + wait_until_bucket_props/3]). + +%% Specify the bucket_types field for the properties record. The list +%% of bucket types may have two forms, a bucket_type or a pair +%% consisting of an integer and a bucket_type. The latter form +%% indicates that a bucket_type should only be applied to the cluster +%% with the given index. The former form is applied to all clusters. +-type bucket_type() :: {binary(), proplists:proplist()}. +-type bucket_types() :: [bucket_type() | {pos_integer(), bucket_type()}]. + +-export_type([bucket_types/0]). + +-spec create_and_wait([node()], binary(), proplists:proplist()) -> ok. +create_and_wait(Nodes, Type, Properties) -> + create_and_activate_bucket_type(hd(Nodes), Type, Properties), + wait_until_bucket_type_status(Type, active, Nodes), + wait_until_bucket_type_visible(Nodes, Type). + +%% @doc create and immediately activate a bucket type +create_and_activate_bucket_type(Node, Type, Props) -> + ok = rpc:call(Node, riak_core_bucket_type, create, [Type, Props]), + wait_until_bucket_type_status(Type, ready, Node), + ok = rpc:call(Node, riak_core_bucket_type, activate, [Type]), + wait_until_bucket_type_status(Type, active, Node). + +wait_until_bucket_type_status(Type, ExpectedStatus, Nodes) when is_list(Nodes) -> + [wait_until_bucket_type_status(Type, ExpectedStatus, Node) || Node <- Nodes]; +wait_until_bucket_type_status(Type, ExpectedStatus, Node) -> + F = fun() -> + ActualStatus = rpc:call(Node, riak_core_bucket_type, status, [Type]), + ExpectedStatus =:= ActualStatus + end, + ?assertEqual(ok, rt:wait_until(F)). + +-spec bucket_type_visible([atom()], binary()|{binary(), binary()}) -> boolean(). +bucket_type_visible(Nodes, Type) -> + MaxTime = rt_config:get(rt_max_receive_wait_time), + IsVisible = fun erlang:is_list/1, + {Res, NodesDown} = rpc:multicall(Nodes, riak_core_bucket_type, get, [Type], MaxTime), + NodesDown == [] andalso lists:all(IsVisible, Res). + +wait_until_bucket_type_visible(Nodes, Type) -> + F = fun() -> bucket_type_visible(Nodes, Type) end, + ?assertEqual(ok, rt:wait_until(F)). + +-spec see_bucket_props([atom()], binary()|{binary(), binary()}, + proplists:proplist()) -> boolean(). +see_bucket_props(Nodes, Bucket, ExpectProps) -> + MaxTime = rt_config:get(rt_max_receive_wait_time), + IsBad = fun({badrpc, _}) -> true; + ({error, _}) -> true; + (Res) when is_list(Res) -> false + end, + HasProps = fun(ResProps) -> + lists:all(fun(P) -> lists:member(P, ResProps) end, + ExpectProps) + end, + case rpc:multicall(Nodes, riak_core_bucket, get_bucket, [Bucket], MaxTime) of + {Res, []} -> + % No nodes down, check no errors + case lists:any(IsBad, Res) of + true -> + false; + false -> + lists:all(HasProps, Res) + end; + {_, _NodesDown} -> + false + end. + +wait_until_bucket_props(Nodes, Bucket, Props) -> + F = fun() -> + see_bucket_props(Nodes, Bucket, Props) + end, + ?assertEqual(ok, rt:wait_until(F)). diff --git a/src/rt_cluster.erl b/src/rt_cluster.erl new file mode 100644 index 000000000..f521ad837 --- /dev/null +++ b/src/rt_cluster.erl @@ -0,0 +1,285 @@ +%% ------------------------------------------------------------------- +%% +%% Copyright (c) 2014 Basho Technologies, Inc. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% ------------------------------------------------------------------- + +-module(rt_cluster). +-include_lib("eunit/include/eunit.hrl"). + +-export([properties/0, + setup/1, + augment_config/3, + clean_cluster/1, + join_cluster/2, + clean_data_dir/1, + clean_data_dir/2, + try_nodes_ready/3, + versions/0, + teardown/0]). + +-export([maybe_wait_for_transfers/3]). + +%% @doc Default properties used if a riak_test module does not specify +%% a custom properties function. +-spec properties() -> rt_properties:properties(). +properties() -> + rt_properties:new(). + +-spec setup(rt_properties:properties()) -> + {ok, rt_properties:properties()} | {error, term()}. +setup(Properties) -> + case form_clusters(Properties) of + {ok, ClusterNodes} -> + maybe_wait_for_transfers(rt_properties:get(node_ids, Properties), + rt_properties:get(node_map, Properties), + rt_properties:get(wait_for_transfers, Properties)), + Clusters = prepare_clusters(ClusterNodes, Properties), + create_bucket_types(Clusters, Properties), + rt_properties:set(clusters, Clusters, Properties); + Error -> + Error + end. + +-spec create_bucket_types([rt_cluster_info:cluster_info()], rt_properties:properties()) -> no_return(). +create_bucket_types(Clusters, Properties) -> + BucketTypes = rt_properties:get(bucket_types, Properties), + create_bucket_types(Clusters, Properties, BucketTypes). + +-spec create_bucket_types([rt_cluster_info:cluster_info()], + rt_properties:properties(), + rt_properties:bucket_types()) -> no_return(). +create_bucket_types(_Clusters, _Properties, []) -> + ok; +create_bucket_types([Cluster], Properties, BucketTypes) -> + NodeMap = rt_properties:get(node_map, Properties), + NodeIds = rt_cluster_info:get(node_ids, Cluster), + Nodes = [rt_node:node_name(NodeId, NodeMap) || NodeId <- NodeIds], + lists:foldl(fun maybe_create_bucket_type/2, {Nodes, 1}, BucketTypes); +create_bucket_types(Clusters, Properties, BucketTypes) -> + NodeMap = rt_properties:get(node_map, Properties), + [begin + NodeIds = rt_cluster_info:get(node_ids, Cluster), + Nodes = [rt_node:node_name(NodeId, NodeMap) || NodeId <- NodeIds], + lists:foldl(fun maybe_create_bucket_type/2, {Nodes, ClusterIndex}, BucketTypes) + end || {Cluster, ClusterIndex} <- lists:zip(Clusters, lists:seq(1, length(Clusters)))]. + +maybe_create_bucket_type({ClusterIndex, {TypeName, TypeProps}}, + {Nodes, ClusterIndex}) -> + rt_bucket_types:create_and_wait(Nodes, TypeName, TypeProps), + {Nodes, ClusterIndex}; +maybe_create_bucket_type({_ApplicableIndex, {_TypeName, _TypeProps}}, + {Nodes, _ClusterIndex}) -> + %% This bucket type does not apply to this cluster + {Nodes, _ClusterIndex}; +maybe_create_bucket_type({TypeName, TypeProps}, {Nodes, _ClusterIndex}) -> + %% This bucket type applies to all clusters + rt_bucket_types:create_and_wait(Nodes, TypeName, TypeProps), + {Nodes, _ClusterIndex}. + +-spec prepare_clusters([list(string())], rt_properties:properties()) -> + [rt_cluster_info:cluster_info()]. +prepare_clusters([ClusterNodes], _Properties) -> + [rt_cluster_info:new([{node_ids, ClusterNodes}])]; +prepare_clusters(ClusterNodesList, Properties) -> + %% If the count of clusters is > 1 the assumption is made that the + %% test is exercising replication and some extra + %% made. This to avoid some noisy and oft-repeated setup + %% boilerplate in every replication test. + NodeMap = rt_properties:get(node_map, Properties), + {Clusters, _, _} = lists:foldl(fun prepare_cluster/2, + {[], 1, NodeMap}, + ClusterNodesList), + lists:reverse(Clusters). + +-type prepare_cluster_acc() :: {[rt_cluster_info:cluster_info()], char(), proplists:proplist()}. +-spec prepare_cluster([string()], prepare_cluster_acc()) -> prepare_cluster_acc(). +prepare_cluster(NodeIds, {Clusters, Name, NodeMap}) -> + Nodes = [rt_node:node_name(NodeId, NodeMap) || NodeId <- NodeIds], + repl_util:name_cluster(hd(Nodes), integer_to_list(Name)), + repl_util:wait_until_leader_converge(Nodes), + Leader = repl_util:get_leader(hd(Nodes)), + Cluster = rt_cluster_info:new([{node_ids, NodeIds}, + {leader, Leader}, + {name, Name}]), + {[Cluster | Clusters], Name+1, NodeMap}. + +-type clusters() :: [rt_cluster_info:cluster_info()]. +-spec form_clusters(rt_properties:properties()) -> clusters(). +form_clusters(Properties) -> + NodeIds = rt_properties:get(node_ids, Properties), + NodeMap = rt_properties:get(node_map, Properties), + ClusterCount = rt_properties:get(cluster_count, Properties), + ClusterWeights = rt_properties:get(cluster_weights, Properties), + MakeCluster = rt_properties:get(make_cluster, Properties), + case divide_nodes(NodeIds, ClusterCount, ClusterWeights) of + {ok, Clusters} -> + maybe_join_clusters(Clusters, NodeMap, MakeCluster), + {ok, Clusters}; + Error -> + Error + end. + +-spec divide_nodes([string()], pos_integer(), [float()]) -> + {ok, [list(string())]} | {error, atom()}. +divide_nodes(Nodes, Count, Weights) + when length(Nodes) < Count; + Weights =/= undefined, length(Weights) =/= Count -> + {error, invalid_cluster_properties}; +divide_nodes(Nodes, 1, _) -> + {ok, [Nodes]}; +divide_nodes(Nodes, Count, Weights) -> + case validate_weights(Weights) of + true -> + TotalNodes = length(Nodes), + NodeCounts = node_counts_from_weights(TotalNodes, Count, Weights), + {_, Clusters, _} = lists:foldl(fun take_nodes/2, {1, [], Nodes}, NodeCounts), + {ok, lists:reverse(Clusters)}; + false -> + {error, invalid_cluster_weights} + end. + +take_nodes(NodeCount, {Index, ClusterAcc, Nodes}) -> + {NewClusterNodes, RestNodes} = lists:split(NodeCount, Nodes), + {Index + 1, [NewClusterNodes | ClusterAcc], RestNodes}. + +validate_weights(undefined) -> + true; +validate_weights(Weights) -> + not lists:sum(Weights) > 1.0 . + +node_counts_from_weights(NodeCount, ClusterCount, undefined) -> + %% Split the nodes evenly. A remainder of nodes is handled by + %% distributing one node per cluster until the remainder is + %% exhausted. + NodesPerCluster = NodeCount div ClusterCount, + Remainder = NodeCount rem ClusterCount, + [NodesPerCluster + remainder_to_apply(Remainder, ClusterIndex) || + ClusterIndex <- lists:seq(1, ClusterCount)]; +node_counts_from_weights(NodeCount, ClusterCount, Weights) -> + InitialNodeCounts = [node_count_from_weight(NodeCount, Weight) || Weight <- Weights], + Remainder = NodeCount - lists:sum(InitialNodeCounts), + [ClusterNodeCount + remainder_to_apply(Remainder, ClusterIndex) || + {ClusterIndex, ClusterNodeCount} + <- lists:zip(lists:seq(1, ClusterCount), InitialNodeCounts)]. + +node_count_from_weight(TotalNodes, Weight) -> + RawNodeCount = TotalNodes * Weight, + IntegerPortion = trunc(RawNodeCount), + Remainder = RawNodeCount - IntegerPortion, + case Remainder >= 0.5 of + true -> + IntegerPortion + 1; + false -> + IntegerPortion + end. + +remainder_to_apply(Remainder, Index) when Remainder > Index; + Remainder =:= 0 -> + 0; +remainder_to_apply(_Remainder, _Index) -> + 1. + +maybe_join_clusters(Clusters, NodeMap, true) -> + [join_cluster(ClusterNodes, NodeMap) || ClusterNodes <- Clusters]; +maybe_join_clusters(_Clusters, _NodeMap, false) -> + ok. + +maybe_wait_for_transfers(NodeIds, NodeMap, true) -> + lager:info("Waiting for transfers"), + rt:wait_until_transfers_complete([rt_node:node_name(NodeId, NodeMap) + || NodeId <- NodeIds]); +maybe_wait_for_transfers(_NodeIds, _NodeMap, false) -> + ok. + +join_cluster(NodeIds, NodeMap) -> + NodeNames = [rt_node:node_name(NodeId, NodeMap) || NodeId <- NodeIds], + %% Ensure each node owns 100% of it's own ring + [?assertEqual([Node], rt_ring:owners_according_to(Node)) || Node <- NodeNames], + + %% Join nodes + [Node1|OtherNodes] = NodeNames, + case OtherNodes of + [] -> + %% no other nodes, nothing to join/plan/commit + + ok; + _ -> + + %% ok do a staged join and then commit it, this eliminates the + %% large amount of redundant handoff done in a sequential join + [rt_node:staged_join(Node, Node1) || Node <- OtherNodes], + rt_node:plan_and_commit(Node1), + try_nodes_ready(NodeNames, 3, 500) + end, + + ?assertEqual(ok, rt_node:wait_until_nodes_ready(NodeNames)), + + %% Ensure each node owns a portion of the ring + rt_node:wait_until_nodes_agree_about_ownership(NodeNames), + ?assertEqual(ok, rt:wait_until_no_pending_changes(NodeNames)), + ok. + +try_nodes_ready([Node1 | _Nodes], 0, _SleepMs) -> + lager:info("Nodes not ready after initial plan/commit, retrying"), + rt_node:plan_and_commit(Node1); +try_nodes_ready(Nodes, N, SleepMs) -> + ReadyNodes = [Node || Node <- Nodes, rt_node:is_ready(Node) =:= true], + case ReadyNodes of + Nodes -> + ok; + _ -> + timer:sleep(SleepMs), + try_nodes_ready(Nodes, N-1, SleepMs) + end. + +%% @doc Stop nodes and wipe out their data directories +clean_cluster(Nodes) when is_list(Nodes) -> + [rt_node:stop_and_wait(Node) || Node <- Nodes], + clean_data_dir(Nodes). + +clean_data_dir(Nodes) -> + clean_data_dir(Nodes, ""). + +clean_data_dir(Nodes, SubDir) when not is_list(Nodes) -> + clean_data_dir([Nodes], SubDir); +clean_data_dir(Nodes, SubDir) when is_list(Nodes) -> + rt_harness:clean_data_dir(Nodes, SubDir). + +%% @doc Shutdown every node, this is for after a test run is complete. +teardown() -> + %% stop all connected nodes, 'cause it'll be faster that + %%lager:info("RPC stopping these nodes ~p", [nodes()]), + %%[ rt_node:stop(Node) || Node <- nodes()], + %% Then do the more exhaustive harness thing, in case something was up + %% but not connected. + rt_harness:teardown(). + +%% TODO: Determine if this is used outside of verify_capabilities +versions() -> + rt_harness:versions(). + +augment_config(Section, Property, Config) -> + UpdSectionConfig = update_section(Section, + Property, + lists:keyfind(Section, 1, Config)), + lists:keyreplace(Section, 1, Config, UpdSectionConfig). + +update_section(Section, Property, false) -> + {Section, [Property]}; +update_section(Section, Property, {Section, SectionConfig}) -> + {Section, [Property | SectionConfig]}. \ No newline at end of file diff --git a/src/rt_cluster_info.erl b/src/rt_cluster_info.erl new file mode 100644 index 000000000..ec1d22346 --- /dev/null +++ b/src/rt_cluster_info.erl @@ -0,0 +1,166 @@ +-module(rt_cluster_info). +%% ------------------------------------------------------------------- +%% +%% Copyright (c) 2013 Basho Technologies, Inc. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% ------------------------------------------------------------------- + +%% @doc Implements a set of functions for accessing and manipulating +%% an `rt_cluster_info' record. + +-record(rt_cluster_info_v1, { + node_ids :: [string()], + leader :: string(), + name :: string() + }). +-type cluster_info() :: #rt_cluster_info_v1{}. + +-export_type([cluster_info/0]). + +-define(RT_CLUSTER_INFO, #rt_cluster_info_v1). +-define(RECORD_FIELDS, record_info(fields, rt_cluster_info_v1)). + +-export([new/0, + new/1, + get/2, + set/2, + set/3]). + +%% @doc Create a new cluster_info record with all fields initialized to +%% the default values. +-spec new() -> cluster_info(). +new() -> + ?RT_CLUSTER_INFO{}. + +%% @doc Create a new cluster_info record with the fields initialized to +%% non-default value. Each field to be initialized should be +%% specified as an entry in a property list (i.e. a list of +%% pairs). Invalid fields are ignored by this function. +-spec new(proplists:proplist()) -> cluster_info(). +new(Defaults) -> + {ClusterInfo, _} = + lists:foldl(fun set_field/2, {?RT_CLUSTER_INFO{}, []}, Defaults), + ClusterInfo. + +%% @doc Get the value of a field from a cluster_info record. An error +%% is returned if `ClusterInfo' is not a valid `rt_cluster_info' record +%% or if the property requested is not a valid property. +-spec get(atom(), cluster_info()) -> term() | {error, atom()}. +get(Field, ClusterInfo) -> + get(Field, ClusterInfo, validate_request(Field, ClusterInfo)). + +%% @doc Set the value for a field in a cluster_info record. An error +%% is returned if `ClusterInfo' is not a valid `rt_cluster_info' record +%% or if any of the fields to be set are not valid. In +%% the case that invalid fields are specified the error returned +%% contains a list of erroneous fields. +-spec set([{atom(), term()}], cluster_info()) -> cluster_info() | {error, atom()}. +set(FieldList, ClusterInfo) when is_list(FieldList) -> + set_fields(FieldList, ClusterInfo, validate_record(ClusterInfo)). + +%% @doc Set the value for a field in a cluster_info record. An error +%% is returned if `ClusterInfo' is not a valid `rt_cluster_info' record +%% or if the field to be set is not valid. +-spec set(atom(), term(), cluster_info()) -> {ok, cluster_info()} | {error, atom()}. +set(Field, Value, ClusterInfo) -> + set_field(Field, Value, ClusterInfo, validate_request(Field, ClusterInfo)). + +-spec get(atom(), cluster_info(), ok | {error, atom()}) -> + term() | {error, atom()}. +get(Field, ClusterInfo, ok) -> + element(field_index(Field), ClusterInfo); +get(_Field, _ClusterInfo, {error, _}=Error) -> + Error. + +%% This function is used by `new/1' to set fields at record +%% creation time and by `set/2' to set multiple properties at once. +%% No cluster_info record validation is done by this function. It is +%% strictly used as a fold function which is the reason for the odd +%% structure of the input parameters. It accumulates any invalid +%% fields that are encountered and the caller may use that +%% information or ignore it. +-spec set_field({atom(), term()}, {cluster_info(), [atom()]}) -> + {cluster_info(), [atom()]}. +set_field({Field, Value}, {ClusterInfo, Invalid}) -> + case is_valid_field(Field) of + true -> + {setelement(field_index(Field), ClusterInfo, Value), Invalid}; + false -> + {ClusterInfo, [Field | Invalid]} + end. + +-spec set_field(atom(), term(), cluster_info(), ok | {error, atom()}) -> + {ok, cluster_info()} | {error, atom()}. +set_field(Field, Value, ClusterInfo, ok) -> + {ok, setelement(field_index(Field), ClusterInfo, Value)}; +set_field(_Field, _Value, _ClusterInfo, {error, _}=Error) -> + Error. + +-spec set_fields([{atom(), term()}], + cluster_info(), + ok | {error, {atom(), [atom()]}}) -> + {cluster_info(), [atom()]}. +set_fields(FieldList, ClusterInfo, ok) -> + case lists:foldl(fun set_field/2, {ClusterInfo, []}, FieldList) of + {UpdClusterInfo, []} -> + UpdClusterInfo; + {_, InvalidClusterInfo} -> + {error, {invalid_properties, InvalidClusterInfo}} + end; +set_fields(_, _, {error, _}=Error) -> + Error. + +-spec validate_request(atom(), term()) -> ok | {error, atom()}. +validate_request(Field, ClusterInfo) -> + validate_field(Field, validate_record(ClusterInfo)). + +-spec validate_record(term()) -> ok | {error, invalid_cluster_info}. +validate_record(Record) -> + case is_valid_record(Record) of + true -> + ok; + false -> + {error, invalid_cluster_info} + end. + +-spec validate_field(atom(), ok | {error, atom()}) -> ok | {error, invalid_field}. +validate_field(Field, ok) -> + case is_valid_field(Field) of + true -> + ok; + false -> + {error, invalid_field} + end; +validate_field(_Field, {error, _}=Error) -> + Error. + +-spec is_valid_record(term()) -> boolean(). +is_valid_record(Record) -> + is_record(Record, rt_cluster_info_v1). + +-spec is_valid_field(atom()) -> boolean(). +is_valid_field(Field) -> + Fields = ?RECORD_FIELDS, + lists:member(Field, Fields). + +-spec field_index(atom()) -> non_neg_integer(). +field_index(node_ids) -> + ?RT_CLUSTER_INFO.node_ids; +field_index(leader) -> + ?RT_CLUSTER_INFO.leader; +field_index(name) -> + ?RT_CLUSTER_INFO.name. diff --git a/src/rt_cmd_line.erl b/src/rt_cmd_line.erl new file mode 100644 index 000000000..ec83c3ff9 --- /dev/null +++ b/src/rt_cmd_line.erl @@ -0,0 +1,83 @@ +%% ------------------------------------------------------------------- +%% +%% Copyright (c) 2014 Basho Technologies, Inc. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% ------------------------------------------------------------------- + +-module(rt_cmd_line). +-include_lib("eunit/include/eunit.hrl"). + +-export([admin/2, + admin/3, + riak/2, + riak_repl/2, + search_cmd/2, + attach/2, + attach_direct/2, + console/2 + ]). + +-define(HARNESS, (rt_config:get(rt_harness))). + +%% @doc Call 'bin/riak-admin' command on `Node' with arguments `Args' +admin(Node, Args) -> + rt_harness:admin(Node, Args). + +%% @doc Call 'bin/riak-admin' command on `Node' with arguments `Args'. +%% The third parameter is a list of options. Valid options are: +%% * `return_exit_code' - Return the exit code along with the command output +admin(Node, Args, Options) -> + rt_harness:admin(Node, Args, Options). + +%% @doc Call 'bin/riak' command on `Node' with arguments `Args' +riak(Node, Args) -> + rt_harness:riak(Node, Args). + + +%% @doc Call 'bin/riak-repl' command on `Node' with arguments `Args' +riak_repl(Node, Args) -> + rt_harness:riak_repl(Node, Args). + +search_cmd(Node, Args) -> + {ok, Cwd} = file:get_cwd(), + rpc:call(Node, riak_search_cmd, command, [[Cwd | Args]]). + +%% @doc Runs `riak attach' on a specific node, and tests for the expected behavoir. +%% Here's an example: ``` +%% rt_cmd_line:attach(Node, [{expect, "erlang.pipe.1 \(^D to exit\)"}, +%% {send, "riak_core_ring_manager:get_my_ring()."}, +%% {expect, "dict,"}, +%% {send, [4]}]), %% 4 = Ctrl + D''' +%% `{expect, String}' scans the output for the existance of the String. +%% These tuples are processed in order. +%% +%% `{send, String}' sends the string to the console. +%% Once a send is encountered, the buffer is discarded, and the next +%% expect will process based on the output following the sent data. +%% +attach(Node, Expected) -> + rt_harness:attach(Node, Expected). + +%% @doc Runs 'riak attach-direct' on a specific node +%% @see rt_cmd_line:attach/2 +attach_direct(Node, Expected) -> + rt_harness:attach_direct(Node, Expected). + +%% @doc Runs `riak console' on a specific node +%% @see rt_cmd_line:attach/2 +console(Node, Expected) -> + rt_harness:console(Node, Expected). diff --git a/src/rt_config.erl b/src/rt_config.erl index 4a916de63..95b531632 100644 --- a/src/rt_config.erl +++ b/src/rt_config.erl @@ -25,12 +25,43 @@ get/2, config_or_os_env/1, config_or_os_env/2, + convert_to_string/1, + get_default_version/0, + get_default_version_product/0, + get_default_version_number/0, + get_previous_version/0, + get_legacy_version/0, get_os_env/1, get_os_env/2, + get_upgrade_path/1, + get_version/1, load/2, - set/2 + set/2, + set_conf/2, + set_advanced_conf/2, + update_app_config/2, + version_to_tag/1 ]). +-define(HARNESS, (rt_config:get(rt_harness))). +-define(CONFIG_NAMESPACE, riak_test). +-define(RECEIVE_WAIT_TIME_KEY, rt_max_receive_wait_time). +-define(GIDDYUP_PLATFORM_KEY, giddyup_platform). +-define(VERSION_KEY, versions). +-define(DEFAULT_VERSION_KEY, default). +-define(PREVIOUS_VERSION_KEY, previous). +-define(LEGACY_VERSION_KEY, legacy). +-define(DEFAULT_VERSION, head). +-define(DEFAULT_ACTUAL_VERSION, {riak_ee, "2.1.0"}). +-define(DEFAULT_ACTUAL_VERSION_TAG, "riak_ee-2.1.0"). +-define(UPGRADE_KEY, upgrade_paths). +-define(PREVIOUS_VERSION, {riak_ee, "1.4.12"}). +-define(PREVIOUS_VERSION_TAG, "riak_ee-1.4.12"). +-define(LEGACY_VERSION, {riak_ee, "1.3.4"}). +-define(LEGACY_VERSION_TAG, "riak_ee-1.3.4"). +-define(CONTINUE_ON_FAIL_KEY, continue_on_fail). +-define(DEFAULT_CONTINUE_ON_FAIL, false). + %% @doc Get the value of an OS Environment variable. The arity 1 version of %% this function will fail the test if it is undefined. get_os_env(Var) -> @@ -69,29 +100,119 @@ load_dot_config(ConfigName, ConfigFile) -> %% Now, overlay the specific project Config = proplists:get_value(list_to_atom(ConfigName), Terms), [set(Key, Value) || {Key, Value} <- Config], - ok; + %% Validate all versions and upgrade paths + Versions=rt_config:get(?VERSION_KEY), + Upgrades=rt_config:get(?UPGRADE_KEY), + RealVersions = [get_version(Name) || {Name, Vsn} <- Versions, is_tuple(Vsn)], + RealUpgrades = lists:merge([get_upgrade_path(Name) || {Name, Upg} <- Upgrades, Upg =/= []]), + rt_harness:validate_config(lists:usort(RealVersions ++ RealUpgrades)); {error, Reason} -> erlang:error("Failed to parse config file", [ConfigFile, Reason]) - end. + end. set(Key, Value) -> ok = application:set_env(riak_test, Key, Value). + +-spec get(rt_max_wait_time | atom()) -> any(). +get(rt_max_wait_time) -> + lager:info("rt_max_wait_time is deprecated. Please use rt_max_receive_wait_time instead."), + rt_config:get(?RECEIVE_WAIT_TIME_KEY); +get(platform) -> + lager:info("platform is deprecated. Please use giddyup_platform instead."), + rt_config:get(?GIDDYUP_PLATFORM_KEY); +get(?CONTINUE_ON_FAIL_KEY) -> + get(?CONTINUE_ON_FAIL_KEY, ?DEFAULT_CONTINUE_ON_FAIL); get(Key) -> - case kvc:path(Key, application:get_all_env(riak_test)) of + case kvc:path(Key, application:get_all_env(?CONFIG_NAMESPACE)) of [] -> lager:warning("Missing configuration key: ~p", [Key]), erlang:error("Missing configuration key", [Key]); - Value -> - Value + Value -> Value end. +-spec get(rt_max_wait_time | atom(), any()) -> any(). +get(rt_max_wait_time, Default) -> + lager:info("rt_max_wait_time is deprecated. Please use rt_max_receive_wait_time instead."), + get(?RECEIVE_WAIT_TIME_KEY, Default); get(Key, Default) -> - case kvc:path(Key, application:get_all_env(riak_test)) of + case kvc:path(Key, application:get_all_env(?CONFIG_NAMESPACE)) of [] -> Default; Value -> Value end. +%% @doc Return the default version +-spec get_default_version() -> string(). +get_default_version() -> + get_version(?DEFAULT_VERSION_KEY). + +%% @doc Return the default product from the version +-spec get_default_version_product() -> string(). +get_default_version_product() -> + DefaultVersion = get_version(?DEFAULT_VERSION_KEY), + string:sub_word(DefaultVersion, 1, $-). + +%% @doc Return the default version number +-spec get_default_version_number() -> string(). +get_default_version_number() -> + DefaultVersion = get_version(?DEFAULT_VERSION_KEY), + string:sub_word(DefaultVersion, 2, $-). + +%% @doc Return the default version +-spec get_previous_version() -> string(). +get_previous_version() -> + get_version(?PREVIOUS_VERSION_KEY). + +%% @doc Return the default version +-spec get_legacy_version() -> string(). +get_legacy_version() -> + get_version(?LEGACY_VERSION_KEY). + +%% @doc Looks up the version in the list of aliases (e.g. riak_ee-latest-2.0) +%% and actual version names (riak_ee-2.0.5) and return the actual +%% name - string, e.g. "riak_ee-1.4.12" +-spec get_version(term()) -> string() | not_found. +get_version(Vsn) -> + Aliases = rt_config:get(?VERSION_KEY), + RealNames = [{convert_to_string(Product) ++ "-" ++ convert_to_string(Tag), {Product, Tag}} || {_, {Product, Tag}} <- Aliases], + resolve_version(Vsn, lists:append(Aliases, RealNames)). + +%% @doc Map logical name of version into a pathname string +-spec resolve_version(term(), [{term(), term()}]) -> string() | no_return(). +resolve_version(Vsn, Versions) -> + case find_atom_or_string(Vsn, Versions) of + undefined -> + erlang:error("Could not find Riak version", [Vsn]); + {Product, Tag} -> + convert_to_string(Product) ++ "-" ++ convert_to_string(Tag); + Version -> + resolve_version(Version, Versions) + end. + +%% @doc Look up values by both atom and by string +find_atom_or_string(Key, Table) -> + case {Key, proplists:get_value(Key, Table)} of + {_, undefined} when is_atom(Key) -> + proplists:get_value(atom_to_list(Key), Table); + {_, undefined} when is_list(Key) -> + proplists:get_value(list_to_atom(Key), Table); + {Key, Value} -> + Value + end. + +%% @doc Look up a named upgrade path and return the resolved list of versions +-spec get_upgrade_path(term()) -> list() | not_found. +get_upgrade_path(Upg) -> + Upgrades = rt_config:get(?UPGRADE_KEY), + case proplists:get_value(Upg, Upgrades) of + undefined -> + erlang:error("Could not find upgrade path version", [Upg]); + UpgradePath when is_list(UpgradePath) -> + [get_version(Vsn) || Vsn <- UpgradePath]; + _ -> + erlang:error("Upgrade path has an invalid definition", [Upg]) + end. + -spec config_or_os_env(atom()) -> term(). config_or_os_env(Config) -> OSEnvVar = to_upper(atom_to_list(Config)), @@ -122,6 +243,103 @@ config_or_os_env(Config, Default) -> V end. + +-spec set_conf(atom(), [{string(), string()}]) -> ok. +set_conf(all, NameValuePairs) -> + ?HARNESS:set_conf(all, NameValuePairs); +set_conf(Node, NameValuePairs) -> + rt:stop(Node), + ?assertEqual(ok, rt:wait_until_unpingable(Node)), + ?HARNESS:set_conf(Node, NameValuePairs), + rt:start(Node). + +-spec set_advanced_conf(atom(), [{string(), string()}]) -> ok. +set_advanced_conf(all, NameValuePairs) -> + ?HARNESS:set_advanced_conf(all, NameValuePairs); +set_advanced_conf(Node, NameValuePairs) -> + rt:stop(Node), + ?assertEqual(ok, rt:wait_until_unpingable(Node)), + ?HARNESS:set_advanced_conf(Node, NameValuePairs), + rt:start(Node). + +%% @doc Rewrite the given node's app.config file, overriding the varialbes +%% in the existing app.config with those in `Config'. +update_app_config(all, Config) -> + ?HARNESS:update_app_config(all, Config); +update_app_config(Node, Config) -> + rt:stop(Node), + ?assertEqual(ok, rt:wait_until_unpingable(Node)), + ?HARNESS:update_app_config(Node, Config), + rt:start(Node). + to_upper(S) -> lists:map(fun char_to_upper/1, S). char_to_upper(C) when C >= $a, C =< $z -> C bxor $\s; char_to_upper(C) -> C. + +%% TODO: Remove after conversion +%% @doc Look up the version by name from the config file +-spec version_to_tag(atom()) -> string(). +version_to_tag(Version) -> + case Version of + default -> rt_config:get_default_version(); + current -> rt_config:get_default_version(); + legacy -> rt_config:get_legacy_version(); + previous -> rt_config:get_previous_version(); + _ -> rt_config:get_version(Version) + end. + +%% @doc: Convert an atom to a string if it is not already +-spec convert_to_string(string()|atom()) -> string(). +convert_to_string(Val) when is_atom(Val) -> + atom_to_list(Val); +convert_to_string(Val) when is_list(Val) -> + Val. + +-ifdef(TEST). + +clear(Key) -> + application:unset_env(?CONFIG_NAMESPACE, Key). + +get_rt_max_wait_time_test() -> + clear(?RECEIVE_WAIT_TIME_KEY), + + ExpectedWaitTime = 10987, + ok = set(?RECEIVE_WAIT_TIME_KEY, ExpectedWaitTime), + ?assertEqual(ExpectedWaitTime, rt_config:get(?RECEIVE_WAIT_TIME_KEY)), + ?assertEqual(ExpectedWaitTime, rt_config:get(rt_max_wait_time)). + +get_rt_max_wait_time_default_test() -> + clear(?RECEIVE_WAIT_TIME_KEY), + + DefaultWaitTime = 20564, + ?assertEqual(DefaultWaitTime, get(?RECEIVE_WAIT_TIME_KEY, DefaultWaitTime)), + ?assertEqual(DefaultWaitTime, get(rt_max_wait_time, DefaultWaitTime)), + + ExpectedWaitTime = 30421, + ok = set(?RECEIVE_WAIT_TIME_KEY, ExpectedWaitTime), + ?assertEqual(ExpectedWaitTime, get(?RECEIVE_WAIT_TIME_KEY, DefaultWaitTime)), + ?assertEqual(ExpectedWaitTime, get(rt_max_wait_time, DefaultWaitTime)). + +get_version_path_test() -> + Versions = [{?DEFAULT_VERSION_KEY, ?DEFAULT_VERSION}, + {?DEFAULT_VERSION, ?DEFAULT_ACTUAL_VERSION}, + {?PREVIOUS_VERSION_KEY, ?PREVIOUS_VERSION}, + {?LEGACY_VERSION_KEY, ?LEGACY_VERSION}], + ok = rt_config:set(?VERSION_KEY, Versions), + + ?assertEqual(version_to_tag(?DEFAULT_VERSION_KEY), ?DEFAULT_ACTUAL_VERSION_TAG), + ?assertEqual(version_to_tag(?PREVIOUS_VERSION_KEY), ?PREVIOUS_VERSION_TAG), + ?assertEqual(version_to_tag(?LEGACY_VERSION_KEY), ?LEGACY_VERSION_TAG). + +get_continue_on_fail_test() -> + clear(?CONTINUE_ON_FAIL_KEY), + ?assertEqual(?DEFAULT_CONTINUE_ON_FAIL, rt_config:get(?CONTINUE_ON_FAIL_KEY)), + + set(?CONTINUE_ON_FAIL_KEY, false), + ?assertEqual(false, rt_config:get(?CONTINUE_ON_FAIL_KEY)), + + clear(?CONTINUE_ON_FAIL_KEY), + set(?CONTINUE_ON_FAIL_KEY, true), + ?assertEqual(true, rt_config:get(?CONTINUE_ON_FAIL_KEY)). + +-endif. diff --git a/src/rt_cover.erl b/src/rt_cover.erl index 5a970039f..dc7ef9545 100644 --- a/src/rt_cover.erl +++ b/src/rt_cover.erl @@ -118,7 +118,7 @@ start2(CoverMods) -> %% These are read, per test, from the test module attributes %% `cover_modules' or `cover_apps'. find_cover_modules(Test) -> - {Mod, _Fun} = riak_test_runner:function_name(Test), + {Mod, _Fun} = riak_test_runner:function_name(confirm, Test), case proplists:get_value(cover_modules, Mod:module_info(attributes), []) of [] -> case proplists:get_value(cover_apps, Mod:module_info(attributes), []) of @@ -176,8 +176,8 @@ find_app_modules(CoverApps) -> %% so only current will do. maybe_start_on_node(Node, Version) -> IsCurrent = case Version of - current -> true; - {current, _} -> true; + head -> true; + {head, _} -> true; _ -> false end, ShouldStart = IsCurrent andalso diff --git a/src/rt_cs_dev.erl b/src/rt_cs_dev.erl index 10e34b344..97e590a65 100644 --- a/src/rt_cs_dev.erl +++ b/src/rt_cs_dev.erl @@ -20,7 +20,42 @@ %% @private -module(rt_cs_dev). --compile(export_all). +-behaviour(test_harness). + +-export([start/1, + stop/1, + deploy_clusters/1, + clean_data_dir/2, + create_dirs/1, + spawn_cmd/1, + spawn_cmd/2, + cmd/1, + cmd/2, + setup_harness/2, + get_version/0, + get_backends/0, + get_deps/0, + set_backend/1, + whats_up/0, + get_ip/1, + node_id/1, + node_version/1, + admin/2, + admin/3, + riak/2, + attach/2, + attach_direct/2, + console/2, + update_app_config/2, + teardown/0, + set_conf/2, + set_advanced_conf/2, + upgrade/2, + deploy_nodes/1, + versions/0, + get_node_logs/0, + get_node_logs/1]). + -include_lib("eunit/include/eunit.hrl"). -define(DEVS(N), lists:concat(["dev", N, "@127.0.0.1"])). @@ -29,7 +64,13 @@ -define(SRC_PATHS, (rt_config:get(src_paths))). get_deps() -> - lists:flatten(io_lib:format("~s/dev/dev1/lib", [relpath(current)])). + lists:flatten(io_lib:format("~s/dev1/lib", [relpath(rt_config:get_default_version())])). + +deploy_clusters(ClusterConfig) -> + rt_harness_util:deploy_clusters(ClusterConfig). + +get_ip(Node) -> + rt_harness_util:get_ip(Node). setup_harness(_Test, _Args) -> confirm_build_type(rt_config:get(build_type, oss)), @@ -76,10 +117,6 @@ relpath(Vsn) -> Path = ?BUILD_PATHS, path(Vsn, Path). -srcpath(Vsn) -> - Path = ?SRC_PATHS, - path(Vsn, Path). - path(Vsn, Paths=[{_,_}|_]) -> orddict:fetch(Vsn, orddict:from_list(Paths)); path(current, Path) -> @@ -99,11 +136,11 @@ upgrade(Node, NewVersion) -> NewPath = relpath(NewVersion), Commands = [ - io_lib:format("cp -p -P -R \"~s/dev/dev~b/data\" \"~s/dev/dev~b\"", + io_lib:format("cp -p -P -R \"~s/dev~b/data\" \"~s/dev~b\"", [OldPath, N, NewPath, N]), - io_lib:format("rm -rf ~s/dev/dev~b/data/*", + io_lib:format("rm -rf ~s/dev~b/data/*", [OldPath, N]), - io_lib:format("cp -p -P -R \"~s/dev/dev~b/etc\" \"~s/dev/dev~b\"", + io_lib:format("cp -p -P -R \"~s/dev~b/etc\" \"~s/dev~b\"", [OldPath, N, NewPath, N]) ], [ begin @@ -120,7 +157,7 @@ all_the_app_configs(DevPath) -> lager:error("The dev path is ~p", [DevPath]), case filelib:is_dir(DevPath) of true -> - Devs = filelib:wildcard(DevPath ++ "/dev/dev*"), + Devs = filelib:wildcard(DevPath ++ "/dev*"), [ Dev ++ "/etc/app.config" || Dev <- Devs]; _ -> lager:debug("~s is not a directory.", [DevPath]), @@ -133,7 +170,7 @@ update_app_config(all, Config) -> update_app_config(Node, Config) when is_atom(Node) -> N = node_id(Node), Path = relpath(node_version(N)), - FileFormatString = "~s/dev/dev~b/etc/~s.config", + FileFormatString = "~s/dev~b/etc/~s.config", AppConfigFile = io_lib:format(FileFormatString, [Path, N, "app"]), AdvConfigFile = io_lib:format(FileFormatString, [Path, N, "advanced"]), @@ -178,7 +215,7 @@ get_backends() -> node_path(Node) -> N = node_id(Node), Path = relpath(node_version(N)), - lists:flatten(io_lib:format("~s/dev/dev~b", [Path, N])). + lists:flatten(io_lib:format("~s/dev~b", [Path, N])). create_dirs(Nodes) -> Snmp = [node_path(Node) ++ "/data/snmp/agent/db" || Node <- Nodes], @@ -193,19 +230,6 @@ rm_dir(Dir) -> ?assertCmd("rm -rf " ++ Dir), ?assertEqual(false, filelib:is_dir(Dir)). -add_default_node_config(Nodes) -> - case rt_config:get(rt_default_config, undefined) of - undefined -> ok; - Defaults when is_list(Defaults) -> - rt:pmap(fun(Node) -> - update_app_config(Node, Defaults) - end, Nodes), - ok; - BadValue -> - lager:error("Invalid value for rt_default_config : ~p", [BadValue]), - throw({invalid_config, {rt_default_config, BadValue}}) - end. - deploy_nodes(NodeConfig) -> Path = relpath(root), lager:info("Riak path: ~p", [Path]), @@ -217,14 +241,14 @@ deploy_nodes(NodeConfig) -> VersionMap = lists:zip(NodesN, Versions), %% Check that you have the right versions available - [ check_node(Version) || Version <- VersionMap ], + [ rt_harness_util:check_node(Version) || Version <- VersionMap ], rt_config:set(rt_nodes, NodeMap), rt_config:set(rt_versions, VersionMap), create_dirs(Nodes), %% Set initial config - add_default_node_config(Nodes), + rt_harness_util:add_default_node_config(Nodes), rt:pmap(fun({_, default}) -> ok; ({Node, Config}) -> @@ -249,7 +273,7 @@ deploy_nodes(NodeConfig) -> [ok = rt:wait_until_registered(N, riak_core_ring_manager) || N <- Nodes], %% Ensure nodes are singleton clusters - [ok = rt:check_singleton_node(?DEV(N)) || {N, Version} <- VersionMap, + [ok = rt_ring:check_singleton_node(?DEV(N)) || {N, Version} <- VersionMap, Version /= "0.14.2"], lager:info("Deployed nodes: ~p", [Nodes]), @@ -379,30 +403,34 @@ interactive_loop(Port, Expected) -> %% We've met every expectation. Yay! If not, it means we've exited before %% something expected happened. ?assertEqual([], Expected) - after rt_config:get(rt_max_wait_time) -> + after rt_config:get(rt_max_receive_wait_time) -> %% interactive_loop is going to wait until it matches expected behavior %% If it doesn't, the test should fail; however, without a timeout it %% will just hang forever in search of expected behavior. See also: Parenting ?assertEqual([], Expected) end. +%% TODO is the correct implementation for admin/2 -- added to pass compilation by jsb +admin(Node, Args) -> + admin(Node, Args, []). + admin(Node, Args, Options) -> - N = node_id(Node), - Path = relpath(node_version(N)), - Cmd = rtdev:riak_admin_cmd(Path, N, Args), - lager:info("Running: ~s", [Cmd]), - Result = execute_admin_cmd(Cmd, Options), - lager:info("~s", [Result]), - {ok, Result}. + N = node_id(Node), + Path = relpath(node_version(N)), + Cmd = rtdev:riak_admin_cmd(Path, N, Args), + lager:info("Running: ~s", [Cmd]), + Result = execute_admin_cmd(Cmd, Options), + lager:info("~s", [Result]), + {ok, Result}. execute_admin_cmd(Cmd, Options) -> - {_ExitCode, Result} = FullResult = wait_for_cmd(spawn_cmd(Cmd)), - case lists:member(return_exit_code, Options) of - true -> - FullResult; - false -> - Result - end. + {_ExitCode, Result} = FullResult = wait_for_cmd(spawn_cmd(Cmd)), + case lists:member(return_exit_code, Options) of + true -> + FullResult; + false -> + Result + end. riak(Node, Args) -> N = node_id(Node), @@ -459,14 +487,6 @@ get_cmd_result(Port, Acc) -> timeout end. -check_node({_N, Version}) -> - case proplists:is_defined(Version, rt_config:get(build_paths)) of - true -> ok; - _ -> - lager:error("You don't have Riak ~s installed or configured", [Version]), - erlang:error("You don't have Riak " ++ atom_to_list(Version) ++ " installed or configured") - end. - set_backend(Backend) -> lager:info("rtdev:set_backend(~p)", [Backend]), update_app_config(all, [{riak_kv, [{storage_backend, Backend}]}]), @@ -508,4 +528,10 @@ get_node_logs(Base) -> [ begin {ok, Port} = file:open(Filename, [read, binary]), {lists:nthtail(RootLen, Filename), Port} - end || Filename <- filelib:wildcard(Root ++ "/*/dev/dev*/log/*") ]. + end || Filename <- filelib:wildcard(Root ++ "/*/dev*/log/*") ]. + +set_advanced_conf(Node, NameValuePairs) -> + rt_harness_util:set_advanced_conf(Node, NameValuePairs). + +set_conf(Node, NameValuePairs) -> + rt_harness_util:set_conf(Node, NameValuePairs). diff --git a/src/rt_driver.erl b/src/rt_driver.erl new file mode 100644 index 000000000..859784b4d --- /dev/null +++ b/src/rt_driver.erl @@ -0,0 +1,39 @@ +-module(rt_driver). + +-define(DRIVER_MODULE, (rt_config:get(driver))). + +-export([behaviour_info/1, + cluster_module/0, + new_configuration/0, + new_configuration/1, + get_configuration_key/2, + set_configuration_key/3]). + +-type configuration() :: term(). +-exporttype([configuration/0]). + +behaviour_info(callbacks) -> + [{new_configuration, 0}, {new_configuration, 1}, {get_configuration_key, 2}, + {set_configuration_key, 3}, {cluster_module, 0}]; +behaviour_info(_) -> + undefined. + +-spec cluster_module() -> module(). +cluster_module() -> + ?DRIVER_MODULE:cluster_module(). + +-spec new_configuration() -> configuration(). +new_configuration() -> + ?DRIVER_MODULE:new_configuration(). + +-spec new_configuration(atom()) -> configuration(). +new_configuration(Profile) -> + ?DRIVER_MODULE:new_configuration(Profile). + +-spec get_configuration_key(atom(), configuration()) -> term(). +get_configuration_key(Key, Configuration) -> + ?DRIVER_MODULE:get_configuration_key(Configuration, Key). + +-spec set_configuration_key(atom(), term(), configuration()) -> {ok, configuration()} | {error, string()}. +set_configuration_key(Configuration, Key, Value) -> + ?DRIVER_MODULE:set_configuration_key(Configuration, Key, Value). diff --git a/src/rt_harness.erl b/src/rt_harness.erl new file mode 100644 index 000000000..d0d79bc0f --- /dev/null +++ b/src/rt_harness.erl @@ -0,0 +1,175 @@ +%% ------------------------------------------------------------------- +%% +%% Copyright (c) 2013-2014 Basho Technologies, Inc. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% ------------------------------------------------------------------- +%% @doc rt_harness provides a level of indirection between the modules +%% calling into the harness and the configured harness, resolving the call +%% to the configured harness. Calls such as `rt_harness:start(Node)' will +%% be resolved to the configured harness. +-module(rt_harness). + +-define(HARNESS_MODULE, (rt_config:get(rt_harness))). + +-export([start/2, + stop/2, + deploy_clusters/1, + clean_data_dir/3, + deploy_nodes/1, + available_resources/0, + spawn_cmd/1, + spawn_cmd/2, + cmd/1, + cmd/2, + setup/0, + get_deps/0, + get_node_logs/0, + get_node_logs/2, + get_version/0, + get_version/1, + get_backends/0, + set_backend/1, + whats_up/0, + get_ip/1, + node_id/1, + node_version/1, + admin/2, + admin/3, + riak/2, + riak_repl/2, + run_riak/3, + attach/2, + attach_direct/2, + console/2, + update_app_config/3, + teardown/0, + set_conf/2, + set_advanced_conf/2, + update_app_config/2, + upgrade/2, + validate_config/1]). + +start(Node, Version) -> + ?HARNESS_MODULE:start(Node, Version). + +stop(Node, Version) -> + ?HARNESS_MODULE:stop(Node, Version). + +deploy_clusters(ClusterConfigs) -> + ?HARNESS_MODULE:deploy_clusters(ClusterConfigs). + +clean_data_dir(Node, Version, SubDir) -> + ?HARNESS_MODULE:clean_data_dir(Node, Version, SubDir). + +spawn_cmd(Cmd) -> + ?HARNESS_MODULE:spawn_cmd(Cmd). + +spawn_cmd(Cmd, Opts) -> + ?HARNESS_MODULE:spawn_cmd(Cmd, Opts). + +cmd(Cmd) -> + ?HARNESS_MODULE:cmd(Cmd). + +cmd(Cmd, Opts) -> + ?HARNESS_MODULE:cmd(Cmd, Opts). + +deploy_nodes(NodeConfig) -> + ?HARNESS_MODULE:deploy_nodes(NodeConfig). + +available_resources() -> + ?HARNESS_MODULE:available_resources(). + +setup() -> + ?HARNESS_MODULE:setup_harness(). + +get_deps() -> + ?HARNESS_MODULE:get_deps(). + +get_version() -> + ?HARNESS_MODULE:get_version(). + +get_version(Node) -> + ?HARNESS_MODULE:get_version(Node). + +get_backends() -> + ?HARNESS_MODULE:get_backends(). + +get_node_logs() -> + ?HARNESS_MODULE:get_node_logs(). + +get_node_logs(LogFile, DestDir) -> + ?HARNESS_MODULE:get_node_logs(LogFile, DestDir). + +set_backend(Backend) -> + ?HARNESS_MODULE:set_backend(Backend). + +whats_up() -> + ?HARNESS_MODULE:whats_up(). + +get_ip(Node) -> + ?HARNESS_MODULE:get_ip(Node). + +node_id(Node) -> + ?HARNESS_MODULE:node_id(Node). + +node_version(N) -> + ?HARNESS_MODULE:node_version(N). + +admin(Node, Args) -> + ?HARNESS_MODULE:admin(Node, Args). + +admin(Node, Args, Options) -> + ?HARNESS_MODULE:admin(Node, Args, Options). + +riak(Node, Args) -> + ?HARNESS_MODULE:riak(Node, Args). + +riak_repl(Node, Args) -> + ?HARNESS_MODULE:riak_repl(Node, Args). + +run_riak(Node, Version, Command) -> + ?HARNESS_MODULE:run_riak(Node, Version, Command). + +attach(Node, Expected) -> + ?HARNESS_MODULE:attach(Node, Expected). + +attach_direct(Node, Expected) -> + ?HARNESS_MODULE:attach_direct(Node, Expected). + +console(Node, Expected) -> + ?HARNESS_MODULE:console(Node, Expected). + +update_app_config(Node, Version, Config) -> + ?HARNESS_MODULE:update_app_config(Node, Version, Config). + +teardown() -> + ?HARNESS_MODULE:teardown(). + +set_conf(Node, NameValuePairs) -> + ?HARNESS_MODULE:set_conf(Node, NameValuePairs). + +set_advanced_conf(Node, NameValuePairs) -> + ?HARNESS_MODULE:set_advanced_conf(Node, NameValuePairs). + +update_app_config(Node, Config) -> + ?HARNESS_MODULE:update_app_config(Node, Config). + +upgrade(Node, NewVersion) -> + ?HARNESS_MODULE:upgrade(Node, NewVersion). + +validate_config(Versions) -> + ?HARNESS_MODULE:validate_config(Versions). diff --git a/src/rt_harness_util.erl b/src/rt_harness_util.erl new file mode 100644 index 000000000..e77a381e1 --- /dev/null +++ b/src/rt_harness_util.erl @@ -0,0 +1,483 @@ +%% ------------------------------------------------------------------- +%% +%% Copyright (c) 2013-2014 Basho Technologies, Inc. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% ------------------------------------------------------------------- +%% @doc The purpose of rt_harness_util is to provide common functions +%% to harness modules implementing the test_harness behaviour. +-module(rt_harness_util). + +-include_lib("eunit/include/eunit.hrl"). +-define(DEVS(N), lists:concat([N, "@127.0.0.1"])). +-define(DEV(N), list_to_atom(?DEVS(N))). +-define(PATH, (rt_config:get(root_path))). + +-export([admin/2, + attach/2, + attach_direct/2, + cmd/1, + cmd/2, + console/2, + deploy_nodes/5, + get_ip/1, + node_id/1, + node_version/1, + riak/2, + set_conf/2, + set_advanced_conf/2, + setup_harness/3, + get_advanced_riak_conf/1, + update_app_config_file/2, + spawn_cmd/1, + spawn_cmd/2, + whats_up/0]). + +admin(Node, Args) -> + N = node_id(Node), + Path = relpath(node_version(N)), + Cmd = riak_admin_cmd(Path, N, Args), + lager:info("Running: ~s", [Cmd]), + Result = os:cmd(Cmd), + lager:info("~s", [Result]), + {ok, Result}. + +attach(Node, Expected) -> + interactive(Node, "attach", Expected). + +attach_direct(Node, Expected) -> + interactive(Node, "attach-direct", Expected). + +console(Node, Expected) -> + interactive(Node, "console", Expected). + +%% deploy_clusters(ClusterConfigs) -> +%% NumNodes = rt_config:get(num_nodes, 6), +%% RequestedNodes = lists:flatten(ClusterConfigs), + +%% case length(RequestedNodes) > NumNodes of +%% true -> +%% erlang:error("Requested more nodes than available"); +%% false -> +%% Nodes = deploy_nodes(RequestedNodes), +%% {DeployedClusters, _} = lists:foldl( +%% fun(Cluster, {Clusters, RemNodes}) -> +%% {A, B} = lists:split(length(Cluster), RemNodes), +%% {Clusters ++ [A], B} +%% end, {[], Nodes}, ClusterConfigs), +%% DeployedClusters +%% end. + +%% deploy_nodes(NodeConfig) -> +deploy_nodes(NodeIds, _NodeMap, _Version, _Config, _Services) when NodeIds =:= [] -> + NodeIds; +deploy_nodes(NodeIds, NodeMap, Version, Config, Services) -> + %% create snmp dirs, for EE + create_dirs(Version, NodeIds), + + %% Set initial config + %% TODO: Cuttlefish should not need to have Nodes be atoms explicitly + ConfigUpdateFun = + case Config of + {cuttlefish, CuttleConfig} -> + fun(Node) -> + rt_harness:set_conf(list_to_atom(Node), CuttleConfig) + end; + _ -> + fun(Node) -> + rt_harness:update_app_config(Node, Version, Config) + end + end, + rt2:pmap(ConfigUpdateFun, NodeIds), + + %% Start nodes + RunRiakFun = + fun(Node) -> + rt_harness:run_riak(Node, Version, "start") + end, + rt2:pmap(RunRiakFun, NodeIds), + + %% Ensure nodes started + lager:debug("Wait until pingable: ~p", [NodeIds]), + [ok = rt2:wait_until_pingable(rt_node:node_name(NodeId, NodeMap)) + || NodeId <- NodeIds], + + %% TODO Rubbish! Fix this. + %% %% Enable debug logging + %% [rpc:call(N, lager, set_loglevel, [lager_console_backend, debug]) + %% || N <- Nodes], + + %% We have to make sure that riak_core_ring_manager is running + %% before we can go on. + [ok = rt2:wait_until_registered(rt_node:node_name(NodeId, NodeMap), + riak_core_ring_manager) + || NodeId <- NodeIds], + + %% Ensure nodes are singleton clusters + case Version =/= "0.14.2" of + true -> + [ok = rt_ring:check_singleton_node(rt_node:node_name(NodeId, NodeMap)) + || NodeId <- NodeIds]; + false -> + ok + end, + + %% Wait for services to start + lager:debug("Waiting for services ~p to start on ~p.", [Services, NodeIds]), + [ ok = rt2:wait_for_service(rt_node:node_name(NodeId, NodeMap), Service) + || NodeId <- NodeIds, + Service <- Services ], + + lager:debug("Deployed nodes: ~p", [NodeIds]), + NodeIds. + +interactive(Node, Command, Exp) -> + N = node_id(Node), + Path = relpath(node_version(N)), + Cmd = riakcmd(Path, N, Command), + lager:info("Opening a port for riak ~s.", [Command]), + lager:debug("Calling open_port with cmd ~s", [binary_to_list(iolist_to_binary(Cmd))]), + P = open_port({spawn, binary_to_list(iolist_to_binary(Cmd))}, + [stream, use_stdio, exit_status, binary, stderr_to_stdout]), + interactive_loop(P, Exp). + +interactive_loop(Port, Expected) -> + receive + {Port, {data, Data}} -> + %% We've gotten some data, so the port isn't done executing + %% Let's break it up by newline and display it. + Tokens = string:tokens(binary_to_list(Data), "\n"), + [lager:debug("~s", [Text]) || Text <- Tokens], + + %% Now we're going to take hd(Expected) which is either {expect, X} + %% or {send, X}. If it's {expect, X}, we foldl through the Tokenized + %% data looking for a partial match via rt:str/2. If we find one, + %% we pop hd off the stack and continue iterating through the list + %% with the next hd until we run out of input. Once hd is a tuple + %% {send, X}, we send that test to the port. The assumption is that + %% once we send data, anything else we still have in the buffer is + %% meaningless, so we skip it. That's what that {sent, sent} thing + %% is about. If there were a way to abort mid-foldl, I'd have done + %% that. {sent, _} -> is just a pass through to get out of the fold. + + NewExpected = lists:foldl(fun(X, Expect) -> + [{Type, Text}|RemainingExpect] = case Expect of + [] -> [{done, "done"}|[]]; + E -> E + end, + case {Type, rt2:str(X, Text)} of + {expect, true} -> + RemainingExpect; + {expect, false} -> + [{Type, Text}|RemainingExpect]; + {send, _} -> + port_command(Port, list_to_binary(Text ++ "\n")), + [{sent, "sent"}|RemainingExpect]; + {sent, _} -> + Expect; + {done, _} -> + [] + end + end, Expected, Tokens), + %% Now that the fold is over, we should remove {sent, sent} if it's there. + %% The fold might have ended not matching anything or not sending anything + %% so it's possible we don't have to remove {sent, sent}. This will be passed + %% to interactive_loop's next iteration. + NewerExpected = case NewExpected of + [{sent, "sent"}|E] -> E; + E -> E + end, + %% If NewerExpected is empty, we've met all expected criteria and in order to boot + %% Otherwise, loop. + case NewerExpected of + [] -> ?assert(true); + _ -> interactive_loop(Port, NewerExpected) + end; + {Port, {exit_status,_}} -> + %% This port has exited. Maybe the last thing we did was {send, [4]} which + %% as Ctrl-D would have exited the console. If Expected is empty, then + %% We've met every expectation. Yay! If not, it means we've exited before + %% something expected happened. + ?assertEqual([], Expected) + after rt_config:get(rt_max_receive_wait_time) -> + %% interactive_loop is going to wait until it matches expected behavior + %% If it doesn't, the test should fail; however, without a timeout it + %% will just hang forever in search of expected behavior. See also: Parenting + ?assertEqual([], Expected) + end. + +node_to_host(Node) -> + case string:tokens(atom_to_list(Node), "@") of + ["riak", Host] -> Host; + _ -> + throw(io_lib:format("rtssh:node_to_host couldn't figure out the host of ~p", [Node])) + end. + +spawn_cmd(Cmd) -> + spawn_cmd(Cmd, []). +spawn_cmd(Cmd, Opts) -> + Port = open_port({spawn, lists:flatten(Cmd)}, [stream, in, exit_status] ++ Opts), + Port. + +wait_for_cmd(Port) -> + rt2:wait_until(node(), + fun(_) -> + receive + {Port, Msg={exit_status, _}} -> + catch port_close(Port), + self() ! {Port, Msg}, + true + after 0 -> + false + end + end), + get_cmd_result(Port, []). + +cmd(Cmd) -> + cmd(Cmd, []). + +cmd(Cmd, Opts) -> + wait_for_cmd(spawn_cmd(Cmd, Opts)). + +get_cmd_result(Port, Acc) -> + receive + {Port, {data, Bytes}} -> + get_cmd_result(Port, [Bytes|Acc]); + {Port, {exit_status, Status}} -> + Output = lists:flatten(lists:reverse(Acc)), + {Status, Output} + after 0 -> + timeout + end. + + +get_host(Node) when is_atom(Node) -> + try orddict:fetch(Node, rt_config:get(rt_hosts)) of + Host -> Host + catch _:_ -> + %% Let's try figuring this out from the node name + node_to_host(Node) + end; +get_host(Host) -> Host. + +get_ip(Node) when is_atom(Node) -> + get_ip(get_host(Node)); +get_ip(Host) -> + {ok, IP} = inet:getaddr(Host, inet), + string:join([integer_to_list(X) || X <- tuple_to_list(IP)], "."). + +node_id(Node) -> + NodeMap = rt_config:get(rt_nodes), + orddict:fetch(Node, NodeMap). + +node_version(N) -> + VersionMap = rt_config:get(rt_versions), + orddict:fetch(N, VersionMap). + +riak(Node, Args) -> + N = node_id(Node), + Path = relpath(node_version(N)), + Result = run_riak(N, Path, Args), + lager:info("~s", [Result]), + {ok, Result}. + +-spec set_conf(atom() | string(), [{string(), string()}]) -> ok. +set_conf(all, NameValuePairs) -> + lager:info("rtdev:set_conf(all, ~p)", [NameValuePairs]), + [ set_conf(DevPath, NameValuePairs) || DevPath <- devpaths()], + ok; +set_conf(Node, NameValuePairs) when is_atom(Node) -> + append_to_conf_file(get_riak_conf(Node), NameValuePairs), + ok; +set_conf(DevPath, NameValuePairs) -> + [append_to_conf_file(RiakConf, NameValuePairs) || RiakConf <- all_the_files(DevPath, "etc/riak.conf")], + ok. + +whats_up() -> + io:format("Here's what's running...~n"), + + Up = [rpc:call(Node, os, cmd, ["pwd"]) || Node <- nodes()], + [io:format(" ~s~n",[string:substr(Dir, 1, length(Dir)-1)]) || Dir <- Up]. + +riak_admin_cmd(Path, N, Args) -> + Quoted = + lists:map(fun(Arg) when is_list(Arg) -> + lists:flatten([$", Arg, $"]); + (_) -> + erlang:error(badarg) + end, Args), + ArgStr = string:join(Quoted, " "), + ExecName = rt_config:get(exec_name, "riak"), + io_lib:format("~s/dev/dev~b/bin/~s-admin ~s", [Path, N, ExecName, ArgStr]). + +% Private functions + +relpath(Vsn) -> + Path = ?PATH, + relpath(Vsn, Path). + +relpath(Vsn, Paths=[{_,_}|_]) -> + orddict:fetch(Vsn, orddict:from_list(Paths)); +relpath(current, Path) -> + Path; +relpath(root, Path) -> + Path; +relpath(_, _) -> + throw("Version requested but only one path provided"). + +riakcmd(Path, N, Cmd) -> + ExecName = rt_config:get(exec_name, "riak"), + io_lib:format("~s/dev/dev~b/bin/~s ~s", [Path, N, ExecName, Cmd]). + +run_riak(N, Path, Cmd) -> + lager:info("Running: ~s", [riakcmd(Path, N, Cmd)]), + R = os:cmd(riakcmd(Path, N, Cmd)), + case Cmd of + "start" -> + rt_cover:maybe_start_on_node(?DEV(N), node_version(N)), + %% Intercepts may load code on top of the cover compiled + %% modules. We'll just get no coverage info then. + case rt_intercept:are_intercepts_loaded(?DEV(N)) of + false -> + ok = rt_intercept:load_intercepts([?DEV(N)]); + true -> + ok + end, + R; + "stop" -> + rt_cover:maybe_stop_on_node(?DEV(N)), + R; + _ -> + R + end. + +append_to_conf_file(File, NameValuePairs) -> + Settings = lists:flatten( + [io_lib:format("~n~s = ~s~n", [Name, Value]) || {Name, Value} <- NameValuePairs]), + file:write_file(File, Settings, [append]). + +get_riak_conf(Node) -> + N = node_id(Node), + Path = relpath(node_version(N)), + io_lib:format("~s/dev/dev~b/etc/riak.conf", [Path, N]). + +all_the_files(DevPath, File) -> + case filelib:is_dir(DevPath) of + true -> + Wildcard = io_lib:format("~s/dev/dev*/~s", [DevPath, File]), + filelib:wildcard(Wildcard); + _ -> + lager:debug("~s is not a directory.", [DevPath]), + [] + end. + +devpaths() -> + lists:usort([ DevPath || {_Name, DevPath} <- proplists:delete(root, rt_config:get(rtdev_path))]). + +create_dirs(Version, NodeIds) -> + VersionPath = filename:join(?PATH, Version), + Snmp = [filename:join([VersionPath, NodeId, "data/snmp/agent/db"]) || + NodeId <- NodeIds], + [?assertCmd("mkdir -p " ++ Dir) || Dir <- Snmp]. + +%% check_node({_N, Version}) -> +%% case proplists:is_defined(Version, rt_config:get(rtdev_path)) of +%% true -> ok; +%% _ -> +%% lager:error("You don't have Riak ~s installed or configured", [Version]), +%% erlang:error("You don't have Riak " ++ atom_to_list(Version) ++ " installed or configured") +%% end. + +%% add_default_node_config(Nodes) -> +%% case rt_config:get(rt_default_config, undefined) of +%% undefined -> ok; +%% Defaults when is_list(Defaults) -> +%% rt:pmap(fun(Node) -> +%% rt_config:update_app_config(Node, Defaults) +%% end, Nodes), +%% ok; +%% BadValue -> +%% lager:error("Invalid value for rt_default_config : ~p", [BadValue]), +%% throw({invalid_config, {rt_default_config, BadValue}}) +%% end. + +%% node_path(Node) -> +%% N = node_id(Node), +%% Path = relpath(node_version(N)), +%% lists:flatten(io_lib:format("~s/dev/dev~b", [Path, N])). + +set_advanced_conf(all, NameValuePairs) -> + lager:info("rtdev:set_advanced_conf(all, ~p)", [NameValuePairs]), + [ set_advanced_conf(DevPath, NameValuePairs) || DevPath <- devpaths()], + ok; +set_advanced_conf(Node, NameValuePairs) when is_atom(Node) -> + append_to_conf_file(get_advanced_riak_conf(Node), NameValuePairs), + ok; +set_advanced_conf(DevPath, NameValuePairs) -> + [update_app_config_file(RiakConf, NameValuePairs) || RiakConf <- all_the_files(DevPath, "etc/advanced.config")], + ok. + +get_advanced_riak_conf(Node) -> + N = node_id(Node), + Path = relpath(node_version(N)), + io_lib:format("~s/dev/dev~b/etc/advanced.config", [Path, N]). + +update_app_config_file(ConfigFile, Config) -> + lager:info("rtdev:update_app_config_file(~s, ~p)", [ConfigFile, Config]), + + BaseConfig = case file:consult(ConfigFile) of + {ok, [ValidConfig]} -> + ValidConfig; + {error, enoent} -> + [] + end, + MergeA = orddict:from_list(Config), + MergeB = orddict:from_list(BaseConfig), + NewConfig = + orddict:merge(fun(_, VarsA, VarsB) -> + MergeC = orddict:from_list(VarsA), + MergeD = orddict:from_list(VarsB), + orddict:merge(fun(_, ValA, _ValB) -> + ValA + end, MergeC, MergeD) + end, MergeA, MergeB), + NewConfigOut = io_lib:format("~p.", [NewConfig]), + ?assertEqual(ok, file:write_file(ConfigFile, NewConfigOut)), + ok. + +%% TODO: This made sense in an earlier iteration, but probably is no +%% longer needed. Original idea was to provide a place for setup that +%% was general to all harnesses to happen. +setup_harness(VersionMap, NodeIds, NodeMap) -> + %% rt_config:set(rt_nodes, Nodes), + %% rt_config:set(rt_nodes_available, Nodes), + %% rt_config:set(rt_version_map, VersionMap), + %% rt_config:set(rt_versions, VersionMap), + %% [create_dirs(Version, VersionNodes) || {Version, VersionNodes} <- VersionMap], + {NodeIds, NodeMap, VersionMap}. + +%% %% @doc Stop nodes and wipe out their data directories +%% stop_and_clean_nodes(Nodes, Version) when is_list(Nodes) -> +%% [rt_node:stop_and_wait(Node) || Node <- Nodes], +%% clean_data_dir(Nodes). + +%% clean_data_dir(Nodes) -> +%% clean_data_dir(Nodes, ""). + +%% clean_data_dir(Nodes, SubDir) when not is_list(Nodes) -> +%% clean_data_dir([Nodes], SubDir); +%% clean_data_dir(Nodes, SubDir) when is_list(Nodes) -> +%% rt_harness:clean_data_dir(Nodes, SubDir). diff --git a/src/rt_host.erl b/src/rt_host.erl new file mode 100644 index 000000000..8cd97c93b --- /dev/null +++ b/src/rt_host.erl @@ -0,0 +1,128 @@ +-module(rt_host). +-export([behaviour_info/1, + command_line_from_command/1, + connect/1, + connect/2, + consult/2, + copy_dir/4, + disconnect/1, + exec/2, + ip_addr/1, + hostname/1, + kill/3, + killall/3, + make_temp_directory_cmd/1, + mkdirs/2, + mvdir/3, + rmdir/2, + rmdir_cmd/1, + mvdir_cmd/2, + temp_dir/1, + write_file/3]). + +-type command() :: {filelib:filename(), [string()]}. +-type command_result() :: {ok, string()} | rt_util:error(). +-type host_id() :: pid(). +-type host() :: {module(), host_id()}. +-type hostname() :: atom(). + +-exporttype([command/0, + command_result/0, + host_id/0, + host/0, + hostname/0]). + +behaviour_info(callbacks) -> + [{connect, 1}, {connect, 2}, {consult, 2}, {copy_dir, 4}, {disconnect, 1}, + {exec, 2}, {ip_addr, 1}, {hostname, 1}, {kill, 3}, {killall, 3}, + {mkdirs, 2}, {mvdir, 3}, {rmdir, 2}, {temp_dir, 1}, {write_file, 3}]; +behaviour_info(_) -> + undefined. + +-spec command_line_from_command(command()) -> string(). +command_line_from_command({Command, Args}) -> + string:join([Command] ++ Args, " "). + +-spec connect(hostname()) -> {ok, host()} | rt_util:error(). +connect(Hostname) -> + connect(Hostname, []). + +%% To add remote host support, add an implementation of this function that +%% handles all atoms != localhost and create a connect_remote function to +%% startup a rt_remote_host gen_server with the appropriate configuration +%% (e.g. user name, creds, ports, etc) read from the r_t hosts file ... +-spec connect(hostname(), proplists:proplist()) -> {ok, host()} | rt_util:error(). +connect(localhost, Options) -> + connect_localhost(rt_local_host:connect(localhost, Options)); +connect(_, _) -> + erlang:error("Remote hosts are not supported."). + +-spec connect_localhost({ok, host_id()} | rt_util:error()) -> {ok, host()} | rt_util:error(). +connect_localhost({ok, Pid}) -> + {ok, {rt_local_host, Pid}}; +connect_localhost(Error) -> + Error. + +-spec consult(host(), filelib:filename()) -> {ok, term()} | rt_util:error(). +consult({HostModule, HostPid}, Filename) -> + HostModule:consult(HostPid, Filename). + +-spec copy_dir(host(), filelib:dirname(), filelib:dirname(), boolean()) -> command_result(). +copy_dir({HostModule, HostPid}, FromDir, ToDir, Recursive) -> + HostModule:copy_dir(HostPid, FromDir, ToDir, Recursive). + +-spec disconnect(host()) -> ok. +disconnect({HostModule, HostPid}) -> + HostModule:disconnect(HostPid). + +-spec exec(host(), rt_host:command()) -> command_result(). +exec({HostModule, HostPid}, Command) -> + HostModule:exec(HostPid, Command). + +-spec hostname(host()) -> rt_host:hostname(). +hostname({HostModule, HostPid}) -> + HostModule:hostname(HostPid). + +-spec ip_addr(host()) -> string(). +ip_addr({HostModule, HostPid}) -> + HostModule:ip_addr(HostPid). + +-spec kill(host(), pos_integer(), pos_integer()) -> rt_util:result(). +kill({HostModule, HostPid}, Signal, OSPid) -> + HostModule:kill(HostPid, Signal, OSPid). + +-spec killall(host(), pos_integer(), string()) -> rt_util:result(). +killall({HostModule, HostPid}, Signal, Name) -> + HostModule:killall(HostPid, Signal, Name). + +-spec make_temp_directory_cmd(string()) -> command(). +make_temp_directory_cmd(Template) -> + {"/usr/bin/mktemp", ["-d", "-t", Template]}. + +-spec mkdirs(host(), filelib:dirname()) -> rt_util:result(). +mkdirs({HostModule, HostPid}, Path) -> + HostModule:mkdirs(HostPid, Path). + +-spec mvdir(host(), filelib:dirname(), filelib:dirname()) -> rt_util:result(). +mvdir({HostModule, HostPid}, FromDir, ToDir) -> + HostModule:mvdir(HostPid, FromDir, ToDir). + +-spec rmdir(host(), filelib:dirname()) -> rt_util:result(). +rmdir({HostModule, HostPid}, Dir) -> + HostModule:rmdir(HostPid, Dir). + +-spec rmdir_cmd(filelib:dirname()) -> command(). +rmdir_cmd(Dir) -> + {"/bin/rm", ["-rf", Dir]}. + +-spec mvdir_cmd(filelib:dirname(), filelib:dirname()) -> command(). +mvdir_cmd(FromDir, ToDir) -> + {"/bin/mv", [FromDir, ToDir]}. + +-spec temp_dir(host()) -> filelib:dirname(). +temp_dir({HostModule, HostPid}) -> + HostModule:temp_dir(HostPid). + +-spec write_file(host(), filelib:filename(), term()) -> rt_util:result(). +write_file({HostModule, HostPid}, Filename, Content) -> + HostModule:write_file(HostPid, Filename, Content). diff --git a/src/rt_http.erl b/src/rt_http.erl new file mode 100644 index 000000000..7135de51b --- /dev/null +++ b/src/rt_http.erl @@ -0,0 +1,81 @@ +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% ------------------------------------------------------------------- +-module(rt_http). +-include_lib("eunit/include/eunit.hrl"). + +-export([http_url/1, + https_url/1, + httpc/1, + httpc_read/3, + httpc_write/4, + get_http_conn_info/1, + get_https_conn_info/1]). + +%% @doc Returns HTTPS URL information for a list of Nodes +https_url(Nodes) when is_list(Nodes) -> + [begin + {Host, Port} = orddict:fetch(https, Connections), + lists:flatten(io_lib:format("https://~s:~b", [Host, Port])) + end || {_Node, Connections} <- rt:connection_info(Nodes)]; +https_url(Node) -> + hd(https_url([Node])). + +%% @doc Returns HTTP URL information for a list of Nodes +http_url(Nodes) when is_list(Nodes) -> + [begin + {Host, Port} = orddict:fetch(http, Connections), + lists:flatten(io_lib:format("http://~s:~b", [Host, Port])) + end || {_Node, Connections} <- rt:connection_info(Nodes)]; +http_url(Node) -> + hd(http_url([Node])). + +%% @doc get me an http client. +-spec httpc(node()) -> term(). +httpc(Node) -> + rt:wait_for_service(Node, riak_kv), + {ok, [{IP, Port}]} = get_http_conn_info(Node), + rhc:create(IP, Port, "riak", []). + +%% @doc does a read via the http erlang client. +-spec httpc_read(term(), binary(), binary()) -> binary(). +httpc_read(C, Bucket, Key) -> + {_, Value} = rhc:get(C, Bucket, Key), + Value. + +%% @doc does a write via the http erlang client. +-spec httpc_write(term(), binary(), binary(), binary()) -> atom(). +httpc_write(C, Bucket, Key, Value) -> + Object = riakc_obj:new(Bucket, Key, Value), + rhc:put(C, Object). + +-spec get_http_conn_info(node()) -> [{inet:ip_address(), pos_integer()}]. +get_http_conn_info(Node) -> + case rt:rpc_get_env(Node, [{riak_api, http}, + {riak_core, http}]) of + {ok, [{IP, Port}|_]} -> + {ok, [{IP, Port}]}; + _ -> + undefined + end. + + +-spec get_https_conn_info(node()) -> [{inet:ip_address(), pos_integer()}]. +get_https_conn_info(Node) -> + case rt:rpc_get_env(Node, [{riak_api, https}, + {riak_core, https}]) of + {ok, [{IP, Port}|_]} -> + {ok, [{IP, Port}]}; + _ -> + undefined + end. diff --git a/src/rt_local.erl b/src/rt_local.erl index b8bfaed8e..f2db1022f 100644 --- a/src/rt_local.erl +++ b/src/rt_local.erl @@ -115,6 +115,6 @@ stream_cmd_loop(Port, Buffer, NewLineBuffer, Time={_MegaSecs, Secs, _MicroSecs}) {Port, {exit_status, Status}} -> catch port_close(Port), {Status, Buffer} - after rt_config:get(rt_max_wait_time) -> + after rt_config:get(rt_max_receive_wait_time) -> {-1, Buffer} - end. \ No newline at end of file + end. diff --git a/src/rt_local_host.erl b/src/rt_local_host.erl new file mode 100644 index 000000000..160c740be --- /dev/null +++ b/src/rt_local_host.erl @@ -0,0 +1,375 @@ +-module(rt_local_host). + +-behaviour(gen_server). +-behaviour(rt_host). + +%% API +-export([connect/1, + connect/2, + copy_dir/4, + consult/2, + disconnect/1, + exec/2, + hostname/1, + ip_addr/1, + killall/3, + kill/3, + mkdirs/2, + mvdir/3, + rmdir/2, + temp_dir/1, + write_file/3]). + +-define(TIMEOUT, infinity). + +%% gen_server callbacks +-export([init/1, handle_call/3, handle_cast/2, handle_info/2, + terminate/2, code_change/3]). + +-record(state, {options=[] :: proplists:proplist(), + temp_dir :: filelib:dirname()}). + +%%%=================================================================== +%%% API +%%%=================================================================== + +-spec connect(rt_host:hostname()) -> {ok, rt_host:host_id()} | rt_util:error(). +connect(Hostname) -> + connect(Hostname, []). + +-spec connect(rt_host:hostname(), proplists:proplist()) -> {ok, rt_host:host_id()} | rt_util:error(). +connect(_Hostname, Options) -> + gen_server:start_link(?MODULE, [Options], []). + +-spec consult(rt_host:host_id(), filelib:filename()) -> {ok, term()} | rt_util:error(). +consult(Pid, Filename) -> + gen_server:call(Pid, {consult, Filename}, ?TIMEOUT). + +-spec copy_dir(rt_host:host_id(), filelib:dirname(), filelib:dirname(), boolean()) -> rt_host:command_result(). +copy_dir(Pid, FromDir, ToDir, Recursive) -> + %% TODO What is the best way to handle timeouts? + gen_server:call(Pid, {copy_dir, FromDir, ToDir, Recursive}, ?TIMEOUT). + +-spec disconnect(rt_host:host_id()) -> ok. +disconnect(Pid) -> + gen_server:call(Pid, stop, ?TIMEOUT). + +-spec exec(rt_host:host_id(), rt_host:command()) -> rt_host:command_result(). +exec(Pid, Command) -> + %% TODO What is the best way to handle timeouts? -> Not the OTP timeout, bot to erlexec + gen_server:call(Pid, {exec, Command}, ?TIMEOUT). + +-spec hostname(rt_host:host_id()) -> rt_host:hostname(). +hostname(Pid) -> + gen_server:call(Pid, hostname, ?TIMEOUT). + +-spec ip_addr(rt_host:host_id()) -> string(). +ip_addr(Pid) -> + gen_server:call(Pid, ip_addr, ?TIMEOUT). + +-spec kill(rt_host:host_id(), pos_integer(), pos_integer()) -> rt_util:result(). +kill(Pid, Signal, OSPid) -> + gen_server:call(Pid, {kill, Signal, OSPid}, ?TIMEOUT). + +-spec killall(rt_host:host_id(), pos_integer(), string()) -> rt_util:result(). +killall(Pid, Signal, Name) -> + gen_server:call(Pid, {killall, Signal, Name}, ?TIMEOUT). + +-spec mvdir(rt_host:host_id(), filelib:dirname(), filelib:dirname()) -> rt_util:result(). +mvdir(Pid, FromDir, ToDir) -> + gen_server:call(Pid, {mvdir, FromDir, ToDir}, ?TIMEOUT). + +-spec mkdirs(rt_host:host_id(), filelib:dirname()) -> {ok, filelib:dirname()} | rt_result:error(). +mkdirs(Pid, Path) -> + gen_server:call(Pid, {mkdirs, Path}, ?TIMEOUT). + +-spec rmdir(rt_host:host_id(), filelib:dirname()) -> rt_result:result(). +rmdir(Pid, Dir) -> + gen_server:call(Pid, {rmdir, Dir}, ?TIMEOUT). + +-spec temp_dir(rt_host:host_id()) -> filelib:dirname(). +temp_dir(Pid) -> + gen_server:call(Pid, temp_dir, ?TIMEOUT). + +-spec write_file(rt_host:host_id(), filelib:filename(), term()) -> rt_util:result(). +write_file(Pid, Filename, Content) -> + gen_server:call(Pid, {write_file, Filename, Content}, ?TIMEOUT). + +%%%=================================================================== +%%% gen_server callbacks +%%%=================================================================== + +%%-------------------------------------------------------------------- +%% @private +%% @doc +%% Initializes the server +%% +%% @spec init(Args) -> {ok, State} | +%% {ok, State, Timeout} | +%% ignore | +%% {stop, Reason} +%% @end +%%-------------------------------------------------------------------- +init([Options]) -> + maybe_init(make_temp_directory(), Options). + +-spec maybe_init({ok, filelib:dirname()} | {error, term()}, proplists:proplist()) -> {ok, #state{}} | {stop, term()}. +maybe_init({ok, TempDir}, Options) -> + State = #state{options=Options, + temp_dir=TempDir}, + lager:debug("Starting localhost gen_server with state ~p", [State]), + {ok, State}; +maybe_init({error, Reason}, _Options) -> + lager:error("Failed to start localhost gen_server -- temp directory failed to be created due to ~p", [Reason]), + {stop, Reason}. + +%%-------------------------------------------------------------------- +%% @private +%% @doc +%% Handling call messages +%% +%% @spec handle_call(Request, From, State) -> +%% {reply, Reply, State} | +%% {reply, Reply, State, Timeout} | +%% {noreply, State} | +%% {noreply, State, Timeout} | +%% {stop, Reason, Reply, State} | +%% {stop, Reason, State} +%% @end +%%-------------------------------------------------------------------- +handle_call({copy_dir, FromDir, ToDir, Recursive}, _From, State) -> + {reply, do_copy_dir(FromDir, ToDir, Recursive), State}; +handle_call({consult, Filename}, _From, State) -> + {reply, file:consult(Filename), State}; +handle_call({exec, Command}, _From, State) -> + {reply, maybe_exec(Command), State}; +handle_call(hostname, _From, State) -> + {reply, localhost, State}; +handle_call(ip_addr, _From, State) -> + {reply, "127.0.0.1", State}; +handle_call({kill, Signal, OSPid}, _From, State) -> + lager:info("Killing process ~p with signal ~p on localhost", [OSPid, Signal]), + {reply, exec:kill(OSPid, Signal), State}; +handle_call({killall, Signal, Name}, _From, State) -> + lager:info("Killing all processes named ~p with signal ~p on localhost", [Name, Signal]), + SignalStr = lists:concat(["-", integer_to_list(Signal)]), + {reply, maybe_exec({"/usr/bin/killall", [SignalStr, Name]}), State}; +handle_call({mkdirs, Path}, _From, State) -> + %% filelib:ensure_dir requires the path to end with a / in order to + %% create a directory ... + SanitizedPath = rt_util:maybe_append_when_not_endswith(Path, "/"), + lager:debug("Creating directory ~p on localhost", [SanitizedPath]), + {reply, filelib:ensure_dir(SanitizedPath), State}; +handle_call({mvdir, FromDir, ToDir}, _From, State) -> + {reply, maybe_exec(rt_host:mvdir_cmd(FromDir, ToDir)), State}; +handle_call({rmdir, Dir}, _From, State) -> + {reply, maybe_exec(rt_host:rmdir_cmd(Dir)), State}; +handle_call(stop, _From, State) -> + {stop, normal, ok, State}; +handle_call(temp_dir, _From, State=#state{temp_dir=TempDir}) -> + {reply, TempDir, State}; +handle_call({write_file, Filename, Content}, _From, State) -> + lager:debug("Writing ~p to file ~p on localhost", [Content, Filename]), + {reply, file:write_file(Filename, Content), State}. + +%%-------------------------------------------------------------------- +%% @private +%% @doc +%% Handling cast messages +%% +%% @spec handle_cast(Msg, State) -> {noreply, State} | +%% {noreply, State, Timeout} | +%% {stop, Reason, State} +%% @end +%%-------------------------------------------------------------------- +handle_cast(_Msg, State) -> + {noreply, State}. + +%%-------------------------------------------------------------------- +%% @private +%% @doc +%% Handling all non call/cast messages +%% +%% @spec handle_info(Info, State) -> {noreply, State} | +%% {noreply, State, Timeout} | +%% {stop, Reason, State} +%% @end +%%-------------------------------------------------------------------- +handle_info(_Info, State) -> + {noreply, State}. + +%%-------------------------------------------------------------------- +%% @private +%% @doc +%% This function is called by a gen_server when it is about to +%% terminate. It should be the opposite of Module:init/1 and do any +%% necessary cleaning up. When it returns, the gen_server terminates +%% with Reason. The return value is ignored. +%% +%% @spec terminate(Reason, State) -> void() +%% @end +%%-------------------------------------------------------------------- +terminate(_Reason, #state{temp_dir=TempDir}) -> + lager:debug("Shuttting down localhost gen_server."), + case maybe_exec(rt_host:rmdir_cmd(TempDir)) of + {ok, _} -> + lager:debug("Removed temporary directory ~p on localhost", [TempDir]), + ok; + {error, Reason} -> + lager:warning("Failed to remove temporary directory ~p due to ~p on localhost", + [TempDir, Reason]), + ok + end. + +%%-------------------------------------------------------------------- +%% @private +%% @doc +%% Convert process state when code is changed +%% +%% @spec code_change(OldVsn, State, Extra) -> {ok, NewState} +%% @end +%%-------------------------------------------------------------------- +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +%%%=================================================================== +%%% Internal functions +%%%=================================================================== +%% TODO Implement stdout and stderr redirection to lager ... +-spec maybe_exec(rt_host:command()) -> rt_host:command_result(). +maybe_exec(Command) -> + maybe_exec(Command, [sync, stdout, stderr]). + +-spec maybe_exec({ok | error, proplists:proplists()} | rt_host:command(), string() | proplists:proplist()) -> rt_host:command_result(). +maybe_exec(Result={ok, Reason}, CommandLine) -> + lager:debug("Command ~s succeeded with result ~p", [CommandLine, Result]), + Output = proplists:get_value(stdout, Reason, []), + {ok, join_binaries_to_string(Output)}; +maybe_exec(Result={error, Reason}, CommandLine) -> + lager:error("Command ~s failed with result ~p", [CommandLine, Result]), + Output = proplists:get_value(stderr, Reason, []), + {error, join_binaries_to_string(Output)}; +maybe_exec(Command, Options) -> + CommandLine = rt_host:command_line_from_command(Command), + lager:debug("Executing command ~p with options ~p", [CommandLine, Options]), + maybe_exec(exec:run(CommandLine, Options), CommandLine). + +%% TODO Consider implementing in pure Erlang to get more granular result info ... +-spec do_copy_dir(filelib:dirname(), filelib:dirname(), boolean()) -> rt_host:command_result(). +do_copy_dir(FromDir, ToDir, true) -> + lager:debug("Copying ~p to ~p recursively", [FromDir, ToDir]), + maybe_exec("cp", ["-R", FromDir, ToDir]); +do_copy_dir(FromDir, ToDir, false) -> + lager:debug("Copying ~p to ~p non-recursively", [FromDir, ToDir]), + maybe_exec("cp", [FromDir, ToDir]). + +-spec make_temp_directory() -> rt_host:command_result(). +make_temp_directory() -> + lager:debug("Creating a temporary directory on localhost"), + maybe_exec(rt_host:make_temp_directory_cmd("riak_test")). + +-spec join_binaries_to_string([binary()]) -> string(). +join_binaries_to_string(Bins) -> + Strings = lists:foldr(fun(Element, List) -> + [binary:bin_to_list(Element)|List] + end, [], Bins), + Result = string:join(Strings, "\n"), + + %% Only strip off the trailing newline ... + string:strip(Result, right, $\n). + +-ifdef(TEST). +-include_lib("eunit/include/eunit.hrl"). + +setup() -> + ok = application:ensure_started(exec), + {Result, Pid} = connect(localhost), + ?assertEqual(ok, Result), + ?assertEqual(true, is_pid(Pid)), + ?assertEqual(true, filelib:is_dir(temp_dir(Pid))), + Pid. + +teardown(Pid) -> + TempDir = temp_dir(Pid), + disconnect(Pid), + ?assertEqual(false, filelib:is_dir(TempDir)). + +verify_connect({Result, Pid}) -> + ?_test(begin + ?assertEqual(ok, Result), + ?assertEqual(true, is_pid(Pid)), + ok = disconnect(Pid) + end). + +connect_test_() -> + {foreach, + fun() -> ok = application:ensure_started(exec) end, + [fun() -> verify_connect(connect(localhost)) end, + fun() -> verify_connect(connect(localhost, [])) end]}. + +check_exec_args(Pid) -> + ?_test(begin + verify_exec_results(exec(Pid, {"echo", ["test"]}), "test") + end). + +check_exec_no_args(Pid) -> + ?_test(begin + verify_exec_results(exec(Pid, {"echo", []}), "") + end). + +verify_exec_results({Result, Output}, Expected) -> + ?assertEqual(ok, Result), + ?assertEqual(Expected, Output). + +exec_success_test_() -> + {foreach, + fun setup/0, + fun teardown/1, + [fun check_exec_no_args/1, + fun check_exec_args/1]}. + +exec_failure_test() -> + Pid = setup(), + {Result, _} = exec(Pid, {"asdfasdf", []}), + ?assertEqual(error, Result), + teardown(Pid). + +check_kill(Pid, Signal) -> + ?_test(begin + {ok, _, OSPid} = exec:run("while true; do sleep 1; done", + [{success_exit_code, Signal}]), + Result = kill(Pid, Signal, OSPid), + ?assertEqual(ok, Result) + end). + +kill_test_() -> + {foreach, + fun setup/0, + fun teardown/1, + [fun(Pid) -> check_kill(Pid, 9) end, + fun(Pid) -> check_kill(Pid, 15) end]}. + +mkdirs_test() -> + Pid = setup(), + TestDir = filename:join([temp_dir(Pid), "test123", "foo", "bar"]), + Result = mkdirs(Pid, TestDir), + ?assertEqual(ok, Result), + ?assertEqual(true, filelib:is_dir(TestDir)), + teardown(Pid). + +hostname_test() -> + Pid = setup(), + ?assertEqual(localhost, hostname(Pid)), + teardown(Pid). + +ip_addr_test() -> + Pid = setup(), + ?assertEqual("127.0.0.1", ip_addr(Pid)), + teardown(Pid). + +join_binaries_to_string_test() -> + Bins = [<<"foo">>, <<"bar">>, <<"zoo">>], + Actual = join_binaries_to_string(Bins), + ?assertEqual("foo\nbar\nzoo", Actual). + +-endif. diff --git a/src/rt_node.erl b/src/rt_node.erl new file mode 100644 index 000000000..4cb68821b --- /dev/null +++ b/src/rt_node.erl @@ -0,0 +1,244 @@ +%% ------------------------------------------------------------------- +%% +%% Copyright (c) 2013-2014 Basho Technologies, Inc. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% ------------------------------------------------------------------- +-module(rt_node). +-include_lib("eunit/include/eunit.hrl"). + +-export([start/2, + start_and_wait/3, + async_start/2, + stop/2, + stop_and_wait/3, + upgrade/4, + is_ready/1, + %% slow_upgrade/3, + join/2, + staged_join/2, + plan_and_commit/1, + do_commit/1, + leave/1, + down/2, + heal/1, + partition/2, + remove/2, + brutal_kill/1, + wait_until_nodes_ready/1, + wait_until_owners_according_to/2, + wait_until_nodes_agree_about_ownership/1, + is_pingable/1, + clean_data_dir/2, + node_name/2]). + +-spec node_name(string(), [{string(), node()}]) -> node() | undefined. +%% @doc Hide the details of underlying data structure of the node map +%% in case it needs to change at some point. +node_name(NodeId, NodeMap) -> + case lists:keyfind(NodeId, 1, NodeMap) of + {NodeId, NodeName} -> + NodeName; + false -> + undefined + end. + +clean_data_dir(Node, Version) -> + clean_data_dir(Node, Version, ""). + +clean_data_dir(Node, Version, SubDir) -> + rt_harness:clean_data_dir(Node, Version, SubDir). + +%% @doc Start the specified Riak node +start(Node, Version) -> + rt_harness:start(Node, Version). + +%% @doc Start the specified Riak `Node' and wait for it to be pingable +start_and_wait(NodeId, NodeName, Version) -> + start(NodeId, Version), + ?assertEqual(ok, rt:wait_until_pingable(NodeName)). + +async_start(Node, Version) -> + spawn(fun() -> start(Node, Version) end). + +%% @doc Stop the specified Riak `Node'. +stop(Node, Version) -> + lager:info("Stopping riak version ~p on ~p", [Version, Node]), + rt_harness:stop(Node, Version). + +%% @doc Stop the specified Riak `Node' and wait until it is not pingable +-spec stop_and_wait(string(), node(), string()) -> ok. +stop_and_wait(NodeId, NodeName, Version) -> + stop(NodeId, Version), + ?assertEqual(ok, rt:wait_until_unpingable(NodeName)). + +%% %% @doc Upgrade a Riak `Node' to the specified `NewVersion'. +%% upgrade(Node, NewVersion) -> +%% rt_harness:upgrade(Node, NewVersion). + +%% @doc Upgrade a Riak `Node' to the specified `NewVersion' and update +%% the config based on entries in `Config'. +upgrade(Node, CurrentVersion, NewVersion, Config) -> + rt_harness:upgrade(Node, CurrentVersion, NewVersion, Config). + +%% @doc Upgrade a Riak node to a specific version using the alternate +%% leave/upgrade/rejoin approach +%% slow_upgrade(Node, NewVersion, Nodes) -> +%% lager:info("Perform leave/upgrade/join upgrade on ~p", [Node]), +%% lager:info("Leaving ~p", [Node]), +%% leave(Node), +%% ?assertEqual(ok, rt:wait_until_unpingable(Node)), +%% upgrade(Node, NewVersion), +%% lager:info("Rejoin ~p", [Node]), +%% join(Node, hd(Nodes -- [Node])), +%% lager:info("Wait until all nodes are ready and there are no pending changes"), +%% ?assertEqual(ok, rt:wait_until_nodes_ready(Nodes)), +%% ?assertEqual(ok, rt:wait_until_no_pending_changes(Nodes)), +%% ok. + +%% @doc Have `Node' send a join request to `PNode' +join(Node, PNode) -> + R = rpc:call(Node, riak_core, join, [PNode]), + lager:info("[join] ~p to (~p): ~p", [Node, PNode, R]), + ?assertEqual(ok, R), + ok. + +%% @doc Have `Node' send a join request to `PNode' +staged_join(Node, PNode) -> + R = rpc:call(Node, riak_core, staged_join, [PNode]), + lager:info("[join] ~p to (~p): ~p", [Node, PNode, R]), + ?assertEqual(ok, R), + ok. + +plan_and_commit(Node) -> + timer:sleep(500), + lager:info("planning and commiting cluster join"), + case rpc:call(Node, riak_core_claimant, plan, []) of + {error, ring_not_ready} -> + lager:info("plan: ring not ready on ~p", [Node]), + timer:sleep(100), + plan_and_commit(Node); + {badrpc, _} -> + lager:info("plan: ring not ready on ~p", [Node]), + timer:sleep(100), + plan_and_commit(Node); + {ok, _, _} -> + lager:info("plan: done"), + do_commit(Node) + end. + +do_commit(Node) -> + case rpc:call(Node, riak_core_claimant, commit, []) of + {error, plan_changed} -> + lager:info("commit: plan changed"), + timer:sleep(100), + rt:maybe_wait_for_changes(Node), + plan_and_commit(Node); + {error, ring_not_ready} -> + lager:info("commit: ring not ready"), + timer:sleep(100), + rt:maybe_wait_for_changes(Node), + do_commit(Node); + {error,nothing_planned} -> + %% Assume plan actually committed somehow + ok; + ok -> + ok + end. + +%% @doc Have the `Node' leave the cluster +leave(Node) -> + R = rpc:call(Node, riak_core, leave, []), + lager:info("[leave] ~p: ~p", [Node, R]), + ?assertEqual(ok, R), + ok. + +%% @doc Have `Node' remove `OtherNode' from the cluster +remove(Node, OtherNode) -> + ?assertEqual(ok, + rpc:call(Node, riak_kv_console, remove, [[atom_to_list(OtherNode)]])). + +%% @doc Have `Node' mark `OtherNode' as down +down(Node, OtherNode) -> + rpc:call(Node, riak_kv_console, down, [[atom_to_list(OtherNode)]]). + +%% @doc partition the `P1' from `P2' nodes +%% note: the nodes remained connected to riak_test@local, +%% which is how `heal/1' can still work. +partition(P1, P2) -> + OldCookie = rpc:call(hd(P1), erlang, get_cookie, []), + NewCookie = list_to_atom(lists:reverse(atom_to_list(OldCookie))), + [true = rpc:call(N, erlang, set_cookie, [N, NewCookie]) || N <- P1], + [[true = rpc:call(N, erlang, disconnect_node, [P2N]) || N <- P1] || P2N <- P2], + rt:wait_until_partitioned(P1, P2), + {NewCookie, OldCookie, P1, P2}. + +%% @doc heal the partition created by call to `partition/2' +%% `OldCookie' is the original shared cookie +heal({_NewCookie, OldCookie, P1, P2}) -> + Cluster = P1 ++ P2, + % set OldCookie on P1 Nodes + [true = rpc:call(N, erlang, set_cookie, [N, OldCookie]) || N <- P1], + rt:wait_until_connected(Cluster), + {_GN, []} = rpc:sbcast(Cluster, riak_core_node_watcher, broadcast), + ok. + +% when you just can't wait +brutal_kill(Node) -> + rt_cover:maybe_stop_on_node(Node), + lager:info("Killing node ~p", [Node]), + OSPidToKill = rpc:call(Node, os, getpid, []), + %% try a normal kill first, but set a timer to + %% kill -9 after 5 seconds just in case + rpc:cast(Node, timer, apply_after, + [5000, os, cmd, [io_lib:format("kill -9 ~s", [OSPidToKill])]]), + rpc:cast(Node, os, cmd, [io_lib:format("kill -15 ~s", [OSPidToKill])]), + ok. + +%% @doc Given a list of nodes, wait until all nodes are considered ready. +%% See {@link wait_until_ready/1} for definition of ready. +wait_until_nodes_ready(Nodes) -> + lager:info("Wait until nodes are ready : ~p", [Nodes]), + [?assertEqual(ok, rt:wait_until(Node, fun is_ready/1)) || Node <- Nodes], + ok. + +is_ready(Node) -> + case rpc:call(Node, riak_core_ring_manager, get_raw_ring, []) of + {ok, Ring} -> + case lists:member(Node, riak_core_ring:ready_members(Ring)) of + true -> true; + false -> {not_ready, Node} + end; + Other -> + Other + end. + +wait_until_owners_according_to(Node, Nodes) -> + SortedNodes = lists:usort(Nodes), + F = fun(N) -> + rt_ring:owners_according_to(N) =:= SortedNodes + end, + ?assertEqual(ok, rt:wait_until(Node, F)), + ok. + +wait_until_nodes_agree_about_ownership(Nodes) -> + lager:info("Wait until nodes agree about ownership ~p", [Nodes]), + Results = [ wait_until_owners_according_to(Node, Nodes) || Node <- Nodes ], + ?assert(lists:all(fun(X) -> ok =:= X end, Results)). + +%% @doc Is the `Node' up according to net_adm:ping +is_pingable(Node) -> + net_adm:ping(Node) =:= pong. diff --git a/src/rt_pb.erl b/src/rt_pb.erl new file mode 100644 index 000000000..21cf14b00 --- /dev/null +++ b/src/rt_pb.erl @@ -0,0 +1,200 @@ +%% ------------------------------------------------------------------- +%% +%% Copyright (c) 2013-2014 Basho Technologies, Inc. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% ------------------------------------------------------------------- +-module(rt_pb). +-include_lib("eunit/include/eunit.hrl"). + +-export([pbc/1, + pbc/2, + stop/1, + pbc_read/3, + pbc_read/4, + pbc_read_check/4, + pbc_read_check/5, + pbc_set_bucket_prop/3, + pbc_write/4, + pbc_write/5, + pbc_put_dir/3, + pbc_put_file/4, + pbc_really_deleted/3, + pbc_systest_write/2, + pbc_systest_write/3, + pbc_systest_write/5, + pbc_systest_read/2, + pbc_systest_read/3, + pbc_systest_read/5, + get_pb_conn_info/1]). + +-define(HARNESS, (rt_config:get(rt_harness))). + +%% @doc get me a protobuf client process and hold the mayo! +-spec pbc(node()) -> pid(). +pbc(Node) -> + pbc(Node, [{auto_reconnect, true}]). + +-spec pbc(node(), proplists:proplist()) -> pid(). +pbc(Node, Options) -> + rt2:wait_for_service(Node, riak_kv), + ConnInfo = get_pb_conn_info(Node), + {ok, [{IP, PBPort}]} = ConnInfo, + {ok, Pid} = riakc_pb_socket:start_link(IP, PBPort, Options), + Pid. + +stop(Pid) -> + riakc_pb_socket:stop(Pid). + +%% @doc does a read via the erlang protobuf client +-spec pbc_read(pid(), binary(), binary()) -> binary(). +pbc_read(Pid, Bucket, Key) -> + pbc_read(Pid, Bucket, Key, []). + +-spec pbc_read(pid(), binary(), binary(), [any()]) -> binary(). +pbc_read(Pid, Bucket, Key, Options) -> + {ok, Value} = riakc_pb_socket:get(Pid, Bucket, Key, Options), + Value. + +-spec pbc_read_check(pid(), binary(), binary(), [any()]) -> boolean(). +pbc_read_check(Pid, Bucket, Key, Allowed) -> + pbc_read_check(Pid, Bucket, Key, Allowed, []). + +-spec pbc_read_check(pid(), binary(), binary(), [any()], [any()]) -> boolean(). +pbc_read_check(Pid, Bucket, Key, Allowed, Options) -> + case riakc_pb_socket:get(Pid, Bucket, Key, Options) of + {ok, _} -> + true = lists:member(ok, Allowed); + Other -> + lists:member(Other, Allowed) orelse throw({failed, Other, Allowed}) + end. + +%% @doc does a write via the erlang protobuf client +-spec pbc_write(pid(), binary(), binary(), binary()) -> atom(). +pbc_write(Pid, Bucket, Key, Value) -> + Object = riakc_obj:new(Bucket, Key, Value), + riakc_pb_socket:put(Pid, Object). + +%% @doc does a write via the erlang protobuf client plus content-type +-spec pbc_write(pid(), binary(), binary(), binary(), list()) -> atom(). +pbc_write(Pid, Bucket, Key, Value, CT) -> + Object = riakc_obj:new(Bucket, Key, Value, CT), + riakc_pb_socket:put(Pid, Object). + +%% @doc sets a bucket property/properties via the erlang protobuf client +-spec pbc_set_bucket_prop(pid(), binary(), [proplists:property()]) -> atom(). +pbc_set_bucket_prop(Pid, Bucket, PropList) -> + riakc_pb_socket:set_bucket(Pid, Bucket, PropList). + +%% @doc Puts the contents of the given file into the given bucket using the +%% filename as a key and assuming a plain text content type. +pbc_put_file(Pid, Bucket, Key, Filename) -> + {ok, Contents} = file:read_file(Filename), + riakc_pb_socket:put(Pid, riakc_obj:new(Bucket, Key, Contents, "text/plain")). + +%% @doc Puts all files in the given directory into the given bucket using the +%% filename as a key and assuming a plain text content type. +pbc_put_dir(Pid, Bucket, Dir) -> + lager:info("Putting files from dir ~p into bucket ~p", [Dir, Bucket]), + {ok, Files} = file:list_dir(Dir), + [pbc_put_file(Pid, Bucket, list_to_binary(F), filename:join([Dir, F])) + || F <- Files]. + +%% @doc True if the given keys have been really, really deleted. +%% Useful when you care about the keys not being there. Delete simply writes +%% tombstones under the given keys, so those are still seen by key folding +%% operations. +pbc_really_deleted(Pid, Bucket, Keys) -> + StillThere = + fun(K) -> + Res = riakc_pb_socket:get(Pid, Bucket, K, + [{r, 1}, + {notfound_ok, false}, + {basic_quorum, false}, + deletedvclock]), + case Res of + {error, notfound} -> + false; + _ -> + %% Tombstone still around + true + end + end, + [] == lists:filter(StillThere, Keys). + +%% @doc PBC-based version of {@link systest_write/1} +pbc_systest_write(Node, Size) -> + pbc_systest_write(Node, Size, 2). + +pbc_systest_write(Node, Size, W) -> + pbc_systest_write(Node, 1, Size, <<"systest">>, W). + +pbc_systest_write(Node, Start, End, Bucket, W) -> + rt:wait_for_service(Node, riak_kv), + Pid = pbc(Node), + F = fun(N, Acc) -> + Obj = riakc_obj:new(Bucket, <>, <>), + try riakc_pb_socket:put(Pid, Obj, W) of + ok -> + Acc; + Other -> + [{N, Other} | Acc] + catch + What:Why -> + [{N, {What, Why}} | Acc] + end + end, + lists:foldl(F, [], lists:seq(Start, End)). + +pbc_systest_read(Node, Size) -> + pbc_systest_read(Node, Size, 2). + +pbc_systest_read(Node, Size, R) -> + pbc_systest_read(Node, 1, Size, <<"systest">>, R). + +pbc_systest_read(Node, Start, End, Bucket, R) -> + rt:wait_for_service(Node, riak_kv), + Pid = pbc(Node), + F = fun(N, Acc) -> + case riakc_pb_socket:get(Pid, Bucket, <>, R) of + {ok, Obj} -> + case riakc_obj:get_value(Obj) of + <> -> + Acc; + WrongVal -> + [{N, {wrong_val, WrongVal}} | Acc] + end; + Other -> + [{N, Other} | Acc] + end + end, + lists:foldl(F, [], lists:seq(Start, End)). + +-spec get_pb_conn_info(node()) -> [{inet:ip_address(), pos_integer()}]. +get_pb_conn_info(Node) -> + lager:debug("Querying connection pb connection information for node ~p", [Node]), + case rt2:rpc_get_env(Node, [{riak_api, pb}, + {riak_api, pb_ip}, + {riak_kv, pb_ip}]) of + {ok, [{NewIP, NewPort}|_]} -> + {ok, [{NewIP, NewPort}]}; + {ok, PB_IP} -> + {ok, PB_Port} = rt2:rpc_get_env(Node, [{riak_api, pb_port}, + {riak_kv, pb_port}]), + {ok, [{PB_IP, PB_Port}]}; + _ -> + undefined + end. diff --git a/src/rt_planner.erl b/src/rt_planner.erl new file mode 100644 index 000000000..14bd2c5be --- /dev/null +++ b/src/rt_planner.erl @@ -0,0 +1,363 @@ +%%------------------------------------------------------------------- +%% +%% Copyright (c) 2015 Basho Technologies, Inc. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% ------------------------------------------------------------------- +%% @author Brett Hazen +%% @copyright (C) 2015, Basho Technologies +%% @doc +%% Module to manage the list of pending test plans and hand off work +%% to the appropriate test scheduler. +%% @end +%% Created : 30. Mar 2015 10:25 AM +%%------------------------------------------------------------------- +-module(rt_planner). +-author("Brett Hazen"). + +-behaviour(gen_server). + +-ifdef(TEST). +-include_lib("eunit/include/eunit.hrl"). +-endif. + +%% API +-export([start_link/0, + load_from_giddyup/2, + add_test_plan/5, + fetch_test_plan/0, + fetch_test_non_runnable_plan/0, + number_of_plans/0, + number_of_non_runable_plans/0, + stop/0]). + +%% gen_server callbacks +-export([init/1, + handle_call/3, + handle_cast/2, + handle_info/2, + terminate/2, + code_change/3]). + +-define(SERVER, ?MODULE). + +-record(state, { + %% Tests which are deemed to be runable + runnable_test_plans :: queue(), + %% Tests which are deemed not to be runable + non_runnable_test_plans :: queue() +}). + +%%%=================================================================== +%%% API +%%%=================================================================== + +%%-------------------------------------------------------------------- +%% @doc +%% Starts the server +%% +%% @end +%%-------------------------------------------------------------------- +-spec(start_link() -> + {ok, Pid :: pid()} | ignore | {error, Reason :: term()}). +start_link() -> + gen_server:start_link({local, ?SERVER}, ?MODULE, [], []). + +%%-------------------------------------------------------------------- +%% @doc +%% Reads the list of test plans from GiddyUp and queues them up +%% +%% @end +%%-------------------------------------------------------------------- +-spec(load_from_giddyup([string()] | undefined, list()) -> ok). +load_from_giddyup(Backends, CommandLineTests) -> + gen_server:call(?MODULE, {load_from_giddyup, Backends, CommandLineTests}). + +%%-------------------------------------------------------------------- +%% @doc +%% Queue up a new test plan +%% +%% @end +%%-------------------------------------------------------------------- +-spec(add_test_plan(string(), string(), [atom()], [rt_properties2:product_version()], rt_properties2:properties()) -> ok). +add_test_plan(Module, Platform, Backends, UpgradePaths, Version) -> + gen_server:call(?MODULE, {add_test_plan, Module, Platform, Backends, UpgradePaths, Version}). + +%%-------------------------------------------------------------------- +%% @doc +%% Fetch a test plan off the queue +%% +%% @end +%%-------------------------------------------------------------------- +-spec(fetch_test_plan() -> rt_test_plan:test_plan() | empty). +fetch_test_plan() -> + gen_server:call(?MODULE, fetch_test_plan). + +%%-------------------------------------------------------------------- +%% @doc +%% Fetch a test plan off the queue +%% +%% @end +%%-------------------------------------------------------------------- +-spec(fetch_test_non_runnable_plan() -> rt_test_plan:test_plan() | empty). +fetch_test_non_runnable_plan() -> + gen_server:call(?MODULE, fetch_test_non_runnable_plan). + +%%-------------------------------------------------------------------- +%% @doc +%% Return the number of runable test plans in the queue +%% +%% @end +%%-------------------------------------------------------------------- +-spec(number_of_plans() -> rt_test_plan:test_plan() | empty). +number_of_plans() -> + gen_server:call(?MODULE, number_of_plans). + +%%-------------------------------------------------------------------- +%% @doc +%% Return the number of non-runable test plans in the queue +%% +%% @end +%%-------------------------------------------------------------------- +-spec(number_of_non_runable_plans() -> rt_test_plan:test_plan() | empty). +number_of_non_runable_plans() -> + gen_server:call(?MODULE, number_of_non_runable_plans). + +%%-------------------------------------------------------------------- +%% @doc +%% Stops the server +%% +%% @end +%%-------------------------------------------------------------------- +-spec stop() -> ok. +stop() -> + gen_server:call(?MODULE, stop, infinity). + +%%%=================================================================== +%%% gen_server callbacks +%%%=================================================================== + +%%-------------------------------------------------------------------- +%% @private +%% @doc +%% Initializes the server +%% +%% @spec init(Args) -> {ok, State} | +%% {ok, State, Timeout} | +%% ignore | +%% {stop, Reason} +%% @end +%%-------------------------------------------------------------------- +-spec(init(Args :: term()) -> + {ok, State :: #state{}} | {ok, State :: #state{}, timeout() | hibernate} | + {stop, Reason :: term()} | ignore). +init([]) -> + {ok, #state{runnable_test_plans=queue:new(), + non_runnable_test_plans=queue:new()}}. + +%%-------------------------------------------------------------------- +%% @private +%% @doc +%% Handling call messages +%% +%% @end +%%-------------------------------------------------------------------- +-spec(handle_call(Request :: term(), From :: {pid(), Tag :: term()}, + State :: #state{}) -> + {reply, Reply :: term(), NewState :: #state{}} | + {reply, Reply :: term(), NewState :: #state{}, timeout() | hibernate} | + {noreply, NewState :: #state{}} | + {noreply, NewState :: #state{}, timeout() | hibernate} | + {stop, Reason :: term(), Reply :: term(), NewState :: #state{}} | + {stop, Reason :: term(), NewState :: #state{}}). +%% Run only those GiddyUp tests which are specified on the command line and are +%% included in the specified backends. +%% If none are specified, run everything +handle_call({load_from_giddyup, Backends, CommandLineTests}, _From, State) -> + AllGiddyupTests = giddyup:get_test_plans(), + {Included, Excluded} = case CommandLineTests of + [] -> + {AllGiddyupTests, []}; + _ -> + Inc = [TestPlan || TestPlan <- AllGiddyupTests, + CName <- CommandLineTests, + rt_test_plan:get_module(TestPlan) =:= CName], + {Inc, lists:filter(fun(Elem) -> not lists:member(Elem, Inc) end, AllGiddyupTests)} + end, + {Included1, Excluded1} = case Backends of + undefined -> + {Included, Excluded}; + _ -> + Inc1 = [TestPlan || TestPlan <- Included, + lists:member(rt_test_plan:get(backend, TestPlan), Backends)], + {Inc1, lists:filter(fun(Elem) -> not lists:member(Elem, Inc1) end, AllGiddyupTests)} + end, + State1 = lists:foldl(fun sort_and_queue/2, State, Included1), + State2 = lists:foldl(fun exclude_test_plan/2, State1, Excluded1), + {reply, ok, State2}; +%% Add a single test plan for each backend to the queue +handle_call({add_test_plan, Module, Platform, Backends, UpgradePaths, _Version}, _From, State) -> + State1 = lists:foldl(fun(Backend, AccState) -> + case UpgradePaths of + undefined -> + TestPlan = rt_test_plan:new([{module, Module}, {platform, Platform}, {backend, Backend}]), + sort_and_queue(TestPlan, AccState); + UpgradePaths -> + lists:foldl(fun(UpgradePath, AccState1) -> + TestPlan = rt_test_plan:new([{module, Module}, {platform, Platform}, {backend, Backend}, {upgrade_path, UpgradePath}]), + sort_and_queue(TestPlan, AccState1) + end, + AccState, UpgradePaths) + end + end, + State, Backends), + {reply, ok, State1}; +handle_call(fetch_test_plan, _From, State) -> + Q = State#state.runnable_test_plans, + {Item, Q1} = queue:out(Q), + Result = case Item of + {value, Value} -> Value; + Empty -> Empty + end, + {reply, Result, State#state{runnable_test_plans=Q1}}; +handle_call(fetch_test_non_runnable_plan, _From, State) -> + Q = State#state.non_runnable_test_plans, + {Item, Q1} = queue:out(Q), + Result = case Item of + {value, Value} -> Value; + Empty -> Empty + end, + {reply, Result, State#state{non_runnable_test_plans=Q1}}; +handle_call(number_of_plans, _From, State) -> + Q = State#state.runnable_test_plans, + {reply, queue:len(Q), State}; +handle_call(number_of_non_runable_plans, _From, State) -> + Q = State#state.non_runnable_test_plans, + {reply, queue:len(Q), State}; +handle_call(_Request, _From, State) -> + {reply, ok, State}. + +%%-------------------------------------------------------------------- +%% @private +%% @doc +%% Handling cast messages +%% +%% @end +%%-------------------------------------------------------------------- +-spec(handle_cast(Request :: term(), State :: #state{}) -> + {noreply, NewState :: #state{}} | + {noreply, NewState :: #state{}, timeout() | hibernate} | + {stop, Reason :: term(), NewState :: #state{}}). +handle_cast(_Request, State) -> + {noreply, State}. + +%%-------------------------------------------------------------------- +%% @private +%% @doc +%% Handling all non call/cast messages +%% +%% @spec handle_info(Info, State) -> {noreply, State} | +%% {noreply, State, Timeout} | +%% {stop, Reason, State} +%% @end +%%-------------------------------------------------------------------- +-spec(handle_info(Info :: timeout() | term(), State :: #state{}) -> + {noreply, NewState :: #state{}} | + {noreply, NewState :: #state{}, timeout() | hibernate} | + {stop, Reason :: term(), NewState :: #state{}}). +handle_info(_Info, State) -> + {noreply, State}. + +%%-------------------------------------------------------------------- +%% @private +%% @doc +%% This function is called by a gen_server when it is about to +%% terminate. It should be the opposite of Module:init/1 and do any +%% necessary cleaning up. When it returns, the gen_server terminates +%% with Reason. The return value is ignored. +%% +%% @spec terminate(Reason, State) -> void() +%% @end +%%-------------------------------------------------------------------- +-spec(terminate(Reason :: (normal | shutdown | {shutdown, term()} | term()), + State :: #state{}) -> term()). +terminate(_Reason, _State) -> + ok. + +%%-------------------------------------------------------------------- +%% @private +%% @doc +%% Convert process state when code is changed +%% +%% @spec code_change(OldVsn, State, Extra) -> {ok, NewState} +%% @end +%%-------------------------------------------------------------------- +-spec(code_change(OldVsn :: term() | {down, term()}, State :: #state{}, + Extra :: term()) -> + {ok, NewState :: #state{}} | {error, Reason :: term()}). +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +%%%=================================================================== +%%% Internal functions +%%%=================================================================== + +%%-------------------------------------------------------------------- +%% @private +%% @doc +%% Add a single test plan +%% +%% @end +%%-------------------------------------------------------------------- + +%%-------------------------------------------------------------------- +%% @private +%% @doc +%% Add a specific test into the proper list +%% +%% @end +%%-------------------------------------------------------------------- +sort_and_queue(TestPlan, State) -> + QR = State#state.runnable_test_plans, + QNR = State#state.non_runnable_test_plans, + {QR2, QNR2} = case is_runnable_test_plan(TestPlan) of + true -> + {queue:in(TestPlan, QR), QNR}; + _ -> + {QR, queue:in(TestPlan, QNR)} + end, + State#state{runnable_test_plans=QR2, + non_runnable_test_plans=QNR2}. + +%% Check for api compatibility +%% TODO: Move into "harness" or "driver" since it might be on a remote node. +is_runnable_test_plan(TestPlan) -> + TestModule = rt_test_plan:get_module(TestPlan), + {Mod, Fun} = riak_test_runner:function_name(confirm, TestModule), + + code:ensure_loaded(Mod), + erlang:function_exported(Mod, Fun, 0) orelse + erlang:function_exported(Mod, Fun, 1). + +%%-------------------------------------------------------------------- +%% @private +%% @doc +%% Add a unused test to the list of non_runnable_test_plans +%% @end +%%-------------------------------------------------------------------- +exclude_test_plan(TestPlan, State) -> + QNR = queue:in(TestPlan, State#state.non_runnable_test_plans), + State#state{non_runnable_test_plans=QNR}. \ No newline at end of file diff --git a/src/rt_properties.erl b/src/rt_properties.erl new file mode 100644 index 000000000..8b1fb66d6 --- /dev/null +++ b/src/rt_properties.erl @@ -0,0 +1,265 @@ +-module(rt_properties). +%% ------------------------------------------------------------------- +%% +%% Copyright (c) 2013 Basho Technologies, Inc. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% ------------------------------------------------------------------- + +%% @doc Implements a set of functions for accessing and manipulating +%% an `rt_properties' record. + +%%-record(rt_cluster_topology_v1, { +%% name :: atom(), +%% connected_to :: [atom()], +%% override_properties :: proplists:proplist() +%% }). +%%-type topology() :: #rt_cluster_topology_v1{}. + +%% A quick note on the distinction between `node_ids' and +%% `node_map'. `node_ids' are short identifers (e.g. dev1) and the +%% `node_map' maps each node_id to a full erlang node names. Both are +%% necessary because the different existing helper functions make use +%% of each one to either compose shell commands or execute rpc calls. +%% The `node_map' is used to make the details of the actual node +%% names, which are harness-specific, opaque to the helper functions +%% and ensure that the helpers are valid for use with any harness. +-record(rt_properties_v1, { + node_ids :: [string()], + node_map :: [{string(), node()}], + node_count=3 :: non_neg_integer(), + metadata=[] :: proplists:proplist(), + rolling_upgrade=false :: boolean(), + start_version=rt_config:get_default_version() :: string(), + current_version :: string(), + upgrade_path=undefined :: [atom()], + wait_for_transfers=false :: boolean(), + valid_backends=all :: all | [atom()], + make_cluster=true :: boolean(), + cluster_count=1 :: pos_integer(), + cluster_weights :: [float()], + clusters :: [rt_cluster_info:cluster_info()], + required_services=[riak_kv] :: [atom()], + bucket_types=[] :: bucket_types(), + config=default_config() :: term(), + external_properties :: term() % arbitrary properties for 3rd party use + }). +%%-record(rt_properties_v2, { +%% description :: string(), +%% supported_products :: [atom()], %% TODO Use the product type when exported ... +%% minimum_version :: string(), %% TODO Use the version types when exported ... +%% maximum_version :: string(), %% TODO Use the version types when exported ... +%% supported_backends=all :: [atom()], +%% node_count :: non_neg_integer(), +%% wait_for_transfers=false :: boolean(), +%% bucket_types=[] :: bucket_types(), +%% indexes=[] :: [index()], +%% ring_size=auto :: [atom() | non_neg_integer()], +%% enable_strong_consistency=false :: boolean(), +%% enable_yokozuna=false :: boolean(), +%% enable_jmx=false :: boolean(), +%% enable_snmp=false :: boolean(), +%% config=default_config() :: term(), +%% required_services=[riak_kv] :: [atom()] + %% TODO What elements are needed to configure security? +%% }). + +-type properties() :: #rt_properties_v1{}. + +%% Specify the bucket_types field for the properties record. The list +%% of bucket types may have two forms, a bucket_type or a pair +%% consisting of an integer and a bucket_type. The latter form +%% indicates that a bucket_type should only be applied to the cluster +%% with the given index. The former form is applied to all clusters. +-type bucket_type() :: {binary(), proplists:proplist()}. +-type bucket_types() :: [bucket_type() | {pos_integer(), bucket_type()}]. +%%-type index() :: {binary(), binary(), binary()}. + +-export_type([properties/0, +%% index/0, + bucket_types/0]). + +-define(RT_PROPERTIES, #rt_properties_v1). +-define(RECORD_FIELDS, record_info(fields, rt_properties_v1)). + +-export([new/0, + new/1, + get/2, + set/2, + set/3, + default_config/0]). + +%% @doc Create a new properties record with all fields initialized to +%% the default values. +-spec new() -> properties(). +new() -> + ?RT_PROPERTIES{}. + +%% @doc Create a new properties record with the fields initialized to +%% non-default value. Each field to be initialized should be +%% specified as an entry in a property list (i.e. a list of +%% pairs). Invalid property fields are ignored by this function. +-spec new(proplists:proplist()) -> properties(). +new(PropertyDefaults) -> + {Properties, _} = + lists:foldl(fun set_property/2, {?RT_PROPERTIES{}, []}, PropertyDefaults), + Properties. + +%% @doc Get the value of a property from a properties record. An error +%% is returned if `Properties' is not a valid `rt_properties' record +%% or if the property requested is not a valid property. +-spec get(atom(), properties()) -> term() | {error, atom()}. +get(Property, Properties) -> + get(Property, Properties, validate_request(Property, Properties)). + +%% @doc Set the value for a property in a properties record. An error +%% is returned if `Properties' is not a valid `rt_properties' record +%% or if any of the properties to be set are not a valid property. In +%% the case that invalid properties are specified the error returned +%% contains a list of erroneous properties. +-spec set([{atom(), term()}], properties()) -> properties() | {error, atom()}. +set(PropertyList, Properties) when is_list(PropertyList) -> + set_properties(PropertyList, Properties, validate_record(Properties)). + +%% @doc Set the value for a property in a properties record. An error +%% is returned if `Properties' is not a valid `rt_properties' record +%% or if the property to be set is not a valid property. +-spec set(atom(), term(), properties()) -> {ok, properties()} | {error, atom()}. +set(Property, Value, Properties) -> + set_property(Property, Value, Properties, validate_request(Property, Properties)). + + +-spec get(atom(), properties(), ok | {error, atom()}) -> + term() | {error, atom()}. +get(Property, Properties, ok) -> + element(field_index(Property), Properties); +get(_Property, _Properties, {error, _}=Error) -> + Error. + +%% This function is used by `new/1' to set properties at record +%% creation time and by `set/2' to set multiple properties at once. +%% Node properties record validation is done by this function. It is +%% strictly used as a fold function which is the reason for the odd +%% structure of the input parameters. It accumulates any invalid +%% properties that are encountered and the caller may use that +%% information or ignore it. +-spec set_property({atom(), term()}, {properties(), [atom()]}) -> + {properties(), [atom()]}. +set_property({Property, Value}, {Properties, Invalid}) -> + case is_valid_property(Property) of + true -> + {setelement(field_index(Property), Properties, Value), Invalid}; + false -> + {Properties, [Property | Invalid]} + end. + +-spec set_property(atom(), term(), properties(), ok | {error, atom()}) -> + {ok, properties()} | {error, atom()}. +set_property(Property, Value, Properties, ok) -> + {ok, setelement(field_index(Property), Properties, Value)}; +set_property(_Property, _Value, _Properties, {error, _}=Error) -> + Error. + +-spec set_properties([{atom(), term()}], + properties(), + ok | {error, {atom(), [atom()]}}) -> + {properties(), [atom()]}. +set_properties(PropertyList, Properties, ok) -> + case lists:foldl(fun set_property/2, {Properties, []}, PropertyList) of + {UpdProperties, []} -> + UpdProperties; + {_, InvalidProperties} -> + {error, {invalid_properties, InvalidProperties}} + end; +set_properties(_, _, {error, _}=Error) -> + Error. + +-spec validate_request(atom(), term()) -> ok | {error, atom()}. +validate_request(Property, Properties) -> + validate_property(Property, validate_record(Properties)). + +-spec validate_record(term()) -> ok | {error, invalid_properties}. +validate_record(Record) -> + case is_valid_record(Record) of + true -> + ok; + false -> + {error, invalid_properties} + end. + +-spec validate_property(atom(), ok | {error, atom()}) -> ok | {error, invalid_property}. +validate_property(Property, ok) -> + case is_valid_property(Property) of + true -> + ok; + false -> + {error, invalid_property} + end; +validate_property(_Property, {error, _}=Error) -> + Error. + +-spec default_config() -> [term()]. +default_config() -> + [{riak_core, [{handoff_concurrency, 11}]}, + {riak_search, [{enabled, true}]}, + {riak_pipe, [{worker_limit, 200}]}]. + +-spec is_valid_record(term()) -> boolean(). +is_valid_record(Record) -> + is_record(Record, rt_properties_v1). + +-spec is_valid_property(atom()) -> boolean(). +is_valid_property(Property) -> + Fields = ?RECORD_FIELDS, + lists:member(Property, Fields). + +-spec field_index(atom()) -> non_neg_integer(). +field_index(node_ids) -> + ?RT_PROPERTIES.node_ids; +field_index(node_map) -> + ?RT_PROPERTIES.node_map; +field_index(node_count) -> + ?RT_PROPERTIES.node_count; +field_index(metadata) -> + ?RT_PROPERTIES.metadata; +field_index(rolling_upgrade) -> + ?RT_PROPERTIES.rolling_upgrade; +field_index(start_version) -> + ?RT_PROPERTIES.start_version; +field_index(current_version) -> + ?RT_PROPERTIES.current_version; +field_index(upgrade_path) -> + ?RT_PROPERTIES.upgrade_path; +field_index(wait_for_transfers) -> + ?RT_PROPERTIES.wait_for_transfers; +field_index(valid_backends) -> + ?RT_PROPERTIES.valid_backends; +field_index(make_cluster) -> + ?RT_PROPERTIES.make_cluster; +field_index(cluster_count) -> + ?RT_PROPERTIES.cluster_count; +field_index(cluster_weights) -> + ?RT_PROPERTIES.cluster_weights; +field_index(bucket_types) -> + ?RT_PROPERTIES.bucket_types; +field_index(clusters) -> + ?RT_PROPERTIES.clusters; +field_index(required_services) -> + ?RT_PROPERTIES.required_services; +field_index(config) -> + ?RT_PROPERTIES.config; +field_index(external_properties) -> + ?RT_PROPERTIES.external_properties. diff --git a/src/rt_properties2.erl b/src/rt_properties2.erl new file mode 100644 index 000000000..b7b52a62f --- /dev/null +++ b/src/rt_properties2.erl @@ -0,0 +1,213 @@ +-module(rt_properties2). +%% ------------------------------------------------------------------- +%% +%% Copyright (c) 2015 Basho Technologies, Inc. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% ------------------------------------------------------------------- + +%% @doc Implements a set of functions for accessing and manipulating +%% an `rt_properties2' record. + +-type cluster_name() :: atom(). +-record(rt_cluster_topology_v1, { + name :: cluster_name(), + %% TODO: Account for full sync/real time connection types, ssl, repl protocol version + connected_to :: [] | [cluster_name()], + nodes :: pos_integer() | [rt_util:version_selector()] +}). + + +-record(rt_properties_v2, { + description :: string(), + supported_products=riak :: [rt_util:products()], + minimum_version=any :: any | rt_util:release(), + maximum_version=any :: any | rt_util:release(), + cluster_topology=default_topology(3) :: [topology()], + groups=[] :: [atom()], + driver_configuration=rt_driver:new_configuration() :: rt_driver:configuration(), + %% TODO Do we need these two properties since the versions are specified in the topology + %% and default will be used when is specified and the upgrade pathes are specified in + %% the configuration ... + default_version=rt_config:get_default_version() :: rt_util:version(), + upgrade_path :: [rt_util:version_selector()] +}). + +%% What if we moved the Riak specific bits to an rt_riak_driver module +%% and provided some additional callback functions for product +%% specific extension points ... + +-type properties() :: #rt_properties_v2{}. +-type topology() :: #rt_cluster_topology_v1{}. +-export_type([properties/0, + topology/0]). + +-define(RT_CLUSTER_TOPOLOGY, #rt_cluster_topology_v1). +-define(RT_PROPERTIES, #rt_properties_v2). +-define(RECORD_FIELDS, record_info(fields, rt_properties_v2)). + +-export([new/0, + new/1, + get/2, + get_configuration_key/2, + set/2, + set/3, + set_configuration_key/3, + default_topology/1]). + +%% @doc Create a new properties record with all fields initialized to +%% the default values. +-spec new() -> properties(). +new() -> + ?RT_PROPERTIES{}. + +%% @doc Create a new properties record with the fields initialized to +%% non-default value. Each field to be initialized should be +%% specified as an entry in a property list (i.e. a list of +%% pairs). Invalid property fields are ignored by this function. +-spec new(proplists:proplist()) -> properties(). +new(PropertyDefaults) -> + {Properties, _} = + lists:foldl(fun set_property/2, {?RT_PROPERTIES{}, []}, PropertyDefaults), + Properties. + +%% @doc Get the value of a property from a properties record. An error +%% is returned if `Properties' is not a valid `rt_properties2' record +%% or if the property requested is not a valid property. +-spec get(atom(), properties()) -> term() | {error, atom()}. +get(Property, Properties) -> + get(Property, Properties, validate_request(Property, Properties)). + +%% @doc Set the value for a property in a properties record. An error +%% is returned if `Properties' is not a valid `rt_properties2' record +%% or if any of the properties to be set are not a valid property. In +%% the case that invalid properties are specified the error returned +%% contains a list of erroneous properties. +-spec set([{atom(), term()}], properties()) -> properties() | {error, atom()}. +set(PropertyList, Properties) when is_list(PropertyList) -> + set_properties(PropertyList, Properties, validate_record(Properties)). + +%% @doc Set the value for a property in a properties record. An error +%% is returned if `Properties' is not a valid `rt_properties2' record +%% or if the property to be set is not a valid property. +-spec set(atom(), term(), properties()) -> {ok, properties()} | {error, atom()}. +set(Property, Value, Properties) -> + set_property(Property, Value, Properties, validate_request(Property, Properties)). + +-spec get(atom(), properties(), ok | {error, atom()}) -> + term() | {error, atom()}. +get(Property, Properties, ok) -> + element(field_index(Property), Properties); +get(_Property, _Properties, {error, _}=Error) -> + Error. + +%% This function is used by `new/1' to set properties at record +%% creation time and by `set/2' to set multiple properties at once. +%% Node properties record validation is done by this function. It is +%% strictly used as a fold function which is the reason for the odd +%% structure of the input parameters. It accumulates any invalid +%% properties that are encountered and the caller may use that +%% information or ignore it. +-spec set_property({atom(), term()}, {properties(), [atom()]}) -> + {properties(), [atom()]}. +set_property({Property, Value}, {Properties, Invalid}) -> + case is_valid_property(Property) of + true -> + {setelement(field_index(Property), Properties, Value), Invalid}; + false -> + {Properties, [Property | Invalid]} + end. + +-spec set_property(atom(), term(), properties(), ok | {error, atom()}) -> + {ok, properties()} | {error, atom()}. +set_property(Property, Value, Properties, ok) -> + {ok, setelement(field_index(Property), Properties, Value)}; +set_property(_Property, _Value, _Properties, {error, _}=Error) -> + Error. + +-spec set_properties([{atom(), term()}], + properties(), + ok | {error, {atom(), [atom()]}}) -> + {properties(), [atom()]}. +set_properties(PropertyList, Properties, ok) -> + case lists:foldl(fun set_property/2, {Properties, []}, PropertyList) of + {UpdProperties, []} -> + UpdProperties; + {_, InvalidProperties} -> + {error, {invalid_properties, InvalidProperties}} + end; +set_properties(_, _, {error, _}=Error) -> + Error. + +-spec get_configuration_key(atom(), properties()) -> term() | {error, string()}. +get_configuration_key(Key, Properties) -> + DriverConfiguration = get(driver_configuration, Properties), + rt_driver:get_configuration_key(DriverConfiguration, Key). + +-spec set_configuration_key(atom(), term(), properties()) -> {ok, term()} | {error, string()}. +set_configuration_key(Key, Value, Properties) -> + DriverConfiguration = get(driver_configuration, Properties), + rt_driver:set_configuration_key(Key, Value, DriverConfiguration). + +-spec validate_request(atom(), term()) -> ok | {error, atom()}. +validate_request(Property, Properties) -> + validate_property(Property, validate_record(Properties)). + +-spec validate_record(term()) -> ok | {error, invalid_properties}. +validate_record(Record) -> + case is_valid_record(Record) of + true -> + ok; + false -> + {error, invalid_properties} + end. + +-spec validate_property(atom(), ok | {error, atom()}) -> ok | {error, invalid_property}. +validate_property(Property, ok) -> + case is_valid_property(Property) of + true -> + ok; + false -> + {error, invalid_property} + end; +validate_property(_Property, {error, _}=Error) -> + Error. + +%% @doc Create a single default cluster topology with default node versions +-spec default_topology(pos_integer()) -> [topology()]. +default_topology(N) -> + ?RT_CLUSTER_TOPOLOGY{name=cluster1, connected_to=[], nodes=[rt_config:get_default_version() || lists:seq(1,N)]}. + +-spec is_valid_record(term()) -> boolean(). +is_valid_record(Record) -> + is_record(Record, rt_properties_v2). + +-spec is_valid_property(atom()) -> boolean(). +is_valid_property(Property) -> + Fields = ?RECORD_FIELDS, + lists:member(Property, Fields). + +-spec field_index(atom()) -> non_neg_integer(). +field_index(description) -> + ?RT_PROPERTIES.description; +field_index(supported_products) -> + ?RT_PROPERTIES.supported_products; +field_index(upgrade_path) -> + ?RT_PROPERTIES.upgrade_path; +field_index(cluster_topology) -> + ?RT_PROPERTIES.cluster_topology; +field_index(default_version) -> + ?RT_PROPERTIES.default_version. diff --git a/src/rt_reporter.erl b/src/rt_reporter.erl new file mode 100644 index 000000000..616251c32 --- /dev/null +++ b/src/rt_reporter.erl @@ -0,0 +1,381 @@ +%%------------------------------------------------------------------- +%% +%% Copyright (c) 2015 Basho Technologies, Inc. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% ------------------------------------------------------------------- +%% @author Brett Hazen +%% @copyright (C) 2015, Basho Technologies +%% @doc +%% +%% @end +%% Created : 31. Mar 2015 10:25 AM +%%------------------------------------------------------------------- +-module(rt_reporter). +-author("Brett Hazen"). + +-behaviour(gen_server). + +-define(HEADER, [<<"Test">>, <<"Result">>, <<"Reason">>, <<"Test Duration">>]). +-ifdef(TEST). +-include_lib("eunit/include/eunit.hrl"). +-endif. + + +%% API +-export([start_link/3, + stop/0, + send_result/1]). + +%% gen_server callbacks +-export([init/1, + handle_call/3, + handle_cast/2, + handle_info/2, + terminate/2, + code_change/3]). + +-define(SERVER, ?MODULE). + +-record(state, { + % Collection of log files + % Running summary of test results: {test, pass/fail, duration} + summary :: list(), + log_dir :: string(), + %% True if results should be uploaded to GiddyUp + giddyup :: boolean(), + %% PID of escript used to update results + notify_pid :: pid() +}). + +%%%=================================================================== +%%% API +%%%=================================================================== + +%%-------------------------------------------------------------------- +%% @doc +%% Starts the server +%% +%% @end +%%-------------------------------------------------------------------- +-spec(start_link(boolean(), string(), pid()) -> + {ok, Pid :: pid()} | ignore | {error, Reason :: term()}). +start_link(UploadToGiddyUp, LogDir, NotifyPid) -> + gen_server:start_link({local, ?SERVER}, ?MODULE, [UploadToGiddyUp, LogDir, NotifyPid], []). + +%%-------------------------------------------------------------------- +%% @doc +%% Stops the server +%% +%% @end +%%-------------------------------------------------------------------- +-spec stop() -> ok. +stop() -> + gen_server:call(?MODULE, stop, infinity). + + +%% @doc Send an asychronous message to the reporter +%% -spec send_cast(term()) -> ok. +%% send_cast(Msg) -> +%% gen_server:cast(?MODULE, Msg). + +%% @doc Send a sychronous message to the reporter +%% -spec send_call(term()) -> ok. +%% send_call(Msg) -> +%% gen_server:call(?MODULE, Msg). + +%% @doc Send the test result to the reporter +-spec send_result(term()) -> ok. +send_result(Msg) -> + %% TODO: Determine proper timeout + gen_server:call(?MODULE, Msg, 30000). + + +%%%=================================================================== +%%% gen_server callbacks +%%%=================================================================== + +%%-------------------------------------------------------------------- +%% @private +%% @doc +%% Initializes the server +%% +%% @spec init(Args) -> {ok, State} | +%% {ok, State, Timeout} | +%% ignore | +%% {stop, Reason} +%% @end +%%-------------------------------------------------------------------- +-spec(init(Args :: term()) -> + {ok, State :: #state{}} | {ok, State :: #state{}, timeout() | hibernate} | + {stop, Reason :: term()} | ignore). +init([UploadToGiddyUp, LogDir, NotifyPid]) -> + {ok, #state{summary=[], + log_dir=LogDir, + giddyup=UploadToGiddyUp, + notify_pid=NotifyPid}}. + +%%-------------------------------------------------------------------- +%% @private +%% @doc +%% Handling call messages +%% +%% @end +%%-------------------------------------------------------------------- +-spec(handle_call(Request :: term(), From :: {pid(), Tag :: term()}, + State :: #state{}) -> + {reply, Reply :: term(), NewState :: #state{}} | + {reply, Reply :: term(), NewState :: #state{}, timeout() | hibernate} | + {noreply, NewState :: #state{}} | + {noreply, NewState :: #state{}, timeout() | hibernate} | + {stop, Reason :: term(), Reply :: term(), NewState :: #state{}} | + {stop, Reason :: term(), NewState :: #state{}}). +handle_call({test_result, Result}, From, State) -> + Results = State#state.summary, + State#state.notify_pid ! {From, {test_result, Result}}, + report_and_gather_logs(State#state.giddyup, State#state.log_dir, Result), + {reply, ok, State#state{summary=[Result|Results]}}; +handle_call(done, From, State) -> + State#state.notify_pid ! {From, done}, + print_summary(State#state.summary, undefined, true), + {reply, ok, State}; +handle_call(_Request, _From, State) -> + {reply, ok, State}. + +%%-------------------------------------------------------------------- +%% @private +%% @doc +%% Handling cast messages +%% +%% @end +%%-------------------------------------------------------------------- +-spec(handle_cast(Request :: term(), State :: #state{}) -> + {noreply, NewState :: #state{}} | + {noreply, NewState :: #state{}, timeout() | hibernate} | + {stop, Reason :: term(), NewState :: #state{}}). +handle_cast(_Request, State) -> + {noreply, State}. + +%%-------------------------------------------------------------------- +%% @private +%% @doc +%% Handling all non call/cast messages +%% +%% @spec handle_info(Info, State) -> {noreply, State} | +%% {noreply, State, Timeout} | +%% {stop, Reason, State} +%% @end +%%-------------------------------------------------------------------- +-spec(handle_info(Info :: timeout() | term(), State :: #state{}) -> + {noreply, NewState :: #state{}} | + {noreply, NewState :: #state{}, timeout() | hibernate} | + {stop, Reason :: term(), NewState :: #state{}}). +handle_info(_Info, State) -> + {noreply, State}. + +%%-------------------------------------------------------------------- +%% @private +%% @doc +%% This function is called by a gen_server when it is about to +%% terminate. It should be the opposite of Module:init/1 and do any +%% necessary cleaning up. When it returns, the gen_server terminates +%% with Reason. The return value is ignored. +%% +%% @spec terminate(Reason, State) -> void() +%% @end +%%-------------------------------------------------------------------- +-spec(terminate(Reason :: (normal | shutdown | {shutdown, term()} | term()), + State :: #state{}) -> term()). +terminate(_Reason, _State) -> + ok. + +%%-------------------------------------------------------------------- +%% @private +%% @doc +%% Convert process state when code is changed +%% +%% @spec code_change(OldVsn, State, Extra) -> {ok, NewState} +%% @end +%%-------------------------------------------------------------------- +-spec(code_change(OldVsn :: term() | {down, term()}, State :: #state{}, + Extra :: term()) -> + {ok, NewState :: #state{}} | {error, Reason :: term()}). +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +%%%=================================================================== +%%% Internal functions +%%%=================================================================== + +%%-------------------------------------------------------------------- +%% @private +%% @doc +%% Dump the summary of all of the log runs to the console +%% +%% @spec print_summary(TestResults, _CoverResult, Verbose) -> ok +%% @end +%%-------------------------------------------------------------------- +-spec(print_summary(list(), term(), boolean()) -> ok). +print_summary(TestResults, _CoverResult, Verbose) -> + %% TODO Log vs console output ... -jsb + lager:notice("Test Results:"), + + {StatusCounts, RowList} = lists:foldl(fun test_summary_fun/2, {{0,0,0}, []}, TestResults), + Rows = lists:reverse(RowList), + + case Verbose of + true -> + Table = clique_table:autosize_create_table(?HEADER, Rows), + [lager:notice(string:tokens(lists:flatten(FormattedRow), "\n")) || FormattedRow <- Table]; + false -> + ok + end, + + {PassCount, FailCount, SkippedCount} = StatusCounts, + lager:notice("---------------------------------------------"), + lager:notice("~w Tests Failed", [FailCount]), + lager:notice("~w Tests Skipped", [SkippedCount]), + lager:notice("~w Tests Passed", [PassCount]), + Percentage = case PassCount == 0 andalso FailCount == 0 of + true -> 0; + false -> (PassCount / (PassCount + FailCount + SkippedCount)) * 100 + end, + lager:notice("That's ~w% for those keeping score", [Percentage]), + + %% case CoverResult of + %% cover_disabled -> + %% ok; + %% {Coverage, AppCov} -> + %% io:format("Coverage : ~.1f%~n", [Coverage]), + %% [io:format(" ~s : ~.1f%~n", [App, Cov]) + %% || {App, Cov, _} <- AppCov] + %% end, + ok. + +%% @doc Convert Microseconds into human-readable string +-spec(test_summary_format_time(integer()) -> string()). +test_summary_format_time(Microseconds) -> + Micros = trunc(((Microseconds / 1000000) - (Microseconds div 1000000)) * 1000000), + TotalSecs = (Microseconds - Micros) div 1000000, + TotalMins = TotalSecs div 60, + Hours = TotalSecs div 3600, + Secs = TotalSecs - (TotalMins * 60), + Mins = TotalMins - (Hours * 60), + Decimal = lists:flatten(io_lib:format("~6..0B", [Micros])), + FirstDigit = string:left(Decimal, 1), + Fractional = string:strip(tl(Decimal), right, $0), + list_to_binary(io_lib:format("~ph ~pm ~p.~s~ss", [Hours, Mins, Secs, FirstDigit, Fractional])). + +%% @doc Count the number of passed, failed and skipped tests +test_summary_fun(Result = {_, pass, _}, {{Pass, _Fail, _Skipped}, Rows}) -> + FormattedRow = format_test_row(Result), + {{Pass+1, _Fail, _Skipped}, [FormattedRow|Rows]}; +test_summary_fun(Result = {_, {fail, _}, _}, {{_Pass, Fail, _Skipped}, Rows}) -> + FormattedRow = format_test_row(Result), + {{_Pass, Fail+1, _Skipped}, [FormattedRow|Rows]}; +test_summary_fun(Result = {_, {skipped, _}, _}, {{_Pass, _Fail, Skipped}, Rows}) -> + FormattedRow = format_test_row(Result), + {{_Pass, _Fail, Skipped+1}, [FormattedRow|Rows]}. + +%% @doc Format a row for clique +format_test_row({TestPlan, Result, Duration}) -> + TestName = rt_test_plan:get_name(TestPlan), + {Status, Reason} = case Result of + {FailOrSkip, Failure} when is_list(Failure) -> + {FailOrSkip, lists:flatten(Failure)}; + {FailOrSkip, Failure} -> + {FailOrSkip, lists:flatten(io_lib:format("~p", [Failure]))}; + pass -> + {pass, "N/A"} + end, + [TestName, Status, Reason, test_summary_format_time(Duration)]. + +-spec(report_to_giddyup(term(), list()) -> list). +report_to_giddyup(TestResult, Logs) -> + {TestPlan, Reason, _Duration} = TestResult, + giddyup:post_result(TestPlan, Reason), + [giddyup:post_artifact(TestPlan, Label, Filename) || {Label, Filename} <- Logs]. + +%%-------------------------------------------------------------------- +%% @private +%% @doc +%% Gather all of the log files from the nodes and either upload to +%% GiddyUp or copy them to the directory of your choice. Also upload +%% latest test result, if necessary. +%% +%% @spec report_and_gather_logs(Directory) -> ok +%% @end +%%-------------------------------------------------------------------- +-spec(report_and_gather_logs(boolean(), string(), term()) -> ok). +report_and_gather_logs(UploadToGiddyUp, LogDir, TestResult = {TestPlan, _, _}) -> + SubDir = filename:join([LogDir, rt_test_plan:get_name(TestPlan)]), + LogFile = filename:join([SubDir, "riak_test.log"]), + Logs = rt:get_node_logs(LogFile, SubDir), + case UploadToGiddyUp of + true -> + report_to_giddyup(TestResult, Logs); + _ -> + Logs + end. + +%% +%% RetList = [{test, TestModule}, {status, Status}, {log, Log}, {backend, Backend} | proplists:delete(backend, TestMetaData)], +%% case Status of +%% fail -> RetList ++ [{reason, iolist_to_binary(io_lib:format("~p", [Reason]))}]; +%% _ -> RetList +%% end. + +-ifdef(TEST). + +format_result_row_pass_test() -> + %% Need to prime the config with any old default version + rt_config:set(versions, [{default, {riak_ee, "1.3.4"}}]), + Plan = rt_test_plan:new([{module,test},{backend,bitcask}]), + ?assertEqual(["test-bitcask", pass, "N/A", <<"0h 0m 0.012345s">>], format_test_row({Plan, pass, 12345})). + +format_result_row_fail_atom_test() -> + %% Need to prime the config with any old default version + rt_config:set(versions, [{default, {riak_ee, "1.3.4"}}]), + Plan = rt_test_plan:new([{module,test},{backend,bitcask}]), + ?assertEqual(["test-bitcask", fail, "timeout", <<"0h 0m 0.012345s">>], format_test_row({Plan, {fail,timeout}, 12345})). + +format_result_row_fail_string_test() -> + %% Need to prime the config with any old default version + rt_config:set(versions, [{default, {riak_ee, "1.3.4"}}]), + Plan = rt_test_plan:new([{module,test},{backend,bitcask}]), + ?assertEqual(["test-bitcask", fail, "some reason", <<"0h 0m 0.012345s">>], format_test_row({Plan, {fail,"some reason"}, 12345})). + +format_result_row_fail_list_test() -> + %% Need to prime the config with any old default version + rt_config:set(versions, [{default, {riak_ee, "1.3.4"}}]), + Plan = rt_test_plan:new([{module,test},{backend,bitcask}]), + ?assertEqual(["test-bitcask", fail, "nested", <<"0h 0m 0.012345s">>], format_test_row({Plan, {fail,[[$n],[$e],[[$s]],[$t],$e,$d]}, 12345})). + +format_time_microsecond_test() -> + ?assertEqual(<<"0h 0m 0.000001s">>, test_summary_format_time(1)). + +format_time_millisecond_test() -> + ?assertEqual(<<"0h 0m 0.001s">>, test_summary_format_time(1000)). + +format_time_second_test() -> + ?assertEqual(<<"0h 0m 1.0s">>, test_summary_format_time(1000000)). + +format_time_minute_test() -> + ?assertEqual(<<"0h 1m 0.0s">>, test_summary_format_time(60000000)). + +format_time_hour_test() -> + ?assertEqual(<<"1h 0m 0.0s">>, test_summary_format_time(3600000000)). +-endif. \ No newline at end of file diff --git a/src/rt_riak_cluster.erl b/src/rt_riak_cluster.erl new file mode 100644 index 000000000..8ac87f240 --- /dev/null +++ b/src/rt_riak_cluster.erl @@ -0,0 +1,323 @@ +%%%------------------------------------------------------------------- +%%% @author John Burwell <> +%%% @copyright (C) 2015, John Burwell +%%% @doc +%%% +%%% @end +%%% Created : 19 Mar 2015 by John Burwell <> +%%%------------------------------------------------------------------- +-module(rt_riak_cluster). + +-behaviour(gen_fsm). + +%% API +-export([activate_bucket_type/2, + clean/1, + create_and_activate_bucket_type/3, + create_bucket_type/3, + is_mixed/1, + join/2, + leave/2, + name/1, + nodes/1, + partition/3, + start/1, + start_link/2, + stop/1, + wait_until_all_members/1, + wait_until_connected/1, + wait_until_legacy_ring_ready/1, + wait_until_no_pending_changes/1, + wait_until_nodes_agree_about_ownership/1, + wait_until_ring_converged/1, + version/1, + upgrade/2]). + +%% gen_fsm callbacks +-export([init/1, state_name/2, state_name/3, handle_event/3, + handle_sync_event/4, handle_info/3, terminate/3, code_change/4]). + +-define(SERVER, ?MODULE). + +-type node_attribute() :: {name, atom()} | {version, rt_util:version()} | + {hostname, rt_host:hostname()} | {config, term()} | + {type, proplists:proplist()}. +-type node_definition() :: [node_attribute()]. + +-record(state, {name :: cluster_name(), + bucket_types :: [term()], %% should this be a string? + indexes :: [atom()], %% should this be a string? + nodes :: [node()], + version :: rt_util:version(), + config :: term(), + is_mixed=false :: boolean()}). + +%%%=================================================================== +%%% API +%%%=================================================================== +-type cluster_name() :: atom(). +-type cluster_id() :: pid(). +%% -type partition() :: [node()]. + +-spec activate_bucket_type(cluster_id(), atom()) -> rt_util:result(). +activate_bucket_type(Cluster, BucketType) -> + gen_fsm:sync_send_event(Cluster, {activate_bucket_type, BucketType}). + +%% @doc Stops cluster, `Cluster', removes all data, and restarts it +-spec clean(cluster_id()) -> rt_util:result(). +clean(Cluster) -> + gen_fsm:sync_send_event(Cluster, clean). + +%% @doc Creates and activates a bucket type, `BucketType', on cluster, `Cluster', using properties, `Properties' +-spec create_and_activate_bucket_type(cluster_id(), atom(), proplists:proplist()) -> rt_util:result(). +create_and_activate_bucket_type(Cluster, BucketType, Properties) -> + ok = create_bucket_type(Cluster, BucketType, Properties), + activate_bucket_type(Cluster, BucketType). + +%% @doc Creates a bucket type, `BucketType', on cluster, `Cluster', with properties, `Properties' +-spec create_bucket_type(cluster_id(), atom(), proplists:proplist()) -> rt_util:result(). +create_bucket_type(Cluster, BucketType, Properties) -> + gen_fsm:create_bucket_type(Cluster, {create_bucket_type, BucketType, Properties}). + +-spec is_mixed(cluster_id()) -> boolean(). +is_mixed(Cluster) -> + gen_fsm:sync_send_all_state_event(Cluster, is_mixed). + +%% @doc Joins a node, `Node', to the cluster, `Cluster' +-spec join(cluster_id(), node()) -> rt_util:result(). +join(Cluster, Node) -> + gen_fsm:sync_send_event(Cluster, {join, Node}). + +-spec leave(cluster_id(), node()) -> rt_util:result(). +leave(Cluster, Node) -> + gen_fsm:sync_send_event(Cluster, {leave, Node}). + +-spec name(cluster_id()) -> atom(). +name(Cluster) -> + gen_fsm:sync_send_all_state_event(Cluster, name). + +%% @doc Returns the list of nodes in the cluster +-spec nodes(cluster_id()) -> [node()]. +nodes(Cluster) -> + gen_fsm:sync_send_all_state_event(Cluster, nodes). + +%% -spec partition(cluster_id(), partition(), partition()) -> [atom(), atom(), partition(), partition()] | rt_util:error(). +partition(Cluster, P1, P2) -> + gen_fsm:sync_send_event(Cluster, {partition, P1, P2}). + +%% @doc Starts each node in the cluster +-spec start(cluster_id()) -> rt_util:result(). +start(Cluster) -> + gen_fsm:sync_send_event(Cluster, start). + +%% @doc Stops each node in the cluster +-spec stop(cluster_id()) -> rt_util:result(). +stop(Cluster) -> + gen_fsm:sync_send_event(Cluster, stop). + +-spec wait_until_all_members(cluster_id()) -> rt_util:result(). +wait_until_all_members(Cluster) -> + gen_fsm:sync_send_event(Cluster, wait_unit_all_members). + +-spec wait_until_connected(cluster_id()) -> rt_util:result(). +wait_until_connected(Cluster) -> + gen_fsm:sync_send_event(Cluster, wait_until_connected). + +-spec wait_until_legacy_ring_ready(cluster_id()) -> rt_util:result(). +wait_until_legacy_ring_ready(Cluster) -> + gen_fsm:sync_send_event(Cluster, wait_until_legacy_ring). + +-spec wait_until_no_pending_changes(cluster_id()) -> rt_util:result(). +wait_until_no_pending_changes(Cluster) -> + gen_fsm:sync_send_event(Cluster, wait_until_no_pending_changes). + +-spec wait_until_nodes_agree_about_ownership(cluster_id()) -> rt_util:result(). +wait_until_nodes_agree_about_ownership(Cluster) -> + gen_fsm:sync_send_event(Cluster, wait_until_nodes_agree_about_ownership). + +-spec wait_until_ring_converged(cluster_id()) -> rt_util:result(). +wait_until_ring_converged(Cluster) -> + gen_fsm:sync_send_event(Cluster, wait_until_ring_converged). + +%% @doc Returns the version of the cluster, `Cluster' +-spec version(cluster_id()) -> rt_util:version(). +version(Cluster) -> + gen_fsm:sync_send_all_state_event(Cluster, version). + +%% @doc Performs a rolling upgrade of the cluster, `Cluster', to version, `NewVersion'. +-spec upgrade(cluster_id(), rt_util:version()) -> rt_util:result(). +upgrade(Cluster, NewVersion) -> + gen_fsm:sync_send_event(Cluster, {upgrade, NewVersion}). + +%% index creation ... + +%% security: users/acls, etc + +%%-------------------------------------------------------------------- +%% @doc +%% Creates a gen_fsm process which calls Module:init/1 to +%% initialize. To ensure a synchronized start-up procedure, this +%% function does not return until Module:init/1 has returned. +%% +%% @spec start_link() -> {ok, Pid} | ignore | {error, Error} +%% @end +%%-------------------------------------------------------------------- +%%-spec start_link([cluster_name(), [{node(), rt_util:version()}], term()]) -> {ok, pid()} | ignore | rt_util:error(). +-spec start_link(cluster_name(), [node_definition()]) -> {ok, pid()} | ignore | rt_util:error(). +start_link(Name, NodeDefinitions) -> + gen_fsm:start_link(?MODULE, [Name, NodeDefinitions], []). + +%%%=================================================================== +%%% gen_fsm callbacks +%%%=================================================================== + +%%-------------------------------------------------------------------- +%% @private +%% @doc +%% Whenever a gen_fsm is started using gen_fsm:start/[3,4] or +%% gen_fsm:start_link/[3,4], this function is called by the new +%% process to initialize. +%% +%% @spec init(Args) -> {ok, StateName, State} | +%% {ok, StateName, State, Timeout} | +%% ignore | +%% {stop, StopReason} +%% @end +%%-------------------------------------------------------------------- +init([Name, NodeDefinitions]) -> + %% TODO Calculate the mixed flag + Nodes = lists:foldl(fun(NodeDefinition, Nodes) -> + Hostname = proplists:get_value(hostname, NodeDefinition), + Type = proplists:get_value(type, NodeDefinition), + Config = proplists:get_value(config, NodeDefinition), + Version = proplists:get_value(version, NodeDefinition), + + {ok, NodeName} = rt_riak_node:start_link(Hostname, Type, + Config, Version), + + lists:apped(NodeName, Nodes) + end, [], NodeDefinitions), + + {ok, allocated, #state{name=Name, nodes=Nodes}}. + +%%-------------------------------------------------------------------- +%% @private +%% @doc +%% There should be one instance of this function for each possible +%% state name. Whenever a gen_fsm receives an event sent using +%% gen_fsm:send_event/2, the instance of this function with the same +%% name as the current state name StateName is called to handle +%% the event. It is also called if a timeout occurs. +%% +%% @spec state_name(Event, State) -> +%% {next_state, NextStateName, NextState} | +%% {next_state, NextStateName, NextState, Timeout} | +%% {stop, Reason, NewState} +%% @end +%%-------------------------------------------------------------------- +state_name(_Event, State) -> + {next_state, state_name, State}. + +%%-------------------------------------------------------------------- +%% @private +%% @doc +%% There should be one instance of this function for each possible +%% state name. Whenever a gen_fsm receives an event sent using +%% gen_fsm:sync_send_event/[2,3], the instance of this function with +%% the same name as the current state name StateName is called to +%% handle the event. +%% +%% @spec state_name(Event, From, State) -> +%% {next_state, NextStateName, NextState} | +%% {next_state, NextStateName, NextState, Timeout} | +%% {reply, Reply, NextStateName, NextState} | +%% {reply, Reply, NextStateName, NextState, Timeout} | +%% {stop, Reason, NewState} | +%% {stop, Reason, Reply, NewState} +%% @end +%%-------------------------------------------------------------------- +state_name(_Event, _From, State) -> + Reply = ok, + {reply, Reply, state_name, State}. + +%%-------------------------------------------------------------------- +%% @private +%% @doc +%% Whenever a gen_fsm receives an event sent using +%% gen_fsm:send_all_state_event/2, this function is called to handle +%% the event. +%% +%% @spec handle_event(Event, StateName, State) -> +%% {next_state, NextStateName, NextState} | +%% {next_state, NextStateName, NextState, Timeout} | +%% {stop, Reason, NewState} +%% @end +%%-------------------------------------------------------------------- +handle_event(_Event, StateName, State) -> + {next_state, StateName, State}. + +%%-------------------------------------------------------------------- +%% @private +%% @doc +%% Whenever a gen_fsm receives an event sent using +%% gen_fsm:sync_send_all_state_event/[2,3], this function is called +%% to handle the event. +%% +%% @spec handle_sync_event(Event, From, StateName, State) -> +%% {next_state, NextStateName, NextState} | +%% {next_state, NextStateName, NextState, Timeout} | +%% {reply, Reply, NextStateName, NextState} | +%% {reply, Reply, NextStateName, NextState, Timeout} | +%% {stop, Reason, NewState} | +%% {stop, Reason, Reply, NewState} +%% @end +%%-------------------------------------------------------------------- +handle_sync_event(_Event, _From, StateName, State) -> + Reply = ok, + {reply, Reply, StateName, State}. + +%%-------------------------------------------------------------------- +%% @private +%% @doc +%% This function is called by a gen_fsm when it receives any +%% message other than a synchronous or asynchronous event +%% (or a system message). +%% +%% @spec handle_info(Info,StateName,State)-> +%% {next_state, NextStateName, NextState} | +%% {next_state, NextStateName, NextState, Timeout} | +%% {stop, Reason, NewState} +%% @end +%%-------------------------------------------------------------------- +handle_info(_Info, StateName, State) -> + {next_state, StateName, State}. + +%%-------------------------------------------------------------------- +%% @private +%% @doc +%% This function is called by a gen_fsm when it is about to +%% terminate. It should be the opposite of Module:init/1 and do any +%% necessary cleaning up. When it returns, the gen_fsm terminates with +%% Reason. The return value is ignored. +%% +%% @spec terminate(Reason, StateName, State) -> void() +%% @end +%%-------------------------------------------------------------------- +terminate(_Reason, _StateName, _State) -> + ok. + +%%-------------------------------------------------------------------- +%% @private +%% @doc +%% Convert process state when code is changed +%% +%% @spec code_change(OldVsn, StateName, State, Extra) -> +%% {ok, StateName, NewState} +%% @end +%%-------------------------------------------------------------------- +code_change(_OldVsn, StateName, State, _Extra) -> + {ok, StateName, State}. + +%%%=================================================================== +%%% Internal functions +%%%=================================================================== diff --git a/src/rt_riak_driver.erl b/src/rt_riak_driver.erl new file mode 100644 index 000000000..2da9cbd3f --- /dev/null +++ b/src/rt_riak_driver.erl @@ -0,0 +1,103 @@ +-module(rt_riak_driver). + +-behavior(rt_driver). + +-export([cluster_module/0, + new_configuration/0, + new_configuration/1, + get_configuration_key/2, + set_configuration_key/3]). + +-record(riak_configuration_v1, { + %% TODO Add support for backend configuration options + supported_backends=all :: [storage_backend()], + wait_for_transfers=false :: boolean(), + bucket_types=[] :: rt_bucket_types:bucket_types(), + indexes=[] :: [index()], + ring_size=auto :: [atom() | pos_integer()], + features=[] :: [feature_flag()], + mr_modules=[] :: [module()], + %% TODO Can rt_riak_node properly calculate the list of required services? + required_services=[riak_kv] :: [atom()], + node_config=default_node_config() :: proplists:proplist() + %% TODO What do we need to configure security? +}). +-define(CONFIGURATION_RECORD, #riak_configuration_v1). + +-type riak_configuration() :: ?CONFIGURATION_RECORD{}. + +-type feature() :: aae | strong_consistency | yokozuna | jmx | snmp | security. +-type feature_flag() :: { feature(), boolean() }. +-type storage_backend() :: bitcask | leveldb | memory | multi_backend. +-type index() :: {binary(), binary(), binary()}. + +-exporttype([feature/0, + feature_flag/0, + storage_backend/0, + index/0]). + +-spec cluster_module() -> module(). +cluster_module() -> + rt_riak_cluster. + +-spec default_node_config() -> [term()]. +default_node_config() -> + [{riak_core, [{handoff_concurrency, 11}]}, + {riak_search, [{enabled, true}]}, + {riak_pipe, [{worker_limit, 200}]}]. + +-spec new_configuration() -> riak_configuration(). +new_configuration() -> + ?CONFIGURATION_RECORD{}. + +-spec new_configuration(atom()) -> riak_configuration(). +new_configuration(default) -> + ?CONFIGURATION_RECORD{}. + +-spec get_configuration_key(atom(), riak_configuration()) -> term() | {error, string()}. +get_configuration_key(Key, Configuration) when is_record(Configuration, riak_configuration_v1) -> + maybe_get_configuration_key(field_index(Key), Configuration); +get_configuration_key(Key, Configuration) -> + {error, io_lib:format("~w is not a riak_configuration_v1 record from which to retrieve ~w", [Configuration, Key])}. + +-spec maybe_get_configuration_key(pos_integer() | {error, string()}, riak_configuration()) -> term() | {error, string()}. +maybe_get_configuration_key(Error={error, _}, _Configuration) -> + Error; +maybe_get_configuration_key(FieldIndex, Configuration) -> + element(FieldIndex, Configuration). + +-spec set_configuration_key(atom(), term(), riak_configuration()) -> {ok, riak_configuration()} | {error, string()}. +set_configuration_key(Key, Value, Configuration) when is_record(Configuration, riak_configuration_v1) -> + maybe_set_configuration_key(field_index(Key), Configuration, Value); +set_configuration_key(Configuration, Key, Value) -> + {error, io_lib:format("~w is not a riak_configuration_v1 record from which to set ~w to ~w", [Configuration, Key, Value])}. + +-spec maybe_set_configuration_key({error, string()} | pos_integer, riak_configuration, term()) -> + {ok, riak_configuration()} | {error, string()}. +maybe_set_configuration_key(Error={error, _}, _Configuration, _Value) -> + Error; +maybe_set_configuration_key(FieldIndex, Configuration, Value) -> + {ok, setelement(FieldIndex, Configuration, Value)}. + +-spec field_index(atom()) -> pos_integer | {error, string()}. +field_index(supported_backends) -> + ?CONFIGURATION_RECORD.supported_backends; +field_index(wait_for_transfers) -> + ?CONFIGURATION_RECORD.supported_backends; +field_index(bucket_types) -> + ?CONFIGURATION_RECORD.bucket_types; +field_index(indexes) -> + ?CONFIGURATION_RECORD.indexes; +field_index(ring_sizes) -> + ?CONFIGURATION_RECORD.ring_size; +field_index(features) -> + ?CONFIGURATION_RECORD.features; +field_index(mr_modules) -> + ?CONFIGURATION_RECORD.mr_modules; +field_index(required_services) -> + ?CONFIGURATION_RECORD.required_services; +field_index(node_config) -> + ?CONFIGURATION_RECORD.node_config; +field_index(Unknown) -> + {error, "Unknown Riak configuration field: " ++ Unknown}. + diff --git a/src/rt_riak_node.erl b/src/rt_riak_node.erl new file mode 100644 index 000000000..5b98c6a6d --- /dev/null +++ b/src/rt_riak_node.erl @@ -0,0 +1,1412 @@ +%%%------------------------------------------------------------------- +%%% @author John Burwell <> +%%% @copyright (C) 2015, John Burwell +%%% @doc +%%% +%%% @end +%%% Created : 19 Mar 2015 by John Burwell <> +%%%------------------------------------------------------------------- +-module(rt_riak_node). + +-behaviour(gen_fsm). + +-include_lib("eunit/include/eunit.hrl"). + +%% TODO Document the following topics: +%% - State model and defintions +%% - Error handling: what is passed back to the client to handle +%% vs. what stops the FSM +%% - + + +%% API +-export([admin/2, + admin/3, + attach/2, + attach_direct/2, + brutal_kill/1, + check_singleton/1, + claimant_according_to/1, + clean_data_dir/1, + clean_data_dir/2, + commit/1, + console/2, + cookie/1, + copy_logs/2, + get_ring/1, + ip/1, + is_invalid/1, + is_ready/1, + is_started/1, + is_stopped/1, + host/1, + join/2, + maybe_wait_for_changes/1, + members_according_to/1, + owners_according_to/1, + partitions/1, + ping/1, + plan/1, + release/1, + riak/2, + riak_repl/2, + search_cmd/2, + set_cookie/2, + staged_join/2, + start/1, + start/2, + start_link/5, + status_of_according_to/1, + stop/1, + stop/2, + upgrade/2, + wait_for_service/2, + wait_until_pingable/1, + wait_until_registered/2, + wait_until_unpingable/1, + version/1]). + +%% gen_fsm callbacks +-export([init/1, handle_event/3, stopped/3, stopped/2, + handle_sync_event/4, handle_info/3, invalid/3, + ready/2, ready/3, started/3, terminate/3, + code_change/4]). + +%% internal exports +-export([do_check_singleton/3, do_wait_until_pingable/2, + do_wait_until_registered/3, do_wait_for_service/3, + do_unpingable/1, get_os_pid/2, start_riak_daemon/3]). + +-define(SERVER, ?MODULE). +-define(TIMEOUT, infinity). + +%% @doc The directory overlay describes the layout of the various +%% directories on the node. While the paths are absolute, they +%% are agnostic of a local/remote installation. The values in +%% this structure correspond to the directory locations in the +%% Riak 2.x+ riak.conf file. Use of this structure allows paths +%% to commands and files to be calculated in an installation/transport +%% neutral manner. +%% +%% For devrel-based installations, the layout will be calculated +%% relative to the root_path provided in the node_type. +%% +%% When support for package-based installations is implemented, +%% the directories will correspond to those used by the package +%% to deploy Riak on the host OS. +%% +%% @since 1.1.0 +-record(directory_overlay, {bin_dir :: filelib:dirname(), + conf_dir :: filelib:dirname(), + data_dir :: filelib:dirname(), + home_dir :: filelib:dirname(), + lib_dir :: filelib:dirname(), + log_dir :: filelib:dirname()}). + +%% @doc Defines the following metadata elements required to initialize +%% and control a Riak devrel node: +%% +%% * id: The devrel atom +%% * root_path: The absolute path to the root directory to +%% the available version/node sets +%% * node_id: The number of the devrel node to be managed by +%% the FSM process (e.g. 1). This number is +%% used to form the base path of node as +%% //dev +%% +%% @since 1.1.0 +-type devrel_node_type() :: [{root_path, filelib:dirname()} | + {id, devrel} | + {node_id, pos_integer()}]. + +%% @doc This record bundles the pieces required to provision a node +%% and attach orchestration to it. +%% +%% @since 1.1.0 +-record(definition, {config=[] :: proplists:proplist(), + hostname=localhost :: rt_host:hostname(), + name :: node(), + type :: devrel_node_type(), + version :: rt_util:version()}). + +-type(node_definition() :: #definition{}). +-exporttype(node_definition/0). + +-record(state, {host :: rt_host:host(), + id :: node_id(), + directory_overlay :: #directory_overlay{}, + name :: node(), + os_pid=0 :: pos_integer(), + required_services=[riak_kv] :: [atom()], + start_command :: rt_host:command(), + stop_command :: rt_host:command(), + version :: rt_util:version()}). + +%%%=================================================================== +%%% API +%%%=================================================================== + +%% @doc Call 'bin/riak-admin' command on `Node' with arguments `Args' +%% +%% @since 1.1.0 +-spec admin(node(), [term()]) -> {ok, term()} | rt_util:error(). +admin(Node, Args) -> + admin(Node, Args, []). + +%% @doc Call 'bin/riak-admin' command on `Node' with arguments `Args'. +%% The third parameter is a list of options. Valid options are: +%% * `return_exit_code' - Return the exit code along with the command output +%% +%% @since 1.1.0 +-spec admin(node(), [term()], [term()]) -> {ok, term()} | rt_util:error(). +admin(Node, Args, Options) -> + gen_fsm:sync_send_event(Node, {admin, Node, Args, Options}, ?TIMEOUT). + +%% @doc Runs `riak attach' on a specific node, and tests for the expected behavoir. +%% Here's an example: ``` +%% rt_riak_node:attach(Node, [{expect, "erlang.pipe.1 \(^D to exit\)"}, +%% {send, "riak_core_ring_manager:get_my_ring()."}, +%% {expect, "dict,"}, +%% {send, [4]}]), %% 4 = Ctrl + D''' +%% `{expect, String}' scans the output for the existance of the String. +%% These tuples are processed in order. +%% +%% `{send, String}' sends the string to the console. +%% Once a send is encountered, the buffer is discarded, and the next +%% expect will process based on the output following the sent data. +%% ``` +%% +%% @since 1.1.0 +-spec attach(node(), {expect, list()} | {send, list()}) -> {ok, term()} | rt_util:error(). +attach(Node, Expected) -> + gen_fsm:sync_send_event(Node, {attach, Expected}, ?TIMEOUT). + +%% @doc Runs 'riak attach-direct' on a specific node +%% @see rt_riak_node:attach/2 +%% +%% @since 1.1.0 +-spec attach_direct(node(), {expect, list()} | {send, list()}) -> {ok, term()} | rt_util:error(). +attach_direct(Node, Expected) -> + gen_fsm:sync_send_event(Node, {attach_direct, Expected}, ?TIMEOUT). + +%% @doc Kills any Riak processes running on the passed `Node', and resets the +%% the state of the FSM to `stopped'. Therefore, this function is the means +%% to reset/resync the state of a Riak node FSM with a running Riak node. +%% +%% @since 1.1.0 +-spec brutal_kill(node()) -> rt_util:result(). +brutal_kill(Node) -> + gen_fsm:sync_send_all_state_event(Node, brutal_kill, ?TIMEOUT). + +%% @doc Ensure that the specified node is a singleton node/cluster -- a node +%% that owns 100% of the ring. +%% +%% @since 1.1.0 +-spec check_singleton(node()) -> boolean(). +check_singleton(Node) -> + gen_fsm:sync_send_event(Node, check_singleton, ?TIMEOUT). + +%% @doc Return a list of nodes that own partitions according to the ring +%% retrieved from the specified node. +%% +%% @since 1.1.0 +claimant_according_to(Node) -> + gen_fsm:sync_send_event(Node, claimant_according_to, ?TIMEOUT). + +-spec clean_data_dir(node()) -> rt_util:result(). +clean_data_dir(Node) -> + gen_fsm:sync_send_event(Node, clean_data_dir, ?TIMEOUT). + +-spec clean_data_dir(node(), list()) -> rt_util:result(). +clean_data_dir(Node, SubDir) -> + gen_fsm:sync_send_event(Node, {clean_data_dir, SubDir}, ?TIMEOUT). + +%% @doc Runs `riak console' on a the passed `Node' +%% @see rt_riak_node:attach/2 +%% +%% @since 1.1.0 +-spec console(node(), {expect, list()} | {send, list()}) -> {ok, term()} | rt_util:error(). +console(Node, Expected) -> + geb_fsm:sync_send_event(Node, {console, Expected}, ?TIMEOUT). + +%% @doc Commits changes to a cluster using the passed `Node' +%% +%% @since 1.1.0 +-spec commit(node()) -> rt_util:result(). +commit(Node) -> + gen_fsm:sync_send_event(Node, commit, ?TIMEOUT). + +%% @doc Retrieves the Erlang cookie current of the passed `Node' +%% +%% @since 1.1.0 +-spec cookie(node()) -> atom() | rt_util:error(). +cookie(Node) -> + gen_fsm:sync_send_event(Node, cookie, ?TIMEOUT). + +%% @doc Copy all logs from the passed node, `Node', to the directory, `ToDir' +%% +%% @since 1.1.0 +-spec copy_logs(node(), string()) -> rt_util:result(). +copy_logs(Node, ToDir) -> + gen_fsm:sync_send_all_state_event(Node, {copy_logs, ToDir}, ?TIMEOUT). + +%% @doc Get the raw ring for `Node'. +%% +%% @since 1.1.0 +-spec get_ring(node()) -> {ok, term()} | rt_util:error(). +get_ring(Node) -> + gen_fsm:sync_send_event(Node, get_ring, ?TIMEOUT). + +%% @doc Returns the IP address of the passed `Node' +%% +%% @since 1.1.0 +-spec ip(node() | string()) -> string(). +ip(Node) -> + gen_fsm:sync_send_all_state_event(Node, ip, ?TIMEOUT). + +is_invalid(Node) -> + gen_fsm:sync_send_all_state_event(Node, is_invalid, ?TIMEOUT). + +%% @doc Returns `true' if the passed node, `Node', is ready to +%% accept requests. If the node is not ready or stopped, +%5 this function returns `false'. +%% +%% @since 1.1.0 +-spec is_ready(node()) -> boolean(). +is_ready(Node) -> + gen_fsm:sync_send_all_state_event(Node, is_ready, ?TIMEOUT). + +is_started(Node) -> + gen_fsm:sync_send_all_state_event(Node, is_started, ?TIMEOUT). + +%% @doc Returns `true' if the passed node, `Node', is not running. +%% If the node is started, ready, or invalid, this function +%% returns `false'. +%% +%% @since 1.1.0 +-spec is_stopped(node()) -> boolean(). +is_stopped(Node) -> + gen_fsm:sync_send_all_state_event(Node, is_stopped, ?TIMEOUT). + +-spec host(node()) -> rt_host:host(). +host(Node) -> + gen_fsm:sync_send_all_state_event(Node, host, ?TIMEOUT). + +-spec join(node(), node()) -> rt_util:result(). +join(Node, ToNode) -> + gen_fsm:sync_send_event(Node, {join, ToNode}, ?TIMEOUT). + +-spec maybe_wait_for_changes(node()) -> rt_util:result(). +maybe_wait_for_changes(Node) -> + gen_fsm:sync_send_event(Node, maybe_wait_for_changes, ?TIMEOUT). + +%% @doc Return a list of cluster members according to the ring retrieved from +%% the specified node. +%% +%% @since 1.1.0 +-spec members_according_to(node()) -> [term()] | rt_util:error(). +members_according_to(Node) -> + gen_fsm:sync_send_event(Node, members_according_to, ?TIMEOUT). + +%% @doc Return a list of nodes that own partitions according to the ring +%% retrieved from the specified node. +%% +%% @since 1.1.0 +-spec owners_according_to(node()) -> [term()]. +owners_according_to(Node) -> + gen_fsm:sync_send_event(Node, owners_according_to, ?TIMEOUT). + +%% @doc Get list of partitions owned by node (primary). +%% +%% @since 1.1.0 +-spec partitions(node()) -> [term()]. +partitions(Node) -> + gen_fsm:sync_send_event(Node, partitions, ?TIMEOUT). + +-spec ping(node()) -> boolean(). +ping(Node) -> + gen_fsm:sync_send_all_state_event(Node, ping, ?TIMEOUT). + +-spec plan(node()) -> rt_util:result(). +plan(Node) -> + gen_fsm:sync_send_event(Node, plan, ?TIMEOUT). + +%% @doc Releases the `Node' for use by another cluster. This function ensures +%% that the node is stopped before returning it for use by another cluster. +%% +%% @since 1.1.0 +-spec release(node()) -> rt_util:result(). +release(Node) -> + case whereis(Node) of + undefined -> + ok; + _ -> + gen_fsm:sync_send_all_state_event(Node, release, ?TIMEOUT) + end. + +%% @doc Call 'bin/riak' command on `Node' with arguments `Args' +%% +%% @since 1.1.0 +-spec riak(node(), [term()]) -> {ok, term()} | rt_util:error(). +riak(Node, Args) -> + gen_fsm:sync_send_all_state_event(Node, {riak, Args}, ?TIMEOUT). + +%% @doc Call 'bin/riak' command on `Node' with arguments `Args' +%% +%% @since 1.1.0 +-spec riak_repl(node(), [term()]) -> {ok, term()} | rt_util:error(). +riak_repl(Node, Args) -> + gen_fsm:sync_send_event(Node, {riak_repl, Args}, ?TIMEOUT). + +-spec search_cmd(node(), [term()]) -> {ok, term()} | rt_util:error(). +search_cmd(Node, Args) -> + gen_fsm:sync_send_event(Node, {search_cmd, Args}, ?TIMEOUT). + +-spec set_cookie(node(), atom()) -> rt_util:result(). +set_cookie(Node, NewCookie) -> + gen_fsm:sync_send_event(Node, {set_cookie, NewCookie}, ?TIMEOUT). + +-spec staged_join(node(), node()) -> rt_util:result(). +staged_join(Node, ToNode) -> + gen_fsm:sync_send_event(Node, {staged_join, ToNode}, ?TIMEOUT). + +%% TODO Document the behavior of start including ready checks +-spec start(node()) -> rt_util:result(). +start(Node) -> + start(Node, true). + +-spec start(node(), boolean()) -> rt_util:result(). +start(Node, true) -> + gen_fsm:sync_send_event(Node, start, ?TIMEOUT); +start(Node, false) -> + gen_fsm:send_event(Node, start). + +%% @doc Starts a gen_fsm process to configure, start, and +%% manage a Riak node on the `Host' identified by `NodeId' +%% and `NodeName' using Riak `Version' ({product, release}) +-spec start_link(rt_host:hostname(), node_definition(), node_id(), proplists:proplist(), rt_util:version()) -> + {ok, node()} | ignore | rt_util:error(). +start_link(Hostname, NodeDefinition, NodeId, Config, Version) -> + %% TODO Re-implement node naming when 1.x and 2.x configuration is propely implemented + %% NodeName = list_to_atom(string:join([dev(NodeId), atom_to_list(Hostname)], "@")), + NodeName = list_to_atom(string:join([dev(NodeId), "127.0.0.1"], "@")), + Args = [Hostname, NodeType, NodeId, NodeName, Config, Version], + case gen_fsm:start({local, NodeName}, ?MODULE, Args, []) of + {ok, _} -> + {ok, NodeName}; + Error={error, _} -> + Error + end. + +%% @doc Return the cluster status of `Member' according to the ring +%% retrieved from `Node'. +%% +%% @since 1.1.0 +-spec status_of_according_to(node()) -> [term()] | rt_util:error(). +status_of_according_to(Node) -> + gen_fsm:sync_send_event(Node, status_of_according_to, ?TIMEOUT). + +-spec stop(node()) -> rt_util:result(). +stop(Node) -> + stop(Node, true). + +-spec stop(node(), boolean()) -> rt_util:result(). +stop(Node, true) -> + gen_fsm:sync_send_event(Node, stop, ?TIMEOUT); +stop(Node, false) -> + gen_fsm:send_event(Node, stop). + +-spec upgrade(node(), rt_util:version()) -> rt_util:result(). +upgrade(Node, NewVersion) -> + gen_fsm:sync_send_event(Node, {upgrade, NewVersion}, ?TIMEOUT). + +-spec wait_for_service(node(), atom() | [atom()]) -> rt_util:wait_result(). +wait_for_service(Node, Services) when is_list(Services) -> + gen_fsm:sync_send_event(Node, {wait_for_services, Services}, ?TIMEOUT); +wait_for_service(Node, Service) -> + wait_for_service(Node, [Service]). + +-spec wait_until_pingable(node()) -> rt_util:wait_result(). +wait_until_pingable(Node) -> + gen_fsm:sync_send_event(Node, wait_until_pingable, ?TIMEOUT). + +-spec wait_until_registered(node(), atom()) -> rt_util:wait_result(). +wait_until_registered(Node, Name) -> + gen_fsm:sync_send_event(Node, {wait_until_registered, Name}, ?TIMEOUT). + +-spec wait_until_unpingable(node()) -> rt_util:wait_result(). +wait_until_unpingable(Node) -> + gen_fsm:sync_send_all_state_event(Node, wait_until_unpingable, ?TIMEOUT). + +-spec version(node()) -> rt_util:version(). +version(Node) -> + gen_fsm:sync_send_all_state_event(Node, version, ?TIMEOUT). + +%%%=================================================================== +%%% gen_fsm callbacks +%%%=================================================================== + +%%-------------------------------------------------------------------- +%% @private +%% @doc +%% Whenever a gen_fsm is started using gen_fsm:start/[3,4] or +%% gen_fsm:start_link/[3,4], this function is called by the new +%% process to initialize. +%% +%% @spec init(Args) -> {ok, StateName, State} | +%% {ok, StateName, State, Timeout} | +%% ignore | +%% {stop, StopReason} +%% @end +%%-------------------------------------------------------------------- +init([Hostname, NodeType, NodeId, NodeName, ConfigOverrides, Version]) -> + DirOverlay = create_directory_overlay(NodeType, Version, NodeId), + case rt_host:connect(Hostname) of + {ok, Host} -> + DirCreateResult = create_snmp_dirs(Host, DirOverlay#directory_overlay.data_dir), + ConfigureResult = maybe_configure_node(DirCreateResult, Version, NodeName, Host, + DirOverlay, ConfigOverrides), + maybe_transition_to_stopped(ConfigureResult, Version, NodeId, NodeName, Host, + DirOverlay, NodeType); + Error -> + maybe_transition_to_stopped(Error, Version, NodeId, NodeName, none, DirOverlay, + NodeType) + end. + +-spec maybe_configure_node(rt_util:result(), rt_util:version(), node(), rt_host:host(), #directory_overlay{}, proplists:proplist()) -> rt_util:result(). +maybe_configure_node(ok, Version, NodeName, Host, DirOverlay, ConfigOverrides) -> + ExistingConfig = case load_configuration(Version, Host, DirOverlay) of + {ok, [Term]} -> + Term; + {error, LoadError} -> + lager:warning("Unable to load existing configuration for node ~p due to ~p. Defaulting to an empty configuration.", + [NodeName, LoadError]), + [] + end, + %% TODO account for properly assigning the node name ... + %% TODO handle backend ... + Config = rt_util:merge_configs(ExistingConfig, ConfigOverrides), + save_configuration(Version, Host, Config, DirOverlay); +maybe_configure_node(Error={error, _}, _Version, _NodeName, _Host, _DirOverlay, _ConfigOverrides) -> + Error. + +-spec maybe_transition_to_stopped(rt_util:result(), rt_util:version(), node_id(), node(), rt_host:host(), + #directory_overlay{}, atom()) -> {ok, stopped, #state{}} | {stop, term()}. +maybe_transition_to_stopped(ok, Version, NodeId, NodeName, Host, DirOverlay, NodeType) -> + State=#state{host=Host, + id=NodeId, + name=NodeName, + directory_overlay=DirOverlay, + start_command=start_command(NodeType, DirOverlay), + stop_command=stop_command(NodeType, DirOverlay), + version=Version}, + {ok, stopped, State}; +maybe_transition_to_stopped({error, Reason}, _Version, _NodeId, _NodeName, _Host, _DirOverlay, _NodeType) -> + {stop, Reason}. + +invalid(Event, _From, State) -> + lager:error("Attempt to perform ~p operation (state: ~p)", [Event, State]), + {reply, {error, node_invalid}, invalid, State}. + +stopped(clean_data_dir, _From, State=#state{directory_overlay=DirOverlay, host=Host}) -> + {reply, do_clean_data_dir(Host, DirOverlay), stopped, State}; +stopped({clean_data_dir, SubDir}, _From, State=#state{directory_overlay=DirOverlay, host=Host}) -> + {reply, do_clean_data_dir(Host, DirOverlay, SubDir), stopped, State}; +stopped(start, _From, State=#state{host=Host, name=Node}) -> + lager:info("Starting node synchronously ~p on ~p", [Node, Host]), + {Result, {NextState, UpdState}} = do_start_and_update_state(State), + {reply, Result, NextState, UpdState}; +stopped(stop, _From, State=#state{host=Host, name=Node}) -> + lager:warning("Stop called on an already stopped node ~p on ~p", [Node, Host]), + {reply, ok, stopped, State}; +stopped(Event, _From, State=#state{host=Host, name=Node}) -> + %% The state of the node is not harmed. Therefore, we leave the FSM running + %% in the stopped state, but refuse to execute the command ... + lager:error("Invalid operation ~p when node ~p on ~p is in stopped state", [Event, Node, Host]), + {reply, {error, invalid_stopped_event}, stopped, State}. + +started(start, _From, State=#state{host=Host, name=Node, required_services=RequiredServices, + version=Version}) -> + case do_wait_until_ready(Host, Node, RequiredServices, Version) of + true -> {reply, ok, ready, State}; + false -> {error, node_not_ready, started, State} + end; +started(Event, _From, State=#state{host=Host, name=Node}) -> + lager:error("Invalid operation ~p when node ~p on ~p is in started state", [Event, Node, Host]), + {reply, {error, invalid_started_event}, started, State}. + +ready({admin, Args, Options}, _From, State=#state{host=Host, directory_overlay=DirOverlay}) -> + {reply, do_admin(Args, Options, DirOverlay, Host), ready, State}; +ready({attach, Expected}, _From, State=#state{directory_overlay=DirOverlay}) -> + {reply, do_attach(DirOverlay, Expected), ready, State}; +ready({attach_direct, Expected}, _From, State=#state{directory_overlay=DirOverlay}) -> + {reply, do_attach_direct(DirOverlay, Expected), ready, State}; +ready(check_singleton, _From, State=#state{host=Host, name=Node, version=Version}) -> + %% TODO consider adding the started state back + transition_to_state_and_reply(do_check_singleton(Host, Node, Version), ready, stopped, State); +ready(commit, _From, State=#state{host=Host, name=Node}) -> + {reply, do_commit(Host, Node), ready, State}; +ready({console, Expected}, _From, State=#state{directory_overlay=DirOverlay}) -> + {reply, do_console(DirOverlay, Expected), ready, State}; +ready(cookie, _From, State=#state{name=Node}) -> + {reply, rt_util:maybe_rpc_call(Node, erlang, get_cookie, []), ready, State}; +%% TODO Determine whether or not it makes sense to support get_ring in the started +%% state ... +ready(get_ring, _From, #state{name=NodeName}=State) -> + Result = maybe_get_ring(NodeName), + {reply, Result, ready, State}; +ready({join, ToNode}, _From, State=#state{host=Host, name=Node}) -> + {reply, do_join(Host, Node, ToNode), ready, State}; +ready(maybe_wait_for_changes, _From, State=#state{host=Host, name=Node}) -> + {reply, do_maybe_wait_for_changes(Host, Node), ready, State}; +ready(members_according_to, _From, #state{name=NodeName}=State) -> + Members = maybe_members_according_to(NodeName), + {reply, Members, ready, State}; +ready(owners_according_to, _From, #state{name=NodeName}=State) -> + Owners = maybe_owners_according_to(NodeName), + {reply, Owners, ready, State}; +ready(partitions, _From, State=#state{name=Node}) -> + Partitions = maybe_partitions(Node), + {reply, Partitions, ready, State}; +ready(plan, _From, State=#state{host=Host, name=Node}) -> + {reply, do_plan(Host, Node), ready, State}; +ready({riak_repl, Args}, _From, State=#state{host=Host, directory_overlay=DirOverlay}) -> + Result = rt_host:exec(Host, riak_repl_path(DirOverlay), Args), + {reply, Result, ready, State}; +ready({set_cookie, NewCookie}, _From, State=#state{name=Node}) -> + {reply, rt_util:maybe_rpc_call(Node, erlang, set_cookie, [Node, NewCookie]), ready, State}; +ready({staged_join, ToNode}, _From, State=#state{host=Host, name=Node}) -> + {reply, do_staged_join(Host, Node, ToNode), ready, State}; +ready(stop, _From, State) -> + {Result, UpdState} = do_stop_and_update_state(State), + %% TODO need an invalid state ... + transition_to_state_and_reply(Result, stopped, ready, UpdState); +ready({wait_for_service, Services}, _From, State=#state{host=Host, name=Node}) -> + %% TODO consider adding back the started state + transition_to_state_and_reply(do_wait_for_service(Host, Node, Services), ready, stopped, State); +ready(wait_until_pingable, _From, State=#state{host=Host, name=Node}) -> + %% TODO consider adding back the started state + transition_to_state_and_reply(do_wait_until_pingable(Host, Node), ready, stopped, State); +ready({wait_until_registered, Name}, _From, State=#state{host=Host, name=Node}) -> + transition_to_state_and_reply(do_wait_until_registered(Host, Node, Name), ready, stopped, State); +ready(Event, _From, State=#state{host=Host, name=Node}) -> + %% The state of the node is not harmed. Therefore, we leave the FSM running + %% in the stopped state, but refuse to execute the command ... + lager:error("Invalid operation ~p when node ~p on ~p is in ready state", [Event, Node, Host]), + {reply, {error, invalid_ready_event}, ready, State}. + +transition_to_state_and_reply(Result={error, _}, _SuccessState, FailedState, State) -> + {reply, Result, FailedState, State}; +transition_to_state_and_reply(Result, SuccessState, _FailedState, State) -> + {reply, Result, SuccessState, State}. + + +transition_to_state({error, _}, _SuccessState, FailedState, State) -> + {next_state, FailedState, State}; +transition_to_state(ok, SuccessState, _FailedState, State) -> + {next_state, SuccessState, State}. + +stopped(start, State=#state{host=Host, name=Node}) -> + lager:info("Starting node asynchronously ~p on ~p", [Node, Host]), + {_, {NextState, UpdState}} = do_start_and_update_state(State), + {next_state, NextState, UpdState}; +stopped(stop, State=#state{host=Host, name=Node}) -> + lager:warning("Stop called on a stopped node ~p on ~p", [Node, Host]), + {next_state, stopped, State}; +stopped(_Event, State) -> + {next_state, stopped, State}. + +ready(stop, State) -> + {Result, UpdState} = do_stop_and_update_state(State), + transition_to_state(Result, stopped, ready, UpdState); +ready(start, State=#state{host=Host, name=Node}) -> + lager:warning("Start called on ready node ~p on ~p", [Node, Host]), + {next_state, ready, State}; +ready(_Event, State) -> + {next_state, ready, State}. + +-spec maybe_get_ring(node()) -> rt_util:rt_rpc_result(). +maybe_get_ring(NodeName) -> + rt_util:maybe_rpc_call(NodeName, riak_core_ring_manager, get_raw_ring, []). + +-spec maybe_partitions(node()) -> rt_util:rt_rpc_result(). +maybe_partitions(NodeName) -> + maybe_partitions(NodeName, maybe_get_ring(NodeName)). + +-spec maybe_partitions(node(), rt_util:rt_rpc_result()) -> [term()] | rt_util:error(). +maybe_partitions(NodeName, {ok, Ring}) -> + [Idx || {Idx, Owner} <- riak_core_ring:all_owners(Ring), Owner == NodeName]; +maybe_partitions(_NodeName, {error, Reason}) -> + {error, Reason}. + +-spec maybe_members_according_to(node() | rt_util:tt_rpc_result()) -> [term()] | rt_util:error(). +maybe_members_according_to({ok, Ring}) -> + riak_core_ring:all_members(Ring); +maybe_members_according_to({error, Reason}) -> + {error, Reason}; +maybe_members_according_to(NodeName) -> + maybe_members_according_to(maybe_get_ring(NodeName)). + +-spec maybe_owners_according_to(node() | rt_util:rt_rpc_result()) -> [term()] | rt_util:error(). +maybe_owners_according_to({ok, Ring}) -> + Owners = [Owner || {_Idx, Owner} <- riak_core_ring:all_owners(Ring)], + lists:usort(Owners); +maybe_owners_according_to({error, Reason}) -> + {error,Reason}; +maybe_owners_according_to(NodeName) -> + maybe_owners_according_to(maybe_get_ring(NodeName)). + +%%-------------------------------------------------------------------- +%% @private +%% @doc +%% Whenever a gen_fsm receives an event sent using +%% gen_fsm:send_all_state_event/2, this function is called to handle +%% the event. +%% +%% @spec handle_event(Event, StateName, State) -> +%% {next_state, NextStateName, NextState} | +%% {next_state, NextStateName, NextState, Timeout} | +%% {stop, Reason, NewState} +%% @end +%%-------------------------------------------------------------------- +handle_event(_Event, StateName, State) -> + {next_state, StateName, State}. + +%%-------------------------------------------------------------------- +%% @private +%% @doc +%% Whenever a gen_fsm receives an event sent using +%% gen_fsm:sync_send_all_state_event/[2,3], this function is called +%% to handle the event. +%% +%% @spec handle_sync_event(Event, From, StateName, State) -> +%% {next_state, NextStateName, NextState} | +%% {next_state, NextStateName, NextState, Timeout} | +%% {reply, Reply, NextStateName, NextState} | +%% {reply, Reply, NextStateName, NextState, Timeout} | +%% {stop, Reason, NewState} | +%% {stop, Reason, Reply, NewState} +%% @end +%%-------------------------------------------------------------------- +%% handle_sync_event(_Event, _From, StateName, State) -> +%% Reply = ok, +%% {reply, Reply, StateName, State}. +handle_sync_event(brutal_kill, _From, _StateName, + State=#state{host=Host, name=Node, os_pid=OSPid}) -> + case do_brutal_kill(Node, Host, OSPid) of + ok -> UpdState=State#state{os_pid=0}, + {reply, ok, stopped, UpdState}; + Error={error, _} -> {stop, Error, State} + end; +handle_sync_event({copy_logs, ToDir}, _From, StateName, + State=#state{host=Host, directory_overlay=DirOverlay}) -> + Result = rt_host:copy_dir(Host, DirOverlay#directory_overlay.log_dir, ToDir), + transition_to_state_and_reply(Result, StateName, StateName, State); +handle_sync_event(host, _From, StateName, State=#state{host=Host}) -> + {reply, Host, StateName, State}; +handle_sync_event(ip, _From, StateName, State=#state{host=Host}) -> + {reply, rt_host:ip_addr(Host), StateName, State}; +handle_sync_event(is_invalid, _From, invalid, State) -> + {reply, true, invalid, State}; +handle_sync_event(is_invalid, _From, StateName, State) -> + {reply, false, StateName, State}; +handle_sync_event(is_ready, _From, ready, State) -> + {reply, true, ready, State}; +handle_sync_event(is_ready, _From, StateName, State) -> + {reply, false, StateName, State}; +handle_sync_event(is_started, _From, started, State) -> + {reply, true, started, State}; +handle_sync_event(is_started, _From, StateName, State) -> + {reply, false, StateName, State}; +handle_sync_event(is_stopped, _From, stopped, State) -> + {reply, true, stopped, State}; +handle_sync_event(is_stopped, _From, StateName, State) -> + {reply, false, StateName, State}; +handle_sync_event(ping, _From, StateName, State=#state{name=Node}) -> + {reply, do_ping(Node), StateName, State}; +handle_sync_event({riak, Args}, _From, StateName, + State=#state{host=Host, directory_overlay=DirOverlay}) -> + {reply, rt_host:exec(Host, riak_path(DirOverlay), Args), StateName, State}; +handle_sync_event(release, _From, _StateName, State=#state{host=Host, name=Node}) -> + lager:info("Releasing node ~p on ~p", [Node, Host]), + {stop, normal, ok, State}; +handle_sync_event(version, _From, StateName, State=#state{version=Version}) -> + {reply, Version, StateName, State}; +handle_sync_event(wait_until_unpingable, _From, StateName, State=#state{host=Host, name=Node}) -> + {reply, do_wait_until_unpingable(Host, Node), StateName, State}. + +%%-------------------------------------------------------------------- +%% @private +%% @doc +%% This function is called by a gen_fsm when it receives any +%% message other than a synchronous or asynchronous event +%% (or a system message). +%% +%% @spec handle_info(Info,StateName,State)-> +%% {next_state, NextStateName, NextState} | +%% {next_state, NextStateName, NextState, Timeout} | +%% {stop, Reason, NewState} +%% @end +%%-------------------------------------------------------------------- +handle_info(_Info, StateName, State) -> + {next_state, StateName, State}. + +%%-------------------------------------------------------------------- +%% @private +%% @doc +%% This function is called by a gen_fsm when it is about to +%% terminate. It should be the opposite of Module:init/1 and do any +%% necessary cleaning up. When it returns, the gen_fsm terminates with +%% Reason. The return value is ignored. +%% +%% @spec terminate(Reason, StateName, State) -> void() +%% @end +%%-------------------------------------------------------------------- +terminate(_Reason, _StateName, + #state{host=Host=Host, name=Node, os_pid=OSPid}) -> + _ = do_brutal_kill(Node, Host, OSPid), + rt_host:disconnect(Host), + ok. + +%%-------------------------------------------------------------------- +%% @private +%% @doc +%% Convert process state when code is changed +%% +%% @spec code_change(OldVsn, StateName, State, Extra) -> +%% {ok, StateName, NewState} +%% @end +%%-------------------------------------------------------------------- +code_change(_OldVsn, StateName, State, _Extra) -> + {ok, StateName, State}. + +%%%=================================================================== +%%% Internal functions +%%%=================================================================== + +-spec create_snmp_dirs(rt_host:host(), filelib:dirname()) -> rt_util:result(). +create_snmp_dirs(Host, DataDir) -> + SnmpDir = filename:join([DataDir, "snmp", "agent", "db"]), + lager:debug("Creating SNMP data directory ~p on localhost", [SnmpDir]), + rt_host:mkdirs(Host, SnmpDir). + +-spec create_directory_overlay({atom(), filelib:dirname()}, rt_util:version(), node()) -> #directory_overlay{}. +create_directory_overlay({devrel, RootPath}, Version, NodeId) -> + HomeDir = filename:join([rt_util:base_dir_for_version(RootPath, Version), + dev(NodeId)]), + #directory_overlay{bin_dir=filename:join([HomeDir, "bin"]), + conf_dir=filename:join([HomeDir, "etc"]), + data_dir=filename:join([HomeDir, "data"]), + home_dir=HomeDir, + lib_dir=filename:join([HomeDir, "lib"]), + log_dir=filename:join([HomeDir, "log"])}. + +-spec dev(pos_integer()) -> string(). +dev(NodeId) -> + lists:concat(["dev", integer_to_list(NodeId)]). + +-spec do_admin([term()], [term()], #directory_overlay{}, rt_host:host()) -> {ok, term()} | rt_util:error(). +do_admin(Args, Options, DirOverlay, Host) -> + rt_host:exec(Host, riak_admin_path(DirOverlay), Args, Options). + +-spec do_attach(#directory_overlay{}, term()) -> term(). +do_attach(DirOverlay, Expected) -> + interactive(DirOverlay, "attach", Expected). + +-spec do_attach_direct(#directory_overlay{}, term()) -> term(). +do_attach_direct(DirOverlay, Expected) -> + interactive(DirOverlay, "attach-direct", Expected). + +-spec do_brutal_kill(node(), rt_util:host(), pos_integer()) -> rt_util:result(). +do_brutal_kill(_Node, _Host, 0) -> + ok; +do_brutal_kill(Node, Host, OSPid) -> + lager:info("Killing node ~p running as PID ~s", [Node, OSPid]), + %% try a normal kill first, but set a timer to + %% kill -9 after 5 seconds just in case + timer:apply_after(5000, rt_host, kill, [Host, 9, OSPid]), + rt_host:kill(Host, 15, OSPid). + +%% @private +-spec do_check_singleton(rt_host:host(), node(), rt_util:version()) -> boolean() | rt_util:error(). +do_check_singleton(Host, Node, {_, Release}) when Release =/= "0.14.2" -> + HostName = rt_host:hostname(Host), + lager:info("Check that node ~p on ~p is a singleton", [Node, HostName]), + Result = case maybe_get_ring(Node) of + {ok, Ring} -> + Owners = lists:usort([Owner || {_Idx, Owner} <- riak_core_ring:all_owners(Ring)]), + [Node] =:= Owners; + Error -> + Error + end, + do_check_singleton(Result); +do_check_singleton(_Host, _Node, _Version) -> + true. + +do_check_singleton(true) -> + true; +do_check_singleton(false) -> + {error, node_not_singleton}; +do_check_singleton(Error) -> + Error. + +-spec do_clean_data_dir(rt_host:host(), #directory_overlay{}) -> rt_util:result(). +do_clean_data_dir(Host, DirOverlay) -> + do_clean_data_dir(Host, DirOverlay, ""). + +-spec do_clean_data_dir(rt_host:host(), #directory_overlay{}, filelib:dirname()) -> rt_util:result(). +do_clean_data_dir(Host, #directory_overlay{data_dir=DataDir}, SubDir) -> + TmpDir = rt_host:temp_dir(Host), + FromDir = filename:join([DataDir, SubDir]), + ToDir = filename:join([TmpDir, "child"]), + HostName = rt_host:hostname(Host), + + lager:info("Cleaning data directory ~p on ~p", [FromDir, HostName]), + rt_util:maybe_call_funs([ + [rt_host, mkdirs, [Host, ToDir]], + [rt_host, mvdir, [Host, FromDir, ToDir]], + [rt_host, rmdir, [Host, ToDir]], + [rt_host, mkdirs, [Host, ToDir]] + ]). + +-spec do_commit(rt_util:host(), node()) -> rt_util:result(). +do_commit(Host, Node) -> + lager:info("Commit cluster plan using node ~p on host ~p", + [Node, Host]), + case rt_util:maybe_rpc_call(Node, riak_core_claimant, commit, []) of + {error, plan_changed} -> + lager:info("commit: plan changed"), + timer:sleep(100), + ok = do_maybe_wait_for_changes(Host, Node), + ok = do_plan(Host, Node), + ok = do_commit(Host, Node); + {error, ring_not_ready} -> + lager:info("commit: ring not ready"), + timer:sleep(100), + ok = do_maybe_wait_for_changes(Host, Node), + ok = do_commit(Host, Node); + {error,nothing_planned} -> + %% Assume plan actually committed somehow + ok; + ok -> + ok + end. + +-spec do_console(#directory_overlay{}, term()) -> term(). +do_console(DirOverlay, Expected) -> + interactive(DirOverlay, "console", Expected). + +-spec do_join(rt_util:host(), node(), node()) -> rt_util:result(). +do_join(Host, Node, ToNode) -> + Result = rt_util:maybe_rpc_call(Node, riak_core, join, [ToNode]), + lager:info("[join] ~p on ~p to ~p: ~p", [Node, Host, ToNode, Result]), + Result. + +-spec do_staged_join(rt_util:host(), node(), node()) -> rt_util:result(). +do_staged_join(Host, Node, ToNode) -> + Result = rt_util:maybe_rpc_call(Node, riak_core, staged_join, [ToNode]), + lager:info("[staged join] ~p on ~p to (~p): ~p", [Node, Host, ToNode, Result]), + Result. + +-spec do_maybe_wait_for_changes(rt_host:host(), node()) -> ok. +do_maybe_wait_for_changes(Host, Node) -> + Ring = rt_ring:get_ring(Node), + Changes = riak_core_ring:pending_changes(Ring), + Joining = riak_core_ring:members(Ring, [joining]), + lager:info("maybe_wait_for_changes node ~p on ~p, changes: ~p joining: ~p ", + [Node, Host, Changes, Joining]), + if Changes =:= [] -> + ok; + Joining =/= [] -> + ok; + true -> + ok = rt_util:wait_until_no_pending_changes(Host, [Node]) + end. + +-spec do_plan(rt_host:host(), node()) -> rt_util:result(). +do_plan(Host, Node) -> + Result = rt_util:maybe_rpc_call(Node, riak_core_claimant, plan, []), + lager:info("Planned cluster using node ~p on ~p with result ~p", [Node, Host, Result]), + case Result of + {ok, _, _} -> ok; + Error -> Error + end. + +-spec do_ping(node()) -> boolean(). +do_ping(Node) -> + net_adm:ping(Node) =:= pong. + + +-spec do_start(rt_util:host(), node(), rt_host:command()) -> pos_integer | rt_util:error(). +do_start(Host, Node, StartCommand) -> + OSPid = rt_util:maybe_call_funs([ + [?MODULE, start_riak_daemon, [Host, Node, StartCommand]], + [?MODULE, get_os_pid, [Host, Node]] + ]), + lager:info("Started node ~p on ~p with OS pid ~p", [Node, Host ,OSPid]), + OSPid. + +-spec do_wait_until_ready(rt_host:host(), node(), [atom()], rt_util:version()) -> boolean. +do_wait_until_ready(Host, Node, RequiredServices, Version) -> + lager:debug("Checking pingable, registered, singleton, and services for node ~p", + [Node]), + Result = rt_util:maybe_call_funs([ + [?MODULE, do_wait_until_pingable, [Host, Node]], + [?MODULE, do_wait_until_registered, [Host, Node, riak_core_ring_manager]], + [?MODULE, do_check_singleton, [Host, Node, Version]], + [?MODULE, do_wait_for_service, [Host, Node, RequiredServices]] + ]), + case Result of + ok -> true; + Error -> lager:warning("Unable to determine that node ~p on ~p is ready due to ~p", + [Node, Host, Error]), + false + end. + +-spec do_start_and_update_state(#state{}) -> {ok | rt_util:error(), {invalid | started | ready, #state{}}}. +do_start_and_update_state(State=#state{host=Host, name=Node, required_services=RequiredServices, + start_command=StartCommand, version=Version}) -> + + OSPid = do_start(Host, Node, StartCommand), + + case OSPid of + Error={error, _} -> {Error, {invalid, State}}; + _ -> UpdState = State#state{os_pid=OSPid}, + Result = do_wait_until_ready(Host, Node, RequiredServices, Version), + case Result of + true -> {ok, {ready, UpdState}}; + false -> {{error, node_not_ready}, {started, UpdState}} + end + end. + + +-spec do_stop_and_update_state(rt_host:host()) -> {ok | {error, term()}, #state{}}. +do_stop_and_update_state(State=#state{host=Host, stop_command=StopCommand}) -> + case do_stop(Host, StopCommand) of + {ok, _} -> {ok, State#state{os_pid=0}}; + {error, Reason} -> {{error, Reason}, State} + end. + +-spec do_stop(rt_host:host(), string()) -> rt_util:result(). +do_stop(Host, StopCommand) -> + rt_host:exec(Host, StopCommand). + +%% @private +-spec do_wait_for_service(rt_host:host(), node(), [atom()]) -> rt_util:wait_result(). +do_wait_for_service(Host, Node, Services) -> + HostName = rt_host:hostname(Host), + lager:info("Waiting for services ~p on node ~p on ~p", [Services, Node, HostName]), + F = fun(N) -> + case rt_util:maybe_rpc_call(N, riak_core_node_watcher, services, [N]) of + Error={error, _} -> + lager:error("Request for the list of services on node ~p on ~p failed due to ~p", + [Node, HostName, Error]), + Error; + CurrServices when is_list(CurrServices) -> + lager:debug("Found services ~p on ~p on ~p", [CurrServices, N, HostName]), + lists:all(fun(Service) -> lists:member(Service, CurrServices) end, Services); + Res -> + Res + end + end, + rt_util:wait_until(Node, F). + +%% @private +-spec do_wait_until_pingable(rt_host:host(), node()) -> rt_util:wait_result(). +do_wait_until_pingable(Host, Node) -> + HostName = rt_host:hostname(Host), + lager:info("Waiting for node ~p on ~p to become pingable", [Node, HostName]), + rt_util:wait_until(Node, fun do_ping/1). + +%% @private +-spec do_wait_until_registered(rt_host:host(), node(), atom()) -> rt_util:wait_result(). +do_wait_until_registered(Host, Node, Name) -> + HostName = rt_host:hostname(Host), + lager:info("Waiting for node ~p on ~p to become registered", [Node, HostName]), + F = fun() -> + Registered = rpc:call(Node, erlang, registered, []), + lists:member(Name, Registered) + end, + case rt_util:wait_until(F) of + ok -> + lager:info("Node ~p on ~p is registered", [Node, HostName]), + ok; + _ -> + lager:error("The server ~p on node ~p on ~p is not coming up.", + [Name, Node, HostName]), + ?assert(registered_name_timed_out) + end. + +-spec do_wait_until_unpingable(rt_host:host(), node()) -> rt_util:result(). +do_wait_until_unpingable(Host, Node) -> + do_wait_until_unpingable(Host, Node, rt_config:get(rt_max_receive_wait_time)). + +-spec do_wait_until_unpingable(rt_host:host(), node(), pos_integer()) -> rt_util:result(). +do_wait_until_unpingable(Host, Node, WaitTime) -> + Delay = rt_config:get(rt_retry_delay), + Retry = WaitTime div Delay, + lager:info("Wait until ~p on ~p is not pingable for ~p seconds with a retry of ~p", + [Node, Host, Delay, Retry]), + %% TODO Move to stop ... + case rt_util:wait_until(fun() -> do_unpingable(Node) end, Retry, Delay) of + ok -> ok; + _ -> + lager:error("Timed out waiting for node ~p on ~p to shutdown", [Node, Host]), + {error, node_shutdown_timed_out} + end. + + +%% @private +-spec do_unpingable(node()) -> boolean(). +do_unpingable(Node) -> + net_adm:ping(Node) =:= pang. + +%% @private +-spec get_os_pid(rt_host:host(), node()) -> pos_integer(). +get_os_pid(Host, Node) -> + HostName = rt_host:hostname(Host), + lager:info("Retrieving the OS pid for node ~p on ~p", [Node, HostName]), + case rt_util:maybe_rpc_call(Node, os, getpid, []) of + Error={error, _} -> + Error; + OSPid -> + list_to_integer(OSPid) + end. + +%% TODO Consider how to capture errors to provide a more meaningul status. Using assertions +%% feels wrong ... +-spec interactive(#directory_overlay{}, string(), term()) -> term(). +interactive(DirOverlay, Command, Expected) -> + Cmd = riak_path(DirOverlay), + lager:info("Opening a port for riak ~s.", [Command]), + lager:debug("Calling open_port with cmd ~s", [binary_to_list(iolist_to_binary(Cmd))]), + Port = open_port({spawn, binary_to_list(iolist_to_binary(Cmd))}, + [stream, use_stdio, exit_status, binary, stderr_to_stdout]), + interactive_loop(Port, Expected). + +interactive_loop(Port, Expected) -> + receive + {Port, {data, Data}} -> + %% We've gotten some data, so the port isn't done executing + %% Let's break it up by newline and display it. + Tokens = string:tokens(binary_to_list(Data), "\n"), + [lager:debug("~s", [Text]) || Text <- Tokens], + + %% Now we're going to take hd(Expected) which is either {expect, X} + %% or {send, X}. If it's {expect, X}, we foldl through the Tokenized + %% data looking for a partial match via rt:str/2. If we find one, + %% we pop hd off the stack and continue iterating through the list + %% with the next hd until we run out of input. Once hd is a tuple + %% {send, X}, we send that test to the port. The assumption is that + %% once we send data, anything else we still have in the buffer is + %% meaningless, so we skip it. That's what that {sent, sent} thing + %% is about. If there were a way to abort mid-foldl, I'd have done + %% that. {sent, _} -> is just a pass through to get out of the fold. + + NewExpected = lists:foldl(fun(X, Expect) -> + [{Type, Text}|RemainingExpect] = case Expect of + [] -> [{done, "done"}|[]]; + E -> E + end, + case {Type, rt2:str(X, Text)} of + {expect, true} -> + RemainingExpect; + {expect, false} -> + [{Type, Text}|RemainingExpect]; + {send, _} -> + port_command(Port, list_to_binary(Text ++ "\n")), + [{sent, "sent"}|RemainingExpect]; + {sent, _} -> + Expect; + {done, _} -> + [] + end + end, Expected, Tokens), + %% Now that the fold is over, we should remove {sent, sent} if it's there. + %% The fold might have ended not matching anything or not sending anything + %% so it's possible we don't have to remove {sent, sent}. This will be passed + %% to interactive_loop's next iteration. + NewerExpected = case NewExpected of + [{sent, "sent"}|E] -> E; + E -> E + end, + %% If NewerExpected is empty, we've met all expected criteria and in order to boot + %% Otherwise, loop. + case NewerExpected of + [] -> ?assert(true); + _ -> interactive_loop(Port, NewerExpected) + end; + {Port, {exit_status,_}} -> + %% This port has exited. Maybe the last thing we did was {send, [4]} which + %% as Ctrl-D would have exited the console. If Expected is empty, then + %% We've met every expectation. Yay! If not, it means we've exited before + %% something expected happened. + ?assertEqual([], Expected) + after rt_config:get(rt_max_receive_wait_time) -> + %% interactive_loop is going to wait until it matches expected behavior + %% If it doesn't, the test should fail; however, without a timeout it + %% will just hang forever in search of expected behavior. See also: Parenting + ?assertEqual([], Expected) + end. + + +-spec start_command({devrel, string()}, #directory_overlay{}) -> rt_host:command(). +start_command({devrel, _}, DirOverlay) -> + {riak_path(DirOverlay), ["start"]}. + +-spec start_riak_daemon(rt_host:host(), node(), rt_host:command()) -> rt_util:result(). +start_riak_daemon(Host, Node, StartCommand) -> + HostName = rt_host:hostname(Host), + lager:notice("Starting riak node ~p on ~p", [Node, HostName]), + rt_host:exec(Host, StartCommand). + +-spec stop_command({devrel, string()}, #directory_overlay{}) -> rt_host:command(). +stop_command({devrel, _}, DirOverlay) -> + {riak_path(DirOverlay), ["stop"]}. + +-spec riak_path(#directory_overlay{}) -> filelib:filename(). +riak_path(#directory_overlay{bin_dir=BinDir}) -> + filename:join([BinDir, "riak"]). + +-spec riak_admin_path(#directory_overlay{}) -> filelib:filename(). +riak_admin_path(#directory_overlay{bin_dir=BinDir}) -> + filename:join([BinDir, "riak-admin"]). + +-spec riak_repl_path(#directory_overlay{}) -> filelib:filename(). +riak_repl_path(#directory_overlay{bin_dir=BinDir}) -> + filename:join([BinDir, "riak-repl"]). + +-spec load_configuration(rt_util:version(), rt_host:host(), #directory_overlay{}) -> + {ok, proplists:proplist()} | rt_util:error(). +load_configuration(Version, Host, #directory_overlay{conf_dir=ConfDir}) -> + rt_host:consult(Host, config_file_path(rt_util:major_release(Version), ConfDir)). + +-spec save_configuration(rt_util:release(), rt_host:host(), term(), #directory_overlay{}) -> rt_util:result(). +save_configuration(Version, Host, Config, #directory_overlay{conf_dir=ConfDir}) -> + MajorRelease = rt_util:major_release(Version), + rt_host:write_file(Host, config_file_path(MajorRelease, ConfDir), rt_util:term_serialized_form(Config)). + +-spec config_file_path(rt_util:release(), filelib:dirname()) -> filelib:filename() | rt_util:error(). +config_file_path(MajorRelease, ConfDir) -> + filename:join(ConfDir, config_file_name(MajorRelease)). + +-spec config_file_name(rt_util:release()) -> string(). +config_file_name(1) -> + "app.config"; +config_file_name(2) -> + "advanced.config"; +config_file_name(Release) -> + erlang:error(io_lib:format("Configuration of release ~p is not supported", [Release])). + +-ifdef(TEST). + +-define(TEST_ROOT_PATH, filename:join([os:getenv("HOME"), "rt", "riak"])). +-define(TEST_VERSION, {riak_ee, "2.0.5"}). + +bootstrap() -> + rt_util:setup_test_env(). + +init_node(HostName, NodeId) -> + {Result, Node} = start_link(HostName, {devrel, ?TEST_ROOT_PATH}, NodeId, [], ?TEST_VERSION), + ?assertEqual(ok, clean_data_dir(Node)), + ?debugFmt("Initialized node id ~p (~p) on ~p as ~p with result ~p", + [NodeId, ?TEST_VERSION, HostName, Node, Result]), + ?assertEqual(ok, Result), + ?assertEqual(true, is_stopped(Node)), + ?assertEqual(false, is_ready(Node)), + + Node. + +setup() -> + true = bootstrap(), + init_node(localhost, 1). + +multi_node_setup(NumNodes) -> + true = bootstrap(), + [ init_node(localhost, NodeId) || NodeId <- lists:seq(1, NumNodes) ]. + +teardown(Nodes) when is_list(Nodes) -> + lists:map(fun(Node) -> release(Node) end, Nodes); +teardown(Node) -> + ?debugFmt("Releasing node ~p", [Node]), + release(Node). + +dev_test_() -> + ?_assertEqual("dev1", dev(1)). + +ip_test_() -> + {timeout, 300, + {foreach, + fun setup/0, + fun teardown/1, + [fun(Node) -> {"get_ip", ?_assertEqual("127.0.0.1", ip(Node))} end]}}. + +verify_async_start(Node) -> + Result = start(Node, false), + ?debugFmt("Started node asynchronously ~p with result ~p", [Node, Result]), + + ?assertEqual(ok, Result), + Result. + +verify_sync_start(Node) -> + Result = start(Node), + ?debugFmt("Started node synchronously ~p with result ~p", [Node, Result]), + + ?assertEqual(ok, Result), + ?assertEqual(false, is_invalid(Node)), + ?assertEqual(false, is_stopped(Node)), + ?assertEqual(false, is_started(Node)), + ?assertEqual(true, is_ready(Node)), + Result. + +verify_sync_stop(Node) -> + Result = stop(Node), + ?debugFmt("Stopped node ~p with result ~p", [Node, Result]), + + ?assertEqual(ok, Result), + ?assertEqual(true, is_stopped(Node)), + ?assertEqual(false, is_ready(Node)), + ?assertEqual(false, is_invalid(Node)), + ?assertEqual(false, is_started(Node)), + Result. + +async_start_test_() -> + {foreach, + fun setup/0, + fun teardown/1, + [fun(Node) -> + {timeout, 600000, + ?_test(begin + ok = verify_async_start(Node), + ?assertEqual(false, is_stopped(Node)), + ?debugFmt("Node ~p asyncronously started .. checking is_ready", [Node]), + ?assertEqual(ok, rt_util:wait_until(Node, fun(Node1) -> is_ready(Node1) end)) + end)} + end]}. + +cookie_test_() -> + {foreach, + fun setup/0, + fun teardown/1, + [fun(Node) -> + {timeout, 600000, + ?_test(begin + ok = verify_sync_start(Node), + ?assertEqual(riak, cookie(Node)) + end)} + end]}. + +load_configuration_test_() -> + {foreach, + fun () -> + application:ensure_started(exec), + {Result, Host} = rt_host:connect(localhost), + ?assertEqual(ok, Result), + Host + end, + fun(Host) -> rt_host:disconnect(Host) end, + [fun(Host) -> + ?_test(begin + DirOverlay = create_directory_overlay({devrel, ?TEST_ROOT_PATH}, ?TEST_VERSION, 1), + {Result, Config} = load_configuration(?TEST_VERSION, Host, DirOverlay), + + ?assertEqual(ok, Result), + ?assertEqual(true, is_list(Config)) + end) + end]}. + +plan_commit_join_test_() -> + {foreach, + fun() -> multi_node_setup(3) end, + fun teardown/1, + [fun(Nodes) -> + {timeout, 600000, + ?_test(begin + %% Start all of the nodes ... + StartResults = rt_util:pmap(fun(Node) -> rt_riak_node:start(Node) end, Nodes), + lists:map(fun(Result) -> ?assertEqual(ok, Result) end, StartResults), + + %% Split the list into a leader and followers ... + [Leader|Followers] = Nodes, + + %% Join the followers to the leader ... + JoinResults = rt_util:pmap(fun(Follower) -> rt_riak_node:join(Follower, Leader) end, Followers), + lists:map(fun(Result) -> ?assertEqual(ok, Result) end, JoinResults), + ?debugFmt("Joined nodes ~p to ~p with result ~p", [Followers, Leader, JoinResults]), + ?assertEqual([ok, ok], JoinResults), + + %% Plan the cluster on the leader ... + PlanResult = plan(Leader), + ?debugFmt("Plan result: ~p", [PlanResult]), + ?assertEqual(ok, PlanResult), + + %% Commit the cluster changes on the leader ... + CommitResult = commit(Leader), + ?assertEqual(ok, CommitResult) + end)} + end]}. + + +sync_start_stop_test_() -> + {foreach, + fun setup/0, + fun teardown/1, + [fun(Node) -> + {timeout, 600000, + ?_test(begin + ok = verify_sync_start(Node), + ok = verify_sync_stop(Node) + end)} + end]}. + +wait_until_pingable_test_() -> + {foreach, + fun setup/0, + fun teardown/1, + [fun(Node) -> + {timeout, 600000, + ?_test(begin + ok = verify_sync_start(Node), + ?assertEqual(ok, wait_until_pingable(Node)) + end)} + end, + fun(Node) -> + {timeout, 600000, + ?_test(begin + ok = verify_async_start(Node), + ?assertEqual(ok, wait_until_pingable(Node)) + end)} + end]}. + +wait_until_unpingable_test_() -> + {foreach, + fun setup/0, + fun teardown/1, + [fun(Node) -> + {timeout, 600000, + ?_test(begin + ok = verify_sync_start(Node), + Result = stop(Node), + ?assertEqual(ok, Result), + WaitResult = wait_until_unpingable(Node), + ?assertEqual(ok, WaitResult) + end)} + end, + fun(Node) -> + {timeout, 600000, + ?_test(begin + ok = verify_sync_start(Node), + Result = stop(Node, false), + ?assertEqual(ok, Result), + WaitResult = wait_until_unpingable(Node), + ?assertEqual(ok, WaitResult) + end)} + end]}. + +version_test_() -> + {foreach, + fun setup/0, + fun teardown/1, + [fun(Node) -> {"get_version", ?_assertEqual(?TEST_VERSION, version(Node))} end]}. + +-endif. diff --git a/src/rt_ring.erl b/src/rt_ring.erl new file mode 100644 index 000000000..1ce488cb3 --- /dev/null +++ b/src/rt_ring.erl @@ -0,0 +1,114 @@ +%% ------------------------------------------------------------------- +%% +%% Copyright (c) 2013-2014 Basho Technologies, Inc. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% ------------------------------------------------------------------- +-module(rt_ring). +-include_lib("eunit/include/eunit.hrl"). + +-export([assert_nodes_agree_about_ownership/1, + check_singleton_node/1, + claimant_according_to/1, + get_ring/1, + members_according_to/1, + nearest_ringsize/1, + nearest_ringsize/2, + owners_according_to/1, + partitions_for_node/1, + status_of_according_to/2]). + +%% @doc Ensure that the specified node is a singleton node/cluster -- a node +%% that owns 100% of the ring. +check_singleton_node(Node) -> + lager:info("Check ~p is a singleton", [Node]), + {ok, Ring} = rpc:call(Node, riak_core_ring_manager, get_raw_ring, []), + Owners = lists:usort([Owner || {_Idx, Owner} <- riak_core_ring:all_owners(Ring)]), + ?assertEqual([Node], Owners), + ok. + +% @doc Get list of partitions owned by node (primary). +partitions_for_node(Node) -> + Ring = get_ring(Node), + [Idx || {Idx, Owner} <- riak_core_ring:all_owners(Ring), Owner == Node]. + +%% @doc Get the raw ring for `Node'. +get_ring(Node) -> + {ok, Ring} = rpc:call(Node, riak_core_ring_manager, get_raw_ring, []), + Ring. + +assert_nodes_agree_about_ownership(Nodes) -> + ?assertEqual(ok, rt:wait_until_ring_converged(Nodes)), + ?assertEqual(ok, rt:wait_until_all_members(Nodes)), + [ ?assertEqual({Node, Nodes}, {Node, owners_according_to(Node)}) || Node <- Nodes]. + +%% @doc Return a list of nodes that own partitions according to the ring +%% retrieved from the specified node. +owners_according_to(Node) -> + case rpc:call(Node, riak_core_ring_manager, get_raw_ring, []) of + {ok, Ring} -> + Owners = [Owner || {_Idx, Owner} <- riak_core_ring:all_owners(Ring)], + lists:usort(Owners); + {badrpc, _}=BadRpc -> + BadRpc + end. + +%% @doc Return a list of cluster members according to the ring retrieved from +%% the specified node. +members_according_to(Node) -> + case rpc:call(Node, riak_core_ring_manager, get_raw_ring, []) of + {ok, Ring} -> + Members = riak_core_ring:all_members(Ring), + Members; + {badrpc, _}=BadRpc -> + BadRpc + end. + +%% @doc Return an appropriate ringsize for the node count passed +%% in. 24 is the number of cores on the bigger intel machines, but this +%% may be too large for the single-chip machines. +nearest_ringsize(Count) -> + nearest_ringsize(Count * 24, 2). + +nearest_ringsize(Count, Power) -> + case Count < trunc(Power * 0.9) of + true -> + Power; + false -> + nearest_ringsize(Count, Power * 2) + end. + +%% @doc Return the cluster status of `Member' according to the ring +%% retrieved from `Node'. +status_of_according_to(Member, Node) -> + case rpc:call(Node, riak_core_ring_manager, get_raw_ring, []) of + {ok, Ring} -> + Status = riak_core_ring:member_status(Ring, Member), + Status; + {badrpc, _}=BadRpc -> + BadRpc + end. + +%% @doc Return a list of nodes that own partitions according to the ring +%% retrieved from the specified node. +claimant_according_to(Node) -> + case rpc:call(Node, riak_core_ring_manager, get_raw_ring, []) of + {ok, Ring} -> + Claimant = riak_core_ring:claimant(Ring), + Claimant; + {badrpc, _}=BadRpc -> + BadRpc + end. diff --git a/src/rt_systest.erl b/src/rt_systest.erl new file mode 100644 index 000000000..1b524c027 --- /dev/null +++ b/src/rt_systest.erl @@ -0,0 +1,179 @@ +%% ------------------------------------------------------------------- +%% +%% Copyright (c) 2013-2014 Basho Technologies, Inc. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% ------------------------------------------------------------------- +-module(rt_systest). +-include_lib("eunit/include/eunit.hrl"). + +-export([read/2, + read/3, + read/5, + read/6, + read/7, + write/2, + write/3, + write/5, + write/6, + verify_systest_value/4]). + +write(Node, Size) -> + write(Node, Size, 2). + +write(Node, Size, W) -> + write(Node, 1, Size, <<"systest">>, W). + +write(Node, Start, End, Bucket, W) -> + write(Node, Start, End, Bucket, W, <<>>). + +%% @doc Write (End-Start)+1 objects to Node. Objects keys will be +%% `Start', `Start+1' ... `End', each encoded as a 32-bit binary +%% (`<>'). Object values are the same as their keys. +%% +%% The return value of this function is a list of errors +%% encountered. If all writes were successful, return value is an +%% empty list. Each error has the form `{N :: integer(), Error :: term()}', +%% where N is the unencoded key of the object that failed to store. +write(Node, Start, End, Bucket, W, CommonValBin) + when is_binary(CommonValBin) -> + rt:wait_for_service(Node, riak_kv), + {ok, C} = riak:client_connect(Node), + F = fun(N, Acc) -> + Obj = riak_object:new(Bucket, <>, + <>), + try C:put(Obj, W) of + ok -> + Acc; + Other -> + [{N, Other} | Acc] + catch + What:Why -> + [{N, {What, Why}} | Acc] + end + end, + lists:foldl(F, [], lists:seq(Start, End)). + +read(Node, Size) -> + read(Node, Size, 2). + +read(Node, Size, R) -> + read(Node, 1, Size, <<"systest">>, R). + +read(Node, Start, End, Bucket, R) -> + read(Node, Start, End, Bucket, R, <<>>). + +read(Node, Start, End, Bucket, R, CommonValBin) + when is_binary(CommonValBin) -> + read(Node, Start, End, Bucket, R, CommonValBin, false). + +%% Read and verify the values of objects written with +%% `systest_write'. The `SquashSiblings' parameter exists to +%% optionally allow handling of siblings whose value and metadata are +%% identical except for the dot. This goal is to facilitate testing +%% with DVV enabled because siblings can be created internally by Riak +%% in cases where testing with DVV disabled would not. Such cases +%% include writes that happen during handoff when a vnode forwards +%% writes, but also performs them locally or when a put coordinator +%% fails to send an acknowledgment within the timeout window and +%% another put request is issued. +read(Node, Start, End, Bucket, R, CommonValBin, SquashSiblings) + when is_binary(CommonValBin) -> + rt:wait_for_service(Node, riak_kv), + {ok, C} = riak:client_connect(Node), + lists:foldl(read_fold_fun(C, Bucket, R, CommonValBin, SquashSiblings), + [], + lists:seq(Start, End)). + +read_fold_fun(C, Bucket, R, CommonValBin, SquashSiblings) -> + fun(N, Acc) -> + GetRes = C:get(Bucket, <>, R), + Val = object_value(GetRes, SquashSiblings), + update_acc(value_matches(Val, N, CommonValBin), Val, N, Acc) + end. + +object_value({error, _}=Error, _) -> + Error; +object_value({ok, Obj}, SquashSiblings) -> + object_value(riak_object:value_count(Obj), Obj, SquashSiblings). + +object_value(1, Obj, _SquashSiblings) -> + riak_object:get_value(Obj); +object_value(_ValueCount, Obj, false) -> + riak_object:get_value(Obj); +object_value(_ValueCount, Obj, true) -> + lager:debug("Siblings detected for ~p:~p", [riak_object:bucket(Obj), riak_object:key(Obj)]), + Contents = riak_object:get_contents(Obj), + case lists:foldl(fun sibling_compare/2, {true, undefined}, Contents) of + {true, {_, _, _, Value}} -> + lager:debug("Siblings determined to be a single value"), + Value; + {false, _} -> + {error, siblings} + end. + +sibling_compare({MetaData, Value}, {true, undefined}) -> + Dot = case dict:find(<<"dot">>, MetaData) of + {ok, DotVal} -> + DotVal; + error -> + {error, no_dot} + end, + VTag = dict:fetch(<<"X-Riak-VTag">>, MetaData), + LastMod = dict:fetch(<<"X-Riak-Last-Modified">>, MetaData), + {true, {element(2, Dot), VTag, LastMod, Value}}; +sibling_compare(_, {false, _}=InvalidMatch) -> + InvalidMatch; +sibling_compare({MetaData, Value}, {true, PreviousElements}) -> + Dot = case dict:find(<<"dot">>, MetaData) of + {ok, DotVal} -> + DotVal; + error -> + {error, no_dot} + end, + VTag = dict:fetch(<<"X-Riak-VTag">>, MetaData), + LastMod = dict:fetch(<<"X-Riak-Last-Modified">>, MetaData), + ComparisonElements = {element(2, Dot), VTag, LastMod, Value}, + {ComparisonElements =:= PreviousElements, ComparisonElements}. + +value_matches(<>, N, CommonValBin) -> + true; +value_matches(_WrongVal, _N, _CommonValBin) -> + false. +update_acc(true, _, _, Acc) -> + Acc; +update_acc(false, {error, _}=Val, N, Acc) -> + [{N, Val} | Acc]; +update_acc(false, Val, N, Acc) -> + [{N, {wrong_val, Val}} | Acc]. + +verify_systest_value(N, Acc, CommonValBin, Obj) -> + Values = riak_object:get_values(Obj), + Res = [begin + case V of + <> -> + ok; + _WrongVal -> + wrong_val + end + end || V <- Values], + case lists:any(fun(X) -> X =:= ok end, Res) of + true -> + Acc; + false -> + [{N, {wrong_val, hd(Values)}} | Acc] + end. + diff --git a/src/rt_test_plan.erl b/src/rt_test_plan.erl new file mode 100644 index 000000000..9679f9899 --- /dev/null +++ b/src/rt_test_plan.erl @@ -0,0 +1,215 @@ +%%------------------------------------------------------------------- +%% +%% Copyright (c) 2015 Basho Technologies, Inc. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% ------------------------------------------------------------------- +%% @author Brett Hazen +%% @copyright (C) 2015, Basho Technologies +%% @doc +%% +%% @end +%% Created : 30. Mar 2015 3:25 PM +%%------------------------------------------------------------------- +-module(rt_test_plan). +-author("Brett Hazen"). + +%% API +-export([new/0, + new/1, + get/2, + get_module/1, + get_name/1, + set/2, + set/3]). + +-record(rt_test_plan_v1, { + id=-1 :: integer(), + module :: atom(), + project=rt_config:get_default_version_product() :: atom() | binary(), + platform :: string(), + version=rt_config:get_default_version() :: string(), + backend=undefined :: atom(), + upgrade_path=undefined :: [rt_properties2:product_version()], + properties :: rt_properties2:properties() +}). + +-type test_plan() :: #rt_test_plan_v1{}. + +-export_type([test_plan/0]). + +-define(RT_TEST_PLAN, #rt_test_plan_v1). +-define(RECORD_FIELDS, record_info(fields, rt_test_plan_v1)). + +%% Internal + +%% @doc Create a new test plan record with all fields initialized to +%% the default values. +-spec new() -> test_plan(). +new() -> + ?RT_TEST_PLAN{}. + +%% @doc Create a new test plan record with the fields initialized to +%% non-default value. Each field to be initialized should be +%% specified as an entry in a property list (i.e. a list of +%% pairs). Invalid field fields are ignored by this function. +-spec new(proplists:proplist()) -> test_plan(). +new(FieldDefaults) -> + {Fields, _} = + lists:foldl(fun set_field/2, {?RT_TEST_PLAN{}, []}, FieldDefaults), + Fields. + +%% @doc Get the value of a field from a test plan record. An error +%% is returned if `TestPlan' is not a valid `rt_test_plan' record +%% or if the field requested is not a valid field. +-spec get(atom(), test_plan()) -> term() | {error, atom()}. +get(Field, TestPlan) -> + get(Field, TestPlan, validate_request(Field, TestPlan)). + +%% @doc Get the value of the test name from a test plan record. An error +%% is returned if `TestPlan' is not a valid `rt_test_plan' record +%% or if the field requested is not a valid field. +-spec get_module(test_plan()) -> term() | {error, atom()}. +get_module(TestPlan) -> + get(module, TestPlan, validate_request(module, TestPlan)). + +%% @doc Get the value of the name, backend and upgrade from a test plan record. An error +%% is returned if `TestPlan' is not a valid `rt_test_plan' record +%% or if the field requested is not a valid field. +-spec get_name(test_plan()) -> term() | {error, atom()}. +get_name(TestPlan) -> + Module = get(module, TestPlan, validate_request(module, TestPlan)), + Backend = get(backend, TestPlan, validate_request(backend, TestPlan)), + ModBackend = atom_to_list(Module) ++ "-" ++ atom_to_list(Backend), + Upgrade = get(upgrade_path, TestPlan, validate_request(upgrade_path, TestPlan)), + case Upgrade of + undefined -> ModBackend; + _ -> ModBackend ++ "-" ++ rt_config:convert_to_string(Upgrade) + end. + +%% @doc Set the value for a field in a test plan record. An error +%% is returned if `TestPlan' is not a valid `rt_test_plan' record +%% or if any of the fields to be set are not a valid field. In +%% the case that invalid fields are specified the error returned +%% contains a list of erroneous fields. +-spec set([{atom(), term()}], test_plan()) -> test_plan() | {error, atom()}. +set(FieldList, TestPlan) when is_list(FieldList) -> + set_fields(FieldList, TestPlan, validate_record(TestPlan)). + +%% @doc Set the value for a field in a test plan record. An error +%% is returned if `TestPlan' is not a valid `rt_test_plan' record +%% or if the field to be set is not a valid field. +-spec set(atom(), term(), test_plan()) -> {ok, test_plan()} | {error, atom()}. +set(Field, Value, TestPlan) -> + set_field(Field, Value, TestPlan, validate_request(Field, TestPlan)). + + +-spec get(atom(), test_plan(), ok | {error, atom()}) -> + term() | {error, atom()}. +get(Field, Fields, ok) -> + element(field_index(Field), Fields); +get(_Field, _Fields, {error, _}=Error) -> + Error. + +%% This function is used by `new/1' to set fields at record +%% creation time and by `set/2' to set multiple fields at once. +%% Test plan record validation is done by this function. It is +%% strictly used as a fold function which is the reason for the odd +%% structure of the input parameters. It accumulates any invalid +%% properties that are encountered and the caller may use that +%% information or ignore it. +-spec set_field({atom(), term()}, {test_plan(), [atom()]}) -> + {test_plan(), [atom()]}. +set_field({Field, Value}, {TestPlan, Invalid}) -> + case is_valid_field(Field) of + true -> + {setelement(field_index(Field), TestPlan, Value), Invalid}; + false -> + {TestPlan, [Field | Invalid]} + end. + +-spec set_field(atom(), term(), test_plan(), ok | {error, atom()}) -> + {ok, test_plan()} | {error, atom()}. +set_field(Field, Value, TestPlan, ok) -> + {ok, setelement(field_index(Field), TestPlan, Value)}; +set_field(_Field, _Value, _Fields, {error, _}=Error) -> + Error. + +-spec set_fields([{atom(), term()}], + test_plan(), + ok | {error, {atom(), [atom()]}}) -> + {test_plan(), [atom()]}. +set_fields(FieldList, TestPlan, ok) -> + case lists:foldl(fun set_field/2, {TestPlan, []}, FieldList) of + {UpdFields, []} -> + UpdFields; + {_, InvalidFields} -> + {error, {invalid_properties, InvalidFields}} + end; +set_fields(_, _, {error, _}=Error) -> + Error. + +-spec validate_request(atom(), test_plan()) -> ok | {error, atom()}. +validate_request(Field, TestPlan) -> + validate_field(Field, validate_record(TestPlan)). + +-spec validate_record(test_plan()) -> ok | {error, invalid_test_plan}. +validate_record(Record) -> + case is_valid_record(Record) of + true -> + ok; + false -> + {error, invalid_test_plan} + end. + +-spec validate_field(atom(), ok | {error, atom()}) -> ok | {error, invalid_field}. +validate_field(Field, ok) -> + case is_valid_field(Field) of + true -> + ok; + false -> + {error, invalid_field} + end; +validate_field(_Field, {error, _}=Error) -> + Error. + +-spec is_valid_record(term()) -> boolean(). +is_valid_record(Record) -> + is_record(Record, rt_test_plan_v1). + +-spec is_valid_field(atom()) -> boolean(). +is_valid_field(Field) -> + Fields = ?RECORD_FIELDS, + lists:member(Field, Fields). + +-spec field_index(atom()) -> non_neg_integer(). +field_index(id) -> + ?RT_TEST_PLAN.id; +field_index(module) -> + ?RT_TEST_PLAN.module; +field_index(project) -> + ?RT_TEST_PLAN.project; +field_index(platform) -> + ?RT_TEST_PLAN.platform; +field_index(version) -> + ?RT_TEST_PLAN.version; +field_index(backend) -> + ?RT_TEST_PLAN.backend; +field_index(upgrade_path) -> + ?RT_TEST_PLAN.upgrade_path; +field_index(properties) -> + ?RT_TEST_PLAN.properties. + diff --git a/src/rt_util.erl b/src/rt_util.erl new file mode 100644 index 000000000..521915c2f --- /dev/null +++ b/src/rt_util.erl @@ -0,0 +1,283 @@ +-module(rt_util). +-include_lib("eunit/include/eunit.hrl"). + +-type error() :: {error(), term()}. +-type result() :: ok | error(). +-type wait_result() ::ok | {fail, term()}. + +-type erl_rpc_result() :: {ok, term()} | {badrpc, term()}. +-type rt_rpc_result() :: {ok, term()} | rt_util:error(). + +-type release() :: string(). +-type products() :: riak | riak_ee | riak_cs | riak_cs_ee. +-type version() :: {products(), release()}. +-type version_selector() :: atom() | version(). + +-export_type([error/0, + products/0, + release/0, + result/0, + rt_rpc_result/0, + version/0, + version_selector/0, + wait_result/0]). + +-export([add_deps/1, + convert_to_atom_list/1, + base_dir_for_version/2, + ip_addr_to_string/1, + maybe_append_when_not_endswith/2, + maybe_call_funs/1, + maybe_rpc_call/4, + major_release/1, + merge_configs/2, + pmap/2, + parse_release/1, + term_serialized_form/1, + version_to_string/1, + wait_until/1, + wait_until/2, + wait_until/3, + wait_until_no_pending_changes/2]). + +-ifdef(TEST). +-include_lib("eunit/include/eunit.hrl"). +-export([setup_test_env/0, + test_success_fun/0]). +-endif. + +%% @doc Convert string or atom to list of atoms +-spec(convert_to_atom_list(atom()|string()) -> undefined | list()). +convert_to_atom_list(undefined) -> + undefined; +convert_to_atom_list(Values) when is_atom(Values) -> + ListOfValues = atom_to_list(Values), + case lists:member($, , ListOfValues) of + true -> + [list_to_atom(X) || X <- string:tokens(ListOfValues, ", ")]; + _ -> + [Values] + end; +convert_to_atom_list(Values) when is_list(Values) -> + case lists:member($, , Values) of + true -> + [list_to_atom(X) || X <- string:tokens(Values, ", ")]; + _ -> + [list_to_atom(Values)] + end. + +-spec add_deps(filelib:dirname()) -> ok. +add_deps(Path) -> + lager:debug("Adding dep path ~p", [Path]), + case file:list_dir(Path) of + {ok, Deps} -> + [code:add_path(lists:append([Path, "/", Dep, "/ebin"])) || Dep <- Deps], + ok; + {error, Reason} -> + lager:error("Failed to add dep path ~p due to ~p.", [Path, Reason]), + erlang:error(Reason) + end. + +-spec base_dir_for_version(filelib:dirname(), version()) -> filelib:dirname(). +base_dir_for_version(RootPath, Version) -> + filename:join(RootPath, version_to_string(Version)). + +-spec ip_addr_to_string({pos_integer(), pos_integer(), pos_integer(), pos_integer()}) -> string(). +ip_addr_to_string(IP) -> + string:join([integer_to_list(X) || X <- tuple_to_list(IP)], "."). + +-spec maybe_append_when_not_endswith(string(), string()) -> string(). +maybe_append_when_not_endswith(String, Suffix) -> + maybe_append_when_not_endswith(lists:suffix(Suffix, String), String, Suffix). + +-spec maybe_append_when_not_endswith(boolean(), string(), string()) -> string(). +maybe_append_when_not_endswith(true, String, _Suffix) -> + String; +maybe_append_when_not_endswith(false, String, Suffix) -> + String ++ Suffix. + +-spec maybe_rpc_call(node(), module(), function(), [term()]) -> erl_rpc_result(). +maybe_rpc_call(NodeName, Module, Function, Args) -> + maybe_rpc_call(rpc:call(NodeName, Module, Function, Args)). + +%% -spec maybe_call_funs([module(), function(), [term()]]) -> term(). +maybe_call_funs(CallSpecs) -> + lists:foldl(fun([Module, Function, Args], PrevResult) -> + maybe_call_fun(PrevResult, Module, Function, Args) + end, ok, CallSpecs). + +-spec maybe_call_fun(ok | {ok, term()} | rt_util:error(), module(), function(), [term()]) -> term(). +maybe_call_fun(ok, Module, Function, Args) -> + erlang:apply(Module, Function, Args); +maybe_call_fun({ok, _}, Module, Function, Args) -> + erlang:apply(Module, Function, Args); +maybe_call_fun(true, Module, Function, Args) -> + erlang:apply(Module, Function, Args); +maybe_call_fun(Error, Module, Function, Args) -> + lager:debug("~p:~p(~p) not called due error ~p", [Module, Function, Args, Error]), + Error. + +-spec maybe_rpc_call(erl_rpc_result()) -> rt_rpc_result(). +maybe_rpc_call({badrpc, _}) -> + {error, badrpc}; +maybe_rpc_call(Result) -> + Result. + +-spec merge_configs(proplists:proplist() | tuple(), proplists:proplist() | tuple()) -> orddict:orddict() | tuple(). +merge_configs(PropList, ThatPropList) when is_list(PropList) and is_list(ThatPropList) -> + MergeA = orddict:from_list(PropList), + MergeB = orddict:from_list(ThatPropList), + orddict:merge(fun(_, VarsA, VarsB) -> + merge_configs(VarsA, VarsB) + end, MergeA, MergeB); +merge_configs(_, Value) -> + Value. + +-spec major_release(version() | release()) -> pos_integer(). +major_release({_, Release}) -> + major_release(Release); +major_release(Release) -> + {Major, _, _} = parse_release(Release), + Major. + +%% @doc Parallel Map: Runs function F for each item in list L, then +%% returns the list of results +-spec pmap(F :: fun(), L :: list()) -> list(). +pmap(F, L) -> + Parent = self(), + lists:foldl( + fun(X, N) -> + spawn_link(fun() -> + Parent ! {pmap, N, F(X)} + end), + N+1 + end, 0, L), + L2 = [receive {pmap, N, R} -> {N,R} end || _ <- L], + {_, L3} = lists:unzip(lists:keysort(1, L2)), + L3. + +-spec parse_release(version() | release()) -> { pos_integer(), pos_integer(), pos_integer() }. +parse_release({_, Release}) -> + parse_release(Release); +parse_release(Release) -> + list_to_tuple([list_to_integer(Token) || Token <- string:tokens(Release, ".")]). + +-spec term_serialized_form(term()) -> string(). +term_serialized_form(Term) -> + io_lib:format("~p.", [Term]). + +-spec version_to_string(version()) -> string(). +version_to_string({Product, Release}) -> + string:join([atom_to_list(Product), Release], "-"). + +%% @doc Utility function used to construct test predicates. Retries the +%% function `Fun' until it returns `true', or until the maximum +%% number of retries is reached. The retry limit is based on the +%% provided `rt_max_receive_wait_time' and `rt_retry_delay' parameters in +%% specified `riak_test' config file. +%% +%% @since 1.1.0 +-spec wait_until(function()) -> wait_result(). +wait_until(Fun) when is_function(Fun) -> + MaxTime = rt_config:get(rt_max_receive_wait_time), + Delay = rt_config:get(rt_retry_delay), + Retry = MaxTime div Delay, + wait_until(Fun, Retry, Delay). + +%% @doc Convenience wrapper for wait_until for the myriad functions that +%% take a node as single argument. +%% +%% @since 1.1.0 +-spec wait_until(node(), function()) -> wait_result(). +wait_until(Node, Fun) when is_atom(Node), is_function(Fun) -> + wait_until(fun() -> Fun(Node) end). + +%% @doc Retry `Fun' until it returns `Retry' times, waiting `Delay' +%% milliseconds between retries. This is our eventual consistency bread +%% and butter +%% +%% @since 1.1.0 +-spec wait_until(function(), pos_integer(), pos_integer()) -> wait_result(). +wait_until(Fun, Retry, Delay) when Retry > 0 -> + Res = Fun(), + case Res of + true -> + ok; + _ when Retry == 1 -> + {fail, Res}; + _ -> + timer:sleep(Delay), + wait_until(Fun, Retry-1, Delay) + end. + +-ifdef(TEST). + +%% Properly covert backends to atoms +convert_to_atom_list_test() -> + ?assertEqual(undefined, convert_to_atom_list(undefined)), + ?assertEqual([memory], convert_to_atom_list(memory)), + ?assertEqual([memory], convert_to_atom_list("memory")), + ?assertEqual([bitcask, eleveldb, memory], lists:sort(convert_to_atom_list("memory, bitcask,eleveldb"))), + ?assertEqual([bitcask, eleveldb, memory], lists:sort(convert_to_atom_list('memory, bitcask,eleveldb'))). + +-spec wait_until_no_pending_changes(rt_host:host(), [node()]) -> ok | fail. +wait_until_no_pending_changes(Host, Nodes) -> + lager:info("Wait until no pending changes for nodes ~p on ~p", [Nodes, Host]), + F = fun() -> + rpc:multicall(Nodes, riak_core_vnode_manager, force_handoffs, []), + {Rings, BadNodes} = rpc:multicall(Nodes, riak_core_ring_manager, get_raw_ring, []), + Changes = [ riak_core_ring:pending_changes(Ring) =:= [] || {ok, Ring} <- Rings ], + BadNodes =:= [] andalso length(Changes) =:= length(Nodes) andalso lists:all(fun(T) -> T end, Changes) + end, + wait_until(F). + +test_success_fun() -> + ok. + +setup_test_env() -> + application:ensure_started(exec), + rt_config:set(rt_max_receive_wait_time, 600000), + rt_config:set(rt_retry_delay, 1000), + + %% TODO Consider loading up the riak_test.config to get this information + add_deps(filename:join([os:getenv("HOME"), "rt", ".riak-builds", "riak_ee-head", "deps"])), + + {ok, _} = exec:run("epmd -daemon", [sync, stdout, stderr]), + net_kernel:start(['riak_test@127.0.0.1']), + erlang:set_cookie(node(), riak). + +base_dir_for_version_test_() -> + [?_assertEqual("foo/riak_ee-2.0.5", base_dir_for_version("foo", {riak_ee, "2.0.5"}))]. + +%% TODO Refactor into an EQC model ... +ip_addr_to_string_test_() -> + [?_assertEqual("127.0.0.1", ip_addr_to_string({127, 0, 0, 1}))]. + +maybe_append_when_not_endswith_test_() -> + [?_assertEqual("foobar", maybe_append_when_not_endswith("foobar", "bar")), + ?_assertEqual("foobar", maybe_append_when_not_endswith("foo", "bar"))]. + +maybe_call_fun_test_() -> + [?_assertEqual(ok, maybe_call_fun(ok, ?MODULE, test_success_fun, [])), + ?_assertEqual({error, test_failure}, maybe_call_fun({error, test_failure}, ?MODULE, test_success_fun, [])) ]. + +%% TODO Refactor into an EQC model ... +major_version_test_() -> + [?_assertEqual(1, major_release("1.3.4")), + ?_assertEqual(1, major_release({riak_ee, "1.3.4"}))]. + +%% TODO Refactor into an EQC model ... +merge_configs_test_() -> + [?_assertEqual([{a,1},{b,2},{c,3},{d,4}], merge_configs([{a,1},{b,2}],[{c,3},{d,4}])), + ?_assertEqual([{a,1},{b,3},{c,3},{d,4}], merge_configs([{a,1},{b,2}],[{b,3},{c,3},{d,4}])), + ?_assertEqual([{a, [{b,3}, {c,5}, {d,6}]}, {e,7}], merge_configs([{a, [{b,2}, {c,5}]}], [{a, [{b,3}, {d,6}]}, {e,7}]))]. + +%% TODO Refactor into an EQC model ... +parse_release_test_() -> + [?_assertEqual({1, 3, 4}, parse_release("1.3.4")), + ?_assertEqual({1, 3, 4}, parse_release({riak_ee, "1.3.4"}))]. + +version_to_string_test_() -> + [?_assertEqual("riak_ee-2.0.5", version_to_string({riak_ee, "2.0.5"}))]. + +-endif. diff --git a/src/rtdev.erl b/src/rtdev.erl index cb725c363..7f5822ca5 100644 --- a/src/rtdev.erl +++ b/src/rtdev.erl @@ -18,29 +18,77 @@ %% %% ------------------------------------------------------------------- -%% @private -module(rtdev). +%% -behaviour(test_harness). +-export([start/2, + stop/2, + deploy_clusters/1, + clean_data_dir/2, + clean_data_dir/3, + spawn_cmd/1, + spawn_cmd/2, + cmd/1, + cmd/2, + setup_harness/0, + %% setup_harness/2, + get_version/0, + get_backends/0, + set_backend/1, + whats_up/0, + get_ip/1, + node_id/1, + node_short_name/1, + node_version/1, + %% admin/2, + riak/2, + attach/2, + attach_direct/2, + console/2, + update_app_config/3, + teardown/0, + set_conf/2, + set_advanced_conf/2, + rm_dir/1, + validate_config/1, + get_node_logs/2, + get_node_logs/0]). + -compile(export_all). -include_lib("eunit/include/eunit.hrl"). --define(DEVS(N), lists:concat(["dev", N, "@127.0.0.1"])). +-define(DEVS(N), lists:concat([N, "@127.0.0.1"])). -define(DEV(N), list_to_atom(?DEVS(N))). --define(PATH, (rt_config:get(rtdev_path))). +-define(PATH, (rt_config:get(root_path))). +-define(SCRATCH_DIR, (rt_config:get(rt_scratch_dir))). + +%% @doc Convert a node number into a devrel node name +-spec devrel_node_name(N :: integer()) -> atom(). +devrel_node_name(N) when is_integer(N) -> + list_to_atom(lists:concat(["dev", N, "@127.0.0.1"])). get_deps() -> - lists:flatten(io_lib:format("~s/dev/dev1/lib", [relpath(current)])). + DefaultVersionPath = filename:join(?PATH, rt_config:get_default_version()), + lists:flatten(io_lib:format("~s/dev1/lib", [DefaultVersionPath])). +%% @doc Create a command-line command +-spec riakcmd(Path :: string(), N :: string(), Cmd :: string()) -> string(). riakcmd(Path, N, Cmd) -> ExecName = rt_config:get(exec_name, "riak"), - io_lib:format("~s/dev/dev~b/bin/~s ~s", [Path, N, ExecName, Cmd]). + io_lib:format("~s/~s/bin/~s ~s", [Path, N, ExecName, Cmd]). +%% @doc Create a command-line command for repl +-spec riakreplcmd(Path :: string(), N :: string(), Cmd :: string()) -> string(). riakreplcmd(Path, N, Cmd) -> - io_lib:format("~s/dev/dev~b/bin/riak-repl ~s", [Path, N, Cmd]). + io_lib:format("~s/~s/bin/riak-repl ~s", [Path, N, Cmd]). gitcmd(Path, Cmd) -> io_lib:format("git --git-dir=\"~s/.git\" --work-tree=\"~s/\" ~s", [Path, Path, Cmd]). +%% @doc Create a command-line command for riak-admin +-spec riak_admin_cmd(Path :: string(), N :: integer() | string(), Args :: string()) -> string(). +riak_admin_cmd(Path, N, Args) when is_integer(N) -> + riak_admin_cmd(Path, node_short_name_to_name(N), Args); riak_admin_cmd(Path, N, Args) -> Quoted = lists:map(fun(Arg) when is_list(Arg) -> @@ -50,34 +98,39 @@ riak_admin_cmd(Path, N, Args) -> end, Args), ArgStr = string:join(Quoted, " "), ExecName = rt_config:get(exec_name, "riak"), - io_lib:format("~s/dev/dev~b/bin/~s-admin ~s", [Path, N, ExecName, ArgStr]). + {NodeId, _} = extract_node_id_and_name(N), + io_lib:format("~s/~s/bin/~s-admin ~s", [Path, NodeId, ExecName, ArgStr]). run_git(Path, Cmd) -> lager:info("Running: ~s", [gitcmd(Path, Cmd)]), {0, Out} = cmd(gitcmd(Path, Cmd)), Out. -run_riak(N, Path, Cmd) -> - lager:info("Running: ~s", [riakcmd(Path, N, Cmd)]), - R = os:cmd(riakcmd(Path, N, Cmd)), - case Cmd of - "start" -> - rt_cover:maybe_start_on_node(?DEV(N), node_version(N)), - %% Intercepts may load code on top of the cover compiled - %% modules. We'll just get no coverage info then. - case rt_intercept:are_intercepts_loaded(?DEV(N)) of - false -> - ok = rt_intercept:load_intercepts([?DEV(N)]); - true -> - ok - end, - R; - "stop" -> - rt_cover:maybe_stop_on_node(?DEV(N)), - R; - _ -> - R - end. +%% @doc Run a riak command line command, returning its result +-spec run_riak(Node :: string(), Version :: string(), string()) -> string(). +run_riak(Node, Version, "start") -> + VersionPath = filename:join(?PATH, Version), + {NodeId, NodeName} = extract_node_id_and_name(Node), + RiakCmd = riakcmd(VersionPath, NodeId, "start"), + lager:info("Running: ~s", [RiakCmd]), + CmdRes = os:cmd(RiakCmd), + %% rt_cover:maybe_start_on_node(?DEV(Node), Version), + %% Intercepts may load code on top of the cover compiled + %% modules. We'll just get no coverage info then. + case rt_intercept:are_intercepts_loaded(NodeName) of + false -> + ok = rt_intercept:load_intercepts([NodeName]); + true -> + ok + end, + CmdRes; +run_riak(Node, Version, "stop") -> + VersionPath = filename:join(?PATH, Version), + %% rt_cover:maybe_stop_on_node(?DEV(Node)), + os:cmd(riakcmd(VersionPath, Node, "stop")); +run_riak(Node, Version, Cmd) -> + VersionPath = filename:join(?PATH, Version), + os:cmd(riakcmd(VersionPath, Node, Cmd)). run_riak_repl(N, Path, Cmd) -> lager:info("Running: ~s", [riakcmd(Path, N, Cmd)]), @@ -85,76 +138,142 @@ run_riak_repl(N, Path, Cmd) -> %% don't mess with intercepts and/or coverage, %% they should already be setup at this point -setup_harness(_Test, _Args) -> - %% make sure we stop any cover processes on any nodes - %% otherwise, if the next test boots a legacy node we'll end up with cover - %% incompatabilities and crash the cover server - rt_cover:maybe_stop_on_nodes(), - Path = relpath(root), - %% Stop all discoverable nodes, not just nodes we'll be using for this test. - rt:pmap(fun(X) -> stop_all(X ++ "/dev") end, devpaths()), +-spec versions() -> [string()]. +versions() -> + RootPath = ?PATH, + case file:list_dir(RootPath) of + {ok, RootFiles} -> + [Version || Version <- RootFiles, + filelib:is_dir(filename:join(RootPath, Version)), + hd(Version) =/= $.]; + {error, _} -> + [] + end. + +-spec harness_node_ids(string()) -> [string()]. +harness_node_ids(Version) -> + VersionPath = filename:join(?PATH, Version), + case file:list_dir(VersionPath) of + {ok, VersionFiles} -> + SortedVersionFiles = lists:sort(VersionFiles), + [Node || Node <- SortedVersionFiles, + filelib:is_dir(filename:join(VersionPath, Node))]; + {error, _} -> + [] + end. + +-spec harness_nodes([string()]) -> [atom()]. +harness_nodes(NodeIds) -> + [list_to_atom(NodeId ++ "@127.0.0.1") || NodeId <- NodeIds]. + +so_fresh_so_clean(VersionMap) -> + ok = stop_all(VersionMap), %% Reset nodes to base state lager:info("Resetting nodes to fresh state"), - _ = run_git(Path, "reset HEAD --hard"), - _ = run_git(Path, "clean -fd"), + _ = run_git(?PATH, "reset HEAD --hard"), + _ = run_git(?PATH, "clean -fd"), lager:info("Cleaning up lingering pipe directories"), - rt:pmap(fun(Dir) -> + rt:pmap(fun({Version, _}) -> %% when joining two absolute paths, filename:join intentionally %% throws away the first one. ++ gets us around that, while %% keeping some of the security of filename:join. %% the extra slashes will be pruned by filename:join, but this %% ensures that there will be at least one between "/tmp" and Dir - PipeDir = filename:join(["/tmp//" ++ Dir, "dev"]), + %% TODO: Double check this is correct + PipeDir = filename:join("/tmp" ++ ?PATH, Version), {0, _} = cmd("rm -rf " ++ PipeDir) - end, devpaths()), + end, VersionMap), ok. +stop_all() -> + [_, _, VersionMap] = available_resources(), + stop_all(VersionMap). + +stop_all(VersionMap) -> + %% make sure we stop any cover processes on any nodes otherwise, + %% if the next test boots a legacy node we'll end up with cover + %% incompatabilities and crash the cover server + %% rt_cover:maybe_stop_on_nodes(), + %% Path = relpath(root), + %% Stop all discoverable nodes, not just nodes we'll be using for + %% this test. + StopAllFun = + fun({Version, VersionNodes}) -> + VersionPath = filename:join([?PATH, Version]), + stop_nodes(VersionPath, VersionNodes) + end, + rt:pmap(StopAllFun, VersionMap), + ok. + +available_resources() -> + VersionMap = [{Version, harness_node_ids(Version)} || Version <- versions()], + NodeIds = harness_node_ids(rt_config:get_default_version()), + NodeMap = lists:zip(NodeIds, harness_nodes(NodeIds)), + [NodeIds, NodeMap, VersionMap]. + +setup_harness() -> + %% Get node names and populate node map + [NodeIds, NodeMap, VersionMap] = available_resources(), + so_fresh_so_clean(VersionMap), + rm_dir(filename:join(?SCRATCH_DIR, "gc")), + rt_harness_util:setup_harness(VersionMap, NodeIds, NodeMap). + +%% @doc Tack the version onto the end of the root path by looking +%% up the root in the configuation +-spec relpath(Vsn :: string()) -> string(). relpath(Vsn) -> Path = ?PATH, - relpath(Vsn, Path). + ActualVersion = rt_config:version_to_tag(Vsn), + relpath(ActualVersion, Path). -relpath(Vsn, Paths=[{_,_}|_]) -> - orddict:fetch(Vsn, orddict:from_list(Paths)); -relpath(current, Path) -> - Path; -relpath(root, Path) -> - Path; -relpath(_, _) -> - throw("Version requested but only one path provided"). +%% @doc Tack the version onto the end of the root path +-spec relpath(Vsn :: string(), Path :: string()) -> string(). +relpath(Vsn, Path) -> + lists:concat([Path, "/", Vsn]). +%% TODO: Need to replace without the node_version/1 map upgrade(Node, NewVersion) -> - upgrade(Node, NewVersion, same). - -upgrade(Node, NewVersion, Config) -> N = node_id(Node), - Version = node_version(N), - lager:info("Upgrading ~p : ~p -> ~p", [Node, Version, NewVersion]), - stop(Node), - rt:wait_until_unpingable(Node), - OldPath = relpath(Version), - NewPath = relpath(NewVersion), + CurrentVersion = node_version(N), + UpgradedVersion = rt_config:version_to_tag(NewVersion), + upgrade(Node, CurrentVersion, UpgradedVersion, same). +%% upgrade(Node, CurrentVersion, NewVersion) -> +%% upgrade(Node, CurrentVersion, NewVersion, same). + +upgrade(Node, CurrentVersion, NewVersion, Config) -> + lager:info("Upgrading ~p : ~p -> ~p", [Node, CurrentVersion, NewVersion]), + stop(Node, CurrentVersion), + rt:wait_until_unpingable(Node), + CurrentPath = filename:join([?PATH, CurrentVersion, node_short_name(Node)]), + NewPath = filename:join([?PATH, NewVersion, node_short_name(Node)]), Commands = [ - io_lib:format("cp -p -P -R \"~s/dev/dev~b/data\" \"~s/dev/dev~b\"", - [OldPath, N, NewPath, N]), - io_lib:format("rm -rf ~s/dev/dev~b/data/*", - [OldPath, N]), - io_lib:format("cp -p -P -R \"~s/dev/dev~b/etc\" \"~s/dev/dev~b\"", - [OldPath, N, NewPath, N]) + io_lib:format("cp -p -P -R \"~s\" \"~s\"", + [filename:join(CurrentPath, "data"), + NewPath]), + %% io_lib:format("rm -rf ~s*", + %% [filename:join([CurrentPath, "data", "*"])]), + io_lib:format("cp -p -P -R \"~s\" \"~s\"", + [filename:join(CurrentPath, "etc"), + NewPath]) ], - [ begin - lager:info("Running: ~s", [Cmd]), - os:cmd(Cmd) - end || Cmd <- Commands], - VersionMap = orddict:store(N, NewVersion, rt_config:get(rt_versions)), + [begin + lager:debug("Running: ~s", [Cmd]), + os:cmd(Cmd) + end || Cmd <- Commands], + clean_data_dir(node_short_name(Node), CurrentVersion, ""), + + %% TODO: This actually is required by old framework + VersionMap = orddict:store(Node, NewVersion, rt_config:get(rt_versions)), rt_config:set(rt_versions, VersionMap), + case Config of same -> ok; - _ -> update_app_config(Node, Config) + _ -> update_app_config(Node, NewVersion, Config) end, - start(Node), + start(Node, NewVersion), rt:wait_until_pingable(Node), ok. @@ -195,7 +314,7 @@ make_advanced_confs(DevPath) -> lager:error("Failed generating advanced.conf ~p is not a directory.", [DevPath]), []; true -> - Wildcard = io_lib:format("~s/dev/dev*/etc", [DevPath]), + Wildcard = io_lib:format("~s/dev*/etc", [DevPath]), ConfDirs = filelib:wildcard(Wildcard), [ begin @@ -207,14 +326,12 @@ make_advanced_confs(DevPath) -> end. get_riak_conf(Node) -> - N = node_id(Node), - Path = relpath(node_version(N)), - io_lib:format("~s/dev/dev~b/etc/riak.conf", [Path, N]). + Path = relpath(node_version(Node)), + io_lib:format("~s/~s/etc/riak.conf", [Path, Node]). get_advanced_riak_conf(Node) -> - N = node_id(Node), - Path = relpath(node_version(N)), - io_lib:format("~s/dev/dev~b/etc/advanced.config", [Path, N]). + Path = relpath(node_version(Node)), + io_lib:format("~s/~s/etc/advanced.config", [Path, Node]). append_to_conf_file(File, NameValuePairs) -> Settings = lists:flatten( @@ -224,7 +341,7 @@ append_to_conf_file(File, NameValuePairs) -> all_the_files(DevPath, File) -> case filelib:is_dir(DevPath) of true -> - Wildcard = io_lib:format("~s/dev/dev*/~s", [DevPath, File]), + Wildcard = io_lib:format("~s/dev*/~s", [DevPath, File]), filelib:wildcard(Wildcard); _ -> lager:debug("~s is not a directory.", [DevPath]), @@ -235,35 +352,43 @@ all_the_app_configs(DevPath) -> AppConfigs = all_the_files(DevPath, "etc/app.config"), case length(AppConfigs) =:= 0 of true -> - AdvConfigs = filelib:wildcard(DevPath ++ "/dev/dev*/etc"), + AdvConfigs = filelib:wildcard(DevPath ++ "/dev*/etc"), [ filename:join(AC, "advanced.config") || AC <- AdvConfigs]; _ -> AppConfigs end. update_app_config(all, Config) -> - lager:info("rtdev:update_app_config(all, ~p)", [Config]), - [ update_app_config(DevPath, Config) || DevPath <- devpaths()]; + lager:info("rtdev:update_app_config(all, ~p)", [Config]), + [ update_app_config(DevPath, Config) || DevPath <- devpaths()]; update_app_config(Node, Config) when is_atom(Node) -> - N = node_id(Node), - Path = relpath(node_version(N)), - FileFormatString = "~s/dev/dev~b/etc/~s.config", + lager:info("rtdev:update_app_config Node(~p, ~p)", [Node, Config]), + Version = node_version(Node), + update_app_config(Node, Version, Config); +update_app_config(DevPath, Config) -> + [update_app_config_file(AppConfig, Config) || AppConfig <- all_the_app_configs(DevPath)]. + +update_app_config(Node, Version, Config) -> + VersionPath = filename:join(?PATH, Version), + FileFormatString = "~s/~s/etc/~s.config", + {NodeId, _} = extract_node_id_and_name(Node), + AppConfigFile = io_lib:format(FileFormatString, + [VersionPath, node_short_name(NodeId), "app"]), + AdvConfigFile = io_lib:format(FileFormatString, + [VersionPath, node_short_name(NodeId), "advanced"]), - AppConfigFile = io_lib:format(FileFormatString, [Path, N, "app"]), - AdvConfigFile = io_lib:format(FileFormatString, [Path, N, "advanced"]), %% If there's an app.config, do it old style - %% if not, use cuttlefish's adavnced.config + %% if not, use cuttlefish's advanced.config case filelib:is_file(AppConfigFile) of true -> update_app_config_file(AppConfigFile, Config); _ -> update_app_config_file(AdvConfigFile, Config) - end; -update_app_config(DevPath, Config) -> - [update_app_config_file(AppConfig, Config) || AppConfig <- all_the_app_configs(DevPath)]. + end. + update_app_config_file(ConfigFile, Config) -> - lager:info("rtdev:update_app_config_file(~s, ~p)", [ConfigFile, Config]), + lager:debug("rtdev:update_app_config_file(~s, ~p)", [ConfigFile, Config]), BaseConfig = case file:consult(ConfigFile) of {ok, [ValidConfig]} -> @@ -271,6 +396,7 @@ update_app_config_file(ConfigFile, Config) -> {error, enoent} -> [] end, + lager:debug("Config: ~p", [Config]), MergeA = orddict:from_list(Config), MergeB = orddict:from_list(BaseConfig), NewConfig = @@ -281,6 +407,7 @@ update_app_config_file(ConfigFile, Config) -> ValA end, MergeC, MergeD) end, MergeA, MergeB), + lager:debug("Writing ~p to ~p", [NewConfig, ConfigFile]), NewConfigOut = io_lib:format("~p.", [NewConfig]), ?assertEqual(ok, file:write_file(ConfigFile, NewConfigOut)), ok. @@ -298,12 +425,12 @@ get_backend(AppConfig) -> ["app.config"| _ ] -> AppConfig; ["advanced.config" | T] -> - ["etc", [$d, $e, $v | N], "dev" | RPath] = T, + ["etc", Node | RPath] = T, Path = filename:join(lists:reverse(RPath)), %% Why chkconfig? It generates an app.config from cuttlefish %% without starting riak. - ChkConfigOutput = string:tokens(run_riak(list_to_integer(N), Path, "chkconfig"), "\n"), + ChkConfigOutput = string:tokens(run_riak(Node, Path, "chkconfig"), "\n"), ConfigFileOutputLine = lists:last(ChkConfigOutput), @@ -323,38 +450,72 @@ get_backend(AppConfig) -> case filename:pathtype(Files) of absolute -> File; relative -> - io_lib:format("~s/dev/dev~s/~s", [Path, N, tl(hd(Files))]) + io_lib:format("~s/~s/~s", [Path, Node, tl(hd(Files))]) end end end, case file:consult(ConfigFile) of {ok, [Config]} -> - rt:get_backend(Config); + rt_backend:get_backend(Config); E -> lager:error("Error reading ~s, ~p", [ConfigFile, E]), error end. node_path(Node) -> - N = node_id(Node), - Path = relpath(node_version(N)), - lists:flatten(io_lib:format("~s/dev/dev~b", [Path, N])). + {NodeId, NodeName} = extract_node_id_and_name(Node), + Path = relpath(node_version(NodeName)), + lists:flatten(io_lib:format("~s/~s", [Path, node_short_name(NodeId)])). get_ip(_Node) -> %% localhost 4 lyfe "127.0.0.1". create_dirs(Nodes) -> + lager:debug("Nodes ~p", [Nodes]), Snmp = [node_path(Node) ++ "/data/snmp/agent/db" || Node <- Nodes], [?assertCmd("mkdir -p " ++ Dir) || Dir <- Snmp]. clean_data_dir(Nodes, SubDir) when is_list(Nodes) -> DataDirs = [node_path(Node) ++ "/data/" ++ SubDir || Node <- Nodes], + lager:debug("Cleaning data directories ~p", [DataDirs]), lists:foreach(fun rm_dir/1, DataDirs). +%% Blocking to delete files is not the best use of time. Generally it +%% is quicker to move directories than to recursively delete them so +%% move the directory to a GC subdirectory in the riak_test scratch +%% directory, recreate the subdirectory, and asynchronously remove the +%% files from the scratch directory. +clean_data_dir(Node, Version, "") -> + DataDir = filename:join([?PATH, Version, Node, "data"]), + TmpDir = filename:join([?SCRATCH_DIR, "gc", Version, Node]), + filelib:ensure_dir(filename:join(TmpDir, "child")), + mv_dir(DataDir, TmpDir), + Pid = spawn(?MODULE, rm_dir, [TmpDir]), + mk_dir(DataDir), + Pid; +clean_data_dir(Node, Version, SubDir) -> + DataDir = filename:join([?PATH, Version, Node, "data", SubDir]), + TmpDir = filename:join([?SCRATCH_DIR, "gc", Version, Node, "data"]), + filelib:ensure_dir(filename:join(TmpDir, "child")), + mv_dir(DataDir, TmpDir), + Pid = spawn(?MODULE, rm_dir, [TmpDir]), + mk_dir(DataDir), + Pid. + +mk_dir(Dir) -> + lager:debug("Making directory ~s", [Dir]), + ?assertCmd("mkdir " ++ Dir), + ?assertEqual(true, filelib:is_dir(Dir)). + +mv_dir(Src, Dest) -> + lager:debug("Moving directory ~s to ~s", [Src, Dest]), + ?assertCmd("mv " ++ Src ++ " " ++ Dest), + ?assertEqual(false, filelib:is_dir(Src)). + rm_dir(Dir) -> - lager:info("Removing directory ~s", [Dir]), + lager:debug("Removing directory ~s", [Dir]), ?assertCmd("rm -rf " ++ Dir), ?assertEqual(false, filelib:is_dir(Dir)). @@ -363,7 +524,7 @@ add_default_node_config(Nodes) -> undefined -> ok; Defaults when is_list(Defaults) -> rt:pmap(fun(Node) -> - update_app_config(Node, Defaults) + update_app_config(Node, version_here, Defaults) end, Nodes), ok; BadValue -> @@ -374,6 +535,7 @@ add_default_node_config(Nodes) -> deploy_clusters(ClusterConfigs) -> NumNodes = rt_config:get(num_nodes, 6), RequestedNodes = lists:flatten(ClusterConfigs), + lager:info("RequestedNodes ~p~n", [RequestedNodes]), case length(RequestedNodes) > NumNodes of true -> @@ -388,73 +550,94 @@ deploy_clusters(ClusterConfigs) -> DeployedClusters end. +configure_nodes(Nodes, Configs) -> + %% Set initial config + add_default_node_config(Nodes), + rt:pmap(fun({_, default}) -> + ok; + ({Node, {cuttlefish, Config}}) -> + set_conf(Node, Config); + ({Node, Config}) -> + update_app_config(Node, version_here, Config) + end, + lists:zip(Nodes, Configs)). + deploy_nodes(NodeConfig) -> - Path = relpath(root), + Path = rt_config:get(root_path), lager:info("Riak path: ~p", [Path]), NumNodes = length(NodeConfig), + %% TODO: The starting index should not be fixed to 1 NodesN = lists:seq(1, NumNodes), - Nodes = [?DEV(N) || N <- NodesN], - NodeMap = orddict:from_list(lists:zip(Nodes, NodesN)), - {Versions, Configs} = lists:unzip(NodeConfig), - VersionMap = lists:zip(NodesN, Versions), - + FullNodes = [devrel_node_name(N) || N <- NodesN], + DevNodes = [list_to_atom(lists:concat(["dev", N])) || N <- NodesN], + NodeMap = orddict:from_list(lists:zip(FullNodes, NodesN)), + DevNodeMap = orddict:from_list(lists:zip(FullNodes, DevNodes)), + {Versions, _} = lists:unzip(NodeConfig), + VersionMap = lists:zip(FullNodes, Versions), + + %% TODO The new node deployment doesn't appear to perform this check ... -jsb %% Check that you have the right versions available - [ check_node(Version) || Version <- VersionMap ], + %%[ check_node(Version) || Version <- VersionMap ], rt_config:set(rt_nodes, NodeMap), + rt_config:set(rt_nodenames, DevNodeMap), rt_config:set(rt_versions, VersionMap), - create_dirs(Nodes), + lager:debug("Set rtnodes: ~p and rt_versions: ~p", [ rt_config:get(rt_nodes), rt_config:get(rt_versions) ]), + + create_dirs(FullNodes), %% Set initial config - add_default_node_config(Nodes), - rt:pmap(fun({_, default}) -> - ok; - ({Node, {cuttlefish, Config}}) -> - set_conf(Node, Config); - ({Node, Config}) -> - update_app_config(Node, Config) - end, - lists:zip(Nodes, Configs)), + add_default_node_config(FullNodes), + rt:pmap(fun({_, {_, default}}) -> + ok; + ({Node, {_, {cuttlefish, Config}}}) -> + set_conf(Node, Config); + ({Node, {Version, Config}}) -> + update_app_config(Node, Version, Config) + end, + lists:zip(FullNodes, NodeConfig)), %% create snmp dirs, for EE - create_dirs(Nodes), + create_dirs(FullNodes), %% Start nodes - %%[run_riak(N, relpath(node_version(N)), "start") || N <- Nodes], - rt:pmap(fun(N) -> run_riak(N, relpath(node_version(N)), "start") end, NodesN), + %%[run_riak(N, relpath(node_version(N)), "start") || N <- NodesN], + rt:pmap(fun(Node) -> run_riak(node_short_name(Node), relpath(node_version(Node)), "start") end, FullNodes), %% Ensure nodes started - [ok = rt:wait_until_pingable(N) || N <- Nodes], + [ok = rt:wait_until_pingable(N) || N <- FullNodes], %% %% Enable debug logging %% [rpc:call(N, lager, set_loglevel, [lager_console_backend, debug]) || N <- Nodes], %% We have to make sure that riak_core_ring_manager is running before we can go on. - [ok = rt:wait_until_registered(N, riak_core_ring_manager) || N <- Nodes], + [ok = rt:wait_until_registered(N, riak_core_ring_manager) || N <- FullNodes], %% Ensure nodes are singleton clusters - [ok = rt:check_singleton_node(?DEV(N)) || {N, Version} <- VersionMap, - Version /= "0.14.2"], - - lager:info("Deployed nodes: ~p", [Nodes]), - Nodes. - -gen_stop_fun(Timeout) -> - fun({C,Node}) -> - net_kernel:hidden_connect_node(Node), - case rpc:call(Node, os, getpid, []) of + [ok = rt_ring:check_singleton_node(FullNode) || {FullNode, Version} <- VersionMap, + Version /= "0.14.2"], + + lager:info("Deployed nodes: ~p", [FullNodes]), + FullNodes. + +gen_stop_fun(Path, Timeout) -> + fun(Node) -> + NodeName = ?DEV(Node), + NodePath = filename:join(Path, Node), + net_kernel:hidden_connect_node(NodeName), + case rpc:call(NodeName, os, getpid, []) of PidStr when is_list(PidStr) -> - lager:info("Preparing to stop node ~p (process ID ~s) with init:stop/0...", - [Node, PidStr]), - rpc:call(Node, init, stop, []), + lager:debug("Preparing to stop node ~p (process ID ~s) with init:stop/0...", + [NodePath, PidStr]), + rpc:call(NodeName, init, stop, []), %% If init:stop/0 fails here, the wait_for_pid/2 call %% below will timeout and the process will get cleaned %% up by the kill_stragglers/2 function wait_for_pid(PidStr, Timeout); BadRpc -> - Cmd = C ++ "/bin/riak stop", - lager:info("RPC to node ~p returned ~p, will try stop anyway... ~s", - [Node, BadRpc, Cmd]), + Cmd = filename:join([Path, Node, "bin/riak stop"]), + lager:debug("RPC to node ~p returned ~p, will try stop anyway... ~s", + [NodeName, BadRpc, Cmd]), Output = os:cmd(Cmd), Status = case Output of "ok\n" -> @@ -469,12 +652,12 @@ gen_stop_fun(Timeout) -> _ -> "wasn't running" end, - lager:info("Stopped node ~p, stop status: ~s.", [Node, Status]) + lager:debug("Stopped node ~p, stop status: ~s.", [NodePath, Status]) end end. -kill_stragglers(DevPath, Timeout) -> - {ok, Re} = re:compile("^\\s*\\S+\\s+(\\d+).+\\d+\\s+"++DevPath++"\\S+/beam"), +kill_stragglers(Path, Timeout) -> + {ok, Re} = re:compile("^\\s*\\S+\\s+(\\d+).+\\d+\\s+"++Path++"\\S+/beam"), ReOpts = [{capture,all_but_first,list}], Pids = tl(string:tokens(os:cmd("ps -ef"), "\n")), Fold = fun(Proc, Acc) -> @@ -482,13 +665,13 @@ kill_stragglers(DevPath, Timeout) -> nomatch -> Acc; {match,[Pid]} -> - lager:info("Process ~s still running, killing...", + lager:debug("Process ~s still running, killing...", [Pid]), os:cmd("kill -15 "++Pid), case wait_for_pid(Pid, Timeout) of ok -> ok; fail -> - lager:info("Process ~s still hasn't stopped, " + lager:debug("Process ~s still hasn't stopped, " "resorting to kill -9...", [Pid]), os:cmd("kill -9 "++Pid) end, @@ -507,45 +690,42 @@ wait_for_pid(PidStr, Timeout) -> _ -> ok end. -stop_all(DevPath) -> - case filelib:is_dir(DevPath) of - true -> - Devs = filelib:wildcard(DevPath ++ "/dev*"), - Nodes = [?DEV(N) || N <- lists:seq(1, length(Devs))], - MyNode = 'riak_test@127.0.0.1', - case net_kernel:start([MyNode, longnames]) of - {ok, _} -> - true = erlang:set_cookie(MyNode, riak); - {error,{already_started,_}} -> - ok - end, - lager:info("Trying to obtain node shutdown_time via RPC..."), - Tmout = case rpc:call(hd(Nodes), init, get_argument, [shutdown_time]) of - {ok,[[Tm]]} -> list_to_integer(Tm)+10000; - _ -> 20000 - end, - lager:info("Using node shutdown_time of ~w", [Tmout]), - rt:pmap(gen_stop_fun(Tmout), lists:zip(Devs, Nodes)), - kill_stragglers(DevPath, Tmout); - _ -> - lager:info("~s is not a directory.", [DevPath]) +stop_nodes(Path, Nodes) -> + MyNode = 'riak_test@127.0.0.1', + case net_kernel:start([MyNode, longnames]) of + {ok, _} -> + true = erlang:set_cookie(MyNode, riak); + {error,{already_started,_}} -> + ok end, + lager:debug("Trying to obtain node shutdown_time via RPC..."), + Tmout = case rpc:call(?DEV(hd(Nodes)), init, get_argument, [shutdown_time]) of + {ok,[[Tm]]} -> list_to_integer(Tm)+10000; + _ -> 20000 + end, + lager:debug("Using node shutdown_time of ~w", [Tmout]), + rt:pmap(gen_stop_fun(Path, Tmout), Nodes), + kill_stragglers(Path, Tmout), ok. -stop(Node) -> - RiakPid = rpc:call(Node, os, getpid, []), - N = node_id(Node), - rt_cover:maybe_stop_on_node(Node), - run_riak(N, relpath(node_version(N)), "stop"), - F = fun(_N) -> - os:cmd("kill -0 " ++ RiakPid) =/= [] - end, - ?assertEqual(ok, rt:wait_until(Node, F)), - ok. +stop(Node, Version) -> + {NodeId, NodeName} = extract_node_id_and_name(Node), + lager:debug("Stopping node ~p using node name ~p", [NodeId, NodeName]), + case rpc:call(NodeName, os, getpid, []) of + {badrpc, nodedown} -> + ok; + RiakPid -> + %% rt_cover:maybe_stop_on_node(Node), + run_riak(NodeId, Version, "stop"), + F = fun(_N) -> + os:cmd("kill -0 " ++ RiakPid) =/= [] + end, + ?assertEqual(ok, rt:wait_until(NodeName, F)), + ok + end. -start(Node) -> - N = node_id(Node), - run_riak(N, relpath(node_version(N)), "start"), +start(Node, Version) -> + run_riak(Node, Version, "start"), ok. attach(Node, Expected) -> @@ -558,10 +738,10 @@ console(Node, Expected) -> interactive(Node, "console", Expected). interactive(Node, Command, Exp) -> - N = node_id(Node), - Path = relpath(node_version(N)), - Cmd = riakcmd(Path, N, Command), - lager:info("Opening a port for riak ~s.", [Command]), + {NodeId, NodeName} = extract_node_id_and_name(Node), + Path = relpath(node_version(NodeName)), + Cmd = riakcmd(Path, NodeId, Command), + lager:debug("Opening a port for riak ~s.", [Command]), lager:debug("Calling open_port with cmd ~s", [binary_to_list(iolist_to_binary(Cmd))]), P = open_port({spawn, binary_to_list(iolist_to_binary(Cmd))}, [stream, use_stdio, exit_status, binary, stderr_to_stdout]), @@ -625,7 +805,7 @@ interactive_loop(Port, Expected) -> %% We've met every expectation. Yay! If not, it means we've exited before %% something expected happened. ?assertEqual([], Expected) - after rt_config:get(rt_max_wait_time) -> + after rt_config:get(rt_max_receive_wait_time) -> %% interactive_loop is going to wait until it matches expected behavior %% If it doesn't, the test should fail; however, without a timeout it %% will just hang forever in search of expected behavior. See also: Parenting @@ -633,9 +813,8 @@ interactive_loop(Port, Expected) -> end. admin(Node, Args, Options) -> - N = node_id(Node), - Path = relpath(node_version(N)), - Cmd = riak_admin_cmd(Path, N, Args), + Path = relpath(node_version(Node)), + Cmd = riak_admin_cmd(Path, Node, Args), lager:info("Running: ~s", [Cmd]), Result = execute_admin_cmd(Cmd, Options), lager:info("~p", [Result]), @@ -651,27 +830,53 @@ execute_admin_cmd(Cmd, Options) -> end. riak(Node, Args) -> - N = node_id(Node), - Path = relpath(node_version(N)), - Result = run_riak(N, Path, Args), + {NodeId, NodeName} = extract_node_id_and_name(Node), + Path = relpath(node_version(NodeName)), + Result = run_riak(NodeId, Path, Args), lager:info("~s", [Result]), {ok, Result}. riak_repl(Node, Args) -> - N = node_id(Node), - Path = relpath(node_version(N)), - Result = run_riak_repl(N, Path, Args), + {NodeId, NodeName} = extract_node_id_and_name(Node), + Path = relpath(node_version(NodeName)), + Result = run_riak_repl(NodeId, Path, Args), lager:info("~s", [Result]), {ok, Result}. +%% @doc Find the node number from the full name +%% Certain tests are storing short names (dev1) and some are storing integers +-spec node_id(atom()) -> integer(). node_id(Node) -> NodeMap = rt_config:get(rt_nodes), + NodeNumber = orddict:fetch(Node, NodeMap), + case is_integer(NodeNumber) of + true -> NodeNumber; + _ -> + list_to_integer(string:right(lists:flatten(NodeNumber), 1)) + end. + +%% @doc Find the short dev node name from the full name +-spec node_short_name(atom() | list()) -> atom(). +node_short_name(Node) when is_list(Node) -> + case lists:member($@, Node) of + true -> + node_short_name(list_to_atom(Node)); + _ -> + Node + end; +node_short_name(Node) when is_atom(Node) -> + NodeMap = rt_config:get(rt_nodenames), orddict:fetch(Node, NodeMap). -node_version(N) -> +%% @doc Return the node version from rt_versions based on full node name +-spec node_version(atom() | integer() | list()) -> string(). +node_version(Node) when is_integer(Node) -> + node_version(node_short_name_to_name(Node)); +node_version(Node) -> + {_, NodeName} = extract_node_id_and_name(Node), VersionMap = rt_config:get(rt_versions), - orddict:fetch(N, VersionMap). + orddict:fetch(NodeName, VersionMap). spawn_cmd(Cmd) -> spawn_cmd(Cmd, []). @@ -711,7 +916,7 @@ get_cmd_result(Port, Acc) -> end. check_node({_N, Version}) -> - case proplists:is_defined(Version, rt_config:get(rtdev_path)) of + case proplists:is_defined(Version, rt_config:get(root_path)) of true -> ok; _ -> lager:error("You don't have Riak ~s installed or configured", [Version]), @@ -722,21 +927,29 @@ set_backend(Backend) -> set_backend(Backend, []). set_backend(Backend, OtherOpts) -> - lager:info("rtdev:set_backend(~p, ~p)", [Backend, OtherOpts]), + lager:debug("rtdev:set_backend(~p, ~p)", [Backend, OtherOpts]), Opts = [{storage_backend, Backend} | OtherOpts], - update_app_config(all, [{riak_kv, Opts}]), + update_app_config(all, version_here, [{riak_kv, Opts}]), get_backends(). +%% WRONG: Seemingly always stuck on the current version get_version() -> - case file:read_file(relpath(current) ++ "/VERSION") of + case file:read_file(relpath(rt_config:get_default_version()) ++ "/VERSION") of + {error, enoent} -> unknown; + {ok, Version} -> Version + end. + +get_version(Node) -> + case file:read_file(filename:join([relpath(node_version(Node)),"VERSION"])) of {error, enoent} -> unknown; {ok, Version} -> Version end. teardown() -> - rt_cover:maybe_stop_on_nodes(), + %% rt_cover:maybe_stop_on_nodes(), %% Stop all discoverable nodes, not just nodes we'll be using for this test. - rt:pmap(fun(X) -> stop_all(X ++ "/dev") end, devpaths()). + %% rt:pmap(fun(X) -> stop_all(X ++ "/dev") end, devpaths()). + stop_all(). whats_up() -> io:format("Here's what's running...~n"), @@ -744,16 +957,133 @@ whats_up() -> Up = [rpc:call(Node, os, cmd, ["pwd"]) || Node <- nodes()], [io:format(" ~s~n",[string:substr(Dir, 1, length(Dir)-1)]) || Dir <- Up]. +%% @doc Gather the devrel directories in the root_path parent directory +-spec devpaths() -> list(). devpaths() -> - lists:usort([ DevPath || {_Name, DevPath} <- proplists:delete(root, rt_config:get(rtdev_path))]). - -versions() -> - proplists:get_keys(rt_config:get(rtdev_path)) -- [root]. + RootDir = rt_config:get(root_path), + {ok, RawDirs} = file:list_dir(RootDir), + %% Remove any dot files in the directory (e.g. .git) + FilteredPaths = lists:filter(fun([$.|_]) -> false; (_) -> true end, RawDirs), + %% Generate fully qualified path names + DevPaths = lists:map(fun(X) -> filename:join(RootDir, X) end, FilteredPaths), + lists:usort(DevPaths). + +%% versions() -> +%% proplists:get_keys(rt_config:get(rtdev_path)) -- [root]. + +% @doc Get the list of log files and config files and pass them back +-spec(get_node_logs(string(), string()) -> list()). +get_node_logs(LogFile, DestDir) -> + Root = filename:absname(?PATH), + RootLen = length(Root) + 1, %% Remove the leading slash + Fun = get_node_log_fun(DestDir, RootLen), + NodeLogs = [ Fun(Filename) || Filename <- filelib:wildcard(Root ++ "/*/dev*/log/*") ++ + filelib:wildcard(Root ++ "/*/dev*/etc/*.conf*") ], + %% Trim the Lager file path slightly differently + LagerFile = filename:absname(LogFile), + LagerLen = length(filename:dirname(LagerFile)) + 1, + LagerFun = get_node_log_fun(DestDir, LagerLen), + LagerLog = LagerFun(LagerFile), + lists:append([LagerLog], NodeLogs). + +% @doc Copy each file to a local directory +-spec(get_node_log_fun(string(), integer()) -> fun()). +get_node_log_fun(DestDir, RootLen) -> + DestRoot = filename:absname(DestDir), + lager:debug("Copying log files to ~p", [DestRoot]), + fun(Filename) -> + Target = filename:join([DestRoot, lists:nthtail(RootLen, Filename)]), + ok = filelib:ensure_dir(Target), + %% Copy the file only if it's a new location + case Target of + Filename -> ok; + _ -> + lager:debug("Copying ~p to ~p", [Filename, Target]), + {ok, _BytesWritten} = file:copy(Filename, Target) + end, + {lists:nthtail(RootLen, Filename), Target} + end. +%% @doc Open all of the nodes' log files to a list of {filename, port} +%% OBSOLETE +-spec(get_node_logs() -> list()). get_node_logs() -> - Root = filename:absname(proplists:get_value(root, ?PATH)), + Root = filename:absname(?PATH), + lager:debug("ROOT ~p", [Root]), RootLen = length(Root) + 1, %% Remove the leading slash [ begin {ok, Port} = file:open(Filename, [read, binary]), + lager:debug("Opening ~p", [lists:nthtail(RootLen, Filename)]), {lists:nthtail(RootLen, Filename), Port} - end || Filename <- filelib:wildcard(Root ++ "/*/dev/dev*/log/*") ]. + end || Filename <- filelib:wildcard(Root ++ "/*/dev*/log/*") ]. + +-type node_tuple() :: {list(), atom()}. + +-spec extract_node_id_and_name(atom() | string()) -> node_tuple(). +extract_node_id_and_name(Node) when is_atom(Node) -> + NodeStr = atom_to_list(Node), + extract_node_id_and_name(NodeStr); +extract_node_id_and_name(Node) when is_list(Node) -> + extract_node_id_and_name(Node, contains(Node, $@)); +extract_node_id_and_name(_Node) -> + erlang:error(unsupported_node_type). + +-spec extract_node_id_and_name(list(), boolean()) -> node_tuple(). +extract_node_id_and_name(Node, true) -> + [NodeId, _] = re:split(Node, "@"), + {binary_to_list(NodeId), list_to_atom(Node)}; +extract_node_id_and_name(Node, false) -> + {Node, ?DEV(lists:flatten(Node))}. + +-spec contains(list(), char()) -> boolean. +contains(Str, Char) -> + maybe_contains(string:chr(Str, Char)). + +-spec maybe_contains(integer()) -> boolean. +maybe_contains(Pos) when Pos > 0 -> + true; +maybe_contains(_) -> + false. + +-spec node_short_name_to_name(integer()) -> atom(). +node_short_name_to_name(N) -> + ?DEV("dev" ++ integer_to_list(N)). + +%% @doc Check to make sure that all versions specified in the config file actually exist +-spec validate_config([term()]) -> ok | no_return(). +validate_config(Versions) -> + Root = rt_config:get(root_path), + Validate = fun(Vsn) -> + {Result, _} = file:read_file_info(filename:join([Root, Vsn, "dev1/bin/riak"])), + case Result of + ok -> ok; + _ -> + erlang:error("Could not find specified devrel version", [Vsn]) + end + end, + [Validate(Vsn) || Vsn <- Versions], + ok. + +-ifdef(TEST). + +extract_node_id_and_name_test() -> + Expected = {"dev2", 'dev2@127.0.0.1'}, + ?assertEqual(Expected, extract_node_id_and_name('dev2@127.0.0.1')), + ?assertEqual(Expected, extract_node_id_and_name("dev2@127.0.0.1")), + ?assertEqual(Expected, extract_node_id_and_name('dev2')), + ?assertEqual(Expected, extract_node_id_and_name("dev2")). + +maybe_contains_test() -> + ?assertEqual(true, maybe_contains(1)), + ?assertEqual(true, maybe_contains(10)), + ?assertEqual(false, maybe_contains(0)). + + +contains_test() -> + ?assertEqual(true, contains("dev2@127.0.0.1", $@)), + ?assertEqual(false, contains("dev2", $@)). + +node_short_name_to_name_test() -> + ?assertEqual('dev1@127.0.0.1', node_short_name_to_name(1)). + +-endif. diff --git a/src/rtperf.erl b/src/rtperf.erl index 7b46a5360..f0a68c65c 100644 --- a/src/rtperf.erl +++ b/src/rtperf.erl @@ -1,9 +1,67 @@ -module(rtperf). +-behaviour(test_harness). + +-export([start/1, + stop/1, + deploy_clusters/1, + clean_data_dir/2, + spawn_cmd/1, + spawn_cmd/2, + cmd/1, + cmd/2, + setup_harness/2, + get_version/0, + get_backends/0, + set_backend/1, + whats_up/0, + get_ip/1, + node_id/1, + node_version/1, + admin/2, + riak/2, + attach/2, + attach_direct/2, + console/2, + update_app_config/2, + teardown/0, + set_conf/2, + set_advanced_conf/2]). + -compile(export_all). -include_lib("eunit/include/eunit.hrl"). -include_lib("kernel/include/file.hrl"). +admin(Node, Args) -> + rt_harness_util:admin(Node, Args). + +attach(Node, Expected) -> + rt_harness_util:attach(Node, Expected). + +attach_direct(Node, Expected) -> + rt_harness_util:attach_direct(Node, Expected). + +cmd(Cmd, Opts) -> + rt_harness_util:cmd(Cmd, Opts). + +console(Node, Expected) -> + rt_harness_util:console(Node, Expected). + +get_ip(Node) -> + rt_harness_util:get_ip(Node). + +node_id(Node) -> + rt_harness_util:get_ip(Node). + +node_version(N) -> + rt_harness_util:node_version(N). + +riak(Node, Args) -> + rt_harness_util:riak(Node, Args). + +set_conf(Node, NameValuePairs) -> + rt_harness_util:set_conf(Node, NameValuePairs). + update_app_config(Node, Config) -> rtssh:update_app_config(Node, Config). @@ -394,10 +452,20 @@ deploy_nodes(NodeConfig, Hosts) -> [ok = rt:wait_until_registered(N, riak_core_ring_manager) || N <- Nodes], %% Ensure nodes are singleton clusters - [ok = rt:check_singleton_node(N) || {N, Version} <- VersionMap, + [ok = rt_ring:check_singleton_node(N) || {N, Version} <- VersionMap, Version /= "0.14.2"], Nodes. start(Node) -> rtssh:start(Node). + +spawn_cmd(Cmd) -> + rt_harness_util:spawn_cmd(Cmd). + +spawn_cmd(Cmd, Opts) -> + rt_harness_util:spawn_cmd(Cmd, Opts). + +whats_up() -> + rt_harness_util:whats_up(). + diff --git a/src/rtssh.erl b/src/rtssh.erl index 4c3e7d228..cbc3db1f4 100644 --- a/src/rtssh.erl +++ b/src/rtssh.erl @@ -1,7 +1,40 @@ -module(rtssh). +-behaviour(test_harness). + +-export([start/1, + stop/1, + deploy_clusters/1, + clean_data_dir/2, + spawn_cmd/1, + spawn_cmd/2, + cmd/1, + cmd/2, + setup_harness/2, + get_deps/0, + get_version/0, + get_backends/0, + set_backend/1, + whats_up/0, + get_ip/1, + node_id/1, + node_version/1, + admin/2, + riak/2, + attach/2, + attach_direct/2, + console/2, + update_app_config/2, + teardown/0, + set_conf/2, + set_advanced_conf/2, + validate_config/1]). + -compile(export_all). -include_lib("eunit/include/eunit.hrl"). +admin(Node, Args) -> + rt_harness_util:admin(Node, Args). + get_version() -> unknown. @@ -88,7 +121,7 @@ get_backend(Host, AppConfig) -> Str = binary_to_list(Bin), {ok, ErlTok, _} = erl_scan:string(Str), {ok, Term} = erl_parse:parse_term(ErlTok), - rt:get_backend(Term). + rt_backend:get_backend(Term). cmd(Cmd) -> cmd(Cmd, []). @@ -119,6 +152,22 @@ node_to_host(Node) -> throw(io_lib:format("rtssh:node_to_host couldn't figure out the host of ~p", [Node])) end. + +nodes(Count) -> + Hosts = rt_config:get(rtssh_hosts), + %% NumNodes = length(NodeConfig), + NodeConfig = busted_stuff, + NumNodes = Count, + NumHosts = length(Hosts), + case NumNodes > NumHosts of + true -> + erlang:error("Not enough hosts available to deploy nodes", + [NumNodes, NumHosts]); + false -> + Hosts2 = lists:sublist(Hosts, NumNodes), + deploy_nodes(NodeConfig, Hosts2) + end. + deploy_nodes(NodeConfig, Hosts) -> Path = relpath(root), lager:info("Riak path: ~p", [Path]), @@ -205,7 +254,7 @@ deploy_nodes(NodeConfig, Hosts) -> [ok = rt:wait_until_registered(N, riak_core_ring_manager) || N <- Nodes], %% Ensure nodes are singleton clusters - [ok = rt:check_singleton_node(N) || {N, Version} <- VersionMap, + [ok = rt_ring:check_singleton_node(N) || {N, Version} <- VersionMap, Version /= "0.14.2"], Nodes. @@ -300,29 +349,6 @@ remote_cmd(Node, Cmd) -> {0, Result} = ssh_cmd(Node, Cmd), {ok, Result}. -admin(Node, Args) -> - Cmd = riak_admin_cmd(Node, Args), - lager:info("Running: ~s :: ~s", [get_host(Node), Cmd]), - {0, Result} = ssh_cmd(Node, Cmd), - lager:info("~s", [Result]), - {ok, Result}. - -admin(Node, Args, Options) -> - Cmd = riak_admin_cmd(Node, Args), - lager:info("Running: ~s :: ~s", [get_host(Node), Cmd]), - Result = execute_admin_cmd(Node, Cmd, Options), - lager:info("~s", [Result]), - {ok, Result}. - -execute_admin_cmd(Node, Cmd, Options) -> - {_ExitCode, Result} = FullResult = ssh_cmd(Node, Cmd), - case lists:member(return_exit_code, Options) of - true -> - FullResult; - false -> - Result - end. - riak(Node, Args) -> Result = run_riak(Node, Args), lager:info("~s", [Result]), @@ -350,6 +376,7 @@ load_hosts() -> Hosts = lists:sort(HostsIn), rt_config:set(rtssh_hosts, Hosts), rt_config:set(rtssh_aliases, Aliases), + rt_config:set(rtssh_nodes, length(Hosts)), Hosts. read_hosts_file(File) -> @@ -652,10 +679,10 @@ scp_from(Host, RemotePath, Path) -> %%% Riak devrel path utilities %%%=================================================================== --define(PATH, (rt_config:get(rtdev_path))). +-define(PATH, (rt_config:get(root_path))). dev_path(Path, N) -> - format("~s/dev/dev~b", [Path, N]). + format("~s/dev~b", [Path, N]). dev_bin_path(Path, N) -> dev_path(Path, N) ++ "/bin". @@ -692,8 +719,32 @@ node_id(_Node) -> %% orddict:fetch(Node, NodeMap). 1. +set_backend(Backend) -> + set_backend(Backend, []). + +set_backend(Backend, OtherOpts) -> + lager:info("rtssh:set_backend(~p, ~p)", [Backend, OtherOpts]), + Opts = [{storage_backend, Backend} | OtherOpts], + update_app_config(all, [{riak_kv, Opts}]), + get_backends(). + +whats_up() -> + io:format("Here's what's running...~n"), + + Up = [rpc:call(Node, os, cmd, ["pwd"]) || Node <- nodes()], + [io:format(" ~s~n",[string:substr(Dir, 1, length(Dir)-1)]) || Dir <- Up]. + node_version(Node) -> - orddict:fetch(Node, rt_config:get(rt_versions)). + rt_harness_util:node_version(Node). + +attach(Node, Expected) -> + rt_harness_util:attach(Node, Expected). + +attach_direct(Node, Expected) -> + rt_harness_util:attach_direct(Node, Expected). + +console(Node, Expected) -> + rt_harness_util:console(Node, Expected). %%%=================================================================== %%% Local command spawning @@ -781,6 +832,24 @@ stop_all(Host, DevPath) -> teardown() -> stop_all(rt_config:get(rt_hostnames)). +%% @doc Check to make sure that all versions specified in the config file actually exist +-spec validate_config([term()]) -> ok | no_return(). +validate_config(Versions) -> + Hosts = load_hosts(), + Root = rt_config:get(root_path), + Validate = fun(Host, Vsn) -> + Cmd = "ls " ++ filename:join([Root, Vsn, "dev1/bin/riak"]), + Result = wait_for_cmd(spawn_ssh_cmd(atom_to_list(Host), Cmd, [], true)), + io:format("Result = ~p~n", [Result]), + case Result of + {0, _} -> ok; + _ -> + erlang:error("Could not find specified devrel version", [Host, Vsn]) + end + end, + [[Validate(Host, Vsn) || Vsn <- Versions] || Host <- Hosts], + ok. + %%%=================================================================== %%% Utilities %%%=================================================================== diff --git a/src/smoke_test_escript.erl b/src/smoke_test_escript.erl index 62a646da3..2d2d2e0cf 100755 --- a/src/smoke_test_escript.erl +++ b/src/smoke_test_escript.erl @@ -60,7 +60,7 @@ main(Args) -> end, rt_config:set(rt_harness, ?MODULE), lager:debug("ParsedArgs ~p", [Parsed]), - Suites = giddyup:get_suite(rt_config:get(platform)), + Suites = giddyup:get_suite(rt_config:get(giddyup_platform)), Jobs = case lists:keyfind(jobs, 1, Parsed) of false -> 1; diff --git a/src/test_harness.erl b/src/test_harness.erl new file mode 100644 index 000000000..760c70202 --- /dev/null +++ b/src/test_harness.erl @@ -0,0 +1,47 @@ +%% ------------------------------------------------------------------- +%% +%% Copyright (c) 2013-2014 Basho Technologies, Inc. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% ------------------------------------------------------------------- +%% @doc behaviour for all test harnesses. +-module(test_harness). + +%% -callback start(Node :: node(), Version :: string()) -> 'ok'. +%% -callback stop(Node :: node()) -> 'ok'. +%% -callback deploy_clusters(ClusterConfigs :: list()) -> list(). +-callback clean_data_dir(Nodes :: list(), SubDir :: string()) -> 'ok'. +-callback spawn_cmd(Cmd :: string()) -> Port :: pos_integer(). +-callback spawn_cmd(Cmd :: string(), Opts :: list()) -> Port :: pos_integer(). +-callback cmd(Cmd :: string()) -> term()|timeout. +-callback cmd(Cmd :: string(), Opts :: [atom()]) -> term()|timeout. +%% -callback setup_harness(Test :: string(), Args :: list()) -> 'ok'. +-callback get_version() -> term(). +-callback get_backends() -> [atom()]. +-callback set_backend(Backend :: atom()) -> [atom()]. +-callback whats_up() -> string(). +-callback get_ip(Node :: node()) -> string(). +-callback node_id(Node :: node()) -> NodeMap :: term(). +-callback node_version(N :: node()) -> VersionMap :: term(). +-callback admin(Node :: node(), Args :: [atom()]) -> {'ok', string()}. +-callback riak(Node :: node(), Args :: [atom()]) -> {'ok', string()}. +-callback attach(Node :: node(), Expected:: list()) -> 'ok'. +-callback attach_direct(Node :: node(), Expected:: list()) -> 'ok'. +-callback console(Node :: node(), Expected:: list()) -> 'ok'. +%% -callback update_app_config(atom()|node(), Config :: term()) -> 'ok'. +-callback teardown() -> list(). +-callback set_conf(atom()|node(), NameValuePairs :: [{string(), string()}]) -> 'ok'. +-callback set_advanced_conf(atom()|node(), NameValuePairs :: [{string(), string()}]) -> 'ok'. diff --git a/tests/always_fail_test.erl b/tests/always_fail_test.erl index 39d6e341a..3cb4fd368 100644 --- a/tests/always_fail_test.erl +++ b/tests/always_fail_test.erl @@ -1,7 +1,18 @@ %% @doc A test that always returns `fail'. -module(always_fail_test). --export([confirm/0]). --spec confirm() -> pass | fail. -confirm() -> +%% -behaviour(riak_test). + +-include_lib("eunit/include/eunit.hrl"). + +-export([properties/0, + confirm/1]). + +properties() -> + rt_properties:new([{make_cluster, false}]). + +-spec confirm(rt_properties:properties()) -> pass | fail. +confirm(_Properties) -> + lager:info("Running test confirm function"), + ?assertEqual(1,2), fail. diff --git a/tests/always_pass_test.erl b/tests/always_pass_test.erl index e71c9645c..b779c96aa 100644 --- a/tests/always_pass_test.erl +++ b/tests/always_pass_test.erl @@ -1,8 +1,19 @@ -%% @doc A test that always returns `pass'. +%% @doc A test that always returns `fail'. -module(always_pass_test). --behavior(riak_test). --export([confirm/0]). --spec confirm() -> pass | fail. -confirm() -> +%% -behaviour(riak_test). + +-export([properties/0, + confirm/1]). + +-include_lib("eunit/include/eunit.hrl"). + +properties() -> + rt_properties:new([{make_cluster, false}]). + +-spec confirm(rt_properties:properties()) -> pass | fail. +confirm(Properties) -> + NodeIds = rt_properties:get(node_ids, Properties), + lager:notice("~p is using ~p nodes", [?MODULE, length(NodeIds)]), + ?assertEqual(1,1), pass. diff --git a/tests/kv679_dataloss.erl b/tests/kv679_dataloss.erl index 500314bac..5973a062a 100644 --- a/tests/kv679_dataloss.erl +++ b/tests/kv679_dataloss.erl @@ -117,7 +117,6 @@ delete_datadir({{Idx, Node}, Type}) -> DataRoot = rpc:call(Node, app_helper, get_env, [BackendName, data_root]), %% get datadir from Idx Path = filename:join([rtdev:relpath(current), - "dev", "dev"++ integer_to_list(rtdev:node_id(Node)), DataRoot, integer_to_list(Idx)]), diff --git a/tests/repl_aae_fullsync.erl b/tests/replication/repl_aae_fullsync.erl similarity index 100% rename from tests/repl_aae_fullsync.erl rename to tests/replication/repl_aae_fullsync.erl diff --git a/tests/repl_aae_fullsync_bench.erl b/tests/replication/repl_aae_fullsync_bench.erl similarity index 100% rename from tests/repl_aae_fullsync_bench.erl rename to tests/replication/repl_aae_fullsync_bench.erl diff --git a/tests/repl_aae_fullsync_custom_n.erl b/tests/replication/repl_aae_fullsync_custom_n.erl similarity index 100% rename from tests/repl_aae_fullsync_custom_n.erl rename to tests/replication/repl_aae_fullsync_custom_n.erl diff --git a/tests/repl_aae_fullsync_util.erl b/tests/replication/repl_aae_fullsync_util.erl similarity index 100% rename from tests/repl_aae_fullsync_util.erl rename to tests/replication/repl_aae_fullsync_util.erl diff --git a/tests/repl_bucket_types.erl b/tests/replication/repl_bucket_types.erl similarity index 100% rename from tests/repl_bucket_types.erl rename to tests/replication/repl_bucket_types.erl diff --git a/tests/repl_cancel_fullsync.erl b/tests/replication/repl_cancel_fullsync.erl similarity index 100% rename from tests/repl_cancel_fullsync.erl rename to tests/replication/repl_cancel_fullsync.erl diff --git a/tests/repl_consistent_object_filter.erl b/tests/replication/repl_consistent_object_filter.erl similarity index 100% rename from tests/repl_consistent_object_filter.erl rename to tests/replication/repl_consistent_object_filter.erl diff --git a/tests/repl_fs_bench.erl b/tests/replication/repl_fs_bench.erl similarity index 100% rename from tests/repl_fs_bench.erl rename to tests/replication/repl_fs_bench.erl diff --git a/tests/repl_fs_stat_caching.erl b/tests/replication/repl_fs_stat_caching.erl similarity index 100% rename from tests/repl_fs_stat_caching.erl rename to tests/replication/repl_fs_stat_caching.erl diff --git a/tests/repl_location_failures.erl b/tests/replication/repl_location_failures.erl similarity index 100% rename from tests/repl_location_failures.erl rename to tests/replication/repl_location_failures.erl diff --git a/tests/repl_rt_cascading_rtq.erl b/tests/replication/repl_rt_cascading_rtq.erl similarity index 100% rename from tests/repl_rt_cascading_rtq.erl rename to tests/replication/repl_rt_cascading_rtq.erl diff --git a/tests/repl_rt_heartbeat.erl b/tests/replication/repl_rt_heartbeat.erl similarity index 100% rename from tests/repl_rt_heartbeat.erl rename to tests/replication/repl_rt_heartbeat.erl diff --git a/tests/repl_rt_overload.erl b/tests/replication/repl_rt_overload.erl similarity index 100% rename from tests/repl_rt_overload.erl rename to tests/replication/repl_rt_overload.erl diff --git a/tests/repl_rt_pending.erl b/tests/replication/repl_rt_pending.erl similarity index 100% rename from tests/repl_rt_pending.erl rename to tests/replication/repl_rt_pending.erl diff --git a/tests/repl_util.erl b/tests/replication/repl_util.erl similarity index 100% rename from tests/repl_util.erl rename to tests/replication/repl_util.erl diff --git a/tests/replication.erl b/tests/replication/replication.erl similarity index 100% rename from tests/replication.erl rename to tests/replication/replication.erl diff --git a/tests/replication2.erl b/tests/replication/replication2.erl similarity index 100% rename from tests/replication2.erl rename to tests/replication/replication2.erl diff --git a/tests/replication2_connections.erl b/tests/replication/replication2_connections.erl similarity index 100% rename from tests/replication2_connections.erl rename to tests/replication/replication2_connections.erl diff --git a/tests/replication2_console_tests.erl b/tests/replication/replication2_console_tests.erl similarity index 100% rename from tests/replication2_console_tests.erl rename to tests/replication/replication2_console_tests.erl diff --git a/tests/replication2_dirty.erl b/tests/replication/replication2_dirty.erl similarity index 100% rename from tests/replication2_dirty.erl rename to tests/replication/replication2_dirty.erl diff --git a/tests/replication2_fsschedule.erl b/tests/replication/replication2_fsschedule.erl similarity index 100% rename from tests/replication2_fsschedule.erl rename to tests/replication/replication2_fsschedule.erl diff --git a/tests/replication2_pg.erl b/tests/replication/replication2_pg.erl similarity index 100% rename from tests/replication2_pg.erl rename to tests/replication/replication2_pg.erl diff --git a/tests/replication2_rt_sink_connection.erl b/tests/replication/replication2_rt_sink_connection.erl similarity index 100% rename from tests/replication2_rt_sink_connection.erl rename to tests/replication/replication2_rt_sink_connection.erl diff --git a/tests/replication2_ssl.erl b/tests/replication/replication2_ssl.erl similarity index 100% rename from tests/replication2_ssl.erl rename to tests/replication/replication2_ssl.erl diff --git a/tests/replication2_upgrade.erl b/tests/replication/replication2_upgrade.erl similarity index 100% rename from tests/replication2_upgrade.erl rename to tests/replication/replication2_upgrade.erl diff --git a/tests/replication_object_reformat.erl b/tests/replication/replication_object_reformat.erl similarity index 100% rename from tests/replication_object_reformat.erl rename to tests/replication/replication_object_reformat.erl diff --git a/tests/replication_ssl.erl b/tests/replication/replication_ssl.erl similarity index 100% rename from tests/replication_ssl.erl rename to tests/replication/replication_ssl.erl diff --git a/tests/replication_stats.erl b/tests/replication/replication_stats.erl similarity index 100% rename from tests/replication_stats.erl rename to tests/replication/replication_stats.erl diff --git a/tests/replication_upgrade.erl b/tests/replication/replication_upgrade.erl similarity index 100% rename from tests/replication_upgrade.erl rename to tests/replication/replication_upgrade.erl diff --git a/tests/rt_cascading.erl b/tests/replication/rt_cascading.erl similarity index 100% rename from tests/rt_cascading.erl rename to tests/replication/rt_cascading.erl diff --git a/tests/yz_core_properties_create_unload.erl b/tests/yz_core_properties_create_unload.erl index 31bc57f3e..a53fde692 100644 --- a/tests/yz_core_properties_create_unload.erl +++ b/tests/yz_core_properties_create_unload.erl @@ -21,9 +21,20 @@ -compile(export_all). -include_lib("eunit/include/eunit.hrl"). --define(CFG, [{yokozuna, [{enabled, true}]}]). +-define(CFG, [{riak_kv, + [ + %% allow AAE to build trees and exchange rapidly + {anti_entropy_build_limit, {100, 1000}}, + {anti_entropy_concurrency, 4} + ]}, + {yokozuna, + [ + {enabled, true}, + {anti_entropy_tick, 1000} + ]}]). -define(INDEX, <<"test_idx_core">>). --define(BUCKET, <<"test_bkt_core">>). +-define(TYPE, <<"data">>). +-define(BUCKET, {?TYPE, <<"test_bkt_core">>}). -define(SEQMAX, 100). confirm() -> @@ -48,7 +59,9 @@ confirm() -> %% Create a search index and associate with a bucket lager:info("Create and set Index ~p for Bucket ~p~n", [?INDEX, ?BUCKET]), ok = riakc_pb_socket:create_search_index(Pid, ?INDEX), - ok = riakc_pb_socket:set_search_index(Pid, ?BUCKET, ?INDEX), + ok = rt:create_and_activate_bucket_type(Node, + ?TYPE, + [{search_index, ?INDEX}]), timer:sleep(1000), %% Write keys and wait for soft commit @@ -58,13 +71,10 @@ confirm() -> verify_count(Pid, KeyCount), - %% Remove core.properties from the selected subset - remove_core_props(RandNodes), + lager:info("Remove core.properties file in each index data dir"), + remove_core_props(RandNodes, ?INDEX), - wait_until(RandNodes, - fun(N) -> - rpc:call(N, yz_index, exists, [?INDEX]) - end), + check_exists(Cluster, ?INDEX), lager:info("Write one more piece of data"), ok = rt:pbc_write(Pid, ?BUCKET, <<"foo">>, <<"foo">>, "text/plain"), @@ -72,6 +82,38 @@ confirm() -> verify_count(Pid, KeyCount + 1), + lager:info("Remove index directories on each node and let them recreate/reindex"), + remove_index_dirs(RandNodes, ?INDEX), + + check_exists(Cluster, ?INDEX), + + yz_rt:expire_trees(Cluster), + yz_rt:wait_for_aae(Cluster), + + lager:info("Write second piece of data"), + ok = rt:pbc_write(Pid, ?BUCKET, <<"food">>, <<"foody">>, "text/plain"), + timer:sleep(1100), + + verify_count(Pid, KeyCount + 2), + + lager:info("Remove segment info files in each index data dir"), + remove_segment_infos(RandNodes, ?INDEX), + + lager:info("To fix, we remove index directories on each node and let them recreate/reindex"), + + remove_index_dirs(RandNodes, ?INDEX), + + check_exists(Cluster, ?INDEX), + + yz_rt:expire_trees(Cluster), + yz_rt:wait_for_aae(Cluster), + + lager:info("Write third piece of data"), + ok = rt:pbc_write(Pid, ?BUCKET, <<"baz">>, <<"bar">>, "text/plain"), + timer:sleep(1100), + + verify_count(Pid, KeyCount + 3), + riakc_pb_socket:stop(Pid), pass. @@ -88,9 +130,9 @@ verify_count(Pid, ExpectedKeyCount) -> end. %% @doc Remove core properties file on nodes. -remove_core_props(Nodes) -> - IndexDirs = [rpc:call(Node, yz_index, index_dir, [?INDEX]) || - Node <- Nodes], +remove_core_props(Nodes, IndexName) -> + IndexDirs = [rpc:call(Node, yz_index, index_dir, [IndexName]) || + Node <- Nodes], PropsFiles = [filename:join([IndexDir, "core.properties"]) || IndexDir <- IndexDirs], lager:info("Remove core.properties files: ~p, on nodes: ~p~n", @@ -98,11 +140,31 @@ remove_core_props(Nodes) -> [file:delete(PropsFile) || PropsFile <- PropsFiles], ok. -%% @doc Wrapper around `rt:wait_until' to verify `F' against multiple -%% nodes. The function `F' is passed one of the `Nodes' as -%% argument and must return a `boolean()' delcaring whether the -%% success condition has been met or not. --spec wait_until([node()], fun((node()) -> boolean())) -> ok. -wait_until(Nodes, F) -> - [?assertEqual(ok, rt:wait_until(Node, F)) || Node <- Nodes], - ok. +%% @doc Check if index/core exists in metadata, disk via yz_index:exists. +check_exists(Nodes, IndexName) -> + rt:wait_until(Nodes, + fun(N) -> + rpc:call(N, yz_index, exists, [IndexName]) + end). + +%% @doc Remove index directories, removing the index. +remove_index_dirs(Nodes, IndexName) -> + IndexDirs = [rpc:call(Node, yz_index, index_dir, [IndexName]) || + Node <- Nodes], + lager:info("Remove index dirs: ~p, on nodes: ~p~n", + [IndexDirs, Nodes]), + [rt:stop(ANode) || ANode <- Nodes], + [rt:del_dir(binary_to_list(IndexDir)) || IndexDir <- IndexDirs], + [rt:start(ANode) || ANode <- Nodes]. + +%% @doc Remove lucence segment info files to check if reindexing will occur +%% on re-creation/re-indexing. +remove_segment_infos(Nodes, IndexName) -> + IndexDirs = [rpc:call(Node, yz_index, index_dir, [IndexName]) || + Node <- Nodes], + SiPaths = [binary_to_list(filename:join([IndexDir, "data/index/*.si"])) || + IndexDir <- IndexDirs], + SiFiles = lists:append([filelib:wildcard(Path) || Path <- SiPaths]), + lager:info("Remove segment info files: ~p, on in dirs: ~p~n", + [SiFiles, IndexDirs]), + [file:delete(SiFile) || SiFile <- SiFiles]. diff --git a/tests/yz_crdt.erl b/tests/yz_crdt.erl index b79fe49d2..815f9bd98 100644 --- a/tests/yz_crdt.erl +++ b/tests/yz_crdt.erl @@ -9,21 +9,24 @@ -define(TYPE, <<"maps">>). -define(KEY, "Chris Meiklejohn"). -define(BUCKET, {?TYPE, <<"testbucket">>}). +-define(GET(K,L), proplists:get_value(K, L)). --define(CONF, [ - {riak_core, - [{ring_creation_size, 8}] - }, - {yokozuna, - [{enabled, true}] - }]). +-define(CONF, + [ + {riak_core, + [{ring_creation_size, 8}] + }, + {yokozuna, + [{enabled, true}] + }]). confirm() -> rt:set_advanced_conf(all, ?CONF), %% Configure cluster. - [Nodes] = rt:build_clusters([1]), - [Node|_] = Nodes, + Nodes = rt:build_cluster(5, ?CONF), + + Node = rt:select_random(Nodes), %% Create PB connection. Pid = rt:pbc(Node), @@ -39,26 +42,69 @@ confirm() -> {search_index, ?INDEX}]), %% Write some sample data. - Map = riakc_map:update( + + Map1 = riakc_map:update( {<<"name">>, register}, fun(R) -> - riakc_register:set( - list_to_binary(?KEY), R) + riakc_register:set(list_to_binary(?KEY), R) end, riakc_map:new()), + Map2 = riakc_map:update( + {<<"interests">>, set}, + fun(S) -> + riakc_set:add_element(<<"thing">>, S) end, + Map1), ok = riakc_pb_socket:update_type( - Pid, - ?BUCKET, - ?KEY, - riakc_map:to_op(Map)), + Pid, + ?BUCKET, + ?KEY, + riakc_map:to_op(Map2)), %% Wait for yokozuna index to trigger. timer:sleep(1000), - %% Perform a simple query. - {ok, {search_results, Results, _, _}} = riakc_pb_socket:search( + %% Perform simple queries, check for register, set fields. + {ok, {search_results, Results1a, _, _}} = riakc_pb_socket:search( Pid, ?INDEX, <<"name_register:Chris*">>), - ?assertEqual(length(Results), 1), - lager:info("~p~n", [Results]), + lager:info("Search name_register:Chris*: ~p~n", [Results1a]), + ?assertEqual(length(Results1a), 1), + ?assertEqual(?GET(<<"name_register">>, ?GET(?INDEX, Results1a)), + list_to_binary(?KEY)), + ?assertEqual(?GET(<<"interests_set">>, ?GET(?INDEX, Results1a)), + <<"thing">>), + + {ok, {search_results, Results2a, _, _}} = riakc_pb_socket:search( + Pid, ?INDEX, <<"interests_set:thing*">>), + lager:info("Search interests_set:thing*: ~p~n", [Results2a]), + ?assertEqual(length(Results2a), 1), + ?assertEqual(?GET(<<"name_register">>, ?GET(?INDEX, Results2a)), + list_to_binary(?KEY)), + ?assertEqual(?GET(<<"interests_set">>, ?GET(?INDEX, Results2a)), + <<"thing">>), + + {ok, {search_results, Results3a, _, _}} = riakc_pb_socket:search( + Pid, ?INDEX, <<"_yz_rb:testbucket">>), + lager:info("Search testbucket: ~p~n", [Results3a]), + ?assertEqual(length(Results3a), 1), + ?assertEqual(?GET(<<"name_register">>, ?GET(?INDEX, Results3a)), + list_to_binary(?KEY)), + ?assertEqual(?GET(<<"interests_set">>, ?GET(?INDEX, Results3a)), + <<"thing">>), + + %% Redo queries and check if results are equal + {ok, {search_results, Results1b, _, _}} = riakc_pb_socket:search( + Pid, ?INDEX, <<"name_register:Chris*">>), + ?assertEqual(number_of_fields(Results1a), + number_of_fields(Results1b)), + + {ok, {search_results, Results2b, _, _}} = riakc_pb_socket:search( + Pid, ?INDEX, <<"interests_set:thing*">>), + ?assertEqual(number_of_fields(Results2a), + number_of_fields(Results2b)), + + {ok, {search_results, Results3b, _, _}} = riakc_pb_socket:search( + Pid, ?INDEX, <<"_yz_rb:testbucket">>), + ?assertEqual(number_of_fields(Results3a), + number_of_fields(Results3b)), %% Stop PB connection. riakc_pb_socket:stop(Pid), @@ -67,3 +113,7 @@ confirm() -> rt:clean_cluster(Nodes), pass. + +%% @private +number_of_fields(Resp) -> + length(?GET(?INDEX, Resp)). diff --git a/tests/yz_handoff.erl b/tests/yz_handoff.erl index 79c9d489b..27a37b2fc 100644 --- a/tests/yz_handoff.erl +++ b/tests/yz_handoff.erl @@ -29,18 +29,26 @@ -define(NUMRUNSTATES, 1). -define(SEQMAX, 1000). -define(TESTCYCLE, 20). +-define(N, 3). -define(CFG, [ {riak_core, [ - {ring_creation_size, 16} + {ring_creation_size, 16}, + {n_val, ?N}, + {handoff_concurrency, 10}, + {vnode_management_timer, 1000} ]}, {riak_kv, [ + %% allow AAE to build trees and exchange rapidly + {anti_entropy_build_limit, {100, 1000}}, + {anti_entropy_concurrency, 8}, {handoff_rejected_max, infinity} ]}, {yokozuna, [ + {anti_entropy_tick, 1000}, {enabled, true} ]} ]). @@ -54,15 +62,13 @@ confirm() -> %% Setup cluster initially - Nodes = rt:build_cluster(5, ?CFG), + [Node1, Node2, _Node3, _Node4, _Node5] = Nodes = rt:build_cluster(5, ?CFG), - %% We're going to always keep Node2 in the cluster. - [Node1, Node2, _Node3, _Node4, _Node5] = Nodes, rt:wait_for_cluster_service(Nodes, yokozuna), ConnInfo = ?GET(Node2, rt:connection_info([Node2])), {Host, Port} = ?GET(http, ConnInfo), - Shards = [begin {ok, P} = node_solr_port(Node), {Node, P} end || Node <- Nodes], + Shards = [{N, node_solr_port(N)} || N <- Nodes], %% Generate keys, YZ only supports UTF-8 compatible keys Keys = [<> || N <- lists:seq(1, ?SEQMAX), @@ -71,16 +77,7 @@ confirm() -> KeyCount = length(Keys), Pid = rt:pbc(Node2), - riakc_pb_socket:set_options(Pid, [queue_if_disconnected]), - - %% Create a search index and associate with a bucket - ok = riakc_pb_socket:create_search_index(Pid, ?INDEX), - ok = riakc_pb_socket:set_search_index(Pid, ?BUCKET, ?INDEX), - timer:sleep(1000), - - %% Write keys and wait for soft commit - lager:info("Writing ~p keys", [KeyCount]), - [ok = rt:pbc_write(Pid, ?BUCKET, Key, Key, "text/plain") || Key <- Keys], + yz_rt:write_data(Pid, ?INDEX, ?BUCKET, Keys), timer:sleep(1100), %% Separate out shards for multiple runs @@ -92,16 +89,20 @@ confirm() -> SearchURL = search_url(Host, Port, ?INDEX), lager:info("Verify Replicas Count = (3 * docs/keys) count"), - verify_count(SolrURL, KeyCount * 3), + verify_count(SolrURL, (KeyCount * ?N)), States = [#trial_state{solr_url_before = SolrURL, solr_url_after = internal_solr_url(Host, SolrPort2, ?INDEX, Shards2Rest), - leave_node = Node1}], + leave_node = Node1}, + #trial_state{solr_url_before = internal_solr_url(Host, SolrPort2, ?INDEX, Shards2Rest), + solr_url_after = SolrURL, + join_node = Node1, + admin_node = Node2}], %% Run Shell Script to count/test # of replicas and leave/join %% nodes from the cluster [[begin - check_data(State, KeyCount, BucketURL, SearchURL), + check_data(Nodes, KeyCount, BucketURL, SearchURL, State), check_counts(Pid, KeyCount, BucketURL) end || State <- States] || _ <- lists:seq(1,?NUMRUNSTATES)], @@ -113,8 +114,9 @@ confirm() -> %%%=================================================================== node_solr_port(Node) -> - riak_core_util:safe_rpc(Node, application, get_env, - [yokozuna, solr_port]). + {ok, P} = riak_core_util:safe_rpc(Node, application, get_env, + [yokozuna, solr_port]), + P. internal_solr_url(Host, Port, Index) -> ?FMT("http://~s:~B/internal_solr/~s", [Host, Port, Index]). @@ -137,7 +139,8 @@ verify_count(Url, ExpectedCount) -> fun() -> {ok, "200", _, DBody} = ibrowse:send_req(Url, [], get, []), FoundCount = get_count(DBody), - lager:info("FoundCount: ~b, ExpectedCount: ~b", [FoundCount, ExpectedCount]), + lager:info("FoundCount: ~b, ExpectedCount: ~b", + [FoundCount, ExpectedCount]), ExpectedCount =:= FoundCount end, ?assertEqual(ok, rt:wait_until(AreUp)), @@ -153,43 +156,53 @@ get_keys_count(BucketURL) -> length(kvc:path([<<"keys">>], Struct)). check_counts(Pid, InitKeyCount, BucketURL) -> - PBCounts = [begin {ok, Resp} = riakc_pb_socket:search(Pid, ?INDEX, <<"*:*">>), + PBCounts = [begin {ok, Resp} = riakc_pb_socket:search( + Pid, ?INDEX, <<"*:*">>), Resp#search_results.num_found end || _ <- lists:seq(1,?TESTCYCLE)], - HTTPCounts = [begin {ok, "200", _, RBody} = ibrowse:send_req(BucketURL, [], get, []), + HTTPCounts = [begin {ok, "200", _, RBody} = ibrowse:send_req( + BucketURL, [], get, []), Struct = mochijson2:decode(RBody), length(kvc:path([<<"keys">>], Struct)) end || _ <- lists:seq(1,?TESTCYCLE)], MinPBCount = lists:min(PBCounts), MinHTTPCount = lists:min(HTTPCounts), - lager:info("Before-Node-Leave PB: ~b, After-Node-Leave PB: ~b", [InitKeyCount, MinPBCount]), + lager:info("Before-Node-Leave PB: ~b, After-Node-Leave PB: ~b", + [InitKeyCount, MinPBCount]), ?assertEqual(InitKeyCount, MinPBCount), - lager:info("Before-Node-Leave PB: ~b, After-Node-Leave HTTP: ~b", [InitKeyCount, MinHTTPCount]), + lager:info("Before-Node-Leave PB: ~b, After-Node-Leave HTTP: ~b", + [InitKeyCount, MinHTTPCount]), ?assertEqual(InitKeyCount, MinHTTPCount). -check_data(S, KeyCount, BucketURL, SearchURL) -> - CheckCount = KeyCount * 3, +check_data(Cluster, KeyCount, BucketURL, SearchURL, S) -> + CheckCount = KeyCount * ?N, KeysBefore = get_keys_count(BucketURL), - leave_or_join(S), + UpdatedCluster = leave_or_join(Cluster, S), + + yz_rt:wait_for_aae(UpdatedCluster), KeysAfter = get_keys_count(BucketURL), lager:info("KeysBefore: ~b, KeysAfter: ~b", [KeysBefore, KeysAfter]), ?assertEqual(KeysBefore, KeysAfter), lager:info("Verify Search Docs Count =:= key count"), + lager:info("Run Search URL: ~s", [SearchURL]), verify_count(SearchURL, KeysAfter), lager:info("Verify Replicas Count = (3 * docs/keys) count"), + lager:info("Run Search URL: ~s", [S#trial_state.solr_url_after]), verify_count(S#trial_state.solr_url_after, CheckCount). -leave_or_join(S=#trial_state{join_node=undefined}) -> +leave_or_join(Cluster, S=#trial_state{join_node=undefined}) -> Node = S#trial_state.leave_node, rt:leave(Node), - ?assertEqual(ok, rt:wait_until_unpingable(Node)); -leave_or_join(S=#trial_state{leave_node=undefined}) -> + ?assertEqual(ok, rt:wait_until_unpingable(Node)), + Cluster -- [Node]; +leave_or_join(Cluster, S=#trial_state{leave_node=undefined}) -> Node = S#trial_state.join_node, NodeAdmin = S#trial_state.admin_node, ok = rt:start_and_wait(Node), ok = rt:join(Node, NodeAdmin), - ?assertEqual(ok, rt:wait_until_nodes_ready([NodeAdmin, Node])), - ?assertEqual(ok, rt:wait_until_no_pending_changes([NodeAdmin, Node])). + ?assertEqual(ok, rt:wait_until_nodes_ready(Cluster)), + ?assertEqual(ok, rt:wait_until_no_pending_changes(Cluster)), + Cluster ++ [Node]. diff --git a/tools.mk b/tools.mk index 445dd0cbc..639899436 100644 --- a/tools.mk +++ b/tools.mk @@ -1,6 +1,6 @@ # ------------------------------------------------------------------- # -# Copyright (c) 2014 Basho Technologies, Inc. +# Copyright (c) 2015 Basho Technologies, Inc. # # This file is provided to you under the Apache License, # Version 2.0 (the "License"); you may not use this file