Add integration tests using Papermill library for RL notebooks. List of notebooks covered in the tests: (#1580)

1. rl_cartpole_coach/rl_cartpole_coach_gymEnv.ipynb
2. rl_cartpole_ray/rl_cartpole_ray_gymEnv.ipynb

Co-authored-by: Akash Goel <[email protected]>

Author: goelakash

reinforcement_learning/rl_cartpole_coach/rl_cartpole_coach_gymEnv.ipynb

@@ -115,7 +115,10 @@
    "cell_type": "code",
    "execution_count": null,
    "metadata": {
-    "collapsed": true
+    "collapsed": true,
+    "tags": [
+     "parameters"
+    ]
    },
    "outputs": [],
    "source": [
@@ -573,7 +576,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.7.7"
+   "version": "3.7.9"
   },
   "notice": "Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the \"License\"). You may not use this file except in compliance with the License. A copy of the License is located at http://aws.amazon.com/apache2.0/ or in the \"license\" file accompanying this file. This file is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License."
  },

reinforcement_learning/rl_cartpole_coach/src/tmp/2018-11-28-05-42-18_episode-35_score-13.0.gif.gif

@@ -114,7 +114,11 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {},
+   "metadata": {
+    "tags": [
+     "parameters"
+    ]
+   },
    "outputs": [],
    "source": [
     "# run in local_mode on this machine, or as a SageMaker TrainingJob?\n",
@@ -239,6 +243,19 @@
     "5. Define the metrics definitions that you are interested in capturing in your logs. These can also be visualized in CloudWatch and SageMaker Notebooks. "
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "tags": [
+     "parameters"
+    ]
+   },
+   "outputs": [],
+   "source": [
+    "train_instance_count = 1"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -259,7 +276,7 @@
     "                        role=role,\n",
     "                        debugger_hook_config=False,\n",
     "                        train_instance_type=instance_type,\n",
-    "                        train_instance_count=1,\n",
+    "                        train_instance_count=train_instance_count,\n",
     "                        output_path=s3_output_path,\n",
     "                        base_job_name=job_name_prefix,\n",
     "                        metric_definitions=metric_definitions,\n",
@@ -556,7 +573,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.7.7"
+   "version": "3.7.9"
   },
   "notice": "Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the \"License\"). You may not use this file except in compliance with the License. A copy of the License is located at http://aws.amazon.com/apache2.0/ or in the \"license\" file accompanying this file. This file is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License."
  },

reinforcement_learning/rl_cartpole_ray/rl_cartpole_ray_gymEnv.ipynb

@@ -0,0 +1,3 @@
+pandas
+papermill
+tabulate

reinforcement_learning/test_requirements.txt

@@ -0,0 +1,5 @@
+train_instance_count,instance_type,local_mode
+1,ml.m4.xlarge,False
+2,ml.m4.xlarge,False
+1,ml.p3.2xlarge,False
+2,ml.p3.2xlarge,False

reinforcement_learning/testconfig.csv

@@ -0,0 +1,2 @@
+rl_cartpole_ray/rl_cartpole_ray_gymEnv.ipynb
+rl_cartpole_coach/rl_cartpole_coach_gymEnv.ipynb

reinforcement_learning/testnotebooks.txt

@@ -0,0 +1,90 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+import itertools
+import json
+import os
+import subprocess
+import sys
+import time
+try:
+    import pandas as pd
+    import papermill
+    from tabulate import tabulate
+except ImportError:
+    sys.exit("""Some libraries are missing. Please install them by running `pip install -r test_requirements.txt`.""")
+
+# CONSTANTS
+TEST_NOTEBOOKS_FILE = 'testnotebooks.txt'
+TEST_CONFIG_FILE = 'testconfig.csv'
+SUCCESSES = 0
+EXCEPTIONS = 0
+SUCCESSFUL_EXECUTIONS = []
+FAILED_EXECUTIONS = []
+CELL_EXECUTION_TIMEOUT_SECONDS = 1200
+
+
+# helper functions
+def run_notebook(nb_path, test_config):
+    dir_name = os.path.dirname(nb_path)
+    nb_name = os.path.basename(nb_path)
+    output_nb_name = "output_{}.ipynb".format(nb_name)
+    os.chdir(dir_name)
+    print("Current directory: {}".format(os.getcwd()))
+    global SUCCESSES
+    global EXCEPTIONS
+    for i in range(len(test_config)):
+        params = json.loads(test_config.loc[i].to_json())
+        # Coach notebooks support only single instance training, so skip the tests with multiple EC2 instances
+        if 'coach' in nb_name.lower() and params['train_instance_count'] > 1:
+            continue
+        print("\nTEST: " + nb_name + " with parameters " + str(params))
+        process = None
+        try:
+            papermill.execute_notebook(nb_name, output_nb_name, parameters=params, execution_timeout=CELL_EXECUTION_TIMEOUT_SECONDS, log_output=True)
+            SUCCESSES += 1
+            SUCCESSFUL_EXECUTIONS.append(dict({'notebook':nb_name, 'params':params}))
+        except BaseException as error:
+            print('An exception occurred: {}'.format(error))
+            EXCEPTIONS += 1
+            FAILED_EXECUTIONS.append(dict({'notebook':nb_name, 'params':params}))
+
+def print_notebook_executions(nb_list_with_params):
+    # This expects a list of dict type items.
+    # E.g. [{'nb_name':'foo', 'params':'bar'}]
+    if not nb_list_with_params:
+        print("None")
+        return
+    vals = []
+    for nb_dict in nb_list_with_params:
+        val = []
+        for k,v in nb_dict.items():
+            val.append(v)
+        vals.append(val)
+    keys = [k for k in nb_list_with_params[0].keys()]
+    print(tabulate(pd.DataFrame([v for v in vals], columns=keys), showindex=False))
+
+
+
+notebooks_list = open(TEST_NOTEBOOKS_FILE).readlines()
+config = pd.read_csv(TEST_CONFIG_FILE)
+ROOT = os.path.abspath('.')
+
+# Run tests on each notebook listed in the config.
+print("Test Configuration: ")
+print(config)
+for nb_path in notebooks_list:
+    os.chdir(ROOT)
+    print("Testing: {}".format(nb_path))
+    run_notebook(nb_path.strip(), config)
+
+# Print summary of tests ran.
+print("Summary: {}/{} tests passed.".format(SUCCESSES, SUCCESSES + EXCEPTIONS))
+print("Successful executions: ")
+print_notebook_executions(SUCCESSFUL_EXECUTIONS)
+
+# Throw exception if any test fails, so that the CodeBuild also fails.
+if EXCEPTIONS > 0:
+    print("Failed executions: ")
+    print_notebook_executions(FAILED_EXECUTIONS)
+    raise Exception("Test did not complete successfully")