diff --git a/README.md b/README.md index 51728fa5..c4a2a2ee 100644 --- a/README.md +++ b/README.md @@ -109,7 +109,7 @@ npm run coverage The integration tests for this repo require the google analytics credentials to be set in the environment. This can be setup with the dotenv-cli package as -described in "Setup Environment" section above. +described in the [Setup Environment](#setup-environment) section. Note that these tests make real requests to google analytics APIs and should be run sparingly to avoid being rate limited in our live apps which use the @@ -117,10 +117,10 @@ same account credentials. ```bash # Run cucumber integration tests -dotenv -e .env npm run cucumber +npx dotenv -- npm run cucumber # Run cucumber integration tests with node debugging enabled -dotenv -e .env npm run cucumber:debug +npx dotenv -- npm run cucumber:debug ``` The cucumber features and support files can be found in the `features` directory @@ -129,7 +129,7 @@ The cucumber features and support files can be found in the `features` directory #### Setup environment -See "Configuration and Google Analytics Setup" below for the required environment variables and other setup for Google Analytics auth. +See [Configuration - Google Analytics](#google-analytics) for the required environment variables and other setup for Google Analytics auth. It may be easiest to use the dotenv-cli package to configure the environment for the application. @@ -165,6 +165,17 @@ npx dotenv -e .env.analytics node -- deploy/publisher.js npx dotenv -e .env.analytics node -- deploy/consumer.js ``` +To run a single report for a specific agency, set AGENCY_NAME and ANALYTICS_REPORT_IDS in your .env file (refer to `../deploy/agencies.json`) +and then run: + +```bash +# general format +npx dotenv -- npm start -- --only=REPORT_NAME --json --output=/path/to/output + +# example +npx dotenv -- npm start -- --only=devices --json --output=./output +``` + ## Configuration ### Google Analytics diff --git a/reports/usa.json b/reports/usa.json index fb40f26d..3ea3f1a4 100644 --- a/reports/usa.json +++ b/reports/usa.json @@ -4971,34 +4971,6 @@ "dimensionFilter": { "andGroup": { "expressions": [ - { - "notExpression": { - "andGroup": { - "expressions": [ - { - "filter": { - "fieldName": "hostName", - "stringFilter": { - "matchType": "EXACT", - "value": "etap.nsf.gov", - "caseSensitive": false - } - } - }, - { - "filter": { - "fieldName": "pagePath", - "stringFilter": { - "matchType": "BEGINS_WITH", - "value": "/reference-writer", - "caseSensitive": false - } - } - } - ] - } - } - }, { "notExpression": { "filter": { @@ -5022,6 +4994,7 @@ "fieldName": "pagePath", "inListFilter": { "values": [ + "(data deleted)", "(other)", "other", "(not set)", @@ -5074,34 +5047,6 @@ "dimensionFilter": { "andGroup": { "expressions": [ - { - "notExpression": { - "andGroup": { - "expressions": [ - { - "filter": { - "fieldName": "hostName", - "stringFilter": { - "matchType": "EXACT", - "value": "etap.nsf.gov", - "caseSensitive": false - } - } - }, - { - "filter": { - "fieldName": "pagePath", - "stringFilter": { - "matchType": "BEGINS_WITH", - "value": "/reference-writer", - "caseSensitive": false - } - } - } - ] - } - } - }, { "notExpression": { "filter": { @@ -5125,6 +5070,7 @@ "fieldName": "pagePath", "inListFilter": { "values": [ + "(data deleted)", "(other)", "other", "(not set)", @@ -5177,34 +5123,6 @@ "dimensionFilter": { "andGroup": { "expressions": [ - { - "notExpression": { - "andGroup": { - "expressions": [ - { - "filter": { - "fieldName": "hostName", - "stringFilter": { - "matchType": "EXACT", - "value": "etap.nsf.gov", - "caseSensitive": false - } - } - }, - { - "filter": { - "fieldName": "pagePath", - "stringFilter": { - "matchType": "BEGINS_WITH", - "value": "/reference-writer", - "caseSensitive": false - } - } - } - ] - } - } - }, { "notExpression": { "filter": { @@ -5228,6 +5146,7 @@ "fieldName": "pagePath", "inListFilter": { "values": [ + "(data deleted)", "(other)", "other", "(not set)", @@ -5280,34 +5199,6 @@ "dimensionFilter": { "andGroup": { "expressions": [ - { - "notExpression": { - "andGroup": { - "expressions": [ - { - "filter": { - "fieldName": "hostName", - "stringFilter": { - "matchType": "EXACT", - "value": "etap.nsf.gov", - "caseSensitive": false - } - } - }, - { - "filter": { - "fieldName": "pagePath", - "stringFilter": { - "matchType": "BEGINS_WITH", - "value": "/reference-writer", - "caseSensitive": false - } - } - } - ] - } - } - }, { "notExpression": { "filter": { @@ -5331,6 +5222,7 @@ "fieldName": "pagePath", "inListFilter": { "values": [ + "(data deleted)", "(other)", "other", "(not set)", @@ -5383,34 +5275,6 @@ "dimensionFilter": { "andGroup": { "expressions": [ - { - "notExpression": { - "andGroup": { - "expressions": [ - { - "filter": { - "fieldName": "hostName", - "stringFilter": { - "matchType": "EXACT", - "value": "etap.nsf.gov", - "caseSensitive": false - } - } - }, - { - "filter": { - "fieldName": "pagePath", - "stringFilter": { - "matchType": "BEGINS_WITH", - "value": "/reference-writer", - "caseSensitive": false - } - } - } - ] - } - } - }, { "notExpression": { "filter": { @@ -5434,6 +5298,7 @@ "fieldName": "pagePath", "inListFilter": { "values": [ + "(data deleted)", "(other)", "other", "(not set)", @@ -5486,34 +5351,6 @@ "dimensionFilter": { "andGroup": { "expressions": [ - { - "notExpression": { - "andGroup": { - "expressions": [ - { - "filter": { - "fieldName": "hostName", - "stringFilter": { - "matchType": "EXACT", - "value": "etap.nsf.gov", - "caseSensitive": false - } - } - }, - { - "filter": { - "fieldName": "pagePath", - "stringFilter": { - "matchType": "BEGINS_WITH", - "value": "/reference-writer", - "caseSensitive": false - } - } - } - ] - } - } - }, { "notExpression": { "filter": { @@ -5537,6 +5374,7 @@ "fieldName": "pagePath", "inListFilter": { "values": [ + "(data deleted)", "(other)", "other", "(not set)", @@ -5589,34 +5427,6 @@ "dimensionFilter": { "andGroup": { "expressions": [ - { - "notExpression": { - "andGroup": { - "expressions": [ - { - "filter": { - "fieldName": "hostName", - "stringFilter": { - "matchType": "EXACT", - "value": "etap.nsf.gov", - "caseSensitive": false - } - } - }, - { - "filter": { - "fieldName": "pagePath", - "stringFilter": { - "matchType": "BEGINS_WITH", - "value": "/reference-writer", - "caseSensitive": false - } - } - } - ] - } - } - }, { "notExpression": { "filter": { @@ -5640,6 +5450,7 @@ "fieldName": "pagePath", "inListFilter": { "values": [ + "(data deleted)", "(other)", "other", "(not set)", @@ -5692,34 +5503,6 @@ "dimensionFilter": { "andGroup": { "expressions": [ - { - "notExpression": { - "andGroup": { - "expressions": [ - { - "filter": { - "fieldName": "hostName", - "stringFilter": { - "matchType": "EXACT", - "value": "etap.nsf.gov", - "caseSensitive": false - } - } - }, - { - "filter": { - "fieldName": "pagePath", - "stringFilter": { - "matchType": "BEGINS_WITH", - "value": "/reference-writer", - "caseSensitive": false - } - } - } - ] - } - } - }, { "notExpression": { "filter": { @@ -5743,6 +5526,7 @@ "fieldName": "pagePath", "inListFilter": { "values": [ + "(data deleted)", "(other)", "other", "(not set)", @@ -5795,34 +5579,6 @@ "dimensionFilter": { "andGroup": { "expressions": [ - { - "notExpression": { - "andGroup": { - "expressions": [ - { - "filter": { - "fieldName": "hostName", - "stringFilter": { - "matchType": "EXACT", - "value": "etap.nsf.gov", - "caseSensitive": false - } - } - }, - { - "filter": { - "fieldName": "pagePath", - "stringFilter": { - "matchType": "BEGINS_WITH", - "value": "/reference-writer", - "caseSensitive": false - } - } - } - ] - } - } - }, { "notExpression": { "filter": { @@ -5846,6 +5602,7 @@ "fieldName": "pagePath", "inListFilter": { "values": [ + "(data deleted)", "(other)", "other", "(not set)", @@ -5898,34 +5655,6 @@ "dimensionFilter": { "andGroup": { "expressions": [ - { - "notExpression": { - "andGroup": { - "expressions": [ - { - "filter": { - "fieldName": "hostName", - "stringFilter": { - "matchType": "EXACT", - "value": "etap.nsf.gov", - "caseSensitive": false - } - } - }, - { - "filter": { - "fieldName": "pagePath", - "stringFilter": { - "matchType": "BEGINS_WITH", - "value": "/reference-writer", - "caseSensitive": false - } - } - } - ] - } - } - }, { "notExpression": { "filter": { @@ -5949,6 +5678,7 @@ "fieldName": "pagePath", "inListFilter": { "values": [ + "(data deleted)", "(other)", "other", "(not set)", @@ -6001,34 +5731,6 @@ "dimensionFilter": { "andGroup": { "expressions": [ - { - "notExpression": { - "andGroup": { - "expressions": [ - { - "filter": { - "fieldName": "hostName", - "stringFilter": { - "matchType": "EXACT", - "value": "etap.nsf.gov", - "caseSensitive": false - } - } - }, - { - "filter": { - "fieldName": "pagePath", - "stringFilter": { - "matchType": "BEGINS_WITH", - "value": "/reference-writer", - "caseSensitive": false - } - } - } - ] - } - } - }, { "notExpression": { "filter": { @@ -6052,6 +5754,7 @@ "fieldName": "pagePath", "inListFilter": { "values": [ + "(data deleted)", "(other)", "other", "(not set)", @@ -6104,34 +5807,6 @@ "dimensionFilter": { "andGroup": { "expressions": [ - { - "notExpression": { - "andGroup": { - "expressions": [ - { - "filter": { - "fieldName": "hostName", - "stringFilter": { - "matchType": "EXACT", - "value": "etap.nsf.gov", - "caseSensitive": false - } - } - }, - { - "filter": { - "fieldName": "pagePath", - "stringFilter": { - "matchType": "BEGINS_WITH", - "value": "/reference-writer", - "caseSensitive": false - } - } - } - ] - } - } - }, { "notExpression": { "filter": { @@ -6155,6 +5830,7 @@ "fieldName": "pagePath", "inListFilter": { "values": [ + "(data deleted)", "(other)", "other", "(not set)", @@ -6207,34 +5883,6 @@ "dimensionFilter": { "andGroup": { "expressions": [ - { - "notExpression": { - "andGroup": { - "expressions": [ - { - "filter": { - "fieldName": "hostName", - "stringFilter": { - "matchType": "EXACT", - "value": "etap.nsf.gov", - "caseSensitive": false - } - } - }, - { - "filter": { - "fieldName": "pagePath", - "stringFilter": { - "matchType": "BEGINS_WITH", - "value": "/reference-writer", - "caseSensitive": false - } - } - } - ] - } - } - }, { "notExpression": { "filter": { @@ -6258,6 +5906,7 @@ "fieldName": "pagePath", "inListFilter": { "values": [ + "(data deleted)", "(other)", "other", "(not set)", @@ -6310,34 +5959,6 @@ "dimensionFilter": { "andGroup": { "expressions": [ - { - "notExpression": { - "andGroup": { - "expressions": [ - { - "filter": { - "fieldName": "hostName", - "stringFilter": { - "matchType": "EXACT", - "value": "etap.nsf.gov", - "caseSensitive": false - } - } - }, - { - "filter": { - "fieldName": "pagePath", - "stringFilter": { - "matchType": "BEGINS_WITH", - "value": "/reference-writer", - "caseSensitive": false - } - } - } - ] - } - } - }, { "notExpression": { "filter": { @@ -6361,6 +5982,7 @@ "fieldName": "pagePath", "inListFilter": { "values": [ + "(data deleted)", "(other)", "other", "(not set)", @@ -6413,34 +6035,6 @@ "dimensionFilter": { "andGroup": { "expressions": [ - { - "notExpression": { - "andGroup": { - "expressions": [ - { - "filter": { - "fieldName": "hostName", - "stringFilter": { - "matchType": "EXACT", - "value": "etap.nsf.gov", - "caseSensitive": false - } - } - }, - { - "filter": { - "fieldName": "pagePath", - "stringFilter": { - "matchType": "BEGINS_WITH", - "value": "/reference-writer", - "caseSensitive": false - } - } - } - ] - } - } - }, { "notExpression": { "filter": { @@ -6464,6 +6058,7 @@ "fieldName": "pagePath", "inListFilter": { "values": [ + "(data deleted)", "(other)", "other", "(not set)", @@ -6516,34 +6111,6 @@ "dimensionFilter": { "andGroup": { "expressions": [ - { - "notExpression": { - "andGroup": { - "expressions": [ - { - "filter": { - "fieldName": "hostName", - "stringFilter": { - "matchType": "EXACT", - "value": "etap.nsf.gov", - "caseSensitive": false - } - } - }, - { - "filter": { - "fieldName": "pagePath", - "stringFilter": { - "matchType": "BEGINS_WITH", - "value": "/reference-writer", - "caseSensitive": false - } - } - } - ] - } - } - }, { "notExpression": { "filter": { @@ -6567,6 +6134,7 @@ "fieldName": "pagePath", "inListFilter": { "values": [ + "(data deleted)", "(other)", "other", "(not set)", @@ -6626,29 +6194,21 @@ "expressions": [ { "notExpression": { - "andGroup": { - "expressions": [ - { - "filter": { - "fieldName": "hostName", - "stringFilter": { - "matchType": "EXACT", - "value": "etap.nsf.gov", - "caseSensitive": false - } - } - }, - { - "filter": { - "fieldName": "pagePath", - "stringFilter": { - "matchType": "BEGINS_WITH", - "value": "/reference-writer", - "caseSensitive": false - } - } + "filter": { + "fieldName": "pagePath", + "stringFilter": { + "inListFilter": { + "values": [ + "(data deleted)", + "(other)", + "other", + "(not set)", + "null", + "" + ], + "caseSensitive": false } - ] + } } } },