➕ Add cache_downloads feature, automate start and start-gateway with prestarts of the neo4j server.

Daniel Kaminski de Souza · Daniel Kaminski de Souza · commit 84f71f02f329 · 2021-01-28T04:28:40.000Z
diff --git a/.env b/.env
@@ -0,0 +1,9 @@
+NEO4J_DIST='enterprise'
+NEO4J_VERSION='4.2.0'
+APOC_VERSION='4.2.0.0'
+DATASTORE_VERSION='4_0'
+NEO4J_USER=neo4j
+NEO4J_PASSWORD=letmein
+BOLT_PORT=7687
+HTTP_PORT=3000
+NEO4J_URI="bolt://localhost:{$BOLT_PORT}"
diff --git a/.env.example b/.env.example
@@ -0,0 +1,9 @@
+NEO4J_DIST='enterprise'
+NEO4J_VERSION='4.2.0'
+APOC_VERSION='4.2.0.0'
+DATASTORE_VERSION='4_0'
+NEO4J_USER=neo4j
+NEO4J_PASSWORD=letmein
+BOLT_PORT=7687
+HTTP_PORT=3000
+NEO4J_URI="bolt://localhost:{$BOLT_PORT}"
diff --git a/.gitignore b/.gitignore
@@ -71,8 +71,5 @@ test/tck/*
 
 .history
 
-__MACOSX
-neo4j
-
-neo4j-enterprise*
-recommendations.db.zip*
+.download_cache
+neo4j
diff --git a/.vscode/settings.json b/.vscode/settings.json
@@ -0,0 +1,3 @@
+{
+  "avaExplorer.cwd": "test"
+}
diff --git a/example/apollo-federation/gateway.js b/example/apollo-federation/gateway.js
@@ -5,16 +5,16 @@ import { inventorySchema } from './services/inventory';
 import { productsSchema } from './services/products';
 import { reviewsSchema } from './services/reviews';
 import neo4j from 'neo4j-driver';
+import dotenv from 'dotenv';
+
+dotenv.config();
 
 // The schema and seed data are based on the Apollo Federation demo
 // See: https://github.com/apollographql/federation-demo
 
 const driver = neo4j.driver(
   process.env.NEO4J_URI || 'bolt://localhost:7687',
-  neo4j.auth.basic(
-    process.env.NEO4J_USER || 'neo4j',
-    process.env.NEO4J_PASSWORD || 'letmein'
-  )
+  neo4j.auth.basic(process.env.NEO4J_USER, process.env.NEO4J_PASSWORD)
 );
 
 // Start Accounts
diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -4,12 +4,18 @@
   "description": "A GraphQL to Cypher query execution layer for Neo4j. ",
   "main": "./dist/index.js",
   "scripts": {
+    "reinstall": "sudo rm neo4j -r && npm install",
+    "preinstall": "sudo bash ./scripts/install-neo4j.sh",
+    "neo4j:start": "sudo bash ./scripts/start-neo4j.sh",
+    "neo4j:stop": "sudo bash ./scripts/stop-and-clear-neo4j.sh",
+    "prestart": "npm run neo4j:start",
     "start": "nodemon ./example/apollo-server/movies.js --exec babel-node -e js",
     "autogen": "nodemon ./example/autogenerated/autogen.js --exec babel-node -e js",
     "start-middleware": "nodemon ./example/apollo-server/movies-middleware.js --exec babel-node -e js",
     "start-typedefs": "nodemon ./example/apollo-server/movies-typedefs.js --exec babel-node -e js",
     "start-interface": "DEBUG=neo4j-graphql.js nodemon ./example/apollo-server/interface-union-example.js --exec babel-node -e js",
     "start-gateway": "nodemon ./example/apollo-federation/gateway.js --exec babel-node -e js",
+    "prestart-gateway": "npm run neo4j:start",
     "start-bookmark-example": "nodemon ./example/apollo-server/bookmarks.js --exec babel-node -e js",
     "start-auth-example": "JWT_SECRET=oqldBPU1yMXcrTwcha1a9PGi9RHlPVzQ nodemon ./example/apollo-server/authScopes.js --exec babel-node -e js",
     "build": "babel src --presets @babel/preset-env --out-dir dist",
@@ -71,11 +77,12 @@
     "@babel/runtime-corejs2": "^7.5.5",
     "apollo-server-errors": "^2.4.1",
     "debug": "^4.1.1",
+    "dotenv": "^8.2.0",
+    "graphql": "^15.4.0",
     "graphql-auth-directives": "^2.2.1",
+    "graphql-tools": "^7.0.2",
     "lodash": "^4.17.19",
-    "neo4j-driver": "^4.2.1",
-    "graphql": "^15.4.0",
-    "graphql-tools": "^7.0.2"
+    "neo4j-driver": "^4.2.1"
   },
   "ava": {
     "require": [
diff --git a/scripts/helpers/cache.sh b/scripts/helpers/cache.sh
@@ -0,0 +1,45 @@
+#!/bin/bash
+
+get_latest_download() {
+  local cache="$1"   # Save first argument in a variable
+  local filename_glob="$2"
+  cd $cache
+  { # try
+    latest_download="$(ls -r $filename_glob | head -1)"
+    #save your output
+  } 2> /dev/null || { 
+    # catch
+    # save log for exception
+    latest_download=()
+  }
+  cd ..
+}
+
+check_if_there_is_need_to_download() {
+  local cache="$1"   # Save first argument in a variable
+  local file_URL="$2"   # Save first argument in a variable
+  local filename="$3"   # Save first argument in a variable
+  get_latest_download $cache $filename*
+
+  file_to_download=()
+  if [ ! "$latest_download" ]; then
+    file_to_download=($file_URL/$filename)
+  fi
+}
+get_list_of_files_to_download(){
+  # $1 cache folder
+  # $2..n files to check if available in cache
+  local cache="$1"   # Save first argument in a variable
+  shift            # Shift all arguments to the left (original $1 gets lost)
+  local files_info=("$@") # Rebuild the array with rest of arguments
+  local pattern='(.*):+(.*)'
+
+  files_to_download=()
+
+  for file_info in "${files_info[@]}"
+  do
+    [[ $file_info =~ $pattern ]]
+    check_if_there_is_need_to_download $cache ${BASH_REMATCH[1]} ${BASH_REMATCH[2]}
+    files_to_download+=($file_to_download)
+  done
+}
diff --git a/scripts/helpers/cached_download.sh b/scripts/helpers/cached_download.sh
@@ -0,0 +1,28 @@
+#!/bin/bash
+
+source=${BASH_SOURCE[0]}
+
+cached_download(){
+    . $(dirname $source)/get_source_dir.sh
+    local this_directory=$(get_source_dir $source)
+    . $this_directory/cache.sh
+    . $this_directory/download_files.sh
+    . $this_directory/delete_duplicates.sh
+    # $1 download_info
+    # $2 cache_folder
+    local cache="$1"   # Save first argument in a variable
+    shift            # Shift all arguments to the left (original $1 gets lost)
+    local files_info=("$@") # Rebuild the array with rest of arguments
+    get_list_of_files_to_download $cache "${files_info[@]}"
+    download_files $cache "${files_to_download[@]}"
+    delete_duplicates $cache # just in case
+    cached_downloads=()
+    local pattern='(.*):+(.*)'
+    for file_info in ${files_info[@]}
+    do
+        [[ $file_info =~ $pattern ]]
+        local file_name=${BASH_REMATCH[2]}
+        get_latest_download $cache $file_name
+        cached_downloads+=($cache/$latest_download)
+    done
+}
diff --git a/scripts/helpers/delete_duplicates.sh b/scripts/helpers/delete_duplicates.sh
@@ -0,0 +1,8 @@
+#!/bin/bash
+
+# sudo apt-get install fdupes
+
+delete_duplicates(){
+  local folder_to_look_for_duplicates="$1"
+  fdupes -rdN $folder_to_look_for_duplicates
+}
diff --git a/scripts/helpers/download_files.sh b/scripts/helpers/download_files.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+
+source=${BASH_SOURCE[0]}
+. $(dirname $source)/get_source_dir.sh
+
+download_files(){
+    local this_directory=$(get_source_dir $source)
+    local cache="$1"    # Save first argument in a variable
+    shift               # Shift all arguments to the left (original $1 gets lost)
+    local files_to_download=("$@") # Rebuild the array with rest of arguments
+    local number_of_files_to_download=${#files_to_download[@]}
+
+    if [ $number_of_files_to_download -gt 0 ]; then
+        . $this_directory/get_number_of_logical_processors.sh
+        get_number_of_logical_processors
+        echo ${files_to_download[@]} | xargs -n 1 -P $logical_cpus wget -q -P $cache --show-progress
+    fi
+}
diff --git a/scripts/helpers/execute_start.sh b/scripts/helpers/execute_start.sh
@@ -0,0 +1,35 @@
+#!/bin/dash
+
+this_directory=$1
+localhost=$2
+
+. $this_directory/helpers/load_env_vars.sh
+load_env_vars
+
+if [ ! -d "neo4j/data/databases/graph.db" ]; then
+    echo "Neo4j not installed correctly, run ./scripts/install_neo4j"
+    exit 1
+else
+    echo "dbms.allow_upgrade=true" >> ./neo4j/conf/neo4j.conf
+    echo "dbms.recovery.fail_on_missing_files=false" >> ./neo4j/conf/neo4j.conf
+    # Set initial and max heap to workaround JVM in docker issues
+    dbms_memory_heap_initial_size="2048m" dbms_memory_heap_max_size="2048m" ./neo4j/bin/neo4j start
+    echo "Waiting up to 2 minutes for neo4j bolt port ($BOLT_PORT)"
+
+    echo "Endpoint is $localhost:$BOLT_PORT"
+    for i in {1..120};
+        do
+            nc -z $localhost $BOLT_PORT -w 2
+            is_up=$?
+            if [ $is_up -eq 0 ]; then
+                echo
+                echo "Successfully started, neo4j bolt available on $BOLT_PORT"
+                break
+            fi
+            sleep 1
+            echo -n "."
+    done
+    echo
+    # Wait a further 5 seconds after the port is available
+    sleep 5
+fi
diff --git a/scripts/helpers/execute_stop.sh b/scripts/helpers/execute_stop.sh
@@ -0,0 +1,29 @@
+#!/bin/dash
+
+this_directory=$1
+localhost=$2
+
+. $this_directory/helpers/load_env_vars.sh
+load_env_vars
+
+./neo4j/bin/neo4j stop
+rm -r neo4j/data/databases/graph.db
+./neo4j/bin/neo4j start
+
+echo "Waiting up to 2 minutes for neo4j bolt port ($BOLT_PORT)"
+
+for i in {1..120};
+    do
+        nc -z $localhost $BOLT_PORT -w 2
+        is_up=$?
+        if [ $is_up -eq 0 ]; then
+            echo
+            echo "Successfully started, neo4j bolt available on $BOLT_PORT"
+            break
+        fi
+        sleep 1
+        echo -n "."
+done
+echo
+# Wait a further 5 seconds after the port is available
+sleep 5
diff --git a/scripts/helpers/execute_wait.sh b/scripts/helpers/execute_wait.sh
@@ -0,0 +1,24 @@
+#!/bin/dash
+
+this_directory=$1
+localhost=$2
+
+. $this_directory/helpers/load_env_vars.sh
+load_env_vars
+
+echo "Waiting up to 2 minutes for graphql http port ($HTTP_PORT)"
+
+. $this_directory/helpers/get_local_host.sh
+for i in {1..120};
+    do
+        nc -z $(get_local_host) $HTTP_PORT -w 2
+        is_up=$?
+        if [ $is_up -eq 0 ]; then
+            echo
+            echo "Successfully started, graphql http available on $HTTP_PORT"
+            break
+        fi
+        sleep 1
+        echo -n "."
+done
+echo
diff --git a/scripts/helpers/get_local_host.sh b/scripts/helpers/get_local_host.sh
@@ -0,0 +1,10 @@
+#!/bin/bash
+source=${BASH_SOURCE[0]}
+. $(dirname $source)/get_source_dir.sh
+
+get_local_host(){
+    local this_directory=$(get_source_dir $source)
+    local localhost=$(perl $this_directory/regex.pl "$(cat /etc/resolv.conf)")
+    echo $localhost
+}
+# echo "$(get_local_host)"
diff --git a/scripts/helpers/get_number_of_logical_processors.sh b/scripts/helpers/get_number_of_logical_processors.sh
@@ -0,0 +1,7 @@
+#!/bin/bash
+
+get_number_of_logical_processors(){
+    logical_cpus=$([ $(uname) = 'Darwin' ] && 
+                    sysctl -n hw.logicalcpu_max || 
+                    lscpu -p | egrep -v '^#' | wc -l)
+}
diff --git a/scripts/helpers/get_source_dir.sh b/scripts/helpers/get_source_dir.sh
@@ -0,0 +1,14 @@
+#!/bin/bash
+# Based on:
+# https://stackoverflow.com/questions/59895/how-can-i-get-the-source-directory-of-a-bash-script-from-within-the-script-itsel
+
+get_source_dir(){
+    local this_source=$1
+    while [ -h "$this_source" ]; do # resolve $this_source until the file is no longer a symlink
+        local this_dir="$( cd -P "$( dirname "$this_source" )" >/dev/null 2>&1 && pwd )"
+        this_source="$(readlink "$this_source")"
+        [[ $this_source != /* ]] && this_source="$this_dir/$this_source" # if $this_source was a relative symlink, we need to resolve it relative to the path where the symlink file was located
+    done
+    this_dir="$( cd -P "$( dirname "$this_source" )" >/dev/null 2>&1 && pwd )"
+    echo $this_dir
+}
diff --git a/scripts/helpers/load_env_vars.sh b/scripts/helpers/load_env_vars.sh
@@ -0,0 +1,8 @@
+#!/bin/bash
+
+load_env_vars(){
+    if [ ! -f ../.env ]
+    then
+        export $(cat .env | xargs)
+    fi
+}
diff --git a/scripts/helpers/regex.pl b/scripts/helpers/regex.pl
@@ -0,0 +1,13 @@
+use strict;
+my $regex = qr/(?<=nameserver )(?:[0-9]{1,3}\.){3}[0-9]{1,3}/p;
+
+
+if ( $ARGV[0] =~ /$regex/ ) {
+  print "${^MATCH}";
+  # print "Whole match is ${^MATCH} and its start/end positions can be obtained via \$-[0] and \$+[0]\n";
+  # print "Capture Group 1 is $1 and its start/end positions can be obtained via \$-[1] and \$+[1]\n";
+  # print "Capture Group 2 is $2 ... and so on\n";
+}
+
+# ${^POSTMATCH} and ${^PREMATCH} are also available with the use of '/p'
+# Named capture groups can be called via $+{name}
diff --git a/scripts/helpers/test_get_source_data.sh b/scripts/helpers/test_get_source_data.sh
@@ -0,0 +1,6 @@
+#!/bin/bash
+
+source=${BASH_SOURCE[0]}
+. $(dirname $source)/get_source_dir.sh
+this_directory=$(get_source_dir $source)
+echo $this_directory
diff --git a/scripts/install-neo4j.sh b/scripts/install-neo4j.sh
diff --git a/scripts/start-neo4j.sh b/scripts/start-neo4j.sh
diff --git a/scripts/stop-and-clear-neo4j.sh b/scripts/stop-and-clear-neo4j.sh
diff --git a/scripts/wait-for-graphql.sh b/scripts/wait-for-graphql.sh

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+{`
	`2`	`+ "avaExplorer.cwd": "test"`
	`3`	`+}`