Project Reorganization (#89)

* Project Reorganization (#79)

* chore: move files to new locations and dir structure

* fix: adjust setupvenv for new directory structure

* Removed submodule pulumi/aws/sirius/src

* chore: remove submodule to move it

* Added submodule pulumi/python/kubernetes/applications/sirius/src

* chore: cleaning up misc files

* chore: update paths for reorganization

* fix: bug with how pulumi handles node group options #72

* fix: make paths work with new directory structure

* chore: update dependencies

* fix: make gh action work with new docker location

* fix: changes to GH actions

* fix: adjust to work with kubeconfig abstraction

* feat: adding in kubeconfig file deployment

* fix: updated start/stop scripts for aws/kubeconfig

* feat: changes to extend helm timeout and integrate new layout

* fix: update chart versions for all helm deployments

* fix: upgrade to 2.0.2 IC version

* chore: bump requirements

* fix: re-enable the abort on error for destroy scripts

* feat: created standard start/destroy scripts

* fix: downgrade cert manager chart to fix failures with observability

* fix: updates to start scripts to fix bugs

* fix: remove errant comma from chart version

* fix: change where we pull the fqdn from for sirius app

* fix: updated bash start/destroy scripts

* feat: script to check capabilities

* fix: cleanup unused code in module

* feat: script to test the installation capabilities for MARA

* fix: update helper functions and code

* fix: update to keep config dir for tools

* fix: add path to extras shell script

* fix: change file test for JWT to check size

* fix: cleaning up hostname management

* fix: change pulumi check to work

* chore: bump requirement versions

* fix: downgrade pyaml

* chore: rename eks vars to k8 for consistency and clarity

* chore: refactor out vpc pulumi from mainline logic into sep directory

* fix: manage fqdn with kubeconfig files

* fix: add script dir to extras install

* feat: script updates and cert manager updates

* fix: remove introduced error from otel mainline

* fix: bugfix on config dir name

* fix: removed redundant set of fqdn

* chore: bump otel operator version

* feat: tune replicas for logstore

* feat: update documentation and configs

* chore: general cleanup

* feat: begin addition of directory level docs

* feat: README updates

* feat: add configurable timeout to helm operations

* fix: updates to fossa for path changes, gitignore

* fix: workaround for #72

* feat: updated comments to better reflect the logic

* fix: adjust workflow for AWS

* fix: update scripts with new logic and tag issues

* fix: code cleanup

* fix: testing GH actions

* fix: testing GH actions

* feat: remove centos idockerfile and adjust docker builds

* chore: testing

* chore: testing workflows

* chore: workflows

* chore: dep bump

* chore: adjust requirements

* chore: add python version file

* feat: update getting started guide

* chore: requirements

* fix: change tests to use runner

* fix: gitkeep to hold the manifests directory in place

* chore: debugging actions

* chore: debugging actions

* chore: debugging actions

* chore: debugging actions

* chore: debugging actions

* chore: debugging actions

* chore: debugging actions

* feat: documentation update

* fix: config value timeout is an int, we were pulling it as a string (#88)

* fix: config value timeout is an int, we were pulling it as a string

* feat: creation of additional docs and updates

Author: Jason Schmidt

.fossa.yml

@@ -5,10 +5,10 @@ cli:
   project: [email protected]:nginxinc/kic-reference-architectures.git
 analyze:
   modules:
-  - name: pulumi/aws
+  - name: pulumi/python
     type: pip
-    path: pulumi/aws
-    target: pulumi/aws
+    path: pulumi/python
+    target: pulumi/python
     options:
       strategy: pip
-      requirements: pulumi/aws/requirements.txt
+      requirements: config/pulumi/requirements.txt

.github/workflows/pulumi-aws-container-tests.yml

@@ -12,14 +12,5 @@ jobs:
     - uses: actions/checkout@v2
 
     - name: Build and Test Debian Image
-      working-directory: pulumi/aws
-      run: bash build_dev_docker_image.sh debian
-
-  centos-container-tests:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-
-    - name: Build and Test CentOS Image
-      working-directory: pulumi/aws
-      run: bash build_dev_docker_image.sh centos
+      working-directory:
+      run: bash docker/build_dev_docker_image.sh debian

.github/workflows/pulumi-aws-tests.yml

@@ -13,17 +13,17 @@ jobs:
     - uses: syphar/restore-virtualenv@v1
       id: pulumi-aws-tests-cache-virtualenv
       with:
-        requirement_files: pulumi/aws/requirements.txt
+        requirement_files: config/requirements.txt
 
     - uses: syphar/restore-pip-download-cache@v1
       if: steps.pulumi-aws-tests-cache-virtualenv.outputs.cache-hit != 'true'
 
     - run: ./setup_venv.sh
-      working-directory: pulumi/aws
+      working-directory: bin
 
     - name: Test
-      working-directory: pulumi/aws
-      run: python3 test.py
+      working-directory: bin
+      run: ./test_runner.sh "/home/runner/work/kic-reference-architectures/kic-reference-architectures"
 
   macos-tests:
     runs-on: macos-latest
@@ -33,14 +33,14 @@ jobs:
     - uses: syphar/restore-virtualenv@v1
       id: pulumi-aws-tests-cache-virtualenv
       with:
-        requirement_files: pulumi/aws/requirements.txt
+        requirement_files: config/requirements.txt
 
     - uses: syphar/restore-pip-download-cache@v1
       if: steps.pulumi-aws-tests-cache-virtualenv.outputs.cache-hit != 'true'
 
     - run: ./setup_venv.sh
-      working-directory: pulumi/aws
+      working-directory: bin
 
     - name: Test
-      working-directory: pulumi/aws
-      run: python3 test.py
+      working-directory: bin
+      run: ./test_runner.sh "/Users/runner/work/kic-reference-architectures/kic-reference-architectures"

.gitignore

@@ -261,3 +261,7 @@ override.tf.json
 # End of https://www.toptal.com/developers/gitignore/api/python,pycharm+all,terraform
 
 *.pyc
+!/extras/jwt.token
+/pulumi/python/tools/common/config/*.yaml
+/pulumi/python/tools/common/config/environment
+!/pulumi/python/kubernetes/nginx/ingress-controller-repo-only/manifests/regcred.yaml

.gitmodules

@@ -1,3 +1,3 @@
-[submodule "pulumi/aws/sirius/src"]
-	path = pulumi/aws/sirius/src
-	url = https://github.com/nginxinc/bank-of-sirius.git
+[submodule "pulumi/python/kubernetes/applications/sirius/src"]
+	path = pulumi/python/kubernetes/applications/sirius/src
+	url = https://github.com/nginxinc/bank-of-sirius.git

README.md

@@ -2,18 +2,27 @@
 
 # NGINX Modern Reference Architectures
 
-This repository has the basics for a common way to deploy and manage a modern app. Over time we'll be building more 
+This repository has the basics for a common way to deploy and manage a modern app. Over time we'll be building more
 example architectures using different deployment models and options including other clouds, and you’ll be able to find
 those from here.
 
+## Nomenclature
+
+Internally, we refer to this project as MARA for Modern Application Reference Architecture. The current repository name
+reflects the humble origins of this project, as it was started for the purpose of allowing users to build custom
+versions of the NGINX Ingress Controller. This went so well we decided to expand it out to the project you are now
+viewing.
+
 ## Modern App Architectures
+
 We define modern app architectures as those driven by four characteristics: *scalability*, *portability*, *resiliency*,
 and *agility*. While of course there are many different aspects of a modern architecture, these are fundamental.
 
 This diagram is an example of what we mean by a modern app architecture:
- ![Modern Apps Architecture Example Diagram](DIAG-NGINX-ModernAppsRefArch-04-web-1024x776.svg)
+![Modern Apps Architecture Example Diagram](./docs/DIAG-NGINX-ModernAppsRefArch-04-web-1024x776.svg)
 
 To satisfy the four key characteristics, many modern app architectures employ:
+
 * platform agnosticism,
 * prioritize OSS,
 * everything defined by Code,
@@ -22,40 +31,44 @@ To satisfy the four key characteristics, many modern app architectures employ:
 * built on containers, and
 * distributed storage.
 
-
 ## What's being built
-To see details on the current state of this project, please see the 
-[readme](https://github.com/nginxinc/kic-reference-architectures/tree/master/pulumi/aws), as this project is under 
-constant development.
+
+To see details on the current state of this project, please see the
+[readme](./pulumi/python/README.md) in the [`pulumi/python`](./pulumi/python) subdirectory. This project is under active
+development, and the current work is using [Pulumi](https://www.pulumi.com/) with Python. Additionally, please see
+[Status and Issues](./docs/status-and-issues.md) for the current state of the project including build status and known
+issues.
 
 This project was started to provide a complete, stealable, easy to deploy, and standalone example of how a modern app
-architecture can be built.  It was driven by the need to be flexible and not require a long list of dependencies to get 
-started.  It needs to provide examples of tooling used to build this sort of architecture in the real world. Most
-importantly, it needs to work. Hopefully this provides a ‘jumping off’ point for someone to build their own infrastructure.
+architecture can be built. It was driven by the need to be flexible and not require a long list of dependencies to get
+started. It needs to provide examples of tooling used to build this sort of architecture in the real world. Most
+importantly, it needs to work. Hopefully this provides a ‘jumping off’ point for someone to build their own
+infrastructure.
 
-Subdirectories contained within the root directory separate reference architectures by infrastructure deployment tooling.
+Subdirectories contained within the root directory separate reference architectures by infrastructure deployment tooling
+with additional subdirectories as needed. For example, Pulumi allows the use of multiple languages for deployment. As we
+have decided to use Python in our first build, there is a `python` subdirectory under the `pulumi` directory.
 
 ## Deployment Tools
 
 ### Pulumi
 
-[Pulumi](https://www.pulumi.com/) is an infrastructure as code tool that
-allows you to write code (node, Python, Go, etc) that defines cloud
-infrastructure. Within the [`pulumi`](./pulumi) folder are examples for
-building and deploying KIC on different cloud environments.
+[Pulumi](https://www.pulumi.com/) is an infrastructure as code tool that allows you to write code (node, Python, Go,
+etc) that defines cloud infrastructure. Within the [`pulumi`](./pulumi) folder are examples of the pulumi being used to
+stand up MARA.
 
 ## Contribution
 
 We welcome PRs and issues!
 
-Please refer to the [Contributing Guidelines](CONTRIBUTING.md) when doing a
-pull request.  
+Please refer to the [Contributing Guidelines](CONTRIBUTING.md) when doing a pull request.
 
 ## License
 
 All code in this repository is licensed under the
 [Apache License v2 license](./LICENSE).
 
-Open source license notices for all projects in this repository can be found [here](https://app.fossa.com/reports/92595e16-c0b8-4c68-8c76-59696b6ac219).
+Open source license notices for all projects in this repository can be
+found [here](https://app.fossa.com/reports/92595e16-c0b8-4c68-8c76-59696b6ac219).
 
 [![FOSSA Status](https://app.fossa.com/api/projects/custom%2B5618%2Fgit%40github.com%3Anginxinc%2Fkic-reference-architectures.git.svg?type=large)](https://app.fossa.com/projects/custom%2B5618%2Fgit%40github.com%3Anginxinc%2Fkic-reference-architectures.git?ref=badge_large)

bin/destroy.sh

@@ -0,0 +1,76 @@
+#!/usr/bin/env bash
+
+set -o errexit  # abort on nonzero exit status
+set -o nounset  # abort on unbound variable
+set -o pipefail # don't hide errors within pipes
+
+# Don't pollute console output with upgrade notifications
+export PULUMI_SKIP_UPDATE_CHECK=true
+# Run Pulumi non-interactively
+export PULUMI_SKIP_CONFIRMATIONS=true
+
+script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
+
+if ! command -v pulumi >/dev/null; then
+  if [ -x "${script_dir}/venv/bin/pulumi" ]; then
+    echo "Adding to [${script_dir}/venv/bin] to PATH"
+    export PATH="${script_dir}/venv/bin:$PATH"
+  fi
+fi
+
+if ! command -v pulumi >/dev/null; then
+  echo >&2 "Pulumi must be installed to continue"
+  exit 1
+fi
+
+if ! command -v python3 >/dev/null; then
+  echo >&2 "Python 3 must be installed to continue"
+  exit 1
+fi
+
+# Check to see if the user is logged into Pulumi
+if ! pulumi whoami --non-interactive >/dev/null 2>&1; then
+  pulumi login
+
+  if ! pulumi whoami --non-interactive >/dev/null 2>&1; then
+    echo >&2 "Unable to login to Pulumi - exiting"
+    exit 2
+  fi
+fi
+
+echo " "
+echo "Notice! This shell script will read the config/environment file to determine which pulumi stack to destroy."
+echo "Based on the type of stack it will either run the ./bin/destroy_kube.sh or the ./bin/destroy_aws.sh script."
+echo "If this is not what you want to do, please abort the script by typing ctrl-c and running the appropriate "
+echo "script manually."
+echo " "
+
+# Sleep so we are seen...
+sleep 5
+
+source "${script_dir}/../config/pulumi/environment"
+echo "Configuring all Pulumi projects to use the stack: ${PULUMI_STACK}"
+
+#
+# Determine what destroy script we need to run
+#
+if pulumi config get kubernetes:infra_type -C ${script_dir}/../pulumi/python/config>/dev/null 2>&1; then
+  INFRA="$(pulumi config get kubernetes:infra_type -C ${script_dir}/../pulumi/python/config)"
+  if [ $INFRA == 'AWS' ]; then
+    echo "Destroying an AWS based stack; if this is not right please type ctrl-c to abort this script."
+    sleep 5
+    ${script_dir}/destroy_aws.sh
+    exit 0
+  elif [ $INFRA == 'kubeconfig' ]; then
+    echo "Destroying a kubeconfig based stack; if this is not right please type ctrl-c to abort this script."
+    sleep 5
+    ${script_dir}/destroy_kube.sh
+    exit 0
+  else
+    print "No infrastructure set in config file; aborting!"
+    exit 1
+  fi
+else
+  print "No infrastructure set in config file; aborting!"
+  exit 2
+fi