doc: improvement in one-click-dev.md and some changes in dockerFile

Author: Rahul Kumar Yadav

Dockerfile

@@ -31,13 +31,12 @@ RUN pip install -r /tmp/requirements.txt --no-cache-dir --ignore-installed
 COPY --chown=app:root . .
 
 FROM base as test
-ENTRYPOINT [ "python" ]
-CMD [ "-m", "pytest", "-v", "--cov", "--disable-warnings" ]
+CMD ["python" "-m", "pytest", "-v", "--cov", "--disable-warnings" ]
 
 
 FROM base as celery
-ENTRYPOINT [ "python" ]
-CMD ["-m","celery", "-A", "BrowserStackAutomation", "worker" ,"-n","worker1" ,"-l","DEBUG"]
+COPY ./docker-entrypoint-celery.sh /tmp/entrypoint-celery.sh
+ENTRYPOINT ["/tmp/entrypoint-celery.sh"]
 
 FROM base as web
 COPY ./docker-entrypoint.sh /tmp/entrypoint.sh

Makefile

@@ -5,7 +5,7 @@ all: build test lint
 ## make dev : Build and start docker containers - (web/test/db)
 .PHONY: dev
 dev:
-	@export DJANGO_SETTINGS_MODULE=BrowserStackAutomation.settings && docker-compose build && docker-compose up -d
+	@docker-compose build && docker-compose up -d
 
 ## make build : Build and start docker containers - (web and db)
 .PHONY: build

config.json.sample

@@ -9,7 +9,7 @@
   },
   "access_modules": {
     "git_urls": [
-      "https://<user>:<github-token>@github.com/browserstack/enigma-public-access-modules.git"
+      "https://github.com/browserstack/enigma-public-access-modules.git"
     ]
   },
   "enigmaGroup": {

docker-entrypoint-celery.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+python manage.py createcachetable
+python manage.py migrate        # Apply database migrations
+python manage.py collectstatic --clear --noinput # clearstatic files
+python manage.py collectstatic --noinput  # collect static files
+
+# directory for gunicorn logs and django app logs
+mkdir -p /ebs/logs
+touch /ebs/logs/bstack.log
+tail -n 0 -f /ebs/logs/bstack.log &
+
+# run scripts/clone_access_modules.py to clone access modules
+python scripts/clone_access_modules.py
+pip install -r Access/access_modules/requirements.txt --no-cache-dir --ignore-installed
+
+echo Starting celery
+python3 -m celery -A BrowserStackAutomation worker -n worker1 -l DEBUG

docker-entrypoint.sh

@@ -11,11 +11,10 @@ touch /ebs/logs/bstack.log
 tail -n 0 -f /ebs/logs/bstack.log &
 
 # run scripts/clone_access_modules.py to clone access modules
+
 python scripts/clone_access_modules.py
-python scripts/make_config_and_readme.py
 pip install -r Access/access_modules/requirements.txt --no-cache-dir --ignore-installed
-python scripts/github_test.py
-python scripts/merge_config.py
+
 
 echo Starting Django runserver.
 python manage.py runserver 0.0.0.0:8000

docs/one-click-dev.md

@@ -1,14 +1,17 @@
-# one-click-dev-setup
+# Setup for Local Development
 
+## Pre-requisistes
 
-### Pre-requisistes
+### Install Docker
 
-- Install Docker
-```
-brew install docker docker-compose
-```
+- For macOS(brew)
+    ```
+    brew install docker docker-compose
+    ```
+-  For windows/linux/macOS refer to
+  https://docs.docker.com/get-docker/ 
 
-- Install Docker Container Runtime
+### Install Docker Container Runtime
 https://github.com/abiosoft/colima
 ```bash
 brew install colima
@@ -22,7 +25,7 @@ git clone https://github.com/browserstack/enigma-public-central.git
 ```
 1. Create .env file from .env.sample file. Edit the DOCKERIMAGE to the latest image URL.
 2. copy config.json.sample to config.json
-```pan
+```bash
 {
   "googleapi": {
     "SOCIAL_AUTH_GOOGLE_OAUTH2_KEY": "",
@@ -34,7 +37,7 @@ git clone https://github.com/browserstack/enigma-public-central.git
   },
   "access_modules": {
     "git_urls": [
-      "https://<user>:<github-token>@github.com/browserstack/enigma-public-access-modules.git"
+      "https://github.com/browserstack/enigma-public-access-modules.git"
     ]
   },
   "enigmaGroup": {
@@ -55,12 +58,45 @@ git clone https://github.com/browserstack/enigma-public-central.git
     }
   }
 }
+```
+### for private repo in git_urls in `config.json`
+```bash
+ "https://<git-username>:<github-token>@github.com/browserstack/enigma-public-access-modules.git"
+```
+where `github-token` is a pat token from https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token
 
-replace this values in config.json
-celery-broker-url= "redis://localhost:6379"
-celery-result-backend-url = "redis://localhost:6379"
 
+### Add the config of broker and database in `config.json`
+-  For self-hosted celery
+    ```bash
+    celery-broker-url="redis://localhost:6379/0"
+    celery-result-backend-url="redis://localhost:6379/0"
+    ```
+-  For self-hosted databases
+      ```bash
+    celery-broker-url="redis://localhost:6379/0"
+    celery-result-backend-url="db+mysql://root@localhost:3306/<name_of_db>"
+      ```
+### Add the config of custom access-module repos `config.json`
+
+```bash
+   "access_modules": {
+    ....
+
+        "aws_access": {
+            "aws_accounts": [
+                {
+                    "account": "<your-account>",
+                    "access_key_id": "<access_key_id>",
+                    "secret_access_key": "<secret_access_key>"
+                }
+            ]
+        },
+        .....
+    },
 ```
+Add configuration for all access_module as here added for aws_access.
+
 3. Start the service
 ```bash
 make dev
@@ -69,5 +105,3 @@ make dev
 ```bash
 make logs
 ```
-
-

scripts/clone_access_modules.py

@@ -17,48 +17,44 @@
         folder_name = url.split("/").pop()[:-4]
         folder_path = "./Access/access_modules/" + folder_name
         try:
-               Repo.clone_from(url, folder_path)
-                # move all folders, not files in the cloned repo to the access_modules
-                # folder except the .git, .github and secrets folder
-               for file in os.listdir(folder_path):
-                    if (
+            Repo.clone_from(url, folder_path)
+            # move all folders, not files in the cloned repo to the access_modules
+            # folder except the .git, .github and secrets folder
+            for file in os.listdir(folder_path):
+                if (
                         os.path.isdir(folder_path + "/" + file)
                         and file != ".git"
                         and file != ".github"
                         and file != "secrets"
                     ) :
-                            os.rename(
-                                folder_path + "/" + file, "./Access/access_modules/" + file
-                            )
+                        os.rename(
+                            folder_path + "/" + file, "./Access/access_modules/" + file
+                        )
 
-                    if(file == "requirements.txt"):
-
-                       current_requirements_file = folder_path + "/" + file
-                       #Read the requirements
-                       with open(requirements_file, 'r') as f1:
+                if(file == "requirements.txt"):
+                    current_requirements_file = folder_path + "/" + file
+                    #Read the requirements
+                    with open(requirements_file, 'r') as f1:
                              requirements1 = f1.readlines()
 
-                       with open(current_requirements_file, 'r') as f1:
+                    with open(current_requirements_file, 'r') as f1:
                              requirements2 = f1.readlines()
 
-                       # Merge the requirements
-                       merged_requirements = list(set(requirements1 + requirements2))
+                    # Merge the requirements
+                    merged_requirements = list(set(requirements1 + requirements2))
 
-                       #update the requirements.txt
-                       with open(requirements_file, 'w') as out_file:
+                    #update the requirements.txt
+                    with open(requirements_file, 'w') as out_file:
                         for requirement in sorted(merged_requirements):
                              out_file.write(requirement)
 
-
-
-               print("Cloning successful!")
-
+            print("Cloning successful!")
         except Exception as e:
            print(e)
-
         print("failed cloning " + folder_name + ".")
-         # remove the cloned repo folder entirely with all its contents which
-         # includes folders and files using shutil.rmtree()
+
+        # remove the cloned repo folder entirely with all its contents which
+        # includes folders and files using shutil.rmtree()
         # shutil.rmtree() sometimes throws an error on windows,
         # so we use try and except to ignore the error
         try: