Merge pull request #77 from browserstack/one-click

CI: One click

Author: vedharish

Access/migrations/0003_useraccessmapping_fail_reason_and_more.py

@@ -0,0 +1,24 @@
+# Generated by Django 4.1.3 on 2023-02-19 12:26
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('Access', '0002_useridentity_delete_gitacces_remove_user_gitusername_and_more'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='useraccessmapping',
+            name='fail_reason',
+            field=models.TextField(blank=True, null=True),
+        ),
+        migrations.AlterField(
+            model_name='groupaccessmapping',
+            name='group',
+            field=models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='group_access_mapping', to='Access.groupv2'),
+        ),
+    ]

Access/models.py

@@ -272,8 +272,8 @@ def update_revoker(self, revoker):
         self.save()
 
     def offboard(self, revoker):
-        self.state = "offboarding"
-        self.revoker = revoker
+        self.change_state("offboarding")
+        self.update_revoker(revoker)
         self.offbaord_date = datetime.datetime.now()
         self.user.is_active = False
         self.save()

Dockerfile

@@ -5,6 +5,7 @@
 # Set the base image to use to Ubuntu
 FROM python:3.11-slim-buster AS base
 
+ENV DJANGO_SETTINGS_MODULE=BrowserStackAutomation.settings 
 RUN DEBIAN_FRONTEND=noninteractive \
   apt-get update -y \
   && apt-get install --no-install-recommends -y \
@@ -14,13 +15,14 @@ RUN DEBIAN_FRONTEND=noninteractive \
 
 # Set env variables used in this Dockerfile (add a unique prefix, such as DEV)
 RUN apt update && apt install -y netcat dnsutils libmariadbclient-dev
-
 RUN useradd -rm -d /home/app -s /bin/bash -g root -G sudo -u 1001 app
 WORKDIR /srv/code/dev
 RUN mkdir -p logs
 RUN chown -R app /srv/code/dev
 USER app
 
+
+
 # Copy just requirements.txt
 COPY requirements.txt /tmp/requirements.txt
 COPY config.json.sample config.json
@@ -31,8 +33,7 @@ RUN pip install -r /tmp/requirements.txt --no-cache-dir --ignore-installed
 COPY --chown=app:root . .
 
 FROM base as test
-ENTRYPOINT [ "python" ]
-CMD [ "-m", "pytest", "-v", "--cov", "--disable-warnings" ]
+CMD ["python" "-m", "pytest", "-v", "--cov", "--disable-warnings" ]
 
 FROM base as web
 COPY ./docker-entrypoint.sh /tmp/entrypoint.sh

docker-compose.yml

@@ -3,28 +3,21 @@ version: '3'
 services:
   web:
     container_name: dev
-    image: ${DOCKERIMAGE}
     build:
       context: .
       dockerfile: Dockerfile
       target: web
     env_file:
     - ./secrets/ops_app_dev.env
     volumes:
-    - .:/srv/code/dev
     - ./mounts/logs/:/ebs/logs/
     ports:
     - 8000:8000
     depends_on:
     - db
     command: >
-      /bin/bash -c "
-        while ! nc -z db 3306;
-        do
-          echo sleeping;
-          sleep 1;
-        done;
-        echo Connected!;"
+      bash -c "echo Starting Django runserver;
+      python manage.py runserver 0.0.0.0:8000"
   db:
     container_name: db
     image: mysql/mysql-server:8.0.31
@@ -34,6 +27,30 @@ services:
     - ./secrets/ops_mysql_dev.env
     volumes:
     - ./dbs:/var/lib/mysql --socket=/tmp/mysql.sock
+  redis:
+    container_name: redis
+    image: redis:alpine
+    command: --port 6379
+    ports:
+      - "6379:6379"
+  celery:
+    container_name: celery
+    build:
+      context: .
+      dockerfile: Dockerfile
+      target: web
+    volumes:
+      - ./:/app/
+    env_file:
+    - ./secrets/ops_app_celery.env
+    depends_on:
+      - db
+      - redis
+      - web
+    command: >
+      bash -c "
+      echo Starting celery;
+      python3 -m celery -A BrowserStackAutomation worker -n worker1 -l DEBUG"
   test:
     container_name: test
     build:

docker-entrypoint.sh

@@ -11,7 +11,8 @@ touch /ebs/logs/bstack.log
 tail -n 0 -f /ebs/logs/bstack.log &
 
 # run scripts/clone_access_modules.py to clone access modules
+
 python scripts/clone_access_modules.py
+pip install -r Access/access_modules/requirements.txt --no-cache-dir --ignore-installed
 
-echo Starting Django runserver.
-python manage.py runserver 0.0.0.0:8000
+eval "$@"

docs/one-click-dev.md

@@ -0,0 +1,107 @@
+# Setup for Local Development
+
+## Pre-requisistes
+
+### Install Docker
+
+- For macOS(brew)
+    ```
+    brew install docker docker-compose
+    ```
+-  For windows/linux/macOS refer to
+  https://docs.docker.com/get-docker/ 
+
+### Install Docker Container Runtime
+https://github.com/abiosoft/colima
+```bash
+brew install colima
+colima start
+```
+
+### Setup
+Clone the repo in local system
+```bash
+git clone https://github.com/browserstack/enigma-public-central.git
+```
+1. Create .env file from .env.sample file. Edit the DOCKERIMAGE to the latest image URL.
+2. copy config.json.sample to config.json
+```bash
+{
+  "googleapi": {
+    "SOCIAL_AUTH_GOOGLE_OAUTH2_KEY": "",
+    "SOCIAL_AUTH_GOOGLE_OAUTH2_SECRET": "",
+    "SOCIAL_AUTH_GOOGLE_OAUTH2_WHITELISTED_DOMAINS": ""
+  },
+  "database": {
+    "engine": "sqlite3"
+  },
+  "access_modules": {
+    "git_urls": [
+      "https://github.com/browserstack/enigma-public-access-modules.git"
+    ]
+  },
+  "enigmaGroup": {
+    "MAIL_APPROVER_GROUPS": [
+      "[email protected]"
+    ]
+  },
+  "emails": {
+    "access-approve": "<access-approve-email>"
+  },
+  "background_task_manager": {
+    "type": "celery",
+    "config": {
+      "broker": "<celery-broker-url>",
+      "backend": "<celery-result-backend-url>",
+      "need_monitoring": true,
+      "monitoring_apps": "django_celery_results"
+    }
+  }
+}
+```
+### for private repo in git_urls in `config.json`
+```bash
+ "https://<git-username>:<github-token>@github.com/browserstack/enigma-public-access-modules.git"
+```
+where `github-token` is a pat token from https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token
+
+
+### Add the config of broker and database in `config.json`
+-  For self-hosted celery
+    ```bash
+    celery-broker-url="redis://localhost:6379/0"
+    celery-result-backend-url="redis://localhost:6379/0"
+    ```
+-  For self-hosted databases
+      ```bash
+    celery-broker-url="redis://localhost:6379/0"
+    celery-result-backend-url="db+mysql://root@localhost:3306/<name_of_db>"
+      ```
+### Add the config of custom access-module repos `config.json`
+
+```bash
+   "access_modules": {
+    ....
+
+        "aws_access": {
+            "aws_accounts": [
+                {
+                    "account": "<your-account>",
+                    "access_key_id": "<access_key_id>",
+                    "secret_access_key": "<secret_access_key>"
+                }
+            ]
+        },
+        .....
+    },
+```
+Add configuration for all access_module as here added for aws_access.
+
+3. Start the service
+```bash
+make dev
+```
+4. Check logs with
+```bash
+make logs
+```

requirements.txt

@@ -118,5 +118,7 @@ eradicate==2.1.0
 mypy==1.0.0
 mypy-extensions==1.0.0
 vulture==2.7
+google-api-python-client==2.78.0
+google-auth-httplib2==0.1.0
 django-cid==2.3
-flake8==6.0.0
+flake8==6.0.0

scripts/clone_access_modules.py

@@ -8,42 +8,64 @@
     f = open("./config.json", "r")
     config = json.load(f)
     urls = config["access_modules"]["git_urls"]
+
+    requirements_file = 'Access/access_modules/requirements.txt'
+    if not os.path.exists(requirements_file):
+          open(requirements_file, 'w').close()
+
     for url in urls:
         folder_name = url.split("/").pop()[:-4]
         folder_path = "./Access/access_modules/" + folder_name
         try:
-            if os.path.exists(folder_path):
-                Repo(folder_path).remotes.origin.pull()
-            else:
-                Repo.clone_from(url, folder_path)
-                # move all folders, not files in the cloned repo to the access_modules
-                # folder except the .git, .github and secrets folder
-                for file in os.listdir(folder_path):
-                    if (
+            Repo.clone_from(url, folder_path)
+            # move all folders, not files in the cloned repo to the access_modules
+            # folder except the .git, .github and secrets folder
+            for file in os.listdir(folder_path):
+                if (
                         os.path.isdir(folder_path + "/" + file)
                         and file != ".git"
                         and file != ".github"
                         and file != "secrets"
-                    ):
+                    ) :
+                    try :
                         os.rename(
                             folder_path + "/" + file, "./Access/access_modules/" + file
                         )
+                    except:
+                         print("File is already present.")
+                            
+                if(file == "requirements.txt"):
+                    current_requirements_file = folder_path + "/" + file
+                    #Read the requirements
+                    with open(requirements_file, 'r') as f1:
+                             requirements1 = f1.readlines()
 
-                # remove the cloned repo folder entirely with all its contents which
-                # includes folders and files using shutil.rmtree()
-                # shutil.rmtree() sometimes throws an error on windows,
-                # so we use try and except to ignore the error
-                try:
-                    shutil.rmtree(folder_path)
-                except Exception as e:
-                    print(e)
-                    print("failed to remove " + folder_path + " folder.")
+                    with open(current_requirements_file, 'r') as f1:
+                             requirements2 = f1.readlines()
 
+                    # Merge the requirements
+                    merged_requirements = list(set(requirements1 + requirements2))
+                       
+                    #update the requirements.txt
+                    with open(requirements_file, 'w') as out_file:
+                        for requirement in sorted(merged_requirements):
+                             out_file.write(requirement)
+                       
             print("Cloning successful!")
+        except Exception as e:
+           print("error-->",e)
+           print("failed cloning " + folder_name + ".")
 
+        # remove the cloned repo folder entirely with all its contents which
+        # includes folders and files using shutil.rmtree()
+        # shutil.rmtree() sometimes throws an error on windows,
+        # so we use try and except to ignore the error
+        try:
+            shutil.rmtree(folder_path)
         except Exception as e:
             print(e)
-            print("failed cloning " + folder_name + ".")
+            print("failed to remove " + folder_path + " folder.")
+        
 except Exception as e:
     print("Access module cloning failed!")
     print(str(e))

secrets/ops_app_celery.env

@@ -0,0 +1,6 @@
+CELERY_BROKER_URL=redis://redis:6379/0
+CELERY_RESULT_BACKEND=redis://redis:6379/0
+C_FORCE_ROOT=true
+MYSQL_ROOT_PASSWORD=testtest
+MYSQL_DATABASE=enigma
+MYSQL_ROOT_HOST=%