Merge pull request #2420 from IFRCGo/feature/s3-storage

Feature/s3 storage

Author: szabozoltan69

.github/workflows/build-publish-docker-helm.yml

@@ -42,7 +42,7 @@ jobs:
     - name: 🐳 Helm dependency
       working-directory: deploy/helm/ifrcgo-helm
       run: |
-        yq --indent 0 '.dependencies | map(["helm", "repo", "add", .name, .repository] | join(" ")) | .[]' ./helm/Chart.lock  | sh --
+        yq --indent 0 '.dependencies | map(["helm", "repo", "add", .name, .repository] | join(" ")) | .[]' ./Chart.lock | sh --
         helm dependency build ./
 
     - name: Tag docker image in Helm Chart values.yaml

.github/workflows/ci.yml

@@ -136,8 +136,16 @@ jobs:
       - name: Install Helm
         uses: azure/setup-helm@v4
 
+      - name: 🐳 Helm dependency
+        working-directory: deploy/helm/ifrcgo-helm
+        run: |
+          yq --indent 0 '.dependencies | map(["helm", "repo", "add", .name, .repository] | join(" ")) | .[]' ./Chart.lock | sh --
+          helm dependency build ./
+
       - name: 🐳 Helm lint
-        run: helm lint deploy/helm/ifrcgo-helm --values ./deploy/helm/ifrcgo-helm/values-staging.yaml
+        working-directory: deploy/helm/ifrcgo-helm
+        run: helm lint ./ --values ./values-staging.yaml
 
       - name: 🐳 Helm template
-        run: helm template deploy/helm/ifrcgo-helm --values ./deploy/helm/ifrcgo-helm/values-staging.yaml
+        working-directory: deploy/helm/ifrcgo-helm
+        run: helm template ./ --values ./values-staging.yaml

README.md

@@ -293,3 +293,7 @@ To import GEC codes along with country ids, run `python manage.py import-gec-cod
 ## SSO setup
 
 For more info checkout [GO-SSO](./docs/go-sso.md)
+
+## Playwright exports
+
+For more info checkout [Playwright exports](./docs/playwright-exports.md)

api/admin.py

@@ -1044,5 +1044,5 @@ class ExportTokenAdmin(admin.ModelAdmin):
 admin.site.register(models.CountryOfFieldReportToReview, CountryOfFieldReportToReviewAdmin)
 # admin.site.register(Revision, RevisionAdmin)
 
-admin.site.site_url = "https://" + settings.FRONTEND_URL
+admin.site.site_url = settings.GO_WEB_URL
 admin.widgets.RelatedFieldWidgetWrapper.template_name = "related_widget_wrapper.html"

api/management/commands/cron_job_monitor.py

@@ -36,7 +36,7 @@ def handle(self, *args, **options):
         project_id = parsed_url.path.strip("/")
         api_key = parsed_url.username
 
-        SENTRY_INGEST = f"https://{parsed_url.hostname}"
+        SENTRY_INGEST = f"{parsed_url.scheme}://{parsed_url.hostname}"
 
         for cronjob in SentryMonitor.choices:
             job, schedule = cronjob

api/management/commands/index_and_notify.py

@@ -264,8 +264,8 @@ def get_admin_uri(self, record, rtype):
             RecordType.SURGE_DEPLOYMENT_MESSAGES: "deployments/personneldeployment",
             RecordType.SURGE_ALERT: "notifications/surgealert",
         }[rtype]
-        return "https://%s/admin/%s/%s/change" % (
-            settings.BASE_URL,
+        return "%s/admin/%s/%s/change" % (
+            settings.GO_API_URL,
             admin_page,
             record.id,
         )
@@ -960,8 +960,8 @@ def check_ingest_issues(self, having_ingest_issue):
                 (
                     "Ingest issue(s) occured, one of them is "
                     + ingestor_name
-                    + ", via CronJob log record id: https://"
-                    + settings.BASE_URL
+                    + ", via CronJob log record id: "
+                    + settings.GO_API_URL
                     + "/admin/api/cronjob/"
                     + str(ingest_issue_id)
                     + ". Please fix it ASAP."

api/serializers.py

@@ -2436,9 +2436,9 @@ def create(self, validated_data):
             title = "Export"
         user = self.context["request"].user
         if export_type == Export.ExportType.PER:
-            validated_data["url"] = f"https://{settings.FRONTEND_URL}/countries/{country_id}/{export_type}/{export_id}/export/"
+            validated_data["url"] = f"{settings.GO_WEB_INTERNAL_URL}/countries/{country_id}/{export_type}/{export_id}/export/"
         else:
-            validated_data["url"] = f"https://{settings.FRONTEND_URL}/{export_type}/{export_id}/export/"
+            validated_data["url"] = f"{settings.GO_WEB_INTERNAL_URL}/{export_type}/{export_id}/export/"
 
         # Adding is_pga to the url
         is_pga = validated_data.pop("is_pga")

api/tasks.py

@@ -12,10 +12,11 @@
 from playwright.sync_api import sync_playwright
 from rest_framework.authtoken.models import Token
 
+from main.utils import logger_context
+
 from .logger import logger
 from .models import Export
-
-# from .utils import DebugPlaywright
+from .utils import DebugPlaywright
 
 
 def build_storage_state(tmp_dir, user, token):
@@ -25,7 +26,7 @@ def build_storage_state(tmp_dir, user, token):
     state = {
         "origins": [
             {
-                "origin": "https://" + settings.FRONTEND_URL + "/",
+                "origin": settings.GO_WEB_INTERNAL_URL + "/",
                 "localStorage": [
                     {
                         "name": "user",
@@ -51,9 +52,10 @@ def build_storage_state(tmp_dir, user, token):
 
 @shared_task
 def generate_url(url, export_id, user, title):
-    export = Export.objects.filter(id=export_id).first()
-    user = User.objects.filter(id=user).first()
+    export = Export.objects.get(id=export_id)
+    user = User.objects.get(id=user)
     token = Token.objects.filter(user=user).last()
+
     footer_template = """
         <div class="footer" style="width: 100%;font-size: 8px;color: #FEFEFE; bottom: 10px; position: absolute;">
         <div style="float: left; margin-top: 10px; margin-left: 40px;">
@@ -80,6 +82,7 @@ def generate_url(url, export_id, user, title):
         </div>
         </div>
     """  # noqa: E501
+
     try:
         with tempfile.TemporaryDirectory() as tmp_dir:
             with sync_playwright() as p:
@@ -103,7 +106,8 @@ def generate_url(url, export_id, user, title):
                 )
                 context = browser.new_context(storage_state=storage_state)
                 page = context.new_page()
-                # DebugPlaywright.debug(page)
+                if settings.DEBUG_PLAYWRIGHT:
+                    DebugPlaywright.debug(page)
                 timeout = 300000
                 page.goto(url, timeout=timeout)
                 time.sleep(5)
@@ -131,7 +135,11 @@ def generate_url(url, export_id, user, title):
                     "completed_at",
                 ]
             )
-    except Exception as e:
-        logger.error(e)
+    except Exception:
+        logger.error(
+            f"Failed to export PDF: {export.export_type}",
+            exc_info=True,
+            extra=logger_context(dict(export_id=export.pk)),
+        )
         export.status = Export.ExportStatus.ERRORED
         export.save(update_fields=["status"])

api/templates/admin/base_site.html

@@ -8,7 +8,7 @@
 
 <h1 id="site-name">
     <a href="{% url 'admin:index' %}">
-        <img height="40" src="/static/images/logo/go-logo-2020-6cdc2b0c.svg">
+        <img height="40" src="{% static "images/logo/go-logo-2020-6cdc2b0c.svg" %}" />
     </a>
     {% if HAVING_INGEST_ISSUE and request.user.is_superuser %}
         <span title="Ingest issue – please check erroneous api/CronJob item">

api/views.py

@@ -958,9 +958,9 @@ def post(self, request):
 
                 # Construct and re-send the email
                 email_context = {
-                    "confirmation_link": "https://%s/verify_email/?token=%s&user=%s"
+                    "confirmation_link": "%s/verify_email/?token=%s&user=%s"
                     % (
-                        settings.BASE_URL,  # on PROD it should point to goadmin...
+                        settings.GO_API_URL,  # on PROD it should point to goadmin...
                         pending_user.token,
                         username,
                     )