back to single background_job.yaml, just pass scale as param

Author: ikreymer

backend/btrixcloud/crawlmanager.py

@@ -168,8 +168,7 @@ async def run_re_add_org_pages_job(
         )
 
     async def run_optimize_pages_job(
-        self,
-        existing_job_id: Optional[str] = None,
+        self, existing_job_id: Optional[str] = None, scale=3
     ) -> str:
         """run job to recalculate storage stats for the org"""
 
@@ -179,17 +178,14 @@ async def run_optimize_pages_job(
             job_id = f"optimize-pages-{secrets.token_hex(5)}"
 
         return await self._run_bg_job_with_ops_classes(
-            job_id,
-            job_type=BgJobType.OPTIMIZE_PAGES.value,
-            migration_job=True,
+            job_id, job_type=BgJobType.OPTIMIZE_PAGES.value, scale=scale
         )
 
     async def _run_bg_job_with_ops_classes(
         self,
         job_id: str,
         job_type: str,
         oid: Optional[str] = None,
-        migration_job: bool = False,
         **kwargs,
     ) -> str:
         """run background job with access to ops classes"""
@@ -204,9 +200,7 @@ async def _run_bg_job_with_ops_classes(
         if oid:
             params["oid"] = oid
 
-        template = "migration_job.yaml" if migration_job else "background_job.yaml"
-
-        data = self.templates.env.get_template(template).render(params)
+        data = self.templates.env.get_template("background_job.yaml").render(params)
 
         await self.create_from_yaml(data, namespace=DEFAULT_NAMESPACE)
 

chart/app-templates/background_job.yaml

@@ -5,13 +5,18 @@ metadata:
   labels:
     role: "background-job"
     job_type: {{ job_type }}
+{% if oid %}
     btrix.org: {{ oid }}
+{% endif %}
 
 spec:
   ttlSecondsAfterFinished: 90
   backoffLimit: 3
   template:
     spec:
+  {% if scale %}
+      parallelism: {{ scale }}
+  {% endif %}
       restartPolicy: Never
       priorityClassName: bg-job
       podFailurePolicy:
@@ -35,9 +40,10 @@ spec:
           - name: BG_JOB_TYPE
             value: {{ job_type }}
 
+{% if oid %}
           - name: OID
             value: {{ oid }}
-
+{% endif %}
           - name: CRAWL_TYPE
             value: {{ crawl_type }}