update worker deployment

Author: AlexPatrie

.github/workflows/deploy-worker.yml

@@ -10,13 +10,21 @@ jobs:
     steps:
       - name: Checkout code
         uses: actions/checkout@v2
+
       - name: Log in to GHCR
         run: echo "${{ secrets.REPO_ADMIN_GHCR_TOKEN }}" | docker login ghcr.io -u "${{ secrets.REPO_ADMIN_GH_USERNAME }}" --password-stdin
+
+      - name: Create gcloud config
+        run: |
+          echo "$BIOSIMULATIONS_GCLOUD_CONFIG" > ./assets/docker/config/.biosimulations.json
+        env:
+          BIOSIMULATIONS_GCLOUD_CONFIG: ${{ secrets.BIO_JSON_CONTENT }}
+
       - name: Build worker service
         run: |
           WORKER_VERSION=$(python -B .github/parse_container_version.py worker)
           WORKER_IMG="${{ vars.IMG_ROOT }}-worker:$WORKER_VERSION"
-          docker build --no-cache -f ./worker/Dockerfile-worker -t $WORKER_IMG .
+          docker build --no-cache -f ./gateway/Dockerfile-worker -t $WORKER_IMG .
           docker push $WORKER_IMG
           LATEST_IMG="${{ vars.IMG_ROOT }}-worker:latest"
           docker tag $WORKER_IMG $LATEST_IMG

docker-compose.yaml

@@ -19,17 +19,17 @@ services:
     depends_on:
       - mongodb
 
-  # worker:
-  #   platform: linux/amd64
-  #   build:
-  #     context: .
-  #     dockerfile: ./worker/Dockerfile-worker
-  #   image: ghcr.io/biosimulators/compose-worker:0.0.1
-  #   container_name: worker
-  #   networks:
-  #     - compnet
-  #   depends_on:
-  #     - mongodb
+  worker:
+    platform: linux/amd64
+    build:
+      context: .
+      dockerfile: ./worker/Dockerfile-worker
+    image: ghcr.io/biosimulators/compose-worker:0.0.1
+    container_name: worker
+    networks:
+      - compnet
+    depends_on:
+      - mongodb
 
 networks:
   compnet:

environment.yml

@@ -14,6 +14,6 @@ dependencies:
       - cobra
       - copasi-basico
       - smoldyn
-      - tellurium
+      # - tellurium
       - imageio
       - nbformat

kustomize/overlays/compose/kustomization.yaml

@@ -8,7 +8,7 @@ images:
 - name: mongo
   newTag: 8.0-rc
 - name: ghcr.io/biosimulators/compose-gateway
-  newTag: 0.0.4  # stable is 0.0.3
+  newTag: 0.0.5  # stable is 0.0.3
 - name: ghcr.io/biosimulators/bio-compose-server-base
   newTag: 0.0.7-dev
 

worker/dispatch.py

@@ -80,28 +80,22 @@ async def run(self, limit: int = 5, wait: int = 5):
 
     def create_dynamic_environment(self, job: Mapping[str, Any]) -> int:
         # TODO: implement this
-        print(f'Creating dynamic environment for job {job["job_id"]}')
         return 0
 
     async def dispatch_composition(self, job: Mapping[str, Any]):
-        print(f'Registered processes in dispatcher.dispatch: {app_registrar.registered_addresses}')
-
         job_status = job["status"]
         job_id = job["job_id"]
         if job_status.lower() == "pending":
             try:
                 # install simulators required TODO: implement this
                 self.create_dynamic_environment(job)
 
-                # TODO: Now, search job spec for possible file uploads and read them in locally
-
                 # change job status to IN_PROGRESS
-                # await self.db_connector.update_job(job_id=job_id, status="IN_PROGRESS")
+                await self.db_connector.update_job(job_id=job_id, status="IN_PROGRESS")
 
                 # get request params and parse remote file uploads if needed
                 input_state = job["spec"]
                 duration = job.get("duration", 1)
-
                 for process_name, process_spec in input_state.items():
                     print(f'Process {process_name} has spec {process_spec}')
                     process_config = process_spec["config"]
@@ -118,51 +112,39 @@ async def dispatch_composition(self, job: Mapping[str, Any]):
                             local_fp = download_file_from_bucket(source_blob_path=source_fp, out_dir=temp_dest, bucket_name=DEFAULT_BUCKET_NAME)
                             process_spec["config"]["mesh_file"] = local_fp
 
-                print(f'Now the input state is:\n{input_state}')
-                process = SimpleMembraneProcess(config=input_state['membrane']['config'], core=app_registrar.core)
-                print(process)
                 # generate composition instance
                 composition = self.generate_composite(input_state)
-                print(f'Composition state is:\n{composition.state}')
-                # get composition results and state
 
-                # results = self.generate_composition_results(input_state, duration)
+                # get composition results and state
+                results = self.generate_composition_results(input_state, duration)
                 state = self.generate_composition_state(composition)
-                print(f'Composition state is:\n{state}')
-                process = composition.state['membrane']['instance']
-                print(f'Process state is:\n{process.initial_state()}')
-                process.update(process.initial_state(), 1)
-
-                # print(f'Generated results: {results} and state: {state}')
 
                 # change status to complete and write results in DB
-                # await self.db_connector.update_job(
-                #     job_id=job_id,
-                #     status="COMPLETE",
-                #     results=results
-                # )
+                await self.db_connector.update_job(
+                    job_id=job_id,
+                    status="COMPLETE",
+                    results=results
+                )
 
                 # write new result state to states collection
-                # await self.db_connector.write(
-                #     collection_name="result_states",
-                #     job_id=job_id,
-                #     data=state,
-                #     last_updated=self.db_connector.timestamp()
-                # )
+                await self.db_connector.write(
+                    collection_name="result_states",
+                    job_id=job_id,
+                    data=state,
+                    last_updated=self.db_connector.timestamp()
+                )
             except Exception as e:
                 logger.error(f"Exception while dispatching {job_id}: {e}")
                 failed_job = self.generate_failed_job(job_id, str(e))
-                # await self.db_connector.update_job(**failed_job)
+                await self.db_connector.update_job(**failed_job)
 
     def generate_composite(self, input_state) -> Composite:
-        print(f'Got the input state in generate_composite:\n{input_state}')
         return Composite(
             config={"state": input_state},
             core=app_registrar.core
         )
 
     def generate_composition_results(self, composition: Composite, duration: int) -> ResultData:
-        print(f'Got the composition before results\n{composition.state}')
         # run the composition
         composition.run(duration)