Try to push to ecr

Author: singhals

.github/workflows/deploy.yml

@@ -2,108 +2,35 @@ name: Deploy
 on:
   push:
     branches:
-      - main
-env:
-  TF_CLOUD_ORGANIZATION: "shughesuk"
-  CONFIG_DIRECTORY: "./"
+      - singhals/add-some-cube
+permissions:
+  id-token: write # This is required for requesting the JWT
+  contents: read # This is required for actions/checkout
 jobs:
-  deploy-backend:
-    outputs:
-      sha: ${{ steps.short_sha.outputs.sha }}
-    environment: backend-production
-    concurrency: backend-production
-    permissions:
-      id-token: write
-      contents: read
+  deploy-cube:
     runs-on: ubuntu-latest
     steps:
-      - name: Checkout code
+      - name: Checkout repo
         uses: actions/checkout@v4
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-      - name: Login to Docker Hub
-        uses: docker/login-action@v3
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v4
         with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: Get short SHA
-        id: short_sha
-        run: echo "sha=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
-      - name: Build and push Docker image
-        uses: docker/build-push-action@v5
+          role-to-assume: arn:aws:iam::471881062455:role/system/github_actions_role
+          role-session-name: GitHub_to_AWS_sync_svc_cube
+          aws-region: us-east-1
+
+      - name: Login to Amazon ECR
+        id: login-ecr
+        uses: aws-actions/[email protected]
         with:
-          context: .
-          push: true
-          tags: shughesuk/backend:${{ steps.short_sha.outputs.sha }}
-  run-migrations:
-    name: "Run Migrations"
-    runs-on: ubuntu-latest
-    needs: deploy-backend
-    permissions:
-      contents: read
-      id-token: write
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-      - name: configure aws credentials
-        uses: aws-actions/[email protected]
-        with:
-          role-to-assume: arn:aws:iam::905418398753:role/github-actions-role
-          role-session-name: GitHub_to_AWS_via_FederatedOIDC
-          aws-region: "us-east-1"
-      - uses: prefix-dev/[email protected]
-        with:
-          cache: true
-          locked: true
-      - name: Update task
-        run: pixi run python scripts/update_task.py --task-definition production --container-name backend-api --image shughesuk/backend:${{ needs.deploy-backend.outputs.sha }}
-      - name: Run migrations
-        run: pixi run python scripts/run_task.py --task-definition production --cluster production --command "pixi run python manage.py migrate"
-  terraform:
-    needs:
-      - deploy-backend
-      - run-migrations
-    name: "Terraform Apply"
-    runs-on: ubuntu-latest
-    permissions:
-      contents: read
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-      - name: terraform-apply
-        uses: dflook/[email protected]
-        with:
-          path: ./terraform
-          auto_approve: true
-          workspace: resume-workspace
-          variables: |
-            app_image = "shughesuk/backend:${{ needs.deploy-backend.outputs.sha }}"
+          mask-password: "false"
+
+      - name: Build, tag, and push docker image to Amazon ECR
         env:
-          TERRAFORM_CLOUD_TOKENS: app.terraform.io=${{ secrets.TF_API_TOKEN }}
-  deploy-frontend:
-    needs: terraform
-    environment: frontend-production
-    concurrency: frontend-production
-    permissions:
-      id-token: write
-      contents: read
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-      - name: configure aws credentials
-        uses: aws-actions/[email protected]
-        with:
-          role-to-assume: arn:aws:iam::905418398753:role/github-actions-role
-          role-session-name: GitHub_to_AWS_via_FederatedOIDC
-          aws-region: "us-east-1"
-      - uses: prefix-dev/[email protected]
-        env:
-          ACTIONS_STEP_DEBUG: true
-        with:
-          cache: true
-          locked: true
-      - name: Deploy
-        run: pixi run frontend-deploy
+          REGISTRY: ${{ steps.login-ecr.outputs.registry }}
+          REPOSITORY: sync-svc-cube-prod
+          IMAGE_TAG: ${{ github.sha }}
+        run: |
+          docker build -t $REGISTRY/$REPOSITORY:$IMAGE_TAG .
+          docker push $REGISTRY/$REPOSITORY:$IMAGE_TAG

Dockerfile

@@ -1,8 +1,6 @@
-FROM ghcr.io/prefix-dev/pixi:0.18.0-bookworm-slim
+FROM cubejs/cube:v1.1.9
 
-COPY ./backend /opt/backend
-COPY ./pixi.lock /opt/backend/pixi.lock
-COPY ./pixi.toml /opt/backend/pixi.toml
-WORKDIR /opt/backend/
-RUN pixi install
-CMD pixi run uvicorn --port 80 --host 0.0.0.0 resume.asgi:application --log-config logging.yaml
+COPY cube.js cube.js
+COPY fetch.js fetch.js
+RUN mkdir model
+COPY model/ model/

cube.js

@@ -0,0 +1,145 @@
+const fetchUniqueTenants = require("./fetch").fetchUniqueTenants;
+const fetch = require("node-fetch");
+const tenantIdClaim = "https://synccomputing.com/sync_tenant_id";
+const tenantIdOverrideClaim = "https://synccomputing.com/sync_tenant_id_override";
+const cubeBasePath = "/sync-query";
+const port = process.env.PORT || 4000;
+
+exports.logger = (message, params) => {
+  console.log(JSON.stringify({ message, params }));
+};
+
+exports.telemetry = false;
+exports.basePath = cubeBasePath;
+exports.http = {
+  "cors": {
+    "allowedHeaders": ["*"],
+  }
+};
+exports.scheduledRefreshTimer = 60 * 60 * 24; // this refreshs our data models every 24 hours
+
+
+exports.contextToAppId = ({ securityContext }) => {
+  const syncTenantId = securityContext[tenantIdOverrideClaim] || 
+                         securityContext[tenantIdClaim];
+
+  if (!syncTenantId) {
+      throw new Error("You shall not pass! 🧙");
+  }
+
+  return `tenant_${syncTenantId}`;
+};
+
+exports.extendContext = (req) => {
+  return {
+    securityContext: {
+      ...req.securityContext,
+      token: req.headers.authorization,    
+    }
+  }
+}
+
+exports.scheduledRefreshContexts = async () => {
+  console.log("Running refresh contexts");
+  const uniqueTenants = await fetchUniqueTenants();
+  console.log(uniqueTenants);
+  return uniqueTenants;
+};
+
+
+async function getCubeMeta(token) {
+  const CUBEJS_API_URL = `http://localhost:${port}`;
+
+  try {
+    const response = await fetch(`${CUBEJS_API_URL}${cubeBasePath}/v1/meta`, {
+      method: "GET",
+      headers: {
+        Authorization: token,
+      },
+    });
+
+    if (!response.ok) {
+      throw new Error(`HTTP error trying to retrieve cube metadata: ${response.status}`);
+    }
+
+    const metaData = await response.json();
+    return metaData;
+  } catch (err) {
+    console.error("Error fetching cube metadata:", err.message);
+  }
+}
+
+const findAuxiliarySortDimFromCube = (cube, dim) => {
+  const auxDim = cube.dimensions.find( (cubeDim) => {
+    console.debug(`comparing ${cube.name}._sort_${dim} to ${cubeDim.name}`);
+    return `${cube.name}._sort_${dim}` == cubeDim.name;
+  });
+  console.log(`Found aux sort dim ${JSON.stringify(auxDim)}`);
+  return auxDim;
+}
+
+const findAuxiliarySortDim = (cubeOrViewName, dimension, metadata) => {
+  console.debug(`finding aux dims for: ${cubeOrViewName} ${dimension}`);
+
+  let cube = metadata.cubes.find((model) => model.name == cubeOrViewName);
+  if (cube.type == "view") {
+    // get the og cube from the view's dimension
+    const viewDimension = cube.dimensions.find( (cubeDim) => cubeDim.name == `${cubeOrViewName}.${dimension}` );
+    const parts = viewDimension.aliasMember.split(".");
+    const cubeFromAliasMember = parts[0];
+    cube = metadata.cubes.find((model) => model.name == cubeFromAliasMember);
+    dimension = parts[1];
+    console.debug(`Found og cube from view: ${cube.name} ${dimension}`)
+  }
+  
+  const auxSortDim = findAuxiliarySortDimFromCube(cube, dimension);
+
+  return auxSortDim;
+}
+
+const maybeUseAuxilarySortDim = (orderByClause, metadata) => {
+  const orderByPath = orderByClause[0]
+  const orderByDirection = orderByClause[1] // desc vs asc
+  const [cubeOrViewName, dim] = orderByPath.split(".");
+  const auxSortByDim = findAuxiliarySortDim(cubeOrViewName, dim, metadata);
+
+  if (auxSortByDim && orderByDirection == "desc") {
+    // We only need to use an auxilary sort dimension to sort nulls last if descending order.
+    // Ascending order will already sort nulls last.
+    return [auxSortByDim.name, orderByDirection];
+  } else {
+    return orderByClause;
+  }
+};
+
+const replaceOrderBy = (order, metadata) => {
+  const orderByQueries = [];
+  if (order && order.length > 0) {
+    order.forEach((orderByClause) => {
+      let newOrderDim = maybeUseAuxilarySortDim(orderByClause, metadata)
+      if (newOrderDim) {
+        orderByQueries.push(newOrderDim);
+      } else {
+        orderByQueries.push(orderByClause);
+      }
+     
+    });
+  }
+  return orderByQueries;
+};
+
+exports.queryRewrite = async (query, { securityContext }) => {
+  if (query.order && query.order.length > 0 && query.ungrouped) { // we can skip if customer isn't ordering by anything
+    const metadata = await getCubeMeta(securityContext.token);
+    query.order = replaceOrderBy(query.order, metadata);
+    query.order.forEach((orderClause) => {
+      let orderByDimName = orderClause[0];
+      if (!query.dimensions.includes(orderByDimName)) {
+        // We must add the auxiliary sort dimension in order for sorting to work
+        query.dimensions.push(orderByDimName);
+      }
+    });
+    console.log(`Rewritten query: ${JSON.stringify(query)}`)
+  }
+  return query;
+};

docker-compose.yml

@@ -0,0 +1,10 @@
+version: "2.2"
+
+services:
+  cube:
+    image: sync-cube-image
+    ports:
+      - 4000:4000
+      - 15432:15432
+    volumes:
+      - .:/cube/conf

fetch.js

@@ -0,0 +1,70 @@
+const { Pool } = require("pg");
+
+const pool = new Pool({
+  host: process.env.CUBEJS_DB_HOST,
+  port: process.env.CUBEJS_DB_PORT,
+  user: process.env.CUBEJS_DB_USER,
+  password: process.env.CUBEJS_DB_PASS,
+  database: process.env.CUBEJS_DB_NAME,
+  ssl: {
+    rejectUnauthorized: false  // Heroku requires SSL but with this flag off
+  },
+});
+const tenantIdClaim = "https://synccomputing.com/sync_tenant_id";
+
+ 
+exports.fetchUniqueTagKeys = async (sync_tenant_id) => {
+  let client;
+  let tagKeys = [];
+  try {
+    console.log("looking up tags for: ", sync_tenant_id);
+    client = await pool.connect();
+    const uniqueTagKeysQuery = `
+SELECT DISTINCT tag_key
+FROM public.databricks_cluster_tags
+WHERE sync_tenant_id = '${sync_tenant_id}'
+`;
+    const result = await client.query(uniqueTagKeysQuery);
+    // remove special characters from the tag key name so we can expose as a dimension
+    tagKeys = result.rows.map((row) => row.tag_key.replace(/[^a-zA-Z0-9_]/g, '_'));
+  } catch(error) {
+    console.error(error)
+  } finally {
+    if (client) {
+      client.release();
+    }
+  }
+
+  return tagKeys;
+};
+
+exports.fetchUniqueTenants = async () => {
+    console.log("trying to fetch unique tenants")
+    let client;
+    let uniqueTenants = [];
+    try {
+        client = await pool.connect();
+        const uniqueTenantsQuery = `
+SELECT DISTINCT sync_tenant_id
+FROM public.user
+WHERE sync_tenant_id IS NOT NULL and last_login > NOW() - INTERVAL '30 days';
+`;
+        const result = await client.query(uniqueTenantsQuery);
+        console.log(result);
+        uniqueTenants = result.rows.map((row) => {
+            const secContext = { "securityContext": {}};
+            secContext["securityContext"][tenantIdClaim] = row.sync_tenant_id;
+
+            return secContext;
+        });
+    } catch(error) {
+        console.error('Error fetching unique tenants:', error);
+    } finally {
+        if (client) {
+            client.release();
+        }
+    }
+    console.log("Found tenants: " + uniqueTenants)
+    return uniqueTenants
+}
+