Merge pull request #174 from bettersg/stg

Stg

Author: yevkim

.github/workflows/deploy_functions_dev.yml

@@ -14,13 +14,12 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       # Step 1: Checkout the code
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4 # Updated to v4 for better compatibility
 
-      # Step 1.5:  Environment variables
-      - name: create env file
-        working-directory: backend
+      # Step 1.5: Environment variables
+      - name: Create env file
+        working-directory: backend/functions
         run: |
-          cd functions
           touch .env
           echo "${{ secrets.FUNCTIONS_ENV_VARS }}" >> .env
 
@@ -32,9 +31,9 @@ jobs:
 
       # Step 3: Set up Node.js environment
       - name: Set up Node
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v4 # Updated to v4
         with:
-          node-version: 19.7.0
+          node-version: '20' # Upgraded to 20 to meet firebase-tools@14.2.0 requirements
 
       # Step 3.5: Download the ZIP file from Google Drive
       - name: Download Models ZIP
@@ -58,7 +57,7 @@ jobs:
       # Step 4: Create Service Account Key and Set Env Variables
       - name: Create SA key and set env variables
         run: |
-          echo '${{ secrets.FIREBASE_SERVICE_ACCOUNT_SCHEMESSG_V3_DEV }}' > ${{ runner.temp }}/gcloud.json
+          echo "${{ secrets.FIREBASE_SERVICE_ACCOUNT_SCHEMESSG_V3_DEV }}" > ${{ runner.temp }}/gcloud.json
           echo "GOOGLE_APPLICATION_CREDENTIALS=${{ runner.temp }}/gcloud.json" >> $GITHUB_ENV
 
       # Step 5: Prepare the environment
@@ -69,8 +68,10 @@ jobs:
           . venv/bin/activate
           npx firebase-tools --version
           python3.10 -m pip install -r requirements.txt
+          npm install -g firebase-tools@14.2.0 # Install firebase-tools globally
+          npx firebase-tools --version # Verify version
 
-      # Step 6: deploy functions
+      # Step 6: Deploy functions
       - name: Deploy functions
         working-directory: backend
         env:

.github/workflows/firebase-hosting-staging.yml

@@ -1,6 +1,3 @@
-# This file was auto-generated by the Firebase CLI
-# https://github.com/firebase/firebase-tools
-
 name: Deploy Frontend to Firebase Hosting
 on:
     push:
@@ -17,9 +14,9 @@ jobs:
         steps:
             - uses: actions/checkout@v4
             - name: Use Node.js
-              uses: actions/setup-node@v3
+              uses: actions/setup-node@v4 # Updated to v4
               with:
-                  node-version: "18"
+                  node-version: "20"
             - name: Cache dependencies
               uses: actions/cache@v3
               with:
@@ -43,14 +40,13 @@ jobs:
               run: |
                   cd frontend
                   npm ci
-                  npm install sharp
                   npm run build
             - name: Debug Variables
               run: |
-                echo "Using vars context: ${{ vars.NEXT_PUBLIC_API_BASE_URL }}"
-                echo "Current branch: ${{ github.ref_name }}"
-                echo "All environment variables:"
-                env
+                  echo "Using vars context: ${{ vars.NEXT_PUBLIC_API_BASE_URL }}"
+                  echo "Current branch: ${{ github.ref_name }}"
+                  echo "All environment variables:"
+                  env
             - uses: FirebaseExtended/action-hosting-deploy@v0
               with:
                   repoToken: ${{ secrets.GITHUB_TOKEN }}
@@ -65,4 +61,4 @@ jobs:
                   NEXT_PUBLIC_FIREBASE_MEASUREMENT_ID: ${{ vars.NEXT_PUBLIC_FIREBASE_MEASUREMENT_ID }}
                   NEXT_PUBLIC_FB_API_KEY: ${{ vars.NEXT_PUBLIC_FB_API_KEY }}
                   NEXT_PUBLIC_FIREBASE_PROJECT_ID: ${{ vars.NEXT_PUBLIC_FIREBASE_PROJECT_ID }}
-                  NEXT_PUBLIC_FIREBASE_APP_ID: ${{ vars.NEXT_PUBLIC_FIREBASE_APP_ID }}
+                  NEXT_PUBLIC_FIREBASE_APP_ID: ${{ vars.NEXT_PUBLIC_FIREBASE_APP_ID }}

backend/functions/ml_logic/chatbotManager.py

@@ -63,10 +63,21 @@ def dataframe_to_text(df: pd.DataFrame) -> str:
     """
     text_summary = ""
     for _, row in df.iterrows():
-        cleanScrape = row["scraped_text"]
-        sentence = clean_scraped_text(cleanScrape)
-
-        text_summary += f"Scheme Name: {row['Scheme']}, Agency: {row['Agency']}, Description: {row['Description']}, Link: {row['Link']}, Scraped Text from website: {sentence}\n"
+        # Handle both uppercase and lowercase column names
+        scheme = row.get("scheme", "")
+        agency = row.get("agency", "")
+        description = row.get("description", "")
+        link = row.get("link", "")
+        phone = row.get("phone", "")
+        address = row.get("address", "")
+        eligibility = row.get("eligibility", "")
+        email = row.get("email", "")
+        how_to_apply = row.get("how_to_apply", "")
+        scraped_text = row.get("scraped_text", "")
+
+        sentence = clean_scraped_text(scraped_text)
+
+        text_summary += f"Scheme Name: {scheme}, Agency: {agency}, Phone: {phone}, Address: {address}, Eligibility: {eligibility}, Email: {email}, How to Apply: {how_to_apply}, Description: {description}, Link: {link}, Scraped Text from website: {sentence}\n"
     return text_summary
 
 
@@ -123,7 +134,7 @@ def initialise(cls):
                 top_p=0.9,
                 presence_penalty=0.2,
                 frequency_penalty=0.2,
-                max_tokens=512
+                max_tokens=512,
             )
             logger.info("Chatbot initialised")
         except Exception as e:  # TODO: logger
@@ -272,7 +283,6 @@ def chatbot(
 
         template_text = system_instructions + top_schemes_text + "<END OF SCHEMES RESULTS>"
 
-
         prompt_template = ChatPromptTemplate.from_messages(
             [
                 ("system", template_text),
@@ -369,7 +379,6 @@ def chatbot_stream(self, top_schemes_text: str, input_text: str, session_id: str
 
         template_text = system_instructions + top_schemes_text + "<END OF SCHEMES RESULTS>"
 
-
         prompt_template = ChatPromptTemplate.from_messages(
             [
                 ("system", template_text),

backend/functions/ml_logic/searchModelManager.py

@@ -322,7 +322,7 @@ def combine_and_aggregate_results(
 
         # Drop duplicates and sort by similarity
         aggregated_results = combined_results.sort_values("Similarity", ascending=False).drop_duplicates(
-            subset=["Scheme"]
+            subset=["scheme"]
         )
 
         # Filter by similarity threshold