Avoid checking whole list everytime to reduce transfer cost to DB

Author: polyrabbit

.github/workflows/probe.yml

@@ -35,6 +35,13 @@ jobs:
           python-version: 3.9
           cache: 'pip'
 
+      - name: Python Runtime Cache
+        id: python-runtime-cache
+        uses: actions/cache@v3
+        with:
+          path: ${{ env.pythonLocation }}
+          key: ${{ runner.os }}-${{ env.pythonLocation }}-${{ hashFiles('requirements.txt') }}
+
       - name: Install Python Dependencies
         run: pip install --upgrade -r requirements.txt
 

.github/workflows/static.yml

@@ -13,7 +13,7 @@ on:
     types: [ generate-gh-pages ]
 
     schedule:
-    - cron: "*/20 * * * *"
+    - cron: "*/15 * * * *"
 
 # Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages
 permissions:
@@ -40,6 +40,7 @@ jobs:
     steps:
       - name: My Host
         run: |
+          date
           sudo apt install -y -qq moreutils  # https://unix.stackexchange.com/questions/26728/prepending-a-timestamp-to-each-line-of-output-from-a-command
           echo -e "free -h:\n`free -h`" | ts
           echo
@@ -51,7 +52,7 @@ jobs:
           echo
           echo "nproc: `nproc`" | ts
           echo
-          echo -e "curl -s ifconfig.me/all:\n`curl -s ifconfig.me/all`" | ts
+          echo -e "curl -s ifconfig.me/all:\n`curl -s --max-time 30 ifconfig.me/all`" | ts
 
       - name: Export vars to env
         env:

db/image.py

@@ -1,6 +1,7 @@
 # TODO: need a separate model?
 import logging
 import os
+import random
 import time
 
 from sqlalchemy import String, column, Values, select
@@ -21,7 +22,10 @@
 def expire():
     start = time.time()
     removed = 0
-    for img_files in chunks(os.listdir(config.image_dir), 500):
+    all_files = list(os.listdir(config.image_dir))
+    random.shuffle(all_files) # avoid checking whole list everytime to reduce transfer cost to DB
+    candidates = all_files[:1000]
+    for img_files in chunks(candidates, 500):
         values = Values(column('name', String), name='v').data(list(map(lambda x: (x,), img_files)))
         stmt = select(values).join(Summary, Summary.image_name == values.c.name,
                                    isouter=True  # Add this to implement left outer join
@@ -32,7 +36,7 @@ def expire():
                 os.remove(os.path.join(config.image_dir, image_name[0]))
                 removed += 1
     cost = (time.time() - start) * 1000
-    logger.info(f'removed {removed} feature images, cost(ms): {cost:.2f}')
+    logger.info(f'removed {removed}/{len(candidates)} feature images, cost(ms): {cost:.2f}')
 
 
 def chunks(lst, n):

requirements.txt

@@ -17,6 +17,7 @@ openai==0.28.1
 torch==2.1.0
 bert-extractive-summarizer==0.10.1
 transformers==4.36.0
+numpy==1.26.4
 python-dotenv==1.0.0
 python_slugify==8.0.1
 sqlalchemy==2.0.21