Avoid checking whole list everytime to reduce transfer cost to DB

Author: polyrabbit

.github/workflows/static.yml

@@ -13,7 +13,7 @@ on:
     types: [ generate-gh-pages ]
 
     schedule:
-    - cron: "*/20 * * * *"
+    - cron: "*/10 * * * *"
 
 # Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages
 permissions:

db/image.py

@@ -1,6 +1,7 @@
 # TODO: need a separate model?
 import logging
 import os
+import random
 import time
 
 from sqlalchemy import String, column, Values, select
@@ -21,7 +22,10 @@
 def expire():
     start = time.time()
     removed = 0
-    for img_files in chunks(os.listdir(config.image_dir), 500):
+    all_files = list(os.listdir(config.image_dir))
+    random.shuffle(all_files) # avoid checking whole list everytime to reduce transfer cost to DB
+    candidates = all_files[:1000]
+    for img_files in chunks(candidates, 500):
         values = Values(column('name', String), name='v').data(list(map(lambda x: (x,), img_files)))
         stmt = select(values).join(Summary, Summary.image_name == values.c.name,
                                    isouter=True  # Add this to implement left outer join
@@ -32,7 +36,7 @@ def expire():
                 os.remove(os.path.join(config.image_dir, image_name[0]))
                 removed += 1
     cost = (time.time() - start) * 1000
-    logger.info(f'removed {removed} feature images, cost(ms): {cost:.2f}')
+    logger.info(f'removed {removed}/{len(candidates)} feature images, cost(ms): {cost:.2f}')
 
 
 def chunks(lst, n):

requirements.txt

@@ -17,6 +17,7 @@ openai==0.28.1
 torch==2.1.0
 bert-extractive-summarizer==0.10.1
 transformers==4.36.0
+numpy==1.26.4
 python-dotenv==1.0.0
 python_slugify==8.0.1
 sqlalchemy==2.0.21