Code refactoring august 2022 (#22)

Author: odulcy

Dockerfile

@@ -2,6 +2,11 @@ FROM python:3.9.6-buster
 
 WORKDIR /root/Horoscope
 
+# INSTALL DEPENDENCIES FOR DOCTR
+RUN apt-get update \
+ && apt-get install -y libgl1-mesa-dev libsm6 libxext6 libxrender-dev \
+    tesseract-ocr tesseract-ocr-fra
+
 # INSTALL PIPENV
 RUN pip3 install pipenv
 
@@ -12,7 +17,8 @@ RUN pipenv install --dev --system --deploy
 #--system — Use the system pip command rather than the one from your virtualenv.
 #--deploy — Make sure the packages are properly locked in Pipfile.lock, and abort if the lock file is out-of-date.
 
-RUN apt-get update &&\
-    apt-get install -y tesseract-ocr tesseract-ocr-fra
+# FIX THIS
+# Unable to add extras in Pipfile for DocTR, pipenv raises an error
+RUN pip3 install python-doctr['torch']
 
-CMD python3 -u horoscope_bot.py
+CMD python3 -u bot.py

Makefile

@@ -1,8 +1,16 @@
 build:
-	docker build -t horoscope .
+	docker-compose build
+up:
+	docker-compose -p horoscope up -d
+down:
+	docker-compose -p horoscope down -t 0
+
 run:
 	docker run -d -v $(PWD):/root/Horoscope --name horoscope horoscope
 build_lock:
 	docker build -t lock -f Dockerfile.lock .
 lock:
 	docker run -v $(PWD):/root/Horoscope --rm lock pipenv lock
+
+run_selenium:
+	docker run --rm -d -p 4444:4444 --name selenium-horoscope --shm-size=2g selenium/standalone-chrome

Pipfile

@@ -4,7 +4,7 @@ verify_ssl = true
 name = "pypi"
 
 [packages]
-discord = "*"
+discord.py = "*"
 asyncio = "*"
 aiohttp = "*"
 pytz = "*"
@@ -18,6 +18,8 @@ requests = "*"
 matplotlib = "*"
 Pillow = "*"
 nest-asyncio = "*"
+selenium = "*"
+python-doctr = "*"
 
 [dev-packages]
 

Pipfile.lock

@@ -1,24 +1,27 @@
 # coding: utf8
 import discord
+import logging
 import asyncio
 import re
 import os
-import pytz
 import pickle
 import datetime as dt
-import numpy as np
+
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s | %(name)s | %(levelname)s | %(message)s'
+)
 
 from PIL import Image
-from collections import Counter
 from typing import Optional, List
 
 from my_constants import TOKEN, IMG_FOLDER, channel_horoscope
-from scraper import get_last_images, download_image
-from parse import parse_horoscope, reformat_horoscope
-from utils import convert_timedelta, md5
+from rtl2_horoscope.scraper.facebook import FacebookScraper
+from rtl2_horoscope.parse import parse_horoscope, reformat_horoscope
+from rtl2_horoscope.utils import now
 
-import nest_asyncio
-nest_asyncio.apply()
+#import nest_asyncio
+#nest_asyncio.apply()
 
 
 manual  = """
@@ -29,78 +32,29 @@
 ```
 """
 
-tz_paris = pytz.timezone("Europe/Paris")
 TIMESTAMP_FORMAT = "%Y-%m-%d"
-USERNAME = "RTL2officiel"
-
-# top,left,bottow,right
-true_width, true_height = 2362, 3431
-# True Horoscope has the following
-# color proportions
-true_occurences = Counter({1: 960179, 0: 750054, 2: 179367})
-# Tested on a header of size true_width * crop_height
-crop_height = 800
-true_proportions = np.array([true_occurences[0], true_occurences[1], true_occurences[2]])/(true_width * crop_height)
-rtl2_header = np.array([0, 0, true_width, crop_height])
-
-kmeans = pickle.load(open("horoscope_kmeans.pickle", "rb"))
-
-def now():
-    return dt.datetime.now().astimezone(tz_paris)
-
-def is_horoscope(filename, verbose=False):
-    """Check if it is a horoscope or not
-    Step 1 : check the picture size
-    Step 2 : use pretrained KMeans to compare color proporitons
-
-    Args:
-        filename (str) : path to horoscope
-
-    Return:
-        Bool : return True if it is an horoscope, False otherwise
-    """
-    assert os.path.isfile(filename), "Invalid file name"
-
-    # Step 1
-    photo = Image.open(filename)
-    width, height = photo.size
-
-    print(f"Image size : {width}x{height}")
-    if abs(width/true_width - height/true_height) > 0.05 :
-        return False
-    print(f"Ratio de l'image correct.")
-
-    # Step 2
-    k = width/true_width
-    pixels = np.array(photo.crop(tuple(k*rtl2_header))).reshape(-1, 3)
-    occurences = Counter(kmeans.predict(pixels))
-    proportions = np.array([occurences[0], occurences[1], occurences[2]])/(k*true_width * k*crop_height)
-    if verbose:
-        print(proportions, "Image proportions")
-        print(true_proportions, "True proportions")
-        print(np.sum(np.abs(true_proportions - proportions)), "Distance")
-    return np.sum(np.abs(true_proportions - proportions)) < 0.03
-
 
 
 class HoroscopeDiscordBot(discord.Client):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
+        self.scraper = FacebookScraper()
 
+    async def setup_hook(self):
         # create the background task and run it in the background
-        self.bg_task = self.loop.create_task(self.job())
+        self.loop.create_task(self.job())
 
     async def on_ready(self):
         """Initial check"""
         if not os.path.isdir(IMG_FOLDER):
-            print(f"Création du dossier {IMG_FOLDER}")
+            logging.info(f"Création du dossier {IMG_FOLDER}")
             os.mkdir(IMG_FOLDER)
 
-        print(f"[{now().ctime()}] - Bot ready :-)")
-        print('Logged in as')
-        print(self.user.name)
-        print(self.user.id)
-        print('------')
+        logging.info("Bot ready :-)")
+        logging.info('Logged in as')
+        logging.info(self.user.name)
+        logging.info(self.user.id)
+        logging.info('------')
 
     def is_for_bot(self, message) -> bool:
         """Check if the message is for the bot.
@@ -147,9 +101,8 @@ async def job(self, fetch_interval=300, days=[0,1,2,3,4], hours=[9,10,11,12]):
                 await asyncio.sleep(fetch_interval)
 
             time_to_wait = self.get_time_to_wait(hours).total_seconds()
-            time_to_wait_message = f"[{now().ctime()}] - " +\
-                f"Reprise de l'activité dans {time_to_wait} secondes."
-            print(time_to_wait_message)
+            time_to_wait_message = f"Reprise de l'activité dans {time_to_wait} secondes."
+            logging.info(time_to_wait_message)
 
             await asyncio.sleep(time_to_wait)
 
@@ -167,9 +120,8 @@ async def on_message(self, message):
             img_href = message.content.split(" ")[-1]
             if img_href.startswith("http") and await self.fetch_new_horoscope(img_href=img_href):
                 time_to_wait = self.get_time_to_wait([10,11,12]).total_seconds()
-                time_to_wait_message = f"[{now().ctime()}] - " +\
-                    f"Reprise de l'activité dans {time_to_wait} secondes."
-                print(time_to_wait_message)
+                time_to_wait_message = f"Reprise de l'activité dans {time_to_wait} secondes."
+                logging.info(time_to_wait_message)
                 await asyncio.sleep(time_to_wait)
 
         if self.command(message, "last"):
@@ -182,62 +134,23 @@ async def on_message(self, message):
 
     async def parse_and_send_horoscope(self, filename):
         """Parse the image and send the image and the text found through OCR"""
-        print("OCR : en cours.")
+        logging.info("OCR : en cours.")
         horoscope_dict = parse_horoscope(filename, threads=1)
         horoscope_str = reformat_horoscope(horoscope_dict)
-        print("OCR : terminé.")
+        logging.info("OCR : terminé.")
         await self.get_channel(channel_horoscope).send(file=discord.File(filename))
         await self.get_channel(channel_horoscope).send(horoscope_str)
 
-    async def fetch_new_horoscope(self, img_href: Optional[str] = None):
-        """Get last image from RTL2 Twitter page, check if it's a new horoscope (using md5)
+    async def fetch_new_horoscope(self, img_href: Optional[str] = None) -> bool:
+        """Get last image from RTL2 social media page, check if it's a new horoscope
         and send the file on Discord
         Args:
             img_href : if not None, download the image from <img_href> url
         """
-
-        print(f"[{now().ctime()}] - Fetch Horoscope")
-        if img_href:
-            print(f"[{now().ctime()}] - Lien fourni par l'utilisateur : {img_href}.")
-            img_hrefs = [img_href]
-        else:
-            print(f"[{now().ctime()}] - Récupération des dernières images depuis Twitter.")
-            today = now().strftime("%Y-%m-%d")
-            img_hrefs = get_last_images(username=USERNAME, since=today)
-
-        files = sorted(os.listdir(IMG_FOLDER + "/"), reverse=True)
-
-        if len(img_hrefs) > 0:
-            print("Téléchargement des images...")
-        else:
-            print("Pas d'images tweetées aujourd'hui !")
-
-        for img_href in img_hrefs:
-            filename = await download_image(img_href)
-
-            new_image = IMG_FOLDER + "/" + files[0]
-
-            if len(files) >= 1:
-                old_image = IMG_FOLDER + "/" + files[1]
-            else:
-                old_image = ""
-
-            print(f"Test de l'image {img_href}")
-            if is_horoscope(new_image, verbose=True):
-                print("C'est un horoscope !")
-                if md5(new_image) == md5(old_image):
-                    print("C'est l'horoscope d'hier")
-                    # Stop research
-                    return False
-                else:
-                    print("C'est l'horoscope du jour")
-                    await self.parse_and_send_horoscope(new_image)
-                    # Stop research
-                    return True
-            else:
-                print("Ce n'est pas un nouveau horoscope")
-                # Continue research
-
+        horoscope = await self.scraper.fetch_new_horoscope(img_href)
+        if horoscope:
+            await self.parse_and_send_horoscope(horoscope)
+            return True
         return False
 
     def get_time_to_wait(self, hours):
@@ -258,5 +171,7 @@ def get_time_to_wait(self, hours):
         return next_day-today
 
 if __name__ == "__main__":
-    client = HoroscopeDiscordBot()
+    intents = discord.Intents.default()
+    intents.message_content = True
+    client = HoroscopeDiscordBot(intents=intents)
     client.run(TOKEN)

horoscope_bot.py bot.pyhoroscope_bot.py renamed to bot.py

@@ -0,0 +1,20 @@
+version: '3.7'
+
+services:
+
+  selenium-horoscope:
+    image: selenium/standalone-chrome
+    container_name: selenium-horoscope
+    shm_size: "2gb"
+
+  bot:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    image: horoscope-bot
+    volumes:
+      - ./:/root/Horoscope/
+    container_name: horoscope
+    depends_on:
+      - "selenium-horoscope"
+

docker-compose.yml

@@ -2,6 +2,7 @@
 from concurrent.futures import ThreadPoolExecutor
 from itertools import repeat
 import re
+from pathlib import Path
 
 import pandas as pd
 pd.options.mode.chained_assignment = None
@@ -56,6 +57,7 @@ def read_crop(img, crop_region=None, pb=None):
     Args:
         img (PIL.Image): Image to read from.
         crop_region (tuple of ints): Coordinates of the rectangle containing the text to read.
+        pb (tqdm progress bar)
     
     Returns:
         str: The text as read by Tesseract.
@@ -232,24 +234,24 @@ def parse_horoscope(img, threads=12, verbose=True):
          'poisson': ('bronze',
           'Votre corps réclame une pause, ne tirez \\pas trop sur la corde.')}
     """
-    if isinstance(img, str):
+    if isinstance(img, str) or isinstance(img, Path):
         img = Image.open(img)
     img.load()
-    
+ 
     # Rescale regions if necessary
     factor =  img.width/true_width
     print("scale factor:", factor)
     if factor != 1:
         scaled_regions = scale_regions(factor, regions)
     else:
         scaled_regions = regions
-    
+
     # Read and clean up texts
     texts = read_texts(img, threads=threads, regions=scaled_regions, verbose=verbose)
     texts = {sign: clean_up_text(text) for sign, text in texts.items()}
-    
+
     stars = find_star_colors(img, regions=scaled_regions, robust=True)
-    
+ 
     keys = texts.keys()
     values = zip(stars.values(), texts.values())
     out = dict(zip(keys, values))

parse.py rtl2_horoscope/parse.pyparse.py renamed to rtl2_horoscope/parse.py

@@ -0,0 +1 @@
+from .scraper import Scraper

rtl2_horoscope/scraper/__init__.py

@@ -0,0 +1,42 @@
+import logging
+
+from selenium import webdriver
+from bs4 import BeautifulSoup
+from typing import List
+
+from rtl2_horoscope.scraper import Scraper
+
+ALBUM_URL = "https://www.facebook.com/pg/rtl2/photos/?tab=album&album_id=248389291078&ref=page_internal"
+WEBDRIVER_URL = 'http://selenium-horoscope:4444/wd/hub'
+
+class FacebookScraper(Scraper):
+
+    def __init__(self, album_url: str = ALBUM_URL, webdriver_url: str = WEBDRIVER_URL):
+        super().__init__(social_media="facebook")
+        self.album_url = album_url
+        self.webdriver_url = webdriver_url
+
+    def get_last_images(self, **kwargs) -> List[str]:
+
+        logging.info("Initialize Webdriver")
+        driver = webdriver.Remote(
+            self.webdriver_url,
+            options=webdriver.ChromeOptions()
+        )
+        driver.set_window_size(1280, 1024)
+        logging.info(f"Get {self.album_url} ...")
+        driver.get(self.album_url)
+        logging.info("Done")
+        page_source = driver.page_source
+        driver.close()
+
+        soup = BeautifulSoup(page_source, features="html5lib")
+
+        hrefs = []
+
+        for a in soup.find_all("a"):
+            for img in a.find_all("img"):
+                hrefs.append(img['src'])
+
+        return hrefs
+