Code refactoring : use Twitter instead of Facebook (#17)

Fetch horoscope from Twitter using https://github.com/twintproject/twint

Author: odulcy

Dockerfile

@@ -1,13 +1,18 @@
-FROM python:3.6.9
+FROM python:3.9.6-buster
 
 WORKDIR /root/Horoscope
 
-RUN pip3 install --upgrade pip
+# INSTALL PIPENV
+RUN pip3 install pipenv
 
-ADD requirements.txt requirements.txt
+COPY Pipfile Pipfile
+COPY Pipfile.lock Pipfile.lock
+RUN pipenv install --dev --system --deploy
+#--dev — Install both develop and default packages from Pipfile.
+#--system — Use the system pip command rather than the one from your virtualenv.
+#--deploy — Make sure the packages are properly locked in Pipfile.lock, and abort if the lock file is out-of-date.
 
-RUN pip3 install -r requirements.txt
-
-RUN apt-get update && apt-get install -y tesseract-ocr tesseract-ocr-fra
+RUN apt-get update &&\
+    apt-get install -y tesseract-ocr tesseract-ocr-fra
 
 CMD python3 -u horoscope_bot.py

Dockerfile.lock

@@ -0,0 +1,6 @@
+FROM python:3.9.6-buster
+
+WORKDIR /root/Horoscope
+
+# INSTALL PIPENV
+RUN pip3 install pipenv

Makefile

@@ -0,0 +1,8 @@
+build:
+	docker build -t horoscope .
+run:
+	docker run -d -v $(PWD):/root/Horoscope --name horoscope horoscope
+build_lock:
+	docker build -t lock -f Dockerfile.lock .
+lock:
+	docker run -v $(PWD):/root/Horoscope --rm lock pipenv lock

Pipfile

@@ -0,0 +1,25 @@
+[[source]]
+url = "https://pypi.org/simple"
+verify_ssl = true
+name = "pypi"
+
+[packages]
+discord = "*"
+asyncio = "*"
+aiohttp = "*"
+pytz = "*"
+numpy = "*"
+pandas = "*"
+tqdm = "*"
+pytesseract = "*"
+scikit-learn = "*"
+twint = {editable = true, ref = "origin/master", git = "https://github.com/twintproject/twint.git"}
+requests = "*"
+matplotlib = "*"
+Pillow = "*"
+nest-asyncio = "*"
+
+[dev-packages]
+
+[requires]
+python_version = "3.9"

Pipfile.lock

@@ -1,22 +1,25 @@
 # coding: utf8
 import discord
-from discord.ext import commands
 import asyncio
-import aiohttp
-import datetime as dt
-import hashlib
+import re
 import os
 import pytz
+import pickle
+import datetime as dt
+import numpy as np
+
+from PIL import Image
+from collections import Counter
+from typing import Optional, List
 
 from my_constants import TOKEN, IMG_FOLDER, channel_horoscope
-from scraper import get_last_image, download_image
+from scraper import get_last_images, download_image
 from parse import parse_horoscope, reformat_horoscope
 from utils import convert_timedelta, md5
 
-import pickle
-from PIL import Image
-from collections import Counter
-import numpy as np
+import nest_asyncio
+nest_asyncio.apply()
+
 
 manual  = """
 ```Help
@@ -27,19 +30,24 @@
 """
 
 tz_paris = pytz.timezone("Europe/Paris")
+TIMESTAMP_FORMAT = "%Y-%m-%d"
+USERNAME = "RTL2officiel"
 
 # top,left,bottow,right
-true_width, true_height = 1181, 1716
+true_width, true_height = 2362, 3431
 # True Horoscope has the following
 # color proportions
-#Counter({0: 134576, 1: 132231, 2: 28443})
-# Tested on a header of size 1181*250
-crop_height = 250
-true_prop   = np.array([134576, 132231, 28443])/(true_width * crop_height)
-rtl2_header = np.array([0, 0, 1181, 250])
+true_occurences = Counter({1: 960179, 0: 750054, 2: 179367})
+# Tested on a header of size true_width * crop_height
+crop_height = 800
+true_proportions = np.array([true_occurences[0], true_occurences[1], true_occurences[2]])/(true_width * crop_height)
+rtl2_header = np.array([0, 0, true_width, crop_height])
 
 kmeans = pickle.load(open("horoscope_kmeans.pickle", "rb"))
 
+def now():
+    return dt.datetime.now().astimezone(tz_paris)
+
 def is_horoscope(filename, verbose=False):
     """Check if it is a horoscope or not
     Step 1 : check the picture size
@@ -64,18 +72,18 @@ def is_horoscope(filename, verbose=False):
 
     # Step 2
     k = width/true_width
-    pixels = np.array(photo.crop(tuple(k*rtl2_header)).getdata())
+    pixels = np.array(photo.crop(tuple(k*rtl2_header))).reshape(-1, 3)
     occurences = Counter(kmeans.predict(pixels))
-    proportions = np.array([occ for occ in occurences.values()])/(k*true_width * k*crop_height)
+    proportions = np.array([occurences[0], occurences[1], occurences[2]])/(k*true_width * k*crop_height)
     if verbose:
         print(proportions, "Image proportions")
-        print(true_prop, "True proportions")
-        print(np.sum(np.abs(true_prop - proportions)), "Distance")
-    return np.sum(np.abs(true_prop - proportions)) < 0.03
+        print(true_proportions, "True proportions")
+        print(np.sum(np.abs(true_proportions - proportions)), "Distance")
+    return np.sum(np.abs(true_proportions - proportions)) < 0.03
 
 
 
-class MyClient(discord.Client):
+class HoroscopeDiscordBot(discord.Client):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
 
@@ -88,13 +96,28 @@ async def on_ready(self):
             print(f"Création du dossier {IMG_FOLDER}")
             os.mkdir(IMG_FOLDER)
 
-        print("[" + dt.datetime.now().astimezone(tz_paris).ctime() + "] - Bot ready :-)")
+        print(f"[{now().ctime()}] - Bot ready :-)")
         print('Logged in as')
         print(self.user.name)
         print(self.user.id)
         print('------')
 
-    async def job(self, fetch_interval=300, days=[0,1,2,3,4], hours=[10,11,12,13]):
+    def is_for_bot(self, message) -> bool:
+        """Check if the message is for the bot.
+        Args:
+            message: Discord message
+        """
+        return re.match(f"^<@!?{self.user.id}>", message.content)
+
+    def command(self, message, cmd: str) -> str:
+        """Check if the command match the one found in message.
+        Args:
+            message: Discord message
+            cmd: command people sent
+        """
+        return re.match(f"^<@!?{self.user.id}> {cmd}", message.content)
+
+    async def job(self, fetch_interval=300, days=[0,1,2,3,4], hours=[9,10,11,12]):
         """ Job to run evey `fetch_interval` seconds,
         each day in days, between hours
         Args:
@@ -115,18 +138,20 @@ async def job(self, fetch_interval=300, days=[0,1,2,3,4], hours=[10,11,12,13]):
         assert min(hours) >= 0, "Need number between 0 and 23"
 
         while not self.is_closed():
-            today = dt.datetime.today().astimezone(tz_paris)
-            while today.weekday() in days and today.hour in hours and not await self.fetch_new_horoscope():
+            today = now()
+
+            while today.weekday() in days and today.hour in hours \
+                and not await self.fetch_new_horoscope():
                 # while (it's time to fetch horoscope) AND (the horoscope has not been published yet)
-                # wait fetch_interval to not spam Facebook
+                # wait fetch_interval to not spam Twitter
                 await asyncio.sleep(fetch_interval)
+
             time_to_wait = self.get_time_to_wait(hours).total_seconds()
-            print("[" + dt.datetime.now().astimezone(tz_paris).ctime() + "] - " + f"Reprise de l'activité dans {time_to_wait} secondes.")
-            await asyncio.sleep(time_to_wait)
+            time_to_wait_message = f"[{now().ctime()}] - " +\
+                f"Reprise de l'activité dans {time_to_wait} secondes."
+            print(time_to_wait_message)
 
-    def command(self, cmd):
-        """Wrapper for bot prefix"""
-        return f"<@{self.user.id}> " + str(cmd)
+            await asyncio.sleep(time_to_wait)
 
     async def on_message(self, message):
         """Handle messages
@@ -135,20 +160,19 @@ async def on_message(self, message):
         if message.author == client.user:
             return
 
-        if message.content == self.command("help"):
+        if self.command(message, "help"):
             await self.get_channel(channel_horoscope).send(manual.format(id=self.user.id))
 
-        if message.content == self.command("test"):
-            await self.fetch_new_horoscope(force=True)
-
-        if message.content.startswith(self.command("download")):
+        if self.command(message, "download"):
             img_href = message.content.split(" ")[-1]
             if img_href.startswith("http") and await self.fetch_new_horoscope(img_href=img_href):
                 time_to_wait = self.get_time_to_wait([10,11,12]).total_seconds()
-                print("[" + dt.datetime.now().astimezone(tz_paris).ctime() + "] - " + f"Reprise de l'activité dans {time_to_wait} secondes.")
+                time_to_wait_message = f"[{now().ctime()}] - " +\
+                    f"Reprise de l'activité dans {time_to_wait} secondes."
+                print(time_to_wait_message)
                 await asyncio.sleep(time_to_wait)
 
-        if message.content == self.command("last"):
+        if self.command(message, "last"):
             files = sorted(os.listdir(IMG_FOLDER), reverse=True)
             if len(files) == 0:
                 await self.get_channel(channel_horoscope).send("Aucun horoscope en stock :-(")
@@ -157,69 +181,82 @@ async def on_message(self, message):
             await self.parse_and_send_horoscope(horoscope_img)
 
     async def parse_and_send_horoscope(self, filename):
-        """Parse the image and send the image and the text
-        found through OCR
-        """
+        """Parse the image and send the image and the text found through OCR"""
         print("OCR : en cours.")
         horoscope_dict = parse_horoscope(filename, threads=1)
         horoscope_str = reformat_horoscope(horoscope_dict)
         print("OCR : terminé.")
         await self.get_channel(channel_horoscope).send(file=discord.File(filename))
         await self.get_channel(channel_horoscope).send(horoscope_str)
 
-    async def fetch_new_horoscope(self, img_href=None, force=False):
-        """Get last image from RTL2 Facebook page,
-        check if it's a new horoscope (using md5)
+    async def fetch_new_horoscope(self, img_href: Optional[str] = None):
+        """Get last image from RTL2 Twitter page, check if it's a new horoscope (using md5)
         and send the file on Discord
         Args:
-            img_href (str) : if not None, download the image
-                           from <img_href> url
-            force (bool) : if True, download the last image
-                           and send it as it is
+            img_href : if not None, download the image from <img_href> url
         """
 
-        print("[" + dt.datetime.now().astimezone(tz_paris).ctime() + "] - Fetch Horoscope")
-        if not img_href:
-            print("Récupération du dernier lien.")
-            img_href = await get_last_image()
-            print("Téléchargement de l'image...")
-
-        if force:
-            filename = await download_image(img_href, filename=IMG_FOLDER + "/" + "9999-99-99_test.jpg")
-            await self.get_channel(channel_horoscope).send(file=discord.File(filename))
-            return True
+        print(f"[{now().ctime()}] - Fetch Horoscope")
+        if img_href:
+            print(f"[{now().ctime()}] - Lien fourni par l'utilisateur : {img_href}.")
+            img_hrefs = [img_href]
+        else:
+            print(f"[{now().ctime()}] - Récupération des dernières images depuis Twitter.")
+            today = now().strftime("%Y-%m-%d")
+            img_hrefs = get_last_images(username=USERNAME, since=today)
 
-        filename = await download_image(img_href)
         files = sorted(os.listdir(IMG_FOLDER + "/"), reverse=True)
 
-        f1 = IMG_FOLDER + "/" + files[0]
-        f2 = ""
-
-        if len(files) >= 1:
-            f2 = IMG_FOLDER + "/" + files[1]
+        if len(img_hrefs) > 0:
+            print("Téléchargement des images...")
+        else:
+            print("Pas d'images tweetées aujourd'hui !")
+
+        for img_href in img_hrefs:
+            filename = await download_image(img_href)
+
+            new_image = IMG_FOLDER + "/" + files[0]
+
+            if len(files) >= 1:
+                old_image = IMG_FOLDER + "/" + files[1]
+            else:
+                old_image = ""
+
+            print(f"Test de l'image {img_href}")
+            if is_horoscope(new_image, verbose=True):
+                print("C'est un horoscope !")
+                if md5(new_image) == md5(old_image):
+                    print("C'est l'horoscope d'hier")
+                    # Stop research
+                    return False
+                else:
+                    print("C'est l'horoscope du jour")
+                    await self.parse_and_send_horoscope(new_image)
+                    # Stop research
+                    return True
+            else:
+                print("Ce n'est pas un nouveau horoscope")
+                # Continue research
 
-        print("Test de l'image : est-ce l'horoscope ?")
-        if is_horoscope(f1) and (md5(f1) != md5(f2)):
-            print("C'est l'horoscope !")
-            await self.parse_and_send_horoscope(f1)
-            return True
-        print("Ce n'est pas l'horoscope")
         return False
 
     def get_time_to_wait(self, hours):
         """How many time to wait before checking
         for a new horoscope ?
         """
 
-        today = dt.datetime.today().astimezone(tz_paris)
+        today = now()
         # Wait until tomorrow
         days_to_wait = 1
         if today.weekday() == 4:
             # it's Friday -> wait two more days
             days_to_wait += 2
+        if today.weekday() == 5:
+            # it's Saturday -> wait one more day
+            days_to_wait += 1
         next_day = today.replace(hour=hours[0],minute=0,second=0,microsecond=0) + dt.timedelta(days=days_to_wait)
         return next_day-today
 
 if __name__ == "__main__":
-    client = MyClient()
+    client = HoroscopeDiscordBot()
     client.run(TOKEN)