Publish Infomate articles to Telegram channel

boards/migrations/0011_boardblock_is_publishing_to_telegram.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.2.13 on 2022-01-18 03:57
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('boards', '0010_boardfeed_filters'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='boardblock',
+            name='is_publishing_to_telegram',
+            field=models.BooleanField(default=False),
+        ),
+    ]

boards/models.py

@@ -71,6 +71,8 @@ class BoardBlock(models.Model):
 
     index = models.PositiveIntegerField(default=0)
 
+    is_publishing_to_telegram = models.BooleanField(default=False)
+
     class Meta:
         db_table = "board_blocks"
         ordering = ["index"]

docker-compose.yml

@@ -11,6 +11,7 @@ services:
     environment:
       - DEBUG=True
       - PYTHONUNBUFFERED=1
+      - TELEGRAM_TOKEN=${TELEGRAM_TOKEN}
     restart: always
     volumes:
       - .:/app:delegated

infomate/settings.py

@@ -17,7 +17,8 @@
     "django.contrib.humanize",
     "django_bleach",
     "boards",
-    "parsing"
+    "parsing",
+    "notifications",
 ]
 
 MIDDLEWARE = [
@@ -30,6 +31,7 @@
     {
         "BACKEND": "django.template.backends.django.DjangoTemplates",
         "DIRS": [
+            os.path.join(BASE_DIR, "notifications/telegram/templates"),
             os.path.join(BASE_DIR, "templates"),
         ],
         "APP_DIRS": True,
@@ -118,6 +120,7 @@
 TELEGRAM_APP_ID = None  # should set in private_settings.py
 TELEGRAM_APP_HASH = None  # should set in private_settings.py
 TELEGRAM_SESSION_FILE = None  # should set in private settings.py
+TELEGRAM_TOKEN = None  # should set in private settings.py
 TELEGRAM_CACHE_SECONDS = 10 * 60  # 10 min
 
 BLEACH_STRIP_TAGS = True

notifications/apps.py

@@ -0,0 +1,5 @@
+from django.apps import AppConfig
+
+
+class NotificationsConfig(AppConfig):
+    name = 'notifications'

notifications/migrations/0001_initial.py

@@ -0,0 +1,50 @@
+# Generated by Django 2.2.13 on 2022-01-22 14:50
+
+from django.db import migrations, models
+import django.db.models.deletion
+import uuid
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+        ('boards', '0011_boardblock_is_publishing_to_telegram'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='PublishHistory',
+            fields=[
+                ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
+                ('channel_id', models.CharField(max_length=256)),
+                ('published_at', models.DateTimeField(auto_now_add=True)),
+                ('telegram_message_id', models.IntegerField()),
+                ('article', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='published', to='boards.Article')),
+            ],
+            options={
+                'db_table': 'publish_history',
+                'ordering': ['-published_at'],
+            },
+        ),
+        migrations.CreateModel(
+            name='BoardTelegramChannel',
+            fields=[
+                ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
+                ('telegram_channel_id', models.CharField(max_length=256)),
+                ('created_at', models.DateTimeField(auto_now_add=True)),
+                ('updated_at', models.DateTimeField()),
+                ('board', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='telegram_channel', to='boards.Board')),
+            ],
+            options={
+                'verbose_name': "telegram channel to publish Board's updates",
+                'db_table': 'board_telegram_channel',
+                'ordering': ['-updated_at'],
+            },
+        ),
+        migrations.AddConstraint(
+            model_name='boardtelegramchannel',
+            constraint=models.UniqueConstraint(fields=('board', 'telegram_channel_id'), name='unique_board_telegram_channel'),
+        ),
+    ]

notifications/models.py

@@ -0,0 +1,52 @@
+from uuid import uuid4
+from datetime import datetime
+
+from django.db import models
+
+from boards.models import Article, Board
+
+
+class PublishHistory(models.Model):
+    id = models.UUIDField(primary_key=True, default=uuid4, editable=False)
+
+    article = models.ForeignKey(
+        Article,
+        related_name='published',
+        on_delete=models.CASCADE
+    )
+    channel_id = models.CharField(max_length=256)
+    published_at = models.DateTimeField(auto_now_add=True)
+
+    telegram_message_id = models.IntegerField()
+
+    class Meta:
+        db_table = "publish_history"
+        ordering = ["-published_at"]
+
+
+class BoardTelegramChannel(models.Model):
+    id = models.UUIDField(primary_key=True, default=uuid4, editable=False)
+
+    board = models.ForeignKey(
+        Board,
+        related_name='telegram_channel',
+        on_delete=models.CASCADE
+    )
+    telegram_channel_id = models.CharField(max_length=256)
+    created_at = models.DateTimeField(auto_now_add=True)
+    updated_at = models.DateTimeField()
+
+    class Meta:
+        verbose_name = 'telegram channel to publish Board\'s updates'
+        db_table = "board_telegram_channel"
+        ordering = ["-updated_at"]
+        constraints = [models.UniqueConstraint(
+            fields=('board', 'telegram_channel_id'),
+            name='unique_board_telegram_channel',
+        )]
+
+    def save(self, *args, **kwargs):
+        if not self.created_at:
+            self.created_at = datetime.utcnow()
+        self.updated_at = datetime.utcnow()
+        return super().save(*args, **kwargs)

notifications/telegram/bot.py

@@ -0,0 +1,15 @@
+import os
+import logging
+
+import telegram
+from django.conf import settings
+
+log = logging.getLogger()
+
+TELEGRAM_TOKEN = settings.TELEGRAM_TOKEN or os.environ.get('TELEGRAM_TOKEN')
+
+try:
+    bot = telegram.Bot(token=TELEGRAM_TOKEN) if TELEGRAM_TOKEN else None
+except telegram.error.InvalidToken as e:
+    log.error(e)
+    bot = None

notifications/telegram/common.py

@@ -0,0 +1,106 @@
+from collections import namedtuple
+from time import sleep
+from random import random
+
+import telegram
+from django.conf import settings
+from django.template import loader
+from telegram import ParseMode
+
+from notifications.telegram.bot import bot, log
+
+Chat = namedtuple("Chat", ["id"])
+
+
+def send_telegram_message(
+    chat: Chat,
+    text: str,
+    parse_mode: ParseMode = telegram.ParseMode.HTML,
+    disable_preview: bool = True,
+    **kwargs
+):
+    if not bot:
+        log.warning("No telegram token. Skipping")
+        return
+
+    log.info(f"Telegram: sending the message: {text}")
+
+    try:
+        return bot.send_message(
+            chat_id=chat.id,
+            text=text,
+            parse_mode=parse_mode,
+            disable_web_page_preview=disable_preview,
+            **kwargs
+        )
+
+    except telegram.error.RetryAfter as ex:
+        log.warning(f"Telegram error: {ex}")
+        sleep_seconds = ex.retry_after + random()
+        log.warning(f"Sleeping for {sleep_seconds:.2f}")
+        sleep(sleep_seconds)
+
+        return send_telegram_message(
+            chat=Chat(id=chat.id),
+            text=text,
+            parse_mode=parse_mode,
+            disable_web_page_preview=disable_preview,
+            **kwargs
+        )
+
+    except telegram.error.TelegramError as ex:
+        log.warning(f"Telegram error: {ex}")
+        return False
+
+
+def send_telegram_image(
+    chat: Chat,
+    image_url: str,
+    text: str,
+    parse_mode: ParseMode = telegram.ParseMode.HTML,
+    **kwargs
+):
+    if not bot:
+        log.warning("No telegram token. Skipping")
+        return
+
+    log.info(f"Telegram: sending the image: {image_url} {text}")
+
+    try:
+        return bot.send_photo(
+            chat_id=chat.id,
+            photo=image_url,
+            caption=text[:1024],
+            parse_mode=parse_mode,
+            **kwargs
+        )
+    except telegram.error.TelegramError as ex:
+        log.warning(f"Telegram error: {ex}")
+
+
+def remove_action_buttons(chat: Chat, message_id: str, **kwargs):
+    try:
+        return bot.edit_message_reply_markup(
+            chat_id=chat.id,
+            message_id=message_id,
+            reply_markup=None,
+            **kwargs
+        )
+    except telegram.error.TelegramError:
+        log.info("Buttons are already removed. Skipping")
+        return None
+
+
+def render_html_message(template, **data):
+    template = loader.get_template(f"messages/{template}")
+    return template.render({
+        **data,
+        "settings": settings
+    })
+
+
+def get_telergam_channel_name_at(channel_name):
+    if not channel_name:
+        return None
+
+    return channel_name if channel_name[0] == '@' else '@' + channel_name

notifications/telegram/templates/messages/article_as_post.html

@@ -0,0 +1,6 @@
+<b>{{ article.title }}</b>
+[<a href="{{ article.url }}">{{ article.feed.name }}</a>]
+{% for para in paragraphs %}
+{{ para }}
+{% endfor %}
+{{ tg_channel }}

requirements.txt

@@ -10,4 +10,5 @@ feedparser==6
 sentry-sdk==0.14.1
 nltk==3.4.5
 newspaper3k>=0.2.8
-django-bleach==0.6.1
+django-bleach==0.6.1
+python-telegram-bot==12.5.1

scripts/common.py

@@ -21,6 +21,11 @@
 DEFAULT_REQUEST_TIMEOUT = 10
 MAX_PARSABLE_CONTENT_LENGTH = 15 * 1024 * 1024  # 15Mb
 
+WEBSITES_FALLBACK_MESSAGES = [
+    # if try to parse Medium articles
+    'This website is using a security service to protect itself from online attacks'
+]
+
 socket.setdefaulttimeout(DEFAULT_REQUEST_TIMEOUT)
 requests.packages.urllib3.disable_warnings(category=InsecureRequestWarning)
 
@@ -105,4 +110,12 @@ def parse_rss_text_and_image(entry):
         if src:
             return text, src
 
-    return text, ""
+    return text, ""
+
+
+def if_fallback_message_in_text(text: str) -> bool:
+    for message in WEBSITES_FALLBACK_MESSAGES:
+        if message in text:
+            return True
+
+    return False