Keep track of RSS item to Humhub post relationship, never run more than once

Events.php

@@ -11,6 +11,10 @@
 
 class Events
 {
+    /**
+     * @var string mutex to acquire
+     */
+    const MUTEX_ID = 'rss-cron';
 
     /**
      * Defines what to do if admin menu is initialized.
@@ -40,8 +44,16 @@ public static function onAdminMenuInit($event)
      */
     public static function onCron($event)
     {
+        if ($event->sender->requestedRoute != "cron/run")
+            return;
+
+        if (! Yii::$app->mutex->acquire(static::MUTEX_ID)) {
+            Console::stdout("RSS cron execution skipped - already running!\n");
+            return;
+        }
+
         try {
-            Console::stdout("Updating RSS news feeds...\n");
+            Console::stdout("Queueing RSS feeds for spaces:");
             $ccmsEnabled = ContentContainerModuleState::find()->
                 where(['module_id' => 'rss'])->
                 andWhere(['module_state' => 1])->
@@ -54,14 +66,17 @@ public static function onCron($event)
                 if (! empty($lastrun) && time() < ($interval * 60 + $lastrun))
                     continue;
                 $space->setSetting('lastrun', time(), 'rss');
-                Console::stdout("  Queueing update for space \"" . $space->name . "\"\n");
+#                Console::stdout("  Queueing update for space \"" . $space->name . "\"\n");
+                Console::stdout(' "' . $space->name . '"');
                 Yii::$app->queue->push(new jobs\GetFeedUpdates(['space' => $space, 'force' => false]));
             }
-            Console::stdout(Console::renderColoredString("%gdone.%n\n", 1));
+            Console::stdout(Console::renderColoredString(" %gdone.%n\n", 1));
         } catch (\Throwable $e) {
             $event->sender->stderr($e->getMessage()."\n");
             Yii::error($e);
         }
+
+        Yii::$app->mutex->release(static::MUTEX_ID);
     }
 
 }

jobs/GetFeedUpdates.php

@@ -3,6 +3,8 @@
 namespace sij\humhub\modules\rss\jobs;
 
 use Yii;
+#use yii\BaseYii;
+use yii\helpers\Console;
 
 use humhub\modules\queue\ActiveJob;
 use humhub\modules\post\models\Post;
@@ -11,6 +13,7 @@
 use sij\humhub\modules\rss\components\MarkdownHelper;
 use sij\humhub\modules\rss\components\RssElement;
 use sij\humhub\modules\rss\controllers\RssController;
+use sij\humhub\modules\rss\models\RssPosts;
 
 /**
  * Reads the RSS Feed URL for this space
@@ -49,19 +52,27 @@ class GetFeedUpdates extends ActiveJob
     private $newest; # newest date we are accepting
     private $items; # array of sij\humhub\modules\rss\components\RssElement keyed by pubDate
 
+    /**
+     * @var string mutex to acquire
+     */
+    const MUTEX_ID = 'rss-queue';
+
     private function log($message) {
         if ( $this->logFileHandle ) {
             fwrite($this->logFileHandle, $message);
             fflush($this->logFileHandle);
         }
     }
 
-/**
- * Creates a new Post in the Space
- */
+    /**
+     * Creates a new Post in the Space
+     */
     private function postError($title, $info)
     {
 
+        Yii::error("RSS-Reader: ".$title."\n".$info."\n", "RSS-Reader");
+        return;
+
         $post = new Post($this->space);
 
         $post->created_by =
@@ -83,32 +94,52 @@ private function postError($title, $info)
 
     }
 
-/**
- * Creates a new Post in the Space
- */
-    private function postMessage($message, $datePublished = false)
+    /**
+     * Creates a new Post in the Space
+     */
+    private function postMessage($message, $link = false, $datePublished = false)
     {
-
         $post = null;
 
-        // attempt to locate a previous version of the post
         if ( $datePublished ) {
             $stamp = $datePublished->format("Y-m-d H:i:s");
+        }
+
+        // find previous version of the post via db
+        if ( $link ) {
+            $url2id = RssPosts::findOne(['rss_link' => $link]);
+            if ( $url2id !== null )
+                $post = Post::findOne($url2id->post_id);
+        }
+
+        // attempt to locate a previous version of the post
+        // guess this should go some day - themroc
+        if ( $post === null && $stamp ) {
             $oldContent = Content::findAll([
                 'contentcontainer_id' => $this->space->contentcontainer_id,
 //              'created_by' => $this->created_by, // cant rely on this field if config changed
                 'created_at' => $stamp,
             ]);
             if ( count($oldContent) == 1 ) {
                 $post = Post::findOne($oldContent[0]->object_id);
-                $this->log("\n\n### update Post\n");
             }
         }
 
         // if no previous version of the post, create a new one
         if ( $post === null ) {
             $post = new Post($this->space);
             $this->log("\n\n### new Post\n");
+            Console::stdout("RSS queue: creating new post ".$link);
+        } else {
+            if ( ! $stamp ) {
+                return; // we assume it hasn't changed - better miss an update than rewrite the post every time.
+            }
+            if ($stamp > $post->created_at) {
+                $this->log("\n\n### update Post\n");
+                Console::stdout("RSS queue: updating post ".$link);
+            } else {
+                return; // not changed
+            }
         }
 
         $post->created_by =
@@ -126,6 +157,13 @@ private function postMessage($message, $datePublished = false)
         $post->save();
         $this->log(print_r($post, true));
 
+        if (! $url2id ) {
+            $url2id = new RssPosts();
+            $url2id->rss_link = $link;
+            $url2id->post_id = $post->id;
+            $url2id->save();
+        }
+
         // make it look like the space post was created at the same time as the RSS article
         // note $post->save() always sets the time stamps to "now"
         if ( $datePublished ) {
@@ -144,6 +182,7 @@ private function postMessage($message, $datePublished = false)
                 ->query();
         }
 
+        Console::stdout(Console::renderColoredString(" %gdone.%n\n", 1));
     }
 
     /**
@@ -250,7 +289,7 @@ private function parseNewsItem($item)
         }
 
         // post the message in the stream
-        $this->postMessage($message, $datePublished);
+        $this->postMessage($message, $link, $datePublished);
 
     }
 
@@ -444,6 +483,10 @@ private function downloadNewsFeed()
  */
     public function run()
     {
+        if (! Yii::$app->mutex->acquire(static::MUTEX_ID)) {
+            Console::stdout("RSS queue execution skipped - already running!\n");
+            return;
+        }
 
 ####### $this->logFileHandle = fopen(dirname(__FILE__) . '/log.txt', 'w');
 
@@ -478,5 +521,6 @@ public function run()
             fclose($this->logFileHandle);
         }
 
+        Yii::$app->mutex->release(static::MUTEX_ID);
     }
 }

migrations/m220202_180401_initial.php

@@ -0,0 +1,31 @@
+<?php
+
+use yii\db\Migration;
+
+class m220202_180401_initial extends Migration
+{
+
+    public function up()
+    {
+        $this->createTable('rss_posts', [
+            'id' => 'pk',
+            'rss_link' => 'varchar(1024) NOT NULL',
+            'post_id' => 'int(11) NOT NULL',
+        ], '');
+
+        // creates index for column `rss_link`
+        // hopefully mysql is smart enaugh to handle varchar(1024) decently
+        $this->createIndex(
+            'idx-rss_posts-rss_link',
+            'rss_posts',
+            'rss_link',
+        );
+    }
+
+    public function down()
+    {
+        echo "m220202_180401_initial does not support migration down.\n";
+        return false;
+    }
+
+}

migrations/uninstall.php

@@ -0,0 +1,18 @@
+<?php
+
+use yii\db\Migration;
+
+class uninstall extends Migration
+{
+    public function up()
+    {
+        $this->dropTable('rss_posts');
+    }
+
+    public function down()
+    {
+        echo "uninstall does not support migration down.\n";
+        return false;
+    }
+
+}

models/RssPosts.php

@@ -0,0 +1,39 @@
+<?php
+
+namespace sij\humhub\modules\rss\models;
+
+use Yii;
+use humhub\components\ActiveRecord;
+
+/**
+ * This is the model class for table "rss_posts".
+ *
+ * The followings are the available columns in table 'rss_posts':
+ * @property integer $id
+ * @property string $rss_link
+ * @property integer $post_id
+ */
+class RssPosts extends ActiveRecord
+{
+
+    /**
+     * @return string the associated database table name
+     */
+    public static function tableName()
+    {
+        return 'rss_posts';
+    }
+
+    /**
+     * @return array validation rules for model attributes.
+     */
+    public function rules()
+    {
+        return [
+            [ ['rss_link', 'post_id'], 'required' ],
+            [ ['rss_link'], 'string' ],
+            [ ['post_id'], 'integer' ],
+        ];
+    }
+
+}

module.json

@@ -4,7 +4,7 @@
     "description": "RSS News Reader for a HumHub Space.",
     "keywords": [
     ],
-    "version": "0.1.2",
+    "version": "0.1.4",
     "humhub": {
         "minVersion": "1.2"
     },