diff --git a/Events.php b/Events.php index e7235b4..8ead483 100644 --- a/Events.php +++ b/Events.php @@ -11,6 +11,10 @@ class Events { + /** + * @var string mutex to acquire + */ + const MUTEX_ID = 'rss-cron'; /** * Defines what to do if admin menu is initialized. @@ -40,8 +44,16 @@ public static function onAdminMenuInit($event) */ public static function onCron($event) { + if ($event->sender->requestedRoute != "cron/run") + return; + + if (! Yii::$app->mutex->acquire(static::MUTEX_ID)) { + Console::stdout("RSS cron execution skipped - already running!\n"); + return; + } + try { - Console::stdout("Updating RSS news feeds...\n"); + Console::stdout("Queueing RSS feeds for spaces:"); $ccmsEnabled = ContentContainerModuleState::find()-> where(['module_id' => 'rss'])-> andWhere(['module_state' => 1])-> @@ -54,14 +66,17 @@ public static function onCron($event) if (! empty($lastrun) && time() < ($interval * 60 + $lastrun)) continue; $space->setSetting('lastrun', time(), 'rss'); - Console::stdout(" Queueing update for space \"" . $space->name . "\"\n"); +# Console::stdout(" Queueing update for space \"" . $space->name . "\"\n"); + Console::stdout(' "' . $space->name . '"'); Yii::$app->queue->push(new jobs\GetFeedUpdates(['space' => $space, 'force' => false])); } - Console::stdout(Console::renderColoredString("%gdone.%n\n", 1)); + Console::stdout(Console::renderColoredString(" %gdone.%n\n", 1)); } catch (\Throwable $e) { $event->sender->stderr($e->getMessage()."\n"); Yii::error($e); } + + Yii::$app->mutex->release(static::MUTEX_ID); } } diff --git a/jobs/GetFeedUpdates.php b/jobs/GetFeedUpdates.php index 40b5903..faf45ee 100644 --- a/jobs/GetFeedUpdates.php +++ b/jobs/GetFeedUpdates.php @@ -3,6 +3,8 @@ namespace sij\humhub\modules\rss\jobs; use Yii; +#use yii\BaseYii; +use yii\helpers\Console; use humhub\modules\queue\ActiveJob; use humhub\modules\post\models\Post; @@ -11,6 +13,7 @@ use sij\humhub\modules\rss\components\MarkdownHelper; use sij\humhub\modules\rss\components\RssElement; use sij\humhub\modules\rss\controllers\RssController; +use sij\humhub\modules\rss\models\RssPosts; /** * Reads the RSS Feed URL for this space @@ -49,6 +52,11 @@ class GetFeedUpdates extends ActiveJob private $newest; # newest date we are accepting private $items; # array of sij\humhub\modules\rss\components\RssElement keyed by pubDate + /** + * @var string mutex to acquire + */ + const MUTEX_ID = 'rss-queue'; + private function log($message) { if ( $this->logFileHandle ) { fwrite($this->logFileHandle, $message); @@ -56,12 +64,15 @@ private function log($message) { } } -/** - * Creates a new Post in the Space - */ + /** + * Creates a new Post in the Space + */ private function postError($title, $info) { + Yii::error("RSS-Reader: ".$title."\n".$info."\n", "RSS-Reader"); + return; + $post = new Post($this->space); $post->created_by = @@ -83,17 +94,27 @@ private function postError($title, $info) } -/** - * Creates a new Post in the Space - */ - private function postMessage($message, $datePublished = false) + /** + * Creates a new Post in the Space + */ + private function postMessage($message, $link = false, $datePublished = false) { - $post = null; - // attempt to locate a previous version of the post if ( $datePublished ) { $stamp = $datePublished->format("Y-m-d H:i:s"); + } + + // find previous version of the post via db + if ( $link ) { + $url2id = RssPosts::findOne(['rss_link' => $link]); + if ( $url2id !== null ) + $post = Post::findOne($url2id->post_id); + } + + // attempt to locate a previous version of the post + // guess this should go some day - themroc + if ( $post === null && $stamp ) { $oldContent = Content::findAll([ 'contentcontainer_id' => $this->space->contentcontainer_id, // 'created_by' => $this->created_by, // cant rely on this field if config changed @@ -101,7 +122,6 @@ private function postMessage($message, $datePublished = false) ]); if ( count($oldContent) == 1 ) { $post = Post::findOne($oldContent[0]->object_id); - $this->log("\n\n### update Post\n"); } } @@ -109,6 +129,17 @@ private function postMessage($message, $datePublished = false) if ( $post === null ) { $post = new Post($this->space); $this->log("\n\n### new Post\n"); + Console::stdout("RSS queue: creating new post ".$link); + } else { + if ( ! $stamp ) { + return; // we assume it hasn't changed - better miss an update than rewrite the post every time. + } + if ($stamp > $post->created_at) { + $this->log("\n\n### update Post\n"); + Console::stdout("RSS queue: updating post ".$link); + } else { + return; // not changed + } } $post->created_by = @@ -126,6 +157,13 @@ private function postMessage($message, $datePublished = false) $post->save(); $this->log(print_r($post, true)); + if (! $url2id ) { + $url2id = new RssPosts(); + $url2id->rss_link = $link; + $url2id->post_id = $post->id; + $url2id->save(); + } + // make it look like the space post was created at the same time as the RSS article // note $post->save() always sets the time stamps to "now" if ( $datePublished ) { @@ -144,6 +182,7 @@ private function postMessage($message, $datePublished = false) ->query(); } + Console::stdout(Console::renderColoredString(" %gdone.%n\n", 1)); } /** @@ -250,7 +289,7 @@ private function parseNewsItem($item) } // post the message in the stream - $this->postMessage($message, $datePublished); + $this->postMessage($message, $link, $datePublished); } @@ -444,6 +483,10 @@ private function downloadNewsFeed() */ public function run() { + if (! Yii::$app->mutex->acquire(static::MUTEX_ID)) { + Console::stdout("RSS queue execution skipped - already running!\n"); + return; + } ####### $this->logFileHandle = fopen(dirname(__FILE__) . '/log.txt', 'w'); @@ -478,5 +521,6 @@ public function run() fclose($this->logFileHandle); } + Yii::$app->mutex->release(static::MUTEX_ID); } } diff --git a/migrations/m220202_180401_initial.php b/migrations/m220202_180401_initial.php new file mode 100644 index 0000000..0a9f5c9 --- /dev/null +++ b/migrations/m220202_180401_initial.php @@ -0,0 +1,31 @@ +createTable('rss_posts', [ + 'id' => 'pk', + 'rss_link' => 'varchar(1024) NOT NULL', + 'post_id' => 'int(11) NOT NULL', + ], ''); + + // creates index for column `rss_link` + // hopefully mysql is smart enaugh to handle varchar(1024) decently + $this->createIndex( + 'idx-rss_posts-rss_link', + 'rss_posts', + 'rss_link', + ); + } + + public function down() + { + echo "m220202_180401_initial does not support migration down.\n"; + return false; + } + +} diff --git a/migrations/uninstall.php b/migrations/uninstall.php new file mode 100644 index 0000000..fa579d1 --- /dev/null +++ b/migrations/uninstall.php @@ -0,0 +1,18 @@ +dropTable('rss_posts'); + } + + public function down() + { + echo "uninstall does not support migration down.\n"; + return false; + } + +} diff --git a/models/RssPosts.php b/models/RssPosts.php new file mode 100644 index 0000000..8707a39 --- /dev/null +++ b/models/RssPosts.php @@ -0,0 +1,39 @@ +