Merge pull request #1722 from hbontempo-br/feature/keep_repos_up_to_date

Adds routine to keep repo up to date

Author: schneems

app/models/github_fetcher/resource.rb

@@ -56,6 +56,10 @@ def success?
       false
     end
 
+    def not_found?
+      status == 404 ? true : false
+    end
+
     def bad_token?
       if status == 401 && response.body.match?(/Bad credentials/)
         @error = @error_message = "Bad credentials"

app/models/repo.rb

@@ -37,7 +37,7 @@ def fetcher
   end
 
   def fetcher_json
-    fetcher.as_json
+    @fetcher_json ||= fetcher.as_json
   end
 
   def issues_fetcher
@@ -192,12 +192,27 @@ def self.exists_with_name?(name)
   end
 
   def update_from_github
-    json = fetcher.as_json
-    self.update(
-      language: json.fetch('language', language),
-      description: json.fetch('description', description)&.first(255),
-      stars_count: json.fetch('stargazers_count', stars_count)
-    )
+    if fetcher.not_found?
+      self.update!(removed_from_github: true)
+    elsif fetcher.success?
+      repo_full_name = fetcher_json.fetch('full_name', full_name)
+
+      if repo_full_name != full_name && self.class.exists_with_name?(repo_full_name)
+        # TODO: Add deduplication step
+        return
+      end
+
+      repo_user_name, repo_name = repo_full_name.split("/")
+      self.update!(
+        name: repo_name,
+        user_name: repo_user_name,
+        language: fetcher_json.fetch('language', language),
+        description: fetcher_json.fetch('description', description)&.first(255),
+        full_name: repo_full_name,
+        removed_from_github: false,
+        archived: fetcher_json.fetch('archived', archived)
+      )
+    end
   end
 
   def repo_path

db/migrate/20221013212959_add_archived_column_to_repo.rb

@@ -0,0 +1,6 @@
+class AddArchivedColumnToRepo < ActiveRecord::Migration[7.0]
+  def change
+    add_column :repos, :archived, :boolean, default: false
+    add_index :repos, :archived
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema[7.0].define(version: 2022_10_10_011048) do
+ActiveRecord::Schema[7.0].define(version: 2022_10_13_212959) do
   # These are extensions that must be enabled in order to support this database
   enable_extension "pg_stat_statements"
   enable_extension "plpgsql"
@@ -155,6 +155,8 @@
     t.integer "subscribers_count", default: 0
     t.integer "docs_subscriber_count", default: 0
     t.boolean "removed_from_github", default: false
+    t.boolean "archived", default: false
+    t.index ["archived"], name: "index_repos_on_archived"
     t.index ["full_name"], name: "index_repos_on_full_name"
     t.index ["issues_count"], name: "index_repos_on_issues_count"
     t.index ["language"], name: "index_repos_on_language"

lib/tasks/schedule.rake

@@ -38,17 +38,12 @@ namespace :schedule do
     return GithubFetcher::Repo.new(user_name: "rails", name: "rails").success?
   end
 
-  desc "Checks if repos have been deleted on GitHub"
-  task mark_removed_repos: :environment do
+  desc "Update repos information"
+  task update_repos: :environment do
     raise "GITHUB API APPEARS TO BE DOWN" unless github_api_up?
 
-    Repo.select(:id, :user_name, :name).find_each(batch_size: 100) do |repo|
-      fetcher = GithubFetcher::Repo.new(user_name: repo.user_name, name: repo.name)
-      fetcher.call(retry_on_bad_token: 5)
-
-      if fetcher.response.status == 404
-        repo.update!(removed_from_github: true)
-      end
+    Repo.find_each(batch_size: 100) do |repo|
+      repo.update_repo_info!
     end
   end
 

test/jobs/update_repo_info_job_test.rb

@@ -1,9 +1,74 @@
 # frozen_string_literal: true
 
 require 'test_helper'
-
 class UpdateRepoInfoJobTest < ActiveJob::TestCase
-  # test "the truth" do
-  #   assert true
-  # end
+  test 'repo deleted or made private' do
+    GithubFetcher::Resource.any_instance.stubs(:status).returns(404)
+    repo = repos(:node)
+    assert_changes -> {
+      [
+        repo.removed_from_github,
+      ]
+    } do
+      UpdateRepoInfoJob.perform_now(repo)
+      repo.reload
+    end
+    assert repo.removed_from_github
+  end
+
+  test 'repo with information updated' do
+    GithubFetcher::Resource.any_instance.stubs(:status).returns(200)
+    GithubFetcher::Resource.any_instance.stubs(:as_json).returns(
+      {
+        'full_name' => 'test_owner/test_repo',
+        'language' => 'test_language',
+        'description' => 'test_description',
+        'archived' => true
+      }
+    )
+    repo = repos(:node)
+    assert_changes -> {
+      [
+        repo.full_name,
+        repo.name,
+        repo.user_name,
+        repo.language,
+        repo.description,
+        repo.archived
+      ]
+    } do
+      UpdateRepoInfoJob.perform_now(repo)
+      repo.reload
+    end
+    assert_equal false, repo.removed_from_github
+    assert_equal 'test_owner/test_repo', repo.full_name
+    assert_equal 'test_repo', repo.name
+    assert_equal 'test_owner', repo.user_name
+    assert_equal 'test_language', repo.language
+    assert_equal 'test_description', repo.description
+    assert_equal true, repo.archived
+  end
+
+  test 'repo rename conflict' do
+    GithubFetcher::Resource.any_instance.stubs(:status).returns(200)
+    GithubFetcher::Resource.any_instance.stubs(:as_json).returns(
+      {
+        'full_name' => 'sinatra/sinatra',
+      }
+    )
+    repo = repos(:node)
+    assert_no_changes -> {
+      [
+        repo.full_name,
+        repo.name,
+        repo.user_name,
+        repo.language,
+        repo.description,
+        repo.archived
+      ]
+    } do
+      UpdateRepoInfoJob.perform_now(repo)
+      repo.reload
+    end
+  end
 end

test/unit/github_fetcher/resource_test.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+require 'test_helper'
+
+class GithubFetcher::ResourceTest < ActiveSupport::TestCase
+  def status_validation(mocked_status, method, expected_response)
+    GithubFetcher::Resource.any_instance.stubs(:status).returns(mocked_status)
+    resource = GithubFetcher::Resource.new({})
+    assert_equal expected_response, resource.send(method)
+  end
+
+  test '#not_found? is true on 404 status responses' do
+    status_validation(404, :not_found?, true)
+  end
+
+  test '#not_found? is false on non 404 status responses' do
+    [200, 201, 204, 400, 403, 500, 502].each { |status| status_validation(status, :not_found?, false) }
+  end
+end