Add hexdocs.pm/package_names.csv (#54)

Author: wojtekmach

.github/workflows/main.yml

@@ -29,7 +29,7 @@ jobs:
 
       - name: Run tests
         run: |
-          mix test --include typesense
+          mix test.all
 
   docker:
     name: Docker

config/config.exs

@@ -12,6 +12,7 @@ config :hexdocs,
   cdn_impl: Hexdocs.CDN.Local,
   search_impl: Hexdocs.Search.Local,
   source_repo_impl: Hexdocs.SourceRepo.GitHub,
+  hex_repo_impl: Hexdocs.HexRepo.HTTP,
   tmp_dir: "tmp",
   queue_id: "test",
   queue_producer: Broadway.DummyProducer,

config/test.exs

@@ -7,6 +7,7 @@ config :hexdocs,
   store_impl: Hexdocs.Store.Local,
   cdn_impl: Hexdocs.CDN.Local,
   search_impl: Hexdocs.Search.Local,
-  source_repo_impl: Hexdocs.SourceRepo.Mock
+  source_repo_impl: Hexdocs.SourceRepo.Mock,
+  hex_repo_impl: Hexdocs.HexRepo.Mock
 
 config :logger, level: :warning

lib/hexdocs/bucket.ex

@@ -5,14 +5,18 @@ defmodule Hexdocs.Bucket do
   @gcs_put_debounce Application.compile_env!(:hexdocs, :gcs_put_debounce)
 
   def upload_index_sitemap(sitemap) do
-    upload_sitemap("sitemap", "sitemap.xml", sitemap)
+    upload_content("sitemap", "sitemap.xml", sitemap)
   end
 
   def upload_package_sitemap(package, sitemap) do
-    upload_sitemap("sitemap/#{package}", "#{package}/sitemap.xml", sitemap)
+    upload_content("sitemap/#{package}", "#{package}/sitemap.xml", sitemap)
   end
 
-  defp upload_sitemap(key, path, sitemap) do
+  def upload_package_names_csv(contents) do
+    upload_content("package_names.csv", "package_names.csv", contents)
+  end
+
+  defp upload_content(key, path, content) do
     opts = [
       content_type: "text/xml",
       cache_control: "public, max-age=3600",
@@ -21,7 +25,7 @@ defmodule Hexdocs.Bucket do
 
     Logger.info("Uploading docs_public_bucket #{path}")
 
-    case Hexdocs.Store.put(:docs_public_bucket, path, sitemap, opts) do
+    case Hexdocs.Store.put(:docs_public_bucket, path, content, opts) do
       {:ok, 200, _headers, _body} ->
         :ok
 

lib/hexdocs/hex_repo.ex

@@ -0,0 +1,32 @@
+defmodule Hexdocs.HexRepo do
+  @callback get_names() :: {:ok, [binary()]} | {:error, term()}
+
+  @module Application.fetch_env!(:hexdocs, :hex_repo_impl)
+
+  defdelegate get_names(), to: @module
+end
+
+defmodule Hexdocs.HexRepo.HTTP do
+  require Logger
+
+  def get_names do
+    case :hex_repo.get_names(:hex_core.default_config()) do
+      {:ok, {200, _, body}} ->
+        %{repository: "hexpm", packages: packages} = body
+        names = for package <- packages, do: package.name
+        {:ok, names}
+
+      {:ok, {status, _headers, body}} ->
+        message = """
+        unexpected HTTP #{status}
+
+        #{body}\
+        """
+
+        {:error, message}
+
+      {:error, _reason} = error ->
+        error
+    end
+  end
+end

lib/hexdocs/queue.ex

@@ -133,6 +133,7 @@ defmodule Hexdocs.Queue do
             if Hexdocs.Utils.latest_version?(package, version, all_versions) do
               update_index_sitemap(repository, key)
               update_package_sitemap(repository, key, package, files)
+              update_package_names_csv(repository)
             end
 
             if repository == "hexpm" do
@@ -253,6 +254,25 @@ defmodule Hexdocs.Queue do
     :ok
   end
 
+  defp update_package_names_csv("hexpm") do
+    Logger.info("UPDATING package_names.csv")
+
+    case Hexdocs.HexRepo.get_names() do
+      {:ok, names} ->
+        csv = for name <- names, do: [name, "\n"]
+        Hexdocs.Bucket.upload_package_names_csv(csv)
+
+      {:error, reason} ->
+        Logger.error(inspect(reason))
+    end
+
+    Logger.info("UPDATED package_names.csv")
+  end
+
+  defp update_package_names_csv(_repository) do
+    :ok
+  end
+
   defp update_search_index(key, package, version, files) do
     with {proglang, items} <- Hexdocs.Search.find_search_items(package, version, files) do
       Logger.info("UPDATING SEARCH INDEX #{key}")

mix.exs

@@ -8,17 +8,32 @@ defmodule Hexdocs.MixProject do
       elixir: "~> 1.18",
       start_permanent: Mix.env() == :prod,
       releases: releases(),
+      aliases: aliases(),
       deps: deps()
     ]
   end
 
   def application do
     [
-      extra_applications: [:eex, :logger, :runtime_tools],
+      extra_applications: [:eex, :logger, :runtime_tools, :inets],
       mod: {Hexdocs.Application, []}
     ]
   end
 
+  def cli do
+    [
+      preferred_envs: [
+        "test.all": :test
+      ]
+    ]
+  end
+
+  defp aliases do
+    [
+      "test.all": ["test --include typesense --include integration"]
+    ]
+  end
+
   defp deps do
     [
       {:broadway, "~> 1.0"},
@@ -34,7 +49,7 @@ defmodule Hexdocs.MixProject do
       {:sentry, "~> 10.8"},
       {:ssl_verify_fun, "~> 1.1", manager: :rebar3, override: true},
       {:sweet_xml, "~> 0.7.0"},
-      {:hex_core, "~> 0.9.0", only: [:dev, :test]},
+      {:hex_core, "~> 0.11.0"},
       {:mox, "~> 1.0", only: :test}
     ]
   end

mix.lock

@@ -13,7 +13,7 @@
   "gen_stage": {:hex, :gen_stage, "1.2.1", "19d8b5e9a5996d813b8245338a28246307fd8b9c99d1237de199d21efc4c76a1", [:mix], [], "hexpm", "83e8be657fa05b992ffa6ac1e3af6d57aa50aace8f691fcf696ff02f8335b001"},
   "goth": {:hex, :goth, "1.4.0", "29713dd3390021c6ccd2b8aa9fe7d8570b0816d807c5d84fa3eb06a4abfa29ce", [:mix], [{:finch, "~> 0.9", [hex: :finch, repo: "hexpm", optional: false]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:jose, "~> 1.10", [hex: :jose, repo: "hexpm", optional: false]}], "hexpm", "1bf684f1fed7d31c46e49c1dcd4dc5cca454ef748abea2410a49cd8621fbede1"},
   "hackney": {:hex, :hackney, "1.20.1", "8d97aec62ddddd757d128bfd1df6c5861093419f8f7a4223823537bad5d064e2", [:rebar3], [{:certifi, "~> 2.12.0", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "~> 6.1.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "~> 1.0.0", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~> 1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:parse_trans, "3.4.1", [hex: :parse_trans, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "~> 1.1.0", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}, {:unicode_util_compat, "~> 0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "fe9094e5f1a2a2c0a7d10918fee36bfec0ec2a979994cff8cfe8058cd9af38e3"},
-  "hex_core": {:hex, :hex_core, "0.9.0", "ba2b3b8f3ff821b9bad0c9daf4d6f76807bb4a13185e2a2ee6ad89f76fc5753c", [:rebar3], [], "hexpm", "f160418b48511a08dfdb473814a8c8f95ed109878a74649464f13816036cc2f1"},
+  "hex_core": {:hex, :hex_core, "0.11.0", "d1c6bbf2a4ee6b5f002bec6fa52b5080c53c8b63b7caf6eb88b943687547bff4", [:rebar3], [], "hexpm", "707893677a425491962a2db522f1d2b1f85f97ea27418b06f7929f1d30cde0b0"},
   "hpax": {:hex, :hpax, "0.1.2", "09a75600d9d8bbd064cdd741f21fc06fc1f4cf3d0fcc335e5aa19be1a7235c84", [:mix], [], "hexpm", "2c87843d5a23f5f16748ebe77969880e29809580efdaccd615cd3bed628a8c13"},
   "idna": {:hex, :idna, "6.1.1", "8a63070e9f7d0c62eb9d9fcb360a7de382448200fbbd1b106cc96d3d8099df8d", [:rebar3], [{:unicode_util_compat, "~> 0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "92376eb7894412ed19ac475e4a86f7b413c1b9fbb5bd16dccd57934157944cea"},
   "jason": {:hex, :jason, "1.4.4", "b9226785a9aa77b6857ca22832cffa5d5011a667207eb2a0ad56adb5db443b8a", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "c5eb0cab91f094599f94d55bc63409236a8ec69a21a67814529e8d5f6cc90b3b"},

test/hexdocs/hex_repo_test.exs

@@ -0,0 +1,10 @@
+defmodule Hexdocs.HexRepo.HTTPTest do
+  use ExUnit.Case, async: true
+
+  @moduletag :integration
+
+  test "it works" do
+    assert {:ok, names} = Hexdocs.HexRepo.HTTP.get_names()
+    assert "hex_core" in names
+  end
+end

test/hexdocs/queue_test.exs

@@ -12,6 +12,10 @@ defmodule Hexdocs.QueueTest do
       "this is the sitemap"
     end)
 
+    Mox.stub(Hexdocs.HexRepo.Mock, :get_names, fn ->
+      {:ok, ["package1", "package2"]}
+    end)
+
     :ok
   end
 
@@ -58,6 +62,7 @@ defmodule Hexdocs.QueueTest do
       Store.put!(:repo_bucket, key, tar)
 
       ref = Broadway.test_message(Hexdocs.Queue, put_message(key))
+
       assert_receive {:ack, ^ref, [_], []}
 
       assert ls(@public_bucket, "#{test}/") == [
@@ -69,6 +74,7 @@ defmodule Hexdocs.QueueTest do
 
       assert Store.get(@public_bucket, "#{test}/index.html") == "contents"
       assert Store.get(@public_bucket, "#{test}/1.0.0/index.html") == "contents"
+      assert Store.get(@public_bucket, "package_names.csv") == "package1\npackage2\n"
     end
 
     @tag :capture_log