diff --git a/lib/hexdocs.ex b/lib/hexdocs.ex index cbd8a75..59c6d74 100644 --- a/lib/hexdocs.ex +++ b/lib/hexdocs.ex @@ -1,32 +1,32 @@ defmodule Hexdocs do @key_regex ~r"docs/(.*)-(.*).tar.gz$" - def process_object(key) do - key - |> build_message() - |> send_message() + def process_object(type, key) when type in [:upload, :search] do + publish_message(type, key) end - def process_all_objects() do + def process_all_objects(type) when type in [:upload, :search] do (Hexdocs.Store.list(:repo_bucket, "docs/") ++ Hexdocs.Store.list(:repo_bucket, "repos/")) |> Enum.shuffle() - |> batched_send() - end - - defp build_message(key) do - %{ - "Records" => [%{"eventName" => "ObjectCreated:Put", "s3" => %{"object" => %{"key" => key}}}] - } + |> batched_send(type) end - def batched_send(keys) do + def batched_send(keys, type) when type in [:upload, :search] do keys |> Stream.filter(&Regex.match?(@key_regex, &1)) - |> Stream.map(&build_message/1) + |> Stream.map(&build_message(type, &1)) |> Task.async_stream(&send_message/1, max_concurrency: 10, ordered: false, timeout: 60_000) |> Stream.run() end + defp build_message(:upload, key) do + %{"hexdocs:upload" => key} + end + + defp build_message(:search, key) do + %{"hexdocs:search" => key} + end + defp send_message(map) do queue = Application.fetch_env!(:hexdocs, :queue_id) message = Jason.encode!(map) @@ -41,4 +41,22 @@ defmodule Hexdocs do |> Task.async_stream(&send_message/1, max_concurrency: 10, ordered: false, timeout: 60_000) |> Stream.run() end + + def publish_upload_message(key) do + publish_message(:upload, key) + end + + def publish_search_message(key) do + publish_message(:search, key) + end + + def publish_message(:upload, key) do + %{"hexdocs:upload" => key} + |> send_message() + end + + def publish_message(:search, key) do + %{"hexdocs:search" => key} + |> send_message() + end end diff --git a/lib/hexdocs/bucket.ex b/lib/hexdocs/bucket.ex index 206b307..6c3db11 100644 --- a/lib/hexdocs/bucket.ex +++ b/lib/hexdocs/bucket.ex @@ -305,10 +305,6 @@ defmodule Hexdocs.Bucket do |> purge() end - defp purge_hexdocs_cache("hexpm", package, _versions, :unversioned) do - purge([docspage_unversioned_cdn_key("hexpm", package)]) - end - defp purge_hexdocs_cache(_repository, _package, _version, _publish_unversioned?) do :ok end diff --git a/lib/hexdocs/hex_repo.ex b/lib/hexdocs/hex_repo.ex index 3198c12..bee1aaf 100644 --- a/lib/hexdocs/hex_repo.ex +++ b/lib/hexdocs/hex_repo.ex @@ -1,8 +1,9 @@ defmodule Hexdocs.HexRepo do @callback get_names() :: {:ok, [binary()]} | {:error, term()} - @module Application.fetch_env!(:hexdocs, :hex_repo_impl) + @module Application.compile_env(:hexdocs, :hex_repo_impl) + @compile {:no_warn_undefined, [{@module, :get_names, 0}]} defdelegate get_names(), to: @module end diff --git a/lib/hexdocs/queue.ex b/lib/hexdocs/queue.ex index 6b2e1d0..7745f6b 100644 --- a/lib/hexdocs/queue.ex +++ b/lib/hexdocs/queue.ex @@ -77,20 +77,28 @@ defmodule Hexdocs.Queue do message end - @impl true - def handle_batch(_batcher, messages, _batch_info, _context) do - messages + def handle_message(%{data: %{"hexdocs:upload" => key}} = message) do + process_docs(key, :upload) + message end - defp handle_record(%{"eventName" => "ObjectCreated:" <> _, "s3" => s3}) do + def handle_message(%{data: %{"hexdocs:search" => key}} = message) do + process_docs(key, :search) + message + end + + defp process_docs(key, type) do start = System.os_time(:millisecond) - key = s3["object"]["key"] - Logger.info("OBJECT CREATED #{key}") + event_name = if type == :upload, do: "hexdocs:upload", else: "hexdocs:search" + log_prefix = if type == :upload, do: "UPLOAD", else: "SEARCH INDEX" + + Sentry.Context.set_extra_context(%{queue_event: event_name}) + Logger.info("#{log_prefix} #{key}") case key_components(key) do {:ok, repository, package, version} -> Sentry.Context.set_extra_context(%{ - queue_event: "ObjectCreated", + queue_event: event_name, repository: repository, package: package, version: version @@ -98,55 +106,107 @@ defmodule Hexdocs.Queue do body = Hexdocs.Store.get(:repo_bucket, key) - {version, all_versions} = - if package in @special_package_names do - version = - case Version.parse(version) do - {:ok, version} -> - version - - # main or MAJOR.MINOR - :error -> - version - end - - all_versions = Hexdocs.SourceRepo.versions!(Map.fetch!(@special_packages, package)) - {version, all_versions} - else - version = Version.parse!(version) - all_versions = all_versions(repository, package) - {version, all_versions} + case type do + :upload -> + process_upload(key, repository, package, version, body, start) + + :search -> + process_search(key, package, version, body, start) + end + + :error -> + Logger.info("#{key}: skip") + end + end + + defp process_upload(key, repository, package, version, body, start) do + {version, all_versions} = + if package in @special_package_names do + version = + case Version.parse(version) do + {:ok, version} -> + version + + # main or MAJOR.MINOR + :error -> + version end - case Hexdocs.Tar.unpack(body, repository: repository, package: package, version: version) do - {:ok, files} -> - files = rewrite_files(files) + all_versions = Hexdocs.SourceRepo.versions!(Map.fetch!(@special_packages, package)) + {version, all_versions} + else + version = Version.parse!(version) + all_versions = all_versions(repository, package) + {version, all_versions} + end + + case Hexdocs.Tar.unpack(body, repository: repository, package: package, version: version) do + {:ok, files} -> + files = rewrite_files(files) + + Hexdocs.Bucket.upload( + repository, + package, + version, + all_versions, + files + ) + + if Hexdocs.Utils.latest_version?(package, version, all_versions) do + update_index_sitemap(repository, key) + update_package_sitemap(repository, key, package, files) + update_package_names_csv(repository) + end - Hexdocs.Bucket.upload( - repository, - package, - version, - all_versions, - files - ) + elapsed = System.os_time(:millisecond) - start + Logger.info("FINISHED UPLOADING DOCS #{key} #{elapsed}ms") - if Hexdocs.Utils.latest_version?(package, version, all_versions) do - update_index_sitemap(repository, key) - update_package_sitemap(repository, key, package, files) - update_package_names_csv(repository) - end + {:error, reason} -> + Logger.error("Failed unpack #{repository}/#{package} #{version}: #{reason}") + end + end - if repository == "hexpm" do - update_search_index(key, package, version, files) - end + defp process_search(key, package, version, body, start) do + version = Version.parse!(version) - elapsed = System.os_time(:millisecond) - start - Logger.info("FINISHED UPLOADING AND INDEXING DOCS #{key} #{elapsed}ms") + case Hexdocs.Tar.unpack(body, package: package, version: version) do + {:ok, files} -> + update_search_index(key, package, version, files) - {:error, reason} -> - Logger.error("Failed unpack #{repository}/#{package} #{version}: #{reason}") + elapsed = System.os_time(:millisecond) - start + Logger.info("FINISHED INDEXING DOCS #{key} #{elapsed}ms") + + {:error, reason} -> + Logger.error("Failed unpack #{package} #{version}: #{reason}") + end + end + + @impl true + def handle_batch(_batcher, messages, _batch_info, _context) do + messages + end + + defp handle_record(%{"eventName" => "ObjectCreated:" <> _, "s3" => s3}) do + key = s3["object"]["key"] + Logger.info("OBJECT CREATED #{key}") + + case key_components(key) do + {:ok, repository, _package, _version} -> + Sentry.Context.set_extra_context(%{ + queue_event: "ObjectCreated", + repository: repository + }) + + # Publish upload message + publish_message(%{"hexdocs:upload" => key}) + + # Publish search message for hexpm repository + if repository == "hexpm" do + publish_message(%{"hexdocs:search" => key}) end + Logger.info("PUBLISHED MESSAGES FOR #{key}") + :error -> :skip end @@ -281,6 +341,14 @@ defmodule Hexdocs.Queue do end end + defp publish_message(map) do + queue = Application.fetch_env!(:hexdocs, :queue_id) + message = Jason.encode!(map) + + ExAws.SQS.send_message(queue, message) + |> ExAws.request!() + end + @doc false def paths_for_sitemaps() do key_regex = ~r"docs/(.*)-(.*).tar.gz$" diff --git a/lib/hexdocs/search/search.ex b/lib/hexdocs/search/search.ex index b2e859f..b85e507 100644 --- a/lib/hexdocs/search/search.ex +++ b/lib/hexdocs/search/search.ex @@ -42,8 +42,7 @@ defmodule Hexdocs.Search do json _ when is_binary(search_data_js) -> - Logger.error("Unexpected search_data format for #{package} #{version}") - nil + raise "Unexpected search_data format for #{package} #{version}" nil -> nil @@ -55,12 +54,7 @@ defmodule Hexdocs.Search do :json.decode(search_data_json) catch _kind, reason -> - Logger.error( - "Failed to decode search data json for #{package} #{version}: " <> - inspect(reason) - ) - - nil + raise "Failed to decode search data json for #{package} #{version}: #{inspect(reason)}" end end @@ -73,11 +67,7 @@ defmodule Hexdocs.Search do nil _ -> - Logger.error( - "Failed to extract search items and proglang from search data for #{package} #{version}" - ) - - nil + raise "Failed to extract search items and proglang from search data for #{package} #{version}" end end diff --git a/lib/hexdocs/search/typesense.ex b/lib/hexdocs/search/typesense.ex index 8c3b43e..c38e77a 100644 --- a/lib/hexdocs/search/typesense.ex +++ b/lib/hexdocs/search/typesense.ex @@ -33,17 +33,15 @@ defmodule Hexdocs.Search.Typesense do :ok %{"success" => false, "error" => error, "document" => document} -> - Logger.error( - "Failed to index search item for #{package} #{version} for document #{inspect(document)}: #{inspect(error)}" - ) + raise "Failed to index search item for #{package} #{version} for document #{inspect(document)}: #{inspect(error)}" end end) {:ok, status, _resp_headers, _body} -> - Logger.error("Failed to index search items for #{package} #{version}: status=#{status}") + raise "Failed to index search items for #{package} #{version}: status=#{status}" {:error, reason} -> - Logger.error("Failed to index search items #{package} #{version}: #{inspect(reason)}") + raise "Failed to index search items #{package} #{version}: #{inspect(reason)}" end end @@ -60,12 +58,10 @@ defmodule Hexdocs.Search.Typesense do :ok {:ok, status, _resp_headers, _body} -> - Logger.error("Failed to delete search items for #{package} #{version}: status=#{status}") + raise "Failed to delete search items for #{package} #{version}: status=#{status}" {:error, reason} -> - Logger.error( - "Failed to delete search items for #{package} #{version}: #{inspect(reason)}" - ) + raise "Failed to delete search items for #{package} #{version}: #{inspect(reason)}" end end diff --git a/mix.exs b/mix.exs index 7b78b7d..1bedf2f 100644 --- a/mix.exs +++ b/mix.exs @@ -46,7 +46,7 @@ defmodule Hexdocs.MixProject do {:jason, "~> 1.1"}, {:logster, "~> 1.0"}, {:plug_cowboy, "~> 2.0"}, - {:sentry, "~> 10.8"}, + {:sentry, "~> 11.0"}, {:ssl_verify_fun, "~> 1.1", manager: :rebar3, override: true}, {:sweet_xml, "~> 0.7.0"}, {:hex_core, "~> 0.11.0"}, diff --git a/mix.lock b/mix.lock index 5e58bc0..42b860d 100644 --- a/mix.lock +++ b/mix.lock @@ -1,41 +1,40 @@ %{ - "broadway": {:hex, :broadway, "1.1.0", "8ed3aea01fd6f5640b3e1515b90eca51c4fc1fac15fb954cdcf75dc054ae719c", [:mix], [{:gen_stage, "~> 1.0", [hex: :gen_stage, repo: "hexpm", optional: false]}, {:nimble_options, "~> 0.3.7 or ~> 0.4 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "25e315ef1afe823129485d981dcc6d9b221cea30e625fd5439e9b05f44fb60e4"}, + "broadway": {:hex, :broadway, "1.2.1", "83a1567423c26885e15f6cd8670ca790370af2fcff2ede7fa88c5ea793087a67", [:mix], [{:gen_stage, "~> 1.0", [hex: :gen_stage, repo: "hexpm", optional: false]}, {:nimble_options, "~> 0.3.7 or ~> 0.4 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "68ae63d83b55bdca0f95cd49feee5fb74c5a6bec557caf940860fe07dbc8a4fb"}, "broadway_sqs": {:hex, :broadway_sqs, "0.7.4", "ab89b298f9253adb8534f92095b56d4879e35fe2f5a0730256f7e824572c637f", [:mix], [{:broadway, "~> 1.0", [hex: :broadway, repo: "hexpm", optional: false]}, {:ex_aws_sqs, "~> 3.2.1 or ~> 3.3", [hex: :ex_aws_sqs, repo: "hexpm", optional: false]}, {:nimble_options, "~> 0.3.7 or ~> 0.4 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:saxy, "~> 1.1", [hex: :saxy, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "7140085c4f7c4b27886b3a8f3d0942976f39f195fdbc2f652c5d7b157f93ae28"}, - "castore": {:hex, :castore, "1.0.1", "240b9edb4e9e94f8f56ab39d8d2d0a57f49e46c56aced8f873892df8ff64ff5a", [:mix], [], "hexpm", "b4951de93c224d44fac71614beabd88b71932d0b1dea80d2f80fb9044e01bbb3"}, - "certifi": {:hex, :certifi, "2.12.0", "2d1cca2ec95f59643862af91f001478c9863c2ac9cb6e2f89780bfd8de987329", [:rebar3], [], "hexpm", "ee68d85df22e554040cdb4be100f33873ac6051387baf6a8f6ce82272340ff1c"}, - "cowboy": {:hex, :cowboy, "2.10.0", "ff9ffeff91dae4ae270dd975642997afe2a1179d94b1887863e43f681a203e26", [:make, :rebar3], [{:cowlib, "2.12.1", [hex: :cowlib, repo: "hexpm", optional: false]}, {:ranch, "1.8.0", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "3afdccb7183cc6f143cb14d3cf51fa00e53db9ec80cdcd525482f5e99bc41d6b"}, + "certifi": {:hex, :certifi, "2.15.0", "0e6e882fcdaaa0a5a9f2b3db55b1394dba07e8d6d9bcad08318fb604c6839712", [:rebar3], [], "hexpm", "b147ed22ce71d72eafdad94f055165c1c182f61a2ff49df28bcc71d1d5b94a60"}, + "cowboy": {:hex, :cowboy, "2.14.2", "4008be1df6ade45e4f2a4e9e2d22b36d0b5aba4e20b0a0d7049e28d124e34847", [:make, :rebar3], [{:cowlib, ">= 2.16.0 and < 3.0.0", [hex: :cowlib, repo: "hexpm", optional: false]}, {:ranch, ">= 1.8.0 and < 3.0.0", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "569081da046e7b41b5df36aa359be71a0c8874e5b9cff6f747073fc57baf1ab9"}, "cowboy_telemetry": {:hex, :cowboy_telemetry, "0.4.0", "f239f68b588efa7707abce16a84d0d2acf3a0f50571f8bb7f56a15865aae820c", [:rebar3], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "7d98bac1ee4565d31b62d59f8823dfd8356a169e7fcbb83831b8a5397404c9de"}, - "cowlib": {:hex, :cowlib, "2.12.1", "a9fa9a625f1d2025fe6b462cb865881329b5caff8f1854d1cbc9f9533f00e1e1", [:make, :rebar3], [], "hexpm", "163b73f6367a7341b33c794c4e88e7dbfe6498ac42dcd69ef44c5bc5507c8db0"}, - "ex_aws": {:hex, :ex_aws, "2.5.8", "0393cfbc5e4a9e7017845451a015d836a670397100aa4c86901980e2a2c5f7d4", [:mix], [{:configparser_ex, "~> 4.0", [hex: :configparser_ex, repo: "hexpm", optional: true]}, {:hackney, "~> 1.16", [hex: :hackney, repo: "hexpm", optional: true]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: true]}, {:jsx, "~> 2.8 or ~> 3.0", [hex: :jsx, repo: "hexpm", optional: true]}, {:mime, "~> 1.2 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:req, "~> 0.3", [hex: :req, repo: "hexpm", optional: true]}, {:sweet_xml, "~> 0.7", [hex: :sweet_xml, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "8f79777b7932168956c8cc3a6db41f5783aa816eb50de356aed3165a71e5f8c3"}, - "ex_aws_s3": {:hex, :ex_aws_s3, "2.4.0", "ce8decb6b523381812798396bc0e3aaa62282e1b40520125d1f4eff4abdff0f4", [:mix], [{:ex_aws, "~> 2.0", [hex: :ex_aws, repo: "hexpm", optional: false]}, {:sweet_xml, ">= 0.0.0", [hex: :sweet_xml, repo: "hexpm", optional: true]}], "hexpm", "85dda6e27754d94582869d39cba3241d9ea60b6aa4167f9c88e309dc687e56bb"}, + "cowlib": {:hex, :cowlib, "2.16.0", "54592074ebbbb92ee4746c8a8846e5605052f29309d3a873468d76cdf932076f", [:make, :rebar3], [], "hexpm", "7f478d80d66b747344f0ea7708c187645cfcc08b11aa424632f78e25bf05db51"}, + "ex_aws": {:hex, :ex_aws, "2.6.0", "346e87e35e5df0b3c016a96fb30adf6001de102981a71648dfc3ce3ad04765af", [:mix], [{:configparser_ex, "~> 5.0", [hex: :configparser_ex, repo: "hexpm", optional: true]}, {:hackney, "~> 1.16", [hex: :hackney, repo: "hexpm", optional: true]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: true]}, {:jsx, "~> 2.8 or ~> 3.0", [hex: :jsx, repo: "hexpm", optional: true]}, {:mime, "~> 1.2 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:req, "~> 0.5.10 or ~> 0.6 or ~> 1.0", [hex: :req, repo: "hexpm", optional: true]}, {:sweet_xml, "~> 0.7", [hex: :sweet_xml, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "30729ee9cbaacda674a4e4260d74206fa89bcd712267c4eaf42a0fc34592c0b3"}, + "ex_aws_s3": {:hex, :ex_aws_s3, "2.5.8", "5ee7407bc8252121ad28fba936b3b293f4ecef93753962351feb95b8a66096fa", [:mix], [{:ex_aws, "~> 2.0", [hex: :ex_aws, repo: "hexpm", optional: false]}, {:sweet_xml, ">= 0.0.0", [hex: :sweet_xml, repo: "hexpm", optional: true]}], "hexpm", "84e512ca2e0ae6a6c497036dff06d4493ffb422cfe476acc811d7c337c16691c"}, "ex_aws_sqs": {:hex, :ex_aws_sqs, "3.4.0", "f7c4d0177c1c954776363d3dc05e5dfd37ddf0e2c65ec3f047e5c9c7dd1b71ac", [:mix], [{:ex_aws, "~> 2.1", [hex: :ex_aws, repo: "hexpm", optional: false]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: true]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: true]}, {:saxy, "~> 1.1", [hex: :saxy, repo: "hexpm", optional: true]}, {:sweet_xml, ">= 0.0.0", [hex: :sweet_xml, repo: "hexpm", optional: true]}], "hexpm", "b504482206ccaf767b714888e9d41a1cfcdcb241577985517114191c812f155a"}, - "finch": {:hex, :finch, "0.16.0", "40733f02c89f94a112518071c0a91fe86069560f5dbdb39f9150042f44dcfb1a", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: false]}, {:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.3", [hex: :mint, repo: "hexpm", optional: false]}, {:nimble_options, "~> 0.4 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:nimble_pool, "~> 0.2.6 or ~> 1.0", [hex: :nimble_pool, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "f660174c4d519e5fec629016054d60edd822cdfe2b7270836739ac2f97735ec5"}, - "gen_stage": {:hex, :gen_stage, "1.2.1", "19d8b5e9a5996d813b8245338a28246307fd8b9c99d1237de199d21efc4c76a1", [:mix], [], "hexpm", "83e8be657fa05b992ffa6ac1e3af6d57aa50aace8f691fcf696ff02f8335b001"}, - "goth": {:hex, :goth, "1.4.0", "29713dd3390021c6ccd2b8aa9fe7d8570b0816d807c5d84fa3eb06a4abfa29ce", [:mix], [{:finch, "~> 0.9", [hex: :finch, repo: "hexpm", optional: false]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:jose, "~> 1.10", [hex: :jose, repo: "hexpm", optional: false]}], "hexpm", "1bf684f1fed7d31c46e49c1dcd4dc5cca454ef748abea2410a49cd8621fbede1"}, - "hackney": {:hex, :hackney, "1.20.1", "8d97aec62ddddd757d128bfd1df6c5861093419f8f7a4223823537bad5d064e2", [:rebar3], [{:certifi, "~> 2.12.0", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "~> 6.1.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "~> 1.0.0", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~> 1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:parse_trans, "3.4.1", [hex: :parse_trans, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "~> 1.1.0", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}, {:unicode_util_compat, "~> 0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "fe9094e5f1a2a2c0a7d10918fee36bfec0ec2a979994cff8cfe8058cd9af38e3"}, + "finch": {:hex, :finch, "0.20.0", "5330aefb6b010f424dcbbc4615d914e9e3deae40095e73ab0c1bb0968933cadf", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.6.2 or ~> 1.7", [hex: :mint, repo: "hexpm", optional: false]}, {:nimble_options, "~> 0.4 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:nimble_pool, "~> 1.1", [hex: :nimble_pool, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "2658131a74d051aabfcba936093c903b8e89da9a1b63e430bee62045fa9b2ee2"}, + "gen_stage": {:hex, :gen_stage, "1.3.2", "7c77e5d1e97de2c6c2f78f306f463bca64bf2f4c3cdd606affc0100b89743b7b", [:mix], [], "hexpm", "0ffae547fa777b3ed889a6b9e1e64566217413d018cabd825f786e843ffe63e7"}, + "goth": {:hex, :goth, "1.4.5", "ee37f96e3519bdecd603f20e7f10c758287088b6d77c0147cd5ee68cf224aade", [:mix], [{:finch, "~> 0.17", [hex: :finch, repo: "hexpm", optional: false]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:jose, "~> 1.11", [hex: :jose, repo: "hexpm", optional: false]}], "hexpm", "0fc2dce5bd710651ed179053d0300ce3a5d36afbdde11e500d57f05f398d5ed5"}, + "hackney": {:hex, :hackney, "1.25.0", "390e9b83f31e5b325b9f43b76e1a785cbdb69b5b6cd4e079aa67835ded046867", [:rebar3], [{:certifi, "~> 2.15.0", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "~> 6.1.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "~> 1.0.0", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~> 1.4", [hex: :mimerl, repo: "hexpm", optional: false]}, {:parse_trans, "3.4.1", [hex: :parse_trans, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "~> 1.1.0", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}, {:unicode_util_compat, "~> 0.7.1", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "7209bfd75fd1f42467211ff8f59ea74d6f2a9e81cbcee95a56711ee79fd6b1d4"}, "hex_core": {:hex, :hex_core, "0.11.0", "d1c6bbf2a4ee6b5f002bec6fa52b5080c53c8b63b7caf6eb88b943687547bff4", [:rebar3], [], "hexpm", "707893677a425491962a2db522f1d2b1f85f97ea27418b06f7929f1d30cde0b0"}, - "hpax": {:hex, :hpax, "0.1.2", "09a75600d9d8bbd064cdd741f21fc06fc1f4cf3d0fcc335e5aa19be1a7235c84", [:mix], [], "hexpm", "2c87843d5a23f5f16748ebe77969880e29809580efdaccd615cd3bed628a8c13"}, + "hpax": {:hex, :hpax, "1.0.3", "ed67ef51ad4df91e75cc6a1494f851850c0bd98ebc0be6e81b026e765ee535aa", [:mix], [], "hexpm", "8eab6e1cfa8d5918c2ce4ba43588e894af35dbd8e91e6e55c817bca5847df34a"}, "idna": {:hex, :idna, "6.1.1", "8a63070e9f7d0c62eb9d9fcb360a7de382448200fbbd1b106cc96d3d8099df8d", [:rebar3], [{:unicode_util_compat, "~> 0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "92376eb7894412ed19ac475e4a86f7b413c1b9fbb5bd16dccd57934157944cea"}, "jason": {:hex, :jason, "1.4.4", "b9226785a9aa77b6857ca22832cffa5d5011a667207eb2a0ad56adb5db443b8a", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "c5eb0cab91f094599f94d55bc63409236a8ec69a21a67814529e8d5f6cc90b3b"}, - "jose": {:hex, :jose, "1.11.5", "3bc2d75ffa5e2c941ca93e5696b54978323191988eb8d225c2e663ddfefd515e", [:mix, :rebar3], [], "hexpm", "dcd3b215bafe02ea7c5b23dafd3eb8062a5cd8f2d904fd9caa323d37034ab384"}, - "logster": {:hex, :logster, "1.1.0", "e4a11299473ea500adf534366ba53a4d8fbf64709e0e6e49fb16c85b1b23d91e", [:mix], [{:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "875ea901c44d36b82d6f11ff344513faaa92fdf1aa212265e46296e9c78e2da3"}, + "jose": {:hex, :jose, "1.11.10", "a903f5227417bd2a08c8a00a0cbcc458118be84480955e8d251297a425723f83", [:mix, :rebar3], [], "hexpm", "0d6cd36ff8ba174db29148fc112b5842186b68a90ce9fc2b3ec3afe76593e614"}, + "logster": {:hex, :logster, "1.1.1", "d6fddac540dd46adde0c894024500867fe63b0043713f842c62da5815e21db10", [:mix], [{:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "d18e852c430812ad1c9756998ebe46ec814c724e6eb551a512d7e3f8dee24cef"}, "metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], [], "hexpm", "69b09adddc4f74a40716ae54d140f93beb0fb8978d8636eaded0c31b6f099f16"}, - "mime": {:hex, :mime, "2.0.6", "8f18486773d9b15f95f4f4f1e39b710045fa1de891fada4516559967276e4dc2", [:mix], [], "hexpm", "c9945363a6b26d747389aac3643f8e0e09d30499a138ad64fe8fd1d13d9b153e"}, - "mimerl": {:hex, :mimerl, "1.3.0", "d0cd9fc04b9061f82490f6581e0128379830e78535e017f7780f37fea7545726", [:rebar3], [], "hexpm", "a1e15a50d1887217de95f0b9b0793e32853f7c258a5cd227650889b38839fe9d"}, - "mint": {:hex, :mint, "1.5.1", "8db5239e56738552d85af398798c80648db0e90f343c8469f6c6d8898944fb6f", [:mix], [{:castore, "~> 0.1.0 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:hpax, "~> 0.1.1", [hex: :hpax, repo: "hexpm", optional: false]}], "hexpm", "4a63e1e76a7c3956abd2c72f370a0d0aecddc3976dea5c27eccbecfa5e7d5b1e"}, - "mox": {:hex, :mox, "1.0.2", "dc2057289ac478b35760ba74165b4b3f402f68803dd5aecd3bfd19c183815d64", [:mix], [], "hexpm", "f9864921b3aaf763c8741b5b8e6f908f44566f1e427b2630e89e9a73b981fef2"}, + "mime": {:hex, :mime, "2.0.7", "b8d739037be7cd402aee1ba0306edfdef982687ee7e9859bee6198c1e7e2f128", [:mix], [], "hexpm", "6171188e399ee16023ffc5b76ce445eb6d9672e2e241d2df6050f3c771e80ccd"}, + "mimerl": {:hex, :mimerl, "1.4.0", "3882a5ca67fbbe7117ba8947f27643557adec38fa2307490c4c4207624cb213b", [:rebar3], [], "hexpm", "13af15f9f68c65884ecca3a3891d50a7b57d82152792f3e19d88650aa126b144"}, + "mint": {:hex, :mint, "1.7.1", "113fdb2b2f3b59e47c7955971854641c61f378549d73e829e1768de90fc1abf1", [:mix], [{:castore, "~> 0.1.0 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:hpax, "~> 0.1.1 or ~> 0.2.0 or ~> 1.0", [hex: :hpax, repo: "hexpm", optional: false]}], "hexpm", "fceba0a4d0f24301ddee3024ae116df1c3f4bb7a563a731f45fdfeb9d39a231b"}, + "mox": {:hex, :mox, "1.2.0", "a2cd96b4b80a3883e3100a221e8adc1b98e4c3a332a8fc434c39526babafd5b3", [:mix], [{:nimble_ownership, "~> 1.0", [hex: :nimble_ownership, repo: "hexpm", optional: false]}], "hexpm", "c7b92b3cc69ee24a7eeeaf944cd7be22013c52fcb580c1f33f50845ec821089a"}, "nimble_options": {:hex, :nimble_options, "1.1.1", "e3a492d54d85fc3fd7c5baf411d9d2852922f66e69476317787a7b2bb000a61b", [:mix], [], "hexpm", "821b2470ca9442c4b6984882fe9bb0389371b8ddec4d45a9504f00a66f650b44"}, - "nimble_ownership": {:hex, :nimble_ownership, "1.0.1", "f69fae0cdd451b1614364013544e66e4f5d25f36a2056a9698b793305c5aa3a6", [:mix], [], "hexpm", "3825e461025464f519f3f3e4a1f9b68c47dc151369611629ad08b636b73bb22d"}, - "nimble_pool": {:hex, :nimble_pool, "1.0.0", "5eb82705d138f4dd4423f69ceb19ac667b3b492ae570c9f5c900bb3d2f50a847", [:mix], [], "hexpm", "80be3b882d2d351882256087078e1b1952a28bf98d0a287be87e4a24a710b67a"}, + "nimble_ownership": {:hex, :nimble_ownership, "1.0.2", "fa8a6f2d8c592ad4d79b2ca617473c6aefd5869abfa02563a77682038bf916cf", [:mix], [], "hexpm", "098af64e1f6f8609c6672127cfe9e9590a5d3fcdd82bc17a377b8692fd81a879"}, + "nimble_pool": {:hex, :nimble_pool, "1.1.0", "bf9c29fbdcba3564a8b800d1eeb5a3c58f36e1e11d7b7fb2e084a643f645f06b", [:mix], [], "hexpm", "af2e4e6b34197db81f7aad230c1118eac993acc0dae6bc83bac0126d4ae0813a"}, "parse_trans": {:hex, :parse_trans, "3.4.1", "6e6aa8167cb44cc8f39441d05193be6e6f4e7c2946cb2759f015f8c56b76e5ff", [:rebar3], [], "hexpm", "620a406ce75dada827b82e453c19cf06776be266f5a67cff34e1ef2cbb60e49a"}, - "plug": {:hex, :plug, "1.14.2", "cff7d4ec45b4ae176a227acd94a7ab536d9b37b942c8e8fa6dfc0fff98ff4d80", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "842fc50187e13cf4ac3b253d47d9474ed6c296a8732752835ce4a86acdf68d13"}, - "plug_cowboy": {:hex, :plug_cowboy, "2.6.1", "9a3bbfceeb65eff5f39dab529e5cd79137ac36e913c02067dba3963a26efe9b2", [:mix], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:cowboy_telemetry, "~> 0.3", [hex: :cowboy_telemetry, repo: "hexpm", optional: false]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "de36e1a21f451a18b790f37765db198075c25875c64834bcc82d90b309eb6613"}, - "plug_crypto": {:hex, :plug_crypto, "1.2.5", "918772575e48e81e455818229bf719d4ab4181fcbf7f85b68a35620f78d89ced", [:mix], [], "hexpm", "26549a1d6345e2172eb1c233866756ae44a9609bd33ee6f99147ab3fd87fd842"}, - "ranch": {:hex, :ranch, "1.8.0", "8c7a100a139fd57f17327b6413e4167ac559fbc04ca7448e9be9057311597a1d", [:make, :rebar3], [], "hexpm", "49fbcfd3682fab1f5d109351b61257676da1a2fdbe295904176d5e521a2ddfe5"}, + "plug": {:hex, :plug, "1.18.1", "5067f26f7745b7e31bc3368bc1a2b818b9779faa959b49c934c17730efc911cf", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2 or ~> 2.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "57a57db70df2b422b564437d2d33cf8d33cd16339c1edb190cd11b1a3a546cc2"}, + "plug_cowboy": {:hex, :plug_cowboy, "2.7.4", "729c752d17cf364e2b8da5bdb34fb5804f56251e88bb602aff48ae0bd8673d11", [:mix], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:cowboy_telemetry, "~> 0.3", [hex: :cowboy_telemetry, repo: "hexpm", optional: false]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "9b85632bd7012615bae0a5d70084deb1b25d2bcbb32cab82d1e9a1e023168aa3"}, + "plug_crypto": {:hex, :plug_crypto, "2.1.1", "19bda8184399cb24afa10be734f84a16ea0a2bc65054e23a62bb10f06bc89491", [:mix], [], "hexpm", "6470bce6ffe41c8bd497612ffde1a7e4af67f36a15eea5f921af71cf3e11247c"}, + "ranch": {:hex, :ranch, "2.2.0", "25528f82bc8d7c6152c57666ca99ec716510fe0925cb188172f41ce93117b1b0", [:make, :rebar3], [], "hexpm", "fa0b99a1780c80218a4197a59ea8d3bdae32fbff7e88527d7d8a4787eff4f8e7"}, "saxy": {:hex, :saxy, "1.6.0", "02cb4e9bd045f25ac0c70fae8164754878327ee393c338a090288210b02317ee", [:mix], [], "hexpm", "ef42eb4ac983ca77d650fbdb68368b26570f6cc5895f0faa04d34a6f384abad3"}, - "sentry": {:hex, :sentry, "10.8.1", "aa45309785e1521416225adb16e0b4d8b957578804527f3c7babb6fefbc5e456", [:mix], [{:hackney, "~> 1.8", [hex: :hackney, repo: "hexpm", optional: true]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: true]}, {:nimble_options, "~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:nimble_ownership, "~> 0.3.0 or ~> 1.0", [hex: :nimble_ownership, repo: "hexpm", optional: false]}, {:phoenix, "~> 1.6", [hex: :phoenix, repo: "hexpm", optional: true]}, {:phoenix_live_view, "~> 0.20 or ~> 1.0", [hex: :phoenix_live_view, repo: "hexpm", optional: true]}, {:plug, "~> 1.6", [hex: :plug, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: true]}], "hexpm", "495b3cdadad90ba72eef973aa3dec39b3b8b2a362fe87e2f4ef32133ac3b4097"}, + "sentry": {:hex, :sentry, "11.0.4", "60371c96cefd247e0fc98840bba2648f64f19aa0b8db8e938f5a98421f55b619", [:mix], [{:hackney, "~> 1.8", [hex: :hackney, repo: "hexpm", optional: true]}, {:igniter, "~> 0.5", [hex: :igniter, repo: "hexpm", optional: true]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: true]}, {:nimble_options, "~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:nimble_ownership, "~> 0.3.0 or ~> 1.0", [hex: :nimble_ownership, repo: "hexpm", optional: false]}, {:opentelemetry, ">= 0.0.0", [hex: :opentelemetry, repo: "hexpm", optional: true]}, {:opentelemetry_api, ">= 0.0.0", [hex: :opentelemetry_api, repo: "hexpm", optional: true]}, {:opentelemetry_exporter, ">= 0.0.0", [hex: :opentelemetry_exporter, repo: "hexpm", optional: true]}, {:opentelemetry_semantic_conventions, ">= 0.0.0", [hex: :opentelemetry_semantic_conventions, repo: "hexpm", optional: true]}, {:phoenix, "~> 1.6", [hex: :phoenix, repo: "hexpm", optional: true]}, {:phoenix_live_view, "~> 0.20 or ~> 1.0", [hex: :phoenix_live_view, repo: "hexpm", optional: true]}, {:plug, "~> 1.6", [hex: :plug, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: true]}], "hexpm", "feaafc284dc204c82aadaddc884227aeaa3480decb274d30e184b9d41a700c66"}, "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.7", "354c321cf377240c7b8716899e182ce4890c5938111a1296add3ec74cf1715df", [:make, :mix, :rebar3], [], "hexpm", "fe4c190e8f37401d30167c8c405eda19469f34577987c76dde613e838bbc67f8"}, - "sweet_xml": {:hex, :sweet_xml, "0.7.4", "a8b7e1ce7ecd775c7e8a65d501bc2cd933bff3a9c41ab763f5105688ef485d08", [:mix], [], "hexpm", "e7c4b0bdbf460c928234951def54fe87edf1a170f6896675443279e2dbeba167"}, + "sweet_xml": {:hex, :sweet_xml, "0.7.5", "803a563113981aaac202a1dbd39771562d0ad31004ddbfc9b5090bdcd5605277", [:mix], [], "hexpm", "193b28a9b12891cae351d81a0cead165ffe67df1b73fe5866d10629f4faefb12"}, "telemetry": {:hex, :telemetry, "1.3.0", "fedebbae410d715cf8e7062c96a1ef32ec22e764197f70cda73d82778d61e7a2", [:rebar3], [], "hexpm", "7015fc8919dbe63764f4b4b87a95b7c0996bd539e0d499be6ec9d7f3875b79e6"}, - "unicode_util_compat": {:hex, :unicode_util_compat, "0.7.0", "bc84380c9ab48177092f43ac89e4dfa2c6d62b40b8bd132b1059ecc7232f9a78", [:rebar3], [], "hexpm", "25eee6d67df61960cf6a794239566599b09e17e668d3700247bc498638152521"}, + "unicode_util_compat": {:hex, :unicode_util_compat, "0.7.1", "a48703a25c170eedadca83b11e88985af08d35f37c6f664d6dcfb106a97782fc", [:rebar3], [], "hexpm", "b3a917854ce3ae233619744ad1e0102e05673136776fb2fa76234f3e03b23642"}, } diff --git a/test/hexdocs/plug_test.exs b/test/hexdocs/plug_test.exs index 0c1cb71..39809b7 100644 --- a/test/hexdocs/plug_test.exs +++ b/test/hexdocs/plug_test.exs @@ -1,8 +1,12 @@ defmodule Hexdocs.PlugTest do use ExUnit.Case, async: true - use Plug.Test + import Plug.Conn + import Plug.Test + import Mox alias Hexdocs.{HexpmMock, Store} + setup :verify_on_exit! + @bucket :docs_private_bucket test "requests without subdomain not supported" do diff --git a/test/hexdocs/queue_test.exs b/test/hexdocs/queue_test.exs index ec1747d..6aee0b8 100644 --- a/test/hexdocs/queue_test.exs +++ b/test/hexdocs/queue_test.exs @@ -1,7 +1,10 @@ defmodule Hexdocs.QueueTest do use ExUnit.Case + import Mox alias Hexdocs.{HexpmMock, Store} + setup :verify_on_exit! + @bucket :docs_private_bucket @public_bucket :docs_public_bucket @@ -374,6 +377,48 @@ defmodule Hexdocs.QueueTest do end end + describe "search index" do + setup do + # Set up a mock search implementation for these tests + original_impl = Application.get_env(:hexdocs, :search_impl) + Application.put_env(:hexdocs, :search_impl, Hexdocs.Search.Local) + on_exit(fn -> Application.put_env(:hexdocs, :search_impl, original_impl) end) + :ok + end + + test "indexes search data for hexpm packages", %{test: test} do + key = "docs/#{test}-1.0.0.tar.gz" + + tar = + Hexdocs.Tar.create([ + {"index.html", "contents"}, + {"dist/search_data-ABC123.js", + """ + searchData={"items":[{"type":"module","title":"Example","doc":"example text","ref":"Example.html"}]} + """} + ]) + + Store.put!(:repo_bucket, key, tar) + + ref = Broadway.test_message(Hexdocs.Queue, search_message(key)) + assert_receive {:ack, ^ref, [_], []} + end + + test "handles missing search data gracefully", %{test: test} do + key = "docs/#{test}-1.0.0.tar.gz" + tar = Hexdocs.Tar.create([{"index.html", "contents"}]) + Store.put!(:repo_bucket, key, tar) + + ref = Broadway.test_message(Hexdocs.Queue, search_message(key)) + assert_receive {:ack, ^ref, [_], []} + end + + test "skips invalid keys", %{test: test} do + ref = Broadway.test_message(Hexdocs.Queue, search_message("invalid/key/#{test}")) + assert_receive {:ack, ^ref, [_], []} + end + end + describe "delete object" do test "delete all docs when removing only version", %{test: test} do Mox.expect(HexpmMock, :get_package, fn repo, package -> @@ -558,14 +603,7 @@ defmodule Hexdocs.QueueTest do end defp put_message(key) do - Jason.encode!(%{ - "Records" => [ - %{ - "eventName" => "ObjectCreated:Put", - "s3" => %{"object" => %{"key" => key}} - } - ] - }) + Jason.encode!(%{"hexdocs:upload" => key}) end defp delete_message(key) do @@ -579,6 +617,10 @@ defmodule Hexdocs.QueueTest do }) end + defp search_message(key) do + Jason.encode!(%{"hexdocs:search" => key}) + end + defp ls(bucket, prefix) do Store.list(bucket, prefix) |> Enum.map(&String.trim_leading(&1, prefix)) diff --git a/test/hexdocs/search_test.exs b/test/hexdocs/search_test.exs index d555e31..435fa31 100644 --- a/test/hexdocs/search_test.exs +++ b/test/hexdocs/search_test.exs @@ -29,7 +29,7 @@ defmodule Hexdocs.SearchTest do tar = Hexdocs.Tar.create(files) key = "docs/#{package}-#{version}.tar.gz" Hexdocs.Store.put!(:repo_bucket, key, tar) - ref = Broadway.test_message(Hexdocs.Queue, queue_put_message(key)) + ref = Broadway.test_message(Hexdocs.Queue, queue_search_message(key)) assert_receive {:ack, ^ref, [_], []} end @@ -131,60 +131,99 @@ defmodule Hexdocs.SearchTest do assert typesense_search(%{"q" => package, "query_by" => "package"}) == [] end - test "logs an error message if search_data.js file has unexpected format", %{package: package} do + test "raises an error if search_data.js file has unexpected format", %{package: package} do files = [ {"index.html", "contents"}, {"dist/search_data-0F918FFD.js", "unexpected format"} ] - log = capture_log(fn -> run_upload(package, "1.0.0", files) end) - assert log =~ "[error] Unexpected search_data format for #{package} 1.0.0" + tar = Hexdocs.Tar.create(files) + key = "docs/#{package}-1.0.0.tar.gz" + Hexdocs.Store.put!(:repo_bucket, key, tar) + ref = Broadway.test_message(Hexdocs.Queue, queue_search_message(key)) + + assert_receive {:ack, ^ref, [], + [ + %Broadway.Message{ + status: {:error, %RuntimeError{message: msg}, _stacktrace} + } + ]} + + assert msg == "Unexpected search_data format for #{package} 1.0.0" assert typesense_search(%{"q" => package, "query_by" => "package"}) == [] end - test "logs an error message if search_data.json cannot be decoded", %{package: package} do + test "raises an error if search_data.json cannot be decoded", %{package: package} do files = [ {"index.html", "contents"}, {"dist/search_data-0F918FFD.js", "searchData={\"items\":["} ] - log = capture_log(fn -> run_upload(package, "1.0.0", files) end) + tar = Hexdocs.Tar.create(files) + key = "docs/#{package}-1.0.0.tar.gz" + Hexdocs.Store.put!(:repo_bucket, key, tar) + ref = Broadway.test_message(Hexdocs.Queue, queue_search_message(key)) - assert log =~ - "[error] Failed to decode search data json for #{package} 1.0.0: :unexpected_end" + assert_receive {:ack, ^ref, [], + [ + %Broadway.Message{ + status: {:error, %RuntimeError{message: msg}, _stacktrace} + } + ]} + assert msg =~ "Failed to decode search data json for #{package} 1.0.0: :unexpected_end" assert typesense_search(%{"q" => package, "query_by" => "package"}) == [] end - test "logs an error message if search_data has empty items", %{package: package} do + test "raises an error if search_data has empty items", %{package: package} do files = [ {"index.html", "contents"}, {"dist/search_data-0F918FFD.js", "searchData={\"items\":[]}"} ] - log = capture_log(fn -> run_upload(package, "1.0.0", files) end) + tar = Hexdocs.Tar.create(files) + key = "docs/#{package}-1.0.0.tar.gz" + Hexdocs.Store.put!(:repo_bucket, key, tar) + ref = Broadway.test_message(Hexdocs.Queue, queue_search_message(key)) + + assert_receive {:ack, ^ref, [], + [ + %Broadway.Message{ + status: {:error, %RuntimeError{message: msg}, _stacktrace} + } + ]} - assert log =~ - "[error] Failed to extract search items and proglang from search data for #{package} 1.0.0" + assert msg == + "Failed to extract search items and proglang from search data for #{package} 1.0.0" assert typesense_search(%{"q" => package, "query_by" => "package"}) == [] end - test "logs an error message if search_data has no items", %{package: package} do + test "raises an error if search_data has no items", %{package: package} do files = [ {"index.html", "contents"}, {"dist/search_data-0F918FFD.js", "searchData={\"not_items\":[]}"} ] - log = capture_log(fn -> run_upload(package, "1.0.0", files) end) + tar = Hexdocs.Tar.create(files) + key = "docs/#{package}-1.0.0.tar.gz" + Hexdocs.Store.put!(:repo_bucket, key, tar) + ref = Broadway.test_message(Hexdocs.Queue, queue_search_message(key)) + + assert_receive {:ack, ^ref, [], + [ + %Broadway.Message{ + status: {:error, %RuntimeError{message: msg}, _stacktrace} + } + ]} - assert log =~ - "[error] Failed to extract search items and proglang from search data for #{package} 1.0.0" + assert msg == + "Failed to extract search items and proglang from search data for #{package} 1.0.0" assert typesense_search(%{"q" => package, "query_by" => "package"}) == [] end - test "logs errors when indexing incomplete search items", %{package: package} do + test "raises an error when indexing incomplete search items", %{package: package} do files = [ {"index.html", "contents"}, {"dist/search_data-0F918FFD.js", @@ -198,21 +237,23 @@ defmodule Hexdocs.SearchTest do """} ] - log = capture_log(fn -> run_upload(package, "1.0.0", files) end) + tar = Hexdocs.Tar.create(files) + key = "docs/#{package}-1.0.0.tar.gz" + Hexdocs.Store.put!(:repo_bucket, key, tar) + ref = Broadway.test_message(Hexdocs.Queue, queue_search_message(key)) - assert log =~ "[error] Failed to index search item for #{package} 1.0.0 for document " - assert log =~ "Field `doc` has been declared in the schema, but is not found in the document." + assert_receive {:ack, ^ref, [], + [ + %Broadway.Message{ + status: {:error, %RuntimeError{message: msg}, _stacktrace} + } + ]} - # the valid documents should still be indexed - assert [_, _] = - typesense_search(%{ - "q" => "example", - "query_by" => "title", - "filter" => "proglang:elixir" - }) + assert msg =~ "Failed to index search item for #{package} 1.0.0 for document " + assert msg =~ "Field `doc` has been declared in the schema, but is not found in the document." end - test "logs errors when indexing invalid search items", %{package: package} do + test "raises an error when indexing invalid search items", %{package: package} do files = [ {"index.html", "contents"}, @@ -228,30 +269,24 @@ defmodule Hexdocs.SearchTest do """} ] - log = capture_log(fn -> run_upload(package, "1.0.0", files) end) + tar = Hexdocs.Tar.create(files) + key = "docs/#{package}-1.0.0.tar.gz" + Hexdocs.Store.put!(:repo_bucket, key, tar) + ref = Broadway.test_message(Hexdocs.Queue, queue_search_message(key)) - assert log =~ "[error] Failed to index search item for #{package} 1.0.0 for document " - assert log =~ "Field `type` must be a string." - assert log =~ "Field `doc` must be a string." + assert_receive {:ack, ^ref, [], + [ + %Broadway.Message{ + status: {:error, %RuntimeError{message: msg}, _stacktrace} + } + ]} - # the valid documents should still be indexed - assert [_, _] = - typesense_search(%{ - "q" => "example", - "query_by" => "title", - "filter" => "proglang:elixir" - }) + assert msg =~ "Failed to index search item for #{package} 1.0.0 for document " + assert msg =~ "Field `type` must be a string." end - defp queue_put_message(key) do - Jason.encode!(%{ - "Records" => [ - %{ - "eventName" => "ObjectCreated:Put", - "s3" => %{"object" => %{"key" => key}} - } - ] - }) + defp queue_search_message(key) do + Jason.encode!(%{"hexdocs:search" => key}) end defp queue_delete_message(key) do