diff --git a/gleam.toml b/gleam.toml index d8e7b69..eed1f64 100644 --- a/gleam.toml +++ b/gleam.toml @@ -12,6 +12,7 @@ gleam_stdlib = ">= 0.54.0 and < 1.0.0" grille_pain = ">= 1.1.0 and < 2.0.0" lustre = ">= 5.0.0 and < 6.0.0" modem = ">= 2.1.1 and < 3.0.0" +gleam_json = ">= 3.0.2 and < 4.0.0" [dev-dependencies] gleeunit = ">= 1.0.0 and < 2.0.0" diff --git a/manifest.toml b/manifest.toml index 3252b95..a62b82a 100644 --- a/manifest.toml +++ b/manifest.toml @@ -52,6 +52,7 @@ gleam_fetch = { version = ">= 1.2.0 and < 2.0.0" } gleam_hexpm = { version = ">= 3.0.0 and < 4.0.0" } gleam_http = { version = ">= 4.0.0 and < 5.0.0" } gleam_javascript = { version = ">= 1.0.0 and < 2.0.0" } +gleam_json = { version = ">= 3.0.2 and < 4.0.0" } gleam_regexp = { version = ">= 1.1.0 and < 2.0.0" } gleam_stdlib = { version = ">= 0.54.0 and < 1.0.0" } gleeunit = { version = ">= 1.0.0 and < 2.0.0" } diff --git a/src/hexdocs.gleam b/src/hexdocs.gleam index af6e81d..b1ac95f 100644 --- a/src/hexdocs.gleam +++ b/src/hexdocs.gleam @@ -156,15 +156,17 @@ fn api_returned_packages( } fn api_returned_typesense_search(model: Model, response: Loss(decode.Dynamic)) { - response - |> result.try(fn(search_result) { - search_result - |> decode.run(hexdocs.typesense_decoder()) - |> result.map_error(loss.DecodeError) - }) - |> result.map(model.set_search_results(model, _)) - |> result.map(pair.new(_, effect.none())) - |> result.unwrap(#(model, effect.none())) + let response = + response + |> result.try(fn(search_result) { + search_result + |> decode.run(hexdocs.typesense_decoder()) + |> result.map_error(loss.DecodeError) + }) + case response { + Error(_) -> #(model, toast.error("Server error. Retry later.")) + Ok(results) -> #(model.set_search_results(model, results), effect.none()) + } } fn api_returned_initial_latest_packages( diff --git a/src/hexdocs/endpoints.gleam b/src/hexdocs/endpoints.gleam index 9e31acb..281d336 100644 --- a/src/hexdocs/endpoints.gleam +++ b/src/hexdocs/endpoints.gleam @@ -2,7 +2,7 @@ import gleam/uri.{type Uri} import hexdocs/config pub fn search() -> Uri { - let assert Ok(uri) = uri.parse(config.search_url()) + let assert Ok(uri) = uri.parse(config.search_url() <> "/multi_search") uri } diff --git a/src/hexdocs/services/hexdocs.gleam b/src/hexdocs/services/hexdocs.gleam index 46f6c37..51ba282 100644 --- a/src/hexdocs/services/hexdocs.gleam +++ b/src/hexdocs/services/hexdocs.gleam @@ -1,14 +1,14 @@ import gleam/bool import gleam/dynamic/decode import gleam/fetch +import gleam/http import gleam/http/request import gleam/int import gleam/javascript/promise +import gleam/json import gleam/list -import gleam/option.{Some} import gleam/result import gleam/string -import gleam/uri import hexdocs/config import hexdocs/data/model/version import hexdocs/endpoints @@ -44,45 +44,63 @@ pub fn typesense_search( packages: List(version.Package), page: Int, ) { - let query = new_search_query_params(query, packages, page) - let endpoint = uri.Uri(..endpoints.search(), query: Some(query)) - let assert Ok(request) = request.from_uri(endpoint) + let body = new_search_body(query, packages, page) + let endpoint = endpoints.search() + let assert Ok(req) = request.from_uri(endpoint) + let request = + req + |> request.set_method(http.Post) + |> request.set_header("content-type", "application/json") + |> request.set_body(body) fetch.send(request) |> promise.try_await(fetch.read_json_body) |> promise.map(result.map_error(_, loss.FetchError)) } pub fn typesense_decoder() { - use found <- decode.field("found", decode.int) - use hits <- decode.field("hits", { + use results <- decode.field( + "results", decode.list({ - use document <- decode.field("document", { - use doc <- decode.field("doc", decode.string) - use id <- decode.field("id", decode.string) - use package <- decode.field("package", decode.string) - use proglang <- decode.field("proglang", decode.string) - use ref <- decode.field("ref", decode.string) - use title <- decode.field("title", decode.string) - use type_ <- decode.field("type", decode.string) - Document( - doc:, - id:, - package:, - proglang:, - ref:, - title:, - type_:, - headers: [], - ) - |> decode.success + use found <- decode.field("found", decode.int) + use hits <- decode.field("hits", { + decode.list({ + use document <- decode.field("document", { + use doc <- decode.field("doc", decode.string) + use id <- decode.field("id", decode.string) + use package <- decode.field("package", decode.string) + use proglang <- decode.field("proglang", decode.string) + use ref <- decode.field("ref", decode.string) + use title <- decode.field("title", decode.string) + use type_ <- decode.field("type", decode.string) + Document( + doc:, + id:, + package:, + proglang:, + ref:, + title:, + type_:, + headers: [], + ) + |> decode.success + }) + decode.success(document) + }) }) - decode.success(document) - }) - }) - let grouped_results = group_headers(hits) - let removed_count = list.length(hits) - list.length(grouped_results) - let max_results = list.take(grouped_results, config.per_page()) - decode.success(#(found - removed_count, max_results)) + decode.success(#(found, hits)) + }), + ) + + // Extract first result (we only send one search) + case results { + [#(found, hits), ..] -> { + let grouped_results = group_headers(hits) + let removed_count = list.length(hits) - list.length(grouped_results) + let max_results = list.take(grouped_results, config.per_page()) + decode.success(#(found - removed_count, max_results)) + } + [] -> decode.success(#(0, [])) + } } fn group_headers(documents: List(Document)) -> List(Document) { @@ -151,28 +169,33 @@ fn group_headers(documents: List(Document)) -> List(Document) { }) } -fn new_search_query_params( +fn new_search_body( query: String, packages: List(version.Package), page: Int, -) { - list.new() - |> list.key_set("q", query) - |> list.key_set("query_by", "title,doc") - |> list.key_set("query_by_weights", "3,1") - |> list.key_set("page", int.to_string(page)) - // We multiply per 2 because we group results - |> list.key_set("per_page", int.to_string(config.per_page() * 2)) - |> list.key_set("highlight_fields", "none") - |> add_filter_by_packages_param(packages) - |> uri.query_to_string +) -> String { + let filter_by = case get_filter_by_packages(packages) { + "" -> [] + filter -> [#("filter_by", json.string(filter))] + } + + let search_params = + json.object([ + #("q", json.string(query)), + #("query_by", json.string("title,doc")), + #("query_by_weights", json.string("3,1")), + #("page", json.int(page)), + #("per_page", json.int(config.per_page() * 2)), + #("highlight_fields", json.string("none")), + ..filter_by + ]) + + json.object([#("searches", json.array([search_params], fn(x) { x }))]) + |> json.to_string } -fn add_filter_by_packages_param( - query: List(#(String, String)), - packages: List(version.Package), -) -> List(#(String, String)) { - use <- bool.guard(when: list.is_empty(packages), return: query) +fn get_filter_by_packages(packages: List(version.Package)) -> String { + use <- bool.guard(when: list.is_empty(packages), return: "") packages |> list.filter_map(fn(p) { case p.status { @@ -182,7 +205,6 @@ fn add_filter_by_packages_param( }) |> list.map(string.append("package:=", _)) |> string.join("||") - |> list.key_set(query, "filter_by", _) } pub fn snippet(doc: String, search_input: String) -> String {