Merge pull request #743 from Hubs-Foundation/icosa-cleanup-2

Replacing google poly integration with Icosa Gallery integration

Author: Exairnous

.github/workflows/lint-and-test.yml

@@ -193,4 +193,5 @@ jobs:
             _build
           key: mix-${{ hashFiles('mix.lock') }}
 
+      - run: mix deps.get
       - run: mix format --check-formatted '{lib,priv,test,config}/**/*.{ex,exs}'

lib/ret/media_resolver.ex

@@ -20,12 +20,14 @@ defmodule Ret.MediaResolver do
 
   @youtube_rate_limit %{scale: 8_000, limit: 1}
   @sketchfab_rate_limit %{scale: 60_000, limit: 15}
+  @icosa_rate_limit %{scale: 60_000, limit: 1000}
   @max_await_for_rate_limit_s 120
 
   @non_video_root_hosts [
     "sketchfab.com",
     "giphy.com",
-    "tenor.com"
+    "tenor.com",
+    "icosa.gallery"
   ]
 
   @deviant_id_regex ~r/\"DeviantArt:\/\/deviation\/([^"]+)/
@@ -79,6 +81,12 @@ defmodule Ret.MediaResolver do
      |> resolved()}
   end
 
+  def resolve(%MediaResolverQuery{url: %URI{host: "api.icosa.gallery"}} = query, root_host) do
+    rate_limited_resolve(query, root_host, @icosa_rate_limit, fn ->
+      resolve_non_video(query, root_host)
+    end)
+  end
+
   def resolve(%MediaResolverQuery{} = query, root_host) when root_host in @non_video_root_hosts do
     resolve_non_video(query, root_host)
   end
@@ -327,6 +335,46 @@ defmodule Ret.MediaResolver do
     {:commit, (resolved_url || uri) |> resolved(meta)}
   end
 
+  defp resolve_non_video(
+         %MediaResolverQuery{
+           url: %URI{host: "api.icosa.gallery", path: "/v1/assets/" <> asset_id}
+         },
+         "icosa.gallery"
+       ) do
+    # Increment stat for the request
+    Statix.increment("ret.media_resolver.icosa.requests")
+
+    # Make the API call to get the asset data
+    payload =
+      "https://api.icosa.gallery/v1/assets/#{asset_id}"
+      # Assuming this function sends the request and handles retries
+      |> retry_get_until_success()
+      |> Map.get(:body)
+      |> Poison.decode!()
+
+    # Create the meta information based on the payload
+    meta =
+      %{
+        expected_content_type: "model/gltf",
+        name: payload["displayName"],
+        author: payload["authorName"],
+        license: payload["license"]
+      }
+
+    # Extract the GLTF2 format URL from the payload
+    uri =
+      payload["formats"]
+      |> Enum.find(&(&1["formatType"] == "GLTF2"))
+      |> Kernel.get_in(["root", "url"])
+      |> URI.parse()
+
+    # Increment stat for successful resolution
+    Statix.increment("ret.media_resolver.icosa.ok")
+
+    # Return the URI and meta data for further processing
+    {:commit, resolved(uri, meta)}
+  end
+
   defp resolve_non_video(
          %MediaResolverQuery{url: %URI{path: "/models/" <> model_id}} = query,
          "sketchfab.com" = root_host

lib/ret/media_search.ex

@@ -249,6 +249,47 @@ defmodule Ret.MediaSearch do
     sketchfab_search(query)
   end
 
+  def search(%Ret.MediaSearchQuery{source: "icosa", cursor: cursor, filter: filter, q: q}) do
+    query =
+      URI.encode_query(
+        pageSize: @page_size,
+        maxComplexity: :MEDIUM,
+        format: :GLTF2,
+        pageToken: cursor,
+        category: filter,
+        licence: :REMIXABLE,
+        orderBy: :BEST,
+        keywords: q
+      )
+
+    url = "https://api.icosa.gallery/v1/assets?#{query}"
+    # Remove empty category queries as the Icosa API returns validation errors instead of "any"
+    url = String.replace(url, "category=&", "")
+
+    res =
+      url
+      |> retry_get_until_success()
+
+    case res do
+      :error ->
+        :error
+
+      res ->
+        decoded_res = res |> Map.get(:body) |> Poison.decode!()
+        entries = decoded_res |> Map.get("assets") |> Enum.map(&icosa_api_result_to_entry/1)
+        next_cursor = decoded_res |> Map.get("nextPageToken")
+
+        {:commit,
+         %Ret.MediaSearchResult{
+           meta: %Ret.MediaSearchResultMeta{
+             next_cursor: next_cursor,
+             source: :icosa
+           },
+           entries: entries
+         }}
+    end
+  end
+
   def search(%Ret.MediaSearchQuery{
         source: "youtube_videos",
         cursor: cursor,
@@ -382,6 +423,8 @@ defmodule Ret.MediaSearch do
     end
   end
 
+  # Icosa does not currently require an API key
+  def available?(:icosa), do: true
   def available?(:bing_images), do: has_resolver_config?(:bing_search_api_key)
   def available?(:bing_videos), do: has_resolver_config?(:bing_search_api_key)
   def available?(:youtube_videos), do: has_resolver_config?(:youtube_api_key)
@@ -963,6 +1006,18 @@ defmodule Ret.MediaSearch do
     }
   end
 
+  defp icosa_api_result_to_entry(result) do
+    %{
+      id: result["assetId"],
+      type: "icosa_model",
+      name: result["displayName"],
+      attributions: %{creator: %{name: result["authorName"]}},
+      url: "http://api.icosa.gallery/v1/assets/" <> result["assetId"],
+      # url: result["url"],
+      images: %{preview: %{url: result["thumbnail"]["url"]}}
+    }
+  end
+
   defp youtube_api_result_to_entry(result) do
     %{
       id: result["id"]["videoId"],

lib/ret/meta.ex

@@ -44,6 +44,7 @@ defmodule Ret.Meta do
   def available_integrations_meta do
     %{
       twitter: Ret.TwitterClient.available?(),
+      icosa: Ret.MediaSearch.available?(:icosa),
       bing_images: Ret.MediaSearch.available?(:bing_images),
       bing_videos: Ret.MediaSearch.available?(:bing_videos),
       youtube_videos: Ret.MediaSearch.available?(:youtube_videos),

lib/ret_web/controllers/api/v1/media_search_controller.ex

@@ -98,6 +98,7 @@ defmodule RetWeb.Api.V1.MediaSearchController do
   def index(conn, %{"source" => source} = params)
       when source in [
              "sketchfab",
+             "icosa",
              "tenor",
              "youtube_videos",
              "bing_videos",

lib/ret_web/controllers/page_controller.ex

@@ -18,6 +18,7 @@ defmodule RetWeb.PageController do
   alias Plug.Conn
   import Ret.ConnUtils
   import Ret.HttpUtils
+  require Logger
 
   ##
   # NOTE: In addition to adding a route, you must add static html pages to the page_origin_warmer.ex
@@ -745,17 +746,23 @@ defmodule RetWeb.PageController do
     do: cors_proxy(conn, "#{url}?#{qs}")
 
   defp cors_proxy(conn, url) do
+    cors_proxy_with_redirects(conn, url, 0)
+  end
+
+  defp cors_proxy_with_redirects(conn, url, redirect_count) when redirect_count > 5 do
+    Logger.error("CORS Proxy: Too many redirects (#{redirect_count}) for URL: #{url}")
+    conn |> send_resp(400, "Too many redirects")
+  end
+
+  defp cors_proxy_with_redirects(conn, url, redirect_count) do
     %URI{authority: authority, host: host} = uri = URI.parse(url)
 
     resolved_ip = HttpUtils.resolve_ip(host)
 
     if HttpUtils.internal_ip?(resolved_ip) do
+      Logger.warning("CORS Proxy: Blocking internal IP #{inspect(resolved_ip)} for host #{host}")
       conn |> send_resp(401, "Bad request.")
     else
-      # We want to ensure that the URL we request hits the same IP that we verified above,
-      # so we replace the host with the IP address here and use this url to make the proxy request.
-      ip_url = URI.to_string(HttpUtils.replace_host(uri, resolved_ip))
-
       # Disallow CORS proxying unless request was made to the cors proxy url
       cors_proxy_url = Application.get_env(:ret, RetWeb.Endpoint)[:cors_proxy_url]
 
@@ -780,7 +787,7 @@ defmodule RetWeb.PageController do
             upstream: url,
             allowed_origins: allowed_origins,
             proxy_url: "#{cors_scheme}://#{cors_host}:#{cors_port}",
-            # Since we replaced the host with the IP address in ip_url above, we need to force the host
+            # We need to force the host
             # used for ssl verification here so that the connection isn't rejected.
             # Note that we have to convert the authority to a charlist, since this uses Erlang's `ssl` module
             # internally, which expects a charlist.
@@ -789,33 +796,91 @@ defmodule RetWeb.PageController do
                 {:server_name_indication, to_charlist(authority)},
                 {:versions, [:"tlsv1.2", :"tlsv1.3"]}
               ]
-            ]
-            # preserve_host_header: true
+            ],
+            error_callback: fn error -> Logger.error("CORS-proxy error: #{inspect(error)}") end
           )
 
         body = ReverseProxyPlug.read_body(conn)
         is_head = conn |> Conn.get_req_header("x-original-method") == ["HEAD"]
 
-        %Conn{}
-        |> Map.merge(conn)
-        |> Map.put(
-          :method,
-          if is_head do
-            "HEAD"
-          else
-            conn.method
+        try do
+          # First, make a HEAD request to check for redirects using HTTPoison
+          case HTTPoison.head(url, [],
+                 follow_redirect: false,
+                 ssl: [
+                   {:server_name_indication, to_charlist(authority)},
+                   {:versions, [:"tlsv1.2", :"tlsv1.3"]}
+                 ],
+                 timeout: 15_000,
+                 recv_timeout: 15_000
+               ) do
+            {:ok, %HTTPoison.Response{status_code: status_code, headers: headers}}
+            when status_code in [301, 302, 303, 307, 308] ->
+              # Found a redirect
+              location_header =
+                headers
+                |> Enum.find(fn {k, _v} -> String.downcase(k) == "location" end)
+                |> elem(1)
+
+              if location_header do
+                # Resolve relative URLs against the current URL
+                redirect_url = URI.merge(uri, location_header) |> URI.to_string()
+                cors_proxy_with_redirects(conn, redirect_url, redirect_count + 1)
+              else
+                Logger.warning("CORS Proxy: Redirect response missing location header")
+                # Fall back to ReverseProxyPlug for this response
+                make_reverse_proxy_request(conn, url, body, is_head, opts)
+              end
+
+            {:ok, %HTTPoison.Response{}} ->
+              # Not a redirect, use ReverseProxyPlug for the actual request
+              make_reverse_proxy_request(conn, url, body, is_head, opts)
+
+            {:error, reason} ->
+              Logger.error("CORS Proxy: HEAD request failed: #{inspect(reason)}")
+              # Fall back to ReverseProxyPlug anyway
+              make_reverse_proxy_request(conn, url, body, is_head, opts)
           end
-        )
-        # Need to strip path_info since proxy plug reads it
-        |> Map.put(:path_info, [])
-        |> ReverseProxyPlug.request(body, opts)
-        |> ReverseProxyPlug.response(conn, opts)
+        rescue
+          error ->
+            Logger.error("CORS Proxy: Request failed with exception: #{inspect(error)}")
+            conn |> send_resp(500, "Proxy request failed: #{inspect(error)}")
+        catch
+          :exit, reason ->
+            Logger.error("CORS Proxy: Request exited with reason: #{inspect(reason)}")
+            conn |> send_resp(500, "Proxy request timed out or failed")
+
+          kind, reason ->
+            Logger.error("CORS Proxy: Request failed with #{kind}: #{inspect(reason)}")
+            conn |> send_resp(500, "Proxy request failed")
+        end
       else
+        Logger.warning("CORS Proxy: Request rejected - invalid host or scheme")
         conn |> send_resp(401, "Bad request.")
       end
     end
   end
 
+  defp make_reverse_proxy_request(conn, _url, body, is_head, opts) do
+    proxy_conn =
+      %Conn{}
+      |> Map.merge(conn)
+      |> Map.put(
+        :method,
+        if is_head do
+          "HEAD"
+        else
+          conn.method
+        end
+      )
+      # Need to strip path_info since proxy plug reads it
+      |> Map.put(:path_info, [])
+      |> ReverseProxyPlug.request(body, opts)
+      |> ReverseProxyPlug.response(conn, opts)
+
+    proxy_conn
+  end
+
   defp render_static_asset(conn) do
     static_options = Plug.Static.init(at: "/", from: :ret, gzip: true, brotli: true)
     Plug.Static.call(conn, static_options)