From a254079b0472edcf5617eefe03fdccc613d28816 Mon Sep 17 00:00:00 2001 From: Andy Baker Date: Fri, 30 Aug 2024 10:30:27 +0100 Subject: [PATCH 01/26] Revert "removed poly references" 1e85db47f6d99d983e7198a9c55f04c11eb316cb --- config/dev.exs | 1 + config/test.exs | 1 + habitat/config/config.toml | 3 ++ lib/ret/media_resolver.ex | 45 ++++++++++++++++ lib/ret/media_search.ex | 52 +++++++++++++++++++ lib/ret/meta.ex | 1 + .../api/v1/media_search_controller.ex | 3 +- 7 files changed, 105 insertions(+), 1 deletion(-) diff --git a/config/dev.exs b/config/dev.exs index 2da1f4fc3..702575eb3 100644 --- a/config/dev.exs +++ b/config/dev.exs @@ -128,6 +128,7 @@ config :ret, Ret.MediaResolver, deviantart_client_secret: nil, imgur_mashape_api_key: nil, imgur_client_id: nil, + google_poly_api_key: nil, youtube_api_key: nil, sketchfab_api_key: nil, ytdl_host: nil, diff --git a/config/test.exs b/config/test.exs index 9c1cf1b9c..d4d2ee4bc 100644 --- a/config/test.exs +++ b/config/test.exs @@ -66,6 +66,7 @@ config :ret, Ret.MediaResolver, deviantart_client_secret: nil, imgur_mashape_api_key: nil, imgur_client_id: nil, + google_poly_api_key: nil, youtube_api_key: nil, sketchfab_api_key: nil, ytdl_host: nil, diff --git a/habitat/config/config.toml b/habitat/config/config.toml index 0b36a5f62..c1fad4c91 100644 --- a/habitat/config/config.toml +++ b/habitat/config/config.toml @@ -197,6 +197,9 @@ deviantart_client_secret = {{ toToml cfg.resolver.deviantart_client_secret }} imgur_mashape_api_key = {{ toToml cfg.resolver.imgur_mashape_api_key }} imgur_client_id = {{ toToml cfg.resolver.imgur_client_id }} {{/if}} +{{ #if cfg.resolver.google_poly_api_key }} +google_poly_api_key = {{ toToml cfg.resolver.google_poly_api_key }} +{{/if}} {{ #if cfg.resolver.youtube_api_key }} youtube_api_key = {{ toToml cfg.resolver.youtube_api_key }} {{/if}} diff --git a/lib/ret/media_resolver.ex b/lib/ret/media_resolver.ex index 31e91f606..f7326e9e8 100644 --- a/lib/ret/media_resolver.ex +++ b/lib/ret/media_resolver.ex @@ -20,6 +20,7 @@ defmodule Ret.MediaResolver do @youtube_rate_limit %{scale: 8_000, limit: 1} @sketchfab_rate_limit %{scale: 60_000, limit: 15} + @poly_rate_limit %{scale: 60_000, limit: 1000} @max_await_for_rate_limit_s 120 @non_video_root_hosts [ @@ -79,6 +80,13 @@ defmodule Ret.MediaResolver do |> resolved()} end + # Necessary short circuit around google.com root_host to skip YT-DL check for Poly + def resolve(%MediaResolverQuery{url: %URI{host: "poly.google.com"}} = query, root_host) do + rate_limited_resolve(query, root_host, @poly_rate_limit, fn -> + resolve_non_video(query, root_host) + end) + end + def resolve(%MediaResolverQuery{} = query, root_host) when root_host in @non_video_root_hosts do resolve_non_video(query, root_host) end @@ -327,6 +335,43 @@ defmodule Ret.MediaResolver do {:commit, (resolved_url || uri) |> resolved(meta)} end + defp resolve_non_video( + %MediaResolverQuery{url: %URI{host: "poly.google.com", path: "/view/" <> asset_id} = uri}, + "google.com" + ) do + [uri, meta] = + with api_key when is_binary(api_key) <- module_config(:google_poly_api_key) do + Statix.increment("ret.media_resolver.poly.requests") + + payload = + "https://poly.googleapis.com/v1/assets/#{asset_id}?key=#{api_key}" + |> retry_get_until_success + |> Map.get(:body) + |> Poison.decode!() + + meta = + %{expected_content_type: "model/gltf"} + |> Map.put(:name, payload["displayName"]) + |> Map.put(:author, payload["authorName"]) + |> Map.put(:license, payload["license"]) + + formats = payload |> Map.get("formats") + + uri = + (Enum.find(formats, &(&1["formatType"] == "GLTF2")) || Enum.find(formats, &(&1["formatType"] == "GLTF"))) + |> Kernel.get_in(["root", "url"]) + |> URI.parse() + + Statix.increment("ret.media_resolver.poly.ok") + + [uri, meta] + else + _err -> [uri, nil] + end + + {:commit, uri |> resolved(meta)} + end + defp resolve_non_video( %MediaResolverQuery{url: %URI{path: "/models/" <> model_id}} = query, "sketchfab.com" = root_host diff --git a/lib/ret/media_search.ex b/lib/ret/media_search.ex index 03a4f615c..5e4f8cf57 100644 --- a/lib/ret/media_search.ex +++ b/lib/ret/media_search.ex @@ -249,6 +249,46 @@ defmodule Ret.MediaSearch do sketchfab_search(query) end + def search(%Ret.MediaSearchQuery{source: "poly", cursor: cursor, filter: filter, q: q}) do + with api_key when is_binary(api_key) <- resolver_config(:google_poly_api_key) do + query = + URI.encode_query( + pageSize: @page_size, + maxComplexity: :MEDIUM, + format: :GLTF2, + pageToken: cursor, + category: filter, + keywords: q, + key: api_key + ) + + res = + "https://poly.googleapis.com/v1/assets?#{query}" + |> retry_get_until_success() + + case res do + :error -> + :error + + res -> + decoded_res = res |> Map.get(:body) |> Poison.decode!() + entries = decoded_res |> Map.get("assets") |> Enum.map(&poly_api_result_to_entry/1) + next_cursor = decoded_res |> Map.get("nextPageToken") + + {:commit, + %Ret.MediaSearchResult{ + meta: %Ret.MediaSearchResultMeta{ + next_cursor: next_cursor, + source: :poly + }, + entries: entries + }} + end + else + _ -> nil + end + end + def search(%Ret.MediaSearchQuery{source: "youtube_videos", cursor: cursor, filter: filter, q: q}) do with api_key when is_binary(api_key) <- resolver_config(:youtube_api_key) do query = @@ -377,6 +417,7 @@ defmodule Ret.MediaSearch do end end + def available?(:poly), do: has_resolver_config?(:google_poly_api_key) def available?(:bing_images), do: has_resolver_config?(:bing_search_api_key) def available?(:bing_videos), do: has_resolver_config?(:bing_search_api_key) def available?(:youtube_videos), do: has_resolver_config?(:youtube_api_key) @@ -957,6 +998,17 @@ defmodule Ret.MediaSearch do } end + defp poly_api_result_to_entry(result) do + %{ + id: result["name"], + type: "poly_model", + name: result["displayName"], + attributions: %{creator: %{name: result["authorName"]}}, + url: "https://poly.google.com/view/#{result["name"] |> String.replace("assets/", "")}", + images: %{preview: %{url: result["thumbnail"]["url"]}} + } + end + defp youtube_api_result_to_entry(result) do %{ id: result["id"]["videoId"], diff --git a/lib/ret/meta.ex b/lib/ret/meta.ex index adc310317..07b474513 100644 --- a/lib/ret/meta.ex +++ b/lib/ret/meta.ex @@ -44,6 +44,7 @@ defmodule Ret.Meta do def available_integrations_meta do %{ twitter: Ret.TwitterClient.available?(), + poly: Ret.MediaSearch.available?(:poly), bing_images: Ret.MediaSearch.available?(:bing_images), bing_videos: Ret.MediaSearch.available?(:bing_videos), youtube_videos: Ret.MediaSearch.available?(:youtube_videos), diff --git a/lib/ret_web/controllers/api/v1/media_search_controller.ex b/lib/ret_web/controllers/api/v1/media_search_controller.ex index f21ca1460..516f5d5fc 100644 --- a/lib/ret_web/controllers/api/v1/media_search_controller.ex +++ b/lib/ret_web/controllers/api/v1/media_search_controller.ex @@ -98,6 +98,7 @@ defmodule RetWeb.Api.V1.MediaSearchController do def index(conn, %{"source" => source} = params) when source in [ "sketchfab", + "poly", "tenor", "youtube_videos", "bing_videos", @@ -127,7 +128,7 @@ defmodule RetWeb.Api.V1.MediaSearchController do # For Google services, increase cache duration for landing pages by using long-lived cache, due to quotas. defp cache_for_query(%Ret.MediaSearchQuery{source: source, q: nil}) - when source == "youtube_videos", + when source == "youtube_videos" or source == "poly", do: :media_search_results_long defp cache_for_query(_query), do: :media_search_results From a435ae2cccc945615fb5dccf753dc530852c563f Mon Sep 17 00:00:00 2001 From: Craig Moore Date: Fri, 30 Aug 2024 10:47:43 +0100 Subject: [PATCH 02/26] Replacing hardcoded references to poly.googleapis.com with icosa-api.ixxy.co.uk --- lib/ret/media_resolver.ex | 2 +- lib/ret/media_search.ex | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/ret/media_resolver.ex b/lib/ret/media_resolver.ex index f7326e9e8..bce63c9ac 100644 --- a/lib/ret/media_resolver.ex +++ b/lib/ret/media_resolver.ex @@ -344,7 +344,7 @@ defmodule Ret.MediaResolver do Statix.increment("ret.media_resolver.poly.requests") payload = - "https://poly.googleapis.com/v1/assets/#{asset_id}?key=#{api_key}" + "https://icosa-api.ixxy.co.uk/v1/assets/#{asset_id}?key=#{api_key}" |> retry_get_until_success |> Map.get(:body) |> Poison.decode!() diff --git a/lib/ret/media_search.ex b/lib/ret/media_search.ex index 5e4f8cf57..bd5c2eaf2 100644 --- a/lib/ret/media_search.ex +++ b/lib/ret/media_search.ex @@ -263,7 +263,7 @@ defmodule Ret.MediaSearch do ) res = - "https://poly.googleapis.com/v1/assets?#{query}" + "https://icosa-api.ixxy.co.uk/v1/assets?#{query}" |> retry_get_until_success() case res do From a496cfdef11202938861844a58216077791d4399 Mon Sep 17 00:00:00 2001 From: Andy Baker Date: Fri, 30 Aug 2024 10:58:10 +0100 Subject: [PATCH 03/26] Poly to Icosa --- lib/ret/media_resolver.ex | 11 +++++++++-- lib/ret/media_search.ex | 8 ++++---- lib/ret/meta.ex | 2 +- .../controllers/api/v1/media_search_controller.ex | 4 ++-- 4 files changed, 16 insertions(+), 9 deletions(-) diff --git a/lib/ret/media_resolver.ex b/lib/ret/media_resolver.ex index bce63c9ac..fddd25834 100644 --- a/lib/ret/media_resolver.ex +++ b/lib/ret/media_resolver.ex @@ -81,7 +81,14 @@ defmodule Ret.MediaResolver do end # Necessary short circuit around google.com root_host to skip YT-DL check for Poly - def resolve(%MediaResolverQuery{url: %URI{host: "poly.google.com"}} = query, root_host) do + def resolve(%MediaResolverQuery{url: %URI{host: "icosa-api.ixxy.co.uk"}} = query, root_host) do + rate_limited_resolve(query, root_host, @poly_rate_limit, fn -> + resolve_non_video(query, root_host) + end) + end + + # Necessary short circuit around google.com root_host to skip YT-DL check for Poly + def resolve(%MediaResolverQuery{url: %URI{host: "icosa.ixxy.co.uk"}} = query, root_host) do rate_limited_resolve(query, root_host, @poly_rate_limit, fn -> resolve_non_video(query, root_host) end) @@ -336,7 +343,7 @@ defmodule Ret.MediaResolver do end defp resolve_non_video( - %MediaResolverQuery{url: %URI{host: "poly.google.com", path: "/view/" <> asset_id} = uri}, + %MediaResolverQuery{url: %URI{host: "icosa.ixxy.co.uk", path: "/view/" <> asset_id} = uri}, "google.com" ) do [uri, meta] = diff --git a/lib/ret/media_search.ex b/lib/ret/media_search.ex index bd5c2eaf2..8af3771d5 100644 --- a/lib/ret/media_search.ex +++ b/lib/ret/media_search.ex @@ -249,7 +249,7 @@ defmodule Ret.MediaSearch do sketchfab_search(query) end - def search(%Ret.MediaSearchQuery{source: "poly", cursor: cursor, filter: filter, q: q}) do + def search(%Ret.MediaSearchQuery{source: "icosa", cursor: cursor, filter: filter, q: q}) do with api_key when is_binary(api_key) <- resolver_config(:google_poly_api_key) do query = URI.encode_query( @@ -279,7 +279,7 @@ defmodule Ret.MediaSearch do %Ret.MediaSearchResult{ meta: %Ret.MediaSearchResultMeta{ next_cursor: next_cursor, - source: :poly + source: :icosa }, entries: entries }} @@ -417,7 +417,7 @@ defmodule Ret.MediaSearch do end end - def available?(:poly), do: has_resolver_config?(:google_poly_api_key) + def available?(:icosa), do: has_resolver_config?(:google_poly_api_key) def available?(:bing_images), do: has_resolver_config?(:bing_search_api_key) def available?(:bing_videos), do: has_resolver_config?(:bing_search_api_key) def available?(:youtube_videos), do: has_resolver_config?(:youtube_api_key) @@ -1004,7 +1004,7 @@ defmodule Ret.MediaSearch do type: "poly_model", name: result["displayName"], attributions: %{creator: %{name: result["authorName"]}}, - url: "https://poly.google.com/view/#{result["name"] |> String.replace("assets/", "")}", + url: "https://icosa.ixxy.co.uk/view/#{result["name"] |> String.replace("assets/", "")}", images: %{preview: %{url: result["thumbnail"]["url"]}} } end diff --git a/lib/ret/meta.ex b/lib/ret/meta.ex index 07b474513..be3df373e 100644 --- a/lib/ret/meta.ex +++ b/lib/ret/meta.ex @@ -44,7 +44,7 @@ defmodule Ret.Meta do def available_integrations_meta do %{ twitter: Ret.TwitterClient.available?(), - poly: Ret.MediaSearch.available?(:poly), + icosa: Ret.MediaSearch.available?(:icosa), bing_images: Ret.MediaSearch.available?(:bing_images), bing_videos: Ret.MediaSearch.available?(:bing_videos), youtube_videos: Ret.MediaSearch.available?(:youtube_videos), diff --git a/lib/ret_web/controllers/api/v1/media_search_controller.ex b/lib/ret_web/controllers/api/v1/media_search_controller.ex index 516f5d5fc..946eaad61 100644 --- a/lib/ret_web/controllers/api/v1/media_search_controller.ex +++ b/lib/ret_web/controllers/api/v1/media_search_controller.ex @@ -98,7 +98,7 @@ defmodule RetWeb.Api.V1.MediaSearchController do def index(conn, %{"source" => source} = params) when source in [ "sketchfab", - "poly", + "icosa", "tenor", "youtube_videos", "bing_videos", @@ -128,7 +128,7 @@ defmodule RetWeb.Api.V1.MediaSearchController do # For Google services, increase cache duration for landing pages by using long-lived cache, due to quotas. defp cache_for_query(%Ret.MediaSearchQuery{source: source, q: nil}) - when source == "youtube_videos" or source == "poly", + when source == "youtube_videos" or source == "icosa", do: :media_search_results_long defp cache_for_query(_query), do: :media_search_results From 85bb2d0b7b4257748d1bc2d6e4a5ca8adf5dfd24 Mon Sep 17 00:00:00 2001 From: Andy Baker Date: Fri, 30 Aug 2024 11:34:27 +0100 Subject: [PATCH 04/26] Update media_resolver.ex --- lib/ret/media_resolver.ex | 7 ------- 1 file changed, 7 deletions(-) diff --git a/lib/ret/media_resolver.ex b/lib/ret/media_resolver.ex index fddd25834..57e14157c 100644 --- a/lib/ret/media_resolver.ex +++ b/lib/ret/media_resolver.ex @@ -80,13 +80,6 @@ defmodule Ret.MediaResolver do |> resolved()} end - # Necessary short circuit around google.com root_host to skip YT-DL check for Poly - def resolve(%MediaResolverQuery{url: %URI{host: "icosa-api.ixxy.co.uk"}} = query, root_host) do - rate_limited_resolve(query, root_host, @poly_rate_limit, fn -> - resolve_non_video(query, root_host) - end) - end - # Necessary short circuit around google.com root_host to skip YT-DL check for Poly def resolve(%MediaResolverQuery{url: %URI{host: "icosa.ixxy.co.uk"}} = query, root_host) do rate_limited_resolve(query, root_host, @poly_rate_limit, fn -> From aa396187774c94298d099b71fdbfef253d76164a Mon Sep 17 00:00:00 2001 From: Craig Moore Date: Fri, 6 Sep 2024 16:18:42 +0100 Subject: [PATCH 05/26] Replacing MediaSearchQuery to not check that the google_poly api key was available --- lib/ret/media_search.ex | 65 +++++++++++++++++++---------------------- 1 file changed, 30 insertions(+), 35 deletions(-) diff --git a/lib/ret/media_search.ex b/lib/ret/media_search.ex index 8af3771d5..040346e71 100644 --- a/lib/ret/media_search.ex +++ b/lib/ret/media_search.ex @@ -250,42 +250,37 @@ defmodule Ret.MediaSearch do end def search(%Ret.MediaSearchQuery{source: "icosa", cursor: cursor, filter: filter, q: q}) do - with api_key when is_binary(api_key) <- resolver_config(:google_poly_api_key) do - query = - URI.encode_query( - pageSize: @page_size, - maxComplexity: :MEDIUM, - format: :GLTF2, - pageToken: cursor, - category: filter, - keywords: q, - key: api_key - ) - - res = - "https://icosa-api.ixxy.co.uk/v1/assets?#{query}" - |> retry_get_until_success() - - case res do - :error -> - :error - - res -> - decoded_res = res |> Map.get(:body) |> Poison.decode!() - entries = decoded_res |> Map.get("assets") |> Enum.map(&poly_api_result_to_entry/1) - next_cursor = decoded_res |> Map.get("nextPageToken") + query = + URI.encode_query( + pageSize: @page_size, + maxComplexity: :MEDIUM, + format: :GLTF2, + pageToken: cursor, + category: filter, + keywords: q + ) - {:commit, - %Ret.MediaSearchResult{ - meta: %Ret.MediaSearchResultMeta{ - next_cursor: next_cursor, - source: :icosa - }, - entries: entries - }} - end - else - _ -> nil + res = + "https://icosa-api.ixxy.co.uk/v1/assets?#{query}" + |> retry_get_until_success() + + case res do + :error -> + :error + + res -> + decoded_res = res |> Map.get(:body) |> Poison.decode!() + entries = decoded_res |> Map.get("assets") |> Enum.map(&poly_api_result_to_entry/1) + next_cursor = decoded_res |> Map.get("nextPageToken") + + {:commit, + %Ret.MediaSearchResult{ + meta: %Ret.MediaSearchResultMeta{ + next_cursor: next_cursor, + source: :icosa + }, + entries: entries + }} end end From ee8ce21eb325f0adf9cc4008d33fd98b72ef0812 Mon Sep 17 00:00:00 2001 From: Andy Baker Date: Fri, 6 Sep 2024 16:23:58 +0100 Subject: [PATCH 06/26] remove api key from single asset api call --- lib/ret/media_resolver.ex | 49 ++++++++++++++++++--------------------- 1 file changed, 23 insertions(+), 26 deletions(-) diff --git a/lib/ret/media_resolver.ex b/lib/ret/media_resolver.ex index 57e14157c..6ed140d0b 100644 --- a/lib/ret/media_resolver.ex +++ b/lib/ret/media_resolver.ex @@ -336,37 +336,34 @@ defmodule Ret.MediaResolver do end defp resolve_non_video( - %MediaResolverQuery{url: %URI{host: "icosa.ixxy.co.uk", path: "/view/" <> asset_id} = uri}, - "google.com" - ) do - [uri, meta] = - with api_key when is_binary(api_key) <- module_config(:google_poly_api_key) do - Statix.increment("ret.media_resolver.poly.requests") - - payload = - "https://icosa-api.ixxy.co.uk/v1/assets/#{asset_id}?key=#{api_key}" - |> retry_get_until_success - |> Map.get(:body) - |> Poison.decode!() + %MediaResolverQuery{url: %URI{host: "icosa.ixxy.co.uk", path: "/view/" <> asset_id} = uri}, + "google.com" + ) do + [uri, meta] = + Statix.increment("ret.media_resolver.poly.requests") + + payload = + "https://icosa-api.ixxy.co.uk/v1/assets/#{asset_id}?" + |> retry_get_until_success + |> Map.get(:body) + |> Poison.decode!() - meta = - %{expected_content_type: "model/gltf"} - |> Map.put(:name, payload["displayName"]) - |> Map.put(:author, payload["authorName"]) - |> Map.put(:license, payload["license"]) + meta = + %{expected_content_type: "model/gltf"} + |> Map.put(:name, payload["displayName"]) + |> Map.put(:author, payload["authorName"]) + |> Map.put(:license, payload["license"]) - formats = payload |> Map.get("formats") + formats = payload |> Map.get("formats") - uri = - (Enum.find(formats, &(&1["formatType"] == "GLTF2")) || Enum.find(formats, &(&1["formatType"] == "GLTF"))) - |> Kernel.get_in(["root", "url"]) - |> URI.parse() + uri = + (Enum.find(formats, &(&1["formatType"] == "GLTF2")) || Enum.find(formats, &(&1["formatType"] == "GLTF"))) + |> Kernel.get_in(["root", "url"]) + |> URI.parse() - Statix.increment("ret.media_resolver.poly.ok") + Statix.increment("ret.media_resolver.poly.ok") - [uri, meta] - else - _err -> [uri, nil] + [uri, meta] end {:commit, uri |> resolved(meta)} From 08f741ec2c4d85dca6c5ff6bb4920cd107441437 Mon Sep 17 00:00:00 2001 From: Andy Baker Date: Fri, 13 Sep 2024 17:10:23 +0100 Subject: [PATCH 07/26] switch resolver to api instead of front end urls --- lib/ret/media_resolver.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/ret/media_resolver.ex b/lib/ret/media_resolver.ex index 6ed140d0b..294015b40 100644 --- a/lib/ret/media_resolver.ex +++ b/lib/ret/media_resolver.ex @@ -336,8 +336,8 @@ defmodule Ret.MediaResolver do end defp resolve_non_video( - %MediaResolverQuery{url: %URI{host: "icosa.ixxy.co.uk", path: "/view/" <> asset_id} = uri}, - "google.com" + %MediaResolverQuery{url: %URI{host: "icosa-api.ixxy.co.uk", path: "/assets/" <> asset_id} = uri}, + "ixxy.co.uk" ) do [uri, meta] = Statix.increment("ret.media_resolver.poly.requests") From dd7eed2a5e7b508cf0e25e459e1e78a21d331d8b Mon Sep 17 00:00:00 2001 From: Andy Baker Date: Fri, 13 Sep 2024 17:10:41 +0100 Subject: [PATCH 08/26] only ask for GLTF2 --- lib/ret/media_resolver.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/ret/media_resolver.ex b/lib/ret/media_resolver.ex index 294015b40..bf803b31e 100644 --- a/lib/ret/media_resolver.ex +++ b/lib/ret/media_resolver.ex @@ -357,7 +357,7 @@ defmodule Ret.MediaResolver do formats = payload |> Map.get("formats") uri = - (Enum.find(formats, &(&1["formatType"] == "GLTF2")) || Enum.find(formats, &(&1["formatType"] == "GLTF"))) + (Enum.find(formats, &(&1["formatType"] == "GLTF2"))) |> Kernel.get_in(["root", "url"]) |> URI.parse() From 8069c31a52c668bd00d123170b6efbc8faccd4b5 Mon Sep 17 00:00:00 2001 From: Andy Baker Date: Fri, 13 Sep 2024 17:30:01 +0100 Subject: [PATCH 09/26] Back to front end url but add root to non_video_root_hosts --- lib/ret/media_resolver.ex | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/ret/media_resolver.ex b/lib/ret/media_resolver.ex index bf803b31e..2cb751bcd 100644 --- a/lib/ret/media_resolver.ex +++ b/lib/ret/media_resolver.ex @@ -26,7 +26,8 @@ defmodule Ret.MediaResolver do @non_video_root_hosts [ "sketchfab.com", "giphy.com", - "tenor.com" + "tenor.com", + "ixxy.co.uk" ] @deviant_id_regex ~r/\"DeviantArt:\/\/deviation\/([^"]+)/ @@ -336,7 +337,7 @@ defmodule Ret.MediaResolver do end defp resolve_non_video( - %MediaResolverQuery{url: %URI{host: "icosa-api.ixxy.co.uk", path: "/assets/" <> asset_id} = uri}, + %MediaResolverQuery{url: %URI{host: "icosa.ixxy.co.uk", path: "/views/" <> asset_id} = uri}, "ixxy.co.uk" ) do [uri, meta] = From 3cea8428d0c05233968ba7664e9d4eacfecc4d97 Mon Sep 17 00:00:00 2001 From: Andy Baker Date: Fri, 13 Sep 2024 17:46:25 +0100 Subject: [PATCH 10/26] Sigh. Back to api urls --- lib/ret/media_resolver.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/ret/media_resolver.ex b/lib/ret/media_resolver.ex index 2cb751bcd..1f224a059 100644 --- a/lib/ret/media_resolver.ex +++ b/lib/ret/media_resolver.ex @@ -82,7 +82,7 @@ defmodule Ret.MediaResolver do end # Necessary short circuit around google.com root_host to skip YT-DL check for Poly - def resolve(%MediaResolverQuery{url: %URI{host: "icosa.ixxy.co.uk"}} = query, root_host) do + def resolve(%MediaResolverQuery{url: %URI{host: "icosa-api.ixxy.co.uk"}} = query, root_host) do rate_limited_resolve(query, root_host, @poly_rate_limit, fn -> resolve_non_video(query, root_host) end) @@ -337,7 +337,7 @@ defmodule Ret.MediaResolver do end defp resolve_non_video( - %MediaResolverQuery{url: %URI{host: "icosa.ixxy.co.uk", path: "/views/" <> asset_id} = uri}, + %MediaResolverQuery{url: %URI{host: "icosa-api.ixxy.co.uk", path: "/views/" <> asset_id} = uri}, "ixxy.co.uk" ) do [uri, meta] = From 2e82b1f061bfa7b5076053de78a931e2dfeb321f Mon Sep 17 00:00:00 2001 From: Andy Baker Date: Fri, 13 Sep 2024 17:52:17 +0100 Subject: [PATCH 11/26] Try moving all urls to be api urls --- lib/ret/media_resolver.ex | 2 +- lib/ret/media_search.ex | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/ret/media_resolver.ex b/lib/ret/media_resolver.ex index 1f224a059..a0a676a7c 100644 --- a/lib/ret/media_resolver.ex +++ b/lib/ret/media_resolver.ex @@ -337,7 +337,7 @@ defmodule Ret.MediaResolver do end defp resolve_non_video( - %MediaResolverQuery{url: %URI{host: "icosa-api.ixxy.co.uk", path: "/views/" <> asset_id} = uri}, + %MediaResolverQuery{url: %URI{host: "icosa-api.ixxy.co.uk", path: "v1/assets/" <> asset_id} = uri}, "ixxy.co.uk" ) do [uri, meta] = diff --git a/lib/ret/media_search.ex b/lib/ret/media_search.ex index 040346e71..57a88ed1e 100644 --- a/lib/ret/media_search.ex +++ b/lib/ret/media_search.ex @@ -995,11 +995,11 @@ defmodule Ret.MediaSearch do defp poly_api_result_to_entry(result) do %{ - id: result["name"], + id: result["assetId"], type: "poly_model", name: result["displayName"], attributions: %{creator: %{name: result["authorName"]}}, - url: "https://icosa.ixxy.co.uk/view/#{result["name"] |> String.replace("assets/", "")}", + url: result["url"], images: %{preview: %{url: result["thumbnail"]["url"]}} } end From 8bae0a626d8dce26ff08c9c25f4fbd6a24876cba Mon Sep 17 00:00:00 2001 From: Andy Baker Date: Fri, 13 Sep 2024 17:53:54 +0100 Subject: [PATCH 12/26] quick fix for ".co.uk" --- lib/ret/media_resolver.ex | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/lib/ret/media_resolver.ex b/lib/ret/media_resolver.ex index a0a676a7c..b3d1c90aa 100644 --- a/lib/ret/media_resolver.ex +++ b/lib/ret/media_resolver.ex @@ -665,10 +665,19 @@ defmodule Ret.MediaResolver do nil end - defp get_root_host(host) do - # Drop subdomains - host |> String.split(".") |> Enum.slice(-2..-1) |> Enum.join(".") +defp get_root_host(host) do + if String.ends_with?(host, ".co.uk") do + host + |> String.split(".") + |> Enum.slice(-3..-1) + |> Enum.join(".") + else + host + |> String.split(".") + |> Enum.slice(-2..-1) + |> Enum.join(".") end +end defp get_imgur_headers() do with client_id when is_binary(client_id) <- module_config(:imgur_client_id), From 525a1a065312ce7085d84ee514167f1c541d0861 Mon Sep 17 00:00:00 2001 From: Craig Moore Date: Fri, 27 Sep 2024 13:45:47 +0100 Subject: [PATCH 13/26] Minor fixes for resolving icosa assets --- lib/ret/media_resolver.ex | 66 +++++++++++++++++++++------------------ 1 file changed, 35 insertions(+), 31 deletions(-) diff --git a/lib/ret/media_resolver.ex b/lib/ret/media_resolver.ex index b3d1c90aa..9df633a6d 100644 --- a/lib/ret/media_resolver.ex +++ b/lib/ret/media_resolver.ex @@ -337,39 +337,43 @@ defmodule Ret.MediaResolver do end defp resolve_non_video( - %MediaResolverQuery{url: %URI{host: "icosa-api.ixxy.co.uk", path: "v1/assets/" <> asset_id} = uri}, - "ixxy.co.uk" - ) do - [uri, meta] = - Statix.increment("ret.media_resolver.poly.requests") - - payload = - "https://icosa-api.ixxy.co.uk/v1/assets/#{asset_id}?" - |> retry_get_until_success - |> Map.get(:body) - |> Poison.decode!() - - meta = - %{expected_content_type: "model/gltf"} - |> Map.put(:name, payload["displayName"]) - |> Map.put(:author, payload["authorName"]) - |> Map.put(:license, payload["license"]) - - formats = payload |> Map.get("formats") - - uri = - (Enum.find(formats, &(&1["formatType"] == "GLTF2"))) - |> Kernel.get_in(["root", "url"]) - |> URI.parse() - - Statix.increment("ret.media_resolver.poly.ok") - - [uri, meta] - end - - {:commit, uri |> resolved(meta)} + %MediaResolverQuery{url: %URI{host: "icosa-api.ixxy.co.uk", path: "/v1/assets/" <> asset_id} = uri}, + "ixxy.co.uk" + ) do + # Increment stat for the request + Statix.increment("ret.media_resolver.poly.requests") + + # Make the API call to get the asset data + payload = + "https://icosa-api.ixxy.co.uk/v1/assets/#{asset_id}" + |> retry_get_until_success() # Assuming this function sends the request and handles retries + |> Map.get(:body) + |> Poison.decode!() + + # Create the meta information based on the payload + meta = + %{ + expected_content_type: "model/gltf", + name: payload["displayName"], + author: payload["authorName"], + license: payload["license"] + } + + # Extract the GLTF2 format URL from the payload + uri = + payload["formats"] + |> Enum.find(&(&1["formatType"] == "GLTF2")) + |> Kernel.get_in(["root", "url"]) + |> URI.parse() + + # Increment stat for successful resolution + Statix.increment("ret.media_resolver.poly.ok") + + # Return the URI and meta data for further processing + {:commit, resolved(uri, meta)} end + defp resolve_non_video( %MediaResolverQuery{url: %URI{path: "/models/" <> model_id}} = query, "sketchfab.com" = root_host From c8b356ebc32b8b191e37c9cce99a2dad8527394d Mon Sep 17 00:00:00 2001 From: Craig Moore Date: Fri, 27 Sep 2024 13:53:33 +0100 Subject: [PATCH 14/26] Adding temporary fix for icosa asset url --- lib/ret/media_search.ex | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/ret/media_search.ex b/lib/ret/media_search.ex index 57a88ed1e..b687f009b 100644 --- a/lib/ret/media_search.ex +++ b/lib/ret/media_search.ex @@ -999,7 +999,8 @@ defmodule Ret.MediaSearch do type: "poly_model", name: result["displayName"], attributions: %{creator: %{name: result["authorName"]}}, - url: result["url"], + url: "http://icosa-api.ixxy.co.uk/v1/assets/" <> result["assetId"], + # url: result["url"], images: %{preview: %{url: result["thumbnail"]["url"]}} } end From 5a1b0a72827cbec57a33c2f084babb1b4f43907c Mon Sep 17 00:00:00 2001 From: Andy Baker Date: Fri, 25 Oct 2024 10:25:14 +0100 Subject: [PATCH 15/26] Remove references to google_poly_api_key This reverts commit 3cea8428d0c05233968ba7664e9d4eacfecc4d97. --- config/dev.exs | 1 - config/test.exs | 1 - habitat/config/config.toml | 3 --- lib/ret/media_search.ex | 2 +- 4 files changed, 1 insertion(+), 6 deletions(-) diff --git a/config/dev.exs b/config/dev.exs index 702575eb3..2da1f4fc3 100644 --- a/config/dev.exs +++ b/config/dev.exs @@ -128,7 +128,6 @@ config :ret, Ret.MediaResolver, deviantart_client_secret: nil, imgur_mashape_api_key: nil, imgur_client_id: nil, - google_poly_api_key: nil, youtube_api_key: nil, sketchfab_api_key: nil, ytdl_host: nil, diff --git a/config/test.exs b/config/test.exs index d4d2ee4bc..9c1cf1b9c 100644 --- a/config/test.exs +++ b/config/test.exs @@ -66,7 +66,6 @@ config :ret, Ret.MediaResolver, deviantart_client_secret: nil, imgur_mashape_api_key: nil, imgur_client_id: nil, - google_poly_api_key: nil, youtube_api_key: nil, sketchfab_api_key: nil, ytdl_host: nil, diff --git a/habitat/config/config.toml b/habitat/config/config.toml index c1fad4c91..0b36a5f62 100644 --- a/habitat/config/config.toml +++ b/habitat/config/config.toml @@ -197,9 +197,6 @@ deviantart_client_secret = {{ toToml cfg.resolver.deviantart_client_secret }} imgur_mashape_api_key = {{ toToml cfg.resolver.imgur_mashape_api_key }} imgur_client_id = {{ toToml cfg.resolver.imgur_client_id }} {{/if}} -{{ #if cfg.resolver.google_poly_api_key }} -google_poly_api_key = {{ toToml cfg.resolver.google_poly_api_key }} -{{/if}} {{ #if cfg.resolver.youtube_api_key }} youtube_api_key = {{ toToml cfg.resolver.youtube_api_key }} {{/if}} diff --git a/lib/ret/media_search.ex b/lib/ret/media_search.ex index b687f009b..d226a1ae7 100644 --- a/lib/ret/media_search.ex +++ b/lib/ret/media_search.ex @@ -412,7 +412,7 @@ defmodule Ret.MediaSearch do end end - def available?(:icosa), do: has_resolver_config?(:google_poly_api_key) + def available?(:icosa), do: true # Icosa does not currently require an API key def available?(:bing_images), do: has_resolver_config?(:bing_search_api_key) def available?(:bing_videos), do: has_resolver_config?(:bing_search_api_key) def available?(:youtube_videos), do: has_resolver_config?(:youtube_api_key) From d6fe31e7991190355805639a091f2b4c487cb4c5 Mon Sep 17 00:00:00 2001 From: Andy Baker Date: Fri, 25 Oct 2024 10:40:27 +0100 Subject: [PATCH 16/26] rename another poly to icosa --- lib/ret/media_resolver.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/ret/media_resolver.ex b/lib/ret/media_resolver.ex index 9df633a6d..ed244694c 100644 --- a/lib/ret/media_resolver.ex +++ b/lib/ret/media_resolver.ex @@ -20,7 +20,7 @@ defmodule Ret.MediaResolver do @youtube_rate_limit %{scale: 8_000, limit: 1} @sketchfab_rate_limit %{scale: 60_000, limit: 15} - @poly_rate_limit %{scale: 60_000, limit: 1000} + @icosa_rate_limit %{scale: 60_000, limit: 1000} @max_await_for_rate_limit_s 120 @non_video_root_hosts [ @@ -83,7 +83,7 @@ defmodule Ret.MediaResolver do # Necessary short circuit around google.com root_host to skip YT-DL check for Poly def resolve(%MediaResolverQuery{url: %URI{host: "icosa-api.ixxy.co.uk"}} = query, root_host) do - rate_limited_resolve(query, root_host, @poly_rate_limit, fn -> + rate_limited_resolve(query, root_host, @icosa_rate_limit, fn -> resolve_non_video(query, root_host) end) end From 59e8127f6fdaf687945aaecc8e690095c25eabc7 Mon Sep 17 00:00:00 2001 From: Andy Baker Date: Fri, 25 Oct 2024 10:43:28 +0100 Subject: [PATCH 17/26] More poly > icosa renaming --- lib/ret/media_resolver.ex | 4 ++-- lib/ret/media_search.ex | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/ret/media_resolver.ex b/lib/ret/media_resolver.ex index ed244694c..873b93916 100644 --- a/lib/ret/media_resolver.ex +++ b/lib/ret/media_resolver.ex @@ -341,7 +341,7 @@ defmodule Ret.MediaResolver do "ixxy.co.uk" ) do # Increment stat for the request - Statix.increment("ret.media_resolver.poly.requests") + Statix.increment("ret.media_resolver.icosa.requests") # Make the API call to get the asset data payload = @@ -367,7 +367,7 @@ defmodule Ret.MediaResolver do |> URI.parse() # Increment stat for successful resolution - Statix.increment("ret.media_resolver.poly.ok") + Statix.increment("ret.media_resolver.icosa.ok") # Return the URI and meta data for further processing {:commit, resolved(uri, meta)} diff --git a/lib/ret/media_search.ex b/lib/ret/media_search.ex index d226a1ae7..3f396d7c6 100644 --- a/lib/ret/media_search.ex +++ b/lib/ret/media_search.ex @@ -270,7 +270,7 @@ defmodule Ret.MediaSearch do res -> decoded_res = res |> Map.get(:body) |> Poison.decode!() - entries = decoded_res |> Map.get("assets") |> Enum.map(&poly_api_result_to_entry/1) + entries = decoded_res |> Map.get("assets") |> Enum.map(&icosa_api_result_to_entry/1) next_cursor = decoded_res |> Map.get("nextPageToken") {:commit, @@ -993,10 +993,10 @@ defmodule Ret.MediaSearch do } end - defp poly_api_result_to_entry(result) do + defp icosa_api_result_to_entry(result) do %{ id: result["assetId"], - type: "poly_model", + type: "icosa_model", name: result["displayName"], attributions: %{creator: %{name: result["authorName"]}}, url: "http://icosa-api.ixxy.co.uk/v1/assets/" <> result["assetId"], From cb896bb0d334c04ce378e2f3487cbba4fec0c345 Mon Sep 17 00:00:00 2001 From: Andy Baker Date: Wed, 13 Nov 2024 10:47:15 +0000 Subject: [PATCH 18/26] Switch from staging to production urls --- lib/ret/media_resolver.ex | 10 +++++----- lib/ret/media_search.ex | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/ret/media_resolver.ex b/lib/ret/media_resolver.ex index 873b93916..a6892f0d3 100644 --- a/lib/ret/media_resolver.ex +++ b/lib/ret/media_resolver.ex @@ -27,7 +27,7 @@ defmodule Ret.MediaResolver do "sketchfab.com", "giphy.com", "tenor.com", - "ixxy.co.uk" + "icosa.gallery" ] @deviant_id_regex ~r/\"DeviantArt:\/\/deviation\/([^"]+)/ @@ -82,7 +82,7 @@ defmodule Ret.MediaResolver do end # Necessary short circuit around google.com root_host to skip YT-DL check for Poly - def resolve(%MediaResolverQuery{url: %URI{host: "icosa-api.ixxy.co.uk"}} = query, root_host) do + def resolve(%MediaResolverQuery{url: %URI{host: "api.icosa.gallery"}} = query, root_host) do rate_limited_resolve(query, root_host, @icosa_rate_limit, fn -> resolve_non_video(query, root_host) end) @@ -337,15 +337,15 @@ defmodule Ret.MediaResolver do end defp resolve_non_video( - %MediaResolverQuery{url: %URI{host: "icosa-api.ixxy.co.uk", path: "/v1/assets/" <> asset_id} = uri}, - "ixxy.co.uk" + %MediaResolverQuery{url: %URI{host: "api.icosa.gallery", path: "/v1/assets/" <> asset_id} = uri}, + "icosa.gallery" ) do # Increment stat for the request Statix.increment("ret.media_resolver.icosa.requests") # Make the API call to get the asset data payload = - "https://icosa-api.ixxy.co.uk/v1/assets/#{asset_id}" + "https://api.icosa.gallery/v1/assets/#{asset_id}" |> retry_get_until_success() # Assuming this function sends the request and handles retries |> Map.get(:body) |> Poison.decode!() diff --git a/lib/ret/media_search.ex b/lib/ret/media_search.ex index 3f396d7c6..54de43ce9 100644 --- a/lib/ret/media_search.ex +++ b/lib/ret/media_search.ex @@ -261,7 +261,7 @@ defmodule Ret.MediaSearch do ) res = - "https://icosa-api.ixxy.co.uk/v1/assets?#{query}" + "https://api.icosa.gallery/v1/assets?#{query}" |> retry_get_until_success() case res do @@ -999,7 +999,7 @@ defmodule Ret.MediaSearch do type: "icosa_model", name: result["displayName"], attributions: %{creator: %{name: result["authorName"]}}, - url: "http://icosa-api.ixxy.co.uk/v1/assets/" <> result["assetId"], + url: "http://api.icosa.gallery/v1/assets/" <> result["assetId"], # url: result["url"], images: %{preview: %{url: result["thumbnail"]["url"]}} } From 4343a30ae3122569de8ba35cfb31f5ac1a112b82 Mon Sep 17 00:00:00 2001 From: Andy Baker Date: Wed, 13 Nov 2024 10:47:45 +0000 Subject: [PATCH 19/26] Revert "quick fix for ".co.uk"" This reverts commit 8bae0a626d8dce26ff08c9c25f4fbd6a24876cba. --- lib/ret/media_resolver.ex | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/lib/ret/media_resolver.ex b/lib/ret/media_resolver.ex index a6892f0d3..015cb554f 100644 --- a/lib/ret/media_resolver.ex +++ b/lib/ret/media_resolver.ex @@ -669,19 +669,10 @@ defmodule Ret.MediaResolver do nil end -defp get_root_host(host) do - if String.ends_with?(host, ".co.uk") do - host - |> String.split(".") - |> Enum.slice(-3..-1) - |> Enum.join(".") - else - host - |> String.split(".") - |> Enum.slice(-2..-1) - |> Enum.join(".") + defp get_root_host(host) do + # Drop subdomains + host |> String.split(".") |> Enum.slice(-2..-1) |> Enum.join(".") end -end defp get_imgur_headers() do with client_id when is_binary(client_id) <- module_config(:imgur_client_id), From bfadcb4ac2c0b360e665a16e5649afcbddf6c7b3 Mon Sep 17 00:00:00 2001 From: Andy Baker Date: Fri, 5 Sep 2025 12:42:22 +0100 Subject: [PATCH 20/26] Only show remixable and order by "best" --- lib/ret/media_search.ex | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/ret/media_search.ex b/lib/ret/media_search.ex index 54de43ce9..762052fc9 100644 --- a/lib/ret/media_search.ex +++ b/lib/ret/media_search.ex @@ -257,6 +257,8 @@ defmodule Ret.MediaSearch do format: :GLTF2, pageToken: cursor, category: filter, + licence: :REMIXABLE, + orderBy: :BEST, keywords: q ) From 653df227b63c10de35194bce23ceb1fe5ddb1887 Mon Sep 17 00:00:00 2001 From: Andy Baker Date: Fri, 5 Sep 2025 12:42:46 +0100 Subject: [PATCH 21/26] Remove empty category param --- lib/ret/media_search.ex | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/ret/media_search.ex b/lib/ret/media_search.ex index 762052fc9..6ca567018 100644 --- a/lib/ret/media_search.ex +++ b/lib/ret/media_search.ex @@ -262,8 +262,12 @@ defmodule Ret.MediaSearch do keywords: q ) + url = "https://api.icosa.gallery/v1/assets?#{query}" + # Remove empty category queries as the Icosa API returns validation errors instead of "any" + url = String.replace(url, "category=&", "") + res = - "https://api.icosa.gallery/v1/assets?#{query}" + url |> retry_get_until_success() case res do From 05143e2ee2146007a2ee34295f1c435ec631f13a Mon Sep 17 00:00:00 2001 From: Andy Baker Date: Mon, 8 Sep 2025 15:09:16 +0100 Subject: [PATCH 22/26] Handle redirects in the CORS proxy Note: The code for this commit was generated by one or more AI tools: probably Github Copilot (GPT-5 Mini?) and potentially also Claude Code CLI (Sonnet 4.5?). --- lib/ret_web/controllers/page_controller.ex | 99 +++++++++++++++++----- 1 file changed, 77 insertions(+), 22 deletions(-) diff --git a/lib/ret_web/controllers/page_controller.ex b/lib/ret_web/controllers/page_controller.ex index 141a277bb..48a5a5b5f 100644 --- a/lib/ret_web/controllers/page_controller.ex +++ b/lib/ret_web/controllers/page_controller.ex @@ -18,6 +18,7 @@ defmodule RetWeb.PageController do alias Plug.Conn import Ret.ConnUtils import Ret.HttpUtils + require Logger ## # NOTE: In addition to adding a route, you must add static html pages to the page_origin_warmer.ex @@ -716,17 +717,24 @@ defmodule RetWeb.PageController do do: cors_proxy(conn, "#{url}?#{qs}") defp cors_proxy(conn, url) do + cors_proxy_with_redirects(conn, url, 0) + end + + defp cors_proxy_with_redirects(conn, url, redirect_count) when redirect_count > 5 do + Logger.error("CORS Proxy: Too many redirects (#{redirect_count}) for URL: #{url}") + conn |> send_resp(400, "Too many redirects") + end + + defp cors_proxy_with_redirects(conn, url, redirect_count) do + %URI{authority: authority, host: host} = uri = URI.parse(url) resolved_ip = HttpUtils.resolve_ip(host) if HttpUtils.internal_ip?(resolved_ip) do + Logger.warning("CORS Proxy: Blocking internal IP #{inspect(resolved_ip)} for host #{host}") conn |> send_resp(401, "Bad request.") else - # We want to ensure that the URL we request hits the same IP that we verified above, - # so we replace the host with the IP address here and use this url to make the proxy request. - ip_url = URI.to_string(HttpUtils.replace_host(uri, resolved_ip)) - # Disallow CORS proxying unless request was made to the cors proxy url cors_proxy_url = Application.get_env(:ret, RetWeb.Endpoint)[:cors_proxy_url] @@ -745,43 +753,90 @@ defmodule RetWeb.PageController do if is_cors_proxy_url do allowed_origins = Application.get_env(:ret, RetWeb.Endpoint)[:allowed_origins] |> String.split(",") + opts = ReverseProxyPlug.init( upstream: url, allowed_origins: allowed_origins, proxy_url: "#{cors_scheme}://#{cors_host}:#{cors_port}", - # Since we replaced the host with the IP address in ip_url above, we need to force the host - # used for ssl verification here so that the connection isn't rejected. - # Note that we have to convert the authority to a charlist, since this uses Erlang's `ssl` module - # internally, which expects a charlist. + # Since we need to use the host for SSL verification, we provide the authority client_options: [ ssl: [{:server_name_indication, to_charlist(authority)}, {:versions, [:"tlsv1.2",:"tlsv1.3"]}] - ], - # preserve_host_header: true + ] ) body = ReverseProxyPlug.read_body(conn) is_head = conn |> Conn.get_req_header("x-original-method") == ["HEAD"] - %Conn{}|> Map.merge(conn) - |> Map.put( - :method, - if is_head do - "HEAD" - else - conn.method + try do + # First, make a HEAD request to check for redirects using HTTPoison + case HTTPoison.head(url, [], [ + follow_redirect: false, + ssl: [{:server_name_indication, to_charlist(authority)}, {:versions, [:"tlsv1.2", :"tlsv1.3"]}], + timeout: 15_000, + recv_timeout: 15_000 + ]) do + {:ok, %HTTPoison.Response{status_code: status_code, headers: headers}} when status_code in [301, 302, 303, 307, 308] -> + # Found a redirect + location_header = headers |> Enum.find(fn {k, _v} -> String.downcase(k) == "location" end) |> elem(1) + + if location_header do + # Resolve relative URLs against the current URL + redirect_url = URI.merge(uri, location_header) |> URI.to_string() + cors_proxy_with_redirects(conn, redirect_url, redirect_count + 1) + else + Logger.warning("CORS Proxy: Redirect response missing location header") + # Fall back to ReverseProxyPlug for this response + make_reverse_proxy_request(conn, url, body, is_head, opts) + end + + {:ok, %HTTPoison.Response{status_code: status_code}} -> + # Not a redirect, use ReverseProxyPlug for the actual request + make_reverse_proxy_request(conn, url, body, is_head, opts) + + {:error, reason} -> + Logger.error("CORS Proxy: HEAD request failed: #{inspect(reason)}") + # Fall back to ReverseProxyPlug anyway + make_reverse_proxy_request(conn, url, body, is_head, opts) end - ) - # Need to strip path_info since proxy plug reads it - |> Map.put(:path_info, []) - |> ReverseProxyPlug.request(body, opts) - |> ReverseProxyPlug.response(conn, opts) + rescue + error -> + Logger.error("CORS Proxy: Request failed with exception: #{inspect(error)}") + conn |> send_resp(500, "Proxy request failed: #{inspect(error)}") + catch + :exit, reason -> + Logger.error("CORS Proxy: Request exited with reason: #{inspect(reason)}") + conn |> send_resp(500, "Proxy request timed out or failed") + kind, reason -> + Logger.error("CORS Proxy: Request failed with #{kind}: #{inspect(reason)}") + conn |> send_resp(500, "Proxy request failed") + end else + Logger.warning("CORS Proxy: Request rejected - invalid host or scheme") conn |> send_resp(401, "Bad request.") end end end + defp make_reverse_proxy_request(conn, _url, body, is_head, opts) do + proxy_conn = %Conn{} + |> Map.merge(conn) + |> Map.put( + :method, + if is_head do + "HEAD" + else + conn.method + end + ) + # Need to strip path_info since proxy plug reads it + |> Map.put(:path_info, []) + |> ReverseProxyPlug.request(body, opts) + |> ReverseProxyPlug.response(conn, opts) + + proxy_conn + end + defp render_static_asset(conn) do static_options = Plug.Static.init(at: "/", from: :ret, gzip: true, brotli: true) Plug.Static.call(conn, static_options) From a636aed78a9bce4bf27b4c6129b67505743e821f Mon Sep 17 00:00:00 2001 From: Andy Baker Date: Wed, 10 Sep 2025 12:08:25 +0100 Subject: [PATCH 23/26] mix format --- lib/ret/discord_client.ex | 81 +++++---- lib/ret/http_utils.ex | 15 +- lib/ret/media_resolver.ex | 70 ++++---- lib/ret/media_search.ex | 4 +- lib/ret/scene.ex | 11 +- lib/ret/storage.ex | 8 +- lib/ret/storage_used.ex | 2 +- .../api/v1/media_search_controller.ex | 34 ++-- lib/ret_web/controllers/file_controller.ex | 12 +- lib/ret_web/controllers/page_controller.ex | 162 +++++++++++------- lib/ret_web/email.ex | 4 +- ...190114232257_add_reviewed_at_to_scenes.exs | 4 +- test/ret_web/channels/entity_test.exs | 4 +- 13 files changed, 247 insertions(+), 164 deletions(-) diff --git a/lib/ret/discord_client.ex b/lib/ret/discord_client.ex index 1e0fea6c3..8f220098c 100644 --- a/lib/ret/discord_client.ex +++ b/lib/ret/discord_client.ex @@ -76,21 +76,23 @@ defmodule Ret.DiscordClient do {status, result} when status in [:commit, :ok] -> "#{result["nick"]}" end - nickname = if !nickname or nickname == "" do - case Cachex.fetch(:discord_api, "/users/#{provider_account_id}") do - {status, result} when status in [:commit, :ok] -> "#{result["global_name"]}" - end + nickname = + if !nickname or nickname == "" do + case Cachex.fetch(:discord_api, "/users/#{provider_account_id}") do + {status, result} when status in [:commit, :ok] -> "#{result["global_name"]}" + end else nickname - end - - nickname = if !nickname or nickname == "" do - case Cachex.fetch(:discord_api, "/users/#{provider_account_id}") do - {status, result} when status in [:commit, :ok] -> "#{result["username"]}" end + + nickname = + if !nickname or nickname == "" do + case Cachex.fetch(:discord_api, "/users/#{provider_account_id}") do + {status, result} when status in [:commit, :ok] -> "#{result["username"]}" + end else nickname - end + end end def fetch_community_identifier(%Ret.OAuthProvider{ @@ -169,18 +171,21 @@ defmodule Ret.DiscordClient do case Cachex.fetch(:discord_api, "/guilds/#{community_id}/roles") do {status, result} when status in [:commit, :ok] -> result |> Map.new(&{&1["id"], &1}) end - # Note: Whether the bitfield values in guild_roles are represented as strings or integers is inconsistent (possibly based on what permissions the user has), so every time they're used they need to be checked and, if needed, converted to integers. + + # Note: Whether the bitfield values in guild_roles are represented as strings or integers is inconsistent (possibly based on what permissions the user has), so every time they're used they need to be checked and, if needed, converted to integers. role_everyone = guild_roles[community_id] permissions = role_everyone["permissions"] user_permissions = user_roles |> Enum.map(&guild_roles[&1]["permissions"]) - permissions = user_permissions |> - Enum.reduce(permissions, &( - (if is_binary(&1), do: String.to_integer(&1), else: &1) ||| - (if is_binary(&2), do: String.to_integer(&2), else: &2) - )) + permissions = + user_permissions + |> Enum.reduce( + permissions, + &(if(is_binary(&1), do: String.to_integer(&1), else: &1) ||| + if(is_binary(&2), do: String.to_integer(&2), else: &2)) + ) if (permissions &&& @administrator) == @administrator do @all @@ -203,15 +208,21 @@ defmodule Ret.DiscordClient do |> Map.get("permission_overwrites") |> Map.new(&{&1["id"], &1}) end - # Note: Whether the bitfield values in channel_overwrites are represented as strings or integers is inconsistent (possibly based on what permissions the user has), so every time they're used they need to be checked and, if needed, converted to integers. + + # Note: Whether the bitfield values in channel_overwrites are represented as strings or integers is inconsistent (possibly based on what permissions the user has), so every time they're used they need to be checked and, if needed, converted to integers. overwrite_everyone = channel_overwrites[community_id] permissions = if overwrite_everyone do (permissions &&& - ~~~(if is_binary(overwrite_everyone["deny"]), do: String.to_integer(overwrite_everyone["deny"]), else: overwrite_everyone["deny"])) ||| - (if is_binary(overwrite_everyone["allow"]), do: String.to_integer(overwrite_everyone["allow"]), else: overwrite_everyone["allow"]) + ~~~if(is_binary(overwrite_everyone["deny"]), + do: String.to_integer(overwrite_everyone["deny"]), + else: overwrite_everyone["deny"] + )) ||| + if is_binary(overwrite_everyone["allow"]), + do: String.to_integer(overwrite_everyone["allow"]), + else: overwrite_everyone["allow"] else permissions end @@ -220,14 +231,21 @@ defmodule Ret.DiscordClient do user_permissions = user_roles |> Enum.map(&channel_overwrites[&1]) |> Enum.filter(&(&1 != nil)) - allow = user_permissions |> Enum.reduce(@none, &( - (if is_binary(&1["allow"]), do: String.to_integer(&1["allow"]), else: &1["allow"]) ||| - &2 - )) - deny = user_permissions |> Enum.reduce(@none, &( - (if is_binary(&1["deny"]), do: String.to_integer(&1["deny"]), else: &1["deny"]) ||| - &2 - )) + allow = + user_permissions + |> Enum.reduce( + @none, + &(if(is_binary(&1["allow"]), do: String.to_integer(&1["allow"]), else: &1["allow"]) ||| + &2) + ) + + deny = + user_permissions + |> Enum.reduce( + @none, + &(if(is_binary(&1["deny"]), do: String.to_integer(&1["deny"]), else: &1["deny"]) ||| + &2) + ) permissions = (permissions &&& ~~~deny) ||| allow @@ -237,8 +255,13 @@ defmodule Ret.DiscordClient do permissions = if overwrite_member do (permissions &&& - ~~~(if is_binary(overwrite_member["deny"]), do: String.to_integer(overwrite_member["deny"]), else: overwrite_member["deny"])) ||| - (if is_binary(overwrite_member["allow"]), do: String.to_integer(overwrite_member["allow"]), else: overwrite_member["allow"]) + ~~~if(is_binary(overwrite_member["deny"]), + do: String.to_integer(overwrite_member["deny"]), + else: overwrite_member["deny"] + )) ||| + if is_binary(overwrite_member["allow"]), + do: String.to_integer(overwrite_member["allow"]), + else: overwrite_member["allow"] else permissions end diff --git a/lib/ret/http_utils.ex b/lib/ret/http_utils.ex index 71366aba8..8bacdebfe 100644 --- a/lib/ret/http_utils.ex +++ b/lib/ret/http_utils.ex @@ -177,12 +177,14 @@ defmodule Ret.HttpUtils do end def join_smart(enum) do - Enum.reduce(enum, "", fn(x, acc) -> - x = cond do - !x -> nil - is_binary(x) -> String.trim(x) - true -> "#{x}" - end + Enum.reduce(enum, "", fn x, acc -> + x = + cond do + !x -> nil + is_binary(x) -> String.trim(x) + true -> "#{x}" + end + if x && x != "" do if acc && acc != "", do: acc <> " — " <> x, else: x else @@ -190,5 +192,4 @@ defmodule Ret.HttpUtils do end end) end - end diff --git a/lib/ret/media_resolver.ex b/lib/ret/media_resolver.ex index 015cb554f..43d162e14 100644 --- a/lib/ret/media_resolver.ex +++ b/lib/ret/media_resolver.ex @@ -337,42 +337,44 @@ defmodule Ret.MediaResolver do end defp resolve_non_video( - %MediaResolverQuery{url: %URI{host: "api.icosa.gallery", path: "/v1/assets/" <> asset_id} = uri}, + %MediaResolverQuery{ + url: %URI{host: "api.icosa.gallery", path: "/v1/assets/" <> asset_id} = uri + }, "icosa.gallery" - ) do - # Increment stat for the request - Statix.increment("ret.media_resolver.icosa.requests") - - # Make the API call to get the asset data - payload = - "https://api.icosa.gallery/v1/assets/#{asset_id}" - |> retry_get_until_success() # Assuming this function sends the request and handles retries - |> Map.get(:body) - |> Poison.decode!() - - # Create the meta information based on the payload - meta = - %{ - expected_content_type: "model/gltf", - name: payload["displayName"], - author: payload["authorName"], - license: payload["license"] - } - - # Extract the GLTF2 format URL from the payload - uri = - payload["formats"] - |> Enum.find(&(&1["formatType"] == "GLTF2")) - |> Kernel.get_in(["root", "url"]) - |> URI.parse() - - # Increment stat for successful resolution - Statix.increment("ret.media_resolver.icosa.ok") - - # Return the URI and meta data for further processing - {:commit, resolved(uri, meta)} - end + ) do + # Increment stat for the request + Statix.increment("ret.media_resolver.icosa.requests") + + # Make the API call to get the asset data + payload = + "https://api.icosa.gallery/v1/assets/#{asset_id}" + # Assuming this function sends the request and handles retries + |> retry_get_until_success() + |> Map.get(:body) + |> Poison.decode!() + + # Create the meta information based on the payload + meta = + %{ + expected_content_type: "model/gltf", + name: payload["displayName"], + author: payload["authorName"], + license: payload["license"] + } + # Extract the GLTF2 format URL from the payload + uri = + payload["formats"] + |> Enum.find(&(&1["formatType"] == "GLTF2")) + |> Kernel.get_in(["root", "url"]) + |> URI.parse() + + # Increment stat for successful resolution + Statix.increment("ret.media_resolver.icosa.ok") + + # Return the URI and meta data for further processing + {:commit, resolved(uri, meta)} + end defp resolve_non_video( %MediaResolverQuery{url: %URI{path: "/models/" <> model_id}} = query, diff --git a/lib/ret/media_search.ex b/lib/ret/media_search.ex index 6ca567018..55de070fd 100644 --- a/lib/ret/media_search.ex +++ b/lib/ret/media_search.ex @@ -418,7 +418,8 @@ defmodule Ret.MediaSearch do end end - def available?(:icosa), do: true # Icosa does not currently require an API key + # Icosa does not currently require an API key + def available?(:icosa), do: true def available?(:bing_images), do: has_resolver_config?(:bing_search_api_key) def available?(:bing_videos), do: has_resolver_config?(:bing_search_api_key) def available?(:youtube_videos), do: has_resolver_config?(:youtube_api_key) @@ -601,6 +602,7 @@ defmodule Ret.MediaSearch do defp created_rooms_search(cursor, account_id, _query) do page_number = (cursor || "1") |> Integer.parse() |> elem(0) + ecto_query = from h in Hub, where: h.created_by_account_id == ^account_id, diff --git a/lib/ret/scene.ex b/lib/ret/scene.ex index 5927034a3..18485c713 100644 --- a/lib/ret/scene.ex +++ b/lib/ret/scene.ex @@ -226,7 +226,7 @@ defmodule Ret.Scene do model_owned_file: old_model_owned_file, account: account } = scene - + new_scene_owned_file = Storage.create_new_owned_file_with_replaced_string( old_scene_owned_file, @@ -234,29 +234,30 @@ defmodule Ret.Scene do old_domain_url, new_domain_url ) - + {:ok, new_model_owned_file} = Storage.duplicate_and_transform(old_model_owned_file, account, fn glb_stream, _total_bytes -> GLTFUtils.replace_in_glb(glb_stream, old_domain_url, new_domain_url) end) - + scene |> change() |> put_change(:scene_owned_file_id, new_scene_owned_file.owned_file_id) |> put_change(:model_owned_file_id, new_model_owned_file.owned_file_id) |> Repo.update!() - + for old_owned_file <- [old_scene_owned_file, old_model_owned_file] do OwnedFile.set_inactive(old_owned_file) Storage.rm_files_for_owned_file(old_owned_file) Repo.delete(old_owned_file) end rescue - e -> + e -> IO.warn("Failed to process scene due to an error: #{inspect(e)}") end end) + :ok end) end diff --git a/lib/ret/storage.ex b/lib/ret/storage.ex index 05c064dcc..34dfe6e03 100644 --- a/lib/ret/storage.ex +++ b/lib/ret/storage.ex @@ -288,7 +288,7 @@ defmodule Ret.Storage do _account, _require_token ) do - {:ok, owned_file} + {:ok, owned_file} end # Promoting a stored file to being owned has two side effects: the file is moved @@ -310,7 +310,11 @@ defmodule Ret.Storage do "promotion_token" => actual_promotion_token } <- File.read!(meta_file_path) |> Poison.decode!(), - {:ok} <- (if require_token, do: check_promotion_token(actual_promotion_token, promotion_token), else: {:ok}), + {:ok} <- + if(require_token, + do: check_promotion_token(actual_promotion_token, promotion_token), + else: {:ok} + ), {:ok} <- check_blob_file_key(blob_file_path, key) ) do owned_file_params = %{ diff --git a/lib/ret/storage_used.ex b/lib/ret/storage_used.ex index bbe6e5f63..53931bf57 100644 --- a/lib/ret/storage_used.ex +++ b/lib/ret/storage_used.ex @@ -29,7 +29,7 @@ defmodule Ret.StorageUsed do line = lines |> String.split("\n") |> Enum.at(1) {:ok, [_FS, _kb, used, _Avail], _RestStr} = - :io_lib.fread('~s~d~d~d', line |> to_charlist) + :io_lib.fread(~c"~s~d~d~d", line |> to_charlist) {:ok, [{:storage_used, used}]} diff --git a/lib/ret_web/controllers/api/v1/media_search_controller.ex b/lib/ret_web/controllers/api/v1/media_search_controller.ex index 946eaad61..7aa49ae09 100644 --- a/lib/ret_web/controllers/api/v1/media_search_controller.ex +++ b/lib/ret_web/controllers/api/v1/media_search_controller.ex @@ -3,25 +3,25 @@ defmodule RetWeb.Api.V1.MediaSearchController do use Retry def index(conn, %{"source" => source, "filter" => "created", "user" => user} = params) - when source in ["rooms"] do - account = conn |> Guardian.Plug.current_resource() - - if account && account.account_id == String.to_integer(user) do - {:commit, results} = - %Ret.MediaSearchQuery{ - source: "rooms", - cursor: params["cursor"] || "1", - user: account.account_id, - filter: "created", - q: params["q"] - } - |> Ret.MediaSearch.search() + when source in ["rooms"] do + account = conn |> Guardian.Plug.current_resource() - conn |> render("index.json", results: results) - else - conn |> send_resp(401, "You can only search created rooms for your own account.") - end + if account && account.account_id == String.to_integer(user) do + {:commit, results} = + %Ret.MediaSearchQuery{ + source: "rooms", + cursor: params["cursor"] || "1", + user: account.account_id, + filter: "created", + q: params["q"] + } + |> Ret.MediaSearch.search() + + conn |> render("index.json", results: results) + else + conn |> send_resp(401, "You can only search created rooms for your own account.") end + end def index(conn, %{"source" => "rooms"} = params) do {:commit, results} = diff --git a/lib/ret_web/controllers/file_controller.ex b/lib/ret_web/controllers/file_controller.ex index 6b6d8ca20..a61efff1f 100644 --- a/lib/ret_web/controllers/file_controller.ex +++ b/lib/ret_web/controllers/file_controller.ex @@ -26,7 +26,9 @@ defmodule RetWeb.FileController do app_name = AppConfig.get_cached_config_value("translations|en|app-full-name") || - AppConfig.get_cached_config_value("translations|en|app-name") || RetWeb.Endpoint.host() + AppConfig.get_cached_config_value("translations|en|app-name") || + RetWeb.Endpoint.host() + title = "Photo taken in #{app_name} immersive space" config = AppConfig.get_config() @@ -36,9 +38,11 @@ defmodule RetWeb.FileController do content_type: content_type |> RetWeb.ContentType.sanitize_content_type(), content_length: content_length, title: title, - description_social_media: Ret.HttpUtils.join_smart([ - config["translations"]["en"]["app-tagline"], - "powered by Hubs"]), + description_social_media: + Ret.HttpUtils.join_smart([ + config["translations"]["en"]["app-tagline"], + "powered by Hubs" + ]), translations: config["translations"]["en"], app_name: app_name, images: config["images"], diff --git a/lib/ret_web/controllers/page_controller.ex b/lib/ret_web/controllers/page_controller.ex index 48a5a5b5f..941d275b3 100644 --- a/lib/ret_web/controllers/page_controller.ex +++ b/lib/ret_web/controllers/page_controller.ex @@ -65,19 +65,24 @@ defmodule RetWeb.PageController do defp render_scene_content(%t{} = scene, conn) when t in [Scene, SceneListing] do {app_config, app_config_script} = generate_app_config() - app_name = app_config["translations"]["en"]["app-full-name"] || app_config["translations"]["en"]["app-name"] || RetWeb.Endpoint.host() + app_name = + app_config["translations"]["en"]["app-full-name"] || + app_config["translations"]["en"]["app-name"] || RetWeb.Endpoint.host() + scene_meta_tags = Phoenix.View.render_to_string(RetWeb.PageView, "scene-meta.html", scene: scene, ret_meta: Ret.Meta.get_meta(include_repo: false), translations: app_config["translations"]["en"], - title: join_smart([ scene.name, app_name ]), + title: join_smart([scene.name, app_name]), name: scene.name, - description: join_smart([ - scene.description, - "A scene you can use in the #{app_name} immersive spaces and others powered by Hubs", - app_config["translations"]["en"]["app-description"] - ]) |> String.replace("\\n", " "), + description: + join_smart([ + scene.description, + "A scene you can use in the #{app_name} immersive spaces and others powered by Hubs", + app_config["translations"]["en"]["app-description"] + ]) + |> String.replace("\\n", " "), app_config_script: {:safe, app_config_script |> with_script_tags}, extra_script: {:safe, get_extra_script(:scene) |> with_script_tags}, extra_html: {:safe, get_extra_html(:scene) || ""} @@ -106,13 +111,17 @@ defmodule RetWeb.PageController do defp render_avatar_content(%t{} = avatar, conn) when t in [Avatar, AvatarListing] do {app_config, app_config_script} = generate_app_config() - app_name = app_config["translations"]["en"]["app-full-name"] || app_config["translations"]["en"]["app-name"] || RetWeb.Endpoint.host() + app_name = + app_config["translations"]["en"]["app-full-name"] || + app_config["translations"]["en"]["app-name"] || RetWeb.Endpoint.host() + avatar_meta_tags = Phoenix.View.render_to_string(RetWeb.PageView, "avatar-meta.html", avatar: avatar, title: join_smart([avatar.name, app_name]), name: avatar.name, - description: "An avatar you can use in the #{app_name} immersive spaces and others powered by Hubs.", + description: + "An avatar you can use in the #{app_name} immersive spaces and others powered by Hubs.", ret_meta: Ret.Meta.get_meta(include_repo: false), translations: app_config["translations"]["en"], root_url: RetWeb.Endpoint.url(), @@ -144,7 +153,9 @@ defmodule RetWeb.PageController do defp render_homepage_content(conn, nil = _public_room_id) do {app_config, app_config_script} = generate_app_config() - app_name = app_config["translations"]["en"]["app-full-name"] || app_config["translations"]["en"]["app-name"] || RetWeb.Endpoint.host() + app_name = + app_config["translations"]["en"]["app-full-name"] || + app_config["translations"]["en"]["app-name"] || RetWeb.Endpoint.host() index_meta_tags = Phoenix.View.render_to_string( @@ -154,10 +165,11 @@ defmodule RetWeb.PageController do translations: app_config["translations"]["en"], app_name: app_name, title: join_smart([app_name, app_config["translations"]["en"]["app-tagline"]]), - description: join_smart( - [app_config["translations"]["en"]["app-description"], - "Immersive spaces, right in your browser, powered by Hubs" - ]), + description: + join_smart([ + app_config["translations"]["en"]["app-description"], + "Immersive spaces, right in your browser, powered by Hubs" + ]), images: app_config["images"], app_config_script: {:safe, app_config_script |> with_script_tags}, extra_script: {:safe, get_extra_script(:index) |> with_script_tags}, @@ -291,11 +303,14 @@ defmodule RetWeb.PageController do manifest = Phoenix.View.render_to_string(RetWeb.PageView, "manifest.webmanifest", root_url: RetWeb.Endpoint.url(), - app_name: get_app_config_value("translations|en|app-name") || RetWeb.Endpoint.host(), - app_description: join_smart([ - get_app_config_value("translations|en|app-description"), - "Immersive spaces, right in your browser, powered by Hubs" - ]) |> String.replace("\\n", " ") + app_name: + get_app_config_value("translations|en|app-name") || RetWeb.Endpoint.host(), + app_description: + join_smart([ + get_app_config_value("translations|en|app-description"), + "Immersive spaces, right in your browser, powered by Hubs" + ]) + |> String.replace("\\n", " ") ) unless module_config(:skip_cache) do @@ -484,12 +499,22 @@ defmodule RetWeb.PageController do app_name = app_config["translations"]["en"]["app-name"] || RetWeb.Endpoint.host() scene = hub.scene || hub.scene_listing - name = cond do - hub.name && scene && scene.name && scene.name != hub.name -> join_smart([hub.name, scene.name]) - hub.name -> hub.name - scene && scene.name -> scene.name - true -> "a room on " <> app_name - end + + name = + cond do + hub.name && scene && scene.name && scene.name != hub.name -> + join_smart([hub.name, scene.name]) + + hub.name -> + hub.name + + scene && scene.name -> + scene.name + + true -> + "a room on " <> app_name + end + hub_meta_tags = Phoenix.View.render_to_string(RetWeb.PageView, "hub-meta.html", hub: hub, @@ -499,18 +524,22 @@ defmodule RetWeb.PageController do translations: app_config["translations"]["en"], title: join_smart([hub.name, app_name]), name: name, - description: join_smart([ - hub.description, - "an immersive space in #{app_name}, right in your browser", - app_config["translations"]["en"]["app-description"], - "powered by Hubs." - ]) |> String.replace("\\n", " "), - description_social_media: join_smart([ - "Join others in an immersive space in #{app_name}, right in your browser", - hub.description, - app_config["translations"]["en"]["app-description"], - "powered by Hubs." - ]) |> String.replace("\\n", " "), + description: + join_smart([ + hub.description, + "an immersive space in #{app_name}, right in your browser", + app_config["translations"]["en"]["app-description"], + "powered by Hubs." + ]) + |> String.replace("\\n", " "), + description_social_media: + join_smart([ + "Join others in an immersive space in #{app_name}, right in your browser", + hub.description, + app_config["translations"]["en"]["app-description"], + "powered by Hubs." + ]) + |> String.replace("\\n", " "), app_config_script: {:safe, app_config_script |> with_script_tags}, extra_script: {:safe, get_extra_script(:room) |> with_script_tags}, extra_html: {:safe, get_extra_html(:room) || ""} @@ -726,7 +755,6 @@ defmodule RetWeb.PageController do end defp cors_proxy_with_redirects(conn, url, redirect_count) do - %URI{authority: authority, host: host} = uri = URI.parse(url) resolved_ip = HttpUtils.resolve_ip(host) @@ -761,7 +789,10 @@ defmodule RetWeb.PageController do proxy_url: "#{cors_scheme}://#{cors_host}:#{cors_port}", # Since we need to use the host for SSL verification, we provide the authority client_options: [ - ssl: [{:server_name_indication, to_charlist(authority)}, {:versions, [:"tlsv1.2",:"tlsv1.3"]}] + ssl: [ + {:server_name_indication, to_charlist(authority)}, + {:versions, [:"tlsv1.2", :"tlsv1.3"]} + ] ] ) @@ -770,15 +801,22 @@ defmodule RetWeb.PageController do try do # First, make a HEAD request to check for redirects using HTTPoison - case HTTPoison.head(url, [], [ - follow_redirect: false, - ssl: [{:server_name_indication, to_charlist(authority)}, {:versions, [:"tlsv1.2", :"tlsv1.3"]}], - timeout: 15_000, - recv_timeout: 15_000 - ]) do - {:ok, %HTTPoison.Response{status_code: status_code, headers: headers}} when status_code in [301, 302, 303, 307, 308] -> + case HTTPoison.head(url, [], + follow_redirect: false, + ssl: [ + {:server_name_indication, to_charlist(authority)}, + {:versions, [:"tlsv1.2", :"tlsv1.3"]} + ], + timeout: 15_000, + recv_timeout: 15_000 + ) do + {:ok, %HTTPoison.Response{status_code: status_code, headers: headers}} + when status_code in [301, 302, 303, 307, 308] -> # Found a redirect - location_header = headers |> Enum.find(fn {k, _v} -> String.downcase(k) == "location" end) |> elem(1) + location_header = + headers + |> Enum.find(fn {k, _v} -> String.downcase(k) == "location" end) + |> elem(1) if location_header do # Resolve relative URLs against the current URL @@ -807,6 +845,7 @@ defmodule RetWeb.PageController do :exit, reason -> Logger.error("CORS Proxy: Request exited with reason: #{inspect(reason)}") conn |> send_resp(500, "Proxy request timed out or failed") + kind, reason -> Logger.error("CORS Proxy: Request failed with #{kind}: #{inspect(reason)}") conn |> send_resp(500, "Proxy request failed") @@ -819,20 +858,21 @@ defmodule RetWeb.PageController do end defp make_reverse_proxy_request(conn, _url, body, is_head, opts) do - proxy_conn = %Conn{} - |> Map.merge(conn) - |> Map.put( - :method, - if is_head do - "HEAD" - else - conn.method - end - ) - # Need to strip path_info since proxy plug reads it - |> Map.put(:path_info, []) - |> ReverseProxyPlug.request(body, opts) - |> ReverseProxyPlug.response(conn, opts) + proxy_conn = + %Conn{} + |> Map.merge(conn) + |> Map.put( + :method, + if is_head do + "HEAD" + else + conn.method + end + ) + # Need to strip path_info since proxy plug reads it + |> Map.put(:path_info, []) + |> ReverseProxyPlug.request(body, opts) + |> ReverseProxyPlug.response(conn, opts) proxy_conn end diff --git a/lib/ret_web/email.ex b/lib/ret_web/email.ex index 0c2f6072b..7ba8b12d5 100644 --- a/lib/ret_web/email.ex +++ b/lib/ret_web/email.ex @@ -3,7 +3,9 @@ defmodule RetWeb.Email do alias Ret.{AppConfig} def auth_email(to_address, signin_args) do - app_name = AppConfig.get_cached_config_value("translations|en|app-name") || RetWeb.Endpoint.host() + app_name = + AppConfig.get_cached_config_value("translations|en|app-name") || RetWeb.Endpoint.host() + app_full_name = AppConfig.get_cached_config_value("translations|en|app-full-name") || app_name admin_email = Application.get_env(:ret, Ret.Account)[:admin_email] custom_login_subject = AppConfig.get_cached_config_value("auth|login_subject") diff --git a/priv/repo/migrations/20190114232257_add_reviewed_at_to_scenes.exs b/priv/repo/migrations/20190114232257_add_reviewed_at_to_scenes.exs index 8e5a26da1..ea1707548 100644 --- a/priv/repo/migrations/20190114232257_add_reviewed_at_to_scenes.exs +++ b/priv/repo/migrations/20190114232257_add_reviewed_at_to_scenes.exs @@ -6,6 +6,8 @@ defmodule Ret.Repo.Migrations.AddReviewedAtToScenes do add :reviewed_at, :utc_datetime, null: true end - create index(:scenes, [:reviewed_at], where: "reviewed_at is null or reviewed_at < updated_at") + create index(:scenes, [:reviewed_at], + where: "reviewed_at is null or reviewed_at < updated_at" + ) end end diff --git a/test/ret_web/channels/entity_test.exs b/test/ret_web/channels/entity_test.exs index c18d1e28a..0cbc6b556 100644 --- a/test/ret_web/channels/entity_test.exs +++ b/test/ret_web/channels/entity_test.exs @@ -8,7 +8,9 @@ defmodule RetWeb.EntityTest do @payload_save_entity_state read_json("save_entity_state_payload.json") @payload_save_entity_state_2 read_json("save_entity_state_payload_2.json") - @payload_save_entity_state_promotable_no_token read_json("save_entity_state_payload_promotable_no_token.json") + @payload_save_entity_state_promotable_no_token read_json( + "save_entity_state_payload_promotable_no_token.json" + ) @payload_save_entity_state_promotable read_json("save_entity_state_payload_promotable.json") @payload_save_entity_state_unpromotable read_json("save_entity_state_payload_unpromotable.json") @payload_update_entity_state read_json("update_entity_state_payload.json") From fc784c43c0308bcae57c996815f11e49e74d366f Mon Sep 17 00:00:00 2001 From: Andy Baker Date: Thu, 11 Sep 2025 10:36:18 +0100 Subject: [PATCH 24/26] Revert "mix format" This reverts commit 76b5e1cc3f0881ecdfc938ca3a63ed790739fd34. --- lib/ret/discord_client.ex | 81 ++++----- lib/ret/http_utils.ex | 15 +- lib/ret/media_resolver.ex | 70 ++++---- lib/ret/media_search.ex | 4 +- lib/ret/scene.ex | 11 +- lib/ret/storage.ex | 8 +- lib/ret/storage_used.ex | 2 +- .../api/v1/media_search_controller.ex | 34 ++-- lib/ret_web/controllers/file_controller.ex | 12 +- lib/ret_web/controllers/page_controller.ex | 162 +++++++----------- lib/ret_web/email.ex | 4 +- ...190114232257_add_reviewed_at_to_scenes.exs | 4 +- test/ret_web/channels/entity_test.exs | 4 +- 13 files changed, 164 insertions(+), 247 deletions(-) diff --git a/lib/ret/discord_client.ex b/lib/ret/discord_client.ex index 8f220098c..1e0fea6c3 100644 --- a/lib/ret/discord_client.ex +++ b/lib/ret/discord_client.ex @@ -76,23 +76,21 @@ defmodule Ret.DiscordClient do {status, result} when status in [:commit, :ok] -> "#{result["nick"]}" end - nickname = - if !nickname or nickname == "" do - case Cachex.fetch(:discord_api, "/users/#{provider_account_id}") do - {status, result} when status in [:commit, :ok] -> "#{result["global_name"]}" - end + nickname = if !nickname or nickname == "" do + case Cachex.fetch(:discord_api, "/users/#{provider_account_id}") do + {status, result} when status in [:commit, :ok] -> "#{result["global_name"]}" + end else nickname - end + end - nickname = - if !nickname or nickname == "" do - case Cachex.fetch(:discord_api, "/users/#{provider_account_id}") do - {status, result} when status in [:commit, :ok] -> "#{result["username"]}" - end + nickname = if !nickname or nickname == "" do + case Cachex.fetch(:discord_api, "/users/#{provider_account_id}") do + {status, result} when status in [:commit, :ok] -> "#{result["username"]}" + end else nickname - end + end end def fetch_community_identifier(%Ret.OAuthProvider{ @@ -171,21 +169,18 @@ defmodule Ret.DiscordClient do case Cachex.fetch(:discord_api, "/guilds/#{community_id}/roles") do {status, result} when status in [:commit, :ok] -> result |> Map.new(&{&1["id"], &1}) end - - # Note: Whether the bitfield values in guild_roles are represented as strings or integers is inconsistent (possibly based on what permissions the user has), so every time they're used they need to be checked and, if needed, converted to integers. + # Note: Whether the bitfield values in guild_roles are represented as strings or integers is inconsistent (possibly based on what permissions the user has), so every time they're used they need to be checked and, if needed, converted to integers. role_everyone = guild_roles[community_id] permissions = role_everyone["permissions"] user_permissions = user_roles |> Enum.map(&guild_roles[&1]["permissions"]) - permissions = - user_permissions - |> Enum.reduce( - permissions, - &(if(is_binary(&1), do: String.to_integer(&1), else: &1) ||| - if(is_binary(&2), do: String.to_integer(&2), else: &2)) - ) + permissions = user_permissions |> + Enum.reduce(permissions, &( + (if is_binary(&1), do: String.to_integer(&1), else: &1) ||| + (if is_binary(&2), do: String.to_integer(&2), else: &2) + )) if (permissions &&& @administrator) == @administrator do @all @@ -208,21 +203,15 @@ defmodule Ret.DiscordClient do |> Map.get("permission_overwrites") |> Map.new(&{&1["id"], &1}) end - - # Note: Whether the bitfield values in channel_overwrites are represented as strings or integers is inconsistent (possibly based on what permissions the user has), so every time they're used they need to be checked and, if needed, converted to integers. + # Note: Whether the bitfield values in channel_overwrites are represented as strings or integers is inconsistent (possibly based on what permissions the user has), so every time they're used they need to be checked and, if needed, converted to integers. overwrite_everyone = channel_overwrites[community_id] permissions = if overwrite_everyone do (permissions &&& - ~~~if(is_binary(overwrite_everyone["deny"]), - do: String.to_integer(overwrite_everyone["deny"]), - else: overwrite_everyone["deny"] - )) ||| - if is_binary(overwrite_everyone["allow"]), - do: String.to_integer(overwrite_everyone["allow"]), - else: overwrite_everyone["allow"] + ~~~(if is_binary(overwrite_everyone["deny"]), do: String.to_integer(overwrite_everyone["deny"]), else: overwrite_everyone["deny"])) ||| + (if is_binary(overwrite_everyone["allow"]), do: String.to_integer(overwrite_everyone["allow"]), else: overwrite_everyone["allow"]) else permissions end @@ -231,21 +220,14 @@ defmodule Ret.DiscordClient do user_permissions = user_roles |> Enum.map(&channel_overwrites[&1]) |> Enum.filter(&(&1 != nil)) - allow = - user_permissions - |> Enum.reduce( - @none, - &(if(is_binary(&1["allow"]), do: String.to_integer(&1["allow"]), else: &1["allow"]) ||| - &2) - ) - - deny = - user_permissions - |> Enum.reduce( - @none, - &(if(is_binary(&1["deny"]), do: String.to_integer(&1["deny"]), else: &1["deny"]) ||| - &2) - ) + allow = user_permissions |> Enum.reduce(@none, &( + (if is_binary(&1["allow"]), do: String.to_integer(&1["allow"]), else: &1["allow"]) ||| + &2 + )) + deny = user_permissions |> Enum.reduce(@none, &( + (if is_binary(&1["deny"]), do: String.to_integer(&1["deny"]), else: &1["deny"]) ||| + &2 + )) permissions = (permissions &&& ~~~deny) ||| allow @@ -255,13 +237,8 @@ defmodule Ret.DiscordClient do permissions = if overwrite_member do (permissions &&& - ~~~if(is_binary(overwrite_member["deny"]), - do: String.to_integer(overwrite_member["deny"]), - else: overwrite_member["deny"] - )) ||| - if is_binary(overwrite_member["allow"]), - do: String.to_integer(overwrite_member["allow"]), - else: overwrite_member["allow"] + ~~~(if is_binary(overwrite_member["deny"]), do: String.to_integer(overwrite_member["deny"]), else: overwrite_member["deny"])) ||| + (if is_binary(overwrite_member["allow"]), do: String.to_integer(overwrite_member["allow"]), else: overwrite_member["allow"]) else permissions end diff --git a/lib/ret/http_utils.ex b/lib/ret/http_utils.ex index 8bacdebfe..71366aba8 100644 --- a/lib/ret/http_utils.ex +++ b/lib/ret/http_utils.ex @@ -177,14 +177,12 @@ defmodule Ret.HttpUtils do end def join_smart(enum) do - Enum.reduce(enum, "", fn x, acc -> - x = - cond do - !x -> nil - is_binary(x) -> String.trim(x) - true -> "#{x}" - end - + Enum.reduce(enum, "", fn(x, acc) -> + x = cond do + !x -> nil + is_binary(x) -> String.trim(x) + true -> "#{x}" + end if x && x != "" do if acc && acc != "", do: acc <> " — " <> x, else: x else @@ -192,4 +190,5 @@ defmodule Ret.HttpUtils do end end) end + end diff --git a/lib/ret/media_resolver.ex b/lib/ret/media_resolver.ex index 43d162e14..015cb554f 100644 --- a/lib/ret/media_resolver.ex +++ b/lib/ret/media_resolver.ex @@ -337,45 +337,43 @@ defmodule Ret.MediaResolver do end defp resolve_non_video( - %MediaResolverQuery{ - url: %URI{host: "api.icosa.gallery", path: "/v1/assets/" <> asset_id} = uri - }, + %MediaResolverQuery{url: %URI{host: "api.icosa.gallery", path: "/v1/assets/" <> asset_id} = uri}, "icosa.gallery" - ) do - # Increment stat for the request - Statix.increment("ret.media_resolver.icosa.requests") - - # Make the API call to get the asset data - payload = - "https://api.icosa.gallery/v1/assets/#{asset_id}" - # Assuming this function sends the request and handles retries - |> retry_get_until_success() - |> Map.get(:body) - |> Poison.decode!() - - # Create the meta information based on the payload - meta = - %{ - expected_content_type: "model/gltf", - name: payload["displayName"], - author: payload["authorName"], - license: payload["license"] - } - - # Extract the GLTF2 format URL from the payload - uri = - payload["formats"] - |> Enum.find(&(&1["formatType"] == "GLTF2")) - |> Kernel.get_in(["root", "url"]) - |> URI.parse() - - # Increment stat for successful resolution - Statix.increment("ret.media_resolver.icosa.ok") - - # Return the URI and meta data for further processing - {:commit, resolved(uri, meta)} + ) do + # Increment stat for the request + Statix.increment("ret.media_resolver.icosa.requests") + + # Make the API call to get the asset data + payload = + "https://api.icosa.gallery/v1/assets/#{asset_id}" + |> retry_get_until_success() # Assuming this function sends the request and handles retries + |> Map.get(:body) + |> Poison.decode!() + + # Create the meta information based on the payload + meta = + %{ + expected_content_type: "model/gltf", + name: payload["displayName"], + author: payload["authorName"], + license: payload["license"] + } + + # Extract the GLTF2 format URL from the payload + uri = + payload["formats"] + |> Enum.find(&(&1["formatType"] == "GLTF2")) + |> Kernel.get_in(["root", "url"]) + |> URI.parse() + + # Increment stat for successful resolution + Statix.increment("ret.media_resolver.icosa.ok") + + # Return the URI and meta data for further processing + {:commit, resolved(uri, meta)} end + defp resolve_non_video( %MediaResolverQuery{url: %URI{path: "/models/" <> model_id}} = query, "sketchfab.com" = root_host diff --git a/lib/ret/media_search.ex b/lib/ret/media_search.ex index 55de070fd..6ca567018 100644 --- a/lib/ret/media_search.ex +++ b/lib/ret/media_search.ex @@ -418,8 +418,7 @@ defmodule Ret.MediaSearch do end end - # Icosa does not currently require an API key - def available?(:icosa), do: true + def available?(:icosa), do: true # Icosa does not currently require an API key def available?(:bing_images), do: has_resolver_config?(:bing_search_api_key) def available?(:bing_videos), do: has_resolver_config?(:bing_search_api_key) def available?(:youtube_videos), do: has_resolver_config?(:youtube_api_key) @@ -602,7 +601,6 @@ defmodule Ret.MediaSearch do defp created_rooms_search(cursor, account_id, _query) do page_number = (cursor || "1") |> Integer.parse() |> elem(0) - ecto_query = from h in Hub, where: h.created_by_account_id == ^account_id, diff --git a/lib/ret/scene.ex b/lib/ret/scene.ex index 18485c713..5927034a3 100644 --- a/lib/ret/scene.ex +++ b/lib/ret/scene.ex @@ -226,7 +226,7 @@ defmodule Ret.Scene do model_owned_file: old_model_owned_file, account: account } = scene - + new_scene_owned_file = Storage.create_new_owned_file_with_replaced_string( old_scene_owned_file, @@ -234,30 +234,29 @@ defmodule Ret.Scene do old_domain_url, new_domain_url ) - + {:ok, new_model_owned_file} = Storage.duplicate_and_transform(old_model_owned_file, account, fn glb_stream, _total_bytes -> GLTFUtils.replace_in_glb(glb_stream, old_domain_url, new_domain_url) end) - + scene |> change() |> put_change(:scene_owned_file_id, new_scene_owned_file.owned_file_id) |> put_change(:model_owned_file_id, new_model_owned_file.owned_file_id) |> Repo.update!() - + for old_owned_file <- [old_scene_owned_file, old_model_owned_file] do OwnedFile.set_inactive(old_owned_file) Storage.rm_files_for_owned_file(old_owned_file) Repo.delete(old_owned_file) end rescue - e -> + e -> IO.warn("Failed to process scene due to an error: #{inspect(e)}") end end) - :ok end) end diff --git a/lib/ret/storage.ex b/lib/ret/storage.ex index 34dfe6e03..05c064dcc 100644 --- a/lib/ret/storage.ex +++ b/lib/ret/storage.ex @@ -288,7 +288,7 @@ defmodule Ret.Storage do _account, _require_token ) do - {:ok, owned_file} + {:ok, owned_file} end # Promoting a stored file to being owned has two side effects: the file is moved @@ -310,11 +310,7 @@ defmodule Ret.Storage do "promotion_token" => actual_promotion_token } <- File.read!(meta_file_path) |> Poison.decode!(), - {:ok} <- - if(require_token, - do: check_promotion_token(actual_promotion_token, promotion_token), - else: {:ok} - ), + {:ok} <- (if require_token, do: check_promotion_token(actual_promotion_token, promotion_token), else: {:ok}), {:ok} <- check_blob_file_key(blob_file_path, key) ) do owned_file_params = %{ diff --git a/lib/ret/storage_used.ex b/lib/ret/storage_used.ex index 53931bf57..bbe6e5f63 100644 --- a/lib/ret/storage_used.ex +++ b/lib/ret/storage_used.ex @@ -29,7 +29,7 @@ defmodule Ret.StorageUsed do line = lines |> String.split("\n") |> Enum.at(1) {:ok, [_FS, _kb, used, _Avail], _RestStr} = - :io_lib.fread(~c"~s~d~d~d", line |> to_charlist) + :io_lib.fread('~s~d~d~d', line |> to_charlist) {:ok, [{:storage_used, used}]} diff --git a/lib/ret_web/controllers/api/v1/media_search_controller.ex b/lib/ret_web/controllers/api/v1/media_search_controller.ex index 7aa49ae09..946eaad61 100644 --- a/lib/ret_web/controllers/api/v1/media_search_controller.ex +++ b/lib/ret_web/controllers/api/v1/media_search_controller.ex @@ -3,25 +3,25 @@ defmodule RetWeb.Api.V1.MediaSearchController do use Retry def index(conn, %{"source" => source, "filter" => "created", "user" => user} = params) - when source in ["rooms"] do - account = conn |> Guardian.Plug.current_resource() - - if account && account.account_id == String.to_integer(user) do - {:commit, results} = - %Ret.MediaSearchQuery{ - source: "rooms", - cursor: params["cursor"] || "1", - user: account.account_id, - filter: "created", - q: params["q"] - } - |> Ret.MediaSearch.search() + when source in ["rooms"] do + account = conn |> Guardian.Plug.current_resource() + + if account && account.account_id == String.to_integer(user) do + {:commit, results} = + %Ret.MediaSearchQuery{ + source: "rooms", + cursor: params["cursor"] || "1", + user: account.account_id, + filter: "created", + q: params["q"] + } + |> Ret.MediaSearch.search() - conn |> render("index.json", results: results) - else - conn |> send_resp(401, "You can only search created rooms for your own account.") + conn |> render("index.json", results: results) + else + conn |> send_resp(401, "You can only search created rooms for your own account.") + end end - end def index(conn, %{"source" => "rooms"} = params) do {:commit, results} = diff --git a/lib/ret_web/controllers/file_controller.ex b/lib/ret_web/controllers/file_controller.ex index a61efff1f..6b6d8ca20 100644 --- a/lib/ret_web/controllers/file_controller.ex +++ b/lib/ret_web/controllers/file_controller.ex @@ -26,9 +26,7 @@ defmodule RetWeb.FileController do app_name = AppConfig.get_cached_config_value("translations|en|app-full-name") || - AppConfig.get_cached_config_value("translations|en|app-name") || - RetWeb.Endpoint.host() - + AppConfig.get_cached_config_value("translations|en|app-name") || RetWeb.Endpoint.host() title = "Photo taken in #{app_name} immersive space" config = AppConfig.get_config() @@ -38,11 +36,9 @@ defmodule RetWeb.FileController do content_type: content_type |> RetWeb.ContentType.sanitize_content_type(), content_length: content_length, title: title, - description_social_media: - Ret.HttpUtils.join_smart([ - config["translations"]["en"]["app-tagline"], - "powered by Hubs" - ]), + description_social_media: Ret.HttpUtils.join_smart([ + config["translations"]["en"]["app-tagline"], + "powered by Hubs"]), translations: config["translations"]["en"], app_name: app_name, images: config["images"], diff --git a/lib/ret_web/controllers/page_controller.ex b/lib/ret_web/controllers/page_controller.ex index 941d275b3..48a5a5b5f 100644 --- a/lib/ret_web/controllers/page_controller.ex +++ b/lib/ret_web/controllers/page_controller.ex @@ -65,24 +65,19 @@ defmodule RetWeb.PageController do defp render_scene_content(%t{} = scene, conn) when t in [Scene, SceneListing] do {app_config, app_config_script} = generate_app_config() - app_name = - app_config["translations"]["en"]["app-full-name"] || - app_config["translations"]["en"]["app-name"] || RetWeb.Endpoint.host() - + app_name = app_config["translations"]["en"]["app-full-name"] || app_config["translations"]["en"]["app-name"] || RetWeb.Endpoint.host() scene_meta_tags = Phoenix.View.render_to_string(RetWeb.PageView, "scene-meta.html", scene: scene, ret_meta: Ret.Meta.get_meta(include_repo: false), translations: app_config["translations"]["en"], - title: join_smart([scene.name, app_name]), + title: join_smart([ scene.name, app_name ]), name: scene.name, - description: - join_smart([ - scene.description, - "A scene you can use in the #{app_name} immersive spaces and others powered by Hubs", - app_config["translations"]["en"]["app-description"] - ]) - |> String.replace("\\n", " "), + description: join_smart([ + scene.description, + "A scene you can use in the #{app_name} immersive spaces and others powered by Hubs", + app_config["translations"]["en"]["app-description"] + ]) |> String.replace("\\n", " "), app_config_script: {:safe, app_config_script |> with_script_tags}, extra_script: {:safe, get_extra_script(:scene) |> with_script_tags}, extra_html: {:safe, get_extra_html(:scene) || ""} @@ -111,17 +106,13 @@ defmodule RetWeb.PageController do defp render_avatar_content(%t{} = avatar, conn) when t in [Avatar, AvatarListing] do {app_config, app_config_script} = generate_app_config() - app_name = - app_config["translations"]["en"]["app-full-name"] || - app_config["translations"]["en"]["app-name"] || RetWeb.Endpoint.host() - + app_name = app_config["translations"]["en"]["app-full-name"] || app_config["translations"]["en"]["app-name"] || RetWeb.Endpoint.host() avatar_meta_tags = Phoenix.View.render_to_string(RetWeb.PageView, "avatar-meta.html", avatar: avatar, title: join_smart([avatar.name, app_name]), name: avatar.name, - description: - "An avatar you can use in the #{app_name} immersive spaces and others powered by Hubs.", + description: "An avatar you can use in the #{app_name} immersive spaces and others powered by Hubs.", ret_meta: Ret.Meta.get_meta(include_repo: false), translations: app_config["translations"]["en"], root_url: RetWeb.Endpoint.url(), @@ -153,9 +144,7 @@ defmodule RetWeb.PageController do defp render_homepage_content(conn, nil = _public_room_id) do {app_config, app_config_script} = generate_app_config() - app_name = - app_config["translations"]["en"]["app-full-name"] || - app_config["translations"]["en"]["app-name"] || RetWeb.Endpoint.host() + app_name = app_config["translations"]["en"]["app-full-name"] || app_config["translations"]["en"]["app-name"] || RetWeb.Endpoint.host() index_meta_tags = Phoenix.View.render_to_string( @@ -165,11 +154,10 @@ defmodule RetWeb.PageController do translations: app_config["translations"]["en"], app_name: app_name, title: join_smart([app_name, app_config["translations"]["en"]["app-tagline"]]), - description: - join_smart([ - app_config["translations"]["en"]["app-description"], - "Immersive spaces, right in your browser, powered by Hubs" - ]), + description: join_smart( + [app_config["translations"]["en"]["app-description"], + "Immersive spaces, right in your browser, powered by Hubs" + ]), images: app_config["images"], app_config_script: {:safe, app_config_script |> with_script_tags}, extra_script: {:safe, get_extra_script(:index) |> with_script_tags}, @@ -303,14 +291,11 @@ defmodule RetWeb.PageController do manifest = Phoenix.View.render_to_string(RetWeb.PageView, "manifest.webmanifest", root_url: RetWeb.Endpoint.url(), - app_name: - get_app_config_value("translations|en|app-name") || RetWeb.Endpoint.host(), - app_description: - join_smart([ - get_app_config_value("translations|en|app-description"), - "Immersive spaces, right in your browser, powered by Hubs" - ]) - |> String.replace("\\n", " ") + app_name: get_app_config_value("translations|en|app-name") || RetWeb.Endpoint.host(), + app_description: join_smart([ + get_app_config_value("translations|en|app-description"), + "Immersive spaces, right in your browser, powered by Hubs" + ]) |> String.replace("\\n", " ") ) unless module_config(:skip_cache) do @@ -499,22 +484,12 @@ defmodule RetWeb.PageController do app_name = app_config["translations"]["en"]["app-name"] || RetWeb.Endpoint.host() scene = hub.scene || hub.scene_listing - - name = - cond do - hub.name && scene && scene.name && scene.name != hub.name -> - join_smart([hub.name, scene.name]) - - hub.name -> - hub.name - - scene && scene.name -> - scene.name - - true -> - "a room on " <> app_name - end - + name = cond do + hub.name && scene && scene.name && scene.name != hub.name -> join_smart([hub.name, scene.name]) + hub.name -> hub.name + scene && scene.name -> scene.name + true -> "a room on " <> app_name + end hub_meta_tags = Phoenix.View.render_to_string(RetWeb.PageView, "hub-meta.html", hub: hub, @@ -524,22 +499,18 @@ defmodule RetWeb.PageController do translations: app_config["translations"]["en"], title: join_smart([hub.name, app_name]), name: name, - description: - join_smart([ - hub.description, - "an immersive space in #{app_name}, right in your browser", - app_config["translations"]["en"]["app-description"], - "powered by Hubs." - ]) - |> String.replace("\\n", " "), - description_social_media: - join_smart([ - "Join others in an immersive space in #{app_name}, right in your browser", - hub.description, - app_config["translations"]["en"]["app-description"], - "powered by Hubs." - ]) - |> String.replace("\\n", " "), + description: join_smart([ + hub.description, + "an immersive space in #{app_name}, right in your browser", + app_config["translations"]["en"]["app-description"], + "powered by Hubs." + ]) |> String.replace("\\n", " "), + description_social_media: join_smart([ + "Join others in an immersive space in #{app_name}, right in your browser", + hub.description, + app_config["translations"]["en"]["app-description"], + "powered by Hubs." + ]) |> String.replace("\\n", " "), app_config_script: {:safe, app_config_script |> with_script_tags}, extra_script: {:safe, get_extra_script(:room) |> with_script_tags}, extra_html: {:safe, get_extra_html(:room) || ""} @@ -755,6 +726,7 @@ defmodule RetWeb.PageController do end defp cors_proxy_with_redirects(conn, url, redirect_count) do + %URI{authority: authority, host: host} = uri = URI.parse(url) resolved_ip = HttpUtils.resolve_ip(host) @@ -789,10 +761,7 @@ defmodule RetWeb.PageController do proxy_url: "#{cors_scheme}://#{cors_host}:#{cors_port}", # Since we need to use the host for SSL verification, we provide the authority client_options: [ - ssl: [ - {:server_name_indication, to_charlist(authority)}, - {:versions, [:"tlsv1.2", :"tlsv1.3"]} - ] + ssl: [{:server_name_indication, to_charlist(authority)}, {:versions, [:"tlsv1.2",:"tlsv1.3"]}] ] ) @@ -801,22 +770,15 @@ defmodule RetWeb.PageController do try do # First, make a HEAD request to check for redirects using HTTPoison - case HTTPoison.head(url, [], - follow_redirect: false, - ssl: [ - {:server_name_indication, to_charlist(authority)}, - {:versions, [:"tlsv1.2", :"tlsv1.3"]} - ], - timeout: 15_000, - recv_timeout: 15_000 - ) do - {:ok, %HTTPoison.Response{status_code: status_code, headers: headers}} - when status_code in [301, 302, 303, 307, 308] -> + case HTTPoison.head(url, [], [ + follow_redirect: false, + ssl: [{:server_name_indication, to_charlist(authority)}, {:versions, [:"tlsv1.2", :"tlsv1.3"]}], + timeout: 15_000, + recv_timeout: 15_000 + ]) do + {:ok, %HTTPoison.Response{status_code: status_code, headers: headers}} when status_code in [301, 302, 303, 307, 308] -> # Found a redirect - location_header = - headers - |> Enum.find(fn {k, _v} -> String.downcase(k) == "location" end) - |> elem(1) + location_header = headers |> Enum.find(fn {k, _v} -> String.downcase(k) == "location" end) |> elem(1) if location_header do # Resolve relative URLs against the current URL @@ -845,7 +807,6 @@ defmodule RetWeb.PageController do :exit, reason -> Logger.error("CORS Proxy: Request exited with reason: #{inspect(reason)}") conn |> send_resp(500, "Proxy request timed out or failed") - kind, reason -> Logger.error("CORS Proxy: Request failed with #{kind}: #{inspect(reason)}") conn |> send_resp(500, "Proxy request failed") @@ -858,21 +819,20 @@ defmodule RetWeb.PageController do end defp make_reverse_proxy_request(conn, _url, body, is_head, opts) do - proxy_conn = - %Conn{} - |> Map.merge(conn) - |> Map.put( - :method, - if is_head do - "HEAD" - else - conn.method - end - ) - # Need to strip path_info since proxy plug reads it - |> Map.put(:path_info, []) - |> ReverseProxyPlug.request(body, opts) - |> ReverseProxyPlug.response(conn, opts) + proxy_conn = %Conn{} + |> Map.merge(conn) + |> Map.put( + :method, + if is_head do + "HEAD" + else + conn.method + end + ) + # Need to strip path_info since proxy plug reads it + |> Map.put(:path_info, []) + |> ReverseProxyPlug.request(body, opts) + |> ReverseProxyPlug.response(conn, opts) proxy_conn end diff --git a/lib/ret_web/email.ex b/lib/ret_web/email.ex index 7ba8b12d5..0c2f6072b 100644 --- a/lib/ret_web/email.ex +++ b/lib/ret_web/email.ex @@ -3,9 +3,7 @@ defmodule RetWeb.Email do alias Ret.{AppConfig} def auth_email(to_address, signin_args) do - app_name = - AppConfig.get_cached_config_value("translations|en|app-name") || RetWeb.Endpoint.host() - + app_name = AppConfig.get_cached_config_value("translations|en|app-name") || RetWeb.Endpoint.host() app_full_name = AppConfig.get_cached_config_value("translations|en|app-full-name") || app_name admin_email = Application.get_env(:ret, Ret.Account)[:admin_email] custom_login_subject = AppConfig.get_cached_config_value("auth|login_subject") diff --git a/priv/repo/migrations/20190114232257_add_reviewed_at_to_scenes.exs b/priv/repo/migrations/20190114232257_add_reviewed_at_to_scenes.exs index ea1707548..8e5a26da1 100644 --- a/priv/repo/migrations/20190114232257_add_reviewed_at_to_scenes.exs +++ b/priv/repo/migrations/20190114232257_add_reviewed_at_to_scenes.exs @@ -6,8 +6,6 @@ defmodule Ret.Repo.Migrations.AddReviewedAtToScenes do add :reviewed_at, :utc_datetime, null: true end - create index(:scenes, [:reviewed_at], - where: "reviewed_at is null or reviewed_at < updated_at" - ) + create index(:scenes, [:reviewed_at], where: "reviewed_at is null or reviewed_at < updated_at") end end diff --git a/test/ret_web/channels/entity_test.exs b/test/ret_web/channels/entity_test.exs index 0cbc6b556..c18d1e28a 100644 --- a/test/ret_web/channels/entity_test.exs +++ b/test/ret_web/channels/entity_test.exs @@ -8,9 +8,7 @@ defmodule RetWeb.EntityTest do @payload_save_entity_state read_json("save_entity_state_payload.json") @payload_save_entity_state_2 read_json("save_entity_state_payload_2.json") - @payload_save_entity_state_promotable_no_token read_json( - "save_entity_state_payload_promotable_no_token.json" - ) + @payload_save_entity_state_promotable_no_token read_json("save_entity_state_payload_promotable_no_token.json") @payload_save_entity_state_promotable read_json("save_entity_state_payload_promotable.json") @payload_save_entity_state_unpromotable read_json("save_entity_state_payload_unpromotable.json") @payload_update_entity_state read_json("update_entity_state_payload.json") From 8acf9cfebdac46d5f176af64dd8c37e2912eccae Mon Sep 17 00:00:00 2001 From: Andy Baker Date: Wed, 12 Nov 2025 14:46:29 +0000 Subject: [PATCH 25/26] Reverts Icosa to use usual caching period, and reverts a comment By @DougReeder Why: Icosa doesn't have the rate limits that YouTube has. The original comment contained useful context. --- lib/ret_web/controllers/api/v1/media_search_controller.ex | 2 +- lib/ret_web/controllers/page_controller.ex | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/ret_web/controllers/api/v1/media_search_controller.ex b/lib/ret_web/controllers/api/v1/media_search_controller.ex index 946eaad61..a4e07dd8a 100644 --- a/lib/ret_web/controllers/api/v1/media_search_controller.ex +++ b/lib/ret_web/controllers/api/v1/media_search_controller.ex @@ -128,7 +128,7 @@ defmodule RetWeb.Api.V1.MediaSearchController do # For Google services, increase cache duration for landing pages by using long-lived cache, due to quotas. defp cache_for_query(%Ret.MediaSearchQuery{source: source, q: nil}) - when source == "youtube_videos" or source == "icosa", + when source == "youtube_videos", do: :media_search_results_long defp cache_for_query(_query), do: :media_search_results diff --git a/lib/ret_web/controllers/page_controller.ex b/lib/ret_web/controllers/page_controller.ex index 48a5a5b5f..70ac82a02 100644 --- a/lib/ret_web/controllers/page_controller.ex +++ b/lib/ret_web/controllers/page_controller.ex @@ -759,7 +759,10 @@ defmodule RetWeb.PageController do upstream: url, allowed_origins: allowed_origins, proxy_url: "#{cors_scheme}://#{cors_host}:#{cors_port}", - # Since we need to use the host for SSL verification, we provide the authority + # We need to force the host + # used for ssl verification here so that the connection isn't rejected. + # Note that we have to convert the authority to a charlist, since this uses Erlang's `ssl` module + # internally, which expects a charlist. client_options: [ ssl: [{:server_name_indication, to_charlist(authority)}, {:versions, [:"tlsv1.2",:"tlsv1.3"]}] ] From facd9be736ab15bacb7f3bd6a823e1b4ccb23ddc Mon Sep 17 00:00:00 2001 From: Andy Baker Date: Thu, 20 Nov 2025 10:18:32 +0000 Subject: [PATCH 26/26] Mix format --- lib/ret/account.ex | 5 +- lib/ret/discord_client.ex | 81 +++++---- lib/ret/http_utils.ex | 15 +- lib/ret/media_resolver.ex | 70 ++++---- lib/ret/media_search.ex | 11 +- lib/ret/scene.ex | 11 +- lib/ret/storage.ex | 8 +- .../api/v1/media_search_controller.ex | 34 ++-- lib/ret_web/controllers/file_controller.ex | 12 +- lib/ret_web/controllers/page_controller.ex | 162 +++++++++++------- lib/ret_web/email.ex | 4 +- ...190114232257_add_reviewed_at_to_scenes.exs | 4 +- test/ret_web/channels/entity_test.exs | 4 +- test/support/test_helpers.ex | 6 +- 14 files changed, 261 insertions(+), 166 deletions(-) diff --git a/lib/ret/account.ex b/lib/ret/account.ex index 853b23811..7f2f95311 100644 --- a/lib/ret/account.ex +++ b/lib/ret/account.ex @@ -68,7 +68,10 @@ defmodule Ret.Account do _ -> false end - Repo.insert!(%Account{login: %Login{identifier_hash: identifier_hash}, is_admin: is_admin}) + Repo.insert!(%Account{ + login: %Login{identifier_hash: identifier_hash}, + is_admin: is_admin + }) true -> nil diff --git a/lib/ret/discord_client.ex b/lib/ret/discord_client.ex index 1e0fea6c3..8f220098c 100644 --- a/lib/ret/discord_client.ex +++ b/lib/ret/discord_client.ex @@ -76,21 +76,23 @@ defmodule Ret.DiscordClient do {status, result} when status in [:commit, :ok] -> "#{result["nick"]}" end - nickname = if !nickname or nickname == "" do - case Cachex.fetch(:discord_api, "/users/#{provider_account_id}") do - {status, result} when status in [:commit, :ok] -> "#{result["global_name"]}" - end + nickname = + if !nickname or nickname == "" do + case Cachex.fetch(:discord_api, "/users/#{provider_account_id}") do + {status, result} when status in [:commit, :ok] -> "#{result["global_name"]}" + end else nickname - end - - nickname = if !nickname or nickname == "" do - case Cachex.fetch(:discord_api, "/users/#{provider_account_id}") do - {status, result} when status in [:commit, :ok] -> "#{result["username"]}" end + + nickname = + if !nickname or nickname == "" do + case Cachex.fetch(:discord_api, "/users/#{provider_account_id}") do + {status, result} when status in [:commit, :ok] -> "#{result["username"]}" + end else nickname - end + end end def fetch_community_identifier(%Ret.OAuthProvider{ @@ -169,18 +171,21 @@ defmodule Ret.DiscordClient do case Cachex.fetch(:discord_api, "/guilds/#{community_id}/roles") do {status, result} when status in [:commit, :ok] -> result |> Map.new(&{&1["id"], &1}) end - # Note: Whether the bitfield values in guild_roles are represented as strings or integers is inconsistent (possibly based on what permissions the user has), so every time they're used they need to be checked and, if needed, converted to integers. + + # Note: Whether the bitfield values in guild_roles are represented as strings or integers is inconsistent (possibly based on what permissions the user has), so every time they're used they need to be checked and, if needed, converted to integers. role_everyone = guild_roles[community_id] permissions = role_everyone["permissions"] user_permissions = user_roles |> Enum.map(&guild_roles[&1]["permissions"]) - permissions = user_permissions |> - Enum.reduce(permissions, &( - (if is_binary(&1), do: String.to_integer(&1), else: &1) ||| - (if is_binary(&2), do: String.to_integer(&2), else: &2) - )) + permissions = + user_permissions + |> Enum.reduce( + permissions, + &(if(is_binary(&1), do: String.to_integer(&1), else: &1) ||| + if(is_binary(&2), do: String.to_integer(&2), else: &2)) + ) if (permissions &&& @administrator) == @administrator do @all @@ -203,15 +208,21 @@ defmodule Ret.DiscordClient do |> Map.get("permission_overwrites") |> Map.new(&{&1["id"], &1}) end - # Note: Whether the bitfield values in channel_overwrites are represented as strings or integers is inconsistent (possibly based on what permissions the user has), so every time they're used they need to be checked and, if needed, converted to integers. + + # Note: Whether the bitfield values in channel_overwrites are represented as strings or integers is inconsistent (possibly based on what permissions the user has), so every time they're used they need to be checked and, if needed, converted to integers. overwrite_everyone = channel_overwrites[community_id] permissions = if overwrite_everyone do (permissions &&& - ~~~(if is_binary(overwrite_everyone["deny"]), do: String.to_integer(overwrite_everyone["deny"]), else: overwrite_everyone["deny"])) ||| - (if is_binary(overwrite_everyone["allow"]), do: String.to_integer(overwrite_everyone["allow"]), else: overwrite_everyone["allow"]) + ~~~if(is_binary(overwrite_everyone["deny"]), + do: String.to_integer(overwrite_everyone["deny"]), + else: overwrite_everyone["deny"] + )) ||| + if is_binary(overwrite_everyone["allow"]), + do: String.to_integer(overwrite_everyone["allow"]), + else: overwrite_everyone["allow"] else permissions end @@ -220,14 +231,21 @@ defmodule Ret.DiscordClient do user_permissions = user_roles |> Enum.map(&channel_overwrites[&1]) |> Enum.filter(&(&1 != nil)) - allow = user_permissions |> Enum.reduce(@none, &( - (if is_binary(&1["allow"]), do: String.to_integer(&1["allow"]), else: &1["allow"]) ||| - &2 - )) - deny = user_permissions |> Enum.reduce(@none, &( - (if is_binary(&1["deny"]), do: String.to_integer(&1["deny"]), else: &1["deny"]) ||| - &2 - )) + allow = + user_permissions + |> Enum.reduce( + @none, + &(if(is_binary(&1["allow"]), do: String.to_integer(&1["allow"]), else: &1["allow"]) ||| + &2) + ) + + deny = + user_permissions + |> Enum.reduce( + @none, + &(if(is_binary(&1["deny"]), do: String.to_integer(&1["deny"]), else: &1["deny"]) ||| + &2) + ) permissions = (permissions &&& ~~~deny) ||| allow @@ -237,8 +255,13 @@ defmodule Ret.DiscordClient do permissions = if overwrite_member do (permissions &&& - ~~~(if is_binary(overwrite_member["deny"]), do: String.to_integer(overwrite_member["deny"]), else: overwrite_member["deny"])) ||| - (if is_binary(overwrite_member["allow"]), do: String.to_integer(overwrite_member["allow"]), else: overwrite_member["allow"]) + ~~~if(is_binary(overwrite_member["deny"]), + do: String.to_integer(overwrite_member["deny"]), + else: overwrite_member["deny"] + )) ||| + if is_binary(overwrite_member["allow"]), + do: String.to_integer(overwrite_member["allow"]), + else: overwrite_member["allow"] else permissions end diff --git a/lib/ret/http_utils.ex b/lib/ret/http_utils.ex index 71366aba8..8bacdebfe 100644 --- a/lib/ret/http_utils.ex +++ b/lib/ret/http_utils.ex @@ -177,12 +177,14 @@ defmodule Ret.HttpUtils do end def join_smart(enum) do - Enum.reduce(enum, "", fn(x, acc) -> - x = cond do - !x -> nil - is_binary(x) -> String.trim(x) - true -> "#{x}" - end + Enum.reduce(enum, "", fn x, acc -> + x = + cond do + !x -> nil + is_binary(x) -> String.trim(x) + true -> "#{x}" + end + if x && x != "" do if acc && acc != "", do: acc <> " — " <> x, else: x else @@ -190,5 +192,4 @@ defmodule Ret.HttpUtils do end end) end - end diff --git a/lib/ret/media_resolver.ex b/lib/ret/media_resolver.ex index 015cb554f..43d162e14 100644 --- a/lib/ret/media_resolver.ex +++ b/lib/ret/media_resolver.ex @@ -337,42 +337,44 @@ defmodule Ret.MediaResolver do end defp resolve_non_video( - %MediaResolverQuery{url: %URI{host: "api.icosa.gallery", path: "/v1/assets/" <> asset_id} = uri}, + %MediaResolverQuery{ + url: %URI{host: "api.icosa.gallery", path: "/v1/assets/" <> asset_id} = uri + }, "icosa.gallery" - ) do - # Increment stat for the request - Statix.increment("ret.media_resolver.icosa.requests") - - # Make the API call to get the asset data - payload = - "https://api.icosa.gallery/v1/assets/#{asset_id}" - |> retry_get_until_success() # Assuming this function sends the request and handles retries - |> Map.get(:body) - |> Poison.decode!() - - # Create the meta information based on the payload - meta = - %{ - expected_content_type: "model/gltf", - name: payload["displayName"], - author: payload["authorName"], - license: payload["license"] - } - - # Extract the GLTF2 format URL from the payload - uri = - payload["formats"] - |> Enum.find(&(&1["formatType"] == "GLTF2")) - |> Kernel.get_in(["root", "url"]) - |> URI.parse() - - # Increment stat for successful resolution - Statix.increment("ret.media_resolver.icosa.ok") - - # Return the URI and meta data for further processing - {:commit, resolved(uri, meta)} - end + ) do + # Increment stat for the request + Statix.increment("ret.media_resolver.icosa.requests") + + # Make the API call to get the asset data + payload = + "https://api.icosa.gallery/v1/assets/#{asset_id}" + # Assuming this function sends the request and handles retries + |> retry_get_until_success() + |> Map.get(:body) + |> Poison.decode!() + + # Create the meta information based on the payload + meta = + %{ + expected_content_type: "model/gltf", + name: payload["displayName"], + author: payload["authorName"], + license: payload["license"] + } + # Extract the GLTF2 format URL from the payload + uri = + payload["formats"] + |> Enum.find(&(&1["formatType"] == "GLTF2")) + |> Kernel.get_in(["root", "url"]) + |> URI.parse() + + # Increment stat for successful resolution + Statix.increment("ret.media_resolver.icosa.ok") + + # Return the URI and meta data for further processing + {:commit, resolved(uri, meta)} + end defp resolve_non_video( %MediaResolverQuery{url: %URI{path: "/models/" <> model_id}} = query, diff --git a/lib/ret/media_search.ex b/lib/ret/media_search.ex index 6ca567018..a5b28318f 100644 --- a/lib/ret/media_search.ex +++ b/lib/ret/media_search.ex @@ -290,7 +290,12 @@ defmodule Ret.MediaSearch do end end - def search(%Ret.MediaSearchQuery{source: "youtube_videos", cursor: cursor, filter: filter, q: q}) do + def search(%Ret.MediaSearchQuery{ + source: "youtube_videos", + cursor: cursor, + filter: filter, + q: q + }) do with api_key when is_binary(api_key) <- resolver_config(:youtube_api_key) do query = URI.encode_query( @@ -418,7 +423,8 @@ defmodule Ret.MediaSearch do end end - def available?(:icosa), do: true # Icosa does not currently require an API key + # Icosa does not currently require an API key + def available?(:icosa), do: true def available?(:bing_images), do: has_resolver_config?(:bing_search_api_key) def available?(:bing_videos), do: has_resolver_config?(:bing_search_api_key) def available?(:youtube_videos), do: has_resolver_config?(:youtube_api_key) @@ -601,6 +607,7 @@ defmodule Ret.MediaSearch do defp created_rooms_search(cursor, account_id, _query) do page_number = (cursor || "1") |> Integer.parse() |> elem(0) + ecto_query = from h in Hub, where: h.created_by_account_id == ^account_id, diff --git a/lib/ret/scene.ex b/lib/ret/scene.ex index 5927034a3..18485c713 100644 --- a/lib/ret/scene.ex +++ b/lib/ret/scene.ex @@ -226,7 +226,7 @@ defmodule Ret.Scene do model_owned_file: old_model_owned_file, account: account } = scene - + new_scene_owned_file = Storage.create_new_owned_file_with_replaced_string( old_scene_owned_file, @@ -234,29 +234,30 @@ defmodule Ret.Scene do old_domain_url, new_domain_url ) - + {:ok, new_model_owned_file} = Storage.duplicate_and_transform(old_model_owned_file, account, fn glb_stream, _total_bytes -> GLTFUtils.replace_in_glb(glb_stream, old_domain_url, new_domain_url) end) - + scene |> change() |> put_change(:scene_owned_file_id, new_scene_owned_file.owned_file_id) |> put_change(:model_owned_file_id, new_model_owned_file.owned_file_id) |> Repo.update!() - + for old_owned_file <- [old_scene_owned_file, old_model_owned_file] do OwnedFile.set_inactive(old_owned_file) Storage.rm_files_for_owned_file(old_owned_file) Repo.delete(old_owned_file) end rescue - e -> + e -> IO.warn("Failed to process scene due to an error: #{inspect(e)}") end end) + :ok end) end diff --git a/lib/ret/storage.ex b/lib/ret/storage.ex index 05c064dcc..34dfe6e03 100644 --- a/lib/ret/storage.ex +++ b/lib/ret/storage.ex @@ -288,7 +288,7 @@ defmodule Ret.Storage do _account, _require_token ) do - {:ok, owned_file} + {:ok, owned_file} end # Promoting a stored file to being owned has two side effects: the file is moved @@ -310,7 +310,11 @@ defmodule Ret.Storage do "promotion_token" => actual_promotion_token } <- File.read!(meta_file_path) |> Poison.decode!(), - {:ok} <- (if require_token, do: check_promotion_token(actual_promotion_token, promotion_token), else: {:ok}), + {:ok} <- + if(require_token, + do: check_promotion_token(actual_promotion_token, promotion_token), + else: {:ok} + ), {:ok} <- check_blob_file_key(blob_file_path, key) ) do owned_file_params = %{ diff --git a/lib/ret_web/controllers/api/v1/media_search_controller.ex b/lib/ret_web/controllers/api/v1/media_search_controller.ex index a4e07dd8a..5e6db6ae7 100644 --- a/lib/ret_web/controllers/api/v1/media_search_controller.ex +++ b/lib/ret_web/controllers/api/v1/media_search_controller.ex @@ -3,25 +3,25 @@ defmodule RetWeb.Api.V1.MediaSearchController do use Retry def index(conn, %{"source" => source, "filter" => "created", "user" => user} = params) - when source in ["rooms"] do - account = conn |> Guardian.Plug.current_resource() - - if account && account.account_id == String.to_integer(user) do - {:commit, results} = - %Ret.MediaSearchQuery{ - source: "rooms", - cursor: params["cursor"] || "1", - user: account.account_id, - filter: "created", - q: params["q"] - } - |> Ret.MediaSearch.search() + when source in ["rooms"] do + account = conn |> Guardian.Plug.current_resource() - conn |> render("index.json", results: results) - else - conn |> send_resp(401, "You can only search created rooms for your own account.") - end + if account && account.account_id == String.to_integer(user) do + {:commit, results} = + %Ret.MediaSearchQuery{ + source: "rooms", + cursor: params["cursor"] || "1", + user: account.account_id, + filter: "created", + q: params["q"] + } + |> Ret.MediaSearch.search() + + conn |> render("index.json", results: results) + else + conn |> send_resp(401, "You can only search created rooms for your own account.") end + end def index(conn, %{"source" => "rooms"} = params) do {:commit, results} = diff --git a/lib/ret_web/controllers/file_controller.ex b/lib/ret_web/controllers/file_controller.ex index 6b6d8ca20..a61efff1f 100644 --- a/lib/ret_web/controllers/file_controller.ex +++ b/lib/ret_web/controllers/file_controller.ex @@ -26,7 +26,9 @@ defmodule RetWeb.FileController do app_name = AppConfig.get_cached_config_value("translations|en|app-full-name") || - AppConfig.get_cached_config_value("translations|en|app-name") || RetWeb.Endpoint.host() + AppConfig.get_cached_config_value("translations|en|app-name") || + RetWeb.Endpoint.host() + title = "Photo taken in #{app_name} immersive space" config = AppConfig.get_config() @@ -36,9 +38,11 @@ defmodule RetWeb.FileController do content_type: content_type |> RetWeb.ContentType.sanitize_content_type(), content_length: content_length, title: title, - description_social_media: Ret.HttpUtils.join_smart([ - config["translations"]["en"]["app-tagline"], - "powered by Hubs"]), + description_social_media: + Ret.HttpUtils.join_smart([ + config["translations"]["en"]["app-tagline"], + "powered by Hubs" + ]), translations: config["translations"]["en"], app_name: app_name, images: config["images"], diff --git a/lib/ret_web/controllers/page_controller.ex b/lib/ret_web/controllers/page_controller.ex index 70ac82a02..028649b19 100644 --- a/lib/ret_web/controllers/page_controller.ex +++ b/lib/ret_web/controllers/page_controller.ex @@ -65,19 +65,24 @@ defmodule RetWeb.PageController do defp render_scene_content(%t{} = scene, conn) when t in [Scene, SceneListing] do {app_config, app_config_script} = generate_app_config() - app_name = app_config["translations"]["en"]["app-full-name"] || app_config["translations"]["en"]["app-name"] || RetWeb.Endpoint.host() + app_name = + app_config["translations"]["en"]["app-full-name"] || + app_config["translations"]["en"]["app-name"] || RetWeb.Endpoint.host() + scene_meta_tags = Phoenix.View.render_to_string(RetWeb.PageView, "scene-meta.html", scene: scene, ret_meta: Ret.Meta.get_meta(include_repo: false), translations: app_config["translations"]["en"], - title: join_smart([ scene.name, app_name ]), + title: join_smart([scene.name, app_name]), name: scene.name, - description: join_smart([ - scene.description, - "A scene you can use in the #{app_name} immersive spaces and others powered by Hubs", - app_config["translations"]["en"]["app-description"] - ]) |> String.replace("\\n", " "), + description: + join_smart([ + scene.description, + "A scene you can use in the #{app_name} immersive spaces and others powered by Hubs", + app_config["translations"]["en"]["app-description"] + ]) + |> String.replace("\\n", " "), app_config_script: {:safe, app_config_script |> with_script_tags}, extra_script: {:safe, get_extra_script(:scene) |> with_script_tags}, extra_html: {:safe, get_extra_html(:scene) || ""} @@ -106,13 +111,17 @@ defmodule RetWeb.PageController do defp render_avatar_content(%t{} = avatar, conn) when t in [Avatar, AvatarListing] do {app_config, app_config_script} = generate_app_config() - app_name = app_config["translations"]["en"]["app-full-name"] || app_config["translations"]["en"]["app-name"] || RetWeb.Endpoint.host() + app_name = + app_config["translations"]["en"]["app-full-name"] || + app_config["translations"]["en"]["app-name"] || RetWeb.Endpoint.host() + avatar_meta_tags = Phoenix.View.render_to_string(RetWeb.PageView, "avatar-meta.html", avatar: avatar, title: join_smart([avatar.name, app_name]), name: avatar.name, - description: "An avatar you can use in the #{app_name} immersive spaces and others powered by Hubs.", + description: + "An avatar you can use in the #{app_name} immersive spaces and others powered by Hubs.", ret_meta: Ret.Meta.get_meta(include_repo: false), translations: app_config["translations"]["en"], root_url: RetWeb.Endpoint.url(), @@ -144,7 +153,9 @@ defmodule RetWeb.PageController do defp render_homepage_content(conn, nil = _public_room_id) do {app_config, app_config_script} = generate_app_config() - app_name = app_config["translations"]["en"]["app-full-name"] || app_config["translations"]["en"]["app-name"] || RetWeb.Endpoint.host() + app_name = + app_config["translations"]["en"]["app-full-name"] || + app_config["translations"]["en"]["app-name"] || RetWeb.Endpoint.host() index_meta_tags = Phoenix.View.render_to_string( @@ -154,10 +165,11 @@ defmodule RetWeb.PageController do translations: app_config["translations"]["en"], app_name: app_name, title: join_smart([app_name, app_config["translations"]["en"]["app-tagline"]]), - description: join_smart( - [app_config["translations"]["en"]["app-description"], - "Immersive spaces, right in your browser, powered by Hubs" - ]), + description: + join_smart([ + app_config["translations"]["en"]["app-description"], + "Immersive spaces, right in your browser, powered by Hubs" + ]), images: app_config["images"], app_config_script: {:safe, app_config_script |> with_script_tags}, extra_script: {:safe, get_extra_script(:index) |> with_script_tags}, @@ -291,11 +303,14 @@ defmodule RetWeb.PageController do manifest = Phoenix.View.render_to_string(RetWeb.PageView, "manifest.webmanifest", root_url: RetWeb.Endpoint.url(), - app_name: get_app_config_value("translations|en|app-name") || RetWeb.Endpoint.host(), - app_description: join_smart([ - get_app_config_value("translations|en|app-description"), - "Immersive spaces, right in your browser, powered by Hubs" - ]) |> String.replace("\\n", " ") + app_name: + get_app_config_value("translations|en|app-name") || RetWeb.Endpoint.host(), + app_description: + join_smart([ + get_app_config_value("translations|en|app-description"), + "Immersive spaces, right in your browser, powered by Hubs" + ]) + |> String.replace("\\n", " ") ) unless module_config(:skip_cache) do @@ -484,12 +499,22 @@ defmodule RetWeb.PageController do app_name = app_config["translations"]["en"]["app-name"] || RetWeb.Endpoint.host() scene = hub.scene || hub.scene_listing - name = cond do - hub.name && scene && scene.name && scene.name != hub.name -> join_smart([hub.name, scene.name]) - hub.name -> hub.name - scene && scene.name -> scene.name - true -> "a room on " <> app_name - end + + name = + cond do + hub.name && scene && scene.name && scene.name != hub.name -> + join_smart([hub.name, scene.name]) + + hub.name -> + hub.name + + scene && scene.name -> + scene.name + + true -> + "a room on " <> app_name + end + hub_meta_tags = Phoenix.View.render_to_string(RetWeb.PageView, "hub-meta.html", hub: hub, @@ -499,18 +524,22 @@ defmodule RetWeb.PageController do translations: app_config["translations"]["en"], title: join_smart([hub.name, app_name]), name: name, - description: join_smart([ - hub.description, - "an immersive space in #{app_name}, right in your browser", - app_config["translations"]["en"]["app-description"], - "powered by Hubs." - ]) |> String.replace("\\n", " "), - description_social_media: join_smart([ - "Join others in an immersive space in #{app_name}, right in your browser", - hub.description, - app_config["translations"]["en"]["app-description"], - "powered by Hubs." - ]) |> String.replace("\\n", " "), + description: + join_smart([ + hub.description, + "an immersive space in #{app_name}, right in your browser", + app_config["translations"]["en"]["app-description"], + "powered by Hubs." + ]) + |> String.replace("\\n", " "), + description_social_media: + join_smart([ + "Join others in an immersive space in #{app_name}, right in your browser", + hub.description, + app_config["translations"]["en"]["app-description"], + "powered by Hubs." + ]) + |> String.replace("\\n", " "), app_config_script: {:safe, app_config_script |> with_script_tags}, extra_script: {:safe, get_extra_script(:room) |> with_script_tags}, extra_html: {:safe, get_extra_html(:room) || ""} @@ -726,7 +755,6 @@ defmodule RetWeb.PageController do end defp cors_proxy_with_redirects(conn, url, redirect_count) do - %URI{authority: authority, host: host} = uri = URI.parse(url) resolved_ip = HttpUtils.resolve_ip(host) @@ -764,7 +792,10 @@ defmodule RetWeb.PageController do # Note that we have to convert the authority to a charlist, since this uses Erlang's `ssl` module # internally, which expects a charlist. client_options: [ - ssl: [{:server_name_indication, to_charlist(authority)}, {:versions, [:"tlsv1.2",:"tlsv1.3"]}] + ssl: [ + {:server_name_indication, to_charlist(authority)}, + {:versions, [:"tlsv1.2", :"tlsv1.3"]} + ] ] ) @@ -773,15 +804,22 @@ defmodule RetWeb.PageController do try do # First, make a HEAD request to check for redirects using HTTPoison - case HTTPoison.head(url, [], [ - follow_redirect: false, - ssl: [{:server_name_indication, to_charlist(authority)}, {:versions, [:"tlsv1.2", :"tlsv1.3"]}], - timeout: 15_000, - recv_timeout: 15_000 - ]) do - {:ok, %HTTPoison.Response{status_code: status_code, headers: headers}} when status_code in [301, 302, 303, 307, 308] -> + case HTTPoison.head(url, [], + follow_redirect: false, + ssl: [ + {:server_name_indication, to_charlist(authority)}, + {:versions, [:"tlsv1.2", :"tlsv1.3"]} + ], + timeout: 15_000, + recv_timeout: 15_000 + ) do + {:ok, %HTTPoison.Response{status_code: status_code, headers: headers}} + when status_code in [301, 302, 303, 307, 308] -> # Found a redirect - location_header = headers |> Enum.find(fn {k, _v} -> String.downcase(k) == "location" end) |> elem(1) + location_header = + headers + |> Enum.find(fn {k, _v} -> String.downcase(k) == "location" end) + |> elem(1) if location_header do # Resolve relative URLs against the current URL @@ -810,6 +848,7 @@ defmodule RetWeb.PageController do :exit, reason -> Logger.error("CORS Proxy: Request exited with reason: #{inspect(reason)}") conn |> send_resp(500, "Proxy request timed out or failed") + kind, reason -> Logger.error("CORS Proxy: Request failed with #{kind}: #{inspect(reason)}") conn |> send_resp(500, "Proxy request failed") @@ -822,20 +861,21 @@ defmodule RetWeb.PageController do end defp make_reverse_proxy_request(conn, _url, body, is_head, opts) do - proxy_conn = %Conn{} - |> Map.merge(conn) - |> Map.put( - :method, - if is_head do - "HEAD" - else - conn.method - end - ) - # Need to strip path_info since proxy plug reads it - |> Map.put(:path_info, []) - |> ReverseProxyPlug.request(body, opts) - |> ReverseProxyPlug.response(conn, opts) + proxy_conn = + %Conn{} + |> Map.merge(conn) + |> Map.put( + :method, + if is_head do + "HEAD" + else + conn.method + end + ) + # Need to strip path_info since proxy plug reads it + |> Map.put(:path_info, []) + |> ReverseProxyPlug.request(body, opts) + |> ReverseProxyPlug.response(conn, opts) proxy_conn end diff --git a/lib/ret_web/email.ex b/lib/ret_web/email.ex index 0c2f6072b..7ba8b12d5 100644 --- a/lib/ret_web/email.ex +++ b/lib/ret_web/email.ex @@ -3,7 +3,9 @@ defmodule RetWeb.Email do alias Ret.{AppConfig} def auth_email(to_address, signin_args) do - app_name = AppConfig.get_cached_config_value("translations|en|app-name") || RetWeb.Endpoint.host() + app_name = + AppConfig.get_cached_config_value("translations|en|app-name") || RetWeb.Endpoint.host() + app_full_name = AppConfig.get_cached_config_value("translations|en|app-full-name") || app_name admin_email = Application.get_env(:ret, Ret.Account)[:admin_email] custom_login_subject = AppConfig.get_cached_config_value("auth|login_subject") diff --git a/priv/repo/migrations/20190114232257_add_reviewed_at_to_scenes.exs b/priv/repo/migrations/20190114232257_add_reviewed_at_to_scenes.exs index 8e5a26da1..ea1707548 100644 --- a/priv/repo/migrations/20190114232257_add_reviewed_at_to_scenes.exs +++ b/priv/repo/migrations/20190114232257_add_reviewed_at_to_scenes.exs @@ -6,6 +6,8 @@ defmodule Ret.Repo.Migrations.AddReviewedAtToScenes do add :reviewed_at, :utc_datetime, null: true end - create index(:scenes, [:reviewed_at], where: "reviewed_at is null or reviewed_at < updated_at") + create index(:scenes, [:reviewed_at], + where: "reviewed_at is null or reviewed_at < updated_at" + ) end end diff --git a/test/ret_web/channels/entity_test.exs b/test/ret_web/channels/entity_test.exs index c18d1e28a..0cbc6b556 100644 --- a/test/ret_web/channels/entity_test.exs +++ b/test/ret_web/channels/entity_test.exs @@ -8,7 +8,9 @@ defmodule RetWeb.EntityTest do @payload_save_entity_state read_json("save_entity_state_payload.json") @payload_save_entity_state_2 read_json("save_entity_state_payload_2.json") - @payload_save_entity_state_promotable_no_token read_json("save_entity_state_payload_promotable_no_token.json") + @payload_save_entity_state_promotable_no_token read_json( + "save_entity_state_payload_promotable_no_token.json" + ) @payload_save_entity_state_promotable read_json("save_entity_state_payload_promotable.json") @payload_save_entity_state_unpromotable read_json("save_entity_state_payload_unpromotable.json") @payload_update_entity_state read_json("update_entity_state_payload.json") diff --git a/test/support/test_helpers.ex b/test/support/test_helpers.ex index eb00c8483..77df6036b 100644 --- a/test/support/test_helpers.ex +++ b/test/support/test_helpers.ex @@ -216,7 +216,11 @@ defmodule Ret.TestHelpers do {:ok, asset: asset} end - def create_project_asset(%{account: account, project: project, thumbnail_owned_file: owned_file}) do + def create_project_asset(%{ + account: account, + project: project, + thumbnail_owned_file: owned_file + }) do {:ok, asset} = %Asset{} |> Asset.changeset(account, owned_file, owned_file, %{