diff --git a/lib/ret/account.ex b/lib/ret/account.ex index 853b23811..7f2f95311 100644 --- a/lib/ret/account.ex +++ b/lib/ret/account.ex @@ -68,7 +68,10 @@ defmodule Ret.Account do _ -> false end - Repo.insert!(%Account{login: %Login{identifier_hash: identifier_hash}, is_admin: is_admin}) + Repo.insert!(%Account{ + login: %Login{identifier_hash: identifier_hash}, + is_admin: is_admin + }) true -> nil diff --git a/lib/ret/discord_client.ex b/lib/ret/discord_client.ex index 1e0fea6c3..8f220098c 100644 --- a/lib/ret/discord_client.ex +++ b/lib/ret/discord_client.ex @@ -76,21 +76,23 @@ defmodule Ret.DiscordClient do {status, result} when status in [:commit, :ok] -> "#{result["nick"]}" end - nickname = if !nickname or nickname == "" do - case Cachex.fetch(:discord_api, "/users/#{provider_account_id}") do - {status, result} when status in [:commit, :ok] -> "#{result["global_name"]}" - end + nickname = + if !nickname or nickname == "" do + case Cachex.fetch(:discord_api, "/users/#{provider_account_id}") do + {status, result} when status in [:commit, :ok] -> "#{result["global_name"]}" + end else nickname - end - - nickname = if !nickname or nickname == "" do - case Cachex.fetch(:discord_api, "/users/#{provider_account_id}") do - {status, result} when status in [:commit, :ok] -> "#{result["username"]}" end + + nickname = + if !nickname or nickname == "" do + case Cachex.fetch(:discord_api, "/users/#{provider_account_id}") do + {status, result} when status in [:commit, :ok] -> "#{result["username"]}" + end else nickname - end + end end def fetch_community_identifier(%Ret.OAuthProvider{ @@ -169,18 +171,21 @@ defmodule Ret.DiscordClient do case Cachex.fetch(:discord_api, "/guilds/#{community_id}/roles") do {status, result} when status in [:commit, :ok] -> result |> Map.new(&{&1["id"], &1}) end - # Note: Whether the bitfield values in guild_roles are represented as strings or integers is inconsistent (possibly based on what permissions the user has), so every time they're used they need to be checked and, if needed, converted to integers. + + # Note: Whether the bitfield values in guild_roles are represented as strings or integers is inconsistent (possibly based on what permissions the user has), so every time they're used they need to be checked and, if needed, converted to integers. role_everyone = guild_roles[community_id] permissions = role_everyone["permissions"] user_permissions = user_roles |> Enum.map(&guild_roles[&1]["permissions"]) - permissions = user_permissions |> - Enum.reduce(permissions, &( - (if is_binary(&1), do: String.to_integer(&1), else: &1) ||| - (if is_binary(&2), do: String.to_integer(&2), else: &2) - )) + permissions = + user_permissions + |> Enum.reduce( + permissions, + &(if(is_binary(&1), do: String.to_integer(&1), else: &1) ||| + if(is_binary(&2), do: String.to_integer(&2), else: &2)) + ) if (permissions &&& @administrator) == @administrator do @all @@ -203,15 +208,21 @@ defmodule Ret.DiscordClient do |> Map.get("permission_overwrites") |> Map.new(&{&1["id"], &1}) end - # Note: Whether the bitfield values in channel_overwrites are represented as strings or integers is inconsistent (possibly based on what permissions the user has), so every time they're used they need to be checked and, if needed, converted to integers. + + # Note: Whether the bitfield values in channel_overwrites are represented as strings or integers is inconsistent (possibly based on what permissions the user has), so every time they're used they need to be checked and, if needed, converted to integers. overwrite_everyone = channel_overwrites[community_id] permissions = if overwrite_everyone do (permissions &&& - ~~~(if is_binary(overwrite_everyone["deny"]), do: String.to_integer(overwrite_everyone["deny"]), else: overwrite_everyone["deny"])) ||| - (if is_binary(overwrite_everyone["allow"]), do: String.to_integer(overwrite_everyone["allow"]), else: overwrite_everyone["allow"]) + ~~~if(is_binary(overwrite_everyone["deny"]), + do: String.to_integer(overwrite_everyone["deny"]), + else: overwrite_everyone["deny"] + )) ||| + if is_binary(overwrite_everyone["allow"]), + do: String.to_integer(overwrite_everyone["allow"]), + else: overwrite_everyone["allow"] else permissions end @@ -220,14 +231,21 @@ defmodule Ret.DiscordClient do user_permissions = user_roles |> Enum.map(&channel_overwrites[&1]) |> Enum.filter(&(&1 != nil)) - allow = user_permissions |> Enum.reduce(@none, &( - (if is_binary(&1["allow"]), do: String.to_integer(&1["allow"]), else: &1["allow"]) ||| - &2 - )) - deny = user_permissions |> Enum.reduce(@none, &( - (if is_binary(&1["deny"]), do: String.to_integer(&1["deny"]), else: &1["deny"]) ||| - &2 - )) + allow = + user_permissions + |> Enum.reduce( + @none, + &(if(is_binary(&1["allow"]), do: String.to_integer(&1["allow"]), else: &1["allow"]) ||| + &2) + ) + + deny = + user_permissions + |> Enum.reduce( + @none, + &(if(is_binary(&1["deny"]), do: String.to_integer(&1["deny"]), else: &1["deny"]) ||| + &2) + ) permissions = (permissions &&& ~~~deny) ||| allow @@ -237,8 +255,13 @@ defmodule Ret.DiscordClient do permissions = if overwrite_member do (permissions &&& - ~~~(if is_binary(overwrite_member["deny"]), do: String.to_integer(overwrite_member["deny"]), else: overwrite_member["deny"])) ||| - (if is_binary(overwrite_member["allow"]), do: String.to_integer(overwrite_member["allow"]), else: overwrite_member["allow"]) + ~~~if(is_binary(overwrite_member["deny"]), + do: String.to_integer(overwrite_member["deny"]), + else: overwrite_member["deny"] + )) ||| + if is_binary(overwrite_member["allow"]), + do: String.to_integer(overwrite_member["allow"]), + else: overwrite_member["allow"] else permissions end diff --git a/lib/ret/http_utils.ex b/lib/ret/http_utils.ex index 71366aba8..8bacdebfe 100644 --- a/lib/ret/http_utils.ex +++ b/lib/ret/http_utils.ex @@ -177,12 +177,14 @@ defmodule Ret.HttpUtils do end def join_smart(enum) do - Enum.reduce(enum, "", fn(x, acc) -> - x = cond do - !x -> nil - is_binary(x) -> String.trim(x) - true -> "#{x}" - end + Enum.reduce(enum, "", fn x, acc -> + x = + cond do + !x -> nil + is_binary(x) -> String.trim(x) + true -> "#{x}" + end + if x && x != "" do if acc && acc != "", do: acc <> " — " <> x, else: x else @@ -190,5 +192,4 @@ defmodule Ret.HttpUtils do end end) end - end diff --git a/lib/ret/media_resolver.ex b/lib/ret/media_resolver.ex index 31e91f606..43d162e14 100644 --- a/lib/ret/media_resolver.ex +++ b/lib/ret/media_resolver.ex @@ -20,12 +20,14 @@ defmodule Ret.MediaResolver do @youtube_rate_limit %{scale: 8_000, limit: 1} @sketchfab_rate_limit %{scale: 60_000, limit: 15} + @icosa_rate_limit %{scale: 60_000, limit: 1000} @max_await_for_rate_limit_s 120 @non_video_root_hosts [ "sketchfab.com", "giphy.com", - "tenor.com" + "tenor.com", + "icosa.gallery" ] @deviant_id_regex ~r/\"DeviantArt:\/\/deviation\/([^"]+)/ @@ -79,6 +81,13 @@ defmodule Ret.MediaResolver do |> resolved()} end + # Necessary short circuit around google.com root_host to skip YT-DL check for Poly + def resolve(%MediaResolverQuery{url: %URI{host: "api.icosa.gallery"}} = query, root_host) do + rate_limited_resolve(query, root_host, @icosa_rate_limit, fn -> + resolve_non_video(query, root_host) + end) + end + def resolve(%MediaResolverQuery{} = query, root_host) when root_host in @non_video_root_hosts do resolve_non_video(query, root_host) end @@ -327,6 +336,46 @@ defmodule Ret.MediaResolver do {:commit, (resolved_url || uri) |> resolved(meta)} end + defp resolve_non_video( + %MediaResolverQuery{ + url: %URI{host: "api.icosa.gallery", path: "/v1/assets/" <> asset_id} = uri + }, + "icosa.gallery" + ) do + # Increment stat for the request + Statix.increment("ret.media_resolver.icosa.requests") + + # Make the API call to get the asset data + payload = + "https://api.icosa.gallery/v1/assets/#{asset_id}" + # Assuming this function sends the request and handles retries + |> retry_get_until_success() + |> Map.get(:body) + |> Poison.decode!() + + # Create the meta information based on the payload + meta = + %{ + expected_content_type: "model/gltf", + name: payload["displayName"], + author: payload["authorName"], + license: payload["license"] + } + + # Extract the GLTF2 format URL from the payload + uri = + payload["formats"] + |> Enum.find(&(&1["formatType"] == "GLTF2")) + |> Kernel.get_in(["root", "url"]) + |> URI.parse() + + # Increment stat for successful resolution + Statix.increment("ret.media_resolver.icosa.ok") + + # Return the URI and meta data for further processing + {:commit, resolved(uri, meta)} + end + defp resolve_non_video( %MediaResolverQuery{url: %URI{path: "/models/" <> model_id}} = query, "sketchfab.com" = root_host diff --git a/lib/ret/media_search.ex b/lib/ret/media_search.ex index 03a4f615c..a5b28318f 100644 --- a/lib/ret/media_search.ex +++ b/lib/ret/media_search.ex @@ -249,7 +249,53 @@ defmodule Ret.MediaSearch do sketchfab_search(query) end - def search(%Ret.MediaSearchQuery{source: "youtube_videos", cursor: cursor, filter: filter, q: q}) do + def search(%Ret.MediaSearchQuery{source: "icosa", cursor: cursor, filter: filter, q: q}) do + query = + URI.encode_query( + pageSize: @page_size, + maxComplexity: :MEDIUM, + format: :GLTF2, + pageToken: cursor, + category: filter, + licence: :REMIXABLE, + orderBy: :BEST, + keywords: q + ) + + url = "https://api.icosa.gallery/v1/assets?#{query}" + # Remove empty category queries as the Icosa API returns validation errors instead of "any" + url = String.replace(url, "category=&", "") + + res = + url + |> retry_get_until_success() + + case res do + :error -> + :error + + res -> + decoded_res = res |> Map.get(:body) |> Poison.decode!() + entries = decoded_res |> Map.get("assets") |> Enum.map(&icosa_api_result_to_entry/1) + next_cursor = decoded_res |> Map.get("nextPageToken") + + {:commit, + %Ret.MediaSearchResult{ + meta: %Ret.MediaSearchResultMeta{ + next_cursor: next_cursor, + source: :icosa + }, + entries: entries + }} + end + end + + def search(%Ret.MediaSearchQuery{ + source: "youtube_videos", + cursor: cursor, + filter: filter, + q: q + }) do with api_key when is_binary(api_key) <- resolver_config(:youtube_api_key) do query = URI.encode_query( @@ -377,6 +423,8 @@ defmodule Ret.MediaSearch do end end + # Icosa does not currently require an API key + def available?(:icosa), do: true def available?(:bing_images), do: has_resolver_config?(:bing_search_api_key) def available?(:bing_videos), do: has_resolver_config?(:bing_search_api_key) def available?(:youtube_videos), do: has_resolver_config?(:youtube_api_key) @@ -559,6 +607,7 @@ defmodule Ret.MediaSearch do defp created_rooms_search(cursor, account_id, _query) do page_number = (cursor || "1") |> Integer.parse() |> elem(0) + ecto_query = from h in Hub, where: h.created_by_account_id == ^account_id, @@ -957,6 +1006,18 @@ defmodule Ret.MediaSearch do } end + defp icosa_api_result_to_entry(result) do + %{ + id: result["assetId"], + type: "icosa_model", + name: result["displayName"], + attributions: %{creator: %{name: result["authorName"]}}, + url: "http://api.icosa.gallery/v1/assets/" <> result["assetId"], + # url: result["url"], + images: %{preview: %{url: result["thumbnail"]["url"]}} + } + end + defp youtube_api_result_to_entry(result) do %{ id: result["id"]["videoId"], diff --git a/lib/ret/meta.ex b/lib/ret/meta.ex index adc310317..be3df373e 100644 --- a/lib/ret/meta.ex +++ b/lib/ret/meta.ex @@ -44,6 +44,7 @@ defmodule Ret.Meta do def available_integrations_meta do %{ twitter: Ret.TwitterClient.available?(), + icosa: Ret.MediaSearch.available?(:icosa), bing_images: Ret.MediaSearch.available?(:bing_images), bing_videos: Ret.MediaSearch.available?(:bing_videos), youtube_videos: Ret.MediaSearch.available?(:youtube_videos), diff --git a/lib/ret/scene.ex b/lib/ret/scene.ex index 5927034a3..18485c713 100644 --- a/lib/ret/scene.ex +++ b/lib/ret/scene.ex @@ -226,7 +226,7 @@ defmodule Ret.Scene do model_owned_file: old_model_owned_file, account: account } = scene - + new_scene_owned_file = Storage.create_new_owned_file_with_replaced_string( old_scene_owned_file, @@ -234,29 +234,30 @@ defmodule Ret.Scene do old_domain_url, new_domain_url ) - + {:ok, new_model_owned_file} = Storage.duplicate_and_transform(old_model_owned_file, account, fn glb_stream, _total_bytes -> GLTFUtils.replace_in_glb(glb_stream, old_domain_url, new_domain_url) end) - + scene |> change() |> put_change(:scene_owned_file_id, new_scene_owned_file.owned_file_id) |> put_change(:model_owned_file_id, new_model_owned_file.owned_file_id) |> Repo.update!() - + for old_owned_file <- [old_scene_owned_file, old_model_owned_file] do OwnedFile.set_inactive(old_owned_file) Storage.rm_files_for_owned_file(old_owned_file) Repo.delete(old_owned_file) end rescue - e -> + e -> IO.warn("Failed to process scene due to an error: #{inspect(e)}") end end) + :ok end) end diff --git a/lib/ret/storage.ex b/lib/ret/storage.ex index 05c064dcc..34dfe6e03 100644 --- a/lib/ret/storage.ex +++ b/lib/ret/storage.ex @@ -288,7 +288,7 @@ defmodule Ret.Storage do _account, _require_token ) do - {:ok, owned_file} + {:ok, owned_file} end # Promoting a stored file to being owned has two side effects: the file is moved @@ -310,7 +310,11 @@ defmodule Ret.Storage do "promotion_token" => actual_promotion_token } <- File.read!(meta_file_path) |> Poison.decode!(), - {:ok} <- (if require_token, do: check_promotion_token(actual_promotion_token, promotion_token), else: {:ok}), + {:ok} <- + if(require_token, + do: check_promotion_token(actual_promotion_token, promotion_token), + else: {:ok} + ), {:ok} <- check_blob_file_key(blob_file_path, key) ) do owned_file_params = %{ diff --git a/lib/ret_web/controllers/api/v1/media_search_controller.ex b/lib/ret_web/controllers/api/v1/media_search_controller.ex index f21ca1460..5e6db6ae7 100644 --- a/lib/ret_web/controllers/api/v1/media_search_controller.ex +++ b/lib/ret_web/controllers/api/v1/media_search_controller.ex @@ -3,25 +3,25 @@ defmodule RetWeb.Api.V1.MediaSearchController do use Retry def index(conn, %{"source" => source, "filter" => "created", "user" => user} = params) - when source in ["rooms"] do - account = conn |> Guardian.Plug.current_resource() - - if account && account.account_id == String.to_integer(user) do - {:commit, results} = - %Ret.MediaSearchQuery{ - source: "rooms", - cursor: params["cursor"] || "1", - user: account.account_id, - filter: "created", - q: params["q"] - } - |> Ret.MediaSearch.search() + when source in ["rooms"] do + account = conn |> Guardian.Plug.current_resource() - conn |> render("index.json", results: results) - else - conn |> send_resp(401, "You can only search created rooms for your own account.") - end + if account && account.account_id == String.to_integer(user) do + {:commit, results} = + %Ret.MediaSearchQuery{ + source: "rooms", + cursor: params["cursor"] || "1", + user: account.account_id, + filter: "created", + q: params["q"] + } + |> Ret.MediaSearch.search() + + conn |> render("index.json", results: results) + else + conn |> send_resp(401, "You can only search created rooms for your own account.") end + end def index(conn, %{"source" => "rooms"} = params) do {:commit, results} = @@ -98,6 +98,7 @@ defmodule RetWeb.Api.V1.MediaSearchController do def index(conn, %{"source" => source} = params) when source in [ "sketchfab", + "icosa", "tenor", "youtube_videos", "bing_videos", diff --git a/lib/ret_web/controllers/file_controller.ex b/lib/ret_web/controllers/file_controller.ex index 6b6d8ca20..a61efff1f 100644 --- a/lib/ret_web/controllers/file_controller.ex +++ b/lib/ret_web/controllers/file_controller.ex @@ -26,7 +26,9 @@ defmodule RetWeb.FileController do app_name = AppConfig.get_cached_config_value("translations|en|app-full-name") || - AppConfig.get_cached_config_value("translations|en|app-name") || RetWeb.Endpoint.host() + AppConfig.get_cached_config_value("translations|en|app-name") || + RetWeb.Endpoint.host() + title = "Photo taken in #{app_name} immersive space" config = AppConfig.get_config() @@ -36,9 +38,11 @@ defmodule RetWeb.FileController do content_type: content_type |> RetWeb.ContentType.sanitize_content_type(), content_length: content_length, title: title, - description_social_media: Ret.HttpUtils.join_smart([ - config["translations"]["en"]["app-tagline"], - "powered by Hubs"]), + description_social_media: + Ret.HttpUtils.join_smart([ + config["translations"]["en"]["app-tagline"], + "powered by Hubs" + ]), translations: config["translations"]["en"], app_name: app_name, images: config["images"], diff --git a/lib/ret_web/controllers/page_controller.ex b/lib/ret_web/controllers/page_controller.ex index 141a277bb..028649b19 100644 --- a/lib/ret_web/controllers/page_controller.ex +++ b/lib/ret_web/controllers/page_controller.ex @@ -18,6 +18,7 @@ defmodule RetWeb.PageController do alias Plug.Conn import Ret.ConnUtils import Ret.HttpUtils + require Logger ## # NOTE: In addition to adding a route, you must add static html pages to the page_origin_warmer.ex @@ -64,19 +65,24 @@ defmodule RetWeb.PageController do defp render_scene_content(%t{} = scene, conn) when t in [Scene, SceneListing] do {app_config, app_config_script} = generate_app_config() - app_name = app_config["translations"]["en"]["app-full-name"] || app_config["translations"]["en"]["app-name"] || RetWeb.Endpoint.host() + app_name = + app_config["translations"]["en"]["app-full-name"] || + app_config["translations"]["en"]["app-name"] || RetWeb.Endpoint.host() + scene_meta_tags = Phoenix.View.render_to_string(RetWeb.PageView, "scene-meta.html", scene: scene, ret_meta: Ret.Meta.get_meta(include_repo: false), translations: app_config["translations"]["en"], - title: join_smart([ scene.name, app_name ]), + title: join_smart([scene.name, app_name]), name: scene.name, - description: join_smart([ - scene.description, - "A scene you can use in the #{app_name} immersive spaces and others powered by Hubs", - app_config["translations"]["en"]["app-description"] - ]) |> String.replace("\\n", " "), + description: + join_smart([ + scene.description, + "A scene you can use in the #{app_name} immersive spaces and others powered by Hubs", + app_config["translations"]["en"]["app-description"] + ]) + |> String.replace("\\n", " "), app_config_script: {:safe, app_config_script |> with_script_tags}, extra_script: {:safe, get_extra_script(:scene) |> with_script_tags}, extra_html: {:safe, get_extra_html(:scene) || ""} @@ -105,13 +111,17 @@ defmodule RetWeb.PageController do defp render_avatar_content(%t{} = avatar, conn) when t in [Avatar, AvatarListing] do {app_config, app_config_script} = generate_app_config() - app_name = app_config["translations"]["en"]["app-full-name"] || app_config["translations"]["en"]["app-name"] || RetWeb.Endpoint.host() + app_name = + app_config["translations"]["en"]["app-full-name"] || + app_config["translations"]["en"]["app-name"] || RetWeb.Endpoint.host() + avatar_meta_tags = Phoenix.View.render_to_string(RetWeb.PageView, "avatar-meta.html", avatar: avatar, title: join_smart([avatar.name, app_name]), name: avatar.name, - description: "An avatar you can use in the #{app_name} immersive spaces and others powered by Hubs.", + description: + "An avatar you can use in the #{app_name} immersive spaces and others powered by Hubs.", ret_meta: Ret.Meta.get_meta(include_repo: false), translations: app_config["translations"]["en"], root_url: RetWeb.Endpoint.url(), @@ -143,7 +153,9 @@ defmodule RetWeb.PageController do defp render_homepage_content(conn, nil = _public_room_id) do {app_config, app_config_script} = generate_app_config() - app_name = app_config["translations"]["en"]["app-full-name"] || app_config["translations"]["en"]["app-name"] || RetWeb.Endpoint.host() + app_name = + app_config["translations"]["en"]["app-full-name"] || + app_config["translations"]["en"]["app-name"] || RetWeb.Endpoint.host() index_meta_tags = Phoenix.View.render_to_string( @@ -153,10 +165,11 @@ defmodule RetWeb.PageController do translations: app_config["translations"]["en"], app_name: app_name, title: join_smart([app_name, app_config["translations"]["en"]["app-tagline"]]), - description: join_smart( - [app_config["translations"]["en"]["app-description"], - "Immersive spaces, right in your browser, powered by Hubs" - ]), + description: + join_smart([ + app_config["translations"]["en"]["app-description"], + "Immersive spaces, right in your browser, powered by Hubs" + ]), images: app_config["images"], app_config_script: {:safe, app_config_script |> with_script_tags}, extra_script: {:safe, get_extra_script(:index) |> with_script_tags}, @@ -290,11 +303,14 @@ defmodule RetWeb.PageController do manifest = Phoenix.View.render_to_string(RetWeb.PageView, "manifest.webmanifest", root_url: RetWeb.Endpoint.url(), - app_name: get_app_config_value("translations|en|app-name") || RetWeb.Endpoint.host(), - app_description: join_smart([ - get_app_config_value("translations|en|app-description"), - "Immersive spaces, right in your browser, powered by Hubs" - ]) |> String.replace("\\n", " ") + app_name: + get_app_config_value("translations|en|app-name") || RetWeb.Endpoint.host(), + app_description: + join_smart([ + get_app_config_value("translations|en|app-description"), + "Immersive spaces, right in your browser, powered by Hubs" + ]) + |> String.replace("\\n", " ") ) unless module_config(:skip_cache) do @@ -483,12 +499,22 @@ defmodule RetWeb.PageController do app_name = app_config["translations"]["en"]["app-name"] || RetWeb.Endpoint.host() scene = hub.scene || hub.scene_listing - name = cond do - hub.name && scene && scene.name && scene.name != hub.name -> join_smart([hub.name, scene.name]) - hub.name -> hub.name - scene && scene.name -> scene.name - true -> "a room on " <> app_name - end + + name = + cond do + hub.name && scene && scene.name && scene.name != hub.name -> + join_smart([hub.name, scene.name]) + + hub.name -> + hub.name + + scene && scene.name -> + scene.name + + true -> + "a room on " <> app_name + end + hub_meta_tags = Phoenix.View.render_to_string(RetWeb.PageView, "hub-meta.html", hub: hub, @@ -498,18 +524,22 @@ defmodule RetWeb.PageController do translations: app_config["translations"]["en"], title: join_smart([hub.name, app_name]), name: name, - description: join_smart([ - hub.description, - "an immersive space in #{app_name}, right in your browser", - app_config["translations"]["en"]["app-description"], - "powered by Hubs." - ]) |> String.replace("\\n", " "), - description_social_media: join_smart([ - "Join others in an immersive space in #{app_name}, right in your browser", - hub.description, - app_config["translations"]["en"]["app-description"], - "powered by Hubs." - ]) |> String.replace("\\n", " "), + description: + join_smart([ + hub.description, + "an immersive space in #{app_name}, right in your browser", + app_config["translations"]["en"]["app-description"], + "powered by Hubs." + ]) + |> String.replace("\\n", " "), + description_social_media: + join_smart([ + "Join others in an immersive space in #{app_name}, right in your browser", + hub.description, + app_config["translations"]["en"]["app-description"], + "powered by Hubs." + ]) + |> String.replace("\\n", " "), app_config_script: {:safe, app_config_script |> with_script_tags}, extra_script: {:safe, get_extra_script(:room) |> with_script_tags}, extra_html: {:safe, get_extra_html(:room) || ""} @@ -716,17 +746,23 @@ defmodule RetWeb.PageController do do: cors_proxy(conn, "#{url}?#{qs}") defp cors_proxy(conn, url) do + cors_proxy_with_redirects(conn, url, 0) + end + + defp cors_proxy_with_redirects(conn, url, redirect_count) when redirect_count > 5 do + Logger.error("CORS Proxy: Too many redirects (#{redirect_count}) for URL: #{url}") + conn |> send_resp(400, "Too many redirects") + end + + defp cors_proxy_with_redirects(conn, url, redirect_count) do %URI{authority: authority, host: host} = uri = URI.parse(url) resolved_ip = HttpUtils.resolve_ip(host) if HttpUtils.internal_ip?(resolved_ip) do + Logger.warning("CORS Proxy: Blocking internal IP #{inspect(resolved_ip)} for host #{host}") conn |> send_resp(401, "Bad request.") else - # We want to ensure that the URL we request hits the same IP that we verified above, - # so we replace the host with the IP address here and use this url to make the proxy request. - ip_url = URI.to_string(HttpUtils.replace_host(uri, resolved_ip)) - # Disallow CORS proxying unless request was made to the cors proxy url cors_proxy_url = Application.get_env(:ret, RetWeb.Endpoint)[:cors_proxy_url] @@ -745,43 +781,105 @@ defmodule RetWeb.PageController do if is_cors_proxy_url do allowed_origins = Application.get_env(:ret, RetWeb.Endpoint)[:allowed_origins] |> String.split(",") + opts = ReverseProxyPlug.init( upstream: url, allowed_origins: allowed_origins, proxy_url: "#{cors_scheme}://#{cors_host}:#{cors_port}", - # Since we replaced the host with the IP address in ip_url above, we need to force the host + # We need to force the host # used for ssl verification here so that the connection isn't rejected. # Note that we have to convert the authority to a charlist, since this uses Erlang's `ssl` module # internally, which expects a charlist. client_options: [ - ssl: [{:server_name_indication, to_charlist(authority)}, {:versions, [:"tlsv1.2",:"tlsv1.3"]}] - ], - # preserve_host_header: true + ssl: [ + {:server_name_indication, to_charlist(authority)}, + {:versions, [:"tlsv1.2", :"tlsv1.3"]} + ] + ] ) body = ReverseProxyPlug.read_body(conn) is_head = conn |> Conn.get_req_header("x-original-method") == ["HEAD"] - %Conn{}|> Map.merge(conn) - |> Map.put( - :method, - if is_head do - "HEAD" - else - conn.method + try do + # First, make a HEAD request to check for redirects using HTTPoison + case HTTPoison.head(url, [], + follow_redirect: false, + ssl: [ + {:server_name_indication, to_charlist(authority)}, + {:versions, [:"tlsv1.2", :"tlsv1.3"]} + ], + timeout: 15_000, + recv_timeout: 15_000 + ) do + {:ok, %HTTPoison.Response{status_code: status_code, headers: headers}} + when status_code in [301, 302, 303, 307, 308] -> + # Found a redirect + location_header = + headers + |> Enum.find(fn {k, _v} -> String.downcase(k) == "location" end) + |> elem(1) + + if location_header do + # Resolve relative URLs against the current URL + redirect_url = URI.merge(uri, location_header) |> URI.to_string() + cors_proxy_with_redirects(conn, redirect_url, redirect_count + 1) + else + Logger.warning("CORS Proxy: Redirect response missing location header") + # Fall back to ReverseProxyPlug for this response + make_reverse_proxy_request(conn, url, body, is_head, opts) + end + + {:ok, %HTTPoison.Response{status_code: status_code}} -> + # Not a redirect, use ReverseProxyPlug for the actual request + make_reverse_proxy_request(conn, url, body, is_head, opts) + + {:error, reason} -> + Logger.error("CORS Proxy: HEAD request failed: #{inspect(reason)}") + # Fall back to ReverseProxyPlug anyway + make_reverse_proxy_request(conn, url, body, is_head, opts) end - ) - # Need to strip path_info since proxy plug reads it - |> Map.put(:path_info, []) - |> ReverseProxyPlug.request(body, opts) - |> ReverseProxyPlug.response(conn, opts) + rescue + error -> + Logger.error("CORS Proxy: Request failed with exception: #{inspect(error)}") + conn |> send_resp(500, "Proxy request failed: #{inspect(error)}") + catch + :exit, reason -> + Logger.error("CORS Proxy: Request exited with reason: #{inspect(reason)}") + conn |> send_resp(500, "Proxy request timed out or failed") + + kind, reason -> + Logger.error("CORS Proxy: Request failed with #{kind}: #{inspect(reason)}") + conn |> send_resp(500, "Proxy request failed") + end else + Logger.warning("CORS Proxy: Request rejected - invalid host or scheme") conn |> send_resp(401, "Bad request.") end end end + defp make_reverse_proxy_request(conn, _url, body, is_head, opts) do + proxy_conn = + %Conn{} + |> Map.merge(conn) + |> Map.put( + :method, + if is_head do + "HEAD" + else + conn.method + end + ) + # Need to strip path_info since proxy plug reads it + |> Map.put(:path_info, []) + |> ReverseProxyPlug.request(body, opts) + |> ReverseProxyPlug.response(conn, opts) + + proxy_conn + end + defp render_static_asset(conn) do static_options = Plug.Static.init(at: "/", from: :ret, gzip: true, brotli: true) Plug.Static.call(conn, static_options) diff --git a/lib/ret_web/email.ex b/lib/ret_web/email.ex index 0c2f6072b..7ba8b12d5 100644 --- a/lib/ret_web/email.ex +++ b/lib/ret_web/email.ex @@ -3,7 +3,9 @@ defmodule RetWeb.Email do alias Ret.{AppConfig} def auth_email(to_address, signin_args) do - app_name = AppConfig.get_cached_config_value("translations|en|app-name") || RetWeb.Endpoint.host() + app_name = + AppConfig.get_cached_config_value("translations|en|app-name") || RetWeb.Endpoint.host() + app_full_name = AppConfig.get_cached_config_value("translations|en|app-full-name") || app_name admin_email = Application.get_env(:ret, Ret.Account)[:admin_email] custom_login_subject = AppConfig.get_cached_config_value("auth|login_subject") diff --git a/priv/repo/migrations/20190114232257_add_reviewed_at_to_scenes.exs b/priv/repo/migrations/20190114232257_add_reviewed_at_to_scenes.exs index 8e5a26da1..ea1707548 100644 --- a/priv/repo/migrations/20190114232257_add_reviewed_at_to_scenes.exs +++ b/priv/repo/migrations/20190114232257_add_reviewed_at_to_scenes.exs @@ -6,6 +6,8 @@ defmodule Ret.Repo.Migrations.AddReviewedAtToScenes do add :reviewed_at, :utc_datetime, null: true end - create index(:scenes, [:reviewed_at], where: "reviewed_at is null or reviewed_at < updated_at") + create index(:scenes, [:reviewed_at], + where: "reviewed_at is null or reviewed_at < updated_at" + ) end end diff --git a/test/ret_web/channels/entity_test.exs b/test/ret_web/channels/entity_test.exs index c18d1e28a..0cbc6b556 100644 --- a/test/ret_web/channels/entity_test.exs +++ b/test/ret_web/channels/entity_test.exs @@ -8,7 +8,9 @@ defmodule RetWeb.EntityTest do @payload_save_entity_state read_json("save_entity_state_payload.json") @payload_save_entity_state_2 read_json("save_entity_state_payload_2.json") - @payload_save_entity_state_promotable_no_token read_json("save_entity_state_payload_promotable_no_token.json") + @payload_save_entity_state_promotable_no_token read_json( + "save_entity_state_payload_promotable_no_token.json" + ) @payload_save_entity_state_promotable read_json("save_entity_state_payload_promotable.json") @payload_save_entity_state_unpromotable read_json("save_entity_state_payload_unpromotable.json") @payload_update_entity_state read_json("update_entity_state_payload.json") diff --git a/test/support/test_helpers.ex b/test/support/test_helpers.ex index eb00c8483..77df6036b 100644 --- a/test/support/test_helpers.ex +++ b/test/support/test_helpers.ex @@ -216,7 +216,11 @@ defmodule Ret.TestHelpers do {:ok, asset: asset} end - def create_project_asset(%{account: account, project: project, thumbnail_owned_file: owned_file}) do + def create_project_asset(%{ + account: account, + project: project, + thumbnail_owned_file: owned_file + }) do {:ok, asset} = %Asset{} |> Asset.changeset(account, owned_file, owned_file, %{