feat: media matching (#18)

- add blake2b and conversation aggregates; 
- add some unit tests
- add genserver for rabbitmq
- add sentry for exception reporting

Author: dennyabrain

.formatter.exs

@@ -2,5 +2,5 @@
   import_deps: [:ecto, :ecto_sql, :phoenix],
   subdirectories: ["priv/*/migrations"],
   plugins: [Phoenix.LiveView.HTMLFormatter],
-  inputs: ["*.{heex,ex,exs}", "{config,lib,test}/**/*.{heex,ex,exs}", "priv/*/seeds.exs"]
+  inputs: ["*.{heex,ex,exs}", "{config,lib,test}/**/*.{heex,ex,exs}", "priv/*/*.exs"]
 ]

.iex.exs

@@ -5,3 +5,11 @@ import Ecto.Query
 alias DAU.Feed
 alias DAU.Feed.Common
 alias DAU.UserMessage.Inbox
+
+alias DAU.MediaMatch.{
+  Hash,
+  HashMeta,
+  HashWorkerGenServer,
+  HashWorkerRequest,
+  HashWorkerResponse
+}

config/config.exs

@@ -10,7 +10,8 @@ import Config
 config :dau,
   namespace: DAU,
   ecto_repos: [DAU.Repo],
-  generators: [timestamp_type: :utc_datetime]
+  generators: [timestamp_type: :utc_datetime],
+  env: config_env()
 
 # Configures the endpoint
 config :dau, DAUWeb.Endpoint,
@@ -62,6 +63,15 @@ config :logger, :console,
 # Use Jason for JSON parsing in Phoenix
 config :phoenix, :json_library, Jason
 
+config :sentry,
+  dsn:
+    "https://38b2487d72b7a30d5bf6ca606dfd5c25@o4507162180976640.ingest.de.sentry.io/4507163464237136",
+  environment_name: Mix.env(),
+  enable_source_code_context: true,
+  root_source_code_paths: [File.cwd!()]
+
+config :dau, AWSS3, file_prefix: "temp"
+
 # Import environment specific config. This must remain at the bottom
 # of this file so it overrides the configuration defined above.
 import_config "#{config_env()}.exs"

config/dev.exs

@@ -85,4 +85,4 @@ config :ex_aws,
   secret_access_key: {:system, "AWS_SECRET_ACCESS_KEY"},
   region: "ap-south-1"
 
-config :dau, RabbitMQ, url: "amqp://admin:Admin123@localhost"
+config :dau, AWSS3, file_prefix: "app-data"

config/prod.exs

@@ -18,3 +18,5 @@ config :logger, level: :info
 
 # Runtime production configuration, including reading
 # of environment variables, is done on config/runtime.exs.
+
+config :dau, AWSS3, file_prefix: "app-data"

config/runtime.exs

@@ -83,6 +83,14 @@ config :ex_aws,
   access_key_id: {:system, "AWS_ACCESS_KEY_ID"},
   secret_access_key: {:system, "AWS_SECRET_ACCESS_KEY"}
 
+rabbit_mq_url =
+  System.get_env("RABBITMQ_URL") ||
+    raise """
+    Slack webhook url is missing. Please contact tattle admin
+    """
+
+config :dau, RabbitMQ, url: rabbit_mq_url
+
 if config_env() == :prod do
   # database_url =
   #   System.get_env("DATABASE_URL") ||

config/test.exs

@@ -34,3 +34,5 @@ config :logger, level: :warning
 
 # Initialize plugs at runtime for faster test compilation
 config :phoenix, :plug_init_mode, :runtime
+
+config :dau, RabbitMQ, url: "amqp://admin:Admin123@localhost"

docker-compose.yml

@@ -21,3 +21,44 @@ services:
     restart: always
     ports:
       - 8080:8080
+
+  queue:
+    image: rabbitmq@sha256:e2505f78d58dca8c372cde3930e4d6bee5e02ac9001ce85ece5a11df606c1fa3 # rabbitmq:3.12.12-management
+    container_name: rabbitmq
+    hostname: rabbitmq
+    volumes:
+      - ./.docker/rabbitmq/data:/var/lib/rabbitmq
+      - ./.docker/rabbitmq/logs:/var/log/rabbitmq
+    environment:
+      RABBITMQ_ERLANG_COOKIE: "secret-cookie"
+      RABBITMQ_DEFAULT_USER: "admin"
+      RABBITMQ_DEFAULT_PASS: "Admin123"
+    ports:
+      - 5672:5672
+      - 15672:15672
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:15672"]
+      interval: 30s
+      timeout: 10s
+      retries: 5
+
+  hash_worker:
+    image: tattletech/feluda-operator-hash:worker-amd64-latest
+    container_name: hash_worker
+    depends_on:
+      - queue
+    environment:
+      MQ_USERNAME: admin
+      MQ_PASSWORD: Admin123
+      MQ_HOST: rabbitmq
+      AWS_ACCESS_KEY_ID: AKIAZEOQEMFTJS244746
+      AWS_SECRET_ACCESS_KEY: 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
+      AWS_BUCKET: staging.dau.tattle.co.in
+      AWS_REGION: ap-south-1
+    command:
+      - /bin/sh
+      - -c
+      - |
+        echo "sleep for 10sec"
+        sleep 10 
+        python -m worker.hash.hash_worker

lib/aws_s3.ex

@@ -1,7 +1,9 @@
 defmodule AWSS3 do
+  require Logger
   alias ExAws.S3
 
   @opts [query_params: [{"x-amz-acl", "public-read"}]]
+  @file_prefix Application.compile_env(:dau, [AWSS3, :file_prefix])
 
   @doc """
   Get a short lived URL to file on S3
@@ -28,7 +30,7 @@ defmodule AWSS3 do
   def upload_to_s3(url) do
     {:ok, _, _headers, ref} = :hackney.get(url)
     {:ok, body} = :hackney.body(ref)
-    file_key = "app-data/#{UUID.uuid4()}"
+    file_key = "#{@file_prefix}/#{UUID.uuid4()}"
     file_hash = :crypto.hash(:sha256, body) |> Base.encode16() |> String.downcase()
     S3.put_object("staging.dau.tattle.co.in", file_key, body) |> ExAws.request!()
     {file_key, file_hash}

lib/dau/application.ex

@@ -7,13 +7,18 @@ defmodule DAU.Application do
 
   @impl true
   def start(_type, _args) do
+    :logger.add_handler(:my_sentry_handler, Sentry.LoggerHandler, %{
+      config: %{metadata: [:file, :line]}
+    })
+
     children = [
       DAUWeb.Telemetry,
       DAU.Repo,
       {DNSCluster, query: Application.get_env(:dau, :dns_cluster_query) || :ignore},
       {Phoenix.PubSub, name: DAU.PubSub},
       # Start the Finch HTTP client for sending emails
       {Finch, name: DAU.Finch},
+      DAU.MediaMatch.HashWorkerGenServer,
       # Start a worker by calling: DAU.Worker.start_link(arg)
       # {DAU.Worker, arg},
       # Start to serve requests, typically the last entry