add llm

Author: zcesur

lib/algora/admin/admin.ex

@@ -24,6 +24,154 @@ defmodule Algora.Admin do
 
   require Logger
 
+  defmodule JobPostingPrediction do
+    @moduledoc false
+    use Ecto.Schema
+    use Instructor.Validator
+
+    alias Algora.Organizations
+
+    @llm_doc """
+    ## Field Descriptions:
+    - tech_stack: List of technologies used in the job posting (e.g. ["Ruby", "Rails", "PostgreSQL"])
+    - countries: List of 2-letter ISO country codes (e.g. ["US", "CA"])
+    - regions: List of regions (e.g. ["EMEA", "LATAM"])
+    - location: Location of the job posting (e.g. "Remote", "San Francisco, CA", "London/Berlin")
+    - seniority: Seniority level (e.g. "Senior", "Mid-Senior", "Entry-Level")
+    - company_url: Company website URL which can be derived from email (e.g. example.com)
+    """
+    @primary_key false
+    embedded_schema do
+      embeds_many :job_postings, JobPosting, primary_key: false do
+        field(:title, :string)
+        field(:description, :string)
+        field(:tech_stack, {:array, :string})
+        field(:company_name, :string)
+        field(:company_url, :string)
+        field(:location, :string)
+        field(:countries, {:array, :string})
+        field(:regions, {:array, :string})
+        field(:compensation, :string)
+        field(:seniority, :string)
+      end
+    end
+
+    @impl true
+    def validate_changeset(changeset) do
+      changeset
+    end
+
+    def seed_jobs(jobs) do
+      jobs
+      |> Task.async_stream(&seed/1, timeout: :infinity, max_concurrency: 50)
+      |> Enum.to_list()
+    end
+
+    def seed(job) do
+      with domain when not is_nil(domain) <- to_domain(job.company_url),
+           {:ok, org} <- fetch_or_create_user(domain, %{hiring: true, tech_stack: job.tech_stack}),
+           {:ok, org} <-
+             org
+             |> change(
+               Map.merge(
+                 %{
+                   domain: org.domain || domain,
+                   hiring_subscription: :trial,
+                   billing_name: org.billing_name || job.company_name,
+                   billing_address: org.billing_address || job.location,
+                   executive_name: org.executive_name || job.company_name,
+                   executive_role: org.executive_role || job.seniority
+                 },
+                 if org.handle do
+                   %{}
+                 else
+                   %{handle: Organizations.ensure_unique_org_handle(job.company_name)}
+                 end
+               )
+             )
+             |> Repo.update() do
+        Repo.insert(%JobPosting{
+          status: :processing,
+          id: Nanoid.generate(),
+          user_id: org.id,
+          company_name: org.name,
+          company_url: org.website_url,
+          title: job.title,
+          description: job.description,
+          tech_stack: job.tech_stack,
+          location: job.location,
+          compensation: job.compensation,
+          seniority: job.seniority,
+          countries: job.countries,
+          regions: job.regions
+        })
+      end
+    end
+
+    defp to_domain(nil), do: nil
+
+    defp to_domain(url) do
+      url
+      |> String.trim_leading("https://")
+      |> String.trim_leading("http://")
+      |> String.trim_leading("www.")
+    end
+
+    def fetch_or_create_user(domain, opts) do
+      case Repo.one(from o in User, where: o.domain == ^domain, limit: 1) do
+        %User{} = user ->
+          {:ok, user}
+
+        _ ->
+          res = Organizations.onboard_organization_from_domain(domain, opts)
+          res
+      end
+    end
+  end
+
+  def classify_jobs(jobs) do
+    batches = Enum.chunk_every(jobs, 10)
+
+    batches
+    |> Enum.with_index()
+    |> Enum.flat_map(fn {jobs, index} ->
+      case classify_batch(jobs, index) do
+        {:ok, predictions} ->
+          predictions.job_postings
+
+        {:error, error} ->
+          Logger.error("Failed to classify batch #{index}: #{inspect(error)}")
+          []
+      end
+    end)
+  end
+
+  def classify_batch(jobs, index) do
+    text = Enum.join(jobs, "\n\n")
+
+    Github.Client.run_cached("classify_jobs_#{index}", fn ->
+      Instructor.chat_completion(
+        model: "gpt-4o-mini",
+        response_model: JobPostingPrediction,
+        max_retries: 2,
+        messages: [
+          %{
+            role: "user",
+            content: """
+            Your purpose is to turn arbitrary job postings into structured data.
+
+            Return a distinct entry for each job. Some paragraphs may contain multiple jobs.
+
+            Turn following job postings into structured data:
+
+            #{text}
+            """
+          }
+        ]
+      )
+    end)
+  end
+
   def seed_job(opts \\ %{}) do
     with {:ok, user} <- Repo.fetch_by(User, handle: opts.org.handle),
          {:ok, user} <- user |> change(opts.org) |> Repo.update(),

lib/algora/organizations/organizations.ex

@@ -163,6 +163,8 @@ defmodule Algora.Organizations do
             org -> org.handle
           end
 
+        dbg(params)
+
         case org do
           nil ->
             %User{type: :organization}

lib/algora_web/live/hn_jobs_live.ex

@@ -21,7 +21,7 @@ defmodule AlgoraWeb.HNJobsLive do
   def mount(_params, _session, socket) do
     jobs =
       JobPosting
-      |> where([j], j.id in ^Settings.get_hn_job_ids())
+      # |> where([j], j.id in ^Settings.get_hn_job_ids())
       |> order_by([j], desc: j.inserted_at)
       |> Repo.all()
       |> Repo.preload(:user)

mix.exs

@@ -99,6 +99,7 @@ defmodule Algora.MixProject do
       {:sobelow, "~> 0.13", only: [:dev, :test], runtime: false},
       {:cmark, "~> 0.10"},
       {:csv, "~> 3.2"},
+      {:instructor, "~> 0.1.0"},
       # ex_aws
       {:ex_aws, "~> 2.1"},
       {:ex_aws_s3, "~> 2.0"},

mix.lock

@@ -46,7 +46,9 @@
   "hpax": {:hex, :hpax, "1.0.2", "762df951b0c399ff67cc57c3995ec3cf46d696e41f0bba17da0518d94acd4aac", [:mix], [], "hexpm", "2f09b4c1074e0abd846747329eaa26d535be0eb3d189fa69d812bfb8bfefd32f"},
   "httpoison": {:hex, :httpoison, "2.2.1", "87b7ed6d95db0389f7df02779644171d7319d319178f6680438167d7b69b1f3d", [:mix], [{:hackney, "~> 1.17", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "51364e6d2f429d80e14fe4b5f8e39719cacd03eb3f9a9286e61e216feac2d2df"},
   "idna": {:hex, :idna, "6.1.1", "8a63070e9f7d0c62eb9d9fcb360a7de382448200fbbd1b106cc96d3d8099df8d", [:rebar3], [{:unicode_util_compat, "~> 0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "92376eb7894412ed19ac475e4a86f7b413c1b9fbb5bd16dccd57934157944cea"},
+  "instructor": {:hex, :instructor, "0.1.0", "ca587fa11b9de7dff68b6f0a28ee17682d35f67efa20f71aef61bbb528444562", [:mix], [{:ecto, "~> 3.12", [hex: :ecto, repo: "hexpm", optional: false]}, {:jason, "~> 1.4.0", [hex: :jason, repo: "hexpm", optional: false]}, {:jaxon, "~> 2.0", [hex: :jaxon, repo: "hexpm", optional: false]}, {:req, "~> 0.5 or ~> 1.0", [hex: :req, repo: "hexpm", optional: false]}], "hexpm", "05a7020a460ca43dc1123c7903e928b678360cd37eac761f472bd8e0787fefcb"},
   "jason": {:hex, :jason, "1.4.4", "b9226785a9aa77b6857ca22832cffa5d5011a667207eb2a0ad56adb5db443b8a", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "c5eb0cab91f094599f94d55bc63409236a8ec69a21a67814529e8d5f6cc90b3b"},
+  "jaxon": {:hex, :jaxon, "2.0.8", "00951a79d354260e28d7e36f956c3de94818124768a4b22e0fc55559d1b3bfe7", [:make, :mix], [{:elixir_make, "~> 0.4", [hex: :elixir_make, repo: "hexpm", optional: false]}], "hexpm", "74532853b1126609615ea98f0ceb5009e70465ca98027afbbd8ed314d887e82d"},
   "joken": {:hex, :joken, "2.6.2", "5daaf82259ca603af4f0b065475099ada1b2b849ff140ccd37f4b6828ca6892a", [:mix], [{:jose, "~> 1.11.10", [hex: :jose, repo: "hexpm", optional: false]}], "hexpm", "5134b5b0a6e37494e46dbf9e4dad53808e5e787904b7c73972651b51cce3d72b"},
   "jose": {:hex, :jose, "1.11.10", "a903f5227417bd2a08c8a00a0cbcc458118be84480955e8d251297a425723f83", [:mix, :rebar3], [], "hexpm", "0d6cd36ff8ba174db29148fc112b5842186b68a90ce9fc2b3ec3afe76593e614"},
   "live_svelte": {:hex, :live_svelte, "0.14.1", "78fcd3bb7eb1c349138ebcaef5b61653bf1a818e42129c847482717895af8f70", [:mix], [{:jason, "~> 1.2", [hex: :jason, repo: "hexpm", optional: false]}, {:nodejs, "~> 3.1", [hex: :nodejs, repo: "hexpm", optional: false]}, {:phoenix, ">= 1.7.0", [hex: :phoenix, repo: "hexpm", optional: false]}, {:phoenix_html, ">= 3.3.1", [hex: :phoenix_html, repo: "hexpm", optional: false]}, {:phoenix_live_view, ">= 0.18.0", [hex: :phoenix_live_view, repo: "hexpm", optional: false]}], "hexpm", "f96b06456957fbc6d25f71fc984f8f949664d5b61fbfcb3db43d5b82699c39a8"},
@@ -82,6 +84,7 @@
   "postgrex": {:hex, :postgrex, "0.20.0", "363ed03ab4757f6bc47942eff7720640795eb557e1935951c1626f0d303a3aed", [:mix], [{:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "d36ef8b36f323d29505314f704e21a1a038e2dc387c6409ee0cd24144e187c0f"},
   "ranch": {:hex, :ranch, "2.2.0", "25528f82bc8d7c6152c57666ca99ec716510fe0925cb188172f41ce93117b1b0", [:make, :rebar3], [], "hexpm", "fa0b99a1780c80218a4197a59ea8d3bdae32fbff7e88527d7d8a4787eff4f8e7"},
   "redirect": {:hex, :redirect, "0.4.0", "98b46053504ee517bc3ad2fd04c064b64b48d339e1e18266355b30c4f8bb52b0", [:mix], [{:phoenix, "~> 1.4", [hex: :phoenix, repo: "hexpm", optional: false]}, {:plug, "~> 1.8.3 or ~> 1.9", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "dfa29a8ecbad066ed0b73b34611cf24c78101719737f37bdf750f39197d67b97"},
+  "req": {:hex, :req, "0.5.10", "a3a063eab8b7510785a467f03d30a8d95f66f5c3d9495be3474b61459c54376c", [:mix], [{:brotli, "~> 0.3.1", [hex: :brotli, repo: "hexpm", optional: true]}, {:ezstd, "~> 1.0", [hex: :ezstd, repo: "hexpm", optional: true]}, {:finch, "~> 0.17", [hex: :finch, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mime, "~> 2.0.6 or ~> 2.1", [hex: :mime, repo: "hexpm", optional: false]}, {:nimble_csv, "~> 1.0", [hex: :nimble_csv, repo: "hexpm", optional: true]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "8a604815743f8a2d3b5de0659fa3137fa4b1cffd636ecb69b30b2b9b2c2559be"},
   "reverse_proxy_plug": {:hex, :reverse_proxy_plug, "3.0.2", "38fde2f59bca8b219ef4f1ec0c0849a67c6d9705160e426a2354f35399db5c7b", [:mix], [{:finch, "~> 0.18", [hex: :finch, repo: "hexpm", optional: true]}, {:httpoison, "~> 1.2 or ~> 2.0", [hex: :httpoison, repo: "hexpm", optional: true]}, {:plug, "~> 1.6", [hex: :plug, repo: "hexpm", optional: false]}, {:req, "~> 0.3.0 or ~> 0.4.0 or ~> 0.5.0", [hex: :req, repo: "hexpm", optional: true]}, {:tesla, "~> 1.4", [hex: :tesla, repo: "hexpm", optional: true]}], "hexpm", "31ae5e068f7f504fba1b5c17c31c87966c720809ac15140c6c181440fbd24eda"},
   "rustler": {:hex, :rustler, "0.36.1", "2d4b1ff57ea2789a44756a40dbb5fbb73c6ee0a13d031dcba96d0a5542598a6a", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:toml, "~> 0.7", [hex: :toml, repo: "hexpm", optional: false]}], "hexpm", "f3fba4ad272970e0d1bc62972fc4a99809651e54a125c5242de9bad4574b2d02"},
   "rustler_precompiled": {:hex, :rustler_precompiled, "0.8.2", "5f25cbe220a8fac3e7ad62e6f950fcdca5a5a5f8501835d2823e8c74bf4268d5", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: false]}, {:rustler, "~> 0.23", [hex: :rustler, repo: "hexpm", optional: true]}], "hexpm", "63d1bd5f8e23096d1ff851839923162096364bac8656a4a3c00d1fff8e83ee0a"},

priv/repo/migrations/20250504200111_update_job_posting_description.exs

@@ -0,0 +1,26 @@
+defmodule Algora.Repo.Migrations.UpdateJobPostingDescription do
+  use Ecto.Migration
+
+  def up do
+    alter table(:job_postings) do
+      modify :description, :text
+      modify :company_name, :text
+      modify :company_url, :text
+    end
+
+    alter table(:users) do
+      modify :linkedin_url, :text
+      modify :website_url, :text
+      modify :discord_url, :text
+      modify :github_url, :text
+      modify :twitter_url, :text
+      modify :youtube_url, :text
+      modify :slack_url, :text
+      modify :twitch_url, :text
+      modify :og_image_url, :text
+    end
+  end
+
+  def down do
+  end
+end