Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion .sampo/changesets/heroic-bard-ilmarinen.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,4 +2,6 @@
hex/posthog: minor
---

Add LLMAnalytics (experimental)
Add LLM Analytics

This release introduces a lightweight LLM analytics toolkit for instrumenting, recording, and analyzing large language model usage across applications that use this repository's SDK. It provides practical observability for teams running LLMs in production and during development.
5 changes: 5 additions & 0 deletions .sampo/changesets/majestic-seer-goulven.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
hex/posthog: minor
---

Add `uuid` to events on every request to guarantee idempotency in the backend
1 change: 1 addition & 0 deletions lib/posthog.ex
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,7 @@ defmodule PostHog do
event = %{
event: event,
distinct_id: distinct_id,
uuid: UUIDv7.generate(),
timestamp: DateTime.utc_now() |> DateTime.to_iso8601(),
properties: properties
}
Expand Down
13 changes: 7 additions & 6 deletions test/integration_test.exs
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ defmodule PostHog.IntegrationTest do

require Logger

alias PostHog.Integrations.LLMAnalytics.Req, as: LLMReq
alias PostHog.LLMAnalytics

@moduletag integration: true
Expand Down Expand Up @@ -102,7 +103,7 @@ defmodule PostHog.IntegrationTest do

test "OpenAI Responses", %{wait_fun: wait} do
Req.new()
|> PostHog.Integrations.LLMAnalytics.Req.attach()
|> LLMReq.attach()
|> Req.post!(
url: "https://api.openai.com/v1/responses",
auth: {:bearer, Application.get_env(:posthog, :openai_key)},
Expand All @@ -117,7 +118,7 @@ defmodule PostHog.IntegrationTest do

test "OpenAI Chat Completions", %{wait_fun: wait} do
Req.new()
|> PostHog.Integrations.LLMAnalytics.Req.attach()
|> LLMReq.attach()
|> Req.post!(
url: "https://api.openai.com/v1/chat/completions",
auth: {:bearer, Application.get_env(:posthog, :openai_key)},
Expand All @@ -137,7 +138,7 @@ defmodule PostHog.IntegrationTest do

test "OpenAI Responses with tool", %{wait_fun: wait} do
Req.new()
|> PostHog.Integrations.LLMAnalytics.Req.attach()
|> LLMReq.attach()
|> Req.post!(
url: "https://api.openai.com/v1/responses",
auth: {:bearer, Application.get_env(:posthog, :openai_key)},
Expand Down Expand Up @@ -173,7 +174,7 @@ defmodule PostHog.IntegrationTest do

test "OpenAI Chat Completions with tool", %{wait_fun: wait} do
Req.new()
|> PostHog.Integrations.LLMAnalytics.Req.attach()
|> LLMReq.attach()
|> Req.post!(
url: "https://api.openai.com/v1/chat/completions",
auth: {:bearer, Application.get_env(:posthog, :openai_key)},
Expand Down Expand Up @@ -211,7 +212,7 @@ defmodule PostHog.IntegrationTest do

test "Gemini", %{wait_fun: wait} do
Req.new()
|> PostHog.Integrations.LLMAnalytics.Req.attach()
|> LLMReq.attach()
|> Req.post!(
url: "https://generativelanguage.googleapis.com/v1beta/models/{model}:generateContent",
headers: [{"x-goog-api-key", Application.get_env(:posthog, :gemini_key)}],
Expand All @@ -231,7 +232,7 @@ defmodule PostHog.IntegrationTest do

test "Gemini with tool", %{wait_fun: wait} do
Req.new()
|> PostHog.Integrations.LLMAnalytics.Req.attach()
|> LLMReq.attach()
|> Req.post!(
url: "https://generativelanguage.googleapis.com/v1beta/models/{model}:generateContent",
headers: [{"x-goog-api-key", Application.get_env(:posthog, :gemini_key)}],
Expand Down
10 changes: 10 additions & 0 deletions test/posthog/handler_test.exs
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ defmodule PostHog.HandlerTest do

assert %{
event: "$exception",
uuid: _,
distinct_id: "foo",
properties: %{
"$exception_list": [
Expand All @@ -45,6 +46,7 @@ defmodule PostHog.HandlerTest do

assert %{
event: "$exception",
uuid: _,
properties: %{
"$lib": "posthog-elixir",
"$lib_version": _,
Expand Down Expand Up @@ -83,6 +85,7 @@ defmodule PostHog.HandlerTest do

assert %{
event: "$exception",
uuid: _,
properties: %{
"$exception_list": [
%{
Expand All @@ -103,6 +106,7 @@ defmodule PostHog.HandlerTest do

assert %{
event: "$exception",
uuid: _,
properties: %{
"$exception_list": [
%{
Expand All @@ -123,6 +127,7 @@ defmodule PostHog.HandlerTest do

assert %{
event: "$exception",
uuid: _,
properties: %{
"$exception_list": [
%{
Expand All @@ -143,6 +148,7 @@ defmodule PostHog.HandlerTest do

assert %{
event: "$exception",
uuid: _,
properties: %{
"$exception_list": [
%{
Expand Down Expand Up @@ -189,6 +195,7 @@ defmodule PostHog.HandlerTest do

assert %{
event: "$exception",
uuid: _,
properties: %{
"$exception_list": [
%{
Expand All @@ -209,6 +216,7 @@ defmodule PostHog.HandlerTest do

assert %{
event: "$exception",
uuid: _,
properties: %{
"$exception_list": [
%{
Expand All @@ -229,6 +237,7 @@ defmodule PostHog.HandlerTest do

assert %{
event: "$exception",
uuid: _,
properties: %{
"$exception_list": [
%{
Expand All @@ -249,6 +258,7 @@ defmodule PostHog.HandlerTest do

assert %{
event: "$exception",
uuid: _,
properties: %{
"$exception_list": [
%{
Expand Down
6 changes: 6 additions & 0 deletions test/posthog/integrations/plug_test.exs
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ defmodule PostHog.Integrations.PlugTest do

assert %{
event: "$exception",
uuid: _,
properties: properties
} = event

Expand Down Expand Up @@ -79,6 +80,7 @@ defmodule PostHog.Integrations.PlugTest do

assert %{
event: "$exception",
uuid: _,
properties: properties
} = event

Expand Down Expand Up @@ -109,6 +111,7 @@ defmodule PostHog.Integrations.PlugTest do

assert %{
event: "$exception",
uuid: _,
properties: properties
} = event

Expand Down Expand Up @@ -139,6 +142,7 @@ defmodule PostHog.Integrations.PlugTest do

assert %{
event: "$exception",
uuid: _,
properties: properties
} = event

Expand Down Expand Up @@ -168,6 +172,7 @@ defmodule PostHog.Integrations.PlugTest do

assert %{
event: "$exception",
uuid: _,
properties: properties
} = event

Expand Down Expand Up @@ -197,6 +202,7 @@ defmodule PostHog.Integrations.PlugTest do

assert %{
event: "$exception",
uuid: _,
properties: properties
} = event

Expand Down
21 changes: 21 additions & 0 deletions test/posthog_test.exs
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ defmodule PostHogTest do

assert %{
event: "case tested",
uuid: _,
distinct_id: "distinct_id",
properties: %{},
timestamp: _
Expand All @@ -42,6 +43,7 @@ defmodule PostHogTest do

assert %{
event: "case tested",
uuid: _,
distinct_id: "distinct_id",
properties: %{foo: "bar"},
timestamp: _
Expand All @@ -59,6 +61,7 @@ defmodule PostHogTest do

assert %{
event: "case tested",
uuid: _,
distinct_id: "distinct_id",
properties: %{
egg: "spam",
Expand All @@ -80,6 +83,7 @@ defmodule PostHogTest do

assert %{
event: "case tested",
uuid: _,
distinct_id: "distinct_id",
properties: %{},
timestamp: _
Expand All @@ -94,6 +98,7 @@ defmodule PostHogTest do

assert %{
event: "case tested",
uuid: _,
distinct_id: "distinct_id",
properties: %{foo: "bar"},
timestamp: _
Expand All @@ -120,13 +125,27 @@ defmodule PostHogTest do

assert %{
event: "case tested",
uuid: _,
distinct_id: "distinct_id",
properties: %{struct: %{hello: nil}, ref: _} = properties,
timestamp: _
} = event

Jason.encode!(properties)
end

test "uuid is valid v7" do
PostHog.bare_capture("uuid test", "distinct_id")

assert [event] = all_captured()

assert is_binary(event.uuid)

assert Regex.match?(
~r/^[0-9a-f]{8}-[0-9a-f]{4}-7[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i,
event.uuid
)
end
end

describe "capture/4" do
Expand All @@ -137,6 +156,7 @@ defmodule PostHogTest do

assert %{
event: "case tested",
uuid: _,
distinct_id: "distinct_id",
properties: %{},
timestamp: _
Expand All @@ -154,6 +174,7 @@ defmodule PostHogTest do

assert %{
event: "case tested",
uuid: _,
distinct_id: "distinct_id",
properties: %{foo: "bar"},
timestamp: _
Expand Down
Loading