diff --git a/admin/assets/javascripts/discourse/routes/admin-plugins-show-discourse-ai-usage.js b/admin/assets/javascripts/discourse/routes/admin-plugins-show-discourse-ai-usage.js new file mode 100644 index 000000000..08a0399c8 --- /dev/null +++ b/admin/assets/javascripts/discourse/routes/admin-plugins-show-discourse-ai-usage.js @@ -0,0 +1,11 @@ +import { service } from "@ember/service"; +import { ajax } from "discourse/lib/ajax"; +import DiscourseRoute from "discourse/routes/discourse"; + +export default class DiscourseAiUsageRoute extends DiscourseRoute { + @service store; + + model() { + return ajax("/admin/plugins/discourse-ai/ai-usage.json"); + } +} diff --git a/admin/assets/javascripts/discourse/templates/admin-plugins/show/discourse-ai-usage.hbs b/admin/assets/javascripts/discourse/templates/admin-plugins/show/discourse-ai-usage.hbs new file mode 100644 index 000000000..e1592fc9f --- /dev/null +++ b/admin/assets/javascripts/discourse/templates/admin-plugins/show/discourse-ai-usage.hbs @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/app/controllers/discourse_ai/admin/ai_usage_controller.rb b/app/controllers/discourse_ai/admin/ai_usage_controller.rb new file mode 100644 index 000000000..606db6cab --- /dev/null +++ b/app/controllers/discourse_ai/admin/ai_usage_controller.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module DiscourseAi + module Admin + class AiUsageController < ::Admin::AdminController + requires_plugin "discourse-ai" + + def show + render json: AiUsageSerializer.new(create_report, root: false) + end + + private + + def create_report + report = + DiscourseAi::Completions::Report.new( + start_date: params[:start_date]&.to_date || 30.days.ago, + end_date: params[:end_date]&.to_date || Time.current, + ) + + report = report.filter_by_feature(params[:feature]) if params[:feature].present? + report = report.filter_by_model(params[:model]) if params[:model].present? + report + end + end + end +end diff --git a/app/models/ai_api_audit_log.rb b/app/models/ai_api_audit_log.rb index c8d02c513..ca33a2dee 100644 --- a/app/models/ai_api_audit_log.rb +++ b/app/models/ai_api_audit_log.rb @@ -3,6 +3,7 @@ class AiApiAuditLog < ActiveRecord::Base belongs_to :post belongs_to :topic + belongs_to :user module Provider OpenAI = 1 @@ -43,3 +44,10 @@ def prev_log_id # feature_name :string(255) # language_model :string(255) # feature_context :jsonb +# cached_tokens :integer +# +# Indexes +# +# index_ai_api_audit_logs_on_created_at_and_feature_name (created_at,feature_name) +# index_ai_api_audit_logs_on_created_at_and_language_model (created_at,language_model) +# diff --git a/app/serializers/ai_usage_serializer.rb b/app/serializers/ai_usage_serializer.rb new file mode 100644 index 000000000..38969b05b --- /dev/null +++ b/app/serializers/ai_usage_serializer.rb @@ -0,0 +1,69 @@ +# frozen_string_literal: true + +class AiUsageSerializer < ApplicationSerializer + attributes :data, :features, :models, :users, :summary, :period + + def data + object.tokens_by_period.as_json( + only: %i[period total_tokens total_cached_tokens total_request_tokens total_response_tokens], + ) + end + + def period + object.guess_period + end + + def features + object.feature_breakdown.as_json( + only: %i[ + feature_name + usage_count + total_tokens + total_cached_tokens + total_request_tokens + total_response_tokens + ], + ) + end + + def models + object.model_breakdown.as_json( + only: %i[ + llm + usage_count + total_tokens + total_cached_tokens + total_request_tokens + total_response_tokens + ], + ) + end + + def users + object.user_breakdown.map do |user| + { + avatar_template: User.avatar_template(user.username, user.uploaded_avatar_id), + username: user.username, + usage_count: user.usage_count, + total_tokens: user.total_tokens, + total_cached_tokens: user.total_cached_tokens, + total_request_tokens: user.total_request_tokens, + total_response_tokens: user.total_response_tokens, + } + end + end + + def summary + { + total_tokens: object.total_tokens, + total_cached_tokens: object.total_cached_tokens, + total_request_tokens: object.total_request_tokens, + total_response_tokens: object.total_response_tokens, + total_requests: object.total_requests, + date_range: { + start: object.start_date, + end: object.end_date, + }, + } + end +end diff --git a/assets/javascripts/discourse/admin-discourse-ai-plugin-route-map.js b/assets/javascripts/discourse/admin-discourse-ai-plugin-route-map.js index f71f313a0..0a9de7f1e 100644 --- a/assets/javascripts/discourse/admin-discourse-ai-plugin-route-map.js +++ b/assets/javascripts/discourse/admin-discourse-ai-plugin-route-map.js @@ -18,5 +18,6 @@ export default { this.route("new"); this.route("show", { path: "/:id" }); }); + this.route("discourse-ai-usage", { path: "ai-usage" }); }, }; diff --git a/assets/javascripts/discourse/components/ai-usage.gjs b/assets/javascripts/discourse/components/ai-usage.gjs new file mode 100644 index 000000000..668a41b13 --- /dev/null +++ b/assets/javascripts/discourse/components/ai-usage.gjs @@ -0,0 +1,484 @@ +import Component from "@glimmer/component"; +import { tracked } from "@glimmer/tracking"; +import { fn, hash } from "@ember/helper"; +import { on } from "@ember/modifier"; +import { action } from "@ember/object"; +import { LinkTo } from "@ember/routing"; +import { service } from "@ember/service"; +import { eq } from "truth-helpers"; +import DateTimeInputRange from "discourse/components/date-time-input-range"; +import avatar from "discourse/helpers/avatar"; +import { ajax } from "discourse/lib/ajax"; +import i18n from "discourse-common/helpers/i18n"; +import Chart from "admin/components/chart"; +import ComboBox from "select-kit/components/combo-box"; + +export default class AiUsage extends Component { + @service store; + @tracked startDate = moment().subtract(30, "days").toDate(); + @tracked endDate = new Date(); + @tracked data = this.args.model; + @tracked selectedFeature; + @tracked selectedModel; + @tracked selectedPeriod = "month"; + @tracked isCustomDateActive = false; + + @action + async fetchData() { + const response = await ajax("/admin/plugins/discourse-ai/ai-usage.json", { + data: { + start_date: moment(this.startDate).format("YYYY-MM-DD"), + end_date: moment(this.endDate).format("YYYY-MM-DD"), + feature: this.selectedFeature, + model: this.selectedModel, + }, + }); + this.data = response; + } + + @action + async onFilterChange() { + await this.fetchData(); + } + + @action + onFeatureChanged(value) { + this.selectedFeature = value; + this.onFilterChange(); + } + + @action + onModelChanged(value) { + this.selectedModel = value; + this.onFilterChange(); + } + + normalizeTimeSeriesData(data) { + if (!data?.length) { + return []; + } + + const startDate = moment(this.startDate); + const endDate = moment(this.endDate); + const normalized = []; + let interval; + let format; + + if (this.data.period === "hour") { + interval = "hour"; + format = "YYYY-MM-DD HH:00:00"; + } else if (this.data.period === "day") { + interval = "day"; + format = "YYYY-MM-DD"; + } else { + interval = "month"; + format = "YYYY-MM"; + } + const dataMap = new Map( + data.map((d) => [moment(d.period).format(format), d]) + ); + + for ( + let m = moment(startDate); + m.isSameOrBefore(endDate); + m.add(1, interval) + ) { + const dateKey = m.format(format); + const existingData = dataMap.get(dateKey); + + normalized.push( + existingData || { + period: m.format(), + total_tokens: 0, + total_cached_tokens: 0, + total_request_tokens: 0, + total_response_tokens: 0, + } + ); + } + + return normalized; + } + + get chartConfig() { + if (!this.data?.data) { + return; + } + + const normalizedData = this.normalizeTimeSeriesData(this.data.data); + + const chartEl = document.querySelector(".ai-usage__chart"); + const computedStyle = getComputedStyle(chartEl); + + const colors = { + response: computedStyle.getPropertyValue("--chart-response-color").trim(), + request: computedStyle.getPropertyValue("--chart-request-color").trim(), + cached: computedStyle.getPropertyValue("--chart-cached-color").trim(), + }; + + return { + type: "bar", + data: { + labels: normalizedData.map((row) => { + const date = moment(row.period); + if (this.data.period === "hour") { + return date.format("HH:00"); + } else if (this.data.period === "day") { + return date.format("DD-MMM"); + } else { + return date.format("MMM-YY"); + } + }), + datasets: [ + { + label: "Response Tokens", + data: normalizedData.map((row) => row.total_response_tokens), + backgroundColor: colors.response, + }, + { + label: "Net Request Tokens", + data: normalizedData.map( + (row) => row.total_request_tokens - row.total_cached_tokens + ), + backgroundColor: colors.request, + }, + { + label: "Cached Request Tokens", + data: normalizedData.map((row) => row.total_cached_tokens), + backgroundColor: colors.cached, + }, + ], + }, + options: { + responsive: true, + scales: { + x: { + stacked: true, + }, + y: { + stacked: true, + beginAtZero: true, + }, + }, + }, + }; + } + + get availableFeatures() { + // when you switch we don't want the list to change + // only when you switch durations + this._cachedFeatures = + this._cachedFeatures || + (this.data?.features || []).map((f) => ({ + id: f.feature_name, + name: f.feature_name, + })); + + return this._cachedFeatures; + } + + get availableModels() { + this._cachedModels = + this._cachedModels || + (this.data?.models || []).map((m) => ({ + id: m.llm, + name: m.llm, + })); + + return this._cachedModels; + } + + get periodOptions() { + return [ + { id: "day", name: "Last 24 Hours" }, + { id: "week", name: "Last Week" }, + { id: "month", name: "Last Month" }, + ]; + } + + @action + setPeriodDates(period) { + const now = moment(); + + switch (period) { + case "day": + this.startDate = now.clone().subtract(1, "day").toDate(); + this.endDate = now.toDate(); + break; + case "week": + this.startDate = now.clone().subtract(7, "days").toDate(); + this.endDate = now.toDate(); + break; + case "month": + this.startDate = now.clone().subtract(30, "days").toDate(); + this.endDate = now.toDate(); + break; + } + } + + @action + onPeriodSelect(period) { + this.selectedPeriod = period; + this.isCustomDateActive = false; + this.setPeriodDates(period); + this.fetchData(); + } + + @action + onCustomDateClick() { + this.isCustomDateActive = !this.isCustomDateActive; + if (this.isCustomDateActive) { + this.selectedPeriod = null; + } + } + + @action + onDateChange() { + this.isCustomDateActive = true; + this.selectedPeriod = null; + this.fetchData(); + } + + @action + onChangeDateRange({ from, to }) { + this._startDate = from; + this._endDate = to; + } + + @action + onRefreshDateRange() { + this.startDate = this._startDate; + this.endDate = this._endDate; + this.fetchData(); + } + + +} diff --git a/assets/javascripts/initializers/admin-plugin-configuration-nav.js b/assets/javascripts/initializers/admin-plugin-configuration-nav.js index 6fa22aca2..d667b4688 100644 --- a/assets/javascripts/initializers/admin-plugin-configuration-nav.js +++ b/assets/javascripts/initializers/admin-plugin-configuration-nav.js @@ -24,6 +24,10 @@ export default { label: "discourse_ai.tools.short_title", route: "adminPlugins.show.discourse-ai-tools", }, + { + label: "discourse_ai.usage.short_title", + route: "adminPlugins.show.discourse-ai-usage", + }, ]); }); }, diff --git a/assets/stylesheets/modules/llms/common/usage.scss b/assets/stylesheets/modules/llms/common/usage.scss new file mode 100644 index 000000000..04564d380 --- /dev/null +++ b/assets/stylesheets/modules/llms/common/usage.scss @@ -0,0 +1,168 @@ +.ai-usage { + --chart-response-color: rgba(75, 192, 192, 0.8); + --chart-request-color: rgba(153, 102, 255, 0.8); + --chart-cached-color: rgba(153, 102, 255, 0.4); + + padding: 1em; + + &__filters-dates { + display: flex; + flex-direction: column; + gap: 1em; + margin-bottom: 1em; + } + + &__period-buttons { + display: flex; + gap: 0.5em; + align-items: center; + + .btn { + padding: 0.5em 1em; + + &.btn-primary { + background: var(--tertiary); + color: var(--secondary); + } + } + } + + &__custom-date-pickers { + display: flex; + gap: 1em; + align-items: center; + margin-top: 0.5em; + } + + &__filters { + margin-bottom: 2em; + } + + &__filters-period { + display: flex; + align-items: center; + gap: 1em; + } + + .d-date-time-input-range { + display: flex; + gap: 1em; + align-items: center; + } + + .d-date-time-input-range .from { + margin: 0; + } + + &__period-label { + font-weight: bold; + } + + &__summary { + margin: 2em 0; + padding: 1.5em; + background: var(--primary-very-low); + border-radius: 0.5em; + } + + &__summary-title { + margin-bottom: 1em; + color: var(--primary); + font-size: 1.2em; + } + + &__summary-stats { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(200px, 1fr)); + gap: 1em; + } + + &__summary-stat { + display: flex; + flex-direction: column; + padding: 1em; + background: var(--secondary); + border-radius: 0.25em; + + .label { + color: var(--primary-medium); + font-size: 0.875em; + margin-bottom: 0.5em; + } + + .value { + color: var(--primary); + font-size: 1.5em; + font-weight: bold; + } + } + + &__charts { + margin-top: 2em; + } + + &__chart { + position: relative; + } + + &__chart-container { + margin-bottom: 2em; + } + + &__chart-title { + margin-bottom: 1em; + } + + &__breakdowns { + display: grid; + grid-template-columns: 1fr 1fr; + gap: 2em; + margin-top: 2em; + + @media (max-width: 768px) { + grid-template-columns: 1fr; + } + } + + &__features, + &__users, + &__models { + background: var(--primary-very-low); + padding: 1em; + border-radius: 0.5em; + } + + &__features-title, + &__users-title, + &__models-title { + margin-bottom: 1em; + } + + &__features-table, + &__users-table, + &__models-table { + width: 100%; + border-collapse: collapse; + + th { + text-align: left; + padding: 0.5em; + border-bottom: 2px solid var(--primary-low); + } + } + + &__features-row, + &__users-row, + &__models-row { + &:hover { + background: var(--primary-low); + } + } + + &__features-cell, + &__users-cell, + &__models-cell { + padding: 0.5em; + border-bottom: 1px solid var(--primary-low); + } +} diff --git a/config/locales/client.en.yml b/config/locales/client.en.yml index badcf6b37..b2aab1bd8 100644 --- a/config/locales/client.en.yml +++ b/config/locales/client.en.yml @@ -126,6 +126,26 @@ en: modals: select_option: "Select an option..." + usage: + short_title: "Usage" + summary: "Summary" + total_tokens: "Total tokens" + tokens_over_time: "Tokens over time" + features_breakdown: "Usage per feature" + feature: "Feature" + usage_count: "Usage count" + model: "Model" + models_breakdown: "Usage per model" + users_breakdown: "Usage per user" + all_features: "All features" + all_models: "All models" + username: "Username" + total_requests: "Total requests" + request_tokens: "Request tokens" + response_tokens: "Response tokens" + cached_tokens: "Cached tokens" + + ai_persona: tool_strategies: all: "Apply to all replies" diff --git a/config/routes.rb b/config/routes.rb index b0a84bf2a..1c442309f 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -77,6 +77,8 @@ get "/rag-document-fragments/files/status", to: "discourse_ai/admin/rag_document_fragments#indexing_status_check" + get "/ai-usage", to: "discourse_ai/admin/ai_usage#show" + resources :ai_llms, only: %i[index create show update destroy], path: "ai-llms", diff --git a/db/migrate/20241128010221_add_cached_tokens_to_ai_api_audit_log.rb b/db/migrate/20241128010221_add_cached_tokens_to_ai_api_audit_log.rb new file mode 100644 index 000000000..a100bb84a --- /dev/null +++ b/db/migrate/20241128010221_add_cached_tokens_to_ai_api_audit_log.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +class AddCachedTokensToAiApiAuditLog < ActiveRecord::Migration[7.2] + def change + add_column :ai_api_audit_logs, :cached_tokens, :integer + add_index :ai_api_audit_logs, %i[created_at feature_name] + add_index :ai_api_audit_logs, %i[created_at language_model] + end +end diff --git a/lib/ai_bot/bot.rb b/lib/ai_bot/bot.rb index a8975b9ce..6160bff6f 100644 --- a/lib/ai_bot/bot.rb +++ b/lib/ai_bot/bot.rb @@ -22,7 +22,7 @@ def initialize(bot_user, persona, model = nil) attr_reader :bot_user attr_accessor :persona - def get_updated_title(conversation_context, post) + def get_updated_title(conversation_context, post, user) system_insts = <<~TEXT.strip You are titlebot. Given a conversation, you will suggest a title. @@ -61,7 +61,7 @@ def get_updated_title(conversation_context, post) DiscourseAi::Completions::Llm .proxy(model) - .generate(title_prompt, user: post.user, feature_name: "bot_title") + .generate(title_prompt, user: user, feature_name: "bot_title") .strip .split("\n") .last diff --git a/lib/ai_bot/playground.rb b/lib/ai_bot/playground.rb index 5ec35039f..f0ed19403 100644 --- a/lib/ai_bot/playground.rb +++ b/lib/ai_bot/playground.rb @@ -245,11 +245,11 @@ def conversation_context(post) builder.to_a end - def title_playground(post) + def title_playground(post, user) context = conversation_context(post) bot - .get_updated_title(context, post) + .get_updated_title(context, post, user) .tap do |new_title| PostRevisor.new(post.topic.first_post, post.topic).revise!( bot.bot_user, @@ -544,7 +544,7 @@ def reply_to(post, custom_instructions: nil, &blk) post_streamer&.finish(skip_callback: true) publish_final_update(reply_post) if stream_reply if reply_post && post.post_number == 1 && post.topic.private_message? - title_playground(reply_post) + title_playground(reply_post, post.user) end end diff --git a/lib/completions/endpoints/open_ai.rb b/lib/completions/endpoints/open_ai.rb index 46382732c..4e7dd84b6 100644 --- a/lib/completions/endpoints/open_ai.rb +++ b/lib/completions/endpoints/open_ai.rb @@ -97,6 +97,7 @@ def prepare_request(payload) def final_log_update(log) log.request_tokens = processor.prompt_tokens if processor.prompt_tokens log.response_tokens = processor.completion_tokens if processor.completion_tokens + log.cached_tokens = processor.cached_tokens if processor.cached_tokens end def decode(response_raw) diff --git a/lib/completions/open_ai_message_processor.rb b/lib/completions/open_ai_message_processor.rb index 7b7378db2..33182995b 100644 --- a/lib/completions/open_ai_message_processor.rb +++ b/lib/completions/open_ai_message_processor.rb @@ -1,13 +1,14 @@ # frozen_string_literal: true module DiscourseAi::Completions class OpenAiMessageProcessor - attr_reader :prompt_tokens, :completion_tokens + attr_reader :prompt_tokens, :completion_tokens, :cached_tokens def initialize(partial_tool_calls: false) @tool = nil @tool_arguments = +"" @prompt_tokens = nil @completion_tokens = nil + @cached_tokens = nil @partial_tool_calls = partial_tool_calls end @@ -121,6 +122,7 @@ def process_arguments def update_usage(json) @prompt_tokens ||= json.dig(:usage, :prompt_tokens) @completion_tokens ||= json.dig(:usage, :completion_tokens) + @cached_tokens ||= json.dig(:usage, :prompt_tokens_details, :cached_tokens) end end end diff --git a/lib/completions/report.rb b/lib/completions/report.rb new file mode 100644 index 000000000..acb33f518 --- /dev/null +++ b/lib/completions/report.rb @@ -0,0 +1,148 @@ +# frozen_string_literal: true +module DiscourseAi + module Completions + class Report + UNKNOWN_FEATURE = "unknown" + USER_LIMIT = 50 + + attr_reader :start_date, :end_date, :base_query + + def initialize(start_date: 30.days.ago, end_date: Time.current) + @start_date = start_date.beginning_of_day + @end_date = end_date.end_of_day + @base_query = AiApiAuditLog.where(created_at: @start_date..@end_date) + end + + def total_tokens + stats.total_tokens + end + + def total_cached_tokens + stats.total_cached_tokens + end + + def total_request_tokens + stats.total_request_tokens + end + + def total_response_tokens + stats.total_response_tokens + end + + def total_requests + stats.total_requests + end + + def stats + @stats ||= + base_query.select( + "COUNT(*) as total_requests", + "SUM(request_tokens + response_tokens) as total_tokens", + "SUM(COALESCE(cached_tokens,0)) as total_cached_tokens", + "SUM(request_tokens) as total_request_tokens", + "SUM(response_tokens) as total_response_tokens", + )[ + 0 + ] + end + + def guess_period(period = nil) + period = nil if %i[day month hour].include?(period) + period || + case @end_date - @start_date + when 0..3.days + :hour + when 3.days..90.days + :day + else + :month + end + end + + def tokens_by_period(period = nil) + period = guess_period(period) + base_query + .group("DATE_TRUNC('#{period}', created_at)") + .order("DATE_TRUNC('#{period}', created_at)") + .select( + "DATE_TRUNC('#{period}', created_at) as period", + "SUM(request_tokens + response_tokens) as total_tokens", + "SUM(COALESCE(cached_tokens,0)) as total_cached_tokens", + "SUM(request_tokens) as total_request_tokens", + "SUM(response_tokens) as total_response_tokens", + ) + end + + def user_breakdown + base_query + .joins(:user) + .group(:user_id, "users.username", "users.uploaded_avatar_id") + .order("usage_count DESC") + .limit(USER_LIMIT) + .select( + "users.username", + "users.uploaded_avatar_id", + "COUNT(*) as usage_count", + "SUM(request_tokens + response_tokens) as total_tokens", + "SUM(COALESCE(cached_tokens,0)) as total_cached_tokens", + "SUM(request_tokens) as total_request_tokens", + "SUM(response_tokens) as total_response_tokens", + ) + end + + def feature_breakdown + base_query + .group(:feature_name) + .order("usage_count DESC") + .select( + "case when coalesce(feature_name, '') = '' then '#{UNKNOWN_FEATURE}' else feature_name end as feature_name", + "COUNT(*) as usage_count", + "SUM(request_tokens + response_tokens) as total_tokens", + "SUM(COALESCE(cached_tokens,0)) as total_cached_tokens", + "SUM(request_tokens) as total_request_tokens", + "SUM(response_tokens) as total_response_tokens", + ) + end + + def model_breakdown + base_query + .group(:language_model) + .order("usage_count DESC") + .select( + "language_model as llm", + "COUNT(*) as usage_count", + "SUM(request_tokens + response_tokens) as total_tokens", + "SUM(COALESCE(cached_tokens,0)) as total_cached_tokens", + "SUM(request_tokens) as total_request_tokens", + "SUM(response_tokens) as total_response_tokens", + ) + end + + def tokens_per_hour + tokens_by_period(:hour) + end + + def tokens_per_day + tokens_by_period(:day) + end + + def tokens_per_month + tokens_by_period(:month) + end + + def filter_by_feature(feature_name) + if feature_name == UNKNOWN_FEATURE + @base_query = base_query.where("coalesce(feature_name, '') = ''") + else + @base_query = base_query.where(feature_name: feature_name) + end + self + end + + def filter_by_model(model_name) + @base_query = base_query.where(language_model: model_name) + self + end + end + end +end diff --git a/plugin.rb b/plugin.rb index d331672f7..cd2c2bcb3 100644 --- a/plugin.rb +++ b/plugin.rb @@ -37,6 +37,8 @@ register_asset "stylesheets/modules/llms/common/ai-llms-editor.scss" +register_asset "stylesheets/modules/llms/common/usage.scss" + register_asset "stylesheets/modules/ai-bot/common/ai-tools.scss" register_asset "stylesheets/modules/ai-bot/common/ai-artifact.scss" diff --git a/spec/fixtures/bot/openai_artifact_call.txt b/spec/fixtures/bot/openai_artifact_call.txt index 3939500c4..ce712111c 100644 --- a/spec/fixtures/bot/openai_artifact_call.txt +++ b/spec/fixtures/bot/openai_artifact_call.txt @@ -292,7 +292,7 @@ data: {"id":"chatcmpl-ATimVYagKnCWQ0VXY0Hn2SDjRuN6B","object":"chat.completion.c data: {"id":"chatcmpl-ATimVYagKnCWQ0VXY0Hn2SDjRuN6B","object":"chat.completion.chunk","created":1731647015,"model":"gpt-4o-2024-08-06","system_fingerprint":"fp_45cf54deae","choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"tool_calls"}],"usage":null} -data: {"id":"chatcmpl-ATimVYagKnCWQ0VXY0Hn2SDjRuN6B","object":"chat.completion.chunk","created":1731647015,"model":"gpt-4o-2024-08-06","system_fingerprint":"fp_45cf54deae","choices":[],"usage":{"prompt_tokens":735,"completion_tokens":156,"total_tokens":891,"prompt_tokens_details":{"cached_tokens":0,"audio_tokens":0},"completion_tokens_details":{"reasoning_tokens":0,"audio_tokens":0,"accepted_prediction_tokens":0,"rejected_prediction_tokens":0}}} +data: {"id":"chatcmpl-ATimVYagKnCWQ0VXY0Hn2SDjRuN6B","object":"chat.completion.chunk","created":1731647015,"model":"gpt-4o-2024-08-06","system_fingerprint":"fp_45cf54deae","choices":[],"usage":{"prompt_tokens":735,"completion_tokens":156,"total_tokens":891,"prompt_tokens_details":{"cached_tokens":33,"audio_tokens":0},"completion_tokens_details":{"reasoning_tokens":0,"audio_tokens":0,"accepted_prediction_tokens":0,"rejected_prediction_tokens":0}}} data: [DONE] diff --git a/spec/lib/completions/endpoints/open_ai_spec.rb b/spec/lib/completions/endpoints/open_ai_spec.rb index 331f08e62..d5b6668be 100644 --- a/spec/lib/completions/endpoints/open_ai_spec.rb +++ b/spec/lib/completions/endpoints/open_ai_spec.rb @@ -657,6 +657,9 @@ def request_body(prompt, stream: false, tool_call: false) end end end + + audit_log = AiApiAuditLog.order("id desc").first + expect(audit_log.cached_tokens).to eq(33) end it "properly handles spaces in tools payload and partial tool calls" do diff --git a/spec/lib/modules/ai_bot/playground_spec.rb b/spec/lib/modules/ai_bot/playground_spec.rb index 3078e2112..b04d36553 100644 --- a/spec/lib/modules/ai_bot/playground_spec.rb +++ b/spec/lib/modules/ai_bot/playground_spec.rb @@ -829,8 +829,7 @@ it "updates the title using bot suggestions" do DiscourseAi::Completions::Llm.with_prepared_responses([expected_response]) do - playground.title_playground(third_post) - + playground.title_playground(third_post, user) expect(pm.reload.title).to eq(expected_response) end end diff --git a/spec/requests/admin/ai_usage_controller_spec.rb b/spec/requests/admin/ai_usage_controller_spec.rb new file mode 100644 index 000000000..dacb17c32 --- /dev/null +++ b/spec/requests/admin/ai_usage_controller_spec.rb @@ -0,0 +1,140 @@ +# frozen_string_literal: true + +require "rails_helper" + +RSpec.describe DiscourseAi::Admin::AiUsageController do + fab!(:admin) + fab!(:user) + let(:usage_path) { "/admin/plugins/discourse-ai/ai-usage.json" } + + before { SiteSetting.discourse_ai_enabled = true } + + context "when logged in as admin" do + before { sign_in(admin) } + + describe "#show" do + fab!(:log1) do + AiApiAuditLog.create!( + provider_id: 1, + feature_name: "summarize", + language_model: "gpt-4", + request_tokens: 100, + response_tokens: 50, + created_at: 1.day.ago, + ) + end + + fab!(:log2) do + AiApiAuditLog.create!( + provider_id: 1, + feature_name: "translate", + language_model: "gpt-3.5", + request_tokens: 200, + response_tokens: 100, + created_at: 2.days.ago, + ) + end + + it "returns correct data structure" do + get usage_path + + expect(response.status).to eq(200) + + json = response.parsed_body + expect(json).to have_key("data") + expect(json).to have_key("features") + expect(json).to have_key("models") + expect(json).to have_key("summary") + end + + it "respects date filters" do + get usage_path, params: { start_date: 3.days.ago.to_date, end_date: 1.day.ago.to_date } + + json = response.parsed_body + expect(json["summary"]["total_tokens"]).to eq(450) # sum of all tokens + end + + it "filters by feature" do + get usage_path, params: { feature: "summarize" } + + json = response.parsed_body + + features = json["features"] + expect(features.length).to eq(1) + expect(features.first["feature_name"]).to eq("summarize") + expect(features.first["total_tokens"]).to eq(150) + end + + it "filters by model" do + get usage_path, params: { model: "gpt-3.5" } + + json = response.parsed_body + models = json["models"] + expect(models.length).to eq(1) + expect(models.first["llm"]).to eq("gpt-3.5") + expect(models.first["total_tokens"]).to eq(300) + end + + it "handles different period groupings" do + get usage_path, params: { period: "hour" } + expect(response.status).to eq(200) + + get usage_path, params: { period: "month" } + expect(response.status).to eq(200) + end + end + + # spec/requests/admin/ai_usage_controller_spec.rb + context "with hourly data" do + before do + freeze_time Time.parse("2021-02-01 00:00:00") + # Create data points across different hours + [23.hours.ago, 22.hours.ago, 21.hours.ago, 20.hours.ago].each do |time| + AiApiAuditLog.create!( + provider_id: 1, + feature_name: "summarize", + language_model: "gpt-4", + request_tokens: 100, + response_tokens: 50, + created_at: time, + ) + end + end + + it "returns hourly data when period is day" do + get usage_path, params: { start_date: 1.day.ago.to_date, end_date: Time.current.to_date } + + expect(response.status).to eq(200) + json = response.parsed_body + + expect(json["data"].length).to eq(4) + + data_by_hour = json["data"].index_by { |d| Time.parse(d["period"]).hour } + + expect(data_by_hour.keys.length).to eq(4) + expect(data_by_hour.first[1]["total_tokens"]).to eq(150) + end + end + end + + context "when not admin" do + before { sign_in(user) } + + it "blocks access" do + get usage_path + expect(response.status).to eq(404) + end + end + + context "when plugin disabled" do + before do + SiteSetting.discourse_ai_enabled = false + sign_in(admin) + end + + it "returns error" do + get usage_path + expect(response.status).to eq(404) + end + end +end