Skip to content

Commit c5d6900

Browse files
committed
wip: do more otel things
1 parent 319e15c commit c5d6900

File tree

8 files changed

+246
-53
lines changed

8 files changed

+246
-53
lines changed

app/controllers/application_controller.rb

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,16 @@ def set_request_logging_attributes
3535
CurrentRequestLoggingAttributes.page_slug = params[:page_slug] if params[:page_slug].present?
3636
CurrentRequestLoggingAttributes.session_id_hash = session_id_hash
3737
CurrentRequestLoggingAttributes.trace_id = request.env["HTTP_X_AMZN_TRACE_ID"] if request.env["HTTP_X_AMZN_TRACE_ID"].present?
38+
39+
# Add same attributes to OpenTelemetry span for journey tracking
40+
TelemetryService.set_request_attributes({
41+
"session.id_hash" => session_id_hash,
42+
"request.host" => request.host,
43+
"request.id" => request.request_id,
44+
"form.id" => params[:form_id],
45+
"page.id" => params[:page_slug]&.match(Page::PAGE_ID_REGEX) ? params[:page_slug] : nil,
46+
"page.slug" => params[:page_slug],
47+
})
3848
end
3949

4050
def log_rescued_exception(exception)

app/controllers/forms/base_controller.rb

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,14 @@ def set_request_logging_attributes
2323
super
2424
CurrentRequestLoggingAttributes.form_name = @form.name
2525
CurrentRequestLoggingAttributes.preview = mode.preview?
26+
27+
# Add form-level attributes to OpenTelemetry span
28+
TelemetryService.set_request_attributes({
29+
"form.name" => @form.name,
30+
"form.slug" => @form.form_slug,
31+
"mode.type" => mode.to_s,
32+
"mode.preview" => mode.preview?,
33+
})
2634
end
2735

2836
private

app/controllers/forms/page_controller.rb

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,9 @@ def set_request_logging_attributes
66
super
77
CurrentRequestLoggingAttributes.question_number = @step.page_number if @step&.page_number
88
CurrentRequestLoggingAttributes.answer_type = @step&.page&.answer_type if @step&.page&.answer_type
9+
10+
# Add question-level attributes to OpenTelemetry span
11+
TelemetryService.set_question_attributes(@step, @form) if @step && @form
912
end
1013

1114
def show

app/lib/flow/context.rb

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,15 @@ def initialize(form:, store:)
1515
delegate :save_submission_details, :get_submission_reference, :requested_email_confirmation?, :clear_submission_details, to: :confirmation_details_store
1616

1717
def save_step(step, context: nil)
18-
return false unless step.valid?(context)
18+
is_valid = step.valid?(context)
19+
20+
if is_valid
21+
TelemetryService.record_validation_success
22+
else
23+
TelemetryService.record_validation_failure(step)
24+
end
25+
26+
return false unless is_valid
1927

2028
step.save_to_store(@answer_store)
2129
end

app/services/api/v2/form_document_repository.rb

Lines changed: 20 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3,11 +3,26 @@ class << self
33
def find(form_id:, tag:, language: :en)
44
raise ActiveResource::ResourceNotFound.new(404, "Not Found") unless form_id.to_s =~ /^[[:alnum:]]+$/
55

6-
form_document = Api::V2::FormDocumentResource.get(form_id, tag, **options_for_language(language))
7-
form = Form.new(form_document, true)
8-
form.document_json = form_document
9-
form.prefix_options = { form_id:, tag: }
10-
form
6+
TelemetryService.trace("api.forms_admin.fetch_form", attributes: {
7+
"api.endpoint" => "#{Settings.forms_api.base_url}/api/v2/form/#{form_id}/#{tag}",
8+
"api.method" => "GET",
9+
"form.id" => form_id.to_s,
10+
"form.tag" => tag.to_s,
11+
"form.language" => language.to_s,
12+
}) do |span|
13+
form_document = Api::V2::FormDocumentResource.get(form_id, tag, **options_for_language(language))
14+
15+
span.set_attribute("api.response.status", 200)
16+
span.set_attribute("form.name", form_document.name) if form_document.respond_to?(:name)
17+
18+
form = Form.new(form_document, true)
19+
form.document_json = form_document
20+
form.prefix_options = { form_id:, tag: }
21+
form
22+
end
23+
rescue ActiveResource::ResourceNotFound => e
24+
# Re-raise but let the span record the error
25+
raise
1126
end
1227

1328
def find_with_mode(form_id:, mode:, language: :en)

app/services/form_submission_service.rb

Lines changed: 49 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -23,14 +23,22 @@ def initialize(current_context:, email_confirmation_input:, mode:)
2323
end
2424

2525
def submit
26-
ensure_form_english
27-
validate_submission
28-
29-
confirmation_mail = setup_confirmation_email if requested_confirmation?
30-
deliver_submission
31-
send_confirmation_email(confirmation_mail) if confirmation_mail.present?
32-
33-
submission_reference
26+
TelemetryService.trace("form.submission.process", attributes: {
27+
"submission.type" => form.submission_type,
28+
"submission.format" => form.submission_format,
29+
"submission.reference" => submission_reference,
30+
"form.id" => form.id.to_s,
31+
"confirmation.requested" => requested_confirmation?,
32+
}) do
33+
ensure_form_english
34+
validate_submission
35+
36+
confirmation_mail = setup_confirmation_email if requested_confirmation?
37+
deliver_submission
38+
send_confirmation_email(confirmation_mail) if confirmation_mail.present?
39+
40+
submission_reference
41+
end
3442
end
3543

3644
private
@@ -71,31 +79,42 @@ def deliver_submission
7179
end
7280

7381
def deliver_submission_via_s3
74-
s3_submission_service = S3SubmissionService.new(
75-
journey: current_context.journey,
76-
form: form,
77-
timestamp: timestamp,
78-
submission_reference: submission_reference,
79-
is_preview: mode.preview?,
80-
)
81-
82-
s3_submission_service.submit
82+
TelemetryService.trace("form.submission.deliver_s3", attributes: {
83+
"submission.reference" => submission_reference,
84+
"submission.format" => form.submission_format,
85+
"form.id" => form.id.to_s,
86+
}) do
87+
s3_submission_service = S3SubmissionService.new(
88+
journey: current_context.journey,
89+
form: form,
90+
timestamp: timestamp,
91+
submission_reference: submission_reference,
92+
is_preview: mode.preview?,
93+
)
94+
95+
s3_submission_service.submit
96+
end
8397
end
8498

8599
def deliver_submission_via_email
86-
submission = Submission.create!(
87-
reference: submission_reference,
88-
form_id: form.id,
89-
answers: current_context.answers,
90-
mode: mode,
91-
form_document: form.document_json,
92-
)
93-
94-
SendSubmissionJob.perform_later(submission) do |job|
95-
unless job.successfully_enqueued?
96-
submission.destroy!
97-
message_suffix = ": #{job.enqueue_error&.message}" if job.enqueue_error
98-
raise StandardError, "Failed to enqueue submission for reference #{submission_reference}#{message_suffix}"
100+
TelemetryService.trace("form.submission.deliver_email", attributes: {
101+
"submission.reference" => submission_reference,
102+
"form.id" => form.id.to_s,
103+
}) do
104+
submission = Submission.create!(
105+
reference: submission_reference,
106+
form_id: form.id,
107+
answers: current_context.answers,
108+
mode: mode,
109+
form_document: form.document_json,
110+
)
111+
112+
SendSubmissionJob.perform_later(submission) do |job|
113+
unless job.successfully_enqueued?
114+
submission.destroy!
115+
message_suffix = ": #{job.enqueue_error&.message}" if job.enqueue_error
116+
raise StandardError, "Failed to enqueue submission for reference #{submission_reference}#{message_suffix}"
117+
end
99118
end
100119
end
101120
end

app/services/s3_submission_service.rb

Lines changed: 30 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -17,23 +17,36 @@ def submit
1717
raise StandardError, "S3 bucket account ID is not set on the form" if @form.s3_bucket_aws_account_id.nil?
1818
raise StandardError, "S3 bucket region is not set on the form" if @form.s3_bucket_region.nil?
1919

20-
# We send the uploaded files before the submissions CSV so that processors can have automations run when the CSV
21-
# file arrives and the referenced files will already be present
22-
copy_uploaded_files_to_bucket
23-
24-
submission_content, key =
25-
case @form.submission_format
26-
when %w[csv]
27-
[generate_csv_submission, generate_key("form_submission.csv")]
28-
when %w[json]
29-
[generate_json_submission, generate_key("form_submission.json")]
30-
else
31-
raise StandardError, "Unsupported submission format: #{@form.submission_format.inspect}"
32-
end
33-
34-
upload_submission_to_s3(submission_content, key)
35-
36-
delete_uploaded_files_from_our_bucket
20+
file_count = @journey.completed_file_upload_questions.count
21+
22+
TelemetryService.trace("submission.s3.upload", attributes: {
23+
"s3.bucket" => @form.s3_bucket_name,
24+
"s3.region" => @form.s3_bucket_region,
25+
"submission.format" => @form.submission_format.join(","),
26+
"submission.reference" => @submission_reference,
27+
"submission.file_count" => file_count,
28+
}) do |span|
29+
# We send the uploaded files before the submissions CSV so that processors can have automations run when the CSV
30+
# file arrives and the referenced files will already be present
31+
copy_uploaded_files_to_bucket
32+
33+
submission_content, key =
34+
case @form.submission_format
35+
when %w[csv]
36+
[generate_csv_submission, generate_key("form_submission.csv")]
37+
when %w[json]
38+
[generate_json_submission, generate_key("form_submission.json")]
39+
else
40+
raise StandardError, "Unsupported submission format: #{@form.submission_format.inspect}"
41+
end
42+
43+
span.set_attribute("submission.size_bytes", submission_content.bytesize)
44+
span.set_attribute("s3.key", key)
45+
46+
upload_submission_to_s3(submission_content, key)
47+
48+
delete_uploaded_files_from_our_bucket
49+
end
3750
end
3851

3952
private

app/services/telemetry_service.rb

Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,117 @@
1+
class TelemetryService
2+
# OpenTelemetry tracing service for adding span attributes
3+
# Follows the same pattern as CloudWatchService and LogEventService
4+
#
5+
# We are heavily using attributes, not events because X-Ray does not support events
6+
# see: https://github.com/aws-observability/aws-otel-collector/issues/821
7+
8+
# Set request-level attributes for journey tracking
9+
# Call from ApplicationController to add form/session context to all spans
10+
def self.set_request_attributes(attrs)
11+
return unless defined?(OpenTelemetry)
12+
13+
# Ensure all values are primitives (string, number, boolean, nil)
14+
sanitized = attrs.compact.transform_values { |v| sanitize_attribute_value(v) }
15+
current_span.add_attributes(sanitized.transform_keys(&:to_s))
16+
rescue StandardError => e
17+
Sentry.capture_exception(e) if defined?(Sentry)
18+
end
19+
20+
# Set question-level attributes on page requests
21+
# Call from PageController to add question context to all page spans
22+
def self.set_question_attributes(step, form)
23+
return unless defined?(OpenTelemetry)
24+
25+
attrs = {
26+
"question.type" => step.question.class.name,
27+
"question.id" => step.page_id,
28+
"question.text" => step.question_text,
29+
"question.answer_type" => step.page&.answer_type,
30+
"question.number" => step.page_number,
31+
"question.is_optional" => step.question.is_optional?,
32+
"question.is_repeatable" => step.repeatable?,
33+
"form.submission_type" => form.submission_type,
34+
}.compact
35+
36+
sanitized = attrs.transform_values { |v| sanitize_attribute_value(v) }
37+
current_span.add_attributes(sanitized)
38+
rescue StandardError => e
39+
Sentry.capture_exception(e) if defined?(Sentry)
40+
end
41+
42+
def self.record_validation_failure(step)
43+
return unless defined?(OpenTelemetry)
44+
45+
attrs = {
46+
"validation.failed" => true,
47+
"validation.error_count" => step.question.errors.count,
48+
"validation.errors" => step.question.errors.full_messages.join(", "),
49+
"validation.error_attributes" => step.question.errors.attribute_names.map(&:to_s).join(", "),
50+
}
51+
52+
sanitized = attrs.transform_values { |v| sanitize_attribute_value(v) }
53+
current_span.add_attributes(sanitized)
54+
rescue StandardError => e
55+
# Silently fail - don't break the app if telemetry has issues
56+
Sentry.capture_exception(e) if defined?(Sentry)
57+
end
58+
59+
def self.record_validation_success
60+
return unless defined?(OpenTelemetry)
61+
62+
current_span.set_attribute("validation.passed", true)
63+
rescue StandardError => e
64+
Sentry.capture_exception(e) if defined?(Sentry)
65+
end
66+
67+
# Create a custom span for wrapping important operations
68+
# Usage: TelemetryService.trace('operation.name', attributes: {...}) { ... }
69+
def self.trace(span_name, attributes: {}, &block)
70+
return yield(NoOpSpan.new) unless defined?(OpenTelemetry)
71+
72+
# Get tracer
73+
tracer = OpenTelemetry.tracer_provider.tracer("forms-runner")
74+
75+
# Sanitize attributes to ensure they're primitives
76+
sanitized = attributes.compact.transform_values { |v| sanitize_attribute_value(v) }
77+
78+
tracer.in_span(span_name, attributes: sanitized, &block)
79+
rescue StandardError => e
80+
Sentry.capture_exception(e) if defined?(Sentry)
81+
# If tracing fails, still execute the block with a no-op span
82+
# This ensures business logic runs even if telemetry breaks
83+
yield(NoOpSpan.new)
84+
end
85+
86+
def self.current_span
87+
OpenTelemetry::Trace.current_span
88+
end
89+
private_class_method :current_span
90+
91+
# Sanitize attribute values to ensure they're primitives (String, Integer, Float, Boolean)
92+
# OpenTelemetry requires attribute values to be primitives, not complex objects
93+
def self.sanitize_attribute_value(value)
94+
case value
95+
when String, Integer, Float, TrueClass, FalseClass, NilClass
96+
value
97+
when Array
98+
value.join(", ")
99+
else
100+
value.to_s
101+
end
102+
end
103+
private_class_method :sanitize_attribute_value
104+
105+
# No-op span that safely ignores all method calls
106+
# Used as a fallback when tracing is disabled or fails
107+
class NoOpSpan
108+
def method_missing(_method_name, *_args, **_kwargs, &_block)
109+
# Silently ignore all method calls (set_attribute, add_event, etc.)
110+
nil
111+
end
112+
113+
def respond_to_missing?(_method_name, _include_private = false)
114+
true
115+
end
116+
end
117+
end

0 commit comments

Comments
 (0)