Skip to content

[o11y] Report SpanOpen event earlier #5370

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open

[o11y] Report SpanOpen event earlier #5370

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
62 changes: 31 additions & 31 deletions src/cloudflare/internal/test/d1/d1-api-instrumentation-test.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,18 +31,6 @@ export const test = {
};

const expectedSpans = [
{
name: 'fetch',
'network.protocol.name': 'http',
'network.protocol.version': 'HTTP/1.1',
'http.request.method': 'POST',
'url.full': 'http://d1/execute?resultsFormat=NONE',
'http.request.header.content-type': 'application/json',
'http.request.body.size': 231n,
'http.response.status_code': 200n,
'http.response.body.size': 0n,
closed: true,
},
{
name: 'd1_run',
'db.system.name': 'cloudflare-d1',
Expand Down Expand Up @@ -70,9 +58,9 @@ const expectedSpans = [
'network.protocol.name': 'http',
'network.protocol.version': 'HTTP/1.1',
'http.request.method': 'POST',
'url.full': 'http://d1/query?resultsFormat=ROWS_AND_COLUMNS',
'url.full': 'http://d1/execute?resultsFormat=NONE',
'http.request.header.content-type': 'application/json',
'http.request.body.size': 42n,
'http.request.body.size': 231n,
'http.response.status_code': 200n,
'http.response.body.size': 0n,
closed: true,
Expand All @@ -96,7 +84,7 @@ const expectedSpans = [
'network.protocol.name': 'http',
'network.protocol.version': 'HTTP/1.1',
'http.request.method': 'POST',
'url.full': 'http://d1/execute?resultsFormat=NONE',
'url.full': 'http://d1/query?resultsFormat=ROWS_AND_COLUMNS',
'http.request.header.content-type': 'application/json',
'http.request.body.size': 42n,
'http.response.status_code': 200n,
Expand Down Expand Up @@ -124,7 +112,7 @@ const expectedSpans = [
'http.request.method': 'POST',
'url.full': 'http://d1/execute?resultsFormat=NONE',
'http.request.header.content-type': 'application/json',
'http.request.body.size': 40n,
'http.request.body.size': 42n,
'http.response.status_code': 200n,
'http.response.body.size': 0n,
closed: true,
Expand All @@ -148,9 +136,9 @@ const expectedSpans = [
'network.protocol.name': 'http',
'network.protocol.version': 'HTTP/1.1',
'http.request.method': 'POST',
'url.full': 'http://d1/query?resultsFormat=ROWS_AND_COLUMNS',
'url.full': 'http://d1/execute?resultsFormat=NONE',
'http.request.header.content-type': 'application/json',
'http.request.body.size': 223n,
'http.request.body.size': 40n,
'http.response.status_code': 200n,
'http.response.body.size': 0n,
closed: true,
Expand Down Expand Up @@ -180,9 +168,9 @@ const expectedSpans = [
'network.protocol.name': 'http',
'network.protocol.version': 'HTTP/1.1',
'http.request.method': 'POST',
'url.full': 'http://d1/execute?resultsFormat=NONE',
'url.full': 'http://d1/query?resultsFormat=ROWS_AND_COLUMNS',
'http.request.header.content-type': 'application/json',
'http.request.body.size': 40n,
'http.request.body.size': 223n,
'http.response.status_code': 200n,
'http.response.body.size': 0n,
closed: true,
Expand All @@ -208,7 +196,7 @@ const expectedSpans = [
'http.request.method': 'POST',
'url.full': 'http://d1/execute?resultsFormat=NONE',
'http.request.header.content-type': 'application/json',
'http.request.body.size': 223n,
'http.request.body.size': 40n,
'http.response.status_code': 200n,
'http.response.body.size': 0n,
closed: true,
Expand Down Expand Up @@ -238,9 +226,9 @@ const expectedSpans = [
'network.protocol.name': 'http',
'network.protocol.version': 'HTTP/1.1',
'http.request.method': 'POST',
'url.full': 'http://d1/query?resultsFormat=ROWS_AND_COLUMNS',
'url.full': 'http://d1/execute?resultsFormat=NONE',
'http.request.header.content-type': 'application/json',
'http.request.body.size': 31n,
'http.request.body.size': 223n,
'http.response.status_code': 200n,
'http.response.body.size': 0n,
closed: true,
Expand Down Expand Up @@ -344,7 +332,7 @@ const expectedSpans = [
'http.request.method': 'POST',
'url.full': 'http://d1/query?resultsFormat=ROWS_AND_COLUMNS',
'http.request.header.content-type': 'application/json',
'http.request.body.size': 42n,
'http.request.body.size': 31n,
'http.response.status_code': 200n,
'http.response.body.size': 0n,
closed: true,
Expand Down Expand Up @@ -448,7 +436,7 @@ const expectedSpans = [
'http.request.method': 'POST',
'url.full': 'http://d1/query?resultsFormat=ROWS_AND_COLUMNS',
'http.request.header.content-type': 'application/json',
'http.request.body.size': 61n,
'http.request.body.size': 42n,
'http.response.status_code': 200n,
'http.response.body.size': 0n,
closed: true,
Expand Down Expand Up @@ -656,7 +644,7 @@ const expectedSpans = [
'http.request.method': 'POST',
'url.full': 'http://d1/query?resultsFormat=ROWS_AND_COLUMNS',
'http.request.header.content-type': 'application/json',
'http.request.body.size': 125n,
'http.request.body.size': 61n,
'http.response.status_code': 200n,
'http.response.body.size': 0n,
closed: true,
Expand Down Expand Up @@ -685,7 +673,7 @@ const expectedSpans = [
'http.request.method': 'POST',
'url.full': 'http://d1/query?resultsFormat=ROWS_AND_COLUMNS',
'http.request.header.content-type': 'application/json',
'http.request.body.size': 79n,
'http.request.body.size': 125n,
'http.response.status_code': 200n,
'http.response.body.size': 0n,
closed: true,
Expand Down Expand Up @@ -815,7 +803,7 @@ const expectedSpans = [
'http.request.method': 'POST',
'url.full': 'http://d1/query?resultsFormat=ROWS_AND_COLUMNS',
'http.request.header.content-type': 'application/json',
'http.request.body.size': 298n,
'http.request.body.size': 79n,
'http.response.status_code': 200n,
'http.response.body.size': 0n,
closed: true,
Expand Down Expand Up @@ -846,7 +834,7 @@ const expectedSpans = [
'http.request.method': 'POST',
'url.full': 'http://d1/query?resultsFormat=ROWS_AND_COLUMNS',
'http.request.header.content-type': 'application/json',
'http.request.body.size': 45n,
'http.request.body.size': 298n,
'http.response.status_code': 200n,
'http.response.body.size': 0n,
closed: true,
Expand Down Expand Up @@ -950,7 +938,7 @@ const expectedSpans = [
'http.request.method': 'POST',
'url.full': 'http://d1/query?resultsFormat=ROWS_AND_COLUMNS',
'http.request.header.content-type': 'application/json',
'http.request.body.size': 76n,
'http.request.body.size': 45n,
'http.response.status_code': 200n,
'http.response.body.size': 0n,
closed: true,
Expand All @@ -976,7 +964,7 @@ const expectedSpans = [
'http.request.method': 'POST',
'url.full': 'http://d1/query?resultsFormat=ROWS_AND_COLUMNS',
'http.request.header.content-type': 'application/json',
'http.request.body.size': 117n,
'http.request.body.size': 76n,
'http.response.status_code': 200n,
'http.response.body.size': 0n,
closed: true,
Expand All @@ -996,4 +984,16 @@ const expectedSpans = [
'cloudflare.d1.response.changes': 0,
closed: true,
},
{
name: 'fetch',
'network.protocol.name': 'http',
'network.protocol.version': 'HTTP/1.1',
'http.request.method': 'POST',
'url.full': 'http://d1/query?resultsFormat=ROWS_AND_COLUMNS',
'http.request.header.content-type': 'application/json',
'http.request.body.size': 117n,
'http.response.status_code': 200n,
'http.response.body.size': 0n,
closed: true,
},
];
2 changes: 1 addition & 1 deletion src/workerd/api/actor-kv-test-tail.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ export default testTailHandler;
export const test = {
async test(ctrl, env, ctx) {
const expected = [
{ name: 'durable_object_subrequest', closed: true },
{ name: 'durable_object_storage_put', closed: true },
{ name: 'durable_object_storage_put', closed: true },
{ name: 'durable_object_storage_get', closed: true },
Expand All @@ -23,7 +24,6 @@ export const test = {
{ name: 'durable_object_storage_deleteAlarm', closed: true },
{ name: 'durable_object_storage_transaction', closed: true },
{ name: 'durable_object_storage_sync', closed: true },
{ name: 'durable_object_subrequest', closed: true },
];

await Promise.allSettled(invocationPromises);
Expand Down
26 changes: 13 additions & 13 deletions src/workerd/api/r2-instrumentation-test.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -192,12 +192,19 @@ export const test = {
closed: true,
},
{
name: 'r2_put',
name: 'r2_get',
'cloudflare.binding.type': 'r2',
'cloudflare.binding.name': 'BUCKET',
'cloudflare.r2.operation': 'PutObject',
'cloudflare.r2.operation': 'GetObject',
'cloudflare.r2.bucket': 'r2-test',
'cloudflare.r2.request.key': 'throwOnInvalidEtag',
'cloudflare.r2.request.key': 'rangeSuff',
'cloudflare.r2.request.range.suffix': 2n,
'cloudflare.r2.response.success': true,
'cloudflare.r2.response.etag': 'objectEtag',
'cloudflare.r2.response.size': 123,
'cloudflare.r2.response.uploaded': '2024-08-27T14:00:57.918Z',
'cloudflare.r2.response.storage_class': 'Standard',
'cloudflare.r2.response.custom_metadata': true,
closed: true,
},
{
Expand All @@ -210,19 +217,12 @@ export const test = {
closed: true,
},
{
name: 'r2_get',
name: 'r2_put',
'cloudflare.binding.type': 'r2',
'cloudflare.binding.name': 'BUCKET',
'cloudflare.r2.operation': 'GetObject',
'cloudflare.r2.operation': 'PutObject',
'cloudflare.r2.bucket': 'r2-test',
'cloudflare.r2.request.key': 'rangeSuff',
'cloudflare.r2.request.range.suffix': 2n,
'cloudflare.r2.response.success': true,
'cloudflare.r2.response.etag': 'objectEtag',
'cloudflare.r2.response.size': 123,
'cloudflare.r2.response.uploaded': '2024-08-27T14:00:57.918Z',
'cloudflare.r2.response.storage_class': 'Standard',
'cloudflare.r2.response.custom_metadata': true,
'cloudflare.r2.request.key': 'throwOnInvalidEtag',
closed: true,
},
{
Expand Down
2 changes: 1 addition & 1 deletion src/workerd/api/tail-worker-test.js
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

34 changes: 15 additions & 19 deletions src/workerd/io/trace.c++
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1469,7 +1469,10 @@ SpanBuilder::SpanBuilder(kj::Maybe<kj::Own<SpanObserver>> observer,
KJ_IF_SOME(obs, observer) {
// TODO(o11y): Once we report the user tracing spanOpen event as soon as a span is created, we
// should be able to fold this virtual call and just get the timestamp directly.
span.emplace(kj::mv(operationName), startTime.orDefault(obs->getTime()));
kj::Date time = startTime.orDefault([&]() { return obs->getTime(); });
// Report spanOpen event for user tracing spans
obs->reportStart(operationName, time);
span.emplace(kj::mv(operationName), time);
this->observer = kj::mv(obs);
}
}
Expand Down Expand Up @@ -1612,7 +1615,6 @@ void CompleteSpan::copyTo(rpc::UserSpanData::Builder builder) const {
builder.setStartTimeNs((startTime - kj::UNIX_EPOCH) / kj::NANOSECONDS);
builder.setEndTimeNs((endTime - kj::UNIX_EPOCH) / kj::NANOSECONDS);
builder.setSpanId(spanId);
builder.setParentSpanId(parentSpanId);

auto tagsParam = builder.initTags(tags.size());
auto i = 0;
Expand All @@ -1622,10 +1624,15 @@ void CompleteSpan::copyTo(rpc::UserSpanData::Builder builder) const {
serializeTagValue(tagParam.initValue(), tag.value);
}
}
void SpanOpenData::copyTo(rpc::SpanOpenData::Builder builder) const {
builder.setOperationName(operationName.asPtr());
builder.setStartTimeNs((startTime - kj::UNIX_EPOCH) / kj::NANOSECONDS);
builder.setSpanId(spanId);
builder.setParentSpanId(parentSpanId);
}

CompleteSpan::CompleteSpan(rpc::UserSpanData::Reader reader)
: spanId(reader.getSpanId()),
parentSpanId(reader.getParentSpanId()),
operationName(kj::str(reader.getOperationName())),
startTime(kj::UNIX_EPOCH + reader.getStartTimeNs() * kj::NANOSECONDS),
endTime(kj::UNIX_EPOCH + reader.getEndTimeNs() * kj::NANOSECONDS) {
Expand All @@ -1636,22 +1643,11 @@ CompleteSpan::CompleteSpan(rpc::UserSpanData::Reader reader)
deserializeTagValue(tagParam.getValue()));
}
}

CompleteSpan CompleteSpan::clone() const {
CompleteSpan copy(
spanId, parentSpanId, kj::ConstString(kj::str(operationName)), startTime, endTime);
copy.tags.reserve(tags.size());
for (auto& tag: tags) {
copy.tags.insert(kj::ConstString(kj::str(tag.key)), spanTagClone(tag.value));
}
return copy;
}

kj::String CompleteSpan::toString() const {
return kj::str("CompleteSpan: ", operationName,
kj::strArray(
KJ_MAP(tag, tags) { return kj::str("(", tag.key, ", ", tag.value, ")"); }, ", "));
}
SpanOpenData::SpanOpenData(rpc::SpanOpenData::Reader reader)
: spanId(reader.getSpanId()),
parentSpanId(reader.getParentSpanId()),
operationName(kj::str(reader.getOperationName())),
startTime(kj::UNIX_EPOCH + reader.getStartTimeNs() * kj::NANOSECONDS) {}

ScopedDurationTagger::ScopedDurationTagger(
SpanBuilder& span, kj::ConstString key, const kj::MonotonicClock& timer)
Expand Down
11 changes: 11 additions & 0 deletions src/workerd/io/trace.capnp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,3 +42,14 @@ struct UserSpanData {
parentSpanId @5 :UInt64;
}

struct SpanOpenData {
# Representation of a SpanOpen event
operationName @0 :Text;

startTimeNs @1 :Int64;
# Nanoseconds since Unix epoch

spanId @2 :UInt64;
parentSpanId @3 :UInt64;
}

29 changes: 24 additions & 5 deletions src/workerd/io/trace.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -611,8 +611,10 @@ kj::String KJ_STRINGIFY(const CustomInfo& customInfo);
struct CompleteSpan {
// Represents a completed span within user tracing.
tracing::SpanId spanId;
tracing::SpanId parentSpanId;

// TODO(cleanup): operationName and startTime are not used in the spanClose event, but still used
// for diagnostics and a fallback timestamp in the implementation. Get rid of them once that is no
// longer needed.
kj::ConstString operationName;
kj::Date startTime;
kj::Date endTime;
Expand All @@ -621,21 +623,37 @@ struct CompleteSpan {

CompleteSpan(rpc::UserSpanData::Reader reader);
void copyTo(rpc::UserSpanData::Builder builder) const;
CompleteSpan clone() const;
explicit CompleteSpan(tracing::SpanId spanId,
tracing::SpanId parentSpanId,
kj::ConstString operationName,
kj::Date startTime,
kj::Date endTime,
kj::HashMap<kj::ConstString, tracing::Attribute::Value> tags =
kj::HashMap<kj::ConstString, tracing::Attribute::Value>())
: spanId(spanId),
parentSpanId(parentSpanId),
operationName(kj::mv(operationName)),
startTime(startTime),
endTime(endTime),
tags(kj::mv(tags)) {}
kj::String toString() const;
};

struct SpanOpenData {
// Represents the data needed for a SpanOpen event
tracing::SpanId spanId;
tracing::SpanId parentSpanId;

kj::ConstString operationName;
kj::Date startTime;

SpanOpenData(rpc::SpanOpenData::Reader reader);
void copyTo(rpc::SpanOpenData::Builder builder) const;
explicit SpanOpenData(tracing::SpanId spanId,
tracing::SpanId parentSpanId,
kj::ConstString operationName,
kj::Date startTime)
: spanId(spanId),
parentSpanId(parentSpanId),
operationName(kj::mv(operationName)),
startTime(startTime) {}
};

namespace tracing {
Expand Down Expand Up @@ -1099,6 +1117,7 @@ class SpanObserver: public kj::Refcounted {
//
// This should always be called exactly once per observer.
virtual void report(const Span& span) = 0;
virtual void reportStart(kj::ConstString& operationName, kj::Date startTime) = 0;

// The current time to be provided for the span. For user tracing, we will override this to
// provide I/O time. This *requires* that spans are only created when an IOContext is available
Expand Down
Loading
Loading