Tracing and metrics not working using AWS Collector

[mfolivas]

Getting an error:

Cannot read property 'context' of undefined

When calling the following:

const currentSpan = api.getSpan(api.context.active())
const traceId = currentSpan.context.traceId;

Below is the code:

Configured the following handler.js

'use strict';

const tracer = require('./tracer')('aws-otel-integ-test');
const meter = require('./metric-emitter');
const AWS = require('aws-sdk');
const api = require('@opentelemetry/api');


module.exports.hello = async (event) => {
  console.log('Calling counter')
  const requestStartTime = new Date().getMilliseconds();
  //TODO fix error - Cannot read property 'context' of undefined
  // const traceID = returnTraceIdJson()
  // console.log('tracerId', traceID)
  const statusCode = 200
  meter.emitsPayloadMetric(mimicPayLoadSize(), '/outgoing-http-call', statusCode);
  meter.emitReturnTimeMetric(new Date().getMilliseconds() - requestStartTime, '/outgoing-http-call', statusCode);
  console.log('Counter called')
  return {
    statusCode: 200,
    body: JSON.stringify(
      {
        message: 'Go Serverless v1.0! Your function executed successfully!',
        input: event,
      },
      null,
      2
    ),
  };

  // Use this code if you don't use the http event with the LAMBDA-PROXY integration
  // return { message: 'Go Serverless v1.0! Your function executed successfully!', event };
};

const returnTraceIdJson = () => {
  const currentSpan = api.getSpan(api.context.active())
  const traceId = currentSpan.context.traceId;
  const xrayTraceId = "1-" + traceId.substring(0, 8) + "-" + traceId.substring(8);
  const traceIdJson = JSON.stringify({"traceId": xrayTraceId});
  return traceIdJson;
}

const mimicPayLoadSize = () => {
  return Math.random() * 1000;
}

Using the following trace.js

'use strict'

const { DiagConsoleLogger, DiagLogLevel, diag } = require('@opentelemetry/api')
const { SimpleSpanProcessor, ConsoleSpanExporter } = require("@opentelemetry/tracing");
const { NodeTracerProvider } = require('@opentelemetry/node');
const { CollectorTraceExporter } = require('@opentelemetry/exporter-collector-grpc');

const { AWSXRayPropagator } = require('@aws/otel-aws-xray-propagator');
const { AwsXRayIdGenerator } = require('@aws/otel-aws-xray-id-generator');

const { propagation, trace } = require("@opentelemetry/api");

// Optional and only needed to see the internal diagnostic logging (during development)
diag.setLogger(new DiagConsoleLogger(), DiagLogLevel.DEBUG);

module.exports = (serviceName) => {
  // set global propagator
  propagation.setGlobalPropagator(new AWSXRayPropagator());

  // create a provider for activating and tracking with AWS IdGenerator
  const tracerConfig = {
    idGenerator: new AwsXRayIdGenerator(),
    plugins: {
      https: {
        enabled: true,
        // You may use a package name or absolute path to the file.
        path: '@opentelemetry/plugin-https',
        // https plugin options
      },
      "aws-sdk": {
        enabled: true,
        // You may use a package name or absolute path to the file.
        path: "opentelemetry-plugin-aws-sdk",
      },
    },
  };
  const tracerProvider = new NodeTracerProvider(tracerConfig);
  
  // add OTLP exporter
  const otlpExporter = new CollectorTraceExporter({
    serviceName: serviceName,
    url: (process.env.OTEL_EXPORTER_OTLP_ENDPOINT) ? process.env.OTEL_EXPORTER_OTLP_ENDPOINT : "localhost:55680"
  });
  tracerProvider.addSpanProcessor(new SimpleSpanProcessor(otlpExporter));
  tracerProvider.addSpanProcessor(new SimpleSpanProcessor(new ConsoleSpanExporter()));

  // Register the tracer
  tracerProvider.register();

  // Return an tracer instance
  return trace.getTracer("awsxray-tests");
}

And the following metric-emitter.js

const { DiagConsoleLogger, DiagLogLevel, diag } = require('@opentelemetry/api')
const { CollectorMetricExporter } = require('@opentelemetry/exporter-collector-grpc');
const { MeterProvider } = require('@opentelemetry/metrics');

const API_COUNTER_METRIC = 'apiBytesSent';
const API_LATENCY_METRIC = 'latency';

// Optional and only needed to see the internal diagnostic logging (during development)
diag.setLogger(new DiagConsoleLogger(), DiagLogLevel.DEBUG);

/** The OTLP Metrics gRPC Collector */
const metricExporter = new CollectorMetricExporter({
    serviceName: process.env.OTEL_RESOURCE_ATTRIBUTES ? process.env.OTEL_RESOURCE_ATTRIBUTES : 'aws-otel-integ-test',
    url: process.env.OTEL_EXPORTER_OTLP_ENDPOINT ? process.env.OTEL_EXPORTER_OTLP_ENDPOINT : 'localhost:55680'
});

/** The OTLP Metrics Provider with OTLP gRPC Metric Exporter and Metrics collection Interval  */
const meter = new MeterProvider({
    exporter: metricExporter,
    interval: 1000,
}).getMeter('aws-otel');

/**  grabs instanceId and append to metric name to check individual metric for integration test */
const latencyMetricName = API_LATENCY_METRIC;
const  apiBytesSentMetricName = API_COUNTER_METRIC;
// const instanceId = process.env.INSTANCE_ID || '';
// if (instanceId && instanceId.trim() !== '') {
//     latencyMetricName += '_' + instanceId;
//     apiBytesSentMetricName += '_' + instanceId;
// }

/** Counter Metrics */
const payloadMetric = meter.createCounter(apiBytesSentMetricName, {
    description: 'Metric for counting request payload size',
});

/** Value Recorder Metrics with Histogram */
const requestLatency = meter.createValueRecorder(latencyMetricName, {
    description: 'Metric for record request latency'
});

//** binds request latency metric with returnTime */
const emitReturnTimeMetric = (returnTime, apiName, statusCode) => {
    console.log('emit metric with return time ' + returnTime + ', ' + apiName + ', ' + statusCode);
    const labels = { 'apiName': apiName, 'statusCode': statusCode };
    requestLatency.bind(labels).record(returnTime);
}

//** emitsPayLoadMetrics() Binds payload Metric with number of bytes */
const emitsPayloadMetric = (bytes, apiName, statusCode) => {
    console.log('emit metric with http request size ' + bytes + ' byte, ' + apiName);
    const labels = { 'apiName': apiName, 'statusCode': statusCode };
    payloadMetric.bind(labels).add(bytes);
}

module.exports = {
    emitReturnTimeMetric: emitReturnTimeMetric,
    emitsPayloadMetric: emitsPayloadMetric
}

My package.json dependencies are the following:

"dependencies": {
    "@aws/otel-aws-xray-id-generator": "^0.13.1",
    "@aws/otel-aws-xray-propagator": "^0.13.0",
    "@opentelemetry/api": "^1.0.0-rc.0",
    "@opentelemetry/core": "^0.16.0",
    "@opentelemetry/exporter-collector-grpc": "^0.18.0",
    "@opentelemetry/exporter-zipkin": "^0.17.0",
    "@opentelemetry/instrumentation": "^0.18.0",
    "@opentelemetry/metrics": "^0.18.0",
    "@opentelemetry/node": "^0.17.0",
    "@opentelemetry/plugin-express": "^0.14.0",
    "@opentelemetry/plugin-http": "^0.18.0",
    "@opentelemetry/plugin-https": "^0.18.0",
    "@opentelemetry/tracing": "^0.17.0"
  }

[Flarna]

I think it should be currentSpan.context().traceId - but if currentSpan is undefined it will not change anything.

What I miss in this sample is the code which calls the hello function. This part of the code should create a Span and then call hello like this to have this span in current context: api.context.with(api.setSpan(api.ROOT_CONTEXT, theSpan), hello, undefined, event).

If there is no span created and set in context it's clear that api.getSpan() will return undefined.

[mfolivas]

Yes, all the spans should be created by x-ray.

[dyladan]

x-ray does not create the traces automatically, it just provides a propagator you can use to manually extract xray headers and an id generator which is used to generate xray compatible ids

[mfolivas]

Thanks @dyladan & @Flarna . I got this from the AWS example:

function returnTraceIdJson() {
  const traceId = tracer.getCurrentSpan().context().traceId;
  const xrayTraceId = "1-" + traceId.substring(0, 8) + "-" + traceId.substring(8);
  const traceIdJson = JSON.stringify({"traceId": xrayTraceId});
  return traceIdJson;
}

but I have to do a bit more digging or if you can provide me with some example, I would be greatly appreciated.

I also don't understand why I am not seeing any metrics on my cloudwatch. Are my metrics ok? I don't see any errors and they look good to me. But again, I don't see anything in my cloudwatch's metric with the name: "apiBytesSent" or "latency".

[Flarna]

I'm not an AWS X-Ray expert therefore I can't give more hints what needs to be done on AWS side.

But it seems that you mix incompatible versions of the opentelemetry components, e.g. @opentelemetry/api 1.0.0-rc.0 is not compatible to @opentelemetry/node 0.17.0 see compatibility-matrix.