Fixing stackdriver export interval propagation (#66)

* Introducing Stackdriver Exporter for Opencensus C# library

- Current implementation can only store string values
- Added the exporter and trace handler only
- The exporter relies on newest Trace API from Stackdriver.

* Updating translation from ISpan to Stackdriver's Span to cover more fields

* Fixing the issue that prevented Stackdriver API call to succeed:
now construction of Span resource is taken care of by SpanName class that
is part of Stackdriver Trace V2 API.

* - Added support for capturing all types of trace spans (long/bool/string)
- Fixed csproj, so it produces both .NET Core and .NET versions. It also means signing the assembly using the same mechanism as other assemblies in the solution

* - Added support for storing links
- Minor fixes to proto<->opencensus translation methods

* Fixing merge issue

* Added command line support for the Samples project.
This will make it easier to script testing multiple exporters as
well as prevent commenting and uncommenting different test procedures
for different exporters. Finally, it will making test code cleaner for
showing samples embedded in documentation website.

In order to test the exporter, you can either execute it from the command line:
samples.dll prometheus
or to debug using Visual Studio:
Debug => Sample Properties => Debug.
Now create a profile for your testing execution and fill application arguments
if needed.
Now choose your profile in running configuration of Visual Studio.
Running settings are local to the machine and are ignored in PRs.

* Adding sceleton of metrics support in Stackdriver Exporter.

Currently we can't detect all types of monitoring resources but we
can ship the batches of metrics with time series to Stackdriver.
Thorough testing TBD

* Fixing a few bugs in Stackdriver metrics exporter

- Small refactoring
- Assinging default values for time intervals

* - Fixing a few bugs around metric creation and labels
- Refactoring MetricsConversions
- Fixing stackdriver stats configuration - population of default resource and projects

* First working version of Stackdriver Stats Exporter.

1) Fixed bugs related to Stackdriver Metrics Descriptor creation
2) Renamed Metrics Exporter to Stats Exporter to follow Opencensus terminology
3) Cache Stackdriver Metrics Descriptors locally so it's easy which descriptor holds the time series to upload
4) Fixed test code that was producing the metrics using Opencensus and uploading it to Stackdriver
5) Minor refactoring

Current limitations:

1) Supporting only string labels
2) We support only "Global" monitored resource. We need to detect dynamically where the code is running and attach
corresponding labels to the monitored resource, so metrics are stored correctly
3) More Tests

* Removing commented lines

* Fixed Stackdriver export interval propagation
Added Opencensus headers into every call into Stackdriver APIs
Added consts for resources that Stackdriver Exporter should auto-detect when running on GCP

* - Added test project for Stackdriver Exporter
- Added tests for StackdriverStatsConfiguration

* Trying to fix the tests that fail on CI as a result of wrong signing key for InternalVisibleTo

* Updating Travis build definition to latest .netcore sdk and running all tests

* Adding an option to set google cloud projectId in environment variable for testing purposes(Stackdriver exporter).

* Temporarily commenting out lines that read from a friend assembly as it doesn't work on a build server

* Added AssemblyInfo.cs into Stackdriver Exporter, so it's visible to tests on both signed and unsigned builds

Author: Simon Zeltser

.travis.yml

@@ -1,7 +1,7 @@
 language: csharp
 mono: none
-dotnet: 2.1.402
+dotnet: 2.1.500
 script:
   - dotnet restore
   - dotnet build 
-  - dotnet test ./test/OpenCensus.Tests/OpenCensus.Tests.csproj
+  - dotnet test

OpenCensus.sln

@@ -59,6 +59,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TestApp.AspNetCore.2.0", "t
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "OpenCensus.Exporter.Ocagent", "src\OpenCensus.Exporter.Ocagent\OpenCensus.Exporter.Ocagent.csproj", "{56B0ED25-8A14-4AA2-B59D-FAAFCBACDD4A}"
 EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "OpenCensus.Exporter.Stackdriver.Tests", "test\OpenCensus.Exporter.Stackdriver.Tests\OpenCensus.Exporter.Stackdriver.Tests.csproj", "{6875032B-DFDC-4CDE-A283-37CA7F99926A}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -121,11 +123,16 @@ Global
 		{56B0ED25-8A14-4AA2-B59D-FAAFCBACDD4A}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{56B0ED25-8A14-4AA2-B59D-FAAFCBACDD4A}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{56B0ED25-8A14-4AA2-B59D-FAAFCBACDD4A}.Release|Any CPU.Build.0 = Release|Any CPU
+		{6875032B-DFDC-4CDE-A283-37CA7F99926A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{6875032B-DFDC-4CDE-A283-37CA7F99926A}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{6875032B-DFDC-4CDE-A283-37CA7F99926A}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{6875032B-DFDC-4CDE-A283-37CA7F99926A}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
 	EndGlobalSection
 	GlobalSection(NestedProjects) = preSolution
+		{7CB2F02E-03FA-4FFF-89A5-C51F107623FD} = {61188153-47FB-4567-AC9B-79B2435853EB}
 		{F2F81E76-6A0E-466B-B673-EBBF1A9ED075} = {77C7929A-2EED-4AA6-8705-B5C443C8AA0F}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution

src/OpenCensus.Exporter.Stackdriver/AssemblyInfo.cs

@@ -0,0 +1,34 @@
+// <copyright file="AssemblyInfo.cs" company="OpenCensus Authors">
+// Copyright 2018, OpenCensus Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of theLicense at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+// </copyright>
+
+using System.Runtime.CompilerServices;
+
+[assembly: InternalsVisibleTo("OpenCensus.Exporter.Stackdriver.Tests" + AssemblyInfo.PublicKey)]
+[assembly: InternalsVisibleTo("DynamicProxyGenAssembly2" + AssemblyInfo.MoqPublicKey)]
+
+#if SIGNED
+internal static class AssemblyInfo
+{
+    public const string PublicKey = ", PublicKey=002400000480000094000000060200000024000052534131000400000100010051C1562A090FB0C9F391012A32198B5E5D9A60E9B80FA2D7B434C9E5CCB7259BD606E66F9660676AFC6692B8CDC6793D190904551D2103B7B22FA636DCBB8208839785BA402EA08FC00C8F1500CCEF28BBF599AA64FFB1E1D5DC1BF3420A3777BADFE697856E9D52070A50C3EA5821C80BEF17CA3ACFFA28F89DD413F096F898";
+    public const string MoqPublicKey = ", PublicKey=0024000004800000940000000602000000240000525341310004000001000100c547cac37abd99c8db225ef2f6c8a3602f3b3606cc9891605d02baa56104f4cfc0734aa39b93bf7852f7d9266654753cc297e7d2edfe0bac1cdcf9f717241550e0a7b191195b7667bb4f64bcb8e2121380fd1d9d46ad2d92d2d15605093924cceaf74c4861eff62abf69b9291ed0a340e113be11e6a7d3113e92484cf7045cc7";
+}
+#else
+internal static class AssemblyInfo
+{
+    public const string PublicKey = "";
+    public const string MoqPublicKey = "";
+}
+#endif

src/OpenCensus.Exporter.Stackdriver/Implementation/Constants.cs

@@ -17,8 +17,6 @@
 namespace OpenCensus.Exporter.Stackdriver.Implementation
 {
     using System;
-    using System.Collections.Generic;
-    using System.Collections.Specialized;
 
     internal class Constants
     {
@@ -34,6 +32,17 @@ internal class Constants
         public const string PROJECT_ID_LABEL_KEY = "project_id";
         public static readonly string OPENCENSUS_TASK_VALUE_DEFAULT = generateDefaultTaskValue();
 
+        public const string GCP_GCE_INSTANCE_TYPE = "cloud.google.com/gce/instance";
+        public const string GCP_INSTANCE_ID_KEY = "cloud.google.com/gce/instance_id";
+        public const string GCP_ACCOUNT_ID_KEY = "cloud.google.com/gce/project_id";
+        public const string GCP_ZONE_KEY = "cloud.google.com/gce/zone";
+
+        public const string K8S_CONTAINER_TYPE = "k8s.io/container";
+        public const string K8S_CLUSTER_NAME_KEY = "k8s.io/cluster/name";
+        public const string K8S_CONTAINER_NAME_KEY = "k8s.io/container/name";
+        public const string K8S_NAMESPACE_NAME_KEY = "k8s.io/namespace/name";
+        public const string K8S_POD_NAME_KEY = "k8s.io/pod/name";
+
         private static string generateDefaultTaskValue()
         {
             // Something like '<pid>@<hostname>'

src/OpenCensus.Exporter.Stackdriver/Implementation/GoogleCloudResourceUtils.cs

@@ -17,32 +17,44 @@
 namespace OpenCensus.Exporter.Stackdriver.Implementation
 {
     using Google.Api;
+    using System;
     using System.Collections.Generic;
+    using System.IO;
 
     /// <summary>
     /// Utility methods for working with Google Cloud Resources
     /// </summary>
     public static class GoogleCloudResourceUtils
     {
-        private static Dictionary<string, string> gcpResourceLabelMappings = new Dictionary<string, string>()
-        {
-            { "project_id", Constants.PROJECT_ID_LABEL_KEY },
-            { "instance_id", Constants.GCP_GCE_INSTANCE },
-            { "zone", null }
-        };
-
         /// <summary>
         /// Detects Google Cloud ProjectId based on the environment on which the code runs.
-        /// Supports GCE/GKE/GAE
+        /// Supports GCE/GKE/GAE and projectId tied to service account
         /// In case the code runs in a different environment,
         /// the method returns null
         /// </summary>
         /// <returns>Google Cloud Project ID</returns>
         public static string GetProjectId()
         {
+            // Try to detect projectId from the environment where the code is running
             var instance = Google.Api.Gax.Platform.Instance();
             var projectId = instance?.ProjectId;
+            if (!string.IsNullOrEmpty(projectId))
+            {
+                return projectId;
+            }
+
+            // Try to detect projectId from service account credential if it exists
+            string serviceAccountFilePath = Environment.GetEnvironmentVariable("GOOGLE_APPLICATION_CREDENTIALS");
+            if (!string.IsNullOrEmpty(serviceAccountFilePath) && File.Exists(serviceAccountFilePath))
+            {
+                using (var stream = new FileStream(serviceAccountFilePath, FileMode.Open))
+                {
+                    var credential = Google.Apis.Auth.OAuth2.ServiceAccountCredential.FromServiceAccountData(stream);
+                    return credential.ProjectId;
+                }
+            }
 
+            projectId = Environment.GetEnvironmentVariable("GOOGLE_PROJECT_ID");
             return projectId;
         }
 

src/OpenCensus.Exporter.Stackdriver/Implementation/MetricsConversions.cs

@@ -23,8 +23,8 @@ namespace OpenCensus.Exporter.Stackdriver.Implementation
     using OpenCensus.Exporter.Stackdriver.Utils;
     using OpenCensus.Stats;
     using OpenCensus.Stats.Aggregations;
+    using OpenCensus.Stats.Measures;
     using OpenCensus.Tags;
-    using System;
     using System.Collections.Generic;
     using static Google.Api.Distribution.Types;
     using static Google.Api.MetricDescriptor.Types;
@@ -52,17 +52,30 @@ public static MetricKind ToMetricKind(
         }
 
         /// <summary>
-        /// Converts from Opencensus Measure to Stackdriver ValueType
+        /// Converts from Opencensus Measure+Aggregation to Stackdriver's ValueType
         /// </summary>
-        /// <param name="measure"></param>
+        /// <param name="measure">Opencensus Measure definition</param>
+        /// <param name="aggregation">Opencensus Aggregation definition</param>
         /// <returns></returns>
-        public static MetricDescriptor.Types.ValueType ToValueType(
-            this IMeasure measure)
+        public static ValueType ToValueType(
+            this IMeasure measure, IAggregation aggregation)
         {
-            return measure.Match(
-                v => MetricDescriptor.Types.ValueType.Double,        // Double
-                v => MetricDescriptor.Types.ValueType.Int64,         // Long
-                v => MetricDescriptor.Types.ValueType.Unspecified);  // Unrecognized
+            MetricKind metricKind = aggregation.ToMetricKind();
+            if (aggregation is IDistribution && (metricKind == MetricKind.Cumulative || metricKind == MetricKind.Gauge))
+                return ValueType.Distribution;
+
+            if (measure is IMeasureDouble && (metricKind == MetricKind.Cumulative || metricKind == MetricKind.Gauge))
+            {
+                return ValueType.Double;
+            }
+
+            if (measure is IMeasureLong && (metricKind == MetricKind.Cumulative || metricKind == MetricKind.Gauge))
+            {
+                return ValueType.Int64;
+            }
+
+            // TODO - zeltser - we currently don't support money and string as Opencensus doesn't support them yet
+            return ValueType.Unspecified;
         }
 
         public static LabelDescriptor ToLabelDescriptor(this ITagKey tagKey)
@@ -135,8 +148,8 @@ public static MetricDescriptor CreateMetricDescriptor(
             var unit = GetUnit(view.Aggregation, view.Measure);
             metricDescriptor.Unit = unit;
             metricDescriptor.MetricKind = view.Aggregation.ToMetricKind();
-            metricDescriptor.ValueType = view.Measure.ToValueType();
-            
+            metricDescriptor.ValueType = view.Measure.ToValueType(view.Aggregation);
+
             return metricDescriptor;
         }
 
@@ -236,6 +249,7 @@ public static Metric GetMetric(
         /// Convert ViewData to a list of TimeSeries, so that ViewData can be uploaded to Stackdriver.
         /// </summary>
         /// <param name="viewData">OpenCensus View</param>
+        /// <param name="metricDescriptor">Stackdriver Metric Descriptor</param>
         /// <param name="monitoredResource">Stackdriver Resource to which the metrics belong</param>
         /// <param name="domain">The metrics domain (namespace)</param>
         /// <returns></returns>
@@ -262,7 +276,7 @@ public static List<TimeSeries> CreateTimeSeriesList(
                 IAggregationData points = entry.Value;
 
                 timeSeries.Resource = monitoredResource;
-                timeSeries.ValueType = view.Measure.ToValueType();
+                timeSeries.ValueType = view.Measure.ToValueType(view.Aggregation);
                 timeSeries.MetricKind = view.Aggregation.ToMetricKind();
 
                 timeSeries.Metric = GetMetric(view, labels, metricDescriptor, domain);
@@ -288,25 +302,6 @@ private static Point ExtractPointInInterval(
                 Value = CreateTypedValue(aggregation, points),
                 Interval = CreateTimeInterval(startTime, endTime)
             };
-            /*
-            var ret = points.Match(
-                v => new Point { Value = new TypedValue { DoubleValue = v.Sum }, Interval = CreateTimeInterval(startTime, endTime) }, // ISumDataDouble
-                v => new Point { Value = new TypedValue { Int64Value = v.Sum }, Interval = CreateTimeInterval(startTime, endTime) }, // ISumDataLong
-                v => new Point { Value = new TypedValue { Int64Value = v.Count }, Interval = CreateTimeInterval(startTime, endTime) }, // ICountData
-                v => new Point { Value = new TypedValue { DoubleValue = v.Mean }, Interval = CreateTimeInterval(startTime, endTime) }, // IMeanData
-                v => new Point {
-                    Value = new TypedValue
-                    {
-                        DistributionValue = CreateDistribution(v, ((IDistribution)aggregation).BucketBoundaries),
-                    },
-                    Interval = CreateTimeInterval(startTime, endTime)
-                }, // IDistributionData
-                v => new Point { Value = new TypedValue { DoubleValue = v.LastValue }, Interval = CreateTimeInterval(startTime, endTime) }, // ILastValueDataDouble
-                v => new Point { Value = new TypedValue { Int64Value = v.LastValue }, Interval = CreateTimeInterval(startTime, endTime) }, // ILastValueDataLong
-                v => throw new InvalidOperationException());
-
-            return ret;
-            */
         }
 
         private static TimeInterval CreateTimeInterval(ITimestamp start, ITimestamp end)

src/OpenCensus.Exporter.Stackdriver/Implementation/StackdriverStatsConfiguration.cs

@@ -52,11 +52,20 @@ public class StackdriverStatsConfiguration
         /// <summary>
         /// Default Stats Configuration for Stackdriver
         /// </summary>
-        public static StackdriverStatsConfiguration Default { get; } = new StackdriverStatsConfiguration
+        public static StackdriverStatsConfiguration Default
         {
-            ExportInterval = DEFAULT_INTERVAL,
-            ProjectId = GoogleCloudResourceUtils.GetProjectId(),
-            MetricNamePrefix = string.Empty,
-        };
+            get
+            {
+                var defaultConfig = new StackdriverStatsConfiguration
+                {
+                    ExportInterval = DEFAULT_INTERVAL,
+                    ProjectId = GoogleCloudResourceUtils.GetProjectId(),
+                    MetricNamePrefix = string.Empty,
+                };
+
+                defaultConfig.MonitoredResource = GoogleCloudResourceUtils.GetDefaultResource(defaultConfig.ProjectId);
+                return defaultConfig;
+            }
+        }
     }
 }

src/OpenCensus.Exporter.Stackdriver/Implementation/StackdriverStatsExporter.cs

@@ -18,6 +18,7 @@ namespace OpenCensus.Exporter.Stackdriver.Implementation
 {
     using Google.Api;
     using Google.Api.Gax;
+    using Google.Api.Gax.Grpc;
     using Google.Cloud.Monitoring.V3;
     using Grpc.Core;
     using OpenCensus.Exporter.Stackdriver.Utils;
@@ -44,15 +45,19 @@ internal class StackdriverStatsExporter
         private const string CUSTOM_METRIC_DOMAIN = "custom.googleapis.com/";
         private const string CUSTOM_OPENCENSUS_DOMAIN = CUSTOM_METRIC_DOMAIN + "opencensus/";
 
+        // TODO - zeltser - figure out how to extract OpenCensus package version
+        private const string USER_AGENT = "opencensus-csharp [0.0]";
+        private const string USER_AGENT_KEY = "user-agent";
+
         private readonly string domain;
         private readonly string displayNamePrefix;
 
         private bool isStarted;
 
         /// <summary>
-        /// Interval between two subsequent metrics collection operations
+        /// Interval between two subsequent stats collection operations
         /// </summary>
-        private readonly TimeSpan collectionInterval = TimeSpan.FromSeconds(15);
+        private TimeSpan collectionInterval = TimeSpan.FromMinutes(1);
 
         /// <summary>
         /// Interval within which the cancellation should be honored
@@ -69,11 +74,16 @@ public StackdriverStatsExporter(
             GaxPreconditions.CheckNotNull(configuration, "configuration");
             GaxPreconditions.CheckNotNullOrEmpty(configuration.ProjectId, "configuration.ProjectId");
             GaxPreconditions.CheckNotNull(configuration.MonitoredResource, "configuration.MonitoredResource");
+            GaxPreconditions.CheckArgument(
+                configuration.ExportInterval != TimeSpan.Zero,
+                paramName: "configuration.ExportInterval", 
+                message: "Export interval can't be zero. Typically it's 1 minute");
 
             this.viewManager = viewManager;
             monitoredResource = configuration.MonitoredResource;
+            collectionInterval = configuration.ExportInterval;
             project = new ProjectName(configuration.ProjectId);
-            metricServiceClient = MetricServiceClient.Create();
+
             domain = GetDomain(configuration.MetricNamePrefix);
             displayNamePrefix = GetDisplayNamePrefix(configuration.MetricNamePrefix);
         }
@@ -85,6 +95,7 @@ public void Start()
                 if (!isStarted)
                 {
                     tokenSource = new CancellationTokenSource();
+                    metricServiceClient = CreateMetricServiceClient(tokenSource);
 
                     Task.Factory.StartNew(DoWork, tokenSource.Token);
 
@@ -248,6 +259,25 @@ private void Export()
             }
         }
 
+        private static MetricServiceClient CreateMetricServiceClient(CancellationTokenSource tokenSource)
+        {
+            // Make sure to add Opencensus header to every outgoing call to Stackdriver APIs
+            Action<Metadata> addOpencensusHeader = m => m.Add(USER_AGENT_KEY, USER_AGENT);
+            var callSettings = new CallSettings(
+                cancellationToken: tokenSource.Token,
+                credentials: null,
+                timing: null,
+                headerMutation: addOpencensusHeader,
+                writeOptions: WriteOptions.Default,
+                propagationToken: null);
+            var metricServiceSettings = new MetricServiceSettings()
+            {
+                CallSettings = callSettings
+            };
+
+            return MetricServiceClient.Create(settings: metricServiceSettings);
+        }
+
         private static string GetDomain(string metricNamePrefix)
         {
             string domain;

src/OpenCensus.Exporter.Stackdriver/Implementation/StackdriverTraceExporter.cs

@@ -27,7 +27,7 @@ static class SpanExtensions
     {
         /// <summary>
         /// Translating <see cref="ISpanData"/> to Stackdriver's Span
-        /// According to <see cref="https://cloud.google.com/trace/docs/reference/v2/rpc/google.devtools.cloudtrace.v2"/> specifications
+        /// According to <see href="https://cloud.google.com/trace/docs/reference/v2/rpc/google.devtools.cloudtrace.v2"/> specifications
         /// </summary>
         /// <param name="spanData">Span in OpenCensus format</param>
         /// <param name="projectId">Google Cloud Platform Project Id</param>