Added Anonymous Telemetry (#223)

Author: ezodude

README.md

@@ -1,6 +1,6 @@
 # 🪡 orra
 
-Move beyond simple Crews and Agents. Use orra to build production-ready multi-agent applications that handle complex real-world interactions.
+Move beyond simple Crews and Agents. Use orra's production-grade planning to reliably run multi-agent workflows that require complex real-world interactions.
 
 ![](images/orra-diagram.png)
 
@@ -9,11 +9,11 @@ orra coordinates tasks across your existing stack, agents and any tools run as s
 * 🧠 Smart pre-evaluated execution plans
 * 🎯 Domain grounded
 * 🗿 Durable execution
-* 🚀 Go fast with tools as services
+* 🚀 Go fast and save cost with tools as services
 * ↩️ Revert state to handle failures
 * ⛑️ Automatic service health monitoring
 * 🔮 Real-time status tracking
-* 🪝 Webhooks for completion and failure notifications
+* 🪝 Webhooks for completion and failure monitoring
 
 [Learn why we built orra →](https://tinyurl.com/orra-launch-blog-post)
 
@@ -33,6 +33,7 @@ orra coordinates tasks across your existing stack, agents and any tools run as s
 - [Docs](#docs)
 - [Self Hosting & On-premises Deployment](#self-hosting--on-premises-deployment)
 - [Support](#support)
+- [Telemetry](#telemetry)
 - [License](#license)
 
 ## Installation
@@ -96,11 +97,11 @@ Download the latest CLI binary for your platform from our [releases page](https:
 
 ```shell
 # macOS
-curl -L https://github.com/orra-dev/orra/releases/download/v0.2.5/orra-darwin-arm64 -o /usr/local/bin/orra
+curl -L https://github.com/orra-dev/orra/releases/download/v0.2.6/orra-darwin-arm64 -o /usr/local/bin/orra
 chmod +x /usr/local/bin/orra
 
 # Linux
-curl -L https://github.com/ezodude/orra/releases/download/v0.2.5/orra-linux-amd64 -o /usr/local/bin/orra
+curl -L https://github.com/ezodude/orra/releases/download/v0.2.6/orra-linux-amd64 -o /usr/local/bin/orra
 chmod +x /usr/local/bin/orra
 
 # Verify installation
@@ -291,6 +292,10 @@ Need help? We're here to support you:
 - Report a bug or request a feature by creating an [issue](https://github.com/orra-dev/orra/issues/new?template=bug-report-feature-request.yml)
 - Start a [discussion](https://github.com/orra-dev/orra/discussions) about your ideas or questions
 
+## Telemetry
+
+See [telemetry.md](./docs/telemetry.md) for details on what is collected and how to opt out.
+
 ## License
 
 Orra is MPL-2.0 licensed.

docs/telemetry.md

@@ -0,0 +1,42 @@
+# Telemetry
+
+To help us improve orra, we collect minimal, privacy-preserving usage analytics. **Telemetry is anonymous by design, and you can easily opt out.**
+
+## What We Collect
+
+- **[Anonymous usage events](../planengine/events.go)** (e.g., server start/stop, project/service changes, execution plan outcomes).
+- **Non-identifiable execution plan and project tracking:**
+    - For events related to projects, we only track a **hashed version of the project ID**.
+    - For execution plan events, we only track a **hashed version of the execution plan ID**.
+- **No personal or sensitive data** is ever collected.
+- **No IP addresses** or environment details are sent.
+
+## How IDs Are Handled
+
+- We use a **SHA-256 hash** of each project or execution plan ID.  
+  This ensures IDs cannot be reversed or linked to your actual data.
+
+- Each orra instance also generates a random, anonymous identifier stored locally, used solely to distinguish unique installations.
+
+## How to Opt Out
+
+You can fully disable telemetry at any time:
+
+- **Via environment variable:**
+  Set the environment variable before starting orra:
+  ```shell
+  export ANONYMIZED_TELEMETRY=false
+  ```
+- **Via `.env` file:**  
+  Add the following line to your project's `.env` file:
+  ```text
+  ANONYMIZED_TELEMETRY=false
+  ```
+
+## Transparency and Trust
+
+- All telemetry code is open source and can be reviewed [here](../planengine/telemetry.go).
+- We follow industry best practices for privacy and transparency.
+- Telemetry helps us prioritize features, fix bugs, and ensure orra works reliably for everyone.
+
+**Thank you for helping us make orra better!**

planengine/_env

@@ -30,8 +30,18 @@ EMBEDDINGS_API_BASE_URL=https://api.openai.com/v1  # Default for OpenAI, change
 # Server port (default: 8005)
 # PORT=8005
 
+#========================================
+# PDDL CONFIGURATION
+#========================================
+
 # PDDL validator path (default: /usr/local/bin/Validate)
 # PDDL_VALIDATOR_PATH=/usr/local/bin/Validate
 
 # PDDL validation timeout (default: 30s)
 # PDDL_VALIDATION_TIMEOUT=30s
+
+#========================================
+# Telemetry CONFIGURATION
+#========================================
+
+ANONYMIZED_TELEMETRY=true

planengine/app.go

@@ -25,13 +25,14 @@ import (
 )
 
 type App struct {
-	Engine     *PlanEngine
-	Router     *mux.Router
-	Db         *BadgerDB
-	Cfg        Config
-	RootCtx    context.Context
-	RootCancel context.CancelFunc
-	Logger     zerolog.Logger
+	Engine       *PlanEngine
+	Router       *mux.Router
+	Db           *BadgerDB
+	Cfg          Config
+	TelemetrySvc *TelemetryService
+	RootCtx      context.Context
+	RootCancel   context.CancelFunc
+	Logger       zerolog.Logger
 }
 
 func NewApp(cfg Config, args []string) (*App, error) {
@@ -125,6 +126,10 @@ func (app *App) Run() {
 		if err := srv.ListenAndServe(); err != nil {
 			app.Logger.Info().Msg(err.Error())
 		}
+
+		app.TelemetrySvc.TrackEvent(EventServerStart, map[string]any{
+			"version": Version,
+		})
 	}()
 
 	c := make(chan os.Signal, 1)
@@ -162,6 +167,10 @@ func (app *App) gracefulShutdown(srv *http.Server, ctx context.Context) {
 		app.Logger.Error().Err(err).Msg("Error shutting down plan engine server")
 	}
 	app.Logger.Debug().Msg("http: All connections drained")
+
+	app.TelemetrySvc.TrackEvent(EventServerStop, map[string]any{
+		"version": Version,
+	})
 }
 
 func (app *App) RegisterProject(w http.ResponseWriter, r *http.Request) {

planengine/app_test.go

@@ -47,13 +47,20 @@ func setupTestApp(t *testing.T) (*App, *Project, func()) {
 		assert.NoError(t, err)
 	}
 
+	telemetrySvc := &TelemetryService{
+		client:  nil,
+		enabled: false,
+		logger:  logger,
+	}
+
 	plane := NewPlanEngine()
-	plane.Initialise(context.Background(), db, db, db, db, db, nil, nil, nil, &fakePddlValidator{}, nil, logger)
+	plane.Initialise(context.Background(), db, db, db, db, db, nil, nil, nil, &fakePddlValidator{}, nil, logger, telemetrySvc)
 
 	app := &App{
-		Router: mux.NewRouter(),
-		Engine: plane,
-		Logger: logger,
+		Router:       mux.NewRouter(),
+		Engine:       plane,
+		Logger:       logger,
+		TelemetrySvc: telemetrySvc,
 	}
 
 	app.configureRoutes()

planengine/compworker.go

@@ -47,6 +47,11 @@ func NewCompensationWorker(projectID, orchestrationID string, logManager *LogMan
 }
 
 func (w *CompensationWorker) Start(ctx context.Context, orchestrationID string) {
+	w.LogManager.planEngine.TelemetrySvc.TrackEvent(EventCompensationStarted, map[string]any{
+		"version":           Version,
+		"execution_plan_id": HashUUID(orchestrationID),
+	})
+
 	for _, candidate := range w.Candidates {
 		if err := w.processCandidate(ctx, candidate); err != nil {
 			w.LogManager.Logger.Error().
@@ -61,6 +66,15 @@ func (w *CompensationWorker) Start(ctx context.Context, orchestrationID string)
 			if strings.Contains(err.Error(), "expired") {
 				status = CompensationExpired
 				logType = CompensationExpiredLogType
+				w.LogManager.planEngine.TelemetrySvc.TrackEvent(EventCompensationExpired, map[string]any{
+					"version":           Version,
+					"execution_plan_id": HashUUID(orchestrationID),
+				})
+			} else {
+				w.LogManager.planEngine.TelemetrySvc.TrackEvent(EventCompensationFailed, map[string]any{
+					"version":           Version,
+					"execution_plan_id": HashUUID(orchestrationID),
+				})
 			}
 
 			// Log the compensation failure
@@ -109,6 +123,11 @@ func (w *CompensationWorker) Start(ctx context.Context, orchestrationID string)
 					go w.sendWebhookNotification(webhook, payload)
 				}
 			}
+		} else {
+			w.LogManager.planEngine.TelemetrySvc.TrackEvent(EventCompensationCompleted, map[string]any{
+				"version":           Version,
+				"execution_plan_id": HashUUID(orchestrationID),
+			})
 		}
 	}
 }

planengine/config.go

@@ -41,6 +41,7 @@ const (
 	PauseExecutionCode            = "PAUSE_EXECUTION"
 	StartJsonMarker               = "```json"
 	EndJsonMarker                 = "```"
+	AnonymouseIDFilename          = "orra.telemetry.uuid"
 )
 
 // Supported LLM and Embeddings models
@@ -100,6 +101,7 @@ type Config struct {
 	PddlValidatorPath     string        `envconfig:"default=/usr/local/bin/Validate"`
 	PddlValidationTimeout time.Duration `envconfig:"default=30s"`
 	StoragePath           string        `envconfig:"optional"`
+	AnonymizedTelemetry   bool          `envconfig:"default=true"`
 }
 
 func Load() (Config, error) {

planengine/engine.go

@@ -51,20 +51,7 @@ func NewPlanEngine() *PlanEngine {
 	return plane
 }
 
-func (p *PlanEngine) Initialise(
-	ctx context.Context,
-	pStorage ProjectStorage,
-	svcStorage ServiceStorage,
-	orchestrationStorage OrchestrationStorage,
-	groundingStorage GroundingStorage,
-	failedCompStorage FailedCompensationStorage,
-	logMgr *LogManager,
-	wsManager *WebSocketManager,
-	vCache *VectorCache,
-	pddlValid PddlValidator,
-	matcher SimilarityMatcher,
-	Logger zerolog.Logger,
-) {
+func (p *PlanEngine) Initialise(ctx context.Context, pStorage ProjectStorage, svcStorage ServiceStorage, orchestrationStorage OrchestrationStorage, groundingStorage GroundingStorage, failedCompStorage FailedCompensationStorage, logMgr *LogManager, wsManager *WebSocketManager, vCache *VectorCache, pddlValid PddlValidator, matcher SimilarityMatcher, Logger zerolog.Logger, telemetry *TelemetryService) {
 	p.pStorage = pStorage
 	p.svcStorage = svcStorage
 	p.orchestrationStorage = orchestrationStorage
@@ -76,6 +63,7 @@ func (p *PlanEngine) Initialise(
 	p.VectorCache = vCache
 	p.PddlValidator = pddlValid
 	p.SimilarityMatcher = matcher
+	p.TelemetrySvc = telemetry
 
 	if projects, err := pStorage.ListProjects(); err == nil {
 		p.Logger.Trace().Interface("Projects", projects).Msg("Loaded projects from DB")
@@ -180,6 +168,10 @@ func (p *PlanEngine) RegisterOrUpdateService(service *ServiceInfo) error {
 			Str("ProjectID", service.ProjectID).
 			Str("ServiceName", service.Name).
 			Msgf("Generating new service ID")
+		p.TelemetrySvc.TrackEvent(EventProjectServiceRegistered, map[string]any{
+			"version":    Version,
+			"project_id": HashUUID(service.ProjectID),
+		})
 	} else {
 		// Load existing service
 		existingService, err := p.svcStorage.LoadServiceByProjectID(service.ProjectID, service.ID)
@@ -194,6 +186,10 @@ func (p *PlanEngine) RegisterOrUpdateService(service *ServiceInfo) error {
 			Str("ServiceName", service.Name).
 			Int64("ServiceVersion", service.Version).
 			Msgf("Updating existing service")
+		p.TelemetrySvc.TrackEvent(EventProjectServiceUpdated, map[string]any{
+			"version":    Version,
+			"project_id": HashUUID(service.ProjectID),
+		})
 	}
 
 	if err := p.svcStorage.StoreService(service); err != nil {
@@ -341,6 +337,11 @@ func (p *PlanEngine) ApplyGroundingSpec(ctx context.Context, spec *GroundingSpec
 		Str("domain", spec.Domain).
 		Msgf("Added grounding spec with %d action uses cases", len(spec.UseCases))
 
+	p.TelemetrySvc.TrackEvent(EventProjectGroundingApplied, map[string]any{
+		"version":    Version,
+		"project_id": HashUUID(projectID),
+	})
+
 	return nil
 }
 
@@ -415,6 +416,11 @@ func (p *PlanEngine) RemoveGroundingSpecByName(projectID string, name string) er
 		Str("name", name).
 		Msg("Removed grounding spec")
 
+	p.TelemetrySvc.TrackEvent(EventProjectGroundingRemoved, map[string]any{
+		"version":    Version,
+		"project_id": HashUUID(projectID),
+	})
+
 	return nil
 }
 
@@ -468,6 +474,7 @@ func (p *PlanEngine) AddProject(project *Project) error {
 	}
 
 	p.projects[project.ID] = project
+	p.TelemetrySvc.TrackEvent(EventProjectAdded, map[string]any{"version": Version})
 	return nil
 }
 

planengine/events.go

@@ -0,0 +1,30 @@
+/*
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ *  License, v. 2.0. If a copy of the MPL was not distributed with this
+ *  file, You can obtain one at https://mozilla.org/MPL/2.0/.
+ */
+
+package main
+
+// Tracked telemetry events
+
+const (
+	EventServerStart                   = "server_start"
+	EventServerStop                    = "server_stop"
+	EventProjectAdded                  = "project_added"
+	EventProjectServiceRegistered      = "service_registered"
+	EventProjectServiceUpdated         = "service_updated"
+	EventExecutionPlanNotActionable    = "execution_plan_not_actionable"
+	EventExecutionPlanFailedCreation   = "execution_plan_failed_creation"
+	EventExecutionPlanFailedValidation = "execution_plan_failed_validation"
+	EventExecutionPlanAttempted        = "execution_plan_attempted"
+	EventExecutionPlanCompleted        = "execution_plan_completed"
+	EventExecutionPlanAborted          = "execution_plan_aborted"
+	EventExecutionPlanFailed           = "execution_plan_failed"
+	EventCompensationStarted           = "execution_plan_compensation_started"
+	EventCompensationCompleted         = "execution_plan_compensation_completed"
+	EventCompensationFailed            = "execution_plan_compensation_failed"
+	EventCompensationExpired           = "execution_plan_compensation_expired"
+	EventProjectGroundingApplied       = "project_grounding_applied"
+	EventProjectGroundingRemoved       = "project_grounding_removed"
+)