merge main

Author: briangregoryholmes

README.md

@@ -28,7 +28,7 @@
 
 <div align="center">
 
-[Docs](https://docs.rilldata.com/) • [Install](https://docs.rilldata.com/home/install) • [Quickstart](https://docs.rilldata.com/get-started/quickstart) • [Guides](https://docs.rilldata.com/guides) • [Reference](https://docs.rilldata.com/reference/project-files)
+[Docs](https://docs.rilldata.com/) • [Install](https://docs.rilldata.com/developers/get-started/install) • [Quickstart](https://docs.rilldata.com/developers/get-started/quickstart) • [Guides](https://docs.rilldata.com/developers/guides) • [Reference](https://docs.rilldata.com/reference/project-files)
 
 </div>
 

admin/database/postgres/migrations/0092.sql

@@ -0,0 +1,8 @@
+UPDATE deployments 
+SET desired_status = CASE 
+    WHEN status IN (1, 2, 4, 6) THEN 2  -- Pending, Running, Errored, Updating -> Running
+    WHEN status IN (5, 7) THEN 5        -- Stopped, Stopping -> Stopped
+    WHEN status IN (8, 9) THEN 9        -- Deleting, Deleted -> Deleted
+    ELSE 2                              -- Default to Running for any other case
+END
+WHERE desired_status = 0;

admin/github.go

@@ -355,8 +355,9 @@ func (s *Service) processGithubPush(ctx context.Context, event *github.PushEvent
 				// Don't trigger runtime reconcile for dev deployments, let the user manually pull changes to avoid any conflicts
 				continue
 			}
-			// We don't trigger runtime reconcile if the deployment is not ready
-			if depl.Status != database.DeploymentStatusRunning {
+			// Only trigger a reconcile for running/updating deployments.
+			// NOTE: Including "updating" here to avoid race conditions when there's a push and env config update happening simultaneously.
+			if depl.Status != database.DeploymentStatusRunning && depl.Status != database.DeploymentStatusUpdating {
 				s.Logger.Info("process github event: runtime reconcile not triggered, deployment is not ready", zap.String("project_id", project.ID), zap.String("deployment_id", depl.ID), zap.String("deployment_status", depl.Status.String()), observability.ZapCtx(ctx))
 				continue
 			}

admin/jobs/river/reconcile_deployment.go

@@ -9,6 +9,7 @@ import (
 	"github.com/rilldata/rill/admin/database"
 	"github.com/rilldata/rill/runtime/pkg/observability"
 	"github.com/riverqueue/river"
+	"go.opentelemetry.io/otel/attribute"
 	"go.uber.org/zap"
 )
 
@@ -28,6 +29,7 @@ type ReconcileDeploymentWorker struct {
 // such as starting, updating, stopping, and deleting deployments.
 // We handle all deployment state transitions in this job to ensure consistency and to avoid concurrent conflicting operations on the same deployment.
 func (w *ReconcileDeploymentWorker) Work(ctx context.Context, job *river.Job[ReconcileDeploymentArgs]) error {
+	observability.AddRequestAttributes(ctx, attribute.String("args.deployment_id", job.Args.DeploymentID))
 	depl, err := w.admin.DB.FindDeployment(ctx, job.Args.DeploymentID)
 	if err != nil {
 		if errors.Is(err, database.ErrNotFound) {

admin/server/server.go

@@ -179,12 +179,20 @@ func (s *Server) HTTPHandler(ctx context.Context) (http.Handler, error) {
 	adminv1.RegisterAIServiceServer(grpcServer, s)
 	adminv1.RegisterTelemetryServiceServer(grpcServer, s)
 
+	// Prepare CORS config.
+	// We currently only add CORS config on the transcoder and runtime proxy handlers.
+	// Other endpoints here don't need CORS config because they should not be used in cross-origin browser requests.
+	transcoderCORSMiddleware := cors.New(newCORSOptions(s.opts.AllowedOrigins, true)).Handler // Allow cookies for calls from ui.rilldata.com to admin.rilldata.com
+	runtimeProxyCORSMiddleware := cors.New(newCORSOptions([]string{"*"}, false)).Handler      // Allow any origin but no cookies. In the longer term, we should add explicit domain allowlisting per org or project.
+
 	// Add gRPC and gRPC-to-REST transcoder.
 	// This will be the fallback for REST routes like `/v1/ping` and GPRC routes like `/rill.admin.v1.AdminService/Ping`.
+	var transcoder http.Handler
 	transcoder, err := vanguardgrpc.NewTranscoder(grpcServer)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create transcoder: %w", err)
 	}
+	transcoder = transcoderCORSMiddleware(transcoder)
 
 	// Add auth cookie refresh specifically for the GetCurrentUser RPC.
 	// This is sufficient to refresh the cookie on each page load without unnecessarily refreshing cookies in each API call.
@@ -195,14 +203,14 @@ func (s *Server) HTTPHandler(ctx context.Context) (http.Handler, error) {
 	mux.Handle("/rill.admin.v1.AIService/", transcoder)
 	mux.Handle("/rill.admin.v1.TelemetryService/", transcoder)
 
-	// Add runtime proxy
+	// Add runtime proxy.
 	proxyHandler := observability.Middleware(
 		"runtime-proxy",
 		s.logger,
-		s.authenticator.HTTPMiddlewareLenient(httputil.Handler(s.runtimeProxyForOrgAndProject)),
+		runtimeProxyCORSMiddleware(s.authenticator.HTTPMiddlewareLenient(httputil.Handler(s.runtimeProxyForOrgAndProject))),
 	)
-	observability.MuxHandle(mux, "/v1/orgs/{org}/projects/{project}/runtime/{path...}", proxyHandler) // Backwards compatibility
-	observability.MuxHandle(mux, "/v1/organizations/{org}/projects/{project}/runtime/{path...}", proxyHandler)
+	observability.MuxHandle(mux, "/v1/orgs/{org}/projects/{project}/runtime/{path...}", proxyHandler)
+	observability.MuxHandle(mux, "/v1/organizations/{org}/projects/{project}/runtime/{path...}", proxyHandler) // Backwards compatibility
 
 	// Add backwards compatibility alias for iframe endpoint
 	observability.MuxHandle(mux, "/v1/organizations/{org}/projects/{project}/iframe", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -260,44 +268,9 @@ func (s *Server) HTTPHandler(ctx context.Context) (http.Handler, error) {
 	// 	}
 	// })
 
-	// Build CORS options for admin server
-
-	// If the AllowedOrigins contains a "*" we want to return the requester's origin instead of "*" in the "Access-Control-Allow-Origin" header.
-	// This is useful in development. In production, we set AllowedOrigins to non-wildcard values, so this does not have security implications.
-	// Details: https://github.com/rs/cors#allow--with-credentials-security-protection
-	var allowedOriginFunc func(string) bool
-	allowedOrigins := s.opts.AllowedOrigins
-	for _, origin := range s.opts.AllowedOrigins {
-		if origin == "*" {
-			allowedOriginFunc = func(origin string) bool { return true }
-			allowedOrigins = nil
-			break
-		}
-	}
-
-	corsOpts := cors.Options{
-		AllowedOrigins:  allowedOrigins,
-		AllowOriginFunc: allowedOriginFunc,
-		AllowedMethods: []string{
-			http.MethodHead,
-			http.MethodGet,
-			http.MethodPost,
-			http.MethodPut,
-			http.MethodPatch,
-			http.MethodDelete,
-		},
-		AllowedHeaders: []string{"*"},
-		// We use cookies for browser sessions, so this is required to allow ui.rilldata.com to make authenticated requests to admin.rilldata.com
-		AllowCredentials: true,
-		// Set max age to 1 hour (default if not set is 5 seconds)
-		MaxAge: 60 * 60,
-	}
-
-	// Wrap mux with CORS middleware
-	handlerWithAuth := s.authenticator.CookieToAuthHeader(mux)
-	handlerWithCors := cors.New(corsOpts).Handler(handlerWithAuth)
-	handler := middleware.TraceMiddleware(handlerWithCors)
-
+	// Wrap mux with final middleware and return
+	handler := s.authenticator.CookieToAuthHeader(mux) // Convert auth cookies to Authorization headers
+	handler = middleware.TraceMiddleware(handler)      // OpenTelemetry tracing
 	return handler, nil
 }
 
@@ -505,3 +478,38 @@ func checkUserAgent(ctx context.Context) (context.Context, error) {
 
 	return ctx, nil
 }
+
+// newCORSOptions creates config for the CORS middleware.
+// It supports passing a wildcard "*" in allowed origins to allow all origins.
+// Pass allowCredentials=true to allow cookies to be sent in CORS requests (necessary for endpoints accessed from ui.rilldata.com).
+func newCORSOptions(allowedOrigins []string, allowCredentials bool) cors.Options {
+	// If the AllowedOrigins contains a "*" we want to return the requester's origin instead of "*" in the "Access-Control-Allow-Origin" header.
+	// This is useful in development. In production, we set AllowedOrigins to non-wildcard values, so this does not have security implications.
+	// Details: https://github.com/rs/cors#allow--with-credentials-security-protection
+	var allowedOriginFunc func(string) bool
+	for _, origin := range allowedOrigins {
+		if origin == "*" {
+			allowedOriginFunc = func(origin string) bool { return true }
+			allowedOrigins = nil
+			break
+		}
+	}
+
+	return cors.Options{
+		AllowedOrigins:  allowedOrigins,
+		AllowOriginFunc: allowedOriginFunc,
+		AllowedMethods: []string{
+			http.MethodHead,
+			http.MethodGet,
+			http.MethodPost,
+			http.MethodPut,
+			http.MethodPatch,
+			http.MethodDelete,
+		},
+		AllowedHeaders: []string{"*"},
+		// We use cookies for browser sessions, so this must be true on endpoints that ui.rilldata.com access on admin.rilldata.com.
+		AllowCredentials: allowCredentials,
+		// Set max age to 1 hour (default if not set is 5 seconds)
+		MaxAge: 60 * 60,
+	}
+}

docs/README.md

@@ -1,6 +1,6 @@
 # `docs/`
 
-[![Netlify Status](https://api.netlify.com/api/v1/badges/23baf08e-2d3e-44db-8bd4-938e54467a29/deploy-status)](https://app.netlify.com/sites/rill-developer/deploys)
+[![Netlify Status](https://api.netlify.com/api/v1/badges/23baf08e-2d3e-44db-8bd4-938e54467a29/deploy-status)](https://app.netlify.com/sites/rill-developers/deploys)
 
 This folder contains docs for Rill, generated using [Docusaurus](https://docusaurus.io/) and deployed to [https://docs.rilldata.com](https://docs.rilldata.com).
 

docs/blog/0.10.md

@@ -10,14 +10,14 @@ description: Rill is the world's first AI-native business intelligence tool. Tru
 :::note
 ⚡ Rill Developer is a tool that makes it effortless to transform your datasets with SQL and create powerful, opinionated dashboards. These are our release notes for our `0.10` release, still in Tech Preview.
 
-[To try out Rill Developer, check out these instructions](/get-started/install) and [let us know over on Discord](https://discord.gg/2ubRfjC7Rh) if you encounter any problems or have ideas about how to improve Rill Developer!
+[To try out Rill Developer, check out these instructions](/developers/get-started/install) and [let us know over on Discord](https://discord.gg/2ubRfjC7Rh) if you encounter any problems or have ideas about how to improve Rill Developer!
 :::
 
 In our 0.6 release of Rill Developer we introduced dashboard leaderboards that present the top 7 dimension values for a given measure. Understanding the leads of each dimension helps us understand major players, but it is is hard to go deeper when you can only select 1 metric at a time for the very top of the pack. In the 0.10 release of Rill Developer we introduce *expanded leaderboard tables* to help you make these comparisons easily and focus on the impact of more segments.
 
 - **Expanded leaderboard metrics —** When we are exploring aggregated measures we want to be able to compare dimension values and see additional values beyond the top 7 when they are focused on a specific dimension’s impact. Our new dimension tables expand your leaderboards to provide a deeper view into all of your metrics for the top 250 values. You can use these values as filters.
 
-- **Nightly binaries —** Want to stay on the bleeding edge of our development? We have added support for nightly builds of our fast install binaries. You can easily switch between the [production release](/get-started/install) and the nightly build by re-running the script for the version you want. Try it yourself using our nightly build script:
+- **Nightly binaries —** Want to stay on the bleeding edge of our development? We have added support for nightly builds of our fast install binaries. You can easily switch between the [production release](/developers/get-started/install) and the nightly build by re-running the script for the version you want. Try it yourself using our nightly build script:
 
   `curl -s [https://cdn.rilldata.com/install.sh](https://cdn.rilldata.com/install.sh) | bash -s -- --nightly`
 

docs/blog/0.11.md

@@ -11,7 +11,7 @@ description: Rill is the world's first AI-native business intelligence tool. Tru
 :::note
 ⚡ Rill Developer is a tool that makes it effortless to transform your datasets with SQL and create powerful, opinionated dashboards. These are our release notes for our `0.11` release, still in Tech Preview.
 
-[To try out Rill Developer, check out these instructions](/get-started/install) and [let us know over on Discord](https://discord.gg/2ubRfjC7Rh) if you encounter any problems or have ideas about how to improve Rill Developer!
+[To try out Rill Developer, check out these instructions](/developers/get-started/install) and [let us know over on Discord](https://discord.gg/2ubRfjC7Rh) if you encounter any problems or have ideas about how to improve Rill Developer!
 :::
 
 

docs/blog/0.12.md

@@ -11,7 +11,7 @@ description: Rill is the world's first AI-native business intelligence tool. Tru
 :::note
 ⚡ Rill Developer is a tool that makes it effortless to transform your datasets with SQL and create powerful, opinionated dashboards. These are our release notes for our `0.12` release, still in Tech Preview.
 
-[To try out Rill Developer, check out these instructions](/get-started/install) and [let us know over on Discord](https://discord.gg/2ubRfjC7Rh) if you encounter any problems or have ideas about how to improve Rill Developer!
+[To try out Rill Developer, check out these instructions](/developers/get-started/install) and [let us know over on Discord](https://discord.gg/2ubRfjC7Rh) if you encounter any problems or have ideas about how to improve Rill Developer!
 :::
 
 

docs/blog/0.13.md

@@ -11,15 +11,15 @@ description: Rill is the world's first AI-native business intelligence tool. Tru
 :::note
 ⚡ Rill Developer is a tool that makes it effortless to transform your datasets with SQL and create powerful, opinionated dashboards. These are our release notes for our `0.13` release, still in Tech Preview.
 
-[To try out Rill Developer, check out these instructions](/get-started/install) and [let us know over on Discord](https://discord.gg/2ubRfjC7Rh) if you encounter any problems or have ideas about how to improve Rill Developer!
+[To try out Rill Developer, check out these instructions](/developers/get-started/install) and [let us know over on Discord](https://discord.gg/2ubRfjC7Rh) if you encounter any problems or have ideas about how to improve Rill Developer!
 :::
 
 
 Historically in Rill Developer it could be cumbersome to maintain fresh data sources through the local file system. Keeping data sources fresh and relevant to your current questions helps you proactively address what is happening in real time. If data is being collected over time and stored in a data warehouse, you would need to repeatedly track down, download, and import the files you care about to utilize the power of Rill. The process of refreshing data shouldn’t feel like a barrier to exploring the current trends. 
 
 In this release of Rill Developer we have added the ability to bring remotely stored files from GCS / S3 into your local Rill application through a remote connection. This gives you all of the incredible analytical power of DuckDB with Rill's powerful profiling and exploratory dashboard. Try Rill connections and spend less time digging in your data warehouse for the latest files and more time exploring what insights they hold.
 
-NOTE: We will not support Docker and npm install paths moving forward to help us focus on a high quality binary install experience. Please try Rill with our [binary](/get-started/install) moving forward.
+NOTE: We will not support Docker and npm install paths moving forward to help us focus on a high quality binary install experience. Please try Rill with our [binary](/developers/get-started/install) moving forward.
 
 - **S3 and GCS stored file connections —**  Connections keep your local data fresh and evergreen by establishing the location and credentials needed to access your data stored in the cloud. Our S3 and GCS connectors bring remote parquet and CSV files into Rill Developer’s sources for exploration with the click of a button. Each time the source is refreshed it will fully replace the current source with the remote file keeping your models and dashboards current over time. Rill picks up your local environment credentials to access protected remote files making it extremely low-touch to get started. Check out [our documentation](https://docs.rilldata.com/) to learn more.
 - **Making your experience better —** This release included many fixes to ensure we are providing a more cohesive application experience. We aligned interactions across different surfaces, reviewed our use of color, revised our copy, fixed browser some jank, and corrected performance regressions. A few of our notable contributions include: