Add a new /deploy2/evaluate_push/ endpoint (#42524)

This PR adds a new endpoint, `/deploy2/evaluate_push/`, which allows the user to push modules and determine what are the indexes that are being added/deleted. Unlike `/deploy2/start_push`, this doesn’t start index backfill or schema validation.

GitOrigin-RevId: 39774de5f9d73490bb3cbd189ab9ac813d7fbefc

Author: Nicolapps

crates/application/src/deploy_config.rs

@@ -150,9 +150,63 @@ pub struct PushMetrics {
     pub occ_stats: OccRetryStats,
 }
 
+struct EvaluatedPushContents {
+    app: CheckedComponent,
+    auth_info: Vec<AuthInfo>,
+    component_definition_packages: BTreeMap<ComponentDefinitionPath, SourcePackage>,
+    evaluated_components: BTreeMap<ComponentDefinitionPath, EvaluatedComponentDefinition>,
+    external_deps_id: Option<ExternalDepsPackageId>,
+    user_environment_variables: BTreeMap<EnvVarName, EnvVarValue>,
+}
+
 impl<RT: Runtime> Application<RT> {
     #[fastrace::trace]
     pub async fn start_push(&self, config: &ProjectConfig) -> anyhow::Result<StartPushResponse> {
+        let EvaluatedPushContents {
+            app,
+            auth_info,
+            component_definition_packages,
+            mut evaluated_components,
+            external_deps_id,
+            user_environment_variables,
+        } = self.evaluate_push_contents(config).await?;
+
+        let schema_change = self
+            .handle_schema_change_in_start_push(&app, &evaluated_components)
+            .await?;
+        self.database
+            .load_indexes_into_memory(btreeset! { SCHEMAS_TABLE.clone() })
+            .await?;
+
+        // TODO(ENG-7533): Clean up exports from the start push response when we've
+        // updated clients to use `functions` directly.
+        for (path, definition) in evaluated_components.iter_mut() {
+            // We don't need to include exports for the root since we don't use codegen
+            // for the app's `api` object.
+            if path.is_root() {
+                continue;
+            }
+            anyhow::ensure!(definition.definition.exports.is_empty());
+            definition.definition.exports = file_based_exports(&definition.functions)?;
+        }
+
+        let resp = StartPushResponse {
+            environment_variables: user_environment_variables,
+            external_deps_id,
+            component_definition_packages,
+            app_auth: auth_info,
+            analysis: evaluated_components,
+            app,
+            schema_change,
+        };
+        Ok(resp)
+    }
+
+    #[fastrace::trace]
+    async fn evaluate_push_contents(
+        &self,
+        config: &ProjectConfig,
+    ) -> anyhow::Result<EvaluatedPushContents> {
         let unix_timestamp = self.runtime.unix_timestamp();
         let (external_deps_id, component_definition_packages) =
             self.upload_packages(config).await?;
@@ -206,7 +260,7 @@ impl<RT: Runtime> Application<RT> {
         )
         .await?;
 
-        let mut evaluated_components = self
+        let evaluated_components = self
             .evaluate_components(
                 config,
                 &component_definition_packages,
@@ -231,39 +285,18 @@ impl<RT: Runtime> Application<RT> {
         let ctx = TypecheckContext::new(&evaluated_components, &initializer_evaluator);
         let app = ctx.instantiate_root().await?;
 
-        let schema_change = self
-            ._handle_schema_change_in_start_push(&app, &evaluated_components)
-            .await?;
-        self.database
-            .load_indexes_into_memory(btreeset! { SCHEMAS_TABLE.clone() })
-            .await?;
-
-        // TODO(ENG-7533): Clean up exports from the start push response when we've
-        // updated clients to use `functions` directly.
-        for (path, definition) in evaluated_components.iter_mut() {
-            // We don't need to include exports for the root since we don't use codegen
-            // for the app's `api` object.
-            if path.is_root() {
-                continue;
-            }
-            anyhow::ensure!(definition.definition.exports.is_empty());
-            definition.definition.exports = file_based_exports(&definition.functions)?;
-        }
-
-        let resp = StartPushResponse {
-            environment_variables: user_environment_variables,
-            external_deps_id,
-            component_definition_packages,
-            app_auth: auth_info,
-            analysis: evaluated_components,
+        Ok(EvaluatedPushContents {
             app,
-            schema_change,
-        };
-        Ok(resp)
+            auth_info,
+            component_definition_packages,
+            evaluated_components,
+            external_deps_id,
+            user_environment_variables,
+        })
     }
 
     #[fastrace::trace]
-    async fn _handle_schema_change_in_start_push(
+    async fn handle_schema_change_in_start_push(
         &self,
         app: &CheckedComponent,
         evaluated_components: &BTreeMap<ComponentDefinitionPath, EvaluatedComponentDefinition>,
@@ -289,6 +322,20 @@ impl<RT: Runtime> Application<RT> {
         Ok(schema_change)
     }
 
+    #[fastrace::trace]
+    async fn handle_schema_change_in_evaluate_push(
+        &self,
+        app: &CheckedComponent,
+        evaluated_components: &BTreeMap<ComponentDefinitionPath, EvaluatedComponentDefinition>,
+    ) -> anyhow::Result<SchemaChange> {
+        let mut tx = self.begin(Identity::system()).await?;
+        let schema_change = ComponentConfigModel::new(&mut tx)
+            .start_component_schema_changes(app, evaluated_components)
+            .await?;
+        drop(tx);
+        Ok(schema_change)
+    }
+
     #[fastrace::trace]
     async fn evaluate_components(
         &self,
@@ -451,6 +498,24 @@ impl<RT: Runtime> Application<RT> {
             .await
     }
 
+    #[fastrace::trace]
+    pub async fn evaluate_push(
+        &self,
+        config: &ProjectConfig,
+    ) -> anyhow::Result<EvaluatePushResponse> {
+        let EvaluatedPushContents {
+            app,
+            evaluated_components,
+            ..
+        } = self.evaluate_push_contents(config).await?;
+
+        let schema_change = self
+            .handle_schema_change_in_evaluate_push(&app, &evaluated_components)
+            .await?;
+
+        Ok(EvaluatePushResponse { schema_change })
+    }
+
     #[fastrace::trace]
     pub async fn wait_for_schema(
         &self,
@@ -860,6 +925,11 @@ pub struct StartPushResponse {
     pub schema_change: SchemaChange,
 }
 
+#[derive(Debug)]
+pub struct EvaluatePushResponse {
+    pub schema_change: SchemaChange,
+}
+
 impl From<NodeDependencyJson> for NodeDependency {
     fn from(value: NodeDependencyJson) -> Self {
         Self {

crates/common/src/knobs.rs

@@ -821,7 +821,7 @@ pub static APPLICATION_MAX_CONCURRENT_HTTP_ACTIONS: LazyLock<usize> = LazyLock::
 });
 
 /// The maximum number of concurrent package uploads during
-/// `/api/deploy2/start_push`.
+/// `/api/deploy2/start_push` + `/api/deploy2/evaluate_push`.
 pub static APPLICATION_MAX_CONCURRENT_UPLOADS: LazyLock<usize> =
     LazyLock::new(|| env_config("APPLICATION_MAX_CONCURRENT_UPLOADS", 4));
 

crates/local_backend/src/deploy_config2.rs

@@ -4,6 +4,7 @@ use std::{
 };
 
 use application::deploy_config::{
+    EvaluatePushResponse,
     FinishPushDiff,
     SchemaStatusJson,
     StartPushRequest,
@@ -169,6 +170,22 @@ pub struct SerializedStartPushResponse {
     schema_change: SerializedSchemaChange,
 }
 
+impl TryFrom<EvaluatePushResponse> for SerializedEvaluatePushResponse {
+    type Error = anyhow::Error;
+
+    fn try_from(value: EvaluatePushResponse) -> Result<Self, Self::Error> {
+        Ok(Self {
+            schema_change: value.schema_change.try_into()?,
+        })
+    }
+}
+
+#[derive(Deserialize, Serialize)]
+#[serde(rename_all = "camelCase")]
+pub struct SerializedEvaluatePushResponse {
+    schema_change: SerializedSchemaChange,
+}
+
 #[derive(Deserialize, Serialize)]
 #[serde(rename_all = "camelCase")]
 struct SerializedIndexDiff {
@@ -205,6 +222,32 @@ pub async fn start_push(
     Ok(Json(SerializedStartPushResponse::try_from(resp)?))
 }
 
+// This endpoint is similar to `start_push`, but it doesn’t save the schema (so
+// it won’t start schema validation/index backfill). It can be used to determine
+// what will be the effects of a large push without starting work that can take
+// a long time on large instances.
+#[debug_handler]
+pub async fn evaluate_push(
+    State(st): State<LocalAppState>,
+    Json(req): Json<StartPushRequest>,
+) -> Result<impl IntoResponse, HttpResponseError> {
+    let _identity = must_be_admin_from_key_with_write_access(
+        st.application.app_auth(),
+        st.instance_name.clone(),
+        req.admin_key.clone(),
+    )
+    .await?;
+    let config = req.into_project_config().map_err(|e| {
+        anyhow::Error::new(ErrorMetadata::bad_request("InvalidConfig", e.to_string()))
+    })?;
+    let resp =
+        st.application.evaluate_push(&config).await.map_err(|e| {
+            e.wrap_error_message(|msg| format!("Hit an error while pushing:\n{msg}"))
+        })?;
+
+    Ok(Json(SerializedEvaluatePushResponse::try_from(resp)?))
+}
+
 const DEFAULT_SCHEMA_TIMEOUT_MS: u32 = 10_000;
 
 #[derive(Deserialize)]

crates/local_backend/src/router.rs

@@ -320,6 +320,10 @@ pub fn router(st: LocalAppState) -> Router {
         .route("/push_config", post(push_config))
         .route("/prepare_schema", post(prepare_schema))
         .route("/deploy2/start_push", post(deploy_config2::start_push))
+        .route(
+            "/deploy2/evaluate_push",
+            post(deploy_config2::evaluate_push),
+        )
         .route("/run_test_function", post(run_test_function))
         .route(
             "/deploy2/wait_for_schema",

npm-packages/convex/src/cli/lib/config.ts

@@ -1103,6 +1103,7 @@ export function diffConfig(
  * legacy push path:
  * - /api/push_config
  * modern push paths:
+ * - /api/deploy2/evaluate_push
  * - /api/deploy2/start_push
  * - /api/deploy2/finish_push
  *