Run the migration on this iteration.

Author: ChrisPenner

sql/2025-02-25_causal_depth_migration.sql …rs/2025-02-25_causal_depth_migration.sqlsql/2025-02-25_causal_depth_migration.sql renamed to sql/migration-helpers/2025-02-25_causal_depth_migration.sql sql/2025-02-25_causal_depth_migration.sql renamed to sql/migration-helpers/2025-02-25_causal_depth_migration.sql

@@ -43,3 +43,8 @@ INSERT INTO unfinished_component_depths (id)
       SELECT FROM component_depth cd WHERE cd.component_hash_id = ch.id
     );  
 
+-- Afterwards
+DROP TABLE unfinished_causal_depths;
+DROP TABLE unfinished_namespace_depths;
+DROP TABLE unfinished_patch_depths;
+DROP TABLE unfinished_component_depths;

src/Share/BackgroundJobs/EntityDepthMigration/Queries.hs

@@ -38,6 +38,7 @@ updateComponentDepths = do
               t.component_hash_id = ch.id
               AND cr_sub.type_id = t.id AND cd.depth IS NULL
         )
+        LIMIT 10000
     ), updated(component_hash_id, x) AS (
         SELECT ch.component_hash_id, update_component_depth(ch.component_hash_id)
           FROM updatable_components ch
@@ -89,6 +90,7 @@ updatePatchDepths = do
               WHERE ptm.patch_id = p.id
                 AND cd.depth IS NULL
           )
+        LIMIT 10000
     ), updated(patch_id, x) AS (
         SELECT up.patch_id, update_patch_depth(up.patch_id)
           FROM updatable_patches up
@@ -176,6 +178,7 @@ updateNamespaceDepths = do
               WHERE nt.namespace_hash_id = n.id
               AND cd.depth IS NULL
           )
+        LIMIT 10000
     ), updated(namespace_hash_id, x) AS (
         SELECT un.namespace_hash_id, update_namespace_depth(un.namespace_hash_id)
           FROM updatable_namespaces un
@@ -210,6 +213,7 @@ updateCausalDepths = do
               WHERE ca.causal_id = c.id
               AND cd.depth IS NULL
         )
+        LIMIT 10000
     ), updated(causal_id) AS (
         SELECT c.id, update_causal_depth(c.id)
           FROM updatable_causals c

src/Share/BackgroundJobs/EntityDepthMigration/Worker.hs

@@ -32,6 +32,7 @@ worker scope = do
 -- NOTE: this process doesn't insert the row into serialized_components, you'll need to do that manually after the automated migration is finished.
 computeComponentDepths :: AuthZ.AuthZReceipt -> Background ()
 computeComponentDepths !_authZReceipt = do
+  Logging.logInfoText $ "Computing component depth for batch"
   PG.runTransaction Q.updateComponentDepths >>= \case
     0 -> do
       Logging.logInfoText $ "Done processing component depth"
@@ -43,6 +44,7 @@ computeComponentDepths !_authZReceipt = do
 
 computePatchDepths :: AuthZ.AuthZReceipt -> Background ()
 computePatchDepths !_authZReceipt = do
+  Logging.logInfoText $ "Computing patch depth for batch"
   PG.runTransaction Q.updatePatchDepths >>= \case
     0 -> do
       Logging.logInfoText $ "Done processing patch depth"
@@ -54,6 +56,7 @@ computePatchDepths !_authZReceipt = do
 
 computeNamespaceAndCausalDepths :: AuthZ.AuthZReceipt -> Background ()
 computeNamespaceAndCausalDepths !authZReceipt = do
+  Logging.logInfoText $ "Computing namespace and causal depth for batch"
   PG.runTransaction Q.updateNamespaceDepths >>= \case
     namespaceN -> do
       PG.runTransaction Q.updateCausalDepths >>= \case

src/Share/Web/UCM/SyncV2/Impl.hs

@@ -48,7 +48,7 @@ batchSize :: Int32
 batchSize = 1000
 
 streamSettings :: Hash32 -> Maybe SyncV2.BranchRef -> StreamInitInfo
-streamSettings rootCausalHash rootBranchRef = StreamInitInfo {version = SyncV2.Version 1, entitySorting = SyncV2.DependenciesFirst, numEntities = Nothing, rootCausalHash, rootBranchRef}
+streamSettings rootCausalHash rootBranchRef = StreamInitInfo {version = SyncV2.Version 1, entitySorting = SyncV2.Unsorted, numEntities = Nothing, rootCausalHash, rootBranchRef}
 
 server :: Maybe UserId -> SyncV2.Routes WebAppServer
 server mayUserId =

src/Share/Web/UCM/SyncV2/Queries.hs

@@ -50,7 +50,7 @@ allSerializedDependenciesOfCausalCursor cid exceptCausalHashes = do
         JOIN component_hashes ch ON dh.hash = ch.base32
     |]
   cursor <-
-    PGCursor.newRowCursor
+    PGCursor.newRowCursor @(CBORBytes TempEntity, Hash32, Maybe Int32)
       "serialized_entities"
       [sql|
         WITH RECURSIVE transitive_causals(causal_id, causal_hash, causal_namespace_hash_id) AS (
@@ -178,32 +178,44 @@ allSerializedDependenciesOfCausalCursor cid exceptCausalHashes = do
                JOIN serialized_components sc ON sc.user_id = #{ownerUserId} AND tc.component_hash_id = sc.component_hash_id
                JOIN bytes ON sc.bytes_id = bytes.id
                JOIN component_hashes ch ON tc.component_hash_id = ch.id
-               JOIN component_depth cd ON ch.id = cd.component_hash_id
+               LEFT JOIN component_depth cd ON ch.id = cd.component_hash_id
            )
            UNION ALL
            (SELECT bytes.bytes, ap.patch_hash, pd.depth
              FROM all_patches ap
                JOIN serialized_patches sp ON ap.patch_id = sp.patch_id
                JOIN bytes ON sp.bytes_id = bytes.id
-               JOIN patch_depth pd ON ap.patch_id = pd.patch_id
+               LEFT JOIN patch_depth pd ON ap.patch_id = pd.patch_id
            )
            UNION ALL
            (SELECT bytes.bytes, an.namespace_hash, nd.depth
              FROM all_namespaces an
                JOIN serialized_namespaces sn ON an.namespace_hash_id = sn.namespace_hash_id
                JOIN bytes ON sn.bytes_id = bytes.id
-               JOIN namespace_depth nd ON an.namespace_hash_id = nd.namespace_hash_id
+               LEFT JOIN namespace_depth nd ON an.namespace_hash_id = nd.namespace_hash_id
            )
            UNION ALL
            (SELECT bytes.bytes, tc.causal_hash, cd.depth
              FROM transitive_causals tc
                JOIN serialized_causals sc ON tc.causal_id = sc.causal_id
                JOIN bytes ON sc.bytes_id = bytes.id
-               JOIN causal_depth cd ON tc.causal_id = cd.causal_id
+               LEFT JOIN causal_depth cd ON tc.causal_id = cd.causal_id
            )
-           ORDER BY depth ASC
+           -- Re-add this once the migration is done.
+           -- Put them in dependency order, nulls come first because we want to bail and
+           -- report an error
+           -- if we somehow are missing a depth.
+           -- ORDER BY depth ASC NULLS FIRST
   |]
-  pure cursor
+  -- pure
+  --   ( cursor <&> \(bytes, hash, depth) -> case depth of
+  --       -- This should never happen, but is a sanity check in case we're missing a depth.
+  --       -- Better than silently omitting a required result.
+  --       Nothing -> error $ "allSerializedDependenciesOfCausalCursor: Missing depth for entity: " <> show hash
+  --       Just _ -> (bytes, hash)
+  --   )
+  pure
+    (cursor <&> \(bytes, hash, _depth) -> (bytes, hash))
 
 spineAndLibDependenciesOfCausalCursor :: CausalId -> CodebaseM e (PGCursor (Hash32, IsCausalSpine, IsLibRoot))
 spineAndLibDependenciesOfCausalCursor cid = do