Skip to content

Speed up the dependents endpoints #147

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ChrisPenner merged 2 commits into from
Sep 17, 2025
Merged

Speed up the dependents endpoints #147

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
f1a3783
Add ability to skip deps and transitive deps when looking up names
ChrisPenner Sep 17, 2025
b7742dc
Add Name Search Scope direcives to endpoints
ChrisPenner Sep 17, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
196 changes: 196 additions & 0 deletions sql/2025-09-17_optional_transitive_names.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,196 @@
-- Add an additional argument for whether to include dependencies or transitive dependencies.
CREATE FUNCTION term_names_for_ref(
arg_bh_id integer,
arg_namespace_prefix text,
arg_reversed_name_prefix text,
arg_should_suffixify boolean,
arg_referent_builtin text,
arg_referent_component_hash_id integer,
arg_referent_component_index bigint,
arg_referent_constructor_index bigint,
arg_include_dependencies boolean,
arg_include_transitive_dependencies boolean
) RETURNS TABLE (
reversed_name text,
suffixified_name text
) AS $$
DECLARE
names name_with_suffix[];
BEGIN
SELECT array_agg(ROW(names.reversed_name, names.suffixified_name)::name_with_suffix) INTO names
FROM (
SELECT stnl.reversed_name,
CASE
WHEN arg_should_suffixify THEN suffixify_term_fqn(arg_bh_id, arg_namespace_prefix, '', ROW(stnl.*))
ELSE stnl.reversed_name
END AS suffixified_name
FROM scoped_term_name_lookup stnl
WHERE root_branch_hash_id = arg_bh_id
-- This may seem overly verbose, but it nudges the query planner to use the
-- correct partial index, which is keyed on whether the refBuiltin is null or not.
AND (
(arg_referent_builtin IS NULL
AND referent_builtin IS NULL
AND referent_component_hash_id = arg_referent_component_hash_id
AND referent_component_index = arg_referent_component_index
AND referent_constructor_index IS NOT DISTINCT FROM arg_referent_constructor_index
)
OR
( arg_referent_builtin IS NOT NULL
AND referent_builtin = arg_referent_builtin
)
)
AND (arg_namespace_prefix = '' OR namespace LIKE like_escape(arg_namespace_prefix) || '%')
AND (arg_reversed_name_prefix = '' OR stnl.reversed_name LIKE like_escape(arg_reversed_name_prefix) || '%')
UNION ALL
SELECT (names.reversed_name || mount.reversed_mount_path) AS reversed_name,
CASE
WHEN arg_should_suffixify THEN suffixify_term_fqn(arg_bh_id, arg_namespace_prefix, mount.reversed_mount_path, ROW(names.*))
ELSE names.reversed_name || mount.reversed_mount_path
END AS suffixified_name
FROM name_lookup_mounts mount
INNER JOIN scoped_term_name_lookup names ON names.root_branch_hash_id = mount.mounted_root_branch_hash_id
WHERE arg_include_dependencies
AND mount.parent_root_branch_hash_id = arg_bh_id
AND (arg_namespace_prefix = '' OR mount.mount_path LIKE like_escape(arg_namespace_prefix) || '%')
AND (
(arg_referent_builtin IS NULL
AND referent_builtin IS NULL
AND referent_component_hash_id = arg_referent_component_hash_id
AND referent_component_index = arg_referent_component_index
AND referent_constructor_index IS NOT DISTINCT FROM arg_referent_constructor_index
)
OR
( arg_referent_builtin IS NOT NULL
AND referent_builtin = arg_referent_builtin
)
)
AND (arg_reversed_name_prefix = '' OR names.reversed_name LIKE like_escape(arg_reversed_name_prefix) || '%')
) AS names;

IF NOT arg_include_transitive_dependencies OR (names IS NOT NULL AND array_length(names, 1) > 0) THEN
RETURN QUERY
SELECT n.reversed_name, n.suffixified_name
FROM unnest(names) AS n(reversed_name, suffixified_name);
ELSE
RETURN QUERY
SELECT (stnl.reversed_name || reversed_mount_path) AS reversed_name,
CASE
WHEN arg_should_suffixify THEN suffixify_term_fqn(arg_bh_id, arg_namespace_prefix, reversed_mount_path, ROW(stnl.*))
ELSE stnl.reversed_name || reversed_mount_path
END AS suffixified_name
FROM transitive_dependency_mounts(arg_bh_id)
INNER JOIN scoped_term_name_lookup stnl
ON stnl.root_branch_hash_id = transitive_dependency_mounts.root_branch_hash_id
WHERE (
(arg_referent_builtin IS NULL
AND referent_builtin IS NULL
AND referent_component_hash_id = arg_referent_component_hash_id
AND referent_component_index = arg_referent_component_index
AND referent_constructor_index IS NOT DISTINCT FROM arg_referent_constructor_index
)
OR
( arg_referent_builtin IS NOT NULL
AND referent_builtin = arg_referent_builtin
)
) AND (arg_reversed_name_prefix = '' OR stnl.reversed_name LIKE like_escape(arg_reversed_name_prefix) || '%')
LIMIT 1;
END IF;
END;
$$ LANGUAGE plpgsql STABLE;


CREATE FUNCTION type_names_for_ref(
arg_bh_id integer,
arg_namespace_prefix text,
arg_reversed_name_prefix text,
arg_should_suffixify boolean,
arg_reference_builtin text,
arg_reference_component_hash_id integer,
arg_reference_component_index bigint,
arg_include_dependencies boolean,
arg_include_transitive_dependencies boolean
) RETURNS TABLE (
reversed_name text,
suffixified_name text
) AS $$
DECLARE
names name_with_suffix[];
BEGIN
SELECT array_agg(ROW(names.reversed_name, names.suffixified_name)::name_with_suffix) INTO names
FROM (
SELECT stnl.reversed_name,
CASE
WHEN arg_should_suffixify THEN suffixify_type_fqn(arg_bh_id, arg_namespace_prefix, '', ROW(stnl.*))
ELSE stnl.reversed_name
END AS suffixified_name
FROM scoped_type_name_lookup stnl
WHERE root_branch_hash_id = arg_bh_id
-- This may seem overly verbose, but it nudges the query planner to use the
-- correct partial index, which is keyed on whether the refBuiltin is null or not.
AND (
(arg_reference_builtin IS NULL
AND reference_builtin IS NULL
AND reference_component_hash_id = arg_reference_component_hash_id
AND reference_component_index = arg_reference_component_index
)
OR
( arg_reference_builtin IS NOT NULL
AND reference_builtin = arg_reference_builtin
)
)
AND (arg_namespace_prefix = '' OR namespace LIKE like_escape(arg_namespace_prefix) || '%')
AND (arg_reversed_name_prefix = '' OR stnl.reversed_name LIKE like_escape(arg_reversed_name_prefix) || '%')
UNION ALL
SELECT (names.reversed_name || mount.reversed_mount_path) AS reversed_name,
CASE
WHEN arg_should_suffixify THEN suffixify_type_fqn(arg_bh_id, arg_namespace_prefix, mount.reversed_mount_path, ROW(names.*))
ELSE names.reversed_name || mount.reversed_mount_path
END AS suffixified_name
FROM name_lookup_mounts mount
INNER JOIN scoped_type_name_lookup names ON names.root_branch_hash_id = mount.mounted_root_branch_hash_id
WHERE arg_include_dependencies
AND mount.parent_root_branch_hash_id = arg_bh_id
AND (arg_namespace_prefix = '' OR mount.mount_path LIKE like_escape(arg_namespace_prefix) || '%')
AND (
(arg_reference_builtin IS NULL
AND reference_builtin IS NULL
AND reference_component_hash_id = arg_reference_component_hash_id
AND reference_component_index = arg_reference_component_index
)
OR
( arg_reference_builtin IS NOT NULL
AND reference_builtin = arg_reference_builtin
)
) AND (arg_reversed_name_prefix = '' OR names.reversed_name LIKE like_escape(arg_reversed_name_prefix) || '%')
) AS names;

IF NOT arg_include_transitive_dependencies OR (names IS NOT NULL AND array_length(names, 1) > 0) THEN
RETURN QUERY
SELECT n.reversed_name, n.suffixified_name
FROM unnest(names) AS n(reversed_name, suffixified_name);
ELSE
RETURN QUERY
SELECT (stnl.reversed_name || reversed_mount_path) AS reversed_name,
CASE
WHEN arg_should_suffixify THEN suffixify_type_fqn(arg_bh_id, arg_namespace_prefix, reversed_mount_path, ROW(stnl.*))
ELSE stnl.reversed_name || reversed_mount_path
END AS suffixified_name
FROM transitive_dependency_mounts(arg_bh_id)
INNER JOIN scoped_type_name_lookup stnl
ON stnl.root_branch_hash_id = transitive_dependency_mounts.root_branch_hash_id
WHERE (
(arg_reference_builtin IS NULL
AND reference_builtin IS NULL
AND reference_component_hash_id = arg_reference_component_hash_id
AND reference_component_index = arg_reference_component_index
)
OR
( arg_reference_builtin IS NOT NULL
AND reference_builtin = arg_reference_builtin
)
) AND (arg_reversed_name_prefix = '' OR stnl.reversed_name LIKE like_escape(arg_reversed_name_prefix) || '%')
LIMIT 1;
END IF;
END;
$$ LANGUAGE plpgsql STABLE;
5 changes: 3 additions & 2 deletions src/Share/BackgroundJobs/Search/DefinitionSync.hs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ import Share.Postgres qualified as PG
import Share.Postgres.Cursors qualified as Cursors
import Share.Postgres.IDs (BranchHashId)
import Share.Postgres.NameLookups.Ops qualified as NLOps
import Share.Postgres.NameLookups.Queries (NameSearchScope (..))
import Share.Postgres.NamesPerspective.Ops qualified as NPOps
import Share.Postgres.NamesPerspective.Types (NamesPerspective (..))
import Share.Postgres.Notifications qualified as Notif
Expand Down Expand Up @@ -243,7 +244,7 @@ syncTerms codebase namesPerspective rootBranchHashId termsCursor = do

-- It's much more efficient to build only one PPE per batch.
let allDeps = setOf (folded . folding tokens . folded . to LD.TypeReference) refDocs
pped <- PG.timeTransaction "Build PPED" $ PPEPostgres.ppedForReferences namesPerspective allDeps
pped <- PG.timeTransaction "Build PPED" $ PPEPostgres.ppedForReferences TransitiveDependencies namesPerspective allDeps
let ppe = PPED.unsuffixifiedPPE pped
let namedDocs :: [DefinitionDocument Name (Name, ShortHash)]
namedDocs =
Expand Down Expand Up @@ -426,7 +427,7 @@ syncTypes codebase codeCache namesPerspective rootBranchHashId typesCursor = do
}
-- It's much more efficient to build only one PPE per batch.
let allDeps = setOf (folded . folding tokens . folded . to LD.TypeReference) defDocuments
pped <- PPEPostgres.ppedForReferences namesPerspective allDeps
pped <- PPEPostgres.ppedForReferences TransitiveDependencies namesPerspective allDeps
let ppe = PPED.unsuffixifiedPPE pped
let namedDocs :: V.Vector (DefinitionDocument Name (Name, ShortHash))
namedDocs =
Expand Down
10 changes: 5 additions & 5 deletions src/Share/Names/Postgres.hs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ import Data.Set qualified as Set
import Share.Postgres qualified as PG
import Share.Postgres.NameLookups.Conversions qualified as CV
import Share.Postgres.NameLookups.Ops qualified as NameLookupOps
import Share.Postgres.NameLookups.Queries (ShouldSuffixify (NoSuffixify))
import Share.Postgres.NameLookups.Queries (NameSearchScope, ShouldSuffixify (NoSuffixify))
import Share.Postgres.NameLookups.Types qualified as NameLookups
import Share.Postgres.NamesPerspective.Types (NamesPerspective)
import Share.Postgres.Refs.Types
Expand All @@ -19,8 +19,8 @@ import Unison.Names qualified as Names
import Unison.Reference qualified as V1
import Unison.Referent qualified as V1

namesForReferences :: forall m. (PG.QueryM m) => NamesPerspective m -> Set LabeledDependency -> m Names
namesForReferences namesPerspective refs = do
namesForReferences :: forall m. (PG.QueryM m) => NameSearchScope -> NamesPerspective m -> Set LabeledDependency -> m Names
namesForReferences namesScope namesPerspective refs = do
(pgRefTerms, pgRefTypes) <-
Set.toList refs
& CV.labeledDependencies1ToPG
Expand All @@ -38,7 +38,7 @@ namesForReferences namesPerspective refs = do
& asListOf trav %%~ \refs -> do
let pgRefs = snd <$> refs
typeNames :: [[(NameLookups.ReversedName, NameLookups.ReversedName)]] <-
NameLookupOps.typeNamesForRefsWithinNamespaceOf namesPerspective Nothing NoSuffixify traversed pgRefs
NameLookupOps.typeNamesForRefsWithinNamespaceOf namesPerspective Nothing NoSuffixify namesScope traversed pgRefs
pure $ do
((ref, _pgRef), names) <- zip refs typeNames
pure $ do
Expand All @@ -51,7 +51,7 @@ namesForReferences namesPerspective refs = do
& asListOf trav %%~ \refs -> do
let pgRefs = snd <$> refs
termNames :: [[(NameLookups.ReversedName, NameLookups.ReversedName)]] <-
NameLookupOps.termNamesForRefsWithinNamespaceOf namesPerspective Nothing NoSuffixify traversed pgRefs
NameLookupOps.termNamesForRefsWithinNamespaceOf namesPerspective Nothing NoSuffixify namesScope traversed pgRefs
pure $ do
((ref, _pgRef), names) <- zip refs termNames
pure $ do
Expand Down
3 changes: 2 additions & 1 deletion src/Share/NamespaceDiffs.hs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@ import Share.Postgres qualified as PG
import Share.Postgres.Definitions.Queries qualified as DefnsQ
import Share.Postgres.IDs (BranchHashId)
import Share.Postgres.NameLookups.Ops qualified as NL
import Share.Postgres.NameLookups.Queries (NameSearchScope (TransitiveDependencies))
import Share.Postgres.NameLookups.Types (NameLookupReceipt)
import Share.Postgres.NameLookups.Types qualified as NL
import Share.Postgres.NamesPerspective.Ops qualified as NPOps
Expand Down Expand Up @@ -298,7 +299,7 @@ computeThreeWayNamespaceDiff codebaseEnvs2 aliceCodeCache bobCodeCache branchHas
traverse NPOps.namesPerspectiveForRoot branchHashIds
names <-
PG.transactionSpan "load names" mempty do
sequence (PGNames.namesForReferences <$> perspectives <*> ThreeWay.toTwoOrThreeWay dependencies)
sequence (PGNames.namesForReferences TransitiveDependencies <$> perspectives <*> ThreeWay.toTwoOrThreeWay dependencies)
pure (TwoOrThreeWay.toThreeWay Names.empty names)

Merge.makeDiffblob
Expand Down
12 changes: 6 additions & 6 deletions src/Share/Postgres/NameLookups/Ops.hs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,20 +74,20 @@ fuzzySearchDefinitions includeDependencies namesPerspective@NamesPerspective {re

-- | Get the list of (fqn, suffixified) names for a given Referent.
-- If 'shouldSuffixify' is 'NoSuffixify', the suffixified name will be the same as the fqn.
termNamesForRefsWithinNamespaceOf :: (PG.QueryM m) => NamesPerspective m -> Maybe ReversedName -> ShouldSuffixify -> Traversal s t PGReferent [(ReversedName, ReversedName)] -> s -> m t
termNamesForRefsWithinNamespaceOf namesPerspective maySuffix shouldSuffixify trav s = do
termNamesForRefsWithinNamespaceOf :: (PG.QueryM m) => NamesPerspective m -> Maybe ReversedName -> ShouldSuffixify -> Q.NameSearchScope -> Traversal s t PGReferent [(ReversedName, ReversedName)] -> s -> m t
termNamesForRefsWithinNamespaceOf namesPerspective maySuffix shouldSuffixify nameScope trav s = do
s
& asListOf trav %%~ \refs -> do
NameLookupQ.termNamesForRefsWithinNamespaceOf namesPerspective maySuffix shouldSuffixify traversed refs
NameLookupQ.termNamesForRefsWithinNamespaceOf namesPerspective maySuffix shouldSuffixify nameScope traversed refs
<&> (fmap . fmap) \(NameWithSuffix {reversedName, suffixifiedName}) -> (reversedName, suffixifiedName)

-- | Get the list of (fqn, suffixified) names for a given Reference.
-- If 'shouldSuffixify' is 'NoSuffixify', the suffixified name will be the same as the fqn.
typeNamesForRefsWithinNamespaceOf :: (PG.QueryM m) => NamesPerspective m -> Maybe ReversedName -> ShouldSuffixify -> Traversal s t PGReference [(ReversedName, ReversedName)] -> s -> m t
typeNamesForRefsWithinNamespaceOf namesPerspective maySuffix shouldSuffixify trav s = do
typeNamesForRefsWithinNamespaceOf :: (PG.QueryM m) => NamesPerspective m -> Maybe ReversedName -> ShouldSuffixify -> Q.NameSearchScope -> Traversal s t PGReference [(ReversedName, ReversedName)] -> s -> m t
typeNamesForRefsWithinNamespaceOf namesPerspective maySuffix shouldSuffixify nameScope trav s = do
s
& asListOf trav %%~ \refs -> do
NameLookupQ.typeNamesForRefsWithinNamespaceOf namesPerspective maySuffix shouldSuffixify traversed refs
NameLookupQ.typeNamesForRefsWithinNamespaceOf namesPerspective maySuffix shouldSuffixify nameScope traversed refs
<&> (fmap . fmap) \(NameWithSuffix {reversedName, suffixifiedName}) -> (reversedName, suffixifiedName)

termRefsForExactNamesOf :: (PG.QueryM m) => NamesPerspective m -> Traversal s t ReversedName [NamedRef V1.Referent] -> s -> m t
Expand Down
Loading
Loading