Improve frontend handling of many projects (#20151)

* Remove project context fetching

* limit projects returned from `ListSuggestedRepositories`

* Get rid of all projects query

* A WIP state

* Enhance search and normalize links

* Revert find project DB changes

* Make repo finder responsible for current selection

* remove debug

* Comments

* Re-use pagination flattening

* Query improvements

* limit pagination

* Add test about keeping length

* Fix test

* Fix repo ordering

* add normalize comment

Author: filiptronicek

components/dashboard/package.json

@@ -97,7 +97,7 @@
     "react-scripts": "^5.0.1",
     "tailwind-underline-utils": "^1.1.2",
     "tailwindcss-filters": "^3.0.0",
-    "typescript": "^5.4.5",
+    "typescript": "^5.5.4",
     "web-vitals": "^1.1.1"
   },
   "scripts": {

components/dashboard/src/components/RepositoryFinder.tsx

@@ -11,12 +11,17 @@ import { ReactComponent as RepositoryIcon } from "../icons/RepositoryWithColor.s
 import { ReactComponent as GitpodRepositoryTemplate } from "../icons/GitpodRepositoryTemplate.svg";
 import GitpodRepositoryTemplateSVG from "../icons/GitpodRepositoryTemplate.svg";
 import { MiddleDot } from "./typography/MiddleDot";
-import { useUnifiedRepositorySearch } from "../data/git-providers/unified-repositories-search-query";
+import {
+    deduplicateAndFilterRepositories,
+    flattenPagedConfigurations,
+    useUnifiedRepositorySearch,
+} from "../data/git-providers/unified-repositories-search-query";
 import { useAuthProviderDescriptions } from "../data/auth-providers/auth-provider-descriptions-query";
 import { ReactComponent as Exclamation2 } from "../images/exclamation2.svg";
 import { AuthProviderType } from "@gitpod/public-api/lib/gitpod/v1/authprovider_pb";
 import { SuggestedRepository } from "@gitpod/public-api/lib/gitpod/v1/scm_pb";
 import { PREDEFINED_REPOS } from "../data/git-providers/predefined-repos";
+import { useConfiguration, useListConfigurations } from "../data/configurations/configuration-queries";
 
 interface RepositoryFinderProps {
     selectedContextURL?: string;
@@ -41,7 +46,7 @@ export default function RepositoryFinder({
 }: RepositoryFinderProps) {
     const [searchString, setSearchString] = useState("");
     const {
-        data: repos,
+        data: unifiedRepos,
         isLoading,
         isSearching,
         hasMore,
@@ -52,6 +57,59 @@ export default function RepositoryFinder({
         showExamples,
     });
 
+    // We search for the current context URL in order to have data for the selected suggestion
+    const selectedItemSearch = useListConfigurations({
+        sortBy: "name",
+        sortOrder: "desc",
+        pageSize: 30,
+        searchTerm: selectedContextURL,
+    });
+    const flattenedSelectedItem = useMemo(() => {
+        if (excludeConfigurations) {
+            return [];
+        }
+
+        const flattened = flattenPagedConfigurations(selectedItemSearch.data);
+        return flattened.map(
+            (repo) =>
+                new SuggestedRepository({
+                    configurationId: repo.id,
+                    configurationName: repo.name,
+                    url: repo.cloneUrl,
+                }),
+        );
+    }, [excludeConfigurations, selectedItemSearch.data]);
+
+    // We get the configuration by ID if one is selected
+    const selectedConfiguration = useConfiguration(selectedConfigurationId);
+    const selectedConfigurationSuggestion = useMemo(() => {
+        if (!selectedConfiguration.data) {
+            return undefined;
+        }
+
+        return new SuggestedRepository({
+            configurationId: selectedConfiguration.data.id,
+            configurationName: selectedConfiguration.data.name,
+            url: selectedConfiguration.data.cloneUrl,
+        });
+    }, [selectedConfiguration.data]);
+
+    const repos = useMemo(() => {
+        return deduplicateAndFilterRepositories(
+            searchString,
+            excludeConfigurations,
+            onlyConfigurations,
+            [unifiedRepos, selectedConfigurationSuggestion, flattenedSelectedItem].flat().filter((r) => !!r),
+        );
+    }, [
+        searchString,
+        excludeConfigurations,
+        onlyConfigurations,
+        selectedConfigurationSuggestion,
+        flattenedSelectedItem,
+        unifiedRepos,
+    ]);
+
     const authProviders = useAuthProviderDescriptions();
 
     // This approach creates a memoized Map of the predefined repos,
@@ -126,6 +184,11 @@ export default function RepositoryFinder({
                 return repo.configurationId === selectedConfigurationId;
             }
 
+            // todo(ft): normalize this more centrally
+            if (repo.url.endsWith(".git")) {
+                repo.url = repo.url.slice(0, -4);
+            }
+
             return repo.url === selectedContextURL;
         });
 

components/dashboard/src/data/featureflag-query.ts

@@ -7,7 +7,6 @@
 import { getPrimaryEmail } from "@gitpod/public-api-common/lib/user-utils";
 import { useQuery } from "@tanstack/react-query";
 import { getExperimentsClient } from "../experiments/client";
-import { useCurrentProject } from "../projects/project-context";
 import { useCurrentUser } from "../user-context";
 import { useCurrentOrg } from "./organizations/orgs-query";
 
@@ -32,17 +31,15 @@ type FeatureFlags = typeof featureFlags;
 export const useFeatureFlag = <K extends keyof FeatureFlags>(featureFlag: K): FeatureFlags[K] | boolean => {
     const user = useCurrentUser();
     const org = useCurrentOrg().data;
-    const project = useCurrentProject().project;
 
-    const queryKey = ["featureFlag", featureFlag, user?.id || "", org?.id || "", project?.id || ""];
+    const queryKey = ["featureFlag", featureFlag, user?.id || "", org?.id || ""];
 
     const query = useQuery(queryKey, async () => {
         const flagValue = await getExperimentsClient().getValueAsync(featureFlag, featureFlags[featureFlag], {
             user: user && {
                 id: user.id,
                 email: getPrimaryEmail(user),
             },
-            projectId: project?.id,
             teamId: org?.id,
             teamName: org?.name,
             gitpodHost: window.location.host,

components/dashboard/src/data/git-providers/suggested-repositories-query.ts

@@ -24,6 +24,9 @@ export const useSuggestedRepositories = ({ excludeConfigurations }: Props) => {
             const { repositories } = await scmClient.listSuggestedRepositories({
                 organizationId: org.id,
                 excludeConfigurations,
+                pagination: {
+                    pageSize: 100,
+                },
             });
             return repositories;
         },

components/dashboard/src/data/git-providers/unified-repositories-search-query.test.ts

@@ -45,7 +45,7 @@ test("it should exclude project entries", () => {
     expect(deduplicated[1].repoName).toEqual("foo2");
 });
 
-test("it should match entries in url as well as poject name", () => {
+test("it should match entries in url as well as project name", () => {
     const suggestedRepos: SuggestedRepository[] = [
         repo("somefOOtest"),
         repo("Footest"),
@@ -54,7 +54,7 @@ test("it should match entries in url as well as poject name", () => {
         repo("bar", "someFootest"),
         repo("bar", "FOOtest"),
     ];
-    var deduplicated = deduplicateAndFilterRepositories("foo", false, false, suggestedRepos);
+    let deduplicated = deduplicateAndFilterRepositories("foo", false, false, suggestedRepos);
     expect(deduplicated.length).toEqual(6);
     deduplicated = deduplicateAndFilterRepositories("foot", false, false, suggestedRepos);
     expect(deduplicated.length).toEqual(4);
@@ -70,12 +70,16 @@ test("it keeps the order", () => {
         repo("bar", "somefOO"),
         repo("bar", "someFootest"),
         repo("bar", "FOOtest"),
+        repo("bar", "somefOO"),
     ];
     const deduplicated = deduplicateAndFilterRepositories("foot", false, false, suggestedRepos);
     expect(deduplicated[0].repoName).toEqual("somefOOtest");
     expect(deduplicated[1].repoName).toEqual("Footest");
     expect(deduplicated[2].configurationName).toEqual("someFootest");
     expect(deduplicated[3].configurationName).toEqual("FOOtest");
+
+    const deduplicatedNoSearch = deduplicateAndFilterRepositories("", false, false, suggestedRepos);
+    expect(deduplicatedNoSearch.length).toEqual(6);
 });
 
 test("it should return all repositories without duplicates when excludeProjects is true", () => {

components/dashboard/src/data/git-providers/unified-repositories-search-query.ts

@@ -9,6 +9,15 @@ import { useSearchRepositories } from "./search-repositories-query";
 import { useSuggestedRepositories } from "./suggested-repositories-query";
 import { PREDEFINED_REPOS } from "./predefined-repos";
 import { useMemo } from "react";
+import { useListConfigurations } from "../configurations/configuration-queries";
+import type { UseInfiniteQueryResult } from "@tanstack/react-query";
+import { Configuration } from "@gitpod/public-api/lib/gitpod/v1/configuration_pb";
+
+export const flattenPagedConfigurations = (
+    data: UseInfiniteQueryResult<{ configurations: Configuration[] }>["data"],
+): Configuration[] => {
+    return data?.pages.flatMap((p) => p.configurations) ?? [];
+};
 
 type UnifiedRepositorySearchArgs = {
     searchString: string;
@@ -26,9 +35,34 @@ export const useUnifiedRepositorySearch = ({
     onlyConfigurations = false,
     showExamples = false,
 }: UnifiedRepositorySearchArgs) => {
+    // 1st data source: suggested SCM repos + up to 100 imported repos.
+    // todo(ft): look into deduplicating and merging these on the server
     const suggestedQuery = useSuggestedRepositories({ excludeConfigurations });
     const searchLimit = 30;
+    // 2nd data source: SCM repos according to `searchString`
     const searchQuery = useSearchRepositories({ searchString, limit: searchLimit });
+    // 3rd data source: imported repos according to `searchString`
+    const configurationSearch = useListConfigurations({
+        sortBy: "name",
+        sortOrder: "desc",
+        pageSize: searchLimit,
+        searchTerm: searchString,
+    });
+    const flattenedConfigurations = useMemo(() => {
+        if (excludeConfigurations) {
+            return [];
+        }
+
+        const flattened = flattenPagedConfigurations(configurationSearch.data);
+        return flattened.map(
+            (repo) =>
+                new SuggestedRepository({
+                    configurationId: repo.id,
+                    configurationName: repo.name,
+                    url: repo.cloneUrl,
+                }),
+        );
+    }, [configurationSearch.data, excludeConfigurations]);
 
     const filteredRepos = useMemo(() => {
         if (showExamples && searchString.length === 0) {
@@ -41,17 +75,25 @@ export const useUnifiedRepositorySearch = ({
             );
         }
 
-        const repos = [suggestedQuery.data || [], searchQuery.data || []].flat();
+        const repos = [suggestedQuery.data || [], searchQuery.data || [], flattenedConfigurations ?? []].flat();
         return deduplicateAndFilterRepositories(searchString, excludeConfigurations, onlyConfigurations, repos);
-    }, [excludeConfigurations, onlyConfigurations, showExamples, searchQuery.data, searchString, suggestedQuery.data]);
+    }, [
+        showExamples,
+        searchString,
+        suggestedQuery.data,
+        searchQuery.data,
+        flattenedConfigurations,
+        excludeConfigurations,
+        onlyConfigurations,
+    ]);
 
     return {
         data: filteredRepos,
-        hasMore: searchQuery.data?.length === searchLimit,
+        hasMore: (searchQuery.data?.length ?? 0) >= searchLimit,
         isLoading: suggestedQuery.isLoading,
         isSearching: searchQuery.isFetching,
-        isError: suggestedQuery.isError || searchQuery.isError,
-        error: suggestedQuery.error || searchQuery.error,
+        isError: suggestedQuery.isError || searchQuery.isError || configurationSearch.isError,
+        error: suggestedQuery.error || searchQuery.error || configurationSearch.error,
     };
 };
 
@@ -72,6 +114,11 @@ export function deduplicateAndFilterRepositories(
         });
     }
     for (const repo of suggestedRepos) {
+        // normalize URLs
+        if (repo.url.endsWith(".git")) {
+            repo.url = repo.url.slice(0, -4);
+        }
+
         // filter out configuration-less entries if an entry with a configuration exists, and we're not excluding configurations
         if (!repo.configurationId) {
             if (reposWithConfiguration.has(repo.url) || onlyConfigurations) {

components/dashboard/src/data/projects/create-project-mutation.ts

@@ -7,13 +7,11 @@
 import { useMutation } from "@tanstack/react-query";
 import { getGitpodService } from "../../service/service";
 import { useCurrentOrg } from "../organizations/orgs-query";
-import { useRefreshAllProjects } from "./list-all-projects-query";
 import { CreateProjectParams, Project } from "@gitpod/gitpod-protocol";
 
 export type CreateProjectArgs = Omit<CreateProjectParams, "teamId">;
 
 export const useCreateProject = () => {
-    const refreshProjects = useRefreshAllProjects();
     const { data: org } = useCurrentOrg();
 
     return useMutation<Project, Error, CreateProjectArgs>(async ({ name, slug, cloneUrl, appInstallationId }) => {
@@ -32,11 +30,6 @@ export const useCreateProject = () => {
             appInstallationId,
         });
 
-        // TODO: remove this once we delete ProjectContext
-        // wait for projects to refresh before returning
-        // this ensures that the new project is included in the list before we navigate to it
-        await refreshProjects(org.id);
-
         return newProject;
     });
 };

components/dashboard/src/data/projects/list-all-projects-query.ts

@@ -23,7 +23,6 @@ import { ConfettiContextProvider } from "./contexts/ConfettiContext";
 import { setupQueryClientProvider } from "./data/setup";
 import "./index.css";
 import { PaymentContextProvider } from "./payment-context";
-import { ProjectContextProvider } from "./projects/project-context";
 import { ThemeContextProvider } from "./theme-context";
 import { UserContextProvider } from "./user-context";
 import { getURLHash, isGitpodIo, isWebsiteSlug } from "./utils";
@@ -69,9 +68,7 @@ const bootApp = () => {
                                     <ToastContextProvider>
                                         <UserContextProvider>
                                             <PaymentContextProvider>
-                                                <ProjectContextProvider>
-                                                    <RootAppRouter />
-                                                </ProjectContextProvider>
+                                                <RootAppRouter />
                                             </PaymentContextProvider>
                                         </UserContextProvider>
                                     </ToastContextProvider>