836 public search option (#964)

* the issue is that on this branch the status is not indexed

* working on putting public search in

* search not hidden

* use public search endpoint, still need to remove cookie and authentication

* trying to use name

* lowerecase matches

* lowerecase matches

* indexing for file status was wrong

* error in public search because of admin mode, not sure why its still happening

* public layout not layout

* public search is not commented out

* formatting, fix test for search

* need a public search result with public routes

* removing creator filter
unused imports

* added missing import

* inconsistent use of public/datasets and public_datasets fixed

* _ for public files instead of /

* _ for public datasets instead of /

* file status should be fixed now

* formatting

* fixing the link for public file result

Author: tcnichol

backend/app/routers/datasets.py

@@ -681,7 +681,15 @@ async def save_file(
                 raise HTTPException(
                     status_code=404, detail=f"Folder {folder_id} not found"
                 )
-
+        file_public = False
+        file_authenticated = False
+        file_private = False
+        if dataset.status == DatasetStatus.PUBLIC:
+            file_public = True
+        elif dataset.status == DatasetStatus.AUTHENTICATED:
+            file_authenticated = True
+        else:
+            file_private = True
         await add_file_entry(
             new_file,
             user,
@@ -690,6 +698,8 @@ async def save_file(
             rabbitmq_client,
             file.file,
             content_type=file.content_type,
+            authenticated=file_authenticated,
+            public=file_public,
         )
         return new_file.dict()
     raise HTTPException(status_code=404, detail=f"Dataset {dataset_id} not found")

backend/app/routers/elasticsearch.py

@@ -23,8 +23,8 @@ def _add_permissions_clause(
             "should": [
                 {"term": {"creator": username}},
                 {"term": {"user_ids": username}},
-                {"term": {"status": "AUTHENTICATED"}},
-                {"term": {"status": "PUBLIC"}},
+                {"term": {"status": "authenticated"}},
+                {"term": {"status": "public"}},
             ]
         }
     }

backend/app/routers/public_elasticsearch.py

@@ -10,7 +10,7 @@
 def _add_public_clause(query):
     """Append filter to Elasticsearch object that restricts permissions based on the requesting user."""
     # TODO: Add public filter once added
-    public_clause = {"bool": {"should": [{"term": {"public": True}}]}}
+    public_clause = {"bool": {"should": [{"term": {"status": "public"}}]}}
 
     updated_query = ""
     for content in query.decode().split("\n"):

backend/app/search/index.py

@@ -66,8 +66,6 @@ async def index_file(
     file: FileOut,
     user_ids: Optional[List[str]] = None,
     update: bool = False,
-    public: bool = False,
-    authenticated: bool = False,
 ):
     """Create or update an Elasticsearch entry for the file. user_ids is the list of users
     with permission to at least view the file's dataset, it will be queried if not provided.
@@ -89,11 +87,7 @@ async def index_file(
     ):
         metadata.append(md.content)
 
-    status = None
-    if authenticated:
-        status = "AUTHENTICATED"
-    if public:
-        status = "PUBLIC"
+    status = file.status
 
     # Add en entry to the file index
     doc = ElasticsearchEntry(
@@ -109,7 +103,7 @@ async def index_file(
         folder_id=str(file.folder_id),
         bytes=file.bytes,
         metadata=metadata,
-        status=status,
+        status=file.status,
     ).dict()
     if update:
         try:

backend/app/tests/test_elastic_search.py

@@ -193,11 +193,12 @@ async def test_public_files():
         time.sleep(1)
         dummy_file_query = []
 
+        query_body = {"query": {"match": {"creator": "xyz"}}}
+
         user_public_clause = {
             "bool": {
                 "should": [
-                    {"term": {"creator": "xyz"}},
-                    {"term": {"status": "PUBLIC"}},
+                    {"term": {"status": "public"}},
                 ]
             }
         }
@@ -252,7 +253,7 @@ async def test_public_datasets():
             "bool": {
                 "should": [
                     {"term": {"creator": "abcd"}},
-                    {"term": {"status": "PUBLIC"}},
+                    {"term": {"status": "public"}},
                 ]
             }
         }

frontend/src/app.config.ts

@@ -15,6 +15,7 @@ interface Config {
 	KeycloakRefresh: string;
 	KeycloakRegister: string;
 	searchEndpoint: string;
+	publicSearchEndpoint: string;
 	refreshTokenInterval: number;
 	extractorStatusInterval: number;
 	extractorLivelihoodInterval: number;
@@ -61,6 +62,7 @@ config["KeycloakRegister"] = `${config.KeycloakBaseURL}/register`;
 
 // elasticsearch
 config["searchEndpoint"] = `${hostname}/api/v2/elasticsearch`;
+config["publicSearchEndpoint"] = `${hostname}/api/v2/public_elasticsearch`;
 
 // refresh token time interval
 config["refreshTokenInterval"] = 1000 * 60; // 1 minute

frontend/src/components/PublicLayout.tsx

@@ -22,6 +22,9 @@ import { Explore } from "@material-ui/icons";
 import PersonIcon from "@mui/icons-material/Person";
 import VpnKeyIcon from "@mui/icons-material/VpnKey";
 import LogoutIcon from "@mui/icons-material/Logout";
+import { EmbeddedPublicSearch } from "./search/EmbeddedPublicSearch";
+import { AppVersion } from "./versions/AppVersion";
+import SearchDatasetIcon from "@mui/icons-material/Search";
 import { EmbeddedSearch } from "./search/EmbeddedSearch";
 import { Footer } from "./navigation/Footer";
 
@@ -46,7 +49,7 @@ const Main = styled("main", { shouldForwardProp: (prop) => prop !== "open" })<{
 	}),
 }));
 
-const SearchDiv = styled("div")(({ theme }) => ({
+const PublicSearchDiv = styled("div")(({ theme }) => ({
 	position: "relative",
 	marginLeft: theme.spacing(3),
 	marginBottom: "-5px", // to compoensate the tags div
@@ -94,7 +97,7 @@ export default function PersistentDrawerLeft(props) {
 	const { children } = props;
 	const theme = useTheme();
 	const [open, setOpen] = React.useState(false);
-	const [embeddedSearchHidden, setEmbeddedSearchHidden] = React.useState(false);
+	const [embeddedPublicSearchHidden, setEmbeddedPublicSearchHidden] = React.useState(false);
 	const [anchorEl, setAnchorEl] = React.useState(null);
 	const isMenuOpen = Boolean(anchorEl);
 
@@ -114,9 +117,9 @@ export default function PersistentDrawerLeft(props) {
 
 	useEffect(() => {
 		if (location.pathname.includes("search")) {
-			setEmbeddedSearchHidden(true);
+			setEmbeddedPublicSearchHidden(true);
 		} else {
-			setEmbeddedSearchHidden(false);
+			setEmbeddedPublicSearchHidden(false);
 		}
 	}, [location]);
 
@@ -151,10 +154,10 @@ export default function PersistentDrawerLeft(props) {
 					</Link>
 
 					{/*for searching*/}
-					<SearchDiv hidden={true}>
+					<PublicSearchDiv hidden={embeddedPublicSearchHidden}>
 						{/*	<InputSearchBox />*/}
-						<EmbeddedSearch />
-					</SearchDiv>
+						<EmbeddedPublicSearch />
+					</PublicSearchDiv>
 					<Box sx={{ flexGrow: 1 }} />
 					<Box sx={{ marginLeft: "auto" }}>
 						<Link href="/auth/register" sx={link}>
@@ -233,6 +236,18 @@ export default function PersistentDrawerLeft(props) {
 						</ListItemButton>
 					</ListItem>
 				</List>
+				<Divider/>
+				<List>
+					<ListItem key={"public_search"} disablePadding>
+						<ListItemButton component={RouterLink} to="/public_search">
+							<ListItemIcon>
+								<SearchDatasetIcon />
+							</ListItemIcon>
+							<ListItemText primary={"Public Search"} />
+						</ListItemButton>
+					</ListItem>
+				</List>
+				{/*<Divider />*/}
 				<Divider />
 			</Drawer>
 			<Main open={open}>

frontend/src/components/datasets/DatasetCard.tsx

@@ -52,7 +52,7 @@ export default function DatasetCard(props: DatasetCardProps) {
 			{publicView ? (
 				<CardActionArea
 					component={Link}
-					to={`/public/datasets/${id}`}
+					to={`/public_datasets/${id}`}
 					sx={{ height: "100%" }}
 				>
 					<CardHeader title={name} subheader={subheader} />

frontend/src/components/datasets/PublicDataset.tsx

@@ -133,15 +133,15 @@ export const PublicDataset = (): JSX.Element => {
 		const tmpPaths = [
 			{
 				name: about["name"],
-				url: `/public/datasets/${datasetId}`,
+				url: `/public_datasets/${datasetId}`,
 			},
 		];
 
 		if (publicFolderPath != null) {
 			for (const folderBread of publicFolderPath) {
 				tmpPaths.push({
 					name: folderBread["folder_name"],
-					url: `/public/datasets/${datasetId}?folder=${folderBread["folder_id"]}`,
+					url: `/public_datasets/${datasetId}?folder=${folderBread["folder_id"]}`,
 				});
 			}
 		} else {

frontend/src/components/datasets/PublicDatasetCard.tsx

@@ -52,7 +52,7 @@ export default function PublicDatasetCard(props: PublicDatasetCardProps) {
 			{publicView ? (
 				<CardActionArea
 					component={Link}
-					to={`/public/datasets/${id}`}
+					to={`/public_datasets/${id}`}
 					sx={{ height: "100%" }}
 				>
 					<CardHeader title={name} subheader={subheader} />