add control over indexability (#679)

nichtsam · web-flow · commit b637a2b2974b · 2024-04-01T10:54:16.000-06:00
diff --git a/.env.example b/.env.example
@@ -13,3 +13,7 @@ SENTRY_DSN="your-dsn"
 GITHUB_CLIENT_ID="MOCK_GITHUB_CLIENT_ID"
 GITHUB_CLIENT_SECRET="MOCK_GITHUB_CLIENT_SECRET"
 GITHUB_TOKEN="MOCK_GITHUB_TOKEN"
+
+# set this to false to prevent search engines from indexing the website
+# default to allow indexing for seo safety
+ALLOW_INDEXING="true"
diff --git a/app/root.tsx b/app/root.tsx
@@ -181,11 +181,13 @@ function Document({
 	nonce,
 	theme = 'light',
 	env = {},
+	allowIndexing = true,
 }: {
 	children: React.ReactNode
 	nonce: string
 	theme?: Theme
 	env?: Record<string, string>
+	allowIndexing?: boolean
 }) {
 	return (
 		<html lang="en" className={`${theme} h-full overflow-x-hidden`}>
@@ -194,6 +196,9 @@ function Document({
 				<Meta />
 				<meta charSet="utf-8" />
 				<meta name="viewport" content="width=device-width,initial-scale=1" />
+				{allowIndexing ? null : (
+					<meta name="robots" content="noindex, nofollow" />
+				)}
 				<Links />
 			</head>
 			<body className="bg-background text-foreground">
@@ -219,10 +224,16 @@ function App() {
 	const matches = useMatches()
 	const isOnSearchPage = matches.find(m => m.id === 'routes/users+/index')
 	const searchBar = isOnSearchPage ? null : <SearchBar status="idle" />
+	const allowIndexing = data.ENV.ALLOW_INDEXING !== 'false'
 	useToast(data.toast)
 
 	return (
-		<Document nonce={nonce} theme={theme} env={data.ENV}>
+		<Document
+			nonce={nonce}
+			theme={theme}
+			allowIndexing={allowIndexing}
+			env={data.ENV}
+		>
 			<div className="flex h-screen flex-col justify-between">
 				<header className="container py-6">
 					<nav className="flex flex-wrap items-center justify-between gap-4 sm:flex-nowrap md:gap-8">
diff --git a/app/utils/env.server.ts b/app/utils/env.server.ts
@@ -16,6 +16,7 @@ const schema = z.object({
 	GITHUB_CLIENT_ID: z.string().default('MOCK_GITHUB_CLIENT_ID'),
 	GITHUB_CLIENT_SECRET: z.string().default('MOCK_GITHUB_CLIENT_SECRET'),
 	GITHUB_TOKEN: z.string().default('MOCK_GITHUB_TOKEN'),
+	ALLOW_INDEXING: z.enum(['true', 'false']).optional(),
 })
 
 declare global {
@@ -50,6 +51,7 @@ export function getEnv() {
 	return {
 		MODE: process.env.NODE_ENV,
 		SENTRY_DSN: process.env.SENTRY_DSN,
+		ALLOW_INDEXING: process.env.ALLOW_INDEXING,
 	}
 }
 
diff --git a/docs/deployment.md b/docs/deployment.md
@@ -67,6 +67,14 @@ Prior to your first deployment, you'll need to do a few things:
   > [1Password](https://1password.com/password-generator) to generate a random
   > secret, just replace `$(openssl rand -hex 32)` with the generated secret.
 
+- Add a `ALLOW_INDEXING` with `false` value to your non-production fly app
+  secrets, this is to prevent duplicate content from being indexed multiple
+  times by search engines. To do this you can run the following commands:
+
+  ```sh
+  fly secrets set ALLOW_INDEXING=false --app [YOUR_APP_NAME]-staging
+  ```
+
 6. Create production database:
 
    Create a persistent volume for the sqlite database for both your staging and
diff --git a/remix.init/index.mjs b/remix.init/index.mjs
@@ -160,7 +160,7 @@ async function setupDeployment({ rootDirectory }) {
 	await $I`fly apps create ${APP_NAME}`
 
 	console.log(`🤫 Setting secrets in apps`)
-	await $I`fly secrets set SESSION_SECRET=${getRandomString32()} INTERNAL_COMMAND_TOKEN=${getRandomString32()} HONEYPOT_SECRET=${getRandomString32()} --app ${APP_NAME}-staging`
+	await $I`fly secrets set SESSION_SECRET=${getRandomString32()} INTERNAL_COMMAND_TOKEN=${getRandomString32()} HONEYPOT_SECRET=${getRandomString32()} ALLOW_INDEXING=false --app ${APP_NAME}-staging`
 	await $I`fly secrets set SESSION_SECRET=${getRandomString32()} INTERNAL_COMMAND_TOKEN=${getRandomString32()} HONEYPOT_SECRET=${getRandomString32()} --app ${APP_NAME}`
 
 	console.log(
diff --git a/server/index.ts b/server/index.ts
@@ -16,7 +16,8 @@ installGlobals()
 
 const MODE = process.env.NODE_ENV ?? 'development'
 const IS_PROD = MODE === 'production'
-const IS_DEV = MODE === "development"
+const IS_DEV = MODE === 'development'
+const ALLOW_INDEXING = process.env.ALLOW_INDEXING !== 'false'
 
 const createRequestHandler = IS_PROD
 	? Sentry.wrapExpressCreateRequestHandler(_createRequestHandler)
@@ -205,6 +206,13 @@ async function getBuild() {
 	return build as unknown as ServerBuild
 }
 
+if (!ALLOW_INDEXING) {
+	app.use((_, res, next) => {
+		res.set('X-Robots-Tag', 'noindex, nofollow')
+		next()
+	})
+}
+
 app.all(
 	'*',
 	createRequestHandler({

Original file line number	Diff line number	Diff line change
`@@ -16,6 +16,7 @@ const schema = z.object({`
`16`	`16`	`GITHUB_CLIENT_ID: z.string().default('MOCK_GITHUB_CLIENT_ID'),`
`17`	`17`	`GITHUB_CLIENT_SECRET: z.string().default('MOCK_GITHUB_CLIENT_SECRET'),`
`18`	`18`	`GITHUB_TOKEN: z.string().default('MOCK_GITHUB_TOKEN'),`
	`19`	`+ ALLOW_INDEXING: z.enum(['true', 'false']).optional(),`
`19`	`20`	`})`
`20`	`21`
`21`	`22`	`declare global {`
`@@ -50,6 +51,7 @@ export function getEnv() {`
`50`	`51`	`return {`
`51`	`52`	`MODE: process.env.NODE_ENV,`
`52`	`53`	`SENTRY_DSN: process.env.SENTRY_DSN,`
	`54`	`+ ALLOW_INDEXING: process.env.ALLOW_INDEXING,`
`53`	`55`	`}`
`54`	`56`	`}`
`55`	`57`