Generate robots.txt (#11)

* Generate robot.txt

* Generate robots.txt during prerender

* Fix types

Author: d0rich

app.config.ts

@@ -15,6 +15,8 @@
  */
 
 import type { ModuleOptions } from 'nuxt-icon'
+import type { DeepPartial } from './utils/DeepPartial'
+import type { RobotsTxtOptions } from './server/routes/robots.txt'
 
 interface ExactproDocsOptions {
   /**
@@ -35,20 +37,26 @@ interface ExactproDocsOptions {
      *
      * @default 'master'
      */
-    branch?: string
+    branch: string
     /**
      * Path to the directory with documentation files on GitHub
      * Specify if the documentation is stored in a subdirectory of the repository.
      *
      * @default '/'
      */
-    docsDir?: string
+    docsDir: string
+  }
+  seo?: {
+    sitemap?: {
+      baseUrl?: string
+    }
+    robots?: RobotsTxtOptions[]
   }
 }
 
 declare module 'nuxt/schema' {
   interface AppConfigInput {
-    exactproDocs?: ExactproDocsOptions
+    exactproDocs?: DeepPartial<ExactproDocsOptions>
     // TODO: Workaround for nuxt-icon types module, delete when https://github.com/nuxt-modules/icon/pull/63 is resolved
     nuxtIcon?: ModuleOptions
   }
@@ -62,7 +70,7 @@ export default defineAppConfig({
       branch: 'master',
       docsDir: '/'
     }
-  },
+  } as ExactproDocsOptions,
   // TODO: Workaround for nuxt-icon types module, delete when https://github.com/nuxt-modules/icon/pull/63 is resolved
   nuxtIcon: {}
 })

docs/app.config.ts

@@ -17,6 +17,12 @@
 export default defineAppConfig({
   exactproDocs: {
     title: 'Docs Template Project',
+    seo: {
+      sitemap: {
+        baseUrl: 'https://exactpro.github.io/docs-toolkit'
+      },
+      robots: [{ UserAgent: '*' }, { Allow: '/' }]
+    },
     github: {
       repoLink: 'https://github.com/exactpro/docs-toolkit',
       branch: 'master',

nuxt.config.ts

@@ -36,6 +36,11 @@ export default defineNuxtConfig({
     }
   },
   modules: ['@nuxt/content', '@nuxtjs/tailwindcss', 'nuxt-icon'],
+  nitro: {
+    prerender: {
+      routes: ['/robots.txt']
+    }
+  },
   content: {
     documentDriven: true,
     highlight: {

server/routes/robots.txt.ts

@@ -0,0 +1,56 @@
+/*
+ * Copyright 2023 Exactpro (Exactpro Systems Limited)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+const keysToTemplates = {
+  UserAgent: (input: string) => `User-agent: ${input}`,
+  CrawlDelay: (input: string) => `Crawl-delay: ${input}`,
+  Disallow: (input: string) => `Disallow: ${input}`,
+  Allow: (input: string) => `Allow: ${input}`,
+  Host: (input: string) => `Host: ${input}`,
+  Sitemap: (input: string) => `Sitemap: ${input}`,
+  CleanParam: (input: string) => `Clean-param: ${input}`,
+  Comment: (input: string) => `# ${input}`
+}
+
+export type RobotsTxtOptions = {
+  [key in keyof typeof keysToTemplates]?: string
+} & {
+  BlankLine?: boolean
+}
+
+export default defineEventHandler((event) => {
+  setHeader(event, 'Content-Type', 'text/plain')
+  const lines: string[] = []
+  const config = useAppConfig()
+  for (const robotOptionsItem of config.exactproDocs?.seo?.robots ?? []) {
+    for (const [key, value] of Object.entries(robotOptionsItem)) {
+      if (typeof value === 'string') {
+        lines.push(keysToTemplates[key as keyof typeof keysToTemplates](value))
+      }
+      // If BlankLine is true, add blank line
+      if (typeof value === 'boolean' && value) {
+        lines.push('')
+      }
+    }
+  }
+  if (config.exactproDocs?.seo?.sitemap?.baseUrl) {
+    lines.push('')
+    lines.push(
+      `Sitemap: ${config.exactproDocs.seo.sitemap.baseUrl}/sitemap.xml`
+    )
+  }
+  return lines.join('\n')
+})

utils/DeepPartial.d.ts

@@ -0,0 +1,21 @@
+/*
+ * Copyright 2023 Exactpro (Exactpro Systems Limited)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export type DeepPartial<T> = T extends object
+  ? {
+      [P in keyof T]?: DeepPartial<T[P]>
+    }
+  : T