Hide sample tld domains from search engines: getting DMCA notices on them

Author: fileformat

package-lock.json

@@ -9,6 +9,7 @@
     "convict": "^6.2.3",
     "haikunator": "^2.1.2",
     "handlebars": "^4.7.3",
+    "isbot": "^5.1.18",
     "js-yaml": "^4.0.0",
     "koa": "^2.7.0",
     "koa-better-flash": "0.0.4",

package.json

@@ -1,7 +1,8 @@
-import * as punycode from 'punycode';
-import Router from '@koa/router';
-import path from 'path';
-import * as wsw from 'whoisserver-world'
+import { isbot } from "isbot";
+import * as punycode from "punycode";
+import Router from "@koa/router";
+import path from "path";
+import * as wsw from "whoisserver-world";
 import * as url from "url";
 const __dirname = url.fileURLToPath(new URL(".", import.meta.url));
 
@@ -12,39 +13,44 @@ import * as domainData from "../data/domainData.js";
 
 const tldsRouter = new Router();
 
-tldsRouter.get('/tlds/index.html', async (ctx: any) => {
-
+tldsRouter.get("/tlds/index.html", async (ctx: any) => {
     const tldMap = wsw.tlds();
-    const unsortedTlds:any[] = [];
-    Object.getOwnPropertyNames(tldMap).forEach(x => unsortedTlds.push(tldMap[x]));
-    unsortedTlds.forEach(x => { x.unicode = punycode.toUnicode(x.tld) })
-    const tlds = unsortedTlds.sort((a, b) => { return a.unicode.localeCompare(b.unicode) });
-    ctx.body = await ctx.render('tlds/index.hbs', {
+    const unsortedTlds: any[] = [];
+    Object.getOwnPropertyNames(tldMap).forEach((x) =>
+        unsortedTlds.push(tldMap[x])
+    );
+    unsortedTlds.forEach((x) => {
+        x.unicode = punycode.toUnicode(x.tld);
+    });
+    const tlds = unsortedTlds.sort((a, b) => {
+        return a.unicode.localeCompare(b.unicode);
+    });
+    ctx.body = await ctx.render("tlds/index.hbs", {
         tlds,
-        title: 'Top Level Domains',
+        title: "Top Level Domains",
     });
-
 });
 
-tldsRouter.get('/tlds/', async (ctx) => {
-    ctx.redirect('/tlds/index.html');
+tldsRouter.get("/tlds/", async (ctx) => {
+    ctx.redirect("/tlds/index.html");
 });
 
 // this must come after base /tlds/ routes
-tldsRouter.get('/tlds/:tld', async (ctx) => {
+tldsRouter.get("/tlds/:tld", async (ctx) => {
     ctx.redirect(`${encodeURIComponent(ctx.params.tld)}/index.html`);
 });
 
-tldsRouter.get('/tlds/:tld/', async (ctx) => {
-    ctx.redirect('index.html');
+tldsRouter.get("/tlds/:tld/", async (ctx) => {
+    ctx.redirect("index.html");
 });
 
-tldsRouter.get('/tlds/:tld/index.html', async (ctx:any) => {
-
+tldsRouter.get("/tlds/:tld/index.html", async (ctx: any) => {
     const tld = ctx.params.tld;
 
     if (tld.startsWith("xn--")) {
-        ctx.redirect(`/tlds/${encodeURIComponent(punycode.toUnicode(tld))}/index.html`);
+        ctx.redirect(
+            `/tlds/${encodeURIComponent(punycode.toUnicode(tld))}/index.html`
+        );
         return;
     }
 
@@ -56,28 +62,33 @@ tldsRouter.get('/tlds/:tld/index.html', async (ctx:any) => {
 
     const tldInfo = wsw.tldDetails(punycode.toASCII(tld));
     if (!tldInfo) {
-      ctx.flash('error', `Sorry, we don't have any info for the domain "${tld}".`)
-      ctx.redirect("/tlds/index.html");
-      return;
+        ctx.flash(
+            "error",
+            `Sorry, we don't have any info for the domain "${tld}".`
+        );
+        ctx.redirect("/tlds/index.html");
+        return;
     }
 
-    let rdapUnofficial:string|undefined;
+    let rdapUnofficial: string | undefined;
     if (!tldInfo.rdap) {
         const rdapInfo = rdapData.get(tld);
         if (rdapInfo && !rdapInfo.official && rdapInfo.working) {
             rdapUnofficial = rdapInfo.rdap;
         }
     }
 
-    const sampleDomains:string[] = [];
+    const sampleDomains: string[] = [];
 
-    Object.getOwnPropertyNames(tldInfo.sampleDomains).forEach(propName => {
-        sampleDomains.push(...tldInfo.sampleDomains[propName]);
-    });
+    if (!isbot(ctx.get("user-agent"))) {
+        Object.getOwnPropertyNames(tldInfo.sampleDomains).forEach((propName) => {
+            sampleDomains.push(...tldInfo.sampleDomains[propName]);
+        });
+    }
 
     tldInfo.unicode = punycode.toUnicode(tld);
 
-    ctx.body = await ctx.render('_tlds/index.hbs', {
+    ctx.body = await ctx.render("_tlds/index.hbs", {
         publicSuffixes: domainData.pslTlds[tld],
         rdapUnofficial,
         sampleDomains,
@@ -86,14 +97,12 @@ tldsRouter.get('/tlds/:tld/index.html', async (ctx:any) => {
     });
 });
 
-
-
 const tldChangeLogUI: ChangeLogUI = new ChangeLogUI(
-    new ChangeLog(path.join(__dirname, '../../data/icann/deltas')),
-    '/tlds/changelog/index.html',
-    '/tlds/changelog',
-    'ICANN TLD',
-    'https://botsin.space/@TLDChanges',
+    new ChangeLog(path.join(__dirname, "../../data/icann/deltas")),
+    "/tlds/changelog/index.html",
+    "/tlds/changelog",
+    "ICANN TLD",
+    "https://botsin.space/@TLDChanges"
 );
 const tldsChangeLogRouter = tldChangeLogUI.changelogRouter;
 

src/routers/tldsRouter.ts

@@ -68,21 +68,19 @@
                 {{/each}}
             </td>
         </tr>
+        {{#if sampleDomains.length}}
         <tr>
             <td>Sample domains</td>
             <td>
-              {{#if sampleDomains.length}}
                 <details>
                   <summary>{{sampleDomains.length}}</summary>
                   {{#each sampleDomains}}
                   <a href="https://{{this}}/">{{this}}</a><br/>
                   {{/each}}
                 </details>
-                {{else}}
-                <i>(none)</i>
-                {{/if}}
             </td>
         </tr>
+        {{/if}}
         <tr>
             <td>Public Suffixes</td>
             <td>
@@ -108,4 +106,4 @@
 <pre>{{toJson tld}}</pre>
 </details>
 
-{{>below}}
+{{>below}}