Skip to content

Commit 629cd5c

Browse files
bhavabhuthibhavabhuthi
committed
setup crawler SEO for application
1 parent 4b48c4c commit 629cd5c

File tree

4 files changed

+447
-24
lines changed

4 files changed

+447
-24
lines changed

‎app/robots.txt/route.ts‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
// app/robots.txt/route.ts
2+
export async function GET() {
3+
const baseUrl = process.env.NEXTAUTH_URL;
4+
5+
const robotsTxt = `User-agent: *
6+
Allow: /
7+
8+
Sitemap: ${baseUrl}/sitemap.xml
9+
`;
10+
11+
return new Response(robotsTxt, {
12+
headers: {
13+
'Content-Type': 'text/plain',
14+
},
15+
});
16+
}

‎app/sitemap-[entity]-[page].xml/route.ts‎

Lines changed: 141 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,141 @@
1+
// app/sitemap-[entity]-[page].xml/route.ts
2+
import { type NextRequest } from 'next/server';
3+
4+
import { ENTITY_CONFIG, getSiteMapConfig } from '@/lib/utils';
5+
import {
6+
getGraphqlEntityCount,
7+
getSearchEntityCount,
8+
} from '../sitemap.xml/route';
9+
10+
// Check if sitemaps are enabled
11+
const isSitemapEnabled = () => {
12+
return (
13+
process.env.FEATURE_SITEMAPS === 'true' ||
14+
process.env.NODE_ENV === 'production'
15+
);
16+
};
17+
18+
interface EntityItem {
19+
id: string;
20+
slug?: string;
21+
updated_at?: string;
22+
}
23+
24+
async function fetchEntityData(
25+
entity: string,
26+
page: number
27+
): Promise<EntityItem[]> {
28+
const config = ENTITY_CONFIG[entity];
29+
if (!config) return [];
30+
if (config.source === 'search') {
31+
const response = await getSearchEntityCount(entity, 5, page);
32+
if (!response || !response.list) return [];
33+
console.log(entity, response);
34+
return response.list;
35+
} else if (config.source === 'graphql') {
36+
const response = await getGraphqlEntityCount(entity, config);
37+
if (!response || !response.list) return [];
38+
console.log(entity, response);
39+
return response.list;
40+
} else {
41+
return [];
42+
}
43+
// try {
44+
// const flags = getSiteMapConfig();
45+
// const response = await fetch(
46+
// `${process.env.FEATURE_SITEMAP_BACKEND_BASE_URL}${config.endpoint}?page=${page}&page_size=${flags.itemsPerPage}`,
47+
// {
48+
// method: 'GET',
49+
// headers: {
50+
// 'Content-Type': 'application/json',
51+
// },
52+
// next: { revalidate: flags.childCacheDuration },
53+
// }
54+
// );
55+
56+
// if (!response.ok) return [];
57+
58+
// const data = await response.json();
59+
// return data.results || data;
60+
}
61+
62+
function generateEntitySitemap(items: EntityItem[], entity: string): string {
63+
const baseUrl = process.env.NEXTAUTH_URL;
64+
const config = ENTITY_CONFIG[entity];
65+
66+
if (!config) {
67+
return `<?xml version="1.0" encoding="UTF-8"?>
68+
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
69+
</urlset>`;
70+
}
71+
72+
const urls = items
73+
?.map((item) => {
74+
const loc = `${baseUrl}/${config.path}/${item.slug || item.id}`;
75+
const lastmod = item.updated_at
76+
? new Date(item.updated_at).toISOString()
77+
: new Date().toISOString();
78+
79+
return `
80+
<url>
81+
<loc>${loc}</loc>
82+
<lastmod>${lastmod}</lastmod>
83+
<changefreq>weekly</changefreq>
84+
<priority>${config.priority}</priority>
85+
</url>
86+
`;
87+
})
88+
.join('');
89+
90+
return `<?xml version="1.0" encoding="UTF-8"?>
91+
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">\n${urls}\n</urlset>`;
92+
}
93+
94+
export async function GET(
95+
request: NextRequest,
96+
{ params }: { params: { entity: string; page: string } }
97+
) {
98+
// Check if sitemaps are enabled via feature flag
99+
if (!isSitemapEnabled()) {
100+
return new Response('Sitemaps are not enabled', { status: 404 });
101+
}
102+
103+
try {
104+
const { entity, page } = params;
105+
106+
if (!ENTITY_CONFIG[entity]) {
107+
return new Response('Entity not found', { status: 404 });
108+
}
109+
110+
const pageNumber = parseInt(page);
111+
if (isNaN(pageNumber) || pageNumber < 1) {
112+
return new Response('Invalid page number', { status: 400 });
113+
}
114+
115+
const items = await fetchEntityData(entity, pageNumber);
116+
const sitemap = generateEntitySitemap(items, entity);
117+
118+
const flags = getSiteMapConfig();
119+
return new Response(sitemap, {
120+
headers: {
121+
'Content-Type': 'application/xml',
122+
'Cache-Control': `public, max-age=${flags.childCacheDuration}`,
123+
},
124+
});
125+
} catch (error) {
126+
console.error('Error generating entity sitemap:', error);
127+
128+
const errorSitemap = `<?xml version="1.0" encoding="UTF-8"?>
129+
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
130+
</urlset>`;
131+
132+
return new Response(errorSitemap, {
133+
status: 500,
134+
headers: {
135+
'Content-Type': 'application/xml',
136+
},
137+
});
138+
}
139+
}
140+
141+
export const dynamic = 'force-dynamic';

‎app/sitemap.xml/route.ts‎

Lines changed: 216 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,216 @@
1+
// app/sitemap.xml/route.ts
2+
import { type NextRequest } from 'next/server';
3+
4+
import {
5+
ENTITY_CONFIG,
6+
ENTITY_CONFIG_TYPE,
7+
getSiteMapConfig,
8+
} from '@/lib/utils';
9+
10+
// Check if sitemaps are enabled
11+
const isSitemapEnabled = () => {
12+
return (
13+
process.env.FEATURE_SITEMAPS === 'true' ||
14+
process.env.NODE_ENV === 'production'
15+
);
16+
};
17+
18+
const getAllEntityCounts = async (): Promise<Record<string, number>> => {
19+
const counts: Record<string, number> = {};
20+
21+
const countPromises: Promise<{ entityName: string; count: number }>[] = [];
22+
23+
Object.entries(ENTITY_CONFIG).forEach(([entityName, config]) => {
24+
if (config.source === 'graphql' && config.graphqlQuery) {
25+
countPromises.push(getGraphqlEntityCount(entityName, config));
26+
}
27+
if (config.source === 'search' && config.endpoint) {
28+
countPromises.push(getSearchEntityCount(entityName, 5, 1));
29+
}
30+
});
31+
32+
const results = await Promise.all(countPromises);
33+
34+
results.forEach(({ entityName, count }) => {
35+
counts[entityName] = count;
36+
});
37+
38+
return counts;
39+
};
40+
41+
export async function getGraphqlEntityCount(
42+
entity: string,
43+
config: ENTITY_CONFIG_TYPE[string]
44+
): Promise<{ entityName: string; count: number; list: any }> {
45+
try {
46+
const response = await fetch(
47+
`${process.env.FEATURE_SITEMAP_BACKEND_BASE_URL}/graphql`,
48+
{
49+
method: 'POST',
50+
headers: {
51+
'Content-Type': 'application/json',
52+
},
53+
body: JSON.stringify({
54+
query: config.graphqlQuery,
55+
variables: {},
56+
}),
57+
}
58+
);
59+
const data = await response.json();
60+
return {
61+
entityName: entity,
62+
count: data?.data?.[config.queryResKey as string]?.length || 0,
63+
list: data?.data?.[config.queryResKey as string] || [],
64+
};
65+
} catch (error) {
66+
console.error(`Error fetching count for ${entity}:`, error);
67+
return { entityName: entity, count: 0, list: [] };
68+
}
69+
}
70+
71+
export async function getSearchEntityCount(
72+
entity: string,
73+
size: number,
74+
page: number
75+
): Promise<{ entityName: string; count: number; list: any }> {
76+
try {
77+
const response = await fetch(
78+
`${process.env.FEATURE_SITEMAP_BACKEND_BASE_URL}/search/dataset/?sort=recent&size=${size}&page=${page}`,
79+
{
80+
method: 'GET',
81+
headers: {
82+
'Content-Type': 'application/json',
83+
},
84+
next: { revalidate: 3600 },
85+
}
86+
);
87+
const data = await response.json();
88+
return { entityName: entity, count: data.total, list: data.results };
89+
} catch (error) {
90+
console.error(`Error fetching count for ${entity}:`, error);
91+
return { entityName: entity, count: 0, list: [] };
92+
}
93+
}
94+
95+
function generateStaticUrls(): string {
96+
const baseUrl = process.env.NEXTAUTH_URL;
97+
98+
const staticPages = [
99+
{ path: '', priority: '1.0', changefreq: 'daily' },
100+
{ path: '/datasets', priority: '0.9', changefreq: 'daily' },
101+
{ path: '/usecases', priority: '0.8', changefreq: 'weekly' },
102+
{ path: '/publishers', priority: '0.7', changefreq: 'weekly' },
103+
{ path: '/sectors', priority: '0.7', changefreq: 'weekly' },
104+
];
105+
106+
return staticPages
107+
.map(
108+
(page) => `
109+
<url>
110+
<loc>${baseUrl}${page.path}</loc>
111+
<changefreq>${page.changefreq}</changefreq>
112+
<priority>${page.priority}</priority>
113+
</url>`
114+
)
115+
.join('');
116+
}
117+
118+
function generateSitemapIndex(
119+
sitemapUrls: string[],
120+
staticUrls: string
121+
): string {
122+
const sitemapEntries = sitemapUrls
123+
.map(
124+
(url) =>
125+
`
126+
<url>
127+
<loc>${url}</loc>
128+
<lastmod>${new Date().toISOString()}</lastmod>
129+
</url>`
130+
)
131+
.join('');
132+
133+
return `<?xml version="1.0" encoding="UTF-8"?><urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">${staticUrls}\n${sitemapEntries}\n</urlset>`;
134+
}
135+
136+
export async function GET(request: NextRequest) {
137+
// Check if sitemaps are enabled via feature flag
138+
if (!isSitemapEnabled()) {
139+
return new Response('Sitemaps are not enabled', { status: 404 });
140+
}
141+
142+
try {
143+
const flags = getSiteMapConfig();
144+
const ITEMS_PER_SITEMAP = flags.itemsPerPage;
145+
146+
// Fetch counts for all entities
147+
// const [sectorsCount] = await Promise.all([
148+
// getGraphqlEntityCount({ sectors: ENTITY_CONFIG.sectors }),
149+
// ]);
150+
151+
const baseUrl = process.env.NEXTAUTH_URL;
152+
153+
// Generate sitemap URLs for each entity
154+
const sitemapUrls: string[] = [];
155+
156+
const entityCounts = await getAllEntityCounts();
157+
158+
// Datasets sitemaps
159+
if (entityCounts.datasets > 0) {
160+
const datasetPages = Math.ceil(entityCounts.datasets / ITEMS_PER_SITEMAP);
161+
for (let i = 1; i <= datasetPages; i++) {
162+
sitemapUrls.push(`${baseUrl}/sitemap-datasets-${i}.xml`);
163+
}
164+
}
165+
166+
// Usecases sitemaps
167+
// const usecasePages = Math.ceil(usecasesCount / ITEMS_PER_SITEMAP);
168+
// for (let i = 1; i <= usecasePages; i++) {
169+
// sitemapUrls.push(`${baseUrl}/sitemap-usecases-${i}.xml`);
170+
// }
171+
172+
// // Publishers sitemaps
173+
// const publisherPages = Math.ceil(publishersCount / ITEMS_PER_SITEMAP);
174+
// for (let i = 1; i <= publisherPages; i++) {
175+
// sitemapUrls.push(`${baseUrl}/sitemap-publishers-${i}.xml`);
176+
// }
177+
178+
// Sectors sitemaps
179+
if (entityCounts.sectors > 0) {
180+
const sectorPages = Math.ceil(entityCounts.sectors / ITEMS_PER_SITEMAP);
181+
for (let i = 1; i <= sectorPages; i++) {
182+
sitemapUrls.push(`${baseUrl}/sitemap-sectors-${i}.xml`);
183+
}
184+
}
185+
186+
const sitemapIndex = generateSitemapIndex(
187+
sitemapUrls,
188+
generateStaticUrls()
189+
);
190+
191+
return new Response(sitemapIndex, {
192+
status: 200,
193+
headers: {
194+
'Content-Type': 'application/xml',
195+
'Cache-Control': `public, max-age=${flags.cacheDuration}`,
196+
},
197+
});
198+
199+
// return new Response(JSON.stringify(entityCounts), { status: 200 });
200+
} catch (error) {
201+
console.error('Error generating sitemap index:', error);
202+
203+
const errorSitemap = `<?xml version="1.0" encoding="UTF-8"?>
204+
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
205+
</urlset>`;
206+
207+
return new Response(errorSitemap, {
208+
status: 500,
209+
headers: {
210+
'Content-Type': 'application/xml',
211+
},
212+
});
213+
}
214+
}
215+
216+
export const dynamic = 'force-dynamic';

0 commit comments

Comments
 (0)