Skip to content

Commit 67ae62b

Browse files
Deepthi-ChandDeepthi-Chand
authored
Merge pull request #308 from CivicDataLab/286-implement-dynamic-seo-sitemap-generation-with-feature-flag-control
286 implement dynamic seo sitemap generation with feature flag control
2 parents e8ec52c + 51c378a commit 67ae62b

File tree

4 files changed

+475
-24
lines changed

4 files changed

+475
-24
lines changed

‎app/robots.txt/route.ts‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
// app/robots.txt/route.ts
2+
import { isSitemapEnabled } from '@/lib/utils';
3+
4+
export async function GET() {
5+
const baseUrl = process.env.NEXTAUTH_URL;
6+
7+
const robotsTxt = `User-agent: *
8+
Allow: /
9+
10+
Sitemap: ${baseUrl}/sitemap/main.xml
11+
`;
12+
13+
if (!isSitemapEnabled()) {
14+
return new Response('Sitemaps are not enabled', { status: 404 });
15+
}
16+
17+
return new Response(robotsTxt, {
18+
headers: {
19+
'Content-Type': 'text/plain',
20+
},
21+
});
22+
}

‎app/sitemap/[entityPage]/route.ts‎

Lines changed: 140 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,140 @@
1+
// app/sitemap-[entity]-[page].xml/route.ts
2+
import { type NextRequest } from 'next/server';
3+
4+
import { ENTITY_CONFIG, getSiteMapConfig, isSitemapEnabled } from '@/lib/utils';
5+
import { getGraphqlEntityCount, getSearchEntityCount } from '../main.xml/route';
6+
7+
interface EntityItem {
8+
id: string;
9+
slug?: string;
10+
updated_at?: string;
11+
__typename?: 'TypeUser' | 'TypeOrganization';
12+
}
13+
14+
async function fetchEntityData(
15+
entity: string,
16+
page: number
17+
): Promise<EntityItem[]> {
18+
const config = ENTITY_CONFIG[entity];
19+
20+
// If no config is found, return empty array
21+
if (!config) return [];
22+
23+
if (config.source === 'search') {
24+
// Fetch entity based on general rest query
25+
const response = await getSearchEntityCount(
26+
entity,
27+
getSiteMapConfig().itemsPerPage,
28+
page
29+
);
30+
if (!response || !response.list) return [];
31+
return response.list;
32+
} else if (config.source === 'graphql') {
33+
// Fetch entity based on graphql query
34+
const response = await getGraphqlEntityCount(entity, config);
35+
if (!response || !response.list) return [];
36+
return response.list;
37+
} else {
38+
return [];
39+
}
40+
}
41+
42+
function generateEntitySitemap(items: EntityItem[], entity: string): string {
43+
const baseUrl = process.env.NEXTAUTH_URL;
44+
const config = ENTITY_CONFIG[entity];
45+
46+
if (!config) {
47+
return `<?xml version="1.0" encoding="UTF-8"?>
48+
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
49+
</urlset>`;
50+
}
51+
52+
const urls = items
53+
?.map((item) => {
54+
console.log(item, entity);
55+
56+
// Function to handle loc or URLs for different types of entities especially for contributors or organizations
57+
const getLoc = () => {
58+
if (item.__typename === 'TypeOrganization') {
59+
return `${baseUrl}/${config.path}/organization/${item.id}`;
60+
} else if (item.__typename === 'TypeUser') {
61+
return `${baseUrl}/${config.path}/${item.id}`;
62+
} else {
63+
return `${baseUrl}/${config.path}/${item.slug || item.id}`;
64+
}
65+
};
66+
67+
const loc = getLoc();
68+
const lastmod = item.updated_at
69+
? new Date(item.updated_at).toISOString()
70+
: new Date().toISOString();
71+
72+
return `
73+
<url>
74+
<loc>${loc}</loc>
75+
<lastmod>${lastmod}</lastmod>
76+
<changefreq>weekly</changefreq>
77+
<priority>${config.priority}</priority>
78+
</url>
79+
`;
80+
})
81+
.join('');
82+
83+
return `<?xml version="1.0" encoding="UTF-8"?>\n<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">\n${urls}\n</urlset>`;
84+
}
85+
86+
export async function GET(
87+
request: NextRequest,
88+
{ params }: { params: { entityPage: string } }
89+
) {
90+
// Check if sitemaps are enabled via feature flag
91+
if (!isSitemapEnabled()) {
92+
return new Response('Sitemaps are not enabled', { status: 404 });
93+
}
94+
95+
try {
96+
const { entityPage } = params;
97+
98+
const m = entityPage.match(/^([a-zA-Z0-9_]+)-(\d+)\.xml$/);
99+
if (!m) {
100+
return new Response('Invalid Route', { status: 404 });
101+
}
102+
103+
const entity = m[1];
104+
const pageNumber = Number(m[2]);
105+
106+
if (!ENTITY_CONFIG[entity]) {
107+
return new Response('Entity not found', { status: 404 });
108+
}
109+
110+
if (isNaN(pageNumber) || pageNumber < 1) {
111+
return new Response('Invalid page number', { status: 400 });
112+
}
113+
114+
const items = await fetchEntityData(entity, pageNumber);
115+
const sitemap = generateEntitySitemap(items, entity);
116+
117+
const flags = getSiteMapConfig();
118+
return new Response(sitemap, {
119+
headers: {
120+
'Content-Type': 'application/xml',
121+
'Cache-Control': `public, max-age=${flags.childCacheDuration}`,
122+
},
123+
});
124+
} catch (error) {
125+
console.error('Error generating entity sitemap:', error);
126+
127+
const errorSitemap = `<?xml version="1.0" encoding="UTF-8"?>
128+
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
129+
</urlset>`;
130+
131+
return new Response(errorSitemap, {
132+
status: 500,
133+
headers: {
134+
'Content-Type': 'application/xml',
135+
},
136+
});
137+
}
138+
}
139+
140+
export const dynamic = 'force-dynamic';

‎app/sitemap/main.xml/route.ts‎

Lines changed: 214 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,214 @@
1+
// app/sitemap.xml/route.ts
2+
import { type NextRequest } from 'next/server';
3+
4+
import {
5+
ENTITY_CONFIG,
6+
ENTITY_CONFIG_TYPE,
7+
getSiteMapConfig,
8+
isSitemapEnabled,
9+
} from '@/lib/utils';
10+
11+
const getAllEntityCounts = async (): Promise<Record<string, number>> => {
12+
const counts: Record<string, number> = {};
13+
14+
const countPromises: Promise<{ entityName: string; count: number }>[] = [];
15+
16+
Object.entries(ENTITY_CONFIG).forEach(([entityName, config]) => {
17+
if (config.source === 'graphql' && config.graphqlQuery) {
18+
countPromises.push(getGraphqlEntityCount(entityName, config));
19+
}
20+
if (config.source === 'search' && config.endpoint) {
21+
countPromises.push(getSearchEntityCount(entityName, 5, 1));
22+
}
23+
});
24+
25+
const results = await Promise.all(countPromises);
26+
27+
results.forEach(({ entityName, count }) => {
28+
counts[entityName] = count;
29+
});
30+
31+
return counts;
32+
};
33+
34+
export async function getGraphqlEntityCount(
35+
entity: string,
36+
config: ENTITY_CONFIG_TYPE[string]
37+
): Promise<{ entityName: string; count: number; list: any }> {
38+
try {
39+
const response = await fetch(
40+
`${process.env.FEATURE_SITEMAP_BACKEND_BASE_URL}/graphql`,
41+
{
42+
method: 'POST',
43+
headers: {
44+
'Content-Type': 'application/json',
45+
},
46+
body: JSON.stringify({
47+
query: config.graphqlQuery,
48+
variables: {},
49+
}),
50+
}
51+
);
52+
const data = await response.json();
53+
54+
return {
55+
entityName: entity,
56+
count: data?.data?.[config.queryResKey as string]?.length || 0,
57+
list: data?.data?.[config.queryResKey as string] || [],
58+
};
59+
} catch (error) {
60+
console.error(`Error fetching count for ${entity}:`, error);
61+
return { entityName: entity, count: 0, list: [] };
62+
}
63+
}
64+
65+
export async function getSearchEntityCount(
66+
entity: string,
67+
size: number,
68+
page: number
69+
): Promise<{ entityName: string; count: number; list: any }> {
70+
try {
71+
const config = ENTITY_CONFIG[entity];
72+
const response = await fetch(
73+
`${process.env.FEATURE_SITEMAP_BACKEND_BASE_URL}${config.endpoint}?sort=recent&size=${size}&page=${page}`,
74+
{
75+
method: 'GET',
76+
headers: {
77+
'Content-Type': 'application/json',
78+
},
79+
next: { revalidate: 3600 },
80+
}
81+
);
82+
const data = await response.json();
83+
return { entityName: entity, count: data.total, list: data.results };
84+
} catch (error) {
85+
console.error(`Error fetching count for ${entity}:`, error);
86+
return { entityName: entity, count: 0, list: [] };
87+
}
88+
}
89+
90+
function generateStaticUrls(): string {
91+
const baseUrl = process.env.NEXTAUTH_URL;
92+
93+
const staticPages = [
94+
{ path: '', priority: '1.0', changefreq: 'daily' },
95+
{ path: '/datasets', priority: '0.9', changefreq: 'daily' },
96+
{ path: '/usecases', priority: '0.8', changefreq: 'weekly' },
97+
{ path: '/publishers', priority: '0.7', changefreq: 'weekly' },
98+
{ path: '/sectors', priority: '0.7', changefreq: 'weekly' },
99+
];
100+
101+
return staticPages
102+
.map(
103+
(page) => `
104+
<url>
105+
<loc>${baseUrl}${page.path}</loc>
106+
<changefreq>${page.changefreq}</changefreq>
107+
<priority>${page.priority}</priority>
108+
</url>`
109+
)
110+
.join('');
111+
}
112+
113+
function generateSitemapIndex(
114+
sitemapUrls: string[],
115+
staticUrls: string
116+
): string {
117+
const sitemapEntries = sitemapUrls
118+
.map(
119+
(url) =>
120+
`
121+
<url>
122+
<loc>${url}</loc>
123+
<lastmod>${new Date().toISOString()}</lastmod>
124+
</url>`
125+
)
126+
.join('');
127+
128+
return `<?xml version="1.0" encoding="UTF-8"?>\n
129+
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">${staticUrls}\n${sitemapEntries}\n</urlset>`;
130+
}
131+
132+
export async function GET(request: NextRequest) {
133+
// Check if sitemaps are enabled via feature flag
134+
if (!isSitemapEnabled()) {
135+
return new Response('Sitemaps are not enabled', { status: 404 });
136+
}
137+
138+
try {
139+
const flags = getSiteMapConfig();
140+
const ITEMS_PER_SITEMAP = flags.itemsPerPage;
141+
142+
// Fetch counts for all entities
143+
// const [sectorsCount] = await Promise.all([
144+
// getGraphqlEntityCount({ sectors: ENTITY_CONFIG.sectors }),
145+
// ]);
146+
147+
const baseUrl = process.env.NEXTAUTH_URL;
148+
149+
// Generate sitemap URLs for each entity
150+
const sitemapUrls: string[] = [];
151+
152+
const entityCounts = await getAllEntityCounts();
153+
154+
// Datasets sitemaps
155+
if (entityCounts.datasets > 0) {
156+
const datasetPages = Math.ceil(entityCounts.datasets / ITEMS_PER_SITEMAP);
157+
for (let i = 1; i <= datasetPages; i++) {
158+
sitemapUrls.push(`${baseUrl}/sitemap/datasets-${i}.xml`);
159+
}
160+
}
161+
162+
// Usecases sitemaps
163+
const usecasePages = Math.ceil(entityCounts.usecases / ITEMS_PER_SITEMAP);
164+
for (let i = 1; i <= usecasePages; i++) {
165+
sitemapUrls.push(`${baseUrl}/sitemap/usecases-${i}.xml`);
166+
}
167+
168+
// Contributors sitemaps
169+
const contributorPages = Math.ceil(
170+
entityCounts.contributors / ITEMS_PER_SITEMAP
171+
);
172+
for (let i = 1; i <= contributorPages; i++) {
173+
sitemapUrls.push(`${baseUrl}/sitemap/contributors-${i}.xml`);
174+
}
175+
176+
// Sectors sitemaps
177+
if (entityCounts.sectors > 0) {
178+
const sectorPages = Math.ceil(entityCounts.sectors / ITEMS_PER_SITEMAP);
179+
for (let i = 1; i <= sectorPages; i++) {
180+
sitemapUrls.push(`${baseUrl}/sitemap/sectors-${i}.xml`);
181+
}
182+
}
183+
184+
const sitemapIndex = generateSitemapIndex(
185+
sitemapUrls,
186+
generateStaticUrls()
187+
);
188+
189+
return new Response(sitemapIndex, {
190+
status: 200,
191+
headers: {
192+
'Content-Type': 'application/xml',
193+
'Cache-Control': `public, max-age=${flags.cacheDuration}`,
194+
},
195+
});
196+
197+
// return new Response(JSON.stringify(entityCounts), { status: 200 });
198+
} catch (error) {
199+
console.error('Error generating sitemap index:', error);
200+
201+
const errorSitemap = `<?xml version="1.0" encoding="UTF-8"?>
202+
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
203+
</urlset>`;
204+
205+
return new Response(errorSitemap, {
206+
status: 500,
207+
headers: {
208+
'Content-Type': 'application/xml',
209+
},
210+
});
211+
}
212+
}
213+
214+
export const dynamic = 'force-dynamic';

0 commit comments

Comments
 (0)