Enhance sitemap generation with validation and error handling

Refactored sitemap.ts to add URL validation, improved error handling, and more robust date parsing for sitemap entries. Updated robots.txt comment to clarify sitemap generation by Next.js. These changes improve the reliability and accuracy of the generated sitemap.

Author: gihanrangana

app/sitemap.ts

@@ -1,5 +1,5 @@
-import { statSync } from "fs";
-import { join } from "path";
+import { statSync } from "node:fs";
+import { join } from "node:path";
 import type { MetadataRoute } from "next";
 
 import { getRegistryBaseUrl, source } from "@/lib/source";
@@ -15,88 +15,148 @@ function getFileLastModified(filePath: string): Date {
     }
 }
 
-const sitemap = (): MetadataRoute.Sitemap => {
+// feat: Validate URL format for sitemap
+function isValidUrl(url: string): boolean {
+    try {
+        new URL(url);
+        return true;
+    } catch {
+        return false;
+    }
+}
+
+// feat: Generate sitemap data for Next.js App Router with enhanced error handling
+function generateSitemapData(): MetadataRoute.Sitemap {
     const baseUrl = getRegistryBaseUrl();
 
+    // feat: Validate base URL
+    if (!isValidUrl(baseUrl)) {
+        console.error("Invalid base URL for sitemap:", baseUrl);
+        return [];
+    }
+
     const sitemap: MetadataRoute.Sitemap = [];
 
-    sitemap.push({
-        url: `${baseUrl}/`,
-        lastModified: new Date(),
-        changeFrequency: "weekly",
-        priority: 1,
-    });
+    // Add homepage with validation
+    const homepageUrl = `${baseUrl}/`;
+    if (isValidUrl(homepageUrl)) {
+        sitemap.push({
+            url: homepageUrl,
+            lastModified: new Date(),
+            changeFrequency: "weekly",
+            priority: 1,
+        });
+    }
 
+    // Add documentation pages with enhanced error handling
     try {
         const pages = source.getPages();
-        pages.forEach((page) => {
-            // feat: Try to get actual file modification date
-            let lastModified = new Date();
-            if (page.data.lastModified) {
-                lastModified = new Date(page.data.lastModified);
-            } else {
-                // Try to get file modification date from content directory
-                try {
-                    const contentPath = join(
-                        process.cwd(),
-                        "content",
-                        "docs",
-                        `${page.url.replace("/docs/", "")}.mdx`,
-                    );
-                    lastModified = getFileLastModified(contentPath);
-                } catch {
-                    // Fallback to current date
-                    lastModified = new Date();
+        if (Array.isArray(pages)) {
+            pages.forEach((page) => {
+                if (!page?.url) return;
+
+                // feat: Try to get actual file modification date
+                let lastModified = new Date();
+                if (page.data?.lastModified) {
+                    try {
+                        lastModified = new Date(page.data.lastModified);
+                        // Validate date
+                        if (Number.isNaN(lastModified.getTime())) {
+                            lastModified = new Date();
+                        }
+                    } catch {
+                        lastModified = new Date();
+                    }
+                } else {
+                    // Try to get file modification date from content directory
+                    try {
+                        const contentPath = join(
+                            process.cwd(),
+                            "content",
+                            "docs",
+                            `${page.url.replace("/docs/", "")}.mdx`,
+                        );
+                        lastModified = getFileLastModified(contentPath);
+                    } catch {
+                        // Fallback to current date
+                        lastModified = new Date();
+                    }
                 }
-            }
 
-            sitemap.push({
-                url: `${baseUrl}${page.url}`,
-                lastModified,
-                changeFrequency: "monthly",
-                priority: 0.9,
+                const pageUrl = `${baseUrl}${page.url}`;
+                if (isValidUrl(pageUrl)) {
+                    sitemap.push({
+                        url: pageUrl,
+                        lastModified,
+                        changeFrequency: "monthly",
+                        priority: 0.9,
+                    });
+                }
             });
-        });
+        }
     } catch (error) {
-        console.warn("Error generating sitemap", error);
+        console.warn("Error generating sitemap for pages:", error);
     }
 
-    Object.keys(Index).forEach((componentName) => {
-        if (componentName.endsWith("-demo")) return;
+    // Add component registry JSON files with validation
+    try {
+        if (Index && typeof Index === "object") {
+            Object.keys(Index).forEach((componentName) => {
+                if (componentName.endsWith("-demo") || !componentName) return;
 
-        // feat: Component registry JSON file with proper modification date
-        const componentPath = join(
-            process.cwd(),
-            "public",
-            "r",
-            `${componentName}.json`,
-        );
-        const lastModified = getFileLastModified(componentPath);
+                // feat: Component registry JSON file with proper modification date
+                const componentPath = join(
+                    process.cwd(),
+                    "public",
+                    "r",
+                    `${componentName}.json`,
+                );
+                const lastModified = getFileLastModified(componentPath);
 
-        sitemap.push({
-            url: `${baseUrl}/r/${componentName}.json`,
-            lastModified,
-            changeFrequency: "monthly",
-            priority: 0.6,
-        });
-    });
+                const componentUrl = `${baseUrl}/r/${componentName}.json`;
+                if (isValidUrl(componentUrl)) {
+                    sitemap.push({
+                        url: componentUrl,
+                        lastModified,
+                        changeFrequency: "monthly",
+                        priority: 0.6,
+                    });
+                }
+            });
+        }
+    } catch (error) {
+        console.warn("Error generating sitemap for components:", error);
+    }
 
-    sitemap.push(
+    // Add API endpoints and other important pages with validation
+    const additionalUrls = [
         {
             url: `${baseUrl}/api/search`,
-            lastModified: new Date(),
-            changeFrequency: "weekly",
+            changeFrequency: "weekly" as const,
             priority: 0.7,
         },
         {
             url: `${baseUrl}/llms-full.txt`,
-            lastModified: new Date(),
-            changeFrequency: "weekly",
+            changeFrequency: "weekly" as const,
             priority: 0.8,
         },
-    );
+    ];
+
+    additionalUrls.forEach(({ url, changeFrequency, priority }) => {
+        if (isValidUrl(url)) {
+            sitemap.push({
+                url,
+                lastModified: new Date(),
+                changeFrequency,
+                priority,
+            });
+        }
+    });
 
     return sitemap;
-};
+}
 
-export default sitemap;
+// feat: Export sitemap function for Next.js App Router
+export default function sitemap(): MetadataRoute.Sitemap {
+    return generateSitemapData();
+}

public/robots.txt

@@ -11,5 +11,5 @@ Disallow: /llms.mdx/
 # Allow registry JSON files for component discovery
 Allow: /r/*.json
 
-# Sitemap location
+# Sitemap location - Next.js automatically generates sitemap.xml
 Sitemap: https://shadix-ui.vercel.app/sitemap.xml