145b1669e7
Adds metadataBase, full OpenGraph + Twitter card tags, keywords, JSON-LD structured data (SoftwareApplication + Organization), sitemap.ts, robots.ts with AI crawler directives, and llms.txt for AI agent discoverability.
23 lines
689 B
TypeScript
23 lines
689 B
TypeScript
import { MetadataRoute } from "next";
|
|
|
|
export default function robots(): MetadataRoute.Robots {
|
|
return {
|
|
rules: [
|
|
{ userAgent: "GPTBot", allow: "/" },
|
|
{ userAgent: "ChatGPT-User", allow: "/" },
|
|
{ userAgent: "ClaudeBot", allow: "/" },
|
|
{ userAgent: "PerplexityBot", allow: "/" },
|
|
{ userAgent: "Applebot-Extended", allow: "/" },
|
|
{ userAgent: "CCBot", disallow: "/" },
|
|
{ userAgent: "Google-Extended", disallow: "/" },
|
|
{ userAgent: "Bytespider", disallow: "/" },
|
|
{
|
|
userAgent: "*",
|
|
allow: "/",
|
|
disallow: ["/api/", "/dashboard/"],
|
|
},
|
|
],
|
|
sitemap: "https://agentlens.vectry.tech/sitemap.xml",
|
|
};
|
|
}
|