feat: comprehensive SEO — meta tags, OG, Twitter cards, JSON-LD, sitemap, robots, llms.txt

Adds metadataBase, full OpenGraph + Twitter card tags, keywords,
JSON-LD structured data (SoftwareApplication + Organization),
sitemap.ts, robots.ts with AI crawler directives, and llms.txt
for AI agent discoverability.

apps/web/src/app/robots.ts

@@ -0,0 +1,22 @@
+import { MetadataRoute } from "next";
+
+export default function robots(): MetadataRoute.Robots {
+  return {
+    rules: [
+      { userAgent: "GPTBot", allow: "/" },
+      { userAgent: "ChatGPT-User", allow: "/" },
+      { userAgent: "ClaudeBot", allow: "/" },
+      { userAgent: "PerplexityBot", allow: "/" },
+      { userAgent: "Applebot-Extended", allow: "/" },
+      { userAgent: "CCBot", disallow: "/" },
+      { userAgent: "Google-Extended", disallow: "/" },
+      { userAgent: "Bytespider", disallow: "/" },
+      {
+        userAgent: "*",
+        allow: "/",
+        disallow: ["/api/", "/dashboard/"],
+      },
+    ],
+    sitemap: "https://agentlens.vectry.tech/sitemap.xml",
+  };
+}