Add robots.txt handler to disallow all web crawlers

Author: Llewellyn500

app/robots.ts

@@ -0,0 +1,14 @@
+import { NextResponse } from 'next/server';
+
+export async function GET() {
+  const content = `
+    User-agent: *
+    Disallow: /
+  `;
+
+  return new NextResponse(content, {
+    headers: {
+      'Content-Type': 'text/plain',
+    },
+  });
+}