AI crawlers, search bots and training agents

// middleware.ts
import { NextResponse, type NextRequest, type NextFetchEvent } from 'next/server';

const MAYBE_BOT = /bot|crawl|spider|-user|agent|fetcher|copilot|grok|notebooklm|read-aloud|externalhit|cohere/i;

export function middleware(request: NextRequest, event: NextFetchEvent) {
  const userAgent = request.headers.get('user-agent') ?? '';
  if (MAYBE_BOT.test(userAgent)) {
    event.waitUntil(
      fetch('https://thebizness.ai/api/crawls', {
        method: 'POST',
        headers: {
          'Content-Type': 'application/json',
          Authorization: `Bearer ${process.env.THEBIZNESS_INGEST_SECRET}`,
        },
        body: JSON.stringify({
          websiteId: process.env.THEBIZNESS_WEBSITE_ID,
          url: request.nextUrl.pathname,
          userAgent,
        }),
      }).catch(() => {})
    );
  }
  return NextResponse.next();
}

export const config = { matcher: ['/((?!_next/|favicon.ico).*)'] };

curl -X POST https://thebizness.ai/api/crawls \
  -H "Authorization: Bearer $THEBIZNESS_INGEST_SECRET" \
  -H "Content-Type: application/json" \
  -d '{"websiteId":"YOUR_WEBSITE_ID","url":"/pricing","userAgent":"Mozilla/5.0 (compatible; GPTBot/1.2)","status":200}'