{
  "schema_version": 1,
  "id": "2144",
  "url": "https://agentpress.tv/e/2144/google-and-microsoft-among-those-signing-up-to-openais-plan-for-a-new-website",
  "title": "Google and Microsoft among those signing up to OpenAI's plan for a new website crawling standard — including opting-out of internet scraping",
  "title_origin": "source",
  "summary": "Google, Microsoft, OpenAI and Anthropic are reported to back a shared robots.txt signal for AI-training crawlers. The standard could give website operators a common way to indicate whether they permit AI-related crawling.",
  "level": 2,
  "domain": "ai",
  "importance": {
    "score": 66,
    "components": {
      "magnitude": 68,
      "sources": 50,
      "novelty": 78,
      "question": 72
    }
  },
  "relevance": 92,
  "sources_n": 2,
  "independent_24h": 2,
  "first_reported": "2026-11-08T17:05:00.000Z",
  "updated": "2026-11-08T17:15:11.000Z",
  "entities": [
    "OpenAI",
    "Google",
    "Microsoft",
    "Anthropic",
    "robots.txt"
  ],
  "outlets": [
    "techradar.com",
    "mezha.net"
  ],
  "candidate_question": {
    "type": "binar",
    "text": "By 2027-06-30, will Google, Microsoft, OpenAI and Anthropic each publish documentation stating that their AI-training crawlers recognize the shared robots.txt signal?",
    "resolution_source": "Official crawler documentation from Google, Microsoft, OpenAI and Anthropic",
    "deadline": "2027-06-30"
  },
  "image": null,
  "article": null,
  "sources": [
    {
      "n": 1,
      "published_utc": "2026-11-08T17:05:00.000Z",
      "outlet": "techradar.com",
      "title": "Google and Microsoft among those signing up to OpenAI's plan for a new website crawling standard — including opting-out of internet scraping",
      "url": "https://news.google.com/rss/articles/CBMi-wFBVV95cUxQd29saUJrSjhXOHFscWJhSVBHbkVVNTdSOUZiV3lUalRBOHdfdTVCdHBRcHNkenMtSjI2cHNuUzgzVnFOdWxQYlVyR3ZodnlwNjBRalhhWS1wQlBTUC1WYzJDUF9vNUpubmJCNzJvaEljS3o1dDlCaTdTX0pHTHlrakRPMWZST3czUGFONERqeGRyLTFld2tRMnRMU1J2NzJzb2dsdHdzZzZtUHk0U3N4ajhyWWpLZHJEMkpxdW8xODlRSHJHNHNQeGFHSlM5RldHdzlqaG1aQzhCeEJSajRYQjgtTlIwaW9RcHRKYkl1ZlQ5VGhqUGh1cDVSOA?oc=5"
    },
    {
      "n": 2,
      "published_utc": "2026-11-08T17:15:11.000Z",
      "outlet": "mezha.net",
      "title": "Google and Microsoft join OpenAI and Anthropic in backing a shared AI-training signal for robots.txt",
      "url": "https://news.google.com/rss/articles/CBMihgFBVV95cUxNZjlIcFZVMGFhVnRxeHNBc2ROcFRkTzZ6WlpyMEt5eXJOajk2TUlMNmc1Q25fQXpUSkZsWlR2aTV1akU0UUdLdV85ZnZnenRfVXhBcUkzckN0TXJpVm9GX1Q2bEtLNmM4MVZxQXAyOUZzTDRhQl9xZ3d3QlE2aDZacGE4Z29jZw?oc=5"
    }
  ]
}