{
  "schema_version": "1",
  "label": "261026",
  "date": "2026-10-26",
  "period": {
    "start": "2026-10-26T00:00:00Z",
    "end": "2026-10-27T00:00:00Z"
  },
  "generated_at": "2026-10-27T03:18:38.045Z",
  "license": "CC BY 4.0",
  "license_url": "https://creativecommons.org/licenses/by/4.0/",
  "citation": "Benes, Peter. Rattlesnakes By Mail: A Field Guide to the Crawlers of the Web. benesthemenace.com.",
  "method_url": "https://rattlesnakesbymail.com/method",
  "caveat": "The numbers come from one website. They describe what these crawlers did here, not across the web.",
  "outage": false,
  "outage_notice": null,
  "totals": {
    "requests": 142,
    "crawlers": 7
  },
  "facts": [
    {
      "id": "F-261026-01",
      "rule": "record_day",
      "text": "ChatGPT-User made 9 requests on 2026-10-26, its highest daily count since the site launched on 2026-09-13; its previous high was 3, on 2026-09-19.",
      "url": "https://rattlesnakesbymail.com/observed/chatgpt-user"
    },
    {
      "id": "F-261026-02",
      "rule": "return",
      "text": "ChatGPT-User returned on 2026-10-26 after 13 days without a request.",
      "url": "https://rattlesnakesbymail.com/observed/chatgpt-user"
    },
    {
      "id": "F-261026-03",
      "rule": "user_triggered",
      "text": "ChatGPT-User fetched 9 pages on 2026-10-26 in response to people's requests; the most fetched was /data.",
      "url": "https://rattlesnakesbymail.com/observed/chatgpt-user"
    },
    {
      "id": "F-261026-04",
      "rule": "snapshot_pickup",
      "text": "Bingbot first fetched the 2026-10-25 snapshot 3 h 59 min after it was published.",
      "url": "https://rattlesnakesbymail.com/observed/bingbot"
    },
    {
      "id": "F-261026-05",
      "rule": "snapshot_pickup",
      "text": "Googlebot first fetched the 2026-10-25 snapshot 10 h 31 min after it was published.",
      "url": "https://rattlesnakesbymail.com/observed/googlebot"
    },
    {
      "id": "F-261026-06",
      "rule": "verification",
      "text": "Every request on 2026-10-26 from ClaudeBot, Googlebot, Bingbot, ChatGPT-User and OAI-SearchBot was verified against the operator's published method.",
      "url": "https://rattlesnakesbymail.com/method"
    }
  ],
  "random_selection": {
    "date": "2026-10-27",
    "urls": [
      "https://rattlesnakesbymail.com/questions",
      "https://rattlesnakesbymail.com/compare/bingbot-vs-claude-searchbot",
      "https://rattlesnakesbymail.com/claims/10",
      "https://rattlesnakesbymail.com/compare/claudebot-vs-oai-adsbot",
      "https://rattlesnakesbymail.com/compare/google-extended-vs-gptbot",
      "https://rattlesnakesbymail.com/compare/claude-user-vs-oai-searchbot",
      "https://rattlesnakesbymail.com/compare/bingbot-vs-oai-searchbot",
      "https://rattlesnakesbymail.com/claims/36",
      "https://rattlesnakesbymail.com/claims/27",
      "https://rattlesnakesbymail.com/claims/60"
    ],
    "pool_size": 156,
    "line": "Ten records from the field guide are chosen at random each day.",
    "shown_for": "3 days after publication, then the block is replaced by a note",
    "method": {
      "summary": "Ten records from the field guide are chosen at random each day. The pool is every content page in /sitemap.xml, minus pages drawn or linked by a snapshot's facts in the previous 5 days. The pool is sorted, shuffled with a Fisher-Yates shuffle driven by a keyed hash stream, and the first 10 go to the snapshot block, the next 10 to the /snapshots block and the next 10 to IndexNow.",
      "seed": "HMAC-SHA256(key = secret, message = \"XS1|\" + YYMMDD). The secret is published after the experiment, so every draw can be rerun.",
      "stream": "block i = HMAC-SHA256(key = seed, message = i as 4 bytes, big-endian); each block is read as eight unsigned 32-bit big-endian integers.",
      "shuffle": "For j from n-1 down to 1: draw r; limit = floor(2^32 / (j+1)) * (j+1); redraw while r >= limit; k = r mod (j+1); swap items j and k.",
      "arms": "Items 0 to 9 form arm S, 10 to 19 arm M, 20 to 29 arm I."
    }
  },
  "crawlers": [
    {
      "slug": "claudebot",
      "name": "ClaudeBot",
      "vendor": "Anthropic",
      "kind": "crawler",
      "purpose": "training",
      "requests": 62,
      "verified_requests": 62,
      "verified_share": 1,
      "distinct_paths": 4,
      "formats": {
        "txt": 15,
        "xml": 47
      },
      "first_path": "/robots.txt",
      "robots_txt_fetches": 15,
      "sitemap_fetches": 47,
      "status_mix": {
        "200": 62
      },
      "hourly": [
        4,
        0,
        4,
        2,
        4,
        3,
        3,
        1,
        4,
        2,
        4,
        4,
        3,
        4,
        0,
        4,
        0,
        4,
        0,
        4,
        0,
        4,
        0,
        4
      ],
      "context": {
        "median_7d": 0,
        "median_28d": 48,
        "history_days": 28,
        "ratio_28d": 1.3,
        "highest_before": {
          "count": 1141,
          "date": "2026-09-24"
        },
        "first_seen": "2026-09-13",
        "last_seen_before": "2026-10-25",
        "days_since_last_seen": 1
      },
      "sweep": {
        "sweep_day": false,
        "ended_today": null,
        "runs": [
          {
            "start": "2026-09-24T00:54:00.000Z",
            "end": "2026-09-24T02:08:00.000Z",
            "requests": 448,
            "days_since_previous": null
          },
          {
            "start": "2026-10-09T20:36:00.000Z",
            "end": "2026-10-09T22:35:00.000Z",
            "requests": 308,
            "days_since_previous": 15
          }
        ]
      },
      "new_formats": []
    },
    {
      "slug": "googlebot",
      "name": "Googlebot",
      "vendor": "Google",
      "kind": "crawler",
      "purpose": "search",
      "requests": 38,
      "verified_requests": 38,
      "verified_share": 1,
      "distinct_paths": 26,
      "formats": {
        "csv": 1,
        "html": 24,
        "json": 1,
        "txt": 12
      },
      "first_path": "/robots.txt",
      "robots_txt_fetches": 12,
      "sitemap_fetches": 0,
      "status_mix": {
        "200": 38
      },
      "hourly": [
        2,
        0,
        0,
        0,
        0,
        2,
        1,
        4,
        3,
        2,
        2,
        2,
        2,
        2,
        5,
        4,
        1,
        2,
        0,
        0,
        2,
        2,
        0,
        0
      ],
      "context": {
        "median_7d": 20,
        "median_28d": 13,
        "history_days": 28,
        "ratio_28d": 2.9,
        "highest_before": {
          "count": 711,
          "date": "2026-09-21"
        },
        "first_seen": "2026-09-14",
        "last_seen_before": "2026-10-25",
        "days_since_last_seen": 1
      },
      "sweep": {
        "sweep_day": false,
        "ended_today": null,
        "runs": [
          {
            "start": "2026-09-20T14:03:00.000Z",
            "end": "2026-09-21T14:51:00.000Z",
            "requests": 241,
            "days_since_previous": null
          }
        ]
      },
      "new_formats": []
    },
    {
      "slug": "bingbot",
      "name": "Bingbot",
      "vendor": "Microsoft",
      "kind": "crawler",
      "purpose": "search",
      "requests": 19,
      "verified_requests": 19,
      "verified_share": 1,
      "distinct_paths": 19,
      "formats": {
        "csv": 1,
        "html": 17,
        "xml": 1
      },
      "first_path": "/sitemap.xml",
      "robots_txt_fetches": 0,
      "sitemap_fetches": 1,
      "status_mix": {
        "200": 18,
        "405": 1
      },
      "hourly": [
        0,
        1,
        1,
        3,
        1,
        1,
        1,
        0,
        3,
        0,
        2,
        0,
        1,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        1,
        0,
        3,
        1
      ],
      "context": {
        "median_7d": 16,
        "median_28d": 11,
        "history_days": 28,
        "ratio_28d": 1.7,
        "highest_before": {
          "count": 23,
          "date": "2026-10-21"
        },
        "first_seen": "2026-09-15",
        "last_seen_before": "2026-10-25",
        "days_since_last_seen": 1
      },
      "sweep": {
        "sweep_day": false,
        "ended_today": null,
        "runs": []
      },
      "new_formats": []
    },
    {
      "slug": "chatgpt-user",
      "name": "ChatGPT-User",
      "vendor": "OpenAI",
      "kind": "fetcher",
      "purpose": "user_fetch",
      "requests": 9,
      "verified_requests": 9,
      "verified_share": 1,
      "distinct_paths": 5,
      "formats": {
        "html": 6,
        "json": 3
      },
      "first_path": "/data",
      "robots_txt_fetches": 0,
      "sitemap_fetches": 0,
      "status_mix": {
        "200": 9
      },
      "hourly": [
        5,
        4,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0
      ],
      "context": {
        "median_7d": 0,
        "median_28d": 0,
        "history_days": 28,
        "ratio_28d": null,
        "highest_before": {
          "count": 3,
          "date": "2026-09-19"
        },
        "first_seen": "2026-09-18",
        "last_seen_before": "2026-10-12",
        "days_since_last_seen": 14
      },
      "sweep": {
        "sweep_day": false,
        "ended_today": null,
        "runs": []
      },
      "new_formats": []
    },
    {
      "slug": "oai-searchbot",
      "name": "OAI-SearchBot",
      "vendor": "OpenAI",
      "kind": "search_bot",
      "purpose": "search",
      "requests": 8,
      "verified_requests": 8,
      "verified_share": 1,
      "distinct_paths": 3,
      "formats": {
        "html": 3,
        "txt": 5
      },
      "first_path": "/robots.txt",
      "robots_txt_fetches": 5,
      "sitemap_fetches": 0,
      "status_mix": {
        "200": 8
      },
      "hourly": [
        0,
        0,
        0,
        1,
        0,
        0,
        1,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        2,
        3,
        1
      ],
      "context": {
        "median_7d": 5,
        "median_28d": 5,
        "history_days": 28,
        "ratio_28d": 1.6,
        "highest_before": {
          "count": 44,
          "date": "2026-10-14"
        },
        "first_seen": "2026-09-13",
        "last_seen_before": "2026-10-25",
        "days_since_last_seen": 1
      },
      "sweep": {
        "sweep_day": false,
        "ended_today": null,
        "runs": []
      },
      "new_formats": []
    },
    {
      "slug": "gptbot",
      "name": "GPTBot",
      "vendor": "OpenAI",
      "kind": "crawler",
      "purpose": "training",
      "requests": 4,
      "verified_requests": 4,
      "verified_share": 1,
      "distinct_paths": 4,
      "formats": {
        "json": 1,
        "xml": 3
      },
      "first_path": "/sitemap-index.xml",
      "robots_txt_fetches": 0,
      "sitemap_fetches": 3,
      "status_mix": {
        "200": 3,
        "405": 1
      },
      "hourly": [
        0,
        0,
        0,
        3,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        1,
        0,
        0
      ],
      "context": {
        "median_7d": 3,
        "median_28d": 3,
        "history_days": 28,
        "ratio_28d": 1.3,
        "highest_before": {
          "count": 1101,
          "date": "2026-09-24"
        },
        "first_seen": "2026-09-13",
        "last_seen_before": "2026-10-25",
        "days_since_last_seen": 1
      },
      "sweep": {
        "sweep_day": false,
        "ended_today": null,
        "runs": [
          {
            "start": "2026-09-24T01:54:00.000Z",
            "end": "2026-09-24T01:59:00.000Z",
            "requests": 462,
            "days_since_previous": null
          }
        ]
      },
      "new_formats": []
    },
    {
      "slug": "applebot",
      "name": "Applebot",
      "vendor": "Apple",
      "kind": "crawler",
      "purpose": "search",
      "requests": 2,
      "verified_requests": 2,
      "verified_share": 1,
      "distinct_paths": 2,
      "formats": {
        "json": 1,
        "txt": 1
      },
      "first_path": "/robots.txt",
      "robots_txt_fetches": 1,
      "sitemap_fetches": 0,
      "status_mix": {
        "200": 2
      },
      "hourly": [
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        2,
        0,
        0,
        0,
        0,
        0,
        0
      ],
      "context": {
        "median_7d": 0,
        "median_28d": 0,
        "history_days": 28,
        "ratio_28d": null,
        "highest_before": {
          "count": 426,
          "date": "2026-09-24"
        },
        "first_seen": "2026-09-14",
        "last_seen_before": "2026-10-21",
        "days_since_last_seen": 5
      },
      "sweep": {
        "sweep_day": false,
        "ended_today": null,
        "runs": [
          {
            "start": "2026-09-20T13:45:00.000Z",
            "end": "2026-09-22T00:42:00.000Z",
            "requests": 320,
            "days_since_previous": null
          }
        ]
      },
      "new_formats": []
    }
  ],
  "mcp": {
    "calls": [
      {
        "client": "BrickBlueBot",
        "tool": "get_snapshot",
        "calls": 1
      },
      {
        "client": "BrickBlueBot",
        "tool": "identify_user_agent",
        "calls": 1
      },
      {
        "client": "BrickBlueBot",
        "tool": "list_changes",
        "calls": 1
      },
      {
        "client": "BrickBlueBot",
        "tool": "lookup_crawler",
        "calls": 1
      },
      {
        "client": "rokmcp-probe",
        "tool": "list_changes",
        "calls": 1
      },
      {
        "client": "verifymcp-probe",
        "tool": "__verifymcp_auth_probe_d57a1fff793c8aa3__",
        "calls": 1
      }
    ],
    "new_clients": []
  },
  "ledger": {
    "added": [],
    "answered": [],
    "open_gaps": [
      {
        "id": 26,
        "hash": "eaaa86558b443873916376dc4d0546f60ee5d890631698c41768fb619582451c",
        "text": "Can robots.txt cleanly separate \"allow this page for search or citation\" from \"block it for training,\" and does that distinction hold across every major AI platform?",
        "count": 5,
        "url": "https://rattlesnakesbymail.com/questions/eaaa86558b443873916376dc4d0546f60ee5d890631698c41768fb619582451c"
      },
      {
        "id": 34,
        "hash": "e61346be3fef727db8cefe0257815575a03a33698acfb1e0059140169dbd35ee",
        "text": "How can AI citation performance be measured accurately, given there is no equivalent of a Search Console report for citations?",
        "count": 5,
        "url": "https://rattlesnakesbymail.com/questions/e61346be3fef727db8cefe0257815575a03a33698acfb1e0059140169dbd35ee"
      },
      {
        "id": 38,
        "hash": "97b24d1729c7f32e2e09cb297328a2f7977235c746960d243536ee8b9e4e494d",
        "text": "Was a specific document used in a model's training, fine-tuning, or evaluation?",
        "count": 4,
        "url": "https://rattlesnakesbymail.com/questions/97b24d1729c7f32e2e09cb297328a2f7977235c746960d243536ee8b9e4e494d"
      },
      {
        "id": 7,
        "hash": "e08c55e6af0f8858dae0af2af4eb4caafe28dfb739f1ed4163e9cb0d954bb2ff",
        "text": "Why wasn't a specific page cited by an AI answer instead of a competitor's page?",
        "count": 3,
        "url": "https://rattlesnakesbymail.com/questions/e08c55e6af0f8858dae0af2af4eb4caafe28dfb739f1ed4163e9cb0d954bb2ff"
      },
      {
        "id": 11,
        "hash": "064f0033dab1091575ab29137f558668ff6193e891221987fd9523c31b4ff78f",
        "text": "How much do backlinks, brand mentions, structured data, reviews, forum discussions, publisher reputation, and content freshness each matter to being cited?",
        "count": 3,
        "url": "https://rattlesnakesbymail.com/questions/064f0033dab1091575ab29137f558668ff6193e891221987fd9523c31b4ff78f"
      }
    ],
    "published_gap_ids": [
      26,
      34,
      38,
      7,
      11,
      30,
      42,
      8,
      10,
      12,
      14,
      23,
      24,
      25,
      31,
      33,
      37,
      45,
      49,
      9,
      13,
      15,
      16,
      17,
      18,
      19,
      20,
      22,
      27,
      28,
      29,
      32,
      35,
      36,
      39,
      40,
      41,
      43,
      44,
      46,
      47,
      48,
      50,
      51,
      52,
      53,
      54,
      55
    ],
    "claim_changes": [],
    "published_question_ids": [
      7,
      8,
      9,
      10,
      11,
      12,
      13,
      14,
      15,
      16,
      17,
      18,
      19,
      20,
      21,
      22,
      23,
      24,
      25,
      26,
      27,
      28,
      29,
      30,
      31,
      32,
      33,
      34,
      35,
      36,
      37,
      38,
      39,
      40,
      41,
      42,
      43,
      44,
      45,
      46,
      47,
      48,
      49,
      50,
      51,
      52,
      53,
      54,
      55
    ]
  },
  "snapshot_pickup": [
    {
      "snapshot_label": "261025",
      "snapshot_date": "2026-10-25",
      "crawler": "bingbot",
      "crawler_name": "Bingbot",
      "first_fetch_ts": "2026-10-26T07:18:19.885Z",
      "minutes_after_publication": 239,
      "channel_tag": null
    },
    {
      "snapshot_label": "261025",
      "snapshot_date": "2026-10-25",
      "crawler": "googlebot",
      "crawler_name": "Googlebot",
      "first_fetch_ts": "2026-10-26T13:49:58.973Z",
      "minutes_after_publication": 631,
      "channel_tag": null
    },
    {
      "snapshot_label": "261024",
      "snapshot_date": "2026-10-24",
      "crawler": "bingbot",
      "crawler_name": "Bingbot",
      "first_fetch_ts": "2026-10-25T07:12:14.827Z",
      "minutes_after_publication": 233,
      "channel_tag": null
    },
    {
      "snapshot_label": "261023",
      "snapshot_date": "2026-10-23",
      "crawler": "bingbot",
      "crawler_name": "Bingbot",
      "first_fetch_ts": "2026-10-24T07:12:04.142Z",
      "minutes_after_publication": 234,
      "channel_tag": null
    },
    {
      "snapshot_label": "261023",
      "snapshot_date": "2026-10-23",
      "crawler": "googlebot",
      "crawler_name": "Googlebot",
      "first_fetch_ts": "2026-10-24T10:21:17.658Z",
      "minutes_after_publication": 423,
      "channel_tag": null
    },
    {
      "snapshot_label": "261023",
      "snapshot_date": "2026-10-23",
      "crawler": "googleother",
      "crawler_name": "GoogleOther",
      "first_fetch_ts": "2026-10-24T13:02:39.809Z",
      "minutes_after_publication": 584,
      "channel_tag": null
    },
    {
      "snapshot_label": "261022",
      "snapshot_date": "2026-10-22",
      "crawler": "claudebot",
      "crawler_name": "ClaudeBot",
      "first_fetch_ts": "2026-10-23T06:52:25.517Z",
      "minutes_after_publication": 214,
      "channel_tag": null
    },
    {
      "snapshot_label": "261022",
      "snapshot_date": "2026-10-22",
      "crawler": "googlebot",
      "crawler_name": "Googlebot",
      "first_fetch_ts": "2026-10-23T07:03:21.907Z",
      "minutes_after_publication": 225,
      "channel_tag": null
    },
    {
      "snapshot_label": "261022",
      "snapshot_date": "2026-10-22",
      "crawler": "googleother",
      "crawler_name": "GoogleOther",
      "first_fetch_ts": "2026-10-23T07:03:22.372Z",
      "minutes_after_publication": 225,
      "channel_tag": "h"
    },
    {
      "snapshot_label": "261022",
      "snapshot_date": "2026-10-22",
      "crawler": "bingbot",
      "crawler_name": "Bingbot",
      "first_fetch_ts": "2026-10-23T07:13:29.556Z",
      "minutes_after_publication": 235,
      "channel_tag": null
    }
  ],
  "user_triggered_fetches": [
    {
      "slug": "chatgpt-user",
      "name": "ChatGPT-User",
      "requests": 9,
      "verified_requests": 9,
      "top_path": "/data",
      "verified_pages": 9,
      "paths": [
        {
          "path": "/data",
          "n": 2,
          "v": 2
        },
        {
          "path": "/data/latest/observation_daily.json",
          "n": 2,
          "v": 2
        },
        {
          "path": "/method",
          "n": 2,
          "v": 2
        },
        {
          "path": "/observed",
          "n": 2,
          "v": 2
        },
        {
          "path": "/data/latest/mcp_calls_daily.json",
          "n": 1,
          "v": 1
        }
      ]
    }
  ],
  "convention_probes": [],
  "absent_crawlers": [],
  "invitation": "What data about web crawlers did you look for and not find? Search https://rattlesnakesbymail.com/search?q=your+question. Searches with no match are recorded as gaps. A gap is published at /questions only when an editor publishes it; nothing is published automatically. To suggest something this daily snapshot should include, send a note with target snapshot:261026.",
  "links": {
    "snapshot": "https://261026.rattlesnakesbymail.com/",
    "json": "https://261026.rattlesnakesbymail.com/snapshot.json",
    "markdown": "https://261026.rattlesnakesbymail.com/snapshot.md",
    "csv": "https://261026.rattlesnakesbymail.com/snapshot.csv",
    "facts_json": "https://261026.rattlesnakesbymail.com/facts.json",
    "facts_markdown": "https://261026.rattlesnakesbymail.com/facts.md",
    "about": "https://261026.rattlesnakesbymail.com/about-this-site.md",
    "previous": "https://261025.rattlesnakesbymail.com/?c=p",
    "all_snapshots": "https://rattlesnakesbymail.com/snapshots",
    "mcp": "https://rattlesnakesbymail.com/mcp",
    "dataset": "https://github.com/crank-box/rattlesnakesbymail-data",
    "method": "https://rattlesnakesbymail.com/method"
  }
}