{
  "kind": "fact",
  "studySlug": "llmstxt-effect",
  "slug": "the-significance-test-comes-back-null",
  "title": "The top-5,000 test is not statistically significant",
  "description": "Only the top-5,000 subset test is non-significant; the full-sample test is significant.",
  "lastUpdated": "2026-09-04",
  "lastTested": "2026-03-14",
  "sourceStudyUrl": "/trakkr-research/llmstxt-effect",
  "sourceStudyTitle": "The llms.txt Effect",
  "claimIds": [
    "llmstxt-effect:p-value",
    "llmstxt-effect:full-p-value",
    "llmstxt-effect:effect-size"
  ],
  "relatedSlugs": [
    "answer:which-industries-adopt-llms-txt-most",
    "answer:does-llms-txt-help-you-even-if-it-does-not-raise-citations",
    "tracker:llmstxt-adoption-by-sector-tracker"
  ],
  "methodologySummary": "Built from HTTP scans of 37,894 AI-cited domains, linked to 337,362 citations and 882 citation snapshots in the Trakkr corpus.",
  "limitations": [
    "This is an observational study. It measures correlation with citation outcomes, not a controlled experiment.",
    "Adoption is uneven by sector, so raw averages can hide category concentration in SaaS and developer tooling.",
    "The top-5,000 test returns p=0.85. The full scanned sample returns p<0.001 with reported r=-0.065. Neither establishes a causal effect. The public aggregates do not include confidence intervals or document the cohort for citation means and medians well enough to recompute the tests."
  ],
  "keywords": [
    "llms.txt",
    "llms txt effect",
    "AI citations",
    "does llms.txt work"
  ],
  "schemaHints": {
    "pageType": "Article",
    "includeDataset": true
  },
  "claim": "The top-5,000 citation comparison returns p=0.85; the full sample returns p<0.001 with reported r=-0.065.",
  "supportingMetrics": [
    {
      "label": "Top-5,000 Mann-Whitney p-value",
      "value": "0.85",
      "detail": "Subset of the top 5,000 cited domains; not the full-sample test."
    },
    {
      "label": "Full-sample Mann-Whitney p-value",
      "value": "<0.001",
      "detail": "Full 37,894-domain sample. The public JSON rounds this value to zero."
    },
    {
      "label": "Full-sample reported effect size r",
      "value": "-0.065",
      "detail": "Small reported magnitude; not an estimate of causal citation lift."
    }
  ],
  "whyItMatters": "Report both cohorts and avoid interpreting a non-significant test as proof of no effect.",
  "citationBlock": {
    "quote": "The top-5,000 citation comparison returns p=0.85; the full sample returns p<0.001 with reported r=-0.065.",
    "attribution": "The llms.txt Effect - Trakkr Research",
    "url": "https://trakkr.ai/trakkr-research/llmstxt-effect/facts/the-significance-test-comes-back-null"
  }
}