{
  "site": "https://neurarch.com",
  "description": "Datasets Neurarch publishes. Every entry names its licence, its downloads, and the limit that would make it the wrong dataset for a question.",
  "count": 9,
  "catalog": "https://neurarch.com/resources.html",
  "datasets": [
    {
      "key": "architectures",
      "name": "Neurarch architecture corpus",
      "description": "36 neural network architectures kept as typed graphs rather than diagrams. Each entry carries every layer's type and parameters, the tensor shape propagated through it, an estimated parameter count, the verdict of 41 structural checks, and a link to the model.json an agent can fetch, edit and submit back for verification. Families span vision, language, recommendation, diffusion, biosignal and speech.",
      "size": "36 architectures",
      "license": "CC0-1.0",
      "licenseUrl": "https://creativecommons.org/publicdomain/zero/1.0/",
      "keywords": [
        "neural architecture",
        "model graph",
        "deep learning",
        "pytorch",
        "machine learning"
      ],
      "variableMeasured": [
        "layer count",
        "estimated parameter count",
        "input shape",
        "output shape",
        "verifier verdict"
      ],
      "caveat": "Every graph here passes our own verifier by construction: a template with a blocking finding is held back from the gallery rather than published. That makes this a corpus of designs that are structurally sound, not a sample of designs people actually write, and the wrong set to measure a checker's recall against. The grounding study is the one with failures in it.",
      "page": "https://neurarch.com/d/architectures.html",
      "markdown": "https://neurarch.com/d/architectures.md",
      "distributions": [
        {
          "name": "Index of every architecture",
          "url": "https://neurarch.com/a/index.json",
          "encodingFormat": "application/json"
        },
        {
          "name": "Hugging Face mirror (dataset viewer, load_dataset)",
          "url": "https://huggingface.co/datasets/neurarch-ai/neurarch-architectures",
          "encodingFormat": "text/html"
        }
      ],
      "sameAs": [
        "https://huggingface.co/datasets/neurarch-ai/neurarch-architectures"
      ]
    },
    {
      "key": "checks",
      "name": "Neurarch structural check catalogue",
      "description": "The 41 structural checks Neurarch runs on a model graph, as data: id, severity, category, the condition that triggers it, why it costs something, and the fix. These are the checks that run in the editor as a design is built, in CI through the GitHub Action, and over the wire at POST /api/v1/check, so the catalogue is the vocabulary any of those three surfaces grades in.",
      "size": "41 checks",
      "license": "CC-BY-4.0",
      "licenseUrl": "https://creativecommons.org/licenses/by/4.0/",
      "keywords": [
        "static analysis",
        "neural architecture",
        "linting",
        "model verification",
        "machine learning"
      ],
      "variableMeasured": [
        "check id",
        "severity",
        "category",
        "trigger condition",
        "provenance"
      ],
      "caveat": "Severity is our editorial judgment, not a measured error rate. 3 of the 41 carry a non-null provenance field naming the study that grounds them; the rest follow from the shape algebra or from convention, and the entry says which. Reading the whole catalogue as measured would overstate it by 38 checks.",
      "page": "https://neurarch.com/d/checks.html",
      "markdown": "https://neurarch.com/d/checks.md",
      "distributions": [
        {
          "name": "The catalogue as JSON",
          "url": "https://neurarch.com/r/index.json",
          "encodingFormat": "application/json"
        },
        {
          "name": "Hugging Face mirror (dataset viewer, load_dataset)",
          "url": "https://huggingface.co/datasets/neurarch-ai/neurarch-structural-checks",
          "encodingFormat": "text/html"
        }
      ],
      "sameAs": [
        "https://huggingface.co/datasets/neurarch-ai/neurarch-structural-checks"
      ]
    },
    {
      "key": "grounding-264",
      "name": "Verifier grounding study (264 graphs)",
      "description": "Clean reference architectures plus systematically corrupted variants (broken attention head divisibility, linear width mismatches, severed connections), each built as a real PyTorch model and run on a GPU. Every row pairs the static verifier verdict with what actually happened at runtime: whether the module constructed, whether the forward pass survived, whether training made progress, and the initial and final loss. 264 graphs, two seeds, torch 2.8.",
      "size": "264 graphs",
      "license": "MIT",
      "licenseUrl": "https://opensource.org/licenses/MIT",
      "keywords": [
        "neural architecture",
        "static analysis",
        "empirical study",
        "pytorch",
        "verification"
      ],
      "variableMeasured": [
        "verifier score",
        "verifier blocked",
        "constructs",
        "forward pass ok",
        "makes training progress",
        "initial loss",
        "final loss"
      ],
      "caveat": "The corruptions are injected, not sampled from real mistakes, so the failure modes in here are the ones we chose to plant. This settles that a blocked graph does not run. It says nothing about how often an engineer writes one, and a paper quoting it as a base rate would be quoting it wrong.",
      "page": "https://neurarch.com/d/grounding-264.html",
      "markdown": "https://neurarch.com/d/grounding-264.md",
      "distributions": [
        {
          "name": "Per-graph results",
          "url": "https://raw.githubusercontent.com/neurarch-ai/neurarch-arch-bench/main/grounding_results.csv",
          "encodingFormat": "text/csv"
        },
        {
          "name": "Hugging Face mirror (dataset viewer, load_dataset)",
          "url": "https://huggingface.co/datasets/neurarch-ai/arch-verifier-grounding-264",
          "encodingFormat": "text/html"
        }
      ],
      "sameAs": [
        "https://github.com/neurarch-ai/neurarch-arch-bench",
        "https://huggingface.co/datasets/neurarch-ai/arch-verifier-grounding-264"
      ]
    },
    {
      "key": "arena",
      "name": "Arch-Bench arena results",
      "description": "Frontier language models scored on design-from-spec tasks by a deterministic verifier rather than a human or an LLM judge. Each row is one model on one split: tasks passed out of tasks attempted, mean graph health score, the date of the run, and the rubric version it was measured under. 18 rows across a curated split, a procedurally generated split, and a grounded split whose designs were actually trained on a GPU. Last updated 2026-08-21.",
      "size": "18 model-split results",
      "license": "CC-BY-4.0",
      "licenseUrl": "https://creativecommons.org/licenses/by/4.0/",
      "keywords": [
        "llm evaluation",
        "benchmark",
        "agents",
        "neural architecture",
        "verifiable rewards"
      ],
      "variableMeasured": [
        "model",
        "tasks passed",
        "tasks attempted",
        "mean graph health score",
        "rubric version",
        "run date"
      ],
      "caveat": "Rows carry the rubric version they were measured under and versions are not comparable. Rubric v2 included a task whose start graph already satisfied every constraint, so an empty plan passed it: a v2 row contains one free pass. Ranking a v2 row against a v3 row is the specific mistake this field exists to prevent, and we made it once ourselves.",
      "page": "https://neurarch.com/d/arena.html",
      "markdown": "https://neurarch.com/d/arena.md",
      "distributions": [
        {
          "name": "Board data",
          "url": "https://neurarch.com/leaderboard-data.json",
          "encodingFormat": "application/json"
        }
      ]
    },
    {
      "key": "bench-tasks",
      "name": "Arch-Bench task set",
      "description": "The task definitions behind the benchmark: design-from-spec and repair-and-extend instances for agents that build neural network architectures. Each task pairs a natural-language design brief with a starting graph the agent edits and a set of programmatic pass criteria (structural blockers, parameter budgets and bands, required layer families on an input-to-output path, KV cache and decode-latency ceilings). Twelve curated tasks with eight starting fixtures, plus a deterministic generator that mints a larger split from a seed.",
      "size": "12 curated tasks, 8 fixtures",
      "license": "MIT",
      "licenseUrl": "https://opensource.org/licenses/MIT",
      "keywords": [
        "benchmark",
        "agents",
        "reinforcement learning",
        "neural architecture search",
        "verifiable rewards"
      ],
      "variableMeasured": [
        "task id",
        "design spec",
        "starting graph",
        "pass constraints",
        "difficulty"
      ],
      "caveat": "Twelve curated tasks is a seed, not a set you can rank models on with confidence. The generated split exists because of that, and the numbers worth comparing are the generated-split ones. The curated tasks are best read as the worked examples that show what a task is.",
      "page": "https://neurarch.com/d/bench-tasks.html",
      "markdown": "https://neurarch.com/d/bench-tasks.md",
      "distributions": [
        {
          "name": "Task definitions",
          "url": "https://raw.githubusercontent.com/neurarch-ai/neurarch-arch-bench/main/tasks.json",
          "encodingFormat": "application/json"
        },
        {
          "name": "Hugging Face mirror (dataset viewer, load_dataset)",
          "url": "https://huggingface.co/datasets/neurarch-ai/arch-bench-tasks",
          "encodingFormat": "text/html"
        }
      ],
      "sameAs": [
        "https://github.com/neurarch-ai/neurarch-arch-bench",
        "https://huggingface.co/datasets/neurarch-ai/arch-bench-tasks"
      ]
    },
    {
      "key": "arch-design-sft",
      "name": "arch-design-sft: verified architecture-design SFT data",
      "description": "Supervised fine-tuning data for neural architecture design treated as structured graph editing. Each row pairs a natural-language design spec and a serialized starting graph with a reference action plan, and every row is re-graded by a deterministic verifier before it is written: structural blockers, parameter budgets and bands, required layer families. Ten task families across six design-from-spec and four edit-in-place, where wholesale rebuilds are forbidden so the data teaches surgical edits rather than generation. Tasks are procedurally minted from a seed, so an evaluation split on a different seed has never existed anywhere.",
      "size": "3,010 verified examples",
      "license": "MIT",
      "licenseUrl": "https://opensource.org/licenses/MIT",
      "keywords": [
        "supervised fine-tuning",
        "synthetic data",
        "agents",
        "neural architecture",
        "verifiable rewards",
        "llm training"
      ],
      "variableMeasured": [
        "task family",
        "seed",
        "design spec",
        "starting graph",
        "reference action plan",
        "verifier grade"
      ],
      "caveat": "The targets are machine-checked, not human-preferred. Every assistant turn provably satisfies its task constraints, and nothing in the pipeline claims it is the design an experienced engineer would have chosen. Procedural generation is what makes the split contamination-free and is also what makes it narrower than real design work.",
      "page": "https://neurarch.com/d/arch-design-sft.html",
      "markdown": "https://neurarch.com/d/arch-design-sft.md",
      "distributions": [
        {
          "name": "Chat format (TRL SFTTrainer drop-in)",
          "url": "https://raw.githubusercontent.com/neurarch-ai/neurarch-arch-bench/main/training/sft-3k.chat.jsonl",
          "encodingFormat": "application/x-ndjson"
        },
        {
          "name": "Raw fields, for custom formatting",
          "url": "https://raw.githubusercontent.com/neurarch-ai/neurarch-arch-bench/main/training/sft-3k.raw.jsonl",
          "encodingFormat": "application/x-ndjson"
        },
        {
          "name": "Hugging Face mirror (dataset viewer, load_dataset)",
          "url": "https://huggingface.co/datasets/neurarch-ai/arch-design-sft",
          "encodingFormat": "text/html"
        }
      ],
      "sameAs": [
        "https://github.com/neurarch-ai/neurarch-arch-bench/blob/main/training/DATASET_CARD.md",
        "https://huggingface.co/datasets/neurarch-ai/arch-design-sft"
      ]
    },
    {
      "key": "outcome-triples",
      "name": "Structure, verdict and trained outcome triples",
      "description": "The corpus that pairs what a design looks like with what it did. Each row carries a structural fingerprint of a graph (task family, parameter count, component count), the static verifier verdict on it (score and blockers), and the outcome of actually training it: the full loss curve, best and final loss, area under the loss curve, step count, and whether it converged. Every graph was trained through the same managed-GPU path the product uses, so the outcome is measured rather than simulated.",
      "size": "80 trained graphs",
      "license": "MIT",
      "licenseUrl": "https://opensource.org/licenses/MIT",
      "keywords": [
        "neural architecture",
        "training dynamics",
        "empirical study",
        "model selection",
        "machine learning"
      ],
      "variableMeasured": [
        "task family",
        "parameter count",
        "component count",
        "verifier score",
        "blockers",
        "loss curve",
        "best loss",
        "final loss",
        "steps",
        "converged"
      ],
      "caveat": "n is 80, and the headline finding is negative: across families the correlation between verifier score and achievable quality is about zero (rho -0.02), undefined inside four families whose graphs score identically, and positive only where the score varies. It is published because that is the result. The verifier tells you whether a graph runs, not how good it will get, and this is the file that shows it.",
      "page": "https://neurarch.com/d/outcome-triples.html",
      "markdown": "https://neurarch.com/d/outcome-triples.md",
      "distributions": [
        {
          "name": "Triples",
          "url": "https://raw.githubusercontent.com/neurarch-ai/neurarch-arch-bench/main/triples.jsonl",
          "encodingFormat": "application/x-ndjson"
        },
        {
          "name": "Hugging Face mirror (dataset viewer, load_dataset)",
          "url": "https://huggingface.co/datasets/neurarch-ai/arch-outcome-triples",
          "encodingFormat": "text/html"
        }
      ],
      "sameAs": [
        "https://github.com/neurarch-ai/neurarch-arch-bench/blob/main/VERIFICATION.md",
        "https://huggingface.co/datasets/neurarch-ai/arch-outcome-triples"
      ]
    },
    {
      "key": "reasoning-claude",
      "name": "Verified architecture-design reasoning traces (Claude)",
      "description": "Spec to reasoning to design triples where the design is re-graded by the same deterministic verifier the benchmark uses, and only passing traces are kept. No LLM judge anywhere. Each row carries the natural-language spec, the starting graph, step-by-step reasoning over the constraints (required layers, shapes, head divisibility, parameter budgets), the structured edits that produce the design, and a chat-format rendering with the reasoning inside reasoning tags. This is verified reasoning data of the shape RLVR and reasoning-model post-training consume, for a domain no code-only corpus contains.",
      "size": "306 verified traces",
      "license": "MIT",
      "licenseUrl": "https://opensource.org/licenses/MIT",
      "keywords": [
        "reasoning",
        "verifiable rewards",
        "rlvr",
        "neural architecture",
        "llm training",
        "rejection sampling"
      ],
      "variableMeasured": [
        "task id",
        "design spec",
        "starting graph",
        "reasoning",
        "actions",
        "verified",
        "source model"
      ],
      "caveat": "Rejection-sampled, so the distribution is what one model got right rather than what the task space looks like: every kept trace passed, and the ones that failed are not here, which makes this useless for measuring how hard the tasks are. The reasoning is the model's own text and was never checked for whether it describes what the actions actually do; only the final graph was verified.",
      "page": "https://neurarch.com/d/reasoning-claude.html",
      "markdown": "https://neurarch.com/d/reasoning-claude.md",
      "distributions": [
        {
          "name": "Hugging Face mirror (dataset viewer, load_dataset)",
          "url": "https://huggingface.co/datasets/neurarch-ai/arch-reasoning-claude",
          "encodingFormat": "text/html"
        }
      ],
      "sameAs": [
        "https://huggingface.co/datasets/neurarch-ai/arch-reasoning-claude"
      ]
    },
    {
      "key": "reasoning-grok",
      "name": "Verified architecture-design reasoning traces (Grok)",
      "description": "The same verified spec to reasoning to design triples as the Claude split, rejection-sampled from a different frontier model, so the two can be pooled for volume or held apart to see how much of the reasoning style is model-specific. Every row's final graph was re-graded by the deterministic verifier and only passing traces were kept, with no LLM judge in the loop. Fields are identical to the Claude split, which is what makes them poolable at all.",
      "size": "376 verified traces",
      "license": "MIT",
      "licenseUrl": "https://opensource.org/licenses/MIT",
      "keywords": [
        "reasoning",
        "verifiable rewards",
        "rlvr",
        "neural architecture",
        "llm training",
        "rejection sampling"
      ],
      "variableMeasured": [
        "task id",
        "design spec",
        "starting graph",
        "reasoning",
        "actions",
        "verified",
        "source model"
      ],
      "caveat": "Same limit as the Claude split, and one more: the two were minted under different rubric versions, so pooling them without re-grading mixes two pass criteria. Rejection sampling means the failures are absent, so neither split says anything about task difficulty, and the reasoning text itself was never verified against the actions it precedes.",
      "page": "https://neurarch.com/d/reasoning-grok.html",
      "markdown": "https://neurarch.com/d/reasoning-grok.md",
      "distributions": [
        {
          "name": "Hugging Face mirror (dataset viewer, load_dataset)",
          "url": "https://huggingface.co/datasets/neurarch-ai/arch-reasoning-grok",
          "encodingFormat": "text/html"
        }
      ],
      "sameAs": [
        "https://huggingface.co/datasets/neurarch-ai/arch-reasoning-grok"
      ]
    }
  ]
}