{
 "schema_version": "1.7.0",
 "id": "x_NXMC-2025-0057",
 "modified": "2026-10-19T02:33:38.839Z",
 "published": "2026-10-18T09:00:52.565Z",
 "summary": "phi-4: tokenizer change at 6fbb3d3 (2025-02-04)",
 "details": "phi-4: EOS changed from \"<|endoftext|>\" to \"<|im_end|>\"; anything fine-tuned or served against the old value stops at a different token; pad token \"<|endoftext|>\" to \"<|dummy_85|>\"; batched training masks a different id; the chat template changed and renders differently (tool-call); prompts rendered with the old one no longer match what the model now expects, so cached prompts and eval sets need re-rendering (2025-02-04).\n\nRepository: https://huggingface.co/microsoft/phi-4. Revisions from 6fbb3d3bbe726c99b4188087b4deeec1bceac5ae onward carry this change; a pin at 381727a5ee103da6c1b14ecd3d39cd09832cbcf8 or earlier does not.\n\nChanged keys:\n- eos_token_id: 100257 -> 100265 (tokenizer)\n- pad_token_id: 100257 -> 100349 (tokenizer)\n- tokenizer chat_template_sha256: a41fb8727c4f1e72 -> fe4120415a3858b8 (template)\n- tokenizer eos_token: <|endoftext|> -> <|im_end|> (tokenizer)\n- tokenizer pad_token: <|endoftext|> -> <|dummy_85|> (tokenizer)\n- generation eos_token_id: [100257,100265] -> 100265 (tokenizer)\n- generation pad_token_id: unset -> 100349 (tokenizer)\n\nThis record says what the change does mechanically. It is not a report that anything broke for anyone.",
 "affected": [
  {
   "ranges": [
    {
     "type": "GIT",
     "repo": "https://huggingface.co/microsoft/phi-4",
     "events": [
      {
       "introduced": "6fbb3d3bbe726c99b4188087b4deeec1bceac5ae"
      }
     ]
    }
   ],
   "database_specific": {
    "model": "microsoft/phi-4",
    "previous_revision": "381727a5ee103da6c1b14ecd3d39cd09832cbcf8"
   }
  }
 ],
 "references": [
  {
   "type": "INTRODUCED",
   "url": "https://huggingface.co/microsoft/phi-4/commit/6fbb3d3bbe726c99b4188087b4deeec1bceac5ae"
  },
  {
   "type": "PACKAGE",
   "url": "https://huggingface.co/microsoft/phi-4"
  },
  {
   "type": "WEB",
   "url": "https://www.neurarch.com/changes-feeds/models/microsoft/phi-4.html"
  }
 ],
 "database_specific": {
  "source": "neurarch model change tracker",
  "model": "microsoft/phi-4",
  "kind": "tokenizer",
  "classes": [
   "tokenizer",
   "template"
  ],
  "changed_keys": [
   {
    "file": "config",
    "key": "eos_token_id",
    "before": 100257,
    "after": 100265,
    "class": "tokenizer"
   },
   {
    "file": "config",
    "key": "pad_token_id",
    "before": 100257,
    "after": 100349,
    "class": "tokenizer"
   },
   {
    "file": "tokenizer",
    "key": "chat_template_sha256",
    "before": "a41fb8727c4f1e72",
    "after": "fe4120415a3858b8",
    "class": "template"
   },
   {
    "file": "tokenizer",
    "key": "eos_token",
    "before": "<|endoftext|>",
    "after": "<|im_end|>",
    "class": "tokenizer"
   },
   {
    "file": "tokenizer",
    "key": "pad_token",
    "before": "<|endoftext|>",
    "after": "<|dummy_85|>",
    "class": "tokenizer"
   },
   {
    "file": "generation",
    "key": "eos_token_id",
    "before": [
     100257,
     100265
    ],
    "after": 100265,
    "class": "tokenizer"
   },
   {
    "file": "generation",
    "key": "pad_token_id",
    "before": null,
    "after": 100349,
    "class": "tokenizer"
   }
  ],
  "impact": {
   "memory": false,
   "serving": true,
   "tokenizer": true,
   "licence": null
  },
  "breaking": true,
  "breaking_rule": "https://www.neurarch.com/changes-feeds/advisories/index.json#breaking_rule",
  "consequence": {
   "params": null,
   "weightsGbBf16": null,
   "trainPeakGb": null,
   "fingerprint": null,
   "context": null,
   "eos": {
    "before": "<|endoftext|>",
    "after": "<|im_end|>"
   },
   "bos": null
  },
  "commit_date": "2025-02-04T11:51:00.000Z",
  "detected_at": "2026-10-18T09:00:52.565Z",
  "detected_by": "backfill",
  "days_since_first_commit": 54,
  "license": "CC-BY-4.0"
 }
}