{
  "id": "qwen2-vl",
  "name": "Qwen2-VL (Native-Resolution VLM)",
  "description": "Wang et al. 2024 - the image is not resized to a fixed grid. A native-resolution ViT produces however many patches the image has, and the patch merger folds each 2x2 group into one token four times as wide before the language model sees it. That step is where a VLM context budget is won or lost.",
  "icon": "🧱",
  "category": "Multimodal",
  "components": [
    {
      "id": "in-img",
      "type": "input",
      "name": "Image (native resolution)",
      "position": {
        "x": 240,
        "y": 60
      },
      "params": {
        "shape": [
          3,
          448,
          448
        ]
      },
      "inputs": [],
      "outputs": []
    },
    {
      "id": "patch",
      "type": "patchEmbed",
      "name": "Patchify 14x14",
      "position": {
        "x": 240,
        "y": 160
      },
      "params": {
        "imgSize": 448,
        "patchSize": 14,
        "embedDim": 1280,
        "inChans": 3
      },
      "inputs": [],
      "outputs": []
    },
    {
      "id": "vrope",
      "type": "rope",
      "name": "2D RoPE",
      "position": {
        "x": 240,
        "y": 260
      },
      "params": {
        "dim": 1280,
        "maxSeqLen": 1024,
        "base": 10000
      },
      "inputs": [],
      "outputs": []
    },
    {
      "id": "vit1",
      "type": "transformerBlock",
      "name": "ViT Block 1",
      "position": {
        "x": 240,
        "y": 360
      },
      "params": {
        "embedDim": 1280,
        "numHeads": 16,
        "ffDim": 5120
      },
      "inputs": [],
      "outputs": []
    },
    {
      "id": "vit2",
      "type": "transformerBlock",
      "name": "ViT Block 2",
      "position": {
        "x": 240,
        "y": 460
      },
      "params": {
        "embedDim": 1280,
        "numHeads": 16,
        "ffDim": 5120
      },
      "inputs": [],
      "outputs": []
    },
    {
      "id": "merge",
      "type": "patchMerger",
      "name": "Patch Merger 2x2",
      "position": {
        "x": 240,
        "y": 560
      },
      "params": {
        "mergeSize": 2,
        "inDim": 1280,
        "outDim": 3584
      },
      "inputs": [],
      "outputs": []
    },
    {
      "id": "in-txt",
      "type": "input",
      "name": "Text tokens",
      "position": {
        "x": 600,
        "y": 60
      },
      "params": {
        "shape": [
          1,
          64
        ]
      },
      "inputs": [],
      "outputs": []
    },
    {
      "id": "txt-emb",
      "type": "embedding",
      "name": "Token Embed",
      "position": {
        "x": 600,
        "y": 160
      },
      "params": {
        "vocabSize": 152064,
        "embeddingDim": 3584
      },
      "inputs": [],
      "outputs": []
    },
    {
      "id": "vis-r",
      "type": "reshape",
      "name": "to [1, 256, 3584]",
      "position": {
        "x": 240,
        "y": 660
      },
      "params": {
        "shape": [
          1,
          256,
          3584
        ]
      },
      "inputs": [],
      "outputs": []
    },
    {
      "id": "cat",
      "type": "concatenate",
      "name": "[image tokens; text tokens]",
      "position": {
        "x": 420,
        "y": 760
      },
      "params": {
        "dim": 1,
        "numInputs": 2
      },
      "inputs": [],
      "outputs": []
    },
    {
      "id": "llm1",
      "type": "transformerBlock",
      "name": "LLM Block 1",
      "position": {
        "x": 420,
        "y": 860
      },
      "params": {
        "embedDim": 3584,
        "numHeads": 28,
        "ffDim": 18944
      },
      "inputs": [],
      "outputs": []
    },
    {
      "id": "llm2",
      "type": "transformerBlock",
      "name": "LLM Block 2",
      "position": {
        "x": 420,
        "y": 960
      },
      "params": {
        "embedDim": 3584,
        "numHeads": 28,
        "ffDim": 18944
      },
      "inputs": [],
      "outputs": []
    },
    {
      "id": "norm",
      "type": "rmsNorm",
      "name": "RMSNorm",
      "position": {
        "x": 420,
        "y": 1060
      },
      "params": {
        "normalizedShape": 3584,
        "eps": 1e-06
      },
      "inputs": [],
      "outputs": []
    },
    {
      "id": "head",
      "type": "lmHead",
      "name": "LM Head",
      "position": {
        "x": 420,
        "y": 1160
      },
      "params": {
        "vocabSize": 152064,
        "hiddenSize": 3584,
        "tieWeights": true
      },
      "inputs": [],
      "outputs": []
    },
    {
      "id": "out",
      "type": "output",
      "name": "next token logits",
      "position": {
        "x": 420,
        "y": 1260
      },
      "params": {},
      "inputs": [],
      "outputs": []
    }
  ],
  "connections": [
    {
      "id": "c1",
      "from": "in-img",
      "to": "patch",
      "fromPort": "bottom",
      "toPort": "top"
    },
    {
      "id": "c2",
      "from": "patch",
      "to": "vrope",
      "fromPort": "bottom",
      "toPort": "top"
    },
    {
      "id": "c3",
      "from": "vrope",
      "to": "vit1",
      "fromPort": "bottom",
      "toPort": "top"
    },
    {
      "id": "c4",
      "from": "vit1",
      "to": "vit2",
      "fromPort": "bottom",
      "toPort": "top"
    },
    {
      "id": "c5",
      "from": "vit2",
      "to": "merge",
      "fromPort": "bottom",
      "toPort": "top"
    },
    {
      "id": "c6",
      "from": "in-txt",
      "to": "txt-emb",
      "fromPort": "bottom",
      "toPort": "top"
    },
    {
      "id": "c7",
      "from": "merge",
      "to": "vis-r",
      "fromPort": "bottom",
      "toPort": "top"
    },
    {
      "id": "c8",
      "from": "vis-r",
      "to": "cat",
      "fromPort": "bottom",
      "toPort": "top"
    },
    {
      "id": "c9",
      "from": "txt-emb",
      "to": "cat",
      "fromPort": "bottom",
      "toPort": "top"
    },
    {
      "id": "c10",
      "from": "cat",
      "to": "llm1",
      "fromPort": "bottom",
      "toPort": "top"
    },
    {
      "id": "c11",
      "from": "llm1",
      "to": "llm2",
      "fromPort": "bottom",
      "toPort": "top"
    },
    {
      "id": "c12",
      "from": "llm2",
      "to": "norm",
      "fromPort": "bottom",
      "toPort": "top"
    },
    {
      "id": "c13",
      "from": "norm",
      "to": "head",
      "fromPort": "bottom",
      "toPort": "top"
    },
    {
      "id": "c14",
      "from": "head",
      "to": "out",
      "fromPort": "bottom",
      "toPort": "top"
    }
  ]
}