{ "id": "llava-1.5-7b", "name": "LLaVA-1.5-7B", "description": "LLaVA-1.5-7B (Microsoft / UW-Madison): full graph; dimensions verified against the official config.json.", "components": [ { "id": "n1", "type": "input", "name": "image", "position": { "x": 150, "y": 50 }, "params": { "shape": [ 3, 336, 336 ] }, "inputs": [], "outputs": [ "n2" ] }, { "id": "n2", "type": "patchEmbed", "name": "clip_patch_14", "position": { "x": 150, "y": 200 }, "params": { "imgSize": 336, "patchSize": 14, "embedDim": 1024, "inChans": 3 }, "inputs": [ "n1" ], "outputs": [ "n3" ] }, { "id": "n3", "type": "positionalEncoding", "name": "vis_pos", "position": { "x": 150, "y": 350 }, "params": { "maxLen": 576, "embedDim": 1024 }, "inputs": [ "n2" ], "outputs": [ "n4" ] }, { "id": "n4", "type": "transformerBlock", "name": "clip_block_1", "position": { "x": 150, "y": 500 }, "params": { "embedDim": 1024, "numHeads": 16, "ffDim": 4096 }, "inputs": [ "n3" ], "outputs": [ "n5" ] }, { "id": "n5", "type": "transformerBlock", "name": "clip_block_2", "position": { "x": 150, "y": 650 }, "params": { "embedDim": 1024, "numHeads": 16, "ffDim": 4096 }, "inputs": [ "n4" ], "outputs": [ "n6" ] }, { "id": "n6", "type": "transformerBlock", "name": "clip_block_3", "position": { "x": 150, "y": 800 }, "params": { "embedDim": 1024, "numHeads": 16, "ffDim": 4096 }, "inputs": [ "n5" ], "outputs": [ "n7" ] }, { "id": "n7", "type": "transformerBlock", "name": "clip_block_4", "position": { "x": 150, "y": 950 }, "params": { "embedDim": 1024, "numHeads": 16, "ffDim": 4096 }, "inputs": [ "n6" ], "outputs": [ "n8" ] }, { "id": "n8", "type": "transformerBlock", "name": "clip_block_5", "position": { "x": 150, "y": 1100 }, "params": { "embedDim": 1024, "numHeads": 16, "ffDim": 4096 }, "inputs": [ "n7" ], "outputs": [ "n9" ] }, { "id": "n9", "type": "transformerBlock", "name": "clip_block_6", "position": { "x": 150, "y": 1250 }, "params": { "embedDim": 1024, "numHeads": 16, "ffDim": 4096 }, "inputs": [ "n8" ], "outputs": [ "n10" ] }, { "id": "n10", "type": "transformerBlock", "name": "clip_block_7", "position": { "x": 150, "y": 1400 }, "params": { "embedDim": 1024, "numHeads": 16, "ffDim": 4096 }, "inputs": [ "n9" ], "outputs": [ "n11" ] }, { "id": "n11", "type": "transformerBlock", "name": "clip_block_8", "position": { "x": 150, "y": 1550 }, "params": { "embedDim": 1024, "numHeads": 16, "ffDim": 4096 }, "inputs": [ "n10" ], "outputs": [ "n12" ] }, { "id": "n12", "type": "transformerBlock", "name": "clip_block_9", "position": { "x": 150, "y": 1700 }, "params": { "embedDim": 1024, "numHeads": 16, "ffDim": 4096 }, "inputs": [ "n11" ], "outputs": [ "n13" ] }, { "id": "n13", "type": "transformerBlock", "name": "clip_block_10", "position": { "x": 150, "y": 1850 }, "params": { "embedDim": 1024, "numHeads": 16, "ffDim": 4096 }, "inputs": [ "n12" ], "outputs": [ "n14" ] }, { "id": "n14", "type": "transformerBlock", "name": "clip_block_11", "position": { "x": 150, "y": 2000 }, "params": { "embedDim": 1024, "numHeads": 16, "ffDim": 4096 }, "inputs": [ "n13" ], "outputs": [ "n15" ] }, { "id": "n15", "type": "transformerBlock", "name": "clip_block_12", "position": { "x": 150, "y": 2150 }, "params": { "embedDim": 1024, "numHeads": 16, "ffDim": 4096 }, "inputs": [ "n14" ], "outputs": [ "n16" ] }, { "id": "n16", "type": "transformerBlock", "name": "clip_block_13", "position": { "x": 150, "y": 2300 }, "params": { "embedDim": 1024, "numHeads": 16, "ffDim": 4096 }, "inputs": [ "n15" ], "outputs": [ "n17" ] }, { "id": "n17", "type": "transformerBlock", "name": "clip_block_14", "position": { "x": 150, "y": 2450 }, "params": { "embedDim": 1024, "numHeads": 16, "ffDim": 4096 }, "inputs": [ "n16" ], "outputs": [ "n18" ] }, { "id": "n18", "type": "transformerBlock", "name": "clip_block_15", "position": { "x": 150, "y": 2600 }, "params": { "embedDim": 1024, "numHeads": 16, "ffDim": 4096 }, "inputs": [ "n17" ], "outputs": [ "n19" ] }, { "id": "n19", "type": "transformerBlock", "name": "clip_block_16", "position": { "x": 150, "y": 2750 }, "params": { "embedDim": 1024, "numHeads": 16, "ffDim": 4096 }, "inputs": [ "n18" ], "outputs": [ "n20" ] }, { "id": "n20", "type": "transformerBlock", "name": "clip_block_17", "position": { "x": 150, "y": 2900 }, "params": { "embedDim": 1024, "numHeads": 16, "ffDim": 4096 }, "inputs": [ "n19" ], "outputs": [ "n21" ] }, { "id": "n21", "type": "transformerBlock", "name": "clip_block_18", "position": { "x": 150, "y": 3050 }, "params": { "embedDim": 1024, "numHeads": 16, "ffDim": 4096 }, "inputs": [ "n20" ], "outputs": [ "n22" ] }, { "id": "n22", "type": "transformerBlock", "name": "clip_block_19", "position": { "x": 150, "y": 3200 }, "params": { "embedDim": 1024, "numHeads": 16, "ffDim": 4096 }, "inputs": [ "n21" ], "outputs": [ "n23" ] }, { "id": "n23", "type": "transformerBlock", "name": "clip_block_20", "position": { "x": 150, "y": 3350 }, "params": { "embedDim": 1024, "numHeads": 16, "ffDim": 4096 }, "inputs": [ "n22" ], "outputs": [ "n24" ] }, { "id": "n24", "type": "transformerBlock", "name": "clip_block_21", "position": { "x": 150, "y": 3500 }, "params": { "embedDim": 1024, "numHeads": 16, "ffDim": 4096 }, "inputs": [ "n23" ], "outputs": [ "n25" ] }, { "id": "n25", "type": "transformerBlock", "name": "clip_block_22", "position": { "x": 150, "y": 3650 }, "params": { "embedDim": 1024, "numHeads": 16, "ffDim": 4096 }, "inputs": [ "n24" ], "outputs": [ "n26" ] }, { "id": "n26", "type": "transformerBlock", "name": "clip_block_23", "position": { "x": 150, "y": 3800 }, "params": { "embedDim": 1024, "numHeads": 16, "ffDim": 4096 }, "inputs": [ "n25" ], "outputs": [ "n27" ] }, { "id": "n27", "type": "transformerBlock", "name": "clip_block_24", "position": { "x": 150, "y": 3950 }, "params": { "embedDim": 1024, "numHeads": 16, "ffDim": 4096 }, "inputs": [ "n26" ], "outputs": [ "n28" ] }, { "id": "n28", "type": "linear", "name": "projector_1", "position": { "x": 150, "y": 4100 }, "params": { "inFeatures": 1024, "outFeatures": 4096 }, "inputs": [ "n27" ], "outputs": [ "n29" ] }, { "id": "n29", "type": "gelu", "name": "projector_gelu", "position": { "x": 150, "y": 4250 }, "params": {}, "inputs": [ "n28" ], "outputs": [ "n30" ] }, { "id": "n30", "type": "linear", "name": "projector_2", "position": { "x": 150, "y": 4400 }, "params": { "inFeatures": 4096, "outFeatures": 4096 }, "inputs": [ "n29" ], "outputs": [ "n30b" ] }, { "id": "n30b", "type": "reshape", "name": "vision_tokens", "position": { "x": 150, "y": 4475 }, "params": { "shape": [ 1, 576, 4096 ] }, "inputs": [ "n30" ], "outputs": [ "n33" ] }, { "id": "n31", "type": "input", "name": "text_tokens", "position": { "x": 600, "y": 50 }, "params": { "shape": [ 1, 2048 ] }, "inputs": [], "outputs": [ "n32" ] }, { "id": "n32", "type": "embedding", "name": "token_embed", "position": { "x": 600, "y": 200 }, "params": { "numEmbeddings": 32000, "embeddingDim": 4096 }, "inputs": [ "n31" ], "outputs": [ "n33" ] }, { "id": "n33", "type": "concatenate", "name": "image+text_tokens", "position": { "x": 380, "y": 4550 }, "params": { "dim": 1 }, "inputs": [ "n30b", "n32" ], "outputs": [ "n34", "n36" ] }, { "id": "n34", "type": "rmsNorm", "name": "llm_attn_norm_1", "position": { "x": 380, "y": 4700 }, "params": { "normalizedShape": 4096 }, "inputs": [ "n33" ], "outputs": [ "n35" ] }, { "id": "n35", "type": "groupedQueryAttention", "name": "llm_attn_1", "position": { "x": 380, "y": 4850 }, "params": { "embedDim": 4096, "numHeads": 32, "numKVHeads": 32 }, "inputs": [ "n34" ], "outputs": [ "n36" ] }, { "id": "n36", "type": "add", "name": "llm_res1_1", "position": { "x": 380, "y": 5000 }, "params": {}, "inputs": [ "n35", "n33" ], "outputs": [ "n37", "n39" ] }, { "id": "n37", "type": "rmsNorm", "name": "llm_ffn_norm_1", "position": { "x": 380, "y": 5150 }, "params": { "normalizedShape": 4096 }, "inputs": [ "n36" ], "outputs": [ "n38" ] }, { "id": "n38", "type": "swiglu", "name": "llm_ffn_1", "position": { "x": 380, "y": 5300 }, "params": { "embedDim": 4096, "intermediateSize": 11008 }, "inputs": [ "n37" ], "outputs": [ "n39" ] }, { "id": "n39", "type": "add", "name": "llm_res2_1", "position": { "x": 380, "y": 5450 }, "params": {}, "inputs": [ "n38", "n36" ], "outputs": [ "n40", "n42" ] }, { "id": "n40", "type": "rmsNorm", "name": "llm_attn_norm_2", "position": { "x": 380, "y": 5600 }, "params": { "normalizedShape": 4096 }, "inputs": [ "n39" ], "outputs": [ "n41" ] }, { "id": "n41", "type": "groupedQueryAttention", "name": "llm_attn_2", "position": { "x": 380, "y": 5750 }, "params": { "embedDim": 4096, "numHeads": 32, "numKVHeads": 32 }, "inputs": [ "n40" ], "outputs": [ "n42" ] }, { "id": "n42", "type": "add", "name": "llm_res1_2", "position": { "x": 380, "y": 5900 }, "params": {}, "inputs": [ "n41", "n39" ], "outputs": [ "n43", "n45" ] }, { "id": "n43", "type": "rmsNorm", "name": "llm_ffn_norm_2", "position": { "x": 380, "y": 6050 }, "params": { "normalizedShape": 4096 }, "inputs": [ "n42" ], "outputs": [ "n44" ] }, { "id": "n44", "type": "swiglu", "name": "llm_ffn_2", "position": { "x": 380, "y": 6200 }, "params": { "embedDim": 4096, "intermediateSize": 11008 }, "inputs": [ "n43" ], "outputs": [ "n45" ] }, { "id": "n45", "type": "add", "name": "llm_res2_2", "position": { "x": 380, "y": 6350 }, "params": {}, "inputs": [ "n44", "n42" ], "outputs": [ "n46", "n48" ] }, { "id": "n46", "type": "rmsNorm", "name": "llm_attn_norm_3", "position": { "x": 380, "y": 6500 }, "params": { "normalizedShape": 4096 }, "inputs": [ "n45" ], "outputs": [ "n47" ] }, { "id": "n47", "type": "groupedQueryAttention", "name": "llm_attn_3", "position": { "x": 380, "y": 6650 }, "params": { "embedDim": 4096, "numHeads": 32, "numKVHeads": 32 }, "inputs": [ "n46" ], "outputs": [ "n48" ] }, { "id": "n48", "type": "add", "name": "llm_res1_3", "position": { "x": 380, "y": 6800 }, "params": {}, "inputs": [ "n47", "n45" ], "outputs": [ "n49", "n51" ] }, { "id": "n49", "type": "rmsNorm", "name": "llm_ffn_norm_3", "position": { "x": 380, "y": 6950 }, "params": { "normalizedShape": 4096 }, "inputs": [ "n48" ], "outputs": [ "n50" ] }, { "id": "n50", "type": "swiglu", "name": "llm_ffn_3", "position": { "x": 380, "y": 7100 }, "params": { "embedDim": 4096, "intermediateSize": 11008 }, "inputs": [ "n49" ], "outputs": [ "n51" ] }, { "id": "n51", "type": "add", "name": "llm_res2_3", "position": { "x": 380, "y": 7250 }, "params": {}, "inputs": [ "n50", "n48" ], "outputs": [ "n52", "n54" ] }, { "id": "n52", "type": "rmsNorm", "name": "llm_attn_norm_4", "position": { "x": 380, "y": 7400 }, "params": { "normalizedShape": 4096 }, "inputs": [ "n51" ], "outputs": [ "n53" ] }, { "id": "n53", "type": "groupedQueryAttention", "name": "llm_attn_4", "position": { "x": 380, "y": 7550 }, "params": { "embedDim": 4096, "numHeads": 32, "numKVHeads": 32 }, "inputs": [ "n52" ], "outputs": [ "n54" ] }, { "id": "n54", "type": "add", "name": "llm_res1_4", "position": { "x": 380, "y": 7700 }, "params": {}, "inputs": [ "n53", "n51" ], "outputs": [ "n55", "n57" ] }, { "id": "n55", "type": "rmsNorm", "name": "llm_ffn_norm_4", "position": { "x": 380, "y": 7850 }, "params": { "normalizedShape": 4096 }, "inputs": [ "n54" ], "outputs": [ "n56" ] }, { "id": "n56", "type": "swiglu", "name": "llm_ffn_4", "position": { "x": 380, "y": 8000 }, "params": { "embedDim": 4096, "intermediateSize": 11008 }, "inputs": [ "n55" ], "outputs": [ "n57" ] }, { "id": "n57", "type": "add", "name": "llm_res2_4", "position": { "x": 380, "y": 8150 }, "params": {}, "inputs": [ "n56", "n54" ], "outputs": [ "n58", "n60" ] }, { "id": "n58", "type": "rmsNorm", "name": "llm_attn_norm_5", "position": { "x": 380, "y": 8300 }, "params": { "normalizedShape": 4096 }, "inputs": [ "n57" ], "outputs": [ "n59" ] }, { "id": "n59", "type": "groupedQueryAttention", "name": "llm_attn_5", "position": { "x": 380, "y": 8450 }, "params": { "embedDim": 4096, "numHeads": 32, "numKVHeads": 32 }, "inputs": [ "n58" ], "outputs": [ "n60" ] }, { "id": "n60", "type": "add", "name": "llm_res1_5", "position": { "x": 380, "y": 8600 }, "params": {}, "inputs": [ "n59", "n57" ], "outputs": [ "n61", "n63" ] }, { "id": "n61", "type": "rmsNorm", "name": "llm_ffn_norm_5", "position": { "x": 380, "y": 8750 }, "params": { "normalizedShape": 4096 }, "inputs": [ "n60" ], "outputs": [ "n62" ] }, { "id": "n62", "type": "swiglu", "name": "llm_ffn_5", "position": { "x": 380, "y": 8900 }, "params": { "embedDim": 4096, "intermediateSize": 11008 }, "inputs": [ "n61" ], "outputs": [ "n63" ] }, { "id": "n63", "type": "add", "name": "llm_res2_5", "position": { "x": 380, "y": 9050 }, "params": {}, "inputs": [ "n62", "n60" ], "outputs": [ "n64", "n66" ] }, { "id": "n64", "type": "rmsNorm", "name": "llm_attn_norm_6", "position": { "x": 380, "y": 9200 }, "params": { "normalizedShape": 4096 }, "inputs": [ "n63" ], "outputs": [ "n65" ] }, { "id": "n65", "type": "groupedQueryAttention", "name": "llm_attn_6", "position": { "x": 380, "y": 9350 }, "params": { "embedDim": 4096, "numHeads": 32, "numKVHeads": 32 }, "inputs": [ "n64" ], "outputs": [ "n66" ] }, { "id": "n66", "type": "add", "name": "llm_res1_6", "position": { "x": 380, "y": 9500 }, "params": {}, "inputs": [ "n65", "n63" ], "outputs": [ "n67", "n69" ] }, { "id": "n67", "type": "rmsNorm", "name": "llm_ffn_norm_6", "position": { "x": 380, "y": 9650 }, "params": { "normalizedShape": 4096 }, "inputs": [ "n66" ], "outputs": [ "n68" ] }, { "id": "n68", "type": "swiglu", "name": "llm_ffn_6", "position": { "x": 380, "y": 9800 }, "params": { "embedDim": 4096, "intermediateSize": 11008 }, "inputs": [ "n67" ], "outputs": [ "n69" ] }, { "id": "n69", "type": "add", "name": "llm_res2_6", "position": { "x": 380, "y": 9950 }, "params": {}, "inputs": [ "n68", "n66" ], "outputs": [ "n70", "n72" ] }, { "id": "n70", "type": "rmsNorm", "name": "llm_attn_norm_7", "position": { "x": 380, "y": 10100 }, "params": { "normalizedShape": 4096 }, "inputs": [ "n69" ], "outputs": [ "n71" ] }, { "id": "n71", "type": "groupedQueryAttention", "name": "llm_attn_7", "position": { "x": 380, "y": 10250 }, "params": { "embedDim": 4096, "numHeads": 32, "numKVHeads": 32 }, "inputs": [ "n70" ], "outputs": [ "n72" ] }, { "id": "n72", "type": "add", "name": "llm_res1_7", "position": { "x": 380, "y": 10400 }, "params": {}, "inputs": [ "n71", "n69" ], "outputs": [ "n73", "n75" ] }, { "id": "n73", "type": "rmsNorm", "name": "llm_ffn_norm_7", "position": { "x": 380, "y": 10550 }, "params": { "normalizedShape": 4096 }, "inputs": [ "n72" ], "outputs": [ "n74" ] }, { "id": "n74", "type": "swiglu", "name": "llm_ffn_7", "position": { "x": 380, "y": 10700 }, "params": { "embedDim": 4096, "intermediateSize": 11008 }, "inputs": [ "n73" ], "outputs": [ "n75" ] }, { "id": "n75", "type": "add", "name": "llm_res2_7", "position": { "x": 380, "y": 10850 }, "params": {}, "inputs": [ "n74", "n72" ], "outputs": [ "n76", "n78" ] }, { "id": "n76", "type": "rmsNorm", "name": "llm_attn_norm_8", "position": { "x": 380, "y": 11000 }, "params": { "normalizedShape": 4096 }, "inputs": [ "n75" ], "outputs": [ "n77" ] }, { "id": "n77", "type": "groupedQueryAttention", "name": "llm_attn_8", "position": { "x": 380, "y": 11150 }, "params": { "embedDim": 4096, "numHeads": 32, "numKVHeads": 32 }, "inputs": [ "n76" ], "outputs": [ "n78" ] }, { "id": "n78", "type": "add", "name": "llm_res1_8", "position": { "x": 380, "y": 11300 }, "params": {}, "inputs": [ "n77", "n75" ], "outputs": [ "n79", "n81" ] }, { "id": "n79", "type": "rmsNorm", "name": "llm_ffn_norm_8", "position": { "x": 380, "y": 11450 }, "params": { "normalizedShape": 4096 }, "inputs": [ "n78" ], "outputs": [ "n80" ] }, { "id": "n80", "type": "swiglu", "name": "llm_ffn_8", "position": { "x": 380, "y": 11600 }, "params": { "embedDim": 4096, "intermediateSize": 11008 }, "inputs": [ "n79" ], "outputs": [ "n81" ] }, { "id": "n81", "type": "add", "name": "llm_res2_8", "position": { "x": 380, "y": 11750 }, "params": {}, "inputs": [ "n80", "n78" ], "outputs": [ "n82", "n84" ] }, { "id": "n82", "type": "rmsNorm", "name": "llm_attn_norm_9", "position": { "x": 380, "y": 11900 }, "params": { "normalizedShape": 4096 }, "inputs": [ "n81" ], "outputs": [ "n83" ] }, { "id": "n83", "type": "groupedQueryAttention", "name": "llm_attn_9", "position": { "x": 380, "y": 12050 }, "params": { "embedDim": 4096, "numHeads": 32, "numKVHeads": 32 }, "inputs": [ "n82" ], "outputs": [ "n84" ] }, { "id": "n84", "type": "add", "name": "llm_res1_9", "position": { "x": 380, "y": 12200 }, "params": {}, "inputs": [ "n83", "n81" ], "outputs": [ "n85", "n87" ] }, { "id": "n85", "type": "rmsNorm", "name": "llm_ffn_norm_9", "position": { "x": 380, "y": 12350 }, "params": { "normalizedShape": 4096 }, "inputs": [ "n84" ], "outputs": [ "n86" ] }, { "id": "n86", "type": "swiglu", "name": "llm_ffn_9", "position": { "x": 380, "y": 12500 }, "params": { "embedDim": 4096, "intermediateSize": 11008 }, "inputs": [ "n85" ], "outputs": [ "n87" ] }, { "id": "n87", "type": "add", "name": "llm_res2_9", "position": { "x": 380, "y": 12650 }, "params": {}, "inputs": [ "n86", "n84" ], "outputs": [ "n88", "n90" ] }, { "id": "n88", "type": "rmsNorm", "name": "llm_attn_norm_10", "position": { "x": 380, "y": 12800 }, "params": { "normalizedShape": 4096 }, "inputs": [ "n87" ], "outputs": [ "n89" ] }, { "id": "n89", "type": "groupedQueryAttention", "name": "llm_attn_10", "position": { "x": 380, "y": 12950 }, "params": { "embedDim": 4096, "numHeads": 32, "numKVHeads": 32 }, "inputs": [ "n88" ], "outputs": [ "n90" ] }, { "id": "n90", "type": "add", "name": "llm_res1_10", "position": { "x": 380, "y": 13100 }, "params": {}, "inputs": [ "n89", "n87" ], "outputs": [ "n91", "n93" ] }, { "id": "n91", "type": "rmsNorm", "name": "llm_ffn_norm_10", "position": { "x": 380, "y": 13250 }, "params": { "normalizedShape": 4096 }, "inputs": [ "n90" ], "outputs": [ "n92" ] }, { "id": "n92", "type": "swiglu", "name": "llm_ffn_10", "position": { "x": 380, "y": 13400 }, "params": { "embedDim": 4096, "intermediateSize": 11008 }, "inputs": [ "n91" ], "outputs": [ "n93" ] }, { "id": "n93", "type": "add", "name": "llm_res2_10", "position": { "x": 380, "y": 13550 }, "params": {}, "inputs": [ "n92", "n90" ], "outputs": [ "n94", "n96" ] }, { "id": "n94", "type": "rmsNorm", "name": "llm_attn_norm_11", "position": { "x": 380, "y": 13700 }, "params": { "normalizedShape": 4096 }, "inputs": [ "n93" ], "outputs": [ "n95" ] }, { "id": "n95", "type": "groupedQueryAttention", "name": "llm_attn_11", "position": { "x": 380, "y": 13850 }, "params": { "embedDim": 4096, "numHeads": 32, "numKVHeads": 32 }, "inputs": [ "n94" ], "outputs": [ "n96" ] }, { "id": "n96", "type": "add", "name": "llm_res1_11", "position": { "x": 380, "y": 14000 }, "params": {}, "inputs": [ "n95", "n93" ], "outputs": [ "n97", "n99" ] }, { "id": "n97", "type": "rmsNorm", "name": "llm_ffn_norm_11", "position": { "x": 380, "y": 14150 }, "params": { "normalizedShape": 4096 }, "inputs": [ "n96" ], "outputs": [ "n98" ] }, { "id": "n98", "type": "swiglu", "name": "llm_ffn_11", "position": { "x": 380, "y": 14300 }, "params": { "embedDim": 4096, "intermediateSize": 11008 }, "inputs": [ "n97" ], "outputs": [ "n99" ] }, { "id": "n99", "type": "add", "name": "llm_res2_11", "position": { "x": 380, "y": 14450 }, "params": {}, "inputs": [ "n98", "n96" ], "outputs": [ "n100", "n102" ] }, { "id": "n100", "type": "rmsNorm", "name": "llm_attn_norm_12", "position": { "x": 380, "y": 14600 }, "params": { "normalizedShape": 4096 }, "inputs": [ "n99" ], "outputs": [ "n101" ] }, { "id": "n101", "type": "groupedQueryAttention", "name": "llm_attn_12", "position": { "x": 380, "y": 14750 }, "params": { "embedDim": 4096, "numHeads": 32, "numKVHeads": 32 }, "inputs": [ "n100" ], "outputs": [ "n102" ] }, { "id": "n102", "type": "add", "name": "llm_res1_12", "position": { "x": 380, "y": 14900 }, "params": {}, "inputs": [ "n101", "n99" ], "outputs": [ "n103", "n105" ] }, { "id": "n103", "type": "rmsNorm", "name": "llm_ffn_norm_12", "position": { "x": 380, "y": 15050 }, "params": { "normalizedShape": 4096 }, "inputs": [ "n102" ], "outputs": [ "n104" ] }, { "id": "n104", "type": "swiglu", "name": "llm_ffn_12", "position": { "x": 380, "y": 15200 }, "params": { "embedDim": 4096, "intermediateSize": 11008 }, "inputs": [ "n103" ], "outputs": [ "n105" ] }, { "id": "n105", "type": "add", "name": "llm_res2_12", "position": { "x": 380, "y": 15350 }, "params": {}, "inputs": [ "n104", "n102" ], "outputs": [ "n106", "n108" ] }, { "id": "n106", "type": "rmsNorm", "name": "llm_attn_norm_13", "position": { "x": 380, "y": 15500 }, "params": { "normalizedShape": 4096 }, "inputs": [ "n105" ], "outputs": [ "n107" ] }, { "id": "n107", "type": "groupedQueryAttention", "name": "llm_attn_13", "position": { "x": 380, "y": 15650 }, "params": { "embedDim": 4096, "numHeads": 32, "numKVHeads": 32 }, "inputs": [ "n106" ], "outputs": [ "n108" ] }, { "id": "n108", "type": "add", "name": "llm_res1_13", "position": { "x": 380, "y": 15800 }, "params": {}, "inputs": [ "n107", "n105" ], "outputs": [ "n109", "n111" ] }, { "id": "n109", "type": "rmsNorm", "name": "llm_ffn_norm_13", "position": { "x": 380, "y": 15950 }, "params": { "normalizedShape": 4096 }, "inputs": [ "n108" ], "outputs": [ "n110" ] }, { "id": "n110", "type": "swiglu", "name": "llm_ffn_13", "position": { "x": 380, "y": 16100 }, "params": { "embedDim": 4096, "intermediateSize": 11008 }, "inputs": [ "n109" ], "outputs": [ "n111" ] }, { "id": "n111", "type": "add", "name": "llm_res2_13", "position": { "x": 380, "y": 16250 }, "params": {}, "inputs": [ "n110", "n108" ], "outputs": [ "n112", "n114" ] }, { "id": "n112", "type": "rmsNorm", "name": "llm_attn_norm_14", "position": { "x": 380, "y": 16400 }, "params": { "normalizedShape": 4096 }, "inputs": [ "n111" ], "outputs": [ "n113" ] }, { "id": "n113", "type": "groupedQueryAttention", "name": "llm_attn_14", "position": { "x": 380, "y": 16550 }, "params": { "embedDim": 4096, "numHeads": 32, "numKVHeads": 32 }, "inputs": [ "n112" ], "outputs": [ "n114" ] }, { "id": "n114", "type": "add", "name": "llm_res1_14", "position": { "x": 380, "y": 16700 }, "params": {}, "inputs": [ "n113", "n111" ], "outputs": [ "n115", "n117" ] }, { "id": "n115", "type": "rmsNorm", "name": "llm_ffn_norm_14", "position": { "x": 380, "y": 16850 }, "params": { "normalizedShape": 4096 }, "inputs": [ "n114" ], "outputs": [ "n116" ] }, { "id": "n116", "type": "swiglu", "name": "llm_ffn_14", "position": { "x": 380, "y": 17000 }, "params": { "embedDim": 4096, "intermediateSize": 11008 }, "inputs": [ "n115" ], "outputs": [ "n117" ] }, { "id": "n117", "type": "add", "name": "llm_res2_14", "position": { "x": 380, "y": 17150 }, "params": {}, "inputs": [ "n116", "n114" ], "outputs": [ "n118", "n120" ] }, { "id": "n118", "type": "rmsNorm", "name": "llm_attn_norm_15", "position": { "x": 380, "y": 17300 }, "params": { "normalizedShape": 4096 }, "inputs": [ "n117" ], "outputs": [ "n119" ] }, { "id": "n119", "type": "groupedQueryAttention", "name": "llm_attn_15", "position": { "x": 380, "y": 17450 }, "params": { "embedDim": 4096, "numHeads": 32, "numKVHeads": 32 }, "inputs": [ "n118" ], "outputs": [ "n120" ] }, { "id": "n120", "type": "add", "name": "llm_res1_15", "position": { "x": 380, "y": 17600 }, "params": {}, "inputs": [ "n119", "n117" ], "outputs": [ "n121", "n123" ] }, { "id": "n121", "type": "rmsNorm", "name": "llm_ffn_norm_15", "position": { "x": 380, "y": 17750 }, "params": { "normalizedShape": 4096 }, "inputs": [ "n120" ], "outputs": [ "n122" ] }, { "id": "n122", "type": "swiglu", "name": "llm_ffn_15", "position": { "x": 380, "y": 17900 }, "params": { "embedDim": 4096, "intermediateSize": 11008 }, "inputs": [ "n121" ], "outputs": [ "n123" ] }, { "id": "n123", "type": "add", "name": "llm_res2_15", "position": { "x": 380, "y": 18050 }, "params": {}, "inputs": [ "n122", "n120" ], "outputs": [ "n124", "n126" ] }, { "id": "n124", "type": "rmsNorm", "name": "llm_attn_norm_16", "position": { "x": 380, "y": 18200 }, "params": { "normalizedShape": 4096 }, "inputs": [ "n123" ], "outputs": [ "n125" ] }, { "id": "n125", "type": "groupedQueryAttention", "name": "llm_attn_16", "position": { "x": 380, "y": 18350 }, "params": { "embedDim": 4096, "numHeads": 32, "numKVHeads": 32 }, "inputs": [ "n124" ], "outputs": [ "n126" ] }, { "id": "n126", "type": "add", "name": "llm_res1_16", "position": { "x": 380, "y": 18500 }, "params": {}, "inputs": [ "n125", "n123" ], "outputs": [ "n127", "n129" ] }, { "id": "n127", "type": "rmsNorm", "name": "llm_ffn_norm_16", "position": { "x": 380, "y": 18650 }, "params": { "normalizedShape": 4096 }, "inputs": [ "n126" ], "outputs": [ "n128" ] }, { "id": "n128", "type": "swiglu", "name": "llm_ffn_16", "position": { "x": 380, "y": 18800 }, "params": { "embedDim": 4096, "intermediateSize": 11008 }, "inputs": [ "n127" ], "outputs": [ "n129" ] }, { "id": "n129", "type": "add", "name": "llm_res2_16", "position": { "x": 380, "y": 18950 }, "params": {}, "inputs": [ "n128", "n126" ], "outputs": [ "n130", "n132" ] }, { "id": "n130", "type": "rmsNorm", "name": "llm_attn_norm_17", "position": { "x": 380, "y": 19100 }, "params": { "normalizedShape": 4096 }, "inputs": [ "n129" ], "outputs": [ "n131" ] }, { "id": "n131", "type": "groupedQueryAttention", "name": "llm_attn_17", "position": { "x": 380, "y": 19250 }, "params": { "embedDim": 4096, "numHeads": 32, "numKVHeads": 32 }, "inputs": [ "n130" ], "outputs": [ "n132" ] }, { "id": "n132", "type": "add", "name": "llm_res1_17", "position": { "x": 380, "y": 19400 }, "params": {}, "inputs": [ "n131", "n129" ], "outputs": [ "n133", "n135" ] }, { "id": "n133", "type": "rmsNorm", "name": "llm_ffn_norm_17", "position": { "x": 380, "y": 19550 }, "params": { "normalizedShape": 4096 }, "inputs": [ "n132" ], "outputs": [ "n134" ] }, { "id": "n134", "type": "swiglu", "name": "llm_ffn_17", "position": { "x": 380, "y": 19700 }, "params": { "embedDim": 4096, "intermediateSize": 11008 }, "inputs": [ "n133" ], "outputs": [ "n135" ] }, { "id": "n135", "type": "add", "name": "llm_res2_17", "position": { "x": 380, "y": 19850 }, "params": {}, "inputs": [ "n134", "n132" ], "outputs": [ "n136", "n138" ] }, { "id": "n136", "type": "rmsNorm", "name": "llm_attn_norm_18", "position": { "x": 380, "y": 20000 }, "params": { "normalizedShape": 4096 }, "inputs": [ "n135" ], "outputs": [ "n137" ] }, { "id": "n137", "type": "groupedQueryAttention", "name": "llm_attn_18", "position": { "x": 380, "y": 20150 }, "params": { "embedDim": 4096, "numHeads": 32, "numKVHeads": 32 }, "inputs": [ "n136" ], "outputs": [ "n138" ] }, { "id": "n138", "type": "add", "name": "llm_res1_18", "position": { "x": 380, "y": 20300 }, "params": {}, "inputs": [ "n137", "n135" ], "outputs": [ "n139", "n141" ] }, { "id": "n139", "type": "rmsNorm", "name": "llm_ffn_norm_18", "position": { "x": 380, "y": 20450 }, "params": { "normalizedShape": 4096 }, "inputs": [ "n138" ], "outputs": [ "n140" ] }, { "id": "n140", "type": "swiglu", "name": "llm_ffn_18", "position": { "x": 380, "y": 20600 }, "params": { "embedDim": 4096, "intermediateSize": 11008 }, "inputs": [ "n139" ], "outputs": [ "n141" ] }, { "id": "n141", "type": "add", "name": "llm_res2_18", "position": { "x": 380, "y": 20750 }, "params": {}, "inputs": [ "n140", "n138" ], "outputs": [ "n142", "n144" ] }, { "id": "n142", "type": "rmsNorm", "name": "llm_attn_norm_19", "position": { "x": 380, "y": 20900 }, "params": { "normalizedShape": 4096 }, "inputs": [ "n141" ], "outputs": [ "n143" ] }, { "id": "n143", "type": "groupedQueryAttention", "name": "llm_attn_19", "position": { "x": 380, "y": 21050 }, "params": { "embedDim": 4096, "numHeads": 32, "numKVHeads": 32 }, "inputs": [ "n142" ], "outputs": [ "n144" ] }, { "id": "n144", "type": "add", "name": "llm_res1_19", "position": { "x": 380, "y": 21200 }, "params": {}, "inputs": [ "n143", "n141" ], "outputs": [ "n145", "n147" ] }, { "id": "n145", "type": "rmsNorm", "name": "llm_ffn_norm_19", "position": { "x": 380, "y": 21350 }, "params": { "normalizedShape": 4096 }, "inputs": [ "n144" ], "outputs": [ "n146" ] }, { "id": "n146", "type": "swiglu", "name": "llm_ffn_19", "position": { "x": 380, "y": 21500 }, "params": { "embedDim": 4096, "intermediateSize": 11008 }, "inputs": [ "n145" ], "outputs": [ "n147" ] }, { "id": "n147", "type": "add", "name": "llm_res2_19", "position": { "x": 380, "y": 21650 }, "params": {}, "inputs": [ "n146", "n144" ], "outputs": [ "n148", "n150" ] }, { "id": "n148", "type": "rmsNorm", "name": "llm_attn_norm_20", "position": { "x": 380, "y": 21800 }, "params": { "normalizedShape": 4096 }, "inputs": [ "n147" ], "outputs": [ "n149" ] }, { "id": "n149", "type": "groupedQueryAttention", "name": "llm_attn_20", "position": { "x": 380, "y": 21950 }, "params": { "embedDim": 4096, "numHeads": 32, "numKVHeads": 32 }, "inputs": [ "n148" ], "outputs": [ "n150" ] }, { "id": "n150", "type": "add", "name": "llm_res1_20", "position": { "x": 380, "y": 22100 }, "params": {}, "inputs": [ "n149", "n147" ], "outputs": [ "n151", "n153" ] }, { "id": "n151", "type": "rmsNorm", "name": "llm_ffn_norm_20", "position": { "x": 380, "y": 22250 }, "params": { "normalizedShape": 4096 }, "inputs": [ "n150" ], "outputs": [ "n152" ] }, { "id": "n152", "type": "swiglu", "name": "llm_ffn_20", "position": { "x": 380, "y": 22400 }, "params": { "embedDim": 4096, "intermediateSize": 11008 }, "inputs": [ "n151" ], "outputs": [ "n153" ] }, { "id": "n153", "type": "add", "name": "llm_res2_20", "position": { "x": 380, "y": 22550 }, "params": {}, "inputs": [ "n152", "n150" ], "outputs": [ "n154", "n156" ] }, { "id": "n154", "type": "rmsNorm", "name": "llm_attn_norm_21", "position": { "x": 380, "y": 22700 }, "params": { "normalizedShape": 4096 }, "inputs": [ "n153" ], "outputs": [ "n155" ] }, { "id": "n155", "type": "groupedQueryAttention", "name": "llm_attn_21", "position": { "x": 380, "y": 22850 }, "params": { "embedDim": 4096, "numHeads": 32, "numKVHeads": 32 }, "inputs": [ "n154" ], "outputs": [ "n156" ] }, { "id": "n156", "type": "add", "name": "llm_res1_21", "position": { "x": 380, "y": 23000 }, "params": {}, "inputs": [ "n155", "n153" ], "outputs": [ "n157", "n159" ] }, { "id": "n157", "type": "rmsNorm", "name": "llm_ffn_norm_21", "position": { "x": 380, "y": 23150 }, "params": { "normalizedShape": 4096 }, "inputs": [ "n156" ], "outputs": [ "n158" ] }, { "id": "n158", "type": "swiglu", "name": "llm_ffn_21", "position": { "x": 380, "y": 23300 }, "params": { "embedDim": 4096, "intermediateSize": 11008 }, "inputs": [ "n157" ], "outputs": [ "n159" ] }, { "id": "n159", "type": "add", "name": "llm_res2_21", "position": { "x": 380, "y": 23450 }, "params": {}, "inputs": [ "n158", "n156" ], "outputs": [ "n160", "n162" ] }, { "id": "n160", "type": "rmsNorm", "name": "llm_attn_norm_22", "position": { "x": 380, "y": 23600 }, "params": { "normalizedShape": 4096 }, "inputs": [ "n159" ], "outputs": [ "n161" ] }, { "id": "n161", "type": "groupedQueryAttention", "name": "llm_attn_22", "position": { "x": 380, "y": 23750 }, "params": { "embedDim": 4096, "numHeads": 32, "numKVHeads": 32 }, "inputs": [ "n160" ], "outputs": [ "n162" ] }, { "id": "n162", "type": "add", "name": "llm_res1_22", "position": { "x": 380, "y": 23900 }, "params": {}, "inputs": [ "n161", "n159" ], "outputs": [ "n163", "n165" ] }, { "id": "n163", "type": "rmsNorm", "name": "llm_ffn_norm_22", "position": { "x": 380, "y": 24050 }, "params": { "normalizedShape": 4096 }, "inputs": [ "n162" ], "outputs": [ "n164" ] }, { "id": "n164", "type": "swiglu", "name": "llm_ffn_22", "position": { "x": 380, "y": 24200 }, "params": { "embedDim": 4096, "intermediateSize": 11008 }, "inputs": [ "n163" ], "outputs": [ "n165" ] }, { "id": "n165", "type": "add", "name": "llm_res2_22", "position": { "x": 380, "y": 24350 }, "params": {}, "inputs": [ "n164", "n162" ], "outputs": [ "n166", "n168" ] }, { "id": "n166", "type": "rmsNorm", "name": "llm_attn_norm_23", "position": { "x": 380, "y": 24500 }, "params": { "normalizedShape": 4096 }, "inputs": [ "n165" ], "outputs": [ "n167" ] }, { "id": "n167", "type": "groupedQueryAttention", "name": "llm_attn_23", "position": { "x": 380, "y": 24650 }, "params": { "embedDim": 4096, "numHeads": 32, "numKVHeads": 32 }, "inputs": [ "n166" ], "outputs": [ "n168" ] }, { "id": "n168", "type": "add", "name": "llm_res1_23", "position": { "x": 380, "y": 24800 }, "params": {}, "inputs": [ "n167", "n165" ], "outputs": [ "n169", "n171" ] }, { "id": "n169", "type": "rmsNorm", "name": "llm_ffn_norm_23", "position": { "x": 380, "y": 24950 }, "params": { "normalizedShape": 4096 }, "inputs": [ "n168" ], "outputs": [ "n170" ] }, { "id": "n170", "type": "swiglu", "name": "llm_ffn_23", "position": { "x": 380, "y": 25100 }, "params": { "embedDim": 4096, "intermediateSize": 11008 }, "inputs": [ "n169" ], "outputs": [ "n171" ] }, { "id": "n171", "type": "add", "name": "llm_res2_23", "position": { "x": 380, "y": 25250 }, "params": {}, "inputs": [ "n170", "n168" ], "outputs": [ "n172", "n174" ] }, { "id": "n172", "type": "rmsNorm", "name": "llm_attn_norm_24", "position": { "x": 380, "y": 25400 }, "params": { "normalizedShape": 4096 }, "inputs": [ "n171" ], "outputs": [ "n173" ] }, { "id": "n173", "type": "groupedQueryAttention", "name": "llm_attn_24", "position": { "x": 380, "y": 25550 }, "params": { "embedDim": 4096, "numHeads": 32, "numKVHeads": 32 }, "inputs": [ "n172" ], "outputs": [ "n174" ] }, { "id": "n174", "type": "add", "name": "llm_res1_24", "position": { "x": 380, "y": 25700 }, "params": {}, "inputs": [ "n173", "n171" ], "outputs": [ "n175", "n177" ] }, { "id": "n175", "type": "rmsNorm", "name": "llm_ffn_norm_24", "position": { "x": 380, "y": 25850 }, "params": { "normalizedShape": 4096 }, "inputs": [ "n174" ], "outputs": [ "n176" ] }, { "id": "n176", "type": "swiglu", "name": "llm_ffn_24", "position": { "x": 380, "y": 26000 }, "params": { "embedDim": 4096, "intermediateSize": 11008 }, "inputs": [ "n175" ], "outputs": [ "n177" ] }, { "id": "n177", "type": "add", "name": "llm_res2_24", "position": { "x": 380, "y": 26150 }, "params": {}, "inputs": [ "n176", "n174" ], "outputs": [ "n178", "n180" ] }, { "id": "n178", "type": "rmsNorm", "name": "llm_attn_norm_25", "position": { "x": 380, "y": 26300 }, "params": { "normalizedShape": 4096 }, "inputs": [ "n177" ], "outputs": [ "n179" ] }, { "id": "n179", "type": "groupedQueryAttention", "name": "llm_attn_25", "position": { "x": 380, "y": 26450 }, "params": { "embedDim": 4096, "numHeads": 32, "numKVHeads": 32 }, "inputs": [ "n178" ], "outputs": [ "n180" ] }, { "id": "n180", "type": "add", "name": "llm_res1_25", "position": { "x": 380, "y": 26600 }, "params": {}, "inputs": [ "n179", "n177" ], "outputs": [ "n181", "n183" ] }, { "id": "n181", "type": "rmsNorm", "name": "llm_ffn_norm_25", "position": { "x": 380, "y": 26750 }, "params": { "normalizedShape": 4096 }, "inputs": [ "n180" ], "outputs": [ "n182" ] }, { "id": "n182", "type": "swiglu", "name": "llm_ffn_25", "position": { "x": 380, "y": 26900 }, "params": { "embedDim": 4096, "intermediateSize": 11008 }, "inputs": [ "n181" ], "outputs": [ "n183" ] }, { "id": "n183", "type": "add", "name": "llm_res2_25", "position": { "x": 380, "y": 27050 }, "params": {}, "inputs": [ "n182", "n180" ], "outputs": [ "n184", "n186" ] }, { "id": "n184", "type": "rmsNorm", "name": "llm_attn_norm_26", "position": { "x": 380, "y": 27200 }, "params": { "normalizedShape": 4096 }, "inputs": [ "n183" ], "outputs": [ "n185" ] }, { "id": "n185", "type": "groupedQueryAttention", "name": "llm_attn_26", "position": { "x": 380, "y": 27350 }, "params": { "embedDim": 4096, "numHeads": 32, "numKVHeads": 32 }, "inputs": [ "n184" ], "outputs": [ "n186" ] }, { "id": "n186", "type": "add", "name": "llm_res1_26", "position": { "x": 380, "y": 27500 }, "params": {}, "inputs": [ "n185", "n183" ], "outputs": [ "n187", "n189" ] }, { "id": "n187", "type": "rmsNorm", "name": "llm_ffn_norm_26", "position": { "x": 380, "y": 27650 }, "params": { "normalizedShape": 4096 }, "inputs": [ "n186" ], "outputs": [ "n188" ] }, { "id": "n188", "type": "swiglu", "name": "llm_ffn_26", "position": { "x": 380, "y": 27800 }, "params": { "embedDim": 4096, "intermediateSize": 11008 }, "inputs": [ "n187" ], "outputs": [ "n189" ] }, { "id": "n189", "type": "add", "name": "llm_res2_26", "position": { "x": 380, "y": 27950 }, "params": {}, "inputs": [ "n188", "n186" ], "outputs": [ "n190", "n192" ] }, { "id": "n190", "type": "rmsNorm", "name": "llm_attn_norm_27", "position": { "x": 380, "y": 28100 }, "params": { "normalizedShape": 4096 }, "inputs": [ "n189" ], "outputs": [ "n191" ] }, { "id": "n191", "type": "groupedQueryAttention", "name": "llm_attn_27", "position": { "x": 380, "y": 28250 }, "params": { "embedDim": 4096, "numHeads": 32, "numKVHeads": 32 }, "inputs": [ "n190" ], "outputs": [ "n192" ] }, { "id": "n192", "type": "add", "name": "llm_res1_27", "position": { "x": 380, "y": 28400 }, "params": {}, "inputs": [ "n191", "n189" ], "outputs": [ "n193", "n195" ] }, { "id": "n193", "type": "rmsNorm", "name": "llm_ffn_norm_27", "position": { "x": 380, "y": 28550 }, "params": { "normalizedShape": 4096 }, "inputs": [ "n192" ], "outputs": [ "n194" ] }, { "id": "n194", "type": "swiglu", "name": "llm_ffn_27", "position": { "x": 380, "y": 28700 }, "params": { "embedDim": 4096, "intermediateSize": 11008 }, "inputs": [ "n193" ], "outputs": [ "n195" ] }, { "id": "n195", "type": "add", "name": "llm_res2_27", "position": { "x": 380, "y": 28850 }, "params": {}, "inputs": [ "n194", "n192" ], "outputs": [ "n196", "n198" ] }, { "id": "n196", "type": "rmsNorm", "name": "llm_attn_norm_28", "position": { "x": 380, "y": 29000 }, "params": { "normalizedShape": 4096 }, "inputs": [ "n195" ], "outputs": [ "n197" ] }, { "id": "n197", "type": "groupedQueryAttention", "name": "llm_attn_28", "position": { "x": 380, "y": 29150 }, "params": { "embedDim": 4096, "numHeads": 32, "numKVHeads": 32 }, "inputs": [ "n196" ], "outputs": [ "n198" ] }, { "id": "n198", "type": "add", "name": "llm_res1_28", "position": { "x": 380, "y": 29300 }, "params": {}, "inputs": [ "n197", "n195" ], "outputs": [ "n199", "n201" ] }, { "id": "n199", "type": "rmsNorm", "name": "llm_ffn_norm_28", "position": { "x": 380, "y": 29450 }, "params": { "normalizedShape": 4096 }, "inputs": [ "n198" ], "outputs": [ "n200" ] }, { "id": "n200", "type": "swiglu", "name": "llm_ffn_28", "position": { "x": 380, "y": 29600 }, "params": { "embedDim": 4096, "intermediateSize": 11008 }, "inputs": [ "n199" ], "outputs": [ "n201" ] }, { "id": "n201", "type": "add", "name": "llm_res2_28", "position": { "x": 380, "y": 29750 }, "params": {}, "inputs": [ "n200", "n198" ], "outputs": [ "n202", "n204" ] }, { "id": "n202", "type": "rmsNorm", "name": "llm_attn_norm_29", "position": { "x": 380, "y": 29900 }, "params": { "normalizedShape": 4096 }, "inputs": [ "n201" ], "outputs": [ "n203" ] }, { "id": "n203", "type": "groupedQueryAttention", "name": "llm_attn_29", "position": { "x": 380, "y": 30050 }, "params": { "embedDim": 4096, "numHeads": 32, "numKVHeads": 32 }, "inputs": [ "n202" ], "outputs": [ "n204" ] }, { "id": "n204", "type": "add", "name": "llm_res1_29", "position": { "x": 380, "y": 30200 }, "params": {}, "inputs": [ "n203", "n201" ], "outputs": [ "n205", "n207" ] }, { "id": "n205", "type": "rmsNorm", "name": "llm_ffn_norm_29", "position": { "x": 380, "y": 30350 }, "params": { "normalizedShape": 4096 }, "inputs": [ "n204" ], "outputs": [ "n206" ] }, { "id": "n206", "type": "swiglu", "name": "llm_ffn_29", "position": { "x": 380, "y": 30500 }, "params": { "embedDim": 4096, "intermediateSize": 11008 }, "inputs": [ "n205" ], "outputs": [ "n207" ] }, { "id": "n207", "type": "add", "name": "llm_res2_29", "position": { "x": 380, "y": 30650 }, "params": {}, "inputs": [ "n206", "n204" ], "outputs": [ "n208", "n210" ] }, { "id": "n208", "type": "rmsNorm", "name": "llm_attn_norm_30", "position": { "x": 380, "y": 30800 }, "params": { "normalizedShape": 4096 }, "inputs": [ "n207" ], "outputs": [ "n209" ] }, { "id": "n209", "type": "groupedQueryAttention", "name": "llm_attn_30", "position": { "x": 380, "y": 30950 }, "params": { "embedDim": 4096, "numHeads": 32, "numKVHeads": 32 }, "inputs": [ "n208" ], "outputs": [ "n210" ] }, { "id": "n210", "type": "add", "name": "llm_res1_30", "position": { "x": 380, "y": 31100 }, "params": {}, "inputs": [ "n209", "n207" ], "outputs": [ "n211", "n213" ] }, { "id": "n211", "type": "rmsNorm", "name": "llm_ffn_norm_30", "position": { "x": 380, "y": 31250 }, "params": { "normalizedShape": 4096 }, "inputs": [ "n210" ], "outputs": [ "n212" ] }, { "id": "n212", "type": "swiglu", "name": "llm_ffn_30", "position": { "x": 380, "y": 31400 }, "params": { "embedDim": 4096, "intermediateSize": 11008 }, "inputs": [ "n211" ], "outputs": [ "n213" ] }, { "id": "n213", "type": "add", "name": "llm_res2_30", "position": { "x": 380, "y": 31550 }, "params": {}, "inputs": [ "n212", "n210" ], "outputs": [ "n214", "n216" ] }, { "id": "n214", "type": "rmsNorm", "name": "llm_attn_norm_31", "position": { "x": 380, "y": 31700 }, "params": { "normalizedShape": 4096 }, "inputs": [ "n213" ], "outputs": [ "n215" ] }, { "id": "n215", "type": "groupedQueryAttention", "name": "llm_attn_31", "position": { "x": 380, "y": 31850 }, "params": { "embedDim": 4096, "numHeads": 32, "numKVHeads": 32 }, "inputs": [ "n214" ], "outputs": [ "n216" ] }, { "id": "n216", "type": "add", "name": "llm_res1_31", "position": { "x": 380, "y": 32000 }, "params": {}, "inputs": [ "n215", "n213" ], "outputs": [ "n217", "n219" ] }, { "id": "n217", "type": "rmsNorm", "name": "llm_ffn_norm_31", "position": { "x": 380, "y": 32150 }, "params": { "normalizedShape": 4096 }, "inputs": [ "n216" ], "outputs": [ "n218" ] }, { "id": "n218", "type": "swiglu", "name": "llm_ffn_31", "position": { "x": 380, "y": 32300 }, "params": { "embedDim": 4096, "intermediateSize": 11008 }, "inputs": [ "n217" ], "outputs": [ "n219" ] }, { "id": "n219", "type": "add", "name": "llm_res2_31", "position": { "x": 380, "y": 32450 }, "params": {}, "inputs": [ "n218", "n216" ], "outputs": [ "n220", "n222" ] }, { "id": "n220", "type": "rmsNorm", "name": "llm_attn_norm_32", "position": { "x": 380, "y": 32600 }, "params": { "normalizedShape": 4096 }, "inputs": [ "n219" ], "outputs": [ "n221" ] }, { "id": "n221", "type": "groupedQueryAttention", "name": "llm_attn_32", "position": { "x": 380, "y": 32750 }, "params": { "embedDim": 4096, "numHeads": 32, "numKVHeads": 32 }, "inputs": [ "n220" ], "outputs": [ "n222" ] }, { "id": "n222", "type": "add", "name": "llm_res1_32", "position": { "x": 380, "y": 32900 }, "params": {}, "inputs": [ "n221", "n219" ], "outputs": [ "n223", "n225" ] }, { "id": "n223", "type": "rmsNorm", "name": "llm_ffn_norm_32", "position": { "x": 380, "y": 33050 }, "params": { "normalizedShape": 4096 }, "inputs": [ "n222" ], "outputs": [ "n224" ] }, { "id": "n224", "type": "swiglu", "name": "llm_ffn_32", "position": { "x": 380, "y": 33200 }, "params": { "embedDim": 4096, "intermediateSize": 11008 }, "inputs": [ "n223" ], "outputs": [ "n225" ] }, { "id": "n225", "type": "add", "name": "llm_res2_32", "position": { "x": 380, "y": 33350 }, "params": {}, "inputs": [ "n224", "n222" ], "outputs": [ "n226" ] }, { "id": "n226", "type": "rmsNorm", "name": "final_norm", "position": { "x": 380, "y": 33500 }, "params": { "normalizedShape": 4096 }, "inputs": [ "n225" ], "outputs": [ "n227" ] }, { "id": "n227", "type": "linear", "name": "lm_head", "position": { "x": 380, "y": 33650 }, "params": { "inFeatures": 4096, "outFeatures": 32000 }, "inputs": [ "n226" ], "outputs": [ "n228" ] }, { "id": "n228", "type": "output", "name": "logits", "position": { "x": 380, "y": 33800 }, "params": {}, "inputs": [ "n227" ], "outputs": [] } ], "connections": [ { "id": "c1", "from": "n1", "to": "n2" }, { "id": "c2", "from": "n2", "to": "n3" }, { "id": "c3", "from": "n3", "to": "n4" }, { "id": "c4", "from": "n4", "to": "n5" }, { "id": "c5", "from": "n5", "to": "n6" }, { "id": "c6", "from": "n6", "to": "n7" }, { "id": "c7", "from": "n7", "to": "n8" }, { "id": "c8", "from": "n8", "to": "n9" }, { "id": "c9", "from": "n9", "to": "n10" }, { "id": "c10", "from": "n10", "to": "n11" }, { "id": "c11", "from": "n11", "to": "n12" }, { "id": "c12", "from": "n12", "to": "n13" }, { "id": "c13", "from": "n13", "to": "n14" }, { "id": "c14", "from": "n14", "to": "n15" }, { "id": "c15", "from": "n15", "to": "n16" }, { "id": "c16", "from": "n16", "to": "n17" }, { "id": "c17", "from": "n17", "to": "n18" }, { "id": "c18", "from": "n18", "to": "n19" }, { "id": "c19", "from": "n19", "to": "n20" }, { "id": "c20", "from": "n20", "to": "n21" }, { "id": "c21", "from": "n21", "to": "n22" }, { "id": "c22", "from": "n22", "to": "n23" }, { "id": "c23", "from": "n23", "to": "n24" }, { "id": "c24", "from": "n24", "to": "n25" }, { "id": "c25", "from": "n25", "to": "n26" }, { "id": "c26", "from": "n26", "to": "n27" }, { "id": "c27", "from": "n27", "to": "n28" }, { "id": "c28", "from": "n28", "to": "n29" }, { "id": "c29", "from": "n29", "to": "n30" }, { "id": "c30", "from": "n31", "to": "n32" }, { "id": "c31", "from": "n30", "to": "n30b" }, { "id": "c31b", "from": "n30b", "to": "n33" }, { "id": "c32", "from": "n32", "to": "n33" }, { "id": "c33", "from": "n33", "to": "n34" }, { "id": "c34", "from": "n34", "to": "n35" }, { "id": "c35", "from": "n35", "to": "n36" }, { "id": "c36", "from": "n33", "to": "n36" }, { "id": "c37", "from": "n36", "to": "n37" }, { "id": "c38", "from": "n37", "to": "n38" }, { "id": "c39", "from": "n38", "to": "n39" }, { "id": "c40", "from": "n36", "to": "n39" }, { "id": "c41", "from": "n39", "to": "n40" }, { "id": "c42", "from": "n40", "to": "n41" }, { "id": "c43", "from": "n41", "to": "n42" }, { "id": "c44", "from": "n39", "to": "n42" }, { "id": "c45", "from": "n42", "to": "n43" }, { "id": "c46", "from": "n43", "to": "n44" }, { "id": "c47", "from": "n44", "to": "n45" }, { "id": "c48", "from": "n42", "to": "n45" }, { "id": "c49", "from": "n45", "to": "n46" }, { "id": "c50", "from": "n46", "to": "n47" }, { "id": "c51", "from": "n47", "to": "n48" }, { "id": "c52", "from": "n45", "to": "n48" }, { "id": "c53", "from": "n48", "to": "n49" }, { "id": "c54", "from": "n49", "to": "n50" }, { "id": "c55", "from": "n50", "to": "n51" }, { "id": "c56", "from": "n48", "to": "n51" }, { "id": "c57", "from": "n51", "to": "n52" }, { "id": "c58", "from": "n52", "to": "n53" }, { "id": "c59", "from": "n53", "to": "n54" }, { "id": "c60", "from": "n51", "to": "n54" }, { "id": "c61", "from": "n54", "to": "n55" }, { "id": "c62", "from": "n55", "to": "n56" }, { "id": "c63", "from": "n56", "to": "n57" }, { "id": "c64", "from": "n54", "to": "n57" }, { "id": "c65", "from": "n57", "to": "n58" }, { "id": "c66", "from": "n58", "to": "n59" }, { "id": "c67", "from": "n59", "to": "n60" }, { "id": "c68", "from": "n57", "to": "n60" }, { "id": "c69", "from": "n60", "to": "n61" }, { "id": "c70", "from": "n61", "to": "n62" }, { "id": "c71", "from": "n62", "to": "n63" }, { "id": "c72", "from": "n60", "to": "n63" }, { "id": "c73", "from": "n63", "to": "n64" }, { "id": "c74", "from": "n64", "to": "n65" }, { "id": "c75", "from": "n65", "to": "n66" }, { "id": "c76", "from": "n63", "to": "n66" }, { "id": "c77", "from": "n66", "to": "n67" }, { "id": "c78", "from": "n67", "to": "n68" }, { "id": "c79", "from": "n68", "to": "n69" }, { "id": "c80", "from": "n66", "to": "n69" }, { "id": "c81", "from": "n69", "to": "n70" }, { "id": "c82", "from": "n70", "to": "n71" }, { "id": "c83", "from": "n71", "to": "n72" }, { "id": "c84", "from": "n69", "to": "n72" }, { "id": "c85", "from": "n72", "to": "n73" }, { "id": "c86", "from": "n73", "to": "n74" }, { "id": "c87", "from": "n74", "to": "n75" }, { "id": "c88", "from": "n72", "to": "n75" }, { "id": "c89", "from": "n75", "to": "n76" }, { "id": "c90", "from": "n76", "to": "n77" }, { "id": "c91", "from": "n77", "to": "n78" }, { "id": "c92", "from": "n75", "to": "n78" }, { "id": "c93", "from": "n78", "to": "n79" }, { "id": "c94", "from": "n79", "to": "n80" }, { "id": "c95", "from": "n80", "to": "n81" }, { "id": "c96", "from": "n78", "to": "n81" }, { "id": "c97", "from": "n81", "to": "n82" }, { "id": "c98", "from": "n82", "to": "n83" }, { "id": "c99", "from": "n83", "to": "n84" }, { "id": "c100", "from": "n81", "to": "n84" }, { "id": "c101", "from": "n84", "to": "n85" }, { "id": "c102", "from": "n85", "to": "n86" }, { "id": "c103", "from": "n86", "to": "n87" }, { "id": "c104", "from": "n84", "to": "n87" }, { "id": "c105", "from": "n87", "to": "n88" }, { "id": "c106", "from": "n88", "to": "n89" }, { "id": "c107", "from": "n89", "to": "n90" }, { "id": "c108", "from": "n87", "to": "n90" }, { "id": "c109", "from": "n90", "to": "n91" }, { "id": "c110", "from": "n91", "to": "n92" }, { "id": "c111", "from": "n92", "to": "n93" }, { "id": "c112", "from": "n90", "to": "n93" }, { "id": "c113", "from": "n93", "to": "n94" }, { "id": "c114", "from": "n94", "to": "n95" }, { "id": "c115", "from": "n95", "to": "n96" }, { "id": "c116", "from": "n93", "to": "n96" }, { "id": "c117", "from": "n96", "to": "n97" }, { "id": "c118", "from": "n97", "to": "n98" }, { "id": "c119", "from": "n98", "to": "n99" }, { "id": "c120", "from": "n96", "to": "n99" }, { "id": "c121", "from": "n99", "to": "n100" }, { "id": "c122", "from": "n100", "to": "n101" }, { "id": "c123", "from": "n101", "to": "n102" }, { "id": "c124", "from": "n99", "to": "n102" }, { "id": "c125", "from": "n102", "to": "n103" }, { "id": "c126", "from": "n103", "to": "n104" }, { "id": "c127", "from": "n104", "to": "n105" }, { "id": "c128", "from": "n102", "to": "n105" }, { "id": "c129", "from": "n105", "to": "n106" }, { "id": "c130", "from": "n106", "to": "n107" }, { "id": "c131", "from": "n107", "to": "n108" }, { "id": "c132", "from": "n105", "to": "n108" }, { "id": "c133", "from": "n108", "to": "n109" }, { "id": "c134", "from": "n109", "to": "n110" }, { "id": "c135", "from": "n110", "to": "n111" }, { "id": "c136", "from": "n108", "to": "n111" }, { "id": "c137", "from": "n111", "to": "n112" }, { "id": "c138", "from": "n112", "to": "n113" }, { "id": "c139", "from": "n113", "to": "n114" }, { "id": "c140", "from": "n111", "to": "n114" }, { "id": "c141", "from": "n114", "to": "n115" }, { "id": "c142", "from": "n115", "to": "n116" }, { "id": "c143", "from": "n116", "to": "n117" }, { "id": "c144", "from": "n114", "to": "n117" }, { "id": "c145", "from": "n117", "to": "n118" }, { "id": "c146", "from": "n118", "to": "n119" }, { "id": "c147", "from": "n119", "to": "n120" }, { "id": "c148", "from": "n117", "to": "n120" }, { "id": "c149", "from": "n120", "to": "n121" }, { "id": "c150", "from": "n121", "to": "n122" }, { "id": "c151", "from": "n122", "to": "n123" }, { "id": "c152", "from": "n120", "to": "n123" }, { "id": "c153", "from": "n123", "to": "n124" }, { "id": "c154", "from": "n124", "to": "n125" }, { "id": "c155", "from": "n125", "to": "n126" }, { "id": "c156", "from": "n123", "to": "n126" }, { "id": "c157", "from": "n126", "to": "n127" }, { "id": "c158", "from": "n127", "to": "n128" }, { "id": "c159", "from": "n128", "to": "n129" }, { "id": "c160", "from": "n126", "to": "n129" }, { "id": "c161", "from": "n129", "to": "n130" }, { "id": "c162", "from": "n130", "to": "n131" }, { "id": "c163", "from": "n131", "to": "n132" }, { "id": "c164", "from": "n129", "to": "n132" }, { "id": "c165", "from": "n132", "to": "n133" }, { "id": "c166", "from": "n133", "to": "n134" }, { "id": "c167", "from": "n134", "to": "n135" }, { "id": "c168", "from": "n132", "to": "n135" }, { "id": "c169", "from": "n135", "to": "n136" }, { "id": "c170", "from": "n136", "to": "n137" }, { "id": "c171", "from": "n137", "to": "n138" }, { "id": "c172", "from": "n135", "to": "n138" }, { "id": "c173", "from": "n138", "to": "n139" }, { "id": "c174", "from": "n139", "to": "n140" }, { "id": "c175", "from": "n140", "to": "n141" }, { "id": "c176", "from": "n138", "to": "n141" }, { "id": "c177", "from": "n141", "to": "n142" }, { "id": "c178", "from": "n142", "to": "n143" }, { "id": "c179", "from": "n143", "to": "n144" }, { "id": "c180", "from": "n141", "to": "n144" }, { "id": "c181", "from": "n144", "to": "n145" }, { "id": "c182", "from": "n145", "to": "n146" }, { "id": "c183", "from": "n146", "to": "n147" }, { "id": "c184", "from": "n144", "to": "n147" }, { "id": "c185", "from": "n147", "to": "n148" }, { "id": "c186", "from": "n148", "to": "n149" }, { "id": "c187", "from": "n149", "to": "n150" }, { "id": "c188", "from": "n147", "to": "n150" }, { "id": "c189", "from": "n150", "to": "n151" }, { "id": "c190", "from": "n151", "to": "n152" }, { "id": "c191", "from": "n152", "to": "n153" }, { "id": "c192", "from": "n150", "to": "n153" }, { "id": "c193", "from": "n153", "to": "n154" }, { "id": "c194", "from": "n154", "to": "n155" }, { "id": "c195", "from": "n155", "to": "n156" }, { "id": "c196", "from": "n153", "to": "n156" }, { "id": "c197", "from": "n156", "to": "n157" }, { "id": "c198", "from": "n157", "to": "n158" }, { "id": "c199", "from": "n158", "to": "n159" }, { "id": "c200", "from": "n156", "to": "n159" }, { "id": "c201", "from": "n159", "to": "n160" }, { "id": "c202", "from": "n160", "to": "n161" }, { "id": "c203", "from": "n161", "to": "n162" }, { "id": "c204", "from": "n159", "to": "n162" }, { "id": "c205", "from": "n162", "to": "n163" }, { "id": "c206", "from": "n163", "to": "n164" }, { "id": "c207", "from": "n164", "to": "n165" }, { "id": "c208", "from": "n162", "to": "n165" }, { "id": "c209", "from": "n165", "to": "n166" }, { "id": "c210", "from": "n166", "to": "n167" }, { "id": "c211", "from": "n167", "to": "n168" }, { "id": "c212", "from": "n165", "to": "n168" }, { "id": "c213", "from": "n168", "to": "n169" }, { "id": "c214", "from": "n169", "to": "n170" }, { "id": "c215", "from": "n170", "to": "n171" }, { "id": "c216", "from": "n168", "to": "n171" }, { "id": "c217", "from": "n171", "to": "n172" }, { "id": "c218", "from": "n172", "to": "n173" }, { "id": "c219", "from": "n173", "to": "n174" }, { "id": "c220", "from": "n171", "to": "n174" }, { "id": "c221", "from": "n174", "to": "n175" }, { "id": "c222", "from": "n175", "to": "n176" }, { "id": "c223", "from": "n176", "to": "n177" }, { "id": "c224", "from": "n174", "to": "n177" }, { "id": "c225", "from": "n177", "to": "n178" }, { "id": "c226", "from": "n178", "to": "n179" }, { "id": "c227", "from": "n179", "to": "n180" }, { "id": "c228", "from": "n177", "to": "n180" }, { "id": "c229", "from": "n180", "to": "n181" }, { "id": "c230", "from": "n181", "to": "n182" }, { "id": "c231", "from": "n182", "to": "n183" }, { "id": "c232", "from": "n180", "to": "n183" }, { "id": "c233", "from": "n183", "to": "n184" }, { "id": "c234", "from": "n184", "to": "n185" }, { "id": "c235", "from": "n185", "to": "n186" }, { "id": "c236", "from": "n183", "to": "n186" }, { "id": "c237", "from": "n186", "to": "n187" }, { "id": "c238", "from": "n187", "to": "n188" }, { "id": "c239", "from": "n188", "to": "n189" }, { "id": "c240", "from": "n186", "to": "n189" }, { "id": "c241", "from": "n189", "to": "n190" }, { "id": "c242", "from": "n190", "to": "n191" }, { "id": "c243", "from": "n191", "to": "n192" }, { "id": "c244", "from": "n189", "to": "n192" }, { "id": "c245", "from": "n192", "to": "n193" }, { "id": "c246", "from": "n193", "to": "n194" }, { "id": "c247", "from": "n194", "to": "n195" }, { "id": "c248", "from": "n192", "to": "n195" }, { "id": "c249", "from": "n195", "to": "n196" }, { "id": "c250", "from": "n196", "to": "n197" }, { "id": "c251", "from": "n197", "to": "n198" }, { "id": "c252", "from": "n195", "to": "n198" }, { "id": "c253", "from": "n198", "to": "n199" }, { "id": "c254", "from": "n199", "to": "n200" }, { "id": "c255", "from": "n200", "to": "n201" }, { "id": "c256", "from": "n198", "to": "n201" }, { "id": "c257", "from": "n201", "to": "n202" }, { "id": "c258", "from": "n202", "to": "n203" }, { "id": "c259", "from": "n203", "to": "n204" }, { "id": "c260", "from": "n201", "to": "n204" }, { "id": "c261", "from": "n204", "to": "n205" }, { "id": "c262", "from": "n205", "to": "n206" }, { "id": "c263", "from": "n206", "to": "n207" }, { "id": "c264", "from": "n204", "to": "n207" }, { "id": "c265", "from": "n207", "to": "n208" }, { "id": "c266", "from": "n208", "to": "n209" }, { "id": "c267", "from": "n209", "to": "n210" }, { "id": "c268", "from": "n207", "to": "n210" }, { "id": "c269", "from": "n210", "to": "n211" }, { "id": "c270", "from": "n211", "to": "n212" }, { "id": "c271", "from": "n212", "to": "n213" }, { "id": "c272", "from": "n210", "to": "n213" }, { "id": "c273", "from": "n213", "to": "n214" }, { "id": "c274", "from": "n214", "to": "n215" }, { "id": "c275", "from": "n215", "to": "n216" }, { "id": "c276", "from": "n213", "to": "n216" }, { "id": "c277", "from": "n216", "to": "n217" }, { "id": "c278", "from": "n217", "to": "n218" }, { "id": "c279", "from": "n218", "to": "n219" }, { "id": "c280", "from": "n216", "to": "n219" }, { "id": "c281", "from": "n219", "to": "n220" }, { "id": "c282", "from": "n220", "to": "n221" }, { "id": "c283", "from": "n221", "to": "n222" }, { "id": "c284", "from": "n219", "to": "n222" }, { "id": "c285", "from": "n222", "to": "n223" }, { "id": "c286", "from": "n223", "to": "n224" }, { "id": "c287", "from": "n224", "to": "n225" }, { "id": "c288", "from": "n222", "to": "n225" }, { "id": "c289", "from": "n225", "to": "n226" }, { "id": "c290", "from": "n226", "to": "n227" }, { "id": "c291", "from": "n227", "to": "n228" } ] }