quiz.json

{
  "lesson": "59-vit-transformer",
  "title": "Vision Transformer Encoder",
  "questions": [
    {
      "stage": "pre",
      "question": "Why does the standard ViT block use pre-LayerNorm placement instead of post-LayerNorm?",
      "options": [
        "Pre-LN runs faster",
        "Post-LN uses more memory",
        "Pre-LN trains stably at depth without learning-rate warm-up while post-LN diverges past about 6 layers",
        "Post-LN is patented"
      ],
      "correct": 2,
      "explanation": "Pre-LN normalizes each sub-layer's input which keeps gradient norms bounded as depth grows."
    },
    {
      "stage": "pre",
      "question": "At hidden=768 with heads=12, what is the per-head dimension and why does it matter?",
      "options": [
        "12, so each head can run on a small GPU",
        "96, set by ResNet-50",
        "768, because all heads share the projection",
        "64, which controls the resolution of each attention map and lets twelve heads attend independently in parallel"
      ],
      "correct": 3,
      "explanation": "head_dim = hidden / heads = 64. Each head projects the token to its own 64-D q/k/v and computes its own attention map."
    },
    {
      "stage": "check",
      "question": "Does the ViT encoder need a causal attention mask?",
      "options": [
        "Only during training",
        "No: the encoder is bidirectional, every token may attend to every other token in the same sequence",
        "Yes, like a GPT decoder",
        "Only for the CLS row"
      ],
      "correct": 1,
      "explanation": "Vision encoders are bidirectional. Causal masking appears later when a text decoder consumes the cross-attention output."
    },
    {
      "stage": "check",
      "question": "What does the CLS token's final hidden state represent after all 12 blocks?",
      "options": [
        "A learned vector summary of the entire image that downstream heads project into class logits or contrastive embeddings",
        "The last patch only",
        "Random noise",
        "A copy of the input embedding"
      ],
      "correct": 0,
      "explanation": "The CLS token starts empty and accumulates information from every patch through self-attention across all 12 blocks."
    },
    {
      "stage": "check",
      "question": "Why is the feed-forward layer expanded to 4x the hidden dim?",
      "options": [
        "It halves FLOPs",
        "PyTorch requires 4x",
        "An empirical sweet spot since the original Transformer; smaller underfits, larger overfits at fixed data budget",
        "It removes the need for attention"
      ],
      "correct": 2,
      "explanation": "The 4x factor has held across language and vision transformers since 2017. Capacity per parameter peaks here."
    },
    {
      "stage": "post",
      "question": "Which family extension keeps the block math unchanged but adds prepended learned tokens?",
      "options": [
        "Causal masking",
        "RoPE rotation",
        "Register tokens (DINOv2): a few learned vectors prepended after CLS that capture global statistics and smooth attention maps",
        "Mixture-of-experts"
      ],
      "correct": 2,
      "explanation": "Register tokens stabilize attention without altering the block's math; they are extra slots in the sequence."
    }
  ]
}