Skip to content

Commit c4c6180

Browse files
danielsobradodanielsobrado
committed
feat: add gpt2 comprehensive animation (phase 1 - steps 1-3)
1 parent 935d23d commit c4c6180

File tree

13 files changed

+845
-0
lines changed

13 files changed

+845
-0
lines changed

‎gpt2-comprehensive-animation/README.md‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
# GPT-2 Comprehensive Animation
2+
3+
An interactive, multi-step educational journey through GPT-2 architecture and training optimizations.
4+
5+
## Overview
6+
7+
This project provides a deep dive into GPT-2, covering:
8+
9+
### Implemented Steps (Phase 1)
10+
1. **Tokenization & Embeddings**: BPE tokenization, learned embeddings
11+
2. **Positional Encoding**: Learned absolute position embeddings
12+
3. **Multi-Head Self-Attention**: Q/K/V projections, causal masking, multi-head mechanism
13+
14+
### Coming Soon (Phase 2)
15+
4. Feed-Forward Network
16+
5. Layer Normalization & Residual Connections
17+
6. Full Architecture Overview
18+
7. Weight Tying Optimization
19+
8. Training Optimizations (gradient accumulation, mixed precision, etc.)
20+
9. Inference Optimizations (KV cache, sampling strategies)
21+
22+
## Features
23+
24+
- **Interactive Visualizations**: Three.js for 3D attention patterns, Canvas for positional encoding
25+
- **Hands-on Exercises**: Quiz questions to validate understanding after each step
26+
- **Progress Tracking**: Navigate through steps, mark completion
27+
- **Dark Theme**: Easy on the eyes during long study sessions
28+
29+
## Getting Started
30+
31+
```bash
32+
# Install dependencies
33+
npm install
34+
35+
# Run development server
36+
npm run dev
37+
```
38+
39+
## Learning Objectives
40+
41+
After completing all steps, you will understand:
42+
- How text is converted to numerical representations
43+
- The self-attention mechanism that powers transformers
44+
- Why GPT-2 uses causal masking for autoregressive generation
45+
- Training and inference optimizations that make GPT-2 practical
46+
47+
## Project Structure
48+
49+
```
50+
src/
51+
├── App.jsx # Main app with navigation
52+
├── stepsConfig.js # Step definitions
53+
└── steps/
54+
├── Step1Tokenization.jsx
55+
├── Step2Positional.jsx
56+
└── Step3Attention.jsx
57+
```
58+
59+
## Extending This Project
60+
61+
To add Step 4 and beyond:
62+
1. Create new component in `src/steps/`
63+
2. Add route in `App.jsx`
64+
3. Update `stepsConfig.js`
65+
66+
## License
67+
68+
MIT

‎gpt2-comprehensive-animation/index.html‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
<!doctype html>
2+
<html lang="en">
3+
4+
<head>
5+
<meta charset="UTF-8" />
6+
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
7+
<title>GPT-2 Comprehensive Animation</title>
8+
</head>
9+
10+
<body>
11+
<div id="root"></div>
12+
<script type="module" src="/src/main.jsx"></script>
13+
</body>
14+
15+
</html>

‎gpt2-comprehensive-animation/package.json‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
{
2+
"name": "gpt2-comprehensive-animation",
3+
"version": "1.0.0",
4+
"type": "module",
5+
"scripts": {
6+
"dev": "vite",
7+
"build": "vite build",
8+
"preview": "vite preview"
9+
},
10+
"dependencies": {
11+
"gsap": "^3.12.2",
12+
"react": "^18.2.0",
13+
"react-dom": "^18.2.0",
14+
"react-router-dom": "^6.16.0",
15+
"three": "^0.158.0",
16+
"prismjs": "^1.29.0"
17+
},
18+
"devDependencies": {
19+
"@types/react": "^18.2.15",
20+
"@types/react-dom": "^18.2.7",
21+
"@vitejs/plugin-react": "^4.0.3",
22+
"autoprefixer": "^10.4.16",
23+
"postcss": "^8.4.31",
24+
"tailwindcss": "^3.3.5",
25+
"vite": "^5.0.0"
26+
}
27+
}

‎gpt2-comprehensive-animation/postcss.config.js‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
export default {
2+
plugins: {
3+
tailwindcss: {},
4+
autoprefixer: {},
5+
},
6+
}

‎gpt2-comprehensive-animation/src/App.jsx‎

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
import React, { useState } from 'react';
2+
import { Routes, Route, useNavigate, useLocation } from 'react-router-dom';
3+
import { steps } from './stepsConfig';
4+
import Step1Tokenization from './steps/Step1Tokenization';
5+
import Step2Positional from './steps/Step2Positional';
6+
import Step3Attention from './steps/Step3Attention';
7+
8+
export default function App() {
9+
const [completedSteps, setCompletedSteps] = useState(new Set());
10+
const navigate = useNavigate();
11+
const location = useLocation();
12+
13+
const currentStepId = steps.find(s => s.path === location.pathname)?.id || 1;
14+
15+
const markComplete = (stepId) => {
16+
setCompletedSteps(prev => new Set([...prev, stepId]));
17+
};
18+
19+
const goToNext = () => {
20+
const currentIndex = steps.findIndex(s => s.id === currentStepId);
21+
if (currentIndex < steps.length - 1) {
22+
navigate(steps[currentIndex + 1].path);
23+
}
24+
};
25+
26+
const goToPrev = () => {
27+
const currentIndex = steps.findIndex(s => s.id === currentStepId);
28+
if (currentIndex > 0) {
29+
navigate(steps[currentIndex - 1].path);
30+
}
31+
};
32+
33+
return (
34+
<div className="min-h-screen bg-gray-900 text-gray-100">
35+
{/* Header */}
36+
<header className="bg-gray-800 border-b border-gray-700 px-6 py-4">
37+
<div className="max-w-7xl mx-auto flex items-center justify-between">
38+
<h1 className="text-2xl font-bold text-emerald-400">GPT-2 Deep Dive</h1>
39+
<div className="text-sm text-gray-400">
40+
Step {currentStepId} of {steps.length}
41+
</div>
42+
</div>
43+
</header>
44+
45+
{/* Main Content */}
46+
<div className="flex">
47+
{/* Sidebar */}
48+
<aside className="w-64 bg-gray-800 min-h-[calc(100vh-73px)] border-r border-gray-700 p-4">
49+
<nav>
50+
<ul className="space-y-2">
51+
{steps.map((step) => (
52+
<li key={step.id}>
53+
<button
54+
onClick={() => navigate(step.path)}
55+
className={`w-full text-left px-4 py-3 rounded-lg transition-colors ${currentStepId === step.id
56+
? 'bg-emerald-600 text-white'
57+
: completedSteps.has(step.id)
58+
? 'bg-gray-700 text-emerald-400'
59+
: 'bg-gray-700 text-gray-300 hover:bg-gray-600'
60+
}`}
61+
>
62+
<div className="flex items-center gap-2">
63+
<span className="font-mono text-xs">{step.id}</span>
64+
<div className="flex-1">
65+
<div className="font-semibold text-sm">{step.title}</div>
66+
<div className="text-xs opacity-75">{step.description}</div>
67+
</div>
68+
{completedSteps.has(step.id) && (
69+
<span className="text-emerald-400">✓</span>
70+
)}
71+
</div>
72+
</button>
73+
</li>
74+
))}
75+
</ul>
76+
</nav>
77+
</aside>
78+
79+
{/* Content */}
80+
<main className="flex-1 p-8">
81+
<div className="max-w-4xl mx-auto">
82+
<Routes>
83+
<Route path="/" element={<Step1Tokenization onComplete={() => markComplete(1)} onNext={goToNext} />} />
84+
<Route path="/step1" element={<Step1Tokenization onComplete={() => markComplete(1)} onNext={goToNext} />} />
85+
<Route path="/step2" element={<Step2Positional onComplete={() => markComplete(2)} onNext={goToNext} onPrev={goToPrev} />} />
86+
<Route path="/step3" element={<Step3Attention onComplete={() => markComplete(3)} onNext={goToNext} onPrev={goToPrev} />} />
87+
</Routes>
88+
</div>
89+
</main>
90+
</div>
91+
</div>
92+
);
93+
}

‎gpt2-comprehensive-animation/src/index.css‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
@tailwind base;
2+
@tailwind components;
3+
@tailwind utilities;
4+
5+
@import 'prismjs/themes/prism-tomorrow.css';
6+
7+
body {
8+
margin: 0;
9+
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', 'Roboto', 'Oxygen',
10+
'Ubuntu', 'Cantarell', 'Fira Sans', 'Droid Sans', 'Helvetica Neue',
11+
sans-serif;
12+
-webkit-font-smoothing: antialiased;
13+
-moz-osx-font-smoothing: grayscale;
14+
}
15+
16+
code {
17+
font-family: source-code-pro, Menlo, Monaco, Consolas, 'Courier New',
18+
monospace;
19+
}

‎gpt2-comprehensive-animation/src/main.jsx‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
import React from 'react'
2+
import ReactDOM from 'react-dom/client'
3+
import { BrowserRouter } from 'react-router-dom'
4+
import App from './App.jsx'
5+
import './index.css'
6+
7+
ReactDOM.createRoot(document.getElementById('root')).render(
8+
<React.StrictMode>
9+
<BrowserRouter>
10+
<App />
11+
</BrowserRouter>
12+
</React.StrictMode>,
13+
)

‎gpt2-comprehensive-animation/src/steps/Step1Tokenization.jsx‎

Lines changed: 132 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,132 @@
1+
import React, { useState } from 'react';
2+
3+
export default function Step1Tokenization({ onComplete, onNext }) {
4+
const [inputText, setInputText] = useState("Hello, GPT-2!");
5+
const [tokens, setTokens] = useState([]);
6+
const [quizAnswer, setQuizAnswer] = useState('');
7+
const [quizFeedback, setQuizFeedback] = useState('');
8+
9+
// Simplified BPE tokenization simulation
10+
const tokenize = (text) => {
11+
// Simple word-level tokenization for demonstration
12+
const simpleTokens = text.split(/(\s+|[,.!?])/g).filter(t => t.trim());
13+
setTokens(simpleTokens);
14+
};
15+
16+
const checkQuiz = () => {
17+
const correct = quizAnswer.toLowerCase().includes('subword');
18+
setQuizFeedback(correct
19+
? '✓ Correct! BPE breaks text into subword units, allowing the model to handle unknown words.'
20+
: '✗ Try again. Think about how BPE handles rare or unknown words.'
21+
);
22+
if (correct) onComplete();
23+
};
24+
25+
return (
26+
<div className="space-y-8">
27+
<div>
28+
<h2 className="text-3xl font-bold mb-2">Step 1: Tokenization & Embeddings</h2>
29+
<p className="text-gray-400">How text becomes numbers that GPT-2 can understand</p>
30+
</div>
31+
32+
{/* Explanation */}
33+
<div className="bg-gray-800 rounded-lg p-6 space-y-4">
34+
<h3 className="text-xl font-semibold text-emerald-400">What is Tokenization?</h3>
35+
<p className="text-gray-300">
36+
GPT-2 can't process raw text - it needs numbers. <strong>Tokenization</strong> converts text into a sequence of tokens (subword units).
37+
</p>
38+
<p className="text-gray-300">
39+
GPT-2 uses <strong>Byte-Pair Encoding (BPE)</strong> with a vocabulary of 50,257 tokens. This allows it to:
40+
</p>
41+
<ul className="list-disc list-inside space-y-1 text-gray-300 ml-4">
42+
<li>Handle any text (including rare words)</li>
43+
<li>Break unknown words into known subwords</li>
44+
<li>Keep common words as single tokens</li>
45+
</ul>
46+
</div>
47+
48+
{/* Interactive Demo */}
49+
<div className="bg-gray-800 rounded-lg p-6 space-y-4">
50+
<h3 className="text-xl font-semibold text-emerald-400">Try it Yourself</h3>
51+
<div>
52+
<label className="block text-sm text-gray-400 mb-2">Enter text:</label>
53+
<input
54+
type="text"
55+
value={inputText}
56+
onChange={(e) => setInputText(e.target.value)}
57+
className="w-full bg-gray-700 text-white px-4 py-2 rounded border border-gray-600 focus:border-emerald-500 focus:outline-none"
58+
placeholder="Type anything..."
59+
/>
60+
</div>
61+
<button
62+
onClick={() => tokenize(inputText)}
63+
className="px-6 py-2 bg-emerald-600 hover:bg-emerald-700 rounded font-semibold transition-colors"
64+
>
65+
Tokenize
66+
</button>
67+
68+
{tokens.length > 0 && (
69+
<div className="mt-4">
70+
<div className="text-sm text-gray-400 mb-2">Tokens ({tokens.length}):</div>
71+
<div className="flex flex-wrap gap-2">
72+
{tokens.map((token, i) => (
73+
<div key={i} className="bg-emerald-900 text-emerald-100 px-3 py-1 rounded text-sm font-mono">
74+
{token}
75+
</div>
76+
))}
77+
</div>
78+
</div>
79+
)}
80+
</div>
81+
82+
{/* Embedding Explanation */}
83+
<div className="bg-gray-800 rounded-lg p-6 space-y-4">
84+
<h3 className="text-xl font-semibold text-emerald-400">Token Embeddings</h3>
85+
<p className="text-gray-300">
86+
Each token is converted to a <strong>learned embedding vector</strong> of size 768 (for GPT-2 Small).
87+
</p>
88+
<div className="bg-gray-900 p-4 rounded font-mono text-sm text-gray-300">
89+
Token "Hello" → Token ID: 15496 → Embedding: [0.23, -0.45, 0.12, ..., 0.67] (768 dimensions)
90+
</div>
91+
<p className="text-gray-300">
92+
These embeddings are <strong>learned during training</strong> so that similar tokens have similar vectors.
93+
</p>
94+
</div>
95+
96+
{/* Exercise */}
97+
<div className="bg-blue-900 bg-opacity-30 border border-blue-700 rounded-lg p-6 space-y-4">
98+
<h3 className="text-xl font-semibold text-blue-400">📝 Exercise</h3>
99+
<p className="text-gray-300">
100+
Why does GPT-2 use Byte-Pair Encoding instead of word-level tokenization?
101+
</p>
102+
<textarea
103+
value={quizAnswer}
104+
onChange={(e) => setQuizAnswer(e.target.value)}
105+
className="w-full bg-gray-700 text-white px-4 py-2 rounded border border-gray-600 focus:border-blue-500 focus:outline-none h-24"
106+
placeholder="Your answer..."
107+
/>
108+
<button
109+
onClick={checkQuiz}
110+
className="px-6 py-2 bg-blue-600 hover:bg-blue-700 rounded font-semibold transition-colors"
111+
>
112+
Check Answer
113+
</button>
114+
{quizFeedback && (
115+
<div className={`p-3 rounded ${quizFeedback.startsWith('✓') ? 'bg-green-900 text-green-200' : 'bg-red-900 text-red-200'}`}>
116+
{quizFeedback}
117+
</div>
118+
)}
119+
</div>
120+
121+
{/* Navigation */}
122+
<div className="flex justify-end">
123+
<button
124+
onClick={onNext}
125+
className="px-6 py-3 bg-emerald-600 hover:bg-emerald-700 rounded font-semibold transition-colors"
126+
>
127+
Next: Positional Encoding →
128+
</button>
129+
</div>
130+
</div>
131+
);
132+
}

0 commit comments

Comments
 (0)