Skip to content

Commit a70929d

Browse files
dbsanftedbsanfte
committed
dev container and testing notes
1 parent 92593e7 commit a70929d

File tree

7 files changed

+754
-0
lines changed

7 files changed

+754
-0
lines changed

‎.devcontainer/Dockerfile‎

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
FROM ubuntu:24.04
2+
3+
# Avoid prompts from apt
4+
ENV DEBIAN_FRONTEND=noninteractive
5+
6+
# Copy in a zscaler.crt if one exists
7+
# This allows the container to access the internet on corporate laptops
8+
COPY zscaler.cr[t] /usr/local/share/ca-certificates/
9+
10+
# This tells various tools to use the system CA certificates
11+
ENV REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt
12+
ENV SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt
13+
ENV NODE_OPTIONS=--use-openssl-ca
14+
15+
# Update and install system dependencies
16+
RUN apt-get update && \
17+
apt-get install -y \
18+
build-essential \
19+
ca-certificates \
20+
cmake \
21+
git \
22+
curl \
23+
wget \
24+
pkg-config \
25+
python3 \
26+
python3-pip \
27+
python3-venv \
28+
libcurl4-openssl-dev \
29+
libnuma-dev \
30+
numactl \
31+
hwloc-nox \
32+
libhwloc-dev \
33+
ccache \
34+
ninja-build \
35+
gdb \
36+
valgrind \
37+
gh && \
38+
update-ca-certificates && \
39+
mkdir -p --mode=0755 /etc/apt/keyrings && \
40+
wget https://repo.radeon.com/rocm/rocm.gpg.key -O - | \
41+
gpg --dearmor | tee /etc/apt/keyrings/rocm.gpg > /dev/null && \
42+
echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/rocm.gpg] https://repo.radeon.com/rocm/apt/6.4.2 noble main" \
43+
| tee /etc/apt/sources.list.d/rocm.list && \
44+
echo 'Package: *' \
45+
| tee /etc/apt/preferences.d/rocm-pin-600 && \
46+
echo 'Pin: release o=repo.radeon.com' \
47+
| tee -a /etc/apt/preferences.d/rocm-pin-600 && \
48+
echo 'Pin-Priority: 600' \
49+
| tee -a /etc/apt/preferences.d/rocm-pin-600 && \
50+
apt-get update && \
51+
apt-get install -y rocm && \
52+
apt-get autoremove -y && \
53+
apt-get clean
54+
55+
# Install Python dependencies for gguf conversion tools
56+
RUN python3 -m pip install --break-system-packages \
57+
numpy \
58+
torch \
59+
transformers \
60+
sentencepiece \
61+
protobuf \
62+
gguf
63+
64+
# Set up ccache for faster compilation
65+
ENV PATH="/usr/lib/ccache:${PATH}"
66+
ENV CCACHE_DIR="/tmp/ccache"
67+
RUN mkdir -p /tmp/ccache
68+
69+
# Create a non-root user
70+
RUN useradd -m -s /bin/bash developer && \
71+
usermod -aG sudo developer && \
72+
echo "developer ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers
73+
74+
# Set working directory
75+
WORKDIR /workspace
76+
77+
# Switch to non-root user
78+
USER developer
79+
80+
# Set up shell environment
81+
RUN echo 'export PS1="\[\033[01;32m\]\u@\h\[\033[00m\]:\[\033[01;34m\]\w\[\033[00m\]\$ "' >> ~/.bashrc && \
82+
echo 'alias ll="ls -alF"' >> ~/.bashrc && \
83+
echo 'alias la="ls -A"' >> ~/.bashrc && \
84+
echo 'alias l="ls -CF"' >> ~/.bashrc
85+
86+
# Expose common ports
87+
EXPOSE 8080 8081
88+
89+
CMD ["/bin/bash"]

‎.devcontainer/README.md‎

Lines changed: 135 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,135 @@
1+
# llama.cpp Development Container
2+
3+
This dev container provides a complete Ubuntu 24.04 environment for building and testing llama.cpp with NUMA support.
4+
5+
## Features
6+
7+
- **Ubuntu 24.04 LTS** base image
8+
- **Complete build toolchain**: gcc, cmake, ninja, ccache
9+
- **NUMA support**: libnuma-dev, numactl, hwloc for CPU topology detection
10+
- **Python environment**: with all necessary packages for GGUF conversion tools
11+
- **VS Code integration**: with C/C++, CMake, and Python extensions
12+
- **Development tools**: gdb, valgrind for debugging
13+
14+
## Quick Start
15+
16+
1. **Open in VS Code**: Make sure you have the "Dev Containers" extension installed, then:
17+
- Open the llama.cpp folder in VS Code
18+
- Press `Ctrl+Shift+P` (or `Cmd+Shift+P` on Mac)
19+
- Type "Dev Containers: Reopen in Container"
20+
- Select it and wait for the container to build and start
21+
22+
2. **Build the project**:
23+
```bash
24+
cmake -B build -DCMAKE_BUILD_TYPE=Release
25+
cmake --build build --parallel
26+
```
27+
28+
3. **Test NUMA functionality**:
29+
```bash
30+
# Check NUMA topology
31+
numactl --hardware
32+
33+
# Test CPU topology detection
34+
./build/bin/llama-server --cpu-topology
35+
36+
# Run with specific NUMA settings
37+
numactl --cpunodebind=0 --membind=0 ./build/bin/llama-server --model path/to/model.gguf
38+
```
39+
40+
## Available Tools
41+
42+
### System Tools
43+
- `numactl`: NUMA policy control
44+
- `hwloc-info`: Hardware locality information
45+
- `lscpu`: CPU information
46+
- `ccache`: Compiler cache for faster rebuilds
47+
48+
### Build Configurations
49+
50+
#### Debug Build (default post-create)
51+
```bash
52+
cmake -B build -DCMAKE_BUILD_TYPE=Debug
53+
cmake --build build --parallel
54+
```
55+
56+
#### Release Build (optimized)
57+
```bash
58+
cmake -B build -DCMAKE_BUILD_TYPE=Release
59+
cmake --build build --parallel
60+
```
61+
62+
#### With Additional Options
63+
```bash
64+
# Enable OpenBLAS
65+
cmake -B build -DGGML_BLAS=ON -DGGML_BLAS_VENDOR=OpenBLAS
66+
67+
# Static build
68+
cmake -B build -DBUILD_SHARED_LIBS=OFF
69+
70+
# Disable CURL if not needed
71+
cmake -B build -DLLAMA_CURL=OFF
72+
```
73+
74+
## Testing NUMA Improvements
75+
76+
The container includes tools to test the NUMA improvements:
77+
78+
### CPU Topology Detection
79+
```bash
80+
# View detailed CPU information
81+
./build/bin/llama-server --cpu-topology
82+
83+
# Check current NUMA configuration
84+
numactl --show
85+
86+
# Display NUMA hardware topology
87+
numactl --hardware
88+
```
89+
90+
### Performance Testing
91+
```bash
92+
# Test with default settings (hyperthreading enabled)
93+
./build/bin/llama-bench -m model.gguf
94+
95+
# Test without hyperthreading
96+
./build/bin/llama-bench -m model.gguf --no-hyperthreading
97+
98+
# Test with specific thread count
99+
./build/bin/llama-bench -m model.gguf --threads 8
100+
101+
# Test with NUMA binding
102+
numactl --cpunodebind=0 --membind=0 ./build/bin/llama-bench -m model.gguf
103+
```
104+
105+
### Environment Variables
106+
```bash
107+
# Disable hyperthreading via environment
108+
LLAMA_NO_HYPERTHREADING=1 ./build/bin/llama-server --model model.gguf
109+
110+
# Enable efficiency cores
111+
LLAMA_USE_EFFICIENCY_CORES=1 ./build/bin/llama-server --model model.gguf
112+
```
113+
114+
## Development Workflow
115+
116+
1. **Code changes**: Edit files in VS Code with full IntelliSense support
117+
2. **Build**: Use `Ctrl+Shift+P` → "CMake: Build" or terminal commands
118+
3. **Debug**: Set breakpoints and use the integrated debugger
119+
4. **Test**: Run executables directly or through the testing framework
120+
121+
## Troubleshooting
122+
123+
### Container Build Issues
124+
- Ensure Docker Desktop is running
125+
- Try rebuilding: `Ctrl+Shift+P` → "Dev Containers: Rebuild Container"
126+
127+
### NUMA Issues
128+
- Check if running on a NUMA system: `numactl --hardware`
129+
- Verify CPU topology detection: `lscpu` and `hwloc-info`
130+
- Test CPU affinity: `taskset -c 0-3 ./your-program`
131+
132+
### Build Issues
133+
- Clear build cache: `rm -rf build && cmake -B build`
134+
- Check ccache stats: `ccache -s`
135+
- Use verbose build: `cmake --build build --verbose`

‎.devcontainer/devcontainer.json‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
{
2+
"name": "llama.cpp Development",
3+
"dockerFile": "Dockerfile",
4+
"customizations": {
5+
"vscode": {
6+
"extensions": [
7+
"ms-vscode.cpptools-extension-pack",
8+
"ms-vscode.cmake-tools",
9+
"ms-python.python",
10+
"ms-python.black-formatter",
11+
"github.copilot",
12+
"github.copilot-chat"
13+
],
14+
"settings": {
15+
"cmake.configureOnOpen": true,
16+
"cmake.buildDirectory": "${workspaceFolder}/build",
17+
"C_Cpp.default.configurationProvider": "ms-vscode.cmake-tools",
18+
"C_Cpp.default.cStandard": "c11",
19+
"C_Cpp.default.cppStandard": "c++14"
20+
}
21+
}
22+
},
23+
"mounts": [
24+
"source=/var/run/docker.sock,target=/var/run/docker.sock,type=bind"
25+
],
26+
"postCreateCommand": "cmake -B build -DCMAKE_BUILD_TYPE=Debug",
27+
"forwardPorts": [8080],
28+
"runArgs": [
29+
"--privileged",
30+
"--cap-add=SYS_ADMIN"
31+
],
32+
"features": {
33+
"ghcr.io/devcontainers/features/git:1": {},
34+
"ghcr.io/devcontainers/features/github-cli:1": {}
35+
}
36+
}

‎.devcontainer/launch.json‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
{
2+
"version": "0.2.0",
3+
"configurations": [
4+
{
5+
"name": "Debug llama-server",
6+
"type": "cppdbg",
7+
"request": "launch",
8+
"program": "${workspaceFolder}/build/bin/llama-server",
9+
"args": [
10+
"--model", "/path/to/your/model.gguf",
11+
"--host", "0.0.0.0",
12+
"--port", "8080",
13+
"--cpu-topology"
14+
],
15+
"stopAtEntry": false,
16+
"cwd": "${workspaceFolder}",
17+
"environment": [],
18+
"externalConsole": false,
19+
"MIMode": "gdb",
20+
"setupCommands": [
21+
{
22+
"description": "Enable pretty-printing for gdb",
23+
"text": "-enable-pretty-printing",
24+
"ignoreFailures": true
25+
},
26+
{
27+
"description": "Set Disassembly Flavor to Intel",
28+
"text": "set disassembly-flavor intel",
29+
"ignoreFailures": true
30+
}
31+
],
32+
"preLaunchTask": "cmake-build",
33+
"miDebuggerPath": "/usr/bin/gdb"
34+
},
35+
{
36+
"name": "Debug llama-cli",
37+
"type": "cppdbg",
38+
"request": "launch",
39+
"program": "${workspaceFolder}/build/bin/llama-cli",
40+
"args": [
41+
"--model", "/path/to/your/model.gguf",
42+
"--prompt", "Hello, world!",
43+
"--no-hyperthreading"
44+
],
45+
"stopAtEntry": false,
46+
"cwd": "${workspaceFolder}",
47+
"environment": [],
48+
"externalConsole": false,
49+
"MIMode": "gdb",
50+
"setupCommands": [
51+
{
52+
"description": "Enable pretty-printing for gdb",
53+
"text": "-enable-pretty-printing",
54+
"ignoreFailures": true
55+
}
56+
],
57+
"preLaunchTask": "cmake-build",
58+
"miDebuggerPath": "/usr/bin/gdb"
59+
},
60+
{
61+
"name": "Test CPU Topology",
62+
"type": "cppdbg",
63+
"request": "launch",
64+
"program": "${workspaceFolder}/build/bin/llama-server",
65+
"args": [
66+
"--cpu-topology"
67+
],
68+
"stopAtEntry": false,
69+
"cwd": "${workspaceFolder}",
70+
"environment": [],
71+
"externalConsole": false,
72+
"MIMode": "gdb",
73+
"preLaunchTask": "cmake-build",
74+
"miDebuggerPath": "/usr/bin/gdb"
75+
}
76+
]
77+
}

0 commit comments

Comments
 (0)