Skip to content

Commit 5fa229d

Browse files
vilsonrodriguesvilsonrodrigues
authored
perf: Optimize settings loading with bulk JSON decoding (3.8x faster)
1 parent 34a59a9 commit 5fa229d

File tree

12 files changed

+1247
-241
lines changed

12 files changed

+1247
-241
lines changed

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,3 +23,4 @@ _build/
2323
/worktrees/
2424
/.ruff_cache/
2525
.DS_Store
26+
.benchmarks/

‎README.md‎

Lines changed: 13 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -138,22 +138,28 @@ class AppSettings(BaseSettings):
138138

139139
## Performance
140140

141-
msgspec-ext leverages msgspec's high-performance serialization for fast settings loading with full type validation.
141+
msgspec-ext leverages msgspec's high-performance serialization with bulk JSON decoding for maximum speed.
142142

143-
**Benchmark Results** (1000 iterations, Python 3.13):
143+
**Benchmark Results** (1000 iterations, Python 3.12):
144144

145145
| Library | Time per load | Relative Performance |
146146
|---------|---------------|---------------------|
147-
| msgspec-ext | 0.933ms | Baseline |
148-
| pydantic-settings | 2.694ms | 2.9x slower |
147+
| msgspec-ext | 0.702ms | Baseline ⚡ |
148+
| pydantic-settings | 2.694ms | 3.8x slower |
149149

150-
msgspec-ext is **2.9x faster** than pydantic-settings while providing the same level of type safety and validation.
150+
msgspec-ext is **3.8x faster** than pydantic-settings while providing the same level of type safety and validation.
151+
152+
**Key optimizations:**
153+
- Bulk JSON decoding in C (via msgspec)
154+
- Cached encoders and decoders
155+
- Automatic field ordering
156+
- Zero Python loops for validation
151157

152158
*Benchmark measures complete settings initialization including .env file parsing and type validation. Run `python benchmark.py` to reproduce.*
153159

154160
## Why msgspec-ext?
155161

156-
- **Performance** - 2.9x faster than pydantic-settings
162+
- **Performance** - 3.8x faster than pydantic-settings
157163
- **Lightweight** - 4x smaller package size (0.49 MB vs 1.95 MB)
158164
- **Type safety** - Full type validation with modern Python type checkers
159165
- **Minimal dependencies** - Only msgspec and python-dotenv
@@ -164,7 +170,7 @@ msgspec-ext is **2.9x faster** than pydantic-settings while providing the same l
164170
|---------|------------|-------------------|
165171
| .env support | ✅ | ✅ |
166172
| Type validation | ✅ | ✅ |
167-
| Performance | 2.9x faster | Baseline |
173+
| Performance | **3.8x faster** ⚡ | Baseline |
168174
| Package size | 0.49 MB | 1.95 MB |
169175
| Nested config | ✅ | ✅ |
170176
| Field aliases | ✅ | ✅ |

‎benchmark.py‎

Lines changed: 1 addition & 63 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,8 @@
11
#!/usr/bin/env python3
2-
"""Benchmark comparing msgspec-ext, pydantic-settings, and dynaconf."""
2+
"""Benchmark comparing msgspec-ext and pydantic-settings."""
33

44
import os
5-
import tempfile
65
import time
7-
from pathlib import Path
86

97

108
# Test with msgspec-ext
@@ -110,49 +108,6 @@ class Config:
110108
os.unlink(".env.benchmark")
111109

112110

113-
def benchmark_dynaconf(iterations: int = 1000) -> float:
114-
"""Benchmark dynaconf settings loading."""
115-
from dynaconf import Dynaconf
116-
117-
# Create settings files
118-
with tempfile.TemporaryDirectory() as tmpdir:
119-
settings_file = Path(tmpdir) / "settings.toml"
120-
settings_file.write_text("""
121-
[default]
122-
app_name = "benchmark-app"
123-
debug = true
124-
api_key = "test-api-key-12345"
125-
max_connections = 200
126-
timeout = 60.0
127-
allowed_hosts = ["localhost", "127.0.0.1"]
128-
129-
[default.database]
130-
host = "db.example.com"
131-
port = 5433
132-
username = "dbuser"
133-
password = "dbpass123"
134-
database = "production"
135-
""")
136-
137-
# Warm up
138-
for _ in range(10):
139-
settings = Dynaconf(
140-
settings_files=[str(settings_file)],
141-
environments=True,
142-
)
143-
144-
# Actual benchmark
145-
start = time.perf_counter()
146-
for _ in range(iterations):
147-
settings = Dynaconf(
148-
settings_files=[str(settings_file)],
149-
environments=True,
150-
)
151-
end = time.perf_counter()
152-
153-
return (end - start) / iterations * 1000 # ms per iteration
154-
155-
156111
def main():
157112
"""Run benchmarks and display results."""
158113
print("=" * 70)
@@ -179,14 +134,6 @@ def main():
179134
print(f"ERROR: {e}")
180135
pydantic_time = None
181136

182-
try:
183-
print("⏱ dynaconf...", end=" ", flush=True)
184-
dynaconf_time = benchmark_dynaconf()
185-
print(f"{dynaconf_time:.3f}ms")
186-
except Exception as e:
187-
print(f"ERROR: {e}")
188-
dynaconf_time = None
189-
190137
print()
191138
print("=" * 70)
192139
print("Results Summary")
@@ -197,19 +144,13 @@ def main():
197144
print(f"msgspec-ext: {msgspec_time:.3f}ms per load")
198145
if pydantic_time:
199146
print(f"pydantic-settings: {pydantic_time:.3f}ms per load")
200-
if dynaconf_time:
201-
print(f"dynaconf: {dynaconf_time:.3f}ms per load")
202147

203148
print()
204149

205150
if msgspec_time and pydantic_time:
206151
speedup = pydantic_time / msgspec_time
207152
print(f"msgspec-ext is {speedup:.1f}x faster than pydantic-settings")
208153

209-
if msgspec_time and dynaconf_time:
210-
speedup = dynaconf_time / msgspec_time
211-
print(f"msgspec-ext is {speedup:.1f}x faster than dynaconf")
212-
213154
print()
214155
print("=" * 70)
215156

@@ -224,9 +165,6 @@ def main():
224165
if pydantic_time:
225166
rel = pydantic_time / msgspec_time if msgspec_time else 1.0
226167
print(f"| pydantic-settings | {pydantic_time:.3f}ms | {rel:.1f}x slower |")
227-
if dynaconf_time:
228-
rel = dynaconf_time / msgspec_time if msgspec_time else 1.0
229-
print(f"| dynaconf | {dynaconf_time:.3f}ms | {rel:.1f}x slower |")
230168
print()
231169

232170

‎examples/01_basic_usage.py‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
"""
2+
Basic usage of msgspec-ext for settings management.
3+
4+
This example shows the simplest use case: defining settings with defaults
5+
and loading from environment variables.
6+
"""
7+
8+
import os
9+
10+
from msgspec_ext import BaseSettings
11+
12+
13+
class AppSettings(BaseSettings):
14+
"""Application settings with sensible defaults."""
15+
16+
app_name: str = "my-app"
17+
debug: bool = False
18+
port: int = 8000
19+
host: str = "0.0.0.0"
20+
21+
22+
def main():
23+
print("=" * 60)
24+
print("Example 1: Basic Usage")
25+
print("=" * 60)
26+
print()
27+
28+
# Create settings with defaults
29+
print("1. Using defaults:")
30+
settings = AppSettings()
31+
print(f" App Name: {settings.app_name}")
32+
print(f" Debug: {settings.debug}")
33+
print(f" Port: {settings.port}")
34+
print(f" Host: {settings.host}")
35+
print()
36+
37+
# Override with environment variables
38+
print("2. Override with environment variables:")
39+
os.environ["APP_NAME"] = "production-app"
40+
os.environ["PORT"] = "9000"
41+
os.environ["DEBUG"] = "true"
42+
43+
settings2 = AppSettings()
44+
print(f" App Name: {settings2.app_name}")
45+
print(f" Debug: {settings2.debug}")
46+
print(f" Port: {settings2.port}")
47+
print(f" Host: {settings2.host}")
48+
print()
49+
50+
# Clean up
51+
os.environ.pop("APP_NAME", None)
52+
os.environ.pop("PORT", None)
53+
os.environ.pop("DEBUG", None)
54+
55+
# Override with explicit values
56+
print("3. Override with explicit values:")
57+
settings3 = AppSettings(app_name="test-app", port=3000, debug=True)
58+
print(f" App Name: {settings3.app_name}")
59+
print(f" Debug: {settings3.debug}")
60+
print(f" Port: {settings3.port}")
61+
print()
62+
63+
print("✅ Basic usage complete!")
64+
65+
66+
if __name__ == "__main__":
67+
main()

‎examples/02_env_prefix.py‎

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
"""
2+
Using env_prefix to namespace environment variables.
3+
4+
This example shows how to use env_prefix to avoid naming conflicts
5+
when multiple applications share the same environment.
6+
"""
7+
8+
import os
9+
10+
from msgspec_ext import BaseSettings, SettingsConfigDict
11+
12+
13+
class DatabaseSettings(BaseSettings):
14+
"""Database settings with DB_ prefix."""
15+
16+
model_config = SettingsConfigDict(env_prefix="DB_")
17+
18+
host: str = "localhost"
19+
port: int = 5432
20+
username: str = "admin"
21+
password: str = "secret"
22+
database: str = "myapp"
23+
24+
25+
class RedisSettings(BaseSettings):
26+
"""Redis settings with REDIS_ prefix."""
27+
28+
model_config = SettingsConfigDict(env_prefix="REDIS_")
29+
30+
host: str = "localhost"
31+
port: int = 6379
32+
database: int = 0
33+
34+
35+
def main():
36+
print("=" * 60)
37+
print("Example 2: Environment Variable Prefixes")
38+
print("=" * 60)
39+
print()
40+
41+
# Set environment variables with different prefixes
42+
os.environ["DB_HOST"] = "db.example.com"
43+
os.environ["DB_PORT"] = "5433"
44+
os.environ["DB_USERNAME"] = "dbuser"
45+
os.environ["DB_PASSWORD"] = "dbpass123"
46+
os.environ["DB_DATABASE"] = "production"
47+
48+
os.environ["REDIS_HOST"] = "redis.example.com"
49+
os.environ["REDIS_PORT"] = "6380"
50+
os.environ["REDIS_DATABASE"] = "1"
51+
52+
try:
53+
# Load database settings
54+
db_settings = DatabaseSettings()
55+
print("Database Settings (DB_ prefix):")
56+
print(f" Host: {db_settings.host}")
57+
print(f" Port: {db_settings.port}")
58+
print(f" Username: {db_settings.username}")
59+
print(f" Password: {db_settings.password}")
60+
print(f" Database: {db_settings.database}")
61+
print()
62+
63+
# Load Redis settings
64+
redis_settings = RedisSettings()
65+
print("Redis Settings (REDIS_ prefix):")
66+
print(f" Host: {redis_settings.host}")
67+
print(f" Port: {redis_settings.port}")
68+
print(f" Database: {redis_settings.database}")
69+
print()
70+
71+
print("✅ Environment prefixes working correctly!")
72+
print()
73+
print("💡 Tip: Use prefixes to organize settings for different")
74+
print(" services in microservice architectures.")
75+
76+
finally:
77+
# Clean up
78+
for key in [
79+
"DB_HOST",
80+
"DB_PORT",
81+
"DB_USERNAME",
82+
"DB_PASSWORD",
83+
"DB_DATABASE",
84+
"REDIS_HOST",
85+
"REDIS_PORT",
86+
"REDIS_DATABASE",
87+
]:
88+
os.environ.pop(key, None)
89+
90+
91+
if __name__ == "__main__":
92+
main()

‎examples/03_dotenv_file.py‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
"""
2+
Loading settings from .env files.
3+
4+
This example shows how to load settings from .env files,
5+
which is useful for local development and deployment.
6+
"""
7+
8+
import tempfile
9+
from pathlib import Path
10+
11+
from msgspec_ext import BaseSettings, SettingsConfigDict
12+
13+
14+
class AppSettings(BaseSettings):
15+
"""Application settings loaded from .env file."""
16+
17+
model_config = SettingsConfigDict(
18+
env_file=".env.example", env_file_encoding="utf-8"
19+
)
20+
21+
app_name: str
22+
environment: str = "development"
23+
api_key: str
24+
database_url: str
25+
max_connections: int = 100
26+
enable_logging: bool = True
27+
28+
29+
def main():
30+
print("=" * 60)
31+
print("Example 3: Loading from .env Files")
32+
print("=" * 60)
33+
print()
34+
35+
# Create a temporary .env file
36+
env_content = """# Application Configuration
37+
APP_NAME=my-awesome-app
38+
ENVIRONMENT=production
39+
API_KEY=sk-1234567890abcdef
40+
DATABASE_URL=postgresql://user:pass@localhost:5432/mydb
41+
MAX_CONNECTIONS=200
42+
ENABLE_LOGGING=false
43+
"""
44+
45+
env_file = Path(".env.example")
46+
env_file.write_text(env_content)
47+
48+
try:
49+
print("Created .env.example file:")
50+
print("-" * 60)
51+
print(env_content)
52+
print("-" * 60)
53+
print()
54+
55+
# Load settings from .env file
56+
settings = AppSettings()
57+
58+
print("Loaded Settings:")
59+
print(f" App Name: {settings.app_name}")
60+
print(f" Environment: {settings.environment}")
61+
print(f" API Key: {settings.api_key}")
62+
print(f" Database URL: {settings.database_url}")
63+
print(f" Max Connections: {settings.max_connections}")
64+
print(f" Enable Logging: {settings.enable_logging}")
65+
print()
66+
67+
print("✅ Settings loaded from .env file!")
68+
print()
69+
print("💡 Tips:")
70+
print(" - Use .env.local for local overrides (add to .gitignore)")
71+
print(" - Use .env.production for production settings")
72+
print(" - Never commit secrets to version control")
73+
74+
finally:
75+
# Clean up
76+
env_file.unlink(missing_ok=True)
77+
78+
79+
if __name__ == "__main__":
80+
main()

0 commit comments

Comments
 (0)