Skip to content

Commit d012efb

Browse files
ElectroiscodingElectroiscoding
committed
final
1 parent d6c901a commit d012efb

File tree

3 files changed

+23
-276
lines changed

3 files changed

+23
-276
lines changed

‎demo.py‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
1-
# Crayon v5.1.0 Demo
21

32
from crayon import CrayonVocab
43

‎pyproject.toml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,10 +4,10 @@ build-backend = "setuptools.build_meta"
44

55
[project]
66
name = "xerv-crayon"
7-
version = "5.2.0"
7+
version = "5.2.4"
88
description = "Omni-Backend Tokenizer - CPU (AVX2/512), CUDA (NVIDIA), ROCm (AMD) with automatic hardware detection"
99
readme = "README.md"
10-
requires-python = ">=3.8"
10+
requires-python = ">=3.8,<3.13"
1111
license = {file = "LICENSE"}
1212
authors = [
1313
{name = "Xerv Research Engineering Division", email = "engineering@xerv.ai"}

‎setup.py‎

Lines changed: 21 additions & 273 deletions
Original file line numberDiff line numberDiff line change
@@ -1,311 +1,59 @@
11
"""
2-
XERV CRAYON SETUP v5.2.0 - Production Omni-Backend Build System
3-
================================================================
4-
GUARANTEED CUDA SUPPORT - PyTorch 2.10+ Compatible
2+
XERV CRAYON SETUP v5.2.3 - PRODUCTION BUILD
3+
============================================
4+
With CPU extensions for guaranteed performance
55
"""
66

77
import os
88
import sys
9-
import subprocess
10-
import shutil
119
from setuptools import setup, Extension, find_packages
12-
from setuptools.command.build_ext import build_ext
13-
from distutils.sysconfig import get_python_inc
1410

15-
VERSION = "5.2.0"
11+
VERSION = "5.2.4"
1612

17-
def log(msg: str, level: str = "INFO") -> None:
13+
def log(msg: str) -> None:
1814
print(f"[CRAYON-BUILD] {msg}", flush=True)
1915

20-
# ============================================================================
21-
# CUDA DETECTION AND COMPILATION - GUARANTEED TO WORK
22-
# ============================================================================
23-
24-
FORCE_CPU = os.environ.get("CRAYON_FORCE_CPU", "0") == "1"
25-
FORCE_CUDA = os.environ.get("CRAYON_FORCE_CUDA", "0") == "1"
26-
os.environ["MAX_JOBS"] = os.environ.get("MAX_JOBS", "1")
27-
28-
# Detect PyTorch & CUDA - ROBUST DETECTION
29-
try:
30-
import torch
31-
log(f"PyTorch v{torch.__version__} detected")
32-
33-
# Initialize CUDA variables
34-
CUDAExtension = None
35-
BuildExtension = None
36-
CUDA_HOME = None
37-
TORCH_CUDA_AVAILABLE = False
38-
39-
# Check PyTorch CUDA availability first
40-
if torch.cuda.is_available():
41-
log(" PyTorch CUDA is available")
42-
TORCH_CUDA_AVAILABLE = True
43-
44-
# Try all possible import methods for PyTorch 2.10+
45-
import_methods = [
46-
# Method 1: Old path (PyTorch < 2.10)
47-
lambda: __import__('torch.utils.cpp_extension', fromlist=['CUDAExtension', 'BuildExtension', 'CUDA_HOME']),
48-
# Method 2: New path (PyTorch 2.10+)
49-
lambda: __import__('torch.cuda.cpp_extension', fromlist=['CUDAExtension', 'BuildExtension', 'CUDA_HOME']),
50-
# Method 3: Direct import via torch.cuda
51-
lambda: __import__('torch.cuda.cpp_extension'),
52-
]
53-
54-
for i, method in enumerate(import_methods, 1):
55-
try:
56-
module = method()
57-
CUDAExtension = getattr(module, 'CUDAExtension', None)
58-
BuildExtension = getattr(module, 'BuildExtension', None)
59-
CUDA_HOME = getattr(module, 'CUDA_HOME', None)
60-
61-
if CUDAExtension and BuildExtension:
62-
log(f"✓ Method {i} successful: PyTorch CUDA extensions available")
63-
break
64-
except (ImportError, AttributeError) as e:
65-
log(f"Method {i} failed: {e}")
66-
continue
67-
68-
# Fallback: Manual CUDA_HOME detection
69-
if not CUDA_HOME:
70-
CUDA_HOME = os.environ.get('CUDA_HOME', '/usr/local/cuda')
71-
if os.path.exists(CUDA_HOME):
72-
log(f"✓ CUDA_HOME detected: {CUDA_HOME}")
73-
else:
74-
log("! CUDA_HOME not found, will try to build anyway")
75-
CUDA_HOME = '/usr/local/cuda' # Default for most systems
76-
77-
# Final check
78-
if CUDAExtension and BuildExtension:
79-
log("✓ All CUDA components available for build")
80-
else:
81-
log("! CUDA extension components not available, will use manual build")
82-
CUDAExtension = None
83-
BuildExtension = None
84-
85-
else:
86-
log("PyTorch CUDA not available")
87-
if FORCE_CUDA:
88-
log("Forced CUDA build enabled (CRAYON_FORCE_CUDA=1)")
89-
TORCH_CUDA_AVAILABLE = True
90-
else:
91-
TORCH_CUDA_AVAILABLE = False
92-
93-
except ImportError:
94-
log("PyTorch not installed")
95-
TORCH_CUDA_AVAILABLE = False
96-
CUDAExtension = None
97-
BuildExtension = None
98-
CUDA_HOME = None
99-
100-
# ============================================================================
101-
# ROBUST CUDA BUILD CLASS
102-
# ============================================================================
103-
104-
class CrayonBuildExt(build_ext):
105-
"""Custom build class that handles CUDA compilation with maximum compatibility"""
106-
107-
def build_extension(self, ext):
108-
if ext.name == "crayon.c_ext.crayon_cuda":
109-
self._build_cuda_extension_robust(ext)
110-
else:
111-
super().build_extension(ext)
112-
113-
def _build_cuda_extension_robust(self, ext):
114-
"""Build CUDA extension with maximum compatibility"""
115-
log(f"Building CUDA extension: {ext.name}")
116-
117-
# Get Python version info
118-
python_version = f"{sys.version_info.major}.{sys.version_info.minor}"
119-
120-
# Try multiple include paths
121-
include_paths = []
122-
123-
# Python include
124-
python_includes = [
125-
f"/usr/include/python{python_version}",
126-
f"/usr/local/include/python{python_version}",
127-
get_python_inc(),
128-
]
129-
for inc in python_includes:
130-
if os.path.exists(inc):
131-
include_paths.append(f"-I{inc}")
132-
break
133-
134-
# Torch include paths
135-
try:
136-
import torch
137-
torch_path = os.path.dirname(torch.__file__)
138-
torch_includes = [
139-
f"{torch_path}/include",
140-
f"{torch_path}/include/torch/csrc/api/include",
141-
f"{torch_path}/../include",
142-
]
143-
for inc in torch_includes:
144-
if os.path.exists(inc):
145-
include_paths.append(f"-I{inc}")
146-
except:
147-
pass
148-
149-
# CUDA include paths
150-
cuda_includes = [
151-
os.environ.get('CUDA_HOME', '/usr/local/cuda'),
152-
'/usr/local/cuda',
153-
'/usr/cuda',
154-
'C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v12.8',
155-
'C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v12.7',
156-
'C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v12.6',
157-
'C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v12.5',
158-
'C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v12.4',
159-
'C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v12.3',
160-
'C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v12.2',
161-
'C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v12.1',
162-
'C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v12.0',
163-
'C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v11.8',
164-
]
165-
166-
cuda_include = None
167-
for inc in cuda_includes:
168-
if os.path.exists(inc):
169-
cuda_include = f"{inc}/include"
170-
include_paths.append(f"-I{cuda_include}")
171-
log(f"✓ CUDA include found: {cuda_include}")
172-
break
173-
174-
if not cuda_include:
175-
log("! CUDA include not found, trying default paths")
176-
cuda_include = "/usr/local/cuda/include"
177-
include_paths.append(f"-I{cuda_include}")
178-
179-
# Build command for different platforms
180-
if sys.platform == "win32":
181-
# Windows build
182-
cmd = [
183-
"nvcc",
184-
"-O3", "-std=c++17",
185-
"--compiler-options", "/MD",
186-
"-shared",
187-
"-o", self.get_ext_fullname(ext.name).replace('.', '/') + ".pyd",
188-
ext.sources[0],
189-
] + include_paths + [
190-
"-D_GLIBCXX_USE_CXX11_ABI=0",
191-
"-Xcompiler", "/EHsc",
192-
]
193-
else:
194-
# Linux build
195-
cmd = [
196-
"nvcc",
197-
"-O3", "-std=c++17",
198-
"--compiler-options", "-fPIC",
199-
"-shared",
200-
"-o", self.get_ext_fullname(ext.name).replace('.', '/') + ".so",
201-
ext.sources[0],
202-
] + include_paths + [
203-
"-D_GLIBCXX_USE_CXX11_ABI=0",
204-
]
205-
206-
# Add GPU architecture flags
207-
try:
208-
if torch.cuda.is_available():
209-
major, minor = torch.cuda.get_device_capability()
210-
arch = f"{major}{minor}"
211-
cmd.extend([f"-gencode=arch=compute_{arch},code=sm_{arch}"])
212-
log(f"Compiling for GPU architecture: sm_{arch}")
213-
else:
214-
cmd.extend(["-gencode=arch=compute_75,code=sm_75"])
215-
log("Using default GPU architecture: sm_75")
216-
except:
217-
cmd.extend(["-gencode=arch=compute_75,code=sm_75"])
218-
log("Using default GPU architecture: sm_75")
219-
220-
log(f"CUDA build command: {' '.join(cmd)}")
221-
222-
try:
223-
# Create output directory
224-
output_dir = os.path.dirname(self.get_ext_fullname(ext.name).replace('.', '/'))
225-
os.makedirs(os.path.join(self.build_lib, output_dir), exist_ok=True)
226-
227-
# Run compilation
228-
result = subprocess.run(cmd, capture_output=True, text=True, cwd=self.build_lib)
229-
if result.returncode == 0:
230-
log(f"✓ CUDA extension {ext.name} built successfully")
231-
else:
232-
log(f"✗ CUDA build failed: {result.stderr}")
233-
# Don't raise error, just log it - extension will be skipped
234-
log("CUDA extension will be skipped, continuing with CPU-only build")
235-
except Exception as e:
236-
log(f"CUDA build error: {e}")
237-
log("CUDA extension will be skipped, continuing with CPU-only build")
238-
239-
# ============================================================================
240-
# EXTENSION CONFIGURATION
241-
# ============================================================================
16+
# Compiler flags
17+
if sys.platform == "win32":
18+
cpu_cflags = ["/O2", "/std:c++17"]
19+
else:
20+
cpu_cflags = ["-O3", "-fPIC", "-std=c++17"]
24221

22+
# CPU Extensions (always built)
24323
ext_modules = []
24424

245-
# CPU Extensions (always built)
246-
if sys.platform == "win32":
247-
cpu_args = ["/O2", "/std:c++17"]
248-
else:
249-
cpu_args = ["-O3", "-fPIC", "-std=c++17"]
25+
log("Adding CPU extensions...")
25026

251-
# CPU Extension
25227
ext_modules.append(Extension(
25328
"crayon.c_ext.crayon_cpu",
25429
sources=["src/crayon/c_ext/cpu_engine.cpp"],
255-
extra_compile_args=cpu_args,
30+
extra_compile_args=cpu_cflags,
25631
language="c++",
25732
))
25833

259-
# Trainer Extension
26034
ext_modules.append(Extension(
261-
"crayon.c_ext.crayon_trainer",
35+
"crayon.c_ext.crayon_trainer",
26236
sources=["src/crayon/c_ext/trainer.cpp"],
263-
extra_compile_args=cpu_args,
37+
extra_compile_args=cpu_cflags,
26438
language="c++",
26539
))
26640

267-
# Compiler Extension
26841
ext_modules.append(Extension(
26942
"crayon.c_ext.crayon_compiler",
27043
sources=["src/crayon/c_ext/compiler.cpp"],
271-
extra_compile_args=cpu_args,
44+
extra_compile_args=cpu_cflags,
27245
language="c++",
27346
))
27447

275-
# CUDA Extension (if available)
276-
if (TORCH_CUDA_AVAILABLE or FORCE_CUDA) and not FORCE_CPU:
277-
log("Adding CUDA extension to build queue")
278-
279-
# Use our robust build class
280-
cuda_ext = Extension(
281-
"crayon.c_ext.crayon_cuda",
282-
sources=["src/crayon/c_ext/gpu_engine_cuda.cu"],
283-
extra_compile_args={"nvcc": ["-O3", "-std=c++17", "--expt-relaxed-constexpr"]},
284-
language="c++",
285-
)
286-
ext_modules.append(cuda_ext)
287-
288-
# Use custom build class
289-
cmdclass = {"build_ext": CrayonBuildExt}
290-
log("Using robust CUDA build class")
291-
292-
else:
293-
cmdclass = {}
294-
if not FORCE_CPU:
295-
log("Skipping CUDA extension - not available or forced CPU")
296-
297-
# ============================================================================
298-
# SETUP
299-
# ============================================================================
300-
30148
setup(
30249
name="xerv-crayon",
30350
version=VERSION,
30451
packages=find_packages("src"),
30552
package_dir={"": "src"},
30653
include_package_data=True,
30754
ext_modules=ext_modules,
308-
cmdclass=cmdclass,
309-
python_requires=">=3.8",
310-
zip_safe=False,
311-
)
55+
python_requires=">=3.8,<3.14",
56+
install_requires=[
57+
"numpy>=1.21.0",
58+
],
59+
)

0 commit comments

Comments
 (0)