Skip to content

Commit 869a33b

Browse files
JYMiracle305JYMiracle305
committed
训练营大模型训练项目
0 parents  commit 869a33b

File tree

120 files changed

+11080
-0
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

120 files changed

+11080
-0
lines changed

‎.clang-format‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
---
2+
BasedOnStyle: LLVM
3+
IndentWidth: 4
4+
AccessModifierOffset: -4
5+
AlignOperands: AlignAfterOperator
6+
BreakBeforeBinaryOperators: All
7+
ColumnLimit: 120
8+
AllowShortBlocksOnASingleLine: Always
9+
AllowShortLoopsOnASingleLine: true
10+
InsertBraces: true
11+
BreakBeforeBraces: Custom
12+
BraceWrapping:
13+
AfterCaseLabel: false
14+
AfterClass: false
15+
AfterControlStatement: Never
16+
AfterEnum: false
17+
AfterFunction: false
18+
AfterNamespace: false
19+
AfterObjCDeclaration: false
20+
AfterStruct: false
21+
AfterUnion: false
22+
AfterExternBlock: false
23+
BeforeCatch: false
24+
BeforeElse: false
25+
BeforeLambdaBody: false
26+
BeforeWhile: false
27+
IndentBraces: false
28+
SplitEmptyFunction: true
29+
SplitEmptyRecord: true
30+
SplitEmptyNamespace: true
31+

‎.github/workflows/build.yml‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
name: Build and test cpu
2+
on:
3+
push:
4+
paths-ignore:
5+
- '**.md'
6+
- 'LICENSE'
7+
pull_request:
8+
paths:
9+
- '**.md'
10+
- 'LICENSE'
11+
12+
jobs:
13+
build:
14+
name: Build
15+
runs-on: ubuntu-latest
16+
steps:
17+
- uses: actions/checkout@v3
18+
with:
19+
submodules: recursive
20+
21+
- name: Build
22+
run: make USE_CUDA=OFF
23+
24+
- name: Download starter pack
25+
run: ./download_starter_pack.sh
26+
27+
- name: Test cpu
28+
run: make test-cpp

‎.github/workflows/format-check.yaml‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
name: Format Check
2+
3+
on:
4+
pull_request:
5+
push:
6+
paths-ignore:
7+
- '**.md'
8+
- 'LICENSE'
9+
10+
jobs:
11+
format-check:
12+
name: Check Code Format
13+
runs-on: ubuntu-latest
14+
steps:
15+
16+
- name: Checkout code
17+
uses: actions/checkout@v4
18+
19+
- name: Install Python dependencies
20+
run: |
21+
python3 -m pip install --upgrade pip
22+
pip install black
23+
24+
- name: Run format check
25+
run: |
26+
python3 scripts/format.py --path infini_train example --check
27+

‎.gitignore‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
build/
2+
.cache/
3+
.vscode/
4+
Data/

‎.gitmodules‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
[submodule "third_party/googletest"]
2+
path = third_party/googletest
3+
url = git@github.com:google/googletest.git
4+
[submodule "third_party/gflags"]
5+
path = third_party/gflags
6+
url = git@github.com:gflags/gflags.git
7+
[submodule "third_party/glog"]
8+
path = third_party/glog
9+
url = git@github.com:google/glog.git
10+
[submodule "third_party/eigen"]
11+
path = third_party/eigen
12+
url = https://gitlab.com/libeigen/eigen.git

‎CMakeLists.txt‎

Lines changed: 105 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,105 @@
1+
option(USE_CUDA "Support NVIDIA CUDA" OFF)
2+
option(BUILD_TEST "Build tests" OFF)
3+
option(BUILD_TESTING "third party tests" OFF)
4+
5+
cmake_minimum_required(VERSION 3.28)
6+
7+
include(CMakeDependentOption)
8+
cmake_dependent_option(BUILD_TEST_CORE "Build tests for core components" ON BUILD_TEST OFF)
9+
project(infini_train VERSION 0.3.0 LANGUAGES CXX)
10+
11+
set(CMAKE_CXX_STANDARD 20)
12+
set(CMAKE_CXX_STANDARD_REQUIRED ON)
13+
set(CMAKE_CXX_EXTENSIONS OFF)
14+
15+
# Generate compile_commands.json
16+
set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
17+
18+
# Add gflags
19+
add_subdirectory(third_party/gflags)
20+
include_directories(${gflags_SOURCE_DIR}/include)
21+
22+
set(WITH_GFLAGS OFF CACHE BOOL "Disable glog finding system gflags" FORCE)
23+
set(WITH_GTEST OFF CACHE BOOL "Disable glog finding system gtest" FORCE)
24+
25+
# Add glog
26+
add_subdirectory(third_party/glog)
27+
include_directories(${glog_SOURCE_DIR}/src)
28+
29+
# Add eigen
30+
find_package(OpenMP REQUIRED)
31+
add_subdirectory(third_party/eigen)
32+
include_directories(${PROJECT_SOURCE_DIR}/third_party/eigen)
33+
34+
include_directories(${PROJECT_SOURCE_DIR})
35+
file(GLOB_RECURSE SRC ${PROJECT_SOURCE_DIR}/infini_train/src/*.cc)
36+
list(FILTER SRC EXCLUDE REGEX ".*kernels/cpu/.*")
37+
38+
file (GLOB_RECURSE CPU_KERNELS ${PROJECT_SOURCE_DIR}/infini_train/src/kernels/cpu/*.cc)
39+
add_library(infini_train_cpu_kernels STATIC ${CPU_KERNELS})
40+
target_link_libraries(infini_train_cpu_kernels glog Eigen3::Eigen OpenMP::OpenMP_CXX)
41+
42+
if(USE_CUDA)
43+
add_compile_definitions(USE_CUDA=1)
44+
enable_language(CUDA)
45+
include(FindCUDAToolkit)
46+
47+
# enable CUDA-related compilation options
48+
set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} --expt-extended-lambda --expt-relaxed-constexpr")
49+
file(GLOB_RECURSE CUDA_KERNELS ${PROJECT_SOURCE_DIR}/infini_train/src/*.cu)
50+
add_library(infini_train_cuda_kernels STATIC ${CUDA_KERNELS})
51+
set_target_properties(infini_train_cuda_kernels PROPERTIES CUDA_ARCHITECTURES "75;80")
52+
target_link_libraries(infini_train_cuda_kernels glog CUDA::cudart CUDA::cublas)
53+
54+
add_library(infini_train STATIC ${SRC})
55+
target_link_libraries(infini_train glog gflags "-Wl,--whole-archive" infini_train_cpu_kernels infini_train_cuda_kernels "-Wl,--no-whole-archive")
56+
else()
57+
add_library(infini_train STATIC ${SRC})
58+
target_link_libraries(infini_train glog gflags "-Wl,--whole-archive" infini_train_cpu_kernels "-Wl,--no-whole-archive")
59+
endif()
60+
61+
if(BUILD_TEST)
62+
set(BUILD_GMOCK
63+
OFF
64+
CACHE BOOL "Do not build gmock" FORCE)
65+
set(INSTALL_GTEST
66+
OFF
67+
CACHE BOOL "Do not install gtest" FORCE)
68+
add_subdirectory(third_party/googletest)
69+
include_directories(third_party/googletest/googletest/include)
70+
endif()
71+
72+
add_library(example_gpt2 STATIC
73+
example/common/tiny_shakespeare_dataset.cc
74+
example/common/tokenizer.cc
75+
example/gpt2/net.cc
76+
)
77+
target_link_libraries(example_gpt2 infini_train)
78+
79+
function(build_test files)
80+
# Non-recursive glob for skip failed tests
81+
file(GLOB TEST_SOURCES ${files})
82+
foreach(testsourcefile ${TEST_SOURCES})
83+
get_filename_component(testname ${testsourcefile} NAME_WE)
84+
add_executable(${testname} ${testsourcefile})
85+
target_link_libraries(${testname} infini_train example_gpt2 GTest::gtest_main)
86+
add_test(NAME ${testname} COMMAND ${testname})
87+
endforeach(testsourcefile ${TEST_SOURCES})
88+
endfunction()
89+
90+
if(BUILD_TEST)
91+
add_compile_definitions(BUILD_TEST=1)
92+
enable_testing()
93+
if(BUILD_TEST_CORE)
94+
build_test(test/autograd/test_elementwise.cc)
95+
build_test(test/kernels/test_matmul.cc)
96+
build_test(test/kernels/test_dispatcher.cc)
97+
build_test(test/tensor/test_tensor.cc)
98+
build_test(test/optimizer/test_adam.cc)
99+
build_test(test/example/test_gpt2.cc)
100+
if(USE_CUDA)
101+
build_test(test/kernels/test_matmul_cuda.cc)
102+
build_test(test/optimizer/test_adam_cuda.cc)
103+
endif()
104+
endif()
105+
endif()

‎LICENSE‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
MIT License
2+
3+
Copyright (c) 2025 InfiniTensor
4+
5+
Permission is hereby granted, free of charge, to any person obtaining a copy
6+
of this software and associated documentation files (the "Software"), to deal
7+
in the Software without restriction, including without limitation the rights
8+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9+
copies of the Software, and to permit persons to whom the Software is
10+
furnished to do so, subject to the following conditions:
11+
12+
The above copyright notice and this permission notice shall be included in all
13+
copies or substantial portions of the Software.
14+
15+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21+
SOFTWARE.

‎Makefile‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
TYPE ?= Release
2+
TEST ?= ON
3+
USE_CUDA ?= ON
4+
5+
CMAKE_OPT = -DBUILD_TEST=$(TEST)
6+
CMAKE_OPT += -DBUILD_TESTING=OFF
7+
CMAKE_OPT += -DUSE_CUDA=$(USE_CUDA)
8+
9+
build:
10+
mkdir -p build/$(TYPE)
11+
cd build/$(TYPE) && cmake $(CMAKE_OPT) ../.. && make -j8
12+
13+
clean:
14+
rm -rf build
15+
16+
test-cpp:
17+
@echo
18+
cd build/$(TYPE) && make test

‎README.md‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
# TinyInfiniTrain
2+
3+
简化版的训练框架，全部由C++实现，包含了autograd机制、kernel层关键算子实现，测例中包含单点验证以及端到端训练GPT2模型流程，支持CPU和CUDA平台。
4+
5+
[环境部署文档](docs/项目部署.md)
6+
7+
[训练营作业介绍文档](docs/训练营作业介绍.md)

‎add_module.sh‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
#!/bin/bash
2+
3+
# 初始化子模块配置
4+
git submodule init
5+
6+
# 添加每个子模块
7+
git submodule add git@github.com:google/glog.git third_party/glog
8+
git submodule add git@github.com:gflags/gflags.git third_party/gflags
9+
git submodule add https://gitlab.com/libeigen/eigen.git third_party/eigen
10+
git submodule add git@github.com:google/googletest.git third_party/googletest

0 commit comments

Comments
 (0)