reorg structure of model implementations

Author: zigzagcai

internlm/checkpoint/checkpoint_manager.py

@@ -16,8 +16,8 @@
     auto_resume_sanity_check,
     ckpt_info_sanity_check,
 )
-from internlm.model.base_model import BaseModel
-from internlm.model.registry import model_initializer
+from internlm.model_implementations.registry import model_initializer
+from internlm.model_implementations.transformers.base_model import BaseModel
 from internlm.monitor import send_alert_message
 from internlm.solver.optimizer import HybridZeroOptimizer, HybridZeroOptimizer_v2
 from internlm.utils.common import get_current_device

internlm/checkpoint/load_funcs.py

@@ -1,8 +1,8 @@
 # Copyright (c) InternLM. All rights reserved.
 
-from internlm.model.modeling_internlm import InternLM1
-from internlm.model.modeling_internlm2 import InternLM2
-from internlm.model.modeling_llama import Llama2
+from internlm.model_implementations.transformers.modeling_internlm import InternLM1
+from internlm.model_implementations.transformers.modeling_internlm2 import InternLM2
+from internlm.model_implementations.transformers.modeling_llama import Llama2
 from internlm.utils.logger import get_logger
 
 logger = get_logger(__file__)

internlm/core/trainer_builder.py

@@ -19,12 +19,12 @@
 from internlm.initialize.initialize_trainer import initialize_trainer
 from internlm.model.losses.ce_loss import InternLoss
 from internlm.model.metrics import AccPerplex
+from internlm.model_inject.inject import inject_model
 from internlm.monitor.monitor import send_alert_message
 from internlm.train.pipeline import (
     get_scheduler_hooks,
     initialize_llm_profile,
     initialize_optimizer,
-    inject_model,
     load_new_batch,
     record_current_batch_training_metrics,
 )

internlm/model_implementations/__init__.py

@@ -6,12 +6,12 @@
 from internlm.core.context import ParallelMode
 from internlm.core.context import global_context as gpc
 from internlm.core.parallel.shard import pipeline_parallel_sharding_wrapper
-from internlm.model.base_model import BaseModel
 from internlm.model.modules.linear import (
     ParallelLinearWithCommExt,
     ScaleColumnParallelLinear,
 )
-from internlm.model.registry import model_initializer
+from internlm.model_implementations.registry import model_initializer
+from internlm.model_implementations.transformers.base_model import BaseModel
 from internlm.utils.common import get_current_device
 from internlm.utils.lazy import LazyObject
 from internlm.utils.logger import get_logger

internlm/model/builder.py internlm/model_implementations/builder.pyinternlm/model/builder.py renamed to internlm/model_implementations/builder.py

@@ -4,16 +4,16 @@
 
 from typing import Callable
 
-from internlm.model.modeling_baichuan2 import Baichuan2
-from internlm.model.modeling_gemma import Gemma
-from internlm.model.modeling_internlm import InternLM1
-from internlm.model.modeling_internlm2 import InternLM2
-from internlm.model.modeling_llama import Llama2
-from internlm.model.modeling_llava import Llava
-from internlm.model.modeling_mixtral import MixtralMoE
-from internlm.model.modeling_moe import Internlm1MoE
-from internlm.model.modeling_qwen2 import Qwen2
-from internlm.model.modeling_qwen2_moe import Qwen2Moe
+from internlm.model_implementations.transformers.modeling_baichuan2 import Baichuan2
+from internlm.model_implementations.transformers.modeling_gemma import Gemma
+from internlm.model_implementations.transformers.modeling_internlm import InternLM1
+from internlm.model_implementations.transformers.modeling_internlm2 import InternLM2
+from internlm.model_implementations.transformers.modeling_llama import Llama2
+from internlm.model_implementations.transformers.modeling_llava import Llava
+from internlm.model_implementations.transformers.modeling_mixtral import MixtralMoE
+from internlm.model_implementations.transformers.modeling_moe import Internlm1MoE
+from internlm.model_implementations.transformers.modeling_qwen2 import Qwen2
+from internlm.model_implementations.transformers.modeling_qwen2_moe import Qwen2Moe
 from internlm.utils.common import SingletonMeta
 from internlm.utils.utils import ModelType
 

internlm/model/registry.py …ternlm/model_implementations/registry.pyinternlm/model/registry.py renamed to internlm/model_implementations/registry.py internlm/model/registry.py renamed to internlm/model_implementations/registry.py

@@ -17,7 +17,6 @@
     scaled_init_method_uniform,
     uniform_,
 )
-from internlm.model.base_model import BaseModel
 from internlm.model.modules.embedding import Embedding1D
 from internlm.model.modules.linear import new_linear
 from internlm.model.modules.mha import MHA
@@ -27,6 +26,7 @@
     convert_attn_args_to_kwargs,
     convert_attn_kwargs_to_args,
 )
+from internlm.model_implementations.transformers.base_model import BaseModel
 from internlm.solver.activation_checkpoint import activation_checkpoint
 from internlm.utils.logger import get_logger
 from internlm.utils.storage_manager import get_fns, llm_load, llm_save

internlm/model_implementations/transformers/__init__.py

@@ -16,7 +16,6 @@
     scaled_init_method_uniform,
     uniform_,
 )
-from internlm.model.base_model import BaseModel
 from internlm.model.modules.embedding import Embedding1D
 from internlm.model.modules.linear import new_linear
 from internlm.model.modules.mha import GQA
@@ -26,6 +25,7 @@
     convert_attn_args_to_kwargs,
     convert_attn_kwargs_to_args,
 )
+from internlm.model_implementations.transformers.base_model import BaseModel
 from internlm.solver.activation_checkpoint import activation_checkpoint
 from internlm.utils.logger import get_logger
 from internlm.utils.storage_manager import get_fns, llm_load, llm_save