Skip to content

Commit dd0651e

Browse files
authored
Remove preprocessing base classes we no longer use (#1824)
For some reason, all our causal lm models had a preprocessing layer base class that packed inputs without setting things up for the causal language model loss. With our new base classes, `CausalLMPreprocessor` and `Seq2SeqLMPreprocessor`, these are no longer used at all. These are public, but I did a search for them on GitHub and the only one I could find with any usage is `GPT2Preprocessor`. The others are complete unused, and not generally very useful. Let's remove them before anyone does depend on them so that we can keep preprocessing one-to-one paired with a task. E.g. ``` BertTextClassifier BertTextClassifierPreprocesssor ```
1 parent 29818cb commit dd0651e

26 files changed

+9
-2767
lines changed

keras_nlp/api/models/__init__.py

Lines changed: 0 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,6 @@
3434
from keras_nlp.src.models.albert.albert_tokenizer import AlbertTokenizer
3535
from keras_nlp.src.models.backbone import Backbone
3636
from keras_nlp.src.models.bart.bart_backbone import BartBackbone
37-
from keras_nlp.src.models.bart.bart_preprocessor import BartPreprocessor
3837
from keras_nlp.src.models.bart.bart_seq_2_seq_lm import BartSeq2SeqLM
3938
from keras_nlp.src.models.bart.bart_seq_2_seq_lm_preprocessor import (
4039
BartSeq2SeqLMPreprocessor,
@@ -58,7 +57,6 @@
5857
from keras_nlp.src.models.bloom.bloom_causal_lm_preprocessor import (
5958
BloomCausalLMPreprocessor,
6059
)
61-
from keras_nlp.src.models.bloom.bloom_preprocessor import BloomPreprocessor
6260
from keras_nlp.src.models.bloom.bloom_tokenizer import BloomTokenizer
6361
from keras_nlp.src.models.causal_lm import CausalLM
6462
from keras_nlp.src.models.causal_lm_preprocessor import CausalLMPreprocessor
@@ -105,9 +103,6 @@
105103
DistilBertTokenizer,
106104
)
107105
from keras_nlp.src.models.electra.electra_backbone import ElectraBackbone
108-
from keras_nlp.src.models.electra.electra_preprocessor import (
109-
ElectraPreprocessor,
110-
)
111106
from keras_nlp.src.models.electra.electra_tokenizer import ElectraTokenizer
112107
from keras_nlp.src.models.f_net.f_net_backbone import FNetBackbone
113108
from keras_nlp.src.models.f_net.f_net_masked_lm import FNetMaskedLM
@@ -127,14 +122,12 @@
127122
from keras_nlp.src.models.falcon.falcon_causal_lm_preprocessor import (
128123
FalconCausalLMPreprocessor,
129124
)
130-
from keras_nlp.src.models.falcon.falcon_preprocessor import FalconPreprocessor
131125
from keras_nlp.src.models.falcon.falcon_tokenizer import FalconTokenizer
132126
from keras_nlp.src.models.gemma.gemma_backbone import GemmaBackbone
133127
from keras_nlp.src.models.gemma.gemma_causal_lm import GemmaCausalLM
134128
from keras_nlp.src.models.gemma.gemma_causal_lm_preprocessor import (
135129
GemmaCausalLMPreprocessor,
136130
)
137-
from keras_nlp.src.models.gemma.gemma_preprocessor import GemmaPreprocessor
138131
from keras_nlp.src.models.gemma.gemma_tokenizer import GemmaTokenizer
139132
from keras_nlp.src.models.gpt2.gpt2_backbone import GPT2Backbone
140133
from keras_nlp.src.models.gpt2.gpt2_causal_lm import GPT2CausalLM
@@ -148,23 +141,18 @@
148141
from keras_nlp.src.models.gpt_neo_x.gpt_neo_x_causal_lm_preprocessor import (
149142
GPTNeoXCausalLMPreprocessor,
150143
)
151-
from keras_nlp.src.models.gpt_neo_x.gpt_neo_x_preprocessor import (
152-
GPTNeoXPreprocessor,
153-
)
154144
from keras_nlp.src.models.gpt_neo_x.gpt_neo_x_tokenizer import GPTNeoXTokenizer
155145
from keras_nlp.src.models.llama3.llama3_backbone import Llama3Backbone
156146
from keras_nlp.src.models.llama3.llama3_causal_lm import Llama3CausalLM
157147
from keras_nlp.src.models.llama3.llama3_causal_lm_preprocessor import (
158148
Llama3CausalLMPreprocessor,
159149
)
160-
from keras_nlp.src.models.llama3.llama3_preprocessor import Llama3Preprocessor
161150
from keras_nlp.src.models.llama3.llama3_tokenizer import Llama3Tokenizer
162151
from keras_nlp.src.models.llama.llama_backbone import LlamaBackbone
163152
from keras_nlp.src.models.llama.llama_causal_lm import LlamaCausalLM
164153
from keras_nlp.src.models.llama.llama_causal_lm_preprocessor import (
165154
LlamaCausalLMPreprocessor,
166155
)
167-
from keras_nlp.src.models.llama.llama_preprocessor import LlamaPreprocessor
168156
from keras_nlp.src.models.llama.llama_tokenizer import LlamaTokenizer
169157
from keras_nlp.src.models.masked_lm import MaskedLM
170158
from keras_nlp.src.models.masked_lm_preprocessor import MaskedLMPreprocessor
@@ -173,16 +161,12 @@
173161
from keras_nlp.src.models.mistral.mistral_causal_lm_preprocessor import (
174162
MistralCausalLMPreprocessor,
175163
)
176-
from keras_nlp.src.models.mistral.mistral_preprocessor import (
177-
MistralPreprocessor,
178-
)
179164
from keras_nlp.src.models.mistral.mistral_tokenizer import MistralTokenizer
180165
from keras_nlp.src.models.opt.opt_backbone import OPTBackbone
181166
from keras_nlp.src.models.opt.opt_causal_lm import OPTCausalLM
182167
from keras_nlp.src.models.opt.opt_causal_lm_preprocessor import (
183168
OPTCausalLMPreprocessor,
184169
)
185-
from keras_nlp.src.models.opt.opt_preprocessor import OPTPreprocessor
186170
from keras_nlp.src.models.opt.opt_tokenizer import OPTTokenizer
187171
from keras_nlp.src.models.pali_gemma.pali_gemma_backbone import (
188172
PaliGemmaBackbone,
@@ -201,7 +185,6 @@
201185
from keras_nlp.src.models.phi3.phi3_causal_lm_preprocessor import (
202186
Phi3CausalLMPreprocessor,
203187
)
204-
from keras_nlp.src.models.phi3.phi3_preprocessor import Phi3Preprocessor
205188
from keras_nlp.src.models.phi3.phi3_tokenizer import Phi3Tokenizer
206189
from keras_nlp.src.models.preprocessor import Preprocessor
207190
from keras_nlp.src.models.roberta.roberta_backbone import RobertaBackbone

keras_nlp/src/models/bart/bart_preprocessor.py

Lines changed: 0 additions & 264 deletions
This file was deleted.

0 commit comments

Comments
 (0)