ValueError: Cannot get dimension 'nO' for model 'transformer-listener': value unset when using auto-generated config

[jefhil]

Error trying to train new ner model using auto generated config file

Info about spaCy

• spaCy version: 3.4.1
• Platform: Linux-5.15.0-1015-aws-x86_64-with-glibc2.29
• Python version: 3.8.10
• Pipelines: en_core_web_trf (3.4.0), en_core_web_sm (3.4.0)
  (sorry if this is not correct place to post this -- please redirect me if needed)
  generated config using:
  python -m spacy init config newest.cfg --optimize accuracy --gpu --force
  added:
  frozen_components = ["transformer", "tagger","parser"]
  Complete config file below:

[paths] train = null dev = null vectors = null init_tok2vec = null

[system]
gpu_allocator = "pytorch"
seed = 0

[nlp]
lang = "en"
pipeline = ["transformer","tagger","parser","ner"]
batch_size = 128
disabled = []
before_creation = null
after_creation = null
after_pipeline_creation = null
tokenizer = {"@Tokenizers":"spacy.Tokenizer.v1"}

[components]

[components.ner]
factory = "ner"
incorrect_spans_key = null
moves = null
scorer = {"@scorers":"spacy.ner_scorer.v1"}
update_with_oracle_cut_size = 100

[components.ner.model]
@architectures = "spacy.TransitionBasedParser.v2"
state_type = "ner"
extra_state_tokens = false
hidden_width = 64
maxout_pieces = 2
use_upper = false
nO = null

[components.ner.model.tok2vec]
@architectures = "spacy-transformers.TransformerListener.v1"
grad_factor = 1.0
pooling = {"@layers":"reduce_mean.v1"}
upstream = "*"
[components.parser]
factory = "parser"
learn_tokens = false
min_action_freq = 30
moves = null
scorer = {"@scorers":"spacy.parser_scorer.v1"}
update_with_oracle_cut_size = 100

[components.parser.model]
@architectures = "spacy.TransitionBasedParser.v2"
state_type = "parser"
extra_state_tokens = false
hidden_width = 128
maxout_pieces = 3
use_upper = false
nO = null
[components.parser.model.tok2vec]
@architectures = "spacy-transformers.TransformerListener.v1"
grad_factor = 1.0
pooling = {"@layers":"reduce_mean.v1"}
upstream = "*"

[components.tagger]
factory = "tagger"
neg_prefix = "!"
overwrite = false
scorer = {"@scorers":"spacy.tagger_scorer.v1"}

[components.tagger.model]
@architectures = "spacy.Tagger.v2"
nO = null
normalize = false

[components.tagger.model.tok2vec]
@architectures = "spacy-transformers.TransformerListener.v1"
grad_factor = 1.0
pooling = {"@layers":"reduce_mean.v1"}
upstream = "*"
[components.transformer]
factory = "transformer"
max_batch_items = 4096
set_extra_annotations = {"@annotation_setters":"spacy-transformers.null_annotation_setter.v1"}

[components.transformer.model]
@architectures = "spacy-transformers.TransformerModel.v3"
name = "roberta-base"
mixed_precision = false

[components.transformer.model.get_spans]
@span_getters = "spacy-transformers.strided_spans.v1"
window = 128
stride = 96
[components.transformer.model.grad_scaler_config]

[components.transformer.model.tokenizer_config]
use_fast = true

[components.transformer.model.transformer_config]

[corpora]

[corpora.dev]
@readers = "spacy.Corpus.v1"
path = ${paths.dev}
max_length = 0
gold_preproc = false
limit = 0
augmenter = null

[corpora.train]
@readers = "spacy.Corpus.v1"
path = ${paths.train}
max_length = 0
gold_preproc = false
limit = 0
augmenter = null

[training]
accumulate_gradient = 3
dev_corpus = "corpora.dev"
train_corpus = "corpora.train"
seed = ${system.seed}
gpu_allocator = ${system.gpu_allocator}
dropout = 0.1
patience = 1600
max_epochs = 0
max_steps = 20000
eval_frequency = 200
frozen_components = ["transformer", "tagger","parser"]
annotating_components = []
before_to_disk = null

[training.batcher]
@batchers = "spacy.batch_by_padded.v1"
discard_oversize = true
size = 2000
buffer = 256
get_length = null

[training.logger]
@Loggers = "spacy.ConsoleLogger.v1"
progress_bar = false

[training.optimizer]
@optimizers = "Adam.v1"
beta1 = 0.9
beta2 = 0.999
L2_is_weight_decay = true
L2 = 0.01
grad_clip = 1.0
use_averages = false
eps = 0.00000001

[training.optimizer.learn_rate]
@schedules = "warmup_linear.v1"
warmup_steps = 250
total_steps = 20000
initial_rate = 0.00005

[adrianeboyd]

For statistical components like transformer and tagger, you can use frozen_components for components that have been sourced from another trained pipeline. It doesn't make sense to include and then freeze uninitialized/untrained components as in this config, which is why you're running into errors. (Also a note, there are bugs with freezing transformer components, so for now we recommend training in separate pipelines and then using spacy assemble to generate your final combined pipeline.)

To back up a step, are you trying to fine-tune ner from a model like en_core_web_trf without modifying the other components? If that's the case, have a look at this demo project:

https://github.com/explosion/projects/tree/v3/pipelines/ner_demo_update

The script that generates a config can be used to generate a config for fine-tuning ner from an existing pipeline. For a trf pipeline it will double the size of the pipeline and be slower because there are now two separate internal transformer models (one for the existing components, and one copied for use in ner training), but it's the best approach if you want to update an existing ner.

[jefhil]

thanks for the quick response. I am training a new ner model. I was using only [transformer, ner] for training. Iy=t seems I should take that NER component and assemple a pipeline that contains the other components I want.