The en_core_web_trf model results in zero output

[HarounAbdelsamad]

I tried training the en_core_web_trf model based on datasets i have but after training and evaluation the fscore, recall and precision are all zero. I tried using the small model works fine. I changed the code so that the transformer component is added to the pipe and also use another config file for this. Here is my code for reference:
Could anybody help me or direct me towards the issue?
code.txt

[rmitsch]

Hi @HarounAbdelsamad, please opst your code and your config as formatted snippets inside this thread.

[HarounAbdelsamad]

Code:

import spacy
import en_core_web_trf
from spacy import displacy
import os
import pandas as pd
from tqdm import tqdm
from spacy.tokens import DocBin as DB
import matplotlib.pyplot as plt
import seaborn as sns
import json
import io
from sklearn.metrics import precision_score, recall_score, f1_score
from spacy.training.example import Example
from sklearn.model_selection import train_test_split
import random 

nlp = spacy.load("en_core_web_trf")
# Load the entire JSON data from a file
with open("SmokingDataset.json", "r") as file:
    data = json.load(file)

# Transform the provided data into Spacy training data format
spacy_training_data = []

for sample in data["samples"]:
    entities = [{"start": entity["start"], "end": entity["end"], "label": entity["label"]} for entity in sample["annotation"]["entities"]]
    spacy_sample = {"text": sample["document"], "entities": entities}
    spacy_training_data.append(spacy_sample)

texts = [sample["text"] for sample in spacy_training_data1]
annotations = [sample["entities"] for sample in spacy_training_data1]

train_texts, test_texts, train_annotations, test_annotations = train_test_split(texts, annotations, test_size=0.3, random_state=42)

# Then you can construct train_data and test_data from these lists
train_data = [{"text": text, "entities": entities} for text, entities in zip(train_texts, train_annotations)]
test_data = [{"text": text, "entities": entities} for text, entities in zip(test_texts, test_annotations)]

# Initialize the NER component of the pipeline if not already loaded
if "ner" not in nlp.pipe_names:
    ner = nlp.create_pipe("ner")
    nlp.add_pipe(ner)
else:
    ner = nlp.get_pipe("ner")

for sample in train_data:
    entities = sample["entities"]
    for ent in entities:
        ner.add_label(ent["label"])
# Disable other components for training efficiency
other_pipes = [pipe for pipe in nlp.pipe_names if pipe != "ner" or pipe != "transformer"]
with nlp.disable_pipes(*other_pipes):
    # Training configuration
    optimizer = nlp.create_optimizer()

    # Training loop
    for itn in range(100):
        random.shuffle(train_data)
        losses = {}
        for sample in train_data:
            text = sample["text"]
            entities = sample["entities"]
    
            # Convert the list of entities to the required format
            annotations = {"entities": []}
            for entity in entities:
                start, end, label = entity["start"], entity["end"], entity["label"]
                annotations["entities"].append((start, end, label))
    
            # Update the training example
            example = Example.from_dict(nlp.make_doc(text), annotations)
            nlp.update([example], drop=0.5, losses=losses)


# Save the trained model
nlp.to_disk("roberta7030")

# Create dictionaries to store metrics for each entity
entity_metrics = {
    "ADDRESS": {"true_positives": 0, "false_positives": 0, "false_negatives": 0},
    "PATIENT": {"true_positives": 0, "false_positives": 0, "false_negatives": 0},
    "BPLACE": {"true_positives": 0, "false_positives": 0, "false_negatives": 0},
    "DEDATE": {"true_positives": 0, "false_positives": 0, "false_negatives": 0},
    "BDATE": {"true_positives": 0, "false_positives": 0, "false_negatives": 0},
    "AGE": {"true_positives": 0, "false_positives": 0, "false_negatives": 0},
    "DIDATE": {"true_positives": 0, "false_positives": 0, "false_negatives": 0},
    "ADATE": {"true_positives": 0, "false_positives": 0, "false_negatives": 0}
}

for sample in test_data:
    text = sample["text"]
    true_entities = {}
    for ent in sample["entities"]:
        entity_label = ent["label"]
        start, end = ent["start"], ent["end"]
        if entity_label not in true_entities:
            true_entities[entity_label] = set()
        true_entities[entity_label].add((start, end))

    
    doc = nlp(text)
    predicted_entities = {}
    for ent in doc.ents:
        entity_label = ent.label_
        start_char, end_char = ent.start_char, ent.end_char
        if entity_label not in predicted_entities:
            predicted_entities[entity_label] = set()
        predicted_entities[entity_label].add((start_char, end_char))

    # Calculate metrics for each entity label
    for entity_label in entity_metrics.keys():
        true_positives = len(true_entities.get(entity_label, set()) & predicted_entities.get(entity_label, set()))
        false_positives = len(predicted_entities.get(entity_label, set()) - true_entities.get(entity_label, set()))
        false_negatives = len(true_entities.get(entity_label, set()) - predicted_entities.get(entity_label, set()))

        
        entity_metrics[entity_label]["true_positives"] += true_positives
        entity_metrics[entity_label]["false_positives"] += false_positives
        entity_metrics[entity_label]["false_negatives"] += false_negatives

# Calculate and print metrics for each entity label
for entity_label, metrics in entity_metrics.items():
    true_positives = metrics["true_positives"]
    false_positives = metrics["false_positives"]
    false_negatives = metrics["false_negatives"]
    
    if true_positives + false_positives == 0:
        precision = 0
    else:
        precision = true_positives / (true_positives + false_positives)
    
    if true_positives + false_negatives == 0:
        recall = 0
    else:
        recall = true_positives / (true_positives + false_negatives)
    
    if precision + recall == 0:
        f1 = 0
    else:
        f1 = 2 * (precision * recall) / (precision + recall)
    
    print(f"Entity: {entity_label}")
    print(f"Precision: {precision:.4f}")
    print(f"Recall: {recall:.4f}")
    print(f"F1-score: {f1:.4f}")
    print("-------------------------------")

[HarounAbdelsamad]

config file:

[paths]
train = null
dev = null
vectors = null
init_tok2vec = null

[system]
gpu_allocator = "pytorch"
seed = 0

[nlp]
lang = "en"
pipeline = ["transformer","ner"]
batch_size = 128
disabled = []
before_creation = null
after_creation = null
after_pipeline_creation = null
tokenizer = {"@tokenizers":"spacy.Tokenizer.v1"}
vectors = {"@vectors":"spacy.Vectors.v1"}

[components]

[components.ner]
factory = "ner"
incorrect_spans_key = null
moves = null
scorer = {"@scorers":"spacy.ner_scorer.v1"}
update_with_oracle_cut_size = 100

[components.ner.model]
@architectures = "spacy.TransitionBasedParser.v2"
state_type = "ner"
extra_state_tokens = false
hidden_width = 64
maxout_pieces = 2
use_upper = false
nO = null

[components.ner.model.tok2vec]
@architectures = "spacy-transformers.TransformerListener.v1"
grad_factor = 1.0
pooling = {"@layers":"reduce_mean.v1"}
upstream = "*"

[components.transformer]
factory = "transformer"
max_batch_items = 4096
set_extra_annotations = {"@annotation_setters":"spacy-transformers.null_annotation_setter.v1"}

[components.transformer.model]
@architectures = "spacy-transformers.TransformerModel.v3"
name = "roberta-base"
mixed_precision = false

[components.transformer.model.get_spans]
@span_getters = "spacy-transformers.strided_spans.v1"
window = 128
stride = 96

[components.transformer.model.grad_scaler_config]

[components.transformer.model.tokenizer_config]
use_fast = true

[components.transformer.model.transformer_config]

[corpora]

[corpora.dev]
@readers = "spacy.Corpus.v1"
path = ${paths.dev}
max_length = 0
gold_preproc = false
limit = 0
augmenter = null

[corpora.train]
@readers = "spacy.Corpus.v1"
path = ${paths.train}
max_length = 0
gold_preproc = false
limit = 0
augmenter = null

[training]
accumulate_gradient = 3
dev_corpus = "corpora.dev"
train_corpus = "corpora.train"
seed = ${system.seed}
gpu_allocator = ${system.gpu_allocator}
dropout = 0.1
patience = 1600
max_epochs = 0
max_steps = 20000
eval_frequency = 200
frozen_components = []
annotating_components = []
before_to_disk = null
before_update = null

[training.batcher]
@batchers = "spacy.batch_by_padded.v1"
discard_oversize = true
size = 2000
buffer = 256
get_length = null

[training.logger]
@loggers = "spacy.ConsoleLogger.v1"
progress_bar = false

[training.optimizer]
@optimizers = "Adam.v1"
beta1 = 0.9
beta2 = 0.999
L2_is_weight_decay = true
L2 = 0.01
grad_clip = 1.0
use_averages = false
eps = 0.00000001

[training.optimizer.learn_rate]
@schedules = "warmup_linear.v1"
warmup_steps = 250
total_steps = 20000
initial_rate = 0.00005

[training.score_weights]
ents_f = 1.0
ents_p = 0.0
ents_r = 0.0
ents_per_type = null

[pretraining]

[initialize]
vectors = ${paths.vectors}
init_tok2vec = ${paths.init_tok2vec}
vocab_data = null
lookups = null
before_init = null
after_init = null

[initialize.components]

[initialize.tokenizer]

[rmitsch]

Nothing stands out in your config at first glance. The first think I recommend you to do is to switch to using spacy train instead of your hand-rolled script and spacy evaluate for evaluating your model, that might eliminate some sources of error. Have a look at this example project for reference and this blog post explaining how spaCy projects work.

[HarounAbdelsamad]

Thanks for the reply and the help, will definitely try it and hopefully it works