How to get Spacy to Respect Bracket Tokens with Regex Match?

[grahamanderson]

The below is what I'm looking for...
How do I get Spacy to respect the regex pattern? In my case, Spacy is recognizing the '{' character as a separate token.
I'd simply like Spacy to respect the regex and create the span, which may/not include brackets or whatever text.
Also, my guess is that there is a simpler/more-elegant way to integrate the regex match into Spacy
I think I'm almost there with thanks to this discussion forum!
I think this might approach might best merge what I currently know Spacy with the pre-existing regex of my project.

import spacy
from spacy.language import Language
import re

@Language.component("regex_component")
def custom_component(doc):
    space = r'[ \t]+'
    extension = r'[ ]?\/[A-Z\/]{1,4}' # /D , /D/I , [ ]/JIG
    open_curly = r'\{{1,3}'
    close_curly = r'\}{1,3}'
    open_bracket = r'\[{1,3}'
    close_bracket = r'\]{1,3}'
    id_pattern = r'[A-Z]{1,4}'
    regex = [
        (
            # {{XP.NAJ THING}}PROY ,  {{XP.NAJ THING}} JIG ,
            r'(?P<text>'
                    fr'(?P<open>{open_curly})'           # 1-3 Open curly braces
                    r'(?P<component>[A-Z-0-9][^}]+)'     # COMPONENT THING
                    fr'(?P<close>{close_curly})'         # 1-3 Close curly braces
                    fr'([ ]?(?P<id>{id_pattern}))?'      # PROG or some ID number or Nothing
            r')'
            r'[ ]+'
       )
    ]
    
    # pattern = re.compile(regex)
    
    new_ents = []
    for r in regex:
        for match in re.finditer(r, doc.text):
            start_char, end_char = match.span()
            d = match.groupdict()
            if len(d['open']) != len(d['close']):   # do the open and close brackets match
                break

            # Convert character-based indices to token-based indices
            start_token = None
            end_token = None
            for token in doc:
                if start_token is None and token.idx >= start_char:
                    start_token = token.i
                if token.idx + len(token.text) <= end_char:
                    end_token = token.i

            if start_token is not None and end_token is not None:
                span = Span(doc, start_token, end_token + 1, label=d.get('id'))
                new_ents.append(span)

    doc.ents = tuple(list(doc.ents) + new_ents)
        
    return doc

nlp = spacy.blank("en")
nlp.add_pipe("regex_component")

doc = nlp("This is a {{Sample Text}}JIG example.")
for ent in doc.ents:
    print(ent.text, ent.label_)

for token in doc:
    print(token.text, token.ent_type_)

Incorrect Output (the brackets aren't merged)

This 
is 
a 
{ JIG
{ JIG
Sample JIG
Text}}JIG JIG
example 
. 

[grahamanderson]

I think I hacked it...any improvements from experts/non-beginners are welcome.
The idea is create modules that define entities (extracted by regex) before NER has its way with it

import spacy
from spacy.language import Language
from spacy.tokens import Span
import re
import pandas as pd


# Custom Pre-processor to expand abbreviations
@Language.component("expand_abbreviations")
def expand_abbreviations(doc):
    abbreviations = {
        "AI": "Artificial Intelligence",
        "ML": "Machine Learning",
        "BB": "Blue Bottle",
        "SF": "Siam Fallout"
    }
    for token in doc:
        if token.text in abbreviations:
            token._.expanded = abbreviations[token.text]
        else:
            token._.expanded = None #token.text
    return doc


# Add the acronyms and definitions detector here
# Coming soon

# Test that adding components is working
@Language.component("tech_term_detector")
def tech_term_detector(doc):
    terms = ["Artificial Intelligence", "Machine Learning", "Deep Learning", "Blue Bottle"]
    matcher = Matcher(nlp.vocab)
    for term in terms:
        pattern = [{"LOWER": word.lower()} for word in term.split()]
        matcher.add(term, [pattern])
    matches = matcher(doc)
    
    new_ents = [ent for ent in doc.ents]  # Initializing new_ents with current entities
    for match_id, start, end in matches:
        span = Span(doc, start, end, label="TECH_TERM")
        overlap = False
        for ent in new_ents:
            if span.start < ent.end and span.end > ent.start:
                overlap = True
                break
        if not overlap:
            new_ents.append(span)
            # Update the ent_type_ attribute of each token in the span
            for token in span:
                token.ent_type_ = "TECH_TERM"
            
    doc.ents = new_ents
    return doc

# Real component with single regex, instead of multiple
@Language.component("engineering_patterns")
def custom_component(doc):
    space = r'[ \t]+'
    extension = r'[ ]?\/[A-Z\/]{1,4}' # /D , /D/I , [ ]/JIG
    open_curly = r'\{{1,3}'
    close_curly = r'\}{1,3}'
    open_bracket = r'\[{1,3}'
    close_bracket = r'\]{1,3}'
    id_pattern = r'[A-Z]{1,4}'
    regex = [
        (
            # {{XP.NAJ THING}}PROY ,  {{XP.NAJ THING}} JIG ,
            r'(?P<text>'
                    fr'(?P<open>{open_curly})'           # 1-3 Open curly braces
                    r'(?P<component>[A-Z-0-9][^}]+)'     # COMPONENT THING
                    fr'(?P<close>{close_curly})'         # 1-3 Close curly braces
                    fr'([ ]?(?P<id>{id_pattern}))?'      # PROG or some ID number or Nothing
            r')'
            r'(\s|\.$)'                                # Space or Ends with Period
       )
    ]
    
    new_ents = []
    for r in regex:
        for match in re.finditer(r, doc.text):
            start_char, end_char = match.span()
            d = match.groupdict()
            d['balanced'] = len(d.get('open')) == len(d.get('open'))
            if not d['balanced']:
                break

            start_token = None
            end_token = None
            for token in doc:
                # Check if token starts within the matched span
                if start_token is None and start_char <= token.idx < end_char:
                    start_token = token.i
                # Check if token ends within the matched span
                if start_char < token.idx + len(token.text) <= end_char:
                    end_token = token.i

            if start_token is not None and end_token is not None:
                span = Span(doc, start_token, end_token + 1, label=d.get('id'))
                new_ents.append(span)

    doc.ents = tuple(list(doc.ents) + new_ents)
    
    with doc.retokenize() as retokenizer:
        for ent in doc.ents:
            # new rules for not merging
            retokenizer.merge(ent)
       
    return doc


def doc2df(doc, merge=True) -> pd.DataFrame:
    tokens = [{
        'text': token.text, 
        'tag_':token.tag_, 
        'pos_':token.pos_, 
        'dep_': token.dep_, 
        'ent_type_': token.ent_type_,
        'expanded': token._.expanded
    } for token in doc]
            
    return pd.DataFrame(tokens).T 


def load_nlp(existing_nlp=None):
    if existing_nlp is None:
        return spacy.load("en_core_web_sm")
    return existing_nlp


# Load
def debug():
    nlp = load_nlp()

    # Clear and Add Pipes
    for component in ["expand_abbreviations", "engineering_patterns", "tech_term_detector"]:
        if component in nlp.pipe_names:
            nlp.remove_pipe(component)      

    
    nlp.add_pipe('expand_abbreviations', before='ner')  # Placing it before NER
    nlp.add_pipe('tech_term_detector', after='expand_abbreviations')   # Placing tech term detector immediately after abbreviation expansion
    nlp.add_pipe('engineering_patterns', after='tech_term_detector')   # Placing it after tech term detector


    # Set Extensions
    Token.set_extension('expanded', default="", force=True)

    sentences = [
        ("I think AI is revolutionary. Blue Bottle and ML and Deep Learning are subsets of AI "
        "in accordance with {{SP.NAV Blah}}PROJ thing blah."), 

        "This is a {{Sample Text}}JIG example."
    ]
    doc = nlp(sentences[0])

    return doc2df(doc)

debug()