Abnormal behaviour after removing an item

[ACodingBadger]

As the title says im currently experiencing issues after removing an item from my adapter which is bound to my recyclerview, im using Omega's swiping to show up a side menu which contains two buttons, edit and delete, the issue happens after "itemRemove" is called as far as I can tell.

--EDIT
After many testing runs I noticed that this behaviour only happens with the items that are above the removed item, for example, you have a list with 4 items, you remove number 2, the issue happens with the item number 1, but not with the 2nd and 3rd item

Even if I remove the animation this bug still happens, I have even tried setting my own ItemAnimator class but nothing changed...

--EDIT 2
I figured out that using setHasStableIds(false); removes the animations but fixes the issue, no idea what is causing this...

import android.view.View;
import android.view.ViewGroup;
import android.widget.TextView;

import androidx.annotation.NonNull;

import com.omega_r.libs.omegarecyclerview.OmegaRecyclerView;
import com.omega_r.libs.omegarecyclerview.swipe_menu.SwipeViewHolder;
import com.wmxapps.opsitron.Database.Note;
import com.wmxapps.opsitron.R;

import java.util.ArrayList;

public class NoteAdapter extends OmegaRecyclerView.Adapter<NoteAdapter.ViewHolder> {

    private ArrayList<Note> dataset;

    public OnNoteClickListener onNoteClickListener;

    public interface Action {
        int EDIT = 0;
        int DELETE = 1;
    }

    public interface OnNoteClickListener {
        void onClick(Note note, int position);
        void onActionClick(int action, int position);
    }

    public NoteAdapter(ArrayList<Note> dataset, OnNoteClickListener onNoteClickListener) {
        setHasStableIds(true);
        this.dataset = dataset;
        this.onNoteClickListener = onNoteClickListener;
    }

    @NonNull
    @Override
    public ViewHolder onCreateViewHolder(@NonNull ViewGroup viewGroup, int viewType) {
        return new ViewHolder(viewGroup);
    }

    @Override
    public void onBindViewHolder(@NonNull ViewHolder holder, final int position) {
        holder.updateView(dataset.get(position));
    }

    @Override
    public int getItemCount() {
        return (null != dataset ? dataset.size() : 0);
    }

    @Override
    public long getItemId(int position) {
        return dataset.get(position).getId();
    }

    public Note getItemByPosition(int pos) {
        return dataset.get(pos);
    }

    public class ViewHolder extends SwipeViewHolder implements View.OnClickListener {
        TextView tvTitle;

        public ViewHolder(ViewGroup parent) {
            super(parent, R.layout.item_note, NO_ID, R.layout.item_swipe_menu_right);

            tvTitle = findViewById(R.id.tv_title);

            findViewById(R.id.btnEdit).setOnClickListener(this);
            findViewById(R.id.btnDelete).setOnClickListener(this);

            contentView.setOnClickListener(new View.OnClickListener() {
                @Override
                public void onClick(View v) {
                    onNoteClickListener.onClick(dataset.get(getLayoutPosition()), getLayoutPosition());
                }
            });
        }

        void updateView(Note note) {
            tvTitle.setText(note.getTitle());
        }

        @Override
        public void onClick(View v) {
            if(v.getId() == R.id.btnEdit)
                onNoteClickListener.onActionClick(Action.EDIT, getLayoutPosition());
            if(v.getId() == R.id.btnDelete) {
                smoothCloseMenu(0);
                itemRemove(getLayoutPosition());
                //onNoteClickListener.onActionClick(Action.DELETE, getLayoutPosition());
            }
        }
    }

    public void itemRemove(int position) {
        if(position != -1 && position < dataset.size()) {
            dataset.remove(position);
            tryNotifyItemRemoved(position);
            //notifyItemRangeChanged(position, getItemCount());
        }
    }
}

This is how the recyclerview is declared on my fragment

oRecyclerView = rootView.findViewById(R.id.oRecyclerView);
        oRecyclerView.setItemAnimator(new CustomItemAnimator());
        oAdapter = new NoteAdapter(mData, new NoteAdapter.OnNoteClickListener() {
            @Override
            public void onClick(Note note, int position) {
                noteDisplay.setNote(note);
                noteDisplay.show();
            }

            @Override
            public void onActionClick(int action, int position) {
                if (action == NoteAdapter.Action.EDIT) {
                    form.setEntryMode(EntryMode.EDIT);
                    clickedNote = oAdapter.getItemByPosition(position);
                    form.doEdit(clickedNote);
                } else if (action == NoteAdapter.Action.DELETE) {
                    if(!task.isWorking())
                        task.doTask(new Tasker.Payload(Tasker.OperationType.NOTE_DELETE, oAdapter.getItemByPosition(position).getId(), position));
                }
            }
        });
        oRecyclerView.setAdapter(oAdapter);
        oRecyclerView.setLayoutManager(new LinearLayoutManager(getActivity()));

This is how its declared via XML

<com.omega_r.libs.omegarecyclerview.OmegaRecyclerView
            android:id="@+id/oRecyclerView"
            android:layout_width="match_parent"
            android:layout_height="match_parent"
            android:descendantFocusability="afterDescendants"
            android:clipToPadding="false"
            android:divider="#888"
            android:dividerHeight="1dp"
            android:clickable="false"
            android:focusable="false"
            android:focusableInTouchMode="false"
            android:paddingTop="10dp"
            android:paddingBottom="10dp"
            android:scrollbarStyle="outsideOverlay"
            app:dividerShow="none"
            app:itemSpace="10dp" />

And this is the error line I get on my logcat when I call ItemRemove:
E/View: isRecyclable decremented below 0: unmatched pair of setIsRecyable() calls for ViewHolder{52c9b80 position=1 id=5, oldPos=-1, pLpos:-1}