cannot improve precision in lightgcn

[sorushsaghari]

hi. i am using light gcn on movilens dataset. i have used the lightgcn example. i see the loss is decrasing but my precision is not improving.
this is the code:

import torch
from tqdm import tqdm
from torch_geometric.data import  Data
from torch_geometric.nn import LightGCN
from torch_geometric.loader import LinkNeighborLoader
from torch_geometric.utils import degree
import pandas as pd
import json
from data import RecDataset

device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

def load_node_csv(path, index_col, n=0, encoders=None, **kwargs):
    df = pd.read_csv(path, **kwargs)
    mapping = {index: i + n for i, index in enumerate(df[index_col].unique())}

    x = None
    if encoders is not None:
        xs = [encoder(df[col]) for col, encoder in encoders.items()]
        x = torch.cat(xs, dim=-1)

    return x, mapping


def load_mapping(path: str) -> dict:
    with open(path, "r") as json_file:
        return {int(key): int(value) for key, value in json.load(json_file).items()}


def load_edge_csv(
    path,
    src_index_col,
    src_mapping,
    dst_index_col,
    dst_mapping,
    **kwargs,
):
    df = pd.read_csv(path, **kwargs)
    

    src = [src_mapping[index] for index in df[src_index_col]]
    dst = [dst_mapping[index] for index in df[dst_index_col]]
    weight = [w for w in (df['rating'] / 5)]
    edge_index = torch.tensor([src, dst]).to(device)

    return edge_index, weight


def genData():
    BASE = "ml-latest-small"

    movie_mapping = RecDataset.load_json("my-data/items.json")
    user_mapping = RecDataset.load_json("my-data/users.json")
    num_items = len(movie_mapping)
    num_users = len(user_mapping)
    print(user_mapping)
    print(movie_mapping)
    edge_index, weight = load_edge_csv(
        f"my-data/dataset.csv",
        src_index_col="userId",
        src_mapping=user_mapping,
        dst_index_col="movieId",
        dst_mapping=movie_mapping,
    )
    edge_label_index, _ = load_edge_csv(
        f"{BASE}/test_data.csv",
        src_index_col="userId",
        src_mapping=user_mapping,
        dst_index_col="movieId",
        dst_mapping=movie_mapping,
    )
    data = Data(x=None, edge_index=edge_index)
    data.edge_weight = torch.tensor(weight).to(device)
    data.num_users = num_users
    data.num_items = num_items

    print(data)
    return data


def get_test_data():
    df = pd.read_csv("ml-latest-small/test_data.csv", usecols=["uId", "mId", "rating"])
    df = df[df["rating"] >= 3]
    df["mId"] = df["mId"] + 610
    df = df.groupby("uId").agg(lambda x: x.tolist()).reset_index()
    df.drop("rating", axis=1, inplace=True)
    # print(df)
    return df


data = genData()
kwargs = {"batch_size": 512, "num_workers": 3, "persistent_workers": True}
loader = LinkNeighborLoader(data, num_neighbors=[10] * 2, shuffle=True, **kwargs)
num_users, num_items = data.num_users, data.num_items
print(data.num_nodes)
model = LightGCN(
    num_nodes=data.num_nodes,
    embedding_dim=64,
    num_layers=3,
).to(device)
optimizer = torch.optim.Adam(model.parameters(), lr=0.001)


# %%
def train():
    total_loss = total_examples = 0

    for x in tqdm(loader):
        # Sample positive and negative labels.
        # print(x)
        pos_edge_label_index = x.n_id.to(device)[x.edge_index]
        neg_edge_label_index = torch.stack(
            [
                pos_edge_label_index[0],
                torch.randint(
                    num_users,
                    num_users + num_items,
                    (pos_edge_label_index.shape[1],),
                    device=device,
                ),
            ],
            dim=0,
        )
        edge_label_index = torch.cat(
            [
                pos_edge_label_index,
                neg_edge_label_index,
            ],
            dim=1,
        )
        optimizer.zero_grad()
        pos_rank, neg_rank = model(data.edge_index, edge_label_index, edge_weight=data.edge_weight).chunk(2)

        loss = model.recommendation_loss(
            pos_rank,
            neg_rank,
            node_id=edge_label_index.unique(),
        )
        loss.backward()
        optimizer.step()

        total_loss += float(loss) * pos_rank.numel()
        total_examples += pos_rank.numel()

    return total_loss / total_examples

@torch.no_grad()
def my_test(test_data, do=False):
    test_data = test_data.sort_values(by='uId')
    results = model.recommend(
        data.edge_index,
        src_index=torch.tensor(test_data['uId'].tolist()),
        dst_index=torch.tensor(
            list(RecDataset.load_json("my-data/items.json").values()), dtype=torch.long
        ).to(device),
        k=20,
    )
    res = []
    for one, two in zip(results, test_data["mId"].tolist()):
        x = torch.isin(
            torch.tensor(one).to(device), torch.tensor(two).to(device)
        ).tolist()
        # print(x)
        res.append(x)
    res = torch.tensor(res)
    precision = res.sum(dim=-1) / 20

    return precision.sum()/precision.shape[0]


test_data = get_test_data()
for epoch in range(1, 20):
    loss = train()
    do = False
    if epoch % 10 == 0:
        do =True

    precision = my_test(test_data, do)
    recall = 0.0
    print(
        f"Epoch: {epoch:03d}, Loss: {loss:.4f}, Precision@20: "
        f"{precision:.4f}, Recall@20: {recall:.4f}"
    )

items ans user json is a user and item mapping

[rusty1s]

It's a bit hard to tell for me what might be the reasoning. Is your precision@20 staying flat for the whole time? What's the final precision you are seeing? Do you see the same behavior in our Light-GCN example?

[sorushsaghari]

yes it changes randomly between 7 and 8 percent. and i could not run your example beacuse it was bigger than my laptop memmory

[rusty1s]

Got it. As I said, it's hard for me to give a good answer why you might be seeing poor performance. In the end, the loss can go down while precision@20 stays flat (e.g., it might be super trivial to detect negatives while hard to move positives to the top-20).