Graph Data

[ShobhaRajanna]

Hello Everyone! I Have one more question I've been working on a project involving a graph neural network and I've encountered a specific issue with the data representation. My dataset contains 155,640 data points. However, when I convert this dataset into a graph data object, the GNN seems to only consider unique nodes. As a result, the final graph representation is significantly reduced (Data(edge index=[2, 100], y=[1, 50, 2], node features=[10, 31, 2])). The data points represent different steps of the same entity, similar to having multiple data points for 'Kobe Bryant' with varying x and y values Do you have any suggestions on how to handle this situation and 50 and 30 are the future and history timestep

[wsad1]

However, when I convert this dataset into a graph data object, the GNN seems to only consider unique nodes. As a result, the final graph representation is significantly reduced (Data(edge index=[2, 100], y=[1, 50, 2], node features=[10, 31, 2])).

Could you please share a code snippet, because its not clear to we what GNN seems to only consider unique nodes means.

Further, your graph seems to a temporal graph, so you might want to check PyG Temporal

[ShobhaRajanna]

@wsad1 I attached the snippet of my code and thank you I'll look into PyG Temporal. unique nodes in the sense I have NBA Player Trajectories where i have 30 columns which is of the form Player_name x_0,y_0 till player_9 x_9 y_9 since in the name of player_0 it consider first name a node and take 31 history trajectory and 50 future trajectory and ignores rest.
`import numpy as np
import pandas as pd
import torch
from torch_geometric.data import Data
import logging
import os
import glob

class STP_Datapre():
logging.basicConfig(filename='gnn.log',filemode='w',level=logging.DEBUG, format='%(asctime)s %(message)s')

def __init__(self,parquet_path,t_h=30,t_f=50,d_s=2):
    print('working on path: {}'.format(parquet_path))
    self.parquet_path = parquet_path
    self.df = pd.read_parquet(parquet_path)
    self.playertraj_all = self.get_player_trajall()
    
def get_player_trajall(self):
    num_players = 10  
    player_columns = []
    for i in range(num_players):
        player_name_col = f'player_{i}'
        player_x_col = f'x_{i}'
        player_y_col = f'y_{i}'
        if player_name_col in self.df.columns and player_x_col in self.df.columns and player_y_col in self.df.columns:
            player_columns += [player_name_col, player_x_col, player_y_col]
    shared_columns = ['ball_x', 'ball_y', 'game_clock', 'shot_clock']
    existing_shared_columns = [col for col in shared_columns if col in self.df.columns]
    all_columns = player_columns + existing_shared_columns
    player_traj = self.df[all_columns]
    logging.debug('player_traj {}'.format(player_traj.shape))
    logging.debug('player_traj {}'.format(player_traj.head()))
    logging.debug('player_traj {}'.format(player_traj.columns))
    return player_traj

def get_hist(self, player_num, player_name, target_num, target_name, hist_len=30):

    player_x_col = f'x_{player_num}'
    player_y_col = f'y_{player_num}'
    target_x_col = f'x_{target_num}'
    target_y_col = f'y_{target_num}'
    player_data = self.playertraj_all[(self.playertraj_all[f'player_{player_num}'] == player_name)]
    target_data = self.playertraj_all[(self.playertraj_all[f'player_{target_num}'] == target_name)]
    
    logging.debug('length of player_data {}'.format(len(player_data)))
    logging.debug('length of target_data {}'.format(len(target_data)))

    if not player_data.empty and not target_data.empty:
        available_len = min(len(player_data), len(target_data))
        hist_len = min(hist_len, available_len)
        logging.debug('hist_len {}'.format(hist_len))
        logging.debug('available_len {}'.format(available_len))

        player_hist = player_data[[player_x_col, player_y_col]].tail(hist_len).values
        ref_pos = target_data[[target_x_col, target_y_col]].iloc[-1].values

        rel_positions = player_hist - ref_pos
        logging.debug('length of rel_positions {}'.format(len(rel_positions)))
        if len(rel_positions) < hist_len:
            logging.warning('Not enough historical data available')
            return np.empty([0, 2])
        logging.debug('rel_positions {}'.format(rel_positions))
        logging.debug('length of rel_positions {}'.format(len(rel_positions)))
        return rel_positions
    else:
        logging.error(f'Data for {player_name} or {target_name} not found.')
        return None

def get_future(self, player_num, player_name, target_num, target_name, future_len=30):
    logging.info(f'Getting future trajectories for {player_name} relative to {target_name}')

    player_x_col = f'x_{player_num}'
    player_y_col = f'y_{player_num}'
    target_x_col = f'x_{target_num}'
    target_y_col = f'y_{target_num}'

    player_data = self.playertraj_all[(self.playertraj_all[f'player_{player_num}'] == player_name)]
    target_data = self.playertraj_all[(self.playertraj_all[f'player_{target_num}'] == target_name)]

    if not player_data.empty and not target_data.empty:
        available_len = min(len(player_data), len(target_data))
        future_len = min(future_len, available_len)

        player_future = player_data.iloc[-future_len:][[player_x_col, player_y_col]].values
        target_future = target_data.iloc[-future_len:][[target_x_col, target_y_col]].values

        rel_future_positions = player_future - target_future

        if len(rel_future_positions) < future_len:
            logging.warning('Not enough future data available')
            return np.empty([0, 2])
        logging.debug('length of rel_future_positions {}'.format(len(rel_future_positions)))
        logging.debug('rel_future_positions {}'.format(rel_future_positions))
        return rel_future_positions
    else:
        logging.error(f'Data for {player_name} or {target_name} not found.')
        return None
    
'''def find_closest_time(self, query_time):
    available_times = self.playertraj_all['game_clock'].unique()
    closest_time = min(available_times, key=lambda x: abs(x - query_time))
    logging.debug('closest_time {}'.format(closest_time))
    return closest_time


def get_neighbors(self, target_num, target_name, game_clock_time, distance_threshold=15):
    closest_time = self.find_closest_time( game_clock_time)
    neighbors = []
    target_x_col = f'x_{target_num}'
    target_y_col = f'y_{target_num}'
    target_filter = (self.playertraj_all[f'player_{target_num}'] == target_name) & \
                    (self.playertraj_all['game_clock'] == closest_time)
    target_data = self.playertraj_all[target_filter]

    if target_data.empty:
        logging.error(f'Data for target player {target_name} not found at game clock time {game_clock_time}.')
        return neighbors

    target_pos = target_data.iloc[0][[target_x_col, target_y_col]].values
    for i in range(10):  
        if str(i) == target_num:
            continue  
        player_col = f'player_{i}'
        player_x_col = f'x_{i}'
        player_y_col = f'y_{i}'

        player_filter = (self.playertraj_all[player_col].notnull()) & \
                        (self.playertraj_all['game_clock'] == game_clock_time)
        player_data = self.playertraj_all[player_filter]

        if player_data.empty:
            continue  
        player_pos = player_data.iloc[0][[player_x_col, player_y_col]].values
        distance = np.linalg.norm(player_pos - target_pos)

        if distance <= distance_threshold:
            player_name = player_data.iloc[0][player_col]
            neighbors.append((i, player_name))  
    logging.debug('total neighbors {}'.format(len(neighbors)))
    return neighbors'''

    

def get_data_item(self, target_num, target_name):
    
    hist_all = np.zeros([0, 31, 2])  
    f1 = np.zeros([0, 50, 2])  

   
    target_data = self.playertraj_all[self.playertraj_all[f'player_{target_num}'] == target_name]
    if target_data.empty:
        return hist_all, f1  

    
    f = target_data.iloc[-50:][[f'x_{target_num}', f'y_{target_num}']].values
    if f.shape[0] < 50:
        return hist_all, np.empty([0, 50, 2])  
    f1 = np.vstack([f1, f[np.newaxis, :, :]])

    
    for i in range(10):  
        player_name_col = f'player_{i}'
        player_x_col = f'x_{i}'
        player_y_col = f'y_{i}'
        player_data = self.playertraj_all[[player_name_col, player_x_col, player_y_col]].dropna()
        h = player_data.iloc[-31:][[player_x_col, player_y_col]].values
        if h.shape[0] < 31:
            continue  
        hist_all = np.vstack([hist_all, h[np.newaxis, :, :]])
    logging.debug('hist_all {}'.format(hist_all.shape))
    logging.debug('f1 {}'.format(f1.shape))
    logging.debug('hist_all {}'.format(hist_all))
    logging.debug('f1 {}'.format(f1))
    return hist_all, f1

def data_topyg(self, hist_all, fut_gt):
    node_features = torch.from_numpy(hist_all).float()
    fut_gt = torch.from_numpy(fut_gt).float()

    num_nodes = hist_all.shape[0]
    edge_st_pt = torch.arange(num_nodes).unsqueeze(1)  
    edge_end_pt = torch.arange(num_nodes).unsqueeze(0)  

    edge_index = torch.cartesian_prod(edge_st_pt.squeeze(), edge_end_pt.squeeze())

    data = Data(node_features=node_features, edge_index=edge_index.t().contiguous(), y=fut_gt)

    logging.debug('data {}'.format(data))
    logging.debug('data.y {}'.format(data.y))
    logging.debug('data.node_features {}'.format(data.x))
    return data

def preprocess_data(self, target_num, target_name):
    print(f"Preprocessing data ")
    game_name = self.parquet_path.split('/')[-1].split('.')[0]
    processed_count = 0

    for player_num in range(10):  
        if player_num == target_num:
            continue  

        player_name = f"Player_{player_num}"
        
       
        hist_neighbors, future_target = self.get_data_item(target_num, target_name)
        if hist_neighbors.shape[0] == 0 or future_target.shape[0] == 0:
            continue

        pyg_data = self.data_topyg(hist_neighbors, future_target)
        logging.info(f'Processed data for {player_name} vs {target_name}: {pyg_data}')
    
        if pyg_data.node_features.shape[0] == 0 or pyg_data.y.shape[0] == 0:
            continue

        save_path = f'./pyg_data/{game_name}_allplayer_target{target_num}.pyg'
        logging.info(f'Saving PyG data to {save_path}')
        torch.save(pyg_data, save_path)
        logging.info(f'Saved PyG data to {save_path}')
        processed_count += 1

    logging.info(f'Processed {processed_count} data items in total')
    return processed_count

os.makedirs('./pyg_data', exist_ok=True)

if name == 'main':
os.makedirs('./pyg_data', exist_ok=True)
data_dir = '/data/beegfs/home/srajanna/data/preproceed_data'
parquet_path = glob.glob(data_dir + '/*.parquet')

target_num = 1  
target_name = 'Roy Hibbert'  
for parquet_path in parquet_path:
    stp_datapre = STP_Datapre(parquet_path=parquet_path)
    processed_files_count = stp_datapre.preprocess_data(target_num, target_name)
    print(f'Processed and saved {processed_files_count} files.')
print('Done')

`