List index error

[kimiaGF]

I'm trying to train a model using a multi-system dataset with the following input script:

Click to view input script

{
  "model": {
    "type_map": [
      "B",
      "C"
    ],
    "descriptor": {
      "type": "se_e2_a",
      "sel": "auto:1.5",
      "rcut": 4.50,
      "rcut_smth": 6.00,
      "neuron": [
        50,
        100
      ],
      "axis_neuron": 16,
      "seed": 1
    },
    "fitting_net": {
      "type": "ener",
      "neuron": [
        240,
        240,
        240
      ],
      "activation_function": "relu",
      "seed": 1
    }
  },
  "learning_rate": {
    "type": "exp",
    "start_lr": 0.001,
    "stop_lr": 3.51e-8,
    "decay_steps": 5000
  },
  "loss": {
    "type": "ener",
    "start_pref_e": 0.2,
    "limit_pref_e": 0.2,
    "start_pref_f": 500,
    "limit_pref_f": 500,
    "start_pref_v": 0.1,
    "limit_pref_v": 0.1
  },
  "training": {
    "training_data": {
      "systems": "./training"
    },
    "validation_data": {
      "systems": "./validation"
    },
    "numb_steps": 100000,
    "seed": 1,
    "disp_file": "lcurve.out",
    "disp_freq": 100,
    "save_freq": 100,
    "save_ckpt": "model_ckpt",
    "disp_training": true,
    "profiling": true,
    "profiling_file": "profile",
    "tensorboard": true,
    "tensorboard_log_dir": "tensorboard_log_dir",
    "tensorboard_freq": 100
  }

And I'm getting: IndexError: list index out of range

I've ran into this issue before but somehow fixed it by moving files around and cannot replicate this solution in this case. My current folder tree is as follows:

Click to view directory structure

▾ training/                    
  ▾ 1_B88C24/                                                                                        
      box.raw                  
      coord.raw                
      energy.raw               
      force.raw                
      type.raw                 
      type_map.raw             
      virial.raw               
  ▾ 2_B96C24/                  
      box.raw                  
      coord.raw                
      energy.raw               
      force.raw                
      type.raw                 
      type_map.raw             
      virial.raw               
▾ validation/                  
  ▾ 1_B88C24/                  
      box.raw                  
      coord.raw                
      energy.raw               
      force.raw                
      type.raw                 
      type_map.raw             
      virial.raw               
  ▾ 2_B96C24/                  
      box.raw                  
      coord.raw                
      energy.raw               
      force.raw                
      type.raw                 
      type_map.raw             
      virial.raw               
  B4C_input.json               
  input.json                   
  out.json                     
  set_training_params.py  

[wanghan-iapcm]

DeePMD-kit does not use the raw box, coord, energy, force and virial fies, the raw files are for human reading. They must be compressed as numpy binary before dump. If you were using dpdata, please specify the output format as deepmd/npy

[kimiaGF]

Thank you! I modified the multi-system parsing script so that it takes in a proportion when splitting the dataset into training and testing. Here it is for anyone who ends up having the same issue:

from dpdata import LabeledSystem,MultiSystems
from glob import glob
import os
import shutil
from sklearn.model_selection import train_test_split
import numpy as np
import random 
import pandas as pd
#%%
#os.chdir('/Users/kimiaghaffari/Desktop/Research/B4C_ML_Potential/data/hydrostatic_strain_temp')

#%%
"""
process multi systems
"""
fs=glob('*/*/OUTCAR')  
ms=MultiSystems()
for f in fs:
    try:
        ls=LabeledSystem(f)
    except:
        print(f)
    if len(ls)>0:
        ms.append(ls)


ms.to_deepmd_npy('deepmd_npy')
#%%
#go into created numpy folder and read in set.000 data 
os.chdir('deepmd_npy')
#%%
#make final data location file for every system
for directory in ['training_data','validation_data']:
    for system in ms.systems.keys():
        path = directory + '/' + system
        
        #if not a directory, create one
        if not os.path.isdir(path):
            os.mkdir(path)
            
        #copy over type.raw and type_map.raw files
        if not os.path.isfile(path+'/type_map.raw'):
            shutil.copy('./'+system+'/type_map.raw',path)
        elif not not os.path.isfile(path+'/type.raw'):
            shutil.copy('./'+system+'/type.raw',path)     
#%%
df = {}
for system in ms.systems.keys():
    #find all files ending in .npy
    files = os.listdir(system+'/set.000/')
    path = os.path.join(os.getcwd(),system)+'/set.000/'
    #get number of frames from number of lines in file
    # n_frames = np.load(system+'/set.000/'+files[0]).sha pe[0]

    sys = {}
    #go through all '.npy' files
    for f in files:
        file = os.path.join(path,f)
        #make sure only npy files in folder
        if not file.endswith('.npy'):
            print('Remove non-npy file!')
            break
        
        #load in file
        with open(file):
            data = np.load(file)
        
        #get first component of filename (energy, force, etc.)
        index = f.split('.npy')[0]
        
        #populate system dictionary with files 
        sys[index] = data
    
    df[system] = sys
#%%
#function for finding all non-training indices in splitting 
def find_missing(lst,stop):
    return [x for x in range(1, stop) 
                               if x not in lst]
#%%
#INPUT TRAIN SPLIT HERE
train_split = 0.9
test_split = 1.0 - train_split

for sys in ms.systems.keys():
    #get number of frames from number of lines in file
    n_frames = ms.systems[sys].get_nframes()
    
    #number of training and validation frames
    n_train = int(train_split * n_frames)
    n_valid = n_frames - n_train
    
    #randomly choose training index based on split 
    train_frames = random.sample(range(0,n_frames),n_train)
    test_frames = find_missing(train_frames,n_frames)
    
    #definine training and test directories
    train_path = os.path.join(os.getcwd(),'training_data/',sys,'set.000')
    test_path = os.path.join(os.getcwd(),'validation_data/',sys,'set.000')
    
    
    for file_type in df[sys].keys():
        #define location of test and train files
        train_file = os.path.join(train_path,file_type+'.npy')
        test_file = os.path.join(test_path,file_type+'.npy')
        
        #extract train frames from files and save them to file
        train_data = df[sys][file_type][train_frames]
        np.save(train_file,train_data)
        
        #extract test frames from files and save them to file
        test_data = df[sys][file_type][test_frames]
        np.save(test_file,test_data)