force error does not converge during training

[BaoWuustc]

I want to train carbon potential under high pressure and temperature, 4 AIMD runs (NPT: 0GP-300K,250GPa-2000K,500GPa-4000K,800GPa-8000K) are conducted to obtain the initial data for training. However, the RMS force error evaluated by training set can not converge during training (oscillate around 0.5 eV/A). When I use only 0GPa-300K data, the force eeror quickly converges to ~5e-2 eV/A. So is it reasonable? and how can i use all these data to train potential to get a convergent result.

following is the input.json file
{
"model": {
"_comment": " model parameters",
"type_map": [
"C"
],
"descriptor": {
"type": "se_ar",
"a": {
"sel": [
200
],
"rcut_smth": 4.5,
"rcut": 6.0,
"neuron": [
60,
120,
240
],
"resnet_dt": false,
"axis_neuron": 16,
"seed": 1,
"_comment": " that's all"
},
"r": {
"sel": [
200
],
"rcut_smth": 4.5,
"rcut": 6.0,
"neuron": [
25,
50,
100
],
"resnet_dt": false,
"seed": 1,
"_comment": " that's all"
},
"seed": 1367870059
},
"fitting_net": {
"neuron": [
240,
240,
240
],
"resnet_dt": true,
"seed": 2753404222
}
},
"learning_rate": {
"type": "exp",
"start_lr": 0.001,
"decay_rate": 0.95,
"decay_steps": 2000,
"_comment": "that's all"
},
"loss": {
"start_pref_e": 0.02,
"limit_pref_e": 1,
"start_pref_f": 1000,
"limit_pref_f": 1,
"start_pref_v": 0.0,
"limit_pref_v": 0.0
},
"training": {
"set_prefix": "set",
"stop_batch": 200000,
"batch_size": [
1,
1,
1,
1
],
"disp_file": "lcurve.out",
"disp_freq": 1000,
"numb_test": 4,
"save_freq": 1000,
"save_ckpt": "model.ckpt",
"load_ckpt": "model.ckpt",
"disp_training": true,
"time_training": true,
"profiling": false,
"profiling_file": "timeline.json",
"_comment": "that's all",
"systems": [
"../data.init/001",
"../data.init/002",
"../data.init/003",
"../data.init/004"
],
"seed": 1354050906
}
}

following is the lcurve.out file

batch l2_tst l2_trn l2_e_tst l2_e_trn l2_f_tst l2_f_trn lr

  0    2.23e+02  2.52e+02    7.70e+00  7.12e+00    7.03e+00  7.95e+00    1.0e-03

1000 8.91e+01 9.10e+01 2.93e+00 2.97e+00 2.81e+00 2.87e+00 1.0e-03
2000 1.79e+01 1.89e+01 3.05e+00 3.05e+00 2.71e-01 3.38e-01 8.9e-04
3000 2.76e+01 2.75e+01 7.80e-01 7.68e-01 9.15e-01 9.12e-01 8.9e-04
4000 4.53e+01 4.64e+01 2.84e+00 2.89e+00 1.45e+00 1.49e+00 7.9e-04
5000 2.16e+01 2.25e+01 8.41e-01 1.16e+00 7.38e-01 7.47e-01 7.9e-04
6000 7.10e+01 7.03e+01 1.28e+00 1.34e+00 2.64e+00 2.61e+00 7.1e-04
7000 2.26e+01 1.92e+01 3.04e-01 3.11e-01 8.46e-01 7.15e-01 7.1e-04
8000 6.11e+01 6.36e+01 1.18e-01 7.09e-02 2.43e+00 2.53e+00 6.3e-04
9000 1.18e+01 1.17e+01 2.56e-02 1.29e-02 4.68e-01 4.67e-01 6.3e-04
10000 2.42e+01 2.64e+01 2.74e-01 2.17e-01 1.01e+00 1.11e+00 5.6e-04
11000 5.78e+00 5.74e+00 3.90e-02 3.42e-02 2.43e-01 2.41e-01 5.6e-04
12000 4.99e+01 4.85e+01 3.34e-01 3.20e-01 2.22e+00 2.16e+00 5.0e-04
13000 4.69e+00 4.48e+00 2.13e-01 2.06e-01 1.84e-01 1.75e-01 5.0e-04
14000 1.46e+01 1.41e+01 1.38e-01 3.30e-02 6.86e-01 6.66e-01 4.5e-04
15000 8.59e+00 8.66e+00 9.01e-02 9.65e-02 4.03e-01 4.06e-01 4.5e-04
16000 3.97e+00 3.93e+00 6.84e-02 6.80e-02 1.95e-01 1.93e-01 4.0e-04
17000 6.14e+01 5.95e+01 1.37e+00 1.31e+00 2.97e+00 2.89e+00 4.0e-04
18000 1.59e+01 1.62e+01 6.08e-01 6.00e-01 7.49e-01 7.74e-01 3.5e-04
19000 8.02e+00 7.93e+00 5.85e-02 5.34e-02 4.24e-01 4.19e-01 3.5e-04
20000 1.43e+01 1.31e+01 1.01e-01 1.04e-01 7.98e-01 7.30e-01 3.2e-04
21000 1.20e+01 1.16e+01 2.48e-02 1.21e-02 6.75e-01 6.51e-01 3.2e-04
22000 3.35e+01 3.25e+01 5.85e-02 6.82e-02 1.99e+00 1.93e+00 2.8e-04
23000 1.50e+01 1.79e+01 3.09e-02 2.43e-03 8.90e-01 1.07e+00 2.8e-04
24000 3.18e+01 3.30e+01 4.12e-02 2.38e-02 2.00e+00 2.08e+00 2.5e-04
25000 3.16e+01 3.20e+01 1.66e-01 1.42e-01 1.98e+00 2.02e+00 2.5e-04
26000 5.76e+00 7.02e+00 6.81e-02 1.15e-02 3.80e-01 4.68e-01 2.2e-04
27000 2.91e+01 3.12e+01 3.61e-02 4.41e-02 1.94e+00 2.08e+00 2.2e-04
28000 2.75e+01 2.63e+01 5.51e-02 3.32e-02 1.94e+00 1.86e+00 2.0e-04
29000 9.73e+00 9.08e+00 2.31e-02 1.45e-02 6.87e-01 6.41e-01 2.0e-04
30000 1.17e+01 1.18e+01 5.24e-01 5.31e-01 7.01e-01 7.04e-01 1.8e-04
31000 1.09e+01 1.38e+01 3.91e-01 3.34e-01 7.15e-01 9.77e-01 1.8e-04
32000 4.72e+00 4.75e+00 3.42e-02 3.29e-02 3.72e-01 3.75e-01 1.6e-04
33000 9.15e+00 9.20e+00 3.16e-01 3.16e-01 6.41e-01 6.45e-01 1.6e-04
34000 2.27e+01 2.26e+01 2.94e-02 7.11e-03 1.91e+00 1.90e+00 1.4e-04
35000 9.49e+00 9.49e+00 3.63e-01 3.29e-01 6.79e-01 7.02e-01 1.4e-04
36000 2.12e+01 2.25e+01 2.25e-02 5.74e-02 1.88e+00 2.00e+00 1.3e-04
37000 7.43e+00 1.51e+01 1.40e-01 2.48e-01 6.37e-01 1.31e+00 1.3e-04
38000 6.66e+00 6.59e+00 1.59e-02 2.07e-02 6.26e-01 6.19e-01 1.1e-04
39000 1.99e+01 1.96e+01 3.05e-02 3.51e-04 1.87e+00 1.85e+00 1.1e-04
40000 2.06e+00 1.93e+00 5.14e-02 4.90e-02 1.92e-01 1.80e-01 1.0e-04
41000 1.87e+01 1.91e+01 7.62e-02 3.73e-02 1.86e+00 1.90e+00 1.0e-04
42000 1.09e+01 1.01e+01 6.50e-01 6.09e-01 6.20e-01 5.75e-01 8.9e-05
43000 6.15e+00 5.58e+00 1.22e-01 1.18e-01 6.23e-01 5.62e-01 8.9e-05
44000 3.40e+00 3.53e+00 5.62e-02 5.55e-02 3.69e-01 3.84e-01 7.9e-05
45000 5.79e+00 5.32e+00 4.48e-02 5.03e-04 6.42e-01 5.93e-01 7.9e-05
46000 3.15e+00 3.39e+00 2.67e-02 3.19e-02 3.69e-01 3.96e-01 7.1e-05
47000 3.19e+00 3.75e+00 5.96e-02 5.95e-02 3.63e-01 4.31e-01 7.1e-05
48000 1.47e+01 1.53e+01 2.31e-02 1.73e-02 1.84e+00 1.91e+00 6.3e-05
49000 2.94e+00 3.10e+00 2.20e-02 1.98e-02 3.65e-01 3.85e-01 6.3e-05
50000 1.38e+01 1.36e+01 3.66e-02 6.31e-02 1.82e+00 1.79e+00 5.6e-05
51000 2.17e+00 2.16e+00 1.28e-01 1.29e-01 1.53e-01 1.47e-01 5.6e-05
52000 1.30e+01 1.36e+01 2.46e-02 4.14e-02 1.81e+00 1.90e+00 5.0e-05
53000 2.84e+00 2.81e+00 1.76e-01 1.74e-01 1.82e-01 1.80e-01 5.0e-05
54000 4.16e+00 5.81e+00 6.87e-02 6.98e-02 5.99e-01 8.48e-01 4.5e-05
55000 2.38e+00 2.30e+00 7.13e-03 4.98e-03 3.51e-01 3.41e-01 4.5e-05
56000 1.36e+00 1.36e+00 7.00e-02 7.01e-02 1.44e-01 1.43e-01 4.0e-05
57000 1.75e+00 1.75e+00 1.03e-01 1.03e-01 1.45e-01 1.46e-01 4.0e-05
58000 3.90e+00 3.76e+00 1.09e-01 1.15e-01 5.91e-01 5.58e-01 3.5e-05
59000 1.14e+00 1.38e+00 5.11e-02 4.73e-02 1.44e-01 1.98e-01 3.5e-05
60000 2.18e+00 2.23e+00 4.39e-02 4.18e-02 3.66e-01 3.75e-01 3.2e-05
61000 3.43e+00 3.52e+00 3.89e-02 3.90e-02 5.93e-01 6.08e-01 3.2e-05
62000 9.74e+00 9.24e+00 3.96e-02 8.37e-02 1.80e+00 1.70e+00 2.8e-05
63000 9.75e+00 9.37e+00 2.94e-02 1.51e-02 1.80e+00 1.73e+00 2.8e-05
64000 2.15e+00 2.11e+00 7.73e-02 6.95e-02 3.60e-01 3.64e-01 2.5e-05
65000 1.81e+00 1.85e+00 1.03e-02 6.14e-03 3.53e-01 3.61e-01 2.5e-05
66000 1.06e+00 1.09e+00 5.71e-02 5.72e-02 1.35e-01 1.45e-01 2.2e-05
67000 2.35e+00 2.31e+00 1.11e-01 1.07e-01 3.54e-01 3.55e-01 2.2e-05
68000 6.41e-01 6.58e-01 9.69e-03 5.33e-03 1.37e-01 1.43e-01 2.0e-05
69000 2.90e+00 2.78e+00 7.65e-02 7.28e-02 5.85e-01 5.62e-01 2.0e-05
70000 7.86e+00 7.88e+00 7.07e-02 7.97e-02 1.80e+00 1.80e+00 1.8e-05
71000 6.33e-01 6.62e-01 1.51e-02 1.77e-02 1.37e-01 1.41e-01 1.8e-05
72000 1.52e+00 1.46e+00 3.33e-02 3.22e-02 3.52e-01 3.37e-01 1.6e-05
73000 2.39e+00 2.37e+00 1.46e-02 1.57e-02 5.79e-01 5.74e-01 1.6e-05
74000 2.39e+00 2.20e+00 5.11e-02 4.98e-02 5.83e-01 5.35e-01 1.4e-05
75000 2.27e+00 2.20e+00 1.48e-02 5.55e-03 5.82e-01 5.66e-01 1.4e-05
76000 1.32e+00 1.29e+00 2.16e-02 1.88e-02 3.48e-01 3.42e-01 1.3e-05
77000 2.16e+00 1.84e+00 2.21e-02 2.53e-03 5.80e-01 5.00e-01 1.3e-05
78000 6.30e+00 6.02e+00 3.20e-02 2.72e-02 1.80e+00 1.72e+00 1.1e-05
79000 6.29e+00 6.31e+00 2.41e-02 3.42e-02 1.80e+00 1.80e+00 1.1e-05
80000 4.41e-01 4.22e-01 3.95e-03 3.31e-03 1.32e-01 1.26e-01 1.0e-05

[Epikurz]

I have met similar problem. I have train a force field with datasets from AIMD at 1000 K, 2000 K, 4000 K, 6000 K and 8000 K. RMSE is ok when 6000 K and 8000 K datasets were excluded.

[BaoWuustc]

Thanks for your reply!

[njzjz]

Forces are usually larger under high pressure and temperature, so it may be expected to get a higher absolute error. You can consider checking relative errors.

[BaoWuustc]

Thanks for your reply!