Is training based on a model trained from compressed model available?

[Epikurz]

Dear DP developers:
I have compressed a model(G0) and trained once sucessfully based on G0 to get G1. Training based on G1 failed with both dp train -f and dp train -i mode. The error information of training with "-i" mode is as follows:

2021-12-02 14:40:48.493629: W tensorflow/core/framework/op_kernel.cc:1767] OP_REQUIRES failed at save_restore_v2_ops.cc:205 : Not found: Key descrpt_attr/t_avg not found in checkpoint
Traceback (most recent call last):
File "/home/user/deepmd-kit/lib/python3.9/site-packages/tensorflow/python/client/session.py", line 1375, in _do_call
return fn(*args)
File "/home/user/deepmd-kit/lib/python3.9/site-packages/tensorflow/python/client/session.py", line 1359, in _run_fn
return self._call_tf_sessionrun(options, feed_dict, fetch_list,
File "/home/user/deepmd-kit/lib/python3.9/site-packages/tensorflow/python/client/session.py", line 1451, in _call_tf_sessionrun
return tf_session.TF_SessionRun_wrapper(self._session, options, feed_dict,
tensorflow.python.framework.errors_impl.NotFoundError: 2 root error(s) found.
(0) Not found: Key descrpt_attr/t_avg not found in checkpoint
[[{{node save/RestoreV2}}]]
(1) Not found: Key descrpt_attr/t_avg not found in checkpoint
[[{{node save/RestoreV2}}]]
[[save/RestoreV2/_301]]
0 successful operations.
0 derived errors ignored.

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/home/user/deepmd-kit/lib/python3.9/site-packages/tensorflow/python/training/saver.py", line 1303, in restore
sess.run(self.saver_def.restore_op_name,
File "/home/user/deepmd-kit/lib/python3.9/site-packages/tensorflow/python/client/session.py", line 967, in run
result = self._run(None, fetches, feed_dict, options_ptr,
File "/home/user/deepmd-kit/lib/python3.9/site-packages/tensorflow/python/client/session.py", line 1190, in _run
results = self._do_run(handle, final_targets, final_fetches,
File "/home/user/deepmd-kit/lib/python3.9/site-packages/tensorflow/python/client/session.py", line 1368, in _do_run
return self._do_call(_run_fn, feeds, fetches, targets, options,
File "/home/user/deepmd-kit/lib/python3.9/site-packages/tensorflow/python/client/session.py", line 1394, in _do_call
raise type(e)(node_def, op, message)
tensorflow.python.framework.errors_impl.NotFoundError: 2 root error(s) found.
(0) Not found: Key descrpt_attr/t_avg not found in checkpoint
[[node save/RestoreV2 (defined at /lib/python3.9/site-packages/deepmd/train/trainer.py:407) ]]
(1) Not found: Key descrpt_attr/t_avg not found in checkpoint
[[node save/RestoreV2 (defined at /lib/python3.9/site-packages/deepmd/train/trainer.py:407) ]]
[[save/RestoreV2/_301]]
0 successful operations.
0 derived errors ignored.

Original stack trace for 'save/RestoreV2':
File "/bin/dp", line 10, in
sys.exit(main())
File "/lib/python3.9/site-packages/deepmd/entrypoints/main.py", line 437, in main
train_dp(**dict_args)
File "/lib/python3.9/site-packages/deepmd/entrypoints/train.py", line 102, in train
_do_work(jdata, run_opt, is_compress)
File "/lib/python3.9/site-packages/deepmd/entrypoints/train.py", line 163, in _do_work
model.train(train_data, valid_data)
File "/lib/python3.9/site-packages/deepmd/train/trainer.py", line 450, in train
self._init_session()
File "/lib/python3.9/site-packages/deepmd/train/trainer.py", line 407, in _init_session
self.saver = tf.train.Saver(save_relative_paths=True)
File "/lib/python3.9/site-packages/tensorflow/python/training/saver.py", line 836, in init
self.build()
File "/lib/python3.9/site-packages/tensorflow/python/training/saver.py", line 848, in build
self._build(self._filename, build_save=True, build_restore=True)
File "/lib/python3.9/site-packages/tensorflow/python/training/saver.py", line 876, in _build
self.saver_def = self._builder._build_internal( # pylint: disable=protected-access
File "/lib/python3.9/site-packages/tensorflow/python/training/saver.py", line 515, in _build_internal
restore_op = self._AddRestoreOps(filename_tensor, saveables,
File "/lib/python3.9/site-packages/tensorflow/python/training/saver.py", line 335, in _AddRestoreOps
all_tensors = self.bulk_restore(filename_tensor, saveables, preferred_shard,
File "/lib/python3.9/site-packages/tensorflow/python/training/saver.py", line 583, in bulk_restore
return io_ops.restore_v2(filename_tensor, names, slices, dtypes)
File "/lib/python3.9/site-packages/tensorflow/python/ops/gen_io_ops.py", line 1490, in restore_v2
_, _, _op, _outputs = _op_def_library._apply_op_helper(
File "/lib/python3.9/site-packages/tensorflow/python/framework/op_def_library.py", line 748, in _apply_op_helper
op = g._create_op_internal(op_type_name, inputs, dtypes=None,
File "/lib/python3.9/site-packages/tensorflow/python/framework/ops.py", line 3557, in _create_op_internal
ret = Operation(
File "/lib/python3.9/site-packages/tensorflow/python/framework/ops.py", line 2045, in init
self._traceback = tf_stack.extract_stack_for_node(self._c_op)

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/home/user/deepmd-kit/lib/python3.9/site-packages/tensorflow/python/training/py_checkpoint_reader.py", line 69, in get_tensor
return CheckpointReader.CheckpointReader_GetTensor(
RuntimeError: Key _CHECKPOINTABLE_OBJECT_GRAPH not found in checkpoint

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/home/user/deepmd-kit/lib/python3.9/site-packages/tensorflow/python/training/saver.py", line 1314, in restore
names_to_keys = object_graph_key_mapping(save_path)
File "/home/user/deepmd-kit/lib/python3.9/site-packages/tensorflow/python/training/saver.py", line 1632, in object_graph_key_mapping
object_graph_string = reader.get_tensor(trackable.OBJECT_GRAPH_PROTO_KEY)
File "/home/user/deepmd-kit/lib/python3.9/site-packages/tensorflow/python/training/py_checkpoint_reader.py", line 74, in get_tensor
error_translator(e)
File "/home/user/deepmd-kit/lib/python3.9/site-packages/tensorflow/python/training/py_checkpoint_reader.py", line 35, in error_translator
raise errors_impl.NotFoundError(None, None, error_message)
tensorflow.python.framework.errors_impl.NotFoundError: Key _CHECKPOINTABLE_OBJECT_GRAPH not found in checkpoint

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/home/user/deepmd-kit/bin/dp", line 10, in
sys.exit(main())
File "/home/user/deepmd-kit/lib/python3.9/site-packages/deepmd/entrypoints/main.py", line 437, in main
train_dp(**dict_args)
File "/home/user/deepmd-kit/lib/python3.9/site-packages/deepmd/entrypoints/train.py", line 102, in train
_do_work(jdata, run_opt, is_compress)
File "/home/user/deepmd-kit/lib/python3.9/site-packages/deepmd/entrypoints/train.py", line 163, in _do_work
model.train(train_data, valid_data)
File "/home/user/deepmd-kit/lib/python3.9/site-packages/deepmd/train/trainer.py", line 450, in train
self._init_session()
File "/home/user/deepmd-kit/lib/python3.9/site-packages/deepmd/train/trainer.py", line 416, in _init_session
self.saver.restore (self.sess, self.run_opt.init_model)
File "/home/user/deepmd-kit/lib/python3.9/site-packages/tensorflow/python/training/saver.py", line 1319, in restore
raise _wrap_restore_error_with_msg(
tensorflow.python.framework.errors_impl.NotFoundError: Restoring from checkpoint failed. This is most likely due to a Variable name or other graph key that is missing from the checkpoint. Please ensure that you have not altered the graph expected based on the checkpoint. Original error:

2 root error(s) found.
(0) Not found: Key descrpt_attr/t_avg not found in checkpoint
[[node save/RestoreV2 (defined at /lib/python3.9/site-packages/deepmd/train/trainer.py:407) ]]
(1) Not found: Key descrpt_attr/t_avg not found in checkpoint
[[node save/RestoreV2 (defined at /lib/python3.9/site-packages/deepmd/train/trainer.py:407) ]]
[[save/RestoreV2/_301]]
0 successful operations.
0 derived errors ignored.

Original stack trace for 'save/RestoreV2':
File "/bin/dp", line 10, in
sys.exit(main())
File "/lib/python3.9/site-packages/deepmd/entrypoints/main.py", line 437, in main
train_dp(**dict_args)
File "/lib/python3.9/site-packages/deepmd/entrypoints/train.py", line 102, in train
_do_work(jdata, run_opt, is_compress)
File "/lib/python3.9/site-packages/deepmd/entrypoints/train.py", line 163, in _do_work
model.train(train_data, valid_data)
File "/lib/python3.9/site-packages/deepmd/train/trainer.py", line 450, in train
self._init_session()
File "/lib/python3.9/site-packages/deepmd/train/trainer.py", line 407, in _init_session
self.saver = tf.train.Saver(save_relative_paths=True)
File "/lib/python3.9/site-packages/tensorflow/python/training/saver.py", line 836, in init
self.build()
File "/lib/python3.9/site-packages/tensorflow/python/training/saver.py", line 848, in build
self._build(self._filename, build_save=True, build_restore=True)
File "/lib/python3.9/site-packages/tensorflow/python/training/saver.py", line 876, in _build
self.saver_def = self._builder._build_internal( # pylint: disable=protected-access
File "/lib/python3.9/site-packages/tensorflow/python/training/saver.py", line 515, in _build_internal
restore_op = self._AddRestoreOps(filename_tensor, saveables,
File "/lib/python3.9/site-packages/tensorflow/python/training/saver.py", line 335, in _AddRestoreOps
all_tensors = self.bulk_restore(filename_tensor, saveables, preferred_shard,
File "/lib/python3.9/site-packages/tensorflow/python/training/saver.py", line 583, in bulk_restore
return io_ops.restore_v2(filename_tensor, names, slices, dtypes)
File "/lib/python3.9/site-packages/tensorflow/python/ops/gen_io_ops.py", line 1490, in restore_v2
_, _, _op, _outputs = _op_def_library._apply_op_helper(
File "/lib/python3.9/site-packages/tensorflow/python/framework/op_def_library.py", line 748, in _apply_op_helper
op = g._create_op_internal(op_type_name, inputs, dtypes=None,
File "/lib/python3.9/site-packages/tensorflow/python/framework/ops.py", line 3557, in _create_op_internal
ret = Operation(
File "/lib/python3.9/site-packages/tensorflow/python/framework/ops.py", line 2045, in init
self._traceback = tf_stack.extract_stack_for_node(self._c_op)

[njzjz]

Probably your model was trained by an old version, where this feature is unsupported?

[Epikurz]

Thanks. The problem was solved. 2.0.3 version is OK