From 613d0325ef2565e83674d6ae9539e278b9bdabf8 Mon Sep 17 00:00:00 2001 From: Lu Fang Date: Tue, 21 Aug 2018 12:03:11 -0700 Subject: [PATCH 1/3] Add the changes --- caffe2/proto/torch.proto | 86 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 84 insertions(+), 2 deletions(-) diff --git a/caffe2/proto/torch.proto b/caffe2/proto/torch.proto index 527b16ff7ce26..de757ff94678d 100644 --- a/caffe2/proto/torch.proto +++ b/caffe2/proto/torch.proto @@ -8,7 +8,7 @@ syntax = "proto2"; -package onnx; +package torch; // Overview // @@ -74,7 +74,10 @@ enum Version { // - Added new message OperatorSetIdProto // - Added opset_import in ModelProto // - For vendor extensions, added domain in NodeProto - IR_VERSION = 0x0000000000000003; + IR_VERSION_NEWEST = 0x0000000000000003; + + // PYTORCH IR VERSION + IR_VERSION_NEWEST = 0x0000000100000003; } // Attributes @@ -174,6 +177,9 @@ message NodeProto { // A human-readable documentation for this node. Markdown is allowed. optional string doc_string = 6; + + // Additional annotations, attributes are defined in Schema + repeated AttributeProto annotation = 8; } // Models @@ -182,6 +188,9 @@ message NodeProto { // associating its computation graph with metadata. // // The semantics of the model are described by the associated GraphProto. +// +// Model ==> Caffe2 MetaNetDef +// ==> PyTorch Module message ModelProto { // The version of the IR this model targets. See Version enum above. // This field MUST be present. @@ -222,10 +231,28 @@ message ModelProto { optional string doc_string = 6; // The parameterized graph that is evaluated to execute the model. + // The main graph, in single graph case, it is ONNX compatible. optional GraphProto graph = 7; + // The remaining nets in MetaNetDef. + // Submodules and methods in PyTorch. + repeated GraphProto methods = 15; + // Named metadata values; keys should be distinct. + // Many meta data in MetaNetDef and preditor are piggy backed here. + // 1) project + // 2) model_class + // 3) internal_version + // 4) predictor_type + // 5) predictor_id + // 6) execute_plan + // 7) applicationSpecificInfo (another string map, need to verify it has no duplicate.) + // 8) engine + // 9) publish time repeated StringStringEntryProto metadata_props = 14; + + // Model name + optional string name = 16; }; // StringStringEntryProto follows the pattern for cross-proto-version maps. @@ -241,6 +268,8 @@ message StringStringEntryProto { // list of nodes that form a directed acyclic graph based on their inputs and outputs. // This is the equivalent of the "network" or "graph" in many deep learning // frameworks. +// Graph ==> NetDef in Caffe2 +// ==> Submodule/Method in PyTorch message GraphProto { // The nodes in the graph, sorted topologically. repeated NodeProto node = 1; @@ -264,6 +293,9 @@ message GraphProto { // must be distinct. It is optional for a value to appear in value_info list. repeated ValueInfoProto value_info = 13; + // Additional annotations. + repeated AttributeProto annotation = 14; + // DO NOT USE the following fields, they were deprecated from earlier versions. // repeated string input = 3; // repeated string output = 4; @@ -302,6 +334,7 @@ message TensorProto { // The shape of the tensor. repeated int64 dims = 1; + repeated int64 strides = 14; // The data type of the tensor. optional DataType data_type = 2; @@ -313,6 +346,7 @@ message TensorProto { optional int64 begin = 1; optional int64 end = 2; } + // Used as offset in the external shared data. optional Segment segment = 3; // Tensor content must be organized in row-major order. @@ -383,6 +417,12 @@ message TensorProto { // When this field is present, the data_type field MUST be // UINT32 or UINT64 repeated uint64 uint64_data = 11 [packed = true]; + + // External data by file name + optional string external_data = 13; + + // Device info + optional DeviceOption device_detail = 51; } // Defines a tensor shape. A dimension can be either an integer value @@ -444,3 +484,45 @@ message OperatorSetIdProto { // This field MUST be present in this version of the IR. optional int64 version = 2; } + +// DeviceType that Caffe2 currently supports. +// Note: if you add a device type, make sure you add the corresponding device +// line in the DeviceTypeName() function in caffe2/utils/proto_utils.cc +// and update ATen/core/DeviceType.h +enum DeviceType { + CPU = 0; // In default, we will use CPU. + CUDA = 1; // CUDA. + MKLDNN = 2; // Reserved for explicit MKLDNN + OPENGL = 3; // OpenGL + OPENCL = 4; // OpenCL + IDEEP = 5; // IDEEP. + HIP = 6; // AMD HIP + // Change the following number if you add more devices in the code. + COMPILE_TIME_MAX_DEVICE_TYPES = 7; + ONLY_FOR_TEST = 20901701; // This device type is only for test. +} + +// Device-specific options. We do not distinguish DeviceOption protos for +// different DeviceTypes, so currently all devices share the same DeviceOption +// proto. Fields that are specific to a device type is ignored if the type does +// not match. +// Note: if you add fields to the DeviceOption, make sure you add the +// corresponding changes to IsSameDevice() function in utils/proto_utils.{h,cc}. +message DeviceOption { + // [general] Options that need to be carried out before running the execution. + // optional DeviceType device_type = 1 [ default = CPU ]; + optional int32 device_type = 1 [ default = 0 ]; // 0 is CPU. + // [CUDA specific] the cuda gpu id. + optional int32 cuda_gpu_id = 2; + // [general] The random seed to start the device random number generator with. + optional uint32 random_seed = 3; + // [general] What node this op should execute on. + // Used for net transformation purposes. Must be empty at execution time. + optional string node_name = 4; + // [CPU and Linux specific] NUMA node id + optional int32 numa_node_id = 5 [default = -1]; + // [general] Extra information passed, not used at execution time currently. + repeated string extra_info = 6; + // [HIP specific] the hip gpu id. + optional int32 hip_gpu_id = 7; +} From 4dde3752aa1d7105ff104cef27ffc848dfaf2c5a Mon Sep 17 00:00:00 2001 From: Lu Fang Date: Thu, 23 Aug 2018 16:15:06 -0700 Subject: [PATCH 2/3] Add type support for list, map, and special blob --- caffe2/proto/torch.proto | 37 ++++++++++++++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/caffe2/proto/torch.proto b/caffe2/proto/torch.proto index de757ff94678d..be950340648ea 100644 --- a/caffe2/proto/torch.proto +++ b/caffe2/proto/torch.proto @@ -330,6 +330,10 @@ message TensorProto { COMPLEX64 = 14; // complex with float32 real and imaginary components COMPLEX128 = 15; // complex with float64 real and imaginary components // Future extensions go here. + + // Special data type, real type information is stored in ValueInfoProto. + // If data_type is SPECIAL, raw_data should be used. + SPECIAL = 51; } // The shape of the tensor. @@ -421,6 +425,11 @@ message TensorProto { // External data by file name optional string external_data = 13; + // If two tensors represent the same weights/content, use alias. + // Must exist a TensorProto named alias in the initializer list. + // To avoid the duplicate tensor in attribute, such as value in Constant node. + optional string alias = 16; + // Device info optional DeviceOption device_detail = 51; } @@ -441,6 +450,7 @@ message TensorShapeProto { // for pre-defined dimension denotations. optional string denotation = 3; }; + // To represent a scalar, using one element dim, and dim[0].dim_value = 0 repeated Dimension dim = 1; } @@ -456,11 +466,36 @@ message TypeProto { optional TensorShapeProto shape = 2; } + // Sequence type: List, Tuple + message Sequence { + // Type for each element. + optional TypeProto elem_type = 1; + enum SequenceType { + UNDEFINED = 0; + LIST = 1; + TUPLE = 2; + } + optional SequenceType sequence_type = 2; + } + + // Map, (not necessary at this moment) + message Map { + optional TensorProto.DataType key_type = 1; + optional TypeProto value_type = 2; + } + + // Special type of blobs, based on the type_name, we can choose the right + // serializer and deserialzier. + message SpecialBlob { + optional string type_name = 1; + } oneof value { // The type of a tensor. Tensor tensor_type = 1; - + Squence sequence_type = 4; + Map map_type = 5; + SpecialBlob = 51; } // An optional denotation can be used to denote the whole From 47f4912970bc30915f9feaa7e41a6ba69426420c Mon Sep 17 00:00:00 2001 From: Lu Fang Date: Tue, 28 Aug 2018 14:13:01 -0700 Subject: [PATCH 3/3] add the missing part --- caffe2/proto/torch.proto | 36 ++++++++++++++++++++++++++---------- 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/caffe2/proto/torch.proto b/caffe2/proto/torch.proto index be950340648ea..74bd79dcdcdf5 100644 --- a/caffe2/proto/torch.proto +++ b/caffe2/proto/torch.proto @@ -35,10 +35,11 @@ package torch; // by sharing our working version of ONNX. // // Protobuf compatibility -// -// To simplify framework compatibility, ONNX is defined using the subset of protobuf -// that is compatible with both protobuf v2 and v3. This means that we do not use any -// protobuf features that are only available in one of the two versions. +// +// To simplify framework compatibility, ONNX is defined using the subset of +// protobuf that is compatible with both protobuf v2 and v3. This means that we +// do not use any protobuf features that are only available in one of the two +// versions. // // Here are the most notable contortions we have to carry out to work around // these limitations: @@ -47,7 +48,6 @@ package torch; // of key-value pairs, where order does not matter and duplicates // are not allowed. - // Versioning // // ONNX versioning is specified in docs/IR.md and elaborated on in docs/Versioning.md @@ -60,8 +60,8 @@ enum Version { _START_VERSION = 0; // The version field is always serialized and we will use it to store the // version that the graph is generated from. This helps us set up version - // control. - // For the IR, we are using simple numbers starting with with 0x00000001, + // control. + // For the IR, we are using simple numbers starting with with 0x00000001, // which was the version we published on Oct 10, 2017. IR_VERSION_2017_10_10 = 0x0000000000000001; @@ -180,6 +180,9 @@ message NodeProto { // Additional annotations, attributes are defined in Schema repeated AttributeProto annotation = 8; + + // Node type, like PythonOp, etc, purely for PyTorch + optional string node_type = 51; } // Models @@ -428,10 +431,18 @@ message TensorProto { // If two tensors represent the same weights/content, use alias. // Must exist a TensorProto named alias in the initializer list. // To avoid the duplicate tensor in attribute, such as value in Constant node. + // This is useful, if everything is stored just in the proto. optional string alias = 16; + // Additional annotations. + repeated AttributeProto annotation = 17; + // Device info optional DeviceOption device_detail = 51; + + // For PyTorch serialized tensor. + optional int64 require_gradient = 52; + optional int64 is_bool = 53; } // Defines a tensor shape. A dimension can be either an integer value @@ -452,6 +463,8 @@ message TensorShapeProto { }; // To represent a scalar, using one element dim, and dim[0].dim_value = 0 repeated Dimension dim = 1; + + repeated Dimension stride = 51; } // Types @@ -468,14 +481,17 @@ message TypeProto { // Sequence type: List, Tuple message Sequence { - // Type for each element. + // elem_type and elem_type_list cannot appear together. + // If all the element types are the same, we use elem_type, + // otherwise, we specify the type of each element in elem_type_list. optional TypeProto elem_type = 1; + repeated TypeProto elem_type_list = 51; enum SequenceType { UNDEFINED = 0; LIST = 1; TUPLE = 2; } - optional SequenceType sequence_type = 2; + optional SequenceType sequence_type = 52; } // Map, (not necessary at this moment) @@ -493,7 +509,7 @@ message TypeProto { oneof value { // The type of a tensor. Tensor tensor_type = 1; - Squence sequence_type = 4; + Sequence sequence_type = 4; Map map_type = 5; SpecialBlob = 51; }