diff --git a/src/frontends/onnx/frontend/include/openvino/frontend/onnx/decoder.hpp b/src/frontends/onnx/frontend/include/openvino/frontend/onnx/decoder.hpp new file mode 100644 index 00000000000000..a7b3fd0f2cac67 --- /dev/null +++ b/src/frontends/onnx/frontend/include/openvino/frontend/onnx/decoder.hpp @@ -0,0 +1,117 @@ +// Copyright (C) 2018-2025 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// +#pragma once + +#include "openvino/core/partial_shape.hpp" +#include "openvino/core/type/element_type.hpp" +#include "openvino/frontend/decoder.hpp" +#include "openvino/frontend/onnx/visibility.hpp" + +namespace ov { +namespace frontend { +namespace onnx { + +struct ONNX_FRONTEND_API TensorMetaInfo { + ov::PartialShape m_partial_shape; + ov::element::Type m_element_type; + const uint8_t* m_tensor_data; + size_t m_tensor_data_size; + const std::string* m_tensor_name; + std::shared_ptr m_external_location; + bool m_is_raw; +}; + +class ONNX_FRONTEND_API DecoderBase : public ov::frontend::DecoderBase { +public: + using Ptr = std::shared_ptr; + ~DecoderBase() override; +}; + +struct ONNX_FRONTEND_API SparseTensorInfo { + ov::PartialShape m_partial_shape; + ov::frontend::onnx::DecoderBase::Ptr m_values, m_indices; +}; + +// DecoderBaseOperation corresponds to operation node to retrieve its attributes and information about input and output +// tensors +class ONNX_FRONTEND_API DecoderBaseOperation : public ov::frontend::onnx::DecoderBase { +public: + /// \brief Get input tensor name by index + /// Operation nodes are connected between each other by tensors. + /// Each tensor must have unique name in a graph. + /// The tensor name uniqueness is provided by developer during GraphIterator construction. + /// This method returns tensor name that comes to this operation node by input index idx + /// If idx is out-of-range, it throws std::exception inherited exception + virtual const std::string& get_input_tensor_name(size_t idx) const = 0; + + /// \brief Get input tensor type by index + /// If idx is out-of-range, it throws std::exception inherited exception + virtual ov::element::Type get_input_tensor_type(size_t idx) const = 0; + + /// \brief Get output tensor name by index + /// Operation nodes are connected between each other by tensors. + /// Each tensor must have unique name in a graph. + /// The tensor name uniqueness is provided by developer during GraphIterator construction. + /// This method returns tensor name that outputs by output index idx from this operation + /// If idx is out-of-range, it throws std::exception inherited exception + virtual const std::string& get_output_tensor_name(size_t idx) const = 0; + + /// \brief Get output tensor type by index + /// If idx is out-of-range, it throws std::exception inherited exception + virtual ov::element::Type get_output_tensor_type(size_t idx) const = 0; + + /// \brief Get input tensor info + /// returns TensorInfo by input idx index that corresponds to a tensor + /// (it can correspond to Constant, Parameter or intermediate tensor connecting a producer and this current node) + /// If idx is out-of-range, it throws std::exception inherited exception + virtual const TensorMetaInfo& get_input_tensor_info(size_t idx) const = 0; + + /// \brief Get output tensor info + /// returns TensorInfo by output idx index that corresponds to a tensor + /// (it can correspond to intermediate tensor connecting this current node and a consumer) + /// If idx is out-of-range, it throws std::exception inherited exception + virtual const TensorMetaInfo& get_output_tensor_info(size_t idx) const = 0; + + /// \brief Get a number of outputs + virtual size_t get_output_size() const = 0; + + /// \brief Returns operation's opset version + virtual uint64_t get_op_set() const = 0; + + /// \brief Returns operation's domain + virtual const std::string& get_domain() const = 0; + + /// \brief Returns true if node has attribute + virtual bool has_attribute(const std::string& name) const = 0; + + virtual void experimental_get_internal_structures(const void** node_def) const = 0; + + ~DecoderBaseOperation() override; +}; + +// DecoderBaseTensor corresponds to tensor node to retrieve information about type, shapem quantization and sparsity +// information +class ONNX_FRONTEND_API DecoderBaseTensor : public ov::frontend::onnx::DecoderBase { +public: + /// \brief Get tensor info + virtual const TensorMetaInfo& get_tensor_info() const = 0; + + /// \brief Get input index for tensor + /// returns index of this input in the list of inputs in the model + /// it must be from 0 to n-1, where n - number of inputs in the model + /// if it is not input, returns -1 + virtual int64_t get_input_idx() const = 0; + + /// \brief Get output index for tensor + /// returns index of this output in the list of outputs in the model + /// it must be from 0 to m-1, where m - number of outputs in the model + /// if it is not input, returns -1 + virtual int64_t get_output_idx() const = 0; + + ~DecoderBaseTensor() override; +}; + +} // namespace onnx +} // namespace frontend +} // namespace ov diff --git a/src/frontends/onnx/frontend/include/openvino/frontend/onnx/frontend.hpp b/src/frontends/onnx/frontend/include/openvino/frontend/onnx/frontend.hpp index 427076a8497075..ee9ef0f7c90f70 100644 --- a/src/frontends/onnx/frontend/include/openvino/frontend/onnx/frontend.hpp +++ b/src/frontends/onnx/frontend/include/openvino/frontend/onnx/frontend.hpp @@ -18,17 +18,25 @@ namespace onnx { class ONNX_FRONTEND_API FrontEnd : public ov::frontend::FrontEnd { public: using Ptr = std::shared_ptr; - std::shared_ptr convert(const InputModel::Ptr& model) const override; + std::shared_ptr convert(const ov::frontend::InputModel::Ptr& model) const override; void convert(const std::shared_ptr& partially_converted) const override; - std::shared_ptr convert_partially(const InputModel::Ptr& input_model) const override; - std::shared_ptr decode(const InputModel::Ptr& input_model) const override; + std::shared_ptr convert_partially(const ov::frontend::InputModel::Ptr& input_model) const override; + std::shared_ptr decode(const ov::frontend::InputModel::Ptr& input_model) const override; std::string get_name() const override; bool supported_impl(const std::vector& variants) const override; void add_extension(const std::shared_ptr& extension) override; void normalize(const std::shared_ptr& model) const override; protected: - InputModel::Ptr load_impl(const std::vector& params) const override; + ov::frontend::InputModel::Ptr load_impl(const std::vector& params) const override; + + void translate_graph(const InputModel::Ptr& model, + bool fail_fast, + bool /* no_conversion */, // future use + std::shared_ptr& ov_model) const; + std::shared_ptr convert_unify(const InputModel::Ptr& model) const; + std::shared_ptr convert_partially_unify(const InputModel::Ptr& input_model) const; + std::shared_ptr decode_unify(const InputModel::Ptr& input_model) const; // m_other_extensions should be the first member here, // m_other_extensions can contain SO Extension (holder for other Extensions), diff --git a/src/frontends/onnx/frontend/include/openvino/frontend/onnx/graph_iterator.hpp b/src/frontends/onnx/frontend/include/openvino/frontend/onnx/graph_iterator.hpp new file mode 100644 index 00000000000000..9521dd7295da56 --- /dev/null +++ b/src/frontends/onnx/frontend/include/openvino/frontend/onnx/graph_iterator.hpp @@ -0,0 +1,56 @@ +// Copyright (C) 2018-2025 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include "openvino/core/runtime_attribute.hpp" +#include "openvino/frontend/onnx/decoder.hpp" +#include "openvino/frontend/onnx/visibility.hpp" + +namespace ov { +namespace frontend { +namespace onnx { + +/// Abstract representation for an input model graph that gives nodes in topologically sorted order +/// It returns decoders for model inputs and outputs (DecoderBaseTensor objects) and for operation nodes +/// (DecoderBaseOperation objects) DecoderBaseOperation objects for operation nodes must be sorted in topological order +/// from producing nodes to consumer nodes when `get_decoder()` is called. DecoderBaseTensor objects for inputs and +/// outputs must be returned first by `get_decoder()` method. Order of DecoderBaseTensor objects for inputs and outputs +/// defines their order in the original model, i.e. model input index and model output index. +/// For example, calling `get_decoder()` during iterating GraphIterator returns +/// DecoderBaseTensor (for input 0), ..., DecoderBaseTensor (for input n-1), +/// DecoderBaseTensor (for output 0), ..., DecoderBaseTensor (for output m-1), +/// DecoderBaseOperation (for op 1), ..., DecoderBaseOperation (for op k), +/// where n - number of inputs in the model, m - number of outputs in the model k - number of operation nodes. +/// NOTE: constants are ignored and no decoder object is returned for constant. +class ONNX_FRONTEND_API GraphIterator : ::ov::RuntimeAttribute { +public: + using Ptr = std::shared_ptr; + + /// \brief Get a number of operation nodes in the graph + virtual size_t size() const = 0; + + /// \brief Set iterator to the start position + virtual void reset() = 0; + + /// \brief Move to the next node in the graph + virtual void next() = 0; + + /// \brief Returns true if iterator goes out of the range of available nodes + virtual bool is_end() const = 0; + + /// \brief Return a pointer to a decoder of the current node + virtual std::shared_ptr get_decoder() const = 0; + + /// \brief Returns opset version of requested domain, stored in a ModelProto + /// If there are no domain found returns -1 + virtual int64_t get_opset_version(const std::string& domain) const = 0; + + /// \brief Destructor + virtual ~GraphIterator(); +}; + +} // namespace onnx +} // namespace frontend +} // namespace ov diff --git a/src/frontends/onnx/frontend/src/core/decoder.cpp b/src/frontends/onnx/frontend/src/core/decoder.cpp new file mode 100644 index 00000000000000..fd5cd3e8d5b99e --- /dev/null +++ b/src/frontends/onnx/frontend/src/core/decoder.cpp @@ -0,0 +1,13 @@ +// Copyright (C) 2018-2025 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "openvino/frontend/onnx/decoder.hpp" + +using namespace ov::frontend::onnx; + +DecoderBase::~DecoderBase() = default; + +DecoderBaseOperation::~DecoderBaseOperation() = default; + +DecoderBaseTensor::~DecoderBaseTensor() = default; diff --git a/src/frontends/onnx/frontend/src/core/decoder_proto.cpp b/src/frontends/onnx/frontend/src/core/decoder_proto.cpp new file mode 100644 index 00000000000000..96c0aa53f8c6ec --- /dev/null +++ b/src/frontends/onnx/frontend/src/core/decoder_proto.cpp @@ -0,0 +1,130 @@ +// Copyright (C) 2018-2025 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "decoder_proto.hpp" + +#include + +#include +#include + +#include "graph_iterator_proto.hpp" +#include "openvino/frontend/onnx/graph_iterator.hpp" +#include "openvino/util/wstring_convert_util.hpp" + +namespace ov { +namespace frontend { +namespace onnx { + +const std::string empty_name = ""; +const std::string DEFAULT_DOMAIN = ""; +const std::string EMPTY_NAME = ""; +const std::string EMPTY_OP_TYPE = ""; + +ov::Any DecoderProto::get_attribute(const std::string& name) const { + for (const auto& attr : m_node->attribute()) { + if (!attr.has_name() || attr.name() != name) + continue; + if (!attr.has_type()) { + throw std::runtime_error("Attribute \"" + name + "\" doesn't have a type"); + } + switch (attr.type()) { + case AttributeProto_AttributeType::AttributeProto_AttributeType_FLOAT: + if (attr.has_f()) + return attr.f(); + else + throw std::runtime_error("Attribute doesn't have value"); + break; + case AttributeProto_AttributeType::AttributeProto_AttributeType_FLOATS: + return std::vector{attr.floats().begin(), attr.floats().end()}; + case AttributeProto_AttributeType::AttributeProto_AttributeType_INT: + if (attr.has_i()) + return attr.i(); + else + throw std::runtime_error("Attribute doesn't have value"); + break; + case AttributeProto_AttributeType::AttributeProto_AttributeType_INTS: + return std::vector{attr.ints().begin(), attr.ints().end()}; + case AttributeProto_AttributeType::AttributeProto_AttributeType_STRING: + if (attr.has_s()) + return attr.s(); + else + throw std::runtime_error("Attribute doesn't have value"); + break; + case AttributeProto_AttributeType::AttributeProto_AttributeType_STRINGS: + return std::vector{attr.strings().begin(), attr.strings().end()}; + case AttributeProto_AttributeType::AttributeProto_AttributeType_GRAPH: + if (attr.has_g()) + return static_cast( + std::make_shared(m_parent, &attr.g())); + else + throw std::runtime_error("Attribute doesn't have value"); + break; + case AttributeProto_AttributeType::AttributeProto_AttributeType_TENSOR: + return static_cast( + std::make_shared(&attr.t(), m_parent, 0, 0)); + case AttributeProto_AttributeType::AttributeProto_AttributeType_SPARSE_TENSOR: { + ov::frontend::onnx::SparseTensorInfo sparse_tensor_info{}; + auto& sparse_tensor = attr.sparse_tensor(); + sparse_tensor_info.m_partial_shape = + ov::PartialShape{std::vector(sparse_tensor.dims().begin(), sparse_tensor.dims().end())}; + if (sparse_tensor.has_values()) { + sparse_tensor_info.m_values = static_cast( + std::make_shared(&sparse_tensor.values(), m_parent, 0, 0)); + } + if (sparse_tensor.has_indices()) { + sparse_tensor_info.m_indices = static_cast( + std::make_shared(&sparse_tensor.indices(), m_parent, 0, 0)); + } + return sparse_tensor_info; + } + default: + throw std::runtime_error("Unsupported attribute type " + + ::ONNX_NAMESPACE::AttributeProto_AttributeType_Name(attr.type())); + } + } + return nullptr; +} + +size_t DecoderProto::get_input_size() const { + return m_input_info.size(); +} + +size_t DecoderProto::get_output_size() const { + return m_output_info.size(); +} + +void DecoderProto::get_input_node(size_t input_port_idx, + std::string& producer_name, + std::string& producer_output_port_name, + size_t& producer_output_port_index) const {} + +const std::string& DecoderProto::get_op_type() const { + if (m_node->has_op_type()) { + return m_node->op_type(); + } else { + return EMPTY_OP_TYPE; + } +} + +const std::string& DecoderProto::get_op_name() const { + if (m_node->has_name()) { + return m_node->name(); + } else { + return EMPTY_NAME; + } +} + +bool DecoderProto::has_attribute(const std::string& name) const { + for (const auto& attr : m_node->attribute()) { + if (attr.has_name() && attr.name() == name) { + return true; + } + } + return false; +} + +} // namespace onnx +} // namespace frontend +} // namespace ov diff --git a/src/frontends/onnx/frontend/src/core/decoder_proto.hpp b/src/frontends/onnx/frontend/src/core/decoder_proto.hpp new file mode 100644 index 00000000000000..946a262e1ccb0a --- /dev/null +++ b/src/frontends/onnx/frontend/src/core/decoder_proto.hpp @@ -0,0 +1,181 @@ +// Copyright (C) 2018-2025 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// +#pragma once +#include + +#include + +#include "graph_iterator_proto.hpp" +#include "openvino/frontend/onnx/decoder.hpp" +#include "openvino/frontend/onnx/graph_iterator.hpp" +#include "openvino/util/wstring_convert_util.hpp" + +using ::ONNX_NAMESPACE::AttributeProto_AttributeType; +using ::ONNX_NAMESPACE::GraphProto; +using ::ONNX_NAMESPACE::ModelProto; +using ::ONNX_NAMESPACE::NodeProto; +using ::ONNX_NAMESPACE::OperatorSetIdProto; +using ::ONNX_NAMESPACE::TensorProto; +using ::ONNX_NAMESPACE::TensorProto_DataLocation; +using ::ONNX_NAMESPACE::TensorProto_DataType; +using ::ONNX_NAMESPACE::ValueInfoProto; +using ::ONNX_NAMESPACE::Version; + +namespace ov { +namespace frontend { +namespace onnx { + +ov::frontend::onnx::TensorMetaInfo extract_tensor_meta_info(const TensorProto* tensor_info, + const ValueInfoProto* value_info, + GraphIteratorProto* graph_iterator); + +extern const std::string empty_name; +extern const std::string DEFAULT_DOMAIN; +extern const std::string EMPTY_NAME; +extern const std::string EMPTY_OP_TYPE; + +class DecoderProtoTensor : public ov::frontend::onnx::DecoderBaseTensor { + ov::frontend::onnx::TensorMetaInfo m_tensor_meta_info; + int64_t m_input_idx, m_output_idx; + +public: + DecoderProtoTensor(const TensorProto* tensor_info, + GraphIteratorProto* parent, + const int64_t input_idx, + const int64_t output_idx) + // Probably, we may need to force it to 0/0 + : m_input_idx(input_idx), + m_output_idx(output_idx) { + m_tensor_meta_info = extract_tensor_meta_info(tensor_info, nullptr, parent); + } + DecoderProtoTensor(const ValueInfoProto* value_info, + GraphIteratorProto* parent, + const int64_t input_idx, + const int64_t output_idx) + : m_input_idx(input_idx), + m_output_idx(output_idx) { + m_tensor_meta_info = extract_tensor_meta_info(nullptr, value_info, parent); + } + DecoderProtoTensor(const std::string& name, + GraphIteratorProto* parent, + const int64_t input_idx, + const int64_t output_idx) + : m_input_idx(input_idx), + m_output_idx(output_idx) { + m_tensor_meta_info.m_tensor_name = &name; + m_tensor_meta_info.m_element_type = ov::element::dynamic; + m_tensor_meta_info.m_partial_shape = ov::PartialShape::dynamic(); + m_tensor_meta_info.m_tensor_data = nullptr; + m_tensor_meta_info.m_tensor_data_size = 0; + } + + const ov::frontend::onnx::TensorMetaInfo& get_tensor_info() const override { + return *const_cast(&m_tensor_meta_info); + } + + int64_t get_input_idx() const override { + return m_input_idx; + } + + int64_t get_output_idx() const override { + return m_output_idx; + } + + ov::Any get_attribute(const std::string& name) const override { + FRONT_END_NOT_IMPLEMENTED(get_attribute); + } + + size_t get_input_size() const override { + FRONT_END_NOT_IMPLEMENTED(get_input_size); + } + + void get_input_node(size_t input_port_idx, + std::string& producer_name, + std::string& producer_output_port_name, + size_t& producer_output_port_index) const override { + FRONT_END_NOT_IMPLEMENTED(get_input_node); + } + + const std::string& get_op_type() const override { + FRONT_END_NOT_IMPLEMENTED(get_op_type); + } + + const std::string& get_op_name() const override { + FRONT_END_NOT_IMPLEMENTED(get_op_name); + } +}; + +class DecoderProto : public ov::frontend::onnx::DecoderBaseOperation { + const NodeProto* m_node; + uint64_t m_opset; + GraphIteratorProto* m_parent; // For future use + // For existence of NodeDef object corresponding to the main graph node, + // GraphDef object must live in the memory + const GraphProto* m_graph; + std::vector m_input_info, m_output_info; + +public: + explicit DecoderProto(const NodeProto* node_def, + const uint64_t opset, + GraphIteratorProto* parent, + const std::vector& input_info, + const std::vector& output_info) + : m_node(node_def), + m_opset(opset), + m_parent(parent), + m_graph(parent->get_graph()), + m_input_info(input_info), + m_output_info(output_info) {} + + size_t get_input_size() const override; + size_t get_output_size() const override; + + const std::string& get_input_tensor_name(size_t idx) const override { + return *m_input_info.at(idx)->m_tensor_name; + } + ov::element::Type get_input_tensor_type(size_t idx) const override { + return m_input_info.at(idx)->m_element_type; + } + const std::string& get_output_tensor_name(size_t idx) const override { + return *m_output_info.at(idx)->m_tensor_name; + } + ov::element::Type get_output_tensor_type(size_t idx) const override { + return m_output_info.at(idx)->m_element_type; + } + const ov::frontend::onnx::TensorMetaInfo& get_input_tensor_info(size_t idx) const override { + return *m_input_info.at(idx); + } + const ov::frontend::onnx::TensorMetaInfo& get_output_tensor_info(size_t idx) const override { + return *m_output_info.at(idx); + } + + ov::Any get_attribute(const std::string& name) const override; + + void get_input_node(size_t input_port_idx, + std::string& producer_name, + std::string& producer_output_port_name, + size_t& producer_output_port_index) const override; + + const std::string& get_op_type() const override; + + const std::string& get_op_name() const override; + + uint64_t get_op_set() const override { + return m_opset; + } + + const std::string& get_domain() const override { + return (m_node->has_domain() && m_node->domain() != "ai.onnx" ? m_node->domain() : DEFAULT_DOMAIN); + } + + bool has_attribute(const std::string& name) const override; + + void experimental_get_internal_structures(const void** node_def) const override { + *node_def = m_node; + } +}; + +} // namespace onnx +} // namespace frontend +} // namespace ov diff --git a/src/frontends/onnx/frontend/src/core/graph_iterator.cpp b/src/frontends/onnx/frontend/src/core/graph_iterator.cpp new file mode 100644 index 00000000000000..7fd89452c40e3e --- /dev/null +++ b/src/frontends/onnx/frontend/src/core/graph_iterator.cpp @@ -0,0 +1,9 @@ +// Copyright (C) 2018-2025 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "openvino/frontend/onnx/graph_iterator.hpp" + +using namespace ov::frontend::onnx; + +GraphIterator::~GraphIterator() = default; diff --git a/src/frontends/onnx/frontend/src/core/graph_iterator_proto.cpp b/src/frontends/onnx/frontend/src/core/graph_iterator_proto.cpp new file mode 100644 index 00000000000000..a8a457b631573a --- /dev/null +++ b/src/frontends/onnx/frontend/src/core/graph_iterator_proto.cpp @@ -0,0 +1,672 @@ +// Copyright (C) 2018-2025 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "graph_iterator_proto.hpp" + +#include + +#include +#include + +#include "decoder_proto.hpp" +#include "openvino/frontend/onnx/graph_iterator.hpp" +#include "openvino/util/file_util.hpp" +#include "openvino/util/wstring_convert_util.hpp" + +namespace { +// THis is copied from utils/common.hpp +const ov::element::Type& get_ov_element_type(int64_t onnx_type) { + switch (onnx_type) { + case TensorProto_DataType::TensorProto_DataType_BOOL: + return ov::element::boolean; + case TensorProto_DataType::TensorProto_DataType_DOUBLE: + return ov::element::f64; + case TensorProto_DataType::TensorProto_DataType_FLOAT16: + return ov::element::f16; + case TensorProto_DataType::TensorProto_DataType_FLOAT: + return ov::element::f32; + case TensorProto_DataType::TensorProto_DataType_INT4: + return ov::element::i4; + case TensorProto_DataType::TensorProto_DataType_INT8: + return ov::element::i8; + case TensorProto_DataType::TensorProto_DataType_INT16: + return ov::element::i16; + case TensorProto_DataType::TensorProto_DataType_INT32: + return ov::element::i32; + case TensorProto_DataType::TensorProto_DataType_INT64: + return ov::element::i64; + case TensorProto_DataType::TensorProto_DataType_UINT4: + return ov::element::u4; + case TensorProto_DataType::TensorProto_DataType_UINT8: + return ov::element::u8; + case TensorProto_DataType::TensorProto_DataType_UINT16: + return ov::element::u16; + case TensorProto_DataType::TensorProto_DataType_UINT32: + return ov::element::u32; + case TensorProto_DataType::TensorProto_DataType_UINT64: + return ov::element::u64; + case TensorProto_DataType::TensorProto_DataType_UNDEFINED: + return ov::element::dynamic; + case TensorProto_DataType::TensorProto_DataType_BFLOAT16: + return ov::element::bf16; + case TensorProto_DataType::TensorProto_DataType_FLOAT8E4M3FN: + return ov::element::f8e4m3; + case TensorProto_DataType::TensorProto_DataType_FLOAT8E5M2: + return ov::element::f8e5m2; + case TensorProto_DataType::TensorProto_DataType_STRING: + return ov::element::string; + } + throw std::runtime_error("Unsupported type"); +} +} // namespace + +namespace ov { +namespace frontend { +namespace onnx { + +namespace { +bool extract_tensor_external_data(ov::frontend::onnx::TensorMetaInfo& tensor_meta_info, + const TensorProto* tensor_info, + GraphIteratorProto* graph_iterator) { + std::string ext_location{}; + uint64_t ext_data_offset = 0; + uint64_t ext_data_length = 0; + std::string m_sha1_digest{}; // for future use + for (const auto& entry : tensor_info->external_data()) { + if (entry.key() == "location") { + ext_location = ov::util::sanitize_path(entry.value()); + } else if (entry.key() == "offset") { + ext_data_offset = std::stoull(entry.value()); + } else if (entry.key() == "length") { + ext_data_length = std::stoull(entry.value()); + } else if (entry.key() == "checksum") { + m_sha1_digest = entry.value(); + } + } + const auto full_path = + ov::util::get_absolute_file_path(ov::util::path_join({graph_iterator->get_model_dir(), ext_location}).string()); + const int64_t file_size = ov::util::file_size(full_path); + if (file_size <= 0 || ext_data_offset + ext_data_length > static_cast(file_size)) { + // not_existed_file.data, offset: 4096, data_length: 16) + std::stringstream ss; + ss << "Invalid usage of method for externally stored data in file (" << ext_location; + ss << ", offset: " << ext_data_offset << ", data_length: " << ext_data_length << ")"; + throw std::runtime_error(ss.str()); + } + auto memory_mode = graph_iterator->get_memory_management_mode(); + if (ext_location == "*/_ORT_MEM_ADDR_/*") { + // Specific ONNX Runtime Case when it passes a model with self-managed data + tensor_meta_info.m_is_raw = true; + tensor_meta_info.m_tensor_data = reinterpret_cast(ext_data_offset); + tensor_meta_info.m_tensor_data_size = ext_data_length; + return true; + } else if (memory_mode == External_MMAP) { + auto cache = graph_iterator->get_mmap_cache(); + auto cached_mapped_memory = cache->find(full_path); + std::shared_ptr mapped_memory; + if (cached_mapped_memory != cache->end()) { + mapped_memory = cached_mapped_memory->second; + } else { + mapped_memory = ov::load_mmap_object(full_path); + (*cache)[full_path] = mapped_memory; + } + tensor_meta_info.m_is_raw = true; + tensor_meta_info.m_tensor_data = + static_cast(static_cast(mapped_memory->data() + ext_data_offset)); + tensor_meta_info.m_tensor_data_size = + ext_data_length > 0 ? ext_data_length : static_cast(file_size) - ext_data_length; + return true; + } else if (memory_mode == External_Stream) { + auto cache = graph_iterator->get_stream_cache(); + auto cached_stream = cache->find(full_path); + std::shared_ptr external_data_stream; + if (cached_stream != cache->end()) { + external_data_stream = cached_stream->second; + } else { + external_data_stream = { + new std::ifstream(full_path.c_str(), std::ios::binary | std::ios::in | std::ios::ate), + [](std::ifstream* p) { + p->close(); + delete p; + }}; + (*cache)[full_path] = external_data_stream; + } + + if (external_data_stream->fail() || !external_data_stream->good()) { + throw std::runtime_error("Failed to open external data stream"); + } + + tensor_meta_info.m_is_raw = true; + tensor_meta_info.m_tensor_data_size = + ext_data_length > 0 ? ext_data_length : static_cast(file_size) - ext_data_length; + uint8_t* data_ptr = graph_iterator->allocate_data(tensor_meta_info.m_tensor_data_size).get(); + tensor_meta_info.m_tensor_data = data_ptr; + + // default value of m_offset is 0 + external_data_stream->seekg(ext_data_offset, std::ios::beg); + + external_data_stream->read(static_cast(static_cast(data_ptr)), + tensor_meta_info.m_tensor_data_size); + return true; + } else if (memory_mode == Internal_MMAP || memory_mode == Internal_Stream) { + tensor_meta_info.m_external_location = std::make_shared(full_path); + tensor_meta_info.m_tensor_data = reinterpret_cast(ext_data_offset); + tensor_meta_info.m_tensor_data_size = ext_data_length; + return true; + } else { + throw std::runtime_error("Unsupported memory management mode"); + } +} +} // namespace + +ov::frontend::onnx::TensorMetaInfo extract_tensor_meta_info(const TensorProto* tensor_info, + const ValueInfoProto* value_info, + GraphIteratorProto* graph_iterator) { + auto graph_def = graph_iterator->get_graph(); + ov::frontend::onnx::TensorMetaInfo tensor_meta_info{}; + tensor_meta_info.m_external_location = nullptr; + tensor_meta_info.m_is_raw = false; + if ((tensor_info == nullptr && value_info == nullptr) || graph_def == nullptr) { + throw std::runtime_error("Wrong usage"); + } + + if (value_info == nullptr && tensor_info->has_name()) { + for (const auto& val : graph_def->value_info()) { + if (val.has_name() && val.name() == tensor_info->name()) { + value_info = &val; + break; + } + } + } + if (value_info != nullptr) { + if (value_info->has_type() && !value_info->type().has_tensor_type()) { + throw std::runtime_error("Unsupported value_info type"); + } + tensor_meta_info.m_tensor_name = value_info->has_name() ? &value_info->name() : &empty_name; + const auto& value_type = value_info->type().tensor_type(); + if (value_type.has_shape()) { + std::vector dims{}; + for (const auto& dim : value_type.shape().dim()) { + if (dim.has_dim_value()) { + dims.push_back(dim.dim_value()); + } else { + dims.push_back(-1); + } + } + tensor_meta_info.m_partial_shape = ov::PartialShape{dims}; + } else { + tensor_meta_info.m_partial_shape = ov::PartialShape::dynamic(); + } + if (value_type.has_elem_type()) { + tensor_meta_info.m_element_type = get_ov_element_type(value_type.elem_type()); + } else { + tensor_meta_info.m_element_type = ov::element::dynamic; + } + } + if (tensor_info != nullptr) { + tensor_meta_info.m_tensor_name = tensor_info->has_name() ? &tensor_info->name() : &empty_name; + std::vector dims(tensor_info->dims().begin(), tensor_info->dims().end()); + if (dims.size() == 0 || (dims.size() == 1 && dims[0] == 0)) { + tensor_meta_info.m_partial_shape = ov::PartialShape{}; + } else { + tensor_meta_info.m_partial_shape = ov::PartialShape{dims}; + } + tensor_meta_info.m_element_type = + tensor_info->has_data_type() ? get_ov_element_type(tensor_info->data_type()) : ov::element::dynamic; + if (tensor_info->has_data_location() && + tensor_info->data_location() == TensorProto_DataLocation::TensorProto_DataLocation_EXTERNAL) { + if (extract_tensor_external_data(tensor_meta_info, tensor_info, graph_iterator)) { + return tensor_meta_info; + } + throw std::runtime_error("Unsupported method for externally stored data"); + } + switch (tensor_info->data_type()) { + case TensorProto_DataType::TensorProto_DataType_FLOAT: + tensor_meta_info.m_tensor_data = + static_cast(static_cast(tensor_info->float_data().data())); + tensor_meta_info.m_tensor_data_size = tensor_info->float_data_size(); + break; + case TensorProto_DataType::TensorProto_DataType_INT4: + case TensorProto_DataType::TensorProto_DataType_INT8: + case TensorProto_DataType::TensorProto_DataType_INT16: + case TensorProto_DataType::TensorProto_DataType_INT32: + case TensorProto_DataType::TensorProto_DataType_UINT4: + case TensorProto_DataType::TensorProto_DataType_UINT8: + case TensorProto_DataType::TensorProto_DataType_UINT16: + case TensorProto_DataType::TensorProto_DataType_BOOL: + case TensorProto_DataType::TensorProto_DataType_BFLOAT16: + case TensorProto_DataType::TensorProto_DataType_FLOAT16: + case TensorProto_DataType::TensorProto_DataType_FLOAT8E4M3FN: + case TensorProto_DataType::TensorProto_DataType_FLOAT8E5M2: + tensor_meta_info.m_tensor_data = + static_cast(static_cast(tensor_info->int32_data().data())); + tensor_meta_info.m_tensor_data_size = tensor_info->int32_data_size(); + break; + case TensorProto_DataType::TensorProto_DataType_INT64: + tensor_meta_info.m_tensor_data = + static_cast(static_cast(tensor_info->int64_data().data())); + tensor_meta_info.m_tensor_data_size = tensor_info->int64_data_size(); + break; + case TensorProto_DataType::TensorProto_DataType_UINT32: + case TensorProto_DataType::TensorProto_DataType_UINT64: + tensor_meta_info.m_tensor_data = + static_cast(static_cast(tensor_info->uint64_data().data())); + tensor_meta_info.m_tensor_data_size = tensor_info->uint64_data_size(); + break; + case TensorProto_DataType::TensorProto_DataType_DOUBLE: + tensor_meta_info.m_tensor_data = + static_cast(static_cast(tensor_info->double_data().data())); + tensor_meta_info.m_tensor_data_size = tensor_info->double_data_size(); + break; + default: + throw std::runtime_error("Unsupported type " + + ::ONNX_NAMESPACE::TensorProto_DataType_Name(tensor_info->data_type())); + break; + } + // Looks like raw_data has bigger priority. but not 100% sure + if (tensor_meta_info.m_tensor_data == nullptr && tensor_info->has_raw_data()) { + tensor_meta_info.m_tensor_data = + static_cast(static_cast(tensor_info->raw_data().data())); + tensor_meta_info.m_tensor_data_size = tensor_info->raw_data().size(); + tensor_meta_info.m_is_raw = true; + } + } + if (tensor_meta_info.m_tensor_name == nullptr) { + tensor_meta_info.m_tensor_name = &empty_name; + } + return tensor_meta_info; +} + +GraphIteratorProto::GraphIteratorProto(const GraphIteratorProtoMemoryManagementMode mode) + : m_graph(nullptr), + m_parent(nullptr), + m_model_dir(nullptr), + m_mode(mode), + m_mmap_cache{mode == External_MMAP ? std::make_shared>>() + : nullptr}, + m_stream_cache{mode == External_Stream ? std::make_shared>>() + : nullptr}, + m_data_holder{mode == External_Stream ? std::make_shared>>() : nullptr} {} + +GraphIteratorProto::GraphIteratorProto(GraphIteratorProto* parent, const GraphProto* graph_def) { + m_graph = graph_def; + m_parent = parent; + m_model_dir = parent->m_model_dir; + m_mode = parent->m_mode; + m_mmap_cache = parent->m_mmap_cache; + m_stream_cache = parent->m_stream_cache; + m_data_holder = parent->m_data_holder; + m_model = parent->m_model; +} + +void GraphIteratorProto::initialize(const std::string& path) { + m_model_dir = std::make_shared(ov::util::get_directory(path).string()); + try { + std::ifstream model_file(path, std::ios::binary | std::ios::in); + FRONT_END_GENERAL_CHECK(model_file && model_file.is_open(), "Model file does not exist: ", path); + + m_model = std::make_shared(); + m_model->ParseFromIstream(&model_file); + model_file.close(); + if (m_model->has_graph()) { + m_graph = &m_model->graph(); + } else { + m_graph = nullptr; + return; + } + } catch (...) { + m_model.reset(); + m_graph = nullptr; + node_index = 0; + m_decoders.clear(); + } +} + +#ifdef OPENVINO_ENABLE_UNICODE_PATH_SUPPORT +void GraphIteratorProto::initialize(const std::wstring& path) { + initialize(ov::util::wstring_to_string(path)); +} +#endif // OPENVINO_ENABLE_UNICODE_PATH_SUPPORT + +std::shared_ptr GraphIteratorProto::get_tensor(const std::string& name, + GraphIteratorProto** owner) { + if (m_tensors.count(name) == 0) { + if (name == empty_name) { + *owner = this; + const auto& tensor_decoder = std::make_shared(empty_name, this, -1, -1); + m_tensors[empty_name] = tensor_decoder; + return tensor_decoder; + } + if (m_parent == nullptr) { + throw std::runtime_error("Input tensor isn't found for node \"" + name + "\""); + } + return m_parent->get_tensor(name, owner); + } + *owner = this; + return m_tensors[name]; +} + +void GraphIteratorProto::reset() { + // In case we have any stored external data - free it before beginning + if (m_data_holder != nullptr) { + m_data_holder->clear(); + } + if (m_stream_cache != nullptr) { + m_stream_cache->clear(); + } + node_index = 0; + if (m_decoders.size() > 0 || m_model == nullptr || m_graph == nullptr) + return; + m_decoders.reserve(m_graph->initializer_size() + m_graph->input_size() + m_graph->output_size() + + m_graph->node_size()); + for (const auto& value : m_graph->input()) { + auto tensor = std::make_shared(&value, this, 0, -1); + m_decoders.push_back(tensor); + if (m_tensors.count(*tensor->get_tensor_info().m_tensor_name) > 0) { + throw std::runtime_error("Tensor already exists \"" + *tensor->get_tensor_info().m_tensor_name + "\""); + } + m_tensors[*tensor->get_tensor_info().m_tensor_name] = tensor; + } + for (const auto& value : m_graph->output()) { + auto tensor = std::make_shared(&value, this, -1, 0); + m_decoders.push_back(tensor); + if (m_tensors.count(*tensor->get_tensor_info().m_tensor_name) > 0) { + throw std::runtime_error("Tensor already exists \"" + *tensor->get_tensor_info().m_tensor_name + "\""); + } + m_tensors[*tensor->get_tensor_info().m_tensor_name] = tensor; + } + for (const auto& initializer : m_graph->initializer()) { + const auto& decoder = + std::find_if(m_decoders.begin(), + m_decoders.end(), + [&initializer](const std::shared_ptr& value) { + const auto& tensor = std::dynamic_pointer_cast(value); + if (tensor == nullptr) + return false; + return initializer.name() == *tensor->get_tensor_info().m_tensor_name; + }); + if (decoder != m_decoders.end()) { + *const_cast( + &std::dynamic_pointer_cast(*decoder)->get_tensor_info()) = + extract_tensor_meta_info(&initializer, nullptr, this); + continue; + } + const auto tensor = std::make_shared(&initializer, this, -1, -1); + m_tensors[*tensor->get_tensor_info().m_tensor_name] = tensor; + } + size_t top_index = 0; + for (const auto& node : m_graph->node()) { + std::vector input_tensors{}; + std::vector output_tensors{}; + input_tensors.reserve(node.input_size()); + output_tensors.reserve(node.output_size()); + GraphIteratorProto* tensor_owner = nullptr; + for (const auto& name : node.input()) { + auto decoder_proto_tensor = this->get_tensor(name, &tensor_owner); + input_tensors.push_back(&decoder_proto_tensor->get_tensor_info()); + if (tensor_owner != this) { + // Need to insert parent's decoders on top of decoders + m_decoders.insert(m_decoders.begin() + top_index, decoder_proto_tensor); + ++top_index; + } + } + for (const auto& name : node.output()) { + if (name != "") { + const auto& found_tensor = m_tensors.find(name); + if (found_tensor == m_tensors.end()) { + const auto& initializer = std::find_if(m_graph->initializer().begin(), + m_graph->initializer().end(), + [&name](const TensorProto& value) { + return value.has_name() && value.name() == name; + }); + std::shared_ptr tensor{nullptr}; + if (initializer != m_graph->initializer().end()) { + tensor = std::make_shared(&*initializer, this, -1, -1); + } else { + const auto& value_info = std::find_if(m_graph->value_info().begin(), + m_graph->value_info().end(), + [&name](const ValueInfoProto& value) { + return value.has_name() && value.name() == name; + }); + if (value_info != m_graph->value_info().end()) + tensor = std::make_shared(&*value_info, this, -1, -1); + } + if (tensor == nullptr) { + tensor = std::make_shared(name, this, -1, -1); + } + m_decoders.push_back(tensor); + m_tensors[name] = tensor; + output_tensors.push_back(&tensor->get_tensor_info()); + } else { + output_tensors.push_back(&found_tensor->second->get_tensor_info()); + } + } else { + output_tensors.push_back(&this->get_tensor(empty_name, &tensor_owner)->get_tensor_info()); + } + } + const std::string& domain = node.has_domain() && node.domain() != "ai.onnx" ? node.domain() : DEFAULT_DOMAIN; + int64_t opset = get_opset_version(domain); + if (opset == -1) { + // Forcing a first opset instead of failing + opset = 1; + } + auto decoder_node = + std::make_shared(&node, static_cast(opset), this, input_tensors, output_tensors); + m_decoders.push_back(decoder_node); + } + // If it is a topmost GraphIterator object, and we are not working in Enable MMAP Mode + // Then we can close all opened streams which are used for accessing to external data files + if (m_stream_cache != nullptr && m_parent == nullptr) { + m_stream_cache->clear(); + } +} + +std::shared_ptr GraphIteratorProto::get_decoder() const { + return m_decoders[node_index]; +} + +std::int64_t GraphIteratorProto::get_opset_version(const std::string& domain) const { + // copy the opsets and sort them (descending order) + // then return the version from the first occurrence of a given domain + auto opset_imports = m_model->opset_import(); + std::sort(std::begin(opset_imports), + std::end(opset_imports), + [](const OperatorSetIdProto& lhs, const OperatorSetIdProto& rhs) { + return lhs.version() > rhs.version(); + }); + + for (const auto& opset_import : opset_imports) { + if (domain == opset_import.domain()) { + return opset_import.version(); + } + } + + return -1; +} + +} // namespace onnx +} // namespace frontend +} // namespace ov + +#include +#include +#include +#include +#include +#include +#include + +namespace ov { +namespace frontend { +namespace onnx { +namespace detail { +namespace { +enum Field { + IR_VERSION = 1, + PRODUCER_NAME = 2, + PRODUCER_VERSION = 3, + DOMAIN_ = 4, // DOMAIN collides with some existing symbol in MSVC thus - underscore + MODEL_VERSION = 5, + DOC_STRING = 6, + GRAPH = 7, + OPSET_IMPORT = 8, + METADATA_PROPS = 14, + TRAINING_INFO = 20, + FUNCTIONS = 25 +}; + +enum WireType { VARINT = 0, BITS_64 = 1, LENGTH_DELIMITED = 2, START_GROUP = 3, END_GROUP = 4, BITS_32 = 5 }; + +// A PB key consists of a field number (defined in onnx.proto) and a type of data that follows this key +using PbKey = std::pair; + +// This pair represents a key found in the encoded model and optional size of the payload +// that follows the key (in bytes). The payload should be skipped for fast check purposes. +using ONNXField = std::pair; + +bool is_correct_onnx_field(const PbKey& decoded_key) { + static const std::map onnx_fields = { + {IR_VERSION, VARINT}, + {PRODUCER_NAME, LENGTH_DELIMITED}, + {PRODUCER_VERSION, LENGTH_DELIMITED}, + {DOMAIN_, LENGTH_DELIMITED}, + {MODEL_VERSION, VARINT}, + {DOC_STRING, LENGTH_DELIMITED}, + {GRAPH, LENGTH_DELIMITED}, + {OPSET_IMPORT, LENGTH_DELIMITED}, + {METADATA_PROPS, LENGTH_DELIMITED}, + {TRAINING_INFO, LENGTH_DELIMITED}, + {FUNCTIONS, LENGTH_DELIMITED}, + }; + + if (!onnx_fields.count(static_cast(decoded_key.first))) { + return false; + } + + return onnx_fields.at(static_cast(decoded_key.first)) == static_cast(decoded_key.second); +} + +/** + * Only 7 bits in each component of a varint count in this algorithm. The components form + * a decoded number when they are concatenated bitwise in reverse order. For example: + * bytes = [b1, b2, b3, b4] + * varint = b4 ++ b3 ++ b2 ++ b1 <== only 7 bits of each byte should be extracted before concat + * + * b1 b2 + * bytes = [00101100, 00000010] + * b2 b1 + * varint = 0000010 ++ 0101100 = 100101100 => decimal: 300 + * Each consecutive varint byte needs to be left-shifted "7 x its position in the vector" + * and bitwise added to the accumulator afterward. + */ +uint32_t varint_bytes_to_number(const std::vector& bytes) { + uint32_t accumulator = 0u; + + for (size_t i = 0; i < bytes.size(); ++i) { + uint32_t b = bytes[i]; + b <<= 7 * i; + accumulator |= b; + } + + return accumulator; +} + +uint32_t decode_varint(std::istream& model) { + std::vector bytes; + // max 4 bytes for a single value because this function returns a 32-bit long decoded varint + const size_t MAX_VARINT_BYTES = 4u; + // optimization to avoid allocations during push_back calls + bytes.reserve(MAX_VARINT_BYTES); + + char key_component = 0; + model.get(key_component); + + // keep reading all bytes which have the MSB on from the stream + while (key_component & 0x80 && bytes.size() < MAX_VARINT_BYTES) { + // drop the most significant bit + const uint8_t component = key_component & ~0x80; + bytes.push_back(component); + model.get(key_component); + } + // add the last byte - the one with MSB off + bytes.push_back(key_component); + + return varint_bytes_to_number(bytes); +} + +PbKey decode_key(const char key) { + // 3 least significant bits + const char wire_type = key & 0b111; + // remaining bits + const char field_number = key >> 3; + return {field_number, wire_type}; +} + +ONNXField decode_next_field(std::istream& model) { + char key = 0; + model.get(key); + + const auto decoded_key = decode_key(key); + + if (!is_correct_onnx_field(decoded_key)) { + throw std::runtime_error{"Incorrect field detected in the processed model"}; + } + + const auto onnx_field = static_cast(decoded_key.first); + + switch (decoded_key.second) { + case VARINT: { + // the decoded varint is the payload in this case but its value does not matter + // in the fast check process so it can be discarded + decode_varint(model); + return {onnx_field, 0}; + } + case LENGTH_DELIMITED: + // the varint following the key determines the payload length + return {onnx_field, decode_varint(model)}; + case BITS_64: + return {onnx_field, 8}; + case BITS_32: + return {onnx_field, 4}; + case START_GROUP: + case END_GROUP: + throw std::runtime_error{"StartGroup and EndGroup are not used in ONNX models"}; + default: + throw std::runtime_error{"Unknown WireType encountered in the model"}; + } +} + +inline void skip_payload(std::istream& model, uint32_t payload_size) { + model.seekg(payload_size, std::ios::cur); +} +} // namespace +} // namespace detail + +bool is_valid_model(std::istream& model) { + // the model usually starts with a 0x08 byte indicating the ir_version value + // so this checker expects at least 3 valid ONNX keys to be found in the validated model + const size_t EXPECTED_FIELDS_FOUND = 3u; + std::unordered_set> onnx_fields_found = {}; + try { + while (!model.eof() && onnx_fields_found.size() < EXPECTED_FIELDS_FOUND) { + const auto field = detail::decode_next_field(model); + + if (onnx_fields_found.count(field.first) > 0) { + // if the same field is found twice, this is not a valid ONNX model + return false; + } else { + onnx_fields_found.insert(field.first); + detail::skip_payload(model, field.second); + } + } + + return onnx_fields_found.size() == EXPECTED_FIELDS_FOUND; + } catch (...) { + return false; + } +} + +} // namespace onnx +} // namespace frontend +} // namespace ov diff --git a/src/frontends/onnx/frontend/src/core/graph_iterator_proto.hpp b/src/frontends/onnx/frontend/src/core/graph_iterator_proto.hpp new file mode 100644 index 00000000000000..46c64cab174fd8 --- /dev/null +++ b/src/frontends/onnx/frontend/src/core/graph_iterator_proto.hpp @@ -0,0 +1,155 @@ +// Copyright (C) 2018-2025 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// +#pragma once +#include + +#include + +#include "openvino/frontend/onnx/decoder.hpp" +#include "openvino/frontend/onnx/graph_iterator.hpp" +#include "openvino/util/file_util.hpp" +#include "openvino/util/mmap_object.hpp" +#include "openvino/util/wstring_convert_util.hpp" + +using ::ONNX_NAMESPACE::AttributeProto_AttributeType; +using ::ONNX_NAMESPACE::GraphProto; +using ::ONNX_NAMESPACE::ModelProto; +using ::ONNX_NAMESPACE::NodeProto; +using ::ONNX_NAMESPACE::OperatorSetIdProto; +using ::ONNX_NAMESPACE::TensorProto; +using ::ONNX_NAMESPACE::TensorProto_DataLocation; +using ::ONNX_NAMESPACE::TensorProto_DataType; +using ::ONNX_NAMESPACE::ValueInfoProto; +using ::ONNX_NAMESPACE::Version; + +namespace ov { +namespace frontend { +namespace onnx { + +class DecoderProtoTensor; +using MappedMemoryHandles = std::shared_ptr>>; +using LocalMemoryHandles = std::shared_ptr>>; +using LocalStreamHandles = std::shared_ptr>>; + +enum GraphIteratorProtoMemoryManagementMode : int { + Undefined = 0, + External_Stream = 1, + External_MMAP = 2, + Internal_Stream = 3, + Internal_MMAP = 4, +}; + +bool is_valid_model(std::istream& model); + +class GraphIteratorProto : public ov::frontend::onnx::GraphIterator { + size_t node_index = 0; + std::shared_ptr m_model; + const GraphProto* m_graph{}; + GraphIteratorProto* m_parent; + std::vector> m_decoders{}; + std::map> m_tensors{}; + std::shared_ptr m_model_dir; + GraphIteratorProtoMemoryManagementMode m_mode; + // This is used for keeping MMAP cache handles + MappedMemoryHandles m_mmap_cache; + // This is used for keeping a readed external data without MMAP + LocalStreamHandles m_stream_cache; + LocalMemoryHandles m_data_holder; + +public: + using Ptr = std::shared_ptr; + + GraphIteratorProto() = default; + explicit GraphIteratorProto(const GraphIteratorProtoMemoryManagementMode mode); + explicit GraphIteratorProto(GraphIteratorProto* parent, const GraphProto* graph_def); + ~GraphIteratorProto() = default; + + void initialize(const std::string& path); +#ifdef OPENVINO_ENABLE_UNICODE_PATH_SUPPORT + void initialize(const std::wstring& path); +#endif + + /// Verifies file is supported + template + static bool is_supported(const std::basic_string& path) { + FRONT_END_GENERAL_CHECK(ov::util::file_exists(path), + "Could not open the file: \"", + ov::util::path_to_string(path), + '"'); + try { + std::streamsize file_size = ov::util::file_size(path); + // Skip files which less than size of file identifier + if (file_size < 1) { + return false; + } +#if defined(__MINGW32__) || defined(__MINGW64__) + std::ifstream onnx_stream(std::filesystem::path(path), std::ios::in | std::ifstream::binary); +#else + std::ifstream onnx_stream(path, std::ios::in | std::ifstream::binary); +#endif + return is_valid_model(onnx_stream); + } catch (...) { + return false; + } + } + + /// Set iterator to the start position + void reset() override; + + size_t size() const override { + return m_decoders.size(); + } + + /// Moves to the next node in the graph + void next() override { + node_index++; + } + + bool is_end() const override { + return node_index >= m_decoders.size(); + } + + /// Return Decoder for the current node that iterator points to + std::shared_ptr get_decoder() const override; + + const GraphProto* get_graph() const { + return m_graph; + } + + std::int64_t get_opset_version(const std::string& domain) const override; + + std::string get_model_dir() const { + return *m_model_dir; + } + + GraphIteratorProtoMemoryManagementMode get_memory_management_mode() const { + return m_mode; + } + + MappedMemoryHandles get_mmap_cache() const { + return m_mmap_cache; + } + + LocalStreamHandles get_stream_cache() const { + return m_stream_cache; + } + + std::shared_ptr allocate_data(const size_t size) { + std::shared_ptr data(new uint8_t[size], [](uint8_t* p) { + delete[] p; + }); + m_data_holder->push_back(data); + return data; + } + +protected: + /// \brief Returns DecoderProtoTensor found in the current scope, or in a parent scope + /// \param name Name of tensor + /// \param owner Returns real owner of the tensor + std::shared_ptr get_tensor(const std::string& name, GraphIteratorProto** owner); +}; + +} // namespace onnx +} // namespace frontend +} // namespace ov diff --git a/src/frontends/onnx/frontend/src/core/node.cpp b/src/frontends/onnx/frontend/src/core/node.cpp index 184faf38fa42bc..97c57c81149018 100644 --- a/src/frontends/onnx/frontend/src/core/node.cpp +++ b/src/frontends/onnx/frontend/src/core/node.cpp @@ -10,6 +10,10 @@ #include "core/graph.hpp" #include "core/null_node.hpp" #include "core/tensor.hpp" +#include "input_model.hpp" +#include "openvino/frontend/onnx/decoder.hpp" +#include "openvino/frontend/onnx/graph_iterator.hpp" +#include "translate_session.hpp" namespace ov { namespace frontend { @@ -66,6 +70,7 @@ class Node::Impl { bool has_subgraphs() const; const std::unordered_map>& get_subgraphs() const; + std::shared_ptr get_subgraph(const std::string name) const; template T get_attribute_value(const std::string& name, T default_value) const; @@ -168,6 +173,13 @@ const std::unordered_map>& Node::Impl::ge return m_subgraphs; } +std::shared_ptr Node::Impl::get_subgraph(const std::string name) const { + auto it = m_subgraphs.find(name); + if (it == m_subgraphs.end()) + return nullptr; + return it->second->decode(); +} + template T Node::Impl::get_attribute_value(const std::string& name, T default_value) const { auto it = std::find_if(std::begin(m_attributes), std::end(m_attributes), [&](const Attribute& attribute) { @@ -190,6 +202,18 @@ T Node::Impl::get_attribute_value(const std::string& name) const { return it->template get_value(); } +template <> +std::shared_ptr Node::Impl::get_attribute_value(const std::string& name, + std::shared_ptr default_value) const { + auto it = std::find_if(std::begin(m_attributes), std::end(m_attributes), [&](const Attribute& attribute) { + return attribute.get_name() == name; + }); + if (it == std::end(m_attributes)) { + return std::forward>(default_value); + } + return get_subgraph(name); +} + template <> Subgraph Node::Impl::get_attribute_value(const std::string& name) const { return get_subgraph_from_attribute(name); @@ -200,6 +224,11 @@ ov::Any Node::get_attribute_value(const std::string& name) const { return get_attribute(name).get_any(); } +template <> +std::shared_ptr Node::Impl::get_attribute_value(const std::string& name) const { + return get_subgraph(name); +} + ov::OutputVector Node::Impl::get_ov_inputs() const { ov::OutputVector result; for (const auto& name : m_node_proto->input()) { @@ -290,345 +319,868 @@ std::shared_ptr Node::Impl::get_attribute_as_constant(cons } Node::Node(const NodeProto& node_proto, Graph* graph) - : m_pimpl{new Impl{node_proto, graph}, [](Impl* impl) { + : m_pimpl{new Impl{node_proto, graph}, + [](Impl* impl) { delete impl; - }} {} - -Node::Node(Node&& other) noexcept : m_pimpl{std::move(other.m_pimpl)} {} + }}, + m_decoder(nullptr), + m_translate_session(nullptr) {} +Node::Node(const DecoderBaseOperation& decoder, TranslateSession* translate_session) + : m_pimpl{nullptr, + [](Impl* impl) { + + }}, + m_decoder(&decoder), + m_translate_session(translate_session) {} + +Node::Node(Node&& other) noexcept + : m_pimpl{std::move(other.m_pimpl)}, + m_decoder(nullptr), + m_translate_session(nullptr) {} Node::Node(const Node& other) - : m_pimpl{new Impl{other.m_pimpl->node_proto(), other.m_pimpl->graph(), other.get_subgraphs()}, [](Impl* impl) { + : m_pimpl{other.m_pimpl != nullptr + ? new Impl{other.m_pimpl->node_proto(), other.m_pimpl->graph(), other.get_subgraphs()} + : nullptr, + [](Impl* impl) { delete impl; - }} {} + }}, + m_decoder(other.m_decoder), + m_translate_session(other.m_translate_session) {} + +#include // For std::runtime_error ov::OutputVector Node::get_ov_inputs() const { - return m_pimpl->get_ov_inputs(); + if (m_pimpl != nullptr) { + return m_pimpl->get_ov_inputs(); + } else if (m_decoder != nullptr) { + ov::OutputVector result; + for (size_t idx = 0; idx < m_decoder->get_input_size(); ++idx) { + const std::string& name = m_decoder->get_input_tensor_name(idx); + if (!name.empty()) { + auto node = m_translate_session->lookup_tensor(name); + FRONT_END_GENERAL_CHECK(node.get_node() != nullptr); + result.push_back(node); + } else { + result.push_back(std::make_shared()->get_default_output()); + } + } + return result; + } + FRONT_END_NOT_IMPLEMENTED(get_ov_inputs); } + const std::string& Node::domain() const { - return m_pimpl->domain(); + static const std::string empty_domain; + if (m_pimpl != nullptr) { + return m_pimpl->domain(); + } else if (m_decoder != nullptr) { + return m_decoder->get_domain(); + } + FRONT_END_NOT_IMPLEMENTED(domain); } + const std::string& Node::op_type() const { - return m_pimpl->op_type(); + static const std::string empty_op_type; + if (m_pimpl != nullptr) { + return m_pimpl->op_type(); + } else if (m_decoder != nullptr) { + return m_decoder->get_op_type(); + } + FRONT_END_NOT_IMPLEMENTED(op_type); } + const std::string& Node::get_description() const { - return m_pimpl->description(); + static const std::string empty_description; + if (m_pimpl != nullptr) { + return m_pimpl->description(); + } else if (m_decoder != nullptr) { + // Workaround + return m_decoder->get_output_tensor_name(0); + } + FRONT_END_NOT_IMPLEMENTED(get_description); } + const std::string& Node::get_name() const { - return m_pimpl->name(); -} -const std::vector>& Node::get_output_names() const { - return m_pimpl->get_output_names(); + static const std::string empty_name; + if (m_pimpl != nullptr) { + return m_pimpl->name(); + } else if (m_decoder != nullptr) { + return m_decoder->get_op_name(); + // Add logic for m_decoder if applicable + } + FRONT_END_NOT_IMPLEMENTED(get_name); +} + +const std::vector> Node::get_output_names() const { + if (m_pimpl != nullptr) { + return m_pimpl->get_output_names(); + } else if (m_decoder != nullptr) { + std::vector> names{}; + names.reserve(m_decoder->get_output_size()); + for (size_t idx = 0; idx < m_decoder->get_output_size(); ++idx) { + const auto& name = m_decoder->get_output_tensor_name(idx); + names.push_back(name); + } + return {names.begin(), names.end()}; + } + FRONT_END_NOT_IMPLEMENTED(get_output_names); } const std::string& Node::input(int index) const { - return m_pimpl->input(index); + static const std::string empty_input; + if (m_pimpl != nullptr) { + return m_pimpl->input(index); + } else if (m_decoder != nullptr) { + return m_decoder->get_input_tensor_name(index); + } + FRONT_END_NOT_IMPLEMENTED(input); } std::size_t Node::get_inputs_size() const { - return m_pimpl->get_inputs_size(); + if (m_pimpl != nullptr) { + return m_pimpl->get_inputs_size(); + } else if (m_decoder != nullptr) { + return m_decoder->get_input_size(); + } + FRONT_END_NOT_IMPLEMENTED(get_inputs_size); } const std::string& Node::output(int index) const { - return m_pimpl->output(index); + static const std::string empty_output; + if (m_pimpl != nullptr) { + return m_pimpl->output(index); + } else if (m_decoder != nullptr) { + return m_decoder->get_output_tensor_name(index); + } + FRONT_END_NOT_IMPLEMENTED(output); } std::size_t Node::get_outputs_size() const { - return m_pimpl->get_outputs_size(); + if (m_pimpl != nullptr) { + return m_pimpl->get_outputs_size(); + } else if (m_decoder != nullptr) { + return m_decoder->get_output_size(); + } + FRONT_END_NOT_IMPLEMENTED(get_outputs_size); } bool Node::has_attribute(const std::string& name) const { - return m_pimpl->has_attribute(name); + if (m_pimpl != nullptr) { + return m_pimpl->has_attribute(name); + } else if (m_decoder != nullptr) { + return m_decoder->has_attribute(name); + } + FRONT_END_NOT_IMPLEMENTED(has_attribute); } bool Node::has_subgraphs() const { - return m_pimpl->has_subgraphs(); + if (m_pimpl != nullptr) { + return m_pimpl->has_subgraphs(); + } else if (m_decoder != nullptr) { + // Add logic for m_decoder if applicable + } + FRONT_END_NOT_IMPLEMENTED(has_subgraphs); } const std::unordered_map>& Node::get_subgraphs() const { - return m_pimpl->get_subgraphs(); + static const std::unordered_map> empty_subgraphs; + if (m_pimpl != nullptr) { + return m_pimpl->get_subgraphs(); + } else if (m_decoder != nullptr) { + // Add logic for m_decoder if applicable + } + FRONT_END_NOT_IMPLEMENTED(get_subgraphs); } std::vector Node::get_attribute_names() const { - std::vector attr_names; - const auto& node_attributes = m_pimpl->attributes(); - attr_names.reserve(node_attributes.size()); - std::transform(std::begin(node_attributes), - std::end(node_attributes), - std::back_inserter(attr_names), - [](const Attribute& a) { - return a.get_name(); - }); - return attr_names; + if (m_pimpl != nullptr) { + std::vector attr_names; + const auto& node_attributes = m_pimpl->attributes(); + attr_names.reserve(node_attributes.size()); + std::transform(std::begin(node_attributes), + std::end(node_attributes), + std::back_inserter(attr_names), + [](const Attribute& a) { + return a.get_name(); + }); + return attr_names; + } else if (m_decoder != nullptr) { + // Add logic for m_decoder if applicable + } + FRONT_END_NOT_IMPLEMENTED(get_attribute_names); } const Attribute& Node::get_attribute(const std::string& name) const { - const auto& node_attributes = m_pimpl->attributes(); - auto found_attr = std::find_if(std::begin(node_attributes), std::end(node_attributes), [&name](const Attribute& a) { - return a.get_name() == name; - }); - if (found_attr == std::end(node_attributes)) { - throw error::node::UnknownAttribute{this->get_name(), name}; + if (m_pimpl != nullptr) { + const auto& node_attributes = m_pimpl->attributes(); + auto found_attr = + std::find_if(std::begin(node_attributes), std::end(node_attributes), [&name](const Attribute& a) { + return a.get_name() == name; + }); + if (found_attr == std::end(node_attributes)) { + throw error::node::UnknownAttribute{this->get_name(), name}; + } + return *found_attr; + } else if (m_decoder != nullptr) { + // Add logic for m_decoder if applicable + } + FRONT_END_NOT_IMPLEMENTED(get_attribute); +} + +ov::Any Node::get_attribute_any(const std::string& name) const { + if (m_pimpl != nullptr) { + } else if (m_decoder != nullptr) { + return m_decoder->get_attribute(name); } - return *found_attr; + FRONT_END_NOT_IMPLEMENTED(get_attribute_any); } template <> float Node::get_attribute_value(const std::string& name, float default_value) const { - return m_pimpl->template get_attribute_value(name, default_value); + if (m_pimpl != nullptr) { + return m_pimpl->template get_attribute_value(name, default_value); + } else if (m_decoder != nullptr) { + if (m_decoder->has_attribute(name)) + return m_decoder->get_attribute(name).as(); + else + return default_value; + } + FRONT_END_NOT_IMPLEMENTED(get_attribute_value); } template <> double Node::get_attribute_value(const std::string& name, double default_value) const { - return m_pimpl->template get_attribute_value(name, default_value); + if (m_pimpl != nullptr) { + return m_pimpl->template get_attribute_value(name, default_value); + } else if (m_decoder != nullptr) { + if (m_decoder->has_attribute(name)) + return m_decoder->get_attribute(name).as(); + else + return default_value; + } + FRONT_END_NOT_IMPLEMENTED(get_attribute_value); } template <> std::int64_t Node::get_attribute_value(const std::string& name, std::int64_t default_value) const { - return m_pimpl->template get_attribute_value(name, default_value); + if (m_pimpl != nullptr) { + return m_pimpl->template get_attribute_value(name, default_value); + } else if (m_decoder != nullptr) { + if (m_decoder->has_attribute(name)) + return m_decoder->get_attribute(name).as(); + else + return default_value; + } + FRONT_END_NOT_IMPLEMENTED(get_attribute_value); } template <> std::string Node::get_attribute_value(const std::string& name, std::string default_value) const { - return m_pimpl->template get_attribute_value(name, std::move(default_value)); + if (m_pimpl != nullptr) { + return m_pimpl->template get_attribute_value(name, std::move(default_value)); + } else if (m_decoder != nullptr) { + if (m_decoder->has_attribute(name)) + return m_decoder->get_attribute(name).as(); + else + return default_value; + } + FRONT_END_NOT_IMPLEMENTED(get_attribute_value); } template <> Tensor Node::get_attribute_value(const std::string& name, Tensor default_value) const { - return m_pimpl->template get_attribute_value(name, std::move(default_value)); + if (m_pimpl != nullptr) { + return m_pimpl->template get_attribute_value(name, std::move(default_value)); + } else if (m_decoder != nullptr) { + // Non-applicable + } + FRONT_END_NOT_IMPLEMENTED(get_attribute_value); } template <> SparseTensor Node::get_attribute_value(const std::string& name, SparseTensor default_value) const { - return m_pimpl->template get_attribute_value(name, std::move(default_value)); + if (m_pimpl != nullptr) { + return m_pimpl->template get_attribute_value(name, std::move(default_value)); + } else if (m_decoder != nullptr) { + // Non-applicable + } + FRONT_END_NOT_IMPLEMENTED(get_attribute_value); } template <> Graph Node::get_attribute_value(const std::string& name, Graph default_value) const { - return m_pimpl->template get_attribute_value(name, std::move(default_value)); + if (m_pimpl != nullptr) { + return m_pimpl->template get_attribute_value(name, std::move(default_value)); + } else if (m_decoder != nullptr) { + // Non-applicable + } + FRONT_END_NOT_IMPLEMENTED(get_attribute_value); } template <> std::vector Node::get_attribute_value(const std::string& name, std::vector default_value) const { - return m_pimpl->template get_attribute_value>(name, std::move(default_value)); + if (m_pimpl != nullptr) { + return m_pimpl->template get_attribute_value>(name, std::move(default_value)); + } else if (m_decoder != nullptr) { + if (m_decoder->has_attribute(name)) + return m_decoder->get_attribute(name).as>(); + else + return default_value; + } + FRONT_END_NOT_IMPLEMENTED(get_attribute_value); } template <> std::vector Node::get_attribute_value(const std::string& name, std::vector default_value) const { - return m_pimpl->template get_attribute_value>(name, std::move(default_value)); + if (m_pimpl != nullptr) { + return m_pimpl->template get_attribute_value>(name, std::move(default_value)); + } else if (m_decoder != nullptr) { + if (m_decoder->has_attribute(name)) + return m_decoder->get_attribute(name).as>(); + else + return default_value; + } + FRONT_END_NOT_IMPLEMENTED(get_attribute_value); } template <> std::vector Node::get_attribute_value(const std::string& name, std::vector default_value) const { - return m_pimpl->template get_attribute_value>(name, std::move(default_value)); + if (m_pimpl != nullptr) { + return m_pimpl->template get_attribute_value>(name, std::move(default_value)); + } else if (m_decoder != nullptr) { + if (m_decoder->has_attribute(name)) + return m_decoder->get_attribute(name).as>(); + else + return default_value; + } + FRONT_END_NOT_IMPLEMENTED(get_attribute_value); } template <> std::vector Node::get_attribute_value(const std::string& name, std::vector default_value) const { - return m_pimpl->template get_attribute_value>(name, std::move(default_value)); + if (m_pimpl != nullptr) { + return m_pimpl->template get_attribute_value>(name, std::move(default_value)); + } else if (m_decoder != nullptr) { + if (m_decoder->has_attribute(name)) { + auto ints = m_decoder->get_attribute(name).as>(); + return {ints.begin(), ints.end()}; + } else { + return default_value; + } + } + FRONT_END_NOT_IMPLEMENTED(get_attribute_value); } template <> std::vector Node::get_attribute_value(const std::string& name, std::vector default_value) const { - return m_pimpl->template get_attribute_value>(name, std::move(default_value)); + if (m_pimpl != nullptr) { + return m_pimpl->template get_attribute_value>(name, std::move(default_value)); + } else if (m_decoder != nullptr) { + if (m_decoder->has_attribute(name)) + return m_decoder->get_attribute(name).as>(); + else + return default_value; + } + FRONT_END_NOT_IMPLEMENTED(get_attribute_value); } template <> std::vector Node::get_attribute_value(const std::string& name, std::vector default_value) const { - return m_pimpl->template get_attribute_value>(name, std::move(default_value)); + if (m_pimpl != nullptr) { + return m_pimpl->template get_attribute_value>(name, std::move(default_value)); + } else if (m_decoder != nullptr) { + // Non-applicable + } + FRONT_END_NOT_IMPLEMENTED(get_attribute_value); } template <> std::vector Node::get_attribute_value(const std::string& name, std::vector default_value) const { - return m_pimpl->template get_attribute_value>(name, std::move(default_value)); + if (m_pimpl != nullptr) { + return m_pimpl->template get_attribute_value>(name, std::move(default_value)); + } else if (m_decoder != nullptr) { + // Non-applicable + } + FRONT_END_NOT_IMPLEMENTED(get_attribute_value); } template <> std::vector Node::get_attribute_value(const std::string& name, std::vector default_value) const { - return m_pimpl->template get_attribute_value>(name, std::move(default_value)); + if (m_pimpl != nullptr) { + return m_pimpl->template get_attribute_value>(name, std::move(default_value)); + } else if (m_decoder != nullptr) { + // Non-applicable + } + FRONT_END_NOT_IMPLEMENTED(get_attribute_value); } +template <> +std::shared_ptr Node::get_attribute_value(const std::string& name, + std::shared_ptr default_value) const { + if (m_pimpl != nullptr) { + return m_pimpl->template get_attribute_value>(name, std::move(default_value)); + } else if (m_decoder != nullptr) { + if (!has_attribute(name)) { + return default_value; + } + return get_attribute_value>(name); + } + FRONT_END_NOT_IMPLEMENTED(get_attribute_value); +} + +// Repeat the same for the non-default_value overloads: + template <> float Node::get_attribute_value(const std::string& name) const { - return m_pimpl->template get_attribute_value(name); + if (m_pimpl != nullptr) { + return m_pimpl->template get_attribute_value(name); + } else if (m_decoder != nullptr) { + return m_decoder->get_attribute(name).as(); + } + FRONT_END_NOT_IMPLEMENTED(get_attribute_value); } template <> double Node::get_attribute_value(const std::string& name) const { - return m_pimpl->template get_attribute_value(name); + if (m_pimpl != nullptr) { + return m_pimpl->template get_attribute_value(name); + } else if (m_decoder != nullptr) { + return m_decoder->get_attribute(name).as(); + } + FRONT_END_NOT_IMPLEMENTED(get_attribute_value); } template <> std::int64_t Node::get_attribute_value(const std::string& name) const { - return m_pimpl->template get_attribute_value(name); + if (m_pimpl != nullptr) { + return m_pimpl->template get_attribute_value(name); + } else if (m_decoder != nullptr) { + return m_decoder->get_attribute(name).as(); + } + FRONT_END_NOT_IMPLEMENTED(get_attribute_value); } template <> std::size_t Node::get_attribute_value(const std::string& name) const { - return m_pimpl->template get_attribute_value(name); + if (m_pimpl != nullptr) { + return m_pimpl->template get_attribute_value(name); + } else if (m_decoder != nullptr) { + return m_decoder->get_attribute(name).as(); + } + FRONT_END_NOT_IMPLEMENTED(get_attribute_value); } template <> std::string Node::get_attribute_value(const std::string& name) const { - return m_pimpl->template get_attribute_value(name); + if (m_pimpl != nullptr) { + return m_pimpl->template get_attribute_value(name); + } else if (m_decoder != nullptr) { + return m_decoder->get_attribute(name).as(); + } + FRONT_END_NOT_IMPLEMENTED(get_attribute_value); } template <> Tensor Node::get_attribute_value(const std::string& name) const { - return m_pimpl->template get_attribute_value(name); + if (m_pimpl != nullptr) { + return m_pimpl->template get_attribute_value(name); + } else if (m_decoder != nullptr) { + auto tensor_decoder = std::dynamic_pointer_cast( + m_decoder->get_attribute(name).as()); + const auto& tensor_meta_info = tensor_decoder->get_tensor_info(); + auto tensor_place = std::make_shared( + *m_translate_session->get_input_model().get(), + tensor_meta_info.m_partial_shape, + tensor_meta_info.m_element_type, + std::vector{*tensor_meta_info.m_tensor_name}, + tensor_meta_info.m_tensor_data, + tensor_meta_info.m_tensor_data_size, + tensor_meta_info.m_external_location, + tensor_meta_info.m_is_raw); + return {tensor_place}; + } + FRONT_END_NOT_IMPLEMENTED(get_attribute_value); } template <> SparseTensor Node::get_attribute_value(const std::string& name) const { - return m_pimpl->template get_attribute_value(name); + if (m_pimpl != nullptr) { + return m_pimpl->template get_attribute_value(name); + } else if (m_decoder != nullptr) { + auto sparse_tensor_info = m_decoder->get_attribute(name).as(); + FRONT_END_GENERAL_CHECK(sparse_tensor_info.m_indices && sparse_tensor_info.m_values, + "Incomplete sparse tensors are not supported"); + + auto values_decoder = + std::dynamic_pointer_cast(sparse_tensor_info.m_values); + const auto& values_meta_info = values_decoder->get_tensor_info(); + auto values_place = std::make_shared( + *m_translate_session->get_input_model().get(), + values_meta_info.m_partial_shape, + values_meta_info.m_element_type, + std::vector{*values_meta_info.m_tensor_name}, + values_meta_info.m_tensor_data, + values_meta_info.m_tensor_data_size, + values_meta_info.m_external_location, + values_meta_info.m_is_raw); + + auto indices_decoder = + std::dynamic_pointer_cast(sparse_tensor_info.m_indices); + const auto& indices_meta_info = indices_decoder->get_tensor_info(); + auto indices_place = std::make_shared( + *m_translate_session->get_input_model().get(), + indices_meta_info.m_partial_shape, + indices_meta_info.m_element_type, + std::vector{*indices_meta_info.m_tensor_name}, + indices_meta_info.m_tensor_data, + indices_meta_info.m_tensor_data_size, + indices_meta_info.m_external_location, + indices_meta_info.m_is_raw); + return {values_place, indices_place, sparse_tensor_info.m_partial_shape}; + } + FRONT_END_NOT_IMPLEMENTED(get_attribute_value); } template <> Subgraph Node::get_attribute_value(const std::string& name) const { - return m_pimpl->template get_attribute_value(name); + if (m_pimpl != nullptr) { + return m_pimpl->template get_attribute_value(name); + } else if (m_decoder != nullptr) { + // Non-applicable + } + FRONT_END_NOT_IMPLEMENTED(get_attribute_value); } template <> std::vector Node::get_attribute_value(const std::string& name) const { - return m_pimpl->template get_attribute_value>(name); + if (m_pimpl != nullptr) { + return m_pimpl->template get_attribute_value>(name); + } else if (m_decoder != nullptr) { + return m_decoder->get_attribute(name).as>(); + } + FRONT_END_NOT_IMPLEMENTED(get_attribute_value); } template <> std::vector Node::get_attribute_value(const std::string& name) const { - return m_pimpl->template get_attribute_value>(name); + if (m_pimpl != nullptr) { + return m_pimpl->template get_attribute_value>(name); + } else if (m_decoder != nullptr) { + return m_decoder->get_attribute(name).as>(); + } + FRONT_END_NOT_IMPLEMENTED(get_attribute_value); } template <> std::vector Node::get_attribute_value(const std::string& name) const { - return m_pimpl->template get_attribute_value>(name); + if (m_pimpl != nullptr) { + return m_pimpl->template get_attribute_value>(name); + } else if (m_decoder != nullptr) { + return m_decoder->get_attribute(name).as>(); + } + FRONT_END_NOT_IMPLEMENTED(get_attribute_value); } template <> std::vector Node::get_attribute_value(const std::string& name) const { - return m_pimpl->template get_attribute_value>(name); + if (m_pimpl != nullptr) { + return m_pimpl->template get_attribute_value>(name); + } else if (m_decoder != nullptr) { + auto ints = m_decoder->get_attribute(name).as>(); + return {ints.begin(), ints.end()}; + } + FRONT_END_NOT_IMPLEMENTED(get_attribute_value); } template <> std::vector Node::get_attribute_value(const std::string& name) const { - return m_pimpl->template get_attribute_value>(name); + if (m_pimpl != nullptr) { + return m_pimpl->template get_attribute_value>(name); + } else if (m_decoder != nullptr) { + return m_decoder->get_attribute(name).as>(); + } + FRONT_END_NOT_IMPLEMENTED(get_attribute_value); } template <> std::vector Node::get_attribute_value(const std::string& name) const { - return m_pimpl->template get_attribute_value>(name); + if (m_pimpl != nullptr) { + return m_pimpl->template get_attribute_value>(name); + } else if (m_decoder != nullptr) { + // Non-applicable + } + FRONT_END_NOT_IMPLEMENTED(get_attribute_value); } template <> std::vector Node::get_attribute_value(const std::string& name) const { - return m_pimpl->template get_attribute_value>(name); + if (m_pimpl != nullptr) { + return m_pimpl->template get_attribute_value>(name); + } else if (m_decoder != nullptr) { + // Non-applicable + } + FRONT_END_NOT_IMPLEMENTED(get_attribute_value); } template <> std::vector Node::get_attribute_value(const std::string& name) const { - return m_pimpl->template get_attribute_value>(name); + if (m_pimpl != nullptr) { + return m_pimpl->template get_attribute_value>(name); + } else if (m_decoder != nullptr) { + // Non-applicable + } + FRONT_END_NOT_IMPLEMENTED(get_attribute_value); +} + +template <> +std::shared_ptr Node::get_attribute_value(const std::string& name) const { + if (m_pimpl != nullptr) { + return m_pimpl->template get_attribute_value>(name); + } else if (m_decoder != nullptr) { + auto graph_iterator = m_decoder->get_attribute(name).as(); + graph_iterator->reset(); + auto input_model = + std::make_shared(graph_iterator, m_translate_session->get_input_model().get()); + TranslateSession translate_session(input_model, m_translate_session, get_name()); + std::shared_ptr ov_model(nullptr); + translate_session.translate_graph(input_model, ov_model); + return ov_model; + } + FRONT_END_NOT_IMPLEMENTED(get_attribute_value); +} + +// get_attribute_as_constant specializations + +// Calls get_decoder_attribute_as_constant is better to rewrite as ov::Any later +// After GraphIterator will become a default interface +template +std::shared_ptr Node::get_decoder_attribute_as_constant(const std::string& name) const { + const auto value = get_attribute_value(name); + const ov::element::Type type = ov::element::from(); + return std::make_shared(type, ov::Shape{}, value); +} + +template <> +std::shared_ptr Node::get_decoder_attribute_as_constant>( + const std::string& name) const { + const auto values = get_attribute_value>(name); + return std::make_shared(ov::element::i64, ov::Shape{values.size()}, values); +} + +template +std::shared_ptr Node::get_decoder_attribute_as_constant(const std::string& name, + T default_value) const { + const auto value = get_attribute_value(name, default_value); + const ov::element::Type type = ov::element::from(); + return std::make_shared(type, ov::Shape{}, value); +} + +template <> +std::shared_ptr Node::get_decoder_attribute_as_constant>( + const std::string& name, + std::vector default_value) const { + const auto values = get_attribute_value>(name, default_value); + return std::make_shared(ov::element::i64, ov::Shape{values.size()}, values); } template <> std::shared_ptr Node::get_attribute_as_constant(const std::string& name) const { - return m_pimpl->template get_attribute_as_constant(name); + if (m_pimpl != nullptr) { + return m_pimpl->template get_attribute_as_constant(name); + } else if (m_decoder != nullptr) { + return get_decoder_attribute_as_constant(name); + } + FRONT_END_NOT_IMPLEMENTED(get_attribute_as_constant); } template <> std::shared_ptr Node::get_attribute_as_constant(const std::string& name, float default_value) const { - return m_pimpl->template get_attribute_as_constant(name, default_value); + if (m_pimpl != nullptr) { + return m_pimpl->template get_attribute_as_constant(name, default_value); + } else if (m_decoder != nullptr) { + float value = default_value; + if (m_decoder->has_attribute(name)) { + value = m_decoder->get_attribute(name).as(); + } + return ov::op::v0::Constant::create(ov::element::f32, {}, {value}); + } + FRONT_END_NOT_IMPLEMENTED(get_attribute_as_constant); } template <> std::shared_ptr Node::get_attribute_as_constant(const std::string& name, float default_value, ov::element::Type type) const { - return m_pimpl->template get_attribute_as_constant(name, default_value, std::move(type)); + if (m_pimpl != nullptr) { + return m_pimpl->template get_attribute_as_constant(name, default_value, std::move(type)); + } else if (m_decoder != nullptr) { + float value = default_value; + if (m_decoder->has_attribute(name)) { + value = m_decoder->get_attribute(name).as(); + } + return ov::op::v0::Constant::create(type == ov::element::dynamic ? ov::element::f32 : type, {}, {value}); + } + FRONT_END_NOT_IMPLEMENTED(get_attribute_as_constant); } template <> std::shared_ptr Node::get_attribute_as_constant(const std::string& name, ov::element::Type type) const { - return m_pimpl->template get_attribute_as_constant(name, std::move(type)); + if (m_pimpl != nullptr) { + return m_pimpl->template get_attribute_as_constant(name, std::move(type)); + } else if (m_decoder != nullptr) { + return get_decoder_attribute_as_constant(name); + } + FRONT_END_NOT_IMPLEMENTED(get_attribute_as_constant); } template <> std::shared_ptr Node::get_attribute_as_constant(const std::string& name) const { - return m_pimpl->template get_attribute_as_constant(name); + if (m_pimpl != nullptr) { + return m_pimpl->template get_attribute_as_constant(name); + } else if (m_decoder != nullptr) { + return get_decoder_attribute_as_constant(name); + } + FRONT_END_NOT_IMPLEMENTED(get_attribute_as_constant); } template <> std::shared_ptr Node::get_attribute_as_constant(const std::string& name, double default_value) const { - return m_pimpl->template get_attribute_as_constant(name, default_value); + if (m_pimpl != nullptr) { + return m_pimpl->template get_attribute_as_constant(name, default_value); + } else if (m_decoder != nullptr) { + return get_decoder_attribute_as_constant(name, default_value); + } + FRONT_END_NOT_IMPLEMENTED(get_attribute_as_constant); } template <> std::shared_ptr Node::get_attribute_as_constant(const std::string& name, double default_value, ov::element::Type type) const { - return m_pimpl->template get_attribute_as_constant(name, default_value, std::move(type)); + if (m_pimpl != nullptr) { + return m_pimpl->template get_attribute_as_constant(name, default_value, std::move(type)); + } else if (m_decoder != nullptr) { + double value = default_value; + if (m_decoder->has_attribute(name)) { + value = m_decoder->get_attribute(name).as(); + } + return ov::op::v0::Constant::create(type == ov::element::dynamic ? ov::element::f64 : type, {0}, {value}); + } + FRONT_END_NOT_IMPLEMENTED(get_attribute_as_constant); } template <> std::shared_ptr Node::get_attribute_as_constant(const std::string& name, ov::element::Type type) const { - return m_pimpl->template get_attribute_as_constant(name, std::move(type)); + if (m_pimpl != nullptr) { + return m_pimpl->template get_attribute_as_constant(name, std::move(type)); + } else if (m_decoder != nullptr) { + return get_decoder_attribute_as_constant(name); + } + FRONT_END_NOT_IMPLEMENTED(get_attribute_as_constant); } template <> std::shared_ptr Node::get_attribute_as_constant(const std::string& name) const { - return m_pimpl->template get_attribute_as_constant(name); + if (m_pimpl != nullptr) { + return m_pimpl->template get_attribute_as_constant(name); + } else if (m_decoder != nullptr) { + return get_decoder_attribute_as_constant(name); + } + FRONT_END_NOT_IMPLEMENTED(get_attribute_as_constant); } template <> std::shared_ptr Node::get_attribute_as_constant(const std::string& name, int64_t default_value) const { - return m_pimpl->template get_attribute_as_constant(name, default_value); + if (m_pimpl != nullptr) { + return m_pimpl->template get_attribute_as_constant(name, default_value); + } else if (m_decoder != nullptr) { + return get_decoder_attribute_as_constant(name, default_value); + } + FRONT_END_NOT_IMPLEMENTED(get_attribute_as_constant); } template <> std::shared_ptr Node::get_attribute_as_constant(const std::string& name, int64_t default_value, ov::element::Type type) const { - return m_pimpl->template get_attribute_as_constant(name, default_value, std::move(type)); + if (m_pimpl != nullptr) { + return m_pimpl->template get_attribute_as_constant(name, default_value, std::move(type)); + } else if (m_decoder != nullptr) { + return get_decoder_attribute_as_constant(name, default_value); + } + FRONT_END_NOT_IMPLEMENTED(get_attribute_as_constant); } template <> std::shared_ptr Node::get_attribute_as_constant(const std::string& name, ov::element::Type type) const { - return m_pimpl->template get_attribute_as_constant(name, std::move(type)); + if (m_pimpl != nullptr) { + return m_pimpl->template get_attribute_as_constant(name, std::move(type)); + } else if (m_decoder != nullptr) { + return get_decoder_attribute_as_constant(name); + } + FRONT_END_NOT_IMPLEMENTED(get_attribute_as_constant); } template <> std::shared_ptr Node::get_attribute_as_constant>( const std::string& name) const { - return m_pimpl->template get_attribute_as_constant>(name); + if (m_pimpl != nullptr) { + return m_pimpl->template get_attribute_as_constant>(name); + } else if (m_decoder != nullptr) { + return get_decoder_attribute_as_constant>(name); + } + FRONT_END_NOT_IMPLEMENTED(get_attribute_as_constant); } template <> std::shared_ptr Node::get_attribute_as_constant>( const std::string& name, ov::element::Type type) const { - return m_pimpl->template get_attribute_as_constant>(name, std::move(type)); + if (m_pimpl != nullptr) { + return m_pimpl->template get_attribute_as_constant>(name, std::move(type)); + } else if (m_decoder != nullptr) { + auto value = m_decoder->get_attribute(name).as>(); + return ov::op::v0::Constant::create(type == ov::element::dynamic ? ov::element::i64 : type, + {value.size()}, + value); + } + FRONT_END_NOT_IMPLEMENTED(get_attribute_as_constant); } template <> std::shared_ptr Node::get_attribute_as_constant(const std::string& name, std::vector default_value) const { - return m_pimpl->template get_attribute_as_constant>(name, std::move(default_value)); + if (m_pimpl != nullptr) { + return m_pimpl->template get_attribute_as_constant>(name, std::move(default_value)); + } else if (m_decoder != nullptr) { + return get_decoder_attribute_as_constant>(name, default_value); + } + FRONT_END_NOT_IMPLEMENTED(get_attribute_as_constant); } template <> std::shared_ptr Node::get_attribute_as_constant(const std::string& name, std::vector default_value, ov::element::Type type) const { - return m_pimpl->template get_attribute_as_constant>(name, - std::move(default_value), - std::move(type)); + if (m_pimpl != nullptr) { + return m_pimpl->template get_attribute_as_constant>(name, + std::move(default_value), + std::move(type)); + } else if (m_decoder != nullptr) { + return get_decoder_attribute_as_constant>(name, default_value); + } + FRONT_END_NOT_IMPLEMENTED(get_attribute_as_constant); } } // namespace onnx diff --git a/src/frontends/onnx/frontend/src/core/node.hpp b/src/frontends/onnx/frontend/src/core/node.hpp index 637d4676ab9235..451c980f309cab 100644 --- a/src/frontends/onnx/frontend/src/core/node.hpp +++ b/src/frontends/onnx/frontend/src/core/node.hpp @@ -38,6 +38,8 @@ class Subgraph; class Tensor; class SparseTensor; class Attribute; +class DecoderBaseOperation; +class TranslateSession; using ::ONNX_NAMESPACE::NodeProto; @@ -46,6 +48,7 @@ class Node { Node() = delete; // TODO: hide this ctor since it uses protobufs generated structures Node(const NodeProto& node_proto, Graph* graph); + Node(const DecoderBaseOperation& decoder, TranslateSession* translate_session); Node(Node&&) noexcept; Node(const Node&); @@ -59,6 +62,7 @@ class Node { const std::string& get_name() const; std::vector get_attribute_names() const; const Attribute& get_attribute(const std::string& name) const; + ov::Any get_attribute_any(const std::string& name) const; /// \brief Describe the ONNX Node to make debugging graphs easier /// Function will return the Node's name if it has one, or the names of its outputs. @@ -68,7 +72,7 @@ class Node { const std::string& input(int index) const; std::size_t get_inputs_size() const; - const std::vector>& get_output_names() const; + const std::vector> get_output_names() const; const std::string& output(int index) const; std::size_t get_outputs_size() const; @@ -98,13 +102,29 @@ class Node { T default_value, ov::element::Type type) const; + inline bool has_decoder() const { + return m_decoder != nullptr; + } + + TranslateSession* get_translate_session() const { + return m_translate_session; + } + private: + template + std::shared_ptr get_decoder_attribute_as_constant(const std::string& name) const; + + template + std::shared_ptr get_decoder_attribute_as_constant(const std::string& name, + T default_value) const; class Impl; // In this case we need custom deleter, because Impl is an incomplete // type. Node's are elements of std::vector. Without custom deleter // compilation fails; the compiler is unable to parameterize an allocator's // default deleter due to incomple type. std::unique_ptr m_pimpl; + const DecoderBaseOperation* m_decoder; + TranslateSession* m_translate_session; }; template <> @@ -156,6 +176,10 @@ std::vector Node::get_attribute_value(const std::string& name, template <> std::vector Node::get_attribute_value(const std::string& name, std::vector default_value) const; +template <> +std::shared_ptr Node::get_attribute_value(const std::string& name, + std::shared_ptr default_value) const; + template <> float Node::get_attribute_value(const std::string& name) const; @@ -204,6 +228,9 @@ std::vector Node::get_attribute_value(const std::string& name) con template <> std::vector Node::get_attribute_value(const std::string& name) const; +template <> +std::shared_ptr Node::get_attribute_value(const std::string& name) const; + template <> std::shared_ptr Node::get_attribute_as_constant>( const std::string& name) const; diff --git a/src/frontends/onnx/frontend/src/core/place.cpp b/src/frontends/onnx/frontend/src/core/place.cpp new file mode 100644 index 00000000000000..7a7672cb5ff926 --- /dev/null +++ b/src/frontends/onnx/frontend/src/core/place.cpp @@ -0,0 +1,320 @@ +// Copyright (C) 2018-2025 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "place.hpp" + +#include "openvino/frontend/exception.hpp" +#include "openvino/frontend/node_context.hpp" + +namespace ov { +namespace frontend { +namespace onnx { +bool Place::is_input() const { + const auto& model_ins = m_input_model.get_inputs(); + + const auto cmp = [this](const ov::frontend::Place::Ptr& p) { + return p.get() == this; + }; + return std::find_if(model_ins.begin(), model_ins.end(), cmp) != model_ins.end(); +} + +bool Place::is_output() const { + const auto& model_outs = m_input_model.get_outputs(); + const auto cmp = [this](const ov::frontend::Place::Ptr& p) { + return p.get() == this; + }; + return std::find_if(model_outs.begin(), model_outs.end(), cmp) != model_outs.end(); +} + +OpPlace::OpPlace(const ov::frontend::InputModel& input_model, std::shared_ptr op_decoder) + : Place(input_model, {op_decoder->get_op_name()}), + m_op_decoder(op_decoder), + m_back_edge_set(false) {} + +void OpPlace::set_next_iteration_back_edge(const std::string& next_iteration_producer_name, + size_t next_iteration_producer_output_port_idx) { + m_next_iteration_producer_name = next_iteration_producer_name; + m_next_iteration_producer_output_port_idx = next_iteration_producer_output_port_idx; + m_back_edge_set = true; +} + +void OpPlace::get_next_iteration_back_edge(std::string& next_iteration_producer_name, + size_t& next_iteration_producer_output_port_idx) const { + FRONT_END_GENERAL_CHECK(m_back_edge_set, "[ONNX Frontend] internal error: back edge for NextIteration is not set"); + next_iteration_producer_name = m_next_iteration_producer_name; + next_iteration_producer_output_port_idx = m_next_iteration_producer_output_port_idx; +} + +const std::vector>& OpPlace::get_output_ports() const { + return m_output_ports; +} + +const std::map>>& OpPlace::get_input_ports() const { + return m_input_ports; +} + +std::shared_ptr OpPlace::get_decoder() const { + return m_op_decoder; +} + +void OpPlace::add_out_port(const std::shared_ptr& output, int idx) { + FRONT_END_GENERAL_CHECK(idx >= 0, "Output port index to be added is negative."); + size_t output_port_index = static_cast(idx); + while (output_port_index >= m_output_ports.size()) { + m_output_ports.push_back(std::shared_ptr()); + } + m_output_ports[output_port_index] = output; +} + +void OpPlace::add_in_port(const std::shared_ptr& input, const std::string& name) { + m_input_ports[name].push_back(input); +} + +ov::frontend::Place::Ptr OpPlace::get_input_port(const std::string& name) const { + FRONT_END_GENERAL_CHECK(m_input_ports.at(name).size() == 1, "Only one input port should exist."); + return m_input_ports.at(name)[0]; +} + +ov::frontend::Place::Ptr OpPlace::get_input_port(int outputPortIndex) const { + FRONT_END_GENERAL_CHECK(m_input_ports.size() == 1, "Only one named input port should exist."); + return m_input_ports.begin()->second[outputPortIndex]; +} + +ov::frontend::Place::Ptr OpPlace::get_output_port(int outputPortIndex) const { + FRONT_END_GENERAL_CHECK(outputPortIndex >= 0, "outputPortIndex is negative."); + size_t output_port_index = static_cast(outputPortIndex); + FRONT_END_GENERAL_CHECK(m_output_ports.size() > output_port_index, "No port with index: ", output_port_index); + return m_output_ports[output_port_index]; +} + +ov::frontend::Place::Ptr OpPlace::get_output_port() const { + FRONT_END_GENERAL_CHECK(m_output_ports.size() == 1, "Only one output port should exist."); + return m_output_ports[0]; +} + +ov::frontend::Place::Ptr OpPlace::get_input_port() const { + FRONT_END_GENERAL_CHECK(m_input_ports.size() == 1 && m_input_ports.begin()->second.size() == 1, + "Only one input port should exist."); + return m_input_ports.begin()->second[0]; +} + +std::vector OpPlace::get_consuming_operations() const { + std::vector consuming_ops; + for (const auto& out_port : m_output_ports) { + auto consuming_ops_out = out_port->get_consuming_operations(); + consuming_ops.insert(consuming_ops.end(), consuming_ops_out.begin(), consuming_ops_out.end()); + } + return consuming_ops; +} + +std::vector OpPlace::get_consuming_operations(int outputPortIndex) const { + return get_output_port(outputPortIndex)->get_consuming_operations(); +} + +std::vector OpPlace::get_consuming_ports() const { + std::vector consuming_ports; + for (const auto& out_port : m_output_ports) { + auto consuming_ops_out = out_port->get_consuming_ports(); + consuming_ports.insert(consuming_ports.end(), consuming_ops_out.begin(), consuming_ops_out.end()); + } + return consuming_ports; +} + +ov::frontend::Place::Ptr OpPlace::get_input_port(const std::string& inputName, int inputPortIndex) const { + FRONT_END_GENERAL_CHECK(inputPortIndex >= 0, "inputPortIndex is negative."); + size_t input_port_index = static_cast(inputPortIndex); + FRONT_END_GENERAL_CHECK(input_port_index <= m_input_ports.at(inputName).size(), "inputPortIndex is out of bounds."); + return m_input_ports.at(inputName)[input_port_index]; +} + +ov::frontend::Place::Ptr OpPlace::get_source_tensor() const { + return get_input_port()->get_source_tensor(); +} + +ov::frontend::Place::Ptr OpPlace::get_source_tensor(const std::string& inputName) const { + return get_input_port(inputName)->get_source_tensor(); +} + +ov::frontend::Place::Ptr OpPlace::get_source_tensor(int inputPortIndex) const { + return get_input_port(inputPortIndex)->get_source_tensor(); +} + +ov::frontend::Place::Ptr OpPlace::get_source_tensor(const std::string& inputName, int inputPortIndex) const { + return get_input_port(inputName, inputPortIndex)->get_source_tensor(); +} + +ov::frontend::Place::Ptr OpPlace::get_target_tensor() const { + return get_output_port()->get_target_tensor(); +} + +ov::frontend::Place::Ptr OpPlace::get_producing_operation(const std::string& inputName) const { + return get_input_port(inputName)->get_producing_operation(); +} + +ov::frontend::Place::Ptr OpPlace::get_producing_operation(const std::string& inputName, int inputPortIndex) const { + return get_input_port(inputName, inputPortIndex)->get_producing_operation(); +} + +ov::frontend::Place::Ptr OpPlace::get_producing_operation() const { + return get_input_port()->get_producing_operation(); +} + +ov::frontend::Place::Ptr OpPlace::get_producing_operation(int inputPortIndex) const { + return get_input_port(inputPortIndex)->get_producing_operation(); +} + +ov::frontend::Place::Ptr OpPlace::get_target_tensor(int outputPortIndex) const { + return get_output_port(outputPortIndex)->get_target_tensor(); +} + +TensorPlace::TensorPlace(const ov::frontend::InputModel& input_model, + const ov::PartialShape& pshape, + ov::element::Type type, + const std::vector& names) + : Place(input_model, names), + m_pshape(pshape), + m_type(type) { + m_operation_name = (names.size() > 0) ? names[0] : m_operation_name; +} + +TensorPlace::TensorPlace(const ov::frontend::InputModel& input_model, + const ov::PartialShape& pshape, + ov::element::Type type, + const std::vector& names, + const std::string& operation_name) + : Place(input_model, names), + m_pshape(pshape), + m_type(type), + m_operation_name(operation_name) {} + +std::vector TensorPlace::get_consuming_ports() const { + std::vector consuming_ports; + for (const auto& consuming_port : m_consuming_ports) { + if (const auto& locked = consuming_port.lock()) { + consuming_ports.push_back(locked); + } else { + FRONT_END_THROW("Consuming Port has expired."); + } + } + return consuming_ports; +} + +ov::frontend::Place::Ptr TensorPlace::get_producing_port() const { + FRONT_END_GENERAL_CHECK(m_producing_ports.size() == 1, "Only one producing port is supported."); + if (const auto& producing_port = m_producing_ports[0].lock()) { + return producing_port; + } + FRONT_END_THROW("Producing Port has expired."); +} + +void TensorPlace::add_producing_port(const std::shared_ptr& out_port) { + m_producing_ports.push_back(out_port); +} + +void TensorPlace::add_consuming_port(const std::shared_ptr& in_port) { + m_consuming_ports.push_back(in_port); +} + +std::vector TensorPlace::get_consuming_operations() const { + std::vector consuming_ops; + for (const auto& consuming_port : m_consuming_ports) { + if (auto port_ptr = consuming_port.lock()) { + auto port_consuming_ops = port_ptr->get_consuming_operations(); + consuming_ops.insert(consuming_ops.end(), port_consuming_ops.begin(), port_consuming_ops.end()); + } else { + FRONT_END_THROW("Port has expired."); + } + } + return consuming_ops; +} + +bool TensorPlace::is_equal_data(const ov::frontend::Place::Ptr& another) const { + auto consuming_ports = get_consuming_ports(); + bool eq_to_consuming_port = + std::any_of(consuming_ports.begin(), consuming_ports.end(), [&another](const Ptr& place) { + return place->is_equal(another); + }); + return is_equal(another) || get_producing_port()->is_equal(another) || eq_to_consuming_port; +} + +ov::frontend::Place::Ptr TensorPlace::get_producing_operation() const { + return get_producing_port()->get_producing_operation(); +} + +std::shared_ptr InPortPlace::get_op() { + if (const auto& op = m_op.lock()) { + return op; + } + FRONT_END_THROW("Operation has expired."); +} + +void InPortPlace::set_source_tensor(const std::weak_ptr& source_tensor) { + m_source_tensor = source_tensor; +} + +std::vector InPortPlace::get_consuming_operations() const { + if (const auto& consuming_op = m_op.lock()) { + return {consuming_op}; + } + FRONT_END_THROW("Operation has expired."); +} + +ov::frontend::Place::Ptr InPortPlace::get_source_tensor() const { + if (const auto& tensor = m_source_tensor.lock()) { + return tensor; + } + FRONT_END_THROW("Source Tensor has expired."); +} + +ov::frontend::Place::Ptr InPortPlace::get_producing_port() const { + return get_source_tensor()->get_producing_port(); +} + +bool InPortPlace::is_equal_data(const ov::frontend::Place::Ptr& another) const { + return get_source_tensor()->is_equal_data(another); +} + +ov::frontend::Place::Ptr InPortPlace::get_producing_operation() const { + return get_producing_port()->get_producing_operation(); +} + +std::vector OutPortPlace::get_consuming_operations() const { + if (auto tensor_ptr = m_target_tensor.lock()) { + return tensor_ptr->get_consuming_operations(); + } + FRONT_END_THROW("Tensor has expired."); +} + +void OutPortPlace::set_target_tensor(const std::weak_ptr& target_tensor) { + m_target_tensor = target_tensor; +} + +std::vector OutPortPlace::get_consuming_ports() const { + if (auto tensor_ptr = m_target_tensor.lock()) { + return tensor_ptr->get_consuming_ports(); + } + FRONT_END_THROW("Tensor has expired."); +} + +bool OutPortPlace::is_equal_data(const ov::frontend::Place::Ptr& another) const { + return get_target_tensor()->is_equal_data(another); +} + +ov::frontend::Place::Ptr OutPortPlace::get_target_tensor() const { + if (const auto& target_tensor = m_target_tensor.lock()) { + return target_tensor; + } + FRONT_END_THROW("Target Tensor has expired."); +} + +ov::frontend::Place::Ptr OutPortPlace::get_producing_operation() const { + if (auto op = m_op.lock()) { + return op; + } + FRONT_END_THROW("Operation has expired."); +} + +} // namespace onnx +} // namespace frontend +} // namespace ov diff --git a/src/frontends/onnx/frontend/src/core/place.hpp b/src/frontends/onnx/frontend/src/core/place.hpp new file mode 100644 index 00000000000000..01b123d1af9a6f --- /dev/null +++ b/src/frontends/onnx/frontend/src/core/place.hpp @@ -0,0 +1,200 @@ +// Copyright (C) 2018-2025 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include "openvino/frontend/decoder.hpp" +#include "openvino/frontend/frontend.hpp" + +namespace ov { +namespace frontend { +namespace onnx { + +class TensorPlace; +class OpPlace; + +class Place : public ov::frontend::Place { +public: + Place(const ov::frontend::InputModel& input_model, const std::vector& names) + : m_input_model(input_model), + m_names(names) {} + + explicit Place(const ov::frontend::InputModel& input_model) : Place(input_model, std::vector{}) {} + + ~Place() override = default; + + bool is_input() const override; + bool is_output() const override; + bool is_equal(const Ptr& another) const override { + return this == another.get(); + } + + std::vector get_names() const override { + return m_names; + } + void set_names(const std::vector& names) { + m_names = names; + } + +private: + const ov::frontend::InputModel& m_input_model; + std::vector m_names; +}; + +class InPortPlace : public Place { +public: + explicit InPortPlace(const ov::frontend::InputModel& input_model) : Place(input_model) {} + + void set_op(const std::weak_ptr& op) { + m_op = op; + } + void set_source_tensor(const std::weak_ptr& source_tensor); + + // Internal usage + std::shared_ptr get_op(); + + // External usage + std::vector get_consuming_operations() const override; + Ptr get_producing_operation() const override; + ov::frontend::Place::Ptr get_source_tensor() const override; + Ptr get_producing_port() const override; + + bool is_equal_data(const Ptr& another) const override; + +private: + std::weak_ptr m_source_tensor; + std::weak_ptr m_op; +}; + +class OutPortPlace : public Place { +public: + explicit OutPortPlace(const ov::frontend::InputModel& input_model) : Place(input_model) {} + + void set_op(const std::weak_ptr& op) { + m_op = op; + } + void set_target_tensor(const std::weak_ptr& target_tensor); + + // External usage + std::vector get_consuming_operations() const override; + ov::frontend::Place::Ptr get_producing_operation() const override; + std::vector get_consuming_ports() const override; + Ptr get_target_tensor() const override; + bool is_equal_data(const Ptr& another) const override; + +private: + std::weak_ptr m_op; + std::weak_ptr m_target_tensor; +}; + +class OpPlace : public Place { +public: + OpPlace(const ov::frontend::InputModel& input_model, std::shared_ptr op_decoder); + + void add_in_port(const std::shared_ptr& input, const std::string& name); + void add_out_port(const std::shared_ptr& output, int idx); + + // Internal usage + const std::vector>& get_output_ports() const; + const std::map>>& get_input_ports() const; + std::shared_ptr get_decoder() const; + + // External API methods + std::vector get_consuming_ports() const override; + + Ptr get_output_port() const override; + Ptr get_output_port(int outputPortIndex) const override; + + Ptr get_input_port() const override; + Ptr get_input_port(int inputPortIndex) const override; + Ptr get_input_port(const std::string& inputName) const override; + Ptr get_input_port(const std::string& inputName, int inputPortIndex) const override; + + std::vector get_consuming_operations() const override; + std::vector get_consuming_operations(int outputPortIndex) const override; + + Ptr get_producing_operation() const override; + Ptr get_producing_operation(int inputPortIndex) const override; + Ptr get_producing_operation(const std::string& inputName) const override; + Ptr get_producing_operation(const std::string& inputName, int inputPortIndex) const override; + + Ptr get_source_tensor() const override; + Ptr get_source_tensor(int inputPortIndex) const override; + Ptr get_source_tensor(const std::string& inputName) const override; + Ptr get_source_tensor(const std::string& inputName, int inputPortIndex) const override; + + Ptr get_target_tensor() const override; + Ptr get_target_tensor(int outputPortIndex) const override; + + // set back edge for OpPlace of NextIteration operation + // this is needed since we break a cycle in a graph + void set_next_iteration_back_edge(const std::string& next_iteration_producer_name, + size_t next_iteration_producer_output_port_idx); + void get_next_iteration_back_edge(std::string& next_iteration_producer_name, + size_t& next_iteration_producer_output_port_idx) const; + +private: + std::shared_ptr m_op_decoder; + std::map>> m_input_ports; + std::vector> m_output_ports; + + // flag if back edge is set + bool m_back_edge_set; + std::string m_next_iteration_producer_name; + size_t m_next_iteration_producer_output_port_idx; +}; + +class TensorPlace : public Place { +public: + TensorPlace(const ov::frontend::InputModel& input_model, + const ov::PartialShape& pshape, + ov::element::Type type, + const std::vector& names); + + TensorPlace(const ov::frontend::InputModel& input_model, + const ov::PartialShape& pshape, + ov::element::Type type, + const std::vector& names, + const std::string& operation_name); + + void add_producing_port(const std::shared_ptr& out_port); + void add_consuming_port(const std::shared_ptr& in_port); + + // Internal usage + const PartialShape& get_partial_shape() const { + return m_pshape; + } + const element::Type& get_element_type() const { + return m_type; + } + const std::string& get_operation_name() const { + return m_operation_name; + } + void set_partial_shape(const PartialShape& pshape) { + m_pshape = pshape; + } + void set_element_type(const element::Type& type) { + m_type = type; + } + + // External usage + Ptr get_producing_operation() const override; + std::vector get_consuming_operations() const override; + std::vector get_consuming_ports() const override; + Ptr get_producing_port() const override; + bool is_equal_data(const Ptr& another) const override; + +private: + PartialShape m_pshape; + element::Type m_type; + // store original node name from which tensor place is created + std::string m_operation_name; + + std::vector> m_producing_ports; + std::vector> m_consuming_ports; +}; + +} // namespace onnx +} // namespace frontend +} // namespace ov diff --git a/src/frontends/onnx/frontend/src/core/sparse_tensor.hpp b/src/frontends/onnx/frontend/src/core/sparse_tensor.hpp index 35563b76777a46..c1f1abec3182cd 100644 --- a/src/frontends/onnx/frontend/src/core/sparse_tensor.hpp +++ b/src/frontends/onnx/frontend/src/core/sparse_tensor.hpp @@ -33,6 +33,19 @@ class SparseTensor { m_shape = ov::Shape{}; } } + SparseTensor(const std::shared_ptr& values, + const std::shared_ptr& indices, + const ov::PartialShape& shape) + : m_values{values}, + m_indices{indices}, + m_shape{shape.get_shape()} { + if (m_shape == ov::Shape{0}) { + // It's possible to construct a sparse tensor in ONNX with "dims: 0" property + // Such tensor contains a scalar. This results in a ov::Shape{0} stored in m_shape. + // In OpenVINO a scalar is represented with ov::Shape{} and thus this replacement. + m_shape = ov::Shape{}; + } + } SparseTensor(const SparseTensor&) = default; SparseTensor(SparseTensor&&) = default; @@ -44,7 +57,7 @@ class SparseTensor { return m_shape; } - const std::string& get_name() const { + const std::string get_name() const { return m_values.get_name(); } diff --git a/src/frontends/onnx/frontend/src/core/tensor.cpp b/src/frontends/onnx/frontend/src/core/tensor.cpp index d99ce876d35e16..6e4574b30dc8f0 100644 --- a/src/frontends/onnx/frontend/src/core/tensor.cpp +++ b/src/frontends/onnx/frontend/src/core/tensor.cpp @@ -4,15 +4,37 @@ #include "core/tensor.hpp" +#include "input_model.hpp" + namespace ov { namespace frontend { namespace onnx { +detail::MappedMemoryHandles TensorONNXPlace::get_mmap_cache() { + const auto model_onnx = dynamic_cast(&m_input_model); + return model_onnx->get_mmap_cache(); +} +detail::LocalStreamHandles TensorONNXPlace::get_stream_cache() { + const auto model_onnx = dynamic_cast(&m_input_model); + return model_onnx->get_stream_cache(); +} + +Tensor::Tensor(const std::shared_ptr& tensor_place) { + m_tensor_proto = nullptr; + m_shape = tensor_place->get_partial_shape().get_shape(); + m_model_dir = ""; + m_mmap_cache = tensor_place->get_mmap_cache(); + m_tensor_place = tensor_place; +} + template <> std::vector Tensor::get_data() const { if (has_external_data()) { return get_external_data(); } + if (m_tensor_place != nullptr) { + return detail::__get_data(m_tensor_place->get_data(), m_tensor_place->get_data_size()); + } if (m_tensor_proto->has_raw_data()) { return detail::__get_raw_data(m_tensor_proto->raw_data(), m_tensor_proto->data_type()); } @@ -27,6 +49,9 @@ std::vector Tensor::get_data() const { if (has_external_data()) { return get_external_data(); } + if (m_tensor_place != nullptr) { + return detail::__get_data(m_tensor_place->get_data(), m_tensor_place->get_data_size()); + } if (m_tensor_proto->has_raw_data()) { return detail::__get_raw_data(m_tensor_proto->raw_data(), m_tensor_proto->data_type()); } @@ -41,6 +66,25 @@ std::vector Tensor::get_data() const { if (has_external_data()) { return get_external_data(); } + if (m_tensor_place != nullptr) { + if (m_tensor_place->is_raw()) { + return detail::__get_data(m_tensor_place->get_data(), + m_tensor_place->get_data_size()); + } + using std::begin; + using std::end; + + const auto& int32_data = std::vector( + static_cast(m_tensor_place->get_data()), + static_cast(m_tensor_place->get_data()) + m_tensor_place->get_data_size()); + std::vector float16_data; + float16_data.reserve(int32_data.size()); + std::transform(begin(int32_data), end(int32_data), std::back_inserter(float16_data), [](int32_t elem) { + return ov::float16::from_bits(static_cast(elem)); + }); + + return detail::__get_data(float16_data); + } if (m_tensor_proto->has_raw_data()) { return detail::__get_raw_data(m_tensor_proto->raw_data(), m_tensor_proto->data_type()); } @@ -65,6 +109,13 @@ std::vector Tensor::get_data() const { if (has_external_data()) { return get_external_data(); } + if (m_tensor_place != nullptr) { + if (m_tensor_place->is_raw()) { + return detail::__get_data(m_tensor_place->get_data(), + m_tensor_place->get_data_size()); + } + return detail::__get_data(m_tensor_place->get_data(), m_tensor_place->get_data_size()); + } if (m_tensor_proto->has_raw_data()) { return detail::__get_raw_data(m_tensor_proto->raw_data(), m_tensor_proto->data_type()); } @@ -79,6 +130,12 @@ std::vector Tensor::get_data() const { if (has_external_data()) { return get_external_data(); } + if (m_tensor_place != nullptr) { + if (m_tensor_place->is_raw()) { + return detail::__get_data(m_tensor_place->get_data(), m_tensor_place->get_data_size()); + } + return detail::__get_data(m_tensor_place->get_data(), m_tensor_place->get_data_size()); + } if (m_tensor_proto->has_raw_data()) { return detail::__get_raw_data(m_tensor_proto->raw_data(), m_tensor_proto->data_type()); } @@ -94,6 +151,12 @@ std::vector Tensor::get_data() const { if (has_external_data()) { return get_external_data(); } + if (m_tensor_place != nullptr) { + if (m_tensor_place->is_raw()) { + return detail::__get_data(m_tensor_place->get_data(), m_tensor_place->get_data_size()); + } + return detail::__get_data(m_tensor_place->get_data(), m_tensor_place->get_data_size()); + } if (m_tensor_proto->has_raw_data()) { return detail::__get_raw_data(m_tensor_proto->raw_data(), m_tensor_proto->data_type()); } @@ -108,6 +171,9 @@ std::vector Tensor::get_data() const { if (has_external_data()) { return get_external_data(); } + if (m_tensor_place != nullptr) { + return detail::__get_data(m_tensor_place->get_data(), m_tensor_place->get_data_size()); + } if (m_tensor_proto->has_raw_data()) { return detail::__get_raw_data(m_tensor_proto->raw_data(), m_tensor_proto->data_type()); } @@ -122,6 +188,9 @@ std::vector Tensor::get_data() const { if (has_external_data()) { return get_external_data(); } + if (m_tensor_place != nullptr) { + return detail::__get_data(m_tensor_place->get_data(), m_tensor_place->get_data_size()); + } if (m_tensor_proto->has_raw_data()) { return detail::__get_raw_data(m_tensor_proto->raw_data(), m_tensor_proto->data_type()); } @@ -136,6 +205,12 @@ std::vector Tensor::get_data() const { if (has_external_data()) { return get_external_data(); } + if (m_tensor_place != nullptr) { + if (m_tensor_place->is_raw()) { + return detail::__get_data(m_tensor_place->get_data(), m_tensor_place->get_data_size()); + } + return detail::__get_data(m_tensor_place->get_data(), m_tensor_place->get_data_size()); + } if (m_tensor_proto->has_raw_data()) { return detail::__get_raw_data(m_tensor_proto->raw_data(), m_tensor_proto->data_type()); } @@ -151,6 +226,12 @@ std::vector Tensor::get_data() const { if (has_external_data()) { return get_external_data(); } + if (m_tensor_place != nullptr) { + if (m_tensor_place->is_raw()) { + return detail::__get_data(m_tensor_place->get_data(), m_tensor_place->get_data_size()); + } + return detail::__get_data(m_tensor_place->get_data(), m_tensor_place->get_data_size()); + } if (m_tensor_proto->has_raw_data()) { return detail::__get_raw_data(m_tensor_proto->raw_data(), m_tensor_proto->data_type()); } @@ -165,6 +246,12 @@ std::vector Tensor::get_data() const { if (has_external_data()) { return get_external_data(); } + if (m_tensor_place != nullptr) { + if (m_tensor_place->is_raw()) { + return detail::__get_data(m_tensor_place->get_data(), m_tensor_place->get_data_size()); + } + return detail::__get_data(m_tensor_place->get_data(), m_tensor_place->get_data_size()); + } if (m_tensor_proto->has_raw_data()) { return detail::__get_raw_data(m_tensor_proto->raw_data(), m_tensor_proto->data_type()); } @@ -179,6 +266,9 @@ std::vector Tensor::get_data() const { if (has_external_data()) { return get_external_data(); } + if (m_tensor_place != nullptr) { + return detail::__get_data(m_tensor_place->get_data(), m_tensor_place->get_data_size()); + } if (m_tensor_proto->has_raw_data()) { return detail::__get_raw_data(m_tensor_proto->raw_data(), m_tensor_proto->data_type()); } @@ -193,6 +283,14 @@ std::vector Tensor::get_data() const { if (has_external_data()) { return get_external_data(); } + if (m_tensor_place != nullptr) { + if (m_tensor_place->is_raw()) { + return detail::__get_data(m_tensor_place->get_data(), + m_tensor_place->get_data_size()); + } + return detail::__get_data(m_tensor_place->get_data(), + m_tensor_place->get_data_size()); + } if (m_tensor_proto->has_raw_data()) { return detail::__get_raw_data(m_tensor_proto->raw_data(), m_tensor_proto->data_type()); } @@ -217,6 +315,14 @@ std::vector Tensor::get_data() const { if (has_external_data()) { return get_external_data(); } + if (m_tensor_place != nullptr) { + if (m_tensor_place->is_raw()) { + return detail::__get_data(m_tensor_place->get_data(), + m_tensor_place->get_data_size()); + } + return detail::__get_data(m_tensor_place->get_data(), + m_tensor_place->get_data_size()); + } if (m_tensor_proto->has_raw_data()) { return detail::__get_raw_data(m_tensor_proto->raw_data(), m_tensor_proto->data_type()); } @@ -243,6 +349,12 @@ std::vector Tensor::get_data() const { if (has_external_data()) { return get_external_data(); } + if (m_tensor_place != nullptr) { + if (m_tensor_place->is_raw()) { + return detail::__get_data(m_tensor_place->get_data(), m_tensor_place->get_data_size()); + } + return detail::__get_data(m_tensor_place->get_data(), m_tensor_place->get_data_size()); + } if (m_tensor_proto->has_raw_data()) { return detail::__get_raw_data(m_tensor_proto->raw_data(), m_tensor_proto->data_type()); } @@ -254,6 +366,10 @@ std::vector Tensor::get_data() const { template <> std::vector Tensor::get_data() const { + if (m_tensor_place != nullptr) { + FRONT_END_NOT_IMPLEMENTED(get_data); + } + if (has_external_data()) { FRONT_END_THROW("External strings are not supported"); } @@ -267,10 +383,10 @@ std::vector Tensor::get_data() const { } std::shared_ptr Tensor::get_ov_constant() const { - if (m_tensor_proto->has_segment()) { + std::shared_ptr constant{nullptr}; + if (m_tensor_proto != nullptr && m_tensor_proto->has_segment()) { FRONT_END_THROW("Loading segments isn't supported"); } - std::shared_ptr constant{nullptr}; ov::element::Type ov_type = get_ov_type(); size_t element_count = get_data_size(); if (ov::element::is_nibble_type(ov_type)) { @@ -281,7 +397,11 @@ std::shared_ptr Tensor::get_ov_constant() const { } } if (has_external_data()) { - const auto ext_data = detail::TensorExternalData(*m_tensor_proto); + const auto ext_data = m_tensor_place != nullptr + ? detail::TensorExternalData(*m_tensor_place->get_data_location(), + reinterpret_cast(m_tensor_place->get_data()), + m_tensor_place->get_data_size()) + : detail::TensorExternalData(*m_tensor_proto); if (ext_data.data_location() == detail::ORT_MEM_ADDR) { constant = std::make_shared(ov_type, m_shape, ext_data.load_external_mem_data()); } else if (m_mmap_cache) { @@ -304,7 +424,7 @@ std::shared_ptr Tensor::get_ov_constant() const { "The size of the external data file does not match the byte size of an initializer '" + get_name() + "' in the model"); } - } else if (element_count == shape_size(m_shape)) { + } else if (element_count == shape_size(m_shape) && m_tensor_proto != nullptr) { switch (m_tensor_proto->data_type()) { case TensorProto_DataType::TensorProto_DataType_FLOAT: case TensorProto_DataType::TensorProto_DataType_DOUBLE: @@ -356,18 +476,85 @@ std::shared_ptr Tensor::get_ov_constant() const { "BOOL, BFLOAT16, FLOAT8E4M3FN, FLOAT8E5M2, FLOAT, FLOAT16, DOUBLE, INT4, INT8, INT16, INT32, INT64, " "UINT4, UINT8, UINT16, UINT32, UINT64, STRING"); } + } else if (element_count == shape_size(m_shape) && m_tensor_place != nullptr) { +#if 0 + constant = std::make_shared(m_tensor_place->get_element_type(), + m_tensor_place->get_partial_shape().get_shape(), + m_tensor_place->get_data()); +#endif + switch (m_tensor_place->get_element_type()) { + case ov::element::f32: + case ov::element::f64: + case ov::element::i32: + case ov::element::i64: + case ov::element::u32: + case ov::element::u64: + constant = std::make_shared(ov_type, m_shape, get_data_ptr()); + break; + case ov::element::i4: + constant = std::make_shared(ov_type, m_shape, get_data().data()); + break; + case ov::element::i8: + constant = std::make_shared(ov_type, m_shape, get_data().data()); + break; + case ov::element::i16: + constant = std::make_shared(ov_type, m_shape, get_data().data()); + break; + case ov::element::u4: + constant = std::make_shared(ov_type, m_shape, get_data().data()); + break; + case ov::element::u8: + constant = std::make_shared(ov_type, m_shape, get_data().data()); + break; + case ov::element::u16: + constant = std::make_shared(ov_type, m_shape, get_data().data()); + break; + case ov::element::boolean: + constant = std::make_shared(ov_type, m_shape, get_data().data()); + break; + case ov::element::bf16: + constant = std::make_shared(ov_type, m_shape, get_data().data()); + break; + case ov::element::f16: + constant = std::make_shared(ov_type, m_shape, get_data().data()); + break; + case ov::element::f8e4m3: + constant = std::make_shared(ov_type, m_shape, get_data().data()); + break; + case ov::element::f8e5m2: + constant = std::make_shared(ov_type, m_shape, get_data().data()); + break; + default: + ONNX_UNSUPPORTED_DATA_TYPE( + m_tensor_proto->data_type(), + "BOOL, BFLOAT16, FLOAT8E4M3FN, FLOAT8E5M2, FLOAT, FLOAT16, DOUBLE, INT4, INT8, INT16, INT32, INT64, " + "UINT4, UINT8, UINT16, UINT32, UINT64, STRING"); + } } else if (element_count == 0 && m_shape.size() == 0) { constant = common::make_failsafe_constant(ov_type); } else { FRONT_END_THROW("Tensor shape doesn't match data size"); } - if (m_tensor_proto->has_name()) { + if (m_tensor_proto != nullptr && m_tensor_proto->has_name()) { constant->set_friendly_name(get_name()); } + if (m_tensor_place != nullptr) { + const auto& names = m_tensor_place->get_names(); + if (names.size() > 0) { + constant->set_friendly_name(names[0]); + constant->get_default_output().set_names({names.begin(), names.end()}); + } + } return constant; } +void ov::frontend::onnx::TensorONNXPlace::translate(ov::Output& output) { + if (get_names().size() > 0) { + output.set_names({*get_names().begin()}); + } +} + } // namespace onnx } // namespace frontend } // namespace ov diff --git a/src/frontends/onnx/frontend/src/core/tensor.hpp b/src/frontends/onnx/frontend/src/core/tensor.hpp index 39dba2b242c5d8..aa904610323189 100644 --- a/src/frontends/onnx/frontend/src/core/tensor.hpp +++ b/src/frontends/onnx/frontend/src/core/tensor.hpp @@ -17,6 +17,7 @@ #include "openvino/core/type/element_type.hpp" #include "openvino/frontend/exception.hpp" #include "openvino/runtime/aligned_buffer.hpp" +#include "place.hpp" #include "utils/common.hpp" #include "utils/tensor_external_data.hpp" @@ -51,6 +52,19 @@ inline std::vector __get_data(const Container& container) { #endif } +template +inline std::vector __get_data(const void* data, const size_t data_size) { +#if defined(_MSC_VER) +# pragma warning(push) +# pragma warning(disable : 4267) +# pragma warning(disable : 4244) +#endif + return std::vector(static_cast(data), static_cast(data) + data_size); +#if defined(_MSC_VER) +# pragma warning(pop) +#endif +} + template inline std::vector __get_raw_data(const std::string& raw_data, int onnx_data_type) { auto it = reinterpret_cast(raw_data.data()); @@ -60,6 +74,77 @@ inline std::vector __get_raw_data(const std::string& raw_data, int onnx_data_ } // namespace } // namespace detail +using MappedMemoryHandles = std::shared_ptr>>; +using LocalStreamHandles = std::shared_ptr>>; + +class TensorONNXPlace : public ov::frontend::onnx::TensorPlace { +public: + TensorONNXPlace(const ov::frontend::InputModel& input_model, + const ov::PartialShape& pshape, + ov::element::Type type, + const std::vector& names, + const void* data, + const size_t data_size, + std::shared_ptr data_location, + const bool is_raw) + : ov::frontend::onnx::TensorPlace(input_model, pshape, type, names), + m_input_model(input_model), + m_data(data), + m_data_size(data_size), + m_data_location(data_location), + m_is_raw(is_raw) {}; + + void translate(ov::Output& output); + + bool is_input() const override { + return m_input_idx >= 0; + } + size_t get_input_index() const { + FRONT_END_GENERAL_CHECK(is_input(), "This is not input TensorPlace. Can not deliver input index"); + return static_cast(m_input_idx); + } + bool is_output() const override { + return m_output_idx >= 0; + } + size_t get_output_index() const { + FRONT_END_GENERAL_CHECK(is_output(), "This is not output TensorPlace. Can not deliver output index"); + return static_cast(m_output_idx); + } + void set_input_index(const int64_t& idx) { + m_input_idx = idx; + } + void set_output_index(const int64_t& idx) { + m_output_idx = idx; + } + + const void* get_data() const { + return m_data; + } + + size_t get_data_size() const { + return m_data_size; + } + + std::shared_ptr get_data_location() const { + return m_data_location; + } + + bool is_raw() const { + return m_is_raw; + } + + detail::MappedMemoryHandles get_mmap_cache(); + detail::LocalStreamHandles get_stream_cache(); + +protected: + int64_t m_input_idx = -1, m_output_idx = -1; + const ov::frontend::InputModel& m_input_model; + const void* m_data; + size_t m_data_size; + std::shared_ptr m_data_location; + bool m_is_raw; +}; + class Tensor { public: enum class Type { @@ -89,6 +174,7 @@ class Tensor { Tensor() = delete; Tensor(const TensorProto& tensor, const std::string& model_dir, detail::MappedMemoryHandles mmap_cache) : m_tensor_proto{&tensor}, + m_tensor_place(nullptr), m_shape{std::begin(tensor.dims()), std::end(tensor.dims())}, m_model_dir{model_dir}, m_mmap_cache{mmap_cache} { @@ -100,6 +186,8 @@ class Tensor { } } + Tensor(const std::shared_ptr& tensor_place); + Tensor(const Tensor&) = default; Tensor(Tensor&&) = default; @@ -117,7 +205,13 @@ class Tensor { ONNX_UNSUPPORTED_DATA_TYPE(m_tensor_proto->data_type(), "[nothing expected]"); } - const std::string& get_name() const { + const std::string get_name() const { + if (m_tensor_place != nullptr) { + const auto& names = m_tensor_place->get_names(); + if (names.size() <= 0) + FRONT_END_THROW("Tensor has no specified name"); + return names[0]; + } if (!m_tensor_proto->has_name()) { FRONT_END_THROW("Tensor has no specified name"); } @@ -125,6 +219,9 @@ class Tensor { } Type get_type() const { + if (m_tensor_place != nullptr) { + FRONT_END_NOT_IMPLEMENTED(get_type); + } if (!m_tensor_proto->has_data_type()) { FRONT_END_THROW("Tensor has no specified data type"); } @@ -132,6 +229,9 @@ class Tensor { } const ov::element::Type& get_ov_type() const { + if (m_tensor_place != nullptr) { + return m_tensor_place->get_element_type(); + } if (!m_tensor_proto->has_data_type()) { FRONT_END_THROW("Tensor has no specified data type"); } @@ -190,13 +290,20 @@ class Tensor { private: bool has_external_data() const { + if (m_tensor_place != nullptr) { + return m_tensor_place->get_data_location() != nullptr; + } return m_tensor_proto->has_data_location() && m_tensor_proto->data_location() == TensorProto_DataLocation::TensorProto_DataLocation_EXTERNAL; } template std::vector get_external_data() const { - const auto ext_data = detail::TensorExternalData(*m_tensor_proto); + const auto ext_data = m_tensor_place != nullptr + ? detail::TensorExternalData(*m_tensor_place->get_data_location(), + reinterpret_cast(m_tensor_place->get_data()), + m_tensor_place->get_data_size()) + : detail::TensorExternalData(*m_tensor_proto); std::shared_ptr buffer = nullptr; if (ext_data.data_location() == detail::ORT_MEM_ADDR) { buffer = ext_data.load_external_mem_data(); @@ -212,6 +319,10 @@ class Tensor { if (has_external_data()) { FRONT_END_THROW("Unexpected usage of method for externally stored data"); } + if (m_tensor_place != nullptr) { + return m_tensor_place->get_data(); + } + if (m_tensor_proto->has_raw_data()) { return m_tensor_proto->raw_data().data(); } @@ -231,6 +342,14 @@ class Tensor { } size_t get_data_size() const { + if (m_tensor_place != nullptr) { + if (m_tensor_place->is_raw()) { + return m_tensor_place->get_data_size() / + get_onnx_data_size(ov_to_onnx_data_type(m_tensor_place->get_element_type())); + } else { + return m_tensor_place->get_data_size(); + } + } if (has_external_data()) { const auto ext_data = detail::TensorExternalData(*m_tensor_proto); return ext_data.size() / get_onnx_data_size(m_tensor_proto->data_type()); @@ -271,6 +390,7 @@ class Tensor { } const TensorProto* m_tensor_proto; + std::shared_ptr m_tensor_place; ov::Shape m_shape; std::string m_model_dir; detail::MappedMemoryHandles m_mmap_cache; diff --git a/src/frontends/onnx/frontend/src/frontend.cpp b/src/frontends/onnx/frontend/src/frontend.cpp index 0165f6f9476478..d87291aace2d22 100644 --- a/src/frontends/onnx/frontend/src/frontend.cpp +++ b/src/frontends/onnx/frontend/src/frontend.cpp @@ -20,8 +20,10 @@ #include #include +#include "core/graph_iterator_proto.hpp" #include "input_model.hpp" #include "onnx_common/onnx_model_validator.hpp" +#include "onnx_framework_node.hpp" #include "openvino/core/rt_info/weightless_caching_attributes.hpp" #include "openvino/core/so_extension.hpp" #include "openvino/frontend/exception.hpp" @@ -30,8 +32,12 @@ #include "openvino/frontend/onnx/extension/conversion.hpp" #include "openvino/frontend/onnx/frontend.hpp" #include "openvino/frontend/onnx/visibility.hpp" +#include "openvino/op/parameter.hpp" +#include "openvino/op/result.hpp" +#include "openvino/util/log.hpp" #include "ops_bridge.hpp" #include "transformations/resolve_names_collisions.hpp" +#include "translate_session.hpp" #include "utils/common.hpp" #include "utils/onnx_internal.hpp" @@ -77,7 +83,8 @@ ONNX_FRONTEND_C_API void* get_front_end_data() { return res; } -InputModel::Ptr FrontEnd::load_impl(const std::vector& variants) const { +ov::frontend::InputModel::Ptr FrontEnd::load_impl(const std::vector& variants) const { + const bool gi_enabled = std::getenv("ONNX_ITERATOR") != nullptr; if (variants.empty()) { return nullptr; } @@ -86,12 +93,28 @@ InputModel::Ptr FrontEnd::load_impl(const std::vector& variants) const if (variants[0].is()) { const auto path = variants[0].as(); - return std::make_shared(path, enable_mmap, m_extensions); + if (!gi_enabled) { + return std::make_shared(path, enable_mmap, m_extensions); + } + OPENVINO_DEBUG("[ONNX Frontend] Enabled an experimental GraphIteratorProto interface!!!"); + GraphIteratorProto::Ptr graph_iterator = + std::make_shared(enable_mmap ? Internal_MMAP : Internal_Stream); + graph_iterator->initialize(path); + graph_iterator->reset(); + return std::make_shared(graph_iterator, enable_mmap); } #if defined(OPENVINO_ENABLE_UNICODE_PATH_SUPPORT) && defined(_WIN32) if (variants[0].is()) { const auto path = variants[0].as(); - return std::make_shared(path, enable_mmap, m_extensions); + if (!gi_enabled) { + return std::make_shared(path, enable_mmap, m_extensions); + } + OPENVINO_DEBUG("[ONNX Frontend] Enabled an experimental GraphIteratorProto interface!!!"); + GraphIteratorProto::Ptr graph_iterator = + std::make_shared(enable_mmap ? Internal_MMAP : Internal_Stream); + graph_iterator->initialize(path); + graph_iterator->reset(); + return std::make_shared(graph_iterator, enable_mmap); } #endif if (variants[0].is()) { @@ -119,10 +142,21 @@ InputModel::Ptr FrontEnd::load_impl(const std::vector& variants) const return std::make_shared(std::make_shared(*model_proto_ptr), m_extensions); } // !!! End of Experimental feature + if (variants[0].is()) { + auto graph_iterator = variants[0].as(); + return std::make_shared( + graph_iterator, + enable_mmap); // enable_mmap is a hint for a fallback in case external GraphIterator cannot work with + // external data + } return nullptr; } -std::shared_ptr FrontEnd::convert_partially(const InputModel::Ptr& input_model) const { +std::shared_ptr FrontEnd::convert_partially(const ov::frontend::InputModel::Ptr& input_model) const { + auto unify_model = std::dynamic_pointer_cast(input_model); + if (unify_model != nullptr) { + return convert_partially_unify(unify_model); + } auto model_onnx = std::dynamic_pointer_cast(input_model); FRONT_END_GENERAL_CHECK(model_onnx != nullptr, "Invalid input model"); @@ -159,6 +193,11 @@ void FrontEnd::normalize(const std::shared_ptr& model) const { } std::shared_ptr FrontEnd::convert(const InputModel::Ptr& input_model) const { + auto unify_model = std::dynamic_pointer_cast(input_model); + if (unify_model != nullptr) { + return convert_unify(unify_model); + } + auto model_onnx = std::dynamic_pointer_cast(input_model); FRONT_END_GENERAL_CHECK(model_onnx != nullptr, "Invalid input model"); @@ -193,7 +232,11 @@ void FrontEnd::convert(const std::shared_ptr& partially_converted) co normalize(partially_converted); } -std::shared_ptr FrontEnd::decode(const InputModel::Ptr& model) const { +std::shared_ptr FrontEnd::decode(const ov::frontend::InputModel::Ptr& model) const { + auto unify_model = std::dynamic_pointer_cast(model); + if (unify_model != nullptr) { + return decode_unify(unify_model); + } auto model_onnx = std::dynamic_pointer_cast(model); FRONT_END_GENERAL_CHECK(model_onnx != nullptr, "Invalid input model"); return model_onnx->decode(); @@ -270,9 +313,80 @@ bool FrontEnd::supported_impl(const std::vector& variants) const { return true; } // !!! End of Experimental feature + if (variants[0].is()) { + return true; + } return false; } +std::shared_ptr FrontEnd::convert_unify(const InputModel::Ptr& input_model) const { + std::shared_ptr ov_model; + if (!m_transformation_extensions.empty()) { + auto ov_model = decode(input_model); + + ov::pass::Manager manager("Frontend:onnx:transformation_extensions"); + for (const auto& transformation : m_transformation_extensions) { + transformation->register_pass(manager); + } + manager.run_passes(ov_model); + convert(ov_model); + return ov_model; + } + + translate_graph(input_model, false, false, ov_model); + + std::stringstream error_messages; + if (ov::frontend::onnx::common::collect_translation_exceptions(ov_model, m_extensions.telemetry, &error_messages)) { + FRONT_END_THROW(error_messages.str()); + } + + normalize(ov_model); + return ov_model; +} + +std::shared_ptr FrontEnd::convert_partially_unify(const InputModel::Ptr& input_model) const { + if (!m_transformation_extensions.empty()) { + auto function = decode_unify(input_model); + ov::pass::Manager manager("Frontend:ONNX:convert_partially"); + for (const auto& transformation : m_transformation_extensions) { + transformation->register_pass(manager); + } + manager.run_passes(function); + convert(function); + return function; + } + + std::shared_ptr ov_model; + translate_graph(input_model, false, false, ov_model); + + ov::frontend::onnx::common::collect_translation_exceptions(ov_model, m_extensions.telemetry); + + normalize(ov_model); + return ov_model; +} +void FrontEnd::translate_graph(const InputModel::Ptr& input_model, + bool fail_fast, + bool /* no_conversion */, + std::shared_ptr& ov_model) const { + auto model_onnx = std::dynamic_pointer_cast(input_model); + FRONT_END_GENERAL_CHECK(model_onnx != nullptr, "Invalid input model"); + auto translators_map = std::make_shared(); + TranslateSession translate_session(input_model, translators_map, "MainGraph"); + translate_session.set_fail_fast(fail_fast); + try { + ov_model = translate_session.get_converted_model(); + } catch (const std::exception& e) { + throw e; + } + return; +} + +std::shared_ptr FrontEnd::decode_unify(const InputModel::Ptr& model) const { + std::shared_ptr ov_model; + translate_graph(model, false, true, ov_model); + return ov_model; +} + void FrontEnd::add_extension(const std::shared_ptr& extension) { if (auto telemetry = std::dynamic_pointer_cast(extension)) { m_extensions.telemetry = telemetry; diff --git a/src/frontends/onnx/frontend/src/input_model.cpp b/src/frontends/onnx/frontend/src/input_model.cpp index 5d318bfd09d231..00d9972f9c5eac 100644 --- a/src/frontends/onnx/frontend/src/input_model.cpp +++ b/src/frontends/onnx/frontend/src/input_model.cpp @@ -5,8 +5,10 @@ #include "input_model.hpp" #include "openvino/frontend/exception.hpp" +#include "openvino/op/constant.hpp" #include "openvino/util/file_util.hpp" #include "openvino/util/log.hpp" +#include "ops_bridge.hpp" #include "place.hpp" using namespace ov; @@ -556,3 +558,426 @@ void InputModel::reshape_model_inputs(std::shared_ptr& model) { model->reshape(actual_inputs_to_reshape); } } + +namespace ov { +namespace frontend { +namespace onnx { +namespace unify { + +class InputModel::InputModelONNXImpl { +public: + InputModelONNXImpl(const GraphIterator::Ptr& graph_iterator, + const ov::frontend::InputModel& input_model, + const bool enable_mmap); + InputModelONNXImpl(const GraphIterator::Ptr& graph_iterator, + const ov::frontend::InputModel& input_model, + const std::shared_ptr& telemetry, + const bool enable_mmap); + InputModelONNXImpl(const GraphIterator::Ptr& graph_iterator, + const ov::frontend::InputModel& input_model, + unify::InputModel* parent_model); + + std::vector get_inputs() const; + std::vector get_outputs() const; + ov::frontend::Place::Ptr get_place_by_tensor_name(const std::string& tensorName) const; + + ///// Searching for places ///// + std::vector>& get_op_places() { + return m_op_places; + } + std::map>& get_tensor_places() { + return m_tensor_places; + } + + ///// Naming and annotation ///// + void set_name_for_tensor(const Place::Ptr& tensor, const std::string& new_name); + void add_name_for_tensor(const Place::Ptr& tensor, const std::string& new_name); + void set_name_for_operation(const Place::Ptr& operation, const std::string& new_name); + + ///// Setting / getting tensor properties ///// + void set_partial_shape(ov::frontend::Place::Ptr place, const ov::PartialShape& shape); + ov::PartialShape get_partial_shape(ov::frontend::Place::Ptr place) const; + void set_element_type(ov::frontend::Place::Ptr place, const ov::element::Type& type); + ov::element::Type get_element_type(ov::frontend::Place::Ptr place) const; + void set_tensor_value(ov::frontend::Place::Ptr place, const void* value); + + ///// Topology Editing ///// + void override_all_outputs(const std::vector& outputs); + void override_all_inputs(const std::vector& inputs); + void extract_subgraph(const std::vector& inputs, + const std::vector& outputs); + + std::shared_ptr get_telemetry_extension() const { + return m_telemetry; + } + + bool is_enabled_mmap() const { + return m_enable_mmap; + } + + detail::MappedMemoryHandles get_mmap_cache() const { + return m_mmap_cache; + } + + detail::LocalStreamHandles get_stream_cache() const { + return m_stream_cache; + } + +private: + void load_model(); + void clean_up(); + + std::vector> m_op_places; + std::map> m_op_places_map; + std::map> m_tensor_places; + std::vector m_inputs; + std::vector m_outputs; + + std::shared_ptr m_graph_iterator; + const ov::frontend::InputModel& m_input_model; + std::vector> m_subgraphs; + std::shared_ptr m_telemetry; + bool m_enable_mmap; + + // This is used for keeping MMAP cache handles + detail::MappedMemoryHandles m_mmap_cache; + // This is used for keeping a readed external data without MMAP + detail::LocalStreamHandles m_stream_cache; +}; + +namespace { +std::shared_ptr decode_tensor_place( + const ov::frontend::onnx::TensorMetaInfo& tensor_meta_info, + const ov::frontend::InputModel& model) { + auto tensor_place = + std::make_shared(model, + tensor_meta_info.m_partial_shape, + tensor_meta_info.m_element_type, + std::vector{*tensor_meta_info.m_tensor_name}, + tensor_meta_info.m_tensor_data, + tensor_meta_info.m_tensor_data_size, + tensor_meta_info.m_external_location, + tensor_meta_info.m_is_raw); + return tensor_place; +} + +std::shared_ptr decode_input_tensor( + const std::shared_ptr& decoder, + size_t idx, + const ov::frontend::InputModel& model) { + const auto& tensor_meta_info = decoder->get_input_tensor_info(idx); + return decode_tensor_place(tensor_meta_info, model); +} + +std::shared_ptr decode_output_tensor( + const std::shared_ptr& decoder, + size_t idx, + const ov::frontend::InputModel& model) { + const auto& tensor_meta_info = decoder->get_output_tensor_info(idx); + return decode_tensor_place(tensor_meta_info, model); +} +} // namespace + +void InputModel::InputModelONNXImpl::load_model() { + std::map op_statistics; // for telemetry + + m_op_places.reserve(m_graph_iterator->size()); + for (; !m_graph_iterator->is_end(); m_graph_iterator->next()) { + const auto& decoder = m_graph_iterator->get_decoder(); + + if (auto tensor_decoder = std::dynamic_pointer_cast(decoder)) { + auto tensor_place = decode_tensor_place(tensor_decoder->get_tensor_info(), m_input_model); + tensor_place->set_input_index(tensor_decoder->get_input_idx()); + tensor_place->set_output_index(tensor_decoder->get_output_idx()); + // Constant with data has been found + if (tensor_place->get_data() != nullptr) + continue; + auto name = tensor_place->get_names()[0]; + if (m_tensor_places.count(name) == 0) { + m_tensor_places[name] = tensor_place; + if (tensor_place->is_input()) + m_inputs.push_back(tensor_place); + if (tensor_place->is_output()) + m_outputs.push_back(tensor_place); + } + continue; + } + m_op_places.push_back(std::make_shared(m_input_model, decoder)); + + if (m_telemetry) { + op_statistics[decoder->get_op_type()]++; + } + + auto operation_decoder = std::dynamic_pointer_cast(decoder); + FRONT_END_GENERAL_CHECK(operation_decoder, "Operation decoder is expected"); + for (size_t i = 0; i < operation_decoder->get_input_size(); ++i) { + auto place = decode_input_tensor(operation_decoder, i, m_input_model); + auto name = place->get_names()[0]; + if (m_tensor_places.count(name) == 0) { + m_tensor_places[name] = place; + } + } + for (size_t i = 0; i < operation_decoder->get_output_size(); ++i) { + auto place = decode_output_tensor(operation_decoder, i, m_input_model); + auto name = place->get_names()[0]; + if (m_tensor_places.count(name) == 0) + m_tensor_places[name] = place; + } + } + + auto sorting_places_by_idx = [](bool are_input_places) { + return + [are_input_places](const ov::frontend::Place::Ptr& lhs_place, const ov::frontend::Place::Ptr& rhs_place) { + auto onnx_lhs_place = std::dynamic_pointer_cast(lhs_place); + auto onnx_rhs_place = std::dynamic_pointer_cast(rhs_place); + FRONT_END_GENERAL_CHECK(onnx_lhs_place != nullptr && onnx_rhs_place != nullptr, + "ONNX Frontend works with TensorONNXPlaces only"); + size_t rhs_idx, lhs_idx; + if (are_input_places) { + lhs_idx = onnx_lhs_place->get_input_index(); + rhs_idx = onnx_rhs_place->get_input_index(); + } else { + lhs_idx = onnx_lhs_place->get_output_index(); + rhs_idx = onnx_rhs_place->get_output_index(); + } + return lhs_idx < rhs_idx; + }; + }; + std::sort(m_inputs.begin(), m_inputs.end(), sorting_places_by_idx(true)); + std::sort(m_outputs.begin(), m_outputs.end(), sorting_places_by_idx(false)); + + if (m_telemetry) { + for (const auto& op : op_statistics) { + m_telemetry->send_event("op_count", "onnx_" + op.first, static_cast(op.second)); + } + } +} + +InputModel::InputModelONNXImpl::InputModelONNXImpl(const GraphIterator::Ptr& graph_iterator, + const ov::frontend::InputModel& input_model, + const bool enable_mmap) + : m_graph_iterator(graph_iterator), + m_input_model(input_model), + m_enable_mmap(enable_mmap) { + FRONT_END_GENERAL_CHECK(m_graph_iterator, "Null pointer specified for GraphIterator"); + if (m_enable_mmap) { + m_mmap_cache = std::make_shared>>(); + m_stream_cache = nullptr; + } else { + m_mmap_cache = nullptr; + m_stream_cache = std::make_shared>>(); + } + load_model(); +} + +InputModel::InputModelONNXImpl::InputModelONNXImpl(const GraphIterator::Ptr& graph_iterator, + const ov::frontend::InputModel& input_model, + const std::shared_ptr& telemetry, + const bool enable_mmap) + : m_graph_iterator(graph_iterator), + m_input_model(input_model), + m_telemetry(telemetry), + m_enable_mmap(enable_mmap) { + FRONT_END_GENERAL_CHECK(m_graph_iterator, "Null pointer specified for GraphIterator"); + if (m_enable_mmap) { + m_mmap_cache = std::make_shared>>(); + m_stream_cache = nullptr; + } else { + m_mmap_cache = nullptr; + m_stream_cache = std::make_shared>>(); + } + load_model(); +} + +InputModel::InputModelONNXImpl::InputModelONNXImpl(const GraphIterator::Ptr& graph_iterator, + const ov::frontend::InputModel& input_model, + unify::InputModel* parent_model) + : m_graph_iterator(graph_iterator), + m_input_model(input_model), + m_telemetry(parent_model->_impl->get_telemetry_extension()), + m_enable_mmap(parent_model->is_enabled_mmap()), + m_mmap_cache(parent_model->_impl->m_mmap_cache), + m_stream_cache(parent_model->_impl->m_stream_cache) { + FRONT_END_GENERAL_CHECK(m_graph_iterator, "Null pointer specified for GraphIterator"); + load_model(); +} + +std::vector InputModel::InputModelONNXImpl::get_inputs() const { + return m_inputs; +} + +std::vector InputModel::InputModelONNXImpl::get_outputs() const { + return m_outputs; +} + +std::shared_ptr castToTensorPlace(const ov::frontend::Place::Ptr& place) { + if (auto var_place = std::dynamic_pointer_cast(place)) { + return var_place; + } + FRONT_END_GENERAL_CHECK(false, "Cannot cast this Place to TensorPlace."); +} + +ov::frontend::Place::Ptr InputModel::InputModelONNXImpl::get_place_by_tensor_name(const std::string& tensorName) const { + if (m_tensor_places.find(tensorName) != m_tensor_places.end()) + return castToTensorPlace(m_tensor_places.at(tensorName)); + else + return nullptr; +} + +std::shared_ptr castToOpPlace(const ov::frontend::Place::Ptr& place) { + if (auto var_place = std::dynamic_pointer_cast(place)) { + return var_place; + } + FRONT_END_GENERAL_CHECK(false, "Cannot cast this Place to TensorPlace."); +} + +void InputModel::InputModelONNXImpl::set_partial_shape(ov::frontend::Place::Ptr place, const PartialShape& shape) { + castToTensorPlace(place)->set_partial_shape(shape); +} + +ov::PartialShape InputModel::InputModelONNXImpl::get_partial_shape(ov::frontend::Place::Ptr place) const { + return castToTensorPlace(place)->get_partial_shape(); +} + +void InputModel::InputModelONNXImpl::set_element_type(ov::frontend::Place::Ptr place, const element::Type& type) { + castToTensorPlace(place)->set_element_type(type); +} + +ov::element::Type InputModel::InputModelONNXImpl::get_element_type(ov::frontend::Place::Ptr place) const { + return castToTensorPlace(place)->get_element_type(); +} + +void InputModel::InputModelONNXImpl::set_tensor_value(ov::frontend::Place::Ptr place, const void* value) { + FRONT_END_NOT_IMPLEMENTED(set_tensor_value); +} + +void InputModel::InputModelONNXImpl::set_name_for_tensor(const Place::Ptr& tensor, const std::string& new_name) { + castToTensorPlace(tensor)->set_names({new_name}); +} + +void InputModel::InputModelONNXImpl::add_name_for_tensor(const Place::Ptr& tensor, const std::string& new_name) { + auto onnx_tensor = castToTensorPlace(tensor); + auto names = onnx_tensor->get_names(); + names.push_back(new_name); + onnx_tensor->set_names(names); +} + +void InputModel::InputModelONNXImpl::set_name_for_operation(const Place::Ptr& operation, const std::string& new_name) { + auto op = castToOpPlace(operation); + auto names = op->get_names(); + names.push_back(new_name); + op->set_names(names); +} + +void InputModel::InputModelONNXImpl::override_all_inputs(const std::vector& inputs) { + FRONT_END_NOT_IMPLEMENTED(override_all_inputs); +} + +void InputModel::InputModelONNXImpl::override_all_outputs(const std::vector& outputs) { + FRONT_END_NOT_IMPLEMENTED(override_all_outputs); +} + +void InputModel::InputModelONNXImpl::extract_subgraph(const std::vector& inputs, + const std::vector& outputs) { + FRONT_END_NOT_IMPLEMENTED(extract_subgraph); +} + +void InputModel::InputModelONNXImpl::clean_up() {} + +InputModel::InputModel(const GraphIterator::Ptr& graph_iterator, + const bool enable_mmap, + const std::shared_ptr& telemetry) + : _impl{std::make_shared(graph_iterator, *this, telemetry, enable_mmap)} {} + +InputModel::InputModel(const GraphIterator::Ptr& graph_iterator, InputModel* parent_model) + : _impl{std::make_shared(graph_iterator, *this, parent_model)} {} + +std::vector> InputModel::get_op_places() const { + return _impl->get_op_places(); +} + +std::map>& InputModel::get_tensor_places() const { + return _impl->get_tensor_places(); +} + +std::vector InputModel::get_inputs() const { + return _impl->get_inputs(); +} + +std::vector InputModel::get_outputs() const { + return _impl->get_outputs(); +} + +ov::frontend::Place::Ptr InputModel::get_place_by_tensor_name(const std::string& tensorName) const { + return _impl->get_place_by_tensor_name(tensorName); +} + +ov::frontend::Place::Ptr InputModel::get_place_by_input_index(size_t input_idx) const { + FRONT_END_NOT_IMPLEMENTED(get_place_by_input_index); +} + +void InputModel::set_partial_shape(const Place::Ptr& place, const PartialShape& shape) { + _impl->set_partial_shape(place, shape); +} + +ov::PartialShape InputModel::get_partial_shape(const Place::Ptr& place) const { + return _impl->get_partial_shape(place); +} + +void InputModel::set_element_type(const Place::Ptr& place, const element::Type& type) { + _impl->set_element_type(place, type); +} + +ov::element::Type InputModel::get_element_type(const Place::Ptr& place) const { + return _impl->get_element_type(place); +} + +void InputModel::set_tensor_value(const Place::Ptr& place, const void* value) { + _impl->set_tensor_value(place, value); +} + +void InputModel::set_name_for_tensor(const Place::Ptr& tensor, const std::string& new_name) { + _impl->set_name_for_tensor(tensor, new_name); +} + +void InputModel::add_name_for_tensor(const Place::Ptr& tensor, const std::string& new_name) { + _impl->add_name_for_tensor(tensor, new_name); +} + +void InputModel::set_name_for_operation(const Place::Ptr& operation, const std::string& new_name) { + _impl->set_name_for_operation(operation, new_name); +} + +void InputModel::override_all_outputs(const std::vector& outputs) { + _impl->override_all_outputs(outputs); +} + +void InputModel::override_all_inputs(const std::vector& inputs) { + _impl->override_all_inputs(inputs); +} + +void InputModel::extract_subgraph(const std::vector& inputs, + const std::vector& outputs) { + _impl->extract_subgraph(inputs, outputs); +} + +std::shared_ptr InputModel::get_telemetry_extension() { + return _impl->get_telemetry_extension(); +} + +bool InputModel::is_enabled_mmap() const { + return _impl->is_enabled_mmap(); +} + +detail::MappedMemoryHandles InputModel::get_mmap_cache() const { + return _impl->get_mmap_cache(); +} + +detail::LocalStreamHandles InputModel::get_stream_cache() const { + return _impl->get_stream_cache(); +} + +} // namespace unify +} // namespace onnx +} // namespace frontend +} // namespace ov diff --git a/src/frontends/onnx/frontend/src/input_model.hpp b/src/frontends/onnx/frontend/src/input_model.hpp index 682a114b4433f7..64fb8f6acd13f2 100644 --- a/src/frontends/onnx/frontend/src/input_model.hpp +++ b/src/frontends/onnx/frontend/src/input_model.hpp @@ -8,7 +8,10 @@ #include #include +#include "core/place.hpp" +#include "core/tensor.hpp" #include "openvino/frontend/extension/holder.hpp" +#include "openvino/frontend/onnx/graph_iterator.hpp" using ::ONNX_NAMESPACE::ModelProto; @@ -96,6 +99,58 @@ class InputModel : public ov::frontend::InputModel { void reshape_model_inputs(std::shared_ptr& model); }; +// Forward declarations +class FrontEnd; + +namespace unify { +// Support of unified GraphIterator interface +class InputModel : public ov::frontend::InputModel { + friend class ov::frontend::onnx::FrontEnd; + friend class ov::frontend::onnx::TranslateSession; + class InputModelONNXImpl; + std::shared_ptr _impl; + + std::vector> get_op_places() const; + std::map>& get_tensor_places() const; + +public: + explicit InputModel(const ov::frontend::onnx::GraphIterator::Ptr& graph_iterator, + const bool enable_mmap, + const std::shared_ptr& telemetry = {}); + explicit InputModel(const ov::frontend::onnx::GraphIterator::Ptr& graph_iterator, InputModel* parent_model); + + ///// Searching for places ///// + std::vector get_inputs() const override; + std::vector get_outputs() const override; + ov::frontend::Place::Ptr get_place_by_tensor_name(const std::string& tensorName) const override; + ov::frontend::Place::Ptr get_place_by_input_index(size_t input_idx) const override; + + ///// Naming and annotation ///// + void set_name_for_tensor(const Place::Ptr& tensor, const std::string& new_name) override; + void add_name_for_tensor(const Place::Ptr& tensor, const std::string& new_name) override; + void set_name_for_operation(const Place::Ptr& operation, const std::string& new_name) override; + + ///// Setting / getting tensor properties ///// + void set_partial_shape(const Place::Ptr& place, const ov::PartialShape& shape) override; + ov::PartialShape get_partial_shape(const Place::Ptr& place) const override; + void set_element_type(const Place::Ptr& place, const ov::element::Type& type) override; + ov::element::Type get_element_type(const Place::Ptr& place) const override; + void set_tensor_value(const Place::Ptr& place, const void* value) override; + + ///// Topology Editing ///// + void override_all_outputs(const std::vector& outputs) override; + void override_all_inputs(const std::vector& inputs) override; + void extract_subgraph(const std::vector& inputs, + const std::vector& outputs) override; + + std::shared_ptr get_telemetry_extension(); + + bool is_enabled_mmap() const; + detail::MappedMemoryHandles get_mmap_cache() const; + detail::LocalStreamHandles get_stream_cache() const; +}; +} // namespace unify + } // namespace onnx } // namespace frontend } // namespace ov diff --git a/src/frontends/onnx/frontend/src/op/constant.cpp b/src/frontends/onnx/frontend/src/op/constant.cpp index 72330d50c3b5cc..515ef6d108a899 100644 --- a/src/frontends/onnx/frontend/src/op/constant.cpp +++ b/src/frontends/onnx/frontend/src/op/constant.cpp @@ -11,6 +11,8 @@ #include "core/sparse_tensor.hpp" #include "core/tensor.hpp" #include "openvino/frontend/exception.hpp" +#include "openvino/frontend/onnx/decoder.hpp" + using namespace ov::op; using ov::Shape; @@ -115,7 +117,8 @@ ONNX_OP("Constant", OPSET_RANGE(1, 12), ai_onnx::opset_1::constant); } // namespace opset_1 namespace opset_13 { -ov::OutputVector constant(const ov::frontend::onnx::Node& node) { +namespace detail { +ov::OutputVector constant_legacy(const ov::frontend::onnx::Node& node) { auto attributes_names = node.get_attribute_names(); FRONT_END_GENERAL_CHECK(attributes_names.size() == 1, "The Constant op expects exactly one attribute." @@ -183,6 +186,77 @@ ov::OutputVector constant(const ov::frontend::onnx::Node& node) { auto tensor = node.get_attribute_value(attributes_names[0]); return {tensor.get_ov_constant()}; } + +ov::OutputVector constant(const ov::frontend::onnx::Node& node) { + if (node.has_attribute("value_float")) { + return {v0::Constant::create(ov::element::f32, ov::Shape{}, {node.get_attribute_value("value_float")})}; + } else if (node.has_attribute("value_floats")) { + auto values = node.get_attribute_value>("value_floats"); + return {v0::Constant::create(ov::element::f32, ov::Shape{values.size()}, values)}; + } else if (node.has_attribute("value_int")) { + return {v0::Constant::create(ov::element::i64, ov::Shape{}, {node.get_attribute_value("value_int")})}; + } else if (node.has_attribute("value_ints")) { + auto values = node.get_attribute_value>("value_ints"); + return {v0::Constant::create(ov::element::i64, ov::Shape{values.size()}, values)}; + } else if (node.has_attribute("sparse_value")) { + auto sparse_tensor = node.get_attribute_value("sparse_value"); + const Tensor& values_tensor = sparse_tensor.get_values(); + const Tensor& indices_tensor = sparse_tensor.get_indices(); + const ov::Shape& shape = sparse_tensor.get_shape(); + auto rank = shape.size(); + // NNZ - the number of non-zero values in the sparse-tensor + auto nnz = values_tensor.get_shape().at(0); + std::vector absolute_indices{}; + + // Check if indices tensor with rank 2 has correct shape [NNZ, rank]. + // [i,j]-th value corresponds to the j-th index of the i-th value (in the + // values tensor) + if (indices_tensor.get_shape().size() == 2) { + FRONT_END_GENERAL_CHECK(indices_tensor.get_shape().at(0) == nnz, + "The number of values and indices is not equal." + " Indices number: ", + indices_tensor.get_shape().at(0), + " Values number: ", + nnz); + + FRONT_END_GENERAL_CHECK(indices_tensor.get_shape().at(1) == rank, + "The indices are incorrect. The second dimension of " + "indices is not equal to the rank of output." + " Second dimension of indices: ", + indices_tensor.get_shape().at(0), + " Rank of output: ", + rank); + + absolute_indices = get_absolute_indices(indices_tensor, shape, nnz); + } + // Check if indices tensor with rank 1 has correct shape [NNZ]. + // i-th value is the linearized-index of the i-th value (in the values + // tensor) + else { + FRONT_END_GENERAL_CHECK(indices_tensor.get_shape().at(0) == nnz, + "The number of values and indices is not equal." + " Indices number: ", + indices_tensor.get_shape().at(0), + " Values number: ", + nnz); + + absolute_indices = indices_tensor.get_data(); + } + return {get_dense_tensor_as_constant(absolute_indices, values_tensor, shape)}; + } + FRONT_END_GENERAL_CHECK(node.has_attribute("value"), "Constant doesn't have a required attribute \"value\""); + auto tensor = node.get_attribute_value("value"); + return {tensor.get_ov_constant()}; +} +} // namespace detail +ov::OutputVector constant(const ov::frontend::onnx::Node& node) { + if (!node.has_decoder()) { + return detail::constant_legacy(node); + } else { + return detail::constant(node); + } +} + ONNX_OP("Constant", OPSET_SINCE(13), ai_onnx::opset_13::constant); } // namespace opset_13 } // namespace ai_onnx diff --git a/src/frontends/onnx/frontend/src/op/if.cpp b/src/frontends/onnx/frontend/src/op/if.cpp index ef429c93a1208c..e8d92c434c04ef 100644 --- a/src/frontends/onnx/frontend/src/op/if.cpp +++ b/src/frontends/onnx/frontend/src/op/if.cpp @@ -8,6 +8,7 @@ #include "core/operator_set.hpp" #include "openvino/core/model.hpp" #include "openvino/frontend/exception.hpp" +#include "translate_session.hpp" using namespace ov::op; namespace ov { @@ -15,7 +16,9 @@ namespace frontend { namespace onnx { namespace ai_onnx { namespace opset_1 { -ov::OutputVector if_op(const ov::frontend::onnx::Node& node) { + +namespace detail { +ov::OutputVector if_legacy(const ov::frontend::onnx::Node& node) { const auto& ng_inputs = node.get_ov_inputs(); FRONT_END_GENERAL_CHECK(ng_inputs.size() == 1, "If operator takes only one input"); @@ -66,6 +69,65 @@ ov::OutputVector if_op(const ov::frontend::onnx::Node& node) { return if_node->outputs(); } + +ov::OutputVector if_op(const ov::frontend::onnx::Node& node) { + const auto& ng_inputs = node.get_ov_inputs(); + FRONT_END_GENERAL_CHECK(ng_inputs.size() == 1, "If operator takes only one input"); + + auto if_node = std::make_shared(ng_inputs[0]); + + auto then_branch = node.get_attribute_value>("then_branch", nullptr); + FRONT_END_GENERAL_CHECK(then_branch != nullptr, "Missing 'then_branch' attribute"); + auto else_branch = node.get_attribute_value>("else_branch", nullptr); + FRONT_END_GENERAL_CHECK(else_branch != nullptr, "Missing 'else_branch' attribute"); + + if_node->set_then_body(then_branch); + if_node->set_else_body(else_branch); + + auto translate_session = node.get_translate_session(); + + const auto& then_params = then_branch->get_parameters(); + const auto& else_params = else_branch->get_parameters(); + + for (auto& input : then_params) { + auto known_input = translate_session->lookup_tensor(input->get_friendly_name()); + if (known_input.get_node() != nullptr) { + if_node->set_input(known_input, input, nullptr); + } else { + FRONT_END_THROW("Non-existent connection in then-branch to " + input->get_friendly_name()); + } + } + + for (auto& input : else_params) { + auto known_input = translate_session->lookup_tensor(input->get_friendly_name()); + if (known_input.get_node() != nullptr) { + if_node->set_input(known_input, nullptr, input); + } else { + FRONT_END_THROW("Non-existent connection in else-branch to " + input->get_friendly_name()); + } + } + + auto then_results = then_branch->get_results(); + auto else_results = else_branch->get_results(); + FRONT_END_GENERAL_CHECK(then_results.size() == else_results.size(), + "'then' and 'else' branches have to have the same number of outputs"); + int output_size = static_cast(then_results.size()); + for (int ind = 0; ind < output_size; ++ind) { + if_node->set_output(then_results[ind], else_results[ind]); + } + + if_node->validate_and_infer_types(); + return if_node->outputs(); +} +} // namespace detail + +ov::OutputVector if_op(const ov::frontend::onnx::Node& node) { + if (!node.has_decoder()) { + return detail::if_legacy(node); + } else { + return detail::if_op(node); + } +} ONNX_OP("If", OPSET_SINCE(1), ai_onnx::opset_1::if_op); } // namespace opset_1 } // namespace ai_onnx diff --git a/src/frontends/onnx/frontend/src/op/loop.cpp b/src/frontends/onnx/frontend/src/op/loop.cpp index 979b064bcbdec6..ae534ae619a5ef 100644 --- a/src/frontends/onnx/frontend/src/op/loop.cpp +++ b/src/frontends/onnx/frontend/src/op/loop.cpp @@ -8,11 +8,14 @@ #include "core/null_node.hpp" #include "core/operator_set.hpp" #include "exceptions.hpp" +#include "openvino/core/graph_util.hpp" #include "openvino/core/model.hpp" #include "openvino/op/constant.hpp" #include "openvino/op/unsqueeze.hpp" #include "openvino/op/util/op_types.hpp" +#include "translate_session.hpp" #include "utils/reshape.hpp" + using namespace ov::op; namespace ov { @@ -38,7 +41,8 @@ bool is_termination_condition_always_true(const ov::Node* cond_in, const ov::Nod } } // namespace -ov::OutputVector loop(const ov::frontend::onnx::Node& node) { +namespace detail { +ov::OutputVector loop_legacy(const ov::frontend::onnx::Node& node) { const auto& ng_inputs = node.get_ov_inputs(); const ov::OutputVector loop_carried_dependencies{std::next(ng_inputs.begin(), 2), ng_inputs.end()}; @@ -172,6 +176,154 @@ ov::OutputVector loop(const ov::frontend::onnx::Node& node) { } return node_outputs; } + +ov::OutputVector loop(const ov::frontend::onnx::Node& node) { + const auto& ng_inputs = node.get_ov_inputs(); + + const ov::OutputVector loop_carried_dependencies{std::next(ng_inputs.begin(), 2), ng_inputs.end()}; + + std::shared_ptr body_subgraph; + auto body_graph = node.get_attribute_value>("body"); + OutputVector body_outputs{}; + for (const auto& res : body_graph->get_results()) { + body_outputs.push_back(res->get_input_source_output(0)); + } + auto body_inputs = body_graph->get_parameters(); + + // Infer loop body inputs' element type based on carried dependencies + for (size_t i = 0; i < loop_carried_dependencies.size(); i++) { + body_inputs[i + 2]->set_element_type(loop_carried_dependencies[i].get_element_type()); + body_inputs[i + 2]->set_partial_shape(loop_carried_dependencies[i].get_partial_shape()); + } + + // optional inputs + ov::Output trip_count; + // trip count skipped or has value max(int64_t) means infinitive loop + if (ov::op::util::is_null(ng_inputs.at(0)) || + (ov::op::util::is_constant(ng_inputs.at(0).get_node_shared_ptr()) && + ov::as_type_ptr(ng_inputs.at(0).get_node_shared_ptr())->cast_vector()[0] == + std::numeric_limits::max())) { + // -1 means infinite Loop + trip_count = v0::Constant::create(ov::element::i64, {1}, {-1}); + } else { + trip_count = ng_inputs.at(0); + } + + ov::Output termination_cond; // true means that first interation should be run + if (ov::op::util::is_null(ng_inputs.at(1).get_node_shared_ptr())) // termination condition skipped + { + termination_cond = v0::Constant::create(ov::element::boolean, {1}, {true}); + } else if (ov::op::util::is_constant(ng_inputs.at(1).get_node_shared_ptr()) && + ov::as_type_ptr(ng_inputs.at(1).get_node_shared_ptr())->cast_vector()[0] == false) { + // no iteration is performed so initial values are returned + ov::OutputVector node_outputs; + // final values + for (const auto& dep : loop_carried_dependencies) { + node_outputs.push_back(dep); + } + // scan outputs + for (const auto& dep : loop_carried_dependencies) { + node_outputs.push_back(dep); + } + return node_outputs; + } else { + termination_cond = ng_inputs.at(1); + } + + const int64_t concat_axis = 0; + const auto concat_axis_const = v0::Constant::create(ov::element::i64, {1}, {concat_axis}); + // add dimension along which scan outputs will be concatenated + for (size_t i = loop_carried_dependencies.size() + 1; i < body_outputs.size(); ++i) { + body_outputs[i] = std::make_shared(body_outputs[i], concat_axis_const); + } + + const auto& cond_in = body_inputs[1]; + if (body_outputs.size() > 0) { + const auto& cond_out = body_outputs[0]; + // optimization allow to improve nG Loop shape inference + if (is_termination_condition_always_true(cond_in.get(), cond_out.get_node())) { + body_outputs[0] = v0::Constant::create(ov::element::boolean, {1}, {true}); + } + } else { + body_outputs.push_back(v0::Constant::create(ov::element::boolean, {1}, {true})); + } + + CHECK_VALID_NODE(node, + body_inputs.size() >= loop_carried_dependencies.size() + 2, + "The provided loop body graph inputs size (", + body_inputs.size(), + "), is not greater than the sum of loop carried dependencies " + "and two mandatory" + " inputs (", + loop_carried_dependencies.size() + 2, + ")"); + + CHECK_VALID_NODE(node, + body_outputs.size() >= loop_carried_dependencies.size() + 1, + "The provided loop body graph outputs size (", + body_outputs.size(), + ") is not greater than number of outputs. Required at least: ", + loop_carried_dependencies.size() + 1); + + ov::ParameterVector body_params(body_inputs.begin() + 2, body_inputs.end()); + body_params.emplace(body_params.begin(), + body_inputs[0]); // current iteration body input + const auto body = std::make_shared(body_outputs, body_params); + auto loop = std::make_shared(trip_count, termination_cond); + v5::Loop::SpecialBodyPorts spec_ports{0, 0}; + loop->set_special_body_ports(spec_ports); + loop->set_function(body); + + // Setting up other Loop body inputs. + // body_inputs[0] is iteration number, body_inputs[1] is termination condition + auto body_inputs_it = std::next(body_inputs.begin(), 2); + // body_outputs[0] is termination condition output + auto body_outputs_it = std::next(body_outputs.begin(), 1); + + // Set-up loop carried dependencies and final output values + ov::OutputVector final_values; + for (const auto& dep : loop_carried_dependencies) { + loop->set_merged_input(*body_inputs_it++, dep, *body_outputs_it); + final_values.push_back(loop->get_iter_value(*body_outputs_it++, -1)); + } + + auto translate_session = node.get_translate_session(); + + for (; body_inputs_it != body_inputs.end(); ++body_inputs_it) { + auto known_input = translate_session->lookup_tensor(body_inputs_it->get()->get_friendly_name()); + if (known_input.get_node() != nullptr) { + loop->set_invariant_input(*body_inputs_it, known_input); + } else { + FRONT_END_THROW("Non-existent connection in body-graph to " + body_inputs_it->get()->get_friendly_name()); + } + } + + // Set-up scan outputs + ov::OutputVector scan_outputs; + for (; body_outputs_it != body_outputs.end(); body_outputs_it++) { + // start=0, stride=1, part_size=1, end=-1, axis=0 + scan_outputs.push_back(loop->get_concatenated_slices(*body_outputs_it, 0, 1, 1, -1, concat_axis)); + } + loop->validate_and_infer_types(); + + ov::OutputVector node_outputs; + for (const auto& v : final_values) { + node_outputs.push_back(v); + } + for (const auto& v : scan_outputs) { + node_outputs.push_back(v); + } + return node_outputs; +} +} // namespace detail + +ov::OutputVector loop(const ov::frontend::onnx::Node& node) { + if (!node.has_decoder()) { + return detail::loop_legacy(node); + } else { + return detail::loop(node); + } +} ONNX_OP("Loop", OPSET_SINCE(1), ai_onnx::opset_1::loop); } // namespace opset_1 } // namespace ai_onnx diff --git a/src/frontends/onnx/frontend/src/op/scan.cpp b/src/frontends/onnx/frontend/src/op/scan.cpp index c0aace40533587..538e588517e98c 100644 --- a/src/frontends/onnx/frontend/src/op/scan.cpp +++ b/src/frontends/onnx/frontend/src/op/scan.cpp @@ -132,11 +132,21 @@ ov::OutputVector import_onnx_scan(const ov::frontend::onnx::Node& node, std::string&& in_directions_attr_name) { const auto& node_inputs = node.get_ov_inputs(); - const auto& subgraphs = node.get_subgraphs(); - auto body_graph = subgraphs.at("body"); - auto body_outputs = body_graph->get_ov_outputs(); - auto body_inputs = body_graph->get_ng_parameters(); - + ParameterVector body_inputs; + OutputVector body_outputs; + if (!node.has_decoder()) { + const auto& subgraphs = node.get_subgraphs(); + auto body_graph = subgraphs.at("body"); + body_outputs = body_graph->get_ov_outputs(); + body_inputs = body_graph->get_ng_parameters(); + + } else { + auto body_graph = node.get_attribute_value>("body"); + for (const auto& res : body_graph->get_results()) { + body_outputs.push_back(res->get_input_source_output(0)); + } + body_inputs = body_graph->get_parameters(); + } const int64_t num_scan_inputs = node.get_attribute_value("num_scan_inputs"); const size_t num_initial_values = body_inputs.size() - num_scan_inputs; const size_t num_scan_outputs = body_outputs.size() - num_initial_values; diff --git a/src/frontends/onnx/frontend/src/op/unsqueeze.cpp b/src/frontends/onnx/frontend/src/op/unsqueeze.cpp index ace532add4c785..a6c54c1592c09f 100644 --- a/src/frontends/onnx/frontend/src/op/unsqueeze.cpp +++ b/src/frontends/onnx/frontend/src/op/unsqueeze.cpp @@ -15,7 +15,7 @@ namespace ai_onnx { namespace opset_1 { ov::OutputVector unsqueeze(const ov::frontend::onnx::Node& node) { auto data = node.get_ov_inputs().at(0); - auto axes_node = node.get_attribute_as_constant>("axes", {}); + auto axes_node = node.get_attribute_as_constant>("axes"); return {std::make_shared(data, axes_node)}; } diff --git a/src/frontends/onnx/frontend/src/ops_bridge.cpp b/src/frontends/onnx/frontend/src/ops_bridge.cpp index e31c2b9663e03b..135c8de9e21d30 100644 --- a/src/frontends/onnx/frontend/src/ops_bridge.cpp +++ b/src/frontends/onnx/frontend/src/ops_bridge.cpp @@ -174,6 +174,34 @@ OperatorSet OperatorsBridge::get_operator_set(const std::string& domain, int64_t return result; } +const Operator* OperatorsBridge::get_operator(const std::string& domain, + const std::string& name, + const int64_t version) const { + const auto dm = m_map.find(domain); + if (dm == std::end(m_map)) { + OPENVINO_DEBUG("Domain not recognized by OpenVINO"); + return nullptr; + } + if (domain == "" && version > LATEST_SUPPORTED_ONNX_OPSET_VERSION) { + OPENVINO_WARN("Currently ONNX operator set version: ", + version, + " is unsupported. Falling back to: ", + LATEST_SUPPORTED_ONNX_OPSET_VERSION); + } + for (const auto& op : dm->second) { + if (op.first != name) + continue; + + const auto& it = find(version, op.second); + if (it == std::end(op.second)) { + OPENVINO_THROW("Unsupported operator version: " + (domain.empty() ? "" : domain + ".") + op.first + ":" + + std::to_string(version)); + } + return &it->second; + } + return nullptr; +} + bool OperatorsBridge::is_operator_registered(const std::string& name, int64_t version, const std::string& domain) const { diff --git a/src/frontends/onnx/frontend/src/ops_bridge.hpp b/src/frontends/onnx/frontend/src/ops_bridge.hpp index 7a18c8fd9ff02d..cea1877f780733 100644 --- a/src/frontends/onnx/frontend/src/ops_bridge.hpp +++ b/src/frontends/onnx/frontend/src/ops_bridge.hpp @@ -28,6 +28,7 @@ class OperatorsBridge { OperatorsBridge& operator=(OperatorsBridge&&) = default; OperatorSet get_operator_set(const std::string& domain, std::int64_t version = -1) const; + const Operator* get_operator(const std::string& domain, const std::string& name, const int64_t version) const; template > Container get_supported_operators(int64_t version, std::string domain) const { diff --git a/src/frontends/onnx/frontend/src/place.hpp b/src/frontends/onnx/frontend/src/place.hpp index 6a0aed6a082432..7b8c5ccef183df 100644 --- a/src/frontends/onnx/frontend/src/place.hpp +++ b/src/frontends/onnx/frontend/src/place.hpp @@ -13,7 +13,7 @@ namespace ov { namespace frontend { namespace onnx { -class PlaceInputEdge : public Place { +class PlaceInputEdge : public ov::frontend::Place { public: PlaceInputEdge(const InputEdge& edge, std::shared_ptr editor); PlaceInputEdge(InputEdge&& edge, std::shared_ptr editor); @@ -39,7 +39,7 @@ class PlaceInputEdge : public Place { std::string m_initial_source_tensor_name; }; -class PlaceOutputEdge : public Place { +class PlaceOutputEdge : public ov::frontend::Place { public: PlaceOutputEdge(const OutputEdge& edge, std::shared_ptr editor); PlaceOutputEdge(OutputEdge&& edge, std::shared_ptr editor); @@ -65,7 +65,7 @@ class PlaceOutputEdge : public Place { std::string m_initial_target_tensor_name; }; -class PlaceTensor : public Place { +class PlaceTensor : public ov::frontend::Place { public: PlaceTensor(const std::string& name, std::shared_ptr editor); PlaceTensor(std::string&& name, std::shared_ptr editor); @@ -89,7 +89,7 @@ class PlaceTensor : public Place { std::shared_ptr m_editor; }; -class PlaceOp : public Place { +class PlaceOp : public ov::frontend::Place { public: PlaceOp(const EditorNode& node, std::shared_ptr editor); PlaceOp(EditorNode&& node, std::shared_ptr editor); diff --git a/src/frontends/onnx/frontend/src/translate_session.cpp b/src/frontends/onnx/frontend/src/translate_session.cpp new file mode 100644 index 00000000000000..6d83fa8a6105de --- /dev/null +++ b/src/frontends/onnx/frontend/src/translate_session.cpp @@ -0,0 +1,215 @@ +// Copyright (C) 2018-2025 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "translate_session.hpp" + +#include "core/null_node.hpp" +#include "input_model.hpp" +#include "onnx_framework_node.hpp" +#include "openvino/frontend/onnx/decoder.hpp" +#include "openvino/frontend/onnx/graph_iterator.hpp" +#include "openvino/op/util/op_types.hpp" +#include "ops_bridge.hpp" +#include "place.hpp" + +using namespace ov::frontend::onnx; + +TranslateSession::TranslateSession(const ov::frontend::InputModel::Ptr& input_model, + const std::shared_ptr& translator_map, + const std::string& model_name) + : m_input_model(input_model), + m_translator_map(translator_map), + m_model_name(model_name), + m_ov_model(nullptr), + m_fail_fast(false), + m_parent_session(nullptr) {} + +TranslateSession::TranslateSession(const ov::frontend::InputModel::Ptr& input_model, + TranslateSession* parent_session, + const std::string& model_name) + : m_input_model(input_model), + m_translator_map(parent_session->m_translator_map), + m_model_name(model_name), + m_ov_model(nullptr), + m_fail_fast(false), + m_parent_session(parent_session) {} + +ov::Output TranslateSession::lookup_tensor(const std::string& name) { + auto local_tensor = m_tensor_values.find(name); + if (local_tensor != m_tensor_values.end()) { + return local_tensor->second; + } + if (m_parent_session != nullptr) { + auto node_from_parent = m_parent_session->lookup_tensor(name); + if (node_from_parent.get_node() == nullptr) { + return {}; + } + if (ov::op::util::is_constant(node_from_parent.get_node_shared_ptr())) { + return node_from_parent; + } + auto new_param = std::make_shared(node_from_parent.get_element_type(), + node_from_parent.get_partial_shape()); + new_param->set_friendly_name(node_from_parent.get_node()->get_friendly_name()); + m_parameters.push_back(new_param); + m_tensor_values[name] = new_param; + return new_param; + } + return {}; +} + +std::shared_ptr TranslateSession::get_converted_model() { + if (m_ov_model) { + return m_ov_model; + } + translate_graph(m_input_model, m_ov_model); + return m_ov_model; +} + +void TranslateSession::translate_graph(const ov::frontend::InputModel::Ptr& input_model, + std::shared_ptr& ov_model) { + const OperatorsBridge translate_map; + const auto model_onnx = std::dynamic_pointer_cast(input_model); + + auto& all_tensor_places = model_onnx->get_tensor_places(); + + // inputs + m_parameters.reserve(model_onnx->get_inputs().size()); + + // Lambda detects type of input_tensor and creates correct node: constant or parameter + auto create_const_or_param = [&](const std::string& name, + const std::shared_ptr& input_tensor) { + std::shared_ptr node; + if (input_tensor->get_data_location() != nullptr || input_tensor->get_data() != nullptr) { + Tensor tensor = Tensor(input_tensor); + node = tensor.get_ov_constant(); + } else if (input_tensor->get_partial_shape() == PartialShape{0}) { // empty constant + node = ov::op::v0::Constant::create(input_tensor->get_element_type(), + input_tensor->get_partial_shape().to_shape(), + {}); + } else { + node = std::make_shared(input_tensor->get_element_type(), + input_tensor->get_partial_shape()); + m_parameters.push_back(std::dynamic_pointer_cast(node)); + } + node->set_friendly_name(name); + m_tensor_values[name] = node->get_default_output(); + input_tensor->translate(m_tensor_values[name]); + }; + + for (const auto& input : model_onnx->get_inputs()) { + const auto input_tensor = std::dynamic_pointer_cast(input); + FRONT_END_GENERAL_CHECK(input_tensor != nullptr, + "Inputs of ov::frontend::onnx::InputModel must be TensorONNXPlace instances"); + const auto name = input_tensor->get_names()[0]; + create_const_or_param(name, input_tensor); + } + + // operations + for (const auto& op_place : model_onnx->get_op_places()) { + const auto decoder = std::dynamic_pointer_cast(op_place->get_decoder()); + FRONT_END_GENERAL_CHECK(decoder != nullptr, "Decoder must be onnx::DecoderBase or its child"); + for (size_t i = 0; i < decoder->get_input_size(); ++i) { + const auto& name = decoder->get_input_tensor_name(i); + if (name == "") { + continue; + } + auto node = lookup_tensor(name); + if (node.get_node() == nullptr) { + auto place_it = all_tensor_places.find(name); + FRONT_END_GENERAL_CHECK(place_it != all_tensor_places.end(), "Tensor place not found in a graph"); + create_const_or_param(name, place_it->second); + } + } + + const auto out_size = decoder->get_output_size(); + ov::OutputVector ov_outputs(out_size); + const Operator* translator = + translate_map.get_operator(decoder->get_domain(), decoder->get_op_type(), decoder->get_op_set()); + ov::frontend::onnx::Node node_context(*decoder, this); + std::string error_message{}; + try { + if (translator == nullptr) { + ov_outputs = std::make_shared(node_context)->outputs(); + } else { + ov_outputs = (*translator)(node_context); + } + for (size_t idx = 0; idx < ov_outputs.size() && idx < out_size; ++idx) { + const std::string& out_name = decoder->get_output_tensor_name(idx); + if (is_optimized_out(ov_outputs[idx])) { + ov_outputs[idx].add_names({out_name}); + } else { + ov_outputs[idx].set_names({out_name}); + ov_outputs[idx].get_node()->set_friendly_name(out_name); + } + } + } catch (const ::ov::frontend::onnx::error::OnnxNodeValidationFailure& e) { + error_message = e.what(); + } catch (const std::exception& exc) { + error_message = error::detail::get_error_msg_prefix(node_context); + error_message += ": " + std::string{exc.what()}; + } catch (...) { + error_message = error::detail::get_error_msg_prefix(node_context); + // Since we do not know anything about current exception data type we can only + // notify user in this way. + error_message += "Unhandled exception type. \n"; + } + if (!error_message.empty()) { + auto telemetry = model_onnx->get_telemetry_extension(); + if (m_fail_fast) { + if (telemetry && translator == nullptr) { + telemetry->send_event("error_cause", "onnx_" + decoder->get_op_type()); + } + FRONT_END_THROW(error_message); + } else { + if (telemetry && !error_message.empty()) { + std::string onnx_domain = decoder->get_domain(); + uint64_t opset_version = decoder->get_op_set(); + error_message = "[ONNX Frontend] Conversion failed for " + + (onnx_domain != "" ? "***." + decoder->get_op_type() + "-X" + : decoder->get_op_type() + "-" + std::to_string(opset_version)) + + "\n" + error_message; + } + auto operation = + std::make_shared(node_context.get_ov_inputs(), + decoder->get_output_size(), + decoder->get_domain(), + decoder->get_op_type(), + error_message); + operation->set_friendly_name(decoder->get_op_name()); + ov_outputs = operation->outputs(); + } + } + for (size_t i = 0; i < ov_outputs.size() && i < decoder->get_output_size(); ++i) { + const auto& name = decoder->get_output_tensor_name(i); + if (name == "") { + // Means - not connected + continue; + } + m_tensor_values[name] = ov_outputs[i]; + all_tensor_places[name]->translate(m_tensor_values[name]); + } + } + + // outputs + ResultVector results; + results.reserve(model_onnx->get_outputs().size()); + for (const auto& output : model_onnx->get_outputs()) { + const auto tensor = std::dynamic_pointer_cast(output); + FRONT_END_GENERAL_CHECK(tensor != nullptr, + "Inputs of ov::frontend::onnx::InputModel must be TensorLitePlace instances"); + const auto name = tensor->get_names()[0]; + if (!m_tensor_values.count(name)) { + continue; + } + const auto& output_value = m_tensor_values[name]; + const auto result = std::make_shared(output_value); + auto input = result->output(0); + tensor->translate(input); + result->set_friendly_name(name + "/sink_port_0"); + results.push_back(result); + } + + auto model_name = "onnx_Frontend_IR"; + ov_model = std::make_shared(results, m_parameters, model_name); +} diff --git a/src/frontends/onnx/frontend/src/translate_session.hpp b/src/frontends/onnx/frontend/src/translate_session.hpp new file mode 100644 index 00000000000000..ef4f06b0169a89 --- /dev/null +++ b/src/frontends/onnx/frontend/src/translate_session.hpp @@ -0,0 +1,66 @@ +// Copyright (C) 2018-2025 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include "openvino/frontend/input_model.hpp" +#include "openvino/op/parameter.hpp" + +namespace ov { +namespace frontend { +namespace onnx { + +class OperatorsBridge; + +/// For one call of convert and decode method of Frontend, it creates one TranslateSession object to save data for the +/// translation session: telemetry statistics, cache of convrted body graph models, operation translators (including +/// extensions) registered for this translation session. +class TranslateSession { +public: + TranslateSession(const ov::frontend::InputModel::Ptr& input_model, + const std::shared_ptr& translator_map, + const std::string& model_name); + TranslateSession(const ov::frontend::InputModel::Ptr& input_model, + TranslateSession* parent_session, + const std::string& model_name); + + std::shared_ptr get_converted_model(); + + void translate_graph(const ov::frontend::InputModel::Ptr& input_model, std::shared_ptr& ov_model); + + ov::frontend::InputModel::Ptr get_input_model(void) const { + return m_input_model; + } + + std::map>& get_tensor_values() { + return m_tensor_values; + } + + void set_fail_fast(const bool fail_fast) { + m_fail_fast = fail_fast; + } + + bool get_fail_fast() const { + return m_fail_fast; + } + + /// \brief Method tries to find an output which is named "name" + /// If a result found in a parent session (it means current GraphIterator/InputModel + /// doesn't have a corresponding node. And if node isn't a constant - then + /// method restores a Parameters chain by adding a Parameter with provided name + ov::Output lookup_tensor(const std::string& name); + +private: + const ov::frontend::InputModel::Ptr m_input_model; + const std::shared_ptr m_translator_map; + const std::string m_model_name; + std::shared_ptr m_ov_model; + std::map> m_tensor_values; + bool m_fail_fast; + TranslateSession* m_parent_session; + ParameterVector m_parameters; +}; +} // namespace onnx +} // namespace frontend +} // namespace ov \ No newline at end of file diff --git a/src/frontends/onnx/frontend/src/utils/tensor_external_data.cpp b/src/frontends/onnx/frontend/src/utils/tensor_external_data.cpp index 5a6d31d4ed6bee..d346cfb164698c 100644 --- a/src/frontends/onnx/frontend/src/utils/tensor_external_data.cpp +++ b/src/frontends/onnx/frontend/src/utils/tensor_external_data.cpp @@ -33,10 +33,17 @@ TensorExternalData::TensorExternalData(const TensorProto& tensor) { } #endif } +TensorExternalData::TensorExternalData(const std::string& location, size_t offset, size_t size) { + m_data_location = location; + m_offset = offset; + m_data_length = size; +} Buffer TensorExternalData::load_external_mmap_data(const std::string& model_dir, MappedMemoryHandles cache) const { - const auto full_path = ov::util::get_absolute_file_path(ov::util::path_join({model_dir, m_data_location}).string()); + const auto full_path = + model_dir == "" ? m_data_location + : ov::util::get_absolute_file_path(ov::util::path_join({model_dir, m_data_location}).string()); const int64_t file_size = ov::util::file_size(full_path); if (file_size <= 0 || m_offset + m_data_length > static_cast(file_size)) { throw error::invalid_external_data{*this}; @@ -59,7 +66,9 @@ Buffer TensorExternalData::load_external_mmap_data(const std:: } Buffer TensorExternalData::load_external_data(const std::string& model_dir) const { - auto full_path = ov::util::get_absolute_file_path(ov::util::path_join({model_dir, m_data_location}).string()); + auto full_path = model_dir == "" + ? m_data_location + : ov::util::get_absolute_file_path(ov::util::path_join({model_dir, m_data_location}).string()); #if defined(OPENVINO_ENABLE_UNICODE_PATH_SUPPORT) && defined(_WIN32) ov::util::convert_path_win_style(full_path); std::ifstream external_data_stream(ov::util::string_to_wstring(full_path).c_str(), diff --git a/src/frontends/onnx/frontend/src/utils/tensor_external_data.hpp b/src/frontends/onnx/frontend/src/utils/tensor_external_data.hpp index a43cd738c0ee21..45f8639bab1b11 100644 --- a/src/frontends/onnx/frontend/src/utils/tensor_external_data.hpp +++ b/src/frontends/onnx/frontend/src/utils/tensor_external_data.hpp @@ -18,10 +18,12 @@ using ::ONNX_NAMESPACE::TensorProto; template using Buffer = std::shared_ptr>>; using MappedMemoryHandles = std::shared_ptr>>; +using LocalStreamHandles = std::shared_ptr>>; /// \brief Helper class used to load tensor data from external files class TensorExternalData { public: TensorExternalData(const TensorProto& tensor); + TensorExternalData(const std::string& location, size_t offset, size_t size); /// \brief Load external data from tensor passed to constructor /// diff --git a/src/frontends/onnx/tests/CMakeLists.txt b/src/frontends/onnx/tests/CMakeLists.txt index 5e70227868a48f..54127c56e93bbc 100644 --- a/src/frontends/onnx/tests/CMakeLists.txt +++ b/src/frontends/onnx/tests/CMakeLists.txt @@ -69,6 +69,7 @@ set(SRC conversion.cpp convert_tests.cpp convert_partially_tests.cpp + graph_iterator.cpp library_extension.cpp load_from.cpp onnx_editor.cpp @@ -84,7 +85,10 @@ set(SRC model_support_tests.cpp onnx_ops_registration.cpp onnx_reader_external_data.cpp - skip_tests_config.cpp) + skip_tests_config.cpp + ../frontend/src/core/graph_iterator_proto.cpp + ../frontend/src/core/decoder_proto.cpp +) foreach(src IN LISTS SRC MULTI_TEST_SRC) if(IS_ABSOLUTE "${src}") @@ -124,6 +128,7 @@ set_property(TEST ov_onnx_frontend_tests PROPERTY LABELS OV UNIT ONNX_FE) add_dependencies(ov_onnx_frontend_tests openvino_template_extension) + target_include_directories(ov_onnx_frontend_tests PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}") target_compile_definitions(ov_onnx_frontend_tests @@ -142,7 +147,8 @@ target_link_libraries(ov_onnx_frontend_tests PRIVATE gtest_main_manifest frontend_shared_test_classes openvino::frontend::onnx - func_test_utils) + func_test_utils +) if(OV_COMPILER_IS_CLANG) target_compile_options(ov_onnx_frontend_tests PRIVATE -Wno-undef -Wno-reserved-id-macro) diff --git a/src/frontends/onnx/tests/graph_iterator.cpp b/src/frontends/onnx/tests/graph_iterator.cpp new file mode 100644 index 00000000000000..6fc9a0ef47fa58 --- /dev/null +++ b/src/frontends/onnx/tests/graph_iterator.cpp @@ -0,0 +1,126 @@ +// Copyright (C) 2018-2025 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// +#include "openvino/frontend/onnx/graph_iterator.hpp" + +#include + +#include +#include +#include + +#include "../frontend/src/core/graph_iterator_proto.hpp" +#include "load_from.hpp" +#include "onnx_utils.hpp" +#include "utils.hpp" + +using ::ONNX_NAMESPACE::ModelProto; +using ::ONNX_NAMESPACE::Version; + +TEST_P(FrontEndLoadFromTest, testLoadUsingSimpleGraphIterator) { + ov::frontend::FrontEnd::Ptr fe; + + class SimpleIterator : public ov::frontend::onnx::GraphIterator { + public: + size_t size() const override { + return 0; + } + void reset() override {}; + void next() override {}; + bool is_end() const override { + return true; + }; + std::shared_ptr get_decoder() const override { + return nullptr; + }; + + int64_t get_opset_version(const std::string& domain) const override { + return 1; + } + + ~SimpleIterator() override {}; + }; + + auto iter = std::make_shared(); + + { + auto graph_iter = std::dynamic_pointer_cast(iter); + ASSERT_NO_THROW(m_frontEnd = m_fem.load_by_framework("onnx")) + << "Could not create the ONNX FE using a pointer GraphIterator"; + ASSERT_NE(m_frontEnd, nullptr); + + ASSERT_EQ(m_frontEnd->supported(graph_iter), true); + + ASSERT_NO_THROW(m_inputModel = m_frontEnd->load(graph_iter)) << "Could not load the model"; + ASSERT_NE(m_inputModel, nullptr); + } + std::shared_ptr model; + ASSERT_NO_THROW(model = m_frontEnd->convert(m_inputModel)) << "Could not convert the model to OV representation"; + ASSERT_NE(model, nullptr); + + ASSERT_EQ(model->get_ordered_ops().size(), 0); +} + +TEST_P(FrontEndLoadFromTest, testLoadUsingGraphIteratorExternalStreams) { + const std::string model_name = "external_data/external_data.onnx"; + const auto path = + ov::util::path_join({ov::test::utils::getExecutableDirectory(), TEST_ONNX_MODELS_DIRNAME, model_name}).string(); + + ov::frontend::FrontEnd::Ptr fe; + + auto iter = std::make_shared( + ov::frontend::onnx::GraphIteratorProtoMemoryManagementMode::External_Stream); + iter->initialize(path); + iter->reset(); + + auto graph_iter = std::dynamic_pointer_cast(iter); + ASSERT_NO_THROW(m_frontEnd = m_fem.load_by_framework("onnx")) + << "Could not create the ONNX FE using a pointer GraphIterator"; + ASSERT_NE(m_frontEnd, nullptr); + + ASSERT_EQ(m_frontEnd->supported(graph_iter), true); + + ASSERT_NO_THROW(m_inputModel = m_frontEnd->load(graph_iter)) << "Could not load the model"; + ASSERT_NE(m_inputModel, nullptr); + + std::shared_ptr model; + ASSERT_NO_THROW(model = m_frontEnd->convert(m_inputModel)) << "Could not convert the model to OV representation"; + ASSERT_NE(model, nullptr); + + ASSERT_EQ(iter->get_mmap_cache(), nullptr); + ASSERT_NE(iter->get_stream_cache(), nullptr); + ASSERT_EQ(iter->get_stream_cache()->size(), 0); // All streams must be closed after work + ASSERT_EQ(model->get_ordered_ops().size(), 6); +} + +TEST_P(FrontEndLoadFromTest, testLoadUsingGraphIteratorExternalMMAP) { + const std::string model_name = "external_data/external_data.onnx"; + const auto path = + ov::util::path_join({ov::test::utils::getExecutableDirectory(), TEST_ONNX_MODELS_DIRNAME, model_name}).string(); + + ov::frontend::FrontEnd::Ptr fe; + + auto iter = std::make_shared( + ov::frontend::onnx::GraphIteratorProtoMemoryManagementMode::External_MMAP); + iter->initialize(path); + iter->reset(); + + auto graph_iter = std::dynamic_pointer_cast(iter); + ASSERT_NO_THROW(m_frontEnd = m_fem.load_by_framework("onnx")) + << "Could not create the ONNX FE using a pointer GraphIterator"; + ASSERT_NE(m_frontEnd, nullptr); + + ASSERT_EQ(m_frontEnd->supported(graph_iter), true); + + ASSERT_NO_THROW(m_inputModel = m_frontEnd->load(graph_iter)) << "Could not load the model"; + ASSERT_NE(m_inputModel, nullptr); + + std::shared_ptr model; + ASSERT_NO_THROW(model = m_frontEnd->convert(m_inputModel)) << "Could not convert the model to OV representation"; + ASSERT_NE(model, nullptr); + + ASSERT_EQ(iter->get_stream_cache(), nullptr); + ASSERT_NE(iter->get_mmap_cache(), nullptr); + ASSERT_EQ(iter->get_mmap_cache()->size(), 1); // MMAP handle must be in cache after work finished + ASSERT_EQ(model->get_ordered_ops().size(), 6); +} diff --git a/src/frontends/onnx/tests/models/bool_init_raw.prototxt b/src/frontends/onnx/tests/models/bool_init_raw.prototxt index e9af296994f359..80c5b3ae5fbd34 100644 --- a/src/frontends/onnx/tests/models/bool_init_raw.prototxt +++ b/src/frontends/onnx/tests/models/bool_init_raw.prototxt @@ -25,8 +25,14 @@ graph { } } } + node { + input: "A" + output: "B" + name: "Identity_10" + op_type: "Identity" + } output { - name: "A" + name: "B" type { tensor_type { elem_type: 9 diff --git a/src/frontends/onnx/tests/onnx_import_controlflow.in.cpp b/src/frontends/onnx/tests/onnx_import_controlflow.in.cpp index 0ec130d026baea..7479caa53b9b0a 100644 --- a/src/frontends/onnx/tests/onnx_import_controlflow.in.cpp +++ b/src/frontends/onnx/tests/onnx_import_controlflow.in.cpp @@ -230,7 +230,8 @@ OPENVINO_TEST(${BACKEND_NAME}, onnx_controlflow_loop_add_value_access_to_body_sc FAIL() << "Incorrect access to body scope not detected"; } catch (const ov::Exception& e) { // patent graph should have no access to subgraph (body Loop) scope - EXPECT_HAS_SUBSTRING(e.what(), std::string("from_body_scope node not found in graph cache")); + EXPECT_HAS_SUBSTRING(e.what(), std::string("from_body_scope")); + EXPECT_HAS_SUBSTRING(e.what(), std::string("t found")); } catch (...) { FAIL() << "Deduced type check failed for unexpected reason"; }