diff --git a/lib/task-spec/include/task-spec/dynamic_graph/dynamic_layer_guid_t.dtg.toml b/lib/task-spec/include/task-spec/dynamic_graph/dynamic_layer_guid_t.dtg.toml new file mode 100644 index 0000000000..a429a958f7 --- /dev/null +++ b/lib/task-spec/include/task-spec/dynamic_graph/dynamic_layer_guid_t.dtg.toml @@ -0,0 +1,21 @@ +namespace = "FlexFlow" +name = "dynamic_layer_guid_t" +type = "variant" +features = [ + "eq", + "hash", + "fmt", +] + +includes = [ + "pcg/layer_guid_t.dtg.h", + "pcg/parallel_computation_graph/parallel_layer_guid_t.dtg.h", +] + +[[values]] +name = "layer_guid" +type = "::FlexFlow::layer_guid_t" + +[[values]] +name = "pcg_layer_guid" +type = "::FlexFlow::parallel_layer_guid_t" diff --git a/lib/task-spec/include/task-spec/dynamic_graph/dynamic_node_attrs.dtg.toml b/lib/task-spec/include/task-spec/dynamic_graph/dynamic_node_attrs.dtg.toml index 14f361ca75..4d58e5f04f 100644 --- a/lib/task-spec/include/task-spec/dynamic_graph/dynamic_node_attrs.dtg.toml +++ b/lib/task-spec/include/task-spec/dynamic_graph/dynamic_node_attrs.dtg.toml @@ -13,7 +13,7 @@ includes = [ "pcg/machine_space_coordinate.dtg.h", "pcg/mapped_parallel_computation_graph/mapped_operator_task_group.h", "op-attrs/pcg_operator_attrs.dtg.h", - "pcg/parallel_computation_graph/parallel_layer_guid_t.dtg.h", + "task-spec/dynamic_graph/dynamic_layer_guid_t.dtg.h", "task-spec/device_specific_per_device_op_state.dtg.h", ] @@ -38,8 +38,8 @@ name = "op_attrs" type = "std::optional<::FlexFlow::PCGOperatorAttrs>" [[fields]] -name = "pcg_layer_guid" -type = "::FlexFlow::parallel_layer_guid_t" +name = "layer_guid" +type = "::FlexFlow::dynamic_layer_guid_t" [[fields]] name = "per_device_op_state" diff --git a/lib/task-spec/include/task-spec/dynamic_graph/dynamic_tensor_guid_t.dtg.toml b/lib/task-spec/include/task-spec/dynamic_graph/dynamic_tensor_guid_t.dtg.toml new file mode 100644 index 0000000000..b0c571871b --- /dev/null +++ b/lib/task-spec/include/task-spec/dynamic_graph/dynamic_tensor_guid_t.dtg.toml @@ -0,0 +1,21 @@ +namespace = "FlexFlow" +name = "dynamic_tensor_guid_t" +type = "variant" +features = [ + "eq", + "hash", + "fmt", +] + +includes = [ + "pcg/tensor_guid_t.dtg.h", + "pcg/parallel_computation_graph/parallel_tensor_guid_t.dtg.h", +] + +[[values]] +name = "tensor_guid" +type = "::FlexFlow::tensor_guid_t" + +[[values]] +name = "pcg_tensor_guid" +type = "::FlexFlow::parallel_tensor_guid_t" diff --git a/lib/task-spec/include/task-spec/dynamic_graph/dynamic_value_attrs.dtg.toml b/lib/task-spec/include/task-spec/dynamic_graph/dynamic_value_attrs.dtg.toml index 2332b2c93b..6638f16e62 100644 --- a/lib/task-spec/include/task-spec/dynamic_graph/dynamic_value_attrs.dtg.toml +++ b/lib/task-spec/include/task-spec/dynamic_graph/dynamic_value_attrs.dtg.toml @@ -9,6 +9,7 @@ features = [ includes = [ "", + "task-spec/dynamic_graph/dynamic_tensor_guid_t.dtg.h", "op-attrs/parallel_tensor_shape.dtg.h", "kernels/accessor.h", "pcg/parallel_computation_graph/parallel_tensor_guid_t.dtg.h", @@ -21,8 +22,8 @@ src_includes = [ ] [[fields]] -name = "pcg_tensor_guid" -type = "::FlexFlow::parallel_tensor_guid_t" +name = "tensor_guid" +type = "::FlexFlow::dynamic_tensor_guid_t" [[fields]] name = "parallel_tensor_shape" diff --git a/lib/task-spec/include/task-spec/dynamic_graph/make_dynamic_open_dataflow_graph_from_cg.h b/lib/task-spec/include/task-spec/dynamic_graph/make_dynamic_open_dataflow_graph_from_cg.h new file mode 100644 index 0000000000..639aef8598 --- /dev/null +++ b/lib/task-spec/include/task-spec/dynamic_graph/make_dynamic_open_dataflow_graph_from_cg.h @@ -0,0 +1,14 @@ +#ifndef _FLEXFLOW_LIB_TASK_SPEC_INCLUDE_TASK_SPEC_DYNAMIC_GRAPH_DYNAMIC_OPEN_DATAFLOW_GRAPH_FROM_CG_H +#define _FLEXFLOW_LIB_TASK_SPEC_INCLUDE_TASK_SPEC_DYNAMIC_GRAPH_DYNAMIC_OPEN_DATAFLOW_GRAPH_FROM_CG_H + +#include "pcg/computation_graph.dtg.h" +#include "task-spec/dynamic_graph/dynamic_open_dataflow_graph.dtg.h" + +namespace FlexFlow { + +DynamicOpenDataflowGraph + make_dynamic_open_dataflow_graph_from_cg(ComputationGraph const &); + +} // namespace FlexFlow + +#endif diff --git a/lib/task-spec/src/task-spec/dynamic_graph/make_dynamic_open_dataflow_graph_from_cg.cc b/lib/task-spec/src/task-spec/dynamic_graph/make_dynamic_open_dataflow_graph_from_cg.cc new file mode 100644 index 0000000000..415151f43a --- /dev/null +++ b/lib/task-spec/src/task-spec/dynamic_graph/make_dynamic_open_dataflow_graph_from_cg.cc @@ -0,0 +1,71 @@ +#include "task-spec/dynamic_graph/make_dynamic_open_dataflow_graph_from_cg.h" +#include "op-attrs/pcg_operator_attrs.h" +#include "pcg/computation_graph.h" +#include "task-spec/dynamic_graph/dynamic_layer_guid_t.dtg.h" +#include "task-spec/dynamic_graph/dynamic_open_dataflow_graph.h" +#include "task-spec/dynamic_graph/dynamic_tensor_role.h" +#include "utils/containers/generate_map.h" +#include +#include +#include + +namespace FlexFlow { + +DynamicOpenDataflowGraph + make_dynamic_open_dataflow_graph_from_cg(ComputationGraph const &cg) { + DynamicOpenDataflowGraph result = make_empty_dynamic_open_dataflow_graph(); + + for (auto const &[layer, attrs] : get_layer_attrs_mapping(cg)) { + DynamicNodeAttrs result_attrs{ + /*task_type=*/std::nullopt, + /*device_coord=*/std::nullopt, + /*mapping=*/std::nullopt, + /*op_attrs=*/pcg_op_attrs_from_compgraph_op_attrs(attrs.op_attrs), + /*pcg_layer_guid=*/dynamic_layer_guid_t{layer}, + /*per_device_op_state=*/std::nullopt, + }; + + std::unordered_map result_inputs = + transform( + get_incoming_tensors(cg, layer), + [](TensorSlotName const &slot_name, tensor_guid_t const &tensor) { + return std::pair{ + DynamicTensorSlot{ + /*slot_name=*/slot_name, + /*slot_tensor_role=*/std::nullopt, + }, + DynamicValueAttrs{ + /*tensor_guid=*/dynamic_tensor_guid_t{tensor}, + /*parallel_tensor_shape=*/std::nullopt, + /*shard_coord=*/std::nullopt, + /*accessor=*/std::nullopt, + /*role=*/std::nullopt, + }, + }; + }); + std::unordered_map result_outputs = + transform( + get_outgoing_tensors(cg, layer), + [](TensorSlotName const &slot_name, tensor_guid_t const &tensor) { + return std::pair{ + DynamicTensorSlot{ + /*slot_name=*/slot_name, + /*slot_tensor_role=*/std::nullopt, + }, + DynamicValueAttrs{ + /*tensor_guid=*/dynamic_tensor_guid_t{tensor}, + /*parallel_tensor_shape=*/std::nullopt, + /*shard_coord=*/std::nullopt, + /*accessor=*/std::nullopt, + /*role=*/std::nullopt, + }, + }; + }); + + result.invocations.emplace(result_outputs, result_attrs, result_outputs); + } + + return result; +} + +} // namespace FlexFlow diff --git a/lib/task-spec/test/src/task-spec/dynamic_graph/dynamic_open_dataflow_graph.cc b/lib/task-spec/test/src/task-spec/dynamic_graph/dynamic_open_dataflow_graph.cc index 58cd8c67ec..fc9110b6e4 100644 --- a/lib/task-spec/test/src/task-spec/dynamic_graph/dynamic_open_dataflow_graph.cc +++ b/lib/task-spec/test/src/task-spec/dynamic_graph/dynamic_open_dataflow_graph.cc @@ -7,12 +7,12 @@ using namespace ::FlexFlow; TEST_SUITE(FF_TEST_SUITE) { TEST_CASE("dynamic_op_dataflow_graph_from_invocation_set") { DynamicValueAttrs value_1 = DynamicValueAttrs{ - /*pcg_tensor_guid=*/parallel_tensor_guid_t{ + /*tensor_guid=*/dynamic_tensor_guid_t{parallel_tensor_guid_t{ KwargDataflowOutput{ Node{1}, TensorSlotName::OUTPUT, }, - }, + }}, /*parallel_tensor_shape=*/std::nullopt, /*shard_coord=*/std::nullopt, /*accessor=*/std::nullopt, @@ -20,12 +20,12 @@ TEST_SUITE(FF_TEST_SUITE) { }; DynamicValueAttrs value_2 = DynamicValueAttrs{ - /*pcg_tensor_guid=*/parallel_tensor_guid_t{ + /*tensor_guid=*/dynamic_tensor_guid_t{parallel_tensor_guid_t{ KwargDataflowOutput{ Node{2}, TensorSlotName::OUTPUT, }, - }, + }}, /*parallel_tensor_shape=*/std::nullopt, /*shard_coord=*/std::nullopt, /*accessor=*/std::nullopt, @@ -33,12 +33,12 @@ TEST_SUITE(FF_TEST_SUITE) { }; DynamicValueAttrs value_3 = DynamicValueAttrs{ - /*pcg_tensor_guid=*/parallel_tensor_guid_t{ + /*tensor_guid=*/dynamic_tensor_guid_t{parallel_tensor_guid_t{ KwargDataflowOutput{ Node{3}, TensorSlotName::OUTPUT, }, - }, + }}, /*parallel_tensor_shape=*/std::nullopt, /*shard_coord=*/std::nullopt, /*accessor=*/std::nullopt, @@ -50,7 +50,7 @@ TEST_SUITE(FF_TEST_SUITE) { /*device_coord=*/std::nullopt, /*mapping=*/std::nullopt, /*op_attrs=*/std::nullopt, - /*pcg_layer_guid=*/parallel_layer_guid_t{Node{4}}, + /*layer_guid=*/dynamic_layer_guid_t{parallel_layer_guid_t{Node{4}}}, /*per_device_op_state=*/std::nullopt, }; diff --git a/lib/task-spec/test/src/task-spec/dynamic_graph/machine_slicing.cc b/lib/task-spec/test/src/task-spec/dynamic_graph/machine_slicing.cc index 1e7162f741..40d37f50df 100644 --- a/lib/task-spec/test/src/task-spec/dynamic_graph/machine_slicing.cc +++ b/lib/task-spec/test/src/task-spec/dynamic_graph/machine_slicing.cc @@ -67,12 +67,12 @@ TEST_SUITE(FF_TEST_SUITE) { std::optional const &shard_coord) -> DynamicValueAttrs { return DynamicValueAttrs{ - /*pcg_tensor_guid=*/parallel_tensor_guid_t{ + /*tensor_guid=*/dynamic_tensor_guid_t{parallel_tensor_guid_t{ KwargDataflowOutput{ Node{src_node_id}, src_slot_name, }, - }, + }}, /*parallel_tensor_shape=*/std::nullopt, /*shard_coord=*/shard_coord, /*accessor=*/std::nullopt, @@ -114,7 +114,8 @@ TEST_SUITE(FF_TEST_SUITE) { /*device_coord=*/mc2, /*mapping=*/std::nullopt, /*op_attrs=*/std::nullopt, - /*pcg_layer_guid=*/parallel_layer_guid_t{Node{invocation1_id}}, + /*layer_guid=*/ + dynamic_layer_guid_t{parallel_layer_guid_t{Node{invocation1_id}}}, /*per_device_op_state=*/std::nullopt, }, /*outputs=*/ @@ -140,7 +141,8 @@ TEST_SUITE(FF_TEST_SUITE) { /*device_coord=*/mc1, /*mapping=*/std::nullopt, /*op_attrs=*/std::nullopt, - /*pcg_layer_guid=*/parallel_layer_guid_t{Node{invocation2_id}}, + /*layer_guid=*/ + dynamic_layer_guid_t{parallel_layer_guid_t{Node{invocation2_id}}}, /*per_device_op_state=*/std::nullopt, }, /*outputs=*/ @@ -173,7 +175,8 @@ TEST_SUITE(FF_TEST_SUITE) { /*device_coord=*/mc2, /*mapping=*/std::nullopt, /*op_attrs=*/std::nullopt, - /*pcg_layer_guid=*/parallel_layer_guid_t{Node{invocation3_id}}, + /*layer_guid=*/ + dynamic_layer_guid_t{parallel_layer_guid_t{Node{invocation3_id}}}, /*per_device_op_state=*/std::nullopt, }, /*outputs=*/ diff --git a/lib/task-spec/test/src/task-spec/dynamic_graph/pass_expansion.cc b/lib/task-spec/test/src/task-spec/dynamic_graph/pass_expansion.cc index 9039f16cd9..e8fcf2e40b 100644 --- a/lib/task-spec/test/src/task-spec/dynamic_graph/pass_expansion.cc +++ b/lib/task-spec/test/src/task-spec/dynamic_graph/pass_expansion.cc @@ -11,12 +11,12 @@ TEST_SUITE(FF_TEST_SUITE) { [](size_t node_id, std::optional const &tensor_role) -> DynamicValueAttrs { return DynamicValueAttrs{ - /*pcg_tensor_guid=*/parallel_tensor_guid_t{ + /*tensor_guid=*/dynamic_tensor_guid_t{parallel_tensor_guid_t{ KwargDataflowOutput{ Node{node_id}, TensorSlotName::OUTPUT, }, - }, + }}, /*parallel_tensor_shape=*/std::nullopt, /*shard_coord=*/std::nullopt, /*accessor=*/std::nullopt, @@ -33,7 +33,7 @@ TEST_SUITE(FF_TEST_SUITE) { }; }; - parallel_layer_guid_t pcg_layer_guid = parallel_layer_guid_t{Node{20}}; + dynamic_layer_guid_t layer_guid{parallel_layer_guid_t{Node{20}}}; DynamicNodeInvocation invocation = [&]() -> DynamicNodeInvocation { DynamicValueAttrs v1 = mk_value_attrs(0, std::nullopt); @@ -53,7 +53,7 @@ TEST_SUITE(FF_TEST_SUITE) { /*device_coord=*/std::nullopt, /*mapping=*/std::nullopt, /*op_attrs=*/std::nullopt, - /*pcg_layer_guid=*/pcg_layer_guid, + /*layer_guid=*/layer_guid, /*per_device_op_state=*/std::nullopt, }, /*outputs=*/ @@ -86,7 +86,7 @@ TEST_SUITE(FF_TEST_SUITE) { /*device_coord=*/std::nullopt, /*mapping=*/std::nullopt, /*op_attrs=*/std::nullopt, - /*pcg_layer_guid=*/pcg_layer_guid, + /*layer_guid=*/layer_guid, /*per_device_op_state=*/std::nullopt, }, /*outputs=*/ @@ -104,12 +104,12 @@ TEST_SUITE(FF_TEST_SUITE) { [](size_t node_id, std::optional const &tensor_role) -> DynamicValueAttrs { return DynamicValueAttrs{ - /*pcg_tensor_guid=*/parallel_tensor_guid_t{ + /*tensor_guid=*/dynamic_tensor_guid_t{parallel_tensor_guid_t{ KwargDataflowOutput{ Node{node_id}, TensorSlotName::OUTPUT, }, - }, + }}, /*parallel_tensor_shape=*/std::nullopt, /*shard_coord=*/std::nullopt, /*accessor=*/std::nullopt, @@ -126,7 +126,7 @@ TEST_SUITE(FF_TEST_SUITE) { }; }; - parallel_layer_guid_t pcg_layer_guid = parallel_layer_guid_t{Node{20}}; + dynamic_layer_guid_t layer_guid{parallel_layer_guid_t{Node{20}}}; DynamicNodeInvocation invocation = [&]() -> DynamicNodeInvocation { DynamicValueAttrs v1 = mk_value_attrs(0, std::nullopt); @@ -146,7 +146,7 @@ TEST_SUITE(FF_TEST_SUITE) { /*device_coord=*/std::nullopt, /*mapping=*/std::nullopt, /*op_attrs=*/std::nullopt, - /*pcg_layer_guid=*/pcg_layer_guid, + /*layer_guid=*/layer_guid, /*per_device_op_state=*/std::nullopt, }, /*outputs=*/ @@ -185,7 +185,7 @@ TEST_SUITE(FF_TEST_SUITE) { /*device_coord=*/std::nullopt, /*mapping=*/std::nullopt, /*op_attrs=*/std::nullopt, - /*pcg_layer_guid=*/pcg_layer_guid, + /*layer_guid=*/layer_guid, /*per_device_op_state=*/std::nullopt, }, /*outputs=*/ @@ -210,7 +210,8 @@ TEST_SUITE(FF_TEST_SUITE) { /*device_coord=*/std::nullopt, /*mapping=*/std::nullopt, /*op_attrs=*/std::nullopt, - /*pcg_layer_guid=*/parallel_layer_guid_t{Node{layer_id}}, + /*layer_guid=*/ + dynamic_layer_guid_t{parallel_layer_guid_t{Node{layer_id}}}, /*per_device_op_state=*/std::nullopt, }; }; @@ -219,12 +220,12 @@ TEST_SUITE(FF_TEST_SUITE) { [](size_t node_id, std::optional const &tensor_type) -> DynamicValueAttrs { return DynamicValueAttrs{ - /*pcg_tensor_guid=*/parallel_tensor_guid_t{ + /*tensor_guid=*/dynamic_tensor_guid_t{parallel_tensor_guid_t{ KwargDataflowOutput{ Node{node_id}, TensorSlotName::OUTPUT, }, - }, + }}, /*parallel_tensor_shape=*/std::nullopt, /*shard_coord=*/std::nullopt, /*accessor=*/std::nullopt, diff --git a/lib/task-spec/test/src/task-spec/dynamic_graph/shard_expansion.cc b/lib/task-spec/test/src/task-spec/dynamic_graph/shard_expansion.cc index f49a496647..23fbb6e514 100644 --- a/lib/task-spec/test/src/task-spec/dynamic_graph/shard_expansion.cc +++ b/lib/task-spec/test/src/task-spec/dynamic_graph/shard_expansion.cc @@ -112,12 +112,12 @@ TEST_SUITE(FF_TEST_SUITE) { std::optional const &shard_coord) -> DynamicValueAttrs { return DynamicValueAttrs{ - /*pcg_tensor_guid=*/parallel_tensor_guid_t{ + /*tensor_guid=*/dynamic_tensor_guid_t{parallel_tensor_guid_t{ KwargDataflowOutput{ Node{src_node_id}, src_slot_name, }, - }, + }}, /*parallel_tensor_shape=*/std::nullopt, /*shard_coord=*/shard_coord, /*accessor=*/std::nullopt, @@ -142,7 +142,8 @@ TEST_SUITE(FF_TEST_SUITE) { /*device_coord=*/std::nullopt, /*mapping=*/mapped_task_group, /*op_attrs=*/std::nullopt, - /*pcg_layer_guid=*/parallel_layer_guid_t{Node{20}}, + /*layer_guid=*/ + dynamic_layer_guid_t{parallel_layer_guid_t{Node{20}}}, /*per_device_op_state=*/std::nullopt, }, /*outputs=*/ @@ -185,7 +186,8 @@ TEST_SUITE(FF_TEST_SUITE) { /*device_coord=*/device_coord, /*mapping=*/mapped_task_group, /*op_attrs=*/std::nullopt, - /*pcg_layer_guid=*/parallel_layer_guid_t{Node{20}}, + /*layer_guid=*/ + dynamic_layer_guid_t{parallel_layer_guid_t{Node{20}}}, /*per_device_op_state=*/std::nullopt, }, /*outputs=*/