Skip to content

Commit 7672a16

Browse files
nitbharambenitbharambe
committed
add decision todos
Signed-off-by: Nitish Bharambe <[email protected]>
1 parent b4f6353 commit 7672a16

File tree

1 file changed

+140
-28
lines changed

1 file changed

+140
-28
lines changed

‎docs/examples/arrow_example.ipynb‎

Lines changed: 140 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -235,6 +235,38 @@
235235
"print(pgm_combined_schema(DatasetType.input, ComponentType.asym_load))"
236236
]
237237
},
238+
{
239+
"cell_type": "code",
240+
"execution_count": 26,
241+
"metadata": {},
242+
"outputs": [
243+
{
244+
"data": {
245+
"text/plain": [
246+
"pyarrow.RecordBatch\n",
247+
"id: int32\n",
248+
"u_rated: double\n",
249+
"----\n",
250+
"id: [1,2,3]\n",
251+
"u_rated: [10500,10500,10500]"
252+
]
253+
},
254+
"execution_count": 26,
255+
"metadata": {},
256+
"output_type": "execute_result"
257+
}
258+
],
259+
"source": [
260+
"#TODO Decisions: Create from schema\n",
261+
"pa.record_batch(\n",
262+
" {\n",
263+
" \"id\": [1, 2, 3],\n",
264+
" \"u_rated\": [10500.0, 10500.0, 10500.0],\n",
265+
" },\n",
266+
" schema=pgm_schema(DatasetType.input, ComponentType.node),\n",
267+
")"
268+
]
269+
},
238270
{
239271
"cell_type": "markdown",
240272
"metadata": {},
@@ -248,7 +280,7 @@
248280
},
249281
{
250282
"cell_type": "code",
251-
"execution_count": 5,
283+
"execution_count": 6,
252284
"metadata": {},
253285
"outputs": [
254286
{
@@ -262,7 +294,7 @@
262294
"u_rated: [10500,10500,10500]"
263295
]
264296
},
265-
"execution_count": 5,
297+
"execution_count": 6,
266298
"metadata": {},
267299
"output_type": "execute_result"
268300
}
@@ -354,7 +386,7 @@
354386
},
355387
{
356388
"cell_type": "code",
357-
"execution_count": 6,
389+
"execution_count": 7,
358390
"metadata": {},
359391
"outputs": [
360392
{
@@ -364,7 +396,7 @@
364396
" dtype={'names': ['id', 'u_rated'], 'formats': ['<i4', '<f8'], 'offsets': [0, 8], 'itemsize': 16, 'aligned': True})"
365397
]
366398
},
367-
"execution_count": 6,
399+
"execution_count": 7,
368400
"metadata": {},
369401
"output_type": "execute_result"
370402
}
@@ -396,7 +428,7 @@
396428
},
397429
{
398430
"cell_type": "code",
399-
"execution_count": 7,
431+
"execution_count": 8,
400432
"metadata": {},
401433
"outputs": [
402434
{
@@ -405,7 +437,7 @@
405437
"{'id': array([1, 2, 3]), 'u_rated': array([10500., 10500., 10500.])}"
406438
]
407439
},
408-
"execution_count": 7,
440+
"execution_count": 8,
409441
"metadata": {},
410442
"output_type": "execute_result"
411443
}
@@ -442,7 +474,7 @@
442474
},
443475
{
444476
"cell_type": "code",
445-
"execution_count": 8,
477+
"execution_count": 9,
446478
"metadata": {},
447479
"outputs": [
448480
{
@@ -474,7 +506,7 @@
474506
" 'q_specified': array([0.5, 1.5])}}"
475507
]
476508
},
477-
"execution_count": 8,
509+
"execution_count": 9,
478510
"metadata": {},
479511
"output_type": "execute_result"
480512
}
@@ -492,7 +524,7 @@
492524
},
493525
{
494526
"cell_type": "code",
495-
"execution_count": 9,
527+
"execution_count": 10,
496528
"metadata": {},
497529
"outputs": [],
498530
"source": [
@@ -513,7 +545,7 @@
513545
},
514546
{
515547
"cell_type": "code",
516-
"execution_count": 10,
548+
"execution_count": 11,
517549
"metadata": {},
518550
"outputs": [
519551
{
@@ -620,7 +652,7 @@
620652
},
621653
{
622654
"cell_type": "code",
623-
"execution_count": 11,
655+
"execution_count": 12,
624656
"metadata": {},
625657
"outputs": [
626658
{
@@ -644,7 +676,7 @@
644676
"q: [-3299418.661306348,-0.5000000701801947,-1.4999998507078594]"
645677
]
646678
},
647-
"execution_count": 11,
679+
"execution_count": 12,
648680
"metadata": {},
649681
"output_type": "execute_result"
650682
}
@@ -679,7 +711,7 @@
679711
},
680712
{
681713
"cell_type": "code",
682-
"execution_count": 12,
714+
"execution_count": 13,
683715
"metadata": {},
684716
"outputs": [
685717
{
@@ -716,7 +748,7 @@
716748
"q_specified_c: [0.1,1500]"
717749
]
718750
},
719-
"execution_count": 12,
751+
"execution_count": 13,
720752
"metadata": {},
721753
"output_type": "execute_result"
722754
}
@@ -766,7 +798,7 @@
766798
},
767799
{
768800
"cell_type": "code",
769-
"execution_count": 13,
801+
"execution_count": 14,
770802
"metadata": {},
771803
"outputs": [
772804
{
@@ -777,7 +809,7 @@
777809
" dtype={'names': ['id', 'node', 'status', 'type', 'p_specified', 'q_specified'], 'formats': ['<i4', '<i4', 'i1', 'i1', ('<f8', (3,)), ('<f8', (3,))], 'offsets': [0, 4, 8, 9, 16, 40], 'itemsize': 64, 'aligned': True})"
778810
]
779811
},
780-
"execution_count": 13,
812+
"execution_count": 14,
781813
"metadata": {},
782814
"output_type": "execute_result"
783815
}
@@ -826,7 +858,7 @@
826858
},
827859
{
828860
"cell_type": "code",
829-
"execution_count": 14,
861+
"execution_count": 15,
830862
"metadata": {},
831863
"outputs": [
832864
{
@@ -842,7 +874,7 @@
842874
" [1.5e+00, 2.5e+00, 1.5e+03]])}"
843875
]
844876
},
845-
"execution_count": 14,
877+
"execution_count": 15,
846878
"metadata": {},
847879
"output_type": "execute_result"
848880
}
@@ -897,7 +929,7 @@
897929
},
898930
{
899931
"cell_type": "code",
900-
"execution_count": 15,
932+
"execution_count": 27,
901933
"metadata": {},
902934
"outputs": [
903935
{
@@ -928,12 +960,15 @@
928960
"q_specified: [[0.5,1500,0.1],[1.5,2.5,1500]]"
929961
]
930962
},
931-
"execution_count": 15,
963+
"execution_count": 27,
932964
"metadata": {},
933965
"output_type": "execute_result"
934966
}
935967
],
936968
"source": [
969+
"# TODO Decisions: Asymmetric attributes should be fixed list array or individual phases or both?\n",
970+
"\n",
971+
"\n",
937972
"asym_load_input_dtype = initialize_array(\"input\", \"asym_load\", 0).dtype\n",
938973
"print(\"asym_load:\", asym_load_input_dtype)\n",
939974
"# asym_float_type = pa.struct([(\"a\", pa.float64()), (\"b\", pa.float64()), (\"c\", pa.float64())])\n",
@@ -955,7 +990,7 @@
955990
},
956991
{
957992
"cell_type": "code",
958-
"execution_count": 16,
993+
"execution_count": 17,
959994
"metadata": {},
960995
"outputs": [],
961996
"source": [
@@ -972,7 +1007,7 @@
9721007
},
9731008
{
9741009
"cell_type": "code",
975-
"execution_count": 17,
1010+
"execution_count": 18,
9761011
"metadata": {},
9771012
"outputs": [
9781013
{
@@ -988,7 +1023,7 @@
9881023
" [1.5e+00, 2.5e+00, 1.5e+03]])}"
9891024
]
9901025
},
991-
"execution_count": 17,
1026+
"execution_count": 18,
9921027
"metadata": {},
9931028
"output_type": "execute_result"
9941029
}
@@ -1036,7 +1071,7 @@
10361071
},
10371072
{
10381073
"cell_type": "code",
1039-
"execution_count": 18,
1074+
"execution_count": 19,
10401075
"metadata": {},
10411076
"outputs": [
10421077
{
@@ -1095,7 +1130,7 @@
10951130
"2 -0.004338 -2.098733 2.090057"
10961131
]
10971132
},
1098-
"execution_count": 18,
1133+
"execution_count": 19,
10991134
"metadata": {},
11001135
"output_type": "execute_result"
11011136
}
@@ -1129,7 +1164,7 @@
11291164
},
11301165
{
11311166
"cell_type": "code",
1132-
"execution_count": 19,
1167+
"execution_count": 20,
11331168
"metadata": {},
11341169
"outputs": [
11351170
{
@@ -1167,7 +1202,7 @@
11671202
"..."
11681203
]
11691204
},
1170-
"execution_count": 19,
1205+
"execution_count": 20,
11711206
"metadata": {},
11721207
"output_type": "execute_result"
11731208
}
@@ -1206,7 +1241,84 @@
12061241
},
12071242
{
12081243
"cell_type": "code",
1209-
"execution_count": 20,
1244+
"execution_count": 21,
1245+
"metadata": {},
1246+
"outputs": [
1247+
{
1248+
"data": {
1249+
"text/plain": [
1250+
"pyarrow.Table\n",
1251+
"id: int32\n",
1252+
"energized: int8\n",
1253+
"u_pu_a: double\n",
1254+
"u_pu_b: double\n",
1255+
"u_pu_c: double\n",
1256+
"u_a: double\n",
1257+
"u_b: double\n",
1258+
"u_c: double\n",
1259+
"u_angle_a: double\n",
1260+
"u_angle_b: double\n",
1261+
"u_angle_c: double\n",
1262+
"p_a: double\n",
1263+
"p_b: double\n",
1264+
"p_c: double\n",
1265+
"q_a: double\n",
1266+
"q_b: double\n",
1267+
"q_c: double\n",
1268+
"----\n",
1269+
"id: [[1,2,3]]\n",
1270+
"energized: [[1,1,1]]\n",
1271+
"u_pu_a: [[1.0003248257977395,1.0028803762176164,1.0041143008174032]]\n",
1272+
"u_pu_b: [[1.0003243769486854,1.0028710993140406,1.0041033583077175]]\n",
1273+
"u_pu_c: [[1.00032436416241,1.0028730789021523,1.0041004935738533]]\n",
1274+
"u_a: [[6064.146978239599,6079.639179329456,6087.119449677845]]\n",
1275+
"u_b: [[6064.144257236815,6079.582941090301,6087.053114238262]]\n",
1276+
"u_c: [[6064.1441797241405,6079.594941705457,6087.035747712152]]\n",
1277+
"u_angle_a: [[-0.00006651848125694397,-0.0029298831864832267,-0.004337685507209373]]\n",
1278+
"u_angle_b: [[-2.094461573665813,-2.0973219974462594,-2.098732840554144]]\n",
1279+
"..."
1280+
]
1281+
},
1282+
"execution_count": 21,
1283+
"metadata": {},
1284+
"output_type": "execute_result"
1285+
}
1286+
],
1287+
"source": [
1288+
"def numpy_columnar_to_arrow_combined(data: np.ndarray) -> pa.lib.table:\n",
1289+
" \"\"\"Convert NumPy data to Arrow data.\"\"\"\n",
1290+
" simple_data_types = []\n",
1291+
" multi_value_data_types = []\n",
1292+
"\n",
1293+
" for name, (dtype, _) in data.dtype.fields.items():\n",
1294+
" if len(dtype.shape) == 0:\n",
1295+
" simple_data_types.append(name)\n",
1296+
" else:\n",
1297+
" multi_value_data_types.append(name)\n",
1298+
"\n",
1299+
" result = pa.table(pd.DataFrame(data[simple_data_types]))\n",
1300+
"\n",
1301+
" phases = (\"a\", \"b\", \"c\")\n",
1302+
" for name in multi_value_data_types:\n",
1303+
" column = data[name]\n",
1304+
"\n",
1305+
" assert column.shape[1] == len(phases), \"Asymmetric data has 3 phase output\"\n",
1306+
"\n",
1307+
" for phase_index, phase in enumerate(phases):\n",
1308+
" sub_column = column[:, phase_index]\n",
1309+
" result = result.append_column(f\"{name}_{phase}\", [pd.Series(sub_column)])\n",
1310+
"\n",
1311+
" return result\n",
1312+
"\n",
1313+
"\n",
1314+
"pa_asym_node_result = numpy_to_arrow(asym_result[\"node\"])\n",
1315+
"\n",
1316+
"pa_asym_node_result"
1317+
]
1318+
},
1319+
{
1320+
"cell_type": "code",
1321+
"execution_count": 22,
12101322
"metadata": {},
12111323
"outputs": [],
12121324
"source": [

0 commit comments

Comments
 (0)