Skip to content

Commit 0af64c5

Browse files
MaxTueckeGlobusMaxTueckeGlobus
committed
Responding to feedback: updated flow examples
1 parent e77c42a commit 0af64c5

File tree

4 files changed

+63
-103
lines changed

4 files changed

+63
-103
lines changed

‎compute_transfer_examples/collection_transfer_requires_flow/compute_transfer_example_1_definition.json‎

Lines changed: 17 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -10,52 +10,15 @@
1010
"compute_output_directory.=": "'/' + _context.run_id + '/'"
1111
},
1212
"ResultPath": "$.constants",
13-
"Next": "GetDestinationPath"
14-
},
15-
"GetDestinationPath": {
16-
"Type": "ExpressionEval",
17-
"Parameters": {
18-
"path.=": "getattr('destination_path', transfer_settings.DATA[0].destination_path)"
19-
},
20-
"ResultPath": "$.destination_path",
21-
"Next": "InitIterateSourcePaths"
13+
"Next": "CollectSourcePaths"
2214
},
23-
"InitIterateSourcePaths": {
15+
"CollectSourcePaths": {
2416
"Type": "ExpressionEval",
2517
"Parameters": {
26-
"src_paths_iterator_pos": 0,
27-
"src_paths": []
18+
"paths.=": "[item.source_path for item in transfer_settings.DATA]"
2819
},
29-
"ResultPath": "$.iterate_source_paths",
30-
"Next": "IterateSourcePathsConditional"
31-
},
32-
"IterateSourcePathsConditional": {
33-
"Type": "ExpressionEval",
34-
"Parameters": {
35-
"conditional.=": "len(transfer_settings.DATA) > iterate_source_paths.src_paths_iterator_pos"
36-
},
37-
"ResultPath": "$.iterate_source_paths_conditional",
38-
"Next": "IterateSourcePaths"
39-
},
40-
"IterateSourcePaths": {
41-
"Type": "Choice",
42-
"Choices": [
43-
{
44-
"Variable": "$.iterate_source_paths_conditional.conditional",
45-
"BooleanEquals": true,
46-
"Next": "AppendSourcePath"
47-
}
48-
],
49-
"Default": "MakeComputeWorkingDir"
50-
},
51-
"AppendSourcePath": {
52-
"Type": "ExpressionEval",
53-
"Parameters": {
54-
"src_paths.=": "iterate_source_paths.src_paths + [transfer_settings.DATA[iterate_source_paths.src_paths_iterator_pos].source_path]",
55-
"src_paths_iterator_pos.=": "iterate_source_paths.src_paths_iterator_pos + 1"
56-
},
57-
"ResultPath": "$.iterate_source_paths",
58-
"Next": "IterateSourcePathsConditional"
20+
"ResultPath": "$.src",
21+
"Next": "MakeComputeWorkingDir"
5922
},
6023
"MakeComputeWorkingDir": {
6124
"Type": "Action",
@@ -77,14 +40,22 @@
7740
"function_id.$": "$.constants.compute_function_id",
7841
"args": [],
7942
"kwargs": {
80-
"src_paths.$": "$.iterate_source_paths.src_paths",
43+
"src_paths.$": "$.src.paths",
8144
"dest_path.$": "$.constants.compute_output_directory",
8245
"gcs_base_path.$": "$.constants.gcs_base_path"
8346
}
8447
}
8548
]
8649
},
87-
"ResultPath": "$.run_compute_result",
50+
"ResultPath": "$.compute_result",
51+
"Next": "GetDestinationPath"
52+
},
53+
"GetDestinationPath": {
54+
"Type": "ExpressionEval",
55+
"Parameters": {
56+
"path.=": "getattr('destination_path', '/~/' + pathsplit(compute_result.details.result[0])[1])"
57+
},
58+
"ResultPath": "$.destination",
8859
"Next": "TransferFromComputeEndpoint"
8960
},
9061
"TransferFromComputeEndpoint": {
@@ -95,8 +66,8 @@
9566
"destination_endpoint.$": "$.transfer_settings.destination_endpoint",
9667
"DATA": [
9768
{
98-
"source_path.=": "run_compute_result.details.result[0]",
99-
"destination_path.$": "$.destination_path.path"
69+
"source_path.=": "compute_result.details.result[0]",
70+
"destination_path.$": "$.destination.path"
10071
}
10172
]
10273
},

‎compute_transfer_examples/collection_transfer_requires_flow/compute_transfer_example_1_schema.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111
"destination_path": {
1212
"type": "string",
1313
"title": "Destination Collection Path",
14-
"description": "The path on the destination collection to transfer the compute output"
14+
"description": "The path on the destination collection for the tarball file"
1515
}
1616
},
1717
"additionalProperties": false

‎compute_transfer_examples/collection_transfer_requires_flow/compute_transfer_example_2_definition.json‎

Lines changed: 40 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -4,68 +4,69 @@
44
"SetConstants": {
55
"Type": "ExpressionEval",
66
"Parameters": {
7+
"gcs_endpoint_id": "<INSERT YOUR GCS ENDPOINT ID HERE>",
78
"gcs_base_path": "/",
89
"compute_endpoint_id": "<INSERT YOUR COMPUTE ENDPOINT ID HERE>",
910
"compute_function_id": "<INSERT YOUR COMPUTE FUNCTION ID HERE>",
1011
"compute_output_directory.=": "'/' + _context.run_id + '/'"
1112
},
1213
"ResultPath": "$.constants",
13-
"Next": "InitIterateSourcePaths"
14+
"Next": "InitIterateTransferData"
1415
},
15-
"InitIterateSourcePaths": {
16+
"InitIterateTransferData": {
1617
"Type": "ExpressionEval",
1718
"Parameters": {
18-
"src_paths_iterator_pos": 0,
19-
"src_paths": [],
20-
"transfer_from_src_data": []
19+
"transfer_data_iterator_pos": 0,
20+
"transfer_from_src_data": [],
21+
"src_paths": []
2122
},
22-
"ResultPath": "$.iterate_source_paths",
23-
"Next": "IterateSourcePathsConditional"
23+
"ResultPath": "$.iterate_transfer_data",
24+
"Next": "IterateTransferDataConditional"
2425
},
25-
"IterateSourcePathsConditional": {
26+
"IterateTransferDataConditional": {
2627
"Type": "ExpressionEval",
2728
"Parameters": {
28-
"conditional.=": "len(transfer_settings.DATA) > iterate_source_paths.src_paths_iterator_pos"
29+
"val.=": "len(transfer_settings.DATA) > iterate_transfer_data.transfer_data_iterator_pos"
2930
},
30-
"ResultPath": "$.iterate_source_paths_conditional",
31-
"Next": "IterateSourcePaths"
31+
"ResultPath": "$.iterate_transfer_data_conditional",
32+
"Next": "IterateTransferData"
3233
},
33-
"IterateSourcePaths": {
34+
"IterateTransferData": {
3435
"Type": "Choice",
3536
"Choices": [
3637
{
37-
"Variable": "$.iterate_source_paths_conditional.conditional",
38+
"Variable": "$.iterate_transfer_data_conditional.val",
3839
"BooleanEquals": true,
39-
"Next": "StatSourcePath"
40+
"Next": "StatTransferData"
4041
}
4142
],
4243
"Default": "MakeComputeWorkingDir"
4344
},
44-
"StatSourcePath": {
45+
"StatTransferData": {
4546
"Type": "Action",
4647
"ActionUrl": "https://transfer.actions.globus.org/stat",
4748
"Parameters": {
4849
"endpoint_id.$": "$.transfer_settings.source_endpoint",
49-
"path.=": "transfer_settings.DATA[iterate_source_paths.src_paths_iterator_pos].source_path"
50+
"path.=": "transfer_settings.DATA[iterate_transfer_data.transfer_data_iterator_pos].source_path"
5051
},
51-
"ResultPath": "$.source_path_stat_result",
52-
"Next": "AppendSourcePath"
52+
"ResultPath": "$.transfer_data_stat_result",
53+
"Next": "AppendTransferData"
5354
},
54-
"AppendSourcePath": {
55+
"AppendTransferData": {
5556
"Type": "ExpressionEval",
5657
"Parameters": {
57-
"src_paths.=": "iterate_source_paths.src_paths + [constants.compute_output_directory + source_path_stat_result.details.name]",
58-
"transfer_from_src_data.=": "iterate_source_paths.transfer_from_src_data + [{'source_path': transfer_settings.DATA[iterate_source_paths.src_paths_iterator_pos].source_path, 'destination_path': constants.compute_output_directory + source_path_stat_result.details.name, 'DATA_TYPE': transfer_settings.DATA[iterate_source_paths.src_paths_iterator_pos].DATA_TYPE, 'recursive': transfer_settings.DATA[iterate_source_paths.src_paths_iterator_pos].recursive}]",
59-
"src_paths_iterator_pos.=": "iterate_source_paths.src_paths_iterator_pos + 1"
58+
"src_paths.=": "iterate_transfer_data.src_paths + [constants.compute_output_directory + transfer_data_stat_result.details.name]",
59+
"transfer_from_src_data.=": "iterate_transfer_data.transfer_from_src_data + [{'source_path': transfer_settings.DATA[iterate_transfer_data.transfer_data_iterator_pos].source_path, 'destination_path': constants.compute_output_directory + transfer_data_stat_result.details.name, 'DATA_TYPE': transfer_settings.DATA[iterate_transfer_data.transfer_data_iterator_pos].DATA_TYPE, 'recursive': transfer_settings.DATA[iterate_transfer_data.transfer_data_iterator_pos].recursive}]",
60+
"transfer_data_iterator_pos.=": "iterate_transfer_data.transfer_data_iterator_pos + 1"
6061
},
61-
"ResultPath": "$.iterate_source_paths",
62-
"Next": "IterateSourcePathsConditional"
62+
"ResultPath": "$.iterate_transfer_data",
63+
"Next": "IterateTransferDataConditional"
6364
},
6465
"MakeComputeWorkingDir": {
6566
"Type": "Action",
6667
"ActionUrl": "https://transfer.actions.globus.org/mkdir",
6768
"Parameters": {
68-
"endpoint_id.$": "$.transfer_settings.destination_endpoint",
69+
"endpoint_id.$": "$.constants.gcs_endpoint_id",
6970
"path.$": "$.constants.compute_output_directory"
7071
},
7172
"ResultPath": "$.mkdir_result",
@@ -76,8 +77,8 @@
7677
"ActionUrl": "https://transfer.actions.globus.org/transfer",
7778
"Parameters": {
7879
"source_endpoint.$": "$.transfer_settings.source_endpoint",
79-
"destination_endpoint.$": "$.transfer_settings.destination_endpoint",
80-
"DATA.$": "$.iterate_source_paths.transfer_from_src_data"
80+
"destination_endpoint.$": "$.constants.gcs_endpoint_id",
81+
"DATA.$": "$.iterate_transfer_data.transfer_from_src_data"
8182
},
8283
"ResultPath": "$.transfer_from_src_result",
8384
"Next": "RunComputeFunction"
@@ -92,22 +93,30 @@
9293
"function_id.$": "$.constants.compute_function_id",
9394
"args": [],
9495
"kwargs": {
95-
"src_paths.$": "$.iterate_source_paths.src_paths",
96+
"src_paths.$": "$.iterate_transfer_data.src_paths",
9697
"dest_path.$": "$.constants.compute_output_directory",
9798
"gcs_base_path.$": "$.constants.gcs_base_path"
9899
}
99100
}
100101
]
101102
},
102103
"ResultPath": "$.compute_result",
104+
"Next": "GetDestinationPath"
105+
},
106+
"GetDestinationPath": {
107+
"Type": "ExpressionEval",
108+
"Parameters": {
109+
"path.=": "getattr('destination_path', '/~/' + pathsplit(compute_result.details.result[0])[1])"
110+
},
111+
"ResultPath": "$.destination",
103112
"Next": "TransferFromComputeEndpoint"
104113
},
105114
"TransferFromComputeEndpoint": {
106115
"Type": "Action",
107116
"ActionUrl": "https://transfer.actions.globus.org/transfer",
108117
"Parameters": {
109-
"source_endpoint.$": "$.transfer_settings.destination_endpoint",
110-
"destination_endpoint.$": "$.destination.id",
118+
"source_endpoint.$": "$.constants.gcs_endpoint_id",
119+
"destination_endpoint.$": "$.transfer_settings.destination_endpoint",
111120
"DATA": [
112121
{
113122
"source_path.=": "compute_result.details.result[0]",
@@ -122,7 +131,7 @@
122131
"Type": "Action",
123132
"ActionUrl": "https://transfer.actions.globus.org/delete",
124133
"Parameters": {
125-
"endpoint.$": "$.transfer_settings.destination_endpoint",
134+
"endpoint.$": "$.constants.gcs_endpoint_id",
126135
"recursive": true,
127136
"DATA": [
128137
{

‎compute_transfer_examples/collection_transfer_requires_flow/compute_transfer_example_2_schema.json‎

Lines changed: 5 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -1,37 +1,17 @@
11
{
22
"type": "object",
33
"required": [
4-
"transfer_settings",
5-
"destination"
4+
"transfer_settings"
65
],
76
"properties": {
87
"transfer_settings": {
98
"type": "object",
109
"format": "globus-transfer-transfer#0.10"
1110
},
12-
"destination": {
13-
"type": "object",
14-
"title": "Destination",
15-
"format": "globus-collection",
16-
"required": [
17-
"id",
18-
"path"
19-
],
20-
"properties": {
21-
"id": {
22-
"type": "string",
23-
"title": "Destination Collection ID",
24-
"format": "uuid",
25-
"description": "The endpoint id of the destination collection"
26-
},
27-
"path": {
28-
"type": "string",
29-
"title": "Destination Collection Path",
30-
"description": "The path on the destination collection to transfer the compute output"
31-
}
32-
},
33-
"description": "The destination for the data",
34-
"additionalProperties": false
11+
"destination_path": {
12+
"type": "string",
13+
"title": "Destination Collection Path",
14+
"description": "The path on the destination collection for the tarball file"
3515
}
3616
},
3717
"additionalProperties": false

0 commit comments

Comments
 (0)