Skip to content

Commit cebe431

Browse files
sirosensirosen
authored
Import the 'looping batched move' example (#12)
This example originates in the globus-flows-trigger-examples repository. Make only minor updates to fit it into this repo (primarily, convert the readme to adoc).
1 parent a997ee0 commit cebe431

File tree

5 files changed

+193
-0
lines changed

5 files changed

+193
-0
lines changed

‎transfer_examples/looping_batched_move/.doc_config.yaml‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
# configuration for conversion to docs.globus.org
2+
title: 'Looping Batched Move'
3+
short_description: |
4+
Use transfer and deletion to 'move' files from a source to a destination, in batches of 100 files.
5+
Repeat this process until the source directory is empty.
6+
7+
example_dir: 'looping_batched_move'
8+
index_source:
9+
copy: "README.adoc"
10+
append_source_blocks: true
11+
12+
menu_weight: 350

‎transfer_examples/looping_batched_move/README.adoc‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
= Looping Batched Move
2+
3+
[TIP]
4+
=====
5+
This example is a demonstration of looping in a **flow**.
6+
For most use cases, moving a directory in a single transfer **task** is a more appropriate solution.
7+
=====
8+
9+
This example demonstrates while-loop-style looping in a **flow**.
10+
It moves files from a source to a destination until the source is empty.
11+
12+
The contents of the given source path are transferred in batches of 100 items and are deleted from the source path as the transfers succeed.
13+
14+
[NOTE]
15+
======
16+
The sample **flow** does not validate the results of the transfer and delete **tasks**.
17+
18+
Even if a transfer **task** fails, the **flow** will attempt to delete the batch of items.
19+
This could lead to data loss.
20+
21+
Even if a delete **task** fails, the **flow** will continue to loop if there are still items to transfer in the source path. This could lead to an infinite loop.
22+
======
23+
24+
== Highlights
25+
26+
The **flow** starts with an `ls` operation on the source in the `List` **state**, and then uses a `Choice` **state**, `CheckIfDone`, to inspect the results.
27+
`Choice` **states** allow **flow**s to dispatch and make loops possible.
28+
29+
Although the **flow** has some known failure modes, it can rely on the guarantee that when the `Transfer` **state** is done executing, the data transfer will be complete.
30+
That makes it relatively safe to delete data in the `Delete` **state**.

‎transfer_examples/looping_batched_move/definition.json‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
{
2+
"StartAt": "List",
3+
"States": {
4+
"List": {
5+
"Comment": "As an improvement, it may be worthwhile to first verify that $.source.path is a directory. This can be accomplished with two states -- an 'Action' state that calls 'stat', and a 'Choice' state that confirms the type is a 'dir'.",
6+
"Type": "Action",
7+
"ActionUrl": "https://transfer.actions.globus.org/ls",
8+
"Parameters": {
9+
"endpoint_id.$": "$.source.id",
10+
"path.$": "$.source.path",
11+
"limit": 100
12+
},
13+
"ResultPath": "$.ls_results",
14+
"Next": "CheckIfDone"
15+
},
16+
"CheckIfDone": {
17+
"Comment": "This is the gatekeeper for looping. This will continue to loop until all files and directories have been transferred and deleted.",
18+
"Type": "Choice",
19+
"Choices": [
20+
{
21+
"Variable": "$.ls_results.details.total",
22+
"NumericEquals": 0,
23+
"Next": "Done"
24+
}
25+
],
26+
"Default": "PrepareTransferDATA"
27+
},
28+
"Done": {
29+
"Type": "Pass",
30+
"End": true
31+
},
32+
"PrepareTransferDATA": {
33+
"Type": "ExpressionEval",
34+
"ResultPath": "$.transformations",
35+
"Next": "Transfer",
36+
"Parameters": {
37+
"transfer_DATA.=": "[{'source_path': source.path.rstrip('/') + '/' + item.name, 'destination_path': destination.path.rstrip('/') + '/' + item.name} for item in ls_results.details.DATA]"
38+
}
39+
},
40+
"Transfer": {
41+
"Comment": "Some validation may need to be added to verify that the items were transferred. This can be accomplished using a Choice state that looks at values in the $.transfer_results object.",
42+
"Type": "Action",
43+
"ActionUrl": "https://transfer.actions.globus.org/transfer",
44+
"ResultPath": "$.transfer_results",
45+
"Next": "PrepareDeleteDATA",
46+
"Parameters": {
47+
"DATA.$": "$.transformations.transfer_DATA",
48+
"source_endpoint.$": "$.source.id",
49+
"destination_endpoint.$": "$.destination.id"
50+
}
51+
},
52+
"PrepareDeleteDATA": {
53+
"Type": "ExpressionEval",
54+
"ResultPath": "$.transformations",
55+
"Next": "Delete",
56+
"Parameters": {
57+
"delete_DATA.=": "[{'path': source.path.rstrip('/') + '/' + item.name} for item in ls_results.details.DATA]"
58+
}
59+
},
60+
"Delete": {
61+
"Comment": "Some validation may need to be added to verify that the items were deleted. This can be accomplished using a Choice state that looks at values in the $.delete_results object. The delete task may also benefit from some options like 'ignore_missing'.",
62+
"Type": "Action",
63+
"ActionUrl": "https://transfer.actions.globus.org/delete",
64+
"ResultPath": "$.delete_results",
65+
"Next": "List",
66+
"Parameters": {
67+
"DATA.$": "$.transformations.delete_DATA",
68+
"endpoint.$": "$.source.id"
69+
}
70+
}
71+
}
72+
}

‎transfer_examples/looping_batched_move/input_schema.json‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
{
2+
"type": "object",
3+
"required": [
4+
"source",
5+
"destination"
6+
],
7+
"properties": {
8+
"source": {
9+
"type": "object",
10+
"title": "Source",
11+
"format": "globus-collection",
12+
"required": [
13+
"id",
14+
"path"
15+
],
16+
"properties": {
17+
"id": {
18+
"type": "string",
19+
"title": "Source Collection ID",
20+
"format": "uuid",
21+
"description": "The UUID for the collection which serves as the source of the Move"
22+
},
23+
"path": {
24+
"type": "string",
25+
"title": "Source Collection Path",
26+
"description": "The path on the source collection for the data"
27+
}
28+
},
29+
"propertyOrder": [
30+
"id",
31+
"path"
32+
],
33+
"additionalProperties": false
34+
},
35+
"destination": {
36+
"description": "NOTE! The *contents* of the source directory will be transferred to the directory selected here!",
37+
"type": "object",
38+
"title": "Destination",
39+
"format": "globus-collection",
40+
"required": [
41+
"id",
42+
"path"
43+
],
44+
"properties": {
45+
"id": {
46+
"type": "string",
47+
"title": "Destination Collection ID",
48+
"format": "uuid",
49+
"description": "The UUID for the collection which serves as the destination for the Move"
50+
},
51+
"path": {
52+
"type": "string",
53+
"title": "Destination Collection Path",
54+
"description": "The path on the destination collection where the data will be stored"
55+
}
56+
},
57+
"propertyOrder": [
58+
"id",
59+
"path"
60+
],
61+
"additionalProperties": false
62+
}
63+
},
64+
"propertyOrder": [
65+
"source",
66+
"destination"
67+
],
68+
"additionalProperties": false
69+
}

‎transfer_examples/looping_batched_move/sample_input.json‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
{
2+
"source": {
3+
"id": "6c54cade-bde5-45c1-bdea-f4bd71dba2cc",
4+
"path": "/~/source-directory"
5+
},
6+
"destination": {
7+
"id": "31ce9ba0-176d-45a5-add3-f37d233ba47d",
8+
"path": "/~/destination-directory"
9+
}
10+
}

0 commit comments

Comments
 (0)