Skip to content

Commit 6a26d59

Browse files
honnixhonnix
authored
Blob and struct support (#258)
* Bring back Blob support Signed-off-by: Hongxin Liang <[email protected]> * Remove BlobTypeDescription Signed-off-by: Hongxin Liang <[email protected]> * Add blob back Signed-off-by: Hongxin Liang <[email protected]> * Blob in list and map Signed-off-by: Hongxin Liang <[email protected]> * Clean up Signed-off-by: Hongxin Liang <[email protected]> * Support struct (#259) Signed-off-by: Hongxin Liang <[email protected]> * Support struct in Scala layer (#262) Signed-off-by: Hongxin Liang <[email protected]> --------- Signed-off-by: Hongxin Liang <[email protected]>
1 parent 73c6c71 commit 6a26d59

File tree

43 files changed

+1003
-354
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

43 files changed

+1003
-354
lines changed

‎.scalafmt.conf‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
1-
version=2.5.2
1+
version=3.7.14
22
runner.dialect=scala212source3
33

‎flytekit-examples-scala/src/main/resources/META-INF/services/org.flyte.flytekit.SdkLaunchPlanRegistry‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
org.flyte.examples.flytekitscala.FibonacciLaunchPlan
1+
org.flyte.examples.flytekitscala.LaunchPlanRegistry

‎flytekit-examples-scala/src/main/resources/META-INF/services/org.flyte.flytekit.SdkRunnableTask‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,3 +3,4 @@ org.flyte.examples.flytekitscala.SumTask
33
org.flyte.examples.flytekitscala.GreetTask
44
org.flyte.examples.flytekitscala.AddQuestionTask
55
org.flyte.examples.flytekitscala.NoInputsTask
6+
org.flyte.examples.flytekitscala.NestedIOTask

‎flytekit-examples-scala/src/main/resources/META-INF/services/org.flyte.flytekit.SdkWorkflow‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,3 @@
11
org.flyte.examples.flytekitscala.FibonacciWorkflow
22
org.flyte.examples.flytekitscala.WelcomeWorkflow
3+
org.flyte.examples.flytekitscala.NestedIOWorkflow

flytekit-examples-scala/src/main/scala/org/flyte/examples/flytekitscala/FibonacciLaunchPlan.scala renamed to flytekit-examples-scala/src/main/scala/org/flyte/examples/flytekitscala/LaunchPlanRegistry.scala

Lines changed: 33 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,9 @@ import org.flyte.flytekit.{SdkLaunchPlan, SimpleSdkLaunchPlanRegistry}
2020
import org.flyte.flytekitscala.SdkScalaType
2121

2222
case class FibonacciLaunchPlanInput(fib0: Long, fib1: Long)
23+
case class NestedIOLaunchPlanInput(name: String, generic: Nested)
2324

24-
class FibonacciLaunchPlan extends SimpleSdkLaunchPlanRegistry {
25+
class LaunchPlanRegistry extends SimpleSdkLaunchPlanRegistry {
2526
// Register default launch plans for all workflows
2627
registerDefaultLaunchPlans()
2728

@@ -53,4 +54,35 @@ class FibonacciLaunchPlan extends SimpleSdkLaunchPlanRegistry {
5354
.withDefaultInput("fib0", 0L)
5455
.withDefaultInput("fib1", 1L)
5556
)
57+
58+
registerLaunchPlan(
59+
SdkLaunchPlan
60+
.of(new NestedIOWorkflow)
61+
.withName("NestedIOWorkflowLaunchPlan")
62+
.withDefaultInput(
63+
SdkScalaType[NestedIOLaunchPlanInput],
64+
NestedIOLaunchPlanInput(
65+
"yo",
66+
Nested(
67+
boolean = true,
68+
1.toByte,
69+
2.toShort,
70+
3,
71+
4L,
72+
5.toFloat,
73+
6.toDouble,
74+
"hello",
75+
List("1", "2"),
76+
List(NestedNested(7.toDouble, NestedNestedNested("world"))),
77+
Map("1" -> "1", "2" -> "2"),
78+
Map("foo" -> NestedNested(7.toDouble, NestedNestedNested("world"))),
79+
Some(false),
80+
None,
81+
Some(List("3", "4")),
82+
Some(Map("3" -> "3", "4" -> "4")),
83+
NestedNested(7.toDouble, NestedNestedNested("world"))
84+
)
85+
)
86+
)
87+
)
5688
}

‎flytekit-examples-scala/src/main/scala/org/flyte/examples/flytekitscala/NestedIOTask.scala‎

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
/*
2+
* Copyright 2023 Flyte Authors.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing,
11+
* software distributed under the License is distributed on an
12+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
13+
* KIND, either express or implied. See the License for the
14+
* specific language governing permissions and limitations
15+
* under the License.
16+
*/
17+
package org.flyte.examples.flytekitscala
18+
19+
import org.flyte.flytekit.{SdkBindingData, SdkRunnableTask, SdkTransform}
20+
import org.flyte.flytekitscala.{
21+
Description,
22+
SdkBindingDataFactory,
23+
SdkScalaType
24+
}
25+
26+
case class NestedNestedNested(string: String)
27+
case class NestedNested(double: Double, nested: NestedNestedNested)
28+
case class Nested(
29+
boolean: Boolean,
30+
byte: Byte,
31+
short: Short,
32+
int: Int,
33+
long: Long,
34+
float: Float,
35+
double: Double,
36+
string: String,
37+
list: List[String],
38+
listOfNested: List[NestedNested],
39+
map: Map[String, String],
40+
mapOfNested: Map[String, NestedNested],
41+
optBoolean: Option[Boolean],
42+
optByte: Option[Byte],
43+
optList: Option[List[String]],
44+
optMap: Option[Map[String, String]],
45+
nested: NestedNested
46+
)
47+
case class NestedIOTaskInput(
48+
@Description("the name of the person to be greeted")
49+
name: SdkBindingData[String],
50+
@Description("a nested input")
51+
generic: SdkBindingData[Nested]
52+
)
53+
case class NestedIOTaskOutput(
54+
@Description("the name of the person to be greeted")
55+
name: SdkBindingData[String],
56+
@Description("a nested input")
57+
generic: SdkBindingData[Nested]
58+
)
59+
60+
/** Example Flyte task that takes a name as the input and outputs a simple
61+
* greeting message.
62+
*/
63+
class NestedIOTask
64+
extends SdkRunnableTask[
65+
NestedIOTaskInput,
66+
NestedIOTaskOutput
67+
](
68+
SdkScalaType[NestedIOTaskInput],
69+
SdkScalaType[NestedIOTaskOutput]
70+
) {
71+
72+
/** Defines task behavior. This task takes a name as the input, wraps it in a
73+
* welcome message, and outputs the message.
74+
*
75+
* @param input
76+
* the name of the person to be greeted
77+
* @return
78+
* the welcome message
79+
*/
80+
override def run(input: NestedIOTaskInput): NestedIOTaskOutput =
81+
NestedIOTaskOutput(
82+
input.name,
83+
input.generic
84+
)
85+
}

‎flytekit-examples-scala/src/main/scala/org/flyte/examples/flytekitscala/NestedIOWorkflow.scala‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
/*
2+
* Copyright 2020-2023 Flyte Authors.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing,
11+
* software distributed under the License is distributed on an
12+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
13+
* KIND, either express or implied. See the License for the
14+
* specific language governing permissions and limitations
15+
* under the License.
16+
*/
17+
package org.flyte.examples.flytekitscala
18+
19+
import org.flyte.flytekitscala.{
20+
SdkScalaType,
21+
SdkScalaWorkflow,
22+
SdkScalaWorkflowBuilder
23+
}
24+
25+
class NestedIOWorkflow
26+
extends SdkScalaWorkflow[NestedIOTaskInput, Unit](
27+
SdkScalaType[NestedIOTaskInput],
28+
SdkScalaType.unit
29+
) {
30+
31+
override def expand(
32+
builder: SdkScalaWorkflowBuilder,
33+
input: NestedIOTaskInput
34+
): Unit = {
35+
builder.apply(new NestedIOTask(), input)
36+
}
37+
}

‎flytekit-examples/src/main/java/org/flyte/examples/AllInputsTask.java‎

Lines changed: 27 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
import java.time.Instant;
2323
import java.util.List;
2424
import java.util.Map;
25+
import org.flyte.api.v1.Blob;
2526
import org.flyte.flytekit.SdkBindingData;
2627
import org.flyte.flytekit.SdkRunnableTask;
2728
import org.flyte.flytekit.jackson.JacksonSdkType;
@@ -34,8 +35,20 @@ public AllInputsTask() {
3435
JacksonSdkType.of(AutoAllInputsInput.class), JacksonSdkType.of(AutoAllInputsOutput.class));
3536
}
3637

38+
@AutoValue
39+
public abstract static class Nested {
40+
public abstract String hello();
41+
42+
public abstract String world();
43+
44+
public static Nested create(String hello, String world) {
45+
return new AutoValue_AllInputsTask_Nested(hello, world);
46+
}
47+
}
48+
3749
@AutoValue
3850
public abstract static class AutoAllInputsInput {
51+
3952
public abstract SdkBindingData<Long> i();
4053

4154
public abstract SdkBindingData<Double> f();
@@ -48,8 +61,9 @@ public abstract static class AutoAllInputsInput {
4861

4962
public abstract SdkBindingData<Duration> d();
5063

51-
// TODO add blobs to sdkbinding data
52-
// public abstract SdkBindingData<Blob> blob();
64+
public abstract SdkBindingData<Blob> blob();
65+
66+
public abstract SdkBindingData<Nested> generic();
5367

5468
public abstract SdkBindingData<List<String>> l();
5569

@@ -66,13 +80,14 @@ public static AutoAllInputsInput create(
6680
SdkBindingData<Boolean> b,
6781
SdkBindingData<Instant> t,
6882
SdkBindingData<Duration> d,
69-
// Blob blob,
83+
SdkBindingData<Blob> blob,
84+
SdkBindingData<Nested> generic,
7085
SdkBindingData<List<String>> l,
7186
SdkBindingData<Map<String, String>> m,
7287
SdkBindingData<List<String>> emptyList,
7388
SdkBindingData<Map<String, Long>> emptyMap) {
7489
return new AutoValue_AllInputsTask_AutoAllInputsInput(
75-
i, f, s, b, t, d, l, m, emptyList, emptyMap);
90+
i, f, s, b, t, d, blob, generic, l, m, emptyList, emptyMap);
7691
}
7792
}
7893

@@ -91,8 +106,9 @@ public abstract static class AutoAllInputsOutput {
91106

92107
public abstract SdkBindingData<Duration> d();
93108

94-
// TODO add blobs to sdkbinding data
95-
// public abstract SdkBindingData<Blob> blob();
109+
public abstract SdkBindingData<Blob> blob();
110+
111+
public abstract SdkBindingData<Nested> generic();
96112

97113
public abstract SdkBindingData<List<String>> l();
98114

@@ -109,12 +125,14 @@ public static AutoAllInputsOutput create(
109125
SdkBindingData<Boolean> b,
110126
SdkBindingData<Instant> t,
111127
SdkBindingData<Duration> d,
128+
SdkBindingData<Blob> blob,
129+
SdkBindingData<Nested> generic,
112130
SdkBindingData<List<String>> l,
113131
SdkBindingData<Map<String, String>> m,
114132
SdkBindingData<List<String>> emptyList,
115133
SdkBindingData<Map<String, Long>> emptyMap) {
116134
return new AutoValue_AllInputsTask_AutoAllInputsOutput(
117-
i, f, s, b, t, d, l, m, emptyList, emptyMap);
135+
i, f, s, b, t, d, blob, generic, l, m, emptyList, emptyMap);
118136
}
119137
}
120138

@@ -127,6 +145,8 @@ public AutoAllInputsOutput run(AutoAllInputsInput input) {
127145
input.b(),
128146
input.t(),
129147
input.d(),
148+
input.blob(),
149+
input.generic(),
130150
input.l(),
131151
input.m(),
132152
input.emptyList(),

‎flytekit-examples/src/main/java/org/flyte/examples/AllInputsWorkflow.java‎

Lines changed: 28 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -24,13 +24,19 @@
2424
import java.util.Collections;
2525
import java.util.List;
2626
import java.util.Map;
27+
import org.flyte.api.v1.Blob;
28+
import org.flyte.api.v1.BlobMetadata;
29+
import org.flyte.api.v1.BlobType;
30+
import org.flyte.api.v1.BlobType.BlobDimensionality;
2731
import org.flyte.examples.AllInputsTask.AutoAllInputsOutput;
32+
import org.flyte.examples.AllInputsTask.Nested;
2833
import org.flyte.flytekit.SdkBindingData;
2934
import org.flyte.flytekit.SdkBindingDataFactory;
3035
import org.flyte.flytekit.SdkNode;
3136
import org.flyte.flytekit.SdkTypes;
3237
import org.flyte.flytekit.SdkWorkflow;
3338
import org.flyte.flytekit.SdkWorkflowBuilder;
39+
import org.flyte.flytekit.jackson.JacksonSdkLiteralType;
3440
import org.flyte.flytekit.jackson.JacksonSdkType;
3541

3642
@AutoService(SdkWorkflow.class)
@@ -57,6 +63,20 @@ public AllInputsWorkflowOutput expand(SdkWorkflowBuilder builder, Void noInput)
5763
SdkBindingDataFactory.of(true),
5864
SdkBindingDataFactory.of(someInstant),
5965
SdkBindingDataFactory.of(Duration.ofDays(1L)),
66+
SdkBindingDataFactory.of(
67+
Blob.builder()
68+
.uri("file://test/test.csv")
69+
.metadata(
70+
BlobMetadata.builder()
71+
.type(
72+
BlobType.builder()
73+
.format("")
74+
.dimensionality(BlobDimensionality.SINGLE)
75+
.build())
76+
.build())
77+
.build()),
78+
SdkBindingDataFactory.of(
79+
JacksonSdkLiteralType.of(Nested.class), Nested.create("hello", "world")),
6080
SdkBindingDataFactory.ofStringCollection(Arrays.asList("foo", "bar")),
6181
SdkBindingDataFactory.ofStringMap(Map.of("test", "test")),
6282
SdkBindingDataFactory.ofStringCollection(Collections.emptyList()),
@@ -71,6 +91,8 @@ public AllInputsWorkflowOutput expand(SdkWorkflowBuilder builder, Void noInput)
7191
outputs.b(),
7292
outputs.t(),
7393
outputs.d(),
94+
outputs.blob(),
95+
outputs.generic(),
7496
outputs.l(),
7597
outputs.m(),
7698
outputs.emptyList(),
@@ -92,8 +114,9 @@ public abstract static class AllInputsWorkflowOutput {
92114

93115
public abstract SdkBindingData<Duration> d();
94116

95-
// TODO add blobs to sdkbinding data
96-
// public abstract SdkBindingData<Blob> blob();
117+
public abstract SdkBindingData<Blob> blob();
118+
119+
public abstract SdkBindingData<Nested> generic();
97120

98121
public abstract SdkBindingData<List<String>> l();
99122

@@ -110,12 +133,14 @@ public static AllInputsWorkflow.AllInputsWorkflowOutput create(
110133
SdkBindingData<Boolean> b,
111134
SdkBindingData<Instant> t,
112135
SdkBindingData<Duration> d,
136+
SdkBindingData<Blob> blob,
137+
SdkBindingData<Nested> generic,
113138
SdkBindingData<List<String>> l,
114139
SdkBindingData<Map<String, String>> m,
115140
SdkBindingData<List<String>> emptyList,
116141
SdkBindingData<Map<String, Long>> emptyMap) {
117142
return new AutoValue_AllInputsWorkflow_AllInputsWorkflowOutput(
118-
i, f, s, b, t, d, l, m, emptyList, emptyMap);
143+
i, f, s, b, t, d, blob, generic, l, m, emptyList, emptyMap);
119144
}
120145
}
121146
}

‎flytekit-jackson/pom.xml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,10 @@
4747
<groupId>com.fasterxml.jackson.datatype</groupId>
4848
<artifactId>jackson-datatype-jsr310</artifactId>
4949
</dependency>
50+
<dependency>
51+
<groupId>com.fasterxml.jackson.datatype</groupId>
52+
<artifactId>jackson-datatype-jdk8</artifactId>
53+
</dependency>
5054
<dependency>
5155
<groupId>com.fasterxml.jackson.module</groupId>
5256
<artifactId>jackson-module-parameter-names</artifactId>

0 commit comments

Comments
 (0)