Skip to content
This repository was archived by the owner on May 14, 2025. It is now read-only.

Commit 65dbaa4

Browse files
David Turanskimarkpollack
David Turanski
authored and
markpollack
committed
Modify REST API, Shell and services to report current task execution by platform instance
- Update Limit the number concurrent task launches section in docs Fixes #2855
1 parent 8c98169 commit 65dbaa4

File tree

26 files changed

+383
-184
lines changed

26 files changed

+383
-184
lines changed

‎pom.xml‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@
1616
<!-- Note: Also make sure the parent pom version in spring-cloud-dataflow-dependencies/pom.xml is in sync -->
1717
<groupId>org.springframework.cloud</groupId>
1818
<artifactId>spring-cloud-build</artifactId>
19-
<version>2.1.3.RELEASE</version>
19+
<version>2.1.4.RELEASE</version>
2020
<relativePath />
2121
</parent>
2222
<scm>
@@ -31,19 +31,19 @@
3131

3232
<spring-cloud-dataflow-ui.version>2.0.1.RELEASE</spring-cloud-dataflow-ui.version>
3333

34-
<spring-cloud-deployer.version>2.0.0.RELEASE</spring-cloud-deployer.version>
35-
<spring-cloud-deployer-local.version>2.0.0.RELEASE</spring-cloud-deployer-local.version>
34+
<spring-cloud-deployer.version>2.0.1.BUILD-SNAPSHOT</spring-cloud-deployer.version>
35+
<spring-cloud-deployer-local.version>2.0.1.BUILD-SNAPSHOT</spring-cloud-deployer-local.version>
3636

3737
<!-- note - check version of reactor at deployer-cf/cf-java-client uses -->
38-
<spring-cloud-deployer-cloudfoundry.version>2.0.0.RELEASE</spring-cloud-deployer-cloudfoundry.version>
38+
<spring-cloud-deployer-cloudfoundry.version>2.0.1.BUILD-SNAPSHOT</spring-cloud-deployer-cloudfoundry.version>
3939
<reactor.version>3.2.0.RELEASE</reactor.version>
40-
<spring-cloud-deployer-kubernetes.version>2.0.0.RELEASE</spring-cloud-deployer-kubernetes.version>
40+
<spring-cloud-deployer-kubernetes.version>2.0.1.BUILD-SNAPSHOT</spring-cloud-deployer-kubernetes.version>
4141
<kubernetes-client.version>4.0.4</kubernetes-client.version>
4242
<spring-cloud-kubernetes.version>0.3.0.RELEASE</spring-cloud-kubernetes.version>
4343

4444
<spring-cloud-skipper.version>2.0.0.RELEASE</spring-cloud-skipper.version>
4545

46-
<spring-cloud-task.version>2.1.1.RELEASE</spring-cloud-task.version>
46+
<spring-cloud-task.version>2.1.2.BUILD-SNAPSHOT</spring-cloud-task.version>
4747

4848
<!-- Note: Make sure to update `spring-cloud-build` version and `spring-cloud-dependencies` version (spring-cloud.version)
4949
are in sync with the https://github.com/spring-cloud/spring-cloud-release/blob/master/pom.xml updates for the respective

‎spring-cloud-dataflow-classic-docs/src/test/java/org/springframework/cloud/dataflow/server/rest/documentation/TaskExecutionsDocumentation.java‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright 2017 the original author or authors.
2+
* Copyright 2017-2019 the original author or authors.
33
*
44
* Licensed under the Apache License, Version 2.0 (the "License");
55
* you may not use this file except in compliance with the License.
@@ -41,6 +41,7 @@
4141
*
4242
* @author Eric Bottard
4343
* @author Glenn Renfro
44+
* @author David Turanski
4445
*/
4546
@FixMethodOrder(MethodSorters.NAME_ASCENDING)
4647
public class TaskExecutionsDocumentation extends BaseDocumentation {
@@ -162,8 +163,10 @@ public void launchTaskCurrentCount() throws Exception {
162163
.andExpect(status().isOk())
163164
.andDo(this.documentationHandler.document(
164165
responseFields(
165-
fieldWithPath("maximumTaskExecutions").description("The number of maximum task execution"),
166-
fieldWithPath("runningExecutionCount").description("The number of number execution")
166+
fieldWithPath("[].name").description("The name of the platform instance (account)"),
167+
fieldWithPath("[].type").description("The platform type"),
168+
fieldWithPath("[].maximumTaskExecutions").description("The number of maximum task execution"),
169+
fieldWithPath("[].runningExecutionCount").description("The number of number execution")
167170
)
168171
));
169172
}

‎spring-cloud-dataflow-core/src/main/java/org/springframework/cloud/dataflow/core/AllPlatformsTaskExecutionInformation.java‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
/*
2+
* Copyright 2019 the original author or authors.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* https://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package org.springframework.cloud.dataflow.core;
18+
19+
import java.util.ArrayList;
20+
import java.util.Collections;
21+
import java.util.Comparator;
22+
import java.util.List;
23+
24+
import org.springframework.cloud.deployer.spi.task.TaskLauncher;
25+
26+
/**
27+
* @author David Turanski
28+
**/
29+
30+
public class AllPlatformsTaskExecutionInformation {
31+
32+
private List<PlatformTaskExecutionInformation> taskExecutionInformation;
33+
34+
public AllPlatformsTaskExecutionInformation(List<TaskPlatform> taskPlatforms) {
35+
this.taskExecutionInformation = buildTaskExecutionInformation(taskPlatforms);
36+
}
37+
38+
private List<PlatformTaskExecutionInformation> buildTaskExecutionInformation(List<TaskPlatform> taskPlatforms) {
39+
taskExecutionInformation = new ArrayList<>();
40+
taskPlatforms.forEach(taskPlatform -> {
41+
taskPlatform.getLaunchers().forEach(launcher -> {
42+
TaskLauncher taskLauncher = launcher.getTaskLauncher();
43+
taskExecutionInformation.add(
44+
new PlatformTaskExecutionInformation(launcher.getName(),
45+
launcher.getType(), taskLauncher.getMaximumConcurrentTasks(),
46+
taskLauncher.getRunningTaskExecutionCount()));
47+
});
48+
});
49+
50+
Collections.sort(taskExecutionInformation,
51+
Comparator.comparing(PlatformTaskExecutionInformation::getType)
52+
);
53+
54+
return taskExecutionInformation;
55+
}
56+
57+
public List<PlatformTaskExecutionInformation> getTaskExecutionInformation() {
58+
return Collections.unmodifiableList(this.taskExecutionInformation);
59+
}
60+
}

‎spring-cloud-dataflow-core/src/main/java/org/springframework/cloud/dataflow/core/PlatformTaskExecutionInformation.java‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
/*
2+
* Copyright 2019 the original author or authors.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* https://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package org.springframework.cloud.dataflow.core;
18+
19+
/**
20+
* @author David Turanski
21+
**/
22+
public class PlatformTaskExecutionInformation {
23+
24+
/**
25+
* The platform instance (account) name.
26+
*/
27+
private final String name;
28+
29+
/**
30+
* The platform type name.
31+
*/
32+
private final String type;
33+
34+
/**
35+
* The maximum concurrently running task executions allowed.
36+
*/
37+
private final int maximumTaskExecutions;
38+
39+
/**
40+
* The current number of running task executions.
41+
*/
42+
private final int runningExecutionCount;
43+
44+
public PlatformTaskExecutionInformation(String name, String type, int maximumTaskExecutions,
45+
int runningExecutionCount) {
46+
this.name = name;
47+
this.type = type;
48+
this.maximumTaskExecutions = maximumTaskExecutions;
49+
this.runningExecutionCount = runningExecutionCount;
50+
}
51+
52+
public String getName() {
53+
return name;
54+
}
55+
56+
public String getType() {
57+
return type;
58+
}
59+
60+
public int getMaximumTaskExecutions() {
61+
return maximumTaskExecutions;
62+
}
63+
64+
public int getRunningExecutionCount() {
65+
return runningExecutionCount;
66+
}
67+
}

‎spring-cloud-dataflow-docs/src/main/asciidoc/tasks.adoc‎

Lines changed: 13 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -229,11 +229,19 @@ overrides the common property).
229229

230230
[[spring-cloud-dataflow-task-limit-concurrent-executions]]
231231
=== Limit the number concurrent task launches
232-
Spring Cloud Data Flow allows a user establish the maximum number of concurrently running tasks to prevent the saturation of IaaS/hardware resources.
233-
This limit can be configured by setting the `spring.cloud.dataflow.task.maximum-concurrent-tasks` property. By default it is set to `20`.
234-
If the number of concurrently running tasks is equal or greater than the value set by `spring.cloud.dataflow.task.maximum-concurrent-tasks` the next
235-
task launch request will be declined and a warning message will be returned via the RESTful API, Shell or UI.
236-
232+
Spring Cloud Data Flow allows a user to limit the maximum number of concurrently running tasks for each configured platform to prevent the saturation of IaaS/hardware resources.
233+
The limit is set to `20` for all supported platforms by default. If the number of concurrently running tasks on a platform instance is greater or equal to the limit, the next task launch request will fail and an error message will be returned via the RESTful API, Shell or UI.
234+
This limit can be configured for a platform instance by setting the corresponding deployer property, `spring.cloud.dataflow.task.platform.<platform-type>.accounts[<account-name>].maximum-concurrent-tasks` property, where `<account-name>` is the name of a configured platform account (`default` if no accounts are explicitly configured).
235+
The `<platform-type>` refers to one of the currently supported deployers: `local`, `cloudfoundry`, or `kubernetes`.
236+
237+
The TaskLauncher implementation for each supported platform determines the number of currently executing tasks by querying the underlying platform's runtime state if possible. The method for identifying a `task` varies by platform.
238+
For example, launching a task on the local host uses the `LocalTaskLauncher`. The LocalTaskLauncher executes a process for each launch request and keeps track of these processes in memory. In this case, we don't query the underlying OS, as it is impractical to identify tasks this way.
239+
For Cloud Foundry, tasks are a core concept supported by its deployment model. The state of all tasks, running, completed, or failed, is available directly via the API.
240+
This means that every running task container in the account's org and space is included in the running execution count, whether or not it was launched using Spring Cloud Data Flow, or invoking the `CloudFoundryTaskLauncher` directly.
241+
For Kubernetes, launching a task via the `KubernetesTaskLauncher`, if successful, results in a running pod which we expect to eventually complete or fail.
242+
In this environment there is generally no easy way to identify pods that correspond to a task.
243+
For this reason, we only count pods that were launched by the `KubernetesTaskLauncher`.
244+
Since the task launcher provides `task-name` label in the pod's metadata, we filter all running pods by the presence of this label.
237245

238246
[[spring-cloud-dataflow-task-review-executions]]
239247
=== Reviewing Task Executions

‎spring-cloud-dataflow-rest-client/src/main/java/org/springframework/cloud/dataflow/rest/client/StreamTemplate.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -211,7 +211,7 @@ public Collection<Deployer> listPlatforms() {
211211
ParameterizedTypeReference<Collection<Deployer>> typeReference = new ParameterizedTypeReference<Collection<Deployer>>() {
212212
};
213213
Map<String, Object> parameters = new HashMap<>();
214-
String url = url = deploymentsLink.getHref() + "/platform/list";
214+
String url = deploymentsLink.getHref() + "/platform/list";
215215
return this.restTemplate.exchange(url, HttpMethod.GET, null, typeReference, parameters).getBody();
216216
}
217217

‎spring-cloud-dataflow-rest-client/src/main/java/org/springframework/cloud/dataflow/rest/client/TaskOperations.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616

1717
package org.springframework.cloud.dataflow.rest.client;
1818

19+
import java.util.Collection;
1920
import java.util.List;
2021
import java.util.Map;
2122

@@ -99,9 +100,9 @@ public interface TaskOperations {
99100
/**
100101
* Return information including the count of currently executing tasks and task execution
101102
* limits.
102-
* @return {@link CurrentTaskExecutionsResource}
103+
* @return Collection of {@link CurrentTaskExecutionsResource}
103104
*/
104-
CurrentTaskExecutionsResource currentTaskExecutions();
105+
Collection<CurrentTaskExecutionsResource> currentTaskExecutions();
105106

106107
/**
107108
* Cleanup any resources associated with the execution for the id specified.

‎spring-cloud-dataflow-rest-client/src/main/java/org/springframework/cloud/dataflow/rest/client/TaskTemplate.java‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616

1717
package org.springframework.cloud.dataflow.rest.client;
1818

19+
import java.util.Collection;
1920
import java.util.Collections;
2021
import java.util.List;
2122
import java.util.Map;
@@ -29,8 +30,10 @@
2930
import org.springframework.cloud.dataflow.rest.resource.TaskDefinitionResource;
3031
import org.springframework.cloud.dataflow.rest.resource.TaskExecutionResource;
3132
import org.springframework.cloud.dataflow.rest.util.DeploymentPropertiesUtils;
33+
import org.springframework.core.ParameterizedTypeReference;
3234
import org.springframework.hateoas.Link;
3335
import org.springframework.hateoas.ResourceSupport;
36+
import org.springframework.http.HttpMethod;
3437
import org.springframework.util.Assert;
3538
import org.springframework.util.LinkedMultiValueMap;
3639
import org.springframework.util.MultiValueMap;
@@ -183,8 +186,12 @@ public TaskExecutionResource taskExecutionStatus(long id) {
183186
}
184187

185188
@Override
186-
public CurrentTaskExecutionsResource currentTaskExecutions() {
187-
return restTemplate.getForObject(executionsCurrentLink.getHref(), CurrentTaskExecutionsResource.class);
189+
public Collection<CurrentTaskExecutionsResource> currentTaskExecutions() {
190+
ParameterizedTypeReference<Collection<CurrentTaskExecutionsResource>> typeReference =
191+
new ParameterizedTypeReference<Collection<CurrentTaskExecutionsResource>>() {
192+
};
193+
return restTemplate
194+
.exchange(executionsCurrentLink.getHref(),HttpMethod.GET,null, typeReference).getBody();
188195
}
189196

190197
@Override

‎spring-cloud-dataflow-rest-resource/src/main/java/org/springframework/cloud/dataflow/rest/resource/CurrentTaskExecutionsResource.java‎

Lines changed: 44 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright 2018 the original author or authors.
2+
* Copyright 2018-2019 the original author or authors.
33
*
44
* Licensed under the Apache License, Version 2.0 (the "License");
55
* you may not use this file except in compliance with the License.
@@ -16,6 +16,8 @@
1616

1717
package org.springframework.cloud.dataflow.rest.resource;
1818

19+
import org.springframework.cloud.dataflow.core.PlatformTaskExecutionInformation;
20+
import org.springframework.hateoas.PagedResources;
1921
import org.springframework.hateoas.ResourceSupport;
2022

2123
/**
@@ -26,48 +28,65 @@
2628
public class CurrentTaskExecutionsResource extends ResourceSupport {
2729

2830
/**
29-
* The maximum concurrently running task executions allowed.
31+
* The platform instance (account) name.
3032
*/
31-
private long maximumTaskExecutions = Long.MAX_VALUE;
33+
private String name;
3234

3335
/**
34-
* The current number of running task executions.
36+
* The platform type name.
3537
*/
36-
private long runningExecutionCount;
38+
private String type;
3739

3840
/**
39-
*
40-
* @return the maximum number of concurrently running task executions allowed.
41+
* The maximum concurrently running task executions allowed.
4142
*/
42-
public long getMaximumTaskExecutions() {
43-
return maximumTaskExecutions;
44-
}
43+
private int maximumTaskExecutions;
4544

4645
/**
47-
*
48-
* Set the maximum number of concurrently running task executions allowed.
49-
* @param maximumTaskExecutions that should be allowed to be executed.
46+
* The current number of running task executions.
5047
*/
51-
public void setMaximumTaskExecutions(long maximumTaskExecutions) {
48+
private int runningExecutionCount;
49+
50+
51+
public String getName() {
52+
return name;
53+
}
54+
public void setName(String name) {
55+
this.name = name;
56+
}
57+
public String getType() {
58+
return type;
59+
}
60+
public void setType(String type) {
61+
this.type = type;
62+
}
63+
public int getMaximumTaskExecutions() {
64+
return maximumTaskExecutions;
65+
}
66+
public void setMaximumTaskExecutions(int maximumTaskExecutions) {
5267
this.maximumTaskExecutions = maximumTaskExecutions;
5368
}
54-
55-
/**
56-
*
57-
* @return the current number of running task executions.
58-
*/
59-
public long getRunningExecutionCount() {
69+
public int getRunningExecutionCount() {
6070
return runningExecutionCount;
6171
}
62-
72+
public void setRunningExecutionCount(int runningExecutionCount) {
73+
this.runningExecutionCount = runningExecutionCount;
74+
}
6375
/**
6476
*
65-
* Set the current number of running task executions.
66-
* @param runningExecutionCount the current count of running executions.
77+
* @param taskExecutionInformation
6778
*/
68-
public void setRunningExecutionCount(long runningExecutionCount) {
69-
this.runningExecutionCount = runningExecutionCount;
79+
public static CurrentTaskExecutionsResource fromTaskExecutionInformation(
80+
PlatformTaskExecutionInformation taskExecutionInformation) {
81+
CurrentTaskExecutionsResource resource = new CurrentTaskExecutionsResource();
82+
resource.setName(taskExecutionInformation.getName());
83+
resource.setType(taskExecutionInformation.getType());
84+
resource.setMaximumTaskExecutions(taskExecutionInformation.getMaximumTaskExecutions());
85+
resource.setRunningExecutionCount(taskExecutionInformation.getRunningExecutionCount());
86+
return resource;
7087
}
7188

89+
public static class Page extends PagedResources<CurrentTaskExecutionsResource> {
90+
}
7291

7392
}

0 commit comments

Comments
 (0)