Skip to content

Commit 1d56f3c

Browse files
james-toussaintjames-toussaint
authored
Merge pull request #334 from iExecBlockchainComputing/feature/replace-docker-library
Feature/replace docker library
2 parents c56dc34 + b694929 commit 1d56f3c

34 files changed

+3421
-1797
lines changed

‎build.gradle‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -76,8 +76,10 @@ dependencies {
7676
implementation 'com.squareup.okhttp3:okhttp:4.3.1'
7777
implementation 'org.jetbrains.kotlin:kotlin-stdlib:1.3.50'
7878

79-
// docker-client
80-
compile 'com.spotify:docker-client:8.13.1'
79+
// docker
80+
implementation 'com.github.docker-java:docker-java:3.2.5'
81+
implementation 'com.github.docker-java:docker-java-transport-httpclient5:3.2.5'
82+
8183
compile 'org.glassfish.jersey.inject:jersey-hk2:2.26'
8284
compile 'org.glassfish.jersey.bundles.repackaged:jersey-guava:2.25.1'
8385
compile 'javax.activation:activation:1.1.1'
@@ -147,7 +149,7 @@ uploadArchives.enabled = canUploadArchives
147149

148150
test {
149151
if (System.properties['test.profile'] == 'skipDocker') {
150-
exclude '**/compute/DockerServiceTests*'
152+
exclude '**/docker/DockerClientServiceTests*'
151153
}
152154
}
153155

‎src/main/java/com/iexec/worker/compute/ComputeManagerService.java‎

Lines changed: 214 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,214 @@
1+
/*
2+
* Copyright 2020 IEXEC BLOCKCHAIN TECH
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.iexec.worker.compute;
18+
19+
import com.iexec.common.chain.WorkerpoolAuthorization;
20+
import com.iexec.common.dapp.DappType;
21+
import com.iexec.common.result.ComputedFile;
22+
import com.iexec.common.task.TaskDescription;
23+
import com.iexec.common.utils.FileHelper;
24+
import com.iexec.common.utils.IexecFileHelper;
25+
import com.iexec.common.worker.result.ResultUtils;
26+
import com.iexec.worker.chain.IexecHubService;
27+
import com.iexec.worker.compute.app.AppComputeResponse;
28+
import com.iexec.worker.compute.app.AppComputeService;
29+
import com.iexec.worker.compute.post.PostComputeResponse;
30+
import com.iexec.worker.compute.post.PostComputeService;
31+
import com.iexec.worker.compute.pre.PreComputeResponse;
32+
import com.iexec.worker.compute.pre.PreComputeService;
33+
import com.iexec.worker.config.WorkerConfigurationService;
34+
import com.iexec.worker.docker.DockerService;
35+
import lombok.extern.slf4j.Slf4j;
36+
import org.springframework.stereotype.Service;
37+
38+
import java.io.File;
39+
40+
41+
@Slf4j
42+
@Service
43+
public class ComputeManagerService {
44+
45+
private static final String STDOUT_FILENAME = "stdout.txt";
46+
47+
private final DockerService dockerService;
48+
private final PreComputeService preComputeService;
49+
private final AppComputeService appComputeService;
50+
private final PostComputeService postComputeService;
51+
private final WorkerConfigurationService workerConfigService;
52+
private final IexecHubService iexecHubService;
53+
54+
public ComputeManagerService(
55+
DockerService dockerService,
56+
PreComputeService preComputeService,
57+
AppComputeService appComputeService,
58+
PostComputeService postComputeService,
59+
WorkerConfigurationService workerConfigService,
60+
IexecHubService iexecHubService
61+
) {
62+
this.dockerService = dockerService;
63+
this.preComputeService = preComputeService;
64+
this.appComputeService = appComputeService;
65+
this.postComputeService = postComputeService;
66+
this.workerConfigService = workerConfigService;
67+
this.iexecHubService = iexecHubService;
68+
}
69+
70+
public boolean downloadApp(TaskDescription taskDescription) {
71+
if (taskDescription == null || taskDescription.getAppType() == null) {
72+
return false;
73+
}
74+
boolean isDockerType =
75+
taskDescription.getAppType().equals(DappType.DOCKER);
76+
if (!isDockerType || taskDescription.getAppUri() == null) {
77+
return false;
78+
}
79+
return dockerService.pullImage(taskDescription.getAppUri());
80+
}
81+
82+
public boolean isAppDownloaded(String imageUri) {
83+
return dockerService.isImagePulled(imageUri);
84+
}
85+
86+
/*
87+
* non TEE: download secrets && decrypt dataset (TODO: rewritte or remove)
88+
* TEE: download post-compute image && create secure session
89+
*/
90+
public PreComputeResponse runPreCompute(TaskDescription taskDescription,
91+
WorkerpoolAuthorization workerpoolAuth) {
92+
log.info("Running pre-compute [chainTaskId:{}, isTee:{}]",
93+
taskDescription.getChainTaskId(),
94+
taskDescription.isTeeTask());
95+
96+
if (taskDescription.isTeeTask()) {
97+
String secureSessionId =
98+
preComputeService.runTeePreCompute(taskDescription,
99+
workerpoolAuth);
100+
return PreComputeResponse.builder()
101+
.isTeeTask(true)
102+
.secureSessionId(secureSessionId)
103+
.build();
104+
}
105+
106+
return PreComputeResponse.builder()
107+
.isSuccessful(
108+
preComputeService.runStandardPreCompute(taskDescription))
109+
.build();
110+
}
111+
112+
public AppComputeResponse runCompute(TaskDescription taskDescription,
113+
String secureSessionId) {
114+
String chainTaskId = taskDescription.getChainTaskId();
115+
log.info("Running compute [chainTaskId:{}, isTee:{}]", chainTaskId,
116+
taskDescription.isTeeTask());
117+
118+
ComputeResponse computeResponse =
119+
appComputeService.runCompute(taskDescription, secureSessionId);
120+
121+
if (computeResponse.isSuccessful() && !computeResponse.getStdout().isEmpty()) {
122+
// save /output/stdout.txt file
123+
String stdoutFilePath =
124+
workerConfigService.getTaskIexecOutDir(chainTaskId) + File.separator + STDOUT_FILENAME;
125+
File stdoutFile = FileHelper.createFileWithContent(stdoutFilePath
126+
, computeResponse.getStdout());
127+
log.info("Saved stdout file [path:{}]",
128+
stdoutFile.getAbsolutePath());
129+
//TODO Make sure stdout is properly written
130+
}
131+
132+
return AppComputeResponse.builder()
133+
.isSuccessful(computeResponse.isSuccessful())
134+
.stdout(computeResponse.getStdout())
135+
.stderr(computeResponse.getStderr())
136+
.build();
137+
}
138+
139+
/*
140+
* - Copy computed.json file produced by the compute stage to /output
141+
* - Zip iexec_out folder
142+
* For TEE tasks, worker-tee-post-compute will do those two steps since
143+
* all files in are protected.
144+
*
145+
* - Save stdout file
146+
*/
147+
public PostComputeResponse runPostCompute(TaskDescription taskDescription,
148+
String secureSessionId) {
149+
String chainTaskId = taskDescription.getChainTaskId();
150+
log.info("Running post-compute [chainTaskId:{}, isTee:{}]",
151+
chainTaskId, taskDescription.isTeeTask());
152+
153+
if (taskDescription.isTeeTask()) {
154+
ComputeResponse computeResponse =
155+
postComputeService.runTeePostCompute(taskDescription,
156+
secureSessionId);
157+
return PostComputeResponse.builder()
158+
.isSuccessful(computeResponse.isSuccessful())
159+
.stdout(computeResponse.getStdout())
160+
.stderr(computeResponse.getStderr())
161+
.build();
162+
}
163+
164+
// TODO Use container
165+
return PostComputeResponse.builder()
166+
.isSuccessful(postComputeService.runStandardPostCompute(taskDescription))
167+
.build();
168+
}
169+
170+
171+
public ComputedFile getComputedFile(String chainTaskId) {
172+
ComputedFile computedFile =
173+
IexecFileHelper.readComputedFile(chainTaskId,
174+
workerConfigService.getTaskOutputDir(chainTaskId));
175+
if (computedFile == null) {
176+
log.error("Failed to getComputedFile (computed.json missing)" +
177+
"[chainTaskId:{}]", chainTaskId);
178+
return null;
179+
}
180+
if (computedFile.getResultDigest() == null || computedFile.getResultDigest().isEmpty()) {
181+
String resultDigest = computeResultDigest(computedFile);
182+
if (resultDigest.isEmpty()) {
183+
log.error("Failed to getComputedFile (resultDigest is empty " +
184+
"but cant compute it)" +
185+
"[chainTaskId:{}, computedFile:{}]",
186+
chainTaskId,
187+
computedFile);
188+
return null;
189+
}
190+
computedFile.setResultDigest(resultDigest);
191+
}
192+
return computedFile;
193+
}
194+
195+
private String computeResultDigest(ComputedFile computedFile) {
196+
String chainTaskId = computedFile.getTaskId();
197+
String resultDigest;
198+
if (iexecHubService.getTaskDescription(chainTaskId).isCallbackRequested()) {
199+
resultDigest = ResultUtils.computeWeb3ResultDigest(computedFile);
200+
} else {
201+
resultDigest = ResultUtils.computeWeb2ResultDigest(computedFile,
202+
workerConfigService.getTaskOutputDir(chainTaskId));
203+
}
204+
if (resultDigest.isEmpty()) {
205+
log.error("Failed to computeResultDigest (resultDigest empty)" +
206+
"[chainTaskId:{}, computedFile:{}]",
207+
chainTaskId, computedFile);
208+
return "";
209+
}
210+
return resultDigest;
211+
}
212+
213+
214+
}

‎src/main/java/com/iexec/worker/compute/ComputeResponse.java‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
/*
2+
* Copyright 2020 IEXEC BLOCKCHAIN TECH
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.iexec.worker.compute;
18+
19+
public interface ComputeResponse {
20+
21+
boolean isSuccessful();
22+
String getStdout();
23+
String getStderr();
24+
25+
}

0 commit comments

Comments
 (0)