Skip to content

Commit 62688c9

Browse files
jglickjglick
committed
KubernetesSlave.getRunListener to improve logging
1 parent d537d1c commit 62688c9

File tree

7 files changed

+73
-26
lines changed

7 files changed

+73
-26
lines changed

‎src/main/java/org/csanchez/jenkins/plugins/kubernetes/KubernetesLauncher.java‎

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727

2828
import edu.umd.cs.findbugs.annotations.CheckForNull;
2929
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
30+
import hudson.Functions;
3031
import hudson.model.TaskListener;
3132
import hudson.slaves.JNLPLauncher;
3233
import hudson.slaves.SlaveComputer;
@@ -112,7 +113,6 @@ public synchronized void launch(SlaveComputer computer, TaskListener listener) {
112113

113114
String cloudName = node.getCloudName();
114115
final PodTemplate template = node.getTemplate();
115-
TaskListener runListener = TaskListener.NULL;
116116
try {
117117
KubernetesCloud cloud = node.getKubernetesCloud();
118118
KubernetesClient client = cloud.connect();
@@ -128,8 +128,6 @@ public synchronized void launch(SlaveComputer computer, TaskListener listener) {
128128
node.setNamespace(namespace);
129129

130130

131-
runListener = template.getListener();
132-
133131
LOGGER.log(FINE, () -> "Creating Pod: " + cloudName + " " + namespace + "/" + podName);
134132
try {
135133
pod = client.pods().inNamespace(namespace).create(pod);
@@ -138,16 +136,16 @@ public synchronized void launch(SlaveComputer computer, TaskListener listener) {
138136
int httpCode = e.getCode();
139137
if (400 <= httpCode && httpCode < 500) { // 4xx
140138
if (httpCode == 403 && e.getMessage().contains("is forbidden: exceeded quota")) {
141-
runListener.getLogger().printf("WARNING: Unable to create pod: %s %s/%s because kubernetes resource quota exceeded. %n%s%nRetrying...%n%n",
139+
node.getRunListener().getLogger().printf("WARNING: Unable to create pod: %s %s/%s because kubernetes resource quota exceeded. %n%s%nRetrying...%n%n",
142140
cloudName, namespace, pod.getMetadata().getName(), e.getMessage());
143141
}
144142
else if (httpCode == 409 && e.getMessage().contains("Operation cannot be fulfilled on resourcequotas")) {
145143
// See: https://github.com/kubernetes/kubernetes/issues/67761 ; A retry usually works.
146-
runListener.getLogger().printf("WARNING: Unable to create pod: %s %s/%s because kubernetes resource quota update conflict. %n%s%nRetrying...%n%n",
144+
node.getRunListener().getLogger().printf("WARNING: Unable to create pod: %s %s/%s because kubernetes resource quota update conflict. %n%s%nRetrying...%n%n",
147145
cloudName, namespace, pod.getMetadata().getName(), e.getMessage());
148146
}
149147
else {
150-
runListener.getLogger().printf("ERROR: Unable to create pod %s %s/%s.%n%s%n", cloudName, namespace, pod.getMetadata().getName(), e.getMessage());
148+
node.getRunListener().getLogger().printf("ERROR: Unable to create pod %s %s/%s.%n%s%n", cloudName, namespace, pod.getMetadata().getName(), e.getMessage());
151149
PodUtils.cancelQueueItemFor(pod, e.getMessage());
152150
}
153151
} else if (500 <= httpCode && httpCode < 600) { // 5xx
@@ -161,7 +159,7 @@ else if (httpCode == 409 && e.getMessage().contains("Operation cannot be fulfill
161159
listener.getLogger().printf("Created Pod: %s %s/%s%n", cloudName, namespace, podName);
162160
Metrics.metricRegistry().counter(MetricNames.PODS_CREATED).inc();
163161

164-
runListener.getLogger().printf("Created Pod: %s %s/%s%n", cloudName, namespace, podName);
162+
node.getRunListener().getLogger().printf("Created Pod: %s %s/%s%n", cloudName, namespace, podName);
165163
kubernetesComputer.setLaunching(true);
166164

167165
ObjectMeta podMetadata = pod.getMetadata();
@@ -253,6 +251,7 @@ else if (httpCode == 409 && e.getMessage().contains("Operation cannot be fulfill
253251
Metrics.metricRegistry().counter(MetricNames.PODS_LAUNCHED).inc();
254252
} catch (Throwable ex) {
255253
setProblem(ex);
254+
Functions.printStackTrace(ex, node.getRunListener().error("Failed to launch " + node.getPodName()));
256255
LOGGER.log(Level.WARNING, String.format("Error in provisioning; agent=%s, template=%s", node, template), ex);
257256
LOGGER.log(Level.FINER, "Removing Jenkins node: {0}", node.getNodeName());
258257
try {

‎src/main/java/org/csanchez/jenkins/plugins/kubernetes/KubernetesSlave.java‎

Lines changed: 37 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,7 @@
11
package org.csanchez.jenkins.plugins.kubernetes;
22

3-
import edu.umd.cs.findbugs.annotations.NonNull;
4-
import io.fabric8.kubernetes.api.model.StatusDetails;
53
import java.io.IOException;
64
import java.util.HashSet;
7-
import java.util.List;
85
import java.util.Objects;
96
import java.util.Optional;
107
import java.util.Set;
@@ -29,6 +26,7 @@
2926
import org.csanchez.jenkins.plugins.kubernetes.pod.retention.PodRetention;
3027
import org.jenkinsci.plugins.durabletask.executors.OnceRetentionStrategy;
3128
import org.jenkinsci.plugins.kubernetes.auth.KubernetesAuthException;
29+
import org.jenkinsci.plugins.workflow.flow.FlowExecutionOwner;
3230
import org.jvnet.localizer.ResourceBundleHolder;
3331
import org.kohsuke.stapler.DataBoundConstructor;
3432

@@ -108,6 +106,42 @@ public PodTemplate getTemplateOrNull() {
108106
return template;
109107
}
110108

109+
/**
110+
* Makes a best effort to find the build log corresponding to this agent.
111+
*/
112+
@NonNull
113+
public TaskListener getRunListener() {
114+
PodTemplate podTemplate = getTemplateOrNull();
115+
if (podTemplate != null) {
116+
TaskListener listener = podTemplate.getListenerOrNull();
117+
if (listener != null) {
118+
return listener;
119+
}
120+
}
121+
Computer c = toComputer();
122+
if (c != null) {
123+
for (Executor executor : c.getExecutors()) {
124+
Queue.Executable executable = executor.getCurrentExecutable();
125+
// If this executor hosts a PlaceholderExecutable, send to the owning build log.
126+
if (executable != null) {
127+
Queue.Executable parentExecutable = executable.getParentExecutable();
128+
if (parentExecutable instanceof FlowExecutionOwner.Executable) {
129+
FlowExecutionOwner flowExecutionOwner = ((FlowExecutionOwner.Executable) parentExecutable).asFlowExecutionOwner();
130+
if (flowExecutionOwner != null) {
131+
try {
132+
return flowExecutionOwner.getListener();
133+
} catch (IOException x) {
134+
LOGGER.log(Level.WARNING, null, x);
135+
}
136+
}
137+
}
138+
}
139+
// TODO handle freestyle and similar if executable instanceof Run, by capturing a TaskListener from RunListener.onStarted
140+
}
141+
}
142+
return TaskListener.NULL;
143+
}
144+
111145
/**
112146
* @deprecated Use {@link Builder} instead.
113147
*/

‎src/main/java/org/csanchez/jenkins/plugins/kubernetes/PodTemplate.java‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -818,11 +818,18 @@ public void setPodRetention(PodRetention podRetention) {
818818
this.podRetention = PodRetention.getPodTemplateDefault().equals(podRetention) ? null : podRetention;
819819
}
820820

821+
/** @see KubernetesSlave#getRunListener */
821822
@NonNull
822823
public TaskListener getListener() {
823824
return listener == null ? TaskListener.NULL : listener;
824825
}
825826

827+
/** @see KubernetesSlave#getRunListener */
828+
@CheckForNull
829+
public TaskListener getListenerOrNull() {
830+
return listener;
831+
}
832+
826833
public void setListener(@CheckForNull TaskListener listener) {
827834
this.listener = listener;
828835
}

‎src/main/java/org/csanchez/jenkins/plugins/kubernetes/pod/retention/Reaper.java‎

Lines changed: 4 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -387,10 +387,7 @@ public void onEvent(@NonNull Watcher.Action action, @NonNull KubernetesSlave nod
387387
String ns = pod.getMetadata().getNamespace();
388388
String name = pod.getMetadata().getName();
389389
LOGGER.info(() -> ns + "/" + name + " was just deleted, so removing corresponding Jenkins agent");
390-
PodTemplate template = node.getTemplateOrNull();
391-
if (template != null) {
392-
template.getListener().getLogger().printf("Pod %s/%s was just deleted%n", ns, name);
393-
}
390+
node.getRunListener().getLogger().printf("Pod %s/%s was just deleted%n", ns, name);
394391
Jenkins.get().removeNode(node);
395392
}
396393
}
@@ -407,8 +404,7 @@ public void onEvent(@NonNull Watcher.Action action, @NonNull KubernetesSlave nod
407404
if (!terminatedContainers.isEmpty()) {
408405
String ns = pod.getMetadata().getNamespace();
409406
String name = pod.getMetadata().getName();
410-
PodTemplate template = node.getTemplateOrNull();
411-
TaskListener runListener = template != null ? template.getListener() : TaskListener.NULL;
407+
TaskListener runListener = node.getRunListener();
412408
terminatedContainers.forEach(c -> {
413409
ContainerStateTerminated t = c.getState().getTerminated();
414410
LOGGER.info(() -> ns + "/" + name + " Container " + c.getName() + " was just terminated, so removing the corresponding Jenkins agent");
@@ -434,8 +430,7 @@ public void onEvent(@NonNull Watcher.Action action, @NonNull KubernetesSlave nod
434430
if ("Failed".equals(pod.getStatus().getPhase())) {
435431
String ns = pod.getMetadata().getNamespace();
436432
String name = pod.getMetadata().getName();
437-
PodTemplate template = node.getTemplateOrNull();
438-
TaskListener runListener = template != null ? template.getListener() : TaskListener.NULL;
433+
TaskListener runListener = node.getRunListener();
439434
String reason = pod.getStatus().getReason();
440435
LOGGER.info(() -> ns + "/" + name + " Pod just failed. Removing the corresponding Jenkins agent. Reason: " + reason + ", Message: " + pod.getStatus().getMessage());
441436
runListener.getLogger().printf("%s/%s Pod just failed (Reason: %s, Message: %s)%n", ns, name, reason, pod.getStatus().getMessage());
@@ -476,12 +471,7 @@ public void onEvent(@NonNull Watcher.Action action, @NonNull KubernetesSlave nod
476471
if (backOffContainers.isEmpty()) {
477472
return;
478473
}
479-
backOffContainers.forEach(cs -> {
480-
PodTemplate template = node.getTemplateOrNull();
481-
if (template != null) {
482-
template.getListener().error("Unable to pull Docker image \"" + cs.getImage() + "\". Check if image tag name is spelled correctly.");
483-
}
484-
});
474+
backOffContainers.forEach(cs -> node.getRunListener().error("Unable to pull Docker image \"" + cs.getImage() + "\". Check if image tag name is spelled correctly."));
485475
terminationReasons.add("ImagePullBackOff");
486476
PodUtils.cancelQueueItemFor(pod, "ImagePullBackOff");
487477
node.terminate();

‎src/test/java/org/csanchez/jenkins/plugins/kubernetes/pipeline/KubernetesPipelineTest.java‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -515,6 +515,17 @@ public void podDeadlineExceeded() throws Exception {
515515
r.waitForMessage("Pod just failed (Reason: DeadlineExceeded, Message: Pod was active on the node longer than the specified deadline)", b);
516516
}
517517

518+
@Test
519+
public void podDeadlineExceededGlobalTemplate() throws Exception {
520+
PodTemplate podTemplate = new PodTemplate("podDeadlineExceededGlobalTemplate");
521+
podTemplate.setLabel("podDeadlineExceededGlobalTemplate");
522+
podTemplate.setActiveDeadlineSeconds(30);
523+
cloud.addTemplate(podTemplate);
524+
r.assertBuildStatus(Result.ABORTED, r.waitForCompletion(b));
525+
r.waitForMessage("Pod just failed (Reason: DeadlineExceeded, Message: Pod was active on the node longer than the specified deadline)", b);
526+
r.waitForMessage("---Logs---", b);
527+
}
528+
518529
@Test
519530
public void interruptedPod() throws Exception {
520531
r.waitForMessage("starting to sleep", b);

‎src/test/resources/org/csanchez/jenkins/plugins/kubernetes/pipeline/podDeadlineExceededGlobalTemplate.groovy‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
node('podDeadlineExceededGlobalTemplate') {
2+
sh 'sleep 120'
3+
}

‎test-in-k8s.sh‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,8 +14,11 @@ tcp_port=$((2001 + port_offset))
1414
kubectl delete --ignore-not-found --now pod jenkins
1515
sed "s/@HTTP_PORT@/$http_port/g; s/@TCP_PORT@/$tcp_port/g" < test-in-k8s.yaml | kubectl apply -f -
1616
kubectl wait --for=condition=Ready --timeout=15m pod/jenkins
17-
# Copy temporary split files
18-
tar cf - "$WORKSPACE_TMP" | kubectl exec -i jenkins -- tar xf -
17+
if [[ -v WORKSPACE_TMP ]]
18+
then
19+
# Copy temporary split files
20+
tar cf - "$WORKSPACE_TMP" | kubectl exec -i jenkins -- tar xf -
21+
fi
1922
# Copy plugin files
2023
kubectl exec jenkins -- mkdir /checkout
2124
tar cf - pom.xml .mvn src | kubectl exec -i jenkins -- tar xf - -C /checkout

0 commit comments

Comments
 (0)