|
| 1 | +// Copyright (c) 2022, Oracle and/or its affiliates. |
| 2 | +// Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. |
| 3 | + |
| 4 | +package oracle.weblogic.kubernetes.utils; |
| 5 | + |
| 6 | +import java.io.IOException; |
| 7 | +import java.nio.charset.StandardCharsets; |
| 8 | +import java.nio.file.Files; |
| 9 | +import java.nio.file.Path; |
| 10 | +import java.nio.file.Paths; |
| 11 | +import java.time.Duration; |
| 12 | +import java.time.OffsetDateTime; |
| 13 | +import java.util.List; |
| 14 | +import java.util.concurrent.TimeUnit; |
| 15 | +import java.util.logging.Level; |
| 16 | +import java.util.logging.Logger; |
| 17 | + |
| 18 | +import io.kubernetes.client.openapi.ApiException; |
| 19 | +import io.kubernetes.client.openapi.models.CoreV1Event; |
| 20 | +import io.kubernetes.client.util.Yaml; |
| 21 | +import oracle.weblogic.kubernetes.TestConstants; |
| 22 | +import oracle.weblogic.kubernetes.actions.impl.primitive.Kubernetes; |
| 23 | +import oracle.weblogic.kubernetes.logging.LoggingFacade; |
| 24 | + |
| 25 | +import static io.kubernetes.client.util.Yaml.dump; |
| 26 | +import static oracle.weblogic.kubernetes.actions.TestActions.now; |
| 27 | +import static oracle.weblogic.kubernetes.utils.ThreadSafeLogger.getLogger; |
| 28 | + |
| 29 | +/** |
| 30 | + * A utility class to collect logs about services and running processes when a port in use error occur. It spawns a |
| 31 | + * thread from ImageBuilders beforeAll method to start watching all events in all namespaces used by the integration |
| 32 | + * tests and will collect services logs and processes logs using lsof command. The thread runs every 10 seconds to look |
| 33 | + * for the port in use error and writes logs only when the error is thrown. |
| 34 | + * |
| 35 | + */ |
| 36 | +public class PortInuseEventWatcher extends Thread { |
| 37 | + |
| 38 | + static LoggingFacade logger = getLogger(); |
| 39 | + OffsetDateTime timestamp; |
| 40 | + OffsetDateTime begin; |
| 41 | + OffsetDateTime end; |
| 42 | + int secondsBetweenCollection = 10; |
| 43 | + int collectionDelay = 0; |
| 44 | + String regex = "(?s).*\\bprovided\\b.*\\bport\\b.*\\bis\\b.*\\balready\\b.*\\ballocated\\b"; |
| 45 | + |
| 46 | + @Override |
| 47 | + public void run() { |
| 48 | + logger.info("Starting port in use debugger thread"); |
| 49 | + timestamp = now(); |
| 50 | + while (!this.isInterrupted()) { |
| 51 | + timestamp = now().minusSeconds(collectionDelay); |
| 52 | + try { |
| 53 | + begin = now(); |
| 54 | + List<String> ns = Kubernetes.listNamespaces(); |
| 55 | + for (String n : ns) { |
| 56 | + if (n.startsWith("ns-") || n.equals("default")) { |
| 57 | + List<CoreV1Event> listNamespacedEvents = Kubernetes.listNamespacedEvents(n); |
| 58 | + for (CoreV1Event event : listNamespacedEvents) { |
| 59 | + if (event != null && event.getLastTimestamp() != null |
| 60 | + && (event.getLastTimestamp().isEqual(timestamp) |
| 61 | + || event.getLastTimestamp().isAfter(timestamp)) |
| 62 | + && event.getMessage() != null |
| 63 | + && event.getMessage().matches(regex)) { |
| 64 | + logger.info("Port in use issue found in namespace {0}, " |
| 65 | + + "collecting services objects across all namespaces....", n); |
| 66 | + logger.info(Yaml.dump(event)); |
| 67 | + collectLogs(ns); |
| 68 | + } |
| 69 | + } |
| 70 | + } |
| 71 | + } |
| 72 | + end = now(); |
| 73 | + } catch (ApiException ex) { |
| 74 | + Logger.getLogger(PortInuseEventWatcher.class.getName()).log(Level.SEVERE, null, ex); |
| 75 | + } |
| 76 | + try { |
| 77 | + TimeUnit.SECONDS.sleep(secondsBetweenCollection); |
| 78 | + collectionDelay = (int) (secondsBetweenCollection + Duration.between(begin, end).getSeconds()); |
| 79 | + } catch (InterruptedException ex) { |
| 80 | + logger.info("Thread interrupted"); |
| 81 | + } |
| 82 | + } |
| 83 | + logger.info("port in use debugger thread ended"); |
| 84 | + } |
| 85 | + |
| 86 | + private void collectLogs(List<String> namespaces) { |
| 87 | + try { |
| 88 | + Path dest = Files.createDirectories(Paths.get(TestConstants.LOGS_DIR, "portinuselogs", now().toString())); |
| 89 | + //grab all processes using lsof |
| 90 | + ExecResult exec = ExecCommand.exec("sudo lsof -i"); |
| 91 | + if (exec.exitValue() == 0 && exec.stdout() != null) { |
| 92 | + writeToFile(exec.stderr(), dest.toString(), "lsof", false); |
| 93 | + } else { |
| 94 | + logger.warning("Failed to collect lsof logs"); |
| 95 | + } |
| 96 | + for (String ns : namespaces) { |
| 97 | + if (ns.startsWith("ns-") || ns.equals("default")) { |
| 98 | + writeToFile(Kubernetes.listServices(ns), dest.toString(), ns + ".list.services.log", true); |
| 99 | + writeToFile(Kubernetes.listNamespacedEvents(ns), dest.toString(), ns + ".list.events.log", true); |
| 100 | + } |
| 101 | + } |
| 102 | + } catch (IOException | InterruptedException | ApiException ex) { |
| 103 | + logger.warning(ex.getMessage()); |
| 104 | + } |
| 105 | + } |
| 106 | + |
| 107 | + private static void writeToFile(Object obj, String resultDir, String fileName, boolean asYaml) |
| 108 | + throws IOException { |
| 109 | + logger.info("Generating {0}", Paths.get(resultDir, fileName)); |
| 110 | + if (obj != null) { |
| 111 | + Files.createDirectories(Paths.get(resultDir)); |
| 112 | + Files.write(Paths.get(resultDir, fileName), |
| 113 | + (asYaml ? dump(obj).getBytes(StandardCharsets.UTF_8) : ((String) obj).getBytes(StandardCharsets.UTF_8)) |
| 114 | + ); |
| 115 | + } else { |
| 116 | + logger.info("Nothing to write in {0} list is empty", Paths.get(resultDir, fileName)); |
| 117 | + } |
| 118 | + } |
| 119 | +} |
0 commit comments