http 429 in jsonresttask

Author: codingPF

src/main/java/de/mediathekview/mserver/crawler/arte/tasks/ArteVideoInfoTask.java

@@ -64,7 +64,9 @@ protected void postProcessingNextPage(PagedElementListDTO<ArteVideoInfoDto> aRes
 
   protected void postProcessingElements(Set<ArteVideoInfoDto> elements, TopicUrlDTO aDTO) {
     for (ArteVideoInfoDto element : elements)  {
-      taskResults.add(element);
+      if(!taskResults.add(element)) {
+        log.debug("Duplicate film");
+      }
     }
   }
 

src/main/java/de/mediathekview/mserver/crawler/arte/tasks/ArteVideoLinkTask.java

@@ -18,7 +18,6 @@
 import de.mediathekview.mserver.crawler.basic.AbstractCrawler;
 import de.mediathekview.mserver.crawler.basic.AbstractJsonRestTask;
 import de.mediathekview.mserver.crawler.basic.AbstractRecursiveConverterTask;
-import de.mediathekview.mserver.crawler.basic.PagedElementListDTO;
 import jakarta.ws.rs.core.Response;
 
 //return T Class from this task, desirialisation of class R , D , Reasearch in this url

src/main/java/de/mediathekview/mserver/crawler/basic/AbstractJsonRestTask.java

@@ -13,12 +13,16 @@
 import java.util.Optional;
 import java.util.Queue;
 
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
 import static jakarta.ws.rs.core.HttpHeaders.ACCEPT_CHARSET;
 import static jakarta.ws.rs.core.HttpHeaders.ACCEPT_ENCODING;
 
 /** A abstract REST api task which requests the given url with the Funk Api settings. */
 public abstract class AbstractJsonRestTask<T, R, D extends CrawlerUrlDTO>
     extends AbstractRestTask<T, D> {
+  protected final transient Logger log = LogManager.getLogger(this.getClass());
   protected static final String ENCODING_GZIP = "gzip";
   private static final long serialVersionUID = -1090560363478964885L;
   protected final transient GsonBuilder gsonBuilder;
@@ -41,24 +45,66 @@ protected AbstractJsonRestTask(
 
   @Override
   protected void processRestTarget(final D aDTO, final WebTarget aTarget) {
-    gsonBuilder.registerTypeAdapter(getType(), getParser(aDTO));
-    final Gson gson = gsonBuilder.create();
-    Builder request = aTarget.request();
-    final Optional<String> authKey = getAuthKey();
-    if (authKey.isPresent()) {
-      request = request.header(HEADER_AUTHORIZATION, authKey.get());
-    }
+      gsonBuilder.registerTypeAdapter(getType(), getParser(aDTO));
+      final Gson gson = gsonBuilder.create();
 
-    final Response response = createResponse(request, aDTO);
-
-    if (response.getStatus() == 200) {
-      final String jsonOutput = response.readEntity(String.class);
-      final R responseObj = gson.fromJson(jsonOutput, getType());
-      postProcessing(responseObj, aDTO);
-    } else {
-      handleHttpError(aDTO, aTarget.getUri(), response);
-    }
+      Builder request = aTarget.request();
+      final Optional<String> authKey = getAuthKey();
+      if (authKey.isPresent()) {
+          request = request.header(HEADER_AUTHORIZATION, authKey.get());
+      }
+      final int maxRetries = 3;
+      int attempt = 0;
+      while (attempt < maxRetries) {
+          attempt++;
+          Response response = null;
+          try {
+              response = createResponse(request, aDTO);
+              int status = response.getStatus();
+              if (status == 200) {
+                  final String jsonOutput = response.readEntity(String.class);
+                  final R responseObj = gson.fromJson(jsonOutput, getType());
+                  postProcessing(responseObj, aDTO);
+                  return;
+              }
+              if (status == 429 && attempt < maxRetries) {
+                  final long proposalWaitMillis = getRetryAfterMillis(response).orElse(1000L) * attempt;
+                  long waitMillis = proposalWaitMillis;
+                  if (waitMillis < 100) {
+                    waitMillis = 100;
+                  } else if (waitMillis > 180000 ) {
+                    waitMillis = 180000;
+                  }
+                  //log.debug("Too Many Requests - propsoal: {} waiting: {} ", proposalWaitMillis, waitMillis);
+                  Thread.sleep(waitMillis);
+                  continue;
+              }
+              handleHttpError(aDTO, aTarget.getUri(), response);
+              return;
+          } catch (InterruptedException e) {
+              Thread.currentThread().interrupt();
+              throw new RuntimeException("Retry interrupted", e);
+          } finally {
+              if (response != null) {
+                  response.close();
+              }
+          }
+      }
   }
+  
+  private Optional<Long> getRetryAfterMillis(Response response) {
+    String retryAfter = response.getHeaderString("Retry-After");
+    if (retryAfter == null) {
+        return Optional.empty();
+    }
+    try {
+        long seconds = Long.parseLong(retryAfter);
+        return Optional.of(seconds * 1000);
+    } catch (NumberFormatException e) {
+        return Optional.empty();
+    }
+}
+
 
   protected Response createResponse(final Builder request, final D aDTO) {
 	  request.header(ACCEPT_CHARSET, StandardCharsets.UTF_8);