zdf: fix null pointer #1117

arte: add retry 429 #1121

Author: alex1702

build.gradle

@@ -27,7 +27,7 @@ sourceCompatibility = JavaVersion.VERSION_17
 targetCompatibility = JavaVersion.VERSION_17
 group = 'de.mediathekview'
 archivesBaseName = "MServer"
-version = '3.1.277'
+version = '3.1.278'
 
 def jarName = 'MServer.jar'
 def mainClass = 'mServer.Main'

src/main/java/mServer/crawler/sender/arte/ArteConstants.java

@@ -1,8 +1,7 @@
 package mServer.crawler.sender.arte;
 
 public class ArteConstants {
-  public static final String VIDEOS_URL ="https://api.arte.tv/api/opa/v3/videos?limit=100&page=%s&sort=-broadcastBegin&language=%s&kind=SHOW,MANUAL_CLIP,BONUS";
-  public static final String VIDEOS_URL_ALT ="https://api.arte.tv/api/opa/v3/videos?limit=100&page=%s&sort=broadcastBegin&language=%s&kind=SHOW,MANUAL_CLIP,BONUS";
+  public static final String VIDEOS_URL ="https://api.arte.tv/api/opa/v3/videos?limit=100&page=%s&sort=-videoRightsBegin&language=%s&kind=SHOW,MANUAL_CLIP,BONUS";
   public static final String VIDEO_URL ="https://www.arte.tv/hbbtvv2/services/web/index.php/OPA/v3/streams/%s/%s/%s"; //PROGRAMID/KIND/LANG
   public static final String API_TOKEN = "Bearer Nzc1Yjc1ZjJkYjk1NWFhN2I2MWEwMmRlMzAzNjI5NmU3NWU3ODg4ODJjOWMxNTMxYzEzZGRjYjg2ZGE4MmIwOA";
   public static final int MAX_POSSIBLE_SUBPAGES = 100;

src/main/java/mServer/crawler/sender/arte/ArteCrawler.java

@@ -74,18 +74,16 @@ private ConcurrentLinkedQueue<TopicUrlDTO> createVideosQueue(String language) {
     final ConcurrentLinkedQueue<TopicUrlDTO> root = new ConcurrentLinkedQueue<>();
     String rootUrl = String.format(ArteConstants.VIDEOS_URL, 1, language);
     root.add(new TopicUrlDTO("all videos sorted up", rootUrl));
-    String rootUrl2 = String.format(ArteConstants.VIDEOS_URL_ALT, 1, language);
-    root.add(new TopicUrlDTO("all videos sorted down", rootUrl2));
     return root;
   }
 
   private int getMaxPagesForOverview(String lang) {
     final int maxAvailablePages = getNumberOfAvailablePages(lang);
     final int configuredMaxPages = getMaximumSubpages();
     if (configuredMaxPages > maxAvailablePages) {
-      return Math.min(ArteConstants.MAX_POSSIBLE_SUBPAGES, maxAvailablePages / 2);
+      return Math.min(ArteConstants.MAX_POSSIBLE_SUBPAGES, maxAvailablePages);
     } else {
-      return Math.min(ArteConstants.MAX_POSSIBLE_SUBPAGES, configuredMaxPages / 2);
+      return Math.min(ArteConstants.MAX_POSSIBLE_SUBPAGES, configuredMaxPages);
     }
   }
 

src/main/java/mServer/crawler/sender/arte/tasks/ArteRateLimitedJsonRestTask.java

@@ -1,31 +1,78 @@
 package mServer.crawler.sender.arte.tasks;
 
 import com.google.common.util.concurrent.RateLimiter;
+import com.google.gson.Gson;
+import de.mediathekview.mlib.Config;
+import de.mediathekview.mlib.tool.Log;
+import jakarta.ws.rs.client.Invocation.Builder;
 import jakarta.ws.rs.client.WebTarget;
+import jakarta.ws.rs.core.Response;
+import mServer.crawler.FilmeSuchen;
+import mServer.crawler.RunSender;
 import mServer.crawler.sender.MediathekReader;
 import mServer.crawler.sender.base.AbstractJsonRestTask;
 import mServer.crawler.sender.base.CrawlerUrlDTO;
 import mServer.tool.MserverDaten;
 
 import java.util.Optional;
 import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.TimeUnit;
 
 public abstract class ArteRateLimitedJsonRestTask<T, R, D extends CrawlerUrlDTO> extends AbstractJsonRestTask<T, R, D> {
   private static final long serialVersionUID = 1L;
   private static final RateLimiter rateLimiter = RateLimiter.create(MserverDaten.getArteRateLimit());
-  private static final RateLimiter opaApirateLimiter = RateLimiter.create(1.0);
+  private static final RateLimiter opaApirateLimiter = RateLimiter.create(0.3);
 
   protected ArteRateLimitedJsonRestTask(MediathekReader aCrawler, ConcurrentLinkedQueue<D> urlToCrawlDTOs, Optional<String> authKey) {
     super(aCrawler, urlToCrawlDTOs, authKey);
   }
 
   @Override
   protected void processRestTarget(final D aDTO, final WebTarget aTarget) {
-    if (aTarget.getUri().toString().contains("api.arte.tv/api/opa/")) {
-      opaApirateLimiter.acquire();
-    } else {
-      rateLimiter.acquire();
+    int retryCount = 0;
+    int maxRetries = 3;
+    boolean stop = false;
+
+    while (!stop && !Config.getStop()) {
+      // Apply rate limiting before each request (including retries)
+      if (aTarget.getUri().toString().contains("api.arte.tv/api/opa/")) {
+        opaApirateLimiter.acquire();
+      } else {
+        rateLimiter.acquire();
+      }
+
+      Builder request = aTarget.request();
+      final Optional<String> authKey = getAuthKey();
+      if (authKey.isPresent()) {
+        request = request.header(HEADER_AUTHORIZATION, authKey.get());
+      }
+
+      try (Response response = createResponse(request, aDTO)) {
+        traceRequest(response.getLength());
+
+        if (response.getStatus() == 200) {
+          gsonBuilder.registerTypeAdapter(getType(), getParser(aDTO));
+          final Gson gson = gsonBuilder.create();
+          final String jsonOutput = response.readEntity(String.class);
+          final R responseObj = gson.fromJson(jsonOutput, getType());
+          postProcessing(responseObj, aDTO);
+          stop = true;
+          // Check if we got a 429 and have retries left
+        } else if (response.getStatus() == 429 && retryCount < maxRetries) {
+          String retryAfter = response.getHeaderString("Retry-After");
+          Log.sysLog("429: " + aDTO.getUrl() + " - retry after: " + retryAfter);
+          retryCount++;
+          try {
+            TimeUnit.MILLISECONDS.sleep(60000);
+          } catch (InterruptedException ignored) {
+          }
+        } else {
+          FilmeSuchen.listeSenderLaufen.inc(crawler.getRunIdentifier(), RunSender.Count.FEHLER);
+          FilmeSuchen.listeSenderLaufen.inc(crawler.getRunIdentifier(), RunSender.Count.FEHLVERSUCHE);
+          handleHttpError(aDTO, aTarget.getUri(), response);
+          stop = true;
+        }
+      }
     }
-    super.processRestTarget(aDTO, aTarget);
   }
 }

src/main/java/mServer/crawler/sender/zdf/tasks/ZdfFilmTask.java

@@ -64,7 +64,7 @@ private static DatenFilm createFilm(final ZdfFilmDto zdfFilmDto, final DownloadD
 
   @Override
   protected void processRestTarget(ZdfFilmDto aDTO, WebTarget aTarget) {
-    final Optional<DownloadDto> downloadDto = deserialize(aTarget, OPTIONAL_DOWNLOAD_DTO_TYPE_TOKEN);
+    final Optional<DownloadDto> downloadDto = deserializeOptional(aTarget, OPTIONAL_DOWNLOAD_DTO_TYPE_TOKEN);
     if (downloadDto.isPresent()) {
       addFilm(aDTO, downloadDto.get());
     } else {