arte logging

dw: fix nullpointerexception in log

Author: alex1702

build.gradle

@@ -27,7 +27,7 @@ sourceCompatibility = JavaVersion.VERSION_17
 targetCompatibility = JavaVersion.VERSION_17
 group = 'de.mediathekview'
 archivesBaseName = "MServer"
-version = '3.1.275'
+version = '3.1.276'
 
 def jarName = 'MServer.jar'
 def mainClass = 'mServer.Main'

dist/mserver.xml

@@ -31,6 +31,7 @@
     <system-proxy-port></system-proxy-port>
     <system-debug>1</system-debug>
     <sr-rate-limit>1.0</sr-rate-limit>
+    <arte-rate-limit>6.0</arte-rate-limit>
 </system>
 
 

src/main/java/de/mediathekview/mlib/tool/Log.java

@@ -218,7 +218,12 @@ public static synchronized void errorLog(int fehlerNummer, String[] text) {
         fehlermeldung_(fehlerNummer, null, text);
     }
 
-//    public static synchronized void systemMeldung(String[] text) {
+    public static synchronized void logHttpError(String url, int statusCode) {
+        Exception e = new RuntimeException(String.valueOf(statusCode));
+        fehlermeldung_(110000000 + statusCode, e, new String[]{ String.format("error %d for URL: %s", statusCode, url)});
+    }
+
+    //    public static synchronized void systemMeldung(String[] text) {
 //        systemmeldung_(text);
 //    }
     public static synchronized void sysLog(String text) {

src/main/java/mServer/crawler/sender/ard/tasks/ArdTaskBase.java

@@ -62,7 +62,7 @@ private <A> Optional<A> deserializeUnsafe(final WebTarget target, final Type typ
                 + target.getUri().toString()
                 + " failed: "
                 + response.getStatus();
-        Log.errorLog(23646387, logText);
+        Log.logHttpError(target.getUri().toString(), response.getStatus());
         LOG.warn(logText);
       }
     } catch (ProcessingException e) {

src/main/java/mServer/crawler/sender/arte/ArteCrawler.java

@@ -101,9 +101,11 @@ private int getNumberOfAvailablePages(String lang) {
       JsonElement element = jsoupConnection.requestBodyAsJsonElement(rootUrl, headers);
       Optional<Integer> pages = JsonUtils.getElementValueAsInteger(element, path);
       if (pages.isPresent()) {
+        Log.sysLog("ARTE."+ lang + ": number of pages: " + pages.get());
         return pages.get();
       }
     } catch (IOException e) {
+      Log.errorLog(45983790, e);
       LOG.error("getMaxPagesForOverview", e);
     }
     return ArteConstants.MAX_POSSIBLE_SUBPAGES;

src/main/java/mServer/crawler/sender/arte/tasks/ArteRateLimitedJsonRestTask.java

@@ -5,13 +5,14 @@
 import mServer.crawler.sender.MediathekReader;
 import mServer.crawler.sender.base.AbstractJsonRestTask;
 import mServer.crawler.sender.base.CrawlerUrlDTO;
+import mServer.tool.MserverDaten;
 
 import java.util.Optional;
 import java.util.concurrent.ConcurrentLinkedQueue;
 
 public abstract class ArteRateLimitedJsonRestTask<T, R, D extends CrawlerUrlDTO> extends AbstractJsonRestTask<T, R, D> {
   private static final long serialVersionUID = 1L;
-  private static final RateLimiter rateLimiter = RateLimiter.create(6.0);
+  private static final RateLimiter rateLimiter = RateLimiter.create(MserverDaten.getArteRateLimit());
 
   protected ArteRateLimitedJsonRestTask(MediathekReader aCrawler, ConcurrentLinkedQueue<D> urlToCrawlDTOs, Optional<String> authKey) {
     super(aCrawler, urlToCrawlDTOs, authKey);

src/main/java/mServer/crawler/sender/arte/tasks/ArteVideoInfoTask.java

@@ -83,7 +83,7 @@ protected AbstractRecursivConverterTask<ArteVideoInfoDto, TopicUrlDTO> createNew
 
   @Override
   protected void handleHttpError(TopicUrlDTO dto, URI url, Response response) {
-    Log.errorLog(45983789, "http error " + response.getStatus() + ": " + url);
+    Log.logHttpError(url.toString(), response.getStatus());
     log.fatal("A HTTP error {} occurred when getting REST VideoInfo information from: \"{}\".", response.getStatus(), url);
   }
 }

src/main/java/mServer/crawler/sender/arte/tasks/ArteVideoLinkTask.java

@@ -8,7 +8,6 @@
 
 import de.mediathekview.mlib.tool.Log;
 import mServer.crawler.sender.MediathekReader;
-import mServer.crawler.sender.base.AbstractJsonRestTask;
 import mServer.crawler.sender.base.AbstractRecursivConverterTask;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
@@ -59,7 +58,7 @@ protected AbstractRecursivConverterTask<ArteVideoInfoDto, ArteVideoInfoDto> crea
 
   @Override
   protected void handleHttpError(ArteVideoInfoDto dto, URI url, Response response) {
-    Log.errorLog(89723823, "http error: " + response.getStatus() + " " + url);
+    Log.logHttpError(url.toString(), response.getStatus());
     log.fatal("A HTTP error {} occurred when getting REST VideoLink information from: \"{}\".", response.getStatus(), url);
   }
 }

src/main/java/mServer/crawler/sender/dw/tasks/DwFilmDetailTask.java

@@ -46,7 +46,8 @@ protected void processRestTarget(final CrawlerUrlDTO aDTO, final WebTarget aTarg
     } catch (Exception e) {
       LOG.error("error processing {} ", aDTO.getUrl(), e);
     }
-    if (!filmDetailDtoOptional.isPresent()) {
+    // Optional can be null if response code is 200 and response body is empty
+    if (filmDetailDtoOptional == null|| filmDetailDtoOptional.isEmpty()) {
       FilmeSuchen.listeSenderLaufen.inc(crawler.getSendername(), RunSender.Count.FEHLER);
       FilmeSuchen.listeSenderLaufen.inc(crawler.getSendername(), RunSender.Count.FEHLVERSUCHE);
       return;

src/main/java/mServer/tool/MserverDaten.java

@@ -77,6 +77,22 @@ public static double getSrRateLimit() {
     }
   }
 
+  public static double getArteRateLimit() {
+    String rateLimitValue = system[MserverKonstanten.SYSTEM_ARTE_RATELIMIT_NR];
+    if (rateLimitValue == null || rateLimitValue.trim().equals("")) {
+      return 6.0;
+    }
+
+    try {
+      double rateLimit = Double.parseDouble(system[MserverKonstanten.SYSTEM_ARTE_RATELIMIT_NR].trim());
+      MserverLog.systemMeldung("ARTE rate limit: " + Double.toString(rateLimit));
+      return rateLimit;
+    } catch (NumberFormatException ignored) {
+      MserverLog.fehlerMeldung(963487218, MserverDaten.class.getName(), new String[]{"ARTE rate limit falsch: ", system[MserverKonstanten.SYSTEM_ARTE_RATELIMIT_NR]});
+      return 6.0;
+    }
+  }
+
   public static void setBasisVerzeichnis(String b) {
     if (b.isEmpty()) {
       basisverzeichnis = getBasisVerzeichnis(b, true);