Skip to content

Commit 0ea970f

Browse files
alex1702alex1702
committed
Fix HR Livestream URL in live-streams.json
Filmliste Feld FILM_URL_HISTORY wird nicht mehr benötigt SR: fix missing host in videoUrl arte adjust search depth and ratelimiter remove funk from day-search ORF - more sources for (AT) urls
2 parents 3a237c7 + 05d1bdc commit 0ea970f

File tree

10 files changed

+72
-112
lines changed

10 files changed

+72
-112
lines changed

‎dist/live-streams.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111
"X" : [ "BR", "Livestream", "BR Nord Livestream", "", "", "", "", "", "https://mcdn.br.de/br/fs/bfs_nord/hls/de/master.m3u8", "https://www.br.de/mediathek/live", "", "", "", "", "", "", "", "", "", "" ],
1212
"X" : [ "BR", "Livestream", "BR Süd Livestream", "", "", "", "", "", "https://mcdn.br.de/br/fs/bfs_sued/hls/de/master.m3u8", "https://www.br.de/mediathek/live", "", "", "", "", "", "", "", "", "", "" ],
1313
"X" : [ "DW", "Livestream", "DW Livestream", "", "", "", "", "", "https://dwamdstream102.akamaized.net/hls/live/2015525/dwstream102/index.m3u8", "https://www.dw.com", "", "", "", "", "", "", "", "", "", "" ],
14-
"X" : [ "HR", "Livestream", "HR Livestream", "", "", "", "", "", "https://hrhls.akamaized.net/hls/live/2024525/hrhls/master.m3u8", "https://www.ardmediathek.de/live/Y3JpZDovL2hyLmRlL0xpdmVzdHJlYW0tSFI/", "", "", "", "", "", "", "", "", "", "" ],
14+
"X" : [ "HR", "Livestream", "HR Livestream", "", "", "", "", "", "https://hr-live.ard-mcdn.de/hr/live/hls/de/master.m3u8", "https://www.ardmediathek.de/live/Y3JpZDovL2hyLmRlL0xpdmVzdHJlYW0tSFI/", "", "", "", "", "", "", "", "", "", "" ],
1515
"X" : [ "KiKA", "Livestream", "KiKA Livestream", "", "", "", "", "", "https://kikageohls.akamaized.net/hls/live/2022693/livetvkika_de/master.m3u8", "https://www.ardmediathek.de/live/Y3JpZDovL2tpa2EuZGUvTGl2ZXN0cmVhbS1LaUth/", "", "", "", "", "", "", "", "", "", "" ],
1616
"X" : [ "MDR", "Livestream", "MDR Sachsen Livestream", "", "", "", "", "", "https://mdrtvsnhls.akamaized.net/hls/live/2016928/mdrtvsn/master.m3u8", "https://www.mdr.de/video/livestreams/fernsehen/index.html", "", "", "", "", "", "", "", "", "", "" ],
1717
"X" : [ "MDR", "Livestream", "MDR Sachsen-Anhalt Livestream", "", "", "", "", "", "https://mdrtvsahls.akamaized.net/hls/live/2016879/mdrtvsa/master.m3u8", "https://www.mdr.de/video/livestreams/sachsen-anhalt/index.html", "", "", "", "", "", "", "", "", "", "" ],

‎src/main/java/de/mediathekview/mlib/daten/DatenFilm.java‎

Lines changed: 0 additions & 76 deletions
Original file line numberDiff line numberDiff line change
@@ -103,7 +103,6 @@ public class DatenFilm implements Comparable<DatenFilm> {
103103
"+++Aus rechtlichen Gründen kann dieses Video nur innerhalb von Deutschland abgerufen werden.+++"
104104
};
105105
private static final long MAX_DAUER = 3600 * 99L; /* Werte über 99 Stunden */
106-
public static boolean[] spaltenAnzeigen = new boolean[MAX_ELEM];
107106
public final String[] arr = new String[]{
108107
"", "", "", "", "", "", "", "", "", "",
109108
"", "", "", "", "", "", "", "", "", "", "", "",
@@ -247,41 +246,6 @@ public String getUrlFuerAufloesung(String aufloesung) {
247246
return arr[DatenFilm.FILM_URL];
248247
}
249248

250-
public String getUrlRtmpFuerAufloesung(String aufloesung) {
251-
if (aufloesung.equals(AUFLOESUNG_KLEIN)) {
252-
return getUrlFlvstreamerKlein();
253-
}
254-
if (aufloesung.equals(AUFLOESUNG_HD)) {
255-
return getUrlFlvstreamerHd();
256-
}
257-
return getUrlFlvstreamer();
258-
}
259-
260-
public String getDateigroesse(String url) {
261-
if (url.equals(arr[DatenFilm.FILM_URL])) {
262-
return arr[DatenFilm.FILM_GROESSE];
263-
} else {
264-
return FileSize.laengeString(url);
265-
}
266-
}
267-
268-
public void setUrlHistory() {
269-
String u = getUrl();
270-
if (u.equals(arr[DatenFilm.FILM_URL])) {
271-
arr[DatenFilm.FILM_URL_HISTORY] = "";
272-
} else {
273-
arr[DatenFilm.FILM_URL_HISTORY] = u;
274-
}
275-
}
276-
277-
public String getUrlHistory() {
278-
if (arr[DatenFilm.FILM_URL_HISTORY].isEmpty()) {
279-
return arr[DatenFilm.FILM_URL];
280-
} else {
281-
return arr[DatenFilm.FILM_URL_HISTORY];
282-
}
283-
}
284-
285249
public String getIndex() {
286250
// liefert einen eindeutigen Index für die Filmliste
287251
String url = getUrl();
@@ -504,46 +468,6 @@ private String getUrlFlvstreamer() {
504468
return ret;
505469
}
506470

507-
private String getUrlFlvstreamerKlein() {
508-
// liefert die kleine flvstreamer URL
509-
String ret;
510-
if (!arr[DatenFilm.FILM_URL_RTMP_KLEIN].isEmpty()) {
511-
// es gibt eine kleine RTMP
512-
try {
513-
int i = Integer.parseInt(arr[DatenFilm.FILM_URL_RTMP_KLEIN].substring(0, arr[DatenFilm.FILM_URL_RTMP_KLEIN].indexOf('|')));
514-
return arr[DatenFilm.FILM_URL_RTMP].substring(0, i) + arr[DatenFilm.FILM_URL_RTMP_KLEIN].substring(arr[DatenFilm.FILM_URL_RTMP_KLEIN].indexOf('|') + 1);
515-
} catch (Exception ignored) {
516-
}
517-
}
518-
// es gibt keine kleine RTMP
519-
if (!arr[DatenFilm.FILM_URL_RTMP].isEmpty()) {
520-
// dann gibts keine kleine
521-
ret = arr[DatenFilm.FILM_URL_RTMP];
522-
} else {
523-
// dann gibts überhaupt nur die normalen URLs
524-
ret = getUrlNormalKlein();
525-
// und jetzt noch "-r" davorsetzten wenn nötig
526-
if (ret.startsWith(Const.RTMP_PRTOKOLL)) {
527-
ret = Const.RTMP_FLVSTREAMER + ret;
528-
}
529-
}
530-
return ret;
531-
}
532-
533-
private String getUrlFlvstreamerHd() {
534-
// liefert die HD flvstreamer URL
535-
if (!arr[DatenFilm.FILM_URL_RTMP_HD].isEmpty()) {
536-
// es gibt eine HD RTMP
537-
try {
538-
final int i = Integer.parseInt(arr[DatenFilm.FILM_URL_RTMP_HD].substring(0, arr[DatenFilm.FILM_URL_RTMP_HD].indexOf('|')));
539-
return arr[DatenFilm.FILM_URL_RTMP].substring(0, i) + arr[DatenFilm.FILM_URL_RTMP_HD].substring(arr[DatenFilm.FILM_URL_RTMP_HD].indexOf('|') + 1);
540-
} catch (Exception ignored) {
541-
}
542-
}
543-
// es gibt keine HD RTMP
544-
return getUrlFlvstreamer();
545-
}
546-
547471
private void checkDatum(String datum, String fehlermeldung) {
548472
datum = datum.trim();
549473
if (datum.contains(".") && datum.length() == 10) {

‎src/main/java/mServer/crawler/sender/MediathekReader.java‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -226,7 +226,6 @@ protected void addFilm(DatenFilm film) {
226226
setFileSize(film);
227227
upgradeUrl(film);
228228

229-
film.setUrlHistory();
230229
setGeo(film);
231230
if (mlibFilmeSuchen.listeFilmeNeu.addFilmVomSender(film)) {
232231
// dann ist er neu

‎src/main/java/mServer/crawler/sender/ard/ArdConstants.java‎

Lines changed: 21 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
package mServer.crawler.sender.ard;
22

3+
import org.apache.commons.lang3.ArrayUtils;
4+
35
public class ArdConstants {
46

57
public static final String API_URL = "https://api.ardmediathek.de";
@@ -17,24 +19,25 @@ public class ArdConstants {
1719

1820
public static final String DEFAULT_CLIENT = "ard";
1921

20-
public static final String[] CLIENTS =
21-
new String[] {
22-
"daserste",
23-
"br",
24-
"hr",
25-
"mdr",
26-
"ndr",
27-
"radiobremen",
28-
"rbb",
29-
"sr",
30-
"swr",
31-
"wdr",
32-
"one",
33-
"alpha",
34-
"tagesschau24",
35-
"funk",
36-
"phoenix"
37-
};
22+
public static final String[] CLIENTS_DAY =
23+
new String[] {
24+
"daserste",
25+
"br",
26+
"hr",
27+
"mdr",
28+
"ndr",
29+
"radiobremen",
30+
"rbb",
31+
"sr",
32+
"swr",
33+
"wdr",
34+
"one",
35+
"alpha",
36+
"tagesschau24",
37+
"phoenix"
38+
};
39+
40+
public static final String[] CLIENTS = ArrayUtils.add(ArdConstants.CLIENTS_DAY, "funk");
3841

3942
public static final String[] IGNORED_SENDER = new String[] {"zdf", "kika", "3sat", "arte"};
4043

‎src/main/java/mServer/crawler/sender/ard/ArdCrawler.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -94,7 +94,7 @@ private ConcurrentLinkedQueue<CrawlerUrlDTO> createDayUrlsToCrawl() {
9494

9595
private void addDayUrls(ConcurrentLinkedQueue<CrawlerUrlDTO> dayUrlsToCrawl, LocalDateTime day) {
9696
final String formattedDay = day.format(DAY_PAGE_DATE_FORMATTER);
97-
for (String client : ArdConstants.CLIENTS) {
97+
for (String client : ArdConstants.CLIENTS_DAY) {
9898
final String url = String.format(ArdConstants.DAY_PAGE_URL, formattedDay, client);
9999
dayUrlsToCrawl.offer(new CrawlerUrlDTO(url));
100100
}

‎src/main/java/mServer/crawler/sender/arte/ArteCrawler.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -111,9 +111,9 @@ private int getNumberOfAvailablePages(String lang) {
111111

112112
protected int getMaximumSubpages() {
113113
if (CrawlerTool.loadLongMax()) {
114-
return 20;
115-
} else {
116114
return 10;
115+
} else {
116+
return 5;
117117
}
118118
}
119119
}

‎src/main/java/mServer/crawler/sender/arte/tasks/ArteRateLimitedJsonRestTask.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@
2121
public abstract class ArteRateLimitedJsonRestTask<T, R, D extends CrawlerUrlDTO> extends AbstractJsonRestTask<T, R, D> {
2222
private static final long serialVersionUID = 1L;
2323
private static final RateLimiter rateLimiter = RateLimiter.create(MserverDaten.getArteRateLimit());
24-
private static final RateLimiter opaApirateLimiter = RateLimiter.create(0.3);
24+
private static final RateLimiter opaApirateLimiter = RateLimiter.create(0.2);
2525

2626
protected ArteRateLimitedJsonRestTask(MediathekReader aCrawler, ConcurrentLinkedQueue<D> urlToCrawlDTOs, Optional<String> authKey) {
2727
super(aCrawler, urlToCrawlDTOs, authKey);

‎src/main/java/mServer/crawler/sender/orfon/json/OrfOnEpisodeDeserializer.java‎

Lines changed: 39 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -103,13 +103,17 @@ public OrfOnVideoInfoDTO deserialize(
103103
private Optional<Map<Qualities, String>> optimizeUrls(Optional<Map<Qualities, String>> urls) {
104104
if (urls.isPresent() && urls.get().size() == 1) {
105105
final Map<Qualities, String> urlMap = urls.get();
106-
final String urlToOptimize = urlMap.get(Qualities.NORMAL);
107-
urlMap.put(Qualities.SMALL, urlToOptimize.replace("QXA", "Q4A"));
108-
urlMap.put(Qualities.NORMAL, urlToOptimize.replace("QXA", "Q6A"));
109-
urlMap.put(Qualities.HD, urlToOptimize.replace("QXA", "Q8C"));
106+
String urlToOptimize = urlMap.get(Qualities.NORMAL);
107+
for (String s : List.of("QXA","QXB")) {
108+
urlToOptimize = urlToOptimize.replace(s, "#Q#");
109+
}
110+
urlMap.put(Qualities.SMALL, urlToOptimize.replace("#Q#", "Q4A"));
111+
urlMap.put(Qualities.NORMAL, urlToOptimize.replace("#Q#", "Q6A"));
112+
urlMap.put(Qualities.HD, urlToOptimize.replace("#Q#", "Q8C"));
110113
}
111114
return urls;
112115
}
116+
113117

114118
private Optional<String> buildOrResolveSubs(JsonElement jsonElement) {
115119
Optional<String> subtitleSource = JsonUtils.getElementValueAsString(jsonElement, TAG_SUBTITLE);
@@ -185,7 +189,15 @@ private Optional<Map<Qualities, String>> parseVideoFromSources(JsonElement root)
185189
}
186190
}
187191
}
188-
return parseVideoFromThumbnail(root);
192+
Optional<Map<Qualities, String>> fallbackThumbnail = parseVideoFromThumbnail(root);
193+
if (fallbackThumbnail.isPresent()) {
194+
return fallbackThumbnail;
195+
}
196+
Optional<Map<Qualities, String>> fallbackGapless = parseVideoFromGapless(root);
197+
if (fallbackGapless.isPresent()) {
198+
return fallbackGapless;
199+
}
200+
return Optional.empty();
189201
}
190202

191203
private Optional<Map<Qualities, String>> parseVideoFromThumbnail(JsonElement root) {
@@ -212,6 +224,28 @@ private Optional<Map<Qualities, String>> parseVideoFromThumbnail(JsonElement roo
212224
return Optional.of(urls);
213225
}
214226

227+
private Optional<Map<Qualities, String>> parseVideoFromGapless(JsonElement root) {
228+
Map<Qualities, String> urls = new EnumMap<>(Qualities.class);
229+
try {
230+
Optional<JsonElement> gaplessSourceAT = JsonUtils.getElement(root, "gapless_sources_austria", "hls");
231+
if (gaplessSourceAT.isPresent()) {
232+
gaplessSourceAT.get().getAsJsonArray().forEach( e -> {
233+
Optional<String> url = JsonUtils.getElementValueAsString(e, "src");
234+
Optional<String> drm = JsonUtils.getElementValueAsString(e, "is_drm_protected");
235+
if (url.isPresent() && drm.orElse("").equalsIgnoreCase("false")) {
236+
urls.put(Qualities.NORMAL, url.get());
237+
}
238+
});
239+
}
240+
} catch (Exception e) {
241+
LOG.error("generateFallbackVideo {}", e);
242+
}
243+
if (urls.size() == 0) {
244+
return Optional.empty();
245+
}
246+
return Optional.of(urls);
247+
}
248+
215249
private Optional<Map<Qualities, String>> readVideoForTargetCodec(JsonElement urlArray, String targetCodec) {
216250
Map<Qualities, String> urls = new EnumMap<>(Qualities.class);
217251
for (JsonElement videoElement : urlArray.getAsJsonArray()) {

‎src/main/java/mServer/crawler/sender/sr/SrCrawler.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,6 @@ protected RecursiveTask<Set<DatenFilm>> createCrawlerTask() {
3939
} catch (InterruptedException | ExecutionException exception) {
4040
Log.errorLog(56146546, exception);
4141
}
42-
return new SrFilmDetailTask(this, filmDtos);
42+
return new SrFilmDetailTask(this, filmDtos, SrConstants.URL_BASE);
4343
}
4444
}

‎src/main/java/mServer/crawler/sender/sr/tasks/SrFilmDetailTask.java‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
import de.mediathekview.mlib.Const;
66
import de.mediathekview.mlib.daten.DatenFilm;
77
import de.mediathekview.mlib.tool.Log;
8+
import mServer.crawler.sender.base.*;
89
import mServer.crawler.sender.sr.SrTopicUrlDTO;
910
import org.apache.logging.log4j.LogManager;
1011
import org.jsoup.nodes.Document;
@@ -25,10 +26,6 @@
2526
import mServer.crawler.sender.MediathekReader;
2627
import mServer.crawler.sender.ard.json.ArdVideoInfoDto;
2728
import mServer.crawler.sender.ard.json.ArdVideoInfoJsonDeserializer;
28-
import mServer.crawler.sender.base.AbstractUrlTask;
29-
import mServer.crawler.sender.base.DateUtils;
30-
import mServer.crawler.sender.base.Qualities;
31-
import mServer.crawler.sender.base.HtmlDocumentUtils;
3229

3330
public class SrFilmDetailTask extends SrRateLimitedDocumentTask<DatenFilm, SrTopicUrlDTO> {
3431

@@ -51,10 +48,12 @@ public class SrFilmDetailTask extends SrRateLimitedDocumentTask<DatenFilm, SrTop
5148
private static final String DESCRIPTION_SELECTOR = "h1.background-title";
5249
private static final String VIDEO_DETAIL_ATTRIBUTE = "data-mediacollection-ardplayer";
5350
private static final String VIDEO_DETAIL_SELECTOR = "div[" + VIDEO_DETAIL_ATTRIBUTE + "]";
51+
private final String baseUrl;
5452

5553
public SrFilmDetailTask(
56-
final MediathekReader aCrawler, final ConcurrentLinkedQueue<SrTopicUrlDTO> aUrlToCrawlDTOs) {
54+
final MediathekReader aCrawler, final ConcurrentLinkedQueue<SrTopicUrlDTO> aUrlToCrawlDTOs, String baseUrl) {
5755
super(aCrawler, aUrlToCrawlDTOs);
56+
this.baseUrl = baseUrl;
5857
}
5958

6059
private static Optional<String> parseDescription(final Document aDocument) {
@@ -172,7 +171,7 @@ protected void processDocument(final SrTopicUrlDTO aUrlDTO, final Document aDocu
172171
@Override
173172
protected AbstractUrlTask<DatenFilm, SrTopicUrlDTO> createNewOwnInstance(
174173
final ConcurrentLinkedQueue<SrTopicUrlDTO> aURLsToCrawl) {
175-
return new SrFilmDetailTask(crawler, aURLsToCrawl);
174+
return new SrFilmDetailTask(crawler, aURLsToCrawl, baseUrl);
176175
}
177176

178177
/**
@@ -199,6 +198,7 @@ private Optional<ArdVideoInfoDto> parseUrls(final Document aDocument) {
199198

200199
String url = videoDetailUrl.get();
201200
url = addMissingProtocol(url);
201+
url = UrlUtils.addDomainIfMissing(url, baseUrl);
202202

203203
try {
204204
final ArdVideoInfoDto dto

0 commit comments

Comments
 (0)