Skip to content

Commit a94230a

Browse files
pidoubleyoupidoubleyou
authored
Merge branch 'develop' into develop2
2 parents 3b0398f + 67a478b commit a94230a

File tree

7 files changed

+776
-46
lines changed

7 files changed

+776
-46
lines changed

‎src/main/java/de/mediathekview/mserver/crawler/ard/ArdConstants.java‎

Lines changed: 20 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55

66
import de.mediathekview.mserver.daten.Resolution;
77
import de.mediathekview.mserver.daten.Sender;
8+
import org.apache.commons.lang3.ArrayUtils;
89

910
public class ArdConstants {
1011

@@ -23,24 +24,25 @@ public class ArdConstants {
2324

2425
public static final String DEFAULT_CLIENT = "ard";
2526

26-
public static final String[] CLIENTS =
27-
new String[] {
28-
"daserste",
29-
"br",
30-
"hr",
31-
"mdr",
32-
"ndr",
33-
"radiobremen",
34-
"rbb",
35-
"sr",
36-
"swr",
37-
"wdr",
38-
"one",
39-
"funk",
40-
"alpha",
41-
"tagesschau24",
42-
"phoenix"
43-
};
27+
public static final String[] CLIENTS_DAY =
28+
new String[] {
29+
"daserste",
30+
"br",
31+
"hr",
32+
"mdr",
33+
"ndr",
34+
"radiobremen",
35+
"rbb",
36+
"sr",
37+
"swr",
38+
"wdr",
39+
"one",
40+
"alpha",
41+
"tagesschau24",
42+
"phoenix"
43+
};
44+
45+
public static final String[] CLIENTS = ArrayUtils.add(ArdConstants.CLIENTS_DAY, "funk");
4446

4547
public static final Map<String, Sender> PARTNER_TO_SENDER = new HashMap<>();
4648

‎src/main/java/de/mediathekview/mserver/crawler/ard/ArdCrawler.java‎

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -41,13 +41,8 @@ public Sender getSender() {
4141
private Queue<CrawlerUrlDTO> createDayUrlsToCrawl() {
4242
final Queue<CrawlerUrlDTO> dayUrlsToCrawl = new ConcurrentLinkedQueue<>();
4343
final List<String> days = DateUtils.generateDaysToCrawl(crawlerConfig);
44-
// funk hat keine program übersicht
45-
final String[] CLIENTS_WITHOUT_FUNK =
46-
Arrays.stream(ArdConstants.CLIENTS)
47-
.filter(c -> !"funk".equals(c))
48-
.toArray(String[]::new);
4944
days.forEach( dateString -> {
50-
for (final String client : CLIENTS_WITHOUT_FUNK) {
45+
for (final String client : ArdConstants.CLIENTS_DAY) {
5146
final String url = String.format(ArdConstants.DAY_PAGE_URL, dateString, client);
5247
dayUrlsToCrawl.offer(new CrawlerUrlDTO(url));
5348
}

‎src/main/java/de/mediathekview/mserver/crawler/sr/SrCrawler.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,7 @@ protected RecursiveTask<Set<Film>> createCrawlerTask() {
6060
ServerMessages.DEBUG_ALL_SENDUNG_FOLGEN_COUNT, getSender().getName(), filmDtosFiltered.size());
6161
getAndSetMaxCount(filmDtosFiltered.size());
6262

63-
return new SrFilmDetailTask(this, filmDtosFiltered);
63+
return new SrFilmDetailTask(this, filmDtosFiltered, SrConstants.URL_BASE);
6464
} catch (final InterruptedException ex) {
6565
LOG.debug("{} crawler interrupted.", getSender().getName(), ex);
6666
Thread.currentThread().interrupt();

‎src/main/java/de/mediathekview/mserver/crawler/sr/tasks/SrFilmDetailTask.java‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
import de.mediathekview.mserver.daten.Resolution;
88
import de.mediathekview.mserver.base.utils.DateUtils;
99
import de.mediathekview.mserver.base.utils.HtmlDocumentUtils;
10+
import de.mediathekview.mserver.base.utils.UrlUtils;
1011
import de.mediathekview.mserver.crawler.ard.json.ArdVideoInfoDto;
1112
import de.mediathekview.mserver.crawler.ard.json.ArdVideoInfoJsonDeserializer;
1213
import de.mediathekview.mserver.crawler.basic.AbstractCrawler;
@@ -44,10 +45,14 @@ public class SrFilmDetailTask extends AbstractDocumentTask<Film, SrTopicUrlDTO>
4445
private static final String VIDEO_DETAIL_ATTRIBUTE = "data-mediacollection-ardplayer";
4546
private static final String VIDEO_DETAIL_SELECTOR = "div[" + VIDEO_DETAIL_ATTRIBUTE + "]";
4647

48+
private final String baseUrl;
49+
4750
public SrFilmDetailTask(
4851
final AbstractCrawler crawler,
49-
final Queue<SrTopicUrlDTO> urlToCrawlDTOs) {
52+
final Queue<SrTopicUrlDTO> urlToCrawlDTOs,
53+
final String aBaseUrl) {
5054
super(crawler, urlToCrawlDTOs);
55+
baseUrl = aBaseUrl;
5156
}
5257

5358
private static Optional<String> parseDescription(final Document aDocument) {
@@ -185,7 +190,7 @@ protected void processDocument(final SrTopicUrlDTO aUrlDTO, final Document aDocu
185190
@Override
186191
protected AbstractUrlTask<Film, SrTopicUrlDTO> createNewOwnInstance(
187192
final Queue<SrTopicUrlDTO> aURLsToCrawl) {
188-
return new SrFilmDetailTask(crawler, aURLsToCrawl);
193+
return new SrFilmDetailTask(crawler, aURLsToCrawl, baseUrl);
189194
}
190195

191196
/**
@@ -223,6 +228,7 @@ private Optional<ArdVideoInfoDto> parseUrls(final Document aDocument) {
223228

224229
String url = videoDetailUrl.get();
225230
url = addMissingProtocol(url);
231+
url = UrlUtils.addDomainIfMissing(url, baseUrl);
226232

227233
try {
228234
final ArdVideoInfoDto dto =

‎src/test/java/de/mediathekview/mserver/crawler/sr/tasks/SrFilmDetailTaskNoFilmTest.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -94,7 +94,7 @@ public void test() {
9494

9595
private Set<Film> executeTask(final SrCrawler crawler, final String aTheme, final String aRequestUrl) {
9696
return new SrFilmDetailTask(
97-
crawler, createCrawlerUrlDto(aTheme, aRequestUrl))
97+
crawler, createCrawlerUrlDto(aTheme, aRequestUrl), getWireMockBaseUrlSafe())
9898
.invoke();
9999
}
100100
}

‎src/test/java/de/mediathekview/mserver/crawler/sr/tasks/SrFilmDetailTaskTest.java‎

Lines changed: 32 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -1,29 +1,28 @@
11
package de.mediathekview.mserver.crawler.sr.tasks;
22

3+
import static org.hamcrest.CoreMatchers.equalTo;
4+
import static org.hamcrest.CoreMatchers.notNullValue;
5+
import static org.hamcrest.MatcherAssert.assertThat;
6+
7+
import de.mediathekview.mserver.base.webaccess.JsoupConnection;
8+
import de.mediathekview.mserver.crawler.sr.SrCrawler;
39
import de.mediathekview.mserver.daten.Film;
410
import de.mediathekview.mserver.daten.GeoLocations;
511
import de.mediathekview.mserver.daten.Sender;
6-
import de.mediathekview.mserver.base.webaccess.JsoupConnection;
7-
import de.mediathekview.mserver.crawler.sr.SrCrawler;
812
import de.mediathekview.mserver.testhelper.AssertFilm;
913
import de.mediathekview.mserver.testhelper.JsoupMock;
14+
import java.time.Duration;
15+
import java.time.LocalDateTime;
16+
import java.util.Arrays;
17+
import java.util.Collection;
18+
import java.util.Set;
1019
import org.junit.Before;
1120
import org.junit.Test;
1221
import org.junit.runner.RunWith;
1322
import org.junit.runners.Parameterized;
1423
import org.mockito.Mock;
1524
import org.mockito.MockitoAnnotations;
1625

17-
import java.time.Duration;
18-
import java.time.LocalDateTime;
19-
import java.util.Arrays;
20-
import java.util.Collection;
21-
import java.util.Set;
22-
23-
import static org.hamcrest.CoreMatchers.equalTo;
24-
import static org.hamcrest.CoreMatchers.notNullValue;
25-
import static org.hamcrest.MatcherAssert.assertThat;
26-
2726
@RunWith(Parameterized.class)
2827
public class SrFilmDetailTaskTest extends SrTaskTestBase {
2928

@@ -43,11 +42,6 @@ public class SrFilmDetailTaskTest extends SrTaskTestBase {
4342

4443
@Mock JsoupConnection jsoupConnection;
4544

46-
@Before
47-
public void setUp() {
48-
MockitoAnnotations.openMocks(this);
49-
}
50-
5145
public SrFilmDetailTaskTest(
5246
final String aRequestUrl,
5347
final String aFilmPageFile,
@@ -111,9 +105,29 @@ public static Collection<Object[]> data() {
111105
"https://srstorage01-a.akamaihd.net/Video/FS/STH/Schengen_-_Wie_entstand_das_Europa_ohne_Grenzen_SENDEFASSUNG_L.mp4",
112106
"https://srstorage01-a.akamaihd.net/Video/FS/STH/Schengen_-_Wie_entstand_das_Europa_ohne_Grenzen_SENDEFASSUNG_P.mp4"
113107
},
108+
{
109+
"https://www.sr-mediathek.de/index.php?seite=7&id=77119",
110+
"/sr/sr_film_page3_missing_host_in_url.html",
111+
"/sr_player/mc.php?id=77119&tbl=&pnr=0&hd=0&devicetype=",
112+
"/sr/sr_film_video_details2.json",
113+
"SAARTHEMA",
114+
"SAARTHEMA - Schengen",
115+
LocalDateTime.of(2019, 8, 15, 0, 0, 0),
116+
Duration.ofMinutes(43).plusSeconds(18),
117+
"An jeder Grenze der Welt ist es ein Begriff: Schengen heißt eines der wichtigsten Visa, die es heute gibt. Am 14. Juni 1985 unterzeichneten die Vertreter der EG-Staaten Deutschland, Frankreich, Belgien, Niederlande und Luxemburg das Schengener-Abkommen, das im Laufe der Jahre von fast allen EU-Staaten ratifiziert wurde und uns in Europa offene Grenzen gebracht hat.",
118+
"https://www.sr-mediathek.de/sr_player/ut.php?file=STH_20190815.xml",
119+
"https://srstorage01-a.akamaihd.net/Video/FS/STH/Schengen_-_Wie_entstand_das_Europa_ohne_Grenzen_SENDEFASSUNG_M.mp4",
120+
"https://srstorage01-a.akamaihd.net/Video/FS/STH/Schengen_-_Wie_entstand_das_Europa_ohne_Grenzen_SENDEFASSUNG_L.mp4",
121+
"https://srstorage01-a.akamaihd.net/Video/FS/STH/Schengen_-_Wie_entstand_das_Europa_ohne_Grenzen_SENDEFASSUNG_P.mp4"
122+
}
114123
});
115124
}
116125

126+
@Before
127+
public void setUp() {
128+
MockitoAnnotations.openMocks(this);
129+
}
130+
117131
@Test
118132
public void test() {
119133
jsoupConnection =
@@ -147,6 +161,6 @@ public void test() {
147161

148162
private Set<Film> executeTask(
149163
final SrCrawler crawler, final String aTheme, final String aRequestUrl) {
150-
return new SrFilmDetailTask(crawler, createCrawlerUrlDto(aTheme, aRequestUrl)).invoke();
164+
return new SrFilmDetailTask(crawler, createCrawlerUrlDto(aTheme, aRequestUrl), getWireMockBaseUrlSafe()).invoke();
151165
}
152166
}

0 commit comments

Comments
 (0)