Skip to content

Adapt url scraper to site changes #8

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
felgru wants to merge 6 commits into
base: master
Choose a base branch
from
Open

Adapt url scraper to site changes #8

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
e03feaa
Fix url scraping in getStreamUrlNew()
sbahling Sep 7, 2014
21dcbbd
Fix URL for new episodes list.
sbahling Oct 24, 2014
96286d2
changes from version 2.1.1
felgru Sep 20, 2014
9d12de4
Merge remote-tracking branch 'sbahling/master'
felgru Nov 1, 2014
05ab260
Fix URL scraping in listCats().
felgru Nov 1, 2014
0f1fe75
Adapt search to site changes
felgru Nov 1, 2014
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion addon.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
<?xml version="1.0" encoding="UTF-8"?>
<addon id="plugin.video.arte_tv" name="Arte.tv" version="2.0.9" provider-name="AddonScriptorDE">
<addon id="plugin.video.arte_tv" name="Arte.tv" version="2.1.1" provider-name="AddonScriptorDE">
<requires>
<import addon="xbmc.python" version="2.1.0"/>
</requires>
Expand Down
4 changes: 4 additions & 0 deletions changelog.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,3 +22,7 @@
- Fixed site changes (concerts)
2.0.9
- Fixed geoblocking filter
2.1.0
- Fixed concert livestreams
2.1.1
- Added support for concert collections
66 changes: 54 additions & 12 deletions default.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@
forceViewMode = addon.getSetting("forceView") == "true"
useThumbAsFanart = addon.getSetting("useThumbAsFanart") == "true"
viewMode = str(addon.getSetting("viewIDNew"))
addonDir = xbmc.translatePath(addon.getAddonInfo('path'))
defaultFanart = os.path.join(addonDir ,'fanart.png')
icon = xbmc.translatePath('special://home/addons/'+addonID+'/icon.png')
baseUrl = "http://www.arte.tv"
baseUrlConcert = "http://concert.arte.tv"
Expand All @@ -39,7 +41,7 @@ def index():
for filter in regionFilters:
regionFilter += filter.replace("'","").strip()+"%2C"
regionFilter = regionFilter[:-3]
addDir(translation(30001), baseUrl+"/guide/"+language+"/plus7/plus_recentes.json?regions="+regionFilter, "listVideosNew", "")
addDir(translation(30001), baseUrl+"/guide/"+language+"/plus7.json", "listVideosNew", "")
addDir(translation(30002), baseUrl+"/guide/"+language+"/plus7/selection.json?regions="+regionFilter, "listVideosNew", "")
addDir(translation(30003), baseUrl+"/guide/"+language+"/plus7/plus_vues.json?regions="+regionFilter, "listVideosNew", "")
addDir(translation(30004), baseUrl+"/guide/"+language+"/plus7/derniere_chance.json?regions="+regionFilter, "listVideosNew", "")
Expand Down Expand Up @@ -106,24 +108,27 @@ def listSearchVideos(urlMain):
xbmcplugin.setContent(pluginhandle, "episodes")
try:
content = getUrl(urlMain)
spl = content.split('<div class="video-block has-play"')
spl = content.split('has-play\'')
for i in range(1, len(spl), 1):
entry = spl[i]
match = re.compile('alt="(.+?)"', re.DOTALL).findall(entry)
match = re.compile('<h4>.+?<a href=\'.+?\'>\n(.+?) \| ARTE', re.DOTALL).findall(entry)
title = cleanTitle(match[0])
# The search keyword is wrapped in <b> </b> in the tile, remove these
title = re.compile('<b>').sub('',title)
title = re.compile('</b>').sub('',title)
match = re.compile('data-description="(.+?)"', re.DOTALL).findall(entry)
desc = ""
if match:
desc = cleanTitle(match[0])
match = re.compile('<p class="time-row">.+?<span class=".+?">.+?</span>(.+?)</p>.+?<p class=".+?">.+?<span class=".+?">.+?</span>(.+?)</p>', re.DOTALL).findall(entry)
match = re.compile('<p class=\'time-row\'>.+?<span class=\'.+?\'>.+?</span>(.+?) \((.+?)\).+?</p>', re.DOTALL).findall(entry)
if match:
date = match[0][0].strip()
desc = date+"\n"+desc
duration = match[0][1].strip()
duration = duration.split(" ")[0]
match = re.compile('href="(.+?)"', re.DOTALL).findall(entry)
match = re.compile('href=\'(.+?)\'', re.DOTALL).findall(entry)
url = match[0]
match = re.compile('src="(.+?)"', re.DOTALL).findall(entry)
match = re.compile('src=\'(.+?)\'', re.DOTALL).findall(entry)
thumb = match[0]
addLink(title, url, 'playVideoNew', thumb, desc, duration)
currentPage = urlMain[urlMain.find("page=")+5:]
Expand All @@ -138,9 +143,9 @@ def listSearchVideos(urlMain):

def listCats(type, regionFilter):
content = getUrl(baseUrl+"/guide/"+language+"/plus7")
content = content[content.find('<ul class="span12" data-filter="'+type+'">'):]
content = content[:content.find('</ul>')]
match = re.compile('<a href="(.+?)" data-controller="catchup" data-action="refresh" >(.+?)</a>', re.DOTALL).findall(content)
content = content[content.find('<div class=\'choice-'+type+' choices row\' data-filter=\''+type+'\'>'):]
content = content[:content.find('</div>\n</div>\n</div>')]
match = re.compile('<a class="ellipsis" data-action="refresh" data-controller="catchup" href="(.+?)">(.+?)</a>', re.DOTALL).findall(content)
for url, title in match:
title = cleanTitle(title)
url = baseUrl+url.replace("?", ".json?").replace("&amp;", "&")+"&regions="+regionFilter
Expand All @@ -164,7 +169,7 @@ def search():

def listConcertsMain():
addDir(translation(30002), "", "listConcerts", "")
addDir(translation(30003), baseUrlConcert+"/"+language+"/videos/all?sort=mostviewed", "listConcerts", "")
addDir("Collections", "", "listCollections", "")
addDir(translation(30011), baseUrlConcert+"/"+language+"/videos/all", "listConcerts", "")
addDir(translation(30013), baseUrlConcert+"/"+language+"/videos/rockpop", "listConcerts", "")
if language=="de":
Expand Down Expand Up @@ -199,7 +204,10 @@ def listConcerts(url=""):
url = baseUrlConcert+match[0]
match = re.compile('src="(.+?)"', re.DOTALL).findall(entry)
thumb = match[0].replace("/alw_rectangle_376/","/alw_rectangle_690/").replace("/alw_highlight_480/","/alw_rectangle_690/")
addLink(title, url, 'playVideoNew', thumb, "")
if "node-eventp" in entry:
addDir(title, url, "listConcerts", thumb)
elif "node-videop" in entry:
addLink(title, url, 'playVideoNew', thumb, "")
match = re.compile('<li class="pager-next">.+?href="(.+?)"', re.DOTALL).findall(content)
if match:
addDir(translation(30010), baseUrlConcert+match[0], "listConcerts", "")
Expand All @@ -208,23 +216,49 @@ def listConcerts(url=""):
xbmc.executebuiltin('Container.SetViewMode('+viewMode+')')


def listCollections():
content = getUrl("http://concert.arte.tv/"+language+"/collections.xml")
spl = content.split('<item>')
for i in range(1, len(spl), 1):
entry = spl[i]
match = re.compile('<title>(.+?)</title>', re.DOTALL).findall(entry)
title = cleanTitle(match[0])
match = re.compile('field-name-eventp-videos-count.*?&gt;(.+?)&lt;', re.DOTALL).findall(entry)
if match:
count = match[0].strip()
if language=="de":
count = count.replace("vidéos","Videos")
title += " ("+count+")"
match = re.compile('<link>(.+?)</link>', re.DOTALL).findall(entry)
url = match[0]
match = re.compile('data-src=&quot;(.+?)&quot;', re.DOTALL).findall(entry)
thumb = match[0].replace("/alw_rectangle_376/","/alw_rectangle_690/").replace("/alw_highlight_480/","/alw_rectangle_690/")
addDir(title, url, "listConcerts", thumb)
xbmcplugin.endOfDirectory(pluginhandle)
if forceViewMode:
xbmc.executebuiltin('Container.SetViewMode('+viewMode+')')


def playVideoNew(url):
listitem = xbmcgui.ListItem(path=getStreamUrlNew(url))
xbmcplugin.setResolvedUrl(pluginhandle, True, listitem)


def getStreamUrlNew(url):
content = getUrl(url)
match = re.compile('arte_vp_url="(.+?)"', re.DOTALL).findall(content)
match = re.compile('arte_vp_url=[\'"](.+?)[\'"]', re.DOTALL).findall(content)
if "concert.arte.tv" in url:
url = match[0]
content = getUrl(url)
match1 = re.compile('"HTTP_SQ_1":.+?"url":"(.+?)"', re.DOTALL).findall(content)
match2 = re.compile('"HTTP_EQ_1":.+?"url":"(.+?)"', re.DOTALL).findall(content)
match3 = re.compile('"RMTP_HQ":.*?"streamer":"(.+?)","url":"(.+?)"', re.DOTALL).findall(content)
if match1 and maxVideoQuality == "720p":
return match1[0].replace("\\","")
elif match2:
return match2[0].replace("\\","")
elif match3:
return match3[0][0].replace("\\","")+match3[0][1].replace("\\","")+" swfUrl=http://www.arte.tv/flash/mediaplayer/mediaplayer.swf live=1 swfVfy=1"
elif streamingType=="HTTP":
url = match[0].replace("/player/","/")
content = getUrl(url)
Expand Down Expand Up @@ -306,6 +340,8 @@ def addLink(name, url, mode, iconimage, desc="", duration=""):
liz.setProperty('IsPlayable', 'true')
if useThumbAsFanart and iconimage!=icon:
liz.setProperty("fanart_image", iconimage)
else:
liz.setProperty("fanart_image", defaultFanart)
liz.addContextMenuItems([(translation(30020), 'RunPlugin(plugin://'+addonID+'/?mode=queueVideo&url='+urllib.quote_plus(u)+'&name='+urllib.quote_plus(name)+')',)])
ok = xbmcplugin.addDirectoryItem(handle=int(sys.argv[1]), url=u, listitem=liz)
return ok
Expand All @@ -316,6 +352,10 @@ def addDir(name, url, mode, iconimage, regionFilter=""):
ok = True
liz = xbmcgui.ListItem(name, iconImage=icon, thumbnailImage=iconimage)
liz.setInfo(type="Video", infoLabels={"Title": name})
if useThumbAsFanart and iconimage and iconimage!=icon:
liz.setProperty("fanart_image", iconimage)
else:
liz.setProperty("fanart_image", defaultFanart)
ok = xbmcplugin.addDirectoryItem(handle=int(sys.argv[1]), url=u, listitem=liz, isFolder=True)
return ok

Expand All @@ -341,6 +381,8 @@ def addDir(name, url, mode, iconimage, regionFilter=""):
playLiveStream()
elif mode == 'listConcerts':
listConcerts(url)
elif mode == 'listCollections':
listCollections()
elif mode == 'listConcertsMain':
listConcertsMain()
elif mode == 'search':
Expand Down