Fix url scraping in getStreamUrlNew()

addon.xml

@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<addon id="plugin.video.arte_tv" name="Arte.tv" version="2.0.9" provider-name="AddonScriptorDE">
+<addon id="plugin.video.arte_tv" name="Arte.tv" version="2.1.1" provider-name="AddonScriptorDE">
     <requires>
         <import addon="xbmc.python" version="2.1.0"/>
     </requires>

changelog.txt

@@ -22,3 +22,7 @@
 - Fixed site changes (concerts)
 2.0.9
 - Fixed geoblocking filter
+2.1.0
+- Fixed concert livestreams
+2.1.1
+- Added support for concert collections

default.py

@@ -1,25 +1,27 @@
 #!/usr/bin/python
 # -*- coding: utf-8 -*-
+import os
 import urllib
 import urllib2
 import socket
 import sys
 import re
-import os
 import json
 import xbmcplugin
 import xbmcaddon
 import xbmcgui
 
-#addon = xbmcaddon.Addon()
-#addonID = addon.getAddonInfo('id')
+# addon = xbmcaddon.Addon()
+# addonID = addon.getAddonInfo('id')
 addonID = "plugin.video.arte_tv"
 addon = xbmcaddon.Addon(id=addonID)
 socket.setdefaulttimeout(30)
 pluginhandle = int(sys.argv[1])
 forceViewMode = addon.getSetting("forceView") == "true"
 useThumbAsFanart = addon.getSetting("useThumbAsFanart") == "true"
 viewMode = str(addon.getSetting("viewIDNew"))
+addonDir = xbmc.translatePath(addon.getAddonInfo('path'))
+defaultFanart = os.path.join(addonDir ,'fanart.png')
 icon = xbmc.translatePath('special://home/addons/'+addonID+'/icon.png')
 baseUrl = "http://www.arte.tv"
 baseUrlConcert = "http://concert.arte.tv"
@@ -30,73 +32,83 @@
 streamingType = addon.getSetting("streamingType")
 streamingType = ["HTTP", "RTMP"][int(streamingType)]
 
+guide = '%s/guide/%s' % (baseUrl, language)
+new_json = 'plus7.json'
+selection_json = 'plus7/selection.json'
+lastchance_json = 'plus7/derniere_chance.json'
+mostviewed_json = 'plus7/plus_vues.json'
+
+
+def jsonUrl(jsonfile, regions=''):
+    url = '%s/%s' % (guide, jsonfile)
+    if regions:
+        url = '%s?regions=%s' % (url, regions)
+    return url
+
 
 def index():
     content = getUrl(baseUrl+"/artews/js/geolocation.js")
-    match = re.compile('arte_geoip_zone_codes.+?return new Array\\((.+?)\\)', re.DOTALL).findall(content)
+    match = re.compile('arte_geoip_zone_codes.+?return new Array\\((.+?)\\)',
+                       re.DOTALL).findall(content)
     regionFilters = match[0].split(",")
     regionFilter = ""
     for filter in regionFilters:
-        regionFilter += filter.replace("'","").strip()+"%2C"
+        regionFilter += filter.replace("'", "").strip()+"%2C"
     regionFilter = regionFilter[:-3]
-    addDir(translation(30001), baseUrl+"/guide/"+language+"/plus7/plus_recentes.json?regions="+regionFilter, "listVideosNew", "")
-    addDir(translation(30002), baseUrl+"/guide/"+language+"/plus7/selection.json?regions="+regionFilter, "listVideosNew", "")
-    addDir(translation(30003), baseUrl+"/guide/"+language+"/plus7/plus_vues.json?regions="+regionFilter, "listVideosNew", "")
-    addDir(translation(30004), baseUrl+"/guide/"+language+"/plus7/derniere_chance.json?regions="+regionFilter, "listVideosNew", "")
+    addDir(translation(30001), jsonUrl(new_json),
+           "listVideosNew", "")
+    addDir(translation(30002), jsonUrl(selection_json, regionFilter),
+           "listVideosNew", "")
+    addDir(translation(30003), jsonUrl(lastchance_json, regionFilter),
+           "listVideosNew", "")
+    addDir(translation(30004), jsonUrl(mostviewed_json, regionFilter),
+           "listVideosNew", "")
     addDir(translation(30005), "by_channel", "listCats", "", regionFilter)
     addDir(translation(30006), "by_cluster", "listCats", "", regionFilter)
     addDir(translation(30007), "by_date", "listCats", "", regionFilter)
     addDir(translation(30008), "", "search", "")
     addDir(translation(30012), "", "listConcertsMain", "")
-    if regionFilter!="ALL":
+    if regionFilter != "ALL":
         addLink(translation(30009), "", "playLiveStream", icon)
     xbmcplugin.endOfDirectory(pluginhandle)
 
 
+def str_item(item):
+    '''Convert item to utf-8 encoded string object'''
+    if not item:
+        item = ''
+    if not isinstance(item, basestring):
+        try:
+            item = str(item)
+        except:
+            item = ''
+    return item.encode('utf-8', 'replace')
+
+
 def listVideosNew(url):
     xbmcplugin.setContent(pluginhandle, "episodes")
     content = getUrl(url)
     content = json.loads(content)
     for item in content["videos"]:
-        title = item["title"].encode('utf-8')
-        try:
-            desc = item["desc"].encode('utf-8')
-        except:
-            desc = ""
-        try:
-            duration = str(item["duration"])
-        except:
-            duration = ""
-        try:
-            date = item["airdate_long"].encode('utf-8')
-        except:
-            date = ""
-        try:
-            url = item["url"]
-        except:
-            url = ""
-        try:
-            thumb = item["image_url"]
-        except:
-            thumb = ""
-        try:
-            channels = item["video_channels"].encode('utf-8')
-        except:
-            channels = ""
-        try:
-            views = str(item["video_views"])
-        except:
-            views = ""
-        try:
-            until = item["video_rights_until"].encode('utf-8')
-        except:
-            until = ""
-        try:
-            rank = str(item["video_rank"])
-        except:
-            rank = ""
+        title = str_item(item.get("title", "No Title"))
+        desc = str_item(item.get("desc", ""))
+        duration = str_item(item.get("duration", ""))
+        date = str_item(item.get("airdate_long", ""))
+        url = str_item(item.get("url", ""))
+        thumb = str_item(item.get("image_url", ""))
+        channels = str_item(item.get("video_channels", ""))
+        views = str_item(item.get("video_views", ""))
+        until = str_item(item.get("video_rights_until", ""))
+        rank = str_item(item.get("video_rank", ""))
+
         desc = views+"   |   "+date+"\n"+channels+"\n"+desc
-        addLink(cleanTitle(title), baseUrl+url, 'playVideoNew', thumb, desc, duration)
+        addLink(title,
+                url,
+                'playVideoNew',
+                thumb,
+                desc,
+                duration,
+                )
     xbmcplugin.endOfDirectory(pluginhandle)
     if forceViewMode:
         xbmc.executebuiltin('Container.SetViewMode('+viewMode+')')
@@ -110,11 +122,11 @@ def listSearchVideos(urlMain):
         for i in range(1, len(spl), 1):
             entry = spl[i]
             match = re.compile('alt="(.+?)"', re.DOTALL).findall(entry)
-            title = cleanTitle(match[0])
+            title = match[0]
             match = re.compile('data-description="(.+?)"', re.DOTALL).findall(entry)
             desc = ""
             if match:
-                desc = cleanTitle(match[0])
+                desc = match[0]
             match = re.compile('<p class="time-row">.+?<span class=".+?">.+?</span>(.+?)</p>.+?<p class=".+?">.+?<span class=".+?">.+?</span>(.+?)</p>', re.DOTALL).findall(entry)
             if match:
                 date = match[0][0].strip()
@@ -142,7 +154,7 @@ def listCats(type, regionFilter):
     content = content[:content.find('</ul>')]
     match = re.compile('<a href="(.+?)" data-controller="catchup" data-action="refresh" >(.+?)</a>', re.DOTALL).findall(content)
     for url, title in match:
-        title = cleanTitle(title)
+        title = title
         url = baseUrl+url.replace("?", ".json?").replace("&amp;", "&")+"&regions="+regionFilter
         addDir(title, url, 'listVideosNew', "")
     xbmcplugin.endOfDirectory(pluginhandle)
@@ -164,7 +176,7 @@ def search():
 
 def listConcertsMain():
     addDir(translation(30002), "", "listConcerts", "")
-    addDir(translation(30003), baseUrlConcert+"/"+language+"/videos/all?sort=mostviewed", "listConcerts", "")
+    addDir("Collections", "", "listCollections", "")
     addDir(translation(30011), baseUrlConcert+"/"+language+"/videos/all", "listConcerts", "")
     addDir(translation(30013), baseUrlConcert+"/"+language+"/videos/rockpop", "listConcerts", "")
     if language=="de":
@@ -194,12 +206,15 @@ def listConcerts(url=""):
     for i in range(1, len(spl), 1):
         entry = spl[i]
         match = re.compile('title="(.+?)"', re.DOTALL).findall(entry)
-        title = cleanTitle(match[0])
+        title = match[0]
         match = re.compile('href="(.+?)"', re.DOTALL).findall(entry)
         url = baseUrlConcert+match[0]
         match = re.compile('src="(.+?)"', re.DOTALL).findall(entry)
         thumb = match[0].replace("/alw_rectangle_376/","/alw_rectangle_690/").replace("/alw_highlight_480/","/alw_rectangle_690/")
-        addLink(title, url, 'playVideoNew', thumb, "")
+        if "node-eventp" in entry:
+            addDir(title, url, "listConcerts", thumb)
+        elif "node-videop" in entry:
+            addLink(title, url, 'playVideoNew', thumb, "")
     match = re.compile('<li class="pager-next">.+?href="(.+?)"', re.DOTALL).findall(content)
     if match:
         addDir(translation(30010), baseUrlConcert+match[0], "listConcerts", "")
@@ -208,44 +223,89 @@ def listConcerts(url=""):
         xbmc.executebuiltin('Container.SetViewMode('+viewMode+')')
 
 
+def listCollections():
+    content = getUrl("http://concert.arte.tv/"+language+"/collections.xml")
+    spl = content.split('<item>')
+    for i in range(1, len(spl), 1):
+        entry = spl[i]
+        match = re.compile('<title>(.+?)</title>', re.DOTALL).findall(entry)
+        title = cleanTitle(match[0])
+        match = re.compile('field-name-eventp-videos-count.*?&gt;(.+?)&lt;', re.DOTALL).findall(entry)
+        if match:
+            count = match[0].strip()
+            if language=="de":
+                count = count.replace("vidéos","Videos")
+            title += " ("+count+")"
+        match = re.compile('<link>(.+?)</link>', re.DOTALL).findall(entry)
+        url = match[0]
+        match = re.compile('data-src=&quot;(.+?)&quot;', re.DOTALL).findall(entry)
+        thumb = match[0].replace("/alw_rectangle_376/","/alw_rectangle_690/").replace("/alw_highlight_480/","/alw_rectangle_690/")
+        addDir(title, url, "listConcerts", thumb)
+    xbmcplugin.endOfDirectory(pluginhandle)
+    if forceViewMode:
+        xbmc.executebuiltin('Container.SetViewMode('+viewMode+')')
+
+
 def playVideoNew(url):
     listitem = xbmcgui.ListItem(path=getStreamUrlNew(url))
     xbmcplugin.setResolvedUrl(pluginhandle, True, listitem)
 
 
+def getBestStream(VSR):
+    '''Return the best quality stream from an arte VSR json structure.
+    Try preferred streamingType first and fall back to other type.
+    '''
+
+    vid_format_tmpl = u'{streamtype}_{qual}_{lang}'
+    lang_code = {'de': '1', 'fr': '2'}
+
+    if maxVideoQuality == '720p':
+        quals = ['SQ', 'HQ', 'EQ', 'MQ']
+    else:
+        quals = ['EQ', 'MQ']
+
+    # Videos use HTTP_MP4 concerts use HTTP. We always try both.
+    streamtypes = {'RTMP': ['RMTP', 'HTTP_MP4', 'HTTP'],
+                   'HTTP': ['HTTP_MP4', 'HTTP', 'RMTP'],
+                   }
+
+    lang = lang_code.get(language.lower(), 'de')
+    for streamtype in streamtypes[streamingType]:
+        streamUrl = None
+        for qual in quals:
+            vid_format = vid_format_tmpl.format(streamtype=streamtype,
+                                                qual=qual,
+                                                lang=lang,
+                                                )
+            try:
+                streamUrl = VSR[vid_format]['url']
+                streamer = VSR[vid_format].get('streamer', '')
+                if streamer:
+                    streamUrl = '%s/%s' % (streamer, streamUrl)
+                break
+            except:
+                pass
+
+        if streamUrl is not None:
+            break
+
+    return streamUrl
+
+
 def getStreamUrlNew(url):
     content = getUrl(url)
-    match = re.compile('arte_vp_url="(.+?)"', re.DOTALL).findall(content)
-    if "concert.arte.tv" in url:
-        url = match[0]
-        content = getUrl(url)
-        match1 = re.compile('"HTTP_SQ_1":.+?"url":"(.+?)"', re.DOTALL).findall(content)
-        match2 = re.compile('"HTTP_EQ_1":.+?"url":"(.+?)"', re.DOTALL).findall(content)
-        if match1 and maxVideoQuality == "720p":
-            return match1[0].replace("\\","")
-        elif match2:
-            return match2[0].replace("\\","")
-    elif streamingType=="HTTP":
-        url = match[0].replace("/player/","/")
-        content = getUrl(url)
-        match1 = re.compile('"HBBTV","VQU":"SQ","VMT":"mp4","VUR":"(.+?)"', re.DOTALL).findall(content)
-        match2 = re.compile('"HBBTV","VQU":"EQ","VMT":"mp4","VUR":"(.+?)"', re.DOTALL).findall(content)
-        if match1 and maxVideoQuality == "720p":
-            return match1[0]
-        elif match2:
-            return match2[0]
-    elif streamingType=="RTMP":
-        url = match[0]
-        content = getUrl(url)
-        match1 = re.compile('"RTMP_SQ_1":.+?"streamer":"(.+?)","url":"(.+?)"', re.DOTALL).findall(content)
-        match2 = re.compile('"RTMP_MQ_1":.+?"streamer":"(.+?)","url":"(.+?)"', re.DOTALL).findall(content)
-        if match1 and maxVideoQuality == "720p":
-            base = match1[0][0]
-            playpath = match1[0][1]
-        elif match2:
-            base = match2[0][0]
-            playpath = match2[0][1]
-        return base+" playpath=mp4:"+playpath
+    vp_url_re = 'arte_vp_url=[\'"](.+?)[\'"]'
+    match = re.compile(vp_url_re, re.DOTALL).findall(content)
+
+    url = match[0]
+    content = getUrl(url)
+    content = json.loads(content)
+
+    VSR = content['videoJsonPlayer']['VSR']
+
+    url = getBestStream(VSR)
+
+    return url
 
 
 def queueVideo(url, name):
@@ -306,6 +366,8 @@ def addLink(name, url, mode, iconimage, desc="", duration=""):
     liz.setProperty('IsPlayable', 'true')
     if useThumbAsFanart and iconimage!=icon:
         liz.setProperty("fanart_image", iconimage)
+    else:
+        liz.setProperty("fanart_image", defaultFanart)
     liz.addContextMenuItems([(translation(30020), 'RunPlugin(plugin://'+addonID+'/?mode=queueVideo&url='+urllib.quote_plus(u)+'&name='+urllib.quote_plus(name)+')',)])
     ok = xbmcplugin.addDirectoryItem(handle=int(sys.argv[1]), url=u, listitem=liz)
     return ok
@@ -316,6 +378,10 @@ def addDir(name, url, mode, iconimage, regionFilter=""):
     ok = True
     liz = xbmcgui.ListItem(name, iconImage=icon, thumbnailImage=iconimage)
     liz.setInfo(type="Video", infoLabels={"Title": name})
+    if useThumbAsFanart and iconimage and iconimage!=icon:
+        liz.setProperty("fanart_image", iconimage)
+    else:
+        liz.setProperty("fanart_image", defaultFanart)
     ok = xbmcplugin.addDirectoryItem(handle=int(sys.argv[1]), url=u, listitem=liz, isFolder=True)
     return ok
 
@@ -341,6 +407,8 @@ def addDir(name, url, mode, iconimage, regionFilter=""):
     playLiveStream()
 elif mode == 'listConcerts':
     listConcerts(url)
+elif mode == 'listCollections':
+    listCollections()
 elif mode == 'listConcertsMain':
     listConcertsMain()
 elif mode == 'search':