From 47e0cef46e9a76e589a2b1cde1b2dfa8bcf01d8e Mon Sep 17 00:00:00 2001
From: Tithen-Firion <tithen.firion.0@gmail.com>
Date: Sun, 16 Apr 2017 00:34:34 +0200
Subject: [openload] rewrite extractor

---
 youtube_dl/extractor/openload.py | 118 +++++++++++++++++++++++----------------
 1 file changed, 71 insertions(+), 47 deletions(-)

diff --git a/youtube_dl/extractor/openload.py b/youtube_dl/extractor/openload.py
index d8036b5..789bf99 100644
--- a/youtube_dl/extractor/openload.py
+++ b/youtube_dl/extractor/openload.py
@@ -1,12 +1,16 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import os
 import re
+import subprocess
+import tempfile
 
 from .common import InfoExtractor
-from ..compat import compat_chr
 from ..utils import (
+    check_executable,
     determine_ext,
+    encodeArgument,
     ExtractorError,
 )
 
@@ -58,6 +62,39 @@ class OpenloadIE(InfoExtractor):
         'only_matching': True,
     }]
 
+    _PHANTOMJS_SCRIPT = r'''
+        phantom.onError = function(msg, trace) {
+          var msgStack = ['PHANTOM ERROR: ' + msg];
+          if(trace && trace.length) {
+            msgStack.push('TRACE:');
+            trace.forEach(function(t) {
+              msgStack.push(' -> ' + (t.file || t.sourceURL) + ': ' + t.line
+                + (t.function ? ' (in function ' + t.function +')' : ''));
+            });
+          }
+          console.error(msgStack.join('\n'));
+          phantom.exit(1);
+        };
+        var page = require('webpage').create();
+        page.settings.resourceTimeout = 10000;
+        page.onInitialized = function() {
+          page.evaluate(function() {
+            delete window._phantom;
+            delete window.callPhantom;
+          });
+        };
+        page.open('https://openload.co/embed/%s/', function(status) {
+          var info = page.evaluate(function() {
+            return {
+              decoded_id: document.getElementById('streamurl').innerHTML,
+              title: document.querySelector('meta[name="og:title"],'
+                + 'meta[name=description]').content
+            };
+          });
+          console.log(info.decoded_id + ' ' + info.title);
+          phantom.exit();
+        });'''
+
     @staticmethod
     def _extract_urls(webpage):
         return re.findall(
@@ -65,61 +102,48 @@ class OpenloadIE(InfoExtractor):
             webpage)
 
     def _real_extract(self, url):
+        exe = check_executable('phantomjs', ['-v'])
+        if not exe:
+            raise ExtractorError('PhantomJS executable not found in PATH, '
+                                 'download it from http://phantomjs.org',
+                                 expected=True)
+
         video_id = self._match_id(url)
-        webpage = self._download_webpage('https://openload.co/embed/%s/' % video_id, video_id)
+        url = 'https://openload.co/embed/%s/' % video_id
+        webpage = self._download_webpage(url, video_id)
 
         if 'File not found' in webpage or 'deleted by the owner' in webpage:
-            raise ExtractorError('File not found', expected=True)
-
-        ol_id = self._search_regex(
-            '<span[^>]+id="[^"]+"[^>]*>([0-9A-Za-z]+)</span>',
-            webpage, 'openload ID')
-
-        decoded = ''
-        a = ol_id[0:24]
-        b = []
-        for i in range(0, len(a), 8):
-            b.append(int(a[i:i + 8] or '0', 16))
-        ol_id = ol_id[24:]
-        j = 0
-        k = 0
-        while j < len(ol_id):
-            c = 128
-            d = 0
-            e = 0
-            f = 0
-            _more = True
-            while _more:
-                if j + 1 >= len(ol_id):
-                    c = 143
-                f = int(ol_id[j:j + 2] or '0', 16)
-                j += 2
-                d += (f & 127) << e
-                e += 7
-                _more = f >= c
-            g = d ^ b[k % 3]
-            for i in range(4):
-                char_dec = (g >> 8 * i) & (c + 127)
-                char = compat_chr(char_dec)
-                if char != '#':
-                    decoded += char
-            k += 1
-
-        video_url = 'https://openload.co/stream/%s?mime=true'
-        video_url = video_url % decoded
-
-        title = self._og_search_title(webpage, default=None) or self._search_regex(
-            r'<span[^>]+class=["\']title["\'][^>]*>([^<]+)', webpage,
-            'title', default=None) or self._html_search_meta(
-            'description', webpage, 'title', fatal=True)
+            raise ExtractorError('File not found', expected=True, video_id=video_id)
+
+        script_file = tempfile.NamedTemporaryFile(mode='w', delete=False)
+
+        # write JS script to file and close it
+        with script_file:
+            script_file.write(self._PHANTOMJS_SCRIPT % video_id)
+
+        self.to_screen('%s: Decoding video ID with PhantomJS' % video_id)
+
+        p = subprocess.Popen([exe, '--ssl-protocol=any', script_file.name],
+            stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        output, err = p.communicate()
+        if p.returncode != 0:
+            raise ExtractorError('Decoding failed\n:'
+                                 + encodeArgument(err))
+        else:
+            decoded_id, title = encodeArgument(output).strip().split(' ', 1)
+
+        os.remove(script_file.name)
+
+        video_url = 'https://openload.co/stream/%s?mime=true' % decoded_id
 
         entries = self._parse_html5_media_entries(url, webpage, video_id)
-        subtitles = entries[0]['subtitles'] if entries else None
+        entry = entries[0] if entries else {}
+        subtitles = entry.get('subtitles')
 
         info_dict = {
             'id': video_id,
             'title': title,
-            'thumbnail': self._og_search_thumbnail(webpage, default=None),
+            'thumbnail': entry.get('thumbnail') or self._og_search_thumbnail(webpage, default=None),
             'url': video_url,
             # Seems all videos have extensions in their titles
             'ext': determine_ext(title, 'mp4'),
-- 
cgit v1.1


From da57ebaf84225240b356530cdf02d12596f0dce8 Mon Sep 17 00:00:00 2001
From: Tithen-Firion <tithen.firion.0@gmail.com>
Date: Tue, 25 Apr 2017 01:06:14 +0200
Subject: [openload] separate PhantomJS code from extractor

---
 youtube_dl/extractor/openload.py |  78 +++++-----------------
 youtube_dl/utils.py              | 141 +++++++++++++++++++++++++++++++++++++++
 2 files changed, 157 insertions(+), 62 deletions(-)

diff --git a/youtube_dl/extractor/openload.py b/youtube_dl/extractor/openload.py
index 789bf99..ac5e0bb 100644
--- a/youtube_dl/extractor/openload.py
+++ b/youtube_dl/extractor/openload.py
@@ -1,17 +1,14 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import os
 import re
-import subprocess
-import tempfile
 
 from .common import InfoExtractor
 from ..utils import (
-    check_executable,
     determine_ext,
-    encodeArgument,
     ExtractorError,
+    get_element_by_id,
+    PhantomJSwrapper,
 )
 
 
@@ -62,38 +59,7 @@ class OpenloadIE(InfoExtractor):
         'only_matching': True,
     }]
 
-    _PHANTOMJS_SCRIPT = r'''
-        phantom.onError = function(msg, trace) {
-          var msgStack = ['PHANTOM ERROR: ' + msg];
-          if(trace && trace.length) {
-            msgStack.push('TRACE:');
-            trace.forEach(function(t) {
-              msgStack.push(' -> ' + (t.file || t.sourceURL) + ': ' + t.line
-                + (t.function ? ' (in function ' + t.function +')' : ''));
-            });
-          }
-          console.error(msgStack.join('\n'));
-          phantom.exit(1);
-        };
-        var page = require('webpage').create();
-        page.settings.resourceTimeout = 10000;
-        page.onInitialized = function() {
-          page.evaluate(function() {
-            delete window._phantom;
-            delete window.callPhantom;
-          });
-        };
-        page.open('https://openload.co/embed/%s/', function(status) {
-          var info = page.evaluate(function() {
-            return {
-              decoded_id: document.getElementById('streamurl').innerHTML,
-              title: document.querySelector('meta[name="og:title"],'
-                + 'meta[name=description]').content
-            };
-          });
-          console.log(info.decoded_id + ' ' + info.title);
-          phantom.exit();
-        });'''
+    _USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36'
 
     @staticmethod
     def _extract_urls(webpage):
@@ -102,40 +68,27 @@ class OpenloadIE(InfoExtractor):
             webpage)
 
     def _real_extract(self, url):
-        exe = check_executable('phantomjs', ['-v'])
-        if not exe:
-            raise ExtractorError('PhantomJS executable not found in PATH, '
-                                 'download it from http://phantomjs.org',
-                                 expected=True)
-
         video_id = self._match_id(url)
         url = 'https://openload.co/embed/%s/' % video_id
-        webpage = self._download_webpage(url, video_id)
+        headers = {
+            'User-Agent': self._USER_AGENT,
+        }
+
+        phantom = PhantomJSwrapper(self)
+        webpage, _ = phantom.get(url, video_id=video_id, headers=headers)
 
         if 'File not found' in webpage or 'deleted by the owner' in webpage:
             raise ExtractorError('File not found', expected=True, video_id=video_id)
 
-        script_file = tempfile.NamedTemporaryFile(mode='w', delete=False)
-
-        # write JS script to file and close it
-        with script_file:
-            script_file.write(self._PHANTOMJS_SCRIPT % video_id)
-
-        self.to_screen('%s: Decoding video ID with PhantomJS' % video_id)
-
-        p = subprocess.Popen([exe, '--ssl-protocol=any', script_file.name],
-            stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-        output, err = p.communicate()
-        if p.returncode != 0:
-            raise ExtractorError('Decoding failed\n:'
-                                 + encodeArgument(err))
-        else:
-            decoded_id, title = encodeArgument(output).strip().split(' ', 1)
-
-        os.remove(script_file.name)
+        decoded_id = get_element_by_id('streamurl', webpage)
 
         video_url = 'https://openload.co/stream/%s?mime=true' % decoded_id
 
+        title = self._og_search_title(webpage, default=None) or self._search_regex(
+            r'<span[^>]+class=["\']title["\'][^>]*>([^<]+)', webpage,
+            'title', default=None) or self._html_search_meta(
+            'description', webpage, 'title', fatal=True)
+
         entries = self._parse_html5_media_entries(url, webpage, video_id)
         entry = entries[0] if entries else {}
         subtitles = entry.get('subtitles')
@@ -148,5 +101,6 @@ class OpenloadIE(InfoExtractor):
             # Seems all videos have extensions in their titles
             'ext': determine_ext(title, 'mp4'),
             'subtitles': subtitles,
+            'http_headers': headers,
         }
         return info_dict
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 2340bc3..94e1b07 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -3652,3 +3652,144 @@ def write_xattr(path, key, value):
                         "Couldn't find a tool to set the xattrs. "
                         "Install either the python 'xattr' module, "
                         "or the 'xattr' binary.")
+
+
+class PhantomJSwrapper(object):
+    """PhantomJS wrapper class"""
+
+    _TEMPLATE = r'''
+        phantom.onError = function(msg, trace) {{
+          var msgStack = ['PHANTOM ERROR: ' + msg];
+          if(trace && trace.length) {{
+            msgStack.push('TRACE:');
+            trace.forEach(function(t) {{
+              msgStack.push(' -> ' + (t.file || t.sourceURL) + ': ' + t.line
+                + (t.function ? ' (in function ' + t.function +')' : ''));
+            }});
+          }}
+          console.error(msgStack.join('\n'));
+          phantom.exit(1);
+        }};
+        var page = require('webpage').create();
+        var fs = require('fs');
+        var read = {{ mode: 'r', charset: 'utf-8' }};
+        var write = {{ mode: 'w', charset: 'utf-8' }};
+        page.settings.resourceTimeout = {timeout};
+        page.settings.userAgent = "{ua}";
+        page.onLoadStarted = function() {{
+          page.evaluate(function() {{
+            delete window._phantom;
+            delete window.callPhantom;
+          }});
+        }};
+        var saveAndExit = function() {{
+          fs.write("{html}", page.content, write);
+          phantom.exit();
+        }};
+        page.onLoadFinished = function(status) {{
+          if(page.url === "") {{
+            page.setContent(fs.read("{html}", read), "{url}");
+          }}
+          else {{
+            {jscode}
+          }}
+        }};
+        page.open("");
+    '''
+
+    _TMP_FILE_NAMES = ['script', 'html']
+
+    def __init__(self, extractor, timeout=10000):
+        self.exe = check_executable('phantomjs', ['-v'])
+        if not self.exe:
+            raise ExtractorError('PhantomJS executable not found in PATH, '
+                                 'download it from http://phantomjs.org',
+                                 expected=True)
+        self.extractor = extractor
+        self.options = {
+            'timeout': timeout,
+        }
+        self._TMP_FILES = {}
+        for name in self._TMP_FILE_NAMES:
+            tmp = tempfile.NamedTemporaryFile(delete=False)
+            tmp.close()
+            self._TMP_FILES[name] = tmp
+
+    def __del__(self):
+        for name in self._TMP_FILE_NAMES:
+            try:
+                os.remove(self._TMP_FILES[name].name)
+            except:
+                pass
+
+    def get(self, url, html=None, video_id=None, note=None, note2='Executing JS on webpage', headers={}, jscode='saveAndExit();'):
+        """
+        Downloads webpage (if needed) and executes JS
+        
+        Params:
+            url: website url
+            html: optional, html code of website
+            video_id: video id
+            note: optional, displayed when downloading webpage
+            note2: optional, displayed when executing JS
+            headers: custom http headers
+            jscode: code to be executed when page is loaded
+        
+        Returns tuple with:
+            * downloaded website (after JS execution)
+            * anything you print with `console.log` (but not inside `page.execute`!)
+        
+        In most cases you don't need to add any `jscode`.
+        It is executed in `page.onLoadFinished`.
+        `saveAndExit();` is mandatory, use it instead of `phantom.exit()`
+        It is possible to wait for some element on the webpage, for example:
+            var check = function() {
+              var elementFound = page.evaluate(function() {
+                return document.querySelector('#b.done') !== null;
+              });
+              if(elementFound)
+                saveAndExit();
+              else
+                window.setTimeout(check, 500);
+            }
+            
+            page.evaluate(function(){
+              document.querySelector('#a').click();
+            });
+            check();
+        """
+        if 'saveAndExit();' not in jscode:
+            raise ExtractorError('`saveAndExit();` not found in `jscode`')
+        if not html:
+            html = self.extractor._download_webpage(url, video_id, note=note, headers=headers)
+        with open(self._TMP_FILES['html'].name, 'wb') as f:
+            f.write(html.encode('utf-8'))
+
+        replaces = self.options
+        replaces['url'] = url
+        user_agent = headers.get('User-Agent') or std_headers['User-Agent']
+        replaces['ua'] = user_agent.replace('"', '\\"')
+        replaces['jscode'] = jscode
+
+        for x in self._TMP_FILE_NAMES:
+            replaces[x] = self._TMP_FILES[x].name.replace('\\', '\\\\').replace('"', '\\"')
+
+        with open(self._TMP_FILES['script'].name, 'wb') as f:
+            f.write(self._TEMPLATE.format(**replaces).encode('utf-8'))
+
+        if video_id is None:
+            self.extractor.to_screen('%s' % (note2,))
+        else:
+            self.extractor.to_screen('%s: %s' % (video_id, note2))
+
+        p = subprocess.Popen([self.exe, '--ssl-protocol=any',
+            self._TMP_FILES['script'].name], stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE)
+        out, err = p.communicate()
+        if p.returncode != 0:
+            raise ExtractorError('Executing JS failed\n:'
+                                 + encodeArgument(err))
+        with open(self._TMP_FILES['html'].name, 'rb') as f:
+            html = f.read().decode('utf-8')
+        return (html, encodeArgument(out))
+
-- 
cgit v1.1


From 40e41780f1d770a355f01e3c1e6fb09ff392f97e Mon Sep 17 00:00:00 2001
From: Tithen-Firion <tithen.firion.0@gmail.com>
Date: Tue, 25 Apr 2017 15:12:54 +0200
Subject: [phantomjs] add cookie support

---
 youtube_dl/extractor/common.py |  8 ++++--
 youtube_dl/utils.py            | 62 +++++++++++++++++++++++++++++++++++++++++-
 2 files changed, 66 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index dcc9d62..e54adc9 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -2343,10 +2343,12 @@ class InfoExtractor(object):
                 self._downloader.report_warning(msg)
         return res
 
-    def _set_cookie(self, domain, name, value, expire_time=None):
+    def _set_cookie(self, domain, name, value, expire_time=None, port=None,
+                    path='/', secure=False, discard=False, rest={}, **kwargs):
         cookie = compat_cookiejar.Cookie(
-            0, name, value, None, None, domain, None,
-            None, '/', True, False, expire_time, '', None, None, None)
+            0, name, value, port, not port is None, domain, True,
+            domain.startswith('.'), path, True, secure, expire_time,
+            discard, None, None, rest)
         self._downloader.cookiejar.set_cookie(cookie)
 
     def _get_cookies(self, url):
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 94e1b07..9c94b7e 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -3654,6 +3654,37 @@ def write_xattr(path, key, value):
                         "or the 'xattr' binary.")
 
 
+def cookie_to_dict(cookie):
+    cookie_dict = {
+        'name': cookie.name,
+        'value': cookie.value,
+    };
+    if cookie.port_specified:
+        cookie_dict['port'] = cookie.port
+    if cookie.domain_specified:
+        cookie_dict['domain'] = cookie.domain
+    if cookie.path_specified:
+        cookie_dict['path'] = cookie.path
+    if not cookie.expires is None:
+        cookie_dict['expires'] = cookie.expires
+    if not cookie.secure is None:
+        cookie_dict['secure'] = cookie.secure
+    if not cookie.discard is None:
+        cookie_dict['discard'] = cookie.discard
+    try:
+        if (cookie.has_nonstandard_attr('httpOnly') or
+            cookie.has_nonstandard_attr('httponly') or
+            cookie.has_nonstandard_attr('HttpOnly')):
+            cookie_dict['httponly'] = True
+    except TypeError:
+        pass
+    return cookie_dict
+
+
+def cookie_jar_to_list(cookie_jar):
+    return [cookie_to_dict(cookie) for cookie in cookie_jar]
+
+
 class PhantomJSwrapper(object):
     """PhantomJS wrapper class"""
 
@@ -3674,6 +3705,9 @@ class PhantomJSwrapper(object):
         var fs = require('fs');
         var read = {{ mode: 'r', charset: 'utf-8' }};
         var write = {{ mode: 'w', charset: 'utf-8' }};
+        JSON.parse(fs.read("{cookies}", read)).forEach(function(x) {{
+          phantom.addCookie(x);
+        }});
         page.settings.resourceTimeout = {timeout};
         page.settings.userAgent = "{ua}";
         page.onLoadStarted = function() {{
@@ -3684,6 +3718,7 @@ class PhantomJSwrapper(object):
         }};
         var saveAndExit = function() {{
           fs.write("{html}", page.content, write);
+          fs.write("{cookies}", JSON.stringify(phantom.cookies), write);
           phantom.exit();
         }};
         page.onLoadFinished = function(status) {{
@@ -3697,7 +3732,7 @@ class PhantomJSwrapper(object):
         page.open("");
     '''
 
-    _TMP_FILE_NAMES = ['script', 'html']
+    _TMP_FILE_NAMES = ['script', 'html', 'cookies']
 
     def __init__(self, extractor, timeout=10000):
         self.exe = check_executable('phantomjs', ['-v'])
@@ -3722,6 +3757,26 @@ class PhantomJSwrapper(object):
             except:
                 pass
 
+    def _save_cookies(self, url):
+        cookies = cookie_jar_to_list(self.extractor._downloader.cookiejar)
+        for cookie in cookies:
+            if 'path' not in cookie:
+                cookie['path'] = '/'
+            if 'domain' not in cookie:
+                cookie['domain'] = compat_urlparse.urlparse(url).netloc
+        with open(self._TMP_FILES['cookies'].name, 'wb') as f:
+            f.write(json.dumps(cookies).encode('utf-8'))
+
+    def _load_cookies(self):
+        with open(self._TMP_FILES['cookies'].name, 'rb') as f:
+            cookies = json.loads(f.read().decode('utf-8'))
+        for cookie in cookies:
+            if cookie['httponly'] is True:
+                cookie['rest'] = { 'httpOnly': None }
+            if 'expiry' in cookie:
+                cookie['expire_time'] = cookie['expiry']
+            self.extractor._set_cookie(**cookie)
+
     def get(self, url, html=None, video_id=None, note=None, note2='Executing JS on webpage', headers={}, jscode='saveAndExit();'):
         """
         Downloads webpage (if needed) and executes JS
@@ -3765,6 +3820,8 @@ class PhantomJSwrapper(object):
         with open(self._TMP_FILES['html'].name, 'wb') as f:
             f.write(html.encode('utf-8'))
 
+        self._save_cookies(url)
+
         replaces = self.options
         replaces['url'] = url
         user_agent = headers.get('User-Agent') or std_headers['User-Agent']
@@ -3791,5 +3848,8 @@ class PhantomJSwrapper(object):
                                  + encodeArgument(err))
         with open(self._TMP_FILES['html'].name, 'rb') as f:
             html = f.read().decode('utf-8')
+
+        self._load_cookies()
+
         return (html, encodeArgument(out))
 
-- 
cgit v1.1


From fcace2d1adac5d1f306b22219fde3a4542bcd719 Mon Sep 17 00:00:00 2001
From: Tithen-Firion <tithen.firion.0@gmail.com>
Date: Sat, 29 Apr 2017 10:30:45 +0200
Subject: [openload] raise `not found` before executing js

---
 youtube_dl/extractor/openload.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/openload.py b/youtube_dl/extractor/openload.py
index ac5e0bb..0adf177 100644
--- a/youtube_dl/extractor/openload.py
+++ b/youtube_dl/extractor/openload.py
@@ -74,12 +74,14 @@ class OpenloadIE(InfoExtractor):
             'User-Agent': self._USER_AGENT,
         }
 
-        phantom = PhantomJSwrapper(self)
-        webpage, _ = phantom.get(url, video_id=video_id, headers=headers)
+        webpage = self._download_webpage(url, video_id, headers=headers)
 
         if 'File not found' in webpage or 'deleted by the owner' in webpage:
             raise ExtractorError('File not found', expected=True, video_id=video_id)
 
+        phantom = PhantomJSwrapper(self)
+        webpage, _ = phantom.get(url, html=webpage, video_id=video_id, headers=headers)
+
         decoded_id = get_element_by_id('streamurl', webpage)
 
         video_url = 'https://openload.co/stream/%s?mime=true' % decoded_id
-- 
cgit v1.1


From 98f9d873814da2a8584cc30c0e197c15ed249db3 Mon Sep 17 00:00:00 2001
From: Tithen-Firion <tithen.firion.0@gmail.com>
Date: Sat, 29 Apr 2017 12:41:42 +0200
Subject: [phantomjs] Add required version checking

---
 youtube_dl/utils.py | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 9c94b7e..84aaac6 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -3734,13 +3734,22 @@ class PhantomJSwrapper(object):
 
     _TMP_FILE_NAMES = ['script', 'html', 'cookies']
 
-    def __init__(self, extractor, timeout=10000):
+    def __init__(self, extractor, required_version=None, timeout=10000):
         self.exe = check_executable('phantomjs', ['-v'])
         if not self.exe:
             raise ExtractorError('PhantomJS executable not found in PATH, '
                                  'download it from http://phantomjs.org',
                                  expected=True)
+
         self.extractor = extractor
+
+        if required_version:
+            version = get_exe_version(self.exe, version_re=r'([0-9.]+)')
+            if is_outdated_version(version, required_version):
+                self.extractor._downloader.report_warning(
+                    'Your copy of PhantomJS is outdated, update it to version '
+                    '%s or newer if you encounter any errors.' % required_version)
+
         self.options = {
             'timeout': timeout,
         }
-- 
cgit v1.1


From 7552f96352f35cd877e52fd0770b77ba1856fc62 Mon Sep 17 00:00:00 2001
From: Tithen-Firion <tithen.firion.0@gmail.com>
Date: Sat, 29 Apr 2017 12:41:57 +0200
Subject: [openload] Add required version

---
 youtube_dl/extractor/openload.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/openload.py b/youtube_dl/extractor/openload.py
index 0adf177..292476e 100644
--- a/youtube_dl/extractor/openload.py
+++ b/youtube_dl/extractor/openload.py
@@ -79,7 +79,7 @@ class OpenloadIE(InfoExtractor):
         if 'File not found' in webpage or 'deleted by the owner' in webpage:
             raise ExtractorError('File not found', expected=True, video_id=video_id)
 
-        phantom = PhantomJSwrapper(self)
+        phantom = PhantomJSwrapper(self, required_version='2.0')
         webpage, _ = phantom.get(url, html=webpage, video_id=video_id, headers=headers)
 
         decoded_id = get_element_by_id('streamurl', webpage)
-- 
cgit v1.1


From 5ff1bc0cc10bf3006834c4b49fc36d733c83ce5c Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Fri, 28 Apr 2017 22:25:20 +0100
Subject: [YoutubeDL] write raw subtitle files

---
 youtube_dl/YoutubeDL.py | 43 ++++++++++++++++++++++---------------------
 1 file changed, 22 insertions(+), 21 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index eb465c4..c7100bb 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1696,29 +1696,30 @@ class YoutubeDL(object):
             ie = self.get_info_extractor(info_dict['extractor_key'])
             for sub_lang, sub_info in subtitles.items():
                 sub_format = sub_info['ext']
-                if sub_info.get('data') is not None:
-                    sub_data = sub_info['data']
+                sub_filename = subtitles_filename(filename, sub_lang, sub_format)
+                if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(sub_filename)):
+                    self.to_screen('[info] Video subtitle %s.%s is already present' % (sub_lang, sub_format))
                 else:
-                    try:
-                        sub_data = ie._download_webpage(
-                            sub_info['url'], info_dict['id'], note=False)
-                    except ExtractorError as err:
-                        self.report_warning('Unable to download subtitle for "%s": %s' %
-                                            (sub_lang, error_to_compat_str(err.cause)))
-                        continue
-                try:
-                    sub_filename = subtitles_filename(filename, sub_lang, sub_format)
-                    if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(sub_filename)):
-                        self.to_screen('[info] Video subtitle %s.%s is already_present' % (sub_lang, sub_format))
+                    self.to_screen('[info] Writing video subtitles to: ' + sub_filename)
+                    if sub_info.get('data') is not None:
+                        try:
+                            # Use newline='' to prevent conversion of newline characters
+                            # See https://github.com/rg3/youtube-dl/issues/10268
+                            with io.open(encodeFilename(sub_filename), 'w', encoding='utf-8', newline='') as subfile:
+                                subfile.write(sub_info['data'])
+                        except (OSError, IOError):
+                            self.report_error('Cannot write subtitles file ' + sub_filename)
+                            return
                     else:
-                        self.to_screen('[info] Writing video subtitles to: ' + sub_filename)
-                        # Use newline='' to prevent conversion of newline characters
-                        # See https://github.com/rg3/youtube-dl/issues/10268
-                        with io.open(encodeFilename(sub_filename), 'w', encoding='utf-8', newline='') as subfile:
-                            subfile.write(sub_data)
-                except (OSError, IOError):
-                    self.report_error('Cannot write subtitles file ' + sub_filename)
-                    return
+                        try:
+                            sub_data = ie._request_webpage(
+                                sub_info['url'], info_dict['id'], note=False).read()
+                            with io.open(encodeFilename(sub_filename), 'wb') as subfile:
+                                subfile.write(sub_data)
+                        except (ExtractorError, IOError, OSError, ValueError) as err:
+                            self.report_warning('Unable to download subtitle for "%s": %s' %
+                                                (sub_lang, error_to_compat_str(err)))
+                            continue
 
         if self.params.get('writeinfojson', False):
             infofn = replace_extension(filename, 'info.json', info_dict.get('ext'))
-- 
cgit v1.1


From feee8d32e45c9521426cf4a089c70f37542f0065 Mon Sep 17 00:00:00 2001
From: Tithen-Firion <tithen.firion.0@gmail.com>
Date: Thu, 3 Aug 2017 14:17:25 +0200
Subject: [phantomjs] add exe version to debug info

---
 youtube_dl/YoutubeDL.py | 2 ++
 youtube_dl/utils.py     | 6 +++++-
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index eb465c4..033b507 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -86,6 +86,7 @@ from .utils import (
     write_string,
     YoutubeDLCookieProcessor,
     YoutubeDLHandler,
+    PhantomJSwrapper,
 )
 from .cache import Cache
 from .extractor import get_info_extractor, gen_extractor_classes, _LAZY_LOADER
@@ -2146,6 +2147,7 @@ class YoutubeDL(object):
 
         exe_versions = FFmpegPostProcessor.get_versions(self)
         exe_versions['rtmpdump'] = rtmpdump_version()
+        exe_versions['phantomjs'] = PhantomJSwrapper._version()
         exe_str = ', '.join(
             '%s %s' % (exe, v)
             for exe, v in sorted(exe_versions.items())
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index c67f95a..4d0685d 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -3895,6 +3895,10 @@ class PhantomJSwrapper(object):
 
     _TMP_FILE_NAMES = ['script', 'html', 'cookies']
 
+    @staticmethod
+    def _version():
+        return get_exe_version('phantomjs', version_re=r'([0-9.]+)')
+
     def __init__(self, extractor, required_version=None, timeout=10000):
         self.exe = check_executable('phantomjs', ['-v'])
         if not self.exe:
@@ -3905,7 +3909,7 @@ class PhantomJSwrapper(object):
         self.extractor = extractor
 
         if required_version:
-            version = get_exe_version(self.exe, version_re=r'([0-9.]+)')
+            version = self._version()
             if is_outdated_version(version, required_version):
                 self.extractor._downloader.report_warning(
                     'Your copy of PhantomJS is outdated, update it to version '
-- 
cgit v1.1


From e7c3e33456e155106ad08347d8ab9a2ecd0c8a11 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 14 Sep 2017 23:19:53 +0700
Subject: [downloader/fragment] Restart inconsistent incomplete fragment
 downloads (#13731)

---
 youtube_dl/downloader/fragment.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/downloader/fragment.py b/youtube_dl/downloader/fragment.py
index bccc8ec..6f6fb4a 100644
--- a/youtube_dl/downloader/fragment.py
+++ b/youtube_dl/downloader/fragment.py
@@ -151,10 +151,15 @@ class FragmentFD(FileDownloader):
         if self.__do_ytdl_file(ctx):
             if os.path.isfile(encodeFilename(self.ytdl_filename(ctx['filename']))):
                 self._read_ytdl_file(ctx)
+                if ctx['fragment_index'] > 0 and resume_len == 0:
+                    self.report_error(
+                        'Inconsistent state of incomplete fragment download. '
+                        'Restarting from the beginning...')
+                    ctx['fragment_index'] = resume_len = 0
+                    self._write_ytdl_file(ctx)
             else:
                 self._write_ytdl_file(ctx)
-            if ctx['fragment_index'] > 0:
-                assert resume_len > 0
+                assert ctx['fragment_index'] == 0
 
         dest_stream, tmpfilename = sanitize_open(tmpfilename, open_mode)
 
-- 
cgit v1.1


From 319fc70676fea19b71437aab4078d4d72cc1ba5f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 14 Sep 2017 23:50:19 +0700
Subject: [tv4] Relax _VALID_URL (closes #14206)

---
 youtube_dl/extractor/tv4.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/tv4.py b/youtube_dl/extractor/tv4.py
index 7aeb2c6..6487039 100644
--- a/youtube_dl/extractor/tv4.py
+++ b/youtube_dl/extractor/tv4.py
@@ -18,7 +18,7 @@ class TV4IE(InfoExtractor):
             tv4\.se/(?:[^/]+)/klipp/(?:.*)-|
             tv4play\.se/
             (?:
-                (?:program|barn)/(?:[^\?]+)\?video_id=|
+                (?:program|barn)/(?:[^/]+/|(?:[^\?]+)\?video_id=)|
                 iframe/video/|
                 film/|
                 sport/|
@@ -63,6 +63,10 @@ class TV4IE(InfoExtractor):
             'url': 'http://www.tv4play.se/barn/looney-tunes?video_id=3062412',
             'only_matching': True,
         },
+        {
+            'url': ' http://www.tv4play.se/program/farang/3922081',
+            'only_matching': True,
+        }
     ]
 
     def _real_extract(self, url):
-- 
cgit v1.1


From 0732a90579091ad60b124fd693bdda8ee526e305 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 14 Sep 2017 20:37:46 +0200
Subject: [orf] Add new extractor for f4m stories

---
 youtube_dl/extractor/extractors.py |   1 +
 youtube_dl/extractor/orf.py        | 114 +++++++++++++++++++++++++++++++++++--
 2 files changed, 111 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index aefadc5..a3a97e9 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -768,6 +768,7 @@ from .ora import OraTVIE
 from .orf import (
     ORFTVthekIE,
     ORFFM4IE,
+    ORFFM4StoryIE,
     ORFOE1IE,
     ORFIPTVIE,
 )
diff --git a/youtube_dl/extractor/orf.py b/youtube_dl/extractor/orf.py
index cc296ea..74fe801 100644
--- a/youtube_dl/extractor/orf.py
+++ b/youtube_dl/extractor/orf.py
@@ -6,14 +6,15 @@ import re
 from .common import InfoExtractor
 from ..compat import compat_str
 from ..utils import (
+    determine_ext,
+    float_or_none,
     HEADRequest,
-    unified_strdate,
-    strip_jsonp,
     int_or_none,
-    float_or_none,
-    determine_ext,
+    orderedSet,
     remove_end,
+    strip_jsonp,
     unescapeHTML,
+    unified_strdate,
 )
 
 
@@ -307,3 +308,108 @@ class ORFIPTVIE(InfoExtractor):
             'upload_date': upload_date,
             'formats': formats,
         }
+
+
+class ORFFM4StoryIE(InfoExtractor):
+    IE_NAME = 'orf:fm4:story'
+    IE_DESC = 'fm4.orf.at stories'
+    _VALID_URL = r'https?://fm4\.orf\.at/stories/(?P<id>\d+)'
+
+    _TEST = {
+        'url': 'http://fm4.orf.at/stories/2865738/',
+        'playlist': [{
+            'md5': 'e1c2c706c45c7b34cf478bbf409907ca',
+            'info_dict': {
+                'id': '547792',
+                'ext': 'flv',
+                'title': 'Manu Delago und Inner Tongue live',
+                'description': 'Manu Delago und Inner Tongue haben bei der FM4 Soundpark Session live alles gegeben. Hier gibt es Fotos und die gesamte Session als Video.',
+                'duration': 1748.52,
+                'thumbnail': r're:^https?://.*\.jpg$',
+                'upload_date': '20170913',
+            },
+        }, {
+            'md5': 'c6dd2179731f86f4f55a7b49899d515f',
+            'info_dict': {
+                'id': '547798',
+                'ext': 'flv',
+                'title': 'Manu Delago und Inner Tongue live (2)',
+                'duration': 1504.08,
+                'thumbnail': r're:^https?://.*\.jpg$',
+                'upload_date': '20170913',
+                'description': 'Manu Delago und Inner Tongue haben bei der FM4 Soundpark Session live alles gegeben. Hier gibt es Fotos und die gesamte Session als Video.',
+            },
+        }],
+    }
+
+    def _real_extract(self, url):
+        story_id = self._match_id(url)
+        webpage = self._download_webpage(url, story_id)
+
+        entries = []
+        all_ids = orderedSet(re.findall(r'data-video(?:id)?="(\d+)"', webpage))
+        for idx, video_id in enumerate(all_ids):
+            data = self._download_json(
+                'http://bits.orf.at/filehandler/static-api/json/current/data.json?file=%s' % video_id,
+                video_id)[0]
+
+            duration = float_or_none(data['duration'], 1000)
+
+            video = data['sources']['q8c']
+            load_balancer_url = video['loadBalancerUrl']
+            abr = int_or_none(video.get('audioBitrate'))
+            vbr = int_or_none(video.get('bitrate'))
+            fps = int_or_none(video.get('videoFps'))
+            width = int_or_none(video.get('videoWidth'))
+            height = int_or_none(video.get('videoHeight'))
+            thumbnail = video.get('preview')
+
+            rendition = self._download_json(
+                load_balancer_url, video_id, transform_source=strip_jsonp)
+
+            f = {
+                'abr': abr,
+                'vbr': vbr,
+                'fps': fps,
+                'width': width,
+                'height': height,
+            }
+
+            formats = []
+            for format_id, format_url in rendition['redirect'].items():
+                if format_id == 'rtmp':
+                    ff = f.copy()
+                    ff.update({
+                        'url': format_url,
+                        'format_id': format_id,
+                    })
+                    formats.append(ff)
+                elif determine_ext(format_url) == 'f4m':
+                    formats.extend(self._extract_f4m_formats(
+                        format_url, video_id, f4m_id=format_id))
+                elif determine_ext(format_url) == 'm3u8':
+                    formats.extend(self._extract_m3u8_formats(
+                        format_url, video_id, 'mp4', m3u8_id=format_id))
+                else:
+                    continue
+            self._sort_formats(formats)
+
+            title = remove_end(self._og_search_title(webpage), ' - fm4.ORF.at')
+            if idx >= 1:
+                # Titles are duplicates, make them unique
+                title += ' (' + str(idx + 1) + ')'
+            description = self._og_search_description(webpage)
+            upload_date = unified_strdate(self._html_search_meta(
+                'dc.date', webpage, 'upload date'))
+
+            entries.append({
+                'id': video_id,
+                'title': title,
+                'description': description,
+                'duration': duration,
+                'thumbnail': thumbnail,
+                'upload_date': upload_date,
+                'formats': formats,
+            })
+
+        return self.playlist_result(entries)
-- 
cgit v1.1


From fad9fc537d8ab5141f402f6ccdf60161a8d0302a Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 14 Sep 2017 20:47:23 +0200
Subject: [tv4] fix a test URL

---
 youtube_dl/extractor/tv4.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/tv4.py b/youtube_dl/extractor/tv4.py
index 6487039..cfcce02 100644
--- a/youtube_dl/extractor/tv4.py
+++ b/youtube_dl/extractor/tv4.py
@@ -64,7 +64,7 @@ class TV4IE(InfoExtractor):
             'only_matching': True,
         },
         {
-            'url': ' http://www.tv4play.se/program/farang/3922081',
+            'url': 'http://www.tv4play.se/program/farang/3922081',
             'only_matching': True,
         }
     ]
-- 
cgit v1.1


From c46680fb2a0ee61faa25863d3c10b7b098cdbe67 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 15 Sep 2017 01:59:47 +0700
Subject: [condenast] Fix extraction (closes #14196, closes #14207)

---
 youtube_dl/extractor/condenast.py | 54 +++++++++++++++++++++++----------------
 1 file changed, 32 insertions(+), 22 deletions(-)

diff --git a/youtube_dl/extractor/condenast.py b/youtube_dl/extractor/condenast.py
index 0c3f0c0..ed278fe 100644
--- a/youtube_dl/extractor/condenast.py
+++ b/youtube_dl/extractor/condenast.py
@@ -116,16 +116,16 @@ class CondeNastIE(InfoExtractor):
         entries = [self.url_result(build_url(path), 'CondeNast') for path in paths]
         return self.playlist_result(entries, playlist_title=title)
 
-    def _extract_video_params(self, webpage):
-        query = {}
-        params = self._search_regex(
-            r'(?s)var params = {(.+?)}[;,]', webpage, 'player params', default=None)
-        if params:
-            query.update({
-                'videoId': self._search_regex(r'videoId: [\'"](.+?)[\'"]', params, 'video id'),
-                'playerId': self._search_regex(r'playerId: [\'"](.+?)[\'"]', params, 'player id'),
-                'target': self._search_regex(r'target: [\'"](.+?)[\'"]', params, 'target'),
-            })
+    def _extract_video_params(self, webpage, display_id):
+        query = self._parse_json(
+            self._search_regex(
+                r'(?s)var\s+params\s*=\s*({.+?})[;,]', webpage, 'player params',
+                default='{}'),
+            display_id, transform_source=js_to_json, fatal=False)
+        if query:
+            query['videoId'] = self._search_regex(
+                r'(?:data-video-id=|currentVideoId\s*=\s*)["\']([\da-f]+)',
+                webpage, 'video id', default=None)
         else:
             params = extract_attributes(self._search_regex(
                 r'(<[^>]+data-js="video-player"[^>]+>)',
@@ -141,17 +141,27 @@ class CondeNastIE(InfoExtractor):
         video_id = params['videoId']
 
         video_info = None
-        if params.get('playerId'):
-            info_page = self._download_json(
-                'http://player.cnevids.com/player/video.js',
-                video_id, 'Downloading video info', fatal=False, query=params)
-            if info_page:
-                video_info = info_page.get('video')
-            if not video_info:
-                info_page = self._download_webpage(
-                    'http://player.cnevids.com/player/loader.js',
-                    video_id, 'Downloading loader info', query=params)
-        else:
+
+        # New API path
+        query = params.copy()
+        query['embedType'] = 'inline'
+        info_page = self._download_json(
+            'http://player.cnevids.com/embed-api.json', video_id,
+            'Downloading embed info', fatal=False, query=query)
+
+        # Old fallbacks
+        if not info_page:
+            if params.get('playerId'):
+                info_page = self._download_json(
+                    'http://player.cnevids.com/player/video.js', video_id,
+                    'Downloading video info', fatal=False, query=params)
+        if info_page:
+            video_info = info_page.get('video')
+        if not video_info:
+            info_page = self._download_webpage(
+                'http://player.cnevids.com/player/loader.js',
+                video_id, 'Downloading loader info', query=params)
+        if not video_info:
             info_page = self._download_webpage(
                 'https://player.cnevids.com/inline/video/%s.js' % video_id,
                 video_id, 'Downloading inline info', query={
@@ -215,7 +225,7 @@ class CondeNastIE(InfoExtractor):
         if url_type == 'series':
             return self._extract_series(url, webpage)
         else:
-            params = self._extract_video_params(webpage)
+            params = self._extract_video_params(webpage, display_id)
             info = self._search_json_ld(
                 webpage, display_id, fatal=False)
             info.update(self._extract_video(params))
-- 
cgit v1.1


From 86e55e317cb70f07792cfe543186ad520cbe3230 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 15 Sep 2017 21:45:18 +0700
Subject: [ChangeLog] Actualize

---
 ChangeLog | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index c286da6..38d5113 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,18 @@
+version <unreleased>
+
+Core
+* [downloader/fragment] Restart inconsistent incomplete fragment downloads
+  (#13731)
+* [YoutubeDL] Download raw subtitles files (#12909, #14191)
+
+Extractors
+* [condenast] Fix extraction (#14196, #14207)
++ [orf] Add support for f4m stories
+* [tv4] Relax URL regular expression (#14206)
+* [animeondemand] Bypass geo restriction
++ [animeondemand] Add support for flash videos (#9944)
+
+
 version 2017.09.11
 
 Extractors
-- 
cgit v1.1


From 159d304a9fa2b91d91a60fe3bdf2211a59bcf346 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 15 Sep 2017 21:48:06 +0700
Subject: release 2017.09.15

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 docs/supportedsites.md    | 1 +
 youtube_dl/version.py     | 2 +-
 4 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index f40cb2c..98ab5b6 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.09.11*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.09.11**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.09.15*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.09.15**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through the [README](https://github.com/rg3/youtube-dl/blob/master/README.md), **most notably** the [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ Add the `-v` flag to **your command line** you run youtube-dl with (`youtube-dl
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.09.11
+[debug] youtube-dl version 2017.09.15
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index 38d5113..041dfd7 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.09.15
 
 Core
 * [downloader/fragment] Restart inconsistent incomplete fragment downloads
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 798a81d..6b01dc9 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -593,6 +593,7 @@
  - **Openload**
  - **OraTV**
  - **orf:fm4**: radio FM4
+ - **orf:fm4:story**: fm4.orf.at stories
  - **orf:iptv**: iptv.ORF.at
  - **orf:oe1**: Radio Österreich 1
  - **orf:tvthek**: ORF TVthek
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index cdcb32e..8399c04 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.09.11'
+__version__ = '2017.09.15'
-- 
cgit v1.1


From cbf85239bbb835162725cd4c8758831ca1003445 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 15 Sep 2017 22:13:30 +0700
Subject: [vgtv] Relax _VALID_URL (closes #14223)

---
 youtube_dl/extractor/vgtv.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/vgtv.py b/youtube_dl/extractor/vgtv.py
index 0f8c156..c21a09c 100644
--- a/youtube_dl/extractor/vgtv.py
+++ b/youtube_dl/extractor/vgtv.py
@@ -42,7 +42,7 @@ class VGTVIE(XstreamIE):
                     )
                     /?
                     (?:
-                        \#!/(?:video|live)/|
+                        (?:\#!/)?(?:video|live)/|
                         embed?.*id=|
                         articles/
                     )|
@@ -146,7 +146,11 @@ class VGTVIE(XstreamIE):
         {
             'url': 'abtv:140026',
             'only_matching': True,
-        }
+        },
+        {
+            'url': 'http://www.vgtv.no/video/84196/hevnen-er-soet-episode-10-abu',
+            'only_matching': True,
+        },
     ]
 
     def _real_extract(self, url):
-- 
cgit v1.1


From b763e1d68c6becc414a802a452f5aa819c5de920 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 15 Sep 2017 22:18:38 +0700
Subject: [twitch] Add support for go.twitch.tv URLs (closes #14215)

---
 youtube_dl/extractor/twitch.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index 2daf9df..c926c99 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -28,7 +28,7 @@ from ..utils import (
 
 
 class TwitchBaseIE(InfoExtractor):
-    _VALID_URL_BASE = r'https?://(?:www\.)?twitch\.tv'
+    _VALID_URL_BASE = r'https?://(?:(?:www|go)\.)?twitch\.tv'
 
     _API_BASE = 'https://api.twitch.tv'
     _USHER_BASE = 'https://usher.ttvnw.net'
@@ -217,7 +217,7 @@ class TwitchVodIE(TwitchItemBaseIE):
     _VALID_URL = r'''(?x)
                     https?://
                         (?:
-                            (?:www\.)?twitch\.tv/(?:[^/]+/v|videos)/|
+                            (?:(?:www|go)\.)?twitch\.tv/(?:[^/]+/v|videos)/|
                             player\.twitch\.tv/\?.*?\bvideo=v
                         )
                         (?P<id>\d+)
@@ -458,7 +458,7 @@ class TwitchStreamIE(TwitchBaseIE):
     _VALID_URL = r'''(?x)
                     https?://
                         (?:
-                            (?:www\.)?twitch\.tv/|
+                            (?:(?:www|go)\.)?twitch\.tv/|
                             player\.twitch\.tv/\?.*?\bchannel=
                         )
                         (?P<id>[^/#?]+)
@@ -489,6 +489,9 @@ class TwitchStreamIE(TwitchBaseIE):
     }, {
         'url': 'https://player.twitch.tv/?channel=lotsofs',
         'only_matching': True,
+    }, {
+        'url': 'https://go.twitch.tv/food',
+        'only_matching': True,
     }]
 
     @classmethod
-- 
cgit v1.1


From 6be44a50edfe2e75e31553e7a128ce1849301958 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 15 Sep 2017 22:25:38 +0700
Subject: [dailymotion:playlist] Relax _VALID_URL (closes #14219)

---
 youtube_dl/extractor/dailymotion.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py
index 74e9913..e9d0dd19 100644
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@@ -325,7 +325,7 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
 
 class DailymotionPlaylistIE(DailymotionBaseInfoExtractor):
     IE_NAME = 'dailymotion:playlist'
-    _VALID_URL = r'(?:https?://)?(?:www\.)?dailymotion\.[a-z]{2,3}/playlist/(?P<id>.+?)/'
+    _VALID_URL = r'(?:https?://)?(?:www\.)?dailymotion\.[a-z]{2,3}/playlist/(?P<id>[^/?#&]+)'
     _MORE_PAGES_INDICATOR = r'(?s)<div class="pages[^"]*">.*?<a\s+class="[^"]*?icon-arrow_right[^"]*?"'
     _PAGE_TEMPLATE = 'https://www.dailymotion.com/playlist/%s/%s'
     _TESTS = [{
-- 
cgit v1.1


From a4245acef85ac2414e77cf2cda4cb39adb617241 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 15 Sep 2017 23:12:19 +0700
Subject: [noovo] Fix extraction (closes #14214)

---
 youtube_dl/extractor/noovo.py | 61 +++++++++++++++++++++++--------------------
 1 file changed, 32 insertions(+), 29 deletions(-)

diff --git a/youtube_dl/extractor/noovo.py b/youtube_dl/extractor/noovo.py
index f7fa098..974de3c 100644
--- a/youtube_dl/extractor/noovo.py
+++ b/youtube_dl/extractor/noovo.py
@@ -6,6 +6,7 @@ from .common import InfoExtractor
 from ..compat import compat_str
 from ..utils import (
     int_or_none,
+    js_to_json,
     smuggle_url,
     try_get,
 )
@@ -24,8 +25,6 @@ class NoovoIE(InfoExtractor):
             'timestamp': 1491399228,
             'upload_date': '20170405',
             'uploader_id': '618566855001',
-            'creator': 'vtele',
-            'view_count': int,
             'series': 'RPM+',
         },
         'params': {
@@ -37,13 +36,11 @@ class NoovoIE(InfoExtractor):
         'info_dict': {
             'id': '5395865725001',
             'title': 'Épisode 13 : Les retrouvailles',
-            'description': 'md5:336d5ebc5436534e61d16e63ddfca327',
+            'description': 'md5:888c3330f0c1b4476c5bc99a1c040473',
             'ext': 'mp4',
             'timestamp': 1492019320,
             'upload_date': '20170412',
             'uploader_id': '618566855001',
-            'creator': 'vtele',
-            'view_count': int,
             'series': "L'amour est dans le pré",
             'season_number': 5,
             'episode': 'Épisode 13',
@@ -58,40 +55,46 @@ class NoovoIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        data = self._download_json(
-            'http://api.noovo.ca/api/v1/pages/single-episode/%s' % video_id,
-            video_id)['data']
+        webpage = self._download_webpage(url, video_id)
 
-        content = try_get(data, lambda x: x['contents'][0])
+        bc_url = BrightcoveNewIE._extract_url(self, webpage)
 
-        brightcove_id = data.get('brightcoveId') or content['brightcoveId']
+        data = self._parse_json(
+            self._search_regex(
+                r'(?s)dataLayer\.push\(\s*({.+?})\s*\);', webpage, 'data',
+                default='{}'),
+            video_id, transform_source=js_to_json, fatal=False)
+
+        title = try_get(
+            data, lambda x: x['video']['nom'],
+            compat_str) or self._html_search_meta(
+            'dcterms.Title', webpage, 'title', fatal=True)
+
+        description = self._html_search_meta(
+            ('dcterms.Description', 'description'), webpage, 'description')
 
         series = try_get(
-            data, (
-                lambda x: x['show']['title'],
-                lambda x: x['season']['show']['title']),
-            compat_str)
+            data, lambda x: x['emission']['nom']) or self._search_regex(
+            r'<div[^>]+class="banner-card__subtitle h4"[^>]*>([^<]+)',
+            webpage, 'series', default=None)
 
-        episode = None
-        og = data.get('og')
-        if isinstance(og, dict) and og.get('type') == 'video.episode':
-            episode = og.get('title')
+        season_el = try_get(data, lambda x: x['emission']['saison'], dict) or {}
+        season = try_get(season_el, lambda x: x['nom'], compat_str)
+        season_number = int_or_none(try_get(season_el, lambda x: x['numero']))
 
-        video = content or data
+        episode_el = try_get(season_el, lambda x: x['episode'], dict) or {}
+        episode = try_get(episode_el, lambda x: x['nom'], compat_str)
+        episode_number = int_or_none(try_get(episode_el, lambda x: x['numero']))
 
         return {
             '_type': 'url_transparent',
             'ie_key': BrightcoveNewIE.ie_key(),
-            'url': smuggle_url(
-                self.BRIGHTCOVE_URL_TEMPLATE % brightcove_id,
-                {'geo_countries': ['CA']}),
-            'id': brightcove_id,
-            'title': video.get('title'),
-            'creator': video.get('source'),
-            'view_count': int_or_none(video.get('viewsCount')),
+            'url': smuggle_url(bc_url, {'geo_countries': ['CA']}),
+            'title': title,
+            'description': description,
             'series': series,
-            'season_number': int_or_none(try_get(
-                data, lambda x: x['season']['seasonNumber'])),
+            'season': season,
+            'season_number': season_number,
             'episode': episode,
-            'episode_number': int_or_none(data.get('episodeNumber')),
+            'episode_number': episode_number,
         }
-- 
cgit v1.1


From 68d43a61b552007a718894967b869c0f1d8ff00f Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 16 Sep 2017 12:14:48 +0800
Subject: Ignore TTML subtitles

---
 .gitignore | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitignore b/.gitignore
index a5b585f..fbf7cec 100644
--- a/.gitignore
+++ b/.gitignore
@@ -22,6 +22,7 @@ cover/
 updates_key.pem
 *.egg-info
 *.srt
+*.ttml
 *.sbv
 *.vtt
 *.flv
-- 
cgit v1.1


From 3869028ffb6be6ab719e5cf1004276dfdfd1216d Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 16 Sep 2017 12:18:38 +0800
Subject: [utils] Use bytes-like objects in dfxp2srt

This fixes handling of non-UTF8 TTML subtitles

Closes #14191
---
 ChangeLog                          |  6 ++++++
 test/test_utils.py                 | 26 +++++++++++++++++++++++---
 youtube_dl/postprocessor/ffmpeg.py |  2 +-
 youtube_dl/utils.py                | 18 +++++++++++-------
 4 files changed, 41 insertions(+), 11 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 041dfd7..ba9260e 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,9 @@
+version <unreleased>
+
+Core
+* [utils] Fix handling raw TTML subtitles (#14191)
+
+
 version 2017.09.15
 
 Core
diff --git a/test/test_utils.py b/test/test_utils.py
index e50f376..efa73d0 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -1064,7 +1064,7 @@ ffmpeg version 2.4.4 Copyright (c) 2000-2014 the FFmpeg ...'''), '2.4.4')
                     <p begin="3" dur="-1">Ignored, three</p>
                 </div>
             </body>
-            </tt>'''
+            </tt>'''.encode('utf-8')
         srt_data = '''1
 00:00:00,000 --> 00:00:01,000
 The following line contains Chinese characters and special symbols
@@ -1089,7 +1089,7 @@ Line
                     <p begin="0" end="1">The first line</p>
                 </div>
             </body>
-            </tt>'''
+            </tt>'''.encode('utf-8')
         srt_data = '''1
 00:00:00,000 --> 00:00:01,000
 The first line
@@ -1115,7 +1115,7 @@ The first line
       <p style="s1" tts:textDecoration="underline" begin="00:00:09.56" id="p2" end="00:00:12.36"><span style="s2" tts:color="lime">inner<br /> </span>style</p>
     </div>
   </body>
-</tt>'''
+</tt>'''.encode('utf-8')
         srt_data = '''1
 00:00:02,080 --> 00:00:05,839
 <font color="white" face="sansSerif" size="16">default style<font color="red">custom style</font></font>
@@ -1138,6 +1138,26 @@ part 3</font></u>
 '''
         self.assertEqual(dfxp2srt(dfxp_data_with_style), srt_data)
 
+        dfxp_data_non_utf8 = '''<?xml version="1.0" encoding="UTF-16"?>
+            <tt xmlns="http://www.w3.org/ns/ttml" xml:lang="en" xmlns:tts="http://www.w3.org/ns/ttml#parameter">
+            <body>
+                <div xml:lang="en">
+                    <p begin="0" end="1">Line 1</p>
+                    <p begin="1" end="2">第二行</p>
+                </div>
+            </body>
+            </tt>'''.encode('utf-16')
+        srt_data = '''1
+00:00:00,000 --> 00:00:01,000
+Line 1
+
+2
+00:00:01,000 --> 00:00:02,000
+第二行
+
+'''
+        self.assertEqual(dfxp2srt(dfxp_data_non_utf8), srt_data)
+
     def test_cli_option(self):
         self.assertEqual(cli_option({'proxy': '127.0.0.1:3128'}, '--proxy', 'proxy'), ['--proxy', '127.0.0.1:3128'])
         self.assertEqual(cli_option({'proxy': None}, '--proxy', 'proxy'), [])
diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 51256a3..f71d413 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -585,7 +585,7 @@ class FFmpegSubtitlesConvertorPP(FFmpegPostProcessor):
                 dfxp_file = old_file
                 srt_file = subtitles_filename(filename, lang, 'srt')
 
-                with io.open(dfxp_file, 'rt', encoding='utf-8') as f:
+                with open(dfxp_file, 'rb') as f:
                     srt_data = dfxp2srt(f.read())
 
                 with io.open(srt_file, 'wt', encoding='utf-8') as f:
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 9e4492d..b724e0b 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -2572,14 +2572,18 @@ def srt_subtitles_timecode(seconds):
 
 
 def dfxp2srt(dfxp_data):
+    '''
+    @param dfxp_data A bytes-like object containing DFXP data
+    @returns A unicode object containing converted SRT data
+    '''
     LEGACY_NAMESPACES = (
-        ('http://www.w3.org/ns/ttml', [
-            'http://www.w3.org/2004/11/ttaf1',
-            'http://www.w3.org/2006/04/ttaf1',
-            'http://www.w3.org/2006/10/ttaf1',
+        (b'http://www.w3.org/ns/ttml', [
+            b'http://www.w3.org/2004/11/ttaf1',
+            b'http://www.w3.org/2006/04/ttaf1',
+            b'http://www.w3.org/2006/10/ttaf1',
         ]),
-        ('http://www.w3.org/ns/ttml#styling', [
-            'http://www.w3.org/ns/ttml#style',
+        (b'http://www.w3.org/ns/ttml#styling', [
+            b'http://www.w3.org/ns/ttml#style',
         ]),
     )
 
@@ -2674,7 +2678,7 @@ def dfxp2srt(dfxp_data):
         for ns in v:
             dfxp_data = dfxp_data.replace(ns, k)
 
-    dfxp = compat_etree_fromstring(dfxp_data.encode('utf-8'))
+    dfxp = compat_etree_fromstring(dfxp_data)
     out = []
     paras = dfxp.findall(_x('.//ttml:p')) or dfxp.findall('.//p')
 
-- 
cgit v1.1


From 790d379e4df2f85ece7cab02e805643234bb5c16 Mon Sep 17 00:00:00 2001
From: Windom <windom@users.noreply.github.com>
Date: Sat, 16 Sep 2017 18:39:46 +0300
Subject: [morningstar] Relax _VALID_URL

---
 youtube_dl/extractor/morningstar.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/morningstar.py b/youtube_dl/extractor/morningstar.py
index 320d27b..0093bcd 100644
--- a/youtube_dl/extractor/morningstar.py
+++ b/youtube_dl/extractor/morningstar.py
@@ -8,8 +8,8 @@ from .common import InfoExtractor
 
 class MorningstarIE(InfoExtractor):
     IE_DESC = 'morningstar.com'
-    _VALID_URL = r'https?://(?:www\.)?morningstar\.com/[cC]over/video[cC]enter\.aspx\?id=(?P<id>[0-9]+)'
-    _TEST = {
+    _VALID_URL = r'https?://(?:(?:www|news)\.)morningstar\.com/[cC]over/video[cC]enter\.aspx\?id=(?P<id>[0-9]+)'
+    _TESTS = [{
         'url': 'http://www.morningstar.com/cover/videocenter.aspx?id=615869',
         'md5': '6c0acface7a787aadc8391e4bbf7b0f5',
         'info_dict': {
@@ -19,7 +19,10 @@ class MorningstarIE(InfoExtractor):
             'description': "Vanguard's Joel Dickson on managing higher tax rates for high-income earners and fund capital-gain distributions in 2013.",
             'thumbnail': r're:^https?://.*m(?:orning)?star\.com/.+thumb\.jpg$'
         }
-    }
+    }, {
+        'url': 'http://news.morningstar.com/cover/videocenter.aspx?id=825556',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
-- 
cgit v1.1


From 8251af63a12cd73cf2578c81dbb869232da2592c Mon Sep 17 00:00:00 2001
From: Vijay Singh <sudovijay@users.noreply.github.com>
Date: Sat, 16 Sep 2017 21:15:23 +0530
Subject: [viki] Update app data (closes #14181)

---
 youtube_dl/extractor/viki.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/viki.py b/youtube_dl/extractor/viki.py
index e9c8bf8..853e5c7 100644
--- a/youtube_dl/extractor/viki.py
+++ b/youtube_dl/extractor/viki.py
@@ -23,9 +23,9 @@ class VikiBaseIE(InfoExtractor):
     _API_QUERY_TEMPLATE = '/v4/%sapp=%s&t=%s&site=www.viki.com'
     _API_URL_TEMPLATE = 'http://api.viki.io%s&sig=%s'
 
-    _APP = '65535a'
+    _APP = '100005a'
     _APP_VERSION = '2.2.5.1428709186'
-    _APP_SECRET = '-$iJ}@p7!G@SyU/je1bEyWg}upLu-6V6-Lg9VD(]siH,r.,m-r|ulZ,U4LC/SeR)'
+    _APP_SECRET = 'MM_d*yP@`&1@]@!AVrXf_o-HVEnoTnm$O-ti4[G~$JDI/Dc-&piU&z&5.;:}95=Iad'
 
     _GEO_BYPASS = False
     _NETRC_MACHINE = 'viki'
-- 
cgit v1.1


From 4ed2d7b7d1f67e499a46e507d957616e364565ca Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 17 Sep 2017 13:53:04 +0800
Subject: Fix flake8 issues after #14225

---
 youtube_dl/extractor/common.py |  2 +-
 youtube_dl/utils.py            | 33 +++++++++++++++++----------------
 2 files changed, 18 insertions(+), 17 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 317a9a7..2bbbf8f 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -2452,7 +2452,7 @@ class InfoExtractor(object):
     def _set_cookie(self, domain, name, value, expire_time=None, port=None,
                     path='/', secure=False, discard=False, rest={}, **kwargs):
         cookie = compat_cookiejar.Cookie(
-            0, name, value, port, not port is None, domain, True,
+            0, name, value, port, port is not None, domain, True,
             domain.startswith('.'), path, True, secure, expire_time,
             discard, None, None, rest)
         self._downloader.cookiejar.set_cookie(cookie)
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index b724e0b..acc4f98 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -3830,23 +3830,23 @@ def cookie_to_dict(cookie):
     cookie_dict = {
         'name': cookie.name,
         'value': cookie.value,
-    };
+    }
     if cookie.port_specified:
         cookie_dict['port'] = cookie.port
     if cookie.domain_specified:
         cookie_dict['domain'] = cookie.domain
     if cookie.path_specified:
         cookie_dict['path'] = cookie.path
-    if not cookie.expires is None:
+    if cookie.expires is not None:
         cookie_dict['expires'] = cookie.expires
-    if not cookie.secure is None:
+    if cookie.secure is not None:
         cookie_dict['secure'] = cookie.secure
-    if not cookie.discard is None:
+    if cookie.discard is not None:
         cookie_dict['discard'] = cookie.discard
     try:
         if (cookie.has_nonstandard_attr('httpOnly') or
-            cookie.has_nonstandard_attr('httponly') or
-            cookie.has_nonstandard_attr('HttpOnly')):
+                cookie.has_nonstandard_attr('httponly') or
+                cookie.has_nonstandard_attr('HttpOnly')):
             cookie_dict['httponly'] = True
     except TypeError:
         pass
@@ -3957,7 +3957,7 @@ class PhantomJSwrapper(object):
             cookies = json.loads(f.read().decode('utf-8'))
         for cookie in cookies:
             if cookie['httponly'] is True:
-                cookie['rest'] = { 'httpOnly': None }
+                cookie['rest'] = {'httpOnly': None}
             if 'expiry' in cookie:
                 cookie['expire_time'] = cookie['expiry']
             self.extractor._set_cookie(**cookie)
@@ -3965,7 +3965,7 @@ class PhantomJSwrapper(object):
     def get(self, url, html=None, video_id=None, note=None, note2='Executing JS on webpage', headers={}, jscode='saveAndExit();'):
         """
         Downloads webpage (if needed) and executes JS
-        
+
         Params:
             url: website url
             html: optional, html code of website
@@ -3974,11 +3974,11 @@ class PhantomJSwrapper(object):
             note2: optional, displayed when executing JS
             headers: custom http headers
             jscode: code to be executed when page is loaded
-        
+
         Returns tuple with:
             * downloaded website (after JS execution)
             * anything you print with `console.log` (but not inside `page.execute`!)
-        
+
         In most cases you don't need to add any `jscode`.
         It is executed in `page.onLoadFinished`.
         `saveAndExit();` is mandatory, use it instead of `phantom.exit()`
@@ -3992,7 +3992,7 @@ class PhantomJSwrapper(object):
               else
                 window.setTimeout(check, 500);
             }
-            
+
             page.evaluate(function(){
               document.querySelector('#a').click();
             });
@@ -4024,13 +4024,14 @@ class PhantomJSwrapper(object):
         else:
             self.extractor.to_screen('%s: %s' % (video_id, note2))
 
-        p = subprocess.Popen([self.exe, '--ssl-protocol=any',
-            self._TMP_FILES['script'].name], stdout=subprocess.PIPE,
-            stderr=subprocess.PIPE)
+        p = subprocess.Popen([
+            self.exe, '--ssl-protocol=any',
+            self._TMP_FILES['script'].name
+        ], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
         out, err = p.communicate()
         if p.returncode != 0:
-            raise ExtractorError('Executing JS failed\n:'
-                                 + encodeArgument(err))
+            raise ExtractorError(
+                'Executing JS failed\n:' + encodeArgument(err))
         with open(self._TMP_FILES['html'].name, 'rb') as f:
             html = f.read().decode('utf-8')
 
-- 
cgit v1.1


From 9c2a17f2ce7b2b9dc45b603be413a943f6637498 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 17 Sep 2017 22:19:57 +0700
Subject: [popcorntv] Add extractor (closes #5914, closes #14211)

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/popcorntv.py  | 78 ++++++++++++++++++++++++++++++++++++++
 2 files changed, 79 insertions(+)
 create mode 100644 youtube_dl/extractor/popcorntv.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index a3a97e9..ab95c85 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -808,6 +808,7 @@ from .polskieradio import (
     PolskieRadioIE,
     PolskieRadioCategoryIE,
 )
+from .popcorntv import PopcornTVIE
 from .porn91 import Porn91IE
 from .porncom import PornComIE
 from .pornflip import PornFlipIE
diff --git a/youtube_dl/extractor/popcorntv.py b/youtube_dl/extractor/popcorntv.py
new file mode 100644
index 0000000..ac901f4
--- /dev/null
+++ b/youtube_dl/extractor/popcorntv.py
@@ -0,0 +1,78 @@
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    extract_attributes,
+    int_or_none,
+    unified_timestamp,
+)
+
+
+class PopcornTVIE(InfoExtractor):
+    _VALID_URL = r'https?://[^/]+\.popcorntv\.it/guarda/(?P<display_id>[^/]+)/(?P<id>\d+)'
+    _TESTS = [{
+        'url': 'https://animemanga.popcorntv.it/guarda/food-wars-battaglie-culinarie-episodio-01/9183',
+        'md5': '47d65a48d147caf692ab8562fe630b45',
+        'info_dict': {
+            'id': '9183',
+            'display_id': 'food-wars-battaglie-culinarie-episodio-01',
+            'ext': 'mp4',
+            'title': 'Food Wars, Battaglie Culinarie | Episodio 01',
+            'description': 'md5:b8bea378faae4651d3b34c6e112463d0',
+            'thumbnail': r're:^https?://.*\.jpg$',
+            'timestamp': 1497610857,
+            'upload_date': '20170616',
+            'duration': 1440,
+            'view_count': int,
+        },
+    }, {
+        'url': 'https://cinema.popcorntv.it/guarda/smash-cut/10433',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        display_id, video_id = mobj.group('display_id', 'id')
+
+        webpage = self._download_webpage(url, display_id)
+
+        m3u8_url = extract_attributes(
+            self._search_regex(
+                r'(<link[^>]+itemprop=["\'](?:content|embed)Url[^>]*>)',
+                webpage, 'content'
+            ))['href']
+
+        formats = self._extract_m3u8_formats(
+            m3u8_url, display_id, 'mp4', entry_protocol='m3u8_native',
+            m3u8_id='hls')
+
+        title = self._search_regex(
+            r'<h1[^>]+itemprop=["\']name[^>]*>([^<]+)', webpage,
+            'title', default=None) or self._og_search_title(webpage)
+
+        description = self._html_search_regex(
+            r'(?s)<article[^>]+itemprop=["\']description[^>]*>(.+?)</article>',
+            webpage, 'description', fatal=False)
+        thumbnail = self._og_search_thumbnail(webpage)
+        timestamp = unified_timestamp(self._html_search_meta(
+            'uploadDate', webpage, 'timestamp'))
+        print(self._html_search_meta(
+            'duration', webpage))
+        duration = int_or_none(self._html_search_meta(
+            'duration', webpage), invscale=60)
+        view_count = int_or_none(self._html_search_meta(
+            'interactionCount', webpage, 'view count'))
+
+        return {
+            'id': video_id,
+            'display_id': display_id,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'timestamp': timestamp,
+            'duration': duration,
+            'view_count': view_count,
+            'formats': formats,
+        }
-- 
cgit v1.1


From 4d8c4b46d5668387cc685123f80fb64bbc7c5aff Mon Sep 17 00:00:00 2001
From: kayb94 <30302445+kayb94@users.noreply.github.com>
Date: Sun, 17 Sep 2017 15:46:52 +0000
Subject: [heise] Add support for YouTube embeds

---
 youtube_dl/extractor/heise.py | 33 +++++++++++++++++++++++++++------
 1 file changed, 27 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/heise.py b/youtube_dl/extractor/heise.py
index 382f327..495ffb7 100644
--- a/youtube_dl/extractor/heise.py
+++ b/youtube_dl/extractor/heise.py
@@ -2,6 +2,7 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from .youtube import YoutubeIE
 from ..utils import (
     determine_ext,
     int_or_none,
@@ -26,6 +27,22 @@ class HeiseIE(InfoExtractor):
             'thumbnail': r're:^https?://.*/gallery/$',
         }
     }, {
+        # YouTube embed
+        'url': 'http://www.heise.de/newsticker/meldung/Netflix-In-20-Jahren-vom-Videoverleih-zum-TV-Revolutionaer-3814130.html',
+        'md5': 'e403d2b43fea8e405e88e3f8623909f1',
+        'info_dict': {
+            'id': '6kmWbXleKW4',
+            'ext': 'mp4',
+            'title': 'NEU IM SEPTEMBER | Netflix',
+            'description': 'md5:2131f3c7525e540d5fd841de938bd452',
+            'upload_date': '20170830',
+            'uploader': 'Netflix Deutschland, Österreich und Schweiz',
+            'uploader_id': 'netflixdach',
+        },
+        'params': {
+            'skip_download': True,
+        },
+    }, {
         'url': 'http://www.heise.de/ct/artikel/c-t-uplink-3-3-Owncloud-Tastaturen-Peilsender-Smartphone-2403911.html',
         'only_matching': True,
     }, {
@@ -40,6 +57,16 @@ class HeiseIE(InfoExtractor):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
+        title = self._html_search_meta('fulltitle', webpage, default=None)
+        if not title or title == "c't":
+            title = self._search_regex(
+                    r'<div[^>]+class="videoplayerjw"[^>]+data-title="([^"]+)"',
+                    webpage, 'title')
+
+        yt_urls = YoutubeIE._extract_urls(webpage)
+        if yt_urls:
+            return self.playlist_from_matches(yt_urls, video_id, title, ie=YoutubeIE.ie_key())
+
         container_id = self._search_regex(
             r'<div class="videoplayerjw"[^>]+data-container="([0-9]+)"',
             webpage, 'container ID')
@@ -47,12 +74,6 @@ class HeiseIE(InfoExtractor):
             r'<div class="videoplayerjw"[^>]+data-sequenz="([0-9]+)"',
             webpage, 'sequenz ID')
 
-        title = self._html_search_meta('fulltitle', webpage, default=None)
-        if not title or title == "c't":
-            title = self._search_regex(
-                r'<div[^>]+class="videoplayerjw"[^>]+data-title="([^"]+)"',
-                webpage, 'title')
-
         doc = self._download_xml(
             'http://www.heise.de/videout/feed', video_id, query={
                 'container': container_id,
-- 
cgit v1.1


From 8a1a60d17397721620e75d83f2aad3a353286f15 Mon Sep 17 00:00:00 2001
From: Kareem Moussa <itstehkman@gmail.com>
Date: Tue, 19 Sep 2017 08:51:20 -0700
Subject: [devscripts/check-porn] Fix gettestcases import

---
 devscripts/check-porn.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/devscripts/check-porn.py b/devscripts/check-porn.py
index 7a219eb..72b2ee4 100644
--- a/devscripts/check-porn.py
+++ b/devscripts/check-porn.py
@@ -14,7 +14,7 @@ import os
 import sys
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
-from test.helper import get_testcases
+from test.helper import gettestcases
 from youtube_dl.utils import compat_urllib_parse_urlparse
 from youtube_dl.utils import compat_urllib_request
 
@@ -24,7 +24,7 @@ if len(sys.argv) > 1:
 else:
     METHOD = 'EURISTIC'
 
-for test in get_testcases():
+for test in gettestcases():
     if METHOD == 'EURISTIC':
         try:
             webpage = compat_urllib_request.urlopen(test['url'], timeout=10).read()
-- 
cgit v1.1


From dc76eef092dc10f7e3f599fa7d85a04de8d84b8f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 19 Sep 2017 23:59:36 +0700
Subject: [tvplay] Bypass geo restriction

---
 youtube_dl/extractor/tvplay.py | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/tvplay.py b/youtube_dl/extractor/tvplay.py
index 99ff82a..46132ed 100644
--- a/youtube_dl/extractor/tvplay.py
+++ b/youtube_dl/extractor/tvplay.py
@@ -15,7 +15,9 @@ from ..utils import (
     int_or_none,
     parse_iso8601,
     qualities,
+    smuggle_url,
     try_get,
+    unsmuggle_url,
     update_url_query,
 )
 
@@ -224,6 +226,9 @@ class TVPlayIE(InfoExtractor):
     ]
 
     def _real_extract(self, url):
+        url, smuggled_data = unsmuggle_url(url, {})
+        self._initialize_geo_bypass(smuggled_data.get('geo_countries'))
+
         video_id = self._match_id(url)
         geo_country = self._search_regex(
             r'https?://[^/]+\.([a-z]{2})', url,
@@ -426,4 +431,9 @@ class ViafreeIE(InfoExtractor):
                 r'currentVideo["\']\s*:\s*.+?["\']id["\']\s*:\s*["\'](\d{6,})',
                 webpage, 'video id')
 
-        return self.url_result('mtg:%s' % video_id, TVPlayIE.ie_key())
+        return self.url_result(
+            smuggle_url(
+                'mtg:%s' % video_id,
+                {'geo_countries': [
+                    compat_urlparse.urlparse(url).netloc.rsplit('.', 1)[-1]]}),
+            ie=TVPlayIE.ie_key(), video_id=video_id)
-- 
cgit v1.1


From 3b65a6fbf31256030ff35210a7be2b50369a6c4f Mon Sep 17 00:00:00 2001
From: capital-G <dennis.scheiba@gmx.de>
Date: Tue, 19 Sep 2017 22:58:06 +0200
Subject: [twitter] Fix duration extraction

---
 youtube_dl/extractor/twitter.py | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/twitter.py b/youtube_dl/extractor/twitter.py
index 6eaf360..7399cf5 100644
--- a/youtube_dl/extractor/twitter.py
+++ b/youtube_dl/extractor/twitter.py
@@ -229,7 +229,7 @@ class TwitterCardIE(TwitterBaseIE):
 
         title = self._search_regex(r'<title>([^<]+)</title>', webpage, 'title')
         thumbnail = config.get('posterImageUrl') or config.get('image_src')
-        duration = float_or_none(config.get('duration')) or duration
+        duration = float_or_none(config.get('duration'), scale=1000) or duration
 
         return {
             'id': video_id,
@@ -255,6 +255,7 @@ class TwitterIE(InfoExtractor):
             'description': 'FREE THE NIPPLE on Twitter: "FTN supporters on Hollywood Blvd today! http://t.co/c7jHH749xJ"',
             'uploader': 'FREE THE NIPPLE',
             'uploader_id': 'freethenipple',
+            'duration': 12.922,
         },
         'params': {
             'skip_download': True,  # requires ffmpeg
@@ -305,11 +306,12 @@ class TwitterIE(InfoExtractor):
         'info_dict': {
             'id': '700207533655363584',
             'ext': 'mp4',
-            'title': 'Donte - BEAT PROD: @suhmeduh #Damndaniel',
-            'description': 'Donte on Twitter: "BEAT PROD: @suhmeduh  https://t.co/HBrQ4AfpvZ #Damndaniel https://t.co/byBooq2ejZ"',
+            'title': 'あかさ - BEAT PROD: @suhmeduh #Damndaniel',
+            'description': 'あかさ on Twitter: "BEAT PROD: @suhmeduh  https://t.co/HBrQ4AfpvZ #Damndaniel https://t.co/byBooq2ejZ"',
             'thumbnail': r're:^https?://.*\.jpg',
-            'uploader': 'Donte',
+            'uploader': 'あかさ',
             'uploader_id': 'jaydingeer',
+            'duration': 30.0,
         },
         'params': {
             'skip_download': True,  # requires ffmpeg
@@ -337,6 +339,7 @@ class TwitterIE(InfoExtractor):
             'description': 'Captain America on Twitter: "@King0fNerd Are you sure you made the right choice? Find out in theaters. https://t.co/GpgYi9xMJI"',
             'uploader_id': 'captainamerica',
             'uploader': 'Captain America',
+            'duration': 3.17,
         },
         'params': {
             'skip_download': True,  # requires ffmpeg
@@ -364,6 +367,7 @@ class TwitterIE(InfoExtractor):
             'description': 'عالم الأخبار on Twitter: "كلمة تاريخية بجلسة الجناسي التاريخية.. النائب خالد مؤنس العتيبي للمعارضين : اتقوا الله .. الظلم ظلمات يوم القيامة   https://t.co/xg6OhpyKfN"',
             'uploader': 'عالم الأخبار',
             'uploader_id': 'news_al3alm',
+            'duration': 277.4,
         },
         'params': {
             'format': 'best[format_id^=http-]',
-- 
cgit v1.1


From 12ea5c79fb0bfa878d62d130cf67057fc230dfa7 Mon Sep 17 00:00:00 2001
From: Parmjit Virk <pvirk@mts.net>
Date: Wed, 20 Sep 2017 14:53:06 -0500
Subject: [nbcsports:vplayer] Correct theplatform URL (closes #13873)

---
 youtube_dl/extractor/nbc.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/nbc.py b/youtube_dl/extractor/nbc.py
index 62db70b..836a41f 100644
--- a/youtube_dl/extractor/nbc.py
+++ b/youtube_dl/extractor/nbc.py
@@ -109,10 +109,10 @@ class NBCSportsVPlayerIE(InfoExtractor):
     _VALID_URL = r'https?://vplayer\.nbcsports\.com/(?:[^/]+/)+(?P<id>[0-9a-zA-Z_]+)'
 
     _TESTS = [{
-        'url': 'https://vplayer.nbcsports.com/p/BxmELC/nbcsports_share/select/9CsDKds0kvHI',
+        'url': 'https://vplayer.nbcsports.com/p/BxmELC/nbcsports_embed/select/9CsDKds0kvHI',
         'info_dict': {
             'id': '9CsDKds0kvHI',
-            'ext': 'flv',
+            'ext': 'mp4',
             'description': 'md5:df390f70a9ba7c95ff1daace988f0d8d',
             'title': 'Tyler Kalinoski hits buzzer-beater to lift Davidson',
             'timestamp': 1426270238,
@@ -120,7 +120,7 @@ class NBCSportsVPlayerIE(InfoExtractor):
             'uploader': 'NBCU-SPORTS',
         }
     }, {
-        'url': 'http://vplayer.nbcsports.com/p/BxmELC/nbc_embedshare/select/_hqLjQ95yx8Z',
+        'url': 'https://vplayer.nbcsports.com/p/BxmELC/nbcsports_embed/select/media/_hqLjQ95yx8Z',
         'only_matching': True,
     }]
 
@@ -134,7 +134,8 @@ class NBCSportsVPlayerIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
-        theplatform_url = self._og_search_video_url(webpage)
+        theplatform_url = self._og_search_video_url(webpage).replace(
+            'vplayer.nbcsports.com', 'player.theplatform.com')
         return self.url_result(theplatform_url, 'ThePlatform')
 
 
-- 
cgit v1.1


From f6ff52b473c9ed969fadb3e3d50852c4a27ba17e Mon Sep 17 00:00:00 2001
From: Giuseppe Fabiano <gfabiano40@gmail.com>
Date: Wed, 20 Sep 2017 23:05:33 +0200
Subject: [beeg] Fix extraction (closes #14275)

---
 youtube_dl/extractor/beeg.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/beeg.py b/youtube_dl/extractor/beeg.py
index d5c5822..bbeae4b 100644
--- a/youtube_dl/extractor/beeg.py
+++ b/youtube_dl/extractor/beeg.py
@@ -9,6 +9,7 @@ from ..compat import (
 from ..utils import (
     int_or_none,
     parse_iso8601,
+    urljoin,
 )
 
 
@@ -36,9 +37,11 @@ class BeegIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
 
         cpl_url = self._search_regex(
-            r'<script[^>]+src=(["\'])(?P<url>(?:https?:)?//static\.beeg\.com/cpl/\d+\.js.*?)\1',
+            r'<script[^>]+src=(["\'])(?P<url>(?:/static|(?:https?:)?//static\.beeg\.com)/cpl/\d+\.js.*?)\1',
             webpage, 'cpl', default=None, group='url')
 
+        cpl_url = urljoin(url, cpl_url)
+
         beeg_version, beeg_salt = [None] * 2
 
         if cpl_url:
@@ -54,7 +57,7 @@ class BeegIE(InfoExtractor):
                     r'beeg_salt\s*=\s*(["\'])(?P<beeg_salt>.+?)\1', cpl, 'beeg salt',
                     default=None, group='beeg_salt')
 
-        beeg_version = beeg_version or '2000'
+        beeg_version = beeg_version or '2185'
         beeg_salt = beeg_salt or 'pmweAkq8lAYKdfWcFCUj0yoVgoPlinamH5UE1CB3H'
 
         video = self._download_json(
-- 
cgit v1.1


From 8c6919e4331e1cd44f50e700e8fc4e630d913a2f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 21 Sep 2017 23:00:35 +0700
Subject: [lynda] Add support for educourse.ga (closes #14286)

---
 youtube_dl/extractor/lynda.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/lynda.py b/youtube_dl/extractor/lynda.py
index d2f7529..1b6f509 100644
--- a/youtube_dl/extractor/lynda.py
+++ b/youtube_dl/extractor/lynda.py
@@ -94,7 +94,7 @@ class LyndaBaseIE(InfoExtractor):
 class LyndaIE(LyndaBaseIE):
     IE_NAME = 'lynda'
     IE_DESC = 'lynda.com videos'
-    _VALID_URL = r'https?://(?:www\.)?lynda\.com/(?:[^/]+/[^/]+/(?P<course_id>\d+)|player/embed)/(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?(?:lynda\.com|educourse\.ga)/(?:[^/]+/[^/]+/(?P<course_id>\d+)|player/embed)/(?P<id>\d+)'
 
     _TIMECODE_REGEX = r'\[(?P<timecode>\d+:\d+:\d+[\.,]\d+)\]'
 
@@ -110,6 +110,9 @@ class LyndaIE(LyndaBaseIE):
     }, {
         'url': 'https://www.lynda.com/player/embed/133770?tr=foo=1;bar=g;fizz=rt&fs=0',
         'only_matching': True,
+    }, {
+        'url': 'https://educourse.ga/Bootstrap-tutorials/Using-exercise-files/110885/114408-4.html',
+        'only_matching': True,
     }]
 
     def _raise_unavailable(self, video_id):
@@ -253,7 +256,7 @@ class LyndaCourseIE(LyndaBaseIE):
 
     # Course link equals to welcome/introduction video link of same course
     # We will recognize it as course link
-    _VALID_URL = r'https?://(?:www|m)\.lynda\.com/(?P<coursepath>[^/]+/[^/]+/(?P<courseid>\d+))-\d\.html'
+    _VALID_URL = r'https?://(?:www|m)\.(?:lynda\.com|educourse\.ga)/(?P<coursepath>[^/]+/[^/]+/(?P<courseid>\d+))-\d\.html'
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
-- 
cgit v1.1


From 8c2895305dc09920055611c8120f5a65fcd2614f Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 23 Sep 2017 02:30:03 +0800
Subject: [options] Accept lrc as a subtitle conversion target format (closes
 #14292)

---
 ChangeLog              | 1 +
 youtube_dl/__init__.py | 2 +-
 youtube_dl/options.py  | 2 +-
 3 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index ba9260e..42ba879 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,6 +1,7 @@
 version <unreleased>
 
 Core
++ [options] Accept lrc as a subtitle conversion target format (#14292)
 * [utils] Fix handling raw TTML subtitles (#14191)
 
 
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index c458941..ba684a0 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -206,7 +206,7 @@ def _real_main(argv=None):
         if opts.recodevideo not in ['mp4', 'flv', 'webm', 'ogg', 'mkv', 'avi']:
             parser.error('invalid video recode format specified')
     if opts.convertsubtitles is not None:
-        if opts.convertsubtitles not in ['srt', 'vtt', 'ass']:
+        if opts.convertsubtitles not in ['srt', 'vtt', 'ass', 'lrc']:
             parser.error('invalid subtitle format specified')
 
     if opts.date is not None:
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 38439c9..4c04550 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -847,7 +847,7 @@ def parseOpts(overrideArguments=None):
     postproc.add_option(
         '--convert-subs', '--convert-subtitles',
         metavar='FORMAT', dest='convertsubtitles', default=None,
-        help='Convert the subtitles to other format (currently supported: srt|ass|vtt)')
+        help='Convert the subtitles to other format (currently supported: srt|ass|vtt|lrc)')
 
     parser.add_option_group(general)
     parser.add_option_group(network)
-- 
cgit v1.1


From 2384f5a64e501d7abb844e8d31fe340b34d8d4e7 Mon Sep 17 00:00:00 2001
From: Tatsuyuki Ishi <ishitatsuyuki@gmail.com>
Date: Wed, 6 Sep 2017 11:24:34 +0900
Subject: [mixcloud] Fix extraction (closes #14088)

---
 youtube_dl/compat.py             |  10 ++-
 youtube_dl/extractor/mixcloud.py | 167 ++++++++++++++++++++++++---------------
 2 files changed, 114 insertions(+), 63 deletions(-)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 9e4e13b..2a62248 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -6,6 +6,7 @@ import collections
 import email
 import getpass
 import io
+import itertools
 import optparse
 import os
 import re
@@ -15,7 +16,6 @@ import socket
 import struct
 import subprocess
 import sys
-import itertools
 import xml.etree.ElementTree
 
 
@@ -2898,6 +2898,13 @@ else:
     compat_struct_pack = struct.pack
     compat_struct_unpack = struct.unpack
 
+try:
+    from future_builtins import zip as compat_zip
+except ImportError:  # not 2.6+ or is 3.x
+    try:
+        from itertools import izip as compat_zip  # < 2.5 or 3.x
+    except ImportError:
+        compat_zip = zip
 
 __all__ = [
     'compat_HTMLParseError',
@@ -2948,5 +2955,6 @@ __all__ = [
     'compat_urlretrieve',
     'compat_xml_parse_error',
     'compat_xpath',
+    'compat_zip',
     'workaround_optparse_bug9161',
 ]
diff --git a/youtube_dl/extractor/mixcloud.py b/youtube_dl/extractor/mixcloud.py
index f6360cc..4811823 100644
--- a/youtube_dl/extractor/mixcloud.py
+++ b/youtube_dl/extractor/mixcloud.py
@@ -9,16 +9,16 @@ from .common import InfoExtractor
 from ..compat import (
     compat_chr,
     compat_ord,
-    compat_str,
     compat_urllib_parse_unquote,
     compat_urlparse,
+    compat_zip
 )
 from ..utils import (
     clean_html,
     ExtractorError,
     OnDemandPagedList,
     str_to_int,
-)
+    try_get)
 
 
 class MixcloudIE(InfoExtractor):
@@ -54,27 +54,19 @@ class MixcloudIE(InfoExtractor):
         'only_matching': True,
     }]
 
-    _keys = [
-        'return { requestAnimationFrame: function(callback) { callback(); }, innerHeight: 500 };',
-        'pleasedontdownloadourmusictheartistswontgetpaid',
-        'window.addEventListener = window.addEventListener || function() {};',
-        '(function() { return new Date().toLocaleDateString(); })()'
-    ]
-    _current_key = None
-
-    # See https://www.mixcloud.com/media/js2/www_js_2.9e23256562c080482435196ca3975ab5.js
-    def _decrypt_play_info(self, play_info, video_id):
-        play_info = base64.b64decode(play_info.encode('ascii'))
-        for num, key in enumerate(self._keys, start=1):
-            try:
-                return self._parse_json(
-                    ''.join([
-                        compat_chr(compat_ord(ch) ^ compat_ord(key[idx % len(key)]))
-                        for idx, ch in enumerate(play_info)]),
-                    video_id)
-            except ExtractorError:
-                if num == len(self._keys):
-                    raise
+    @staticmethod
+    def _decrypt_xor_cipher(key, ciphertext):
+        """Encrypt/Decrypt XOR cipher. Both ways are possible because it's XOR."""
+        return ''.join([
+            compat_chr(compat_ord(ch) ^ compat_ord(k))
+            for ch, k in compat_zip(ciphertext, itertools.cycle(key))])
+
+    @staticmethod
+    def _decrypt_and_extend(stream_info, url_key, getter, key, formats):
+        maybe_url = stream_info.get(url_key)
+        if maybe_url is not None:
+            decrypted = MixcloudIE._decrypt_xor_cipher(key, base64.b64decode(maybe_url))
+            formats.extend(getter(decrypted))
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
@@ -84,54 +76,105 @@ class MixcloudIE(InfoExtractor):
 
         webpage = self._download_webpage(url, track_id)
 
-        if not self._current_key:
-            js_url = self._search_regex(
-                r'<script[^>]+\bsrc=["\"](https://(?:www\.)?mixcloud\.com/media/js2/www_js_4\.[^>]+\.js)',
-                webpage, 'js url', default=None)
-            if js_url:
-                js = self._download_webpage(js_url, track_id, fatal=False)
-                if js:
-                    KEY_RE_TEMPLATE = r'player\s*:\s*{.*?\b%s\s*:\s*(["\'])(?P<key>(?:(?!\1).)+)\1'
-                    for key_name in ('value', 'key_value', 'key_value.*?', '.*?value.*?'):
-                        key = self._search_regex(
-                            KEY_RE_TEMPLATE % key_name, js, 'key',
-                            default=None, group='key')
-                        if key and isinstance(key, compat_str):
-                            self._keys.insert(0, key)
-                            self._current_key = key
+        # Legacy path
+        encrypted_play_info = self._search_regex(
+            r'm-play-info="([^"]+)"', webpage, 'play info', default=None)
+
+        if encrypted_play_info is not None:
+            # Decode
+            encrypted_play_info = base64.b64decode(encrypted_play_info)
+        else:
+            # New path
+            full_info_json = self._parse_json(self._html_search_regex(
+                r'<script id="relay-data" type="text/x-mixcloud">([^<]+)</script>', webpage, 'play info'), 'play info')
+            for item in full_info_json:
+                item_data = try_get(item, lambda x: x['cloudcast']['data']['cloudcastLookup'])
+                if try_get(item_data, lambda x: x['streamInfo']['url']):
+                    info_json = item_data
+                    break
+            else:
+                raise ExtractorError('Failed to extract matching stream info')
 
         message = self._html_search_regex(
             r'(?s)<div[^>]+class="global-message cloudcast-disabled-notice-light"[^>]*>(.+?)<(?:a|/div)',
             webpage, 'error message', default=None)
 
-        encrypted_play_info = self._search_regex(
-            r'm-play-info="([^"]+)"', webpage, 'play info')
-
-        play_info = self._decrypt_play_info(encrypted_play_info, track_id)
-
-        if message and 'stream_url' not in play_info:
-            raise ExtractorError('%s said: %s' % (self.IE_NAME, message), expected=True)
-
-        song_url = play_info['stream_url']
-
-        title = self._html_search_regex(r'm-title="([^"]+)"', webpage, 'title')
-        thumbnail = self._proto_relative_url(self._html_search_regex(
-            r'm-thumbnail-url="([^"]+)"', webpage, 'thumbnail', fatal=False))
-        uploader = self._html_search_regex(
-            r'm-owner-name="([^"]+)"', webpage, 'uploader', fatal=False)
-        uploader_id = self._search_regex(
-            r'\s+"profile": "([^"]+)",', webpage, 'uploader id', fatal=False)
-        description = self._og_search_description(webpage)
-        view_count = str_to_int(self._search_regex(
-            [r'<meta itemprop="interactionCount" content="UserPlays:([0-9]+)"',
-             r'/listeners/?">([0-9,.]+)</a>',
-             r'(?:m|data)-tooltip=["\']([\d,.]+) plays'],
-            webpage, 'play count', default=None))
+        js_url = self._search_regex(
+            r'<script[^>]+\bsrc=["\"](https://(?:www\.)?mixcloud\.com/media/js2/www_js_4\.[^>]+\.js)',
+            webpage, 'js url', default=None)
+        if js_url is None:
+            js_url = self._search_regex(
+                r'<script[^>]+\bsrc=["\"](https://(?:www\.)?mixcloud\.com/media/js/www\.[^>]+\.js)',
+                webpage, 'js url')
+        js = self._download_webpage(js_url, track_id)
+        # Known plaintext attack
+        if encrypted_play_info:
+            kps = ['{"stream_url":']
+            kpa_target = encrypted_play_info
+        else:
+            kps = ['https://', 'http://']
+            kpa_target = base64.b64decode(info_json['streamInfo']['url'])
+        for kp in kps:
+            partial_key = self._decrypt_xor_cipher(kpa_target, kp)
+            for quote in ["'", '"']:
+                key = self._search_regex(r'{0}({1}[^{0}]*){0}'.format(quote, re.escape(partial_key)), js,
+                                         "encryption key", default=None)
+                if key is not None:
+                    break
+            else:
+                continue
+            break
+        else:
+            raise ExtractorError('Failed to extract encryption key')
+
+        if encrypted_play_info is not None:
+            play_info = self._parse_json(self._decrypt_xor_cipher(key, encrypted_play_info), 'play info')
+            if message and 'stream_url' not in play_info:
+                raise ExtractorError('%s said: %s' % (self.IE_NAME, message), expected=True)
+            song_url = play_info['stream_url']
+            formats = [{
+                'format_id': 'normal',
+                'url': song_url
+            }]
+
+            title = self._html_search_regex(r'm-title="([^"]+)"', webpage, 'title')
+            thumbnail = self._proto_relative_url(self._html_search_regex(
+                r'm-thumbnail-url="([^"]+)"', webpage, 'thumbnail', fatal=False))
+            uploader = self._html_search_regex(
+                r'm-owner-name="([^"]+)"', webpage, 'uploader', fatal=False)
+            uploader_id = self._search_regex(
+                r'\s+"profile": "([^"]+)",', webpage, 'uploader id', fatal=False)
+            description = self._og_search_description(webpage)
+            view_count = str_to_int(self._search_regex(
+                [r'<meta itemprop="interactionCount" content="UserPlays:([0-9]+)"',
+                 r'/listeners/?">([0-9,.]+)</a>',
+                 r'(?:m|data)-tooltip=["\']([\d,.]+) plays'],
+                webpage, 'play count', default=None))
+
+        else:
+            title = info_json['name']
+            thumbnail = try_get(info_json,
+                                lambda x: 'https://thumbnailer.mixcloud.com/unsafe/600x600/' + x['picture']['urlRoot'])
+            uploader = try_get(info_json, lambda x: x['owner']['displayName'])
+            uploader_id = try_get(info_json, lambda x: x['owner']['username'])
+            description = try_get(info_json, lambda x: x['description'])
+            view_count = try_get(info_json, lambda x: x['plays'])
+
+            stream_info = info_json['streamInfo']
+            formats = []
+            self._decrypt_and_extend(stream_info, 'url', lambda x: [{
+                'format_id': 'normal',
+                'url': x
+            }], key, formats)
+            self._decrypt_and_extend(stream_info, 'hlsUrl', lambda x: self._extract_m3u8_formats(x, title), key,
+                                     formats)
+            self._decrypt_and_extend(stream_info, 'dashUrl', lambda x: self._extract_mpd_formats(x, title), key,
+                                     formats)
 
         return {
             'id': track_id,
             'title': title,
-            'url': song_url,
+            'formats': formats,
             'description': description,
             'thumbnail': thumbnail,
             'uploader': uploader,
-- 
cgit v1.1


From 095774e59130c999ed8ce132f80a7164c5ee39a7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 23 Sep 2017 05:35:55 +0700
Subject: [mixcloud] Improve and simplify (closes #14132)

---
 youtube_dl/extractor/mixcloud.py | 71 +++++++++++++++++++++++-----------------
 1 file changed, 41 insertions(+), 30 deletions(-)

diff --git a/youtube_dl/extractor/mixcloud.py b/youtube_dl/extractor/mixcloud.py
index 4811823..f331db8 100644
--- a/youtube_dl/extractor/mixcloud.py
+++ b/youtube_dl/extractor/mixcloud.py
@@ -9,6 +9,7 @@ from .common import InfoExtractor
 from ..compat import (
     compat_chr,
     compat_ord,
+    compat_str,
     compat_urllib_parse_unquote,
     compat_urlparse,
     compat_zip
@@ -16,9 +17,12 @@ from ..compat import (
 from ..utils import (
     clean_html,
     ExtractorError,
+    int_or_none,
     OnDemandPagedList,
     str_to_int,
-    try_get)
+    try_get,
+    urljoin,
+)
 
 
 class MixcloudIE(InfoExtractor):
@@ -61,13 +65,6 @@ class MixcloudIE(InfoExtractor):
             compat_chr(compat_ord(ch) ^ compat_ord(k))
             for ch, k in compat_zip(ciphertext, itertools.cycle(key))])
 
-    @staticmethod
-    def _decrypt_and_extend(stream_info, url_key, getter, key, formats):
-        maybe_url = stream_info.get(url_key)
-        if maybe_url is not None:
-            decrypted = MixcloudIE._decrypt_xor_cipher(key, base64.b64decode(maybe_url))
-            formats.extend(getter(decrypted))
-
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         uploader = mobj.group(1)
@@ -86,9 +83,12 @@ class MixcloudIE(InfoExtractor):
         else:
             # New path
             full_info_json = self._parse_json(self._html_search_regex(
-                r'<script id="relay-data" type="text/x-mixcloud">([^<]+)</script>', webpage, 'play info'), 'play info')
+                r'<script id="relay-data" type="text/x-mixcloud">([^<]+)</script>',
+                webpage, 'play info'), 'play info')
             for item in full_info_json:
-                item_data = try_get(item, lambda x: x['cloudcast']['data']['cloudcastLookup'])
+                item_data = try_get(
+                    item, lambda x: x['cloudcast']['data']['cloudcastLookup'],
+                    dict)
                 if try_get(item_data, lambda x: x['streamInfo']['url']):
                     info_json = item_data
                     break
@@ -100,13 +100,9 @@ class MixcloudIE(InfoExtractor):
             webpage, 'error message', default=None)
 
         js_url = self._search_regex(
-            r'<script[^>]+\bsrc=["\"](https://(?:www\.)?mixcloud\.com/media/js2/www_js_4\.[^>]+\.js)',
-            webpage, 'js url', default=None)
-        if js_url is None:
-            js_url = self._search_regex(
-                r'<script[^>]+\bsrc=["\"](https://(?:www\.)?mixcloud\.com/media/js/www\.[^>]+\.js)',
-                webpage, 'js url')
-        js = self._download_webpage(js_url, track_id)
+            r'<script[^>]+\bsrc=["\"](https://(?:www\.)?mixcloud\.com/media/(?:js2/www_js_4|js/www)\.[^>]+\.js)',
+            webpage, 'js url')
+        js = self._download_webpage(js_url, track_id, 'Downloading JS')
         # Known plaintext attack
         if encrypted_play_info:
             kps = ['{"stream_url":']
@@ -117,8 +113,9 @@ class MixcloudIE(InfoExtractor):
         for kp in kps:
             partial_key = self._decrypt_xor_cipher(kpa_target, kp)
             for quote in ["'", '"']:
-                key = self._search_regex(r'{0}({1}[^{0}]*){0}'.format(quote, re.escape(partial_key)), js,
-                                         "encryption key", default=None)
+                key = self._search_regex(
+                    r'{0}({1}[^{0}]*){0}'.format(quote, re.escape(partial_key)),
+                    js, 'encryption key', default=None)
                 if key is not None:
                     break
             else:
@@ -153,23 +150,37 @@ class MixcloudIE(InfoExtractor):
 
         else:
             title = info_json['name']
-            thumbnail = try_get(info_json,
-                                lambda x: 'https://thumbnailer.mixcloud.com/unsafe/600x600/' + x['picture']['urlRoot'])
+            thumbnail = urljoin(
+                'https://thumbnailer.mixcloud.com/unsafe/600x600/',
+                try_get(info_json, lambda x: x['picture']['urlRoot'], compat_str))
             uploader = try_get(info_json, lambda x: x['owner']['displayName'])
             uploader_id = try_get(info_json, lambda x: x['owner']['username'])
             description = try_get(info_json, lambda x: x['description'])
-            view_count = try_get(info_json, lambda x: x['plays'])
+            view_count = int_or_none(try_get(info_json, lambda x: x['plays']))
 
             stream_info = info_json['streamInfo']
             formats = []
-            self._decrypt_and_extend(stream_info, 'url', lambda x: [{
-                'format_id': 'normal',
-                'url': x
-            }], key, formats)
-            self._decrypt_and_extend(stream_info, 'hlsUrl', lambda x: self._extract_m3u8_formats(x, title), key,
-                                     formats)
-            self._decrypt_and_extend(stream_info, 'dashUrl', lambda x: self._extract_mpd_formats(x, title), key,
-                                     formats)
+
+            for url_key in ('url', 'hlsUrl', 'dashUrl'):
+                format_url = stream_info.get(url_key)
+                if not format_url:
+                    continue
+                decrypted = self._decrypt_xor_cipher(key, base64.b64decode(format_url))
+                if not decrypted:
+                    continue
+                if url_key == 'hlsUrl':
+                    formats.extend(self._extract_m3u8_formats(
+                        decrypted, track_id, 'mp4', entry_protocol='m3u8_native',
+                        m3u8_id='hls', fatal=False))
+                elif url_key == 'dashUrl':
+                    formats.extend(self._extract_mpd_formats(
+                        decrypted, track_id, mpd_id='dash', fatal=False))
+                else:
+                    formats.append({
+                        'format_id': 'http',
+                        'url': decrypted,
+                    })
+            self._sort_formats(formats)
 
         return {
             'id': track_id,
-- 
cgit v1.1


From 9ce1ac404648142139f8b231c674d434ad4f9ffe Mon Sep 17 00:00:00 2001
From: kayb94 <30302445+kayb94@users.noreply.github.com>
Date: Fri, 22 Sep 2017 22:49:48 +0000
Subject: [generic] Fix support for multiple HTML5 videos on one page (closes
 #14080)

---
 youtube_dl/extractor/generic.py | 22 +++++++++++++++++++---
 1 file changed, 19 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index b83c183..7d0edf0 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1879,6 +1879,15 @@ class GenericIE(InfoExtractor):
                 'title': 'Building A Business Online: Principal Chairs Q & A',
             },
         },
+        {
+            # multiple HTML5 videos on one page
+            'url': 'https://www.paragon-software.com/home/rk-free/keyscenarios.html',
+            'info_dict': {
+                'id': 'keyscenarios',
+                'title': 'Rescue Kit 14 Free Edition - Getting started',
+            },
+            'playlist_count': 4,
+        }
         # {
         #     # TODO: find another test
         #     # http://schema.org/VideoObject
@@ -2849,13 +2858,20 @@ class GenericIE(InfoExtractor):
         # Look for HTML5 media
         entries = self._parse_html5_media_entries(url, webpage, video_id, m3u8_id='hls')
         if entries:
-            for entry in entries:
-                entry.update({
+            if len(entries) == 1:
+                entries[0].update({
                     'id': video_id,
                     'title': video_title,
                 })
+            else:
+                for num, entry in enumerate(entries, start=1):
+                    entry.update({
+                        'id': '%s-%s' % (video_id, num),
+                        'title': '%s (%d)' % (video_title, num),
+                    })
+            for entry in entries:
                 self._sort_formats(entry['formats'])
-            return self.playlist_result(entries)
+            return self.playlist_result(entries, video_id, video_title)
 
         jwplayer_data = self._find_jwplayer_data(
             webpage, video_id, transform_source=js_to_json)
-- 
cgit v1.1


From 13de91c9e92bd831fee38fddbdabce7f6e82ef91 Mon Sep 17 00:00:00 2001
From: Dan Weber <dan.b.weber@me.com>
Date: Tue, 12 Sep 2017 22:52:54 -0400
Subject: [americastestkitchen] Add extractor (closes #10764)

---
 youtube_dl/extractor/americastestkitchen.py | 85 +++++++++++++++++++++++++++++
 youtube_dl/extractor/extractors.py          |  1 +
 2 files changed, 86 insertions(+)
 create mode 100755 youtube_dl/extractor/americastestkitchen.py

diff --git a/youtube_dl/extractor/americastestkitchen.py b/youtube_dl/extractor/americastestkitchen.py
new file mode 100755
index 0000000..f231e7f
--- /dev/null
+++ b/youtube_dl/extractor/americastestkitchen.py
@@ -0,0 +1,85 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+
+
+class AmericasTestKitchenIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?americastestkitchen\.com/episode/(?P<id>\d+)'
+    _TESTS = [{
+        'url':
+        'https://www.americastestkitchen.com/episode/548-summer-dinner-party',
+        'md5': 'b861c3e365ac38ad319cfd509c30577f',
+        'info_dict': {
+            'id': '1_5g5zua6e',
+            'title': 'atk_s17_e24.mp4',
+            'ext': 'mp4',
+            'description': '<p>Host Julia Collin Davison goes into the test kitchen with test cook Dan Souza to learn how to make the ultimate Grill-Roasted Beef Tenderloin. Next, equipment expert Adam Ried reviews gas grills in the Equipment Corner. Then, gadget guru Lisa McManus uncovers the best quirky gadgets. Finally, test cook Erin McMurrer shows host Bridget Lancaster how to make an elegant Pear-Walnut Upside-Down Cake.</p>',
+            'timestamp': 1497285541,
+            'upload_date': '20170612',
+            'uploader_id': 'roger.metcalf@americastestkitchen.com',
+            'release_date': '2017-06-17',
+            'thumbnail': 'http://d3cizcpymoenau.cloudfront.net/images/35973/e24-tenderloin-16.jpg',
+            'episode_number': 24,
+            'episode': 'Summer Dinner Party',
+            'episode_id': '548-summer-dinner-party',
+            'season_number': 17
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
+    }, {
+        'url':
+        'https://www.americastestkitchen.com/episode/546-a-spanish-affair',
+        'only_matching':
+        True,
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        partner_id = self._search_regex(
+            r'partner_id/(?P<partner_id>\d+)',
+            webpage,
+            'partner_id',
+            group='partner_id')
+
+        video_data = self._parse_json(
+            self._search_regex(
+                r'window\.__INITIAL_STATE__\s*=\s*({.+?});\s*</script>',
+                webpage, 'initial context'),
+            video_id)
+
+        episode_data = video_data['episodeDetail']['content']['data']
+        episode_content_meta = episode_data['full_video']
+        external_id = episode_content_meta['external_id']
+
+        # photo data
+        photo_data = episode_content_meta.get('photo')
+        thumbnail = photo_data.get('image_url') if photo_data else None
+
+        # meta
+        release_date = episode_data.get('aired_at')
+        description = episode_content_meta.get('description')
+        episode_number = int(episode_content_meta.get('episode_number'))
+        episode = episode_content_meta.get('title')
+        episode_id = episode_content_meta.get('episode_slug')
+        season_number = int(episode_content_meta.get('season_number'))
+
+        return {
+            '_type': 'url_transparent',
+            'url': 'kaltura:%s:%s' % (partner_id, external_id),
+            'ie_key': 'Kaltura',
+            'id': video_id,
+            'release_date': release_date,
+            'thumbnail': thumbnail,
+            'description': description,
+            'episode_number': episode_number,
+            'episode': episode,
+            'episode_id': episode_id,
+            'season_number': season_number
+        }
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index ab95c85..5853005 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -39,6 +39,7 @@ from .airmozilla import AirMozillaIE
 from .aljazeera import AlJazeeraIE
 from .alphaporno import AlphaPornoIE
 from .amcnetworks import AMCNetworksIE
+from .americastestkitchen import AmericasTestKitchenIE
 from .animeondemand import AnimeOnDemandIE
 from .anitube import AnitubeIE
 from .anvato import AnvatoIE
-- 
cgit v1.1


From 4bb58fa118a8c75b2ecf05f7b29a0ae27eef6239 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 23 Sep 2017 06:28:46 +0700
Subject: [americastestkitchen] Improve (closes #13996)

---
 youtube_dl/extractor/americastestkitchen.py | 82 ++++++++++++++---------------
 1 file changed, 41 insertions(+), 41 deletions(-)

diff --git a/youtube_dl/extractor/americastestkitchen.py b/youtube_dl/extractor/americastestkitchen.py
index f231e7f..0173687 100755
--- a/youtube_dl/extractor/americastestkitchen.py
+++ b/youtube_dl/extractor/americastestkitchen.py
@@ -1,85 +1,85 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
+from ..utils import (
+    clean_html,
+    int_or_none,
+    try_get,
+    unified_strdate,
+)
 
 
 class AmericasTestKitchenIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?americastestkitchen\.com/episode/(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?americastestkitchen\.com/(?:episode|videos)/(?P<id>\d+)'
     _TESTS = [{
-        'url':
-        'https://www.americastestkitchen.com/episode/548-summer-dinner-party',
+        'url': 'https://www.americastestkitchen.com/episode/548-summer-dinner-party',
         'md5': 'b861c3e365ac38ad319cfd509c30577f',
         'info_dict': {
             'id': '1_5g5zua6e',
-            'title': 'atk_s17_e24.mp4',
+            'title': 'Summer Dinner Party',
             'ext': 'mp4',
-            'description': '<p>Host Julia Collin Davison goes into the test kitchen with test cook Dan Souza to learn how to make the ultimate Grill-Roasted Beef Tenderloin. Next, equipment expert Adam Ried reviews gas grills in the Equipment Corner. Then, gadget guru Lisa McManus uncovers the best quirky gadgets. Finally, test cook Erin McMurrer shows host Bridget Lancaster how to make an elegant Pear-Walnut Upside-Down Cake.</p>',
+            'description': 'md5:858d986e73a4826979b6a5d9f8f6a1ec',
+            'thumbnail': r're:^https?://.*\.jpg',
             'timestamp': 1497285541,
             'upload_date': '20170612',
             'uploader_id': 'roger.metcalf@americastestkitchen.com',
-            'release_date': '2017-06-17',
-            'thumbnail': 'http://d3cizcpymoenau.cloudfront.net/images/35973/e24-tenderloin-16.jpg',
-            'episode_number': 24,
+            'release_date': '20170617',
+            'series': "America's Test Kitchen",
+            'season_number': 17,
             'episode': 'Summer Dinner Party',
-            'episode_id': '548-summer-dinner-party',
-            'season_number': 17
+            'episode_number': 24,
         },
         'params': {
-            # m3u8 download
             'skip_download': True,
         },
     }, {
-        'url':
-        'https://www.americastestkitchen.com/episode/546-a-spanish-affair',
-        'only_matching':
-        True,
+        'url': 'https://www.americastestkitchen.com/videos/3420-pan-seared-salmon',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
+
         webpage = self._download_webpage(url, video_id)
 
         partner_id = self._search_regex(
-            r'partner_id/(?P<partner_id>\d+)',
-            webpage,
-            'partner_id',
-            group='partner_id')
+            r'src=["\'](?:https?:)?//(?:[^/]+\.)kaltura\.com/(?:[^/]+/)*(?:p|partner_id)/(\d+)',
+            webpage, 'kaltura partner id')
 
         video_data = self._parse_json(
             self._search_regex(
-                r'window\.__INITIAL_STATE__\s*=\s*({.+?});\s*</script>',
+                r'window\.__INITIAL_STATE__\s*=\s*({.+?})\s*;\s*</script>',
                 webpage, 'initial context'),
             video_id)
 
-        episode_data = video_data['episodeDetail']['content']['data']
-        episode_content_meta = episode_data['full_video']
-        external_id = episode_content_meta['external_id']
+        ep_data = try_get(
+            video_data,
+            (lambda x: x['episodeDetail']['content']['data'],
+             lambda x: x['videoDetail']['content']['data']), dict)
+        ep_meta = ep_data.get('full_video', {})
+        external_id = ep_data.get('external_id') or ep_meta['external_id']
 
-        # photo data
-        photo_data = episode_content_meta.get('photo')
-        thumbnail = photo_data.get('image_url') if photo_data else None
+        title = ep_data.get('title') or ep_meta.get('title')
+        description = clean_html(ep_meta.get('episode_description') or ep_data.get(
+            'description') or ep_meta.get('description'))
+        thumbnail = try_get(ep_meta, lambda x: x['photo']['image_url'])
+        release_date = unified_strdate(ep_data.get('aired_at'))
 
-        # meta
-        release_date = episode_data.get('aired_at')
-        description = episode_content_meta.get('description')
-        episode_number = int(episode_content_meta.get('episode_number'))
-        episode = episode_content_meta.get('title')
-        episode_id = episode_content_meta.get('episode_slug')
-        season_number = int(episode_content_meta.get('season_number'))
+        season_number = int_or_none(ep_meta.get('season_number'))
+        episode = ep_meta.get('title')
+        episode_number = int_or_none(ep_meta.get('episode_number'))
 
         return {
             '_type': 'url_transparent',
             'url': 'kaltura:%s:%s' % (partner_id, external_id),
             'ie_key': 'Kaltura',
-            'id': video_id,
-            'release_date': release_date,
-            'thumbnail': thumbnail,
+            'title': title,
             'description': description,
-            'episode_number': episode_number,
+            'thumbnail': thumbnail,
+            'release_date': release_date,
+            'series': "America's Test Kitchen",
+            'season_number': season_number,
             'episode': episode,
-            'episode_id': episode_id,
-            'season_number': season_number
+            'episode_number': episode_number,
         }
-- 
cgit v1.1


From 5c1452e8f1e744db14be1baef840e9f531e8f144 Mon Sep 17 00:00:00 2001
From: Giuseppe Fabiano <gfabiano40@gmail.com>
Date: Sat, 23 Sep 2017 01:38:09 +0200
Subject: [twitter] Add support for user_id-less URLs (closes #14270)

---
 youtube_dl/extractor/twitter.py | 30 +++++++++++++++++++++++++-----
 1 file changed, 25 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/twitter.py b/youtube_dl/extractor/twitter.py
index 7399cf5..0df3ad7 100644
--- a/youtube_dl/extractor/twitter.py
+++ b/youtube_dl/extractor/twitter.py
@@ -242,8 +242,9 @@ class TwitterCardIE(TwitterBaseIE):
 
 class TwitterIE(InfoExtractor):
     IE_NAME = 'twitter'
-    _VALID_URL = r'https?://(?:www\.|m\.|mobile\.)?twitter\.com/(?P<user_id>[^/]+)/status/(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.|m\.|mobile\.)?twitter\.com/(?:i/web|(?P<user_id>[^/]+))/status/(?P<id>\d+)'
     _TEMPLATE_URL = 'https://twitter.com/%s/status/%s'
+    _TEMPLATE_STATUSES_URL = 'https://twitter.com/statuses/%s'
 
     _TESTS = [{
         'url': 'https://twitter.com/freethenipple/status/643211948184596480',
@@ -322,9 +323,9 @@ class TwitterIE(InfoExtractor):
         'info_dict': {
             'id': 'MIOxnrUteUd',
             'ext': 'mp4',
-            'title': 'FilmDrunk - Vine of the day',
-            'description': 'FilmDrunk on Twitter: "Vine of the day https://t.co/xmTvRdqxWf"',
-            'uploader': 'FilmDrunk',
+            'title': 'Vince Mancini - Vine of the day',
+            'description': 'Vince Mancini on Twitter: "Vine of the day https://t.co/xmTvRdqxWf"',
+            'uploader': 'Vince Mancini',
             'uploader_id': 'Filmdrunk',
             'timestamp': 1402826626,
             'upload_date': '20140615',
@@ -372,6 +373,21 @@ class TwitterIE(InfoExtractor):
         'params': {
             'format': 'best[format_id^=http-]',
         },
+    }, {
+        'url': 'https://twitter.com/i/web/status/910031516746514432',
+        'info_dict': {
+            'id': '910031516746514432',
+            'ext': 'mp4',
+            'title': 'Préfet de Guadeloupe - [Direct] #Maria Le centre se trouve actuellement au sud de Basse-Terre. Restez confinés. Réfugiez-vous dans la pièce la + sûre.',
+            'thumbnail': r're:^https?://.*\.jpg',
+            'description': 'Préfet de Guadeloupe on Twitter: "[Direct] #Maria Le centre se trouve actuellement au sud de Basse-Terre. Restez confinés. Réfugiez-vous dans la pièce la + sûre. https://t.co/mwx01Rs4lo"',
+            'uploader': 'Préfet de Guadeloupe',
+            'uploader_id': 'Prefet971',
+            'duration': 47.48,
+        },
+        'params': {
+            'skip_download': True,  # requires ffmpeg
+        },
     }]
 
     def _real_extract(self, url):
@@ -380,11 +396,15 @@ class TwitterIE(InfoExtractor):
         twid = mobj.group('id')
 
         webpage, urlh = self._download_webpage_handle(
-            self._TEMPLATE_URL % (user_id, twid), twid)
+            self._TEMPLATE_STATUSES_URL % twid, twid)
 
         if 'twitter.com/account/suspended' in urlh.geturl():
             raise ExtractorError('Account suspended by Twitter.', expected=True)
 
+        if user_id is None:
+            mobj = re.match(self._VALID_URL, urlh.geturl())
+            user_id = mobj.group('user_id')
+
         username = remove_end(self._og_search_title(webpage), ' on Twitter')
 
         title = description = self._og_search_description(webpage).strip('').replace('\n', ' ').strip('“”')
-- 
cgit v1.1


From 1c22d7a7f30917abfd2b7495f7bd02d51cb8528a Mon Sep 17 00:00:00 2001
From: Namnamseo <0201ssw+github@gmail.com>
Date: Thu, 24 Aug 2017 11:32:24 +0900
Subject: [kakao] Add extractor (closes #12298)

---
 youtube_dl/extractor/extractors.py |   1 +
 youtube_dl/extractor/kakao.py      | 140 +++++++++++++++++++++++++++++++++++++
 2 files changed, 141 insertions(+)
 create mode 100644 youtube_dl/extractor/kakao.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 5853005..4232a4f 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -483,6 +483,7 @@ from .jove import JoveIE
 from .joj import JojIE
 from .jwplatform import JWPlatformIE
 from .jpopsukitv import JpopsukiIE
+from .kakao import KakaoIE
 from .kaltura import KalturaIE
 from .kamcord import KamcordIE
 from .kanalplay import KanalPlayIE
diff --git a/youtube_dl/extractor/kakao.py b/youtube_dl/extractor/kakao.py
new file mode 100644
index 0000000..0caa41e
--- /dev/null
+++ b/youtube_dl/extractor/kakao.py
@@ -0,0 +1,140 @@
+# coding: utf-8
+
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    compat_str,
+    unified_timestamp,
+)
+
+
+class KakaoIE(InfoExtractor):
+    _VALID_URL = r'https?://tv.kakao.com/channel/(?P<channel>\d+)/cliplink/(?P<id>\d+)'
+    IE_NAME = 'kakao.com'
+
+    _TESTS = [{
+        'url': 'http://tv.kakao.com/channel/2671005/cliplink/301965083',
+        'md5': '702b2fbdeb51ad82f5c904e8c0766340',
+        'info_dict': {
+            'id': '301965083',
+            'ext': 'mp4',
+            'title': '乃木坂46 バナナマン 「3期生紹介コーナーが始動！顔高低差GPも！」 『乃木坂工事中』',
+            'uploader_id': 2671005,
+            'uploader': '그랑그랑이',
+            'timestamp': 1488160199,
+            'upload_date': '20170227',
+        }
+    }, {
+        'url': 'http://tv.kakao.com/channel/2653210/cliplink/300103180',
+        'md5': 'a8917742069a4dd442516b86e7d66529',
+        'info_dict': {
+            'id': '300103180',
+            'ext': 'mp4',
+            'description': '러블리즈 - Destiny (나의 지구) (Lovelyz - Destiny)\r\n\r\n[쇼! 음악중심] 20160611, 507회',
+            'title': '러블리즈 - Destiny (나의 지구) (Lovelyz - Destiny)',
+            'uploader_id': 2653210,
+            'uploader': '쇼 음악중심',
+            'timestamp': 1485684628,
+            'upload_date': '20170129',
+        }
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        player_url = 'http://tv.kakao.com/embed/player/cliplink/' + video_id + \
+            '?service=kakao_tv&autoplay=1&profile=HIGH&wmode=transparent'
+        player_header = {'Referer': player_url}
+
+        impress = self._download_json(
+            'http://tv.kakao.com/api/v1/ft/cliplinks/%s/impress' % video_id,
+            video_id, 'Downloading video info',
+            query={
+                'player': 'monet_html5',
+                'referer': url,
+                'uuid': '',
+                'service': 'kakao_tv',
+                'section': '',
+                'dteType': 'PC',
+                'fields': 'clipLink,clip,channel,hasPlusFriend,-service,-tagList'
+            }, headers=player_header)
+
+        clipLink = impress['clipLink']
+        clip = clipLink['clip']
+
+        video_info = {
+            'id': video_id,
+            'title': clip['title'],
+            'description': clip.get('description'),
+            'uploader': clipLink.get('channel', {}).get('name'),
+            'uploader_id': clipLink.get('channelId'),
+            'duration': int_or_none(clip.get('duration')),
+            'view_count': int_or_none(clip.get('playCount')),
+            'like_count': int_or_none(clip.get('likeCount')),
+            'comment_count': int_or_none(clip.get('commentCount')),
+        }
+
+        tid = impress.get('tid', '')
+        raw = self._download_json(
+            'http://tv.kakao.com/api/v1/ft/cliplinks/%s/raw' % video_id,
+            video_id, 'Downloading video formats info',
+            query={
+                'player': 'monet_html5',
+                'referer': url,
+                'uuid': '',
+                'service': 'kakao_tv',
+                'section': '',
+                'tid': tid,
+                'profile': 'HIGH',
+                'dteType': 'PC',
+            }, headers=player_header, fatal=False)
+
+        formats = []
+        for fmt in raw.get('outputList', []):
+            try:
+                profile_name = fmt['profile']
+                fmt_url_json = self._download_json(
+                    'http://tv.kakao.com/api/v1/ft/cliplinks/%s/raw/videolocation' % video_id,
+                    video_id, 'Downloading video URL for profile %s' % profile_name,
+                    query={
+                        'service': 'kakao_tv',
+                        'section': '',
+                        'tid': tid,
+                        'profile': profile_name
+                    }, headers=player_header, fatal=False)
+
+                if fmt_url_json is None:
+                    continue
+
+                fmt_url = fmt_url_json['url']
+                formats.append({
+                    'url': fmt_url,
+                    'format_id': profile_name,
+                    'width': int_or_none(fmt.get('width')),
+                    'height': int_or_none(fmt.get('height')),
+                    'format_note': fmt.get('label'),
+                    'filesize': int_or_none(fmt.get('filesize'))
+                })
+            except KeyError:
+                pass
+
+        self._sort_formats(formats)
+        video_info['formats'] = formats
+
+        top_thumbnail = clip.get('thumbnailUrl')
+        thumbs = []
+        for thumb in clip.get('clipChapterThumbnailList', []):
+            thumbs.append({
+                'url': thumb.get('thumbnailUrl'),
+                'id': compat_str(thumb.get('timeInSec')),
+                'preference': -1 if thumb.get('isDefault') else 0
+            })
+        video_info['thumbnail'] = top_thumbnail
+        video_info['thumbnails'] = thumbs
+
+        upload_date = unified_timestamp(clipLink.get('createTime'))
+        video_info['timestamp'] = upload_date
+
+        return video_info
-- 
cgit v1.1


From f70ddd4aebbfb0bdf2f63c1eba5b5614d2cfb70f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 23 Sep 2017 07:25:15 +0700
Subject: [kakao] Improve (closes #14007)

---
 youtube_dl/extractor/kakao.py | 113 +++++++++++++++++++++++-------------------
 1 file changed, 61 insertions(+), 52 deletions(-)

diff --git a/youtube_dl/extractor/kakao.py b/youtube_dl/extractor/kakao.py
index 0caa41e..c9b438e 100644
--- a/youtube_dl/extractor/kakao.py
+++ b/youtube_dl/extractor/kakao.py
@@ -3,16 +3,17 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..compat import compat_str
 from ..utils import (
     int_or_none,
-    compat_str,
     unified_timestamp,
+    update_url_query,
 )
 
 
 class KakaoIE(InfoExtractor):
     _VALID_URL = r'https?://tv.kakao.com/channel/(?P<channel>\d+)/cliplink/(?P<id>\d+)'
-    IE_NAME = 'kakao.com'
+    _API_BASE = 'http://tv.kakao.com/api/v1/ft/cliplinks'
 
     _TESTS = [{
         'url': 'http://tv.kakao.com/channel/2671005/cliplink/301965083',
@@ -44,60 +45,57 @@ class KakaoIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        player_url = 'http://tv.kakao.com/embed/player/cliplink/' + video_id + \
-            '?service=kakao_tv&autoplay=1&profile=HIGH&wmode=transparent'
-        player_header = {'Referer': player_url}
+        player_header = {
+            'Referer': update_url_query(
+                'http://tv.kakao.com/embed/player/cliplink/%s' % video_id, {
+                    'service': 'kakao_tv',
+                    'autoplay': '1',
+                    'profile': 'HIGH',
+                    'wmode': 'transparent',
+                })
+        }
+
+        QUERY_COMMON = {
+            'player': 'monet_html5',
+            'referer': url,
+            'uuid': '',
+            'service': 'kakao_tv',
+            'section': '',
+            'dteType': 'PC',
+        }
 
+        query = QUERY_COMMON.copy()
+        query['fields'] = 'clipLink,clip,channel,hasPlusFriend,-service,-tagList'
         impress = self._download_json(
-            'http://tv.kakao.com/api/v1/ft/cliplinks/%s/impress' % video_id,
+            '%s/%s/impress' % (self._API_BASE, video_id),
             video_id, 'Downloading video info',
-            query={
-                'player': 'monet_html5',
-                'referer': url,
-                'uuid': '',
-                'service': 'kakao_tv',
-                'section': '',
-                'dteType': 'PC',
-                'fields': 'clipLink,clip,channel,hasPlusFriend,-service,-tagList'
-            }, headers=player_header)
-
-        clipLink = impress['clipLink']
-        clip = clipLink['clip']
-
-        video_info = {
-            'id': video_id,
-            'title': clip['title'],
-            'description': clip.get('description'),
-            'uploader': clipLink.get('channel', {}).get('name'),
-            'uploader_id': clipLink.get('channelId'),
-            'duration': int_or_none(clip.get('duration')),
-            'view_count': int_or_none(clip.get('playCount')),
-            'like_count': int_or_none(clip.get('likeCount')),
-            'comment_count': int_or_none(clip.get('commentCount')),
-        }
+            query=query, headers=player_header)
+
+        clip_link = impress['clipLink']
+        clip = clip_link['clip']
+
+        title = clip.get('title') or clip_link.get('displayTitle')
 
         tid = impress.get('tid', '')
+
+        query = QUERY_COMMON.copy()
+        query.update({
+            'tid': tid,
+            'profile': 'HIGH',
+        })
         raw = self._download_json(
-            'http://tv.kakao.com/api/v1/ft/cliplinks/%s/raw' % video_id,
+            '%s/%s/raw' % (self._API_BASE, video_id),
             video_id, 'Downloading video formats info',
-            query={
-                'player': 'monet_html5',
-                'referer': url,
-                'uuid': '',
-                'service': 'kakao_tv',
-                'section': '',
-                'tid': tid,
-                'profile': 'HIGH',
-                'dteType': 'PC',
-            }, headers=player_header, fatal=False)
+            query=query, headers=player_header)
 
         formats = []
         for fmt in raw.get('outputList', []):
             try:
                 profile_name = fmt['profile']
                 fmt_url_json = self._download_json(
-                    'http://tv.kakao.com/api/v1/ft/cliplinks/%s/raw/videolocation' % video_id,
-                    video_id, 'Downloading video URL for profile %s' % profile_name,
+                    '%s/%s/raw/videolocation' % (self._API_BASE, video_id),
+                    video_id,
+                    'Downloading video URL for profile %s' % profile_name,
                     query={
                         'service': 'kakao_tv',
                         'section': '',
@@ -119,11 +117,8 @@ class KakaoIE(InfoExtractor):
                 })
             except KeyError:
                 pass
-
         self._sort_formats(formats)
-        video_info['formats'] = formats
 
-        top_thumbnail = clip.get('thumbnailUrl')
         thumbs = []
         for thumb in clip.get('clipChapterThumbnailList', []):
             thumbs.append({
@@ -131,10 +126,24 @@ class KakaoIE(InfoExtractor):
                 'id': compat_str(thumb.get('timeInSec')),
                 'preference': -1 if thumb.get('isDefault') else 0
             })
-        video_info['thumbnail'] = top_thumbnail
-        video_info['thumbnails'] = thumbs
-
-        upload_date = unified_timestamp(clipLink.get('createTime'))
-        video_info['timestamp'] = upload_date
+        top_thumbnail = clip.get('thumbnailUrl')
+        if top_thumbnail:
+            thumbs.append({
+                'url': top_thumbnail,
+                'preference': 10,
+            })
 
-        return video_info
+        return {
+            'id': video_id,
+            'title': title,
+            'description': clip.get('description'),
+            'uploader': clip_link.get('channel', {}).get('name'),
+            'uploader_id': clip_link.get('channelId'),
+            'thumbnails': thumbs,
+            'timestamp': unified_timestamp(clip_link.get('createTime')),
+            'duration': int_or_none(clip.get('duration')),
+            'view_count': int_or_none(clip.get('playCount')),
+            'like_count': int_or_none(clip.get('likeCount')),
+            'comment_count': int_or_none(clip.get('commentCount')),
+            'formats': formats,
+        }
-- 
cgit v1.1


From 7f4921b38d10c17fe354bab20b741b362c5ae0aa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 23 Sep 2017 07:26:40 +0700
Subject: [heise] PEP 8

---
 youtube_dl/extractor/heise.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/heise.py b/youtube_dl/extractor/heise.py
index 495ffb7..82e11a7 100644
--- a/youtube_dl/extractor/heise.py
+++ b/youtube_dl/extractor/heise.py
@@ -60,8 +60,8 @@ class HeiseIE(InfoExtractor):
         title = self._html_search_meta('fulltitle', webpage, default=None)
         if not title or title == "c't":
             title = self._search_regex(
-                    r'<div[^>]+class="videoplayerjw"[^>]+data-title="([^"]+)"',
-                    webpage, 'title')
+                r'<div[^>]+class="videoplayerjw"[^>]+data-title="([^"]+)"',
+                webpage, 'title')
 
         yt_urls = YoutubeIE._extract_urls(webpage)
         if yt_urls:
-- 
cgit v1.1


From 136507b39a2b48cb775249e9724eeeedb56baed2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 23 Sep 2017 07:41:22 +0700
Subject: [24video] Add support for 24video.adult (closes #14295)

---
 youtube_dl/extractor/twentyfourvideo.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/twentyfourvideo.py b/youtube_dl/extractor/twentyfourvideo.py
index 7af1165..cc51ca0 100644
--- a/youtube_dl/extractor/twentyfourvideo.py
+++ b/youtube_dl/extractor/twentyfourvideo.py
@@ -14,7 +14,7 @@ from ..utils import (
 
 class TwentyFourVideoIE(InfoExtractor):
     IE_NAME = '24video'
-    _VALID_URL = r'https?://(?P<host>(?:www\.)?24video\.(?:net|me|xxx|sex|tube))/(?:video/(?:view|xml)/|player/new24_play\.swf\?id=)(?P<id>\d+)'
+    _VALID_URL = r'https?://(?P<host>(?:www\.)?24video\.(?:net|me|xxx|sex|tube|adult))/(?:video/(?:view|xml)/|player/new24_play\.swf\?id=)(?P<id>\d+)'
 
     _TESTS = [{
         'url': 'http://www.24video.net/video/view/1044982',
-- 
cgit v1.1


From e3440d824a7326d0ba609d8f0896203208ecc558 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 23 Sep 2017 07:42:17 +0700
Subject: [24video] Fix timestamp extraction and make non fatal (#14295)

---
 youtube_dl/extractor/twentyfourvideo.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/twentyfourvideo.py b/youtube_dl/extractor/twentyfourvideo.py
index cc51ca0..96e0b96 100644
--- a/youtube_dl/extractor/twentyfourvideo.py
+++ b/youtube_dl/extractor/twentyfourvideo.py
@@ -60,8 +60,8 @@ class TwentyFourVideoIE(InfoExtractor):
         duration = int_or_none(self._og_search_property(
             'duration', webpage, 'duration', fatal=False))
         timestamp = parse_iso8601(self._search_regex(
-            r'<time id="video-timeago" datetime="([^"]+)" itemprop="uploadDate">',
-            webpage, 'upload date'))
+            r'<time[^>]+\bdatetime="([^"]+)"[^>]+itemprop="uploadDate"',
+            webpage, 'upload date', fatal=False))
 
         uploader = self._html_search_regex(
             r'class="video-uploaded"[^>]*>\s*<a href="/jsecUser/movies/[^"]+"[^>]*>([^<]+)</a>',
@@ -72,7 +72,7 @@ class TwentyFourVideoIE(InfoExtractor):
             webpage, 'view count', fatal=False))
         comment_count = int_or_none(self._html_search_regex(
             r'<a[^>]+href="#tab-comments"[^>]*>(\d+) комментари',
-            webpage, 'comment count', fatal=False))
+            webpage, 'comment count', default=None))
 
         # Sets some cookies
         self._download_xml(
-- 
cgit v1.1


From 47b5dfb047ce78dfb4ca7091efb4f7d0a1af3545 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 23 Sep 2017 23:14:41 +0700
Subject: Credit @luboss for joj (#13268)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 478c787..c2dd506 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -224,3 +224,4 @@ Giuseppe Fabiano
 Örn Guðjónsson
 Parmjit Virk
 Genki Sky
+Ľuboš Katrinec
-- 
cgit v1.1


From 07d1344c85781c911d0c1b1f6aa1804cd69f37d3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 23 Sep 2017 23:16:27 +0700
Subject: Credit @coreynicholson for vlive:playlist (#13613)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index c2dd506..cff6600 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -225,3 +225,4 @@ Giuseppe Fabiano
 Parmjit Virk
 Genki Sky
 Ľuboš Katrinec
+Corey Nicholson
-- 
cgit v1.1


From 16f54d0751553c09a512a69f56920235a186d4bc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 23 Sep 2017 23:20:20 +0700
Subject: Credit @codeasashu for voot (#11814)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index cff6600..295b784 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -226,3 +226,4 @@ Parmjit Virk
 Genki Sky
 Ľuboš Katrinec
 Corey Nicholson
+Ashutosh Chaudhary
-- 
cgit v1.1


From aab20aabfc8098a667b0f1bd820fe85ab2653b8e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 23 Sep 2017 23:23:27 +0700
Subject: Credit @jdong92 for voot (#14059)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 295b784..147c3e9 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -227,3 +227,4 @@ Genki Sky
 Ľuboš Katrinec
 Corey Nicholson
 Ashutosh Chaudhary
+John Dong
-- 
cgit v1.1


From 2f3933aa1e33d7ec24a8701f102a309029a31768 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 23 Sep 2017 23:26:35 +0700
Subject: Credit @ishitatsuyuki for mixcloud fix (#14132)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 147c3e9..d1f3bba 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -228,3 +228,4 @@ Genki Sky
 Corey Nicholson
 Ashutosh Chaudhary
 John Dong
+Tatsuyuki Ishi
-- 
cgit v1.1


From a87d7b4953d46e75e62fe25a5cc5e689297127ae Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 23 Sep 2017 23:27:28 +0700
Subject: Credit @nbppp2 for americastestkitchen (#13996)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index d1f3bba..1d40c86 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -229,3 +229,4 @@ Corey Nicholson
 Ashutosh Chaudhary
 John Dong
 Tatsuyuki Ishi
+Daniel Weber
-- 
cgit v1.1


From 4c54b89e0310abd6be643604dc0f1ec3a608b68d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 24 Sep 2017 00:08:27 +0700
Subject: Hide experimental phantomjs wrapper

---
 youtube_dl/YoutubeDL.py          |   2 +-
 youtube_dl/extractor/openload.py | 228 ++++++++++++++++++++++++++++++++++++++-
 youtube_dl/utils.py              | 214 ------------------------------------
 3 files changed, 228 insertions(+), 216 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index bfb4ff2..0a7f36c 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -89,10 +89,10 @@ from .utils import (
     write_string,
     YoutubeDLCookieProcessor,
     YoutubeDLHandler,
-    PhantomJSwrapper,
 )
 from .cache import Cache
 from .extractor import get_info_extractor, gen_extractor_classes, _LAZY_LOADER
+from .extractor.openload import PhantomJSwrapper
 from .downloader import get_suitable_downloader
 from .downloader.rtmp import rtmpdump_version
 from .postprocessor import (
diff --git a/youtube_dl/extractor/openload.py b/youtube_dl/extractor/openload.py
index 292476e..da1ef02 100644
--- a/youtube_dl/extractor/openload.py
+++ b/youtube_dl/extractor/openload.py
@@ -1,17 +1,243 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import json
+import os
 import re
+import subprocess
+import tempfile
 
 from .common import InfoExtractor
+from ..compat import compat_urlparse
 from ..utils import (
+    check_executable,
     determine_ext,
+    encodeArgument,
     ExtractorError,
     get_element_by_id,
-    PhantomJSwrapper,
+    get_exe_version,
+    is_outdated_version,
+    std_headers,
 )
 
 
+def cookie_to_dict(cookie):
+    cookie_dict = {
+        'name': cookie.name,
+        'value': cookie.value,
+    }
+    if cookie.port_specified:
+        cookie_dict['port'] = cookie.port
+    if cookie.domain_specified:
+        cookie_dict['domain'] = cookie.domain
+    if cookie.path_specified:
+        cookie_dict['path'] = cookie.path
+    if cookie.expires is not None:
+        cookie_dict['expires'] = cookie.expires
+    if cookie.secure is not None:
+        cookie_dict['secure'] = cookie.secure
+    if cookie.discard is not None:
+        cookie_dict['discard'] = cookie.discard
+    try:
+        if (cookie.has_nonstandard_attr('httpOnly') or
+                cookie.has_nonstandard_attr('httponly') or
+                cookie.has_nonstandard_attr('HttpOnly')):
+            cookie_dict['httponly'] = True
+    except TypeError:
+        pass
+    return cookie_dict
+
+
+def cookie_jar_to_list(cookie_jar):
+    return [cookie_to_dict(cookie) for cookie in cookie_jar]
+
+
+class PhantomJSwrapper(object):
+    """PhantomJS wrapper class
+
+    This class is experimental.
+    """
+
+    _TEMPLATE = r'''
+        phantom.onError = function(msg, trace) {{
+          var msgStack = ['PHANTOM ERROR: ' + msg];
+          if(trace && trace.length) {{
+            msgStack.push('TRACE:');
+            trace.forEach(function(t) {{
+              msgStack.push(' -> ' + (t.file || t.sourceURL) + ': ' + t.line
+                + (t.function ? ' (in function ' + t.function +')' : ''));
+            }});
+          }}
+          console.error(msgStack.join('\n'));
+          phantom.exit(1);
+        }};
+        var page = require('webpage').create();
+        var fs = require('fs');
+        var read = {{ mode: 'r', charset: 'utf-8' }};
+        var write = {{ mode: 'w', charset: 'utf-8' }};
+        JSON.parse(fs.read("{cookies}", read)).forEach(function(x) {{
+          phantom.addCookie(x);
+        }});
+        page.settings.resourceTimeout = {timeout};
+        page.settings.userAgent = "{ua}";
+        page.onLoadStarted = function() {{
+          page.evaluate(function() {{
+            delete window._phantom;
+            delete window.callPhantom;
+          }});
+        }};
+        var saveAndExit = function() {{
+          fs.write("{html}", page.content, write);
+          fs.write("{cookies}", JSON.stringify(phantom.cookies), write);
+          phantom.exit();
+        }};
+        page.onLoadFinished = function(status) {{
+          if(page.url === "") {{
+            page.setContent(fs.read("{html}", read), "{url}");
+          }}
+          else {{
+            {jscode}
+          }}
+        }};
+        page.open("");
+    '''
+
+    _TMP_FILE_NAMES = ['script', 'html', 'cookies']
+
+    @staticmethod
+    def _version():
+        return get_exe_version('phantomjs', version_re=r'([0-9.]+)')
+
+    def __init__(self, extractor, required_version=None, timeout=10000):
+        self.exe = check_executable('phantomjs', ['-v'])
+        if not self.exe:
+            raise ExtractorError('PhantomJS executable not found in PATH, '
+                                 'download it from http://phantomjs.org',
+                                 expected=True)
+
+        self.extractor = extractor
+
+        if required_version:
+            version = self._version()
+            if is_outdated_version(version, required_version):
+                self.extractor._downloader.report_warning(
+                    'Your copy of PhantomJS is outdated, update it to version '
+                    '%s or newer if you encounter any errors.' % required_version)
+
+        self.options = {
+            'timeout': timeout,
+        }
+        self._TMP_FILES = {}
+        for name in self._TMP_FILE_NAMES:
+            tmp = tempfile.NamedTemporaryFile(delete=False)
+            tmp.close()
+            self._TMP_FILES[name] = tmp
+
+    def __del__(self):
+        for name in self._TMP_FILE_NAMES:
+            try:
+                os.remove(self._TMP_FILES[name].name)
+            except:
+                pass
+
+    def _save_cookies(self, url):
+        cookies = cookie_jar_to_list(self.extractor._downloader.cookiejar)
+        for cookie in cookies:
+            if 'path' not in cookie:
+                cookie['path'] = '/'
+            if 'domain' not in cookie:
+                cookie['domain'] = compat_urlparse.urlparse(url).netloc
+        with open(self._TMP_FILES['cookies'].name, 'wb') as f:
+            f.write(json.dumps(cookies).encode('utf-8'))
+
+    def _load_cookies(self):
+        with open(self._TMP_FILES['cookies'].name, 'rb') as f:
+            cookies = json.loads(f.read().decode('utf-8'))
+        for cookie in cookies:
+            if cookie['httponly'] is True:
+                cookie['rest'] = {'httpOnly': None}
+            if 'expiry' in cookie:
+                cookie['expire_time'] = cookie['expiry']
+            self.extractor._set_cookie(**cookie)
+
+    def get(self, url, html=None, video_id=None, note=None, note2='Executing JS on webpage', headers={}, jscode='saveAndExit();'):
+        """
+        Downloads webpage (if needed) and executes JS
+
+        Params:
+            url: website url
+            html: optional, html code of website
+            video_id: video id
+            note: optional, displayed when downloading webpage
+            note2: optional, displayed when executing JS
+            headers: custom http headers
+            jscode: code to be executed when page is loaded
+
+        Returns tuple with:
+            * downloaded website (after JS execution)
+            * anything you print with `console.log` (but not inside `page.execute`!)
+
+        In most cases you don't need to add any `jscode`.
+        It is executed in `page.onLoadFinished`.
+        `saveAndExit();` is mandatory, use it instead of `phantom.exit()`
+        It is possible to wait for some element on the webpage, for example:
+            var check = function() {
+              var elementFound = page.evaluate(function() {
+                return document.querySelector('#b.done') !== null;
+              });
+              if(elementFound)
+                saveAndExit();
+              else
+                window.setTimeout(check, 500);
+            }
+
+            page.evaluate(function(){
+              document.querySelector('#a').click();
+            });
+            check();
+        """
+        if 'saveAndExit();' not in jscode:
+            raise ExtractorError('`saveAndExit();` not found in `jscode`')
+        if not html:
+            html = self.extractor._download_webpage(url, video_id, note=note, headers=headers)
+        with open(self._TMP_FILES['html'].name, 'wb') as f:
+            f.write(html.encode('utf-8'))
+
+        self._save_cookies(url)
+
+        replaces = self.options
+        replaces['url'] = url
+        user_agent = headers.get('User-Agent') or std_headers['User-Agent']
+        replaces['ua'] = user_agent.replace('"', '\\"')
+        replaces['jscode'] = jscode
+
+        for x in self._TMP_FILE_NAMES:
+            replaces[x] = self._TMP_FILES[x].name.replace('\\', '\\\\').replace('"', '\\"')
+
+        with open(self._TMP_FILES['script'].name, 'wb') as f:
+            f.write(self._TEMPLATE.format(**replaces).encode('utf-8'))
+
+        if video_id is None:
+            self.extractor.to_screen('%s' % (note2,))
+        else:
+            self.extractor.to_screen('%s: %s' % (video_id, note2))
+
+        p = subprocess.Popen([
+            self.exe, '--ssl-protocol=any',
+            self._TMP_FILES['script'].name
+        ], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        out, err = p.communicate()
+        if p.returncode != 0:
+            raise ExtractorError(
+                'Executing JS failed\n:' + encodeArgument(err))
+        with open(self._TMP_FILES['html'].name, 'rb') as f:
+            html = f.read().decode('utf-8')
+
+        self._load_cookies()
+
+        return (html, encodeArgument(out))
+
+
 class OpenloadIE(InfoExtractor):
     _VALID_URL = r'https?://(?:openload\.(?:co|io)|oload\.tv)/(?:f|embed)/(?P<id>[a-zA-Z0-9-_]+)'
 
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index acc4f98..92b22e6 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -3826,220 +3826,6 @@ def write_xattr(path, key, value):
                         "or the 'xattr' binary.")
 
 
-def cookie_to_dict(cookie):
-    cookie_dict = {
-        'name': cookie.name,
-        'value': cookie.value,
-    }
-    if cookie.port_specified:
-        cookie_dict['port'] = cookie.port
-    if cookie.domain_specified:
-        cookie_dict['domain'] = cookie.domain
-    if cookie.path_specified:
-        cookie_dict['path'] = cookie.path
-    if cookie.expires is not None:
-        cookie_dict['expires'] = cookie.expires
-    if cookie.secure is not None:
-        cookie_dict['secure'] = cookie.secure
-    if cookie.discard is not None:
-        cookie_dict['discard'] = cookie.discard
-    try:
-        if (cookie.has_nonstandard_attr('httpOnly') or
-                cookie.has_nonstandard_attr('httponly') or
-                cookie.has_nonstandard_attr('HttpOnly')):
-            cookie_dict['httponly'] = True
-    except TypeError:
-        pass
-    return cookie_dict
-
-
-def cookie_jar_to_list(cookie_jar):
-    return [cookie_to_dict(cookie) for cookie in cookie_jar]
-
-
-class PhantomJSwrapper(object):
-    """PhantomJS wrapper class"""
-
-    _TEMPLATE = r'''
-        phantom.onError = function(msg, trace) {{
-          var msgStack = ['PHANTOM ERROR: ' + msg];
-          if(trace && trace.length) {{
-            msgStack.push('TRACE:');
-            trace.forEach(function(t) {{
-              msgStack.push(' -> ' + (t.file || t.sourceURL) + ': ' + t.line
-                + (t.function ? ' (in function ' + t.function +')' : ''));
-            }});
-          }}
-          console.error(msgStack.join('\n'));
-          phantom.exit(1);
-        }};
-        var page = require('webpage').create();
-        var fs = require('fs');
-        var read = {{ mode: 'r', charset: 'utf-8' }};
-        var write = {{ mode: 'w', charset: 'utf-8' }};
-        JSON.parse(fs.read("{cookies}", read)).forEach(function(x) {{
-          phantom.addCookie(x);
-        }});
-        page.settings.resourceTimeout = {timeout};
-        page.settings.userAgent = "{ua}";
-        page.onLoadStarted = function() {{
-          page.evaluate(function() {{
-            delete window._phantom;
-            delete window.callPhantom;
-          }});
-        }};
-        var saveAndExit = function() {{
-          fs.write("{html}", page.content, write);
-          fs.write("{cookies}", JSON.stringify(phantom.cookies), write);
-          phantom.exit();
-        }};
-        page.onLoadFinished = function(status) {{
-          if(page.url === "") {{
-            page.setContent(fs.read("{html}", read), "{url}");
-          }}
-          else {{
-            {jscode}
-          }}
-        }};
-        page.open("");
-    '''
-
-    _TMP_FILE_NAMES = ['script', 'html', 'cookies']
-
-    @staticmethod
-    def _version():
-        return get_exe_version('phantomjs', version_re=r'([0-9.]+)')
-
-    def __init__(self, extractor, required_version=None, timeout=10000):
-        self.exe = check_executable('phantomjs', ['-v'])
-        if not self.exe:
-            raise ExtractorError('PhantomJS executable not found in PATH, '
-                                 'download it from http://phantomjs.org',
-                                 expected=True)
-
-        self.extractor = extractor
-
-        if required_version:
-            version = self._version()
-            if is_outdated_version(version, required_version):
-                self.extractor._downloader.report_warning(
-                    'Your copy of PhantomJS is outdated, update it to version '
-                    '%s or newer if you encounter any errors.' % required_version)
-
-        self.options = {
-            'timeout': timeout,
-        }
-        self._TMP_FILES = {}
-        for name in self._TMP_FILE_NAMES:
-            tmp = tempfile.NamedTemporaryFile(delete=False)
-            tmp.close()
-            self._TMP_FILES[name] = tmp
-
-    def __del__(self):
-        for name in self._TMP_FILE_NAMES:
-            try:
-                os.remove(self._TMP_FILES[name].name)
-            except:
-                pass
-
-    def _save_cookies(self, url):
-        cookies = cookie_jar_to_list(self.extractor._downloader.cookiejar)
-        for cookie in cookies:
-            if 'path' not in cookie:
-                cookie['path'] = '/'
-            if 'domain' not in cookie:
-                cookie['domain'] = compat_urlparse.urlparse(url).netloc
-        with open(self._TMP_FILES['cookies'].name, 'wb') as f:
-            f.write(json.dumps(cookies).encode('utf-8'))
-
-    def _load_cookies(self):
-        with open(self._TMP_FILES['cookies'].name, 'rb') as f:
-            cookies = json.loads(f.read().decode('utf-8'))
-        for cookie in cookies:
-            if cookie['httponly'] is True:
-                cookie['rest'] = {'httpOnly': None}
-            if 'expiry' in cookie:
-                cookie['expire_time'] = cookie['expiry']
-            self.extractor._set_cookie(**cookie)
-
-    def get(self, url, html=None, video_id=None, note=None, note2='Executing JS on webpage', headers={}, jscode='saveAndExit();'):
-        """
-        Downloads webpage (if needed) and executes JS
-
-        Params:
-            url: website url
-            html: optional, html code of website
-            video_id: video id
-            note: optional, displayed when downloading webpage
-            note2: optional, displayed when executing JS
-            headers: custom http headers
-            jscode: code to be executed when page is loaded
-
-        Returns tuple with:
-            * downloaded website (after JS execution)
-            * anything you print with `console.log` (but not inside `page.execute`!)
-
-        In most cases you don't need to add any `jscode`.
-        It is executed in `page.onLoadFinished`.
-        `saveAndExit();` is mandatory, use it instead of `phantom.exit()`
-        It is possible to wait for some element on the webpage, for example:
-            var check = function() {
-              var elementFound = page.evaluate(function() {
-                return document.querySelector('#b.done') !== null;
-              });
-              if(elementFound)
-                saveAndExit();
-              else
-                window.setTimeout(check, 500);
-            }
-
-            page.evaluate(function(){
-              document.querySelector('#a').click();
-            });
-            check();
-        """
-        if 'saveAndExit();' not in jscode:
-            raise ExtractorError('`saveAndExit();` not found in `jscode`')
-        if not html:
-            html = self.extractor._download_webpage(url, video_id, note=note, headers=headers)
-        with open(self._TMP_FILES['html'].name, 'wb') as f:
-            f.write(html.encode('utf-8'))
-
-        self._save_cookies(url)
-
-        replaces = self.options
-        replaces['url'] = url
-        user_agent = headers.get('User-Agent') or std_headers['User-Agent']
-        replaces['ua'] = user_agent.replace('"', '\\"')
-        replaces['jscode'] = jscode
-
-        for x in self._TMP_FILE_NAMES:
-            replaces[x] = self._TMP_FILES[x].name.replace('\\', '\\\\').replace('"', '\\"')
-
-        with open(self._TMP_FILES['script'].name, 'wb') as f:
-            f.write(self._TEMPLATE.format(**replaces).encode('utf-8'))
-
-        if video_id is None:
-            self.extractor.to_screen('%s' % (note2,))
-        else:
-            self.extractor.to_screen('%s: %s' % (video_id, note2))
-
-        p = subprocess.Popen([
-            self.exe, '--ssl-protocol=any',
-            self._TMP_FILES['script'].name
-        ], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-        out, err = p.communicate()
-        if p.returncode != 0:
-            raise ExtractorError(
-                'Executing JS failed\n:' + encodeArgument(err))
-        with open(self._TMP_FILES['html'].name, 'rb') as f:
-            html = f.read().decode('utf-8')
-
-        self._load_cookies()
-
-        return (html, encodeArgument(out))
-
-
 def random_birthday(year_field, month_field, day_field):
     return {
         year_field: str(random.randint(1950, 1995)),
-- 
cgit v1.1


From 011da618bdb907cbe02aacefba421dad4cf49a4d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 24 Sep 2017 00:12:40 +0700
Subject: [openload] Fix _load_cookies for python 2.6

---
 youtube_dl/extractor/openload.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/openload.py b/youtube_dl/extractor/openload.py
index da1ef02..b50d6c7 100644
--- a/youtube_dl/extractor/openload.py
+++ b/youtube_dl/extractor/openload.py
@@ -8,7 +8,10 @@ import subprocess
 import tempfile
 
 from .common import InfoExtractor
-from ..compat import compat_urlparse
+from ..compat import (
+    compat_urlparse,
+    compat_kwargs,
+)
 from ..utils import (
     check_executable,
     determine_ext,
@@ -158,7 +161,7 @@ class PhantomJSwrapper(object):
                 cookie['rest'] = {'httpOnly': None}
             if 'expiry' in cookie:
                 cookie['expire_time'] = cookie['expiry']
-            self.extractor._set_cookie(**cookie)
+            self.extractor._set_cookie(**compat_kwargs(cookie))
 
     def get(self, url, html=None, video_id=None, note=None, note2='Executing JS on webpage', headers={}, jscode='saveAndExit();'):
         """
-- 
cgit v1.1


From 4d182955a2bbe95eb4b931f82204f402ffd5552d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 24 Sep 2017 00:19:27 +0700
Subject: [kakao] Fix _VALID_URL

---
 youtube_dl/extractor/kakao.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/kakao.py b/youtube_dl/extractor/kakao.py
index c9b438e..7fa140b 100644
--- a/youtube_dl/extractor/kakao.py
+++ b/youtube_dl/extractor/kakao.py
@@ -12,7 +12,7 @@ from ..utils import (
 
 
 class KakaoIE(InfoExtractor):
-    _VALID_URL = r'https?://tv.kakao.com/channel/(?P<channel>\d+)/cliplink/(?P<id>\d+)'
+    _VALID_URL = r'https?://tv\.kakao\.com/channel/(?P<channel>\d+)/cliplink/(?P<id>\d+)'
     _API_BASE = 'http://tv.kakao.com/api/v1/ft/cliplinks'
 
     _TESTS = [{
-- 
cgit v1.1


From 10cab6613fc52f64f0a5a81281fec2f8d2d92f16 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 24 Sep 2017 00:21:34 +0700
Subject: [ChangeLog] Actualize

---
 ChangeLog | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 42ba879..87ceabe 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -4,6 +4,30 @@ Core
 + [options] Accept lrc as a subtitle conversion target format (#14292)
 * [utils] Fix handling raw TTML subtitles (#14191)
 
+Extractors
+* [24video] Fix timestamp extraction and make non fatal (#14295)
++ [24video] Add support for 24video.adult (#14295)
++ [kakao] Add support for tv.kakao.com (#12298, #14007)
++ [twitter] Add support for URLs without user id (#14270)
++ [americastestkitchen] Add support for americastestkitchen.com (#10764,
+  #13996)
+* [generic] Fix support for multiple HTML5 videos on one page (#14080)
+* [mixcloud] Fix extraction (#14088, #14132)
++ [lynda] Add support for educourse.ga (#14286)
+* [beeg] Fix extraction (#14275)
+* [nbcsports:vplayer] Correct theplatform URL (#13873)
+* [twitter] Fix duration extraction (#14141)
+* [tvplay] Bypass geo restriction
++ [heise] Add support for YouTube embeds (#14109)
++ [popcorntv] Add support for popcorntv.it (#5914, #14211)
+* [viki] Update app data (#14181)
+* [morningstar] Relax URL regular expression (#14222)
+* [openload] Fix extraction (#14225, #14257)
+* [noovo] Fix extraction (#14214)
+* [dailymotion:playlist] Relax URL regular expression (#14219)
++ [twitch] Add support for go.twitch.tv URLs (#14215)
+* [vgtv] Relax URL regular expression (#14223)
+
 
 version 2017.09.15
 
-- 
cgit v1.1


From 9fc41bcb6b94ddb64885b1eb20282c69dc787929 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 24 Sep 2017 00:22:50 +0700
Subject: release 2017.09.24

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 README.md                 | 2 +-
 docs/supportedsites.md    | 3 +++
 youtube_dl/version.py     | 2 +-
 5 files changed, 9 insertions(+), 6 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 98ab5b6..1954528 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.09.15*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.09.15**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.09.24*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.09.24**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through the [README](https://github.com/rg3/youtube-dl/blob/master/README.md), **most notably** the [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ Add the `-v` flag to **your command line** you run youtube-dl with (`youtube-dl
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.09.15
+[debug] youtube-dl version 2017.09.24
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index 87ceabe..da60c1b 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.09.24
 
 Core
 + [options] Accept lrc as a subtitle conversion target format (#14292)
diff --git a/README.md b/README.md
index 28ee63f..7818e58 100644
--- a/README.md
+++ b/README.md
@@ -427,7 +427,7 @@ Alternatively, refer to the [developer instructions](#developer-instructions) fo
                                      syntax. Example: --exec 'adb push {}
                                      /sdcard/Music/ && rm {}'
     --convert-subs FORMAT            Convert the subtitles to other format
-                                     (currently supported: srt|ass|vtt)
+                                     (currently supported: srt|ass|vtt|lrc)
 
 # CONFIGURATION
 
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 6b01dc9..d36a07c 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -43,6 +43,7 @@
  - **Allocine**
  - **AlphaPorno**
  - **AMCNetworks**
+ - **AmericasTestKitchen**
  - **anderetijden**: npo.nl, ntr.nl, omroepwnl.nl, zapp.nl and npo3.nl
  - **AnimeOnDemand**
  - **anitube.se**
@@ -378,6 +379,7 @@
  - **Jove**
  - **jpopsuki.tv**
  - **JWPlatform**
+ - **Kakao**
  - **Kaltura**
  - **Kamcord**
  - **KanalPlay**: Kanal 5/9/11 Play
@@ -627,6 +629,7 @@
  - **Pokemon**
  - **PolskieRadio**
  - **PolskieRadioCategory**
+ - **PopcornTV**
  - **PornCom**
  - **PornerBros**
  - **PornFlip**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 8399c04..0e2e4dc 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.09.15'
+__version__ = '2017.09.24'
-- 
cgit v1.1


From 8b389f7e3ce4466824abc7ce2aaf657abf34a682 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 24 Sep 2017 18:21:38 +0700
Subject: Credit the author of multiple generic HTML5 embeds fix

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 1d40c86..7e01224 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -230,3 +230,4 @@ Ashutosh Chaudhary
 John Dong
 Tatsuyuki Ishi
 Daniel Weber
+Kay Bouché
-- 
cgit v1.1


From db96252831e24ae040de92f4c6668d3eaef82165 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 24 Sep 2017 19:23:08 +0700
Subject: [xhamsterembed] Fix extraction (closes #14308)

---
 youtube_dl/extractor/xhamster.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/xhamster.py b/youtube_dl/extractor/xhamster.py
index c42b59e..be3624e 100644
--- a/youtube_dl/extractor/xhamster.py
+++ b/youtube_dl/extractor/xhamster.py
@@ -221,7 +221,7 @@ class XHamsterEmbedIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
 
         video_url = self._search_regex(
-            r'href="(https?://xhamster\.com/movies/%s/[^"]*\.html[^"]*)"' % video_id,
+            r'href="(https?://xhamster\.com/(?:movies/{0}/[^"]*\.html|videos/[^/]*-{0})[^"]*)"'.format(video_id),
             webpage, 'xhamster url', default=None)
 
         if not video_url:
-- 
cgit v1.1


From 02d01e15f1a52a23cc4991c39219f98aed542136 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 26 Sep 2017 21:47:18 +0700
Subject: [generic] Fix infinite recursion for twitter:player URLs (closes
 #14339)

---
 youtube_dl/extractor/generic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 7d0edf0..096e2aa 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -2973,7 +2973,7 @@ class GenericIE(InfoExtractor):
             # be supported by youtube-dl thus this is checked the very last (see
             # https://dev.twitter.com/cards/types/player#On_twitter.com_via_desktop_browser)
             embed_url = self._html_search_meta('twitter:player', webpage, default=None)
-            if embed_url:
+            if embed_url and embed_url != url:
                 return self.url_result(embed_url)
 
         if not found:
-- 
cgit v1.1


From b14b2283a0b9f66f54e862641b5b1cd22c07bd62 Mon Sep 17 00:00:00 2001
From: Timendum <timedum@gmail.com>
Date: Wed, 27 Sep 2017 17:48:47 +0200
Subject: [gfycat] Add support for /gifs/detail URLs (closes #14322)

---
 youtube_dl/extractor/gfycat.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/gfycat.py b/youtube_dl/extractor/gfycat.py
index 45ccc11..a0670b6 100644
--- a/youtube_dl/extractor/gfycat.py
+++ b/youtube_dl/extractor/gfycat.py
@@ -11,7 +11,7 @@ from ..utils import (
 
 
 class GfycatIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?gfycat\.com/(?:ifr/)?(?P<id>[^/?#]+)'
+    _VALID_URL = r'https?://(?:www\.)?gfycat\.com/(?:ifr/|gifs/detail/)?(?P<id>[^/?#]+)'
     _TESTS = [{
         'url': 'http://gfycat.com/DeadlyDecisiveGermanpinscher',
         'info_dict': {
@@ -44,6 +44,9 @@ class GfycatIE(InfoExtractor):
             'categories': list,
             'age_limit': 0,
         }
+    }, {
+        'url': 'https://gfycat.com/gifs/detail/UnconsciousLankyIvorygull',
+        'only_matching': True
     }]
 
     def _real_extract(self, url):
-- 
cgit v1.1


From 63d990d2859d0e981da2e416097655798334431b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 28 Sep 2017 00:29:42 +0700
Subject: [generic] Add support for Video.js embeds

---
 youtube_dl/extractor/generic.py | 56 +++++++++++++++++++++++++++++++++++++++++
 1 file changed, 56 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 096e2aa..a3d09a0 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -22,6 +22,8 @@ from ..utils import (
     HEADRequest,
     is_html,
     js_to_json,
+    KNOWN_EXTENSIONS,
+    mimetype2ext,
     orderedSet,
     sanitized_Request,
     smuggle_url,
@@ -1130,6 +1132,22 @@ class GenericIE(InfoExtractor):
                 'skip_download': True,
             }
         },
+        {
+            # Video.js embed
+            'url': 'http://ortcam.com/solidworks-урок-6-настройка-чертежа_33f9b7351.html',
+            'info_dict': {
+                'id': 'yygqldloqIk',
+                'ext': 'mp4',
+                'title': 'SolidWorks. Урок 6 Настройка чертежа',
+                'description': 'md5:baf95267792646afdbf030e4d06b2ab3',
+                'upload_date': '20130314',
+                'uploader': 'PROстое3D',
+                'uploader_id': 'PROstoe3D',
+            },
+            'params': {
+                'skip_download': True,
+            },
+        },
         # rtl.nl embed
         {
             'url': 'http://www.rtlnieuws.nl/nieuws/buitenland/aanslagen-kopenhagen',
@@ -2880,6 +2898,44 @@ class GenericIE(InfoExtractor):
                 jwplayer_data, video_id, require_title=False, base_url=url)
             return merge_dicts(info, info_dict)
 
+        # Video.js embed
+        mobj = re.search(
+            r'(?s)\bvideojs\s*\(.+?\bplayer\.src\s*\(\s*(\[.+?\])\s*\)\s*;',
+            webpage)
+        if mobj is not None:
+            sources = self._parse_json(
+                mobj.group(1), video_id, transform_source=js_to_json,
+                fatal=False) or []
+            formats = []
+            for source in sources:
+                src = source.get('src')
+                if not src or not isinstance(src, compat_str):
+                    continue
+                src = compat_urlparse.urljoin(url, src)
+                src_type = source.get('type')
+                if isinstance(src_type, compat_str):
+                    src_type = src_type.lower()
+                ext = determine_ext(src).lower()
+                if src_type == 'video/youtube':
+                    return self.url_result(src, YoutubeIE.ie_key())
+                if src_type == 'application/dash+xml' or ext == 'mpd':
+                    formats.extend(self._extract_mpd_formats(
+                        src, video_id, mpd_id='dash', fatal=False))
+                elif src_type == 'application/x-mpegurl' or ext == 'm3u8':
+                    formats.extend(self._extract_m3u8_formats(
+                        src, video_id, 'mp4', entry_protocol='m3u8_native',
+                        m3u8_id='hls', fatal=False))
+                else:
+                    formats.append({
+                        'url': src,
+                        'ext': (mimetype2ext(src_type) or
+                                ext if ext in KNOWN_EXTENSIONS else 'mp4'),
+                    })
+            if formats:
+                self._sort_formats(formats)
+                info_dict['formats'] = formats
+                return info_dict
+
         # Looking for http://schema.org/VideoObject
         json_ld = self._search_json_ld(
             webpage, video_id, default={}, expected_type='VideoObject')
-- 
cgit v1.1


From 3600fd591dc1ee4963f6940d19ef39d464d3a689 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 28 Sep 2017 00:46:48 +0700
Subject: [YoutubeDL] Document youtube_include_dash_manifest

---
 youtube_dl/YoutubeDL.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 0a7f36c..20aa1e4 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -304,6 +304,12 @@ class YoutubeDL(object):
                        otherwise prefer avconv.
     postprocessor_args: A list of additional command-line arguments for the
                         postprocessor.
+                        
+    The following options are used by the Youtube extractor:
+    youtube_include_dash_manifest: If True (default), DASH manifests and related
+                        data will be downloaded and processed by extractor.
+                        You can reduce network I/O by disabling it if you don't
+                        care about DASH.
     """
 
     _NUMERIC_FIELDS = set((
-- 
cgit v1.1


From eb9a15be6091d09f46d4306516b58cf2fe91e647 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 30 Sep 2017 22:47:03 +0700
Subject: [yahoo] Add support for custom brigthcove embeds (closes #14210)

---
 youtube_dl/extractor/yahoo.py | 29 ++++++++++++++++++++++++++---
 1 file changed, 26 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/yahoo.py b/youtube_dl/extractor/yahoo.py
index 38f82bf..e923379 100644
--- a/youtube_dl/extractor/yahoo.py
+++ b/youtube_dl/extractor/yahoo.py
@@ -28,7 +28,7 @@ from .nbc import NBCSportsVPlayerIE
 
 class YahooIE(InfoExtractor):
     IE_DESC = 'Yahoo screen and movies'
-    _VALID_URL = r'(?P<url>(?P<host>https?://(?:[a-zA-Z]{2}\.)?[\da-zA-Z_-]+\.yahoo\.com)/(?:[^/]+/)*(?P<display_id>.+)?-(?P<id>[0-9]+)(?:-[a-z]+)?(?:\.html)?)'
+    _VALID_URL = r'(?P<host>https?://(?:[a-zA-Z]{2}\.)?[\da-zA-Z_-]+\.yahoo\.com)/(?:[^/]+/)*(?:(?P<display_id>.+)?-)?(?P<id>[0-9]+)(?:-[a-z]+)?(?:\.html)?'
     _TESTS = [
         {
             'url': 'http://screen.yahoo.com/julian-smith-travis-legg-watch-214727115.html',
@@ -227,13 +227,28 @@ class YahooIE(InfoExtractor):
                 'skip_download': True,
             },
         },
+        {
+            # custom brightcove
+            'url': 'https://au.tv.yahoo.com/plus7/sunrise/-/watch/37083565/clown-entertainers-say-it-is-hurting-their-business/',
+            'info_dict': {
+                'id': '5575377707001',
+                'ext': 'mp4',
+                'title': "Clown entertainers say 'It' is hurting their business",
+                'description': 'Stephen King s horror film has much to answer for. Jelby and Mr Loopy the Clowns join us.',
+                'timestamp': 1505341164,
+                'upload_date': '20170913',
+                'uploader_id': '2376984109001',
+            },
+            'params': {
+                'skip_download': True,
+            },
+        }
     ]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
-        display_id = mobj.group('display_id') or self._match_id(url)
         page_id = mobj.group('id')
-        url = mobj.group('url')
+        display_id = mobj.group('display_id') or page_id
         host = mobj.group('host')
         webpage, urlh = self._download_webpage_handle(url, display_id)
         if 'err=404' in urlh.geturl():
@@ -262,6 +277,14 @@ class YahooIE(InfoExtractor):
         if bc_url:
             return self.url_result(bc_url, BrightcoveNewIE.ie_key())
 
+        brightcove_id = self._search_regex(
+            r'<iframe[^>]+data-video-id=["\'](\d+)', webpage, 'brightcove id',
+            default=None)
+        if brightcove_id:
+            return self.url_result(
+                'http://players.brightcove.net/2376984109001/default_default/index.html?videoId=%s' % brightcove_id,
+                BrightcoveNewIE.ie_key())
+
         # Query result is often embedded in webpage as JSON. Sometimes explicit requests
         # to video API results in a failure with geo restriction reason therefore using
         # embedded query result when present sounds reasonable.
-- 
cgit v1.1


From fa3fdeb41fdd637283baeb6d681450a982043a1c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 30 Sep 2017 22:54:22 +0700
Subject: [yahoo] Fix some tests

---
 youtube_dl/extractor/yahoo.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/yahoo.py b/youtube_dl/extractor/yahoo.py
index e923379..a3bb1b0 100644
--- a/youtube_dl/extractor/yahoo.py
+++ b/youtube_dl/extractor/yahoo.py
@@ -50,6 +50,7 @@ class YahooIE(InfoExtractor):
                 'description': 'md5:66b627ab0a282b26352136ca96ce73c1',
                 'duration': 151,
             },
+            'skip': 'HTTP Error 404',
         },
         {
             'url': 'https://screen.yahoo.com/community/community-sizzle-reel-203225340.html?format=embed',
@@ -142,7 +143,7 @@ class YahooIE(InfoExtractor):
             'skip': 'Domain name in.lifestyle.yahoo.com gone',
         }, {
             'url': 'https://www.yahoo.com/movies/v/true-story-trailer-173000497.html',
-            'md5': '2a9752f74cb898af5d1083ea9f661b58',
+            'md5': '989396ae73d20c6f057746fb226aa215',
             'info_dict': {
                 'id': '071c4013-ce30-3a93-a5b2-e0413cd4a9d1',
                 'ext': 'mp4',
-- 
cgit v1.1


From 3836b02ce8df654ea70276f1d196b347ea9bc257 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 30 Sep 2017 22:56:40 +0700
Subject: [YoutubeDL] PEP 8

---
 youtube_dl/YoutubeDL.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 20aa1e4..474d6c9 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -304,7 +304,7 @@ class YoutubeDL(object):
                        otherwise prefer avconv.
     postprocessor_args: A list of additional command-line arguments for the
                         postprocessor.
-                        
+
     The following options are used by the Youtube extractor:
     youtube_include_dash_manifest: If True (default), DASH manifests and related
                         data will be downloaded and processed by extractor.
-- 
cgit v1.1


From 2c53bd51c6b0bef836d4e84d9a8f9127c3b67660 Mon Sep 17 00:00:00 2001
From: Giuseppe Fabiano <gfabiano40@gmail.com>
Date: Sat, 30 Sep 2017 22:21:17 +0200
Subject: [rtve:alacarta] Fix extraction (closes #14290)

---
 youtube_dl/extractor/rtve.py | 36 +++++++++++++++++++++++++++++++++---
 1 file changed, 33 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/rtve.py b/youtube_dl/extractor/rtve.py
index 746677a..d9edf9d 100644
--- a/youtube_dl/extractor/rtve.py
+++ b/youtube_dl/extractor/rtve.py
@@ -10,6 +10,7 @@ from ..compat import (
     compat_struct_unpack,
 )
 from ..utils import (
+    determine_ext,
     ExtractorError,
     float_or_none,
     remove_end,
@@ -85,6 +86,18 @@ class RTVEALaCartaIE(InfoExtractor):
         },
         'skip': 'The f4m manifest can\'t be used yet',
     }, {
+        'url': 'http://www.rtve.es/alacarta/videos/servir-y-proteger/servir-proteger-capitulo-104/4236788/',
+        'md5': 'e55e162379ad587e9640eda4f7353c0f',
+        'info_dict': {
+            'id': '4236788',
+            'ext': 'mp4',
+            'title': 'Servir y proteger - Capítulo 104 ',
+            'duration': 3222.0,
+        },
+        'params': {
+            'skip_download': True,  # requires ffmpeg
+        },
+    }, {
         'url': 'http://www.rtve.es/m/alacarta/videos/cuentame-como-paso/cuentame-como-paso-t16-ultimo-minuto-nuestra-vida-capitulo-276/2969138/?media=tve',
         'only_matching': True,
     }, {
@@ -107,24 +120,41 @@ class RTVEALaCartaIE(InfoExtractor):
             video_id)['page']['items'][0]
         if info['state'] == 'DESPU':
             raise ExtractorError('The video is no longer available', expected=True)
+        title = info['title']
         png_url = 'http://www.rtve.es/ztnr/movil/thumbnail/%s/videos/%s.png' % (self._manager, video_id)
         png_request = sanitized_Request(png_url)
         png_request.add_header('Referer', url)
         png = self._download_webpage(png_request, video_id, 'Downloading url information')
         video_url = _decrypt_url(png)
-        if not video_url.endswith('.f4m'):
+        ext = determine_ext(video_url)
+
+        formats = []
+        if not video_url.endswith('.f4m') and ext != 'm3u8':
             if '?' not in video_url:
                 video_url = video_url.replace('resources/', 'auth/resources/')
             video_url = video_url.replace('.net.rtve', '.multimedia.cdn.rtve')
 
+        if ext == 'm3u8':
+            formats.extend(self._extract_m3u8_formats(
+                video_url, video_id, ext='mp4', entry_protocol='m3u8_native',
+                m3u8_id='hls', fatal=False))
+        elif ext == 'f4m':
+            formats.extend(self._extract_f4m_formats(
+                video_url, video_id, f4m_id='hds', fatal=False))
+        else:
+            formats.append({
+                'url': video_url,
+            })
+        self._sort_formats(formats)
+
         subtitles = None
         if info.get('sbtFile') is not None:
             subtitles = self.extract_subtitles(video_id, info['sbtFile'])
 
         return {
             'id': video_id,
-            'title': info['title'],
-            'url': video_url,
+            'title': title,
+            'formats': formats,
             'thumbnail': info.get('image'),
             'page_url': url,
             'subtitles': subtitles,
-- 
cgit v1.1


From b69ca0ccfcd3bc7b64aa227339e92576baf5fc9a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 1 Oct 2017 04:37:42 +0700
Subject: [yahoo] Use extracted brightcove account id (closes #14210)

---
 youtube_dl/extractor/yahoo.py | 27 ++++++++++++++++++---------
 1 file changed, 18 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/yahoo.py b/youtube_dl/extractor/yahoo.py
index a3bb1b0..87615cd 100644
--- a/youtube_dl/extractor/yahoo.py
+++ b/youtube_dl/extractor/yahoo.py
@@ -12,11 +12,12 @@ from ..compat import (
 )
 from ..utils import (
     clean_html,
-    unescapeHTML,
+    determine_ext,
     ExtractorError,
+    extract_attributes,
     int_or_none,
     mimetype2ext,
-    determine_ext,
+    unescapeHTML,
 )
 
 from .brightcove import (
@@ -278,13 +279,21 @@ class YahooIE(InfoExtractor):
         if bc_url:
             return self.url_result(bc_url, BrightcoveNewIE.ie_key())
 
-        brightcove_id = self._search_regex(
-            r'<iframe[^>]+data-video-id=["\'](\d+)', webpage, 'brightcove id',
-            default=None)
-        if brightcove_id:
-            return self.url_result(
-                'http://players.brightcove.net/2376984109001/default_default/index.html?videoId=%s' % brightcove_id,
-                BrightcoveNewIE.ie_key())
+        brightcove_iframe = self._search_regex(
+            r'(<iframe[^>]+data-video-id=["\']\d+[^>]+>)', webpage,
+            'brightcove iframe', default=None)
+        if brightcove_iframe:
+            attr = extract_attributes(brightcove_iframe)
+            src = attr.get('src')
+            if src:
+                parsed_src = compat_urlparse.urlparse(src)
+                qs = compat_urlparse.parse_qs(parsed_src.query)
+                account_id = qs.get('accountId', ['2376984109001'])[0]
+                brightcove_id = attr.get('data-video-id') or qs.get('videoId', [None])[0]
+                if account_id and brightcove_id:
+                    return self.url_result(
+                        'http://players.brightcove.net/%s/default_default/index.html?videoId=%s' % (account_id, brightcove_id),
+                        BrightcoveNewIE.ie_key())
 
         # Query result is often embedded in webpage as JSON. Sometimes explicit requests
         # to video API results in a failure with geo restriction reason therefore using
-- 
cgit v1.1


From c8da40d834697c343f64609b2cfbb3ff759c18a4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 1 Oct 2017 04:49:27 +0700
Subject: [yahoo] Bypass geo restriction for brightcove (#14210)

---
 youtube_dl/extractor/yahoo.py | 21 ++++++++++++++++-----
 1 file changed, 16 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/yahoo.py b/youtube_dl/extractor/yahoo.py
index 87615cd..552013a 100644
--- a/youtube_dl/extractor/yahoo.py
+++ b/youtube_dl/extractor/yahoo.py
@@ -17,6 +17,7 @@ from ..utils import (
     extract_attributes,
     int_or_none,
     mimetype2ext,
+    smuggle_url,
     unescapeHTML,
 )
 
@@ -29,7 +30,7 @@ from .nbc import NBCSportsVPlayerIE
 
 class YahooIE(InfoExtractor):
     IE_DESC = 'Yahoo screen and movies'
-    _VALID_URL = r'(?P<host>https?://(?:[a-zA-Z]{2}\.)?[\da-zA-Z_-]+\.yahoo\.com)/(?:[^/]+/)*(?:(?P<display_id>.+)?-)?(?P<id>[0-9]+)(?:-[a-z]+)?(?:\.html)?'
+    _VALID_URL = r'(?P<host>https?://(?:(?P<country>[a-zA-Z]{2})\.)?[\da-zA-Z_-]+\.yahoo\.com)/(?:[^/]+/)*(?:(?P<display_id>.+)?-)?(?P<id>[0-9]+)(?:-[a-z]+)?(?:\.html)?'
     _TESTS = [
         {
             'url': 'http://screen.yahoo.com/julian-smith-travis-legg-watch-214727115.html',
@@ -244,6 +245,11 @@ class YahooIE(InfoExtractor):
             'params': {
                 'skip_download': True,
             },
+        },
+        {
+            # custom brightcove, geo-restricted to Australia, bypassable
+            'url': 'https://au.tv.yahoo.com/plus7/sunrise/-/watch/37263964/sunrise-episode-wed-27-sep/',
+            'only_matching': True,
         }
     ]
 
@@ -274,10 +280,15 @@ class YahooIE(InfoExtractor):
         if bc_url:
             return self.url_result(bc_url, BrightcoveLegacyIE.ie_key())
 
+        def brightcove_url_result(bc_url):
+            return self.url_result(
+                smuggle_url(bc_url, {'geo_countries': [mobj.group('country')]}),
+                BrightcoveNewIE.ie_key())
+
         # Look for Brightcove New Studio embeds
         bc_url = BrightcoveNewIE._extract_url(self, webpage)
         if bc_url:
-            return self.url_result(bc_url, BrightcoveNewIE.ie_key())
+            return brightcove_url_result(bc_url)
 
         brightcove_iframe = self._search_regex(
             r'(<iframe[^>]+data-video-id=["\']\d+[^>]+>)', webpage,
@@ -291,9 +302,9 @@ class YahooIE(InfoExtractor):
                 account_id = qs.get('accountId', ['2376984109001'])[0]
                 brightcove_id = attr.get('data-video-id') or qs.get('videoId', [None])[0]
                 if account_id and brightcove_id:
-                    return self.url_result(
-                        'http://players.brightcove.net/%s/default_default/index.html?videoId=%s' % (account_id, brightcove_id),
-                        BrightcoveNewIE.ie_key())
+                    return brightcove_url_result(
+                        'http://players.brightcove.net/%s/default_default/index.html?videoId=%s'
+                        % (account_id, brightcove_id))
 
         # Query result is often embedded in webpage as JSON. Sometimes explicit requests
         # to video API results in a failure with geo restriction reason therefore using
-- 
cgit v1.1


From c5b7014a9c936ab8cd0c3b8514a278575c414aa1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 1 Oct 2017 07:01:21 +0700
Subject: [generic] Add support for single format Video.js embeds (closes
 #14371)

---
 youtube_dl/extractor/generic.py | 19 +++++++++++++++++--
 1 file changed, 17 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index a3d09a0..1721a3d 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1133,7 +1133,7 @@ class GenericIE(InfoExtractor):
             }
         },
         {
-            # Video.js embed
+            # Video.js embed, multiple formats
             'url': 'http://ortcam.com/solidworks-урок-6-настройка-чертежа_33f9b7351.html',
             'info_dict': {
                 'id': 'yygqldloqIk',
@@ -1148,6 +1148,19 @@ class GenericIE(InfoExtractor):
                 'skip_download': True,
             },
         },
+        {
+            # Video.js embed, single format
+            'url': 'https://www.vooplayer.com/v3/watch/watch.php?v=NzgwNTg=',
+            'info_dict': {
+                'id': 'watch',
+                'ext': 'mp4',
+                'title': 'Step 1 -  Good Foundation',
+                'description': 'md5:d1e7ff33a29fc3eb1673d6c270d344f4',
+            },
+            'params': {
+                'skip_download': True,
+            },
+        },
         # rtl.nl embed
         {
             'url': 'http://www.rtlnieuws.nl/nieuws/buitenland/aanslagen-kopenhagen',
@@ -2900,12 +2913,14 @@ class GenericIE(InfoExtractor):
 
         # Video.js embed
         mobj = re.search(
-            r'(?s)\bvideojs\s*\(.+?\bplayer\.src\s*\(\s*(\[.+?\])\s*\)\s*;',
+            r'(?s)\bvideojs\s*\(.+?\.src\s*\(\s*((?:\[.+?\]|{.+?}))\s*\)\s*;',
             webpage)
         if mobj is not None:
             sources = self._parse_json(
                 mobj.group(1), video_id, transform_source=js_to_json,
                 fatal=False) or []
+            if not isinstance(sources, list):
+                sources = [sources]
             formats = []
             for source in sources:
                 src = source.get('src')
-- 
cgit v1.1


From 4e599194d6cf3dbe214dceb3c66ba5f718c6579a Mon Sep 17 00:00:00 2001
From: Rafal Borczuch <rafalq.b+github@gmail.com>
Date: Sun, 1 Oct 2017 12:59:00 +0100
Subject: [tvp] Add support for new URL schema (closes #14368)

---
 youtube_dl/extractor/tvp.py | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/tvp.py b/youtube_dl/extractor/tvp.py
index c5b3288..3954f0b 100644
--- a/youtube_dl/extractor/tvp.py
+++ b/youtube_dl/extractor/tvp.py
@@ -15,16 +15,16 @@ from ..utils import (
 class TVPIE(InfoExtractor):
     IE_NAME = 'tvp'
     IE_DESC = 'Telewizja Polska'
-    _VALID_URL = r'https?://[^/]+\.tvp\.(?:pl|info)/(?:(?!\d+/)[^/]+/)*(?P<id>\d+)'
+    _VALID_URL = r'https?://[^/]+\.tvp\.(?:pl|info)/(?:video/(?:[^,\s]*,)*|(?:(?!\d+/)[^/]+/)*)(?P<id>\d+)'
 
     _TESTS = [{
-        'url': 'http://vod.tvp.pl/194536/i-seria-odc-13',
+        'url': 'https://vod.tvp.pl/video/czas-honoru,i-seria-odc-13,194536',
         'md5': '8aa518c15e5cc32dfe8db400dc921fbb',
         'info_dict': {
             'id': '194536',
             'ext': 'mp4',
             'title': 'Czas honoru, I seria – odc. 13',
-            'description': 'md5:76649d2014f65c99477be17f23a4dead',
+            'description': 'md5:381afa5bca72655fe94b05cfe82bf53d',
         },
     }, {
         'url': 'http://www.tvp.pl/there-can-be-anything-so-i-shortened-it/17916176',
@@ -37,12 +37,13 @@ class TVPIE(InfoExtractor):
         },
     }, {
         # page id is not the same as video id(#7799)
-        'url': 'http://vod.tvp.pl/22704887/08122015-1500',
-        'md5': 'cf6a4705dfd1489aef8deb168d6ba742',
+        'url': 'https://wiadomosci.tvp.pl/33908820/28092017-1930',
+        'md5': '84cd3c8aec4840046e5ab712416b73d0',
         'info_dict': {
-            'id': '22680786',
+            'id': '33908820',
             'ext': 'mp4',
-            'title': 'Wiadomości, 08.12.2015, 15:00',
+            'title': 'Wiadomości, 28.09.2017, 19:30',
+            'description': 'Wydanie główne codziennego serwisu informacyjnego.'
         },
     }, {
         'url': 'http://vod.tvp.pl/seriale/obyczajowe/na-sygnale/sezon-2-27-/odc-39/17834272',
-- 
cgit v1.1


From 1dd126180eb7c3266af5480a95a3e5c4eb81ed1d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 1 Oct 2017 21:45:56 +0700
Subject: [ChangeLog] Actualize

---
 ChangeLog | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index da60c1b..66bb790 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,21 @@
+version <unreleased>
+
+Core
+* [YoutubeDL] Document youtube_include_dash_manifest
+
+Extractors
++ [tvp] Add support for new URL schema (#14368)
++ [generic] Add support for single format Video.js embeds (#14371)
+* [yahoo] Bypass geo restriction for brightcove (#14210)
+* [yahoo] Use extracted brightcove account id (#14210)
+* [rtve:alacarta] Fix extraction (#14290)
++ [yahoo] Add support for custom brigthcove embeds (#14210)
++ [generic] Add support for Video.js embeds
++ [gfycat] Add support for /gifs/detail URLs (#14322)
+* [generic] Fix infinite recursion for twitter:player URLs (#14339)
+* [xhamsterembed] Fix extraction (#14308)
+
+
 version 2017.09.24
 
 Core
-- 
cgit v1.1


From fcdd37d053c97c1bc363635c46634c778ce3eee5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 1 Oct 2017 21:54:11 +0700
Subject: release 2017.10.01

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 youtube_dl/version.py     | 2 +-
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 1954528..3be3062 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.09.24*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.09.24**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.10.01*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.10.01**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through the [README](https://github.com/rg3/youtube-dl/blob/master/README.md), **most notably** the [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ Add the `-v` flag to **your command line** you run youtube-dl with (`youtube-dl
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.09.24
+[debug] youtube-dl version 2017.10.01
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index 66bb790..80299d5 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.10.01
 
 Core
 * [YoutubeDL] Document youtube_include_dash_manifest
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 0e2e4dc..49fa02d 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.09.24'
+__version__ = '2017.10.01'
-- 
cgit v1.1


From 839728f5bfad3a6166be3839009b13963f00dfac Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 2 Oct 2017 03:28:25 +0700
Subject: [afreecatv] Add support for adult videos (closes #14376)

---
 youtube_dl/extractor/afreecatv.py | 27 ++++++++++++++++++++++++++-
 1 file changed, 26 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/afreecatv.py b/youtube_dl/extractor/afreecatv.py
index c8cb91d..2c58f46 100644
--- a/youtube_dl/extractor/afreecatv.py
+++ b/youtube_dl/extractor/afreecatv.py
@@ -139,6 +139,23 @@ class AfreecaTVIE(InfoExtractor):
             'skip_download': True,
         },
     }, {
+        # adult video
+        'url': 'http://vod.afreecatv.com/PLAYER/STATION/26542731',
+        'info_dict': {
+            'id': '20171001_F1AE1711_196617479_1',
+            'ext': 'mp4',
+            'title': '[생]서아 초심 찾기 방송 (part 1)',
+            'thumbnail': 're:^https?://(?:video|st)img.afreecatv.com/.*$',
+            'uploader': 'BJ서아',
+            'uploader_id': 'bjdyrksu',
+            'upload_date': '20171001',
+            'duration': 3600,
+            'age_limit': 18,
+        },
+        'params': {
+            'skip_download': True,
+        },
+    }, {
         'url': 'http://www.afreecatv.com/player/Player.swf?szType=szBjId=djleegoon&nStationNo=11273158&nBbsNo=13161095&nTitleNo=36327652',
         'only_matching': True,
     }, {
@@ -160,7 +177,15 @@ class AfreecaTVIE(InfoExtractor):
 
         video_xml = self._download_xml(
             'http://afbbs.afreecatv.com:8080/api/video/get_video_info.php',
-            video_id, query={'nTitleNo': video_id})
+            video_id, query={
+                'nTitleNo': video_id,
+                'partialView': 'SKIP_ADULT',
+            })
+
+        flag = xpath_text(video_xml, './track/flag', 'flag', default=None)
+        if flag and flag != 'SUCCEED':
+            raise ExtractorError(
+                '%s said: %s' % (self.IE_NAME, flag), expected=True)
 
         video_element = video_xml.findall(compat_xpath('./track/video'))[1]
         if video_element is None or video_element.text is None:
-- 
cgit v1.1


From 117589dfa2ac32566dc705ee7e712139105e6dd8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 2 Oct 2017 04:14:36 +0700
Subject: [canvas] Generalize mediazone.vrt.be extractor and rework canvas and
 een

---
 youtube_dl/extractor/canvas.py     | 142 +++++++++++++++++++++++++------------
 youtube_dl/extractor/extractors.py |   5 +-
 2 files changed, 99 insertions(+), 48 deletions(-)

diff --git a/youtube_dl/extractor/canvas.py b/youtube_dl/extractor/canvas.py
index aada029..6899f84 100644
--- a/youtube_dl/extractor/canvas.py
+++ b/youtube_dl/extractor/canvas.py
@@ -3,24 +3,104 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import float_or_none
+from ..utils import (
+    float_or_none,
+    strip_or_none,
+)
 
 
 class CanvasIE(InfoExtractor):
+    _VALID_URL = r'https?://mediazone\.vrt\.be/api/v1/(?P<site_id>canvas|een|ketnet)/assets/(?P<id>m[dz]-ast-[^/?#&]+)'
+    _TESTS = [{
+        'url': 'https://mediazone.vrt.be/api/v1/ketnet/assets/md-ast-4ac54990-ce66-4d00-a8ca-9eac86f4c475',
+        'md5': '90139b746a0a9bd7bb631283f6e2a64e',
+        'info_dict': {
+            'id': 'md-ast-4ac54990-ce66-4d00-a8ca-9eac86f4c475',
+            'display_id': 'md-ast-4ac54990-ce66-4d00-a8ca-9eac86f4c475',
+            'ext': 'flv',
+            'title': 'Nachtwacht: De Greystook',
+            'description': 'md5:1db3f5dc4c7109c821261e7512975be7',
+            'thumbnail': r're:^https?://.*\.jpg$',
+            'duration': 1468.03,
+        },
+        'expected_warnings': ['is not a supported codec', 'Unknown MIME type'],
+    }, {
+        'url': 'https://mediazone.vrt.be/api/v1/canvas/assets/mz-ast-5e5f90b6-2d72-4c40-82c2-e134f884e93e',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        site_id, video_id = mobj.group('site_id'), mobj.group('id')
+
+        data = self._download_json(
+            'https://mediazone.vrt.be/api/v1/%s/assets/%s'
+            % (site_id, video_id), video_id)
+
+        title = data['title']
+        description = data.get('description')
+
+        formats = []
+        for target in data['targetUrls']:
+            format_url, format_type = target.get('url'), target.get('type')
+            if not format_url or not format_type:
+                continue
+            if format_type == 'HLS':
+                formats.extend(self._extract_m3u8_formats(
+                    format_url, video_id, 'mp4', entry_protocol='m3u8_native',
+                    m3u8_id=format_type, fatal=False))
+            elif format_type == 'HDS':
+                formats.extend(self._extract_f4m_formats(
+                    format_url, video_id, f4m_id=format_type, fatal=False))
+            elif format_type == 'MPEG_DASH':
+                formats.extend(self._extract_mpd_formats(
+                    format_url, video_id, mpd_id=format_type, fatal=False))
+            elif format_type == 'HSS':
+                formats.extend(self._extract_ism_formats(
+                    format_url, video_id, ism_id='mss', fatal=False))
+            else:
+                formats.append({
+                    'format_id': format_type,
+                    'url': format_url,
+                })
+        self._sort_formats(formats)
+
+        subtitles = {}
+        subtitle_urls = data.get('subtitleUrls')
+        if isinstance(subtitle_urls, list):
+            for subtitle in subtitle_urls:
+                subtitle_url = subtitle.get('url')
+                if subtitle_url and subtitle.get('type') == 'CLOSED':
+                    subtitles.setdefault('nl', []).append({'url': subtitle_url})
+
+        return {
+            'id': video_id,
+            'display_id': video_id,
+            'title': title,
+            'description': description,
+            'formats': formats,
+            'duration': float_or_none(data.get('duration'), 1000),
+            'thumbnail': data.get('posterImageUrl'),
+            'subtitles': subtitles,
+        }
+
+
+class CanvasEenIE(InfoExtractor):
     IE_DESC = 'canvas.be and een.be'
     _VALID_URL = r'https?://(?:www\.)?(?P<site_id>canvas|een)\.be/(?:[^/]+/)*(?P<id>[^/?#&]+)'
     _TESTS = [{
         'url': 'http://www.canvas.be/video/de-afspraak/najaar-2015/de-afspraak-veilt-voor-de-warmste-week',
-        'md5': 'ea838375a547ac787d4064d8c7860a6c',
+        'md5': 'ed66976748d12350b118455979cca293',
         'info_dict': {
             'id': 'mz-ast-5e5f90b6-2d72-4c40-82c2-e134f884e93e',
             'display_id': 'de-afspraak-veilt-voor-de-warmste-week',
-            'ext': 'mp4',
+            'ext': 'flv',
             'title': 'De afspraak veilt voor de Warmste Week',
             'description': 'md5:24cb860c320dc2be7358e0e5aa317ba6',
             'thumbnail': r're:^https?://.*\.jpg$',
             'duration': 49.02,
-        }
+        },
+        'expected_warnings': ['is not a supported codec'],
     }, {
         # with subtitles
         'url': 'http://www.canvas.be/video/panorama/2016/pieter-0167',
@@ -40,7 +120,8 @@ class CanvasIE(InfoExtractor):
         },
         'params': {
             'skip_download': True,
-        }
+        },
+        'skip': 'Pagina niet gevonden',
     }, {
         'url': 'https://www.een.be/sorry-voor-alles/herbekijk-sorry-voor-alles',
         'info_dict': {
@@ -54,7 +135,8 @@ class CanvasIE(InfoExtractor):
         },
         'params': {
             'skip_download': True,
-        }
+        },
+        'skip': 'Episode no longer available',
     }, {
         'url': 'https://www.canvas.be/check-point/najaar-2016/de-politie-uw-vriend',
         'only_matching': True,
@@ -66,55 +148,21 @@ class CanvasIE(InfoExtractor):
 
         webpage = self._download_webpage(url, display_id)
 
-        title = (self._search_regex(
+        title = strip_or_none(self._search_regex(
             r'<h1[^>]+class="video__body__header__title"[^>]*>(.+?)</h1>',
             webpage, 'title', default=None) or self._og_search_title(
-            webpage)).strip()
+            webpage, default=None))
 
         video_id = self._html_search_regex(
-            r'data-video=(["\'])(?P<id>(?:(?!\1).)+)\1', webpage, 'video id', group='id')
-
-        data = self._download_json(
-            'https://mediazone.vrt.be/api/v1/%s/assets/%s'
-            % (site_id, video_id), display_id)
-
-        formats = []
-        for target in data['targetUrls']:
-            format_url, format_type = target.get('url'), target.get('type')
-            if not format_url or not format_type:
-                continue
-            if format_type == 'HLS':
-                formats.extend(self._extract_m3u8_formats(
-                    format_url, display_id, entry_protocol='m3u8_native',
-                    ext='mp4', preference=0, fatal=False, m3u8_id=format_type))
-            elif format_type == 'HDS':
-                formats.extend(self._extract_f4m_formats(
-                    format_url, display_id, f4m_id=format_type, fatal=False))
-            elif format_type == 'MPEG_DASH':
-                formats.extend(self._extract_mpd_formats(
-                    format_url, display_id, mpd_id=format_type, fatal=False))
-            else:
-                formats.append({
-                    'format_id': format_type,
-                    'url': format_url,
-                })
-        self._sort_formats(formats)
-
-        subtitles = {}
-        subtitle_urls = data.get('subtitleUrls')
-        if isinstance(subtitle_urls, list):
-            for subtitle in subtitle_urls:
-                subtitle_url = subtitle.get('url')
-                if subtitle_url and subtitle.get('type') == 'CLOSED':
-                    subtitles.setdefault('nl', []).append({'url': subtitle_url})
+            r'data-video=(["\'])(?P<id>(?:(?!\1).)+)\1', webpage, 'video id',
+            group='id')
 
         return {
+            '_type': 'url_transparent',
+            'url': 'https://mediazone.vrt.be/api/v1/%s/assets/%s' % (site_id, video_id),
+            'ie_key': CanvasIE.ie_key(),
             'id': video_id,
             'display_id': display_id,
             'title': title,
             'description': self._og_search_description(webpage),
-            'formats': formats,
-            'duration': float_or_none(data.get('duration'), 1000),
-            'thumbnail': data.get('posterImageUrl'),
-            'subtitles': subtitles,
         }
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 4232a4f..24e9acd 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -150,7 +150,10 @@ from .camdemy import (
 from .camwithher import CamWithHerIE
 from .canalplus import CanalplusIE
 from .canalc2 import Canalc2IE
-from .canvas import CanvasIE
+from .canvas import (
+    CanvasIE,
+    CanvasEenIE,
+)
 from .carambatv import (
     CarambaTVIE,
     CarambaTVPageIE,
-- 
cgit v1.1


From 544ffb7790ea1ac7bbc081d2e486101c0382b900 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 2 Oct 2017 04:15:12 +0700
Subject: [ketnet] Add support for videos without direct sources (closes
 #14377)

---
 youtube_dl/extractor/ketnet.py | 23 ++++++++++++++++++++++-
 1 file changed, 22 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ketnet.py b/youtube_dl/extractor/ketnet.py
index fb9c2db..93a98e1 100644
--- a/youtube_dl/extractor/ketnet.py
+++ b/youtube_dl/extractor/ketnet.py
@@ -1,5 +1,6 @@
 from __future__ import unicode_literals
 
+from .canvas import CanvasIE
 from .common import InfoExtractor
 
 
@@ -7,7 +8,7 @@ class KetnetIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?ketnet\.be/(?:[^/]+/)*(?P<id>[^/?#&]+)'
     _TESTS = [{
         'url': 'https://www.ketnet.be/kijken/zomerse-filmpjes',
-        'md5': 'd907f7b1814ef0fa285c0475d9994ed7',
+        'md5': '6bdeb65998930251bbd1c510750edba9',
         'info_dict': {
             'id': 'zomerse-filmpjes',
             'ext': 'mp4',
@@ -16,6 +17,20 @@ class KetnetIE(InfoExtractor):
             'thumbnail': r're:^https?://.*\.jpg$',
         }
     }, {
+        # mzid in playerConfig instead of sources
+        'url': 'https://www.ketnet.be/kijken/nachtwacht/de-greystook',
+        'md5': '90139b746a0a9bd7bb631283f6e2a64e',
+        'info_dict': {
+            'id': 'md-ast-4ac54990-ce66-4d00-a8ca-9eac86f4c475',
+            'display_id': 'md-ast-4ac54990-ce66-4d00-a8ca-9eac86f4c475',
+            'ext': 'flv',
+            'title': 'Nachtwacht: De Greystook',
+            'description': 'md5:1db3f5dc4c7109c821261e7512975be7',
+            'thumbnail': r're:^https?://.*\.jpg$',
+            'duration': 1468.03,
+        },
+        'expected_warnings': ['is not a supported codec', 'Unknown MIME type'],
+    }, {
         'url': 'https://www.ketnet.be/kijken/karrewiet/uitzending-8-september-2016',
         'only_matching': True,
     }, {
@@ -38,6 +53,12 @@ class KetnetIE(InfoExtractor):
                 'player config'),
             video_id)
 
+        mzid = config.get('mzid')
+        if mzid:
+            return self.url_result(
+                'https://mediazone.vrt.be/api/v1/ketnet/assets/%s' % mzid,
+                CanvasIE.ie_key(), video_id=mzid)
+
         title = config['title']
 
         formats = []
-- 
cgit v1.1


From d2ae7e24e5c574fa45621771a134e58b21443e5f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 2 Oct 2017 04:43:25 +0700
Subject: [postprocessor/ffmpeg] Convert to opus using libopus (closes #14381)

---
 youtube_dl/postprocessor/ffmpeg.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index f71d413..3ea1afc 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -44,7 +44,7 @@ ACODECS = {
     'aac': 'aac',
     'flac': 'flac',
     'm4a': 'aac',
-    'opus': 'opus',
+    'opus': 'libopus',
     'vorbis': 'libvorbis',
     'wav': None,
 }
-- 
cgit v1.1


From b7e14f06a4a4fbaafc593c2f118e4b0f5d8d7937 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Matthias=20Ku=CC=88ch?= <mail@matthias-kuech.de>
Date: Tue, 3 Oct 2017 15:17:28 +0200
Subject: Fix for JSON meta data download

Added fixes according to #13651 and user @remitamine
---
 youtube_dl/extractor/nbc.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nbc.py b/youtube_dl/extractor/nbc.py
index 836a41f..35151f5 100644
--- a/youtube_dl/extractor/nbc.py
+++ b/youtube_dl/extractor/nbc.py
@@ -15,7 +15,7 @@ from ..utils import (
 
 
 class NBCIE(AdobePassIE):
-    _VALID_URL = r'(?P<permalink>https?://(?:www\.)?nbc\.com/[^/]+/video/[^/]+/(?P<id>n?\d+))'
+    _VALID_URL = r'https?(?P<permalink>://(?:www\.)?nbc\.com/[^/]+/video/[^/]+/(?P<id>n?\d+))'
 
     _TESTS = [
         {
@@ -72,6 +72,7 @@ class NBCIE(AdobePassIE):
 
     def _real_extract(self, url):
         permalink, video_id = re.match(self._VALID_URL, url).groups()
+        permalink = 'http' + permalink
         video_data = self._download_json(
             'https://api.nbc.com/v3/videos', video_id, query={
                 'filter[permalink]': permalink,
-- 
cgit v1.1


From 3e4cedf9e8cd3157df2457df7274d0c842421945 Mon Sep 17 00:00:00 2001
From: Jakub Wilk <jwilk@jwilk.net>
Date: Tue, 3 Oct 2017 18:28:13 +0200
Subject: [tvn24] Relax _VALID_URL

---
 youtube_dl/extractor/tvn24.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/tvn24.py b/youtube_dl/extractor/tvn24.py
index 12ed603..6590e1f 100644
--- a/youtube_dl/extractor/tvn24.py
+++ b/youtube_dl/extractor/tvn24.py
@@ -9,7 +9,7 @@ from ..utils import (
 
 
 class TVN24IE(InfoExtractor):
-    _VALID_URL = r'https?://(?:(?:[^/]+)\.)?tvn24(?:bis)?\.pl/(?:[^/]+/)*(?P<id>[^/]+)\.html'
+    _VALID_URL = r'https?://(?:(?:[^/]+)\.)?tvn24(?:bis)?\.pl/(?:[^/]+/)*(?P<id>[^/]+)'
     _TESTS = [{
         'url': 'http://www.tvn24.pl/wiadomosci-z-kraju,3/oredzie-artura-andrusa,702428.html',
         'md5': 'fbdec753d7bc29d96036808275f2130c',
@@ -18,7 +18,7 @@ class TVN24IE(InfoExtractor):
             'ext': 'mp4',
             'title': '"Święta mają być wesołe, dlatego, ludziska, wszyscy pod jemiołę"',
             'description': 'Wyjątkowe orędzie Artura Andrusa, jednego z gości "Szkła kontaktowego".',
-            'thumbnail': 're:http://.*[.]jpeg',
+            'thumbnail': 're:https?://.*[.]jpeg',
         }
     }, {
         'url': 'http://fakty.tvn24.pl/ogladaj-online,60/53-konferencja-bezpieczenstwa-w-monachium,716431.html',
@@ -29,6 +29,9 @@ class TVN24IE(InfoExtractor):
     }, {
         'url': 'http://tvn24bis.pl/poranek,146,m/gen-koziej-w-tvn24-bis-wracamy-do-czasow-zimnej-wojny,715660.html',
         'only_matching': True,
+    }, {
+        'url': 'https://www.tvn24.pl/magazyn-tvn24/angie-w-jednej-czwartej-polka-od-szarej-myszki-do-cesarzowej-europy,119,2158',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
-- 
cgit v1.1


From 9524dca3accc1e60cddd141f06924ed7404db9bd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 4 Oct 2017 02:53:20 +0700
Subject: [README.md] Use revision bound link to YoutubeDL options (closes
 #14401)

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 7818e58..2879aad 100644
--- a/README.md
+++ b/README.md
@@ -1167,7 +1167,7 @@ with youtube_dl.YoutubeDL(ydl_opts) as ydl:
     ydl.download(['https://www.youtube.com/watch?v=BaW_jenozKc'])
 ```
 
-Most likely, you'll want to use various options. For a list of options available, have a look at [`youtube_dl/YoutubeDL.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/YoutubeDL.py#L129-L279). For a start, if you want to intercept youtube-dl's output, set a `logger` object.
+Most likely, you'll want to use various options. For a list of options available, have a look at [`youtube_dl/YoutubeDL.py`](https://github.com/rg3/youtube-dl/blob/3e4cedf9e8cd3157df2457df7274d0c842421945/youtube_dl/YoutubeDL.py#L137-L312). For a start, if you want to intercept youtube-dl's output, set a `logger` object.
 
 Here's a more complete example of a program that outputs only errors (and a short message after the download is finished), and downloads/converts the video to an mp3 file:
 
-- 
cgit v1.1


From c110944fa2f21af733b4f3168764e1b008e11514 Mon Sep 17 00:00:00 2001
From: "M.K" <mk-pmb@users.noreply.github.com>
Date: Tue, 3 Oct 2017 22:50:27 +0200
Subject: [extractor/common] Fix typo in _parse_mpd_formats

---
 youtube_dl/extractor/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 2bbbf8f..a878550 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1920,7 +1920,7 @@ class InfoExtractor(object):
                             # can't be used at the same time
                             if '%(Number' in media_template and 's' not in representation_ms_info:
                                 segment_duration = None
-                                if 'total_number' not in representation_ms_info and 'segment_duration':
+                                if 'total_number' not in representation_ms_info and 'segment_duration' in representation_ms_info:
                                     segment_duration = float_or_none(representation_ms_info['segment_duration'], representation_ms_info['timescale'])
                                     representation_ms_info['total_number'] = int(math.ceil(float(period_duration) / segment_duration))
                                 representation_ms_info['fragments'] = [{
-- 
cgit v1.1


From 6e736d86e7d03d14279c403202fe2a632e6e5023 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 4 Oct 2017 04:27:42 +0700
Subject: [beeg] Fix extraction (closes #14403)

---
 youtube_dl/extractor/beeg.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/beeg.py b/youtube_dl/extractor/beeg.py
index bbeae4b..bf22a41 100644
--- a/youtube_dl/extractor/beeg.py
+++ b/youtube_dl/extractor/beeg.py
@@ -60,9 +60,13 @@ class BeegIE(InfoExtractor):
         beeg_version = beeg_version or '2185'
         beeg_salt = beeg_salt or 'pmweAkq8lAYKdfWcFCUj0yoVgoPlinamH5UE1CB3H'
 
-        video = self._download_json(
-            'https://api.beeg.com/api/v6/%s/video/%s' % (beeg_version, video_id),
-            video_id)
+        for api_path in ('', 'api.'):
+            video = self._download_json(
+                'https://%sbeeg.com/api/v6/%s/video/%s'
+                % (api_path, beeg_version, video_id), video_id,
+                fatal=api_path == 'api.')
+            if video:
+                break
 
         def split(o, e):
             def cut(s, x):
-- 
cgit v1.1


From 6b46285e850f8bc8155a93adf13920752537e55d Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 4 Oct 2017 07:45:13 +0200
Subject: [comedycentral] new shortcut :theopposition for "The Opposition" show

---
 youtube_dl/extractor/comedycentral.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/comedycentral.py b/youtube_dl/extractor/comedycentral.py
index 4cac294..d08b909 100644
--- a/youtube_dl/extractor/comedycentral.py
+++ b/youtube_dl/extractor/comedycentral.py
@@ -120,13 +120,16 @@ class ComedyCentralTVIE(MTVServicesInfoExtractor):
 
 
 class ComedyCentralShortnameIE(InfoExtractor):
-    _VALID_URL = r'^:(?P<id>tds|thedailyshow)$'
+    _VALID_URL = r'^:(?P<id>tds|thedailyshow|theopposition)$'
     _TESTS = [{
         'url': ':tds',
         'only_matching': True,
     }, {
         'url': ':thedailyshow',
         'only_matching': True,
+    }, {
+        'url': ':theopposition',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
@@ -134,5 +137,6 @@ class ComedyCentralShortnameIE(InfoExtractor):
         shortcut_map = {
             'tds': 'http://www.cc.com/shows/the-daily-show-with-trevor-noah/full-episodes',
             'thedailyshow': 'http://www.cc.com/shows/the-daily-show-with-trevor-noah/full-episodes',
+            'theopposition': 'http://www.cc.com/shows/the-opposition-with-jordan-klepper/full-episodes',
         }
         return self.url_result(shortcut_map[video_id])
-- 
cgit v1.1


From cf5f6ed5be8bab252f3ded345b6b1fdf31426661 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 5 Oct 2017 00:27:24 +0700
Subject: [xvideos] Add support for embed URLs and improve extraction (closes
 #14409)

---
 youtube_dl/extractor/xvideos.py | 42 ++++++++++++++++++++++++++++++-----------
 1 file changed, 31 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/xvideos.py b/youtube_dl/extractor/xvideos.py
index eca6030..085c8d4 100644
--- a/youtube_dl/extractor/xvideos.py
+++ b/youtube_dl/extractor/xvideos.py
@@ -14,8 +14,16 @@ from ..utils import (
 
 
 class XVideosIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?xvideos\.com/video(?P<id>[0-9]+)(?:.*)'
-    _TEST = {
+    _VALID_URL = r'''(?x)
+                    https?://
+                        (?:
+                            (?:www\.)?xvideos\.com/video|
+                            flashservice\.xvideos\.com/embedframe/|
+                            static-hw\.xvideos\.com/swf/xv-player\.swf\?.*?\bid_video=
+                        )
+                        (?P<id>[0-9]+)
+                    '''
+    _TESTS = [{
         'url': 'http://www.xvideos.com/video4588838/biker_takes_his_girl',
         'md5': '14cea69fcb84db54293b1e971466c2e1',
         'info_dict': {
@@ -25,21 +33,33 @@ class XVideosIE(InfoExtractor):
             'duration': 108,
             'age_limit': 18,
         }
-    }
+    }, {
+        'url': 'https://flashservice.xvideos.com/embedframe/4588838',
+        'only_matching': True,
+    }, {
+        'url': 'http://static-hw.xvideos.com/swf/xv-player.swf?id_video=4588838',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
+
+        webpage = self._download_webpage(
+            'http://www.xvideos.com/video%s/' % video_id, video_id)
 
         mobj = re.search(r'<h1 class="inlineError">(.+?)</h1>', webpage)
         if mobj:
             raise ExtractorError('%s said: %s' % (self.IE_NAME, clean_html(mobj.group(1))), expected=True)
 
-        video_title = self._html_search_regex(
-            r'<title>(.*?)\s+-\s+XVID', webpage, 'title')
-        video_thumbnail = self._search_regex(
+        title = self._html_search_regex(
+            (r'<title>(?P<title>.+?)\s+-\s+XVID',
+             r'setVideoTitle\s*\(\s*(["\'])(?P<title>(?:(?!\1).)+)\1'),
+            webpage, 'title', default=None,
+            group='title') or self._og_search_title(webpage)
+
+        thumbnail = self._search_regex(
             r'url_bigthumb=(.+?)&amp', webpage, 'thumbnail', fatal=False)
-        video_duration = int_or_none(self._og_search_property(
+        duration = int_or_none(self._og_search_property(
             'duration', webpage, default=None)) or parse_duration(
             self._search_regex(
                 r'<span[^>]+class=["\']duration["\'][^>]*>.*?(\d[^<]+)',
@@ -74,8 +94,8 @@ class XVideosIE(InfoExtractor):
         return {
             'id': video_id,
             'formats': formats,
-            'title': video_title,
-            'duration': video_duration,
-            'thumbnail': video_thumbnail,
+            'title': title,
+            'duration': duration,
+            'thumbnail': thumbnail,
             'age_limit': 18,
         }
-- 
cgit v1.1


From 6be08ce60205a65a6739667783eead56ccc34456 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 6 Oct 2017 23:13:53 +0700
Subject: [utils] Use in OnDemandPagedList by default Not using cache results
 in redundant network I/O due to downloading the same pages while using
 --playlist-items n-m

---
 youtube_dl/extractor/mixcloud.py | 2 +-
 youtube_dl/extractor/nba.py      | 2 +-
 youtube_dl/utils.py              | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/mixcloud.py b/youtube_dl/extractor/mixcloud.py
index f331db8..7b2bb6e 100644
--- a/youtube_dl/extractor/mixcloud.py
+++ b/youtube_dl/extractor/mixcloud.py
@@ -291,7 +291,7 @@ class MixcloudUserIE(MixcloudPlaylistBaseIE):
             functools.partial(
                 self._tracks_page_func,
                 '%s/%s' % (user_id, list_type), video_id, 'list of %s' % list_type),
-            self._PAGE_SIZE, use_cache=True)
+            self._PAGE_SIZE)
 
         return self.playlist_result(
             entries, video_id, '%s (%s)' % (username, list_type), description)
diff --git a/youtube_dl/extractor/nba.py b/youtube_dl/extractor/nba.py
index 5356196..be295a7 100644
--- a/youtube_dl/extractor/nba.py
+++ b/youtube_dl/extractor/nba.py
@@ -122,7 +122,7 @@ class NBAIE(TurnerBaseIE):
         playlist_title = self._og_search_title(webpage, fatal=False)
         entries = OnDemandPagedList(
             functools.partial(self._fetch_page, team, video_id),
-            self._PAGE_SIZE, use_cache=True)
+            self._PAGE_SIZE)
 
         return self.playlist_result(entries, team, playlist_title)
 
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 92b22e6..59fb334 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1933,7 +1933,7 @@ class PagedList(object):
 
 
 class OnDemandPagedList(PagedList):
-    def __init__(self, pagefunc, pagesize, use_cache=False):
+    def __init__(self, pagefunc, pagesize, use_cache=True):
         self._pagefunc = pagefunc
         self._pagesize = pagesize
         self._use_cache = use_cache
-- 
cgit v1.1


From 7e85e8729f1e2ed9817466acef30fa6dc7e03e1e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 6 Oct 2017 23:34:46 +0700
Subject: [YoutubeDL] Fix out of range --playlist-items for iterable playlists
 and reduce code duplication (closes #14425)

---
 youtube_dl/YoutubeDL.py | 27 ++++++++++++++++-----------
 1 file changed, 16 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 474d6c9..9036f0f 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -911,12 +911,22 @@ class YoutubeDL(object):
                 playlistitems = iter_playlistitems(playlistitems_str)
 
             ie_entries = ie_result['entries']
+
+            def make_playlistitems_entries(list_ie_entries):
+                num_entries = len(list_ie_entries)
+                return [
+                    list_ie_entries[i - 1] for i in playlistitems
+                    if -num_entries <= i - 1 < num_entries]
+
+            def report_download(num_entries):
+                self.to_screen(
+                    '[%s] playlist %s: Downloading %d videos' %
+                    (ie_result['extractor'], playlist, num_entries))
+
             if isinstance(ie_entries, list):
                 n_all_entries = len(ie_entries)
                 if playlistitems:
-                    entries = [
-                        ie_entries[i - 1] for i in playlistitems
-                        if -n_all_entries <= i - 1 < n_all_entries]
+                    entries = make_playlistitems_entries(ie_entries)
                 else:
                     entries = ie_entries[playliststart:playlistend]
                 n_entries = len(entries)
@@ -934,20 +944,15 @@ class YoutubeDL(object):
                     entries = ie_entries.getslice(
                         playliststart, playlistend)
                 n_entries = len(entries)
-                self.to_screen(
-                    '[%s] playlist %s: Downloading %d videos' %
-                    (ie_result['extractor'], playlist, n_entries))
+                report_download(n_entries)
             else:  # iterable
                 if playlistitems:
-                    entry_list = list(ie_entries)
-                    entries = [entry_list[i - 1] for i in playlistitems]
+                    entries = make_playlistitems_entries(list(ie_entries))
                 else:
                     entries = list(itertools.islice(
                         ie_entries, playliststart, playlistend))
                 n_entries = len(entries)
-                self.to_screen(
-                    '[%s] playlist %s: Downloading %d videos' %
-                    (ie_result['extractor'], playlist, n_entries))
+                report_download(n_entries)
 
             if self.params.get('playlistreverse', False):
                 entries = entries[::-1]
-- 
cgit v1.1


From 86a15ed64b410336b2145f4a6b8e8a22cbf24f51 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 6 Oct 2017 23:41:28 +0700
Subject: [test_YoutubeDL] Add test for #14425

---
 test/test_YoutubeDL.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index e70cbcd..5ac34e6 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -770,6 +770,9 @@ class TestYoutubeDL(unittest.TestCase):
         result = get_ids({'playlist_items': '10'})
         self.assertEqual(result, [])
 
+        result = get_ids({'playlist_items': '3-10'})
+        self.assertEqual(result, [3, 4])
+
     def test_urlopen_no_file_protocol(self):
         # see https://github.com/rg3/youtube-dl/issues/8227
         ydl = YDL()
-- 
cgit v1.1


From cd6fc19ed76af6687fb2cb5f87d9ed4c3071c203 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 6 Oct 2017 23:46:57 +0700
Subject: [YoutubeDL] Ignore duplicates in --playlist-items E.g.
 '--playlist-items 2-4,3-4,3' should result in '[2,3,4]', not '[2,3,4,3,4,3]'

---
 test/test_YoutubeDL.py  | 3 +++
 youtube_dl/YoutubeDL.py | 3 ++-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index 5ac34e6..db936bf 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -773,6 +773,9 @@ class TestYoutubeDL(unittest.TestCase):
         result = get_ids({'playlist_items': '3-10'})
         self.assertEqual(result, [3, 4])
 
+        result = get_ids({'playlist_items': '2-4,3-4,3'})
+        self.assertEqual(result, [2, 3, 4])
+
     def test_urlopen_no_file_protocol(self):
         # see https://github.com/rg3/youtube-dl/issues/8227
         ydl = YDL()
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 9036f0f..855d6b8 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -65,6 +65,7 @@ from .utils import (
     locked_file,
     make_HTTPS_handler,
     MaxDownloadsReached,
+    orderedSet,
     PagedList,
     parse_filesize,
     PerRequestProxyHandler,
@@ -908,7 +909,7 @@ class YoutubeDL(object):
                                 yield int(item)
                         else:
                             yield int(string_segment)
-                playlistitems = iter_playlistitems(playlistitems_str)
+                playlistitems = orderedSet(iter_playlistitems(playlistitems_str))
 
             ie_entries = ie_result['entries']
 
-- 
cgit v1.1


From ac93c09ab2c4d099a628c1ed59670bef008db89c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 6 Oct 2017 23:53:32 +0700
Subject: [xtube] Add support for embedded URLs (closes #14417)

---
 youtube_dl/extractor/xtube.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/xtube.py b/youtube_dl/extractor/xtube.py
index bea9b87..c6c0b32 100644
--- a/youtube_dl/extractor/xtube.py
+++ b/youtube_dl/extractor/xtube.py
@@ -18,7 +18,7 @@ class XTubeIE(InfoExtractor):
     _VALID_URL = r'''(?x)
                         (?:
                             xtube:|
-                            https?://(?:www\.)?xtube\.com/(?:watch\.php\?.*\bv=|video-watch/(?P<display_id>[^/]+)-)
+                            https?://(?:www\.)?xtube\.com/(?:watch\.php\?.*\bv=|video-watch/(?:embedded/)?(?P<display_id>[^/]+)-)
                         )
                         (?P<id>[^/?&#]+)
                     '''
@@ -64,6 +64,9 @@ class XTubeIE(InfoExtractor):
     }, {
         'url': 'xtube:kVTUy_G222_',
         'only_matching': True,
+    }, {
+        'url': 'https://www.xtube.com/video-watch/embedded/milf-tara-and-teen-shared-and-cum-covered-extreme-bukkake-32203482?embedsize=big',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
-- 
cgit v1.1


From 2e2a8e97d53d7759d663be2a1dc3f4108342ea40 Mon Sep 17 00:00:00 2001
From: Jalaz Kumar <jaykay12@users.noreply.github.com>
Date: Fri, 6 Oct 2017 22:26:31 +0530
Subject: [pornflip] Extend _VALID_URL (closes #14405)

---
 youtube_dl/extractor/pornflip.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/pornflip.py b/youtube_dl/extractor/pornflip.py
index a4a5d39..ee04936 100644
--- a/youtube_dl/extractor/pornflip.py
+++ b/youtube_dl/extractor/pornflip.py
@@ -14,7 +14,7 @@ from ..utils import (
 
 
 class PornFlipIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?pornflip\.com/(?:v|embed)/(?P<id>[0-9A-Za-z]{11})'
+    _VALID_URL = r'https?://(?:www\.)?pornflip\.com/(?:v|embed)/(?P<id>[0-9A-Za-z-]{11})'
     _TESTS = [{
         'url': 'https://www.pornflip.com/v/wz7DfNhMmep',
         'md5': '98c46639849145ae1fd77af532a9278c',
@@ -34,6 +34,12 @@ class PornFlipIE(InfoExtractor):
     }, {
         'url': 'https://www.pornflip.com/embed/wz7DfNhMmep',
         'only_matching': True,
+    }, {
+        'url': 'https://www.pornflip.com/v/EkRD6-vS2-s',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.pornflip.com/embed/EkRD6-vS2-s',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
-- 
cgit v1.1


From b1a7bf44b95dee2803e8638b13c4467a253b5b8f Mon Sep 17 00:00:00 2001
From: remis <r-pankevicius@users.noreply.github.com>
Date: Fri, 6 Oct 2017 19:59:09 +0300
Subject: [lnkgo] Relax _VALID_URL

---
 youtube_dl/extractor/lnkgo.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/lnkgo.py b/youtube_dl/extractor/lnkgo.py
index 068378c..cfec0d3 100644
--- a/youtube_dl/extractor/lnkgo.py
+++ b/youtube_dl/extractor/lnkgo.py
@@ -11,7 +11,7 @@ from ..utils import (
 
 
 class LnkGoIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?lnkgo\.alfa\.lt/visi-video/(?P<show>[^/]+)/ziurek-(?P<id>[A-Za-z0-9-]+)'
+    _VALID_URL = r'https?://(?:www\.)?lnkgo\.(?:alfa\.)?lt/visi-video/(?P<show>[^/]+)/ziurek-(?P<id>[A-Za-z0-9-]+)'
     _TESTS = [{
         'url': 'http://lnkgo.alfa.lt/visi-video/yra-kaip-yra/ziurek-yra-kaip-yra-162',
         'info_dict': {
@@ -42,6 +42,9 @@ class LnkGoIE(InfoExtractor):
         'params': {
             'skip_download': True,  # HLS download
         },
+    }, {
+        'url': 'http://www.lnkgo.lt/visi-video/aktualai-pratesimas/ziurek-putka-trys-klausimai',
+        'only_matching': True,
     }]
     _AGE_LIMITS = {
         'N-7': 7,
-- 
cgit v1.1


From e95284754192ea4b5b4e32f1a2274e5767d980b5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 7 Oct 2017 00:57:09 +0700
Subject: [PULL_REQUEST_TEMPLATE.md] Add explicit entry on flake8

---
 .github/PULL_REQUEST_TEMPLATE.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
index 46fa26f..ba4ca75 100644
--- a/.github/PULL_REQUEST_TEMPLATE.md
+++ b/.github/PULL_REQUEST_TEMPLATE.md
@@ -9,6 +9,7 @@
 ### Before submitting a *pull request* make sure you have:
 - [ ] At least skimmed through [adding new extractor tutorial](https://github.com/rg3/youtube-dl#adding-support-for-a-new-site) and [youtube-dl coding conventions](https://github.com/rg3/youtube-dl#youtube-dl-coding-conventions) sections
 - [ ] [Searched](https://github.com/rg3/youtube-dl/search?q=is%3Apr&type=Issues) the bugtracker for similar pull requests
+- [ ] Checked the code with [flake8](https://pypi.python.org/pypi/flake8)
 
 ### In order to be accepted and merged into youtube-dl each piece of code must be in public domain or released under [Unlicense](http://unlicense.org/). Check one of the following options:
 - [ ] I am the original author of this code and I am willing to release it under [Unlicense](http://unlicense.org/)
-- 
cgit v1.1


From 665f42d8c14626404372b349624632b8d39f3c0e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 7 Oct 2017 01:40:00 +0700
Subject: [reddit] Sort formats (closes #14430)

---
 youtube_dl/extractor/reddit.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/reddit.py b/youtube_dl/extractor/reddit.py
index 01c85ee..4d44b9d 100644
--- a/youtube_dl/extractor/reddit.py
+++ b/youtube_dl/extractor/reddit.py
@@ -35,6 +35,8 @@ class RedditIE(InfoExtractor):
             'https://v.redd.it/%s/DASHPlaylist.mpd' % video_id, video_id,
             mpd_id='dash', fatal=False))
 
+        self._sort_formats(formats)
+
         return {
             'id': video_id,
             'title': video_id,
-- 
cgit v1.1


From 3fc8f5b7c239ddd366012d2f0da754fcfe247297 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 7 Oct 2017 05:01:38 +0700
Subject: [ChangeLog] Actualize

---
 ChangeLog | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 80299d5..760d090 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,27 @@
+version <unreleased>
+
+Core
+* [YoutubeDL] Ignore duplicates in --playlist-items
+* [YoutubeDL] Fix out of range --playlist-items for iterable playlists and
+  reduce code duplication (#14425)
++ [utils] Use cache in OnDemandPagedList by default
+* [postprocessor/ffmpeg] Convert to opus using libopus (#14381)
+
+Extractors
+* [reddit] Sort formats (#14430)
+* [lnkgo] Relax URL regular expression (#14423)
+* [pornflip] Extend URL regular expression (#14405, #14406)
++ [xtube] Add support for embed URLs (#14417)
++ [xvideos] Add support for embed URLs and improve extraction (#14409)
+* [beeg] Fix extraction (#14403)
+* [tvn24] Relax URL regular expression (#14395)
+* [nbc] Fix extraction (#13651, #13715, #14137, #14198, #14312, #14314, #14378,
+  #14392, #14414, #14419, #14431)
++ [ketnet] Add support for videos without direct sources (#14377)
+* [canvas] Generalize mediazone.vrt.be extractor and rework canvas and een
++ [afreecatv] Add support for adult videos (#14376)
+
+
 version 2017.10.01
 
 Core
-- 
cgit v1.1


From 8e751a185c53a81cd35900010118894fd8201b75 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 7 Oct 2017 05:02:53 +0700
Subject: release 2017.10.07

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 docs/supportedsites.md    | 3 ++-
 youtube_dl/version.py     | 2 +-
 4 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 3be3062..9fc425c 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.10.01*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.10.01**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.10.07*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.10.07**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through the [README](https://github.com/rg3/youtube-dl/blob/master/README.md), **most notably** the [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ Add the `-v` flag to **your command line** you run youtube-dl with (`youtube-dl
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.10.01
+[debug] youtube-dl version 2017.10.07
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index 760d090..2feb3dc 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.10.07
 
 Core
 * [YoutubeDL] Ignore duplicates in --playlist-items
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index d36a07c..be9df7e 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -130,7 +130,8 @@
  - **CamWithHer**
  - **canalc2.tv**
  - **Canalplus**: canalplus.fr, piwiplus.fr and d8.tv
- - **Canvas**: canvas.be and een.be
+ - **Canvas**
+ - **CanvasEen**: canvas.be and een.be
  - **CarambaTV**
  - **CarambaTVPage**
  - **CartoonNetwork**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 49fa02d..705fcad 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.10.01'
+__version__ = '2017.10.07'
-- 
cgit v1.1


From 8b561bfc9d15bf487be0e0bb5bb1d7248fd321d3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 7 Oct 2017 21:59:04 +0700
Subject: [youtube] Add support for hooktube.com (closes #14437)

---
 youtube_dl/extractor/youtube.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index ad2e933..edd8713 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -332,6 +332,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                          (?:(?:(?:(?:\w+\.)?[yY][oO][uU][tT][uU][bB][eE](?:-nocookie)?\.com/|
                             (?:www\.)?deturl\.com/www\.youtube\.com/|
                             (?:www\.)?pwnyoutube\.com/|
+                            (?:www\.)?hooktube\.com/|
                             (?:www\.)?yourepeat\.com/|
                             tube\.majestyc\.net/|
                             youtube\.googleapis\.com/)                        # the various hostnames, with wildcard subdomains
-- 
cgit v1.1


From a22ccac1f02b5957f1f083a54e1f83d22a7e6f69 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 8 Oct 2017 01:34:17 +0700
Subject: [fox] Delegate to uplynk:preplay (#14147)

---
 youtube_dl/extractor/fox.py | 33 +++++++++++++++++++++++----------
 1 file changed, 23 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/fox.py b/youtube_dl/extractor/fox.py
index facc665..5f98d01 100644
--- a/youtube_dl/extractor/fox.py
+++ b/youtube_dl/extractor/fox.py
@@ -2,7 +2,10 @@
 from __future__ import unicode_literals
 
 from .adobepass import AdobePassIE
+from .uplynk import UplynkPreplayIE
+from ..compat import compat_str
 from ..utils import (
+    HEADRequest,
     int_or_none,
     parse_age_limit,
     parse_duration,
@@ -53,14 +56,7 @@ class FOXIE(AdobePassIE):
             })
 
         title = video['name']
-
-        m3u8_url = self._download_json(
-            video['videoRelease']['url'], video_id)['playURL']
-
-        formats = self._extract_m3u8_formats(
-            m3u8_url, video_id, 'mp4',
-            entry_protocol='m3u8_native', m3u8_id='hls')
-        self._sort_formats(formats)
+        release_url = video['videoRelease']['url']
 
         description = video.get('description')
         duration = int_or_none(video.get('durationInSeconds')) or int_or_none(
@@ -84,7 +80,7 @@ class FOXIE(AdobePassIE):
             # TODO: AP
             pass
 
-        return {
+        info = {
             'id': video_id,
             'title': title,
             'description': description,
@@ -97,5 +93,22 @@ class FOXIE(AdobePassIE):
             'episode': episode,
             'episode_number': episode_number,
             'release_year': release_year,
-            'formats': formats,
         }
+
+        urlh = self._request_webpage(HEADRequest(release_url), video_id)
+        video_url = compat_str(urlh.geturl())
+
+        if UplynkPreplayIE.suitable(video_url):
+            info.update({
+                '_type': 'url_transparent',
+                'url': video_url,
+                'ie_key': UplynkPreplayIE.ie_key(),
+            })
+        else:
+            m3u8_url = self._download_json(release_url, video_id)['playURL']
+            formats = self._extract_m3u8_formats(
+                m3u8_url, video_id, 'mp4',
+                entry_protocol='m3u8_native', m3u8_id='hls')
+            self._sort_formats(formats)
+            info['formats'] = formats
+        return info
-- 
cgit v1.1


From b0dde6686c7110c9c2515a808d803239a81e6505 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Aleksandar=20Topuzovi=C4=87?=
 <aleksandar.topuzovic@gmail.com>
Date: Sat, 7 Oct 2017 23:40:08 +0100
Subject: [hrti] Relax _VALID_URL

---
 youtube_dl/extractor/hrti.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/hrti.py b/youtube_dl/extractor/hrti.py
index 656ce6d..4f03694 100644
--- a/youtube_dl/extractor/hrti.py
+++ b/youtube_dl/extractor/hrti.py
@@ -104,7 +104,7 @@ class HRTiIE(HRTiBaseIE):
                         (?:
                             hrti:(?P<short_id>[0-9]+)|
                             https?://
-                                hrti\.hrt\.hr/\#/video/show/(?P<id>[0-9]+)/(?P<display_id>[^/]+)?
+                                hrti\.hrt\.hr/(?:\#/)?video/show/(?P<id>[0-9]+)/(?P<display_id>[^/]+)?
                         )
                     '''
     _TESTS = [{
@@ -129,6 +129,9 @@ class HRTiIE(HRTiBaseIE):
     }, {
         'url': 'hrti:2181385',
         'only_matching': True,
+    }, {
+        'url': 'https://hrti.hrt.hr/video/show/3873068/cuvar-dvorca-dramska-serija-14',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
-- 
cgit v1.1


From 89923316210f8e17bb1a085278940e1c56fcff48 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 8 Oct 2017 21:36:50 +0700
Subject: [wdr] Relax media link regex (closes #14447)

---
 youtube_dl/extractor/wdr.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/wdr.py b/youtube_dl/extractor/wdr.py
index 8bb7362..621de1e 100644
--- a/youtube_dl/extractor/wdr.py
+++ b/youtube_dl/extractor/wdr.py
@@ -22,8 +22,13 @@ class WDRBaseIE(InfoExtractor):
         # for wdrmaus, in a tag with the class "videoButton" (previously a link
         # to the page in a multiline "videoLink"-tag)
         json_metadata = self._html_search_regex(
-            r'class=(?:"(?:mediaLink|wdrrPlayerPlayBtn|videoButton)\b[^"]*"[^>]+|"videoLink\b[^"]*"[\s]*>\n[^\n]*)data-extension="([^"]+)"',
-            webpage, 'media link', default=None, flags=re.MULTILINE)
+            r'''(?sx)class=
+                    (?:
+                        (["\'])(?:mediaLink|wdrrPlayerPlayBtn|videoButton)\b.*?\1[^>]+|
+                        (["\'])videoLink\b.*?\2[\s]*>\n[^\n]*
+                    )data-extension=(["\'])(?P<data>(?:(?!\3).)+)\3
+            ''',
+            webpage, 'media link', default=None, group='data')
 
         if not json_metadata:
             return
-- 
cgit v1.1


From 197224b7a4e37a6581bf1a0da18d0f67ea61a476 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 9 Oct 2017 23:50:53 +0700
Subject: Fix some regexes

---
 youtube_dl/extractor/aenetworks.py         | 2 +-
 youtube_dl/extractor/appletrailers.py      | 4 ++--
 youtube_dl/extractor/ard.py                | 2 +-
 youtube_dl/extractor/bbc.py                | 2 +-
 youtube_dl/extractor/dailymotion.py        | 2 +-
 youtube_dl/extractor/deezer.py             | 2 +-
 youtube_dl/extractor/freespeech.py         | 2 +-
 youtube_dl/extractor/generic.py            | 2 +-
 youtube_dl/extractor/googleplus.py         | 2 +-
 youtube_dl/extractor/hrti.py               | 2 +-
 youtube_dl/extractor/ign.py                | 2 +-
 youtube_dl/extractor/infoq.py              | 6 +++---
 youtube_dl/extractor/jeuxvideo.py          | 2 +-
 youtube_dl/extractor/livestream.py         | 2 +-
 youtube_dl/extractor/makertv.py            | 2 +-
 youtube_dl/extractor/mangomolo.py          | 2 +-
 youtube_dl/extractor/meipai.py             | 2 +-
 youtube_dl/extractor/mtv.py                | 2 +-
 youtube_dl/extractor/myvideo.py            | 2 +-
 youtube_dl/extractor/nationalgeographic.py | 2 +-
 youtube_dl/extractor/naver.py              | 2 +-
 youtube_dl/extractor/npo.py                | 2 +-
 youtube_dl/extractor/ruhd.py               | 2 +-
 youtube_dl/extractor/stanfordoc.py         | 4 ++--
 youtube_dl/extractor/theplatform.py        | 2 +-
 youtube_dl/extractor/thisav.py             | 4 ++--
 youtube_dl/extractor/twitter.py            | 2 +-
 youtube_dl/extractor/vice.py               | 2 +-
 youtube_dl/extractor/videopremium.py       | 2 +-
 youtube_dl/extractor/youtube.py            | 2 +-
 30 files changed, 35 insertions(+), 35 deletions(-)

diff --git a/youtube_dl/extractor/aenetworks.py b/youtube_dl/extractor/aenetworks.py
index 2dcdba9..da1b566 100644
--- a/youtube_dl/extractor/aenetworks.py
+++ b/youtube_dl/extractor/aenetworks.py
@@ -131,7 +131,7 @@ class AENetworksIE(AENetworksBaseIE):
              r'data-media-url=(["\'])(?P<url>(?:(?!\1).)+?)\1'],
             webpage, 'video url', group='url')
         theplatform_metadata = self._download_theplatform_metadata(self._search_regex(
-            r'https?://link.theplatform.com/s/([^?]+)', media_url, 'theplatform_path'), video_id)
+            r'https?://link\.theplatform\.com/s/([^?]+)', media_url, 'theplatform_path'), video_id)
         info = self._parse_theplatform_metadata(theplatform_metadata)
         if theplatform_metadata.get('AETN$isBehindWall'):
             requestor_id = self._DOMAIN_TO_REQUESTOR_ID[domain]
diff --git a/youtube_dl/extractor/appletrailers.py b/youtube_dl/extractor/appletrailers.py
index b45b431..a9ef733 100644
--- a/youtube_dl/extractor/appletrailers.py
+++ b/youtube_dl/extractor/appletrailers.py
@@ -117,7 +117,7 @@ class AppleTrailersIE(InfoExtractor):
                             continue
                         formats.append({
                             'format_id': '%s-%s' % (version, size),
-                            'url': re.sub(r'_(\d+p.mov)', r'_h\1', src),
+                            'url': re.sub(r'_(\d+p\.mov)', r'_h\1', src),
                             'width': int_or_none(size_data.get('width')),
                             'height': int_or_none(size_data.get('height')),
                             'language': version[:2],
@@ -179,7 +179,7 @@ class AppleTrailersIE(InfoExtractor):
             formats = []
             for format in settings['metadata']['sizes']:
                 # The src is a file pointing to the real video file
-                format_url = re.sub(r'_(\d*p.mov)', r'_h\1', format['src'])
+                format_url = re.sub(r'_(\d*p\.mov)', r'_h\1', format['src'])
                 formats.append({
                     'url': format_url,
                     'format': format['type'],
diff --git a/youtube_dl/extractor/ard.py b/youtube_dl/extractor/ard.py
index 3f248b1..915f886 100644
--- a/youtube_dl/extractor/ard.py
+++ b/youtube_dl/extractor/ard.py
@@ -195,7 +195,7 @@ class ARDMediathekIE(InfoExtractor):
 
         title = self._html_search_regex(
             [r'<h1(?:\s+class="boxTopHeadline")?>(.*?)</h1>',
-             r'<meta name="dcterms.title" content="(.*?)"/>',
+             r'<meta name="dcterms\.title" content="(.*?)"/>',
              r'<h4 class="headline">(.*?)</h4>'],
             webpage, 'title')
         description = self._html_search_meta(
diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 8b20c03..5525f7c 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -386,7 +386,7 @@ class BBCCoUkIE(InfoExtractor):
                             m3u8_id=format_id, fatal=False))
                         if re.search(self._USP_RE, href):
                             usp_formats = self._extract_m3u8_formats(
-                                re.sub(self._USP_RE, r'/\1.ism/\1.m3u8', href),
+                                re.sub(self._USP_RE, r'/\1\.ism/\1\.m3u8', href),
                                 programme_id, ext='mp4', entry_protocol='m3u8_native',
                                 m3u8_id=format_id, fatal=False)
                             for f in usp_formats:
diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py
index e9d0dd19..21a2d02 100644
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@@ -235,7 +235,7 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
 
         # vevo embed
         vevo_id = self._search_regex(
-            r'<link rel="video_src" href="[^"]*?vevo.com[^"]*?video=(?P<id>[\w]*)',
+            r'<link rel="video_src" href="[^"]*?vevo\.com[^"]*?video=(?P<id>[\w]*)',
             webpage, 'vevo embed', default=None)
         if vevo_id:
             return self.url_result('vevo:%s' % vevo_id, 'Vevo')
diff --git a/youtube_dl/extractor/deezer.py b/youtube_dl/extractor/deezer.py
index ec87b94..a38b268 100644
--- a/youtube_dl/extractor/deezer.py
+++ b/youtube_dl/extractor/deezer.py
@@ -19,7 +19,7 @@ class DeezerPlaylistIE(InfoExtractor):
             'id': '176747451',
             'title': 'Best!',
             'uploader': 'Anonymous',
-            'thumbnail': r're:^https?://cdn-images.deezer.com/images/cover/.*\.jpg$',
+            'thumbnail': r're:^https?://cdn-images\.deezer\.com/images/cover/.*\.jpg$',
         },
         'playlist_count': 30,
         'skip': 'Only available in .de',
diff --git a/youtube_dl/extractor/freespeech.py b/youtube_dl/extractor/freespeech.py
index 0a70ca7..7fa271b 100644
--- a/youtube_dl/extractor/freespeech.py
+++ b/youtube_dl/extractor/freespeech.py
@@ -27,7 +27,7 @@ class FreespeechIE(InfoExtractor):
         mobj = re.match(self._VALID_URL, url)
         title = mobj.group('title')
         webpage = self._download_webpage(url, title)
-        info_json = self._search_regex(r'jQuery.extend\(Drupal.settings, ({.*?})\);', webpage, 'info')
+        info_json = self._search_regex(r'jQuery\.extend\(Drupal\.settings, ({.*?})\);', webpage, 'info')
         info = json.loads(info_json)
 
         return {
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 1721a3d..68b6338 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -2206,7 +2206,7 @@ class GenericIE(InfoExtractor):
         # And then there are the jokers who advertise that they use RTA,
         # but actually don't.
         AGE_LIMIT_MARKERS = [
-            r'Proudly Labeled <a href="http://www.rtalabel.org/" title="Restricted to Adults">RTA</a>',
+            r'Proudly Labeled <a href="http://www\.rtalabel\.org/" title="Restricted to Adults">RTA</a>',
         ]
         if any(re.search(marker, webpage) for marker in AGE_LIMIT_MARKERS):
             age_limit = 18
diff --git a/youtube_dl/extractor/googleplus.py b/youtube_dl/extractor/googleplus.py
index 427499b..6b927bb 100644
--- a/youtube_dl/extractor/googleplus.py
+++ b/youtube_dl/extractor/googleplus.py
@@ -61,7 +61,7 @@ class GooglePlusIE(InfoExtractor):
             'width': int(width),
             'height': int(height),
         } for width, height, video_url in re.findall(
-            r'\d+,(\d+),(\d+),"(https?://[^.]+\.googleusercontent.com.*?)"', webpage)]
+            r'\d+,(\d+),(\d+),"(https?://[^.]+\.googleusercontent\.com.*?)"', webpage)]
         self._sort_formats(formats)
 
         return {
diff --git a/youtube_dl/extractor/hrti.py b/youtube_dl/extractor/hrti.py
index 4f03694..7cef5f6 100644
--- a/youtube_dl/extractor/hrti.py
+++ b/youtube_dl/extractor/hrti.py
@@ -173,7 +173,7 @@ class HRTiIE(HRTiBaseIE):
 
 
 class HRTiPlaylistIE(HRTiBaseIE):
-    _VALID_URL = r'https?://hrti.hrt.hr/#/video/list/category/(?P<id>[0-9]+)/(?P<display_id>[^/]+)?'
+    _VALID_URL = r'https?://hrti\.hrt\.hr/#/video/list/category/(?P<id>[0-9]+)/(?P<display_id>[^/]+)?'
     _TESTS = [{
         'url': 'https://hrti.hrt.hr/#/video/list/category/212/ekumena',
         'info_dict': {
diff --git a/youtube_dl/extractor/ign.py b/youtube_dl/extractor/ign.py
index c1367cf..a96ea80 100644
--- a/youtube_dl/extractor/ign.py
+++ b/youtube_dl/extractor/ign.py
@@ -203,7 +203,7 @@ class PCMagIE(IGNIE):
     _VALID_URL = r'https?://(?:www\.)?pcmag\.com/(?P<type>videos|article2)(/.+)?/(?P<name_or_id>.+)'
     IE_NAME = 'pcmag'
 
-    _EMBED_RE = r'iframe.setAttribute\("src",\s*__util.objToUrlString\("http://widgets\.ign\.com/video/embed/content.html?[^"]*url=([^"]+)["&]'
+    _EMBED_RE = r'iframe\.setAttribute\("src",\s*__util.objToUrlString\("http://widgets\.ign\.com/video/embed/content\.html?[^"]*url=([^"]+)["&]'
 
     _TESTS = [{
         'url': 'http://www.pcmag.com/videos/2015/01/06/010615-whats-new-now-is-gogo-snooping-on-your-data',
diff --git a/youtube_dl/extractor/infoq.py b/youtube_dl/extractor/infoq.py
index fe425e7..57c9b0c 100644
--- a/youtube_dl/extractor/infoq.py
+++ b/youtube_dl/extractor/infoq.py
@@ -69,9 +69,9 @@ class InfoQIE(BokeCCBaseIE):
         }]
 
     def _extract_cookies(self, webpage):
-        policy = self._search_regex(r'InfoQConstants.scp\s*=\s*\'([^\']+)\'', webpage, 'policy')
-        signature = self._search_regex(r'InfoQConstants.scs\s*=\s*\'([^\']+)\'', webpage, 'signature')
-        key_pair_id = self._search_regex(r'InfoQConstants.sck\s*=\s*\'([^\']+)\'', webpage, 'key-pair-id')
+        policy = self._search_regex(r'InfoQConstants\.scp\s*=\s*\'([^\']+)\'', webpage, 'policy')
+        signature = self._search_regex(r'InfoQConstants\.scs\s*=\s*\'([^\']+)\'', webpage, 'signature')
+        key_pair_id = self._search_regex(r'InfoQConstants\.sck\s*=\s*\'([^\']+)\'', webpage, 'key-pair-id')
         return 'CloudFront-Policy=%s; CloudFront-Signature=%s; CloudFront-Key-Pair-Id=%s' % (
             policy, signature, key_pair_id)
 
diff --git a/youtube_dl/extractor/jeuxvideo.py b/youtube_dl/extractor/jeuxvideo.py
index 1a4227f..e9f4ed7 100644
--- a/youtube_dl/extractor/jeuxvideo.py
+++ b/youtube_dl/extractor/jeuxvideo.py
@@ -30,7 +30,7 @@ class JeuxVideoIE(InfoExtractor):
         webpage = self._download_webpage(url, title)
         title = self._html_search_meta('name', webpage) or self._og_search_title(webpage)
         config_url = self._html_search_regex(
-            r'data-src(?:set-video)?="(/contenu/medias/video.php.*?)"',
+            r'data-src(?:set-video)?="(/contenu/medias/video\.php.*?)"',
             webpage, 'config URL')
         config_url = 'http://www.jeuxvideo.com' + config_url
 
diff --git a/youtube_dl/extractor/livestream.py b/youtube_dl/extractor/livestream.py
index 7f946c6..317ebbc 100644
--- a/youtube_dl/extractor/livestream.py
+++ b/youtube_dl/extractor/livestream.py
@@ -338,7 +338,7 @@ class LivestreamOriginalIE(InfoExtractor):
                 info = {
                     'title': self._og_search_title(webpage),
                     'description': self._og_search_description(webpage),
-                    'thumbnail': self._search_regex(r'channelLogo.src\s*=\s*"([^"]+)"', webpage, 'thumbnail', None),
+                    'thumbnail': self._search_regex(r'channelLogo\.src\s*=\s*"([^"]+)"', webpage, 'thumbnail', None),
                 }
             video_data = self._download_json(stream_url, content_id)
             is_live = video_data.get('isLive')
diff --git a/youtube_dl/extractor/makertv.py b/youtube_dl/extractor/makertv.py
index 3c34d46..8eda69c 100644
--- a/youtube_dl/extractor/makertv.py
+++ b/youtube_dl/extractor/makertv.py
@@ -5,7 +5,7 @@ from .common import InfoExtractor
 
 
 class MakerTVIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:(?:www\.)?maker\.tv/(?:[^/]+/)*video|makerplayer.com/embed/maker)/(?P<id>[a-zA-Z0-9]{12})'
+    _VALID_URL = r'https?://(?:(?:www\.)?maker\.tv/(?:[^/]+/)*video|makerplayer\.com/embed/maker)/(?P<id>[a-zA-Z0-9]{12})'
     _TEST = {
         'url': 'http://www.maker.tv/video/Fh3QgymL9gsc',
         'md5': 'ca237a53a8eb20b6dc5bd60564d4ab3e',
diff --git a/youtube_dl/extractor/mangomolo.py b/youtube_dl/extractor/mangomolo.py
index 1885ac7..dbd761a 100644
--- a/youtube_dl/extractor/mangomolo.py
+++ b/youtube_dl/extractor/mangomolo.py
@@ -22,7 +22,7 @@ class MangomoloBaseIE(InfoExtractor):
 
         format_url = self._html_search_regex(
             [
-                r'file\s*:\s*"(https?://[^"]+?/playlist.m3u8)',
+                r'file\s*:\s*"(https?://[^"]+?/playlist\.m3u8)',
                 r'<a[^>]+href="(rtsp://[^"]+)"'
             ], webpage, 'format url')
         formats = self._extract_wowza_formats(
diff --git a/youtube_dl/extractor/meipai.py b/youtube_dl/extractor/meipai.py
index c8eacb4..2445b8b 100644
--- a/youtube_dl/extractor/meipai.py
+++ b/youtube_dl/extractor/meipai.py
@@ -11,7 +11,7 @@ from ..utils import (
 
 class MeipaiIE(InfoExtractor):
     IE_DESC = '美拍'
-    _VALID_URL = r'https?://(?:www\.)?meipai.com/media/(?P<id>[0-9]+)'
+    _VALID_URL = r'https?://(?:www\.)?meipai\.com/media/(?P<id>[0-9]+)'
     _TESTS = [{
         # regular uploaded video
         'url': 'http://www.meipai.com/media/531697625',
diff --git a/youtube_dl/extractor/mtv.py b/youtube_dl/extractor/mtv.py
index 25af5dd..1154a35 100644
--- a/youtube_dl/extractor/mtv.py
+++ b/youtube_dl/extractor/mtv.py
@@ -258,7 +258,7 @@ class MTVServicesInfoExtractor(InfoExtractor):
 
         if mgid is None or ':' not in mgid:
             mgid = self._search_regex(
-                [r'data-mgid="(.*?)"', r'swfobject.embedSWF\(".*?(mgid:.*?)"'],
+                [r'data-mgid="(.*?)"', r'swfobject\.embedSWF\(".*?(mgid:.*?)"'],
                 webpage, 'mgid', default=None)
 
         if not mgid:
diff --git a/youtube_dl/extractor/myvideo.py b/youtube_dl/extractor/myvideo.py
index 6bb64eb..367e811 100644
--- a/youtube_dl/extractor/myvideo.py
+++ b/youtube_dl/extractor/myvideo.py
@@ -160,7 +160,7 @@ class MyVideoIE(InfoExtractor):
         else:
             video_playpath = ''
 
-        video_swfobj = self._search_regex(r'swfobject.embedSWF\(\'(.+?)\'', webpage, 'swfobj')
+        video_swfobj = self._search_regex(r'swfobject\.embedSWF\(\'(.+?)\'', webpage, 'swfobj')
         video_swfobj = compat_urllib_parse_unquote(video_swfobj)
 
         video_title = self._html_search_regex("<h1(?: class='globalHd')?>(.*?)</h1>",
diff --git a/youtube_dl/extractor/nationalgeographic.py b/youtube_dl/extractor/nationalgeographic.py
index b91d865..9e8d28f 100644
--- a/youtube_dl/extractor/nationalgeographic.py
+++ b/youtube_dl/extractor/nationalgeographic.py
@@ -111,7 +111,7 @@ class NationalGeographicIE(ThePlatformIE, AdobePassIE):
         release_url = self._search_regex(
             r'video_auth_playlist_url\s*=\s*"([^"]+)"',
             webpage, 'release url')
-        theplatform_path = self._search_regex(r'https?://link.theplatform.com/s/([^?]+)', release_url, 'theplatform path')
+        theplatform_path = self._search_regex(r'https?://link\.theplatform\.com/s/([^?]+)', release_url, 'theplatform path')
         video_id = theplatform_path.split('/')[-1]
         query = {
             'mbr': 'true',
diff --git a/youtube_dl/extractor/naver.py b/youtube_dl/extractor/naver.py
index e813133..2047d44 100644
--- a/youtube_dl/extractor/naver.py
+++ b/youtube_dl/extractor/naver.py
@@ -43,7 +43,7 @@ class NaverIE(InfoExtractor):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        m_id = re.search(r'var rmcPlayer = new nhn.rmcnmv.RMCVideoPlayer\("(.+?)", "(.+?)"',
+        m_id = re.search(r'var rmcPlayer = new nhn\.rmcnmv\.RMCVideoPlayer\("(.+?)", "(.+?)"',
                          webpage)
         if m_id is None:
             error = self._html_search_regex(
diff --git a/youtube_dl/extractor/npo.py b/youtube_dl/extractor/npo.py
index fa4ef20..b8fe244 100644
--- a/youtube_dl/extractor/npo.py
+++ b/youtube_dl/extractor/npo.py
@@ -469,7 +469,7 @@ class SchoolTVIE(NPODataMidEmbedIE):
 
 class HetKlokhuisIE(NPODataMidEmbedIE):
     IE_NAME = 'hetklokhuis'
-    _VALID_URL = r'https?://(?:www\.)?hetklokhuis.nl/[^/]+/\d+/(?P<id>[^/?#&]+)'
+    _VALID_URL = r'https?://(?:www\.)?hetklokhuis\.nl/[^/]+/\d+/(?P<id>[^/?#&]+)'
 
     _TEST = {
         'url': 'http://hetklokhuis.nl/tv-uitzending/3471/Zwaartekrachtsgolven',
diff --git a/youtube_dl/extractor/ruhd.py b/youtube_dl/extractor/ruhd.py
index 2b830cf..3c8053a 100644
--- a/youtube_dl/extractor/ruhd.py
+++ b/youtube_dl/extractor/ruhd.py
@@ -25,7 +25,7 @@ class RUHDIE(InfoExtractor):
         video_url = self._html_search_regex(
             r'<param name="src" value="([^"]+)"', webpage, 'video url')
         title = self._html_search_regex(
-            r'<title>([^<]+)&nbsp;&nbsp; RUHD.ru - Видео Высокого качества №1 в России!</title>',
+            r'<title>([^<]+)&nbsp;&nbsp; RUHD\.ru - Видео Высокого качества №1 в России!</title>',
             webpage, 'title')
         description = self._html_search_regex(
             r'(?s)<div id="longdesc">(.+?)<span id="showlink">',
diff --git a/youtube_dl/extractor/stanfordoc.py b/youtube_dl/extractor/stanfordoc.py
index cce65fb..ae3dd13 100644
--- a/youtube_dl/extractor/stanfordoc.py
+++ b/youtube_dl/extractor/stanfordoc.py
@@ -66,7 +66,7 @@ class StanfordOpenClassroomIE(InfoExtractor):
                 r'(?s)<description>([^<]+)</description>',
                 coursepage, 'description', fatal=False)
 
-            links = orderedSet(re.findall(r'<a href="(VideoPage.php\?[^"]+)">', coursepage))
+            links = orderedSet(re.findall(r'<a href="(VideoPage\.php\?[^"]+)">', coursepage))
             info['entries'] = [self.url_result(
                 'http://openclassroom.stanford.edu/MainFolder/%s' % unescapeHTML(l)
             ) for l in links]
@@ -84,7 +84,7 @@ class StanfordOpenClassroomIE(InfoExtractor):
             rootpage = self._download_webpage(rootURL, info['id'],
                                               errnote='Unable to download course info page')
 
-            links = orderedSet(re.findall(r'<a href="(CoursePage.php\?[^"]+)">', rootpage))
+            links = orderedSet(re.findall(r'<a href="(CoursePage\.php\?[^"]+)">', rootpage))
             info['entries'] = [self.url_result(
                 'http://openclassroom.stanford.edu/MainFolder/%s' % unescapeHTML(l)
             ) for l in links]
diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py
index de236bb..b1a985f 100644
--- a/youtube_dl/extractor/theplatform.py
+++ b/youtube_dl/extractor/theplatform.py
@@ -216,7 +216,7 @@ class ThePlatformIE(ThePlatformBaseIE, AdobePassIE):
         def hex_to_bytes(hex):
             return binascii.a2b_hex(hex.encode('ascii'))
 
-        relative_path = re.match(r'https?://link.theplatform.com/s/([^?]+)', url).group(1)
+        relative_path = re.match(r'https?://link\.theplatform\.com/s/([^?]+)', url).group(1)
         clear_text = hex_to_bytes(flags + expiration_date + str_to_hex(relative_path))
         checksum = hmac.new(sig_key.encode('ascii'), clear_text, hashlib.sha1).hexdigest()
         sig = flags + expiration_date + checksum + str_to_hex(sig_secret)
diff --git a/youtube_dl/extractor/thisav.py b/youtube_dl/extractor/thisav.py
index 33683b1..dc3dd03 100644
--- a/youtube_dl/extractor/thisav.py
+++ b/youtube_dl/extractor/thisav.py
@@ -57,10 +57,10 @@ class ThisAVIE(InfoExtractor):
                 info_dict = self._extract_jwplayer_data(
                     webpage, video_id, require_title=False)
         uploader = self._html_search_regex(
-            r': <a href="http://www.thisav.com/user/[0-9]+/(?:[^"]+)">([^<]+)</a>',
+            r': <a href="http://www\.thisav\.com/user/[0-9]+/(?:[^"]+)">([^<]+)</a>',
             webpage, 'uploader name', fatal=False)
         uploader_id = self._html_search_regex(
-            r': <a href="http://www.thisav.com/user/[0-9]+/([^"]+)">(?:[^<]+)</a>',
+            r': <a href="http://www\.thisav\.com/user/[0-9]+/([^"]+)">(?:[^<]+)</a>',
             webpage, 'uploader id', fatal=False)
 
         info_dict.update({
diff --git a/youtube_dl/extractor/twitter.py b/youtube_dl/extractor/twitter.py
index 0df3ad7..1b0b963 100644
--- a/youtube_dl/extractor/twitter.py
+++ b/youtube_dl/extractor/twitter.py
@@ -174,7 +174,7 @@ class TwitterCardIE(TwitterBaseIE):
         webpage = self._download_webpage(url, video_id)
 
         iframe_url = self._html_search_regex(
-            r'<iframe[^>]+src="((?:https?:)?//(?:www.youtube.com/embed/[^"]+|(?:www\.)?vine\.co/v/\w+/card))"',
+            r'<iframe[^>]+src="((?:https?:)?//(?:www\.youtube\.com/embed/[^"]+|(?:www\.)?vine\.co/v/\w+/card))"',
             webpage, 'video iframe', default=None)
         if iframe_url:
             return self.url_result(iframe_url)
diff --git a/youtube_dl/extractor/vice.py b/youtube_dl/extractor/vice.py
index b8b8bf9..bcc2869 100644
--- a/youtube_dl/extractor/vice.py
+++ b/youtube_dl/extractor/vice.py
@@ -198,7 +198,7 @@ class ViceShowIE(InfoExtractor):
 
 class ViceArticleIE(InfoExtractor):
     IE_NAME = 'vice:article'
-    _VALID_URL = r'https://www.vice.com/[^/]+/article/(?P<id>[^?#]+)'
+    _VALID_URL = r'https://www\.vice\.com/[^/]+/article/(?P<id>[^?#]+)'
 
     _TESTS = [{
         'url': 'https://www.vice.com/en_us/article/on-set-with-the-woman-making-mormon-porn-in-utah',
diff --git a/youtube_dl/extractor/videopremium.py b/youtube_dl/extractor/videopremium.py
index 5de8273..cf690d7 100644
--- a/youtube_dl/extractor/videopremium.py
+++ b/youtube_dl/extractor/videopremium.py
@@ -26,7 +26,7 @@ class VideoPremiumIE(InfoExtractor):
         webpage_url = 'http://videopremium.tv/' + video_id
         webpage = self._download_webpage(webpage_url, video_id)
 
-        if re.match(r'^<html><head><script[^>]*>window.location\s*=', webpage):
+        if re.match(r'^<html><head><script[^>]*>window\.location\s*=', webpage):
             # Download again, we need a cookie
             webpage = self._download_webpage(
                 webpage_url, video_id,
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index edd8713..54f5d72 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1683,7 +1683,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         video_uploader_id = None
         video_uploader_url = None
         mobj = re.search(
-            r'<link itemprop="url" href="(?P<uploader_url>https?://www.youtube.com/(?:user|channel)/(?P<uploader_id>[^"]+))">',
+            r'<link itemprop="url" href="(?P<uploader_url>https?://www\.youtube\.com/(?:user|channel)/(?P<uploader_id>[^"]+))">',
             video_webpage)
         if mobj is not None:
             video_uploader_id = mobj.group('uploader_id')
-- 
cgit v1.1


From ae5af89079558e0e128dc4d7f033459e68ad81e6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 9 Oct 2017 23:52:39 +0700
Subject: [hrti:playlist] Relax _VALID_URL

---
 youtube_dl/extractor/hrti.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/hrti.py b/youtube_dl/extractor/hrti.py
index 7cef5f6..6424d34 100644
--- a/youtube_dl/extractor/hrti.py
+++ b/youtube_dl/extractor/hrti.py
@@ -173,7 +173,7 @@ class HRTiIE(HRTiBaseIE):
 
 
 class HRTiPlaylistIE(HRTiBaseIE):
-    _VALID_URL = r'https?://hrti\.hrt\.hr/#/video/list/category/(?P<id>[0-9]+)/(?P<display_id>[^/]+)?'
+    _VALID_URL = r'https?://hrti\.hrt\.hr/(?:#/)?video/list/category/(?P<id>[0-9]+)/(?P<display_id>[^/]+)?'
     _TESTS = [{
         'url': 'https://hrti.hrt.hr/#/video/list/category/212/ekumena',
         'info_dict': {
@@ -185,6 +185,9 @@ class HRTiPlaylistIE(HRTiBaseIE):
     }, {
         'url': 'https://hrti.hrt.hr/#/video/list/category/212/',
         'only_matching': True,
+    }, {
+        'url': 'https://hrti.hrt.hr/video/list/category/212/ekumena',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
-- 
cgit v1.1


From 9e71f88105e546573b8615d49f9d0c064496d6e6 Mon Sep 17 00:00:00 2001
From: Silvan Mosberger <infinisil@icloud.com>
Date: Mon, 9 Oct 2017 22:48:26 +0200
Subject: [vvvvid] Fix typo

---
 youtube_dl/extractor/vvvvid.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vvvvid.py b/youtube_dl/extractor/vvvvid.py
index d44ec85..656a4b9 100644
--- a/youtube_dl/extractor/vvvvid.py
+++ b/youtube_dl/extractor/vvvvid.py
@@ -133,7 +133,7 @@ class VVVVIDIE(InfoExtractor):
             'season_id': season_id,
             'season_number': video_data.get('season_number'),
             'episode_id': str_or_none(video_data.get('id')),
-            'epidode_number': int_or_none(video_data.get('number')),
+            'episode_number': int_or_none(video_data.get('number')),
             'episode_title': video_data['title'],
             'view_count': int_or_none(video_data.get('views')),
             'like_count': int_or_none(video_data.get('video_likes')),
-- 
cgit v1.1


From 01c742ecd09be734ca4f3db08aba73424683ac1b Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 10 Oct 2017 23:20:38 +0800
Subject: [facebook] Support thumbnails (closes #14416)

---
 ChangeLog                        |  6 ++++++
 youtube_dl/extractor/facebook.py | 18 ++++++++++++++----
 2 files changed, 20 insertions(+), 4 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 2feb3dc..2684cbc 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,9 @@
+version <unreleased>
+
+Extractors
++ [facebook] Support thumbnails (#14416)
+
+
 version 2017.10.07
 
 Core
diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py
index 4b3f6cc..220ada3 100644
--- a/youtube_dl/extractor/facebook.py
+++ b/youtube_dl/extractor/facebook.py
@@ -67,9 +67,9 @@ class FacebookIE(InfoExtractor):
             'uploader': 'Tennis on Facebook',
             'upload_date': '20140908',
             'timestamp': 1410199200,
-        }
+        },
+        'skip': 'Requires logging in',
     }, {
-        'note': 'Video without discernible title',
         'url': 'https://www.facebook.com/video.php?v=274175099429670',
         'info_dict': {
             'id': '274175099429670',
@@ -78,6 +78,7 @@ class FacebookIE(InfoExtractor):
             'uploader': 'Asif Nawab Butt',
             'upload_date': '20140506',
             'timestamp': 1399398998,
+            'thumbnail': r're:^https?://.*',
         },
         'expected_warnings': [
             'title'
@@ -94,6 +95,7 @@ class FacebookIE(InfoExtractor):
             'upload_date': '20160110',
             'timestamp': 1452431627,
         },
+        'skip': 'Requires logging in',
     }, {
         'url': 'https://www.facebook.com/maxlayn/posts/10153807558977570',
         'md5': '037b1fa7f3c2d02b7a0d7bc16031ecc6',
@@ -121,7 +123,11 @@ class FacebookIE(InfoExtractor):
         'info_dict': {
             'id': '10153664894881749',
             'ext': 'mp4',
-            'title': 'Facebook video #10153664894881749',
+            'title': 'Average time to confirm recent Supreme Court nominees: 67 days Longest it\'s t...',
+            'thumbnail': r're:^https?://.*',
+            'timestamp': 1456259628,
+            'upload_date': '20160223',
+            'uploader': 'Barack Obama',
         },
     }, {
         # have 1080P, but only up to 720p in swf params
@@ -130,10 +136,11 @@ class FacebookIE(InfoExtractor):
         'info_dict': {
             'id': '10155529876156509',
             'ext': 'mp4',
-            'title': 'Holocaust survivor becomes US citizen',
+            'title': 'She survived the holocaust — and years later, she’s getting her citizenship s...',
             'timestamp': 1477818095,
             'upload_date': '20161030',
             'uploader': 'CNN',
+            'thumbnail': r're:^https?://.*',
         },
     }, {
         # bigPipe.onPageletArrive ... onPageletArrive pagelet_group_mall
@@ -158,6 +165,7 @@ class FacebookIE(InfoExtractor):
             'timestamp': 1477305000,
             'upload_date': '20161024',
             'uploader': 'La Guía Del Varón',
+            'thumbnail': r're:^https?://.*',
         },
         'params': {
             'skip_download': True,
@@ -376,6 +384,7 @@ class FacebookIE(InfoExtractor):
         timestamp = int_or_none(self._search_regex(
             r'<abbr[^>]+data-utime=["\'](\d+)', webpage,
             'timestamp', default=None))
+        thumbnail = self._og_search_thumbnail(webpage)
 
         info_dict = {
             'id': video_id,
@@ -383,6 +392,7 @@ class FacebookIE(InfoExtractor):
             'formats': formats,
             'uploader': uploader,
             'timestamp': timestamp,
+            'thumbnail': thumbnail,
         }
 
         return webpage, info_dict
-- 
cgit v1.1


From d0f2d6411406a35c9593bbb85375c2d7f8300c77 Mon Sep 17 00:00:00 2001
From: Jakub Wilk <jwilk@jwilk.net>
Date: Tue, 10 Oct 2017 18:45:10 +0200
Subject: [slideslive] Add extractor (closes #2680)

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/slideslive.py | 34 ++++++++++++++++++++++++++++++++++
 2 files changed, 35 insertions(+)
 create mode 100644 youtube_dl/extractor/slideslive.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 24e9acd..d0f71ae 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -943,6 +943,7 @@ from .skynewsarabia import (
 )
 from .skysports import SkySportsIE
 from .slideshare import SlideshareIE
+from .slideslive import SlidesLiveIE
 from .slutload import SlutloadIE
 from .smotri import (
     SmotriIE,
diff --git a/youtube_dl/extractor/slideslive.py b/youtube_dl/extractor/slideslive.py
new file mode 100644
index 0000000..1045760
--- /dev/null
+++ b/youtube_dl/extractor/slideslive.py
@@ -0,0 +1,34 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import ExtractorError
+
+
+class SlidesLiveIE(InfoExtractor):
+    _VALID_URL = r'https?://slideslive\.com/(?P<id>[0-9]+)'
+    _TESTS = [{
+        'url': 'https://slideslive.com/38902413/gcc-ia16-backend',
+        'md5': 'b29fcd6c6952d0c79c5079b0e7a07e6f',
+        'info_dict': {
+            'id': 'LMtgR8ba0b0',
+            'ext': 'mp4',
+            'title': '38902413: external video',
+            'description': '3890241320170925-9-1yd6ech.mp4',
+            'uploader': 'SlidesLive Administrator',
+            'uploader_id': 'UC62SdArr41t_-_fX40QCLRw',
+            'upload_date': '20170925',
+        }
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        video_data = self._download_json(
+            url, video_id, headers={'Accept': 'application/json'})
+        service_name = video_data['video_service_name']
+        if service_name == 'YOUTUBE':
+            yt_video_id = video_data['video_service_id']
+            return self.url_result(yt_video_id, 'Youtube', video_id=yt_video_id)
+        else:
+            raise ExtractorError(
+                'Unsupported service name: {0}'.format(service_name), expected=True)
-- 
cgit v1.1


From 04af3aca049588b6b3d4d4b57ee47224fdeee90f Mon Sep 17 00:00:00 2001
From: Khang Nguyen <khang.neon.1997@gmail.com>
Date: Thu, 5 Oct 2017 21:37:18 +0700
Subject: Remove YoutubeSharedVideoIE 
 https://github.com/rg3/youtube-dl/issues/14303

---
 youtube_dl/extractor/extractors.py |  1 -
 youtube_dl/extractor/youtube.py    | 33 ---------------------------------
 2 files changed, 34 deletions(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index d0f71ae..d96eafb 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1346,7 +1346,6 @@ from .youtube import (
     YoutubeSearchDateIE,
     YoutubeSearchIE,
     YoutubeSearchURLIE,
-    YoutubeSharedVideoIE,
     YoutubeShowIE,
     YoutubeSubscriptionsIE,
     YoutubeTruncatedIDIE,
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 54f5d72..6e2d57d 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -2040,39 +2040,6 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         }
 
 
-class YoutubeSharedVideoIE(InfoExtractor):
-    _VALID_URL = r'(?:https?:)?//(?:www\.)?youtube\.com/shared\?.*\bci=(?P<id>[0-9A-Za-z_-]{11})'
-    IE_NAME = 'youtube:shared'
-
-    _TEST = {
-        'url': 'https://www.youtube.com/shared?ci=1nEzmT-M4fU',
-        'info_dict': {
-            'id': 'uPDB5I9wfp8',
-            'ext': 'webm',
-            'title': 'Pocoyo: 90 minutos de episódios completos Português para crianças - PARTE 3',
-            'description': 'md5:d9e4d9346a2dfff4c7dc4c8cec0f546d',
-            'upload_date': '20160219',
-            'uploader': 'Pocoyo - Português (BR)',
-            'uploader_id': 'PocoyoBrazil',
-        },
-        'add_ie': ['Youtube'],
-        'params': {
-            # There are already too many Youtube downloads
-            'skip_download': True,
-        },
-    }
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-
-        webpage = self._download_webpage(url, video_id)
-
-        real_video_id = self._html_search_meta(
-            'videoId', webpage, 'YouTube video id', fatal=True)
-
-        return self.url_result(real_video_id, YoutubeIE.ie_key())
-
-
 class YoutubePlaylistIE(YoutubePlaylistBaseInfoExtractor):
     IE_DESC = 'YouTube.com playlists'
     _VALID_URL = r"""(?x)(?:
-- 
cgit v1.1


From dfc80bdd2e4ef3d30f161a93f99f3050537944ab Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 11 Oct 2017 02:03:00 +0800
Subject: [ChangeLog] Update after #14420

---
 ChangeLog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ChangeLog b/ChangeLog
index 2684cbc..c541c4f 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,6 +1,7 @@
 version <unreleased>
 
 Extractors
+- [youtube:shared] Removed extractor (#14420)
 + [facebook] Support thumbnails (#14416)
 
 
-- 
cgit v1.1


From cdab1df91242fb617b09a31c023822ef31ea37b8 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 11 Oct 2017 10:04:46 +0000
Subject: [afreecatv] remove AfreecaTVGlobalIE

the website now show this message
> Global AfreecaTV will be merged and integrated on July 20th, 2017.
Every user around the world are now able to interact with one another on
www.afreecatv.com!
---
 youtube_dl/extractor/afreecatv.py  | 104 -------------------------------------
 youtube_dl/extractor/extractors.py |   5 +-
 2 files changed, 1 insertion(+), 108 deletions(-)

diff --git a/youtube_dl/extractor/afreecatv.py b/youtube_dl/extractor/afreecatv.py
index 2c58f46..e6513c7 100644
--- a/youtube_dl/extractor/afreecatv.py
+++ b/youtube_dl/extractor/afreecatv.py
@@ -271,107 +271,3 @@ class AfreecaTVIE(InfoExtractor):
             })
 
         return info
-
-
-class AfreecaTVGlobalIE(AfreecaTVIE):
-    IE_NAME = 'afreecatv:global'
-    _VALID_URL = r'https?://(?:www\.)?afreeca\.tv/(?P<channel_id>\d+)(?:/v/(?P<video_id>\d+))?'
-    _TESTS = [{
-        'url': 'http://afreeca.tv/36853014/v/58301',
-        'info_dict': {
-            'id': '58301',
-            'title': 'tryhard top100',
-            'uploader_id': '36853014',
-            'uploader': 'makgi Hearthstone Live!',
-        },
-        'playlist_count': 3,
-    }]
-
-    def _real_extract(self, url):
-        channel_id, video_id = re.match(self._VALID_URL, url).groups()
-        video_type = 'video' if video_id else 'live'
-        query = {
-            'pt': 'view',
-            'bid': channel_id,
-        }
-        if video_id:
-            query['vno'] = video_id
-        video_data = self._download_json(
-            'http://api.afreeca.tv/%s/view_%s.php' % (video_type, video_type),
-            video_id or channel_id, query=query)['channel']
-
-        if video_data.get('result') != 1:
-            raise ExtractorError('%s said: %s' % (self.IE_NAME, video_data['remsg']))
-
-        title = video_data['title']
-
-        info = {
-            'thumbnail': video_data.get('thumb'),
-            'view_count': int_or_none(video_data.get('vcnt')),
-            'age_limit': int_or_none(video_data.get('grade')),
-            'uploader_id': channel_id,
-            'uploader': video_data.get('cname'),
-        }
-
-        if video_id:
-            entries = []
-            for i, f in enumerate(video_data.get('flist', [])):
-                video_key = self.parse_video_key(f.get('key', ''))
-                f_url = f.get('file')
-                if not video_key or not f_url:
-                    continue
-                entries.append({
-                    'id': '%s_%s' % (video_id, video_key.get('part', i + 1)),
-                    'title': title,
-                    'upload_date': video_key.get('upload_date'),
-                    'duration': int_or_none(f.get('length')),
-                    'url': f_url,
-                    'protocol': 'm3u8_native',
-                    'ext': 'mp4',
-                })
-
-            info.update({
-                'id': video_id,
-                'title': title,
-                'duration': int_or_none(video_data.get('length')),
-            })
-            if len(entries) > 1:
-                info['_type'] = 'multi_video'
-                info['entries'] = entries
-            elif len(entries) == 1:
-                i = entries[0].copy()
-                i.update(info)
-                info = i
-        else:
-            formats = []
-            for s in video_data.get('strm', []):
-                s_url = s.get('purl')
-                if not s_url:
-                    continue
-                stype = s.get('stype')
-                if stype == 'HLS':
-                    formats.extend(self._extract_m3u8_formats(
-                        s_url, channel_id, 'mp4', m3u8_id=stype, fatal=False))
-                elif stype == 'RTMP':
-                    format_id = [stype]
-                    label = s.get('label')
-                    if label:
-                        format_id.append(label)
-                    formats.append({
-                        'format_id': '-'.join(format_id),
-                        'url': s_url,
-                        'tbr': int_or_none(s.get('bps')),
-                        'height': int_or_none(s.get('brt')),
-                        'ext': 'flv',
-                        'rtmp_live': True,
-                    })
-            self._sort_formats(formats)
-
-            info.update({
-                'id': channel_id,
-                'title': self._live_title(title),
-                'is_live': True,
-                'formats': formats,
-            })
-
-        return info
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index d96eafb..a363d95 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -31,10 +31,7 @@ from .aenetworks import (
     AENetworksIE,
     HistoryTopicIE,
 )
-from .afreecatv import (
-    AfreecaTVIE,
-    AfreecaTVGlobalIE,
-)
+from .afreecatv import AfreecaTVIE
 from .airmozilla import AirMozillaIE
 from .aljazeera import AlJazeeraIE
 from .alphaporno import AlphaPornoIE
-- 
cgit v1.1


From 4fe4bda287f4b506850f0baa6ff86445423751e7 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 11 Oct 2017 11:36:05 +0000
Subject: [tubitv] add support for new url format(fixes #14460)

---
 youtube_dl/extractor/tubitv.py | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/tubitv.py b/youtube_dl/extractor/tubitv.py
index c44018a..36f6c16 100644
--- a/youtube_dl/extractor/tubitv.py
+++ b/youtube_dl/extractor/tubitv.py
@@ -13,11 +13,11 @@ from ..utils import (
 
 
 class TubiTvIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?tubitv\.com/video/(?P<id>[0-9]+)'
+    _VALID_URL = r'https?://(?:www\.)?tubitv\.com/(?:video|movies|tv-shows)/(?P<id>[0-9]+)'
     _LOGIN_URL = 'http://tubitv.com/login'
     _NETRC_MACHINE = 'tubitv'
     _GEO_COUNTRIES = ['US']
-    _TEST = {
+    _TESTS = [{
         'url': 'http://tubitv.com/video/283829/the_comedian_at_the_friday',
         'md5': '43ac06be9326f41912dc64ccf7a80320',
         'info_dict': {
@@ -27,7 +27,13 @@ class TubiTvIE(InfoExtractor):
             'description': 'A stand up comedian is forced to look at the decisions in his life while on a one week trip to the west coast.',
             'uploader_id': 'bc168bee0d18dd1cb3b86c68706ab434',
         },
-    }
+    }, {
+        'url': 'http://tubitv.com/tv-shows/321886/s01_e01_on_nom_stories',
+        'only_matching': True,
+    }, {
+        'url': 'http://tubitv.com/movies/383676/tracker',
+        'only_matching': True,
+    }]
 
     def _login(self):
         (username, password) = self._get_login_info()
-- 
cgit v1.1


From 5fe75f976f6c8da76bfea68e073e5f35c4300442 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 11 Oct 2017 14:14:51 +0000
Subject: [tva] fix extraction(fixes #14328)

---
 youtube_dl/extractor/tva.py | 48 ++++++++++++++++++++++-----------------------
 1 file changed, 23 insertions(+), 25 deletions(-)

diff --git a/youtube_dl/extractor/tva.py b/youtube_dl/extractor/tva.py
index 3ced098..b57abea 100644
--- a/youtube_dl/extractor/tva.py
+++ b/youtube_dl/extractor/tva.py
@@ -3,52 +3,50 @@ from __future__ import unicode_literals
 
 from .common import InfoExtractor
 from ..utils import (
-    int_or_none,
-    parse_iso8601,
+    float_or_none,
     smuggle_url,
 )
 
 
 class TVAIE(InfoExtractor):
-    _VALID_URL = r'https?://videos\.tva\.ca/episode/(?P<id>\d+)'
+    _VALID_URL = r'https?://videos\.tva\.ca/details/_(?P<id>\d+)'
     _TEST = {
-        'url': 'http://videos.tva.ca/episode/85538',
+        'url': 'https://videos.tva.ca/details/_5596811470001',
         'info_dict': {
-            'id': '85538',
+            'id': '5596811470001',
             'ext': 'mp4',
-            'title': 'Épisode du 25 janvier 2017',
-            'description': 'md5:e9e7fb5532ab37984d2dc87229cadf98',
-            'upload_date': '20170126',
-            'timestamp': 1485442329,
+            'title': 'Un extrait de l\'épisode du dimanche 8 octobre 2017 !',
+            'uploader_id': '5481942443001',
+            'upload_date': '20171003',
+            'timestamp': 1507064617,
         },
         'params': {
             # m3u8 download
             'skip_download': True,
         }
     }
+    BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/5481942443001/default_default/index.html?videoId=%s'
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
         video_data = self._download_json(
-            "https://d18jmrhziuoi7p.cloudfront.net/isl/api/v1/dataservice/Items('%s')" % video_id,
-            video_id, query={
-                '$expand': 'Metadata,CustomId',
-                '$select': 'Metadata,Id,Title,ShortDescription,LongDescription,CreatedDate,CustomId,AverageUserRating,Categories,ShowName',
-                '$format': 'json',
+            'https://videos.tva.ca/proxy/item/_' + video_id, video_id, headers={
+                'Accept': 'application/json',
             })
-        metadata = video_data.get('Metadata', {})
+
+        def get_attribute(key):
+            for attribute in video_data.get('attributes', []):
+                if attribute.get('key') == key:
+                    return attribute.get('value')
+            return None
 
         return {
             '_type': 'url_transparent',
             'id': video_id,
-            'title': video_data['Title'],
-            'url': smuggle_url('ooyala:' + video_data['CustomId'], {'supportedformats': 'm3u8,hds'}),
-            'description': video_data.get('LongDescription') or video_data.get('ShortDescription'),
-            'series': video_data.get('ShowName'),
-            'episode': metadata.get('EpisodeTitle'),
-            'episode_number': int_or_none(metadata.get('EpisodeNumber')),
-            'categories': video_data.get('Categories'),
-            'average_rating': video_data.get('AverageUserRating'),
-            'timestamp': parse_iso8601(video_data.get('CreatedDate')),
-            'ie_key': 'Ooyala',
+            'title': get_attribute('title'),
+            'url': smuggle_url(self.BRIGHTCOVE_URL_TEMPLATE % video_id, {'geo_countries': ['CA']}),
+            'description': get_attribute('description'),
+            'thumbnail': get_attribute('image-background') or get_attribute('image-landscape'),
+            'duration': float_or_none(get_attribute('video-duration'), 1000),
+            'ie_key': 'BrightcoveNew',
         }
-- 
cgit v1.1


From 26bae2d96509b5c5ec80c27c1ea754b53c53818c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 11 Oct 2017 21:59:30 +0700
Subject: [generic] Add support for channel9 embeds (closes #14469)

---
 youtube_dl/extractor/channel9.py | 6 ++++++
 youtube_dl/extractor/generic.py  | 6 ++++++
 2 files changed, 12 insertions(+)

diff --git a/youtube_dl/extractor/channel9.py b/youtube_dl/extractor/channel9.py
index e928942..81108e7 100644
--- a/youtube_dl/extractor/channel9.py
+++ b/youtube_dl/extractor/channel9.py
@@ -81,6 +81,12 @@ class Channel9IE(InfoExtractor):
 
     _RSS_URL = 'http://channel9.msdn.com/%s/RSS'
 
+    @staticmethod
+    def _extract_urls(webpage):
+        return re.findall(
+            r'<iframe[^>]+src=["\'](https?://channel9\.msdn\.com/(?:[^/]+/)+)player\b',
+            webpage)
+
     def _extract_list(self, video_id, rss_url=None):
         if not rss_url:
             rss_url = self._RSS_URL % video_id
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 68b6338..6dab4c7 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -101,6 +101,7 @@ from .mediaset import MediasetIE
 from .joj import JojIE
 from .megaphone import MegaphoneIE
 from .vzaar import VzaarIE
+from .channel9 import Channel9IE
 
 
 class GenericIE(InfoExtractor):
@@ -2871,6 +2872,11 @@ class GenericIE(InfoExtractor):
             return self.playlist_from_matches(
                 vzaar_urls, video_id, video_title, ie=VzaarIE.ie_key())
 
+        channel9_urls = Channel9IE._extract_urls(webpage)
+        if channel9_urls:
+            return self.playlist_from_matches(
+                channel9_urls, video_id, video_title, ie=Channel9IE.ie_key())
+
         def merge_dicts(dict1, dict2):
             merged = {}
             for k, v in dict1.items():
-- 
cgit v1.1


From 782195a9d417aab21cff4abe35ad9d705f4d8d83 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 11 Oct 2017 15:48:32 +0000
Subject: [once] add support for vmap urls

---
 youtube_dl/extractor/gamespot.py | 2 +-
 youtube_dl/extractor/once.py     | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/gamespot.py b/youtube_dl/extractor/gamespot.py
index 00d3111..02804d2 100644
--- a/youtube_dl/extractor/gamespot.py
+++ b/youtube_dl/extractor/gamespot.py
@@ -105,7 +105,7 @@ class GameSpotIE(OnceIE):
             onceux_url = self._parse_json(unescapeHTML(onceux_json), page_id).get('metadataUri')
             if onceux_url:
                 formats.extend(self._extract_once_formats(re.sub(
-                    r'https?://[^/]+', 'http://once.unicornmedia.com', onceux_url).replace('ads/vmap/', '')))
+                    r'https?://[^/]+', 'http://once.unicornmedia.com', onceux_url)))
 
         if not formats:
             for quality in ['sd', 'hd']:
diff --git a/youtube_dl/extractor/once.py b/youtube_dl/extractor/once.py
index 1bf96ea..a637c8e 100644
--- a/youtube_dl/extractor/once.py
+++ b/youtube_dl/extractor/once.py
@@ -7,7 +7,7 @@ from .common import InfoExtractor
 
 
 class OnceIE(InfoExtractor):
-    _VALID_URL = r'https?://.+?\.unicornmedia\.com/now/[^/]+/[^/]+/(?P<domain_id>[^/]+)/(?P<application_id>[^/]+)/(?:[^/]+/)?(?P<media_item_id>[^/]+)/content\.(?:once|m3u8|mp4)'
+    _VALID_URL = r'https?://.+?\.unicornmedia\.com/now/(?:ads/vmap/)?[^/]+/[^/]+/(?P<domain_id>[^/]+)/(?P<application_id>[^/]+)/(?:[^/]+/)?(?P<media_item_id>[^/]+)/content\.(?:once|m3u8|mp4)'
     ADAPTIVE_URL_TEMPLATE = 'http://once.unicornmedia.com/now/master/playlist/%s/%s/%s/content.m3u8'
     PROGRESSIVE_URL_TEMPLATE = 'http://once.unicornmedia.com/now/media/progressive/%s/%s/%s/%s/content.mp4'
 
-- 
cgit v1.1


From 9e38dbb19ca6874c7350647ea2883d5bbb3b50a1 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 11 Oct 2017 15:50:00 +0000
Subject: [voxmedia] add support for recode.net(fixes #14173)

---
 youtube_dl/extractor/extractors.py |  5 ++-
 youtube_dl/extractor/voxmedia.py   | 66 ++++++++++++++++++++++++++++++--------
 2 files changed, 57 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index a363d95..5629c76 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1244,7 +1244,10 @@ from .vodpl import VODPlIE
 from .vodplatform import VODPlatformIE
 from .voicerepublic import VoiceRepublicIE
 from .voot import VootIE
-from .voxmedia import VoxMediaIE
+from .voxmedia import (
+    VoxMediaVolumeIE,
+    VoxMediaIE,
+)
 from .vporn import VpornIE
 from .vrt import VRTIE
 from .vrak import VrakIE
diff --git a/youtube_dl/extractor/voxmedia.py b/youtube_dl/extractor/voxmedia.py
index f8e3314..c7a0a88 100644
--- a/youtube_dl/extractor/voxmedia.py
+++ b/youtube_dl/extractor/voxmedia.py
@@ -2,11 +2,44 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from .once import OnceIE
 from ..compat import compat_urllib_parse_unquote
+from ..utils import ExtractorError
+
+
+class VoxMediaVolumeIE(OnceIE):
+    _VALID_URL = r'https?://volume\.vox-cdn\.com/embed/(?P<id>[0-9a-f]{9})'
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+        video_data = self._parse_json(self._search_regex(
+            r'Volume\.createVideo\(({.+})\s*,\s*{.*}\s*,\s*\[.*\]\s*,\s*{.*}\);', webpage, 'video data'), video_id)
+        for provider_video_type in ('ooyala', 'youtube', 'brightcove'):
+            provider_video_id = video_data.get('%s_id' % provider_video_type)
+            if not provider_video_id:
+                continue
+            info = {
+                'id': video_id,
+                'title': video_data.get('title_short'),
+                'description': video_data.get('description_long') or video_data.get('description_short'),
+                'thumbnail': video_data.get('brightcove_thumbnail')
+            }
+            if provider_video_type == 'brightcove':
+                info['formats'] = self._extract_once_formats(provider_video_id)
+                self._sort_formats(info['formats'])
+            else:
+                info.update({
+                    '_type': 'url_transparent',
+                    'url': provider_video_id if provider_video_type == 'youtube' else '%s:%s' % (provider_video_type, provider_video_id),
+                    'ie_key': provider_video_type.capitalize(),
+                })
+            return info
+        raise ExtractorError('Unable to find provider video id')
 
 
 class VoxMediaIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?(?:theverge|vox|sbnation|eater|polygon|curbed|racked)\.com/(?:[^/]+/)*(?P<id>[^/?]+)'
+    _VALID_URL = r'https?://(?:www\.)?(?:(?:theverge|vox|sbnation|eater|polygon|curbed|racked)\.com|recode\.net)/(?:[^/]+/)*(?P<id>[^/?]+)'
     _TESTS = [{
         'url': 'http://www.theverge.com/2014/6/27/5849272/material-world-how-google-discovered-what-software-is-made-of',
         'info_dict': {
@@ -31,6 +64,7 @@ class VoxMediaIE(InfoExtractor):
             'description': 'md5:87a51fe95ff8cea8b5bdb9ac7ae6a6af',
         },
         'add_ie': ['Ooyala'],
+        'skip': 'Video Not Found',
     }, {
         # volume embed
         'url': 'http://www.vox.com/2016/3/31/11336640/mississippi-lgbt-religious-freedom-bill',
@@ -84,6 +118,17 @@ class VoxMediaIE(InfoExtractor):
                 'description': 'md5:e02d56b026d51aa32c010676765a690d',
             },
         }],
+    }, {
+        # volume embed, Brightcove Once
+        'url': 'https://www.recode.net/2014/6/17/11628066/post-post-pc-ceo-the-full-code-conference-video-of-microsofts-satya',
+        'md5': '01571a896281f77dc06e084138987ea2',
+        'info_dict': {
+            'id': '1231c973d',
+            'ext': 'mp4',
+            'title': 'Post-Post-PC CEO: The Full Code Conference Video of Microsoft\'s Satya Nadella',
+            'description': 'The longtime veteran was chosen earlier this year as the software giant\'s third leader in its history.',
+        },
+        'add_ie': ['VoxMediaVolume'],
     }]
 
     def _real_extract(self, url):
@@ -91,9 +136,14 @@ class VoxMediaIE(InfoExtractor):
         webpage = compat_urllib_parse_unquote(self._download_webpage(url, display_id))
 
         def create_entry(provider_video_id, provider_video_type, title=None, description=None):
+            video_url = {
+                'youtube': '%s',
+                'ooyala': 'ooyala:%s',
+                'volume': 'http://volume.vox-cdn.com/embed/%s',
+            }[provider_video_type] % provider_video_id
             return {
                 '_type': 'url_transparent',
-                'url': provider_video_id if provider_video_type == 'youtube' else '%s:%s' % (provider_video_type, provider_video_id),
+                'url': video_url,
                 'title': title or self._og_search_title(webpage),
                 'description': description or self._og_search_description(webpage),
             }
@@ -124,17 +174,7 @@ class VoxMediaIE(InfoExtractor):
         volume_uuid = self._search_regex(
             r'data-volume-uuid="([^"]+)"', webpage, 'volume uuid', default=None)
         if volume_uuid:
-            volume_webpage = self._download_webpage(
-                'http://volume.vox-cdn.com/embed/%s' % volume_uuid, volume_uuid)
-            video_data = self._parse_json(self._search_regex(
-                r'Volume\.createVideo\(({.+})\s*,\s*{.*}\s*,\s*\[.*\]\s*,\s*{.*}\);', volume_webpage, 'video data'), volume_uuid)
-            for provider_video_type in ('ooyala', 'youtube'):
-                provider_video_id = video_data.get('%s_id' % provider_video_type)
-                if provider_video_id:
-                    description = video_data.get('description_long') or video_data.get('description_short')
-                    entries.append(create_entry(
-                        provider_video_id, provider_video_type, video_data.get('title_short'), description))
-                    break
+            entries.append(create_entry(volume_uuid, 'volume'))
 
         if len(entries) == 1:
             return entries[0]
-- 
cgit v1.1


From af0f74288dc1b46147bc8f6b5692d2a21c6e178b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 11 Oct 2017 23:45:03 +0700
Subject: [YoutubeDL] Improve _default_format_spec (closes #14461)

---
 test/test_YoutubeDL.py  | 10 ++++++++--
 youtube_dl/YoutubeDL.py | 27 ++++++++++++++++-----------
 2 files changed, 24 insertions(+), 13 deletions(-)

diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index db936bf..4af92fb 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -466,12 +466,18 @@ class TestFormatSelection(unittest.TestCase):
         ydl = YDL({'simulate': True})
         self.assertEqual(ydl._default_format_spec({}), 'bestvideo+bestaudio/best')
 
+        ydl = YDL({'is_live': True})
+        self.assertEqual(ydl._default_format_spec({}), 'best/bestvideo+bestaudio')
+
+        ydl = YDL({'simulate': True, 'is_live': True})
+        self.assertEqual(ydl._default_format_spec({}), 'bestvideo+bestaudio/best')
+
         ydl = YDL({'outtmpl': '-'})
-        self.assertEqual(ydl._default_format_spec({}), 'best')
+        self.assertEqual(ydl._default_format_spec({}), 'best/bestvideo+bestaudio')
 
         ydl = YDL({})
         self.assertEqual(ydl._default_format_spec({}, download=False), 'bestvideo+bestaudio/best')
-        self.assertEqual(ydl._default_format_spec({'is_live': True}), 'best')
+        self.assertEqual(ydl._default_format_spec({'is_live': True}), 'best/bestvideo+bestaudio')
 
 
 class TestYoutubeDL(unittest.TestCase):
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 855d6b8..342d6b4 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1078,22 +1078,27 @@ class YoutubeDL(object):
         return _filter
 
     def _default_format_spec(self, info_dict, download=True):
-        req_format_list = []
 
-        def can_have_partial_formats():
+        def can_merge():
+            merger = FFmpegMergerPP(self)
+            return merger.available and merger.can_merge()
+
+        def prefer_best():
             if self.params.get('simulate', False):
-                return True
+                return False
             if not download:
-                return True
-            if self.params.get('outtmpl', DEFAULT_OUTTMPL) == '-':
                 return False
+            if self.params.get('outtmpl', DEFAULT_OUTTMPL) == '-':
+                return True
             if info_dict.get('is_live'):
-                return False
-            merger = FFmpegMergerPP(self)
-            return merger.available and merger.can_merge()
-        if can_have_partial_formats():
-            req_format_list.append('bestvideo+bestaudio')
-        req_format_list.append('best')
+                return True
+            if not can_merge():
+                return True
+            return False
+
+        req_format_list = ['bestvideo+bestaudio', 'best']
+        if prefer_best():
+            req_format_list.reverse()
         return '/'.join(req_format_list)
 
     def build_format_selector(self, format_spec):
-- 
cgit v1.1


From 694b61545cc3fa37c31d5f62d26101fb2620a01d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 12 Oct 2017 00:41:20 +0700
Subject: [nexx] Add support for shortcuts and relax domain id extraction

---
 youtube_dl/extractor/nexx.py | 28 +++++++++++++++++++++-------
 1 file changed, 21 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/nexx.py b/youtube_dl/extractor/nexx.py
index d0235fd..071879b 100644
--- a/youtube_dl/extractor/nexx.py
+++ b/youtube_dl/extractor/nexx.py
@@ -18,7 +18,13 @@ from ..utils import (
 
 
 class NexxIE(InfoExtractor):
-    _VALID_URL = r'https?://api\.nexx(?:\.cloud|cdn\.com)/v3/(?P<domain_id>\d+)/videos/byid/(?P<id>\d+)'
+    _VALID_URL = r'''(?x)
+                        (?:
+                            https?://api\.nexx(?:\.cloud|cdn\.com)/v3/(?P<domain_id>\d+)/videos/byid/|
+                            nexx:(?P<domain_id_s>\d+):
+                        )
+                        (?P<id>\d+)
+                    '''
     _TESTS = [{
         # movie
         'url': 'https://api.nexx.cloud/v3/748/videos/byid/128907',
@@ -62,9 +68,19 @@ class NexxIE(InfoExtractor):
     }, {
         'url': 'https://api.nexxcdn.com/v3/748/videos/byid/128907',
         'only_matching': True,
+    }, {
+        'url': 'nexx:748:128907',
+        'only_matching': True,
     }]
 
     @staticmethod
+    def _extract_domain_id(webpage):
+        mobj = re.search(
+            r'<script\b[^>]+\bsrc=["\'](?:https?:)?//require\.nexx(?:\.cloud|cdn\.com)/(?P<id>\d+)',
+            webpage)
+        return mobj.group('id') if mobj else None
+
+    @staticmethod
     def _extract_urls(webpage):
         # Reference:
         # 1. https://nx-s.akamaized.net/files/201510/44.pdf
@@ -72,11 +88,8 @@ class NexxIE(InfoExtractor):
         entries = []
 
         # JavaScript Integration
-        mobj = re.search(
-            r'<script\b[^>]+\bsrc=["\']https?://require\.nexx(?:\.cloud|cdn\.com)/(?P<id>\d+)',
-            webpage)
-        if mobj:
-            domain_id = mobj.group('id')
+        domain_id = NexxIE._extract_domain_id(webpage)
+        if domain_id:
             for video_id in re.findall(
                     r'(?is)onPLAYReady.+?_play\.init\s*\(.+?\s*,\s*["\']?(\d+)',
                     webpage):
@@ -112,7 +125,8 @@ class NexxIE(InfoExtractor):
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
-        domain_id, video_id = mobj.group('domain_id', 'id')
+        domain_id = mobj.group('domain_id') or mobj.group('domain_id_s')
+        video_id = mobj.group('id')
 
         # Reverse engineered from JS code (see getDeviceID function)
         device_id = '%d:%d:%d%d' % (
-- 
cgit v1.1


From ff3f1a62f087332fa6409b5cbc39871d49e74f37 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 12 Oct 2017 00:44:13 +0700
Subject: [funk] Add extractor (closes #14464)

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/funk.py       | 43 ++++++++++++++++++++++++++++++++++++++
 youtube_dl/extractor/generic.py    | 16 --------------
 3 files changed, 44 insertions(+), 16 deletions(-)
 create mode 100644 youtube_dl/extractor/funk.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 5629c76..ecb33bc 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -381,6 +381,7 @@ from .freesound import FreesoundIE
 from .freespeech import FreespeechIE
 from .freshlive import FreshLiveIE
 from .funimation import FunimationIE
+from .funk import FunkIE
 from .funnyordie import FunnyOrDieIE
 from .fusion import FusionIE
 from .fxnetworks import FXNetworksIE
diff --git a/youtube_dl/extractor/funk.py b/youtube_dl/extractor/funk.py
new file mode 100644
index 0000000..ce5c67f
--- /dev/null
+++ b/youtube_dl/extractor/funk.py
@@ -0,0 +1,43 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from .nexx import NexxIE
+from ..utils import extract_attributes
+
+
+class FunkIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?funk\.net/(?:mix|channel)/(?:[^/]+/)*(?P<id>[^?/#]+)'
+    _TESTS = [{
+        'url': 'https://www.funk.net/mix/59d65d935f8b160001828b5b/0/59d517e741dca10001252574/',
+        'md5': '4d40974481fa3475f8bccfd20c5361f8',
+        'info_dict': {
+            'id': '716599',
+            'ext': 'mp4',
+            'title': 'Neue Rechte Welle',
+            'description': 'md5:a30a53f740ffb6bfd535314c2cc5fb69',
+            'timestamp': 1501337639,
+            'upload_date': '20170729',
+        },
+        'params': {
+            'format': 'bestvideo',
+            'skip_download': True,
+        },
+    }, {
+        'url': 'https://www.funk.net/channel/59d5149841dca100012511e3/0/59d52049999264000182e79d/',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        domain_id = NexxIE._extract_domain_id(webpage) or '741'
+        nexx_id = extract_attributes(self._search_regex(
+            r'(<div[^>]id=["\']mediaplayer-funk[^>]+>)',
+            webpage, 'media player'))['data-id']
+
+        return self.url_result(
+            'nexx:%s:%s' % (domain_id, nexx_id), ie=NexxIE.ie_key(),
+            video_id=nexx_id)
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 6dab4c7..39630b6 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1613,22 +1613,6 @@ class GenericIE(InfoExtractor):
             },
             'add_ie': ['BrightcoveLegacy'],
         },
-        # Nexx embed
-        {
-            'url': 'https://www.funk.net/serien/5940e15073f6120001657956/items/593efbb173f6120001657503',
-            'info_dict': {
-                'id': '247746',
-                'ext': 'mp4',
-                'title': "Yesterday's Jam (OV)",
-                'description': 'md5:09bc0984723fed34e2581624a84e05f0',
-                'timestamp': 1492594816,
-                'upload_date': '20170419',
-            },
-            'params': {
-                'format': 'bestvideo',
-                'skip_download': True,
-            },
-        },
         # Facebook <iframe> embed
         {
             'url': 'https://www.hostblogger.de/blog/archives/6181-Auto-jagt-Betonmischer.html',
-- 
cgit v1.1


From 7fee3377dced16cd16660aeb78d18d81508aa28b Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 11 Oct 2017 17:49:48 +0000
Subject: [steam] fix extraction(fixes #14067)

---
 youtube_dl/extractor/steam.py | 129 +++++++++++++++++++++++++-----------------
 1 file changed, 76 insertions(+), 53 deletions(-)

diff --git a/youtube_dl/extractor/steam.py b/youtube_dl/extractor/steam.py
index 1a831ef..e5ac586 100644
--- a/youtube_dl/extractor/steam.py
+++ b/youtube_dl/extractor/steam.py
@@ -4,8 +4,10 @@ import re
 
 from .common import InfoExtractor
 from ..utils import (
+    extract_attributes,
     ExtractorError,
-    unescapeHTML,
+    get_element_by_class,
+    js_to_json,
 )
 
 
@@ -25,35 +27,39 @@ class SteamIE(InfoExtractor):
         'url': 'http://store.steampowered.com/video/105600/',
         'playlist': [
             {
-                'md5': 'f870007cee7065d7c76b88f0a45ecc07',
+                'md5': '6a294ee0c4b1f47f5bb76a65e31e3592',
                 'info_dict': {
-                    'id': '81300',
-                    'ext': 'flv',
-                    'title': 'Terraria 1.1 Trailer',
+                    'id': '2040428',
+                    'ext': 'mp4',
+                    'title': 'Terraria 1.3 Trailer',
                     'playlist_index': 1,
                 }
             },
             {
-                'md5': '61aaf31a5c5c3041afb58fb83cbb5751',
+                'md5': '911672b20064ca3263fa89650ba5a7aa',
                 'info_dict': {
-                    'id': '80859',
-                    'ext': 'flv',
-                    'title': 'Terraria Trailer',
+                    'id': '2029566',
+                    'ext': 'mp4',
+                    'title': 'Terraria 1.2 Trailer',
                     'playlist_index': 2,
                 }
             }
         ],
+        'info_dict': {
+            'id': '105600',
+            'title': 'Terraria',
+        },
         'params': {
             'playlistend': 2,
         }
     }, {
         'url': 'http://steamcommunity.com/sharedfiles/filedetails/?id=242472205',
         'info_dict': {
-            'id': 'WB5DvDOOvAY',
+            'id': 'X8kpJBlzD2E',
             'ext': 'mp4',
-            'upload_date': '20140329',
-            'title': 'FRONTIERS - Final Greenlight Trailer',
-            'description': 'md5:dc96a773669d0ca1b36c13c1f30250d9',
+            'upload_date': '20140617',
+            'title': 'FRONTIERS - Trapping',
+            'description': 'md5:bf6f7f773def614054089e5769c12a6e',
             'uploader': 'AAD Productions',
             'uploader_id': 'AtomicAgeDogGames',
         }
@@ -76,48 +82,65 @@ class SteamIE(InfoExtractor):
             self.report_age_confirmation()
             webpage = self._download_webpage(videourl, playlist_id)
 
+        flash_vars = self._parse_json(self._search_regex(
+            r'(?s)rgMovieFlashvars\s*=\s*({.+?});', webpage,
+            'flash vars'), playlist_id, js_to_json)
+
+        playlist_title = None
+        entries = []
         if fileID:
-            playlist_title = self._html_search_regex(
-                r'<div class="workshopItemTitle">(.+)</div>', webpage, 'title')
-            mweb = re.finditer(r'''(?x)
-                'movie_(?P<videoID>[0-9]+)':\s*\{\s*
-                YOUTUBE_VIDEO_ID:\s*"(?P<youtube_id>[^"]+)",
-                ''', webpage)
-            videos = [{
-                '_type': 'url',
-                'url': vid.group('youtube_id'),
-                'ie_key': 'Youtube',
-            } for vid in mweb]
+            playlist_title = get_element_by_class('workshopItemTitle', webpage)
+            for movie in flash_vars.values():
+                if not movie:
+                    continue
+                youtube_id = movie.get('YOUTUBE_VIDEO_ID')
+                if not youtube_id:
+                    continue
+                entries.append({
+                    '_type': 'url',
+                    'url': youtube_id,
+                    'ie_key': 'Youtube',
+                })
         else:
-            playlist_title = self._html_search_regex(
-                r'<h2 class="pageheader">(.*?)</h2>', webpage, 'game title')
-
-            mweb = re.finditer(r'''(?x)
-                'movie_(?P<videoID>[0-9]+)':\s*\{\s*
-                FILENAME:\s*"(?P<videoURL>[\w:/\.\?=]+)"
-                (,\s*MOVIE_NAME:\s*\"(?P<videoName>[\w:/\.\?=\+-]+)\")?\s*\},
-                ''', webpage)
-            titles = re.finditer(
-                r'<span class="title">(?P<videoName>.+?)</span>', webpage)
-            thumbs = re.finditer(
-                r'<img class="movie_thumb" src="(?P<thumbnail>.+?)">', webpage)
-            videos = []
-
-            for vid, vtitle, thumb in zip(mweb, titles, thumbs):
-                video_id = vid.group('videoID')
-                title = vtitle.group('videoName')
-                video_url = vid.group('videoURL')
-                video_thumb = thumb.group('thumbnail')
-                if not video_url:
-                    raise ExtractorError('Cannot find video url for %s' % video_id)
-                videos.append({
+            playlist_title = get_element_by_class('apphub_AppName', webpage)
+            for movie_id, movie in flash_vars.items():
+                if not movie:
+                    continue
+                video_id = self._search_regex(r'movie_(\d+)', movie_id, 'video id', fatal=False)
+                title = movie.get('MOVIE_NAME')
+                if not title or not video_id:
+                    continue
+                entry = {
                     'id': video_id,
-                    'url': video_url,
-                    'ext': 'flv',
-                    'title': unescapeHTML(title),
-                    'thumbnail': video_thumb
-                })
-        if not videos:
+                    'title': title.replace('+', ' '),
+                }
+                formats = []
+                flv_url = movie.get('FILENAME')
+                if flv_url:
+                    formats.append({
+                        'format_id': 'flv',
+                        'url': flv_url,
+                    })
+                highlight_element = self._search_regex(
+                    r'(<div[^>]+id="highlight_movie_%s"[^>]+>)' % video_id,
+                    webpage, 'highlight element', fatal=False)
+                if highlight_element:
+                    highlight_attribs = extract_attributes(highlight_element)
+                    if highlight_attribs:
+                        entry['thumbnail'] = highlight_attribs.get('data-poster')
+                        for quality in ('', '-hd'):
+                            for ext in ('webm', 'mp4'):
+                                video_url = highlight_attribs.get('data-%s%s-source' % (ext, quality))
+                                if video_url:
+                                    formats.append({
+                                        'format_id': ext + quality,
+                                        'url': video_url,
+                                    })
+                if not formats:
+                    continue
+                entry['formats'] = formats
+                entries.append(entry)
+        if not entries:
             raise ExtractorError('Could not find any videos')
 
-        return self.playlist_result(videos, playlist_id, playlist_title)
+        return self.playlist_result(entries, playlist_id, playlist_title)
-- 
cgit v1.1


From 2f0eb0a68a0711e7ac2c31b6177273433fb42b94 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 12 Oct 2017 01:05:14 +0700
Subject: [ChangeLog] Actualize

---
 ChangeLog | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index c541c4f..7477443 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,8 +1,27 @@
 version <unreleased>
 
+Core
+* [YoutubeDL] Improve _default_format_spec (#14461)
+
 Extractors
+* [steam] Fix extraction (#14067)
++ [funk] Add support for funk.net (#14464)
++ [nexx] Add support for shortcuts and relax domain id extraction
++ [voxmedia] Add support for recode.net (#14173)
++ [once] Add support for vmap URLs
++ [generic] Add support for channel9 embeds (#14469)
+* [tva] Fix extraction (#14328)
++ [tubitv] Add support for new URL format (#14460)
+- [afreecatv:global] Remove extractor
 - [youtube:shared] Removed extractor (#14420)
++ [slideslive] Add support for slideslive.com (#2680)
 + [facebook] Support thumbnails (#14416)
+* [vvvvid] Fix episode number extraction (#14456)
+* [hrti:playlist] Relax URL regular expression
+* [wdr] Relax media link regular expression (#14447)
+* [hrti] Relax URL regular expression (#14443)
+* [fox] Delegate extraction to uplynk:preplay (#14147)
++ [youtube] Add support for hooktube.com (#14437)
 
 
 version 2017.10.07
-- 
cgit v1.1


From 76581082f6fd32787c7fe5c3e5be8442ba9f50a3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 12 Oct 2017 01:06:28 +0700
Subject: release 2017.10.12

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 docs/supportedsites.md    | 5 +++--
 youtube_dl/version.py     | 2 +-
 4 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 9fc425c..5ef0873 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.10.07*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.10.07**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.10.12*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.10.12**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through the [README](https://github.com/rg3/youtube-dl/blob/master/README.md), **most notably** the [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ Add the `-v` flag to **your command line** you run youtube-dl with (`youtube-dl
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.10.07
+[debug] youtube-dl version 2017.10.12
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index 7477443..9b57adc 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.10.12
 
 Core
 * [YoutubeDL] Improve _default_format_spec (#14461)
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index be9df7e..7071450 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -36,7 +36,6 @@
  - **AdultSwim**
  - **aenetworks**: A+E Networks: A&E, Lifetime, History.com, FYI Network
  - **afreecatv**: afreecatv.com
- - **afreecatv:global**: afreecatv.com
  - **AirMozilla**
  - **AliExpressLive**
  - **AlJazeera**
@@ -296,6 +295,7 @@
  - **freespeech.org**
  - **FreshLive**
  - **Funimation**
+ - **Funk**
  - **FunnyOrDie**
  - **Fusion**
  - **Fux**
@@ -738,6 +738,7 @@
  - **skynewsarabia:video**
  - **SkySports**
  - **Slideshare**
+ - **SlidesLive**
  - **Slutload**
  - **smotri**: Smotri.com
  - **smotri:broadcast**: Smotri.com broadcasts
@@ -969,6 +970,7 @@
  - **VoiceRepublic**
  - **Voot**
  - **VoxMedia**
+ - **VoxMediaVolume**
  - **Vporn**
  - **vpro**: npo.nl, ntr.nl, omroepwnl.nl, zapp.nl and npo3.nl
  - **Vrak**
@@ -1044,7 +1046,6 @@
  - **youtube:search**: YouTube.com searches
  - **youtube:search:date**: YouTube.com searches, newest videos first
  - **youtube:search_url**: YouTube.com search URLs
- - **youtube:shared**
  - **youtube:show**: YouTube.com (multi-season) shows
  - **youtube:subscriptions**: YouTube.com subscriptions feed, "ytsubs" keyword (requires authentication)
  - **youtube:user**: YouTube.com user videos (URL or "ytuser" keyword)
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 705fcad..36e6e99 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.10.07'
+__version__ = '2017.10.12'
-- 
cgit v1.1


From 344d1a6794270e67b8bb21b78995a1b116286a07 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 11 Oct 2017 20:51:57 +0000
Subject: [vh1] fix extraction(fixes #9613)

---
 youtube_dl/extractor/vh1.py | 137 ++++++++------------------------------------
 1 file changed, 23 insertions(+), 114 deletions(-)

diff --git a/youtube_dl/extractor/vh1.py b/youtube_dl/extractor/vh1.py
index 570fa45..2341e5f 100644
--- a/youtube_dl/extractor/vh1.py
+++ b/youtube_dl/extractor/vh1.py
@@ -1,131 +1,40 @@
 from __future__ import unicode_literals
 
-from .mtv import MTVIE
+from .mtv import MTVServicesInfoExtractor
 
-import re
-from ..utils import fix_xml_ampersands
 
-
-class VH1IE(MTVIE):
+class VH1IE(MTVServicesInfoExtractor):
     IE_NAME = 'vh1.com'
-    _FEED_URL = 'http://www.vh1.com/player/embed/AS3/fullepisode/rss/'
+    _FEED_URL = 'http://www.vh1.com/feeds/mrss/'
     _TESTS = [{
-        'url': 'http://www.vh1.com/video/metal-evolution/full-episodes/progressive-metal/1678612/playlist.jhtml',
-        'playlist': [
-            {
-                'md5': '7827a7505f59633983165bbd2c119b52',
-                'info_dict': {
-                    'id': '731565',
-                    'ext': 'mp4',
-                    'title': 'Metal Evolution: Ep. 11 Act 1',
-                    'description': 'Many rock academics have proclaimed that the truly progressive musicianship of the last 20 years has been found right here in the world of heavy metal, rather than obvious locales such as jazz, fusion or progressive rock. It stands to reason then, that much of this jaw-dropping virtuosity occurs within what\'s known as progressive metal, a genre that takes root with the likes of Rush in the \'70s, Queensryche and Fates Warning in the \'80s, and Dream Theater in the \'90s. Since then, the genre has exploded with creativity, spawning mind-bending, genre-defying acts such as Tool, Mastodon, Coheed And Cambria, Porcupine Tree, Meshuggah, A Perfect Circle and Opeth. Episode 12 looks at the extreme musicianship of these bands, as well as their often extreme literary prowess and conceptual strength, the end result being a rich level of respect and attention such challenging acts have brought upon the world of heavy metal, from a critical community usually dismissive of the form.'
-                }
-            },
-            {
-                'md5': '34fb4b7321c546b54deda2102a61821f',
-                'info_dict': {
-                    'id': '731567',
-                    'ext': 'mp4',
-                    'title': 'Metal Evolution: Ep. 11 Act 2',
-                    'description': 'Many rock academics have proclaimed that the truly progressive musicianship of the last 20 years has been found right here in the world of heavy metal, rather than obvious locales such as jazz, fusion or progressive rock. It stands to reason then, that much of this jaw-dropping virtuosity occurs within what\'s known as progressive metal, a genre that takes root with the likes of Rush in the \'70s, Queensryche and Fates Warning in the \'80s, and Dream Theater in the \'90s. Since then, the genre has exploded with creativity, spawning mind-bending, genre-defying acts such as Tool, Mastodon, Coheed And Cambria, Porcupine Tree, Meshuggah, A Perfect Circle and Opeth. Episode 11 looks at the extreme musicianship of these bands, as well as their often extreme literary prowess and conceptual strength, the end result being a rich level of respect and attention such challenging acts have brought upon the world of heavy metal, from a critical community usually dismissive of the form.'
-                }
-            },
-            {
-                'md5': '813f38dba4c1b8647196135ebbf7e048',
-                'info_dict': {
-                    'id': '731568',
-                    'ext': 'mp4',
-                    'title': 'Metal Evolution: Ep. 11 Act 3',
-                    'description': 'Many rock academics have proclaimed that the truly progressive musicianship of the last 20 years has been found right here in the world of heavy metal, rather than obvious locales such as jazz, fusion or progressive rock. It stands to reason then, that much of this jaw-dropping virtuosity occurs within what\'s known as progressive metal, a genre that takes root with the likes of Rush in the \'70s, Queensryche and Fates Warning in the \'80s, and Dream Theater in the \'90s. Since then, the genre has exploded with creativity, spawning mind-bending, genre-defying acts such as Tool, Mastodon, Coheed And Cambria, Porcupine Tree, Meshuggah, A Perfect Circle and Opeth. Episode 11 looks at the extreme musicianship of these bands, as well as their often extreme literary prowess and conceptual strength, the end result being a rich level of respect and attention such challenging acts have brought upon the world of heavy metal, from a critical community usually dismissive of the form.'
-                }
-            },
-            {
-                'md5': '51adb72439dfaed11c799115d76e497f',
-                'info_dict': {
-                    'id': '731569',
-                    'ext': 'mp4',
-                    'title': 'Metal Evolution: Ep. 11 Act 4',
-                    'description': 'Many rock academics have proclaimed that the truly progressive musicianship of the last 20 years has been found right here in the world of heavy metal, rather than obvious locales such as jazz, fusion or progressive rock. It stands to reason then, that much of this jaw-dropping virtuosity occurs within what\'s known as progressive metal, a genre that takes root with the likes of Rush in the \'70s, Queensryche and Fates Warning in the \'80s, and Dream Theater in the \'90s. Since then, the genre has exploded with creativity, spawning mind-bending, genre-defying acts such as Tool, Mastodon, Coheed And Cambria, Porcupine Tree, Meshuggah, A Perfect Circle and Opeth. Episode 11 looks at the extreme musicianship of these bands, as well as their often extreme literary prowess and conceptual strength, the end result being a rich level of respect and attention such challenging acts have brought upon the world of heavy metal, from a critical community usually dismissive of the form.'
-                }
-            },
-            {
-                'md5': '93d554aaf79320703b73a95288c76a6e',
-                'info_dict': {
-                    'id': '731570',
-                    'ext': 'mp4',
-                    'title': 'Metal Evolution: Ep. 11 Act 5',
-                    'description': 'Many rock academics have proclaimed that the truly progressive musicianship of the last 20 years has been found right here in the world of heavy metal, rather than obvious locales such as jazz, fusion or progressive rock. It stands to reason then, that much of this jaw-dropping virtuosity occurs within what\'s known as progressive metal, a genre that takes root with the likes of Rush in the \'70s, Queensryche and Fates Warning in the \'80s, and Dream Theater in the \'90s. Since then, the genre has exploded with creativity, spawning mind-bending, genre-defying acts such as Tool, Mastodon, Coheed And Cambria, Porcupine Tree, Meshuggah, A Perfect Circle and Opeth. Episode 11 looks at the extreme musicianship of these bands, as well as their often extreme literary prowess and conceptual strength, the end result being a rich level of respect and attention such challenging acts have brought upon the world of heavy metal, from a critical community usually dismissive of the form.'
-                }
-            }
-        ],
-        'skip': 'Blocked outside the US',
-    }, {
-        # Clip
-        'url': 'http://www.vh1.com/video/misc/706675/metal-evolution-episode-1-pre-metal-show-clip.jhtml#id=1674118',
-        'md5': '7d67cf6d9cdc6b4f3d3ac97a55403844',
+        'url': 'http://www.vh1.com/episodes/0umwpq/hip-hop-squares-kent-jones-vs-nick-young-season-1-ep-120',
         'info_dict': {
-            'id': '706675',
-            'ext': 'mp4',
-            'title': 'Metal Evolution: Episode 1 Pre-Metal Show Clip',
-            'description': 'The greatest documentary ever made about Heavy Metal begins as our host Sam Dunn travels the globe to seek out the origins and influences that helped create Heavy Metal. Sam speaks to legends like Kirk Hammett, Alice Cooper, Slash, Bill Ward, Geezer Butler, Tom Morello, Ace Frehley, Lemmy Kilmister, Dave Davies, and many many more. This episode is the prologue for the 11 hour series, and Sam goes back to the very beginning to reveal how Heavy Metal was created.'
+            'title': 'Kent Jones vs. Nick Young',
+            'description': 'Come to Play. Stay to Party. With Mike Epps, TIP, O’Shea Jackson Jr., T-Pain, Tisha Campbell-Martin and more.',
         },
-        'skip': 'Blocked outside the US',
+        'playlist_mincount': 4,
     }, {
-        # Short link
-        'url': 'http://www.vh1.com/video/play.jhtml?id=1678353',
-        'md5': '853192b87ad978732b67dd8e549b266a',
+        # Clip
+        'url': 'http://www.vh1.com/video-clips/t74mif/scared-famous-scared-famous-extended-preview',
         'info_dict': {
-            'id': '730355',
+            'id': '0a50c2d2-a86b-4141-9565-911c7e2d0b92',
             'ext': 'mp4',
-            'title': 'Metal Evolution: Episode 11 Progressive Metal Sneak',
-            'description': 'In Metal Evolution\'s finale sneak, Sam sits with Michael Giles of King Crimson and gets feedback from Metallica guitarist Kirk Hammett on why the group was influential.'
+            'title': 'Scared Famous|October 9, 2017|1|NO-EPISODE#|Scared Famous + Extended Preview',
+            'description': 'md5:eff5551a274c473a29463de40f7b09da',
+            'upload_date': '20171009',
+            'timestamp': 1507574700,
         },
-        'skip': 'Blocked outside the US',
-    }, {
-        'url': 'http://www.vh1.com/video/macklemore-ryan-lewis/900535/cant-hold-us-ft-ray-dalton.jhtml',
-        'md5': 'b1bcb5b4380c9d7f544065589432dee7',
-        'info_dict': {
-            'id': '900535',
-            'ext': 'mp4',
-            'title': 'Macklemore & Ryan Lewis - "Can\'t Hold Us ft. Ray Dalton"',
-            'description': 'The Heist'
+        'params': {
+            # m3u8 download
+            'skip_download': True,
         },
-        'skip': 'Blocked outside the US',
     }]
 
-    _VALID_URL = r'''(?x)
-        https?://www\.vh1\.com/video/
-        (?:
-            .+?/full-episodes/.+?/(?P<playlist_id>[^/]+)/playlist\.jhtml
-        |
-            (?:
-            play.jhtml\?id=|
-            misc/.+?/.+?\.jhtml\#id=
-            )
-            (?P<video_id>[0-9]+)$
-        |
-            [^/]+/(?P<music_id>[0-9]+)/[^/]+?
-        )
-    '''
+    _VALID_URL = r'https?://(?:www\.)?vh1\.com/(?:video-clips|episodes)/(?P<id>[^/?#.]+)'
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        if mobj.group('music_id'):
-            id_field = 'vid'
-            video_id = mobj.group('music_id')
-        else:
-            video_id = mobj.group('playlist_id') or mobj.group('video_id')
-            id_field = 'id'
-        doc_url = '%s?%s=%s' % (self._FEED_URL, id_field, video_id)
-
-        idoc = self._download_xml(
-            doc_url, video_id,
-            'Downloading info', transform_source=fix_xml_ampersands)
-
-        entries = []
-        for item in idoc.findall('.//item'):
-            info = self._get_video_info(item)
-            if info:
-                entries.append(info)
-
-        return self.playlist_result(entries, playlist_id=video_id)
+        playlist_id = self._match_id(url)
+        webpage = self._download_webpage(url, playlist_id)
+        mgid = self._extract_triforce_mgid(webpage)
+        videos_info = self._get_videos_info(mgid)
+        return videos_info
-- 
cgit v1.1


From 1baba7f4a85652c3844661b48378f188a9a5631f Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 12 Oct 2017 12:02:26 +0800
Subject: [vh1] Adding coding cookie

---
 youtube_dl/extractor/vh1.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/vh1.py b/youtube_dl/extractor/vh1.py
index 2341e5f..dff94a2 100644
--- a/youtube_dl/extractor/vh1.py
+++ b/youtube_dl/extractor/vh1.py
@@ -1,3 +1,4 @@
+# coding: utf-8
 from __future__ import unicode_literals
 
 from .mtv import MTVServicesInfoExtractor
-- 
cgit v1.1


From c0bddd6d6597642c7c4733d0fbf52fbe4137d5a0 Mon Sep 17 00:00:00 2001
From: nyuszika7h <nyuszika7h@gmail.com>
Date: Wed, 11 Oct 2017 22:24:17 +0200
Subject: [kaltura] Ignore Widevine encrypted video (.wvm)

There is currently no public method to decrypt this, and there may be
other streams available that can be downloaded.

Example URL, has `.wvm` and `.mp4` formats:
https://www.voot.com/shows/bigg-boss-s11/11/538936/bigg-boss-extra-dose-arshi-s-quirky-demand/541700
---
 youtube_dl/extractor/kaltura.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/kaltura.py b/youtube_dl/extractor/kaltura.py
index 138d484..bdac2df 100644
--- a/youtube_dl/extractor/kaltura.py
+++ b/youtube_dl/extractor/kaltura.py
@@ -287,6 +287,9 @@ class KalturaIE(InfoExtractor):
             # skip for now.
             if f.get('fileExt') == 'chun':
                 continue
+            # DRM-protected video, cannot be decrypted
+            if f.get('fileExt') == 'wvm':
+                continue
             if not f.get('fileExt'):
                 # QT indicates QuickTime; some videos have broken fileExt
                 if f.get('containerFormat') == 'qt':
-- 
cgit v1.1


From 645ed3e7c90881f4fd66a40e783f4b3deb270262 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 12 Oct 2017 12:12:37 +0800
Subject: [ChangeLog] Update after #14471

[skip ci]
---
 ChangeLog | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 9b57adc..db2c2bf 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,8 @@
+version <unreleased>
+
+Extractors
+* [kaltura] Ignore Widevine encrypted video (.wvm) (#14471)
+
 version 2017.10.12
 
 Core
-- 
cgit v1.1


From 414e709405fe8c653fb58982626284134a86167e Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 12 Oct 2017 09:20:07 +0000
Subject: [shahid] fix extraction(fixes #14448)

---
 youtube_dl/extractor/shahid.py | 61 ++++++++++++++++++++++--------------------
 1 file changed, 32 insertions(+), 29 deletions(-)

diff --git a/youtube_dl/extractor/shahid.py b/youtube_dl/extractor/shahid.py
index 62d41e8..374f7fa 100644
--- a/youtube_dl/extractor/shahid.py
+++ b/youtube_dl/extractor/shahid.py
@@ -18,46 +18,32 @@ from ..utils import (
 
 class ShahidIE(InfoExtractor):
     _NETRC_MACHINE = 'shahid'
-    _VALID_URL = r'https?://shahid\.mbc\.net/ar/(?P<type>episode|movie)/(?P<id>\d+)'
+    _VALID_URL = r'https?://shahid\.mbc\.net/ar/(?:serie|show|movie)s/[^/]+/(?P<type>episode|clip|movie)-(?P<id>\d+)'
     _TESTS = [{
-        'url': 'https://shahid.mbc.net/ar/episode/90574/%D8%A7%D9%84%D9%85%D9%84%D9%83-%D8%B9%D8%A8%D8%AF%D8%A7%D9%84%D9%84%D9%87-%D8%A7%D9%84%D8%A5%D9%86%D8%B3%D8%A7%D9%86-%D8%A7%D9%84%D9%85%D9%88%D8%B3%D9%85-1-%D9%83%D9%84%D9%8A%D8%A8-3.html',
+        'url': 'https://shahid.mbc.net/ar/shows/%D9%85%D8%AC%D9%84%D8%B3-%D8%A7%D9%84%D8%B4%D8%A8%D8%A7%D8%A8-%D8%A7%D9%84%D9%85%D9%88%D8%B3%D9%85-1-%D9%83%D9%84%D9%8A%D8%A8-1/clip-275286',
         'info_dict': {
-            'id': '90574',
+            'id': '275286',
             'ext': 'mp4',
-            'title': 'الملك عبدالله الإنسان الموسم 1 كليب 3',
-            'description': 'الفيلم الوثائقي - الملك عبد الله الإنسان',
-            'duration': 2972,
-            'timestamp': 1422057420,
-            'upload_date': '20150123',
+            'title': 'مجلس الشباب الموسم 1 كليب 1',
+            'timestamp': 1506988800,
+            'upload_date': '20171003',
         },
         'params': {
             # m3u8 download
             'skip_download': True,
         }
     }, {
-        'url': 'https://shahid.mbc.net/ar/movie/151746/%D8%A7%D9%84%D9%82%D9%86%D8%A7%D8%B5%D8%A9.html',
+        'url': 'https://shahid.mbc.net/ar/movies/%D8%A7%D9%84%D9%82%D9%86%D8%A7%D8%B5%D8%A9/movie-151746',
         'only_matching': True
     }, {
         # shahid plus subscriber only
-        'url': 'https://shahid.mbc.net/ar/episode/90511/%D9%85%D8%B1%D8%A7%D9%8A%D8%A7-2011-%D8%A7%D9%84%D9%85%D9%88%D8%B3%D9%85-1-%D8%A7%D9%84%D8%AD%D9%84%D9%82%D8%A9-1.html',
+        'url': 'https://shahid.mbc.net/ar/series/%D9%85%D8%B1%D8%A7%D9%8A%D8%A7-2011-%D8%A7%D9%84%D9%85%D9%88%D8%B3%D9%85-1-%D8%A7%D9%84%D8%AD%D9%84%D9%82%D8%A9-1/episode-90511',
         'only_matching': True
     }]
 
-    def _real_initialize(self):
-        email, password = self._get_login_info()
-        if email is None:
-            return
-
+    def _api2_request(self, *args, **kwargs):
         try:
-            user_data = self._download_json(
-                'https://shahid.mbc.net/wd/service/users/login',
-                None, 'Logging in', data=json.dumps({
-                    'email': email,
-                    'password': password,
-                    'basic': 'false',
-                }).encode('utf-8'), headers={
-                    'Content-Type': 'application/json; charset=UTF-8',
-                })['user']
+            return self._download_json(*args, **kwargs)
         except ExtractorError as e:
             if isinstance(e.cause, compat_HTTPError):
                 fail_data = self._parse_json(
@@ -69,6 +55,21 @@ class ShahidIE(InfoExtractor):
                         raise ExtractorError(faults_message, expected=True)
             raise
 
+    def _real_initialize(self):
+        email, password = self._get_login_info()
+        if email is None:
+            return
+
+        user_data = self._api2_request(
+            'https://shahid.mbc.net/wd/service/users/login',
+            None, 'Logging in', data=json.dumps({
+                'email': email,
+                'password': password,
+                'basic': 'false',
+            }).encode('utf-8'), headers={
+                'Content-Type': 'application/json; charset=UTF-8',
+            })['user']
+
         self._download_webpage(
             'https://shahid.mbc.net/populateContext',
             None, 'Populate Context', data=urlencode_postdata({
@@ -93,15 +94,17 @@ class ShahidIE(InfoExtractor):
 
     def _real_extract(self, url):
         page_type, video_id = re.match(self._VALID_URL, url).groups()
+        if page_type == 'clip':
+            page_type = 'episode'
 
-        player = self._get_api_data(self._download_json(
-            'https://shahid.mbc.net/arContent/getPlayerContent-param-.id-%s.type-player.html' % video_id,
-            video_id, 'Downloading player JSON'))
+        playout = self._api2_request(
+            'https://api2.shahid.net/proxy/v2/playout/url/' + video_id,
+            video_id, 'Downloading player JSON')['playout']
 
-        if player.get('drm'):
+        if playout.get('drm'):
             raise ExtractorError('This video is DRM protected.', expected=True)
 
-        formats = self._extract_m3u8_formats(player['url'], video_id, 'mp4')
+        formats = self._extract_m3u8_formats(playout['url'], video_id, 'mp4')
         self._sort_formats(formats)
 
         video = self._get_api_data(self._download_json(
-- 
cgit v1.1


From 81ce479f4dcc17b83672f5ca3ffcb2a2d1618c96 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 12 Oct 2017 16:04:41 +0000
Subject: [udn] fix extraction

---
 youtube_dl/extractor/udn.py | 32 +++++++++++++++++++-------------
 1 file changed, 19 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/udn.py b/youtube_dl/extractor/udn.py
index daf45d0..2c8e5c7 100644
--- a/youtube_dl/extractor/udn.py
+++ b/youtube_dl/extractor/udn.py
@@ -1,7 +1,6 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import json
 import re
 
 from .common import InfoExtractor
@@ -29,6 +28,7 @@ class UDNEmbedIE(InfoExtractor):
             # m3u8 download
             'skip_download': True,
         },
+        'expected_warnings': ['Failed to parse JSON Expecting value'],
     }, {
         'url': 'https://video.udn.com/embed/news/300040',
         'only_matching': True,
@@ -43,10 +43,21 @@ class UDNEmbedIE(InfoExtractor):
 
         page = self._download_webpage(url, video_id)
 
-        options = json.loads(js_to_json(self._html_search_regex(
-            r'var\s+options\s*=\s*([^;]+);', page, 'video urls dictionary')))
-
-        video_urls = options['video']
+        options_str = self._html_search_regex(
+            r'var\s+options\s*=\s*([^;]+);', page, 'options')
+        trans_options_str = js_to_json(options_str)
+        options = self._parse_json(trans_options_str, 'options', fatal=False) or {}
+        if options:
+            video_urls = options['video']
+            title = options['title']
+            poster = options.get('poster')
+        else:
+            video_urls = self._parse_json(self._html_search_regex(
+                r'"video"\s*:\s*({.+?})\s*,', trans_options_str, 'video urls'), 'video urls')
+            title = self._html_search_regex(
+                r"title\s*:\s*'(.+?)'\s*,", options_str, 'title')
+            poster = self._html_search_regex(
+                r"poster\s*:\s*'(.+?)'\s*,", options_str, 'poster', default=None)
 
         if video_urls.get('youtube'):
             return self.url_result(video_urls.get('youtube'), 'Youtube')
@@ -68,7 +79,7 @@ class UDNEmbedIE(InfoExtractor):
                 formats.extend(self._extract_f4m_formats(
                     video_url, video_id, f4m_id='hds'))
             else:
-                mobj = re.search(r'_(?P<height>\d+)p_(?P<tbr>\d+).mp4', video_url)
+                mobj = re.search(r'_(?P<height>\d+)p_(?P<tbr>\d+)\.mp4', video_url)
                 a_format = {
                     'url': video_url,
                     # video_type may be 'mp4', which confuses YoutubeDL
@@ -83,14 +94,9 @@ class UDNEmbedIE(InfoExtractor):
 
         self._sort_formats(formats)
 
-        thumbnails = [{
-            'url': img_url,
-            'id': img_type,
-        } for img_type, img_url in options.get('gallery', [{}])[0].items() if img_url]
-
         return {
             'id': video_id,
             'formats': formats,
-            'title': options['title'],
-            'thumbnails': thumbnails,
+            'title': title,
+            'thumbnail': poster,
         }
-- 
cgit v1.1


From b0def2c297293dfee3ec48c73bb92116a0ee0154 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 12 Oct 2017 16:05:25 +0000
Subject: [onionstudios] add support for bulbs-video embeds

---
 youtube_dl/extractor/onionstudios.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/onionstudios.py b/youtube_dl/extractor/onionstudios.py
index 1d336cf..c6e3d56 100644
--- a/youtube_dl/extractor/onionstudios.py
+++ b/youtube_dl/extractor/onionstudios.py
@@ -13,11 +13,11 @@ from ..utils import (
 
 
 class OnionStudiosIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?onionstudios\.com/(?:videos/[^/]+-|embed\?.*\bid=)(?P<id>\d+)(?!-)'
+    _VALID_URL = r'https?://(?:www\.)?onionstudios\.com/(?:video(?:s/[^/]+-|/)|embed\?.*\bid=)(?P<id>\d+)(?!-)'
 
     _TESTS = [{
         'url': 'http://www.onionstudios.com/videos/hannibal-charges-forward-stops-for-a-cocktail-2937',
-        'md5': 'e49f947c105b8a78a675a0ee1bddedfe',
+        'md5': '719d1f8c32094b8c33902c17bcae5e34',
         'info_dict': {
             'id': '2937',
             'ext': 'mp4',
@@ -29,12 +29,15 @@ class OnionStudiosIE(InfoExtractor):
     }, {
         'url': 'http://www.onionstudios.com/embed?id=2855&autoplay=true',
         'only_matching': True,
+    }, {
+        'url': 'http://www.onionstudios.com/video/6139.json',
+        'only_matching': True,
     }]
 
     @staticmethod
     def _extract_url(webpage):
         mobj = re.search(
-            r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//(?:www\.)?onionstudios\.com/embed.+?)\1', webpage)
+            r'(?s)<(?:iframe|bulbs-video)[^>]+?src=(["\'])(?P<url>(?:https?:)?//(?:www\.)?onionstudios\.com/(?:embed.+?|video/\d+\.json))\1', webpage)
         if mobj:
             return mobj.group('url')
 
-- 
cgit v1.1


From 7a64c33aee81e3a495c162af61bcd030a514f109 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 12 Oct 2017 16:09:06 +0000
Subject: [rtlnl] add support for another type of embeds

---
 youtube_dl/extractor/rtlnl.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/rtlnl.py b/youtube_dl/extractor/rtlnl.py
index 3e22998..bba25a2 100644
--- a/youtube_dl/extractor/rtlnl.py
+++ b/youtube_dl/extractor/rtlnl.py
@@ -12,10 +12,10 @@ class RtlNlIE(InfoExtractor):
     IE_NAME = 'rtl.nl'
     IE_DESC = 'rtl.nl and rtlxl.nl'
     _VALID_URL = r'''(?x)
-        https?://(?:www\.)?
+        https?://(?:(?:www|static)\.)?
         (?:
             rtlxl\.nl/[^\#]*\#!/[^/]+/|
-            rtl\.nl/(?:system/videoplayer/(?:[^/]+/)+(?:video_)?embed\.html\b.+?\buuid=|video/)
+            rtl\.nl/(?:(?:system/videoplayer/(?:[^/]+/)+(?:video_)?embed\.html|embed)\b.+?\buuid=|video/)
         )
         (?P<id>[0-9a-f-]+)'''
 
@@ -73,6 +73,9 @@ class RtlNlIE(InfoExtractor):
     }, {
         'url': 'https://www.rtl.nl/video/c603c9c2-601d-4b5e-8175-64f1e942dc7d/',
         'only_matching': True,
+    }, {
+        'url': 'https://static.rtl.nl/embed/?uuid=1a2970fc-5c0b-43ff-9fdc-927e39e6d1bc&autoplay=false&publicatiepunt=rtlnieuwsnl',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
-- 
cgit v1.1


From 50d808f5c942b454b3febbf1ef5d78bd204ce469 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 12 Oct 2017 16:12:47 +0000
Subject: [common] add support for jwplayer youtube embeds

---
 youtube_dl/extractor/common.py | 18 +++++++++++++-----
 1 file changed, 13 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index a878550..a692406 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -2322,7 +2322,6 @@ class InfoExtractor(object):
             formats = self._parse_jwplayer_formats(
                 video_data['sources'], video_id=this_video_id, m3u8_id=m3u8_id,
                 mpd_id=mpd_id, rtmp_params=rtmp_params, base_url=base_url)
-            self._sort_formats(formats)
 
             subtitles = {}
             tracks = video_data.get('tracks')
@@ -2339,16 +2338,25 @@ class InfoExtractor(object):
                         'url': self._proto_relative_url(track_url)
                     })
 
-            entries.append({
+            entry = {
                 'id': this_video_id,
-                'title': video_data['title'] if require_title else video_data.get('title'),
+                'title': unescapeHTML(video_data['title'] if require_title else video_data.get('title')),
                 'description': video_data.get('description'),
                 'thumbnail': self._proto_relative_url(video_data.get('image')),
                 'timestamp': int_or_none(video_data.get('pubdate')),
                 'duration': float_or_none(jwplayer_data.get('duration') or video_data.get('duration')),
                 'subtitles': subtitles,
-                'formats': formats,
-            })
+            }
+            # https://github.com/jwplayer/jwplayer/blob/master/src/js/utils/validator.js#L32
+            if len(formats) == 1 and re.search(r'^(?:http|//).*(?:youtube\.com|youtu\.be)/.+', formats[0]['url']):
+                entry.update({
+                    '_type': 'url_transparent',
+                    'url': formats[0]['url'],
+                })
+            else:
+                self._sort_formats(formats)
+                entry['formats'] = formats
+            entries.append(entry)
         if len(entries) == 1:
             return entries[0]
         else:
-- 
cgit v1.1


From 2637fadc38566abdc84c62a05d1a6573e60799bf Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 12 Oct 2017 16:14:43 +0000
Subject: [generic] fix some of the tests

---
 youtube_dl/extractor/generic.py | 36 +++++++++++++++++++++++++++---------
 1 file changed, 27 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 39630b6..2a9c3e2 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1091,7 +1091,7 @@ class GenericIE(InfoExtractor):
                 'ext': 'mp4',
                 'upload_date': '20150212',
                 'uploader': 'The National Archives UK',
-                'description': 'md5:a236581cd2449dd2df4f93412f3f01c6',
+                'description': 'md5:8078af856dca76edc42910b61273dbbf',
                 'uploader_id': 'NationalArchives08',
                 'title': 'Webinar: Using Discovery, The National Archives’ online catalogue',
             },
@@ -1107,7 +1107,8 @@ class GenericIE(InfoExtractor):
             },
             'params': {
                 'skip_download': True,
-            }
+            },
+            'skip': 'does not contain a video anymore',
         },
         # Complex jwplayer
         {
@@ -1116,6 +1117,7 @@ class GenericIE(InfoExtractor):
                 'id': 'videos',
                 'ext': 'mp4',
                 'title': 'king machine trailer 1',
+                'description': 'Browse King Machine videos & audio for sweet media. Your eyes will thank you.',
                 'thumbnail': r're:^https?://.*\.jpg$',
             },
         },
@@ -1168,7 +1170,7 @@ class GenericIE(InfoExtractor):
             'playlist_mincount': 5,
             'info_dict': {
                 'id': 'aanslagen-kopenhagen',
-                'title': 'Aanslagen Kopenhagen | RTL Nieuws',
+                'title': 'Aanslagen Kopenhagen',
             }
         },
         # Zapiks embed
@@ -1300,6 +1302,7 @@ class GenericIE(InfoExtractor):
             'params': {
                 'skip_download': True,
             },
+            'skip': 'This video is unavailable.',
         },
         # Pladform embed
         {
@@ -1313,6 +1316,7 @@ class GenericIE(InfoExtractor):
                 'duration': 694,
                 'age_limit': 0,
             },
+            'skip': 'HTTP Error 404: Not Found',
         },
         # Playwire embed
         {
@@ -1333,6 +1337,14 @@ class GenericIE(InfoExtractor):
                 'id': '518726732',
                 'ext': 'mp4',
                 'title': 'Facebook Creates "On This Day" | Crunch Report',
+                'description': 'Amazon updates Fire TV line, Tesla\'s Model X spotted in the wild',
+                'timestamp': 1427237531,
+                'uploader': 'Crunch Report',
+                'upload_date': '20150324',
+            },
+            'params': {
+                # m3u8 download
+                'skip_download': True,
             },
         },
         # SVT embed
@@ -1384,16 +1396,20 @@ class GenericIE(InfoExtractor):
                 'upload_date': '20140107',
                 'timestamp': 1389118457,
             },
+            'skip': 'Invalid Page URL',
         },
         # NBC News embed
         {
             'url': 'http://www.vulture.com/2016/06/letterman-couldnt-care-less-about-late-night.html',
             'md5': '1aa589c675898ae6d37a17913cf68d66',
             'info_dict': {
-                'id': '701714499682',
+                'id': 'x_dtl_oa_LettermanliftPR_160608',
                 'ext': 'mp4',
-                'title': 'PREVIEW: On Assignment: David Letterman',
+                'title': 'David Letterman: A Preview',
                 'description': 'A preview of Tom Brokaw\'s interview with David Letterman as part of the On Assignment series powered by Dateline. Airs Sunday June 12 at 7/6c.',
+                'upload_date': '20160609',
+                'timestamp': 1465431544,
+                'uploader': 'NBCU-NEWS',
             },
         },
         # UDN embed
@@ -1410,6 +1426,7 @@ class GenericIE(InfoExtractor):
                 # m3u8 download
                 'skip_download': True,
             },
+            'expected_warnings': ['Failed to parse JSON Expecting value'],
         },
         # Ooyala embed
         {
@@ -1417,7 +1434,7 @@ class GenericIE(InfoExtractor):
             'info_dict': {
                 'id': '50YnY4czr4ms1vJ7yz3xzq0excz_pUMs',
                 'ext': 'mp4',
-                'description': 'VIDEO: INDEX/MATCH versus VLOOKUP.',
+                'description': 'Index/Match versus VLOOKUP.',
                 'title': 'This is what separates the Excel masters from the wannabes',
                 'duration': 191.933,
             },
@@ -1455,7 +1472,8 @@ class GenericIE(InfoExtractor):
                 'upload_date': '20150622',
                 'uploader': 'Public Sénat',
                 'uploader_id': 'xa9gza',
-            }
+            },
+            'skip': 'File not found.',
         },
         # OnionStudios embed
         {
@@ -2253,7 +2271,7 @@ class GenericIE(InfoExtractor):
 
         # Look for embedded rtl.nl player
         matches = re.findall(
-            r'<iframe[^>]+?src="((?:https?:)?//(?:www\.)?rtl\.nl/system/videoplayer/[^"]+(?:video_)?embed[^"]+)"',
+            r'<iframe[^>]+?src="((?:https?:)?//(?:(?:www|static)\.)?rtl\.nl/(?:system/videoplayer/[^"]+(?:video_)?)?embed[^"]+)"',
             webpage)
         if matches:
             return self.playlist_from_matches(matches, video_id, video_title, ie='RtlNl')
@@ -2652,7 +2670,7 @@ class GenericIE(InfoExtractor):
 
         # Look for UDN embeds
         mobj = re.search(
-            r'<iframe[^>]+src="(?P<url>%s)"' % UDNEmbedIE._PROTOCOL_RELATIVE_VALID_URL, webpage)
+            r'<iframe[^>]+src="(?:https?:)?(?P<url>%s)"' % UDNEmbedIE._PROTOCOL_RELATIVE_VALID_URL, webpage)
         if mobj is not None:
             return self.url_result(
                 compat_urlparse.urljoin(url, mobj.group('url')), 'UDNEmbed')
-- 
cgit v1.1


From e650659b94c526362be0cf963b0f2241934bcbbe Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 12 Oct 2017 17:39:51 +0000
Subject: [infoq] fix http format downloading

---
 youtube_dl/extractor/infoq.py | 24 +++++++++++++-----------
 1 file changed, 13 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/infoq.py b/youtube_dl/extractor/infoq.py
index 57c9b0c..c3e892f 100644
--- a/youtube_dl/extractor/infoq.py
+++ b/youtube_dl/extractor/infoq.py
@@ -8,7 +8,10 @@ from ..compat import (
     compat_urllib_parse_unquote,
     compat_urlparse,
 )
-from ..utils import determine_ext
+from ..utils import (
+    determine_ext,
+    update_url_query,
+)
 from .bokecc import BokeCCBaseIE
 
 
@@ -68,21 +71,22 @@ class InfoQIE(BokeCCBaseIE):
             'play_path': playpath,
         }]
 
-    def _extract_cookies(self, webpage):
+    def _extract_cf_auth(self, webpage):
         policy = self._search_regex(r'InfoQConstants\.scp\s*=\s*\'([^\']+)\'', webpage, 'policy')
         signature = self._search_regex(r'InfoQConstants\.scs\s*=\s*\'([^\']+)\'', webpage, 'signature')
         key_pair_id = self._search_regex(r'InfoQConstants\.sck\s*=\s*\'([^\']+)\'', webpage, 'key-pair-id')
-        return 'CloudFront-Policy=%s; CloudFront-Signature=%s; CloudFront-Key-Pair-Id=%s' % (
-            policy, signature, key_pair_id)
+        return {
+            'Policy': policy,
+            'Signature': signature,
+            'Key-Pair-Id': key_pair_id,
+        }
 
     def _extract_http_video(self, webpage):
         http_video_url = self._search_regex(r'P\.s\s*=\s*\'([^\']+)\'', webpage, 'video URL')
+        http_video_url = update_url_query(http_video_url, self._extract_cf_auth(webpage))
         return [{
             'format_id': 'http_video',
             'url': http_video_url,
-            'http_headers': {
-                'Cookie': self._extract_cookies(webpage)
-            },
         }]
 
     def _extract_http_audio(self, webpage, video_id):
@@ -91,22 +95,20 @@ class InfoQIE(BokeCCBaseIE):
         if not http_audio_url:
             return []
 
-        cookies_header = {'Cookie': self._extract_cookies(webpage)}
-
         # base URL is found in the Location header in the response returned by
         # GET https://www.infoq.com/mp3download.action?filename=... when logged in.
         http_audio_url = compat_urlparse.urljoin('http://res.infoq.com/downloads/mp3downloads/', http_audio_url)
+        http_audio_url = update_url_query(http_audio_url, self._extract_cf_auth(webpage))
 
         # audio file seem to be missing some times even if there is a download link
         # so probe URL to make sure
-        if not self._is_valid_url(http_audio_url, video_id, headers=cookies_header):
+        if not self._is_valid_url(http_audio_url, video_id):
             return []
 
         return [{
             'format_id': 'http_audio',
             'url': http_audio_url,
             'vcodec': 'none',
-            'http_headers': cookies_header,
         }]
 
     def _real_extract(self, url):
-- 
cgit v1.1


From 94a530c6cb65d555d8e3289de155c2f8f9326dea Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 12 Oct 2017 19:03:47 +0000
Subject: [howstuffworks] add support for more domains

---
 youtube_dl/extractor/howstuffworks.py | 39 +++++++----------------------------
 1 file changed, 7 insertions(+), 32 deletions(-)

diff --git a/youtube_dl/extractor/howstuffworks.py b/youtube_dl/extractor/howstuffworks.py
index 2be68ab..cf90ab3 100644
--- a/youtube_dl/extractor/howstuffworks.py
+++ b/youtube_dl/extractor/howstuffworks.py
@@ -11,45 +11,20 @@ from ..utils import (
 
 
 class HowStuffWorksIE(InfoExtractor):
-    _VALID_URL = r'https?://[\da-z-]+\.howstuffworks\.com/(?:[^/]+/)*(?:\d+-)?(?P<id>.+?)-video\.htm'
+    _VALID_URL = r'https?://[\da-z-]+\.(?:howstuffworks|stuff(?:(?:youshould|theydontwantyouto)know|toblowyourmind|momnevertoldyou)|(?:brain|car)stuffshow|fwthinking|geniusstuff)\.com/(?:[^/]+/)*(?:\d+-)?(?P<id>.+?)-video\.htm'
     _TESTS = [
         {
-            'url': 'http://adventure.howstuffworks.com/5266-cool-jobs-iditarod-musher-video.htm',
+            'url': 'http://www.stufftoblowyourmind.com/videos/optical-illusions-video.htm',
+            'md5': '76646a5acc0c92bf7cd66751ca5db94d',
             'info_dict': {
-                'id': '450221',
-                'ext': 'flv',
-                'title': 'Cool Jobs - Iditarod Musher',
-                'description': 'Cold sleds, freezing temps and warm dog breath... an Iditarod musher\'s dream. Kasey-Dee Gardner jumps on a sled to find out what the big deal is.',
-                'display_id': 'cool-jobs-iditarod-musher',
-                'thumbnail': r're:^https?://.*\.jpg$',
-                'duration': 161,
-            },
-            'skip': 'Video broken',
-        },
-        {
-            'url': 'http://adventure.howstuffworks.com/7199-survival-zone-food-and-water-in-the-savanna-video.htm',
-            'info_dict': {
-                'id': '453464',
-                'ext': 'mp4',
-                'title': 'Survival Zone: Food and Water In the Savanna',
-                'description': 'Learn how to find both food and water while trekking in the African savannah. In this video from the Discovery Channel.',
-                'display_id': 'survival-zone-food-and-water-in-the-savanna',
-                'thumbnail': r're:^https?://.*\.jpg$',
-            },
-        },
-        {
-            'url': 'http://entertainment.howstuffworks.com/arts/2706-sword-swallowing-1-by-dan-meyer-video.htm',
-            'info_dict': {
-                'id': '440011',
+                'id': '855410',
                 'ext': 'mp4',
-                'title': 'Sword Swallowing #1 by Dan Meyer',
-                'description': 'Video footage (1 of 3) used by permission of the owner Dan Meyer through Sword Swallowers Association International <www.swordswallow.org>',
-                'display_id': 'sword-swallowing-1-by-dan-meyer',
-                'thumbnail': r're:^https?://.*\.jpg$',
+                'title': 'Your Trickster Brain: Optical Illusions -- Science on the Web',
+                'description': 'md5:e374ff9561f6833ad076a8cc0a5ab2fb',
             },
         },
         {
-            'url': 'http://shows.howstuffworks.com/stuff-to-blow-your-mind/optical-illusions-video.htm',
+            'url': 'http://shows.howstuffworks.com/more-shows/why-does-balloon-stick-to-hair-video.htm',
             'only_matching': True,
         }
     ]
-- 
cgit v1.1


From c9bd503e7de28e9d4ba9ebee5e509a8abb90fc72 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Fri, 13 Oct 2017 08:41:57 +0000
Subject: [spike] bypass geo restriction

---
 youtube_dl/extractor/spike.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/spike.py b/youtube_dl/extractor/spike.py
index c59896a..a7b1b3b 100644
--- a/youtube_dl/extractor/spike.py
+++ b/youtube_dl/extractor/spike.py
@@ -44,6 +44,7 @@ class SpikeIE(MTVServicesInfoExtractor):
     _FEED_URL = 'http://www.spike.com/feeds/mrss/'
     _MOBILE_TEMPLATE = 'http://m.spike.com/videos/video.rbml?id=%s'
     _CUSTOM_URL_REGEX = re.compile(r'spikenetworkapp://([^/]+/[-a-fA-F0-9]+)')
+    _GEO_COUNTRIES = ['US']
 
     def _extract_mgid(self, webpage):
         mgid = super(SpikeIE, self)._extract_mgid(webpage)
-- 
cgit v1.1


From 685e87b61f785b096745cda5ea64ea0b950f56d1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 14 Oct 2017 20:26:24 +0700
Subject: [youtube] Fix relative URLs in description

---
 youtube_dl/extractor/youtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 6e2d57d..4e8db24 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1630,7 +1630,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                     class="[^"]*"[^>]*>
                 [^<]+\.{3}\s*
                 </a>
-            ''', r'\1', video_description)
+            ''', lambda m: compat_urlparse.urljoin(url, m.group(1)), video_description)
             video_description = clean_html(video_description)
         else:
             fd_mobj = re.search(r'<meta name="description" content="([^"]+)"', video_webpage)
-- 
cgit v1.1


From 210a2720bcdaf4f98561fefea021f42cae39462d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 15 Oct 2017 01:44:57 +0700
Subject: [anvato] Process master m3u8 manifests >>> Individual m3u8 manifests
 are not always present, e.g.
 anvato:anvato_scripps_app_web_prod_0837996dbe373629133857ae9eb72e740424d80a:4173834

---
 youtube_dl/extractor/anvato.py | 16 ++++++++++------
 1 file changed, 10 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/anvato.py b/youtube_dl/extractor/anvato.py
index 8023da7..e443ecf 100644
--- a/youtube_dl/extractor/anvato.py
+++ b/youtube_dl/extractor/anvato.py
@@ -197,12 +197,16 @@ class AnvatoIE(InfoExtractor):
                 'tbr': tbr if tbr != 0 else None,
             }
 
-            if ext == 'm3u8' or media_format in ('m3u8', 'm3u8-variant'):
-                if tbr is not None:
-                    a_format.update({
-                        'format_id': '-'.join(filter(None, ['hls', compat_str(tbr)])),
-                        'ext': 'mp4',
-                    })
+            if media_format == 'm3u8' and tbr is not None:
+                a_format.update({
+                    'format_id': '-'.join(filter(None, ['hls', compat_str(tbr)])),
+                    'ext': 'mp4',
+                })
+            elif media_format == 'm3u8-variant' or ext == 'm3u8':
+                formats.extend(self._extract_m3u8_formats(
+                    video_url, video_id, 'mp4', entry_protocol='m3u8_native',
+                    m3u8_id='hls', fatal=False))
+                continue
             elif ext == 'mp3' or media_format == 'mp3':
                 a_format['vcodec'] = 'none'
             else:
-- 
cgit v1.1


From b21ab85088345323d1e6d988b2cdce8e02fe6bdf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 15 Oct 2017 01:46:05 +0700
Subject: [scrippsnetworks:watch] Fix extraction (closes #14389)

---
 youtube_dl/extractor/scrippsnetworks.py | 211 +++++++++++++++++++++++++-------
 1 file changed, 167 insertions(+), 44 deletions(-)

diff --git a/youtube_dl/extractor/scrippsnetworks.py b/youtube_dl/extractor/scrippsnetworks.py
index 597d6f5..30bb31d 100644
--- a/youtube_dl/extractor/scrippsnetworks.py
+++ b/youtube_dl/extractor/scrippsnetworks.py
@@ -1,60 +1,183 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-from .adobepass import AdobePassIE
+import datetime
+import json
+import hashlib
+import hmac
+import re
+
+from .common import InfoExtractor
+from .anvato import AnvatoIE
 from ..utils import (
-    int_or_none,
-    smuggle_url,
-    update_url_query,
+    urlencode_postdata,
+    xpath_text,
 )
 
 
-class ScrippsNetworksWatchIE(AdobePassIE):
+class ScrippsNetworksWatchIE(InfoExtractor):
     IE_NAME = 'scrippsnetworks:watch'
-    _VALID_URL = r'https?://watch\.(?:hgtv|foodnetwork|travelchannel|diynetwork|cookingchanneltv)\.com/player\.[A-Z0-9]+\.html#(?P<id>\d+)'
-    _TEST = {
-        'url': 'http://watch.hgtv.com/player.HNT.html#0256538',
+    _VALID_URL = r'''(?x)
+                    https?://
+                        watch\.
+                        (?P<site>hgtv|foodnetwork|travelchannel|diynetwork|cookingchanneltv)\.com/
+                        (?:
+                            player\.[A-Z0-9]+\.html\#|
+                            show/(?:[^/]+/){2}
+                        )
+                        (?P<id>\d+)
+                    '''
+    _TESTS = [{
+        'url': 'http://watch.hgtv.com/show/HGTVE/Best-Ever-Treehouses/2241515/Best-Ever-Treehouses/',
         'md5': '26545fd676d939954c6808274bdb905a',
         'info_dict': {
-            'id': '0256538',
+            'id': '4173834',
             'ext': 'mp4',
-            'title': 'Seeking a Wow House',
-            'description': 'Buyers retiring in Palm Springs, California, want a modern house with major wow factor. They\'re also looking for a pool and a large, open floorplan with tall windows looking out at the views.',
-            'uploader': 'SCNI',
-            'upload_date': '20170207',
-            'timestamp': 1486450493,
+            'title': 'Best Ever Treehouses',
+            'description': "We're searching for the most over the top treehouses.",
+            'uploader': 'ANV',
+            'upload_date': '20170922',
+            'timestamp': 1506056400,
+        },
+        'params': {
+            'skip_download': True,
         },
-        'skip': 'requires TV provider authentication',
+        'add_ie': [AnvatoIE.ie_key()],
+    }, {
+        'url': 'http://watch.diynetwork.com/show/DSAL/Salvage-Dawgs/2656646/Covington-Church/',
+        'only_matching': True,
+    }, {
+        'url': 'http://watch.diynetwork.com/player.HNT.html#2656646',
+        'only_matching': True,
+    }]
+
+    _SNI_TABLE = {
+        'hgtv': 'hgtv',
+        'diynetwork': 'diy',
+        'foodnetwork': 'food',
+        'cookingchanneltv': 'cook',
+        'travelchannel': 'trav',
+        'geniuskitchen': 'geniuskitchen',
     }
+    _SNI_HOST = 'web.api.video.snidigital.com'
+
+    _AWS_REGION = 'us-east-1'
+    _AWS_IDENTITY_ID_JSON = json.dumps({
+        'IdentityId': '%s:7655847c-0ae7-4d9b-80d6-56c062927eb3' % _AWS_REGION
+    })
+    _AWS_USER_AGENT = 'aws-sdk-js/2.80.0 callback'
+    _AWS_API_KEY = 'E7wSQmq0qK6xPrF13WmzKiHo4BQ7tip4pQcSXVl1'
+    _AWS_SERVICE = 'execute-api'
+    _AWS_REQUEST = 'aws4_request'
+    _AWS_SIGNED_HEADERS = ';'.join([
+        'host', 'x-amz-date', 'x-amz-security-token', 'x-api-key'])
+    _AWS_CANONICAL_REQUEST_TEMPLATE = '''GET
+%(uri)s
+
+host:%(host)s
+x-amz-date:%(date)s
+x-amz-security-token:%(token)s
+x-api-key:%(key)s
+
+%(signed_headers)s
+%(payload_hash)s'''
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
-        channel = self._parse_json(self._search_regex(
-            r'"channels"\s*:\s*(\[.+\])',
-            webpage, 'channels'), video_id)[0]
-        video_data = next(v for v in channel['videos'] if v.get('nlvid') == video_id)
-        title = video_data['title']
-        release_url = video_data['releaseUrl']
-        if video_data.get('restricted'):
-            requestor_id = self._search_regex(
-                r'requestorId\s*=\s*"([^"]+)";', webpage, 'requestor id')
-            resource = self._get_mvpd_resource(
-                requestor_id, title, video_id,
-                video_data.get('ratings', [{}])[0].get('rating'))
-            auth = self._extract_mvpd_auth(
-                url, video_id, requestor_id, resource)
-            release_url = update_url_query(release_url, {'auth': auth})
-
-        return {
-            '_type': 'url_transparent',
-            'id': video_id,
-            'title': title,
-            'url': smuggle_url(release_url, {'force_smil_url': True}),
-            'description': video_data.get('description'),
-            'thumbnail': video_data.get('thumbnailUrl'),
-            'series': video_data.get('showTitle'),
-            'season_number': int_or_none(video_data.get('season')),
-            'episode_number': int_or_none(video_data.get('episodeNumber')),
-            'ie_key': 'ThePlatform',
+        mobj = re.match(self._VALID_URL, url)
+        site_id, video_id = mobj.group('site', 'id')
+
+        def aws_hash(s):
+            return hashlib.sha256(s.encode('utf-8')).hexdigest()
+
+        token = self._download_json(
+            'https://cognito-identity.us-east-1.amazonaws.com/', video_id,
+            data=self._AWS_IDENTITY_ID_JSON.encode('utf-8'),
+            headers={
+                'Accept': '*/*',
+                'Content-Type': 'application/x-amz-json-1.1',
+                'Referer': url,
+                'X-Amz-Content-Sha256': aws_hash(self._AWS_IDENTITY_ID_JSON),
+                'X-Amz-Target': 'AWSCognitoIdentityService.GetOpenIdToken',
+                'X-Amz-User-Agent': self._AWS_USER_AGENT,
+            })['Token']
+
+        sts = self._download_xml(
+            'https://sts.amazonaws.com/', video_id, data=urlencode_postdata({
+                'Action': 'AssumeRoleWithWebIdentity',
+                'RoleArn': 'arn:aws:iam::710330595350:role/Cognito_WebAPIUnauth_Role',
+                'RoleSessionName': 'web-identity',
+                'Version': '2011-06-15',
+                'WebIdentityToken': token,
+            }), headers={
+                'Referer': url,
+                'X-Amz-User-Agent': self._AWS_USER_AGENT,
+                'Content-Type': 'application/x-www-form-urlencoded; charset=utf-8',
+            })
+
+        def get(key):
+            return xpath_text(
+                sts, './/{https://sts.amazonaws.com/doc/2011-06-15/}%s' % key,
+                fatal=True)
+
+        access_key_id = get('AccessKeyId')
+        secret_access_key = get('SecretAccessKey')
+        session_token = get('SessionToken')
+
+        # Task 1: http://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html
+        uri = '/1/web/brands/%s/episodes/scrid/%s' % (self._SNI_TABLE[site_id], video_id)
+        datetime_now = datetime.datetime.utcnow().strftime('%Y%m%dT%H%M%SZ')
+        date = datetime_now[:8]
+        canonical_string = self._AWS_CANONICAL_REQUEST_TEMPLATE % {
+            'uri': uri,
+            'host': self._SNI_HOST,
+            'date': datetime_now,
+            'token': session_token,
+            'key': self._AWS_API_KEY,
+            'signed_headers': self._AWS_SIGNED_HEADERS,
+            'payload_hash': aws_hash(''),
         }
+
+        # Task 2: http://docs.aws.amazon.com/general/latest/gr/sigv4-create-string-to-sign.html
+        credential_string = '/'.join([date, self._AWS_REGION, self._AWS_SERVICE, self._AWS_REQUEST])
+        string_to_sign = '\n'.join([
+            'AWS4-HMAC-SHA256', datetime_now, credential_string,
+            aws_hash(canonical_string)])
+
+        # Task 3: http://docs.aws.amazon.com/general/latest/gr/sigv4-calculate-signature.html
+        def aws_hmac(key, msg):
+            return hmac.new(key, msg.encode('utf-8'), hashlib.sha256)
+
+        def aws_hmac_digest(key, msg):
+            return aws_hmac(key, msg).digest()
+
+        def aws_hmac_hexdigest(key, msg):
+            return aws_hmac(key, msg).hexdigest()
+
+        k_secret = 'AWS4' + secret_access_key
+        k_date = aws_hmac_digest(k_secret.encode('utf-8'), date)
+        k_region = aws_hmac_digest(k_date, self._AWS_REGION)
+        k_service = aws_hmac_digest(k_region, self._AWS_SERVICE)
+        k_signing = aws_hmac_digest(k_service, self._AWS_REQUEST)
+
+        signature = aws_hmac_hexdigest(k_signing, string_to_sign)
+
+        auth_header = ', '.join([
+            'AWS4-HMAC-SHA256 Credential=%s' % '/'.join(
+                [access_key_id, date, self._AWS_REGION, self._AWS_SERVICE, self._AWS_REQUEST]),
+            'SignedHeaders=%s' % self._AWS_SIGNED_HEADERS,
+            'Signature=%s' % signature,
+        ])
+
+        mcp_id = self._download_json(
+            'https://%s%s' % (self._SNI_HOST, uri), video_id, headers={
+                'Accept': '*/*',
+                'Referer': url,
+                'Authorization': auth_header,
+                'X-Amz-Date': datetime_now,
+                'X-Amz-Security-Token': session_token,
+                'X-Api-Key': self._AWS_API_KEY,
+            })['results'][0]['mcpId']
+
+        return self.url_result(
+            'anvato:anvato_scripps_app_web_prod_0837996dbe373629133857ae9eb72e740424d80a:%s' % mcp_id,
+            AnvatoIE.ie_key(), video_id=mcp_id)
-- 
cgit v1.1


From 57eb45b11110bdc6d99b02b738129565239a12b0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 15 Oct 2017 02:01:16 +0700
Subject: [scrippsnetworks:watch] Add support for geniuskitchen.com

---
 youtube_dl/extractor/scrippsnetworks.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/scrippsnetworks.py b/youtube_dl/extractor/scrippsnetworks.py
index 30bb31d..411fef8 100644
--- a/youtube_dl/extractor/scrippsnetworks.py
+++ b/youtube_dl/extractor/scrippsnetworks.py
@@ -20,10 +20,11 @@ class ScrippsNetworksWatchIE(InfoExtractor):
     _VALID_URL = r'''(?x)
                     https?://
                         watch\.
-                        (?P<site>hgtv|foodnetwork|travelchannel|diynetwork|cookingchanneltv)\.com/
+                        (?P<site>hgtv|foodnetwork|travelchannel|diynetwork|cookingchanneltv|geniuskitchen)\.com/
                         (?:
                             player\.[A-Z0-9]+\.html\#|
-                            show/(?:[^/]+/){2}
+                            show/(?:[^/]+/){2}|
+                            player/
                         )
                         (?P<id>\d+)
                     '''
@@ -49,6 +50,9 @@ class ScrippsNetworksWatchIE(InfoExtractor):
     }, {
         'url': 'http://watch.diynetwork.com/player.HNT.html#2656646',
         'only_matching': True,
+    }, {
+        'url': 'http://watch.geniuskitchen.com/player/3787617/Ample-Hills-Ice-Cream-Bike/',
+        'only_matching': True,
     }]
 
     _SNI_TABLE = {
@@ -57,7 +61,7 @@ class ScrippsNetworksWatchIE(InfoExtractor):
         'foodnetwork': 'food',
         'cookingchanneltv': 'cook',
         'travelchannel': 'trav',
-        'geniuskitchen': 'geniuskitchen',
+        'geniuskitchen': 'genius',
     }
     _SNI_HOST = 'web.api.video.snidigital.com'
 
-- 
cgit v1.1


From ee88c1cbc684a3ef09cf31a33b3adac7b299b670 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 15 Oct 2017 02:26:10 +0700
Subject: [ChangeLog] Actualize

---
 ChangeLog | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index db2c2bf..0c720b2 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,7 +1,22 @@
 version <unreleased>
 
+Core
++ [common] Add support for jwplayer youtube embeds
+
 Extractors
+* [scrippsnetworks:watch] Fix extraction (#14389)
+* [anvato] Process master m3u8 manifests
+* [youtube] Fix relative URLs in description
+* [spike] Bypass geo restriction
++ [howstuffworks] Add support for more domains
+* [infoq] Fix http format downloading
++ [rtlnl] Add support for another type of embeds
++ [onionstudios] Add support for bulbs-video embeds
+* [udn] Fix extraction
+* [shahid] Fix extraction (#14448)
 * [kaltura] Ignore Widevine encrypted video (.wvm) (#14471)
+* [vh1] Fix extraction (#9613)
+
 
 version 2017.10.12
 
-- 
cgit v1.1


From 715534083dda7ecfd7aa0156c34422b394672b5e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 15 Oct 2017 02:26:58 +0700
Subject: release 2017.10.15

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 youtube_dl/version.py     | 2 +-
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 5ef0873..2feb22d 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.10.12*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.10.12**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.10.15*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.10.15**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through the [README](https://github.com/rg3/youtube-dl/blob/master/README.md), **most notably** the [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ Add the `-v` flag to **your command line** you run youtube-dl with (`youtube-dl
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.10.12
+[debug] youtube-dl version 2017.10.15
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index 0c720b2..3b3ff45 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.10.15
 
 Core
 + [common] Add support for jwplayer youtube embeds
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 36e6e99..49d475e 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.10.12'
+__version__ = '2017.10.15'
-- 
cgit v1.1


From 9bb2c7673e45aee95023d980ff307d7a90ac58c0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 15 Oct 2017 03:38:34 +0700
Subject: [redditr] Fix extraction for URLs with query (closes #14495)

---
 youtube_dl/extractor/reddit.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/reddit.py b/youtube_dl/extractor/reddit.py
index 4d44b9d..f36bc64 100644
--- a/youtube_dl/extractor/reddit.py
+++ b/youtube_dl/extractor/reddit.py
@@ -1,5 +1,7 @@
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
@@ -45,7 +47,7 @@ class RedditIE(InfoExtractor):
 
 
 class RedditRIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?reddit\.com/r/[^/]+/comments/(?P<id>[^/]+)'
+    _VALID_URL = r'(?P<url>https?://(?:www\.)?reddit\.com/r/[^/]+/comments/(?P<id>[^/?#&]+))'
     _TESTS = [{
         'url': 'https://www.reddit.com/r/videos/comments/6rrwyj/that_small_heart_attack/',
         'info_dict': {
@@ -83,10 +85,13 @@ class RedditRIE(InfoExtractor):
     }]
 
     def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        url, video_id = mobj.group('url', 'id')
+
         video_id = self._match_id(url)
 
         data = self._download_json(
-            url + '.json', video_id)[0]['data']['children'][0]['data']
+            url + '/.json', video_id)[0]['data']['children'][0]['data']
 
         video_url = data['url']
 
-- 
cgit v1.1


From ee093a0ea04d973cc6dbd0d53b57c976a9e68dad Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 15 Oct 2017 06:11:02 +0700
Subject: [anvato] Add ability to bypass geo restriction

---
 youtube_dl/extractor/anvato.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/anvato.py b/youtube_dl/extractor/anvato.py
index e443ecf..7a29cd2 100644
--- a/youtube_dl/extractor/anvato.py
+++ b/youtube_dl/extractor/anvato.py
@@ -18,6 +18,7 @@ from ..utils import (
     int_or_none,
     strip_jsonp,
     unescapeHTML,
+    unsmuggle_url,
 )
 
 
@@ -275,6 +276,9 @@ class AnvatoIE(InfoExtractor):
             anvplayer_data['accessKey'], anvplayer_data['video'])
 
     def _real_extract(self, url):
+        url, smuggled_data = unsmuggle_url(url, {})
+        self._initialize_geo_bypass(smuggled_data.get('geo_countries'))
+
         mobj = re.match(self._VALID_URL, url)
         access_key, video_id = mobj.group('access_key_or_mcp', 'id')
         if access_key not in self._ANVACK_TABLE:
-- 
cgit v1.1


From 4827270526621cdabe74275deb38b04c2ef1b0d3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 15 Oct 2017 06:11:35 +0700
Subject: [scrippsnetworks:watch] Bypass geo restriction

---
 youtube_dl/extractor/scrippsnetworks.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/scrippsnetworks.py b/youtube_dl/extractor/scrippsnetworks.py
index 411fef8..b446a02 100644
--- a/youtube_dl/extractor/scrippsnetworks.py
+++ b/youtube_dl/extractor/scrippsnetworks.py
@@ -10,6 +10,7 @@ import re
 from .common import InfoExtractor
 from .anvato import AnvatoIE
 from ..utils import (
+    smuggle_url,
     urlencode_postdata,
     xpath_text,
 )
@@ -183,5 +184,7 @@ x-api-key:%(key)s
             })['results'][0]['mcpId']
 
         return self.url_result(
-            'anvato:anvato_scripps_app_web_prod_0837996dbe373629133857ae9eb72e740424d80a:%s' % mcp_id,
+            smuggle_url(
+                'anvato:anvato_scripps_app_web_prod_0837996dbe373629133857ae9eb72e740424d80a:%s' % mcp_id,
+                {'geo_countries': ['US']}),
             AnvatoIE.ie_key(), video_id=mcp_id)
-- 
cgit v1.1


From 5efaf43c93247446c8616454bb3f59b7ae13fcb8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 15 Oct 2017 06:13:07 +0700
Subject: [downloader/fragment] Output ad fragment count

---
 youtube_dl/downloader/fragment.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/downloader/fragment.py b/youtube_dl/downloader/fragment.py
index 6f6fb4a..7e891b9 100644
--- a/youtube_dl/downloader/fragment.py
+++ b/youtube_dl/downloader/fragment.py
@@ -117,9 +117,15 @@ class FragmentFD(FileDownloader):
     def _prepare_frag_download(self, ctx):
         if 'live' not in ctx:
             ctx['live'] = False
+        if not ctx['live']:
+            total_frags_str = '%d' % ctx['total_frags']
+            ad_frags = ctx.get('ad_frags', 0)
+            if ad_frags:
+                total_frags_str += ' (not including %d ad)' % ad_frags
+        else:
+            total_frags_str = 'unknown (live)'
         self.to_screen(
-            '[%s] Total fragments: %s'
-            % (self.FD_NAME, ctx['total_frags'] if not ctx['live'] else 'unknown (live)'))
+            '[%s] Total fragments: %s' % (self.FD_NAME, total_frags_str))
         self.report_destination(ctx['filename'])
         dl = HttpQuietDownloader(
             self.ydl,
-- 
cgit v1.1


From 74c42d9ec3c6e0f0f67923fd151ce2531637a3db Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 15 Oct 2017 06:13:48 +0700
Subject: [downloader/hls] Ignore anvato ad fragments (closes #14496)

---
 youtube_dl/downloader/hls.py | 28 ++++++++++++++++++++++++----
 1 file changed, 24 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/downloader/hls.py b/youtube_dl/downloader/hls.py
index 46308cf..7955ca5 100644
--- a/youtube_dl/downloader/hls.py
+++ b/youtube_dl/downloader/hls.py
@@ -75,15 +75,29 @@ class HlsFD(FragmentFD):
                 fd.add_progress_hook(ph)
             return fd.real_download(filename, info_dict)
 
-        total_frags = 0
+        def anvato_ad(s):
+            return s.startswith('#ANVATO-SEGMENT-INFO') and 'type=ad' in s
+
+        media_frags = 0
+        ad_frags = 0
+        ad_frag_next = False
         for line in s.splitlines():
             line = line.strip()
-            if line and not line.startswith('#'):
-                total_frags += 1
+            if not line:
+                continue
+            if line.startswith('#'):
+                if anvato_ad(line):
+                    ad_frags += 1
+                continue
+            if ad_frag_next:
+                ad_frag_next = False
+                continue
+            media_frags += 1
 
         ctx = {
             'filename': filename,
-            'total_frags': total_frags,
+            'total_frags': media_frags,
+            'ad_frags': ad_frags,
         }
 
         self._prepare_and_start_frag_download(ctx)
@@ -101,10 +115,14 @@ class HlsFD(FragmentFD):
         decrypt_info = {'METHOD': 'NONE'}
         byte_range = {}
         frag_index = 0
+        ad_frag_next = False
         for line in s.splitlines():
             line = line.strip()
             if line:
                 if not line.startswith('#'):
+                    if ad_frag_next:
+                        ad_frag_next = False
+                        continue
                     frag_index += 1
                     if frag_index <= ctx['fragment_index']:
                         continue
@@ -175,6 +193,8 @@ class HlsFD(FragmentFD):
                         'start': sub_range_start,
                         'end': sub_range_start + int(splitted_byte_range[0]),
                     }
+                elif anvato_ad(line):
+                    ad_frag_next = True
 
         self._finish_frag_download(ctx)
 
-- 
cgit v1.1


From bd7e1406b31ed18c48d51486262465c88bd92866 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 15 Oct 2017 06:15:37 +0700
Subject: [ChangeLog] Actualize

---
 ChangeLog | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 3b3ff45..4eb41a7 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,15 @@
+version <unreleased>
+
+Core
+* [downloader/hls] Ignore anvato ad fragments (#14496)
+* [downloader/fragment] Output ad fragment count
+
+Extractors
+* [scrippsnetworks:watch] Bypass geo restriction
++ [anvato] Add ability to bypass geo restriction
+* [redditr] Fix extraction for URLs with query (#14495)
+
+
 version 2017.10.15
 
 Core
-- 
cgit v1.1


From 7e721e35daa954f2c12f1113950ae07f1302f49e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 15 Oct 2017 06:16:41 +0700
Subject: release 2017.10.15.1

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 youtube_dl/version.py     | 2 +-
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 2feb22d..1a72ab2 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.10.15*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.10.15**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.10.15.1*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.10.15.1**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through the [README](https://github.com/rg3/youtube-dl/blob/master/README.md), **most notably** the [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ Add the `-v` flag to **your command line** you run youtube-dl with (`youtube-dl
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.10.15
+[debug] youtube-dl version 2017.10.15.1
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index 4eb41a7..d728e4d 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.10.15.1
 
 Core
 * [downloader/hls] Ignore anvato ad fragments (#14496)
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 49d475e..d01ba30 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.10.15'
+__version__ = '2017.10.15.1'
-- 
cgit v1.1


From aaab8c5e711c773885f67fd53788c89d94d48031 Mon Sep 17 00:00:00 2001
From: Pawit Pornkitprasan <p.pawit@gmail.com>
Date: Sun, 15 Oct 2017 10:40:57 +0700
Subject: [niconico] Improve uploader metadata extraction robustness (closes
 #14135)

---
 youtube_dl/extractor/niconico.py | 29 +++++++++++++++++++++++++----
 1 file changed, 25 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/niconico.py b/youtube_dl/extractor/niconico.py
index 026329d..df7f528 100644
--- a/youtube_dl/extractor/niconico.py
+++ b/youtube_dl/extractor/niconico.py
@@ -40,7 +40,7 @@ class NiconicoIE(InfoExtractor):
             'uploader': 'takuya0301',
             'uploader_id': '2698420',
             'upload_date': '20131123',
-            'timestamp': 1385182762,
+            'timestamp': int,  # timestamp is unstable
             'description': '(c) copyright 2008, Blender Foundation / www.bigbuckbunny.org',
             'duration': 33,
             'view_count': int,
@@ -115,8 +115,8 @@ class NiconicoIE(InfoExtractor):
         'skip': 'Requires an account',
     }, {
         # "New" HTML5 video
+        # md5 is unstable
         'url': 'http://www.nicovideo.jp/watch/sm31464864',
-        'md5': '351647b4917660986dc0fa8864085135',
         'info_dict': {
             'id': 'sm31464864',
             'ext': 'mp4',
@@ -124,7 +124,7 @@ class NiconicoIE(InfoExtractor):
             'description': 'md5:e52974af9a96e739196b2c1ca72b5feb',
             'timestamp': 1498514060,
             'upload_date': '20170626',
-            'uploader': 'ゲス',
+            'uploader': 'ゲスト',
             'uploader_id': '40826363',
             'thumbnail': r're:https?://.*',
             'duration': 198,
@@ -133,6 +133,25 @@ class NiconicoIE(InfoExtractor):
         },
         'skip': 'Requires an account',
     }, {
+        # Video without owner
+        'url': 'http://www.nicovideo.jp/watch/sm18238488',
+        'md5': 'd265680a1f92bdcbbd2a507fc9e78a9e',
+        'info_dict': {
+            'id': 'sm18238488',
+            'ext': 'mp4',
+            'title': '【実写版】ミュータントタートルズ',
+            'description': 'md5:15df8988e47a86f9e978af2064bf6d8e',
+            'timestamp': 1341160408,
+            'upload_date': '20120701',
+            'uploader': None,
+            'uploader_id': None,
+            'thumbnail': r're:https?://.*',
+            'duration': 5271,
+            'view_count': int,
+            'comment_count': int,
+        },
+        'skip': 'Requires an account',
+    }, {
         'url': 'http://sp.nicovideo.jp/watch/sm28964488?ss_pos=1&cp_in=wt_tg',
         'only_matching': True,
     }]
@@ -395,7 +414,9 @@ class NiconicoIE(InfoExtractor):
 
         webpage_url = get_video_info('watch_url') or url
 
-        owner = api_data.get('owner', {})
+        # Note: cannot use api_data.get('owner', {}) because owner may be set to "null"
+        # in the JSON, which will cause None to be returned instead of {}.
+        owner = try_get(api_data, lambda x: x.get('owner'), dict) or {}
         uploader_id = get_video_info(['ch_id', 'user_id']) or owner.get('id')
         uploader = get_video_info(['ch_name', 'user_nickname']) or owner.get('nickname')
 
-- 
cgit v1.1


From a9ee4f6e49a5910b432d0ba4d8dc60231aa7deba Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 15 Oct 2017 11:03:54 +0700
Subject: [downloader/hls] Fix total fragments count when ad fragments exist

---
 youtube_dl/downloader/hls.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/downloader/hls.py b/youtube_dl/downloader/hls.py
index 7955ca5..1a6e226 100644
--- a/youtube_dl/downloader/hls.py
+++ b/youtube_dl/downloader/hls.py
@@ -88,6 +88,7 @@ class HlsFD(FragmentFD):
             if line.startswith('#'):
                 if anvato_ad(line):
                     ad_frags += 1
+                    ad_frag_next = True
                 continue
             if ad_frag_next:
                 ad_frag_next = False
-- 
cgit v1.1


From 8cc1840ccb2f17837fc7f12ceea91826eb925ecc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 15 Oct 2017 22:12:34 +0700
Subject: [arte] Capture and output error message

---
 youtube_dl/extractor/arte.py | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/arte.py b/youtube_dl/extractor/arte.py
index 5cde90c..ffc3218 100644
--- a/youtube_dl/extractor/arte.py
+++ b/youtube_dl/extractor/arte.py
@@ -6,6 +6,7 @@ import re
 from .common import InfoExtractor
 from ..compat import (
     compat_parse_qs,
+    compat_str,
     compat_urllib_parse_urlparse,
 )
 from ..utils import (
@@ -15,6 +16,7 @@ from ..utils import (
     int_or_none,
     NO_DEFAULT,
     qualities,
+    try_get,
     unified_strdate,
 )
 
@@ -80,12 +82,15 @@ class ArteTVBaseIE(InfoExtractor):
         info = self._download_json(json_url, video_id)
         player_info = info['videoJsonPlayer']
 
-        vsr = player_info['VSR']
-
+        vsr = try_get(player_info, lambda x: x['VSR'], dict)
         if not vsr:
-            raise ExtractorError(
-                'Video %s is not available' % player_info.get('VID') or video_id,
-                expected=True)
+            error = None
+            if try_get(player_info, lambda x: x['custom_msg']['type']) == 'error':
+                error = try_get(
+                    player_info, lambda x: x['custom_msg']['msg'], compat_str)
+            if not error:
+                error = 'Video %s is not available' % player_info.get('VID') or video_id
+            raise ExtractorError(error, expected=True)
 
         upload_date_str = player_info.get('shootingDate')
         if not upload_date_str:
-- 
cgit v1.1


From acc4ea6237f7a2c7d923eca5b004aefbb6a6a766 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 16 Oct 2017 05:11:25 +0700
Subject: [eporner] Add support for embed URLs (closes #14507)

---
 youtube_dl/extractor/eporner.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/eporner.py b/youtube_dl/extractor/eporner.py
index f3734e9..81f2e2e 100644
--- a/youtube_dl/extractor/eporner.py
+++ b/youtube_dl/extractor/eporner.py
@@ -15,7 +15,7 @@ from ..utils import (
 
 
 class EpornerIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?eporner\.com/hd-porn/(?P<id>\w+)(?:/(?P<display_id>[\w-]+))?'
+    _VALID_URL = r'https?://(?:www\.)?eporner\.com/(?:hd-porn|embed)/(?P<id>\w+)(?:/(?P<display_id>[\w-]+))?'
     _TESTS = [{
         'url': 'http://www.eporner.com/hd-porn/95008/Infamous-Tiffany-Teen-Strip-Tease-Video/',
         'md5': '39d486f046212d8e1b911c52ab4691f8',
@@ -35,6 +35,9 @@ class EpornerIE(InfoExtractor):
     }, {
         'url': 'http://www.eporner.com/hd-porn/3YRUtzMcWn0',
         'only_matching': True,
+    }, {
+        'url': 'http://www.eporner.com/hd-porn/3YRUtzMcWn0',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
-- 
cgit v1.1


From 83fcf19e2d517a2862fd89dffb1d0abad024a5b9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 16 Oct 2017 05:48:45 +0700
Subject: [drtv] Respect preference for direct http formats (#14509)

---
 youtube_dl/extractor/drtv.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/drtv.py b/youtube_dl/extractor/drtv.py
index 69effba..f757745 100644
--- a/youtube_dl/extractor/drtv.py
+++ b/youtube_dl/extractor/drtv.py
@@ -138,6 +138,7 @@ class DRTVIE(InfoExtractor):
                             'tbr': int_or_none(bitrate),
                             'ext': link.get('FileFormat'),
                             'vcodec': 'none' if kind == 'AudioResource' else None,
+                            'preference': preference,
                         })
                 subtitles_list = asset.get('SubtitlesList')
                 if isinstance(subtitles_list, list):
-- 
cgit v1.1


From c233003afe34ba9db7861bf44d5097ade870c739 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 17 Oct 2017 15:39:06 +0800
Subject: [megaphone] Fix deprecated escape sequence

---
 youtube_dl/extractor/megaphone.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/megaphone.py b/youtube_dl/extractor/megaphone.py
index 60e3caf..5bafa6c 100644
--- a/youtube_dl/extractor/megaphone.py
+++ b/youtube_dl/extractor/megaphone.py
@@ -18,7 +18,7 @@ class MegaphoneIE(InfoExtractor):
             'id': 'GLT9749789991',
             'ext': 'mp3',
             'title': '#97 What Kind Of Idiot Gets Phished?',
-            'thumbnail': 're:^https://.*\.png.*$',
+            'thumbnail': r're:^https://.*\.png.*$',
             'duration': 1776.26375,
             'author': 'Reply All',
         },
-- 
cgit v1.1


From 6b9cbd023f1206f90e60cbed4497e6b107438542 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 17 Oct 2017 09:22:43 +0000
Subject: [pbs] restrict direct video url regex(fixes #14519)

---
 youtube_dl/extractor/pbs.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index 8889e4a..b51dcbe 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -187,7 +187,7 @@ class PBSIE(InfoExtractor):
     _VALID_URL = r'''(?x)https?://
         (?:
            # Direct video URL
-           (?:%s)/(?:viralplayer|video)/(?P<id>[0-9]+)/? |
+           (?:%s)/(?:(?:vir|port)alplayer|video)/(?P<id>[0-9]+)(?:[?/]|$) |
            # Article with embedded player (or direct video)
            (?:www\.)?pbs\.org/(?:[^/]+/){1,5}(?P<presumptive_id>[^/]+?)(?:\.html)?/?(?:$|[?\#]) |
            # Player
@@ -367,6 +367,10 @@ class PBSIE(InfoExtractor):
         {
             'url': 'http://watch.knpb.org/video/2365616055/',
             'only_matching': True,
+        },
+        {
+            'url': 'https://player.pbs.org/portalplayer/3004638221/?uid=',
+            'only_matching': True,
         }
     ]
     _ERRORS = {
-- 
cgit v1.1


From fa4bc6e71261613cf530437a2407ff7b61ea6cb5 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 17 Oct 2017 10:07:37 +0000
Subject: [youtube] replace youtube redirect urls in description(fixes #14517)

---
 youtube_dl/extractor/youtube.py | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 4e8db24..5aef555 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1622,6 +1622,17 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         # description
         description_original = video_description = get_element_by_id("eow-description", video_webpage)
         if video_description:
+
+            def replace_url(m):
+                redir_url = compat_urlparse.urljoin(url, m.group(1))
+                parsed_redir_url = compat_urllib_parse_urlparse(redir_url)
+                if re.search(r'^(?:www\.)?(?:youtube(?:-nocookie)?\.com|youtu\.be)$', parsed_redir_url.netloc) and parsed_redir_url.path == '/redirect':
+                    qs = compat_parse_qs(parsed_redir_url.query)
+                    q = qs.get('q')
+                    if q and q[0]:
+                        return q[0]
+                return redir_url
+
             description_original = video_description = re.sub(r'''(?x)
                 <a\s+
                     (?:[a-zA-Z-]+="[^"]*"\s+)*?
@@ -1630,7 +1641,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                     class="[^"]*"[^>]*>
                 [^<]+\.{3}\s*
                 </a>
-            ''', lambda m: compat_urlparse.urljoin(url, m.group(1)), video_description)
+            ''', replace_url, video_description)
             video_description = clean_html(video_description)
         else:
             fd_mobj = re.search(r'<meta name="description" content="([^"]+)"', video_webpage)
-- 
cgit v1.1


From 6f3b4a98c96c54eb0e3933b3cd6b4e6848582a02 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 17 Oct 2017 22:53:34 +0700
Subject: [downloader/fragment] Report warning instead of error on inconsistent
 download state

---
 youtube_dl/downloader/fragment.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/downloader/fragment.py b/youtube_dl/downloader/fragment.py
index 7e891b9..93002e4 100644
--- a/youtube_dl/downloader/fragment.py
+++ b/youtube_dl/downloader/fragment.py
@@ -158,7 +158,7 @@ class FragmentFD(FileDownloader):
             if os.path.isfile(encodeFilename(self.ytdl_filename(ctx['filename']))):
                 self._read_ytdl_file(ctx)
                 if ctx['fragment_index'] > 0 and resume_len == 0:
-                    self.report_error(
+                    self.report_warning(
                         'Inconsistent state of incomplete fragment download. '
                         'Restarting from the beginning...')
                     ctx['fragment_index'] = resume_len = 0
-- 
cgit v1.1


From ca1c9f26fa8277c0ea75c1d91a74f454cc9aaac5 Mon Sep 17 00:00:00 2001
From: Parmjit Virk <pvirk@mts.net>
Date: Wed, 18 Oct 2017 16:46:06 -0500
Subject: [unity] Add extractor (fixes #14528)

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/unity.py      | 32 ++++++++++++++++++++++++++++++++
 2 files changed, 33 insertions(+)
 create mode 100644 youtube_dl/extractor/unity.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index ecb33bc..1624129 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1138,6 +1138,7 @@ from .udn import UDNEmbedIE
 from .uktvplay import UKTVPlayIE
 from .digiteka import DigitekaIE
 from .unistra import UnistraIE
+from .unity import UnityIE
 from .uol import UOLIE
 from .uplynk import (
     UplynkIE,
diff --git a/youtube_dl/extractor/unity.py b/youtube_dl/extractor/unity.py
new file mode 100644
index 0000000..73daacf
--- /dev/null
+++ b/youtube_dl/extractor/unity.py
@@ -0,0 +1,32 @@
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from .youtube import YoutubeIE
+
+
+class UnityIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?unity3d\.com/learn/tutorials/(?:[^/]+/)*(?P<id>[^/?#&]+)'
+    _TESTS = [{
+        'url': 'https://unity3d.com/learn/tutorials/topics/animation/animate-anything-mecanim',
+        'info_dict': {
+            'id': 'jWuNtik0C8E',
+            'ext': 'mp4',
+            'title': 'Live Training 22nd September 2014 -  Animate Anything',
+            'description': 'md5:e54913114bd45a554c56cdde7669636e',
+            'duration': 2893,
+            'uploader': 'Unity',
+            'uploader_id': 'Unity3D',
+            'upload_date': '20140926',
+        }
+    }, {
+        'url': 'https://unity3d.com/learn/tutorials/projects/2d-ufo-tutorial/following-player-camera?playlist=25844',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+        youtube_id = self._search_regex(
+            r'data-video-id="([_0-9a-zA-Z-]+)"',
+            webpage, 'youtube ID')
+        return self.url_result(youtube_id, ie=YoutubeIE.ie_key(), video_id=video_id)
-- 
cgit v1.1


From e1d168e59243e740b83b02dd4f5b7b28c60ef7d3 Mon Sep 17 00:00:00 2001
From: Alex Seiler <seileralex@gmail.com>
Date: Thu, 19 Oct 2017 17:17:20 +0200
Subject: [servus] Add extractor (closes #14362)

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/servus.py     | 43 ++++++++++++++++++++++++++++++++++++++
 2 files changed, 44 insertions(+)
 create mode 100644 youtube_dl/extractor/servus.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 1624129..09b20a3 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -925,6 +925,7 @@ from .seeker import SeekerIE
 from .senateisvp import SenateISVPIE
 from .sendtonews import SendtoNewsIE
 from .servingsys import ServingSysIE
+from .servus import ServusIE
 from .sexu import SexuIE
 from .shahid import ShahidIE
 from .shared import (
diff --git a/youtube_dl/extractor/servus.py b/youtube_dl/extractor/servus.py
new file mode 100644
index 0000000..264e1dd
--- /dev/null
+++ b/youtube_dl/extractor/servus.py
@@ -0,0 +1,43 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class ServusIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?servus\.com/(?:at|de)/p/[^/]+/(?P<id>AA-\w+|\d+-\d+)'
+    _TESTS = [{
+        'url': 'https://www.servus.com/de/p/Die-Gr%C3%BCnen-aus-Sicht-des-Volkes/AA-1T6VBU5PW1W12/',
+        'md5': '046dee641cda1c4cabe13baef3be2c1c',
+        'info_dict': {
+            'id': 'AA-1T6VBU5PW1W12',
+            'ext': 'mp4',
+            'title': 'Die Grünen aus Volkssicht',
+            'description': 'md5:052b5da1cb2cd7d562ef1f19be5a5cba',
+            'thumbnail': r're:^https?://.*\.jpg$',
+        }
+    }, {
+        'url': 'https://www.servus.com/at/p/Wie-das-Leben-beginnt/1309984137314-381415152/',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        title = self._og_search_title(webpage)
+        description = self._og_search_description(webpage)
+        thumbnail = self._og_search_thumbnail(webpage)
+
+        formats = self._extract_m3u8_formats(
+            'https://stv.rbmbtnx.net/api/v1/manifests/%s.m3u8' % video_id,
+            video_id, 'mp4', entry_protocol='m3u8_native', m3u8_id='hls')
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'formats': formats,
+        }
-- 
cgit v1.1


From 382fa456ea79d0f22d274195f63e7d49b3334697 Mon Sep 17 00:00:00 2001
From: Felix Yan <felixonmars@archlinux.org>
Date: Thu, 19 Oct 2017 11:36:32 -0500
Subject: [ChangeLog] Fix typo

---
 ChangeLog | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ChangeLog b/ChangeLog
index d728e4d..38446ff 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -834,7 +834,7 @@ version 2017.04.14
 
 Core
 + [downloader/hls] Add basic support for EXT-X-BYTERANGE tag (#10955)
-+ [adobepass] Improve Comcast and Verison login code (#10803)
++ [adobepass] Improve Comcast and Verizon login code (#10803)
 + [adobepass] Add support for Verizon (#10803)
 
 Extractors
-- 
cgit v1.1


From a26a3c6d343a4da1a3739e3af39147545c51a05d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 20 Oct 2017 21:43:34 +0700
Subject: [soundcloud] Update client id (closes #14546)

---
 youtube_dl/extractor/soundcloud.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index 1c6799d..8894f4b 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -138,7 +138,7 @@ class SoundcloudIE(InfoExtractor):
         },
     ]
 
-    _CLIENT_ID = 'JlZIsxg2hY5WnBgtn3jfS0UYCl0K8DOg'
+    _CLIENT_ID = 'c6CU49JDMapyrQo06UxU9xouB9ZVzqCn'
     _IPHONE_CLIENT_ID = '376f225bf427445fc4bfb6b99b72e0bf'
 
     @staticmethod
-- 
cgit v1.1


From fc5c47d13cefab662fa031f77a07b94d39018055 Mon Sep 17 00:00:00 2001
From: Alex Seiler <seileralex@gmail.com>
Date: Fri, 20 Oct 2017 18:31:13 +0200
Subject: [parliamentliveuk] Fix extraction (closes #14524)

---
 youtube_dl/extractor/parliamentliveuk.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/parliamentliveuk.py b/youtube_dl/extractor/parliamentliveuk.py
index ebdab8d..bdd5ff5 100644
--- a/youtube_dl/extractor/parliamentliveuk.py
+++ b/youtube_dl/extractor/parliamentliveuk.py
@@ -11,7 +11,7 @@ class ParliamentLiveUKIE(InfoExtractor):
     _TESTS = [{
         'url': 'http://parliamentlive.tv/Event/Index/c1e9d44d-fd6c-4263-b50f-97ed26cc998b',
         'info_dict': {
-            'id': 'c1e9d44d-fd6c-4263-b50f-97ed26cc998b',
+            'id': '1_af9nv9ym',
             'ext': 'mp4',
             'title': 'Home Affairs Committee',
             'uploader_id': 'FFMPEG-01',
@@ -28,14 +28,14 @@ class ParliamentLiveUKIE(InfoExtractor):
         webpage = self._download_webpage(
             'http://vodplayer.parliamentlive.tv/?mid=' + video_id, video_id)
         widget_config = self._parse_json(self._search_regex(
-            r'kWidgetConfig\s*=\s*({.+});',
+            r'(?s)kWidgetConfig\s*=\s*({.+});',
             webpage, 'kaltura widget config'), video_id)
-        kaltura_url = 'kaltura:%s:%s' % (widget_config['wid'][1:], widget_config['entry_id'])
+        kaltura_url = 'kaltura:%s:%s' % (
+            widget_config['wid'][1:], widget_config['entry_id'])
         event_title = self._download_json(
             'http://parliamentlive.tv/Event/GetShareVideo/' + video_id, video_id)['event']['title']
         return {
             '_type': 'url_transparent',
-            'id': video_id,
             'title': event_title,
             'description': '',
             'url': kaltura_url,
-- 
cgit v1.1


From c9dcd4b0c573cee16ee061f6598a5a9330858aab Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 20 Oct 2017 23:36:52 +0700
Subject: [ChangeLog] Actualize

---
 ChangeLog | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 38446ff..27d03c1 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,23 @@
+version <unreleased>
+
+Core
+* [downloader/fragment] Report warning instead of error on inconsistent
+  download state
+* [downloader/hls] Fix total fragments count when ad fragments exist
+
+Extractors
+* [parliamentliveuk] Fix extraction (#14524)
+* [soundcloud] Update client id (#14546)
++ [servus] Add support for servus.com (#14362)
++ [unity] Add support for unity3d.com (#14528)
+* [youtube] Replace youtube redirect URLs in description (#14517)
+* [pbs] Restrict direct video URL regular expression (#14519)
+* [drtv] Respect preference for direct HTTP formats (#14509)
++ [eporner] Add support for embed URLs (#14507)
+* [arte] Capture and output error message
+* [niconico] Improve uploader metadata extraction robustness (#14135)
+
+
 version 2017.10.15.1
 
 Core
-- 
cgit v1.1


From fa3f0fd856a80137d873ab305c061af3e8fc5cb7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 20 Oct 2017 23:40:25 +0700
Subject: release 2017.10.20

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 docs/supportedsites.md    | 2 ++
 youtube_dl/version.py     | 2 +-
 4 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 1a72ab2..81fe10d 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.10.15.1*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.10.15.1**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.10.20*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.10.20**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through the [README](https://github.com/rg3/youtube-dl/blob/master/README.md), **most notably** the [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ Add the `-v` flag to **your command line** you run youtube-dl with (`youtube-dl
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.10.15.1
+[debug] youtube-dl version 2017.10.20
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index 27d03c1..547b559 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.10.20
 
 Core
 * [downloader/fragment] Report warning instead of error on inconsistent
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 7071450..be5de22 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -728,6 +728,7 @@
  - **SenateISVP**
  - **SendtoNews**
  - **ServingSys**
+ - **Servus**
  - **Sexu**
  - **Shahid**
  - **Shared**: shared.sx
@@ -887,6 +888,7 @@
  - **UDNEmbed**: 聯合影音
  - **UKTVPlay**
  - **Unistra**
+ - **Unity**
  - **uol.com.br**
  - **uplynk**
  - **uplynk:preplay**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index d01ba30..4d16866 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.10.15.1'
+__version__ = '2017.10.20'
-- 
cgit v1.1


From 424505df7668234333b823e8ea3d7f2621fec9ae Mon Sep 17 00:00:00 2001
From: Alex Seiler <seileralex@gmail.com>
Date: Fri, 20 Oct 2017 20:10:56 +0200
Subject: [azmedien] Fix test

---
 youtube_dl/extractor/azmedien.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/azmedien.py b/youtube_dl/extractor/azmedien.py
index f4e07d9..68f26e2 100644
--- a/youtube_dl/extractor/azmedien.py
+++ b/youtube_dl/extractor/azmedien.py
@@ -47,7 +47,7 @@ class AZMedienIE(AZMedienBaseIE):
         'url': 'http://www.telezueri.ch/62-show-zuerinews/13772-episode-sonntag-18-dezember-2016/32419-segment-massenabweisungen-beim-hiltl-club-wegen-pelzboom',
         'info_dict': {
             'id': '1_2444peh4',
-            'ext': 'mov',
+            'ext': 'mp4',
             'title': 'Massenabweisungen beim Hiltl Club wegen Pelzboom',
             'description': 'md5:9ea9dd1b159ad65b36ddcf7f0d7c76a8',
             'uploader_id': 'TeleZ?ri',
-- 
cgit v1.1


From 9a9de2d7b2462c9a16bac0ead3f415ef64179833 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 21 Oct 2017 01:32:01 +0700
Subject: [travis] Allow download tests to fail and fast finish

---
 .travis.yml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/.travis.yml b/.travis.yml
index f41e111..b655738 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -11,6 +11,10 @@ sudo: false
 env:
   - YTDL_TEST_SET=core
   - YTDL_TEST_SET=download
+matrix:
+  fast_finish: true
+  allow_failures:
+    - env: YTDL_TEST_SET=download
 script: ./devscripts/run_tests.sh
 notifications:
   email:
-- 
cgit v1.1


From 5c0e5bc4df5beea3fe457e4c2d92643837f1e70e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 21 Oct 2017 02:11:11 +0700
Subject: [README.md] Add build status bagde

---
 README.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/README.md b/README.md
index 2879aad..ea321d5 100644
--- a/README.md
+++ b/README.md
@@ -1,3 +1,5 @@
+[![Build Status](https://travis-ci.org/rg3/youtube-dl.svg?branch=master)](https://travis-ci.org/rg3/youtube-dl)
+
 youtube-dl - download videos from youtube.com or other video platforms
 
 - [INSTALLATION](#installation)
-- 
cgit v1.1


From 21ce434051f104bd270c4d03ddf524adc161f582 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 21 Oct 2017 02:14:25 +0700
Subject: [travis] Enable IRC notifications Let's see how is it verbose now

---
 .travis.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index b655738..7d77e6c 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -20,7 +20,7 @@ notifications:
   email:
     - filippo.valsorda@gmail.com
     - yasoob.khld@gmail.com
-#  irc:
-#    channels:
-#      - "irc.freenode.org#youtube-dl"
-#    skip_join: true
+  irc:
+    channels:
+      - "irc.freenode.org#youtube-dl"
+    skip_join: true
-- 
cgit v1.1


From 3ebbd9991e1218a28dad89c13ffb47ebb7ce174c Mon Sep 17 00:00:00 2001
From: Alex Seiler <seileralex@gmail.com>
Date: Sat, 21 Oct 2017 20:26:58 +0200
Subject: [nick] Add support for more nickelodeon sites (closes #14553)

---
 youtube_dl/extractor/nick.py | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nick.py b/youtube_dl/extractor/nick.py
index 510b1c4..46662fe 100644
--- a/youtube_dl/extractor/nick.py
+++ b/youtube_dl/extractor/nick.py
@@ -75,7 +75,7 @@ class NickIE(MTVServicesInfoExtractor):
 
 class NickDeIE(MTVServicesInfoExtractor):
     IE_NAME = 'nick.de'
-    _VALID_URL = r'https?://(?:www\.)?(?P<host>nick\.(?:de|com\.pl)|nickelodeon\.(?:nl|at))/[^/]+/(?:[^/]+/)*(?P<id>[^/?#&]+)'
+    _VALID_URL = r'https?://(?:www\.)?(?P<host>nick\.(?:de|com\.pl)|nickelodeon\.(?:nl|at|dk|no|se))/[^/]+/(?:[^/]+/)*(?P<id>[^/?#&]+)'
     _TESTS = [{
         'url': 'http://www.nick.de/playlist/3773-top-videos/videos/episode/17306-zu-wasser-und-zu-land-rauchende-erdnusse',
         'only_matching': True,
@@ -91,6 +91,15 @@ class NickDeIE(MTVServicesInfoExtractor):
     }, {
         'url': 'http://www.nick.com.pl/seriale/474-spongebob-kanciastoporty/wideo/17412-teatr-to-jest-to-rodeo-oszolom',
         'only_matching': True,
+    }, {
+        'url': 'http://www.nickelodeon.no/program/2626-bulderhuset/videoer/90947-femteklasse-veronica-vs-vanzilla',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.nickelodeon.dk/serier/2626-hojs-hus/videoer/761-tissepause',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.nickelodeon.se/serier/2626-lugn-i-stormen/videos/998-',
+        'only_matching': True,
     }]
 
     def _extract_mrss_url(self, webpage, host):
-- 
cgit v1.1


From cf6bda312b0bbb821df5f26a1a1e879fda1c557b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 22 Oct 2017 01:30:35 +0700
Subject: [nickde] Add support for nick.ch

---
 youtube_dl/extractor/nick.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nick.py b/youtube_dl/extractor/nick.py
index 46662fe..3f8a16f 100644
--- a/youtube_dl/extractor/nick.py
+++ b/youtube_dl/extractor/nick.py
@@ -75,7 +75,7 @@ class NickIE(MTVServicesInfoExtractor):
 
 class NickDeIE(MTVServicesInfoExtractor):
     IE_NAME = 'nick.de'
-    _VALID_URL = r'https?://(?:www\.)?(?P<host>nick\.(?:de|com\.pl)|nickelodeon\.(?:nl|at|dk|no|se))/[^/]+/(?:[^/]+/)*(?P<id>[^/?#&]+)'
+    _VALID_URL = r'https?://(?:www\.)?(?P<host>nick\.(?:de|com\.pl|ch)|nickelodeon\.(?:nl|at|dk|no|se))/[^/]+/(?:[^/]+/)*(?P<id>[^/?#&]+)'
     _TESTS = [{
         'url': 'http://www.nick.de/playlist/3773-top-videos/videos/episode/17306-zu-wasser-und-zu-land-rauchende-erdnusse',
         'only_matching': True,
@@ -100,6 +100,9 @@ class NickDeIE(MTVServicesInfoExtractor):
     }, {
         'url': 'http://www.nickelodeon.se/serier/2626-lugn-i-stormen/videos/998-',
         'only_matching': True,
+    }, {
+        'url': 'http://www.nick.ch/shows/2304-adventure-time-abenteuerzeit-mit-finn-und-jake',
+        'only_matching': True,
     }]
 
     def _extract_mrss_url(self, webpage, host):
-- 
cgit v1.1


From f03ee0b3725898f321f795d68f4f9c4630b188b3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 22 Oct 2017 01:40:05 +0700
Subject: [nickde] Add support for nickelodeon.be

---
 youtube_dl/extractor/nick.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nick.py b/youtube_dl/extractor/nick.py
index 3f8a16f..f0be298 100644
--- a/youtube_dl/extractor/nick.py
+++ b/youtube_dl/extractor/nick.py
@@ -75,7 +75,7 @@ class NickIE(MTVServicesInfoExtractor):
 
 class NickDeIE(MTVServicesInfoExtractor):
     IE_NAME = 'nick.de'
-    _VALID_URL = r'https?://(?:www\.)?(?P<host>nick\.(?:de|com\.pl|ch)|nickelodeon\.(?:nl|at|dk|no|se))/[^/]+/(?:[^/]+/)*(?P<id>[^/?#&]+)'
+    _VALID_URL = r'https?://(?:www\.)?(?P<host>nick\.(?:de|com\.pl|ch)|nickelodeon\.(?:nl|be|at|dk|no|se))/[^/]+/(?:[^/]+/)*(?P<id>[^/?#&]+)'
     _TESTS = [{
         'url': 'http://www.nick.de/playlist/3773-top-videos/videos/episode/17306-zu-wasser-und-zu-land-rauchende-erdnusse',
         'only_matching': True,
@@ -103,6 +103,9 @@ class NickDeIE(MTVServicesInfoExtractor):
     }, {
         'url': 'http://www.nick.ch/shows/2304-adventure-time-abenteuerzeit-mit-finn-und-jake',
         'only_matching': True,
+    }, {
+        'url': 'http://www.nickelodeon.be/afspeellijst/4530-top-videos/videos/episode/73917-inval-broodschapper-lariekoek-arie',
+        'only_matching': True,
     }]
 
     def _extract_mrss_url(self, webpage, host):
-- 
cgit v1.1


From 9ff6273caecaa3e169d69ed8c480ba36776c7434 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 22 Oct 2017 01:51:01 +0700
Subject: [nickru] Add support for more sites

---
 youtube_dl/extractor/nick.py | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nick.py b/youtube_dl/extractor/nick.py
index f0be298..310eea2 100644
--- a/youtube_dl/extractor/nick.py
+++ b/youtube_dl/extractor/nick.py
@@ -147,13 +147,28 @@ class NickNightIE(NickDeIE):
 
 class NickRuIE(MTVServicesInfoExtractor):
     IE_NAME = 'nickelodeonru'
-    _VALID_URL = r'https?://(?:www\.)nickelodeon\.ru/(?:playlist|shows|videos)/(?:[^/]+/)*(?P<id>[^/?#&]+)'
+    _VALID_URL = r'https?://(?:www\.)nickelodeon\.(?:ru|fr|es|pt|ro|hu)/[^/]+/(?:[^/]+/)*(?P<id>[^/?#&]+)'
     _TESTS = [{
         'url': 'http://www.nickelodeon.ru/shows/henrydanger/videos/episodes/3-sezon-15-seriya-licenziya-na-polyot/pmomfb#playlist/7airc6',
         'only_matching': True,
     }, {
         'url': 'http://www.nickelodeon.ru/videos/smotri-na-nickelodeon-v-iyule/g9hvh7',
         'only_matching': True,
+    }, {
+        'url': 'http://www.nickelodeon.fr/programmes/bob-l-eponge/videos/le-marathon-de-booh-kini-bottom-mardi-31-octobre/nfn7z0',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.nickelodeon.es/videos/nickelodeon-consejos-tortitas/f7w7xy',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.nickelodeon.pt/series/spongebob-squarepants/videos/a-bolha-de-tinta-gigante/xutq1b',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.nickelodeon.ro/emisiuni/shimmer-si-shine/video/nahal-din-bomboane/uw5u2k',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.nickelodeon.hu/musorok/spongyabob-kockanadrag/videok/episodes/buborekfujas-az-elszakadt-nadrag/q57iob#playlist/k6te4y',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
-- 
cgit v1.1


From f7a5038305682a2f41ca08addc338198c3c2e27b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 22 Oct 2017 02:46:28 +0700
Subject: [travis] Disable IRC notifications

---
 .travis.yml | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 7d77e6c..5f4f392 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -20,7 +20,3 @@ notifications:
   email:
     - filippo.valsorda@gmail.com
     - yasoob.khld@gmail.com
-  irc:
-    channels:
-      - "irc.freenode.org#youtube-dl"
-    skip_join: true
-- 
cgit v1.1


From 36e2d3ca43145e5f13874a02447db572173cc057 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 22 Oct 2017 08:16:30 +0700
Subject: [dramafever] Fix login error message extraction

---
 youtube_dl/extractor/dramafever.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/dramafever.py b/youtube_dl/extractor/dramafever.py
index 9a498d7..95883a0 100644
--- a/youtube_dl/extractor/dramafever.py
+++ b/youtube_dl/extractor/dramafever.py
@@ -59,7 +59,7 @@ class DramaFeverBaseIE(AMPIE):
         if all(logout_pattern not in response
                for logout_pattern in ['href="/accounts/logout/"', '>Log out<']):
             error = self._html_search_regex(
-                r'(?s)class="hidden-xs prompt"[^>]*>(.+?)<',
+                r'(?s)<h\d[^>]+\bclass="hidden-xs prompt"[^>]*>(.+?)</h\d',
                 response, 'error message', default=None)
             if error:
                 raise ExtractorError('Unable to login: %s' % error, expected=True)
-- 
cgit v1.1


From 55c727a547a3873396c75b62d55393da79ce0fc9 Mon Sep 17 00:00:00 2001
From: rawcoder <rawcoder@users.noreply.github.com>
Date: Sun, 22 Oct 2017 07:02:20 +0530
Subject: [ndtv] Add support for sub-sites

---
 youtube_dl/extractor/ndtv.py | 105 ++++++++++++++++++++++++++++++++++---------
 1 file changed, 83 insertions(+), 22 deletions(-)

diff --git a/youtube_dl/extractor/ndtv.py b/youtube_dl/extractor/ndtv.py
index 255f608..ddec89f 100644
--- a/youtube_dl/extractor/ndtv.py
+++ b/youtube_dl/extractor/ndtv.py
@@ -1,45 +1,106 @@
+# coding: utf-8
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..compat import (
+    compat_urllib_parse_unquote_plus
+)
 from ..utils import (
-    int_or_none,
+    parse_duration,
     remove_end,
     unified_strdate,
+    urljoin
 )
 
 
 class NDTVIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?ndtv\.com/video/(?:[^/]+/)+[^/?^&]+-(?P<id>\d+)'
-
-    _TEST = {
-        'url': 'http://www.ndtv.com/video/news/news/ndtv-exclusive-don-t-need-character-certificate-from-rahul-gandhi-says-arvind-kejriwal-300710',
-        'md5': '39f992dbe5fb531c395d8bbedb1e5e88',
-        'info_dict': {
-            'id': '300710',
-            'ext': 'mp4',
-            'title': "NDTV exclusive: Don't need character certificate from Rahul Gandhi, says Arvind Kejriwal",
-            'description': 'md5:ab2d4b4a6056c5cb4caa6d729deabf02',
-            'upload_date': '20131208',
-            'duration': 1327,
-            'thumbnail': r're:https?://.*\.jpg',
-        },
-    }
+    _VALID_URL = r'https?://(?:[^/]+\.)?ndtv\.com/(?:[^/]+/)*videos?/?(?:[^/]+/)*[^/?^&]+-(?P<id>\d+)'
+
+    _TESTS = [
+        {
+            'url': 'https://khabar.ndtv.com/video/show/prime-time/prime-time-ill-system-and-poor-education-468818',
+            'md5': '78efcf3880ef3fd9b83d405ca94a38eb',
+            'info_dict': {
+                'id': '468818',
+                'ext': 'mp4',
+                'title': "प्राइम टाइम: सिस्टम बीमार, स्कूल बदहाल",
+                'description': 'md5:f410512f1b49672e5695dea16ef2731d',
+                'upload_date': '20170928',
+                'duration': 2218,
+                'thumbnail': r're:https?://.*\.jpg',
+            }
+        },
+        {
+            # __filename is url
+            'url': 'http://movies.ndtv.com/videos/cracker-free-diwali-wishes-from-karan-johar-kriti-sanon-other-stars-470304',
+            'md5': 'f1d709352305b44443515ac56b45aa46',
+            'info_dict': {
+                'id': '470304',
+                'ext': 'mp4',
+                'title': "Cracker-Free Diwali Wishes From Karan Johar, Kriti Sanon & Other Stars",
+                'description': 'md5:f115bba1adf2f6433fa7c1ade5feb465',
+                'upload_date': '20171019',
+                'duration': 137,
+                'thumbnail': r're:https?://.*\.jpg',
+            }
+        },
+        {
+            'url': 'https://www.ndtv.com/video/news/news/delhi-s-air-quality-status-report-after-diwali-is-very-poor-470372',
+            'only_matching': True
+        },
+        {
+            'url': 'https://auto.ndtv.com/videos/the-cnb-daily-october-13-2017-469935',
+            'only_matching': True
+        },
+        {
+            'url': 'https://sports.ndtv.com/cricket/videos/2nd-t20i-rock-thrown-at-australia-cricket-team-bus-after-win-over-india-469764',
+            'only_matching': True
+        },
+        {
+            'url': 'http://gadgets.ndtv.com/videos/uncharted-the-lost-legacy-review-465568',
+            'only_matching': True
+        },
+        {
+            'url': 'http://profit.ndtv.com/videos/news/video-indian-economy-on-very-solid-track-international-monetary-fund-chief-470040',
+            'only_matching': True
+        },
+        {
+            'url': 'http://food.ndtv.com/video-basil-seeds-coconut-porridge-419083',
+            'only_matching': True
+        },
+        {
+            'url': 'https://doctor.ndtv.com/videos/top-health-stories-of-the-week-467396',
+            'only_matching': True
+        },
+        {
+            'url': 'https://swirlster.ndtv.com/video/how-to-make-friends-at-work-469324',
+            'only_matching': True
+        }
+    ]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        title = remove_end(self._og_search_title(webpage), ' - NDTV')
+        # '__title' does not contain extra words such as sub-site name, "Video" etc.
+        title = compat_urllib_parse_unquote_plus(
+            self._search_regex(r"__title\s*=\s*'([^']+)'", webpage, 'title', default=None) or
+            self._og_search_title(webpage))
 
         filename = self._search_regex(
-            r"__filename='([^']+)'", webpage, 'video filename')
-        video_url = 'http://bitcast-b.bitgravity.com/ndtvod/23372/ndtv/%s' % filename
+            r"(?:__)?filename\s*[:=]\s*'([^']+)'", webpage, 'video filename')
+        # in "movies" sub-site pages, filename is URL
+        video_url = urljoin('https://ndtvod.bc-ssl.cdn.bitgravity.com/23372/ndtv/', filename.lstrip('/'))
 
-        duration = int_or_none(self._search_regex(
-            r"__duration='([^']+)'", webpage, 'duration', fatal=False))
+        # "doctor" sub-site has MM:SS format
+        duration = parse_duration(self._search_regex(
+            r"(?:__)?duration\s*[:=]\s*'([^']+)'", webpage, 'duration', fatal=False))
 
+        # "sports", "doctor", "swirlster" sub-sites don't have 'publish-date'
         upload_date = unified_strdate(self._html_search_meta(
-            'publish-date', webpage, 'upload date', fatal=False))
+            'publish-date', webpage, 'upload date', default=None) or self._html_search_meta(
+            'uploadDate', webpage, 'upload date', default=None) or self._search_regex(
+            r'datePublished"\s*:\s*"([^"]+)"', webpage, 'upload date', fatal=False))
 
         description = remove_end(self._og_search_description(webpage), ' (Read more)')
 
-- 
cgit v1.1


From cdd1ce92c46167c12514a061609308112e79ea84 Mon Sep 17 00:00:00 2001
From: "J.D. Purcell" <jdpurcell@gmail.com>
Date: Mon, 23 Oct 2017 12:12:50 -0400
Subject: [twitch:clips] Fix title extraction

---
 youtube_dl/extractor/twitch.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index c926c99..fefcd28 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -609,7 +609,7 @@ class TwitchClipsIE(InfoExtractor):
                 r'(?s)clipInfo\s*=\s*({.+?});', webpage, 'clip info'),
             video_id, transform_source=js_to_json)
 
-        title = clip.get('channel_title') or self._og_search_title(webpage)
+        title = clip.get('title') or clip.get('channel_title') or self._og_search_title(webpage)
 
         formats = [{
             'url': option['source'],
-- 
cgit v1.1


From 7913e0fca7df6840e8434449f534f9744c9394f2 Mon Sep 17 00:00:00 2001
From: mrBliss <dewinant@gmail.com>
Date: Fri, 20 Oct 2017 14:41:07 +0200
Subject: [canvas] Add support for vrt.be/vrtnu (closes #11873)

---
 youtube_dl/extractor/canvas.py     | 142 ++++++++++++++++++++++++++++++++++++-
 youtube_dl/extractor/extractors.py |   1 +
 youtube_dl/extractor/gigya.py      |  22 ++++++
 youtube_dl/extractor/medialaan.py  |  17 ++---
 4 files changed, 167 insertions(+), 15 deletions(-)
 create mode 100644 youtube_dl/extractor/gigya.py

diff --git a/youtube_dl/extractor/canvas.py b/youtube_dl/extractor/canvas.py
index 6899f84..c7e9b8f 100644
--- a/youtube_dl/extractor/canvas.py
+++ b/youtube_dl/extractor/canvas.py
@@ -1,16 +1,24 @@
 from __future__ import unicode_literals
 
 import re
+import json
 
 from .common import InfoExtractor
+from .gigya import GigyaBaseIE
+
+
+from ..compat import compat_HTTPError
 from ..utils import (
-    float_or_none,
+    ExtractorError,
     strip_or_none,
+    float_or_none,
+    int_or_none,
+    parse_iso8601,
 )
 
 
 class CanvasIE(InfoExtractor):
-    _VALID_URL = r'https?://mediazone\.vrt\.be/api/v1/(?P<site_id>canvas|een|ketnet)/assets/(?P<id>m[dz]-ast-[^/?#&]+)'
+    _VALID_URL = r'https?://mediazone\.vrt\.be/api/v1/(?P<site_id>canvas|een|ketnet|vrtvideo)/assets/(?P<id>[^/?#&]+)'
     _TESTS = [{
         'url': 'https://mediazone.vrt.be/api/v1/ketnet/assets/md-ast-4ac54990-ce66-4d00-a8ca-9eac86f4c475',
         'md5': '90139b746a0a9bd7bb631283f6e2a64e',
@@ -166,3 +174,133 @@ class CanvasEenIE(InfoExtractor):
             'title': title,
             'description': self._og_search_description(webpage),
         }
+
+
+class VrtNUIE(GigyaBaseIE):
+    IE_DESC = 'VrtNU.be'
+    _VALID_URL = r'https?://(?:www\.)?vrt\.be/(?P<site_id>vrtnu)/(?:[^/]+/)*(?P<id>[^/?#&]+)'
+    _TESTS = [{
+        'url': 'https://www.vrt.be/vrtnu/a-z/postbus-x/1/postbus-x-s1a1/',
+        'info_dict': {
+            'id': 'pbs-pub-2e2d8c27-df26-45c9-9dc6-90c78153044d$vid-90c932b1-e21d-4fb8-99b1-db7b49cf74de',
+            'ext': 'flv',
+            'title': 'De zwarte weduwe',
+            'description': 'md5:d90c21dced7db869a85db89a623998d4',
+            'duration': 1457.04,
+            'thumbnail': r're:^https?://.*\.jpg$',
+            'season': '1',
+            'season_number': 1,
+            'episode_number': 1,
+        },
+        # 'skip': 'This video is only available for registered users'
+    }]
+    _NETRC_MACHINE = 'vrtnu'
+    _APIKEY = '3_0Z2HujMtiWq_pkAjgnS2Md2E11a1AwZjYiBETtwNE-EoEHDINgtnvcAOpNgmrVGy'
+    _CONTEXT_ID = 'R3595707040'
+
+    def _real_initialize(self):
+        self._login()
+
+    def _login(self):
+        username, password = self._get_login_info()
+        if username is None:
+            self.raise_login_required()
+
+        auth_data = {
+            'APIKey': self._APIKEY,
+            'targetEnv': 'jssdk',
+            'loginID': username,
+            'password': password,
+            'authMode': 'cookie',
+        }
+
+        auth_info = self._gigya_login(auth_data)
+
+        # Sometimes authentication fails for no good reason, retry
+        login_attempt = 1
+        while login_attempt <= 3:
+            try:
+                # When requesting a token, no actual token is returned, but the
+                # necessary cookies are set.
+                self._request_webpage(
+                    'https://token.vrt.be',
+                    None, note='Requesting a token', errnote='Could not get a token',
+                    headers={
+                        'Content-Type': 'application/json',
+                        'Referer': 'https://www.vrt.be/vrtnu/',
+                    },
+                    data=json.dumps({
+                        'uid': auth_info['UID'],
+                        'uidsig': auth_info['UIDSignature'],
+                        'ts': auth_info['signatureTimestamp'],
+                        'email': auth_info['profile']['email'],
+                    }).encode('utf-8'))
+            except ExtractorError as e:
+                if isinstance(e.cause, compat_HTTPError) and e.cause.code == 401:
+                    login_attempt += 1
+                    self.report_warning('Authentication failed')
+                    self._sleep(1, None, msg_template='Waiting for %(timeout)s seconds before trying again')
+                else:
+                    raise e
+            else:
+                break
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, display_id)
+
+        title = self._html_search_regex(
+            r'(?ms)<h1 class="content__heading">(.+?)</h1>',
+            webpage, 'title').strip()
+
+        description = self._html_search_regex(
+            r'(?ms)<div class="content__description">(.+?)</div>',
+            webpage, 'description', default=None)
+
+        season = self._html_search_regex(
+            [r'''(?xms)<div\ class="tabs__tab\ tabs__tab--active">\s*
+                    <span>seizoen\ (.+?)</span>\s*
+                </div>''',
+             r'<option value="seizoen (\d{1,3})" data-href="[^"]+?" selected>'],
+            webpage, 'season', default=None)
+
+        season_number = int_or_none(season)
+
+        episode_number = int_or_none(self._html_search_regex(
+            r'''(?xms)<div\ class="content__episode">\s*
+                    <abbr\ title="aflevering">afl</abbr>\s*<span>(\d+)</span>
+                </div>''',
+            webpage, 'episode_number', default=None))
+
+        release_date = parse_iso8601(self._html_search_regex(
+            r'(?ms)<div class="content__broadcastdate">\s*<time\ datetime="(.+?)"',
+            webpage, 'release_date', default=None))
+
+        # If there's a ? or a # in the URL, remove them and everything after
+        clean_url = url.split('?')[0].split('#')[0].strip('/')
+        securevideo_url = clean_url + '.mssecurevideo.json'
+
+        json = self._download_json(securevideo_url, display_id)
+        # We are dealing with a '../<show>.relevant' URL
+        redirect_url = json.get('url')
+        if redirect_url:
+            return self.url_result('https:' + redirect_url)
+        else:
+            # There is only one entry, but with an unknown key, so just get
+            # the first one
+            video_id = list(json.values())[0].get('videoid')
+
+        return {
+            '_type': 'url_transparent',
+            'url': 'https://mediazone.vrt.be/api/v1/vrtvideo/assets/%s' % video_id,
+            'ie_key': CanvasIE.ie_key(),
+            'id': video_id,
+            'display_id': display_id,
+            'title': title,
+            'description': description,
+            'season': season,
+            'season_number': season_number,
+            'episode_number': episode_number,
+            'release_date': release_date,
+        }
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 09b20a3..42b312c 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -150,6 +150,7 @@ from .canalc2 import Canalc2IE
 from .canvas import (
     CanvasIE,
     CanvasEenIE,
+    VrtNUIE,
 )
 from .carambatv import (
     CarambaTVIE,
diff --git a/youtube_dl/extractor/gigya.py b/youtube_dl/extractor/gigya.py
new file mode 100644
index 0000000..4121784
--- /dev/null
+++ b/youtube_dl/extractor/gigya.py
@@ -0,0 +1,22 @@
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+from ..utils import (
+    ExtractorError,
+    urlencode_postdata,
+)
+
+
+class GigyaBaseIE(InfoExtractor):
+    def _gigya_login(self, auth_data):
+        auth_info = self._download_json(
+            'https://accounts.eu1.gigya.com/accounts.login', None,
+            note='Logging in', errnote='Unable to log in',
+            data=urlencode_postdata(auth_data))
+
+        error_message = auth_info.get('errorDetails') or auth_info.get('errorMessage')
+        if error_message:
+            raise ExtractorError(
+                'Unable to login: %s' % error_message, expected=True)
+        return auth_info
diff --git a/youtube_dl/extractor/medialaan.py b/youtube_dl/extractor/medialaan.py
index 4c32fbc..f8c3005 100644
--- a/youtube_dl/extractor/medialaan.py
+++ b/youtube_dl/extractor/medialaan.py
@@ -2,19 +2,18 @@ from __future__ import unicode_literals
 
 import re
 
-from .common import InfoExtractor
+from .gigya import GigyaBaseIE
+
 from ..compat import compat_str
 from ..utils import (
-    ExtractorError,
     int_or_none,
     parse_duration,
     try_get,
     unified_timestamp,
-    urlencode_postdata,
 )
 
 
-class MedialaanIE(InfoExtractor):
+class MedialaanIE(GigyaBaseIE):
     _VALID_URL = r'''(?x)
                     https?://
                         (?:www\.|nieuws\.)?
@@ -119,15 +118,7 @@ class MedialaanIE(InfoExtractor):
             'password': password,
         }
 
-        auth_info = self._download_json(
-            'https://accounts.eu1.gigya.com/accounts.login', None,
-            note='Logging in', errnote='Unable to log in',
-            data=urlencode_postdata(auth_data))
-
-        error_message = auth_info.get('errorDetails') or auth_info.get('errorMessage')
-        if error_message:
-            raise ExtractorError(
-                'Unable to login: %s' % error_message, expected=True)
+        auth_info = self._gigya_login(auth_data)
 
         self._uid = auth_info['UID']
         self._uid_signature = auth_info['UIDSignature']
-- 
cgit v1.1


From b8c6ffc51823a9ed6af69da0f9777c6504c1d027 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 25 Oct 2017 23:17:14 +0700
Subject: [vrtnu] Add support for cookies authentication and simplify (#11873)

---
 youtube_dl/extractor/canvas.py | 26 +++++++++++++++-----------
 1 file changed, 15 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/canvas.py b/youtube_dl/extractor/canvas.py
index c7e9b8f..3faa760 100644
--- a/youtube_dl/extractor/canvas.py
+++ b/youtube_dl/extractor/canvas.py
@@ -5,8 +5,6 @@ import json
 
 from .common import InfoExtractor
 from .gigya import GigyaBaseIE
-
-
 from ..compat import compat_HTTPError
 from ..utils import (
     ExtractorError,
@@ -192,7 +190,7 @@ class VrtNUIE(GigyaBaseIE):
             'season_number': 1,
             'episode_number': 1,
         },
-        # 'skip': 'This video is only available for registered users'
+        'skip': 'This video is only available for registered users'
     }]
     _NETRC_MACHINE = 'vrtnu'
     _APIKEY = '3_0Z2HujMtiWq_pkAjgnS2Md2E11a1AwZjYiBETtwNE-EoEHDINgtnvcAOpNgmrVGy'
@@ -204,7 +202,7 @@ class VrtNUIE(GigyaBaseIE):
     def _login(self):
         username, password = self._get_login_info()
         if username is None:
-            self.raise_login_required()
+            return
 
         auth_data = {
             'APIKey': self._APIKEY,
@@ -281,15 +279,21 @@ class VrtNUIE(GigyaBaseIE):
         clean_url = url.split('?')[0].split('#')[0].strip('/')
         securevideo_url = clean_url + '.mssecurevideo.json'
 
-        json = self._download_json(securevideo_url, display_id)
+        try:
+            video = self._download_json(securevideo_url, display_id)
+        except ExtractorError as e:
+            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 401:
+                self.raise_login_required()
+            raise
+
         # We are dealing with a '../<show>.relevant' URL
-        redirect_url = json.get('url')
+        redirect_url = video.get('url')
         if redirect_url:
-            return self.url_result('https:' + redirect_url)
-        else:
-            # There is only one entry, but with an unknown key, so just get
-            # the first one
-            video_id = list(json.values())[0].get('videoid')
+            return self.url_result(self._proto_relative_url(redirect_url, 'https:'))
+
+        # There is only one entry, but with an unknown key, so just get
+        # the first one
+        video_id = list(video.values())[0].get('videoid')
 
         return {
             '_type': 'url_transparent',
-- 
cgit v1.1


From d673ab65628e1c83e31d3396f5090dad26218232 Mon Sep 17 00:00:00 2001
From: Logan B <einstein95@users.noreply.github.com>
Date: Thu, 26 Oct 2017 05:23:27 +1300
Subject: [nbc] Add support for classic-tv videos

---
 youtube_dl/extractor/nbc.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/nbc.py b/youtube_dl/extractor/nbc.py
index 35151f5..554dec3 100644
--- a/youtube_dl/extractor/nbc.py
+++ b/youtube_dl/extractor/nbc.py
@@ -15,7 +15,7 @@ from ..utils import (
 
 
 class NBCIE(AdobePassIE):
-    _VALID_URL = r'https?(?P<permalink>://(?:www\.)?nbc\.com/[^/]+/video/[^/]+/(?P<id>n?\d+))'
+    _VALID_URL = r'https?(?P<permalink>://(?:www\.)?nbc\.com/(?:classic-tv/)?[^/]+/video/[^/]+/(?P<id>n?\d+))'
 
     _TESTS = [
         {
@@ -67,7 +67,11 @@ class NBCIE(AdobePassIE):
                 'skip_download': True,
             },
             'skip': 'Only works from US',
-        }
+        },
+        {
+            'url': 'https://www.nbc.com/classic-tv/charles-in-charge/video/charles-in-charge-pilot/n3310',
+            'only_matching': True,
+        },
     ]
 
     def _real_extract(self, url):
-- 
cgit v1.1


From dc24a7d4a2210d6bf282e234e49193d204c41124 Mon Sep 17 00:00:00 2001
From: Alex Seiler <seileralex@gmail.com>
Date: Wed, 25 Oct 2017 18:27:55 +0200
Subject: [myvideo] Remove extractor (closes #8557)

Redirects to store.maxdome.de
---
 youtube_dl/extractor/extractors.py |   1 -
 youtube_dl/extractor/myvideo.py    | 177 -------------------------------------
 2 files changed, 178 deletions(-)
 delete mode 100644 youtube_dl/extractor/myvideo.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 42b312c..1835081 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -624,7 +624,6 @@ from .mwave import MwaveIE, MwaveMeetGreetIE
 from .myspace import MySpaceIE, MySpaceAlbumIE
 from .myspass import MySpassIE
 from .myvi import MyviIE
-from .myvideo import MyVideoIE
 from .myvidster import MyVidsterIE
 from .nationalgeographic import (
     NationalGeographicVideoIE,
diff --git a/youtube_dl/extractor/myvideo.py b/youtube_dl/extractor/myvideo.py
deleted file mode 100644
index 367e811..0000000
--- a/youtube_dl/extractor/myvideo.py
+++ /dev/null
@@ -1,177 +0,0 @@
-from __future__ import unicode_literals
-
-import binascii
-import base64
-import hashlib
-import re
-import json
-
-from .common import InfoExtractor
-from ..compat import (
-    compat_ord,
-    compat_urllib_parse_unquote,
-    compat_urllib_parse_urlencode,
-)
-from ..utils import (
-    ExtractorError,
-    sanitized_Request,
-)
-
-
-class MyVideoIE(InfoExtractor):
-    _WORKING = False
-    _VALID_URL = r'https?://(?:www\.)?myvideo\.de/(?:[^/]+/)?watch/(?P<id>[0-9]+)/[^?/]+.*'
-    IE_NAME = 'myvideo'
-    _TEST = {
-        'url': 'http://www.myvideo.de/watch/8229274/bowling_fail_or_win',
-        'md5': '2d2753e8130479ba2cb7e0a37002053e',
-        'info_dict': {
-            'id': '8229274',
-            'ext': 'flv',
-            'title': 'bowling-fail-or-win',
-        }
-    }
-
-    # Original Code from: https://github.com/dersphere/plugin.video.myvideo_de.git
-    # Released into the Public Domain by Tristan Fischer on 2013-05-19
-    # https://github.com/rg3/youtube-dl/pull/842
-    def __rc4crypt(self, data, key):
-        x = 0
-        box = list(range(256))
-        for i in list(range(256)):
-            x = (x + box[i] + compat_ord(key[i % len(key)])) % 256
-            box[i], box[x] = box[x], box[i]
-        x = 0
-        y = 0
-        out = ''
-        for char in data:
-            x = (x + 1) % 256
-            y = (y + box[x]) % 256
-            box[x], box[y] = box[y], box[x]
-            out += chr(compat_ord(char) ^ box[(box[x] + box[y]) % 256])
-        return out
-
-    def __md5(self, s):
-        return hashlib.md5(s).hexdigest().encode()
-
-    def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-
-        GK = (
-            b'WXpnME1EZGhNRGhpTTJNM01XVmhOREU0WldNNVpHTTJOakpt'
-            b'TW1FMU5tVTBNR05pWkRaa05XRXhNVFJoWVRVd1ptSXhaVEV3'
-            b'TnpsbA0KTVRkbU1tSTRNdz09'
-        )
-
-        # Get video webpage
-        webpage_url = 'http://www.myvideo.de/watch/%s' % video_id
-        webpage = self._download_webpage(webpage_url, video_id)
-
-        mobj = re.search('source src=\'(.+?)[.]([^.]+)\'', webpage)
-        if mobj is not None:
-            self.report_extraction(video_id)
-            video_url = mobj.group(1) + '.flv'
-
-            video_title = self._html_search_regex('<title>([^<]+)</title>',
-                                                  webpage, 'title')
-
-            return {
-                'id': video_id,
-                'url': video_url,
-                'title': video_title,
-            }
-
-        mobj = re.search(r'data-video-service="/service/data/video/%s/config' % video_id, webpage)
-        if mobj is not None:
-            request = sanitized_Request('http://www.myvideo.de/service/data/video/%s/config' % video_id, '')
-            response = self._download_webpage(request, video_id,
-                                              'Downloading video info')
-            info = json.loads(base64.b64decode(response).decode('utf-8'))
-            return {
-                'id': video_id,
-                'title': info['title'],
-                'url': info['streaming_url'].replace('rtmpe', 'rtmpt'),
-                'play_path': info['filename'],
-                'ext': 'flv',
-                'thumbnail': info['thumbnail'][0]['url'],
-            }
-
-        # try encxml
-        mobj = re.search('var flashvars={(.+?)}', webpage)
-        if mobj is None:
-            raise ExtractorError('Unable to extract video')
-
-        params = {}
-        encxml = ''
-        sec = mobj.group(1)
-        for (a, b) in re.findall('(.+?):\'(.+?)\',?', sec):
-            if not a == '_encxml':
-                params[a] = b
-            else:
-                encxml = compat_urllib_parse_unquote(b)
-        if not params.get('domain'):
-            params['domain'] = 'www.myvideo.de'
-        xmldata_url = '%s?%s' % (encxml, compat_urllib_parse_urlencode(params))
-        if 'flash_playertype=MTV' in xmldata_url:
-            self._downloader.report_warning('avoiding MTV player')
-            xmldata_url = (
-                'http://www.myvideo.de/dynamic/get_player_video_xml.php'
-                '?flash_playertype=D&ID=%s&_countlimit=4&autorun=yes'
-            ) % video_id
-
-        # get enc data
-        enc_data = self._download_webpage(xmldata_url, video_id).split('=')[1]
-        enc_data_b = binascii.unhexlify(enc_data)
-        sk = self.__md5(
-            base64.b64decode(base64.b64decode(GK)) +
-            self.__md5(
-                str(video_id).encode('utf-8')
-            )
-        )
-        dec_data = self.__rc4crypt(enc_data_b, sk)
-
-        # extracting infos
-        self.report_extraction(video_id)
-
-        video_url = None
-        mobj = re.search('connectionurl=\'(.*?)\'', dec_data)
-        if mobj:
-            video_url = compat_urllib_parse_unquote(mobj.group(1))
-            if 'myvideo2flash' in video_url:
-                self.report_warning(
-                    'Rewriting URL to use unencrypted rtmp:// ...',
-                    video_id)
-                video_url = video_url.replace('rtmpe://', 'rtmp://')
-
-        if not video_url:
-            # extract non rtmp videos
-            mobj = re.search('path=\'(http.*?)\' source=\'(.*?)\'', dec_data)
-            if mobj is None:
-                raise ExtractorError('unable to extract url')
-            video_url = compat_urllib_parse_unquote(mobj.group(1)) + compat_urllib_parse_unquote(mobj.group(2))
-
-        video_file = self._search_regex('source=\'(.*?)\'', dec_data, 'video file')
-        video_file = compat_urllib_parse_unquote(video_file)
-
-        if not video_file.endswith('f4m'):
-            ppath, prefix = video_file.split('.')
-            video_playpath = '%s:%s' % (prefix, ppath)
-        else:
-            video_playpath = ''
-
-        video_swfobj = self._search_regex(r'swfobject\.embedSWF\(\'(.+?)\'', webpage, 'swfobj')
-        video_swfobj = compat_urllib_parse_unquote(video_swfobj)
-
-        video_title = self._html_search_regex("<h1(?: class='globalHd')?>(.*?)</h1>",
-                                              webpage, 'title')
-
-        return {
-            'id': video_id,
-            'url': video_url,
-            'tc_url': video_url,
-            'title': video_title,
-            'ext': 'flv',
-            'play_path': video_playpath,
-            'player_url': video_swfobj,
-        }
-- 
cgit v1.1


From 30e6161799dfdf9f53d3c8eaa9e10afe615bc5dd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 26 Oct 2017 23:16:16 +0700
Subject: [soundgasm] Improve extraction (closes #14588)

---
 youtube_dl/extractor/soundgasm.py | 35 ++++++++++++++++++++++++-----------
 1 file changed, 24 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/soundgasm.py b/youtube_dl/extractor/soundgasm.py
index e004e2c..3d78a9d 100644
--- a/youtube_dl/extractor/soundgasm.py
+++ b/youtube_dl/extractor/soundgasm.py
@@ -8,36 +8,49 @@ from .common import InfoExtractor
 
 class SoundgasmIE(InfoExtractor):
     IE_NAME = 'soundgasm'
-    _VALID_URL = r'https?://(?:www\.)?soundgasm\.net/u/(?P<user>[0-9a-zA-Z_\-]+)/(?P<title>[0-9a-zA-Z_\-]+)'
+    _VALID_URL = r'https?://(?:www\.)?soundgasm\.net/u/(?P<user>[0-9a-zA-Z_-]+)/(?P<display_id>[0-9a-zA-Z_-]+)'
     _TEST = {
         'url': 'http://soundgasm.net/u/ytdl/Piano-sample',
         'md5': '010082a2c802c5275bb00030743e75ad',
         'info_dict': {
             'id': '88abd86ea000cafe98f96321b23cc1206cbcbcc9',
             'ext': 'm4a',
-            'title': 'ytdl_Piano-sample',
-            'description': 'Royalty Free Sample Music'
+            'title': 'Piano sample',
+            'description': 'Royalty Free Sample Music',
+            'uploader': 'ytdl',
         }
     }
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
-        display_id = mobj.group('title')
-        audio_title = mobj.group('user') + '_' + mobj.group('title')
+        display_id = mobj.group('display_id')
+
         webpage = self._download_webpage(url, display_id)
+
         audio_url = self._html_search_regex(
-            r'(?s)m4a\:\s"([^"]+)"', webpage, 'audio URL')
-        audio_id = re.split(r'\/|\.', audio_url)[-2]
+            r'(?s)m4a\s*:\s*(["\'])(?P<url>(?:(?!\1).)+)\1', webpage,
+            'audio URL', group='url')
+
+        title = self._search_regex(
+            r'<div[^>]+\bclass=["\']jp-title[^>]+>([^<]+)',
+            webpage, 'title', default=display_id)
+
         description = self._html_search_regex(
-            r'(?s)<li>Description:\s(.*?)<\/li>', webpage, 'description',
-            fatal=False)
+            (r'(?s)<div[^>]+\bclass=["\']jp-description[^>]+>(.+?)</div>',
+             r'(?s)<li>Description:\s(.*?)<\/li>'),
+            webpage, 'description', fatal=False)
+
+        audio_id = self._search_regex(
+            r'/([^/]+)\.m4a', audio_url, 'audio id', default=display_id)
 
         return {
             'id': audio_id,
             'display_id': display_id,
             'url': audio_url,
-            'title': audio_title,
-            'description': description
+            'vcodec': 'none',
+            'title': title,
+            'description': description,
+            'uploader': mobj.group('user'),
         }
 
 
-- 
cgit v1.1


From 7c1f419341ac2dec123eaa0075212edc6af3302b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 27 Oct 2017 22:21:47 +0700
Subject: [vimeo] Restrict iframe embed regex (closes #14600)

---
 youtube_dl/extractor/vimeo.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index c3f71b4..cedb548 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -412,7 +412,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
         urls = []
         # Look for embedded (iframe) Vimeo player
         for mobj in re.finditer(
-                r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//player\.vimeo\.com/video/.+?)\1',
+                r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//player\.vimeo\.com/video/\d+.*?)\1',
                 webpage):
             urls.append(VimeoIE._smuggle_referrer(unescapeHTML(mobj.group('url')), url))
         PLAIN_EMBED_RE = (
-- 
cgit v1.1


From f2332f18e66fc5255d11a2762bfaff02f8221251 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 27 Oct 2017 22:26:43 +0700
Subject: [youtube] Restrict embed regex (#14600)

---
 youtube_dl/extractor/youtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 5aef555..9943ddd 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1391,7 +1391,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             )
             (["\'])
                 (?P<url>(?:https?:)?//(?:www\.)?youtube(?:-nocookie)?\.com/
-                (?:embed|v|p)/.+?)
+                (?:embed|v|p)/[0-9A-Za-z_-]{11}.*?)
             \1''', webpage)]
 
         # lazyYT YouTube embed
-- 
cgit v1.1


From 8e01f3ca811e15aae04c7f2c5345c5eca38f99d3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 28 Oct 2017 22:58:01 +0700
Subject: [dctptv] Fix extraction (closes #14599)

---
 youtube_dl/extractor/dctp.py | 68 ++++++++++++++++++++++++++++++++------------
 1 file changed, 50 insertions(+), 18 deletions(-)

diff --git a/youtube_dl/extractor/dctp.py b/youtube_dl/extractor/dctp.py
index 00fbbff..3a6d056 100644
--- a/youtube_dl/extractor/dctp.py
+++ b/youtube_dl/extractor/dctp.py
@@ -2,53 +2,85 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..utils import unified_strdate
+from ..compat import compat_str
+from ..utils import (
+    float_or_none,
+    unified_strdate,
+)
 
 
 class DctpTvIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?dctp\.tv/(#/)?filme/(?P<id>.+?)/$'
+    _VALID_URL = r'https?://(?:www\.)?dctp\.tv/(?:#/)?filme/(?P<id>[^/?#&]+)'
     _TEST = {
         'url': 'http://www.dctp.tv/filme/videoinstallation-fuer-eine-kaufhausfassade/',
-        'md5': '174dd4a8a6225cf5655952f969cfbe24',
         'info_dict': {
             'id': '95eaa4f33dad413aa17b4ee613cccc6c',
             'display_id': 'videoinstallation-fuer-eine-kaufhausfassade',
-            'ext': 'mp4',
+            'ext': 'flv',
             'title': 'Videoinstallation für eine Kaufhausfassade',
             'description': 'Kurzfilm',
             'upload_date': '20110407',
             'thumbnail': r're:^https?://.*\.jpg$',
+            'duration': 71.24,
+        },
+        'params': {
+            # rtmp download
+            'skip_download': True,
         },
     }
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
+        display_id = self._match_id(url)
 
-        object_id = self._html_search_meta('DC.identifier', webpage)
+        webpage = self._download_webpage(url, display_id)
 
-        servers_json = self._download_json(
-            'http://www.dctp.tv/elastic_streaming_client/get_streaming_server/',
-            video_id, note='Downloading server list')
-        server = servers_json[0]['server']
-        m3u8_path = self._search_regex(
-            r'\'([^\'"]+/playlist\.m3u8)"', webpage, 'm3u8 path')
-        formats = self._extract_m3u8_formats(
-            'http://%s%s' % (server, m3u8_path), video_id, ext='mp4',
-            entry_protocol='m3u8_native')
+        video_id = self._html_search_meta(
+            'DC.identifier', webpage, 'video id',
+            default=None) or self._search_regex(
+            r'id=["\']uuid[^>]+>([^<]+)<', webpage, 'video id')
 
         title = self._og_search_title(webpage)
+
+        servers = self._download_json(
+            'http://www.dctp.tv/streaming_servers/', display_id,
+            note='Downloading server list', fatal=False)
+
+        if servers:
+            endpoint = next(
+                server['endpoint']
+                for server in servers
+                if isinstance(server.get('endpoint'), compat_str) and
+                'cloudfront' in server['endpoint'])
+        else:
+            endpoint = 'rtmpe://s2pqqn4u96e4j8.cloudfront.net/cfx/st/'
+
+        app = self._search_regex(
+            r'^rtmpe?://[^/]+/(?P<app>.*)$', endpoint, 'app')
+
+        formats = [{
+            'url': endpoint,
+            'app': app,
+            'play_path': 'mp4:%s_dctp_0500_4x3.m4v' % video_id,
+            'page_url': url,
+            'player_url': 'http://svm-prod-dctptv-static.s3.amazonaws.com/dctptv-relaunch2012-109.swf',
+            'ext': 'flv',
+        }]
+
         description = self._html_search_meta('DC.description', webpage)
         upload_date = unified_strdate(
             self._html_search_meta('DC.date.created', webpage))
         thumbnail = self._og_search_thumbnail(webpage)
+        duration = float_or_none(self._search_regex(
+            r'id=["\']duration_in_ms[^+]>(\d+)', webpage, 'duration',
+            default=None), scale=1000)
 
         return {
-            'id': object_id,
+            'id': video_id,
             'title': title,
             'formats': formats,
-            'display_id': video_id,
+            'display_id': display_id,
             'description': description,
             'upload_date': upload_date,
             'thumbnail': thumbnail,
+            'duration': duration,
         }
-- 
cgit v1.1


From 47a8587915668ef82632a7a75f8bc9862679623a Mon Sep 17 00:00:00 2001
From: Andrew Udvare <audvare@gmail.com>
Date: Sun, 12 Mar 2017 18:19:32 -0400
Subject: [younow] Add extractor

---
 youtube_dl/extractor/extractors.py |   5 +
 youtube_dl/extractor/younow.py     | 197 +++++++++++++++++++++++++++++++++++++
 2 files changed, 202 insertions(+)
 create mode 100644 youtube_dl/extractor/younow.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 1835081..b6ad50e 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1335,6 +1335,11 @@ from .youku import (
     YoukuIE,
     YoukuShowIE,
 )
+from .younow import (
+    YouNowIE,
+    YouNowChannelIE,
+    YouNowMomentIE,
+)
 from .youporn import YouPornIE
 from .yourupload import YourUploadIE
 from .youtube import (
diff --git a/youtube_dl/extractor/younow.py b/youtube_dl/extractor/younow.py
new file mode 100644
index 0000000..99abd66
--- /dev/null
+++ b/youtube_dl/extractor/younow.py
@@ -0,0 +1,197 @@
+# coding: utf-8
+from __future__ import unicode_literals
+from datetime import date, datetime
+
+from .common import InfoExtractor
+from ..compat import compat_str
+from ..utils import int_or_none, UnsupportedError
+
+MOMENT_URL_FORMAT = 'https://cdn.younow.com/php/api/moment/fetch/id=%s'
+STREAM_URL_FORMAT = 'https://hls.younow.com/momentsplaylists/live/%s/%s.m3u8'
+
+
+class YouNowIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?younow\.com/(?P<id>[^/]+)'
+    _TEST = {
+        'url': 'https://www.younow.com/AmandaPadeezy',
+        'info_dict': {
+            'id': 'AmandaPadeezy',
+            'ext': 'mp4',
+            'is_live': True,
+            'title': 'March 26, 2017',
+            'description': 'YouNow is the best way to broadcast live and get an audience to watch you.',
+            'thumbnail': 'https://ynassets.s3.amazonaws.com/broadcast/live/157869188/157869188.jpg',
+            'tags': ['girls'],
+            'categories': ['girls'],
+            'uploader': 'AmandaPadeezy',
+            'uploader_id': '6716501',
+            'uploader_url': 'https://www.younow.com/AmandaPadeezy',
+            'creator': 'AmandaPadeezy',
+            'formats': [{
+                'url': 'https://cdn.younow.com/php/api/broadcast/videoPath/hls=1/broadcastId=157869188/channelId=6716501',
+                'ext': 'mp4',
+                'protocol': 'm3u8',
+            }],
+        }
+    }
+
+    def _real_extract(self, url):
+        username = self._match_id(url)
+        data = self._download_json('https://api.younow.com/php/api/broadcast/info/curId=0/user=%s' % (username), username)
+
+        if data.get('media'):
+            stream_url = 'https://cdn.younow.com/php/api/broadcast/videoPath/hls=1/broadcastId=%s/channelId=%s' % (
+                data.get('broadcastId'),
+                data.get('userId'),
+            )
+        else:
+            raise UnsupportedError('Unsupported stream or user is not streaming at this time')
+
+        webpage = self._download_webpage(url, username)
+        try:
+            uploader = data['user']['profileUrlString']
+        except KeyError:
+            uploader = username
+        try:
+            title = data['title']
+        except KeyError:
+            title = date.today().strftime('%B %d, %Y')
+
+        return {
+            'id': uploader,
+            'is_live': True,
+            'title': title,
+            'description': self._og_search_description(webpage),
+            'thumbnail': data.get('awsUrl'),
+            'tags': data.get('tags'),
+            'categories': data.get('tags'),
+            'uploader': uploader,
+            'uploader_id': data.get('userId'),
+            'uploader_url': 'https://www.younow.com/%s' % (data['user']['profileUrlString'],),
+            'creator': uploader,
+            'view_count': int_or_none(data.get('viewers')),
+            'like_count': int_or_none(data.get('likes')),
+            'formats': [{
+                'url': stream_url,
+                'ext': 'mp4',
+                'protocol': 'm3u8',
+            }],
+        }
+
+
+def _moment_to_entry(item):
+    title = item.get('text')
+    title_type = item.get('titleType')
+    if not title:
+        if title_type:
+            title = 'YouNow %s' % item.get('titleType')
+        else:
+            title = 'YouNow moment'
+
+    entry = {
+        'id': compat_str(item['momentId']),
+        'title': title,
+        'view_count': int_or_none(item.get('views')),
+        'like_count': int_or_none(item.get('likes')),
+        'timestamp': int_or_none(item.get('created')),
+        'formats': [{
+            'url': STREAM_URL_FORMAT % (item['momentId'], item['momentId']),
+            'ext': 'mp4',
+            'protocol': 'm3u8',
+        }],
+    }
+
+    try:
+        entry['uploader'] = entry['creator'] = item['owner']['name']
+        entry['uploader_url'] = 'https://www.younow.com/%s' % (item['owner']['name'],)
+        entry['uploader_id'] = item['owner']['userId']
+    except KeyError:
+        pass
+
+    return entry
+
+
+class YouNowChannelIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?younow\.com/(?P<id>[^/]+)/channel'
+    _TEST = {
+        'url': 'https://www.younow.com/Kate_Swiz/channel',
+        'info_dict': {
+            'title': 'Kate_Swiz moments'
+        },
+        'playlist_count': 6,
+    }
+
+    MOMENTS_URL_FORMAT = 'https://cdn.younow.com/php/api/moment/profile/channelId=%s/createdBefore=%d/records=20'
+
+    def _real_extract(self, url):
+        entries = []
+        username = self._match_id(url)
+        user_info = self._download_json('https://api.younow.com/php/api/broadcast/info/curId=0/user=%s' % (username), username, note='Downloading user information')
+        channel_id = user_info['userId']
+        created_before = 0
+        moment_ids = []
+        moment_ids_processed = []
+        err = False
+
+        while True:
+            if created_before:
+                cb = datetime.fromtimestamp(created_before)
+            else:
+                cb = datetime.now()
+            info = self._download_json(self.MOMENTS_URL_FORMAT % (channel_id, created_before), username, note='Downloading moments data (created before %s)' % (cb))
+
+            for item in info['items']:
+                if item['type'] == 'moment':
+                    entry = _moment_to_entry(item)
+                    moment_ids_processed.append(entry['id'])
+                    entries.append(entry)
+                elif item['type'] == 'collection':
+                    moment_ids += [compat_str(x) for x in item['momentsIds']]
+
+                try:
+                    created_before = int_or_none(item['created'])
+                except KeyError:
+                    err = True
+                    break
+
+            if (err or
+                    not info['hasMore'] or
+                    'items' not in info or
+                    not info['items']):
+                break
+
+        for mid in set(moment_ids):
+            if mid in moment_ids_processed:
+                continue
+            item = self._download_json(MOMENT_URL_FORMAT % (mid), mid)
+            entries.append(_moment_to_entry(item['item']))
+
+        return self.playlist_result(entries, playlist_title='%s moments' % (username))
+
+
+class YouNowMomentIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?younow\.com/[^/]+/(?P<id>[^/]+)/[^/]+'
+    _TEST = {
+        'url': 'https://www.younow.com/GABO.../20712117/36319236/3b316doc/m',
+        'info_dict': {
+            'id': '20712117',
+            'ext': 'mp4',
+            'title': 'YouNow capture',
+            'view_count': 19,
+            'like_count': 0,
+            'timestamp': 1490432040,
+            'formats': [{
+                'url': 'https://hls.younow.com/momentsplaylists/live/20712117/20712117.m3u8',
+                'ext': 'mp4',
+                'protocol': 'm3u8',
+            }],
+            'upload_date': '20170325',
+            'uploader': 'GABO...',
+            'uploader_id': 35917228,
+        },
+    }
+
+    def _real_extract(self, url):
+        mid = self._match_id(url)
+        item = self._download_json(MOMENT_URL_FORMAT % (mid), mid)
+        return _moment_to_entry(item['item'])
-- 
cgit v1.1


From eb4b5818e2a297bd001eb1b4962d709b1245fd24 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 29 Oct 2017 04:16:07 +0700
Subject: [younow] Fix issues and improve extraction (closes #9255, closes
 #9432, closes #12436)

---
 youtube_dl/extractor/extractors.py |   2 +-
 youtube_dl/extractor/younow.py     | 225 +++++++++++++++++++------------------
 2 files changed, 116 insertions(+), 111 deletions(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index b6ad50e..2eed706 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1336,7 +1336,7 @@ from .youku import (
     YoukuShowIE,
 )
 from .younow import (
-    YouNowIE,
+    YouNowLiveIE,
     YouNowChannelIE,
     YouNowMomentIE,
 )
diff --git a/youtube_dl/extractor/younow.py b/youtube_dl/extractor/younow.py
index 99abd66..04dbc87 100644
--- a/youtube_dl/extractor/younow.py
+++ b/youtube_dl/extractor/younow.py
@@ -1,17 +1,22 @@
 # coding: utf-8
 from __future__ import unicode_literals
-from datetime import date, datetime
+
+import itertools
 
 from .common import InfoExtractor
 from ..compat import compat_str
-from ..utils import int_or_none, UnsupportedError
+from ..utils import (
+    ExtractorError,
+    int_or_none,
+    try_get,
+)
 
-MOMENT_URL_FORMAT = 'https://cdn.younow.com/php/api/moment/fetch/id=%s'
-STREAM_URL_FORMAT = 'https://hls.younow.com/momentsplaylists/live/%s/%s.m3u8'
+CDN_API_BASE = 'https://cdn.younow.com/php/api'
+MOMENT_URL_FORMAT = '%s/moment/fetch/id=%%s' % CDN_API_BASE
 
 
-class YouNowIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?younow\.com/(?P<id>[^/]+)'
+class YouNowLiveIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?younow\.com/(?P<id>[^/?#&]+)'
     _TEST = {
         'url': 'https://www.younow.com/AmandaPadeezy',
         'info_dict': {
@@ -19,179 +24,179 @@ class YouNowIE(InfoExtractor):
             'ext': 'mp4',
             'is_live': True,
             'title': 'March 26, 2017',
-            'description': 'YouNow is the best way to broadcast live and get an audience to watch you.',
-            'thumbnail': 'https://ynassets.s3.amazonaws.com/broadcast/live/157869188/157869188.jpg',
+            'thumbnail': r're:^https?://.*\.jpg$',
             'tags': ['girls'],
             'categories': ['girls'],
             'uploader': 'AmandaPadeezy',
             'uploader_id': '6716501',
             'uploader_url': 'https://www.younow.com/AmandaPadeezy',
             'creator': 'AmandaPadeezy',
-            'formats': [{
-                'url': 'https://cdn.younow.com/php/api/broadcast/videoPath/hls=1/broadcastId=157869188/channelId=6716501',
-                'ext': 'mp4',
-                'protocol': 'm3u8',
-            }],
-        }
+        },
+        'skip': True,
     }
 
+    @classmethod
+    def suitable(cls, url):
+        return (False
+                if YouNowChannelIE.suitable(url) or YouNowMomentIE.suitable(url)
+                else super(YouNowLiveIE, cls).suitable(url))
+
     def _real_extract(self, url):
         username = self._match_id(url)
-        data = self._download_json('https://api.younow.com/php/api/broadcast/info/curId=0/user=%s' % (username), username)
-
-        if data.get('media'):
-            stream_url = 'https://cdn.younow.com/php/api/broadcast/videoPath/hls=1/broadcastId=%s/channelId=%s' % (
-                data.get('broadcastId'),
-                data.get('userId'),
-            )
-        else:
-            raise UnsupportedError('Unsupported stream or user is not streaming at this time')
-
-        webpage = self._download_webpage(url, username)
-        try:
-            uploader = data['user']['profileUrlString']
-        except KeyError:
-            uploader = username
-        try:
-            title = data['title']
-        except KeyError:
-            title = date.today().strftime('%B %d, %Y')
+
+        data = self._download_json(
+            'https://api.younow.com/php/api/broadcast/info/curId=0/user=%s'
+            % username, username)
+
+        if data.get('errorCode') != 0:
+            raise ExtractorError(data['errorMsg'], expected=True)
+
+        uploader = try_get(
+            data, lambda x: x['user']['profileUrlString'],
+            compat_str) or username
 
         return {
             'id': uploader,
             'is_live': True,
-            'title': title,
-            'description': self._og_search_description(webpage),
+            'title': self._live_title(uploader),
             'thumbnail': data.get('awsUrl'),
             'tags': data.get('tags'),
             'categories': data.get('tags'),
             'uploader': uploader,
             'uploader_id': data.get('userId'),
-            'uploader_url': 'https://www.younow.com/%s' % (data['user']['profileUrlString'],),
+            'uploader_url': 'https://www.younow.com/%s' % username,
             'creator': uploader,
             'view_count': int_or_none(data.get('viewers')),
             'like_count': int_or_none(data.get('likes')),
             'formats': [{
-                'url': stream_url,
+                'url': '%s/broadcast/videoPath/hls=1/broadcastId=%s/channelId=%s'
+                       % (CDN_API_BASE, data['broadcastId'], data['userId']),
                 'ext': 'mp4',
                 'protocol': 'm3u8',
             }],
         }
 
 
-def _moment_to_entry(item):
+def _extract_moment(item, fatal=True):
+    moment_id = item.get('momentId')
+    if not moment_id:
+        if not fatal:
+            return
+        raise ExtractorError('Unable to extract moment id')
+
+    moment_id = compat_str(moment_id)
+
     title = item.get('text')
-    title_type = item.get('titleType')
     if not title:
-        if title_type:
-            title = 'YouNow %s' % item.get('titleType')
-        else:
-            title = 'YouNow moment'
+        title = 'YouNow %s' % (
+            item.get('momentType') or item.get('titleType') or 'moment')
+
+    uploader = try_get(item, lambda x: x['owner']['name'], compat_str)
+    uploader_id = try_get(item, lambda x: x['owner']['userId'])
+    uploader_url = 'https://www.younow.com/%s' % uploader if uploader else None
 
     entry = {
-        'id': compat_str(item['momentId']),
+        'extractor_key': 'YouNowMoment',
+        'id': moment_id,
         'title': title,
         'view_count': int_or_none(item.get('views')),
         'like_count': int_or_none(item.get('likes')),
         'timestamp': int_or_none(item.get('created')),
+        'creator': uploader,
+        'uploader': uploader,
+        'uploader_id': uploader_id,
+        'uploader_url': uploader_url,
         'formats': [{
-            'url': STREAM_URL_FORMAT % (item['momentId'], item['momentId']),
+            'url': 'https://hls.younow.com/momentsplaylists/live/%s/%s.m3u8'
+                   % (moment_id, moment_id),
             'ext': 'mp4',
-            'protocol': 'm3u8',
+            'protocol': 'm3u8_native',
         }],
     }
 
-    try:
-        entry['uploader'] = entry['creator'] = item['owner']['name']
-        entry['uploader_url'] = 'https://www.younow.com/%s' % (item['owner']['name'],)
-        entry['uploader_id'] = item['owner']['userId']
-    except KeyError:
-        pass
-
     return entry
 
 
 class YouNowChannelIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?younow\.com/(?P<id>[^/]+)/channel'
     _TEST = {
-        'url': 'https://www.younow.com/Kate_Swiz/channel',
+        'url': 'https://www.younow.com/its_Kateee_/channel',
         'info_dict': {
-            'title': 'Kate_Swiz moments'
+            'id': '14629760',
+            'title': 'its_Kateee_ moments'
         },
-        'playlist_count': 6,
+        'playlist_mincount': 8,
     }
 
-    MOMENTS_URL_FORMAT = 'https://cdn.younow.com/php/api/moment/profile/channelId=%s/createdBefore=%d/records=20'
-
-    def _real_extract(self, url):
-        entries = []
-        username = self._match_id(url)
-        user_info = self._download_json('https://api.younow.com/php/api/broadcast/info/curId=0/user=%s' % (username), username, note='Downloading user information')
-        channel_id = user_info['userId']
+    def _entries(self, username, channel_id):
         created_before = 0
-        moment_ids = []
-        moment_ids_processed = []
-        err = False
-
-        while True:
-            if created_before:
-                cb = datetime.fromtimestamp(created_before)
-            else:
-                cb = datetime.now()
-            info = self._download_json(self.MOMENTS_URL_FORMAT % (channel_id, created_before), username, note='Downloading moments data (created before %s)' % (cb))
-
-            for item in info['items']:
-                if item['type'] == 'moment':
-                    entry = _moment_to_entry(item)
-                    moment_ids_processed.append(entry['id'])
-                    entries.append(entry)
-                elif item['type'] == 'collection':
-                    moment_ids += [compat_str(x) for x in item['momentsIds']]
-
-                try:
-                    created_before = int_or_none(item['created'])
-                except KeyError:
-                    err = True
-                    break
-
-            if (err or
-                    not info['hasMore'] or
-                    'items' not in info or
-                    not info['items']):
+        for page_num in itertools.count(1):
+            if created_before is None:
                 break
+            info = self._download_json(
+                '%s/moment/profile/channelId=%s/createdBefore=%d/records=20'
+                % (CDN_API_BASE, channel_id, created_before), username,
+                note='Downloading moments page %d' % page_num)
+            items = info.get('items')
+            if not items or not isinstance(items, list):
+                break
+            for item in items:
+                if not isinstance(item, dict):
+                    continue
+                item_type = item.get('type')
+                if item_type == 'moment':
+                    entry = _extract_moment(item, fatal=False)
+                    if entry:
+                        yield entry
+                elif item_type == 'collection':
+                    moments = item.get('momentsIds')
+                    if isinstance(moments, list):
+                        for moment_id in moments:
+                            m = self._download_json(
+                                MOMENT_URL_FORMAT % moment_id, username,
+                                note='Downloading %s moment JSON' % moment_id,
+                                fatal=False)
+                            if m and isinstance(m, dict) and m.get('item'):
+                                entry = _extract_moment(m['item'])
+                                if entry:
+                                    yield entry
+                created_before = int_or_none(item.get('created'))
 
-        for mid in set(moment_ids):
-            if mid in moment_ids_processed:
-                continue
-            item = self._download_json(MOMENT_URL_FORMAT % (mid), mid)
-            entries.append(_moment_to_entry(item['item']))
-
-        return self.playlist_result(entries, playlist_title='%s moments' % (username))
+    def _real_extract(self, url):
+        username = self._match_id(url)
+        channel_id = compat_str(self._download_json(
+            'https://api.younow.com/php/api/broadcast/info/curId=0/user=%s'
+            % username, username, note='Downloading user information')['userId'])
+        return self.playlist_result(
+            self._entries(username, channel_id), channel_id,
+            '%s moments' % username)
 
 
 class YouNowMomentIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?younow\.com/[^/]+/(?P<id>[^/]+)/[^/]+'
+    _VALID_URL = r'https?://(?:www\.)?younow\.com/[^/]+/(?P<id>[^/?#&]+)'
     _TEST = {
         'url': 'https://www.younow.com/GABO.../20712117/36319236/3b316doc/m',
+        'md5': 'a30c70eadb9fb39a1aa3c8c0d22a0807',
         'info_dict': {
             'id': '20712117',
             'ext': 'mp4',
             'title': 'YouNow capture',
-            'view_count': 19,
-            'like_count': 0,
+            'view_count': int,
+            'like_count': int,
             'timestamp': 1490432040,
-            'formats': [{
-                'url': 'https://hls.younow.com/momentsplaylists/live/20712117/20712117.m3u8',
-                'ext': 'mp4',
-                'protocol': 'm3u8',
-            }],
             'upload_date': '20170325',
             'uploader': 'GABO...',
             'uploader_id': 35917228,
         },
     }
 
+    @classmethod
+    def suitable(cls, url):
+        return (False
+                if YouNowChannelIE.suitable(url)
+                else super(YouNowMomentIE, cls).suitable(url))
+
     def _real_extract(self, url):
-        mid = self._match_id(url)
-        item = self._download_json(MOMENT_URL_FORMAT % (mid), mid)
-        return _moment_to_entry(item['item'])
+        video_id = self._match_id(url)
+        item = self._download_json(MOMENT_URL_FORMAT % video_id, video_id)
+        return _extract_moment(item['item'])
-- 
cgit v1.1


From c3206d02e94ce98c6467762a228a9e58616c6d8f Mon Sep 17 00:00:00 2001
From: enigmaquip <enigmaquip@users.noreply.github.com>
Date: Sat, 28 Oct 2017 16:20:18 -0600
Subject: [fxnetworks] Extract series metadata

---
 youtube_dl/extractor/fxnetworks.py | 23 +++++++++++++++--------
 1 file changed, 15 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/fxnetworks.py b/youtube_dl/extractor/fxnetworks.py
index 6298973..37549fb 100644
--- a/youtube_dl/extractor/fxnetworks.py
+++ b/youtube_dl/extractor/fxnetworks.py
@@ -3,27 +3,31 @@ from __future__ import unicode_literals
 
 from .adobepass import AdobePassIE
 from ..utils import (
-    update_url_query,
     extract_attributes,
+    int_or_none,
     parse_age_limit,
     smuggle_url,
+    update_url_query,
 )
 
 
 class FXNetworksIE(AdobePassIE):
     _VALID_URL = r'https?://(?:www\.)?(?:fxnetworks|simpsonsworld)\.com/video/(?P<id>\d+)'
     _TESTS = [{
-        'url': 'http://www.fxnetworks.com/video/719841347694',
-        'md5': '1447d4722e42ebca19e5232ab93abb22',
+        'url': 'http://www.fxnetworks.com/video/1032565827847',
+        'md5': '8d99b97b4aa7a202f55b6ed47ea7e703',
         'info_dict': {
-            'id': '719841347694',
+            'id': 'dRzwHC_MMqIv',
             'ext': 'mp4',
-            'title': 'Vanpage',
-            'description': 'F*ck settling down. You\'re the Worst returns for an all new season August 31st on FXX.',
+            'title': 'First Look: Better Things - Season 2',
+            'description': 'Because real life is like a fart. Watch this FIRST LOOK to see what inspired the new season of Better Things.',
             'age_limit': 14,
             'uploader': 'NEWA-FNG-FX',
-            'upload_date': '20160706',
-            'timestamp': 1467844741,
+            'upload_date': '20170825',
+            'timestamp': 1503686274,
+            'episode_number': 0,
+            'season_number': 2,
+            'series': 'Better Things',
         },
         'add_ie': ['ThePlatform'],
     }, {
@@ -64,6 +68,9 @@ class FXNetworksIE(AdobePassIE):
             'id': video_id,
             'title': title,
             'url': smuggle_url(update_url_query(release_url, query), {'force_smil_url': True}),
+            'series': video_data.get('data-show-title'),
+            'episode_number': int_or_none(video_data.get('data-episode')),
+            'season_number': int_or_none(video_data.get('data-season')),
             'thumbnail': video_data.get('data-large-thumb'),
             'age_limit': parse_age_limit(rating),
             'ie_key': 'ThePlatform',
-- 
cgit v1.1


From 056653bbb1b94ba04f331ed4c27a1c0d24fe1e30 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 29 Oct 2017 07:04:48 +0700
Subject: [utils] Add support for zero years and months in parse_duration

---
 test/test_utils.py  |  1 +
 youtube_dl/utils.py | 12 +++++++++++-
 2 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index efa73d0..cc13f79 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -540,6 +540,7 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(parse_duration('87 Min.'), 5220)
         self.assertEqual(parse_duration('PT1H0.040S'), 3600.04)
         self.assertEqual(parse_duration('PT00H03M30SZ'), 210)
+        self.assertEqual(parse_duration('P0Y0M0DT0H4M20.880S'), 260.88)
 
     def test_fix_xml_ampersands(self):
         self.assertEqual(
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 59fb334..34866a5 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1835,10 +1835,20 @@ def parse_duration(s):
         days, hours, mins, secs, ms = m.groups()
     else:
         m = re.match(
-            r'''(?ix)(?:P?T)?
+            r'''(?ix)(?:P?
+                (?:
+                    [0-9]+\s*y(?:ears?)?\s*
+                )?
+                (?:
+                    [0-9]+\s*m(?:onths?)?\s*
+                )?
+                (?:
+                    [0-9]+\s*w(?:eeks?)?\s*
+                )?
                 (?:
                     (?P<days>[0-9]+)\s*d(?:ays?)?\s*
                 )?
+                T)?
                 (?:
                     (?P<hours>[0-9]+)\s*h(?:ours?)?\s*
                 )?
-- 
cgit v1.1


From 9211e3319e6006373d8b5055f7a3d0bbd734c57b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 29 Oct 2017 07:05:55 +0700
Subject: [extractor/common] Prefix format id for audio only HLS formats

---
 youtube_dl/extractor/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index a692406..52f2055 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1401,7 +1401,7 @@ class InfoExtractor(object):
             media_url = media.get('URI')
             if media_url:
                 format_id = []
-                for v in (group_id, name):
+                for v in (m3u8_id, group_id, name):
                     if v:
                         format_id.append(v)
                 f = {
-- 
cgit v1.1


From 514e8aefd488b385122fa989e937e5f9ae62d136 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 29 Oct 2017 07:11:37 +0700
Subject: [egghead] Fix extraction (closes #14388)

---
 youtube_dl/extractor/egghead.py | 79 +++++++++++++++++++++++++++++++----------
 1 file changed, 61 insertions(+), 18 deletions(-)

diff --git a/youtube_dl/extractor/egghead.py b/youtube_dl/extractor/egghead.py
index e4a3046..edabaaf 100644
--- a/youtube_dl/extractor/egghead.py
+++ b/youtube_dl/extractor/egghead.py
@@ -2,7 +2,9 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..compat import compat_str
 from ..utils import (
+    determine_ext,
     int_or_none,
     try_get,
     unified_timestamp,
@@ -17,7 +19,7 @@ class EggheadCourseIE(InfoExtractor):
         'url': 'https://egghead.io/courses/professor-frisby-introduces-composable-functional-javascript',
         'playlist_count': 29,
         'info_dict': {
-            'id': 'professor-frisby-introduces-composable-functional-javascript',
+            'id': '72',
             'title': 'Professor Frisby Introduces Composable Functional JavaScript',
             'description': 're:(?s)^This course teaches the ubiquitous.*You\'ll start composing functionality before you know it.$',
         },
@@ -26,14 +28,28 @@ class EggheadCourseIE(InfoExtractor):
     def _real_extract(self, url):
         playlist_id = self._match_id(url)
 
+        lessons = self._download_json(
+            'https://egghead.io/api/v1/series/%s/lessons' % playlist_id,
+            playlist_id, 'Downloading course lessons JSON')
+
+        entries = []
+        for lesson in lessons:
+            lesson_url = lesson.get('http_url')
+            if not lesson_url or not isinstance(lesson_url, compat_str):
+                continue
+            lesson_id = lesson.get('id')
+            if lesson_id:
+                lesson_id = compat_str(lesson_id)
+            entries.append(self.url_result(
+                lesson_url, ie=EggheadLessonIE.ie_key(), video_id=lesson_id))
+
         course = self._download_json(
-            'https://egghead.io/api/v1/series/%s' % playlist_id, playlist_id)
+            'https://egghead.io/api/v1/series/%s' % playlist_id,
+            playlist_id, 'Downloading course JSON', fatal=False) or {}
 
-        entries = [
-            self.url_result(
-                'wistia:%s' % lesson['wistia_id'], ie='Wistia',
-                video_id=lesson['wistia_id'], video_title=lesson.get('title'))
-            for lesson in course['lessons'] if lesson.get('wistia_id')]
+        playlist_id = course.get('id')
+        if playlist_id:
+            playlist_id = compat_str(playlist_id)
 
         return self.playlist_result(
             entries, playlist_id, course.get('title'),
@@ -43,11 +59,12 @@ class EggheadCourseIE(InfoExtractor):
 class EggheadLessonIE(InfoExtractor):
     IE_DESC = 'egghead.io lesson'
     IE_NAME = 'egghead:lesson'
-    _VALID_URL = r'https://egghead\.io/lessons/(?P<id>[^/?#&]+)'
-    _TEST = {
+    _VALID_URL = r'https://egghead\.io/(?:api/v1/)?lessons/(?P<id>[^/?#&]+)'
+    _TESTS = [{
         'url': 'https://egghead.io/lessons/javascript-linear-data-flow-with-container-style-types-box',
         'info_dict': {
-            'id': 'fv5yotjxcg',
+            'id': '1196',
+            'display_id': 'javascript-linear-data-flow-with-container-style-types-box',
             'ext': 'mp4',
             'title': 'Create linear data flow with container style types (Box)',
             'description': 'md5:9aa2cdb6f9878ed4c39ec09e85a8150e',
@@ -60,25 +77,51 @@ class EggheadLessonIE(InfoExtractor):
         },
         'params': {
             'skip_download': True,
+            'format': 'bestvideo',
         },
-    }
+    }, {
+        'url': 'https://egghead.io/api/v1/lessons/react-add-redux-to-a-react-application',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
-        lesson_id = self._match_id(url)
+        display_id = self._match_id(url)
 
         lesson = self._download_json(
-            'https://egghead.io/api/v1/lessons/%s' % lesson_id, lesson_id)
+            'https://egghead.io/api/v1/lessons/%s' % display_id, display_id)
+
+        lesson_id = compat_str(lesson['id'])
+        title = lesson['title']
+
+        formats = []
+        for _, format_url in lesson['media_urls'].items():
+            if not format_url or not isinstance(format_url, compat_str):
+                continue
+            ext = determine_ext(format_url)
+            if ext == 'm3u8':
+                formats.extend(self._extract_m3u8_formats(
+                    format_url, lesson_id, 'mp4', entry_protocol='m3u8',
+                    m3u8_id='hls', fatal=False))
+            elif ext == 'mpd':
+                formats.extend(self._extract_mpd_formats(
+                    format_url, lesson_id, mpd_id='dash', fatal=False))
+            else:
+                formats.append({
+                    'url': format_url,
+                })
+        self._sort_formats(formats)
 
         return {
-            '_type': 'url_transparent',
-            'ie_key': 'Wistia',
-            'url': 'wistia:%s' % lesson['wistia_id'],
-            'id': lesson['wistia_id'],
-            'title': lesson.get('title'),
+            'id': lesson_id,
+            'display_id': display_id,
+            'title': title,
             'description': lesson.get('summary'),
             'thumbnail': lesson.get('thumb_nail'),
             'timestamp': unified_timestamp(lesson.get('published_at')),
             'duration': int_or_none(lesson.get('duration')),
             'view_count': int_or_none(lesson.get('plays_count')),
             'tags': try_get(lesson, lambda x: x['tag_list'], list),
+            'series': try_get(
+                lesson, lambda x: x['series']['title'], compat_str),
+            'formats': formats,
         }
-- 
cgit v1.1


From 518d357b46cb840224d69cc543667be3a87b9b9f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 29 Oct 2017 07:21:33 +0700
Subject: [ChangeLog] Actualize

---
 ChangeLog | 25 +++++++++++++++++++++++++
 1 file changed, 25 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 547b559..795491d 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,28 @@
+version <unreleased>
+
+Core
+* [extractor/common] Prefix format id for audio only HLS formats
++ [utils] Add support for zero years and months in parse_duration
+
+Extractors
+* [egghead] Fix extraction (#14388)
++ [fxnetworks] Extract series metadata (#14603)
++ [younow] Add support for younow.com (#9255, #9432, #12436)
+* [dctptv] Fix extraction (#14599)
+* [youtube] Restrict embed regex (#14600)
+* [vimeo] Restrict iframe embed regex (#14600)
+* [soundgasm] Improve extraction (#14588)
+- [myvideo] Remove extractor (#8557)
++ [nbc] Add support for classic-tv videos (#14575)
++ [vrtnu] Add support for cookies authentication and simplify (#11873)
++ [canvas] Add support for vrt.be/vrtnu (#11873)
+* [twitch:clips] Fix title extraction (#14566)
++ [ndtv] Add support for sub-sites (#14534)
+* [dramafever] Fix login error message extraction
++ [nick] Add support for more nickelodeon sites (no, dk, se, ch, fr, es, pt,
+  ro, hu) (#14553)
+
+
 version 2017.10.20
 
 Core
-- 
cgit v1.1


From 6d0630d8801fd3278a05fa7e55a73bd454403e5a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 29 Oct 2017 07:22:53 +0700
Subject: release 2017.10.29

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 docs/supportedsites.md    | 5 ++++-
 youtube_dl/version.py     | 2 +-
 4 files changed, 9 insertions(+), 6 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 81fe10d..8814758 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.10.20*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.10.20**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.10.29*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.10.29**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through the [README](https://github.com/rg3/youtube-dl/blob/master/README.md), **most notably** the [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ Add the `-v` flag to **your command line** you run youtube-dl with (`youtube-dl
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.10.20
+[debug] youtube-dl version 2017.10.29
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index 795491d..d33a710 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.10.29
 
 Core
 * [extractor/common] Prefix format id for audio only HLS formats
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index be5de22..7b8e740 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -498,7 +498,6 @@
  - **MySpace:album**
  - **MySpass**
  - **Myvi**
- - **myvideo** (Currently broken)
  - **MyVidster**
  - **n-tv.de**
  - **natgeo**
@@ -977,6 +976,7 @@
  - **vpro**: npo.nl, ntr.nl, omroepwnl.nl, zapp.nl and npo3.nl
  - **Vrak**
  - **VRT**: deredactie.be, sporza.be, cobra.be and cobra.canvas.be
+ - **VrtNU**: VrtNU.be
  - **vrv**
  - **vrv:series**
  - **VShare**
@@ -1035,6 +1035,9 @@
  - **YouJizz**
  - **youku**: 优酷
  - **youku:show**
+ - **YouNowChannel**
+ - **YouNowLive**
+ - **YouNowMoment**
  - **YouPorn**
  - **YourUpload**
  - **youtube**: YouTube.com
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 4d16866..43f080b 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.10.20'
+__version__ = '2017.10.29'
-- 
cgit v1.1


From 8fe767e07261abb8013b18ca2ed31ebb8d95c7d1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 31 Oct 2017 23:05:25 +0700
Subject: [spankbang] Detect unavailable videos (closes #14644)

---
 youtube_dl/extractor/spankbang.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/extractor/spankbang.py b/youtube_dl/extractor/spankbang.py
index 3394c7e..2863e53 100644
--- a/youtube_dl/extractor/spankbang.py
+++ b/youtube_dl/extractor/spankbang.py
@@ -3,6 +3,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..utils import ExtractorError
 
 
 class SpankBangIE(InfoExtractor):
@@ -33,6 +34,10 @@ class SpankBangIE(InfoExtractor):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
+        if re.search(r'<[^>]+\bid=["\']video_removed', webpage):
+            raise ExtractorError(
+                'Video %s is not available' % video_id, expected=True)
+
         stream_key = self._html_search_regex(
             r'''var\s+stream_key\s*=\s*['"](.+?)['"]''',
             webpage, 'stream key')
-- 
cgit v1.1


From 044eeb145556cb41485b6b6644f40b2161a4e0f1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 1 Nov 2017 23:39:26 +0700
Subject: [extractor/common] Respect URL query in _extract_wowza_formats
 (closes #14645)

---
 youtube_dl/extractor/common.py | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 52f2055..a67ac44 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -2233,27 +2233,35 @@ class InfoExtractor(object):
         return formats
 
     def _extract_wowza_formats(self, url, video_id, m3u8_entry_protocol='m3u8_native', skip_protocols=[]):
+        query = compat_urlparse.urlparse(url).query
         url = re.sub(r'/(?:manifest|playlist|jwplayer)\.(?:m3u8|f4m|mpd|smil)', '', url)
         url_base = self._search_regex(
             r'(?:(?:https?|rtmp|rtsp):)?(//[^?]+)', url, 'format url')
         http_base_url = '%s:%s' % ('http', url_base)
         formats = []
+
+        def manifest_url(manifest):
+            m_url = '%s/%s' % (http_base_url, manifest)
+            if query:
+                m_url += '?%s' % query
+            return m_url
+
         if 'm3u8' not in skip_protocols:
             formats.extend(self._extract_m3u8_formats(
-                http_base_url + '/playlist.m3u8', video_id, 'mp4',
+                manifest_url('playlist.m3u8'), video_id, 'mp4',
                 m3u8_entry_protocol, m3u8_id='hls', fatal=False))
         if 'f4m' not in skip_protocols:
             formats.extend(self._extract_f4m_formats(
-                http_base_url + '/manifest.f4m',
+                manifest_url('manifest.f4m'),
                 video_id, f4m_id='hds', fatal=False))
         if 'dash' not in skip_protocols:
             formats.extend(self._extract_mpd_formats(
-                http_base_url + '/manifest.mpd',
+                manifest_url('manifest.mpd'),
                 video_id, mpd_id='dash', fatal=False))
         if re.search(r'(?:/smil:|\.smil)', url_base):
             if 'smil' not in skip_protocols:
                 rtmp_formats = self._extract_smil_formats(
-                    http_base_url + '/jwplayer.smil',
+                    manifest_url('jwplayer.smil'),
                     video_id, fatal=False)
                 for rtmp_format in rtmp_formats:
                     rtsp_format = rtmp_format.copy()
-- 
cgit v1.1


From b0f4331002798522621aff55deaf18406d17e081 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 2 Nov 2017 13:30:01 +0100
Subject: [gamespot] extract formats referenced with new data fields(#14652)

---
 youtube_dl/extractor/gamespot.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/gamespot.py b/youtube_dl/extractor/gamespot.py
index 02804d2..6d177cb 100644
--- a/youtube_dl/extractor/gamespot.py
+++ b/youtube_dl/extractor/gamespot.py
@@ -14,7 +14,7 @@ from ..utils import (
 
 
 class GameSpotIE(OnceIE):
-    _VALID_URL = r'https?://(?:www\.)?gamespot\.com/.*-(?P<id>\d+)/?'
+    _VALID_URL = r'https?://(?:www\.)?gamespot\.com/videos/(?:[^/]+/\d+-|embed/)(?P<id>\d+)'
     _TESTS = [{
         'url': 'http://www.gamespot.com/videos/arma-3-community-guide-sitrep-i/2300-6410818/',
         'md5': 'b2a30deaa8654fcccd43713a6b6a4825',
@@ -35,6 +35,9 @@ class GameSpotIE(OnceIE):
         'params': {
             'skip_download': True,  # m3u8 downloads
         },
+    }, {
+        'url': 'https://www.gamespot.com/videos/embed/6439218/',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
@@ -52,7 +55,7 @@ class GameSpotIE(OnceIE):
             manifest_url = f4m_url
             formats.extend(self._extract_f4m_formats(
                 f4m_url + '?hdcore=3.7.0', page_id, f4m_id='hds', fatal=False))
-        m3u8_url = streams.get('m3u8_stream')
+        m3u8_url = dict_get(streams, ('m3u8_stream', 'adaptive_stream'))
         if m3u8_url:
             manifest_url = m3u8_url
             m3u8_formats = self._extract_m3u8_formats(
@@ -60,7 +63,7 @@ class GameSpotIE(OnceIE):
                 m3u8_id='hls', fatal=False)
             formats.extend(m3u8_formats)
         progressive_url = dict_get(
-            streams, ('progressive_hd', 'progressive_high', 'progressive_low'))
+            streams, ('progressive_hd', 'progressive_high', 'progressive_low', 'other_lr'))
         if progressive_url and manifest_url:
             qualities_basename = self._search_regex(
                 r'/([^/]+)\.csmil/',
-- 
cgit v1.1


From 44cca168cc36a71ea77d8635a44f6ee9d2c33a99 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 2 Nov 2017 14:16:15 +0100
Subject: [skysport] add support ooyala embed_token protected videos(fixes
 #14641)

---
 youtube_dl/extractor/skysports.py | 21 ++++++++++++++++++---
 1 file changed, 18 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/skysports.py b/youtube_dl/extractor/skysports.py
index 4ca9f6b..efcbb36 100644
--- a/youtube_dl/extractor/skysports.py
+++ b/youtube_dl/extractor/skysports.py
@@ -2,7 +2,12 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..utils import strip_or_none
+from ..utils import (
+    extract_attributes,
+    smuggle_url,
+    strip_or_none,
+    urljoin,
+)
 
 
 class SkySportsIE(InfoExtractor):
@@ -22,12 +27,22 @@ class SkySportsIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
+        video_data = extract_attributes(self._search_regex(
+            r'(<div.+?class="sdc-article-video__media-ooyala"[^>]+>)', webpage, 'video data'))
+
+        video_url = 'ooyala:%s' % video_data['data-video-id']
+        if video_data.get('data-token-required') == 'true':
+            token_fetch_options = self._parse_json(video_data.get('data-token-fetch-options', '{}'), video_id, fatal=False) or {}
+            token_fetch_url = token_fetch_options.get('url')
+            if token_fetch_url:
+                embed_token = self._download_webpage(urljoin(url, token_fetch_url), video_id, fatal=False)
+                if embed_token:
+                    video_url = smuggle_url(video_url, {'embed_token': embed_token.strip('"')})
 
         return {
             '_type': 'url_transparent',
             'id': video_id,
-            'url': 'ooyala:%s' % self._search_regex(
-                r'data-video-id="([^"]+)"', webpage, 'ooyala id'),
+            'url': video_url,
             'title': self._og_search_title(webpage),
             'description': strip_or_none(self._og_search_description(webpage)),
             'ie_key': 'Ooyala',
-- 
cgit v1.1


From cd670befc4c823a38a88fffbaa6c493e539dd79d Mon Sep 17 00:00:00 2001
From: Jimbolino <Jimbolino@users.noreply.github.com>
Date: Thu, 2 Nov 2017 17:48:43 +0100
Subject: [22tracks] Remove extractor (closes #11024)

---
 youtube_dl/extractor/extractors.py      |  4 --
 youtube_dl/extractor/twentytwotracks.py | 86 ---------------------------------
 2 files changed, 90 deletions(-)
 delete mode 100644 youtube_dl/extractor/twentytwotracks.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 2eed706..92f7e90 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1110,10 +1110,6 @@ from .tvplayer import TVPlayerIE
 from .tweakers import TweakersIE
 from .twentyfourvideo import TwentyFourVideoIE
 from .twentymin import TwentyMinutenIE
-from .twentytwotracks import (
-    TwentyTwoTracksIE,
-    TwentyTwoTracksGenreIE
-)
 from .twitch import (
     TwitchVideoIE,
     TwitchChapterIE,
diff --git a/youtube_dl/extractor/twentytwotracks.py b/youtube_dl/extractor/twentytwotracks.py
deleted file mode 100644
index d6c0ab1..0000000
--- a/youtube_dl/extractor/twentytwotracks.py
+++ /dev/null
@@ -1,86 +0,0 @@
-from __future__ import unicode_literals
-
-import re
-
-from .common import InfoExtractor
-from ..utils import int_or_none
-
-# 22Tracks regularly replace the audio tracks that can be streamed on their
-# site. The tracks usually expire after 1 months, so we can't add tests.
-
-
-class TwentyTwoTracksIE(InfoExtractor):
-    _VALID_URL = r'https?://22tracks\.com/(?P<city>[a-z]+)/(?P<genre>[\da-z]+)/(?P<id>\d+)'
-    IE_NAME = '22tracks:track'
-
-    _API_BASE = 'http://22tracks.com/api'
-
-    def _extract_info(self, city, genre_name, track_id=None):
-        item_id = track_id if track_id else genre_name
-
-        cities = self._download_json(
-            '%s/cities' % self._API_BASE, item_id,
-            'Downloading cities info',
-            'Unable to download cities info')
-        city_id = [x['id'] for x in cities if x['slug'] == city][0]
-
-        genres = self._download_json(
-            '%s/genres/%s' % (self._API_BASE, city_id), item_id,
-            'Downloading %s genres info' % city,
-            'Unable to download %s genres info' % city)
-        genre = [x for x in genres if x['slug'] == genre_name][0]
-        genre_id = genre['id']
-
-        tracks = self._download_json(
-            '%s/tracks/%s' % (self._API_BASE, genre_id), item_id,
-            'Downloading %s genre tracks info' % genre_name,
-            'Unable to download track info')
-
-        return [x for x in tracks if x['id'] == item_id][0] if track_id else [genre['title'], tracks]
-
-    def _get_track_url(self, filename, track_id):
-        token = self._download_json(
-            'http://22tracks.com/token.php?desktop=true&u=/128/%s' % filename,
-            track_id, 'Downloading token', 'Unable to download token')
-        return 'http://audio.22tracks.com%s?st=%s&e=%d' % (token['filename'], token['st'], token['e'])
-
-    def _extract_track_info(self, track_info, track_id):
-        download_url = self._get_track_url(track_info['filename'], track_id)
-        title = '%s - %s' % (track_info['artist'].strip(), track_info['title'].strip())
-        return {
-            'id': track_id,
-            'url': download_url,
-            'ext': 'mp3',
-            'title': title,
-            'duration': int_or_none(track_info.get('duration')),
-            'timestamp': int_or_none(track_info.get('published_at') or track_info.get('created'))
-        }
-
-    def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-
-        city = mobj.group('city')
-        genre = mobj.group('genre')
-        track_id = mobj.group('id')
-
-        track_info = self._extract_info(city, genre, track_id)
-        return self._extract_track_info(track_info, track_id)
-
-
-class TwentyTwoTracksGenreIE(TwentyTwoTracksIE):
-    _VALID_URL = r'https?://22tracks\.com/(?P<city>[a-z]+)/(?P<genre>[\da-z]+)/?$'
-    IE_NAME = '22tracks:genre'
-
-    def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-
-        city = mobj.group('city')
-        genre = mobj.group('genre')
-
-        genre_title, tracks = self._extract_info(city, genre)
-
-        entries = [
-            self._extract_track_info(track_info, track_info['id'])
-            for track_info in tracks]
-
-        return self.playlist_result(entries, genre, genre_title)
-- 
cgit v1.1


From 48107c198bd76e611e3d4c2486cdc5403829a05a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 4 Nov 2017 22:10:55 +0700
Subject: [f4m] Prefer baseURL for relative URLs (closes #14660)

---
 youtube_dl/downloader/f4m.py   | 25 +++++++++++++++++--------
 youtube_dl/extractor/common.py | 14 +++++++-------
 2 files changed, 24 insertions(+), 15 deletions(-)

diff --git a/youtube_dl/downloader/f4m.py b/youtube_dl/downloader/f4m.py
index c8fde9a..fdb80f4 100644
--- a/youtube_dl/downloader/f4m.py
+++ b/youtube_dl/downloader/f4m.py
@@ -243,8 +243,17 @@ def remove_encrypted_media(media):
                        media))
 
 
-def _add_ns(prop):
-    return '{http://ns.adobe.com/f4m/1.0}%s' % prop
+def _add_ns(prop, ver=1):
+    return '{http://ns.adobe.com/f4m/%d.0}%s' % (ver, prop)
+
+
+def get_base_url(manifest):
+    base_url = xpath_text(
+        manifest, [_add_ns('baseURL'), _add_ns('baseURL', 2)],
+        'base URL', default=None)
+    if base_url:
+        base_url = base_url.strip()
+    return base_url
 
 
 class F4mFD(FragmentFD):
@@ -330,13 +339,13 @@ class F4mFD(FragmentFD):
             rate, media = list(filter(
                 lambda f: int(f[0]) == requested_bitrate, formats))[0]
 
-        base_url = compat_urlparse.urljoin(man_url, media.attrib['url'])
+        # Prefer baseURL for relative URLs as per 11.2 of F4M 3.0 spec.
+        man_base_url = get_base_url(doc) or man_url
+
+        base_url = compat_urlparse.urljoin(man_base_url, media.attrib['url'])
         bootstrap_node = doc.find(_add_ns('bootstrapInfo'))
-        # From Adobe F4M 3.0 spec:
-        # The <baseURL> element SHALL be the base URL for all relative
-        # (HTTP-based) URLs in the manifest. If <baseURL> is not present, said
-        # URLs should be relative to the location of the containing document.
-        boot_info, bootstrap_url = self._parse_bootstrap_node(bootstrap_node, man_url)
+        boot_info, bootstrap_url = self._parse_bootstrap_node(
+            bootstrap_node, man_base_url)
         live = boot_info['live']
         metadata_node = media.find(_add_ns('metadata'))
         if metadata_node is not None:
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index a67ac44..64fb869 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -29,7 +29,10 @@ from ..compat import (
     compat_urlparse,
     compat_xml_parse_error,
 )
-from ..downloader.f4m import remove_encrypted_media
+from ..downloader.f4m import (
+    get_base_url,
+    remove_encrypted_media,
+)
 from ..utils import (
     NO_DEFAULT,
     age_restricted,
@@ -1239,11 +1242,8 @@ class InfoExtractor(object):
         media_nodes = remove_encrypted_media(media_nodes)
         if not media_nodes:
             return formats
-        base_url = xpath_text(
-            manifest, ['{http://ns.adobe.com/f4m/1.0}baseURL', '{http://ns.adobe.com/f4m/2.0}baseURL'],
-            'base URL', default=None)
-        if base_url:
-            base_url = base_url.strip()
+
+        manifest_base_url = get_base_url(manifest)
 
         bootstrap_info = xpath_element(
             manifest, ['{http://ns.adobe.com/f4m/1.0}bootstrapInfo', '{http://ns.adobe.com/f4m/2.0}bootstrapInfo'],
@@ -1275,7 +1275,7 @@ class InfoExtractor(object):
                     continue
                 manifest_url = (
                     media_url if media_url.startswith('http://') or media_url.startswith('https://')
-                    else ((base_url or '/'.join(manifest_url.split('/')[:-1])) + '/' + media_url))
+                    else ((manifest_base_url or '/'.join(manifest_url.split('/')[:-1])) + '/' + media_url))
                 # If media_url is itself a f4m manifest do the recursive extraction
                 # since bitrates in parent manifest (this one) and media_url manifest
                 # may differ leading to inability to resolve the format by requested
-- 
cgit v1.1


From 187ee66c941d9c397a46ffa490375e2c405500e5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 4 Nov 2017 22:11:39 +0700
Subject: [extractor/common] Add protocol for f4m formats

---
 youtube_dl/extractor/common.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 64fb869..e2d9f52 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1310,6 +1310,7 @@ class InfoExtractor(object):
                 'url': manifest_url,
                 'manifest_url': manifest_url,
                 'ext': 'flv' if bootstrap_info is not None else None,
+                'protocol': 'f4m',
                 'tbr': tbr,
                 'width': width,
                 'height': height,
-- 
cgit v1.1


From 181e381fda4ddb9083f3834a8bd1bab72c937545 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 4 Nov 2017 22:12:20 +0700
Subject: [test_InfoExtractor] Add test for #14660

---
 test/test_InfoExtractor.py            | 26 ++++++++++++++++++++++++++
 test/testdata/f4m/custom_base_url.f4m | 10 ++++++++++
 2 files changed, 36 insertions(+)
 create mode 100644 test/testdata/f4m/custom_base_url.f4m

diff --git a/test/test_InfoExtractor.py b/test/test_InfoExtractor.py
index f18a823..686c63e 100644
--- a/test/test_InfoExtractor.py
+++ b/test/test_InfoExtractor.py
@@ -574,6 +574,32 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
                 self.ie._sort_formats(formats)
                 expect_value(self, formats, expected_formats, None)
 
+    def test_parse_f4m_formats(self):
+        _TEST_CASES = [
+            (
+                # https://github.com/rg3/youtube-dl/issues/14660
+                'custom_base_url',
+                'http://api.new.livestream.com/accounts/6115179/events/6764928/videos/144884262.f4m',
+                [{
+                    'manifest_url': 'http://api.new.livestream.com/accounts/6115179/events/6764928/videos/144884262.f4m',
+                    'ext': 'flv',
+                    'format_id': '2148',
+                    'protocol': 'f4m',
+                    'tbr': 2148,
+                    'width': 1280,
+                    'height': 720,
+                }]
+            ),
+        ]
+
+        for f4m_file, f4m_url, expected_formats in _TEST_CASES:
+            with io.open('./test/testdata/f4m/%s.f4m' % f4m_file,
+                         mode='r', encoding='utf-8') as f:
+                formats = self.ie._parse_f4m_formats(
+                    compat_etree_fromstring(f.read().encode('utf-8')),
+                    f4m_url, None)
+                self.ie._sort_formats(formats)
+                expect_value(self, formats, expected_formats, None)
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/test/testdata/f4m/custom_base_url.f4m b/test/testdata/f4m/custom_base_url.f4m
new file mode 100644
index 0000000..74e1539
--- /dev/null
+++ b/test/testdata/f4m/custom_base_url.f4m
@@ -0,0 +1,10 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<manifest xmlns="http://ns.adobe.com/f4m/1.0">
+    <streamType>recorded</streamType>
+    <baseURL>http://vod.livestream.com/events/0000000000673980/</baseURL>
+    <duration>269.293</duration>
+    <bootstrapInfo profile="named" id="bootstrap_1">AAAAm2Fic3QAAAAAAAAAAQAAAAPoAAAAAAAEG+0AAAAAAAAAAAAAAAAAAQAAABlhc3J0AAAAAAAAAAABAAAAAQAAAC4BAAAAVmFmcnQAAAAAAAAD6AAAAAAEAAAAAQAAAAAAAAAAAAAXcAAAAC0AAAAAAAQHQAAAE5UAAAAuAAAAAAAEGtUAAAEYAAAAAAAAAAAAAAAAAAAAAAA=</bootstrapInfo>
+    <media url="b90f532f-b0f6-4f4e-8289-706d490b2fd8_2292" bootstrapInfoId="bootstrap_1" bitrate="2148" width="1280" height="720" videoCodec="avc1.4d401f" audioCodec="mp4a.40.2">
+        <metadata>AgAKb25NZXRhRGF0YQgAAAAIAAhkdXJhdGlvbgBAcNSwIMSbpgAFd2lkdGgAQJQAAAAAAAAABmhlaWdodABAhoAAAAAAAAAJZnJhbWVyYXRlAEA4/7DoLwW3AA12aWRlb2RhdGFyYXRlAECe1DLgjcobAAx2aWRlb2NvZGVjaWQAQBwAAAAAAAAADWF1ZGlvZGF0YXJhdGUAQGSimlvaPKQADGF1ZGlvY29kZWNpZABAJAAAAAAAAAAACQ==</metadata>
+    </media>
+</manifest>
-- 
cgit v1.1


From 6e71bbf4abc729cae3b0e428c3bb321690c9e485 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 5 Nov 2017 16:12:56 +0700
Subject: [hotstar] Bypass geo restriction (closes #14672)

---
 youtube_dl/extractor/hotstar.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/hotstar.py b/youtube_dl/extractor/hotstar.py
index 3a7a66a..9be958b 100644
--- a/youtube_dl/extractor/hotstar.py
+++ b/youtube_dl/extractor/hotstar.py
@@ -11,6 +11,7 @@ from ..utils import (
 
 class HotStarIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?hotstar\.com/(?:.+?[/-])?(?P<id>\d{10})'
+    _GEO_COUNTRIES = ['IN']
     _TESTS = [{
         'url': 'http://www.hotstar.com/on-air-with-aib--english-1000076273',
         'info_dict': {
-- 
cgit v1.1


From 477c97f86b5451f384a84a7a8d8237cfd1bec1d2 Mon Sep 17 00:00:00 2001
From: Alpesh Valia <alpeshvalia727@gmail.com>
Date: Thu, 16 Mar 2017 22:00:11 +0530
Subject: [hotstar:playlist] Add extractor

---
 youtube_dl/extractor/extractors.py |  5 +++-
 youtube_dl/extractor/hotstar.py    | 58 +++++++++++++++++++++++++++++++++++++-
 2 files changed, 61 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 92f7e90..d084707 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -432,7 +432,10 @@ from .hitbox import HitboxIE, HitboxLiveIE
 from .hitrecord import HitRecordIE
 from .hornbunny import HornBunnyIE
 from .hotnewhiphop import HotNewHipHopIE
-from .hotstar import HotStarIE
+from .hotstar import (
+    HotStarIE,
+    HotStarPlaylistIE,
+)
 from .howcast import HowcastIE
 from .howstuffworks import HowStuffWorksIE
 from .hrti import (
diff --git a/youtube_dl/extractor/hotstar.py b/youtube_dl/extractor/hotstar.py
index 9be958b..8d8a80a 100644
--- a/youtube_dl/extractor/hotstar.py
+++ b/youtube_dl/extractor/hotstar.py
@@ -7,6 +7,7 @@ from ..utils import (
     determine_ext,
     int_or_none,
 )
+import re
 
 
 class HotStarIE(InfoExtractor):
@@ -17,7 +18,7 @@ class HotStarIE(InfoExtractor):
         'info_dict': {
             'id': '1000076273',
             'ext': 'mp4',
-            'title': 'On Air With AIB - English',
+            'title': 'On Air With AIB',
             'description': 'md5:c957d8868e9bc793ccb813691cc4c434',
             'timestamp': 1447227000,
             'upload_date': '20151111',
@@ -100,3 +101,58 @@ class HotStarIE(InfoExtractor):
             'episode_number': int_or_none(video_data.get('episodeNumber')),
             'series': video_data.get('contentTitle'),
         }
+
+
+class HotStarPlaylistIE(InfoExtractor):
+    IE_NAME = 'hotstar:playlist'
+    _VALID_URL = r'https?://(?:www\.)?hotstar\.com/tv/(?P<playlist_title>.+)/(?P<series_id>\d+)/episodes/(?P<playlist_id>\d{1,})'
+
+    _TESTS = [{
+        'url': 'http://www.hotstar.com/tv/pow-bandi-yuddh-ke/10999/episodes/10856/9993',
+        'info_dict': {
+            'id': '10856',
+            'title': 'pow-bandi-yuddh-ke',
+        },
+        'playlist_mincount': 0,
+    }, {
+        'url': 'http://www.hotstar.com/tv/pow-bandi-yuddh-ke/10999/episodes/10856/9993',
+        'only_matching': True,
+    }]
+
+    def _extract_episode_info(self, series_id, playlist_title, video):
+
+        picture_url = video.get('urlPictures')
+        thumbnail = ''
+        if picture_url:
+            thumbnail = 'http://media0-starag.startv.in/r1/thumbs/PCTV/%s/%s/PCTV-%s-hs.jpg' % (picture_url[-2:], picture_url, picture_url)
+
+        episode_title = video.get('episodeTitle', '')
+        episode_title = episode_title.lower().replace(' ', '-')
+        url = "http://www.hotstar.com/tv/%s/%s/%s/%s" % (playlist_title, series_id, episode_title, video.get('contentId'))
+
+        info_dict = {
+            'id': video.get('contentId'),
+            'title': video.get('episodeTitle'),
+            'description': video.get('longDescription'),
+            'thumbnail': thumbnail,
+            'url': url,
+            '_type': 'url',
+        }
+        return info_dict
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        series_id = mobj.group('series_id')
+        playlist_id = mobj.group('playlist_id')
+        playlist_title = mobj.group('playlist_title')
+
+        collection = self._download_json(
+            "http://search.hotstar.com/AVS/besc?action=SearchContents&appVersion=5.0.39&channel=PCTV&moreFilters=series:%s;&query=*&searchOrder=last_broadcast_date+desc,year+asc,title+asc&type=EPISODE" % playlist_id,
+            playlist_id
+        )
+
+        videos = collection.get('resultObj', {}).get('response', {}).get('docs', [])
+        entries = [
+            self._extract_episode_info(series_id, playlist_title, video)
+            for video in videos if video.get('contentId')]
+        return self.playlist_result(entries, playlist_id, playlist_title)
-- 
cgit v1.1


From 909191de9154bf289b333cfe01b8e88e3ac1fefc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 5 Nov 2017 19:14:48 +0700
Subject: [hotstar:playlist] Fix issues and improve (closes #12465)

---
 youtube_dl/extractor/hotstar.py | 120 +++++++++++++++++++++-------------------
 1 file changed, 62 insertions(+), 58 deletions(-)

diff --git a/youtube_dl/extractor/hotstar.py b/youtube_dl/extractor/hotstar.py
index 8d8a80a..d28af36 100644
--- a/youtube_dl/extractor/hotstar.py
+++ b/youtube_dl/extractor/hotstar.py
@@ -1,18 +1,41 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
+from ..compat import compat_str
 from ..utils import (
-    ExtractorError,
     determine_ext,
+    ExtractorError,
     int_or_none,
 )
-import re
 
 
-class HotStarIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?hotstar\.com/(?:.+?[/-])?(?P<id>\d{10})'
+class HotStarBaseIE(InfoExtractor):
     _GEO_COUNTRIES = ['IN']
+
+    def _download_json(self, *args, **kwargs):
+        response = super(HotStarBaseIE, self)._download_json(*args, **kwargs)
+        if response['resultCode'] != 'OK':
+            if kwargs.get('fatal'):
+                raise ExtractorError(
+                    response['errorDescription'], expected=True)
+            return None
+        return response['resultObj']
+
+    def _download_content_info(self, content_id):
+        return self._download_json(
+            'https://account.hotstar.com/AVS/besc', content_id, query={
+                'action': 'GetAggregatedContentDetails',
+                'appVersion': '5.0.40',
+                'channel': 'PCTV',
+                'contentId': content_id,
+            })['contentInfo'][0]
+
+
+class HotStarIE(HotStarBaseIE):
+    _VALID_URL = r'https?://(?:www\.)?hotstar\.com/(?:.+?[/-])?(?P<id>\d{10})'
     _TESTS = [{
         'url': 'http://www.hotstar.com/on-air-with-aib--english-1000076273',
         'info_dict': {
@@ -36,23 +59,11 @@ class HotStarIE(InfoExtractor):
         'only_matching': True,
     }]
 
-    def _download_json(self, url_or_request, video_id, note='Downloading JSON metadata', fatal=True, query=None):
-        json_data = super(HotStarIE, self)._download_json(
-            url_or_request, video_id, note, fatal=fatal, query=query)
-        if json_data['resultCode'] != 'OK':
-            if fatal:
-                raise ExtractorError(json_data['errorDescription'])
-            return None
-        return json_data['resultObj']
-
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        video_data = self._download_json(
-            'http://account.hotstar.com/AVS/besc', video_id, query={
-                'action': 'GetAggregatedContentDetails',
-                'channel': 'PCTV',
-                'contentId': video_id,
-            })['contentInfo'][0]
+
+        video_data = self._download_content_info(video_id)
+
         title = video_data['episodeTitle']
 
         if video_data.get('encrypted') == 'Y':
@@ -103,56 +114,49 @@ class HotStarIE(InfoExtractor):
         }
 
 
-class HotStarPlaylistIE(InfoExtractor):
+class HotStarPlaylistIE(HotStarBaseIE):
     IE_NAME = 'hotstar:playlist'
-    _VALID_URL = r'https?://(?:www\.)?hotstar\.com/tv/(?P<playlist_title>.+)/(?P<series_id>\d+)/episodes/(?P<playlist_id>\d{1,})'
-
+    _VALID_URL = r'(?P<url>https?://(?:www\.)?hotstar\.com/tv/[^/]+/(?P<content_id>\d+))/(?P<type>[^/]+)/(?P<id>\d+)'
     _TESTS = [{
-        'url': 'http://www.hotstar.com/tv/pow-bandi-yuddh-ke/10999/episodes/10856/9993',
+        'url': 'http://www.hotstar.com/tv/pratidaan/14982/episodes/14812/9993',
         'info_dict': {
-            'id': '10856',
-            'title': 'pow-bandi-yuddh-ke',
+            'id': '14812',
         },
-        'playlist_mincount': 0,
+        'playlist_mincount': 75,
     }, {
-        'url': 'http://www.hotstar.com/tv/pow-bandi-yuddh-ke/10999/episodes/10856/9993',
+        'url': 'http://www.hotstar.com/tv/pratidaan/14982/popular-clips/9998/9998',
         'only_matching': True,
     }]
-
-    def _extract_episode_info(self, series_id, playlist_title, video):
-
-        picture_url = video.get('urlPictures')
-        thumbnail = ''
-        if picture_url:
-            thumbnail = 'http://media0-starag.startv.in/r1/thumbs/PCTV/%s/%s/PCTV-%s-hs.jpg' % (picture_url[-2:], picture_url, picture_url)
-
-        episode_title = video.get('episodeTitle', '')
-        episode_title = episode_title.lower().replace(' ', '-')
-        url = "http://www.hotstar.com/tv/%s/%s/%s/%s" % (playlist_title, series_id, episode_title, video.get('contentId'))
-
-        info_dict = {
-            'id': video.get('contentId'),
-            'title': video.get('episodeTitle'),
-            'description': video.get('longDescription'),
-            'thumbnail': thumbnail,
-            'url': url,
-            '_type': 'url',
-        }
-        return info_dict
+    _ITEM_TYPES = {
+        'episodes': 'EPISODE',
+        'popular-clips': 'CLIPS',
+    }
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
-        series_id = mobj.group('series_id')
-        playlist_id = mobj.group('playlist_id')
-        playlist_title = mobj.group('playlist_title')
+        base_url = mobj.group('url')
+        content_id = mobj.group('content_id')
+        playlist_type = mobj.group('type')
+
+        content_info = self._download_content_info(content_id)
+        playlist_id = compat_str(content_info['categoryId'])
 
         collection = self._download_json(
-            "http://search.hotstar.com/AVS/besc?action=SearchContents&appVersion=5.0.39&channel=PCTV&moreFilters=series:%s;&query=*&searchOrder=last_broadcast_date+desc,year+asc,title+asc&type=EPISODE" % playlist_id,
-            playlist_id
-        )
+            'https://search.hotstar.com/AVS/besc', playlist_id, query={
+                'action': 'SearchContents',
+                'appVersion': '5.0.40',
+                'channel': 'PCTV',
+                'moreFilters': 'series:%s;' % playlist_id,
+                'query': '*',
+                'searchOrder': 'last_broadcast_date desc,year desc,title asc',
+                'type': self._ITEM_TYPES.get(playlist_type, 'EPISODE'),
+            })
 
-        videos = collection.get('resultObj', {}).get('response', {}).get('docs', [])
         entries = [
-            self._extract_episode_info(series_id, playlist_title, video)
-            for video in videos if video.get('contentId')]
-        return self.playlist_result(entries, playlist_id, playlist_title)
+            self.url_result(
+                '%s/_/%s' % (base_url, video['contentId']),
+                ie=HotStarIE.ie_key(), video_id=video['contentId'])
+            for video in collection['response']['docs']
+            if video.get('contentId')]
+
+        return self.playlist_result(entries, playlist_id)
-- 
cgit v1.1


From e0998333fac2238eff8880992c11f76402c4007c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 6 Nov 2017 22:36:46 +0700
Subject: [ChangeLog] Actualize

---
 ChangeLog | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index d33a710..3cbbdda 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,19 @@
+version <unreleased>
+
+Core
++ [extractor/common] Add protocol for f4m formats
+* [f4m] Prefer baseURL for relative URLs (#14660)
+* [extractor/common] Respect URL query in _extract_wowza_formats (14645)
+
+Extractors
++ [hotstar:playlist] Add support for playlists (#12465)
+* [hotstar] Bypass geo restriction (#14672)
+- [22tracks] Remove extractor (#11024, #14628)
++ [skysport] Sdd support ooyala videos protected with embed_token (#14641)
+* [gamespot] Extract formats referenced with new data fields (#14652)
+* [spankbang] Detect unavailable videos (#14644)
+
+
 version 2017.10.29
 
 Core
-- 
cgit v1.1


From f34b841b51be6872914ffe17b210c54b0d823c3c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 6 Nov 2017 22:39:24 +0700
Subject: release 2017.11.06

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 docs/supportedsites.md    | 3 +--
 youtube_dl/version.py     | 2 +-
 4 files changed, 6 insertions(+), 7 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 8814758..be6e6dd 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.10.29*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.10.29**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.11.06*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.11.06**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through the [README](https://github.com/rg3/youtube-dl/blob/master/README.md), **most notably** the [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ Add the `-v` flag to **your command line** you run youtube-dl with (`youtube-dl
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.10.29
+[debug] youtube-dl version 2017.11.06
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index 3cbbdda..8af3682 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.11.06
 
 Core
 + [extractor/common] Add protocol for f4m formats
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 7b8e740..6009df5 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -3,8 +3,6 @@
  - **1up.com**
  - **20min**
  - **220.ro**
- - **22tracks:genre**
- - **22tracks:track**
  - **24video**
  - **3qsdn**: 3Q SDN
  - **3sat**
@@ -342,6 +340,7 @@
  - **HornBunny**
  - **HotNewHipHop**
  - **HotStar**
+ - **hotstar:playlist**
  - **Howcast**
  - **HowStuffWorks**
  - **HRTi**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 43f080b..8b67d23 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.10.29'
+__version__ = '2017.11.06'
-- 
cgit v1.1


From cc6a960e134614f8af2a42dcd8bf146d63638a3c Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 8 Nov 2017 20:30:05 +0100
Subject: use older login method(closes #11572)

---
 youtube_dl/extractor/crunchyroll.py | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index 8bdaf0c..18ef3da 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -43,6 +43,17 @@ class CrunchyrollBaseIE(InfoExtractor):
         if username is None:
             return
 
+        self._download_webpage(
+            'https://www.crunchyroll.com/?a=formhandler',
+            None, 'Logging in', 'Wrong login info',
+            data=urlencode_postdata({
+                'formname': 'RpcApiUser_Login',
+                'next_url': 'https://www.crunchyroll.com/acct/membership',
+                'name': username,
+                'password': password,
+            }))
+
+        '''
         login_page = self._download_webpage(
             self._LOGIN_URL, None, 'Downloading login page')
 
@@ -86,6 +97,7 @@ class CrunchyrollBaseIE(InfoExtractor):
             raise ExtractorError('Unable to login: %s' % error, expected=True)
 
         raise ExtractorError('Unable to log in')
+        '''
 
     def _real_initialize(self):
         self._login()
-- 
cgit v1.1


From 4222346fb2f42af10ac902cd46469d23923cf114 Mon Sep 17 00:00:00 2001
From: hcwhan <hcwhan@gmail.com>
Date: Tue, 7 Nov 2017 17:59:09 +0800
Subject: [pandatv] Update API URL and sign format URLs

---
 youtube_dl/extractor/pandatv.py | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/pandatv.py b/youtube_dl/extractor/pandatv.py
index c86d707..c99a1bb 100644
--- a/youtube_dl/extractor/pandatv.py
+++ b/youtube_dl/extractor/pandatv.py
@@ -6,6 +6,7 @@ from ..utils import (
     ExtractorError,
     qualities,
 )
+import json
 
 
 class PandaTVIE(InfoExtractor):
@@ -33,7 +34,7 @@ class PandaTVIE(InfoExtractor):
         video_id = self._match_id(url)
 
         config = self._download_json(
-            'https://www.panda.tv/api_room?roomid=%s' % video_id, video_id)
+            'https://www.panda.tv/api_room_v2?roomid=%s' % video_id, video_id)
 
         error_code = config.get('errno', 0)
         if error_code is not 0:
@@ -66,6 +67,11 @@ class PandaTVIE(InfoExtractor):
             plflag1 = '4'
         live_panda = 'live_panda' if plflag0 < 1 else ''
 
+        plflag_auth = json.loads(video_info["plflag_list"])
+        sign = plflag_auth["auth"]["sign"]
+        ts = plflag_auth["auth"]["time"]
+        rid = plflag_auth["auth"]["rid"]
+
         quality_key = qualities(['OD', 'HD', 'SD'])
         suffix = ['_small', '_mid', '']
         formats = []
@@ -77,8 +83,8 @@ class PandaTVIE(InfoExtractor):
                 continue
             for pref, (ext, pl) in enumerate((('m3u8', '-hls'), ('flv', ''))):
                 formats.append({
-                    'url': 'https://pl%s%s.live.panda.tv/live_panda/%s%s%s.%s'
-                    % (pl, plflag1, room_key, live_panda, suffix[quality], ext),
+                    'url': 'https://pl%s%s.live.panda.tv/live_panda/%s%s%s.%s?sign=%s&ts=%s&rid=%s'
+                    % (pl, plflag1, room_key, live_panda, suffix[quality], ext, sign, ts, rid),
                     'format_id': '%s-%s' % (k, ext),
                     'quality': quality,
                     'source_preference': pref,
-- 
cgit v1.1


From 61fb07e156671159353ae19a152926cab277ac87 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 9 Nov 2017 23:25:43 +0700
Subject: [pandatv] Modernize (closes #14693)

---
 youtube_dl/extractor/pandatv.py | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/pandatv.py b/youtube_dl/extractor/pandatv.py
index c99a1bb..13a2e7e 100644
--- a/youtube_dl/extractor/pandatv.py
+++ b/youtube_dl/extractor/pandatv.py
@@ -6,7 +6,6 @@ from ..utils import (
     ExtractorError,
     qualities,
 )
-import json
 
 
 class PandaTVIE(InfoExtractor):
@@ -67,10 +66,10 @@ class PandaTVIE(InfoExtractor):
             plflag1 = '4'
         live_panda = 'live_panda' if plflag0 < 1 else ''
 
-        plflag_auth = json.loads(video_info["plflag_list"])
-        sign = plflag_auth["auth"]["sign"]
-        ts = plflag_auth["auth"]["time"]
-        rid = plflag_auth["auth"]["rid"]
+        plflag_auth = self._parse_json(video_info['plflag_list'], video_id)
+        sign = plflag_auth['auth']['sign']
+        ts = plflag_auth['auth']['time']
+        rid = plflag_auth['auth']['rid']
 
         quality_key = qualities(['OD', 'HD', 'SD'])
         suffix = ['_small', '_mid', '']
-- 
cgit v1.1


From a9543e37c8e460e69a8556c8e5004ebd8e9b4da4 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 11 Nov 2017 00:29:08 +0800
Subject: [wsj] Recognize another URL pattern (closes #14704)

---
 ChangeLog                   | 6 ++++++
 youtube_dl/extractor/wsj.py | 5 ++++-
 2 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/ChangeLog b/ChangeLog
index 8af3682..cedab47 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,9 @@
+version <unreleased>
+
+Extractors
++ [wsj] Recognize another URL pattern (#14704)
+
+
 version 2017.11.06
 
 Core
diff --git a/youtube_dl/extractor/wsj.py b/youtube_dl/extractor/wsj.py
index 9b54877..67236f3 100644
--- a/youtube_dl/extractor/wsj.py
+++ b/youtube_dl/extractor/wsj.py
@@ -13,7 +13,7 @@ class WSJIE(InfoExtractor):
     _VALID_URL = r'''(?x)
                         (?:
                             https?://video-api\.wsj\.com/api-video/player/iframe\.html\?.*?\bguid=|
-                            https?://(?:www\.)?(?:wsj|barrons)\.com/video/[^/]+/|
+                            https?://(?:www\.)?(?:wsj|barrons)\.com/video/(?:[^/]+/)+|
                             wsj:
                         )
                         (?P<id>[a-fA-F0-9-]{36})
@@ -38,6 +38,9 @@ class WSJIE(InfoExtractor):
     }, {
         'url': 'http://www.barrons.com/video/capitalism-deserves-more-respect-from-millennials/F301217E-6F46-43AE-B8D2-B7180D642EE9.html',
         'only_matching': True,
+    }, {
+        'url': 'https://www.wsj.com/video/series/a-brief-history-of/the-modern-cell-carrier-how-we-got-here/980E2187-401D-48A1-B82B-1486CEE06CB9',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
-- 
cgit v1.1


From 59d2e6d04f621f41a72a232b8c93250991b4ae5c Mon Sep 17 00:00:00 2001
From: gkoelln <gkoelln7@gmail.com>
Date: Fri, 10 Nov 2017 15:59:48 -0600
Subject: [cartoonnetwork] Update tokenizer_src (closes #14666)

---
 youtube_dl/extractor/cartoonnetwork.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/cartoonnetwork.py b/youtube_dl/extractor/cartoonnetwork.py
index 086ec90..6aeebd7 100644
--- a/youtube_dl/extractor/cartoonnetwork.py
+++ b/youtube_dl/extractor/cartoonnetwork.py
@@ -31,7 +31,7 @@ class CartoonNetworkIE(TurnerBaseIE):
             'http://www.cartoonnetwork.com/video-seo-svc/episodeservices/getCvpPlaylist?networkName=CN2&' + query, video_id, {
                 'secure': {
                     'media_src': 'http://androidhls-secure.cdn.turner.com/toon/big',
-                    'tokenizer_src': 'http://www.cartoonnetwork.com/cntv/mvpd/processors/services/token_ipadAdobe.do',
+                    'tokenizer_src': 'https://token.vgtf.net/token/token_mobile',
                 },
             }, {
                 'url': url,
-- 
cgit v1.1


From a5203935d6cb753bafaf67164553027b62c01781 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sat, 11 Nov 2017 12:41:15 +0100
Subject: [gamespot] skip Brightcove Once http formats(#14652)

---
 youtube_dl/extractor/gamespot.py | 3 ++-
 youtube_dl/extractor/once.py     | 4 ++--
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/gamespot.py b/youtube_dl/extractor/gamespot.py
index 6d177cb..be1ed8b 100644
--- a/youtube_dl/extractor/gamespot.py
+++ b/youtube_dl/extractor/gamespot.py
@@ -108,7 +108,8 @@ class GameSpotIE(OnceIE):
             onceux_url = self._parse_json(unescapeHTML(onceux_json), page_id).get('metadataUri')
             if onceux_url:
                 formats.extend(self._extract_once_formats(re.sub(
-                    r'https?://[^/]+', 'http://once.unicornmedia.com', onceux_url)))
+                    r'https?://[^/]+', 'http://once.unicornmedia.com', onceux_url),
+                    skip_http_formats=True))
 
         if not formats:
             for quality in ['sd', 'hd']:
diff --git a/youtube_dl/extractor/once.py b/youtube_dl/extractor/once.py
index a637c8e..6ba6fe5 100644
--- a/youtube_dl/extractor/once.py
+++ b/youtube_dl/extractor/once.py
@@ -11,7 +11,7 @@ class OnceIE(InfoExtractor):
     ADAPTIVE_URL_TEMPLATE = 'http://once.unicornmedia.com/now/master/playlist/%s/%s/%s/content.m3u8'
     PROGRESSIVE_URL_TEMPLATE = 'http://once.unicornmedia.com/now/media/progressive/%s/%s/%s/%s/content.mp4'
 
-    def _extract_once_formats(self, url):
+    def _extract_once_formats(self, url, skip_http_formats=False):
         domain_id, application_id, media_item_id = re.match(
             OnceIE._VALID_URL, url).groups()
         formats = self._extract_m3u8_formats(
@@ -27,7 +27,7 @@ class OnceIE(InfoExtractor):
             rendition_id = self._search_regex(
                 r'/now/media/playlist/[^/]+/[^/]+/([^/]+)',
                 adaptive_format['url'], 'redition id', default=None)
-            if rendition_id:
+            if rendition_id and not skip_http_formats:
                 progressive_format = adaptive_format.copy()
                 progressive_format.update({
                     'url': self.PROGRESSIVE_URL_TEMPLATE % (
-- 
cgit v1.1


From 79d1f8ed6803b6097f0f3cd57f72e0378bdc1f34 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sat, 11 Nov 2017 13:02:39 +0100
Subject: [gamespot] add support for article URLS(closes #14652)

---
 youtube_dl/extractor/gamespot.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/gamespot.py b/youtube_dl/extractor/gamespot.py
index be1ed8b..e6d6d9b 100644
--- a/youtube_dl/extractor/gamespot.py
+++ b/youtube_dl/extractor/gamespot.py
@@ -14,7 +14,7 @@ from ..utils import (
 
 
 class GameSpotIE(OnceIE):
-    _VALID_URL = r'https?://(?:www\.)?gamespot\.com/videos/(?:[^/]+/\d+-|embed/)(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?gamespot\.com/(?:video|article)s/(?:[^/]+/\d+-|embed/)(?P<id>\d+)'
     _TESTS = [{
         'url': 'http://www.gamespot.com/videos/arma-3-community-guide-sitrep-i/2300-6410818/',
         'md5': 'b2a30deaa8654fcccd43713a6b6a4825',
-- 
cgit v1.1


From e4d9586562d24cbbea6ee07162290ec602399f37 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 11 Nov 2017 20:49:03 +0700
Subject: Remove sensitive data from logging in messages

---
 youtube_dl/extractor/animeondemand.py | 2 +-
 youtube_dl/extractor/atresplayer.py   | 2 +-
 youtube_dl/extractor/bambuser.py      | 2 +-
 youtube_dl/extractor/dramafever.py    | 2 +-
 youtube_dl/extractor/funimation.py    | 2 +-
 youtube_dl/extractor/noco.py          | 2 +-
 youtube_dl/extractor/patreon.py       | 2 +-
 youtube_dl/extractor/pluralsight.py   | 2 +-
 youtube_dl/extractor/roosterteeth.py  | 2 +-
 youtube_dl/extractor/safari.py        | 2 +-
 youtube_dl/extractor/twitch.py        | 2 +-
 youtube_dl/extractor/udemy.py         | 2 +-
 youtube_dl/extractor/viki.py          | 2 +-
 youtube_dl/extractor/vk.py            | 2 +-
 14 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/animeondemand.py b/youtube_dl/extractor/animeondemand.py
index 69d3633..34c2b36 100644
--- a/youtube_dl/extractor/animeondemand.py
+++ b/youtube_dl/extractor/animeondemand.py
@@ -78,7 +78,7 @@ class AnimeOnDemandIE(InfoExtractor):
             post_url = urljoin(self._LOGIN_URL, post_url)
 
         response = self._download_webpage(
-            post_url, None, 'Logging in as %s' % username,
+            post_url, None, 'Logging in',
             data=urlencode_postdata(login_form), headers={
                 'Referer': self._LOGIN_URL,
             })
diff --git a/youtube_dl/extractor/atresplayer.py b/youtube_dl/extractor/atresplayer.py
index 01fa308..1a31ebe 100644
--- a/youtube_dl/extractor/atresplayer.py
+++ b/youtube_dl/extractor/atresplayer.py
@@ -87,7 +87,7 @@ class AtresPlayerIE(InfoExtractor):
             self._LOGIN_URL, urlencode_postdata(login_form))
         request.add_header('Content-Type', 'application/x-www-form-urlencoded')
         response = self._download_webpage(
-            request, None, 'Logging in as %s' % username)
+            request, None, 'Logging in')
 
         error = self._html_search_regex(
             r'(?s)<ul[^>]+class="[^"]*\blist_error\b[^"]*">(.+?)</ul>',
diff --git a/youtube_dl/extractor/bambuser.py b/youtube_dl/extractor/bambuser.py
index 0eb1930..633c575 100644
--- a/youtube_dl/extractor/bambuser.py
+++ b/youtube_dl/extractor/bambuser.py
@@ -59,7 +59,7 @@ class BambuserIE(InfoExtractor):
             self._LOGIN_URL, urlencode_postdata(login_form))
         request.add_header('Referer', self._LOGIN_URL)
         response = self._download_webpage(
-            request, None, 'Logging in as %s' % username)
+            request, None, 'Logging in')
 
         login_error = self._html_search_regex(
             r'(?s)<div class="messages error">(.+?)</div>',
diff --git a/youtube_dl/extractor/dramafever.py b/youtube_dl/extractor/dramafever.py
index 95883a0..6b60e54 100644
--- a/youtube_dl/extractor/dramafever.py
+++ b/youtube_dl/extractor/dramafever.py
@@ -54,7 +54,7 @@ class DramaFeverBaseIE(AMPIE):
         request = sanitized_Request(
             self._LOGIN_URL, urlencode_postdata(login_form))
         response = self._download_webpage(
-            request, None, 'Logging in as %s' % username)
+            request, None, 'Logging in')
 
         if all(logout_pattern not in response
                for logout_pattern in ['href="/accounts/logout/"', '>Log out<']):
diff --git a/youtube_dl/extractor/funimation.py b/youtube_dl/extractor/funimation.py
index 8c37509..107f658 100644
--- a/youtube_dl/extractor/funimation.py
+++ b/youtube_dl/extractor/funimation.py
@@ -57,7 +57,7 @@ class FunimationIE(InfoExtractor):
         try:
             data = self._download_json(
                 'https://prod-api-funimationnow.dadcdigital.com/api/auth/login/',
-                None, 'Logging in as %s' % username, data=urlencode_postdata({
+                None, 'Logging in', data=urlencode_postdata({
                     'username': username,
                     'password': password,
                 }))
diff --git a/youtube_dl/extractor/noco.py b/youtube_dl/extractor/noco.py
index 8b83e1f..a9f9b10 100644
--- a/youtube_dl/extractor/noco.py
+++ b/youtube_dl/extractor/noco.py
@@ -70,7 +70,7 @@ class NocoIE(InfoExtractor):
             return
 
         login = self._download_json(
-            self._LOGIN_URL, None, 'Logging in as %s' % username,
+            self._LOGIN_URL, None, 'Logging in',
             data=urlencode_postdata({
                 'a': 'login',
                 'cookie': '1',
diff --git a/youtube_dl/extractor/patreon.py b/youtube_dl/extractor/patreon.py
index a6a2c27..d4b1d34 100644
--- a/youtube_dl/extractor/patreon.py
+++ b/youtube_dl/extractor/patreon.py
@@ -67,7 +67,7 @@ class PatreonIE(InfoExtractor):
             'https://www.patreon.com/processLogin',
             compat_urllib_parse_urlencode(login_form).encode('utf-8')
         )
-        login_page = self._download_webpage(request, None, note='Logging in as %s' % username)
+        login_page = self._download_webpage(request, None, note='Logging in')
 
         if re.search(r'onLoginFailed', login_page):
             raise ExtractorError('Unable to login, incorrect username and/or password', expected=True)
diff --git a/youtube_dl/extractor/pluralsight.py b/youtube_dl/extractor/pluralsight.py
index f6a9131..4bf0aa7 100644
--- a/youtube_dl/extractor/pluralsight.py
+++ b/youtube_dl/extractor/pluralsight.py
@@ -116,7 +116,7 @@ class PluralsightIE(PluralsightBaseIE):
             post_url = compat_urlparse.urljoin(self._LOGIN_URL, post_url)
 
         response = self._download_webpage(
-            post_url, None, 'Logging in as %s' % username,
+            post_url, None, 'Logging in',
             data=urlencode_postdata(login_form),
             headers={'Content-Type': 'application/x-www-form-urlencoded'})
 
diff --git a/youtube_dl/extractor/roosterteeth.py b/youtube_dl/extractor/roosterteeth.py
index 46dfc78..8b70380 100644
--- a/youtube_dl/extractor/roosterteeth.py
+++ b/youtube_dl/extractor/roosterteeth.py
@@ -68,7 +68,7 @@ class RoosterTeethIE(InfoExtractor):
 
         login_request = self._download_webpage(
             self._LOGIN_URL, None,
-            note='Logging in as %s' % username,
+            note='Logging in',
             data=urlencode_postdata(login_form),
             headers={
                 'Referer': self._LOGIN_URL,
diff --git a/youtube_dl/extractor/safari.py b/youtube_dl/extractor/safari.py
index 909a6ba..cc6698f 100644
--- a/youtube_dl/extractor/safari.py
+++ b/youtube_dl/extractor/safari.py
@@ -61,7 +61,7 @@ class SafariBaseIE(InfoExtractor):
         request = sanitized_Request(
             self._LOGIN_URL, urlencode_postdata(login_form), headers=headers)
         login_page = self._download_webpage(
-            request, None, 'Logging in as %s' % username)
+            request, None, 'Logging in')
 
         if not is_logged(login_page):
             raise ExtractorError(
diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index fefcd28..bf57eac 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -101,7 +101,7 @@ class TwitchBaseIE(InfoExtractor):
             fail(clean_html(login_page))
 
         redirect_page, handle = login_step(
-            login_page, handle, 'Logging in as %s' % username, {
+            login_page, handle, 'Logging in', {
                 'username': username,
                 'password': password,
             })
diff --git a/youtube_dl/extractor/udemy.py b/youtube_dl/extractor/udemy.py
index 207c4a6..c248ea7 100644
--- a/youtube_dl/extractor/udemy.py
+++ b/youtube_dl/extractor/udemy.py
@@ -164,7 +164,7 @@ class UdemyIE(InfoExtractor):
         })
 
         response = self._download_webpage(
-            self._LOGIN_URL, None, 'Logging in as %s' % username,
+            self._LOGIN_URL, None, 'Logging in',
             data=urlencode_postdata(login_form),
             headers={
                 'Referer': self._ORIGIN_URL,
diff --git a/youtube_dl/extractor/viki.py b/youtube_dl/extractor/viki.py
index 853e5c7..ad2a2a4 100644
--- a/youtube_dl/extractor/viki.py
+++ b/youtube_dl/extractor/viki.py
@@ -99,7 +99,7 @@ class VikiBaseIE(InfoExtractor):
 
         login = self._call_api(
             'sessions.json', None,
-            'Logging in as %s' % username, post_data=login_form)
+            'Logging in', post_data=login_form)
 
         self._token = login.get('token')
         if not self._token:
diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index 105e172..0d83765 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -67,7 +67,7 @@ class VKBaseIE(InfoExtractor):
 
         login_page = self._download_webpage(
             'https://login.vk.com/?act=login', None,
-            note='Logging in as %s' % username,
+            note='Logging in',
             data=urlencode_postdata(login_form))
 
         if re.search(r'onLoginFailed', login_page):
-- 
cgit v1.1


From af85ce29c61749676ab934a2b297505ab33bf4c7 Mon Sep 17 00:00:00 2001
From: Bob Poekert <bob@poekert.com>
Date: Sat, 11 Nov 2017 22:25:21 -0800
Subject: [ccma] Fix typo

---
 youtube_dl/extractor/ccma.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ccma.py b/youtube_dl/extractor/ccma.py
index 39938c9..bec0a82 100644
--- a/youtube_dl/extractor/ccma.py
+++ b/youtube_dl/extractor/ccma.py
@@ -93,7 +93,7 @@ class CCMAIE(InfoExtractor):
             'description': clean_html(informacio.get('descripcio')),
             'duration': duration,
             'timestamp': timestamp,
-            'thumnails': thumbnails,
+            'thumbnails': thumbnails,
             'subtitles': subtitles,
             'formats': formats,
         }
-- 
cgit v1.1


From 5fc12b954971f5f63d1e87b05e8b01a9ae0e3b01 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 12 Nov 2017 18:35:17 +0700
Subject: [instagram:user] Fix extraction (closes #14699)

---
 youtube_dl/extractor/instagram.py | 127 +++++++++++++++++++-------------------
 1 file changed, 62 insertions(+), 65 deletions(-)

diff --git a/youtube_dl/extractor/instagram.py b/youtube_dl/extractor/instagram.py
index 4667335..20db31f 100644
--- a/youtube_dl/extractor/instagram.py
+++ b/youtube_dl/extractor/instagram.py
@@ -1,5 +1,6 @@
 from __future__ import unicode_literals
 
+import itertools
 import re
 
 from .common import InfoExtractor
@@ -7,7 +8,6 @@ from ..compat import compat_str
 from ..utils import (
     get_element_by_attribute,
     int_or_none,
-    limit_length,
     lowercase_escape,
     try_get,
 )
@@ -212,7 +212,7 @@ class InstagramIE(InfoExtractor):
 
 
 class InstagramUserIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?instagram\.com/(?P<username>[^/]{2,})/?(?:$|[?#])'
+    _VALID_URL = r'https?://(?:www\.)?instagram\.com/(?P<id>[^/]{2,})/?(?:$|[?#])'
     IE_DESC = 'Instagram user profile'
     IE_NAME = 'instagram:user'
     _TEST = {
@@ -221,82 +221,79 @@ class InstagramUserIE(InfoExtractor):
             'id': 'porsche',
             'title': 'porsche',
         },
-        'playlist_mincount': 2,
-        'playlist': [{
-            'info_dict': {
-                'id': '614605558512799803_462752227',
-                'ext': 'mp4',
-                'title': '#Porsche Intelligent Performance.',
-                'thumbnail': r're:^https?://.*\.jpg',
-                'uploader': 'Porsche',
-                'uploader_id': 'porsche',
-                'timestamp': 1387486713,
-                'upload_date': '20131219',
-            },
-        }],
+        'playlist_count': 5,
         'params': {
             'extract_flat': True,
             'skip_download': True,
+            'playlistend': 5,
         }
     }
 
-    def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        uploader_id = mobj.group('username')
+    def _entries(self, uploader_id):
+        query = {
+            '__a': 1,
+        }
+
+        def get_count(kind):
+            return int_or_none(try_get(
+                node, lambda x: x['%ss' % kind]['count']))
 
-        entries = []
-        page_count = 0
-        media_url = 'http://instagram.com/%s/media' % uploader_id
-        while True:
+        for page_num in itertools.count(1):
             page = self._download_json(
-                media_url, uploader_id,
-                note='Downloading page %d ' % (page_count + 1),
-            )
-            page_count += 1
+                'https://instagram.com/%s/' % uploader_id, uploader_id,
+                note='Downloading page %d' % page_num,
+                fatal=False, query=query)
+            if not page:
+                break
+
+            nodes = try_get(page, lambda x: x['user']['media']['nodes'], list)
+            if not nodes:
+                break
 
-            for it in page['items']:
-                if it.get('type') != 'video':
+            max_id = None
+
+            for node in nodes:
+                node_id = node.get('id')
+                if node_id:
+                    max_id = node_id
+
+                if node.get('__typename') != 'GraphVideo' and node.get('is_video') is not True:
+                    continue
+                video_id = node.get('code')
+                if not video_id:
                     continue
-                like_count = int_or_none(it.get('likes', {}).get('count'))
-                user = it.get('user', {})
-
-                formats = [{
-                    'format_id': k,
-                    'height': v.get('height'),
-                    'width': v.get('width'),
-                    'url': v['url'],
-                } for k, v in it['videos'].items()]
-                self._sort_formats(formats)
-
-                thumbnails_el = it.get('images', {})
-                thumbnail = thumbnails_el.get('thumbnail', {}).get('url')
-
-                # In some cases caption is null, which corresponds to None
-                # in python. As a result, it.get('caption', {}) gives None
-                title = (it.get('caption') or {}).get('text', it['id'])
-
-                entries.append({
-                    'id': it['id'],
-                    'title': limit_length(title, 80),
-                    'formats': formats,
+
+                info = self.url_result(
+                    'https://instagram.com/p/%s/' % video_id,
+                    ie=InstagramIE.ie_key(), video_id=video_id)
+
+                description = try_get(
+                    node, [lambda x: x['caption'], lambda x: x['text']['id']],
+                    compat_str)
+                thumbnail = node.get('thumbnail_src') or node.get('display_src')
+                timestamp = int_or_none(node.get('date'))
+
+                comment_count = get_count('comment')
+                like_count = get_count('like')
+                view_count = int_or_none(node.get('video_views'))
+
+                info.update({
+                    'description': description,
                     'thumbnail': thumbnail,
-                    'webpage_url': it.get('link'),
-                    'uploader': user.get('full_name'),
-                    'uploader_id': user.get('username'),
+                    'timestamp': timestamp,
+                    'comment_count': comment_count,
                     'like_count': like_count,
-                    'timestamp': int_or_none(it.get('created_time')),
+                    'view_count': view_count,
                 })
 
-            if not page['items']:
+                yield info
+
+            if not max_id:
                 break
-            max_id = page['items'][-1]['id'].split('_')[0]
-            media_url = (
-                'http://instagram.com/%s/media?max_id=%s' % (
-                    uploader_id, max_id))
 
-        return {
-            '_type': 'playlist',
-            'entries': entries,
-            'id': uploader_id,
-            'title': uploader_id,
-        }
+            query['max_id'] = max_id
+
+    def _real_extract(self, url):
+        uploader_id = self._match_id(url)
+        return self.playlist_result(
+            self._entries(uploader_id), uploader_id, uploader_id)
-- 
cgit v1.1


From d4e31b72b971172ffdee7fbe3070d20e4454259c Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 13 Nov 2017 10:24:35 +0100
Subject: [gamespot] lower the preference of http formats(#14652)

---
 youtube_dl/extractor/gamespot.py | 2 +-
 youtube_dl/extractor/once.py     | 5 +++--
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/gamespot.py b/youtube_dl/extractor/gamespot.py
index e6d6d9b..a9606a0 100644
--- a/youtube_dl/extractor/gamespot.py
+++ b/youtube_dl/extractor/gamespot.py
@@ -109,7 +109,7 @@ class GameSpotIE(OnceIE):
             if onceux_url:
                 formats.extend(self._extract_once_formats(re.sub(
                     r'https?://[^/]+', 'http://once.unicornmedia.com', onceux_url),
-                    skip_http_formats=True))
+                    http_formats_preference=-1))
 
         if not formats:
             for quality in ['sd', 'hd']:
diff --git a/youtube_dl/extractor/once.py b/youtube_dl/extractor/once.py
index 6ba6fe5..8ae5fad 100644
--- a/youtube_dl/extractor/once.py
+++ b/youtube_dl/extractor/once.py
@@ -11,7 +11,7 @@ class OnceIE(InfoExtractor):
     ADAPTIVE_URL_TEMPLATE = 'http://once.unicornmedia.com/now/master/playlist/%s/%s/%s/content.m3u8'
     PROGRESSIVE_URL_TEMPLATE = 'http://once.unicornmedia.com/now/media/progressive/%s/%s/%s/%s/content.mp4'
 
-    def _extract_once_formats(self, url, skip_http_formats=False):
+    def _extract_once_formats(self, url, http_formats_preference=None):
         domain_id, application_id, media_item_id = re.match(
             OnceIE._VALID_URL, url).groups()
         formats = self._extract_m3u8_formats(
@@ -27,7 +27,7 @@ class OnceIE(InfoExtractor):
             rendition_id = self._search_regex(
                 r'/now/media/playlist/[^/]+/[^/]+/([^/]+)',
                 adaptive_format['url'], 'redition id', default=None)
-            if rendition_id and not skip_http_formats:
+            if rendition_id:
                 progressive_format = adaptive_format.copy()
                 progressive_format.update({
                     'url': self.PROGRESSIVE_URL_TEMPLATE % (
@@ -35,6 +35,7 @@ class OnceIE(InfoExtractor):
                     'format_id': adaptive_format['format_id'].replace(
                         'hls', 'http'),
                     'protocol': 'http',
+                    'preference': http_formats_preference,
                 })
                 progressive_formats.append(progressive_format)
         self._check_formats(progressive_formats, media_item_id)
-- 
cgit v1.1


From 388beb86e0c8e3f76958aa8a258bd396b8b1e0fe Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 13 Nov 2017 10:30:12 +0100
Subject: [gamespot] add test for #14652

---
 youtube_dl/extractor/gamespot.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/gamespot.py b/youtube_dl/extractor/gamespot.py
index a9606a0..ab647dd 100644
--- a/youtube_dl/extractor/gamespot.py
+++ b/youtube_dl/extractor/gamespot.py
@@ -38,6 +38,9 @@ class GameSpotIE(OnceIE):
     }, {
         'url': 'https://www.gamespot.com/videos/embed/6439218/',
         'only_matching': True,
+    }, {
+        'url': 'https://www.gamespot.com/articles/the-last-of-us-2-receives-new-ps4-trailer/1100-6454469/',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
-- 
cgit v1.1


From 27adc9ec65be412e07f6e55d9d9b56c1c224d1db Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 13 Nov 2017 11:24:15 +0100
Subject: [tva] fix extraction(closes #14736)

---
 youtube_dl/extractor/tva.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/tva.py b/youtube_dl/extractor/tva.py
index b57abea..0b863df 100644
--- a/youtube_dl/extractor/tva.py
+++ b/youtube_dl/extractor/tva.py
@@ -32,6 +32,8 @@ class TVAIE(InfoExtractor):
         video_data = self._download_json(
             'https://videos.tva.ca/proxy/item/_' + video_id, video_id, headers={
                 'Accept': 'application/json',
+            }, query={
+                'appId': '5955fc5f23eec60006c951f1',
             })
 
         def get_attribute(key):
-- 
cgit v1.1


From 05dee6c520eb959316a2e58203cbd5d30e908bc3 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 13 Nov 2017 19:15:16 +0100
Subject: [crunchyroll] extract old rtmp formats

---
 youtube_dl/extractor/crunchyroll.py | 156 +++++++++++++++++++++---------------
 1 file changed, 92 insertions(+), 64 deletions(-)

diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index 18ef3da..b53f2d7 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -38,6 +38,16 @@ class CrunchyrollBaseIE(InfoExtractor):
     _LOGIN_FORM = 'login_form'
     _NETRC_MACHINE = 'crunchyroll'
 
+    def _call_rpc_api(self, method, video_id, note=None, data=None):
+        data = data or {}
+        data['req'] = 'RpcApi' + method
+        data = compat_urllib_parse_urlencode(data).encode('utf-8')
+        return self._download_xml(
+            'http://www.crunchyroll.com/xml/',
+            video_id, note, fatal=False, data=data, headers={
+                'Content-Type': 'application/x-www-form-urlencoded',
+            })
+
     def _login(self):
         (username, password) = self._get_login_info()
         if username is None:
@@ -377,15 +387,19 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
     def _get_subtitles(self, video_id, webpage):
         subtitles = {}
         for sub_id, sub_name in re.findall(r'\bssid=([0-9]+)"[^>]+?\btitle="([^"]+)', webpage):
-            sub_page = self._download_webpage(
-                'http://www.crunchyroll.com/xml/?req=RpcApiSubtitle_GetXml&subtitle_script_id=' + sub_id,
-                video_id, note='Downloading subtitles for ' + sub_name)
-            id = self._search_regex(r'id=\'([0-9]+)', sub_page, 'subtitle_id', fatal=False)
-            iv = self._search_regex(r'<iv>([^<]+)', sub_page, 'subtitle_iv', fatal=False)
-            data = self._search_regex(r'<data>([^<]+)', sub_page, 'subtitle_data', fatal=False)
-            if not id or not iv or not data:
+            sub_doc = self._call_rpc_api(
+                'Subtitle_GetXml', video_id,
+                'Downloading subtitles for ' + sub_name, data={
+                    'subtitle_script_id': sub_id,
+                })
+            if not sub_doc:
                 continue
-            subtitle = self._decrypt_subtitles(data, iv, id).decode('utf-8')
+            sid = sub_doc.get('id')
+            iv = xpath_text(sub_doc, 'iv', 'subtitle iv')
+            data = xpath_text(sub_doc, 'data', 'subtitle data')
+            if not sid or not iv or not data:
+                continue
+            subtitle = self._decrypt_subtitles(data, iv, sid).decode('utf-8')
             lang_code = self._search_regex(r'lang_code=["\']([^"\']+)', subtitle, 'subtitle_lang_code', fatal=False)
             if not lang_code:
                 continue
@@ -456,65 +470,79 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
         for fmt in available_fmts:
             stream_quality, stream_format = self._FORMAT_IDS[fmt]
             video_format = fmt + 'p'
-            streamdata_req = sanitized_Request(
-                'http://www.crunchyroll.com/xml/?req=RpcApiVideoPlayer_GetStandardConfig&media_id=%s&video_format=%s&video_quality=%s'
-                % (video_id, stream_format, stream_quality),
-                compat_urllib_parse_urlencode({'current_page': url}).encode('utf-8'))
-            streamdata_req.add_header('Content-Type', 'application/x-www-form-urlencoded')
-            streamdata = self._download_xml(
-                streamdata_req, video_id,
-                note='Downloading media info for %s' % video_format)
-            stream_info = streamdata.find('./{default}preload/stream_info')
-            video_encode_id = xpath_text(stream_info, './video_encode_id')
-            if video_encode_id in video_encode_ids:
-                continue
-            video_encode_ids.append(video_encode_id)
-
-            video_file = xpath_text(stream_info, './file')
-            if not video_file:
-                continue
-            if video_file.startswith('http'):
-                formats.extend(self._extract_m3u8_formats(
-                    video_file, video_id, 'mp4', entry_protocol='m3u8_native',
-                    m3u8_id='hls', fatal=False))
-                continue
-
-            video_url = xpath_text(stream_info, './host')
-            if not video_url:
-                continue
-            metadata = stream_info.find('./metadata')
-            format_info = {
-                'format': video_format,
-                'format_id': video_format,
-                'height': int_or_none(xpath_text(metadata, './height')),
-                'width': int_or_none(xpath_text(metadata, './width')),
-            }
-
-            if '.fplive.net/' in video_url:
-                video_url = re.sub(r'^rtmpe?://', 'http://', video_url.strip())
-                parsed_video_url = compat_urlparse.urlparse(video_url)
-                direct_video_url = compat_urlparse.urlunparse(parsed_video_url._replace(
-                    netloc='v.lvlt.crcdn.net',
-                    path='%s/%s' % (remove_end(parsed_video_url.path, '/'), video_file.split(':')[-1])))
-                if self._is_valid_url(direct_video_url, video_id, video_format):
-                    format_info.update({
-                        'url': direct_video_url,
-                    })
-                    formats.append(format_info)
+            stream_infos = []
+            streamdata = self._call_rpc_api(
+                'VideoPlayer_GetStandardConfig', video_id,
+                'Downloading media info for %s' % video_format, data={
+                    'media_id': video_id,
+                    'video_format': stream_format,
+                    'video_quality': stream_quality,
+                    'current_page': url,
+                })
+            if streamdata:
+                stream_info = streamdata.find('./{default}preload/stream_info')
+                if stream_info:
+                    stream_infos.append(stream_info)
+            stream_info = self._call_rpc_api(
+                'VideoEncode_GetStreamInfo', video_id,
+                'Downloading stream info for %s' % video_format, data={
+                    'media_id': video_id,
+                    'video_format': stream_format,
+                    'video_encode_quality': stream_quality,
+                })
+            if stream_info:
+                stream_infos.append(stream_info)
+            for stream_info in stream_infos:
+                video_encode_id = xpath_text(stream_info, './video_encode_id')
+                if video_encode_id in video_encode_ids:
                     continue
+                video_encode_ids.append(video_encode_id)
 
-            format_info.update({
-                'url': video_url,
-                'play_path': video_file,
-                'ext': 'flv',
-            })
-            formats.append(format_info)
-        self._sort_formats(formats)
+                video_file = xpath_text(stream_info, './file')
+                if not video_file:
+                    continue
+                if video_file.startswith('http'):
+                    formats.extend(self._extract_m3u8_formats(
+                        video_file, video_id, 'mp4', entry_protocol='m3u8_native',
+                        m3u8_id='hls', fatal=False))
+                    continue
 
-        metadata = self._download_xml(
-            'http://www.crunchyroll.com/xml', video_id,
-            note='Downloading media info', query={
-                'req': 'RpcApiVideoPlayer_GetMediaMetadata',
+                video_url = xpath_text(stream_info, './host')
+                if not video_url:
+                    continue
+                metadata = stream_info.find('./metadata')
+                format_info = {
+                    'format': video_format,
+                    'height': int_or_none(xpath_text(metadata, './height')),
+                    'width': int_or_none(xpath_text(metadata, './width')),
+                }
+
+                if '.fplive.net/' in video_url:
+                    video_url = re.sub(r'^rtmpe?://', 'http://', video_url.strip())
+                    parsed_video_url = compat_urlparse.urlparse(video_url)
+                    direct_video_url = compat_urlparse.urlunparse(parsed_video_url._replace(
+                        netloc='v.lvlt.crcdn.net',
+                        path='%s/%s' % (remove_end(parsed_video_url.path, '/'), video_file.split(':')[-1])))
+                    if self._is_valid_url(direct_video_url, video_id, video_format):
+                        format_info.update({
+                            'format_id': 'http-' + video_format,
+                            'url': direct_video_url,
+                        })
+                        formats.append(format_info)
+                        continue
+
+                format_info.update({
+                    'format_id': 'rtmp-' + video_format,
+                    'url': video_url,
+                    'play_path': video_file,
+                    'ext': 'flv',
+                })
+                formats.append(format_info)
+        self._sort_formats(formats, ('height', 'width', 'tbr', 'fps'))
+
+        metadata = self._call_rpc_api(
+            'VideoPlayer_GetMediaMetadata', video_id,
+            note='Downloading media info', data={
                 'media_id': video_id,
             })
 
-- 
cgit v1.1


From 5871ebac473e723376722a37baecf51d6ae7d781 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 14 Nov 2017 01:43:20 +0700
Subject: [YoutubeDL] Fix playlist range optimization for --playlist-items
 (closes #14740)

---
 youtube_dl/YoutubeDL.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 342d6b4..68721e9 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -948,7 +948,8 @@ class YoutubeDL(object):
                 report_download(n_entries)
             else:  # iterable
                 if playlistitems:
-                    entries = make_playlistitems_entries(list(ie_entries))
+                    entries = make_playlistitems_entries(list(itertools.islice(
+                        ie_entries, 0, max(playlistitems))))
                 else:
                     entries = list(itertools.islice(
                         ie_entries, playliststart, playlistend))
-- 
cgit v1.1


From 0987f2ddb27a27506c697ad9dae2ccbf24fc786d Mon Sep 17 00:00:00 2001
From: Timendum <timedum@gmail.com>
Date: Tue, 14 Nov 2017 16:34:45 +0100
Subject: [vshare] Fix extraction (closes #14473)

---
 youtube_dl/extractor/generic.py | 16 ++++++++++++++++
 youtube_dl/extractor/vshare.py  | 36 ++++++++++++++++++++++++++++--------
 2 files changed, 44 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 2a9c3e2..31564e5 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -102,6 +102,7 @@ from .joj import JojIE
 from .megaphone import MegaphoneIE
 from .vzaar import VzaarIE
 from .channel9 import Channel9IE
+from .vshare import VShareIE
 
 
 class GenericIE(InfoExtractor):
@@ -1921,6 +1922,16 @@ class GenericIE(InfoExtractor):
                 'title': 'Rescue Kit 14 Free Edition - Getting started',
             },
             'playlist_count': 4,
+        },
+        {
+            # vshare embed
+            'url': 'https://youtube-dl-demo.neocities.org/vshare.html',
+            'md5': '17b39f55b5497ae8b59f5fbce8e35886',
+            'info_dict': {
+                'id': '0f64ce6',
+                'title': 'vl14062007715967',
+                'ext': 'mp4',
+            }
         }
         # {
         #     # TODO: find another test
@@ -2879,6 +2890,11 @@ class GenericIE(InfoExtractor):
             return self.playlist_from_matches(
                 channel9_urls, video_id, video_title, ie=Channel9IE.ie_key())
 
+        vshare_urls = VShareIE._extract_urls(webpage)
+        if vshare_urls:
+            return self.playlist_from_matches(
+                vshare_urls, video_id, video_title, ie=VShareIE.ie_key())
+
         def merge_dicts(dict1, dict2):
             merged = {}
             for k, v in dict1.items():
diff --git a/youtube_dl/extractor/vshare.py b/youtube_dl/extractor/vshare.py
index 5addbc2..ea39a90 100644
--- a/youtube_dl/extractor/vshare.py
+++ b/youtube_dl/extractor/vshare.py
@@ -1,14 +1,18 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
+from ..compat import compat_chr
+from ..utils import decode_packed_codes
 
 
 class VShareIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?vshare\.io/[dv]/(?P<id>[^/?#&]+)'
     _TESTS = [{
         'url': 'https://vshare.io/d/0f64ce6',
-        'md5': '16d7b8fef58846db47419199ff1ab3e7',
+        'md5': '17b39f55b5497ae8b59f5fbce8e35886',
         'info_dict': {
             'id': '0f64ce6',
             'title': 'vl14062007715967',
@@ -19,20 +23,36 @@ class VShareIE(InfoExtractor):
         'only_matching': True,
     }]
 
+    def _extract_packed(self, webpage):
+        packed = self._search_regex(r'(eval\(function.+)', webpage, 'packed code')
+        unpacked = decode_packed_codes(packed)
+        digits = self._search_regex(r'\[((?:\d+,?)+)\]', unpacked, 'digits')
+        digits = digits.split(',')
+        digits = [int(digit) for digit in digits]
+        key_digit = self._search_regex(r'fromCharCode\(.+?(\d+)\)}', unpacked, 'key digit')
+        chars = [compat_chr(d - int(key_digit)) for d in digits]
+        return ''.join(chars)
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
         webpage = self._download_webpage(
-            'https://vshare.io/d/%s' % video_id, video_id)
+            'https://vshare.io/v/%s/width-650/height-430/1' % video_id, video_id)
 
-        title = self._html_search_regex(
-            r'(?s)<div id="root-container">(.+?)<br/>', webpage, 'title')
-        video_url = self._search_regex(
-            r'<a[^>]+href=(["\'])(?P<url>(?:https?:)?//.+?)\1[^>]*>[Cc]lick\s+here',
-            webpage, 'video url', group='url')
+        title = self._html_search_regex(r'<title>([^<]+)</title>', webpage, 'title')
+        title = title.split(' - ')[0]
 
+        unpacked = self._extract_packed(webpage)
+        video_urls = re.findall(r'<source src="([^"]+)', unpacked)
+        formats = [{'url': video_url} for video_url in video_urls]
         return {
             'id': video_id,
             'title': title,
-            'url': video_url,
+            'formats': formats,
         }
+
+    @staticmethod
+    def _extract_urls(webpage):
+        return re.findall(
+            r'<iframe[^>]+?src=["\'](?P<url>(?:https?:)?//(?:www\.)?vshare\.io/v/[^/?#&]+)',
+            webpage)
-- 
cgit v1.1


From ff31f2d5c3750364b013a9bf59b85cebd0cee1fd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 14 Nov 2017 22:39:54 +0700
Subject: [vshare] Capture and output error message

---
 youtube_dl/extractor/vshare.py | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vshare.py b/youtube_dl/extractor/vshare.py
index ea39a90..20ce22e 100644
--- a/youtube_dl/extractor/vshare.py
+++ b/youtube_dl/extractor/vshare.py
@@ -5,7 +5,10 @@ import re
 
 from .common import InfoExtractor
 from ..compat import compat_chr
-from ..utils import decode_packed_codes
+from ..utils import (
+    decode_packed_codes,
+    ExtractorError,
+)
 
 
 class VShareIE(InfoExtractor):
@@ -42,6 +45,12 @@ class VShareIE(InfoExtractor):
         title = self._html_search_regex(r'<title>([^<]+)</title>', webpage, 'title')
         title = title.split(' - ')[0]
 
+        error = self._html_search_regex(
+            r'(?s)<div[^>]+\bclass=["\']xxx-error[^>]+>(.+?)</div', webpage,
+            'error', default=None)
+        if error:
+            raise ExtractorError(error, expected=True)
+
         unpacked = self._extract_packed(webpage)
         video_urls = re.findall(r'<source src="([^"]+)', unpacked)
         formats = [{'url': video_url} for video_url in video_urls]
-- 
cgit v1.1


From a2b6aba8ded704f55a6650fc3e20b2e6c8aec094 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 14 Nov 2017 22:49:25 +0700
Subject: [vshare] Improve extraction, fix formats sorting and carry long lines

---
 youtube_dl/extractor/vshare.py | 41 ++++++++++++++++++++++++-----------------
 1 file changed, 24 insertions(+), 17 deletions(-)

diff --git a/youtube_dl/extractor/vshare.py b/youtube_dl/extractor/vshare.py
index 20ce22e..e4ec778 100644
--- a/youtube_dl/extractor/vshare.py
+++ b/youtube_dl/extractor/vshare.py
@@ -26,13 +26,20 @@ class VShareIE(InfoExtractor):
         'only_matching': True,
     }]
 
+    @staticmethod
+    def _extract_urls(webpage):
+        return re.findall(
+            r'<iframe[^>]+?src=["\'](?P<url>(?:https?:)?//(?:www\.)?vshare\.io/v/[^/?#&]+)',
+            webpage)
+
     def _extract_packed(self, webpage):
-        packed = self._search_regex(r'(eval\(function.+)', webpage, 'packed code')
+        packed = self._search_regex(
+            r'(eval\(function.+)', webpage, 'packed code')
         unpacked = decode_packed_codes(packed)
         digits = self._search_regex(r'\[((?:\d+,?)+)\]', unpacked, 'digits')
-        digits = digits.split(',')
-        digits = [int(digit) for digit in digits]
-        key_digit = self._search_regex(r'fromCharCode\(.+?(\d+)\)}', unpacked, 'key digit')
+        digits = [int(digit) for digit in digits.split(',')]
+        key_digit = self._search_regex(
+            r'fromCharCode\(.+?(\d+)\)}', unpacked, 'key digit')
         chars = [compat_chr(d - int(key_digit)) for d in digits]
         return ''.join(chars)
 
@@ -40,9 +47,11 @@ class VShareIE(InfoExtractor):
         video_id = self._match_id(url)
 
         webpage = self._download_webpage(
-            'https://vshare.io/v/%s/width-650/height-430/1' % video_id, video_id)
+            'https://vshare.io/v/%s/width-650/height-430/1' % video_id,
+            video_id)
 
-        title = self._html_search_regex(r'<title>([^<]+)</title>', webpage, 'title')
+        title = self._html_search_regex(
+            r'<title>([^<]+)</title>', webpage, 'title')
         title = title.split(' - ')[0]
 
         error = self._html_search_regex(
@@ -51,17 +60,15 @@ class VShareIE(InfoExtractor):
         if error:
             raise ExtractorError(error, expected=True)
 
-        unpacked = self._extract_packed(webpage)
-        video_urls = re.findall(r'<source src="([^"]+)', unpacked)
-        formats = [{'url': video_url} for video_url in video_urls]
-        return {
+        info = self._parse_html5_media_entries(
+            url, '<video>%s</video>' % self._extract_packed(webpage),
+            video_id)[0]
+
+        self._sort_formats(info['formats'])
+
+        info.update({
             'id': video_id,
             'title': title,
-            'formats': formats,
-        }
+        })
 
-    @staticmethod
-    def _extract_urls(webpage):
-        return re.findall(
-            r'<iframe[^>]+?src=["\'](?P<url>(?:https?:)?//(?:www\.)?vshare\.io/v/[^/?#&]+)',
-            webpage)
+        return info
-- 
cgit v1.1


From ea2295842f79c9efff3a9abce1d0eee7de4953d6 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 14 Nov 2017 17:41:30 +0100
Subject: [common] skip Apple FairPlay m3u8 manifests(closes #14741)

---
 youtube_dl/extractor/common.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index e2d9f52..a9d68fc 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1356,6 +1356,9 @@ class InfoExtractor(object):
         if '#EXT-X-FAXS-CM:' in m3u8_doc:  # Adobe Flash Access
             return []
 
+        if re.search(r'#EXT-X-SESSION-KEY:.*?URI="skd://', m3u8_doc):  # Apple FairPlay
+            return []
+
         formats = []
 
         format_url = lambda u: (
-- 
cgit v1.1


From fae0eb42ec4309fe7fb8476d30621ba1d60fa168 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 14 Nov 2017 23:59:30 +0700
Subject: [ChangeLog] Actualize

---
 ChangeLog | 20 +++++++++++++++++++-
 1 file changed, 19 insertions(+), 1 deletion(-)

diff --git a/ChangeLog b/ChangeLog
index cedab47..6ed0f01 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,7 +1,25 @@
 version <unreleased>
 
-Extractors
+Core
+* [common] Skip Apple FairPlay m3u8 manifests (#14741)
+* [YoutubeDL] Fix playlist range optimization for --playlist-items (#14740)
+
+Extractors
+* [vshare] Capture and output error message
+* [vshare] Fix extraction (#14473)
+* [crunchyroll] Extract old RTMP formats
+* [tva] Fix extraction (#14736)
+* [gamespot] Lower preference of HTTP formats (#14652)
+* [instagram:user] Fix extraction (#14699)
+* [ccma] Fix typo (#14730)
+- Remove sensitive data from logging in messages
+* [instagram:user] Fix extraction (#14699)
++ [gamespot] Add support for article URLs (#14652)
+* [gamespot] Skip Brightcove Once HTTP formats (#14652)
+* [cartoonnetwork] Update tokenizer_src (#14666)
 + [wsj] Recognize another URL pattern (#14704)
+* [pandatv] Update API URL and sign format URLs (#14693)
+* [crunchyroll] Use old login method (#11572)
 
 
 version 2017.11.06
-- 
cgit v1.1


From 08e45b39e76419f63aa43d5008257789d8a30bf8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 15 Nov 2017 00:15:42 +0700
Subject: release 2017.11.15

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 youtube_dl/version.py     | 2 +-
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index be6e6dd..4dd1a6e 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.11.06*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.11.06**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.11.15*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.11.15**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through the [README](https://github.com/rg3/youtube-dl/blob/master/README.md), **most notably** the [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ Add the `-v` flag to **your command line** you run youtube-dl with (`youtube-dl
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.11.06
+[debug] youtube-dl version 2017.11.15
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index 6ed0f01..6b4befb 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.11.15
 
 Core
 * [common] Skip Apple FairPlay m3u8 manifests (#14741)
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 8b67d23..1c3cbef 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.11.06'
+__version__ = '2017.11.15'
-- 
cgit v1.1


From 9cbd4dda10ad248a5268ec1e0e563cf97024a8b9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 15 Nov 2017 22:14:54 +0700
Subject: [instagram] Fix description, timestamp and counters extraction
 (closes #14755)

---
 youtube_dl/extractor/instagram.py | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/instagram.py b/youtube_dl/extractor/instagram.py
index 20db31f..a77f619 100644
--- a/youtube_dl/extractor/instagram.py
+++ b/youtube_dl/extractor/instagram.py
@@ -130,13 +130,21 @@ class InstagramIE(InfoExtractor):
                 video_url = media.get('video_url')
                 height = int_or_none(media.get('dimensions', {}).get('height'))
                 width = int_or_none(media.get('dimensions', {}).get('width'))
-                description = media.get('caption')
+                description = try_get(
+                    media, lambda x: x['edge_media_to_caption']['edges'][0]['node']['text'],
+                    compat_str) or media.get('caption')
                 thumbnail = media.get('display_src')
-                timestamp = int_or_none(media.get('date'))
+                timestamp = int_or_none(media.get('taken_at_timestamp') or media.get('date'))
                 uploader = media.get('owner', {}).get('full_name')
                 uploader_id = media.get('owner', {}).get('username')
-                like_count = int_or_none(media.get('likes', {}).get('count'))
-                comment_count = int_or_none(media.get('comments', {}).get('count'))
+
+                def get_count(key, kind):
+                    return int_or_none(try_get(
+                        media, (lambda x: x['edge_media_%s' % key]['count'],
+                                lambda x: x['%ss' % kind]['count'])))
+                like_count = get_count('preview_like', 'like')
+                comment_count = get_count('to_comment', 'comment')
+
                 comments = [{
                     'author': comment.get('user', {}).get('username'),
                     'author_id': comment.get('user', {}).get('id'),
-- 
cgit v1.1


From 3192d4bc7a063983c3a82bc4320c16d65679307a Mon Sep 17 00:00:00 2001
From: Windom <windom@users.noreply.github.com>
Date: Thu, 16 Nov 2017 20:05:04 +0200
Subject: [spankbang] Add support for mobile URLs and fix test

---
 youtube_dl/extractor/spankbang.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/spankbang.py b/youtube_dl/extractor/spankbang.py
index 2863e53..e6c2dcf 100644
--- a/youtube_dl/extractor/spankbang.py
+++ b/youtube_dl/extractor/spankbang.py
@@ -7,7 +7,7 @@ from ..utils import ExtractorError
 
 
 class SpankBangIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:(?:www|[a-z]{2})\.)?spankbang\.com/(?P<id>[\da-z]+)/video'
+    _VALID_URL = r'https?://(?:(?:www|m|[a-z]{2})\.)?spankbang\.com/(?P<id>[\da-z]+)/video'
     _TESTS = [{
         'url': 'http://spankbang.com/3vvn/video/fantasy+solo',
         'md5': '1cc433e1d6aa14bc376535b8679302f7',
@@ -15,7 +15,7 @@ class SpankBangIE(InfoExtractor):
             'id': '3vvn',
             'ext': 'mp4',
             'title': 'fantasy solo',
-            'description': 'Watch fantasy solo free HD porn video - 05 minutes - dillion harper masturbates on a bed free adult movies.',
+            'description': 'Watch fantasy solo free HD porn video - 05 minutes -  Babe,Masturbation,Solo,Toy  - dillion harper masturbates on a bed free adult movies sexy clips.',
             'thumbnail': r're:^https?://.*\.jpg$',
             'uploader': 'silly2587',
             'age_limit': 18,
@@ -28,6 +28,10 @@ class SpankBangIE(InfoExtractor):
         # no uploader
         'url': 'http://spankbang.com/lklg/video/sex+with+anyone+wedding+edition+2',
         'only_matching': True,
+    }, {
+        # mobile page
+        'url': 'http://m.spankbang.com/1o2de/video/can+t+remember+her+name',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
-- 
cgit v1.1


From 38db52adf35c2134444e5b6b601e9567797e9195 Mon Sep 17 00:00:00 2001
From: Windom <windom@users.noreply.github.com>
Date: Thu, 16 Nov 2017 20:50:07 +0200
Subject: [drtuber] Add support for mobile URLs

---
 youtube_dl/extractor/drtuber.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/drtuber.py b/youtube_dl/extractor/drtuber.py
index c5d56a9..c88b312 100644
--- a/youtube_dl/extractor/drtuber.py
+++ b/youtube_dl/extractor/drtuber.py
@@ -10,7 +10,7 @@ from ..utils import (
 
 
 class DrTuberIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?drtuber\.com/(?:video|embed)/(?P<id>\d+)(?:/(?P<display_id>[\w-]+))?'
+    _VALID_URL = r'https?://(?:(?:www|m)\.)?drtuber\.com/(?:video|embed)/(?P<id>\d+)(?:/(?P<display_id>[\w-]+))?'
     _TESTS = [{
         'url': 'http://www.drtuber.com/video/1740434/hot-perky-blonde-naked-golf',
         'md5': '93e680cf2536ad0dfb7e74d94a89facd',
@@ -28,6 +28,9 @@ class DrTuberIE(InfoExtractor):
     }, {
         'url': 'http://www.drtuber.com/embed/489939',
         'only_matching': True,
+    }, {
+        'url': 'http://m.drtuber.com/video/3893529/lingerie-blowjob-from-beautiful-teen',
+        'only_matching': True,
     }]
 
     @staticmethod
-- 
cgit v1.1


From f610dbb05f8d17cc95437958835a437c3777b38c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 18 Nov 2017 19:02:56 +0700
Subject: [extractor/common] Use final URL when dumping request (closes #14769)

---
 youtube_dl/extractor/common.py | 12 ++----------
 1 file changed, 2 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index a9d68fc..8e4ee0d 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -592,19 +592,11 @@ class InfoExtractor(object):
         if not encoding:
             encoding = self._guess_encoding_from_content(content_type, webpage_bytes)
         if self._downloader.params.get('dump_intermediate_pages', False):
-            try:
-                url = url_or_request.get_full_url()
-            except AttributeError:
-                url = url_or_request
-            self.to_screen('Dumping request to ' + url)
+            self.to_screen('Dumping request to ' + urlh.geturl())
             dump = base64.b64encode(webpage_bytes).decode('ascii')
             self._downloader.to_screen(dump)
         if self._downloader.params.get('write_pages', False):
-            try:
-                url = url_or_request.get_full_url()
-            except AttributeError:
-                url = url_or_request
-            basen = '%s_%s' % (video_id, url)
+            basen = '%s_%s' % (video_id, urlh.geturl())
             if len(basen) > 240:
                 h = '___' + hashlib.md5(basen.encode('utf-8')).hexdigest()
                 basen = basen[:240 - len(h)] + h
-- 
cgit v1.1


From a9efdf3d4a18ec5657ea50f31715e1b88a945820 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sun, 19 Nov 2017 12:59:31 +0100
Subject: [livestream] make smil extraction non fatal(fixes #14792)

---
 youtube_dl/extractor/livestream.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/livestream.py b/youtube_dl/extractor/livestream.py
index 317ebbc..c4776bb 100644
--- a/youtube_dl/extractor/livestream.py
+++ b/youtube_dl/extractor/livestream.py
@@ -114,7 +114,7 @@ class LivestreamIE(InfoExtractor):
 
         smil_url = video_data.get('smil_url')
         if smil_url:
-            formats.extend(self._extract_smil_formats(smil_url, video_id))
+            formats.extend(self._extract_smil_formats(smil_url, video_id, fatal=False))
 
         m3u8_url = video_data.get('m3u8_url')
         if m3u8_url:
-- 
cgit v1.1


From 8f639411042d35cd3be6eeff485e3015bafce4d7 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 22 Nov 2017 22:49:48 +0800
Subject: [youku] Fix extraction; update ccode (closes #14815)

---
 ChangeLog                     | 6 ++++++
 youtube_dl/extractor/youku.py | 2 +-
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/ChangeLog b/ChangeLog
index 6b4befb..e3b7750 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,9 @@
+version <unreleased>
+
+Extractors
+* [youku] Fix extraction; update ccode (#14815)
+
+
 version 2017.11.15
 
 Core
diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index 0c4bc2e..6822a30 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -154,7 +154,7 @@ class YoukuIE(InfoExtractor):
         # request basic data
         basic_data_params = {
             'vid': video_id,
-            'ccode': '0402' if 'tudou.com' in url else '0401',
+            'ccode': '0502',
             'client_ip': '192.168.1.1',
             'utid': cna,
             'client_ts': time.time() / 1000,
-- 
cgit v1.1


From 2688664762f406b1ba2913af25ee3a2d2ba58038 Mon Sep 17 00:00:00 2001
From: enigmaquip <enigmaquip@users.noreply.github.com>
Date: Wed, 22 Nov 2017 16:39:11 -0700
Subject: [culturebox] Fix extraction (closes #14827)

---
 youtube_dl/extractor/francetv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py
index 2bcbb3e..037e538 100644
--- a/youtube_dl/extractor/francetv.py
+++ b/youtube_dl/extractor/francetv.py
@@ -363,6 +363,6 @@ class CultureboxIE(FranceTVBaseInfoExtractor):
             raise ExtractorError('Video %s is not available' % name, expected=True)
 
         video_id, catalogue = self._search_regex(
-            r'"http://videos\.francetv\.fr/video/([^@]+@[^"]+)"', webpage, 'video id').split('@')
+            r'"https?://videos\.francetv\.fr/video/([^@]+@[^"]+)"', webpage, 'video id').split('@')
 
         return self._extract_video(video_id, catalogue)
-- 
cgit v1.1


From 939be9adfe810ada7dbd5e9032bcfec19fafa14b Mon Sep 17 00:00:00 2001
From: John Hawkinson <jhawk@mit.edu>
Date: Wed, 22 Nov 2017 11:47:02 -0500
Subject: [JWPlatform] Support iframes

Support content.jwplatform... src attributes inside <iframe> tags in
addition to <script> tags. Just a regexp change.

Add a test (currently Generic_72).
---
 youtube_dl/extractor/generic.py    | 12 ++++++++++++
 youtube_dl/extractor/jwplatform.py |  2 +-
 2 files changed, 13 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 31564e5..8efe681 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1137,6 +1137,18 @@ class GenericIE(InfoExtractor):
             }
         },
         {
+            # JWPlatform iframe
+            'url': 'https://www.mediaite.com/tv/dem-senator-claims-gary-cohn-faked-a-bad-connection-during-trump-call-to-get-him-off-the-phone/',
+            'md5': 'ca00a040364b5b439230e7ebfd02c4e9',
+            'info_dict': {
+                'id': 'O0c5JcKT',
+                'ext': 'mp4',
+                'upload_date': '20171122',
+                'timestamp': 1511366290,
+                'title': 'Dem Senator Claims Gary Cohn Faked a Bad Connection During Trump Call to Get Him Off the Phone',
+            },
+        },
+        {
             # Video.js embed, multiple formats
             'url': 'http://ortcam.com/solidworks-урок-6-настройка-чертежа_33f9b7351.html',
             'info_dict': {
diff --git a/youtube_dl/extractor/jwplatform.py b/youtube_dl/extractor/jwplatform.py
index 33d55f7..4e74eaf 100644
--- a/youtube_dl/extractor/jwplatform.py
+++ b/youtube_dl/extractor/jwplatform.py
@@ -24,7 +24,7 @@ class JWPlatformIE(InfoExtractor):
     @staticmethod
     def _extract_url(webpage):
         mobj = re.search(
-            r'<script[^>]+?src=["\'](?P<url>(?:https?:)?//content.jwplatform.com/players/[a-zA-Z0-9]{8})',
+            r'<(script|iframe)[^>]+?src=["\'](?P<url>(?:https?:)?//content.jwplatform.com/players/[a-zA-Z0-9]{8})',
             webpage)
         if mobj:
             return mobj.group('url')
-- 
cgit v1.1


From 6899b1d9e87857e0ae2d36f710e0e12885b29c09 Mon Sep 17 00:00:00 2001
From: John Hawkinson <jhawk@mit.edu>
Date: Wed, 22 Nov 2017 11:56:47 -0500
Subject: [Generic] Update test 69 (suffolk/sjc)

suffolk.edu/sjc => suffolk.edu/sjc/live.php

Unfortunately it only transmits video a few mornings per month, so
leaving the 'skip' is probably appropriate. Updating the 'skip- to
include the calendar information though.
---
 youtube_dl/extractor/generic.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 8efe681..d2e4551 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1099,9 +1099,9 @@ class GenericIE(InfoExtractor):
         },
         # jwplayer rtmp
         {
-            'url': 'http://www.suffolk.edu/sjc/',
+            'url': 'http://www.suffolk.edu/sjc/live.php',
             'info_dict': {
-                'id': 'sjclive',
+                'id': 'live',
                 'ext': 'flv',
                 'title': 'Massachusetts Supreme Judicial Court Oral Arguments',
                 'uploader': 'www.suffolk.edu',
@@ -1109,7 +1109,7 @@ class GenericIE(InfoExtractor):
             'params': {
                 'skip_download': True,
             },
-            'skip': 'does not contain a video anymore',
+            'skip': 'Only has video a few mornings per month, see http://www.suffolk.edu/sjc/',
         },
         # Complex jwplayer
         {
-- 
cgit v1.1


From 32ad4f3faf33d7ffedf08b3f8e39acb084b361fa Mon Sep 17 00:00:00 2001
From: John Hawkinson <jhawk@mit.edu>
Date: Fri, 24 Nov 2017 08:10:47 -0500
Subject: [JWPlatform] Use non-capturing group in RE

Per @yan12125.
---
 youtube_dl/extractor/jwplatform.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/jwplatform.py b/youtube_dl/extractor/jwplatform.py
index 4e74eaf..c9bcbb0 100644
--- a/youtube_dl/extractor/jwplatform.py
+++ b/youtube_dl/extractor/jwplatform.py
@@ -24,7 +24,7 @@ class JWPlatformIE(InfoExtractor):
     @staticmethod
     def _extract_url(webpage):
         mobj = re.search(
-            r'<(script|iframe)[^>]+?src=["\'](?P<url>(?:https?:)?//content.jwplatform.com/players/[a-zA-Z0-9]{8})',
+            r'<(?:script|iframe)[^>]+?src=["\'](?P<url>(?:https?:)?//content.jwplatform.com/players/[a-zA-Z0-9]{8})',
             webpage)
         if mobj:
             return mobj.group('url')
-- 
cgit v1.1


From 805f5bf75978ba24d072d040af81bd8d1ec3fed9 Mon Sep 17 00:00:00 2001
From: John Hawkinson <jhawk@mit.edu>
Date: Fri, 24 Nov 2017 08:11:32 -0500
Subject: [Generic] ie_key in JWPlatform test 72

---
 youtube_dl/extractor/generic.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index d2e4551..56df2ab 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1147,6 +1147,7 @@ class GenericIE(InfoExtractor):
                 'timestamp': 1511366290,
                 'title': 'Dem Senator Claims Gary Cohn Faked a Bad Connection During Trump Call to Get Him Off the Phone',
             },
+            'add_ie': [JWPlatformIE.ie_key()],
         },
         {
             # Video.js embed, multiple formats
-- 
cgit v1.1


From 6049176471b4c567c1e9bd8f77b298e323d5b8e7 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 24 Nov 2017 21:40:51 +0800
Subject: [ChangeLog] Update after #14828

[skip ci]
---
 ChangeLog | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ChangeLog b/ChangeLog
index e3b7750..c4a84c5 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -2,7 +2,7 @@ version <unreleased>
 
 Extractors
 * [youku] Fix extraction; update ccode (#14815)
-
++ [JWPlatform] support iframes (#14828)
 
 version 2017.11.15
 
-- 
cgit v1.1


From e0a8686f48d10ed86f7be92132dd37481981adf3 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Fri, 24 Nov 2017 18:42:41 +0100
Subject: [faz] fix extraction and add support for Perform Group embeds(fixes
 #14714)

---
 youtube_dl/extractor/extractors.py   |  1 +
 youtube_dl/extractor/faz.py          | 34 ++++++++++++---
 youtube_dl/extractor/performgroup.py | 83 ++++++++++++++++++++++++++++++++++++
 3 files changed, 111 insertions(+), 7 deletions(-)
 create mode 100644 youtube_dl/extractor/performgroup.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index d084707..aecb84b 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -789,6 +789,7 @@ from .patreon import PatreonIE
 from .pbs import PBSIE
 from .pearvideo import PearVideoIE
 from .people import PeopleIE
+from .performgroup import PerformGroupIE
 from .periscope import (
     PeriscopeIE,
     PeriscopeUserIE,
diff --git a/youtube_dl/extractor/faz.py b/youtube_dl/extractor/faz.py
index 4bc8fc5..312ee2a 100644
--- a/youtube_dl/extractor/faz.py
+++ b/youtube_dl/extractor/faz.py
@@ -1,7 +1,10 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
+from ..compat import compat_etree_fromstring
 from ..utils import (
     xpath_element,
     xpath_text,
@@ -43,10 +46,15 @@ class FazIE(InfoExtractor):
 
         webpage = self._download_webpage(url, video_id)
         description = self._og_search_description(webpage)
-        config_xml_url = self._search_regex(
-            r'videoXMLURL\s*=\s*"([^"]+)', webpage, 'config xml url')
-        config = self._download_xml(
-            config_xml_url, video_id, 'Downloading config xml')
+        media = self._html_search_regex(
+            r"data-videojs-media='([^']+)",
+            webpage, 'media')
+        if media == 'extern':
+            perform_url = self._search_regex(
+                r"<iframe[^>]+?src='((?:http:)?//player\.performgroup\.com/eplayer/eplayer\.html#/?[0-9a-f]{26}\.[0-9a-z]{26})",
+                webpage, 'perform url')
+            return self.url_result(perform_url)
+        config = compat_etree_fromstring(media)
 
         encodings = xpath_element(config, 'ENCODINGS', 'encodings', True)
         formats = []
@@ -55,12 +63,24 @@ class FazIE(InfoExtractor):
             if encoding is not None:
                 encoding_url = xpath_text(encoding, 'FILENAME')
                 if encoding_url:
-                    formats.append({
+                    tbr = xpath_text(encoding, 'AVERAGEBITRATE', 1000)
+                    if tbr:
+                        tbr = int_or_none(tbr.replace(',', '.'))
+                    f = {
                         'url': encoding_url,
                         'format_id': code.lower(),
                         'quality': pref,
-                        'tbr': int_or_none(xpath_text(encoding, 'AVERAGEBITRATE')),
-                    })
+                        'tbr': tbr,
+                        'vcodec': xpath_text(encoding, 'CODEC'),
+                    }
+                    mobj = re.search(r'(\d+)x(\d+)_(\d+)\.mp4', encoding_url)
+                    if mobj:
+                        f.update({
+                            'width': int(mobj.group(1)),
+                            'height': int(mobj.group(2)),
+                            'tbr': tbr or int(mobj.group(3)),
+                        })
+                    formats.append(f)
         self._sort_formats(formats)
 
         return {
diff --git a/youtube_dl/extractor/performgroup.py b/youtube_dl/extractor/performgroup.py
new file mode 100644
index 0000000..26942bf
--- /dev/null
+++ b/youtube_dl/extractor/performgroup.py
@@ -0,0 +1,83 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import int_or_none
+
+
+class PerformGroupIE(InfoExtractor):
+    _VALID_URL = r'https?://player\.performgroup\.com/eplayer(?:/eplayer\.html|\.js)#/?(?P<id>[0-9a-f]{26})\.(?P<auth_token>[0-9a-z]{26})'
+    _TESTS = [{
+        # http://www.faz.net/aktuell/sport/fussball/wm-2018-playoffs-schweiz-besiegt-nordirland-1-0-15286104.html
+        'url': 'http://player.performgroup.com/eplayer/eplayer.html#d478c41c5d192f56b9aa859de8.1w4crrej5w14e1ed4s1ce4ykab',
+        'md5': '259cb03d142e2e52471e8837ecacb29f',
+        'info_dict': {
+            'id': 'xgrwobuzumes1lwjxtcdpwgxd',
+            'ext': 'mp4',
+            'title': 'Liga MX: Keine Einsicht nach Horrorfoul',
+            'description': 'md5:7cd3b459c82725b021e046ab10bf1c5b',
+            'timestamp': 1511533477,
+            'upload_date': '20171124',
+        }
+    }]
+
+    def _call_api(self, service, auth_token, content_id, referer_url):
+        return self._download_json(
+            'http://ep3.performfeeds.com/ep%s/%s/%s/' % (service, auth_token, content_id),
+            content_id, headers={
+                'Referer': referer_url,
+                'Origin': 'http://player.performgroup.com',
+            }, query={
+                '_fmt': 'json',
+            })
+
+    def _real_extract(self, url):
+        player_id, auth_token = re.search(self._VALID_URL, url).groups()
+        bootstrap = self._call_api('bootstrap', auth_token, player_id, url)
+        video = bootstrap['config']['dataSource']['sourceItems'][0]['videos'][0]
+        video_id = video['uuid']
+        vod = self._call_api('vod', auth_token, video_id, url)
+        media = vod['videos']['video'][0]['media']
+
+        formats = []
+        hls_url = media.get('hls', {}).get('url')
+        if hls_url:
+            formats.extend(self._extract_m3u8_formats(hls_url, video_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False))
+
+        hds_url = media.get('hds', {}).get('url')
+        if hds_url:
+            formats.extend(self._extract_f4m_formats(hds_url + '?hdcore', video_id, f4m_id='hds', fatal=False))
+
+        for c in media.get('content', []):
+            c_url = c.get('url')
+            if not c_url:
+                continue
+            tbr = int_or_none(c.get('bitrate'), 1000)
+            format_id = 'http'
+            if tbr:
+                format_id += '-%d' % tbr
+            formats.append({
+                'format_id': format_id,
+                'url': c_url,
+                'tbr': tbr,
+                'width': int_or_none(c.get('width')),
+                'height': int_or_none(c.get('height')),
+                'filesize': int_or_none(c.get('fileSize')),
+                'vcodec': c.get('type'),
+                'fps': int_or_none(c.get('videoFrameRate')),
+                'vbr': int_or_none(c.get('videoRate'), 1000),
+                'abr': int_or_none(c.get('audioRate'), 1000),
+            })
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': video['title'],
+            'description': video.get('description'),
+            'thumbnail': video.get('poster'),
+            'duration': int_or_none(video.get('duration')),
+            'timestamp': int_or_none(video.get('publishedTime'), 1000),
+            'formats': formats,
+        }
-- 
cgit v1.1


From c6c6a64aa58af54d3f1d4ebdd21547e708b29bf6 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Fri, 24 Nov 2017 19:00:56 +0100
Subject: [fox9] fix extraction

---
 youtube_dl/extractor/fox9.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/fox9.py b/youtube_dl/extractor/fox9.py
index 56d9975..17dfffa 100644
--- a/youtube_dl/extractor/fox9.py
+++ b/youtube_dl/extractor/fox9.py
@@ -2,7 +2,6 @@
 from __future__ import unicode_literals
 
 from .anvato import AnvatoIE
-from ..utils import js_to_json
 
 
 class FOX9IE(AnvatoIE):
@@ -34,9 +33,9 @@ class FOX9IE(AnvatoIE):
 
         video_id = self._parse_json(
             self._search_regex(
-                r'AnvatoPlaylist\s*\(\s*(\[.+?\])\s*\)\s*;',
+                r"this\.videosJson\s*=\s*'(\[.+?\])';",
                 webpage, 'anvato playlist'),
-            video_id, transform_source=js_to_json)[0]['video']
+            video_id)[0]['video']
 
         return self._get_anvato_videos(
             'anvato_epfox_app_web_prod_b3373168e12f423f41504f207000188daf88251b',
-- 
cgit v1.1


From cd9ff4ec5be86b4a16afc792ff9cf0d5c3eacd47 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Fri, 24 Nov 2017 20:00:01 +0100
Subject: [massengeschmacktv] Add support for Massengeschmack TV(replaces
 Fernseh Kritik TV)

---
 youtube_dl/extractor/extractors.py        |  2 +-
 youtube_dl/extractor/fktv.py              | 51 --------------------
 youtube_dl/extractor/massengeschmacktv.py | 77 +++++++++++++++++++++++++++++++
 3 files changed, 78 insertions(+), 52 deletions(-)
 delete mode 100644 youtube_dl/extractor/fktv.py
 create mode 100644 youtube_dl/extractor/massengeschmacktv.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index aecb84b..f1a4649 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -348,7 +348,6 @@ from .firstpost import FirstpostIE
 from .firsttv import FirstTVIE
 from .fivemin import FiveMinIE
 from .fivetv import FiveTVIE
-from .fktv import FKTVIE
 from .flickr import FlickrIE
 from .flipagram import FlipagramIE
 from .folketinget import FolketingetIE
@@ -572,6 +571,7 @@ from .mangomolo import (
     MangomoloLiveIE,
 )
 from .manyvids import ManyVidsIE
+from .massengeschmacktv import MassengeschmackTVIE
 from .matchtv import MatchTVIE
 from .mdr import MDRIE
 from .mediaset import MediasetIE
diff --git a/youtube_dl/extractor/fktv.py b/youtube_dl/extractor/fktv.py
deleted file mode 100644
index 2958452..0000000
--- a/youtube_dl/extractor/fktv.py
+++ /dev/null
@@ -1,51 +0,0 @@
-from __future__ import unicode_literals
-
-from .common import InfoExtractor
-from ..utils import (
-    clean_html,
-    determine_ext,
-    js_to_json,
-)
-
-
-class FKTVIE(InfoExtractor):
-    IE_NAME = 'fernsehkritik.tv'
-    _VALID_URL = r'https?://(?:www\.)?fernsehkritik\.tv/folge-(?P<id>[0-9]+)(?:/.*)?'
-
-    _TEST = {
-        'url': 'http://fernsehkritik.tv/folge-1',
-        'md5': '21f0b0c99bce7d5b524eb1b17b1c6d79',
-        'info_dict': {
-            'id': '1',
-            'ext': 'mp4',
-            'title': 'Folge 1 vom 10. April 2007',
-            'thumbnail': r're:^https?://.*\.jpg$',
-        },
-    }
-
-    def _real_extract(self, url):
-        episode = self._match_id(url)
-
-        webpage = self._download_webpage(
-            'http://fernsehkritik.tv/folge-%s/play' % episode, episode)
-        title = clean_html(self._html_search_regex(
-            '<h3>([^<]+)</h3>', webpage, 'title'))
-        thumbnail = self._search_regex(r'POSTER\s*=\s*"([^"]+)', webpage, 'thumbnail', fatal=False)
-        sources = self._parse_json(self._search_regex(r'(?s)MEDIA\s*=\s*(\[.+?\]);', webpage, 'media'), episode, js_to_json)
-
-        formats = []
-        for source in sources:
-            furl = source.get('src')
-            if furl:
-                formats.append({
-                    'url': furl,
-                    'format_id': determine_ext(furl),
-                })
-        self._sort_formats(formats)
-
-        return {
-            'id': episode,
-            'title': title,
-            'formats': formats,
-            'thumbnail': thumbnail,
-        }
diff --git a/youtube_dl/extractor/massengeschmacktv.py b/youtube_dl/extractor/massengeschmacktv.py
new file mode 100644
index 0000000..cfcc6b2
--- /dev/null
+++ b/youtube_dl/extractor/massengeschmacktv.py
@@ -0,0 +1,77 @@
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    clean_html,
+    determine_ext,
+    int_or_none,
+    js_to_json,
+    mimetype2ext,
+    parse_filesize,
+)
+
+
+class MassengeschmackTVIE(InfoExtractor):
+    IE_NAME = 'massengeschmack.tv'
+    _VALID_URL = r'https?://(?:www\.)?massengeschmack\.tv/play/(?P<id>[^?&#]+)'
+
+    _TEST = {
+        'url': 'https://massengeschmack.tv/play/fktv202',
+        'md5': 'a9e054db9c2b5a08f0a0527cc201e8d3',
+        'info_dict': {
+            'id': 'fktv202',
+            'ext': 'mp4',
+            'title': 'Fernsehkritik-TV - Folge 202',
+        },
+    }
+
+    def _real_extract(self, url):
+        episode = self._match_id(url)
+
+        webpage = self._download_webpage(url, episode)
+        title = clean_html(self._html_search_regex(
+            '<h3>([^<]+)</h3>', webpage, 'title'))
+        thumbnail = self._search_regex(r'POSTER\s*=\s*"([^"]+)', webpage, 'thumbnail', fatal=False)
+        sources = self._parse_json(self._search_regex(r'(?s)MEDIA\s*=\s*(\[.+?\]);', webpage, 'media'), episode, js_to_json)
+
+        formats = []
+        for source in sources:
+            furl = source.get('src')
+            if not furl:
+                continue
+            furl = self._proto_relative_url(furl)
+            ext = determine_ext(furl) or mimetype2ext(source.get('type'))
+            if ext == 'm3u8':
+                formats.extend(self._extract_m3u8_formats(
+                    furl, episode, 'mp4', 'm3u8_native',
+                    m3u8_id='hls', fatal=False))
+            else:
+                formats.append({
+                    'url': furl,
+                    'format_id': determine_ext(furl),
+                })
+
+        for (durl, format_id, width, height, filesize) in re.findall(r'''(?x)
+                                   <a[^>]+?href="(?P<url>(?:https:)?//[^"]+)".*?
+                                   <strong>(?P<format_id>.+?)</strong>.*?
+                                   <small>(?:(?P<width>\d+)x(?P<height>\d+))?\s+?\((?P<filesize>[\d,]+\s*[GM]iB)\)</small>
+                                ''', webpage):
+            formats.append({
+                'url': durl,
+                'format_id': format_id,
+                'width': int_or_none(width),
+                'height': int_or_none(height),
+                'filesize': parse_filesize(filesize),
+                'vcodec': 'none' if format_id.startswith('Audio') else None,
+            })
+
+        self._sort_formats(formats, ('width', 'height', 'filesize', 'tbr'))
+
+        return {
+            'id': episode,
+            'title': title,
+            'formats': formats,
+            'thumbnail': thumbnail,
+        }
-- 
cgit v1.1


From 6f5c598a28b2b400ffb4d2bc2d228c2802ab7fd7 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sat, 25 Nov 2017 15:49:49 +0100
Subject: [france2.fr:generation-what] fix extraction

---
 youtube_dl/extractor/francetv.py | 38 +++++++++++++++++++-------------------
 1 file changed, 19 insertions(+), 19 deletions(-)

diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py
index 037e538..5a3abea 100644
--- a/youtube_dl/extractor/francetv.py
+++ b/youtube_dl/extractor/francetv.py
@@ -3,7 +3,6 @@
 from __future__ import unicode_literals
 
 import re
-import json
 
 from .common import InfoExtractor
 from ..compat import compat_urlparse
@@ -308,31 +307,32 @@ class FranceTVInfoIE(FranceTVBaseInfoExtractor):
         return self._extract_video(video_id, catalogue)
 
 
-class GenerationQuoiIE(InfoExtractor):
-    IE_NAME = 'france2.fr:generation-quoi'
-    _VALID_URL = r'https?://generation-quoi\.france2\.fr/portrait/(?P<id>[^/?#]+)'
+class GenerationWhatIE(InfoExtractor):
+    IE_NAME = 'france2.fr:generation-what'
+    _VALID_URL = r'https?://generation-what\.francetv\.fr/[^/]+/video/(?P<id>[^/?#]+)'
 
-    _TEST = {
-        'url': 'http://generation-quoi.france2.fr/portrait/garde-a-vous',
+    _TESTS = [{
+        'url': 'http://generation-what.francetv.fr/portrait/video/present-arms',
         'info_dict': {
-            'id': 'k7FJX8VBcvvLmX4wA5Q',
+            'id': 'wtvKYUG45iw',
             'ext': 'mp4',
-            'title': 'Génération Quoi - Garde à Vous',
-            'uploader': 'Génération Quoi',
-        },
-        'params': {
-            # It uses Dailymotion
-            'skip_download': True,
+            'title': 'Generation What - Garde à vous - FRA',
+            'uploader': 'Generation What',
+            'uploader_id': 'UCHH9p1eetWCgt4kXBYCb3_w',
+            'upload_date': '20160411',
         },
-    }
+    }, {
+        'url': 'http://generation-what.francetv.fr/europe/video/present-arms',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         display_id = self._match_id(url)
-        info_url = compat_urlparse.urljoin(url, '/medias/video/%s.json' % display_id)
-        info_json = self._download_webpage(info_url, display_id)
-        info = json.loads(info_json)
-        return self.url_result('http://www.dailymotion.com/video/%s' % info['id'],
-                               ie='Dailymotion')
+        webpage = self._download_webpage(url, display_id)
+        youtube_id = self._search_regex(
+            r"window\.videoURL\s*=\s*'([0-9A-Za-z_-]{11})';",
+            webpage, 'youtube id')
+        return self.url_result(youtube_id, 'Youtube', youtube_id)
 
 
 class CultureboxIE(FranceTVBaseInfoExtractor):
-- 
cgit v1.1


From fe4bfe36e148c0e1d3180a2992503536b4b4f84b Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sat, 25 Nov 2017 15:58:28 +0100
Subject: [gamersyde] remove extractor

---
 youtube_dl/extractor/extractors.py |  3 +-
 youtube_dl/extractor/gamersyde.py  | 70 --------------------------------------
 2 files changed, 1 insertion(+), 72 deletions(-)
 delete mode 100644 youtube_dl/extractor/gamersyde.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index f1a4649..bc1ca0f 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -374,7 +374,7 @@ from .francetv import (
     FranceTVIE,
     FranceTVEmbedIE,
     FranceTVInfoIE,
-    GenerationQuoiIE,
+    GenerationWhatIE,
     CultureboxIE,
 )
 from .freesound import FreesoundIE
@@ -390,7 +390,6 @@ from .gameone import (
     GameOneIE,
     GameOnePlaylistIE,
 )
-from .gamersyde import GamersydeIE
 from .gamespot import GameSpotIE
 from .gamestar import GameStarIE
 from .gaskrank import GaskrankIE
diff --git a/youtube_dl/extractor/gamersyde.py b/youtube_dl/extractor/gamersyde.py
deleted file mode 100644
index a218a69..0000000
--- a/youtube_dl/extractor/gamersyde.py
+++ /dev/null
@@ -1,70 +0,0 @@
-from __future__ import unicode_literals
-
-import re
-
-from .common import InfoExtractor
-from ..utils import (
-    js_to_json,
-    parse_duration,
-    remove_start,
-)
-
-
-class GamersydeIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?gamersyde\.com/hqstream_(?P<display_id>[\da-z_]+)-(?P<id>\d+)_[a-z]{2}\.html'
-    _TEST = {
-        'url': 'http://www.gamersyde.com/hqstream_bloodborne_birth_of_a_hero-34371_en.html',
-        'md5': 'f38d400d32f19724570040d5ce3a505f',
-        'info_dict': {
-            'id': '34371',
-            'ext': 'mp4',
-            'duration': 372,
-            'title': 'Bloodborne - Birth of a hero',
-            'thumbnail': r're:^https?://.*\.jpg$',
-        }
-    }
-
-    def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-        display_id = mobj.group('display_id')
-
-        webpage = self._download_webpage(url, display_id)
-
-        playlist = self._parse_json(
-            self._search_regex(
-                r'(?s)playlist: \[({.+?})\]\s*}\);', webpage, 'files'),
-            display_id, transform_source=js_to_json)
-
-        formats = []
-        for source in playlist['sources']:
-            video_url = source.get('file')
-            if not video_url:
-                continue
-            format_id = source.get('label')
-            f = {
-                'url': video_url,
-                'format_id': format_id,
-            }
-            m = re.search(r'^(?P<height>\d+)[pP](?P<fps>\d+)fps', format_id)
-            if m:
-                f.update({
-                    'height': int(m.group('height')),
-                    'fps': int(m.group('fps')),
-                })
-            formats.append(f)
-        self._sort_formats(formats)
-
-        title = remove_start(playlist['title'], '%s - ' % video_id)
-        thumbnail = playlist.get('image')
-        duration = parse_duration(self._search_regex(
-            r'Length:</label>([^<]+)<', webpage, 'duration', fatal=False))
-
-        return {
-            'id': video_id,
-            'display_id': display_id,
-            'title': title,
-            'thumbnail': thumbnail,
-            'duration': duration,
-            'formats': formats,
-        }
-- 
cgit v1.1


From dbb25af657166108e7afcd4e1ff197e2a5644724 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 25 Nov 2017 23:08:46 +0700
Subject: [tnaflix] Don't modify download URLs (closes #14811)

---
 youtube_dl/extractor/tnaflix.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/tnaflix.py b/youtube_dl/extractor/tnaflix.py
index 7e6ec34..9f47ce2 100644
--- a/youtube_dl/extractor/tnaflix.py
+++ b/youtube_dl/extractor/tnaflix.py
@@ -91,7 +91,8 @@ class TNAFlixNetworkBaseIE(InfoExtractor):
         formats = []
 
         def extract_video_url(vl):
-            return re.sub(r'speed=\d+', 'speed=', unescapeHTML(vl.text))
+            # Any URL modification now results in HTTP Error 403: Forbidden
+            return unescapeHTML(vl.text)
 
         video_link = cfg_xml.find('./videoLink')
         if video_link is not None:
-- 
cgit v1.1


From 9105523818270cadb7ad90a3b1ea7e6f18d2dc54 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 25 Nov 2017 23:13:17 +0700
Subject: [empflix] Fix extractrion

---
 youtube_dl/extractor/tnaflix.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/tnaflix.py b/youtube_dl/extractor/tnaflix.py
index 9f47ce2..c4ead1f 100644
--- a/youtube_dl/extractor/tnaflix.py
+++ b/youtube_dl/extractor/tnaflix.py
@@ -21,6 +21,8 @@ class TNAFlixNetworkBaseIE(InfoExtractor):
         r'flashvars\.config\s*=\s*escape\("([^"]+)"',
         r'<input[^>]+name="config\d?" value="([^"]+)"',
     ]
+    _HOST = 'tna'
+    _VKEY_SUFFIX = ''
     _TITLE_REGEX = r'<input[^>]+name="title" value="([^"]+)"'
     _DESCRIPTION_REGEX = r'<input[^>]+name="description" value="([^"]+)"'
     _UPLOADER_REGEX = r'<input[^>]+name="username" value="([^"]+)"'
@@ -81,8 +83,8 @@ class TNAFlixNetworkBaseIE(InfoExtractor):
 
         if not cfg_url:
             inputs = self._hidden_inputs(webpage)
-            cfg_url = ('https://cdn-fck.tnaflix.com/tnaflix/%s.fid?key=%s&VID=%s&premium=1&vip=1&alpha'
-                       % (inputs['vkey'], inputs['nkey'], video_id))
+            cfg_url = ('https://cdn-fck.%sflix.com/%sflix/%s%s.fid?key=%s&VID=%s&premium=1&vip=1&alpha'
+                       % (self._HOST, self._HOST, inputs['vkey'], self._VKEY_SUFFIX, inputs['nkey'], video_id))
 
         cfg_xml = self._download_xml(
             cfg_url, display_id, 'Downloading metadata',
@@ -240,6 +242,8 @@ class TNAFlixIE(TNAFlixNetworkBaseIE):
 class EMPFlixIE(TNAFlixNetworkBaseIE):
     _VALID_URL = r'https?://(?:www\.)?empflix\.com/videos/(?P<display_id>.+?)-(?P<id>[0-9]+)\.html'
 
+    _HOST = 'emp'
+    _VKEY_SUFFIX = '-1'
     _UPLOADER_REGEX = r'<span[^>]+class="infoTitle"[^>]*>Uploaded By:</span>(.+?)</li>'
 
     _TESTS = [{
-- 
cgit v1.1


From b7785cf15685f7dfbe96cc6d2dd8089ff1363be3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 25 Nov 2017 23:22:54 +0700
Subject: [empflix] Relax _VALID_URL

---
 youtube_dl/extractor/tnaflix.py | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/tnaflix.py b/youtube_dl/extractor/tnaflix.py
index c4ead1f..e57b8e3 100644
--- a/youtube_dl/extractor/tnaflix.py
+++ b/youtube_dl/extractor/tnaflix.py
@@ -74,7 +74,13 @@ class TNAFlixNetworkBaseIE(InfoExtractor):
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
-        display_id = mobj.group('display_id') if 'display_id' in mobj.groupdict() else video_id
+        for display_id_key in ('display_id', 'display_id_2'):
+            if display_id_key in mobj.groupdict():
+                display_id = mobj.group(display_id_key)
+                if display_id:
+                    break
+        else:
+            display_id = video_id
 
         webpage = self._download_webpage(url, display_id)
 
@@ -240,7 +246,7 @@ class TNAFlixIE(TNAFlixNetworkBaseIE):
 
 
 class EMPFlixIE(TNAFlixNetworkBaseIE):
-    _VALID_URL = r'https?://(?:www\.)?empflix\.com/videos/(?P<display_id>.+?)-(?P<id>[0-9]+)\.html'
+    _VALID_URL = r'https?://(?:www\.)?empflix\.com/(?:videos/(?P<display_id>.+?)-|[^/]+/(?P<display_id_2>[^/]+)/video)(?P<id>[0-9]+)'
 
     _HOST = 'emp'
     _VKEY_SUFFIX = '-1'
@@ -264,6 +270,9 @@ class EMPFlixIE(TNAFlixNetworkBaseIE):
     }, {
         'url': 'http://www.empflix.com/videos/[AROMA][ARMD-718]-Aoi-Yoshino-Sawa-25826.html',
         'only_matching': True,
+    }, {
+        'url': 'https://www.empflix.com/amateur-porn/Amateur-Finger-Fuck/video33051',
+        'only_matching': True,
     }]
 
 
-- 
cgit v1.1


From 8cfbcfab9a94d363dbc8e1b026648c3ccf0d399e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 25 Nov 2017 23:37:50 +0700
Subject: [tnaflix] Extract common parts of tnaflix and empflix

---
 youtube_dl/extractor/tnaflix.py | 20 +++++++++++---------
 1 file changed, 11 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/tnaflix.py b/youtube_dl/extractor/tnaflix.py
index e57b8e3..0c2f8f1 100644
--- a/youtube_dl/extractor/tnaflix.py
+++ b/youtube_dl/extractor/tnaflix.py
@@ -201,18 +201,21 @@ class TNAFlixNetworkEmbedIE(TNAFlixNetworkBaseIE):
             webpage)]
 
 
-class TNAFlixIE(TNAFlixNetworkBaseIE):
+class TNAEMPFlixBaseIE(TNAFlixNetworkBaseIE):
+    _DESCRIPTION_REGEX = r'(?s)>Description:</[^>]+>(.+?)<'
+    _UPLOADER_REGEX = r'<span>by\s*<a[^>]+\bhref=["\']/profile/[^>]+>([^<]+)<'
+    _CATEGORIES_REGEX = r'(?s)<span[^>]*>Categories:</span>(.+?)</div>'
+
+
+class TNAFlixIE(TNAEMPFlixBaseIE):
     _VALID_URL = r'https?://(?:www\.)?tnaflix\.com/[^/]+/(?P<display_id>[^/]+)/video(?P<id>\d+)'
 
     _TITLE_REGEX = r'<title>(.+?) - (?:TNAFlix Porn Videos|TNAFlix\.com)</title>'
-    _DESCRIPTION_REGEX = r'(?s)>Description:</[^>]+>(.+?)<'
-    _UPLOADER_REGEX = r'<i>\s*Verified Member\s*</i>\s*<h\d+>(.+?)<'
-    _CATEGORIES_REGEX = r'(?s)<span[^>]*>Categories:</span>(.+?)</div>'
 
     _TESTS = [{
         # anonymous uploader, no categories
         'url': 'http://www.tnaflix.com/porn-stars/Carmella-Decesare-striptease/video553878',
-        'md5': 'ecf3498417d09216374fc5907f9c6ec0',
+        'md5': '7e569419fe6d69543d01e6be22f5f7c4',
         'info_dict': {
             'id': '553878',
             'display_id': 'Carmella-Decesare-striptease',
@@ -237,7 +240,7 @@ class TNAFlixIE(TNAFlixNetworkBaseIE):
             'duration': 164,
             'age_limit': 18,
             'uploader': 'bobwhite39',
-            'categories': ['Amateur Porn', 'Squirting Videos', 'Teen Girls 18+'],
+            'categories': list,
         }
     }, {
         'url': 'https://www.tnaflix.com/amateur-porn/bunzHD-Ms.Donk/video358632',
@@ -245,16 +248,15 @@ class TNAFlixIE(TNAFlixNetworkBaseIE):
     }]
 
 
-class EMPFlixIE(TNAFlixNetworkBaseIE):
+class EMPFlixIE(TNAEMPFlixBaseIE):
     _VALID_URL = r'https?://(?:www\.)?empflix\.com/(?:videos/(?P<display_id>.+?)-|[^/]+/(?P<display_id_2>[^/]+)/video)(?P<id>[0-9]+)'
 
     _HOST = 'emp'
     _VKEY_SUFFIX = '-1'
-    _UPLOADER_REGEX = r'<span[^>]+class="infoTitle"[^>]*>Uploaded By:</span>(.+?)</li>'
 
     _TESTS = [{
         'url': 'http://www.empflix.com/videos/Amateur-Finger-Fuck-33051.html',
-        'md5': 'b1bc15b6412d33902d6e5952035fcabc',
+        'md5': 'bc30d48b91a7179448a0bda465114676',
         'info_dict': {
             'id': '33051',
             'display_id': 'Amateur-Finger-Fuck',
-- 
cgit v1.1


From 9ef909f2b2bf81e16c4fd2e428a740a523ad61cf Mon Sep 17 00:00:00 2001
From: jahudka <me@subsonic.cz>
Date: Sat, 25 Nov 2017 18:04:13 +0100
Subject: [openload] Add support for openload.link

---
 youtube_dl/extractor/openload.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/openload.py b/youtube_dl/extractor/openload.py
index b50d6c7..efa5d46 100644
--- a/youtube_dl/extractor/openload.py
+++ b/youtube_dl/extractor/openload.py
@@ -242,7 +242,7 @@ class PhantomJSwrapper(object):
 
 
 class OpenloadIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:openload\.(?:co|io)|oload\.tv)/(?:f|embed)/(?P<id>[a-zA-Z0-9-_]+)'
+    _VALID_URL = r'https?://(?:www\.)?(?:openload\.(?:co|io|link)|oload\.tv)/(?:f|embed)/(?P<id>[a-zA-Z0-9-_]+)'
 
     _TESTS = [{
         'url': 'https://openload.co/f/kUEfGclsU9o',
@@ -286,6 +286,9 @@ class OpenloadIE(InfoExtractor):
     }, {
         'url': 'https://oload.tv/embed/KnG-kKZdcfY/',
         'only_matching': True,
+    }, {
+        'url': 'http://www.openload.link/f/KnG-kKZdcfY',
+        'only_matching': True,
     }]
 
     _USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36'
-- 
cgit v1.1


From 6ff27b8d5a11f0960f53b93a1ea2423d3384af21 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 26 Nov 2017 00:05:28 +0700
Subject: [openload] Don't use bare except when removing temp files

---
 youtube_dl/extractor/openload.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/openload.py b/youtube_dl/extractor/openload.py
index efa5d46..a99af12 100644
--- a/youtube_dl/extractor/openload.py
+++ b/youtube_dl/extractor/openload.py
@@ -140,7 +140,7 @@ class PhantomJSwrapper(object):
         for name in self._TMP_FILE_NAMES:
             try:
                 os.remove(self._TMP_FILES[name].name)
-            except:
+            except (IOError, OSError):
                 pass
 
     def _save_cookies(self, url):
-- 
cgit v1.1


From c0f647a179b8d75a5f23a06f208ff8c1c0088a54 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sat, 25 Nov 2017 18:13:17 +0100
Subject: [nexx] extract more formats

---
 youtube_dl/extractor/nexx.py | 69 ++++++++++++++++++++++++++++++++------------
 1 file changed, 50 insertions(+), 19 deletions(-)

diff --git a/youtube_dl/extractor/nexx.py b/youtube_dl/extractor/nexx.py
index 071879b..d79a1ce 100644
--- a/youtube_dl/extractor/nexx.py
+++ b/youtube_dl/extractor/nexx.py
@@ -28,7 +28,7 @@ class NexxIE(InfoExtractor):
     _TESTS = [{
         # movie
         'url': 'https://api.nexx.cloud/v3/748/videos/byid/128907',
-        'md5': '16746bfc28c42049492385c989b26c4a',
+        'md5': '828cea195be04e66057b846288295ba1',
         'info_dict': {
             'id': '128907',
             'ext': 'mp4',
@@ -42,9 +42,6 @@ class NexxIE(InfoExtractor):
             'timestamp': 1384264416,
             'upload_date': '20131112',
         },
-        'params': {
-            'format': 'bestvideo',
-        },
     }, {
         # episode
         'url': 'https://api.nexx.cloud/v3/741/videos/byid/247858',
@@ -62,7 +59,6 @@ class NexxIE(InfoExtractor):
             'season_number': 2,
         },
         'params': {
-            'format': 'bestvideo',
             'skip_download': True,
         },
     }, {
@@ -193,35 +189,70 @@ class NexxIE(InfoExtractor):
         stream_data = video['streamdata']
         language = general.get('language_raw') or ''
 
-        # TODO: reverse more cdns and formats
+        # TODO: reverse more cdns
 
         cdn = stream_data['cdnType']
         assert cdn == 'azure'
 
         azure_locator = stream_data['azureLocator']
 
-        AZURE_URL = 'http://nx-p%02d.akamaized.net/'
+        AZURE_URL = 'http://nx%s%02d.akamaized.net/'
 
-        for secure in ('s', ''):
-            cdn_shield = stream_data.get('cdnShieldHTTP%s' % secure.upper())
-            if cdn_shield:
-                azure_base = 'http%s://%s' % (secure, cdn_shield)
-                break
-        else:
-            azure_base = AZURE_URL % int(stream_data['azureAccount'].replace('nexxplayplus', ''))
+        def get_cdn_base(cdn, prefix='-p'):
+            azure_base = None
+            for secure in ('', 's'):
+                cdn_shield = stream_data.get('cdn%sHTTP%s' % (cdn, secure.upper()))
+                if cdn_shield:
+                    azure_base = 'http%s://%s' % (secure, cdn_shield)
+                    break
+            else:
+                azure_base = AZURE_URL % (prefix, int(stream_data['azureAccount'].replace('nexxplayplus', '')))
+            return azure_base
 
+        azure_stream_base = get_cdn_base('Shield')
         is_ml = ',' in language
-        azure_m3u8_url = '%s%s/%s_src%s.ism/Manifest(format=m3u8-aapl)' % (
-            azure_base, azure_locator, video_id, ('_manifest' if is_ml else ''))
+        azure_manifest_url = '%s%s/%s_src%s.ism/Manifest' % (
+            azure_stream_base, azure_locator, video_id, ('_manifest' if is_ml else '')) + '%s'
 
         protection_token = try_get(
             video, lambda x: x['protectiondata']['token'], compat_str)
         if protection_token:
-            azure_m3u8_url += '?hdnts=%s' % protection_token
+            azure_manifest_url += '?hdnts=%s' % protection_token
 
         formats = self._extract_m3u8_formats(
-            azure_m3u8_url, video_id, 'mp4', entry_protocol='m3u8_native',
-            m3u8_id='%s-hls' % cdn)
+            azure_manifest_url % '(format=m3u8-aapl)',
+            video_id, 'mp4', 'm3u8_native',
+            m3u8_id='%s-hls' % cdn, fatal=False)
+        formats.extend(self._extract_mpd_formats(
+            azure_manifest_url % '(format=mpd-time-csf)',
+            video_id, mpd_id='%s-dash' % cdn, fatal=False))
+        formats.extend(self._extract_ism_formats(
+            azure_manifest_url % '', video_id, ism_id='%s-mss' % cdn, fatal=False))
+
+        azure_progressive_base = get_cdn_base('ShieldProg', '-d')
+        azure_file_distribution = stream_data.get('azureFileDistribution')
+        if azure_file_distribution:
+            fds = azure_file_distribution.split(',')
+            if fds:
+                for fd in fds:
+                    ss = fd.split(':')
+                    if len(ss) == 2:
+                        tbr = int_or_none(ss[0])
+                        if tbr:
+                            f = {
+                                'url': '%s%s/%s_src_%s_%d.mp4' % (
+                                    azure_progressive_base, azure_locator, video_id, ss[1], tbr),
+                                'format_id': 'http-%d' % tbr,
+                                'tbr': tbr,
+                            }
+                            width_height = ss[1].split('x')
+                            if len(width_height) == 2:
+                                f.update({
+                                    'width': int_or_none(width_height[0]),
+                                    'height': int_or_none(width_height[1]),
+                                })
+                            formats.append(f)
+
         self._sort_formats(formats)
 
         return {
-- 
cgit v1.1


From a238a868ba33b4683cea57a05893b339a302ff9f Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sat, 25 Nov 2017 18:25:00 +0100
Subject: [freespeech] fix extraction

---
 youtube_dl/extractor/freespeech.py | 31 ++++++++++++++-----------------
 1 file changed, 14 insertions(+), 17 deletions(-)

diff --git a/youtube_dl/extractor/freespeech.py b/youtube_dl/extractor/freespeech.py
index 7fa271b..486a49c 100644
--- a/youtube_dl/extractor/freespeech.py
+++ b/youtube_dl/extractor/freespeech.py
@@ -1,37 +1,34 @@
 from __future__ import unicode_literals
 
-import re
-import json
-
 from .common import InfoExtractor
 
 
 class FreespeechIE(InfoExtractor):
     IE_NAME = 'freespeech.org'
-    _VALID_URL = r'https?://(?:www\.)?freespeech\.org/video/(?P<title>.+)'
+    _VALID_URL = r'https?://(?:www\.)?freespeech\.org/stories/(?P<id>.+)'
     _TEST = {
         'add_ie': ['Youtube'],
-        'url': 'https://www.freespeech.org/video/obama-romney-campaign-colorado-ahead-debate-0',
+        'url': 'http://www.freespeech.org/stories/fcc-announces-net-neutrality-rollback-whats-stake/',
         'info_dict': {
-            'id': 'poKsVCZ64uU',
-            'ext': 'webm',
-            'title': 'Obama, Romney Campaign in Colorado Ahead of Debate',
-            'description': 'Obama, Romney Campaign in Colorado Ahead of Debate',
-            'uploader': 'freespeechtv',
+            'id': 'waRk6IPqyWM',
+            'ext': 'mp4',
+            'title': 'What\'s At Stake - Net Neutrality Special',
+            'description': 'Presented by MNN and FSTV',
+            'upload_date': '20170728',
             'uploader_id': 'freespeechtv',
-            'upload_date': '20121002',
+            'uploader': 'freespeechtv',
         },
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        title = mobj.group('title')
-        webpage = self._download_webpage(url, title)
-        info_json = self._search_regex(r'jQuery\.extend\(Drupal\.settings, ({.*?})\);', webpage, 'info')
-        info = json.loads(info_json)
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+        youtube_url = self._search_regex(
+            r'data-video-url="([^"]+)"',
+            webpage, 'youtube url')
 
         return {
             '_type': 'url',
-            'url': info['jw_player']['basic_video_node_player']['file'],
+            'url': youtube_url,
             'ie_key': 'Youtube',
         }
-- 
cgit v1.1


From b485d5d6bf2052edf6d7218a0e4afa89884f13c6 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sat, 25 Nov 2017 18:36:31 +0100
Subject: [nexx] make http format ids more consistent

---
 youtube_dl/extractor/nexx.py | 17 +++++++----------
 1 file changed, 7 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/nexx.py b/youtube_dl/extractor/nexx.py
index d79a1ce..9203c04 100644
--- a/youtube_dl/extractor/nexx.py
+++ b/youtube_dl/extractor/nexx.py
@@ -198,18 +198,15 @@ class NexxIE(InfoExtractor):
 
         AZURE_URL = 'http://nx%s%02d.akamaized.net/'
 
-        def get_cdn_base(cdn, prefix='-p'):
-            azure_base = None
+        def get_cdn_shield_base(shield_type='', prefix='-p'):
             for secure in ('', 's'):
-                cdn_shield = stream_data.get('cdn%sHTTP%s' % (cdn, secure.upper()))
+                cdn_shield = stream_data.get('cdnShield%sHTTP%s' % (shield_type, secure.upper()))
                 if cdn_shield:
-                    azure_base = 'http%s://%s' % (secure, cdn_shield)
-                    break
+                    return 'http%s://%s' % (secure, cdn_shield)
             else:
-                azure_base = AZURE_URL % (prefix, int(stream_data['azureAccount'].replace('nexxplayplus', '')))
-            return azure_base
+                return AZURE_URL % (prefix, int(stream_data['azureAccount'].replace('nexxplayplus', '')))
 
-        azure_stream_base = get_cdn_base('Shield')
+        azure_stream_base = get_cdn_shield_base()
         is_ml = ',' in language
         azure_manifest_url = '%s%s/%s_src%s.ism/Manifest' % (
             azure_stream_base, azure_locator, video_id, ('_manifest' if is_ml else '')) + '%s'
@@ -229,7 +226,7 @@ class NexxIE(InfoExtractor):
         formats.extend(self._extract_ism_formats(
             azure_manifest_url % '', video_id, ism_id='%s-mss' % cdn, fatal=False))
 
-        azure_progressive_base = get_cdn_base('ShieldProg', '-d')
+        azure_progressive_base = get_cdn_shield_base('Prog', '-d')
         azure_file_distribution = stream_data.get('azureFileDistribution')
         if azure_file_distribution:
             fds = azure_file_distribution.split(',')
@@ -242,7 +239,7 @@ class NexxIE(InfoExtractor):
                             f = {
                                 'url': '%s%s/%s_src_%s_%d.mp4' % (
                                     azure_progressive_base, azure_locator, video_id, ss[1], tbr),
-                                'format_id': 'http-%d' % tbr,
+                                'format_id': '%s-http-%d' % (cdn, tbr),
                                 'tbr': tbr,
                             }
                             width_height = ss[1].split('x')
-- 
cgit v1.1


From 87dac57cf69f7a4528c511dc7c1b9a690199cf1e Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sat, 25 Nov 2017 18:50:15 +0100
Subject: [firstpost] remove extractor

---
 youtube_dl/extractor/extractors.py |  1 -
 youtube_dl/extractor/firstpost.py  | 50 --------------------------------------
 2 files changed, 51 deletions(-)
 delete mode 100644 youtube_dl/extractor/firstpost.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index bc1ca0f..d3104f3 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -344,7 +344,6 @@ from .filmon import (
     FilmOnIE,
     FilmOnChannelIE,
 )
-from .firstpost import FirstpostIE
 from .firsttv import FirstTVIE
 from .fivemin import FiveMinIE
 from .fivetv import FiveTVIE
diff --git a/youtube_dl/extractor/firstpost.py b/youtube_dl/extractor/firstpost.py
deleted file mode 100644
index e8936cb..0000000
--- a/youtube_dl/extractor/firstpost.py
+++ /dev/null
@@ -1,50 +0,0 @@
-from __future__ import unicode_literals
-
-from .common import InfoExtractor
-
-
-class FirstpostIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?firstpost\.com/[^/]+/.*-(?P<id>[0-9]+)\.html'
-
-    _TEST = {
-        'url': 'http://www.firstpost.com/india/india-to-launch-indigenous-aircraft-carrier-monday-1025403.html',
-        'md5': 'ee9114957692f01fb1263ed87039112a',
-        'info_dict': {
-            'id': '1025403',
-            'ext': 'mp4',
-            'title': 'India to launch indigenous aircraft carrier INS Vikrant today',
-            'description': 'md5:feef3041cb09724e0bdc02843348f5f4',
-        }
-    }
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        page = self._download_webpage(url, video_id)
-
-        title = self._html_search_meta('twitter:title', page, 'title', fatal=True)
-        description = self._html_search_meta('twitter:description', page, 'title')
-
-        data = self._download_xml(
-            'http://www.firstpost.com/getvideoxml-%s.xml' % video_id, video_id,
-            'Downloading video XML')
-
-        item = data.find('./playlist/item')
-        thumbnail = item.find('./image').text
-
-        formats = [
-            {
-                'url': details.find('./file').text,
-                'format_id': details.find('./label').text.strip(),
-                'width': int(details.find('./width').text.strip()),
-                'height': int(details.find('./height').text.strip()),
-            } for details in item.findall('./source/file_details') if details.find('./file').text
-        ]
-        self._sort_formats(formats)
-
-        return {
-            'id': video_id,
-            'title': title,
-            'description': description,
-            'thumbnail': thumbnail,
-            'formats': formats,
-        }
-- 
cgit v1.1


From 93f3f10cdcf40bdeb6c61b36bbbc0d0ab45ba66c Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sat, 25 Nov 2017 19:28:26 +0100
Subject: [fczenit] fix extraction

---
 youtube_dl/extractor/fczenit.py | 40 ++++++++++++++++++++++++----------------
 1 file changed, 24 insertions(+), 16 deletions(-)

diff --git a/youtube_dl/extractor/fczenit.py b/youtube_dl/extractor/fczenit.py
index 8d1010b..8db7c59 100644
--- a/youtube_dl/extractor/fczenit.py
+++ b/youtube_dl/extractor/fczenit.py
@@ -2,7 +2,10 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..compat import compat_urlparse
+from ..utils import (
+    int_or_none,
+    float_or_none,
+)
 
 
 class FczenitIE(InfoExtractor):
@@ -14,6 +17,8 @@ class FczenitIE(InfoExtractor):
             'id': '41044',
             'ext': 'mp4',
             'title': 'Так пишется история: казанский разгром ЦСКА на «Зенит-ТВ»',
+            'timestamp': 1462283735,
+            'upload_date': '20160503',
         },
     }
 
@@ -21,28 +26,31 @@ class FczenitIE(InfoExtractor):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        video_title = self._html_search_regex(
-            r'<[^>]+class=\"photoalbum__title\">([^<]+)', webpage, 'title')
+        msi_id = self._search_regex(
+            r"(?s)config\s*=\s*{.+?video_id\s*:\s*'([^']+)'", webpage, 'msi id')
 
-        video_items = self._parse_json(self._search_regex(
-            r'arrPath\s*=\s*JSON\.parse\(\'(.+)\'\)', webpage, 'video items'),
-            video_id)
-
-        def merge_dicts(*dicts):
-            ret = {}
-            for a_dict in dicts:
-                ret.update(a_dict)
-            return ret
+        msi_data = self._download_json(
+            'http://player.fc-zenit.ru/msi/video', msi_id, query={
+                'video': msi_id,
+            })['data']
+        title = msi_data['name']
 
         formats = [{
-            'url': compat_urlparse.urljoin(url, video_url),
-            'tbr': int(tbr),
-        } for tbr, video_url in merge_dicts(*video_items).items()]
+            'format_id': q.get('label'),
+            'url': q['url'],
+            'height': int_or_none(q.get('label')),
+        } for q in msi_data['qualities'] if q.get('url')]
 
         self._sort_formats(formats)
 
+        tags = [tag['label'] for tag in msi_data.get('tags', []) if tag.get('label')]
+
         return {
             'id': video_id,
-            'title': video_title,
+            'title': title,
+            'thumbnail': msi_data.get('preview'),
             'formats': formats,
+            'duration': float_or_none(msi_data.get('duration')),
+            'timestamp': int_or_none(msi_data.get('date')),
+            'tags': tags,
         }
-- 
cgit v1.1


From 7512aa986f517c301fb3272584d21c8d5cec720f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 26 Nov 2017 20:53:10 +0700
Subject: Fix some only matching tests (closes #14855)

---
 youtube_dl/extractor/orf.py     | 6 +++---
 youtube_dl/extractor/rozhlas.py | 2 +-
 youtube_dl/extractor/vidzi.py   | 4 ++--
 youtube_dl/extractor/youku.py   | 2 +-
 4 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/orf.py b/youtube_dl/extractor/orf.py
index 74fe801..c1fb580 100644
--- a/youtube_dl/extractor/orf.py
+++ b/youtube_dl/extractor/orf.py
@@ -49,13 +49,13 @@ class ORFTVthekIE(InfoExtractor):
         'params': {
             'skip_download': True,  # rtsp downloads
         },
-        '_skip': 'Blocked outside of Austria / Germany',
+        'skip': 'Blocked outside of Austria / Germany',
     }, {
         'url': 'http://tvthek.orf.at/topic/Fluechtlingskrise/10463081/Heimat-Fremde-Heimat/13879132/Senioren-betreuen-Migrantenkinder/13879141',
-        'skip_download': True,
+        'only_matching': True,
     }, {
         'url': 'http://tvthek.orf.at/profile/Universum/35429',
-        'skip_download': True,
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
diff --git a/youtube_dl/extractor/rozhlas.py b/youtube_dl/extractor/rozhlas.py
index f8eda8d..fccf694 100644
--- a/youtube_dl/extractor/rozhlas.py
+++ b/youtube_dl/extractor/rozhlas.py
@@ -21,7 +21,7 @@ class RozhlasIE(InfoExtractor):
         }
     }, {
         'url': 'http://prehravac.rozhlas.cz/audio/3421320/embed',
-        'skip_download': True,
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
diff --git a/youtube_dl/extractor/vidzi.py b/youtube_dl/extractor/vidzi.py
index e64873b..ac35d55 100644
--- a/youtube_dl/extractor/vidzi.py
+++ b/youtube_dl/extractor/vidzi.py
@@ -28,10 +28,10 @@ class VidziIE(InfoExtractor):
         },
     }, {
         'url': 'http://vidzi.tv/embed-4z2yb0rzphe9-600x338.html',
-        'skip_download': True,
+        'only_matching': True,
     }, {
         'url': 'http://vidzi.cc/cghql9yq6emu.html',
-        'skip_download': True,
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index 6822a30..a2b3b4d 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -240,7 +240,7 @@ class YoukuShowIE(InfoExtractor):
     }, {
         # Ongoing playlist. The initial page is the last one
         'url': 'http://list.youku.com/show/id_za7c275ecd7b411e1a19e.html',
-        'only_matchine': True,
+        'only_matching': True,
     }]
 
     def _extract_entries(self, playlist_data_url, show_id, note, query):
-- 
cgit v1.1


From d08dcd2dbd4c0d63ff2f4dd504aceceabfa212cc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 26 Nov 2017 21:06:14 +0700
Subject: [test_YoutubeDL] Fix typo (closes #14856)

---
 test/test_YoutubeDL.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index 4af92fb..f0f5a84 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -466,11 +466,11 @@ class TestFormatSelection(unittest.TestCase):
         ydl = YDL({'simulate': True})
         self.assertEqual(ydl._default_format_spec({}), 'bestvideo+bestaudio/best')
 
-        ydl = YDL({'is_live': True})
-        self.assertEqual(ydl._default_format_spec({}), 'best/bestvideo+bestaudio')
+        ydl = YDL({})
+        self.assertEqual(ydl._default_format_spec({'is_live': True}), 'best/bestvideo+bestaudio')
 
-        ydl = YDL({'simulate': True, 'is_live': True})
-        self.assertEqual(ydl._default_format_spec({}), 'bestvideo+bestaudio/best')
+        ydl = YDL({'simulate': True})
+        self.assertEqual(ydl._default_format_spec({'is_live': True}), 'bestvideo+bestaudio/best')
 
         ydl = YDL({'outtmpl': '-'})
         self.assertEqual(ydl._default_format_spec({}), 'best/bestvideo+bestaudio')
-- 
cgit v1.1


From e94d1adc360d0fbfadced82c72a7f20375a1ec87 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 26 Nov 2017 21:10:32 +0700
Subject: Add testdata to youtube-dl.tar.gz (closes #14854)

---
 Makefile | 1 -
 1 file changed, 1 deletion(-)

diff --git a/Makefile b/Makefile
index c74eea7..141593e 100644
--- a/Makefile
+++ b/Makefile
@@ -110,7 +110,6 @@ youtube-dl.tar.gz: youtube-dl README.md README.txt youtube-dl.1 youtube-dl.bash-
 		--exclude '*~' \
 		--exclude '__pycache__' \
 		--exclude '.git' \
-		--exclude 'testdata' \
 		--exclude 'docs/_build' \
 		-- \
 		bin devscripts test youtube_dl docs \
-- 
cgit v1.1


From 6c07f0b288a674c385308cccef31d034b6bd3788 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 26 Nov 2017 21:37:27 +0700
Subject: [ChangeLog] Actualize

---
 ChangeLog | 27 +++++++++++++++++++++++++--
 1 file changed, 25 insertions(+), 2 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index c4a84c5..1f1b704 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,8 +1,31 @@
 version <unreleased>
 
-Extractors
+Core
+* [extractor/common] Use final URL when dumping request (#14769)
+
+Extractors
+* [fczenit] Fix extraction
+- [firstpost] Remove extractor
+* [freespeech] Fix extraction
+* [nexx] Extract more formats
++ [openload] Add support for openload.link (#14763)
+* [empflix] Relax URL regular expression
+* [empflix] Fix extractrion
+* [tnaflix] Don't modify download URLs (#14811)
+- [gamersyde] Remove extractor
+* [francetv:generationwhat] Fix extraction
++ [massengeschmacktv] Add support for Massengeschmack TV
+* [fox9] Fix extraction
+* [faz] Fix extraction and add support for Perform Group embeds (#14714)
++ [performgroup] Add support for performgroup.com
++ [jwplatform] Add support for iframes (#14828)
+* [culturebox] Fix extraction (#14827)
 * [youku] Fix extraction; update ccode (#14815)
-+ [JWPlatform] support iframes (#14828)
+* [livestream] Make SMIL extraction non fatal (#14792)
++ [drtuber] Add support for mobile URLs (#14772)
++ [spankbang] Add support for mobile URLs (#14771)
+* [instagram] Fix description, timestamp and counters extraction (#14755)
+
 
 version 2017.11.15
 
-- 
cgit v1.1


From 5ddeb7702a2c14d51a2c76178c7abb30a4ea06e7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 26 Nov 2017 21:49:22 +0700
Subject: release 2017.11.26

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 docs/supportedsites.md    | 7 +++----
 youtube_dl/version.py     | 2 +-
 4 files changed, 8 insertions(+), 9 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 4dd1a6e..15211e5 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.11.15*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.11.15**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.11.26*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.11.26**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through the [README](https://github.com/rg3/youtube-dl/blob/master/README.md), **most notably** the [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ Add the `-v` flag to **your command line** you run youtube-dl with (`youtube-dl
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.11.15
+[debug] youtube-dl version 2017.11.26
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index 1f1b704..a568fa2 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.11.26
 
 Core
 * [extractor/common] Use final URL when dumping request (#14769)
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 6009df5..b44c9e7 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -266,10 +266,8 @@
  - **fc2**
  - **fc2:embed**
  - **Fczenit**
- - **fernsehkritik.tv**
  - **filmon**
  - **filmon:channel**
- - **Firstpost**
  - **FiveTV**
  - **Flickr**
  - **Flipagram**
@@ -283,7 +281,7 @@
  - **foxnews:article**
  - **foxnews:insider**
  - **FoxSports**
- - **france2.fr:generation-quoi**
+ - **france2.fr:generation-what**
  - **FranceCulture**
  - **FranceInter**
  - **FranceTV**
@@ -301,7 +299,6 @@
  - **GameInformer**
  - **GameOne**
  - **gameone:playlist**
- - **Gamersyde**
  - **GameSpot**
  - **GameStar**
  - **Gaskrank**
@@ -441,6 +438,7 @@
  - **mangomolo:live**
  - **mangomolo:video**
  - **ManyVids**
+ - **massengeschmack.tv**
  - **MatchTV**
  - **MDR**: MDR.DE and KiKA
  - **media.ccc.de**
@@ -608,6 +606,7 @@
  - **pcmag**
  - **PearVideo**
  - **People**
+ - **PerformGroup**
  - **periscope**: Periscope
  - **periscope:user**: Periscope user videos
  - **PhilharmonieDeParis**: Philharmonie de Paris
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 1c3cbef..81e3f79 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.11.15'
+__version__ = '2017.11.26'
-- 
cgit v1.1


From f58a5060443229e017a04e79cb37d499b6347fe3 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 27 Nov 2017 21:30:47 +0800
Subject: [test_InfoExtractor] Fix flake8

---
 test/test_InfoExtractor.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/test/test_InfoExtractor.py b/test/test_InfoExtractor.py
index 686c63e..7b95f88 100644
--- a/test/test_InfoExtractor.py
+++ b/test/test_InfoExtractor.py
@@ -601,5 +601,6 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
                 self.ie._sort_formats(formats)
                 expect_value(self, formats, expected_formats, None)
 
+
 if __name__ == '__main__':
     unittest.main()
-- 
cgit v1.1


From 82a62de192401157a5eb44cacba16296795b4ee5 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 27 Nov 2017 21:32:06 +0800
Subject: [Makefile,devscripts/run_tests.sh] Actually exclude network tests

Closes #14858
---
 Makefile                | 11 ++++++++++-
 devscripts/run_tests.sh |  3 ++-
 2 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/Makefile b/Makefile
index 141593e..0c39368 100644
--- a/Makefile
+++ b/Makefile
@@ -36,8 +36,17 @@ test:
 
 ot: offlinetest
 
+# Keep this list in sync with devscripts/run_tests.sh
 offlinetest: codetest
-	$(PYTHON) -m nose --verbose test --exclude test_download.py --exclude test_age_restriction.py --exclude test_subtitles.py --exclude test_write_annotations.py --exclude test_youtube_lists.py --exclude test_iqiyi_sdk_interpreter.py --exclude test_socks.py
+	$(PYTHON) -m nose --verbose test \
+		--exclude test_age_restriction.py \
+		--exclude test_download.py \
+		--exclude test_iqiyi_sdk_interpreter.py \
+		--exclude test_socks.py \
+		--exclude test_subtitles.py \
+		--exclude test_write_annotations.py \
+		--exclude test_youtube_lists.py \
+		--exclude test_youtube_signature.py
 
 tar: youtube-dl.tar.gz
 
diff --git a/devscripts/run_tests.sh b/devscripts/run_tests.sh
index 6ba2672..dd37a80 100755
--- a/devscripts/run_tests.sh
+++ b/devscripts/run_tests.sh
@@ -1,6 +1,7 @@
 #!/bin/bash
 
-DOWNLOAD_TESTS="age_restriction|download|subtitles|write_annotations|iqiyi_sdk_interpreter|youtube_lists"
+# Keep this list in sync with the `offlinetest` target in Makefile
+DOWNLOAD_TESTS="age_restriction|download|iqiyi_sdk_interpreter|socks|subtitles|write_annotations|youtube_lists|youtube_signature"
 
 test_set=""
 multiprocess_args=""
-- 
cgit v1.1


From dafb4c66476754a46cc83fa750e9237da8fbf595 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 27 Nov 2017 22:49:35 +0800
Subject: [Makefile] Include setup.cfg in the tarball (closes #14857)

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 0c39368..1c760be 100644
--- a/Makefile
+++ b/Makefile
@@ -124,5 +124,5 @@ youtube-dl.tar.gz: youtube-dl README.md README.txt youtube-dl.1 youtube-dl.bash-
 		bin devscripts test youtube_dl docs \
 		ChangeLog LICENSE README.md README.txt \
 		Makefile MANIFEST.in youtube-dl.1 youtube-dl.bash-completion \
-		youtube-dl.zsh youtube-dl.fish setup.py \
+		youtube-dl.zsh youtube-dl.fish setup.py setup.cfg \
 		youtube-dl
-- 
cgit v1.1


From ffe6979ef925cb8b2e9324c2ad4635cb01d45ab7 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 27 Nov 2017 16:53:42 +0100
Subject: [utils] add hvc1 codec code to parse_codecs

---
 youtube_dl/utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 34866a5..febc9d2 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -2372,7 +2372,7 @@ def parse_codecs(codecs_str):
     vcodec, acodec = None, None
     for full_codec in splited_codecs:
         codec = full_codec.split('.')[0]
-        if codec in ('avc1', 'avc2', 'avc3', 'avc4', 'vp9', 'vp8', 'hev1', 'hev2', 'h263', 'h264', 'mp4v'):
+        if codec in ('avc1', 'avc2', 'avc3', 'avc4', 'vp9', 'vp8', 'hev1', 'hev2', 'h263', 'h264', 'mp4v', 'hvc1'):
             if not vcodec:
                 vcodec = full_codec
         elif codec in ('mp4a', 'opus', 'vorbis', 'mp3', 'aac', 'ac-3', 'ec-3', 'eac3', 'dtsc', 'dtse', 'dtsh', 'dtsl'):
-- 
cgit v1.1


From 53f024e7c5bcea57b2446b3b5550801c011736d1 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 27 Nov 2017 16:55:08 +0100
Subject: [pbs] add another media id regex

---
 youtube_dl/extractor/pbs.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index b51dcbe..f11d5da 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -421,6 +421,7 @@ class PBSIE(InfoExtractor):
                 r'class="coveplayerid">([^<]+)<',                       # coveplayer
                 r'<section[^>]+data-coveid="(\d+)"',                    # coveplayer from http://www.pbs.org/wgbh/frontline/film/real-csi/
                 r'<input type="hidden" id="pbs_video_id_[0-9]+" value="([0-9]+)"/>',  # jwplayer
+                r"(?s)window\.PBS\.playerConfig\s*=\s*{.*?id\s*:\s*'([0-9]+)',",
             ]
 
             media_id = self._search_regex(
-- 
cgit v1.1


From 115afb77ec2d31764307c06866b63ba859d3aafb Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 27 Nov 2017 21:59:27 +0100
Subject: [itv] update hls formats extraction

---
 youtube_dl/extractor/itv.py | 21 +++++++++++----------
 1 file changed, 11 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/itv.py b/youtube_dl/extractor/itv.py
index 26c48e4..413a219 100644
--- a/youtube_dl/extractor/itv.py
+++ b/youtube_dl/extractor/itv.py
@@ -4,6 +4,7 @@ from __future__ import unicode_literals
 import uuid
 import xml.etree.ElementTree as etree
 import json
+import re
 
 from .common import InfoExtractor
 from ..compat import (
@@ -142,9 +143,9 @@ class ITVIE(InfoExtractor):
                 f['url'] = rtmp_url
             formats.append(f)
 
-        ios_playlist_url = params.get('data-video-playlist')
+        ios_playlist_url = params.get('data-video-playlist') or params.get('data-video-id')
         hmac = params.get('data-video-hmac')
-        if ios_playlist_url and hmac:
+        if ios_playlist_url and hmac and re.match(r'https?://', ios_playlist_url):
             headers = self.geo_verification_headers()
             headers.update({
                 'Accept': 'application/vnd.itv.vod.playlist.v2+json',
@@ -159,12 +160,12 @@ class ITVIE(InfoExtractor):
                         'token': ''
                     },
                     'device': {
-                        'manufacturer': 'Apple',
-                        'model': 'iPad',
+                        'manufacturer': 'Safari',
+                        'model': '5',
                         'os': {
-                            'name': 'iPhone OS',
-                            'version': '9.3',
-                            'type': 'ios'
+                            'name': 'Windows NT',
+                            'version': '6.1',
+                            'type': 'desktop'
                         }
                     },
                     'client': {
@@ -173,10 +174,10 @@ class ITVIE(InfoExtractor):
                     },
                     'variantAvailability': {
                         'featureset': {
-                            'min': ['hls', 'aes'],
-                            'max': ['hls', 'aes']
+                            'min': ['hls', 'aes', 'outband-webvtt'],
+                            'max': ['hls', 'aes', 'outband-webvtt']
                         },
-                        'platformTag': 'mobile'
+                        'platformTag': 'dotcom'
                     }
                 }).encode(), headers=headers, fatal=False)
             if ios_playlist:
-- 
cgit v1.1


From a3474aa59e44b15471f46fcac8680d79e82d5116 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 28 Nov 2017 09:04:51 +0100
Subject: [Odnoklassniki] fix api metadata request(fixes #14862)

---
 youtube_dl/extractor/odnoklassniki.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/odnoklassniki.py b/youtube_dl/extractor/odnoklassniki.py
index 854b680..8e13bcf 100644
--- a/youtube_dl/extractor/odnoklassniki.py
+++ b/youtube_dl/extractor/odnoklassniki.py
@@ -14,6 +14,7 @@ from ..utils import (
     int_or_none,
     qualities,
     unescapeHTML,
+    urlencode_postdata,
 )
 
 
@@ -56,7 +57,7 @@ class OdnoklassnikiIE(InfoExtractor):
         'url': 'http://ok.ru/video/64211978996595-1',
         'md5': '2f206894ffb5dbfcce2c5a14b909eea5',
         'info_dict': {
-            'id': '64211978996595-1',
+            'id': 'V_VztHT5BzY',
             'ext': 'mp4',
             'title': 'Космическая среда от 26 августа 2015',
             'description': 'md5:848eb8b85e5e3471a3a803dae1343ed0',
@@ -127,9 +128,14 @@ class OdnoklassnikiIE(InfoExtractor):
         if metadata:
             metadata = self._parse_json(metadata, video_id)
         else:
+            data = {}
+            st_location = flashvars.get('location')
+            if st_location:
+                data['st.location'] = st_location
             metadata = self._download_json(
                 compat_urllib_parse_unquote(flashvars['metadataUrl']),
-                video_id, 'Downloading metadata JSON')
+                video_id, 'Downloading metadata JSON',
+                data=urlencode_postdata(data))
 
         movie = metadata['movie']
 
-- 
cgit v1.1


From 5f699251e9421035cc89baf0b9dc78f3b6eacb77 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 28 Nov 2017 10:55:32 +0100
Subject: [daisuki] add support for motto.daisuki.com(fixes #14681)

---
 youtube_dl/extractor/daisuki.py    | 86 ++++++++++++++++++--------------------
 youtube_dl/extractor/extractors.py |  4 +-
 2 files changed, 42 insertions(+), 48 deletions(-)

diff --git a/youtube_dl/extractor/daisuki.py b/youtube_dl/extractor/daisuki.py
index 58cc986..5c9ac68 100644
--- a/youtube_dl/extractor/daisuki.py
+++ b/youtube_dl/extractor/daisuki.py
@@ -13,33 +13,30 @@ from ..aes import (
 from ..utils import (
     bytes_to_intlist,
     bytes_to_long,
-    clean_html,
+    extract_attributes,
     ExtractorError,
     intlist_to_bytes,
-    get_element_by_id,
     js_to_json,
     int_or_none,
     long_to_bytes,
     pkcs1pad,
-    remove_end,
 )
 
 
-class DaisukiIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?daisuki\.net/[^/]+/[^/]+/[^/]+/watch\.[^.]+\.(?P<id>\d+)\.html'
+class DaisukiMottoIE(InfoExtractor):
+    _VALID_URL = r'https?://motto\.daisuki\.net/framewatch/embed/[^/]+/(?P<id>[0-9a-zA-Z]{3})'
 
     _TEST = {
-        'url': 'http://www.daisuki.net/tw/en/anime/watch.TheIdolMasterCG.11213.html',
+        'url': 'http://motto.daisuki.net/framewatch/embed/embedDRAGONBALLSUPERUniverseSurvivalsaga/V2e/760/428',
         'info_dict': {
-            'id': '11213',
+            'id': 'V2e',
             'ext': 'mp4',
-            'title': '#01 Who is in the pumpkin carriage? - THE IDOLM@STER CINDERELLA GIRLS',
+            'title': '#117 SHOWDOWN OF LOVE! ANDROIDS VS UNIVERSE 2!!',
             'subtitles': {
                 'mul': [{
                     'ext': 'ttml',
                 }],
             },
-            'creator': 'BANDAI NAMCO Entertainment',
         },
         'params': {
             'skip_download': True,  # AES-encrypted HLS stream
@@ -73,15 +70,17 @@ class DaisukiIE(InfoExtractor):
 
             n, e = self._RSA_KEY
             encrypted_aeskey = long_to_bytes(pow(bytes_to_long(padded_aeskey), e, n))
-            init_data = self._download_json('http://www.daisuki.net/bin/bgn/init', video_id, query={
-                's': flashvars.get('s', ''),
-                'c': flashvars.get('ss3_prm', ''),
-                'e': url,
-                'd': base64.b64encode(intlist_to_bytes(aes_cbc_encrypt(
-                    bytes_to_intlist(json.dumps(data)),
-                    aes_key, iv))).decode('ascii'),
-                'a': base64.b64encode(encrypted_aeskey).decode('ascii'),
-            }, note='Downloading JSON metadata' + (' (try #%d)' % (idx + 1) if idx > 0 else ''))
+            init_data = self._download_json(
+                'http://motto.daisuki.net/fastAPI/bgn/init/',
+                video_id, query={
+                    's': flashvars.get('s', ''),
+                    'c': flashvars.get('ss3_prm', ''),
+                    'e': url,
+                    'd': base64.b64encode(intlist_to_bytes(aes_cbc_encrypt(
+                        bytes_to_intlist(json.dumps(data)),
+                        aes_key, iv))).decode('ascii'),
+                    'a': base64.b64encode(encrypted_aeskey).decode('ascii'),
+                }, note='Downloading JSON metadata' + (' (try #%d)' % (idx + 1) if idx > 0 else ''))
 
             if 'rtn' in init_data:
                 encrypted_rtn = init_data['rtn']
@@ -98,14 +97,11 @@ class DaisukiIE(InfoExtractor):
                 aes_key, iv)).decode('utf-8').rstrip('\0'),
             video_id)
 
+        title = rtn['title_str']
+
         formats = self._extract_m3u8_formats(
             rtn['play_url'], video_id, ext='mp4', entry_protocol='m3u8_native')
 
-        title = remove_end(self._og_search_title(webpage), ' - DAISUKI')
-
-        creator = self._html_search_regex(
-            r'Creator\s*:\s*([^<]+)', webpage, 'creator', fatal=False)
-
         subtitles = {}
         caption_url = rtn.get('caption_url')
         if caption_url:
@@ -120,21 +116,18 @@ class DaisukiIE(InfoExtractor):
             'title': title,
             'formats': formats,
             'subtitles': subtitles,
-            'creator': creator,
         }
 
 
-class DaisukiPlaylistIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)daisuki\.net/[^/]+/[^/]+/[^/]+/detail\.(?P<id>[a-zA-Z0-9]+)\.html'
+class DaisukiMottoPlaylistIE(InfoExtractor):
+    _VALID_URL = r'https?://motto\.daisuki\.net/(?P<id>information)/'
 
     _TEST = {
-        'url': 'http://www.daisuki.net/tw/en/anime/detail.TheIdolMasterCG.html',
+        'url': 'http://motto.daisuki.net/information/',
         'info_dict': {
-            'id': 'TheIdolMasterCG',
-            'title': 'THE IDOLM@STER CINDERELLA GIRLS',
-            'description': 'md5:0f2c028a9339f7a2c7fbf839edc5c5d8',
+            'title': 'DRAGON BALL SUPER',
         },
-        'playlist_count': 26,
+        'playlist_mincount': 117,
     }
 
     def _real_extract(self, url):
@@ -142,18 +135,19 @@ class DaisukiPlaylistIE(InfoExtractor):
 
         webpage = self._download_webpage(url, playlist_id)
 
-        episode_pattern = r'''(?sx)
-            <img[^>]+delay="[^"]+/(\d+)/movie\.jpg".+?
-            <p[^>]+class=".*?\bepisodeNumber\b.*?">(?:<a[^>]+>)?([^<]+)'''
-        entries = [{
-            '_type': 'url_transparent',
-            'url': url.replace('detail', 'watch').replace('.html', '.' + movie_id + '.html'),
-            'episode_id': episode_id,
-            'episode_number': int_or_none(episode_id),
-        } for movie_id, episode_id in re.findall(episode_pattern, webpage)]
-
-        playlist_title = remove_end(
-            self._og_search_title(webpage, fatal=False), ' - Anime - DAISUKI')
-        playlist_description = clean_html(get_element_by_id('synopsisTxt', webpage))
-
-        return self.playlist_result(entries, playlist_id, playlist_title, playlist_description)
+        entries = []
+        for li in re.findall(r'(<li[^>]+?data-product_id="[a-zA-Z0-9]{3}"[^>]+>)', webpage):
+            attr = extract_attributes(li)
+            ad_id = attr.get('data-ad_id')
+            product_id = attr.get('data-product_id')
+            if ad_id and product_id:
+                episode_id = attr.get('data-chapter')
+                entries.append({
+                    '_type': 'url_transparent',
+                    'url': 'http://motto.daisuki.net/framewatch/embed/%s/%s/760/428' % (ad_id, product_id),
+                    'episode_id': episode_id,
+                    'episode_number': int_or_none(episode_id),
+                    'ie_key': 'DaisukiMotto',
+                })
+
+        return self.playlist_result(entries, playlist_title='DRAGON BALL SUPER')
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index d3104f3..6db1297 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -246,8 +246,8 @@ from .dailymotion import (
     DailymotionCloudIE,
 )
 from .daisuki import (
-    DaisukiIE,
-    DaisukiPlaylistIE,
+    DaisukiMottoIE,
+    DaisukiMottoPlaylistIE,
 )
 from .daum import (
     DaumIE,
-- 
cgit v1.1


From 07cf18b9c56bf8ce08404b084672f33e860a805e Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 29 Nov 2017 14:21:38 +0100
Subject: [br] add support for BR Mediathek videos(fixes #14560)(fixes #14788)

---
 youtube_dl/extractor/br.py         | 146 ++++++++++++++++++++++++++++++++++++-
 youtube_dl/extractor/extractors.py |   5 +-
 2 files changed, 147 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/br.py b/youtube_dl/extractor/br.py
index 2c32b6a..9bde7f2 100644
--- a/youtube_dl/extractor/br.py
+++ b/youtube_dl/extractor/br.py
@@ -1,20 +1,23 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import json
 import re
 
 from .common import InfoExtractor
 from ..utils import (
+    determine_ext,
     ExtractorError,
     int_or_none,
     parse_duration,
+    parse_iso8601,
     xpath_element,
     xpath_text,
 )
 
 
 class BRIE(InfoExtractor):
-    IE_DESC = 'Bayerischer Rundfunk Mediathek'
+    IE_DESC = 'Bayerischer Rundfunk'
     _VALID_URL = r'(?P<base_url>https?://(?:www\.)?br(?:-klassik)?\.de)/(?:[a-z0-9\-_]+/)+(?P<id>[a-z0-9\-_]+)\.html'
 
     _TESTS = [
@@ -123,10 +126,10 @@ class BRIE(InfoExtractor):
         for asset in assets.findall('asset'):
             format_url = xpath_text(asset, ['downloadUrl', 'url'])
             asset_type = asset.get('type')
-            if asset_type == 'HDS':
+            if asset_type.startswith('HDS'):
                 formats.extend(self._extract_f4m_formats(
                     format_url + '?hdcore=3.2.0', media_id, f4m_id='hds', fatal=False))
-            elif asset_type == 'HLS':
+            elif asset_type.startswith('HLS'):
                 formats.extend(self._extract_m3u8_formats(
                     format_url, media_id, 'mp4', 'm3u8_native', m3u8_id='hds', fatal=False))
             else:
@@ -169,3 +172,140 @@ class BRIE(InfoExtractor):
         } for variant in variants.findall('variant') if xpath_text(variant, 'url')]
         thumbnails.sort(key=lambda x: x['width'] * x['height'], reverse=True)
         return thumbnails
+
+
+class BRMediathekIE(InfoExtractor):
+    IE_DESC = 'Bayerischer Rundfunk Mediathek'
+    _VALID_URL = r'https?://(?:www\.)?br\.de/mediathek/video/[^/?&#]*?-(?P<id>av:[0-9a-f]{24})'
+
+    _TESTS = [{
+        'url': 'https://www.br.de/mediathek/video/gesundheit-die-sendung-vom-28112017-av:5a1e6a6e8fce6d001871cc8e',
+        'md5': 'fdc3d485835966d1622587d08ba632ec',
+        'info_dict': {
+            'id': 'av:5a1e6a6e8fce6d001871cc8e',
+            'ext': 'mp4',
+            'title': 'Die Sendung vom 28.11.2017',
+            'description': 'md5:6000cdca5912ab2277e5b7339f201ccc',
+            'timestamp': 1511942766,
+            'upload_date': '20171129',
+        }
+    }]
+
+    def _real_extract(self, url):
+        clip_id = self._match_id(url)
+
+        clip = self._download_json(
+            'https://proxy-base.master.mango.express/graphql',
+            clip_id, data=json.dumps({
+                "query": """{
+  viewer {
+    clip(id: "%s") {
+      title
+      description
+      duration
+      createdAt
+      ageRestriction
+      videoFiles {
+        edges {
+          node {
+            publicLocation
+            fileSize
+            videoProfile {
+              width
+              height
+              bitrate
+              encoding
+            }
+          }
+        }
+      }
+      captionFiles {
+        edges {
+          node {
+            publicLocation
+          }
+        }
+      }
+      teaserImages {
+        edges {
+          node {
+            imageFiles {
+              edges {
+                node {
+                  publicLocation
+                  width
+                  height
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+}""" % clip_id}).encode(), headers={
+                'Content-Type': 'application/json',
+            })['data']['viewer']['clip']
+        title = clip['title']
+
+        formats = []
+        for edge in clip.get('videoFiles', {}).get('edges', []):
+            node = edge.get('node', {})
+            n_url = node.get('publicLocation')
+            if not n_url:
+                continue
+            ext = determine_ext(n_url)
+            if ext == 'm3u8':
+                formats.extend(self._extract_m3u8_formats(
+                    n_url, clip_id, 'mp4', 'm3u8_native',
+                    m3u8_id='hls', fatal=False))
+            else:
+                video_profile = node.get('videoProfile', {})
+                tbr = int_or_none(video_profile.get('bitrate'))
+                format_id = 'http'
+                if tbr:
+                    format_id += '-%d' % tbr
+                formats.append({
+                    'format_id': format_id,
+                    'url': n_url,
+                    'width': int_or_none(video_profile.get('width')),
+                    'height': int_or_none(video_profile.get('height')),
+                    'tbr': tbr,
+                    'filesize': int_or_none(node.get('fileSize')),
+                })
+        self._sort_formats(formats)
+
+        subtitles = {}
+        for edge in clip.get('captionFiles', {}).get('edges', []):
+            node = edge.get('node', {})
+            n_url = node.get('publicLocation')
+            if not n_url:
+                continue
+            subtitles.setdefault('de', []).append({
+                'url': n_url,
+            })
+
+        thumbnails = []
+        for edge in clip.get('teaserImages', {}).get('edges', []):
+            for image_edge in edge.get('node', {}).get('imageFiles', {}).get('edges', []):
+                node = image_edge.get('node', {})
+                n_url = node.get('publicLocation')
+                if not n_url:
+                    continue
+                thumbnails.append({
+                    'url': n_url,
+                    'width': int_or_none(node.get('width')),
+                    'height': int_or_none(node.get('height')),
+                })
+
+        return {
+            'id': clip_id,
+            'title': title,
+            'description': clip.get('description'),
+            'duration': int_or_none(clip.get('duration')),
+            'timestamp': parse_iso8601(clip.get('createdAt')),
+            'age_limit': int_or_none(clip.get('ageRestriction')),
+            'formats': formats,
+            'subtitles': subtitles,
+            'thumbnails': thumbnails,
+        }
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 6db1297..2cc3bc4 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -127,7 +127,10 @@ from .bloomberg import BloombergIE
 from .bokecc import BokeCCIE
 from .bostonglobe import BostonGlobeIE
 from .bpb import BpbIE
-from .br import BRIE
+from .br import (
+    BRIE,
+    BRMediathekIE,
+)
 from .bravotv import BravoTVIE
 from .breakcom import BreakIE
 from .brightcove import (
-- 
cgit v1.1


From fb61b57d0f4c6422b2722f56f5740ea1b19adfcf Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 29 Nov 2017 16:09:45 +0100
Subject: [vvvvid] fix extraction for kenc videos(fixes #13406)

---
 youtube_dl/extractor/vvvvid.py | 22 ++++++++++++++++++++--
 1 file changed, 20 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/vvvvid.py b/youtube_dl/extractor/vvvvid.py
index 656a4b9..3d0dc40 100644
--- a/youtube_dl/extractor/vvvvid.py
+++ b/youtube_dl/extractor/vvvvid.py
@@ -22,6 +22,9 @@ class VVVVIDIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Ping Pong',
         },
+        'params': {
+            'skip_download': True,
+        },
     }, {
         # video_type == 'video/rcs'
         'url': 'https://www.vvvvid.it/#!show/376/death-note-live-action/377/482493/episodio-01',
@@ -31,6 +34,9 @@ class VVVVIDIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Episodio 01',
         },
+        'params': {
+            'skip_download': True,
+        },
     }]
     _conn_id = None
 
@@ -116,8 +122,20 @@ class VVVVIDIE(InfoExtractor):
             embed_code = ds(embed_code)
             video_type = video_data.get('video_type')
             if video_type in ('video/rcs', 'video/kenc'):
-                formats.extend(self._extract_akamai_formats(
-                    embed_code, video_id))
+                embed_code = re.sub(r'https?://([^/]+)/z/', r'https://\1/i/', embed_code).replace('/manifest.f4m', '/master.m3u8')
+                if video_type == 'video/kenc':
+                    kenc = self._download_json(
+                        'https://www.vvvvid.it/kenc', video_id, query={
+                            'action': 'kt',
+                            'conn_id': self._conn_id,
+                            'url': embed_code,
+                        }, fatal=False) or {}
+                    kenc_message = kenc.get('message')
+                    if kenc_message:
+                        embed_code += '?' + ds(kenc_message)
+                formats.extend(self._extract_m3u8_formats(
+                    embed_code, video_id, 'mp4',
+                    m3u8_id='hls', fatal=False))
             else:
                 formats.extend(self._extract_wowza_formats(
                     'http://sb.top-ix.org/videomg/_definst_/mp4:%s/playlist.m3u8' % embed_code, video_id))
-- 
cgit v1.1


From 5ea765fb720523efb2702358e188e5157d08775b Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 29 Nov 2017 17:50:38 +0100
Subject: [mtv] correct scc extention in extracted subtitles(closes #13730)

---
 youtube_dl/extractor/mtv.py | 15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/mtv.py b/youtube_dl/extractor/mtv.py
index 1154a35..7a3b57a 100644
--- a/youtube_dl/extractor/mtv.py
+++ b/youtube_dl/extractor/mtv.py
@@ -115,10 +115,17 @@ class MTVServicesInfoExtractor(InfoExtractor):
             if transcript.get('kind') != 'captions':
                 continue
             lang = transcript.get('srclang')
-            subtitles[lang] = [{
-                'url': compat_str(typographic.get('src')),
-                'ext': typographic.get('format')
-            } for typographic in transcript.findall('./typographic')]
+            for typographic in transcript.findall('./typographic'):
+                sub_src = typographic.get('src')
+                if not sub_src:
+                    continue
+                ext = typographic.get('format')
+                if ext == 'cea-608':
+                    ext = 'scc'
+                subtitles.setdefault(lang, []).append({
+                    'url': compat_str(sub_src),
+                    'ext': ext
+                })
         return subtitles
 
     def _get_video_info(self, itemdoc, use_hls=True):
-- 
cgit v1.1


From 1663b329460b8b83b4eb0381ccb4f201374647db Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 29 Nov 2017 20:36:55 +0100
Subject: [xiami] add Referer header to api request

---
 youtube_dl/extractor/xiami.py | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/xiami.py b/youtube_dl/extractor/xiami.py
index d017e03..7f871c8 100644
--- a/youtube_dl/extractor/xiami.py
+++ b/youtube_dl/extractor/xiami.py
@@ -40,9 +40,12 @@ class XiamiBaseIE(InfoExtractor):
             'subtitles': subtitles,
         }
 
-    def _extract_tracks(self, item_id, typ=None):
+    def _extract_tracks(self, item_id, referer, typ=None):
         playlist = self._download_json(
-            '%s/%s%s' % (self._API_BASE_URL, item_id, '/type/%s' % typ if typ else ''), item_id)
+            '%s/%s%s' % (self._API_BASE_URL, item_id, '/type/%s' % typ if typ else ''),
+            item_id, headers={
+                'Referer': referer,
+            })
         return [
             self._extract_track(track, item_id)
             for track in playlist['data']['trackList']]
@@ -135,13 +138,13 @@ class XiamiSongIE(XiamiBaseIE):
     }]
 
     def _real_extract(self, url):
-        return self._extract_tracks(self._match_id(url))[0]
+        return self._extract_tracks(self._match_id(url), url)[0]
 
 
 class XiamiPlaylistBaseIE(XiamiBaseIE):
     def _real_extract(self, url):
         item_id = self._match_id(url)
-        return self.playlist_result(self._extract_tracks(item_id, self._TYPE), item_id)
+        return self.playlist_result(self._extract_tracks(item_id, url, self._TYPE), item_id)
 
 
 class XiamiAlbumIE(XiamiPlaylistBaseIE):
-- 
cgit v1.1


From f5ac68d88f3f11b88213ccba5e079d7d5221e243 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 30 Nov 2017 23:37:05 +0100
Subject: [mnet] fix format extraction(fixes #14883)

---
 youtube_dl/extractor/mnet.py | 30 +++++++++++++++++++-----------
 1 file changed, 19 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/mnet.py b/youtube_dl/extractor/mnet.py
index 6a85dcb..0e26ca1 100644
--- a/youtube_dl/extractor/mnet.py
+++ b/youtube_dl/extractor/mnet.py
@@ -40,21 +40,29 @@ class MnetIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
+        # TODO: extract rtmp formats
+        # no stype -> rtmp url
+        # stype=H -> m3u8 url
+        # stype=M -> mpd url
         info = self._download_json(
-            'http://content.api.mnet.com/player/vodConfig?id=%s&ctype=CLIP' % video_id,
-            video_id, 'Downloading vod config JSON')['data']['info']
+            'http://content.api.mnet.com/player/vodConfig',
+            video_id, 'Downloading vod config JSON', query={
+                'id': video_id,
+                'ctype': 'CLIP',
+                'stype': 'H',
+            })['data']['info']
 
         title = info['title']
 
-        rtmp_info = self._download_json(
-            info['cdn'], video_id, 'Downloading vod cdn JSON')
-
-        formats = [{
-            'url': rtmp_info['serverurl'] + rtmp_info['fileurl'],
-            'ext': 'flv',
-            'page_url': url,
-            'player_url': 'http://flvfile.mnet.com/service/player/201602/cjem_player_tv.swf?v=201602191318',
-        }]
+        cdn_data = self._download_json(
+            info['cdn'], video_id, 'Downloading vod cdn JSON')['data'][0]
+        m3u8_url = cdn_data['url']
+        token = cdn_data.get('token')
+        if token and token != '-':
+            m3u8_url += '?' + token
+        formats = self._extract_wowza_formats(
+            m3u8_url, video_id, skip_protocols=['rtmp', 'rtsp', 'f4m'])
+        self._sort_formats(formats)
 
         description = info.get('ment')
         duration = parse_duration(info.get('time'))
-- 
cgit v1.1


From 0981585befc7af01331b72304ed11b9865b40401 Mon Sep 17 00:00:00 2001
From: zcanfly <zcanfly@126.com>
Date: Sat, 2 Dec 2017 19:16:22 +0800
Subject: [youku] Update ccode (closes #14872)

---
 youtube_dl/extractor/youku.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index a2b3b4d..f0ba011 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -154,7 +154,7 @@ class YoukuIE(InfoExtractor):
         # request basic data
         basic_data_params = {
             'vid': video_id,
-            'ccode': '0502',
+            'ccode': '0501',
             'client_ip': '192.168.1.1',
             'utid': cna,
             'client_ts': time.time() / 1000,
-- 
cgit v1.1


From fea92aa65dbc28e3a5cd109a5b9de5048c9cdfc9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 2 Dec 2017 19:03:24 +0700
Subject: [xhamster] Fix extraction (closes #14884)

---
 youtube_dl/extractor/xhamster.py | 87 ++++++++++++++++++++++++++++++++++++++--
 1 file changed, 83 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/xhamster.py b/youtube_dl/extractor/xhamster.py
index be3624e..52f8ded 100644
--- a/youtube_dl/extractor/xhamster.py
+++ b/youtube_dl/extractor/xhamster.py
@@ -6,10 +6,12 @@ from .common import InfoExtractor
 from ..compat import compat_str
 from ..utils import (
     clean_html,
+    determine_ext,
     dict_get,
     ExtractorError,
     int_or_none,
     parse_duration,
+    try_get,
     unified_strdate,
 )
 
@@ -32,6 +34,7 @@ class XHamsterIE(InfoExtractor):
             'display_id': 'femaleagent_shy_beauty_takes_the_bait',
             'ext': 'mp4',
             'title': 'FemaleAgent Shy beauty takes the bait',
+            'timestamp': 1350194821,
             'upload_date': '20121014',
             'uploader': 'Ruseful2011',
             'duration': 893,
@@ -45,6 +48,7 @@ class XHamsterIE(InfoExtractor):
             'display_id': 'britney_spears_sexy_booty',
             'ext': 'mp4',
             'title': 'Britney Spears  Sexy Booty',
+            'timestamp': 1379123460,
             'upload_date': '20130914',
             'uploader': 'jojo747400',
             'duration': 200,
@@ -61,6 +65,7 @@ class XHamsterIE(InfoExtractor):
             'id': '5667973',
             'ext': 'mp4',
             'title': '....',
+            'timestamp': 1454948101,
             'upload_date': '20160208',
             'uploader': 'parejafree',
             'duration': 72,
@@ -96,6 +101,83 @@ class XHamsterIE(InfoExtractor):
         if error:
             raise ExtractorError(error, expected=True)
 
+        age_limit = self._rta_search(webpage)
+
+        def get_height(s):
+            return int_or_none(self._search_regex(
+                r'^(\d+)[pP]', s, 'height', default=None))
+
+        initials = self._parse_json(
+            self._search_regex(
+                r'window\.initials\s*=\s*({.+?})\s*;\s*\n', webpage, 'initials',
+                default='{}'),
+            video_id, fatal=False)
+        if initials:
+            video = initials['videoModel']
+            title = video['title']
+            formats = []
+            for format_id, formats_dict in video['sources'].items():
+                if not isinstance(formats_dict, dict):
+                    continue
+                for quality, format_item in formats_dict.items():
+                    if format_id == 'download':
+                        # Download link takes some time to be generated,
+                        # skipping for now
+                        continue
+                        if not isinstance(format_item, dict):
+                            continue
+                        format_url = format_item.get('link')
+                        filesize = int_or_none(
+                            format_item.get('size'), invscale=1000000)
+                    else:
+                        format_url = format_item
+                        filesize = None
+                    if not isinstance(format_url, compat_str):
+                        continue
+                    formats.append({
+                        'format_id': '%s-%s' % (format_id, quality),
+                        'url': format_url,
+                        'ext': determine_ext(format_url, 'mp4'),
+                        'height': get_height(quality),
+                        'filesize': filesize,
+                    })
+            self._sort_formats(formats)
+
+            categories_list = video.get('categories')
+            if isinstance(categories_list, list):
+                categories = []
+                for c in categories_list:
+                    if not isinstance(c, dict):
+                        continue
+                    c_name = c.get('name')
+                    if isinstance(c_name, compat_str):
+                        categories.append(c_name)
+            else:
+                categories = None
+
+            return {
+                'id': video_id,
+                'display_id': display_id,
+                'title': title,
+                'description': video.get('description'),
+                'timestamp': int_or_none(video.get('created')),
+                'uploader': try_get(
+                    video, lambda x: x['author']['name'], compat_str),
+                'thumbnail': video.get('thumbURL'),
+                'duration': int_or_none(video.get('duration')),
+                'view_count': int_or_none(video.get('views')),
+                'like_count': int_or_none(try_get(
+                    video, lambda x: x['rating']['likes'], int)),
+                'dislike_count': int_or_none(try_get(
+                    video, lambda x: x['rating']['dislikes'], int)),
+                'comment_count': int_or_none(video.get('views')),
+                'age_limit': age_limit,
+                'categories': categories,
+                'formats': formats,
+            }
+
+        # Old layout fallback
+
         title = self._html_search_regex(
             [r'<h1[^>]*>([^<]+)</h1>',
              r'<meta[^>]+itemprop=".*?caption.*?"[^>]+content="(.+?)"',
@@ -119,8 +201,7 @@ class XHamsterIE(InfoExtractor):
             formats.append({
                 'format_id': format_id,
                 'url': format_url,
-                'height': int_or_none(self._search_regex(
-                    r'^(\d+)[pP]', format_id, 'height', default=None))
+                'height': get_height(format_id),
             })
 
         video_url = self._search_regex(
@@ -167,8 +248,6 @@ class XHamsterIE(InfoExtractor):
         mobj = re.search(r'</label>Comments \((?P<commentcount>\d+)\)</div>', webpage)
         comment_count = mobj.group('commentcount') if mobj else 0
 
-        age_limit = self._rta_search(webpage)
-
         categories_html = self._search_regex(
             r'(?s)<table.+?(<span>Categories:.+?)</table>', webpage,
             'categories', default=None)
-- 
cgit v1.1


From 41bf647e895faca42cdc2565ea034ed341593f8e Mon Sep 17 00:00:00 2001
From: Petr Novak <petr.novak@cosmoboy.cz>
Date: Sat, 25 Nov 2017 02:13:23 +0100
Subject: [extractor/common] Add support for DASH manifests with SegmentLists
 with bare SegmentURLs

---
 test/test_InfoExtractor.py      |  83 ++++++++++++++-
 test/testdata/mpd/urls_only.mpd | 218 ++++++++++++++++++++++++++++++++++++++++
 youtube_dl/extractor/common.py  |   9 ++
 3 files changed, 309 insertions(+), 1 deletion(-)
 create mode 100644 test/testdata/mpd/urls_only.mpd

diff --git a/test/test_InfoExtractor.py b/test/test_InfoExtractor.py
index 7b95f88..e58452a 100644
--- a/test/test_InfoExtractor.py
+++ b/test/test_InfoExtractor.py
@@ -562,7 +562,88 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
                     'width': 1920,
                     'height': 1080,
                 }]
-            ),
+            ), (
+                'urls_only',
+                'http://unknown/manifest.mpd',
+                [{
+                    'manifest_url': 'http://unknown/manifest.mpd',
+                    'ext': 'mp4',
+                    'format_id': 'h264_aac_144p_m4s',
+                    'format_note': 'DASH video',
+                    'protocol': 'http_dash_segments',
+                    'acodec': 'mp4a.40.2',
+                    'vcodec': 'avc3.42c01e',
+                    'tbr': 200,
+                    'width': 256,
+                    'height': 144,
+                }, {
+                    'manifest_url': 'http://unknown/manifest.mpd',
+                    'ext': 'mp4',
+                    'format_id': 'h264_aac_240p_m4s',
+                    'format_note': 'DASH video',
+                    'protocol': 'http_dash_segments',
+                    'acodec': 'mp4a.40.2',
+                    'vcodec': 'avc3.42c01e',
+                    'tbr': 400,
+                    'width': 424,
+                    'height': 240,
+                }, {
+                    'manifest_url': 'http://unknown/manifest.mpd',
+                    'ext': 'mp4',
+                    'format_id': 'h264_aac_360p_m4s',
+                    'format_note': 'DASH video',
+                    'protocol': 'http_dash_segments',
+                    'acodec': 'mp4a.40.2',
+                    'vcodec': 'avc3.42c01e',
+                    'tbr': 800,
+                    'width': 640,
+                    'height': 360,
+                }, {
+                    'manifest_url': 'http://unknown/manifest.mpd',
+                    'ext': 'mp4',
+                    'format_id': 'h264_aac_480p_m4s',
+                    'format_note': 'DASH video',
+                    'protocol': 'http_dash_segments',
+                    'acodec': 'mp4a.40.2',
+                    'vcodec': 'avc3.42c01e',
+                    'tbr': 1200,
+                    'width': 856,
+                    'height': 480,
+                }, {
+                    'manifest_url': 'http://unknown/manifest.mpd',
+                    'ext': 'mp4',
+                    'format_id': 'h264_aac_576p_m4s',
+                    'format_note': 'DASH video',
+                    'protocol': 'http_dash_segments',
+                    'acodec': 'mp4a.40.2',
+                    'vcodec': 'avc3.42c01e',
+                    'tbr': 1600,
+                    'width': 1024,
+                    'height': 576,
+                }, {
+                    'manifest_url': 'http://unknown/manifest.mpd',
+                    'ext': 'mp4',
+                    'format_id': 'h264_aac_720p_m4s',
+                    'format_note': 'DASH video',
+                    'protocol': 'http_dash_segments',
+                    'acodec': 'mp4a.40.2',
+                    'vcodec': 'avc3.42c01e',
+                    'tbr': 2400,
+                    'width': 1280,
+                    'height': 720,
+                }, {
+                    'manifest_url': 'http://unknown/manifest.mpd',
+                    'ext': 'mp4',
+                    'format_id': 'h264_aac_1080p_m4s',
+                    'format_note': 'DASH video',
+                    'protocol': 'http_dash_segments',
+                    'acodec': 'mp4a.40.2',
+                    'vcodec': 'avc3.42c01e',
+                    'tbr': 4400,
+                    'width': 1920,
+                    'height': 1080,
+                }]
+            )
         ]
 
         for mpd_file, mpd_url, expected_formats in _TEST_CASES:
diff --git a/test/testdata/mpd/urls_only.mpd b/test/testdata/mpd/urls_only.mpd
new file mode 100644
index 0000000..2b9d595
--- /dev/null
+++ b/test/testdata/mpd/urls_only.mpd
@@ -0,0 +1,218 @@
+<?xml version="1.0" ?>
+<MPD maxSegmentDuration="PT0H0M10.000S" mediaPresentationDuration="PT0H4M1.728S" minBufferTime="PT1.500S" profiles="urn:mpeg:dash:profile:isoff-main:2011" type="static" xmlns="urn:mpeg:dash:schema:mpd:2011">
+  <Period duration="PT0H4M1.728S">
+    <AdaptationSet bitstreamSwitching="true" lang="und" maxHeight="1080" maxWidth="1920" par="16:9" segmentAlignment="true">
+      <ContentComponent contentType="video" id="1"/>
+      <Representation audioSamplingRate="44100" bandwidth="200000" codecs="avc3.42c01e,mp4a.40.2" frameRate="25" height="144" id="h264_aac_144p_m4s" mimeType="video/mp4" sar="1:1" startWithSAP="1" width="256">
+        <SegmentList duration="10000" timescale="1000">
+          <Initialization sourceURL="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_144p_m4s/init/432f65a0.mp4"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_144p_m4s/0/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_144p_m4s/1/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_144p_m4s/2/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_144p_m4s/3/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_144p_m4s/4/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_144p_m4s/5/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_144p_m4s/6/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_144p_m4s/7/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_144p_m4s/8/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_144p_m4s/9/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_144p_m4s/10/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_144p_m4s/11/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_144p_m4s/12/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_144p_m4s/13/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_144p_m4s/14/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_144p_m4s/15/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_144p_m4s/16/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_144p_m4s/17/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_144p_m4s/18/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_144p_m4s/19/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_144p_m4s/20/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_144p_m4s/21/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_144p_m4s/22/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_144p_m4s/23/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_144p_m4s/24/432f65a0.m4s"/>
+        </SegmentList>
+      </Representation>
+      <Representation audioSamplingRate="44100" bandwidth="400000" codecs="avc3.42c01e,mp4a.40.2" frameRate="25" height="240" id="h264_aac_240p_m4s" mimeType="video/mp4" sar="160:159" startWithSAP="1" width="424">
+        <SegmentList duration="10000" timescale="1000">
+          <Initialization sourceURL="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_240p_m4s/init/432f65a0.mp4"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_240p_m4s/0/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_240p_m4s/1/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_240p_m4s/2/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_240p_m4s/3/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_240p_m4s/4/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_240p_m4s/5/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_240p_m4s/6/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_240p_m4s/7/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_240p_m4s/8/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_240p_m4s/9/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_240p_m4s/10/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_240p_m4s/11/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_240p_m4s/12/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_240p_m4s/13/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_240p_m4s/14/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_240p_m4s/15/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_240p_m4s/16/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_240p_m4s/17/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_240p_m4s/18/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_240p_m4s/19/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_240p_m4s/20/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_240p_m4s/21/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_240p_m4s/22/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_240p_m4s/23/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_240p_m4s/24/432f65a0.m4s"/>
+        </SegmentList>
+      </Representation>
+      <Representation audioSamplingRate="44100" bandwidth="800000" codecs="avc3.42c01e,mp4a.40.2" frameRate="25" height="360" id="h264_aac_360p_m4s" mimeType="video/mp4" sar="1:1" startWithSAP="1" width="640">
+        <SegmentList duration="10000" timescale="1000">
+          <Initialization sourceURL="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_360p_m4s/init/432f65a0.mp4"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_360p_m4s/0/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_360p_m4s/1/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_360p_m4s/2/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_360p_m4s/3/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_360p_m4s/4/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_360p_m4s/5/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_360p_m4s/6/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_360p_m4s/7/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_360p_m4s/8/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_360p_m4s/9/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_360p_m4s/10/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_360p_m4s/11/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_360p_m4s/12/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_360p_m4s/13/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_360p_m4s/14/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_360p_m4s/15/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_360p_m4s/16/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_360p_m4s/17/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_360p_m4s/18/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_360p_m4s/19/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_360p_m4s/20/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_360p_m4s/21/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_360p_m4s/22/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_360p_m4s/23/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_360p_m4s/24/432f65a0.m4s"/>
+        </SegmentList>
+      </Representation>
+      <Representation audioSamplingRate="44100" bandwidth="1200000" codecs="avc3.42c01e,mp4a.40.2" frameRate="25" height="480" id="h264_aac_480p_m4s" mimeType="video/mp4" sar="320:321" startWithSAP="1" width="856">
+        <SegmentList duration="10000" timescale="1000">
+          <Initialization sourceURL="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_480p_m4s/init/432f65a0.mp4"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_480p_m4s/0/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_480p_m4s/1/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_480p_m4s/2/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_480p_m4s/3/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_480p_m4s/4/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_480p_m4s/5/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_480p_m4s/6/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_480p_m4s/7/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_480p_m4s/8/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_480p_m4s/9/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_480p_m4s/10/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_480p_m4s/11/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_480p_m4s/12/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_480p_m4s/13/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_480p_m4s/14/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_480p_m4s/15/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_480p_m4s/16/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_480p_m4s/17/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_480p_m4s/18/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_480p_m4s/19/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_480p_m4s/20/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_480p_m4s/21/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_480p_m4s/22/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_480p_m4s/23/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_480p_m4s/24/432f65a0.m4s"/>
+        </SegmentList>
+      </Representation>
+      <Representation audioSamplingRate="44100" bandwidth="1600000" codecs="avc3.42c01e,mp4a.40.2" frameRate="25" height="576" id="h264_aac_576p_m4s" mimeType="video/mp4" sar="1:1" startWithSAP="1" width="1024">
+        <SegmentList duration="10000" timescale="1000">
+          <Initialization sourceURL="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_576p_m4s/init/432f65a0.mp4"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_576p_m4s/0/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_576p_m4s/1/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_576p_m4s/2/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_576p_m4s/3/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_576p_m4s/4/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_576p_m4s/5/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_576p_m4s/6/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_576p_m4s/7/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_576p_m4s/8/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_576p_m4s/9/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_576p_m4s/10/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_576p_m4s/11/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_576p_m4s/12/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_576p_m4s/13/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_576p_m4s/14/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_576p_m4s/15/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_576p_m4s/16/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_576p_m4s/17/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_576p_m4s/18/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_576p_m4s/19/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_576p_m4s/20/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_576p_m4s/21/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_576p_m4s/22/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_576p_m4s/23/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_576p_m4s/24/432f65a0.m4s"/>
+        </SegmentList>
+      </Representation>
+      <Representation audioSamplingRate="44100" bandwidth="2400000" codecs="avc3.42c01e,mp4a.40.2" frameRate="25" height="720" id="h264_aac_720p_m4s" mimeType="video/mp4" sar="1:1" startWithSAP="1" width="1280">
+        <SegmentList duration="10000" timescale="1000">
+          <Initialization sourceURL="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_720p_m4s/init/432f65a0.mp4"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_720p_m4s/0/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_720p_m4s/1/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_720p_m4s/2/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_720p_m4s/3/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_720p_m4s/4/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_720p_m4s/5/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_720p_m4s/6/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_720p_m4s/7/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_720p_m4s/8/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_720p_m4s/9/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_720p_m4s/10/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_720p_m4s/11/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_720p_m4s/12/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_720p_m4s/13/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_720p_m4s/14/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_720p_m4s/15/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_720p_m4s/16/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_720p_m4s/17/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_720p_m4s/18/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_720p_m4s/19/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_720p_m4s/20/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_720p_m4s/21/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_720p_m4s/22/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_720p_m4s/23/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_720p_m4s/24/432f65a0.m4s"/>
+        </SegmentList>
+      </Representation>
+      <Representation audioSamplingRate="44100" bandwidth="4400000" codecs="avc3.42c01e,mp4a.40.2" frameRate="25" height="1080" id="h264_aac_1080p_m4s" mimeType="video/mp4" sar="1:1" startWithSAP="1" width="1920">
+        <SegmentList duration="10000" timescale="1000">
+          <Initialization sourceURL="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_1080p_m4s/init/432f65a0.mp4"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_1080p_m4s/0/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_1080p_m4s/1/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_1080p_m4s/2/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_1080p_m4s/3/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_1080p_m4s/4/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_1080p_m4s/5/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_1080p_m4s/6/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_1080p_m4s/7/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_1080p_m4s/8/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_1080p_m4s/9/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_1080p_m4s/10/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_1080p_m4s/11/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_1080p_m4s/12/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_1080p_m4s/13/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_1080p_m4s/14/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_1080p_m4s/15/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_1080p_m4s/16/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_1080p_m4s/17/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_1080p_m4s/18/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_1080p_m4s/19/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_1080p_m4s/20/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_1080p_m4s/21/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_1080p_m4s/22/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_1080p_m4s/23/432f65a0.m4s"/>
+          <SegmentURL media="../vd_5999c902ea707c67d8e267a9_1503250723/h264_aac_1080p_m4s/24/432f65a0.m4s"/>
+        </SegmentList>
+      </Representation>
+    </AdaptationSet>
+  </Period>
+</MPD>
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 8e4ee0d..1599941 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1975,6 +1975,15 @@ class InfoExtractor(object):
                                     })
                                     segment_index += 1
                             representation_ms_info['fragments'] = fragments
+                        elif 'segment_urls' in representation_ms_info:
+                            # Segment URLs with no SegmentTimeline
+                            # Example: https://www.seznam.cz/zpravy/clanek/cesko-zasahne-vitr-o-sile-vichrice-muze-byt-i-zivotu-nebezpecny-39091
+                            fragments = []
+                            for segment_url in representation_ms_info['segment_urls']:
+                                fragments.append({
+                                    location_key(segment_url): segment_url,
+                                })
+                            representation_ms_info['fragments'] = fragments
                         # NB: MPD manifest may contain direct URLs to unfragmented media.
                         # No fragments key is present in this case.
                         if 'fragments' in representation_ms_info:
-- 
cgit v1.1


From 603fc4e0ea472c7c2a78ff201d69686a9e3fe1f2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 2 Dec 2017 21:10:35 +0700
Subject: [extractor/common] Add durations for DASH fragments with bare
 SegmentURLs

---
 youtube_dl/extractor/common.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 1599941..3baf683 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1979,10 +1979,16 @@ class InfoExtractor(object):
                             # Segment URLs with no SegmentTimeline
                             # Example: https://www.seznam.cz/zpravy/clanek/cesko-zasahne-vitr-o-sile-vichrice-muze-byt-i-zivotu-nebezpecny-39091
                             fragments = []
+                            segment_duration = float_or_none(
+                                representation_ms_info['segment_duration'],
+                                representation_ms_info['timescale']) if 'segment_duration' in representation_ms_info else None
                             for segment_url in representation_ms_info['segment_urls']:
-                                fragments.append({
+                                fragment = {
                                     location_key(segment_url): segment_url,
-                                })
+                                }
+                                if segment_duration:
+                                    fragment['duration'] = segment_duration
+                                fragments.append(fragment)
                             representation_ms_info['fragments'] = fragments
                         # NB: MPD manifest may contain direct URLs to unfragmented media.
                         # No fragments key is present in this case.
-- 
cgit v1.1


From 593f2f798922d54be8d3f20f4a2048493095016d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 2 Dec 2017 21:15:45 +0700
Subject: [downloader/fragment] Commit part file after each fragment In order
 to obtain correct resume_len on next iteration

---
 youtube_dl/downloader/fragment.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/downloader/fragment.py b/youtube_dl/downloader/fragment.py
index 93002e4..7bb61a5 100644
--- a/youtube_dl/downloader/fragment.py
+++ b/youtube_dl/downloader/fragment.py
@@ -107,6 +107,7 @@ class FragmentFD(FileDownloader):
     def _append_fragment(self, ctx, frag_content):
         try:
             ctx['dest_stream'].write(frag_content)
+            ctx['dest_stream'].flush()
         finally:
             if self.__do_ytdl_file(ctx):
                 self._write_ytdl_file(ctx)
-- 
cgit v1.1


From 78593e294cc4623108d17b1b0b2d26d507953006 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 2 Dec 2017 21:22:43 +0700
Subject: Add references for #14844

---
 test/test_InfoExtractor.py     | 1 +
 youtube_dl/extractor/common.py | 1 +
 2 files changed, 2 insertions(+)

diff --git a/test/test_InfoExtractor.py b/test/test_InfoExtractor.py
index e58452a..8a372d2 100644
--- a/test/test_InfoExtractor.py
+++ b/test/test_InfoExtractor.py
@@ -563,6 +563,7 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
                     'height': 1080,
                 }]
             ), (
+                # https://github.com/rg3/youtube-dl/pull/14844
                 'urls_only',
                 'http://unknown/manifest.mpd',
                 [{
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 3baf683..80a9c98 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1978,6 +1978,7 @@ class InfoExtractor(object):
                         elif 'segment_urls' in representation_ms_info:
                             # Segment URLs with no SegmentTimeline
                             # Example: https://www.seznam.cz/zpravy/clanek/cesko-zasahne-vitr-o-sile-vichrice-muze-byt-i-zivotu-nebezpecny-39091
+                            # https://github.com/rg3/youtube-dl/pull/14844
                             fragments = []
                             segment_duration = float_or_none(
                                 representation_ms_info['segment_duration'],
-- 
cgit v1.1


From e25ee72657c3e27a24f62ac54d4b100c558ea662 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 2 Dec 2017 21:29:06 +0700
Subject: [ChangeLog] Actualize

---
 ChangeLog | 23 +++++++++++++++++++++++
 1 file changed, 23 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index a568fa2..81eda95 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,26 @@
+version <unreleased>
+
+Core
++ [downloader/fragment] Commit part file after each fragment
++ [extractor/common] Add durations for DASH fragments with bare SegmentURLs
++ [extractor/common] Add support for DASH manifests with SegmentLists with
+  bare SegmentURLs (#14844)
++ [utils] Add hvc1 codec code to parse_codecs
+
+Extractors
+* [xhamster] Fix extraction (#14884)
+* [youku] Update ccode (#14872)
+* [mnet] Fix format extraction (#14883)
++ [xiami] Add Referer header to API request
+* [mtv] Correct scc extention in extracted subtitles (#13730)
+* [vvvvid] Fix extraction for kenc videos (#13406)
++ [br] Add support for BR Mediathek videos (#14560, #14788)
++ [daisuki] Add support for motto.daisuki.com (#14681)
+* [odnoklassniki] Fix API metadata request (#14862)
+* [itv] Fix HLS formats extraction
++ [pbs] Add another media id regular expression
+
+
 version 2017.11.26
 
 Core
-- 
cgit v1.1


From 0d56eddc5970a9e55334add371f555227b957589 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 2 Dec 2017 21:34:34 +0700
Subject: release 2017.12.02

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 docs/supportedsites.md    | 7 ++++---
 youtube_dl/version.py     | 2 +-
 4 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 15211e5..513823b 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.11.26*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.11.26**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.12.02*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.12.02**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through the [README](https://github.com/rg3/youtube-dl/blob/master/README.md), **most notably** the [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ Add the `-v` flag to **your command line** you run youtube-dl with (`youtube-dl
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.11.26
+[debug] youtube-dl version 2017.12.02
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index 81eda95..63837d6 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.12.02
 
 Core
 + [downloader/fragment] Commit part file after each fragment
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index b44c9e7..0287a40 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -112,11 +112,12 @@
  - **BokeCC**
  - **BostonGlobe**
  - **Bpb**: Bundeszentrale für politische Bildung
- - **BR**: Bayerischer Rundfunk Mediathek
+ - **BR**: Bayerischer Rundfunk
  - **BravoTV**
  - **Break**
  - **brightcove:legacy**
  - **brightcove:new**
+ - **BRMediathek**: Bayerischer Rundfunk Mediathek
  - **bt:article**: Bergens Tidende Articles
  - **bt:vestlendingen**: Bergens Tidende - Vestlendingen
  - **BuzzFeed**
@@ -198,8 +199,8 @@
  - **dailymotion:playlist**
  - **dailymotion:user**
  - **DailymotionCloud**
- - **Daisuki**
- - **DaisukiPlaylist**
+ - **DaisukiMotto**
+ - **DaisukiMottoPlaylist**
  - **daum.net**
  - **daum.net:clip**
  - **daum.net:playlist**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 81e3f79..88bf1d6 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.11.26'
+__version__ = '2017.12.02'
-- 
cgit v1.1


From d7df30898115c4762514868140cfa856618cdfd5 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sat, 2 Dec 2017 20:22:17 +0100
Subject: [toutv] fix login(closes 14614)

---
 youtube_dl/extractor/toutv.py | 28 ++++++++++++++++++++++------
 1 file changed, 22 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/toutv.py b/youtube_dl/extractor/toutv.py
index e59ed26..17c0adc 100644
--- a/youtube_dl/extractor/toutv.py
+++ b/youtube_dl/extractor/toutv.py
@@ -1,6 +1,8 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 from ..utils import (
     int_or_none,
@@ -41,7 +43,7 @@ class TouTvIE(InfoExtractor):
         email, password = self._get_login_info()
         if email is None:
             return
-        state = 'http://ici.tou.tv//'
+        state = 'http://ici.tou.tv/'
         webpage = self._download_webpage(state, None, 'Downloading homepage')
         toutvlogin = self._parse_json(self._search_regex(
             r'(?s)toutvlogin\s*=\s*({.+?});', webpage, 'toutvlogin'), None, js_to_json)
@@ -54,16 +56,30 @@ class TouTvIE(InfoExtractor):
                 'scope': 'media-drmt openid profile email id.write media-validation.read.privileged',
                 'state': state,
             })
-        login_form = self._search_regex(
-            r'(?s)(<form[^>]+(?:id|name)="Form-login".+?</form>)', login_webpage, 'login form')
-        form_data = self._hidden_inputs(login_form)
+
+        def extract_form_url_and_data(wp, default_form_url, form_spec_re=''):
+            form, form_elem = re.search(
+                r'(?s)((<form[^>]+?%s[^>]*?>).+?</form>)' % form_spec_re, wp).groups()
+            form_data = self._hidden_inputs(form)
+            form_url = extract_attributes(form_elem).get('action') or default_form_url
+            return form_url, form_data
+
+        post_url, form_data = extract_form_url_and_data(
+            login_webpage,
+            'https://services.radio-canada.ca/auth/oauth/v2/authorize/login',
+            r'(?:id|name)="Form-login"')
         form_data.update({
             'login-email': email,
             'login-password': password,
         })
-        post_url = extract_attributes(login_form).get('action') or authorize_url
-        _, urlh = self._download_webpage_handle(
+        consent_webpage = self._download_webpage(
             post_url, None, 'Logging in', data=urlencode_postdata(form_data))
+        post_url, form_data = extract_form_url_and_data(
+            consent_webpage,
+            'https://services.radio-canada.ca/auth/oauth/v2/authorize/consent')
+        _, urlh = self._download_webpage_handle(
+            post_url, None, 'Following Redirection',
+            data=urlencode_postdata(form_data))
         self._access_token = self._search_regex(
             r'access_token=([\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})',
             urlh.geturl(), 'access token')
-- 
cgit v1.1


From d4f05d473134d7bd61b054468e6ba297cef3c88f Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sun, 3 Dec 2017 00:04:43 +0100
Subject: [utils] add sami mimetype to mimetype2ext

---
 youtube_dl/utils.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index febc9d2..eccbc0b 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -2350,6 +2350,7 @@ def mimetype2ext(mt):
         'ttml+xml': 'ttml',
         'x-flv': 'flv',
         'x-mp4-fragmented': 'mp4',
+        'x-ms-sami': 'sami',
         'x-ms-wmv': 'wmv',
         'mpegurl': 'm3u8',
         'x-mpegurl': 'm3u8',
-- 
cgit v1.1


From c94427dd60f9836e3752077977cae3b4d0da47d1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 5 Dec 2017 22:34:56 +0700
Subject: [pluralsight] Detect agreement request (#14913)

---
 youtube_dl/extractor/pluralsight.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/youtube_dl/extractor/pluralsight.py b/youtube_dl/extractor/pluralsight.py
index 4bf0aa7..597b112 100644
--- a/youtube_dl/extractor/pluralsight.py
+++ b/youtube_dl/extractor/pluralsight.py
@@ -131,6 +131,13 @@ class PluralsightIE(PluralsightBaseIE):
             if BLOCKED in response:
                 raise ExtractorError(
                     'Unable to login: %s' % BLOCKED, expected=True)
+            MUST_AGREE = 'To continue using Pluralsight, you must agree to'
+            if any(p in response for p in (MUST_AGREE, '>Disagree<', '>Agree<')):
+                raise ExtractorError(
+                    'Unable to login: %s some documents. Go to pluralsight.com, '
+                    'log in and agree with what Pluralsight requires.'
+                    % MUST_AGREE, expected=True)
+
             raise ExtractorError('Unable to log in')
 
     def _get_subtitles(self, author, clip_id, lang, name, duration, video_id):
-- 
cgit v1.1


From 61d18c8a4bc91602738bfd2e506c3cbbc3a3788b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 5 Dec 2017 22:42:02 +0700
Subject: [porncom] Fix metadata extraction (closes #14911)

---
 youtube_dl/extractor/porncom.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/porncom.py b/youtube_dl/extractor/porncom.py
index 8218c7d..60ade06 100644
--- a/youtube_dl/extractor/porncom.py
+++ b/youtube_dl/extractor/porncom.py
@@ -77,12 +77,14 @@ class PornComIE(InfoExtractor):
         self._sort_formats(formats)
 
         view_count = str_to_int(self._search_regex(
-            r'class=["\']views["\'][^>]*><p>([\d,.]+)', webpage,
+            (r'Views:\s*</span>\s*<span>\s*([\d,.]+)',
+             r'class=["\']views["\'][^>]*><p>([\d,.]+)'), webpage,
             'view count', fatal=False))
 
         def extract_list(kind):
             s = self._search_regex(
-                r'(?s)<p[^>]*>%s:(.+?)</p>' % kind.capitalize(),
+                (r'(?s)%s:\s*</span>\s*<span>(.+?)</span>' % kind.capitalize(),
+                 r'(?s)<p[^>]*>%s:(.+?)</p>' % kind.capitalize()),
                 webpage, kind, fatal=False)
             return re.findall(r'<a[^>]+>([^<]+)</a>', s or '')
 
-- 
cgit v1.1


From 91328f26b05084a1ddd890866670a1133564ecd0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 5 Dec 2017 23:01:57 +0700
Subject: [ard] Skip invalid stream URLs (closes #14906)

---
 youtube_dl/extractor/ard.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/ard.py b/youtube_dl/extractor/ard.py
index 915f886..ef73d5a 100644
--- a/youtube_dl/extractor/ard.py
+++ b/youtube_dl/extractor/ard.py
@@ -5,6 +5,7 @@ import re
 
 from .common import InfoExtractor
 from .generic import GenericIE
+from ..compat import compat_str
 from ..utils import (
     determine_ext,
     ExtractorError,
@@ -126,6 +127,8 @@ class ARDMediathekIE(InfoExtractor):
                 quality = stream.get('_quality')
                 server = stream.get('_server')
                 for stream_url in stream_urls:
+                    if not isinstance(stream_url, compat_str) or '//' not in stream_url:
+                        continue
                     ext = determine_ext(stream_url)
                     if quality != 'auto' and ext in ('f4m', 'm3u8'):
                         continue
@@ -146,13 +149,11 @@ class ARDMediathekIE(InfoExtractor):
                                 'play_path': stream_url,
                                 'format_id': 'a%s-rtmp-%s' % (num, quality),
                             }
-                        elif stream_url.startswith('http'):
+                        else:
                             f = {
                                 'url': stream_url,
                                 'format_id': 'a%s-%s-%s' % (num, ext, quality)
                             }
-                        else:
-                            continue
                         m = re.search(r'_(?P<width>\d+)x(?P<height>\d+)\.mp4$', stream_url)
                         if m:
                             f.update({
-- 
cgit v1.1


From d3f8b76b69b9bc986354a83a8d4ee4a8617ce52d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 5 Dec 2017 23:11:15 +0700
Subject: [extractor/generic] Fix typo (closes #14902) Don't pass video_id as
 mpd_id

---
 youtube_dl/extractor/generic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 56df2ab..c8b7c2e 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -2195,7 +2195,7 @@ class GenericIE(InfoExtractor):
                 return self.playlist_result(self._parse_xspf(doc, video_id), video_id)
             elif re.match(r'(?i)^(?:{[^}]+})?MPD$', doc.tag):
                 info_dict['formats'] = self._parse_mpd_formats(
-                    doc, video_id,
+                    doc,
                     mpd_base_url=compat_str(full_response.geturl()).rpartition('/')[0],
                     mpd_url=url)
                 self._sort_formats(info_dict['formats'])
-- 
cgit v1.1


From b271e3352603c523bf2c6762973937fd36925aae Mon Sep 17 00:00:00 2001
From: Windom <windom@users.noreply.github.com>
Date: Tue, 5 Dec 2017 19:08:31 +0200
Subject: [xhamster] Add support for mobile URLs and fix thumbnail extraction

---
 youtube_dl/extractor/xhamster.py | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/xhamster.py b/youtube_dl/extractor/xhamster.py
index 52f8ded..68652a2 100644
--- a/youtube_dl/extractor/xhamster.py
+++ b/youtube_dl/extractor/xhamster.py
@@ -76,6 +76,10 @@ class XHamsterIE(InfoExtractor):
             'skip_download': True,
         },
     }, {
+        # mobile site
+        'url': 'https://m.xhamster.com/videos/cute-teen-jacqueline-solo-masturbation-8559111',
+        'only_matching': True,
+    }, {
         'url': 'https://xhamster.com/movies/2272726/amber_slayed_by_the_knight.html',
         'only_matching': True,
     }, {
@@ -93,7 +97,8 @@ class XHamsterIE(InfoExtractor):
         video_id = mobj.group('id') or mobj.group('id_2')
         display_id = mobj.group('display_id') or mobj.group('display_id_2')
 
-        webpage = self._download_webpage(url, video_id)
+        desktop_url = re.sub(r'^(https?://(?:.+?\.)?)m\.', r'\1', url)
+        webpage = self._download_webpage(desktop_url, video_id)
 
         error = self._html_search_regex(
             r'<div[^>]+id=["\']videoClosed["\'][^>]*>(.+?)</div>',
@@ -229,8 +234,8 @@ class XHamsterIE(InfoExtractor):
             webpage, 'uploader', default='anonymous')
 
         thumbnail = self._search_regex(
-            [r'''thumb\s*:\s*(?P<q>["'])(?P<thumbnail>.+?)(?P=q)''',
-             r'''<video[^>]+poster=(?P<q>["'])(?P<thumbnail>.+?)(?P=q)[^>]*>'''],
+            [r'''["']thumbUrl["']\s*:\s*(?P<q>["'])(?P<thumbnail>.+?)(?P=q)''',
+             r'''<video[^>]+"poster"=(?P<q>["'])(?P<thumbnail>.+?)(?P=q)[^>]*>'''],
             webpage, 'thumbnail', fatal=False, group='thumbnail')
 
         duration = parse_duration(self._search_regex(
@@ -274,15 +279,16 @@ class XHamsterIE(InfoExtractor):
 
 
 class XHamsterEmbedIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?xhamster\.com/xembed\.php\?video=(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:.+?\.)?xhamster\.com/xembed\.php\?video=(?P<id>\d+)'
     _TEST = {
         'url': 'http://xhamster.com/xembed.php?video=3328539',
         'info_dict': {
             'id': '3328539',
             'ext': 'mp4',
             'title': 'Pen Masturbation',
+            'timestamp': 1406581861,
             'upload_date': '20140728',
-            'uploader_id': 'anonymous',
+            'uploader': 'ManyakisArt',
             'duration': 5,
             'age_limit': 18,
         }
-- 
cgit v1.1


From 3c4fbfeca2431b120537b9eaedf5977dd0ab13b1 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 6 Dec 2017 10:54:20 +0100
Subject: [dailymotion] remove dailymotion cloud extractor(closes #6794)

https://web.archive.org/web/20160312110217/https://www.dmcloud.net/
---
 youtube_dl/extractor/dailymotion.py | 49 -------------------------------------
 youtube_dl/extractor/extractors.py  |  1 -
 youtube_dl/extractor/francetv.py    |  9 +------
 youtube_dl/extractor/generic.py     | 27 +-------------------
 4 files changed, 2 insertions(+), 84 deletions(-)

diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py
index 21a2d02..0e7d587 100644
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@@ -413,52 +413,3 @@ class DailymotionUserIE(DailymotionPlaylistIE):
             'title': full_user,
             'entries': self._extract_entries(user),
         }
-
-
-class DailymotionCloudIE(DailymotionBaseInfoExtractor):
-    _VALID_URL_PREFIX = r'https?://api\.dmcloud\.net/(?:player/)?embed/'
-    _VALID_URL = r'%s[^/]+/(?P<id>[^/?]+)' % _VALID_URL_PREFIX
-    _VALID_EMBED_URL = r'%s[^/]+/[^\'"]+' % _VALID_URL_PREFIX
-
-    _TESTS = [{
-        # From http://www.francetvinfo.fr/economie/entreprises/les-entreprises-familiales-le-secret-de-la-reussite_933271.html
-        # Tested at FranceTvInfo_2
-        'url': 'http://api.dmcloud.net/embed/4e7343f894a6f677b10006b4/556e03339473995ee145930c?auth=1464865870-0-jyhsm84b-ead4c701fb750cf9367bf4447167a3db&autoplay=1',
-        'only_matching': True,
-    }, {
-        # http://www.francetvinfo.fr/societe/larguez-les-amarres-le-cobaturage-se-developpe_980101.html
-        'url': 'http://api.dmcloud.net/player/embed/4e7343f894a6f677b10006b4/559545469473996d31429f06?auth=1467430263-0-90tglw2l-a3a4b64ed41efe48d7fccad85b8b8fda&autoplay=1',
-        'only_matching': True,
-    }]
-
-    @classmethod
-    def _extract_dmcloud_url(cls, webpage):
-        mobj = re.search(r'<iframe[^>]+src=[\'"](%s)[\'"]' % cls._VALID_EMBED_URL, webpage)
-        if mobj:
-            return mobj.group(1)
-
-        mobj = re.search(
-            r'<input[^>]+id=[\'"]dmcloudUrlEmissionSelect[\'"][^>]+value=[\'"](%s)[\'"]' % cls._VALID_EMBED_URL,
-            webpage)
-        if mobj:
-            return mobj.group(1)
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-
-        webpage = self._download_webpage_no_ff(url, video_id)
-
-        title = self._html_search_regex(r'<title>([^>]+)</title>', webpage, 'title')
-
-        video_info = self._parse_json(self._search_regex(
-            r'var\s+info\s*=\s*([^;]+);', webpage, 'video info'), video_id)
-
-        # TODO: parse ios_url, which is in fact a manifest
-        video_url = video_info['mp4_url']
-
-        return {
-            'id': video_id,
-            'url': video_url,
-            'title': title,
-            'thumbnail': video_info.get('thumbnail_url'),
-        }
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 2cc3bc4..9c9739a 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -246,7 +246,6 @@ from .dailymotion import (
     DailymotionIE,
     DailymotionPlaylistIE,
     DailymotionUserIE,
-    DailymotionCloudIE,
 )
 from .daisuki import (
     DaisukiMottoIE,
diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py
index 5a3abea..80c5970 100644
--- a/youtube_dl/extractor/francetv.py
+++ b/youtube_dl/extractor/francetv.py
@@ -13,10 +13,7 @@ from ..utils import (
     parse_duration,
     determine_ext,
 )
-from .dailymotion import (
-    DailymotionIE,
-    DailymotionCloudIE,
-)
+from .dailymotion import DailymotionIE
 
 
 class FranceTVBaseInfoExtractor(InfoExtractor):
@@ -290,10 +287,6 @@ class FranceTVInfoIE(FranceTVBaseInfoExtractor):
         page_title = mobj.group('title')
         webpage = self._download_webpage(url, page_title)
 
-        dmcloud_url = DailymotionCloudIE._extract_dmcloud_url(webpage)
-        if dmcloud_url:
-            return self.url_result(dmcloud_url, DailymotionCloudIE.ie_key())
-
         dailymotion_urls = DailymotionIE._extract_urls(webpage)
         if dailymotion_urls:
             return self.playlist_result([
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index c8b7c2e..c7b6092 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -59,10 +59,7 @@ from .tnaflix import TNAFlixNetworkEmbedIE
 from .drtuber import DrTuberIE
 from .redtube import RedTubeIE
 from .vimeo import VimeoIE
-from .dailymotion import (
-    DailymotionIE,
-    DailymotionCloudIE,
-)
+from .dailymotion import DailymotionIE
 from .dailymail import DailyMailIE
 from .onionstudios import OnionStudiosIE
 from .viewlift import ViewLiftEmbedIE
@@ -1472,23 +1469,6 @@ class GenericIE(InfoExtractor):
                 'timestamp': 1432570283,
             },
         },
-        # Dailymotion Cloud video
-        {
-            'url': 'http://replay.publicsenat.fr/vod/le-debat/florent-kolandjian,dominique-cena,axel-decourtye,laurence-abeille,bruno-parmentier/175910',
-            'md5': 'dcaf23ad0c67a256f4278bce6e0bae38',
-            'info_dict': {
-                'id': 'x2uy8t3',
-                'ext': 'mp4',
-                'title': 'Sauvons les abeilles ! - Le débat',
-                'description': 'md5:d9082128b1c5277987825d684939ca26',
-                'thumbnail': r're:^https?://.*\.jpe?g$',
-                'timestamp': 1434970506,
-                'upload_date': '20150622',
-                'uploader': 'Public Sénat',
-                'uploader_id': 'xa9gza',
-            },
-            'skip': 'File not found.',
-        },
         # OnionStudios embed
         {
             'url': 'http://www.clickhole.com/video/dont-understand-bitcoin-man-will-mumble-explanatio-2537',
@@ -2704,11 +2684,6 @@ class GenericIE(InfoExtractor):
         if senate_isvp_url:
             return self.url_result(senate_isvp_url, 'SenateISVP')
 
-        # Look for Dailymotion Cloud videos
-        dmcloud_url = DailymotionCloudIE._extract_dmcloud_url(webpage)
-        if dmcloud_url:
-            return self.url_result(dmcloud_url, 'DailymotionCloud')
-
         # Look for OnionStudios embeds
         onionstudios_url = OnionStudiosIE._extract_url(webpage)
         if onionstudios_url:
-- 
cgit v1.1


From 684ae102360dbef6d1e5fcd75a0f86266030e02d Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 6 Dec 2017 22:56:14 +0100
Subject: [fox] add support for adobe pass auth and extract subtitles(close
 #14489)(closes #14205)

---
 youtube_dl/extractor/fox.py | 25 ++++++++++++++++++++++---
 1 file changed, 22 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/fox.py b/youtube_dl/extractor/fox.py
index 5f98d01..11d6c9c 100644
--- a/youtube_dl/extractor/fox.py
+++ b/youtube_dl/extractor/fox.py
@@ -11,6 +11,7 @@ from ..utils import (
     parse_duration,
     try_get,
     unified_timestamp,
+    update_url_query,
 )
 
 
@@ -62,7 +63,8 @@ class FOXIE(AdobePassIE):
         duration = int_or_none(video.get('durationInSeconds')) or int_or_none(
             video.get('duration')) or parse_duration(video.get('duration'))
         timestamp = unified_timestamp(video.get('datePublished'))
-        age_limit = parse_age_limit(video.get('contentRating'))
+        rating = video.get('contentRating')
+        age_limit = parse_age_limit(rating)
 
         data = try_get(
             video, lambda x: x['trackingData']['properties'], dict) or {}
@@ -77,8 +79,24 @@ class FOXIE(AdobePassIE):
         release_year = int_or_none(video.get('releaseYear'))
 
         if data.get('authRequired'):
-            # TODO: AP
-            pass
+            resource = self._get_mvpd_resource(
+                'fbc-fox', title, video.get('guid'), rating)
+            release_url = update_url_query(
+                release_url, {
+                    'auth': self._extract_mvpd_auth(
+                        url, video_id, 'fbc-fox', resource)
+                })
+
+        subtitles = {}
+        for doc_rel in video.get('documentReleases', []):
+            rel_url = doc_rel.get('url')
+            if not url or doc_rel.get('format') != 'SCC':
+                continue
+            subtitles['en'] = [{
+                'url': rel_url,
+                'ext': 'scc',
+            }]
+            break
 
         info = {
             'id': video_id,
@@ -93,6 +111,7 @@ class FOXIE(AdobePassIE):
             'episode': episode,
             'episode_number': episode_number,
             'release_year': release_year,
+            'subtitles': subtitles,
         }
 
         urlh = self._request_webpage(HEADRequest(release_url), video_id)
-- 
cgit v1.1


From 1bd4fc96e6f8978f83e90ca913df18a979445028 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 7 Dec 2017 08:46:30 +0100
Subject: [sonyliv] extract higher quality formats and bypass geo
 restriction(closes #14922)

---
 youtube_dl/extractor/sonyliv.py | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/sonyliv.py b/youtube_dl/extractor/sonyliv.py
index accd112..c3078e2 100644
--- a/youtube_dl/extractor/sonyliv.py
+++ b/youtube_dl/extractor/sonyliv.py
@@ -2,6 +2,7 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..utils import smuggle_url
 
 
 class SonyLIVIE(InfoExtractor):
@@ -10,12 +11,12 @@ class SonyLIVIE(InfoExtractor):
         'url': "http://www.sonyliv.com/details/episodes/5024612095001/Ep.-1---Achaari-Cheese-Toast---Bachelor's-Delight",
         'info_dict': {
             'title': "Ep. 1 - Achaari Cheese Toast - Bachelor's Delight",
-            'id': '5024612095001',
+            'id': 'ref:5024612095001',
             'ext': 'mp4',
-            'upload_date': '20160707',
+            'upload_date': '20170923',
             'description': 'md5:7f28509a148d5be9d0782b4d5106410d',
-            'uploader_id': '4338955589001',
-            'timestamp': 1467870968,
+            'uploader_id': '5182475815001',
+            'timestamp': 1506200547,
         },
         'params': {
             'skip_download': True,
@@ -26,9 +27,11 @@ class SonyLIVIE(InfoExtractor):
         'only_matching': True,
     }]
 
-    BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/4338955589001/default_default/index.html?videoId=%s'
+    # BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/4338955589001/default_default/index.html?videoId=%s'
+    BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/5182475815001/default_default/index.html?videoId=ref:%s'
 
     def _real_extract(self, url):
         brightcove_id = self._match_id(url)
         return self.url_result(
-            self.BRIGHTCOVE_URL_TEMPLATE % brightcove_id, 'BrightcoveNew', brightcove_id)
+            smuggle_url(self.BRIGHTCOVE_URL_TEMPLATE % brightcove_id, {'geo_countries': ['IN']}),
+            'BrightcoveNew', brightcove_id)
-- 
cgit v1.1


From a670b1ba266533c520a436469585904321808a1b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 7 Dec 2017 22:16:41 +0700
Subject: [README.md] Add is_live, start_time and end_time to output template
 section (closes #14926)

---
 README.md | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/README.md b/README.md
index ea321d5..cd30d14 100644
--- a/README.md
+++ b/README.md
@@ -511,6 +511,9 @@ The basic usage is not to set any template arguments when downloading a single f
  - `average_rating` (numeric): Average rating give by users, the scale used depends on the webpage
  - `comment_count` (numeric): Number of comments on the video
  - `age_limit` (numeric): Age restriction for the video (years)
+ - `is_live` (boolean): Whether this video is a live stream or a fixed-length video
+ - `start_time` (numeric): Time in seconds where the reproduction should start, as specified in the URL
+ - `end_time` (numeric): Time in seconds where the reproduction should end, as specified in the URL
  - `format` (string): A human-readable description of the format 
  - `format_id` (string): Format code specified by `--format`
  - `format_note` (string): Additional info about the format
-- 
cgit v1.1


From d21d0ba6c14e8a6696130090641da4e2028e1bb3 Mon Sep 17 00:00:00 2001
From: Timendum <timedum@gmail.com>
Date: Mon, 23 Oct 2017 15:32:45 +0200
Subject: [raiplay:playlist] Add extractor

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/rai.py        | 26 ++++++++++++++++++++++++++
 2 files changed, 27 insertions(+)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 9c9739a..d8f9f94 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -857,6 +857,7 @@ from .rai import (
     RaiPlayIE,
     RaiPlayLiveIE,
     RaiIE,
+    RaiPlaylistIE,
 )
 from .rbmaradio import RBMARadioIE
 from .rds import RDSIE
diff --git a/youtube_dl/extractor/rai.py b/youtube_dl/extractor/rai.py
index 5bf64a5..6254583 100644
--- a/youtube_dl/extractor/rai.py
+++ b/youtube_dl/extractor/rai.py
@@ -455,3 +455,29 @@ class RaiIE(RaiBaseIE):
         info.update(relinker_info)
 
         return info
+
+
+class RaiPlaylistIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?raiplay\.it/programmi/(?P<id>[^/]+)'
+    _TESTS = [{
+        'url': 'http://www.raiplay.it/programmi/nondirloalmiocapo/',
+        'info_dict': {
+            'id': 'nondirloalmiocapo',
+            'title': 'Non dirlo al mio capo',
+        },
+        'playlist_mincount': 12,
+    }]
+
+    def _real_extract(self, url):
+        playlist_id = self._match_id(url)
+        webpage = self._download_webpage(url, playlist_id)
+        title = self._html_search_meta('programma', webpage, default=None)
+        video_urls = re.findall(' href="(/raiplay/video.+)"', webpage)
+        video_urls = [urljoin(url, video_url) for video_url in video_urls]
+        entries = [
+            self.url_result(
+                video_url,
+                RaiPlayIE.ie_key())
+            for video_url in video_urls if RaiPlayIE.suitable(video_url)
+        ]
+        return self.playlist_result(entries, playlist_id, title)
-- 
cgit v1.1


From 1115271ac61b89cc4ac1ca922eff8a4bed0fbf57 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 9 Dec 2017 00:46:28 +0700
Subject: [raiplay:playlist] Fix issues and improve (closes #14563)

---
 youtube_dl/extractor/extractors.py |  2 +-
 youtube_dl/extractor/rai.py        | 62 ++++++++++++++++++++++----------------
 2 files changed, 37 insertions(+), 27 deletions(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index d8f9f94..b9c97fa 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -856,8 +856,8 @@ from .radiofrance import RadioFranceIE
 from .rai import (
     RaiPlayIE,
     RaiPlayLiveIE,
+    RaiPlayPlaylistIE,
     RaiIE,
-    RaiPlaylistIE,
 )
 from .rbmaradio import RBMARadioIE
 from .rds import RDSIE
diff --git a/youtube_dl/extractor/rai.py b/youtube_dl/extractor/rai.py
index 6254583..d223110 100644
--- a/youtube_dl/extractor/rai.py
+++ b/youtube_dl/extractor/rai.py
@@ -17,6 +17,7 @@ from ..utils import (
     parse_duration,
     strip_or_none,
     try_get,
+    unescapeHTML,
     unified_strdate,
     unified_timestamp,
     update_url_query,
@@ -249,6 +250,41 @@ class RaiPlayLiveIE(RaiBaseIE):
         }
 
 
+class RaiPlayPlaylistIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?raiplay\.it/programmi/(?P<id>[^/?#&]+)'
+    _TESTS = [{
+        'url': 'http://www.raiplay.it/programmi/nondirloalmiocapo/',
+        'info_dict': {
+            'id': 'nondirloalmiocapo',
+            'title': 'Non dirlo al mio capo',
+            'description': 'md5:9f3d603b2947c1c7abb098f3b14fac86',
+        },
+        'playlist_mincount': 12,
+    }]
+
+    def _real_extract(self, url):
+        playlist_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, playlist_id)
+
+        title = self._html_search_meta(
+            ('programma', 'nomeProgramma'), webpage, 'title')
+        description = unescapeHTML(self._html_search_meta(
+            ('description', 'og:description'), webpage, 'description'))
+        print(description)
+
+        entries = []
+        for mobj in re.finditer(
+                r'<a\b[^>]+\bhref=(["\'])(?P<path>/raiplay/video/.+?)\1',
+                webpage):
+            video_url = urljoin(url, mobj.group('path'))
+            entries.append(self.url_result(
+                video_url, ie=RaiPlayIE.ie_key(),
+                video_id=RaiPlayIE._match_id(video_url)))
+
+        return self.playlist_result(entries, playlist_id, title, description)
+
+
 class RaiIE(RaiBaseIE):
     _VALID_URL = r'https?://[^/]+\.(?:rai\.(?:it|tv)|rainews\.it)/dl/.+?-(?P<id>%s)(?:-.+?)?\.html' % RaiBaseIE._UUID_RE
     _TESTS = [{
@@ -455,29 +491,3 @@ class RaiIE(RaiBaseIE):
         info.update(relinker_info)
 
         return info
-
-
-class RaiPlaylistIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?raiplay\.it/programmi/(?P<id>[^/]+)'
-    _TESTS = [{
-        'url': 'http://www.raiplay.it/programmi/nondirloalmiocapo/',
-        'info_dict': {
-            'id': 'nondirloalmiocapo',
-            'title': 'Non dirlo al mio capo',
-        },
-        'playlist_mincount': 12,
-    }]
-
-    def _real_extract(self, url):
-        playlist_id = self._match_id(url)
-        webpage = self._download_webpage(url, playlist_id)
-        title = self._html_search_meta('programma', webpage, default=None)
-        video_urls = re.findall(' href="(/raiplay/video.+)"', webpage)
-        video_urls = [urljoin(url, video_url) for video_url in video_urls]
-        entries = [
-            self.url_result(
-                video_url,
-                RaiPlayIE.ie_key())
-            for video_url in video_urls if RaiPlayIE.suitable(video_url)
-        ]
-        return self.playlist_result(entries, playlist_id, title)
-- 
cgit v1.1


From e2707a832cd53e2cfa68b99db997890a6a5bd685 Mon Sep 17 00:00:00 2001
From: Alex Seiler <seileralex@gmail.com>
Date: Mon, 23 Oct 2017 21:15:48 +0200
Subject: [ellentube] Fix extraction (closes #14407)

---
 youtube_dl/extractor/ellentube.py  | 140 +++++++++++++++++++++++++++++++++++++
 youtube_dl/extractor/ellentv.py    | 101 --------------------------
 youtube_dl/extractor/extractors.py |   8 ++-
 3 files changed, 145 insertions(+), 104 deletions(-)
 create mode 100644 youtube_dl/extractor/ellentube.py
 delete mode 100644 youtube_dl/extractor/ellentv.py

diff --git a/youtube_dl/extractor/ellentube.py b/youtube_dl/extractor/ellentube.py
new file mode 100644
index 0000000..68fe172
--- /dev/null
+++ b/youtube_dl/extractor/ellentube.py
@@ -0,0 +1,140 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    clean_html,
+    int_or_none,
+)
+
+
+class EllenTubeIE(InfoExtractor):
+    _VALID_URL = r'''(?x)
+                    (?:
+                        https://api-prod\.ellentube\.com/ellenapi/api/item/
+                        |ellentube:
+                    )
+                    (?P<id>
+                        [\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12}
+                    )'''
+
+    _TESTS = [{
+        'url': 'https://api-prod.ellentube.com/ellenapi/api/item/75c64c16-aefd-4558-b4f5-3de09b22e6fc',
+        'match_only': True,
+    }, {
+        'url': 'ellentube:734a3353-f697-4e79-9ca9-bfc3002dc1e0',
+        'match_only': True,
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        data = self._download_json(
+            'https://api-prod.ellentube.com/ellenapi/api/item/%s' % video_id, video_id)
+        title = data['title']
+        description = data.get('description')
+        publish_time = int_or_none(data.get('publishTime'))
+        thumbnail = data.get('thumbnail')
+
+        formats = []
+        duration = None
+        for entry in data.get('media'):
+            if entry.get('id') == 'm3u8':
+                formats = self._extract_m3u8_formats(
+                    entry.get('url'), video_id, 'mp4', entry_protocol='m3u8_native', m3u8_id='hls')
+                duration = int_or_none(entry.get('duration'))
+                break
+        self._sort_formats(formats)
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'duration': duration,
+            'thumbnail': thumbnail,
+            'timestamp': publish_time,
+            'formats': formats,
+        }
+
+
+class EllenTubeVideoIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?ellentube\.com/video/(?P<id>.+)\.html'
+
+    _TEST = {
+        'url': 'https://www.ellentube.com/video/ellen-meets-las-vegas-survivors-jesus-campos-and-stephen-schuck.html',
+        'md5': '2fabc277131bddafdd120e0fc0f974c9',
+        'info_dict': {
+            'id': '0822171c-3829-43bf-b99f-d77358ae75e3',
+            'ext': 'mp4',
+            'title': 'Ellen Meets Las Vegas Survivors Jesus Campos and Stephen Schuck',
+            'description': 'md5:76e3355e2242a78ad9e3858e5616923f',
+            'duration': 514,
+            'timestamp': 1508505120000,
+            'thumbnail': 'https://warnerbros-h.assetsadobe.com/is/image/content/dam/ellen/videos/episodes/season15/32/video--2728751654987218111',
+        }
+    }
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+        video_id = self._html_search_regex(
+            r'(?s)<!--\s*CONTENT\s*-->.*data-config.+([\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})',
+            webpage, 'video id')
+        return self.url_result('ellentube:%s' % video_id, 'EllenTube')
+
+
+class EllenTubePlaylistIE(InfoExtractor):
+    def _extract_videos_from_json(self, data, display_id):
+        return [self.url_result('ellentube:%s' % elem['id'], 'EllenTube')
+                for elem in data if elem.get('type') == 'VIDEO']
+
+    def _extract_playlist(self, url, display_id, extract_description=True):
+        webpage = self._download_webpage(url, display_id)
+        playlist_data = self._html_search_regex(
+            r'<div\s+data-component\s*=\s*"Details"(.+)</div>', webpage, 'playlist data')
+        playlist_title = self._search_regex(
+            r'"title"\s*:\s*"(.+?)"', playlist_data, 'playlist title')
+        playlist_description = clean_html(self._search_regex(
+            r'"description"\s*:\s*"(.+?)"', playlist_data, 'playlist description',
+            fatal=False)) if extract_description else None
+        api_search = self._search_regex(
+            r'"filter"\s*:\s*"(.+?)"', playlist_data, 'playlist api request')
+        api_data = self._download_json(
+            'https://api-prod.ellentube.com/ellenapi/api/feed/?%s' % api_search,
+            display_id)
+        return self.playlist_result(
+            self._extract_videos_from_json(api_data, display_id),
+            display_id, playlist_title, playlist_description)
+
+
+class EllenTubeEpisodeIE(EllenTubePlaylistIE):
+    _VALID_URL = r'https?://(?:www\.)?ellentube\.com/episode/(?P<id>.+)\.html'
+
+    _TEST = {
+        'url': 'https://www.ellentube.com/episode/dax-shepard-jordan-fisher-haim.html',
+        'info_dict': {
+            'id': 'dax-shepard-jordan-fisher-haim',
+            'title': 'Dax Shepard, \'DWTS\' Team Jordan Fisher & Lindsay Arnold, HAIM',
+            'description': 'md5:aed85d42892f6126e71ec5ed2aea2a0d'
+        },
+        'playlist_count': 6,
+    }
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        return self._extract_playlist(url, display_id)
+
+
+class EllenTubeStudioIE(EllenTubePlaylistIE):
+    _VALID_URL = r'https?://(?:www\.)?ellentube\.com/studios/(?P<id>.+)\.html'
+
+    _TEST = {
+        'url': 'https://www.ellentube.com/studios/macey-goes-rving0.html',
+        'info_dict': {
+            'id': 'macey-goes-rving0',
+            'title': 'Macey Goes RVing',
+        },
+        'playlist_mincount': 3,
+    }
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        return self._extract_playlist(url, display_id, False)
diff --git a/youtube_dl/extractor/ellentv.py b/youtube_dl/extractor/ellentv.py
deleted file mode 100644
index e0a13dd..0000000
--- a/youtube_dl/extractor/ellentv.py
+++ /dev/null
@@ -1,101 +0,0 @@
-# coding: utf-8
-from __future__ import unicode_literals
-
-from .common import InfoExtractor
-from .kaltura import KalturaIE
-from ..utils import NO_DEFAULT
-
-
-class EllenTVIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?(?:ellentv|ellentube)\.com/videos/(?P<id>[a-z0-9_-]+)'
-    _TESTS = [{
-        'url': 'http://www.ellentv.com/videos/0-ipq1gsai/',
-        'md5': '4294cf98bc165f218aaa0b89e0fd8042',
-        'info_dict': {
-            'id': '0_ipq1gsai',
-            'ext': 'mov',
-            'title': 'Fast Fingers of Fate',
-            'description': 'md5:3539013ddcbfa64b2a6d1b38d910868a',
-            'timestamp': 1428035648,
-            'upload_date': '20150403',
-            'uploader_id': 'batchUser',
-        },
-    }, {
-        # not available via http://widgets.ellentube.com/
-        'url': 'http://www.ellentv.com/videos/1-szkgu2m2/',
-        'info_dict': {
-            'id': '1_szkgu2m2',
-            'ext': 'flv',
-            'title': "Ellen's Amazingly Talented Audience",
-            'description': 'md5:86ff1e376ff0d717d7171590e273f0a5',
-            'timestamp': 1255140900,
-            'upload_date': '20091010',
-            'uploader_id': 'ellenkaltura@gmail.com',
-        },
-        'params': {
-            'skip_download': True,
-        },
-    }]
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-
-        URLS = ('http://widgets.ellentube.com/videos/%s' % video_id, url)
-
-        for num, url_ in enumerate(URLS, 1):
-            webpage = self._download_webpage(
-                url_, video_id, fatal=num == len(URLS))
-
-            default = NO_DEFAULT if num == len(URLS) else None
-
-            partner_id = self._search_regex(
-                r"var\s+partnerId\s*=\s*'([^']+)", webpage, 'partner id',
-                default=default)
-
-            kaltura_id = self._search_regex(
-                [r'id="kaltura_player_([^"]+)"',
-                 r"_wb_entry_id\s*:\s*'([^']+)",
-                 r'data-kaltura-entry-id="([^"]+)'],
-                webpage, 'kaltura id', default=default)
-
-            if partner_id and kaltura_id:
-                break
-
-        return self.url_result('kaltura:%s:%s' % (partner_id, kaltura_id), KalturaIE.ie_key())
-
-
-class EllenTVClipsIE(InfoExtractor):
-    IE_NAME = 'EllenTV:clips'
-    _VALID_URL = r'https?://(?:www\.)?ellentv\.com/episodes/(?P<id>[a-z0-9_-]+)'
-    _TEST = {
-        'url': 'http://www.ellentv.com/episodes/meryl-streep-vanessa-hudgens/',
-        'info_dict': {
-            'id': 'meryl-streep-vanessa-hudgens',
-            'title': 'Meryl Streep, Vanessa Hudgens',
-        },
-        'playlist_mincount': 5,
-    }
-
-    def _real_extract(self, url):
-        playlist_id = self._match_id(url)
-
-        webpage = self._download_webpage(url, playlist_id)
-        playlist = self._extract_playlist(webpage, playlist_id)
-
-        return {
-            '_type': 'playlist',
-            'id': playlist_id,
-            'title': self._og_search_title(webpage),
-            'entries': self._extract_entries(playlist)
-        }
-
-    def _extract_playlist(self, webpage, playlist_id):
-        json_string = self._search_regex(r'playerView.addClips\(\[\{(.*?)\}\]\);', webpage, 'json')
-        return self._parse_json('[{' + json_string + '}]', playlist_id)
-
-    def _extract_entries(self, playlist):
-        return [
-            self.url_result(
-                'kaltura:%s:%s' % (item['kaltura_partner_id'], item['kaltura_entry_id']),
-                KalturaIE.ie_key(), video_id=item['kaltura_entry_id'])
-            for item in playlist]
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index b9c97fa..55d2dd1 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -311,9 +311,11 @@ from .ehow import EHowIE
 from .eighttracks import EightTracksIE
 from .einthusan import EinthusanIE
 from .eitb import EitbIE
-from .ellentv import (
-    EllenTVIE,
-    EllenTVClipsIE,
+from .ellentube import (
+    EllenTubeIE,
+    EllenTubeEpisodeIE,
+    EllenTubeStudioIE,
+    EllenTubeVideoIE,
 )
 from .elpais import ElPaisIE
 from .embedly import EmbedlyIE
-- 
cgit v1.1


From 2a57b62b8007973b5b4974a1d9f5ab06ae78c86e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 9 Dec 2017 02:16:23 +0700
Subject: [ellentube] Fix issues, improve and simplify (closes #14570)

---
 youtube_dl/extractor/ellentube.py  | 165 ++++++++++++++++++-------------------
 youtube_dl/extractor/extractors.py |   3 +-
 2 files changed, 80 insertions(+), 88 deletions(-)

diff --git a/youtube_dl/extractor/ellentube.py b/youtube_dl/extractor/ellentube.py
index 68fe172..5444732 100644
--- a/youtube_dl/extractor/ellentube.py
+++ b/youtube_dl/extractor/ellentube.py
@@ -4,137 +4,130 @@ from __future__ import unicode_literals
 from .common import InfoExtractor
 from ..utils import (
     clean_html,
+    extract_attributes,
+    float_or_none,
     int_or_none,
+    try_get,
 )
 
 
-class EllenTubeIE(InfoExtractor):
-    _VALID_URL = r'''(?x)
-                    (?:
-                        https://api-prod\.ellentube\.com/ellenapi/api/item/
-                        |ellentube:
-                    )
-                    (?P<id>
-                        [\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12}
-                    )'''
-
-    _TESTS = [{
-        'url': 'https://api-prod.ellentube.com/ellenapi/api/item/75c64c16-aefd-4558-b4f5-3de09b22e6fc',
-        'match_only': True,
-    }, {
-        'url': 'ellentube:734a3353-f697-4e79-9ca9-bfc3002dc1e0',
-        'match_only': True,
-    }]
+class EllenTubeBaseIE(InfoExtractor):
+    def _extract_data_config(self, webpage, video_id):
+        details = self._search_regex(
+            r'(<[^>]+\bdata-component=(["\'])[Dd]etails.+?></div>)', webpage,
+            'details')
+        return self._parse_json(
+            extract_attributes(details)['data-config'], video_id)
 
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        data = self._download_json(
-            'https://api-prod.ellentube.com/ellenapi/api/item/%s' % video_id, video_id)
+    def _extract_video(self, data, video_id):
         title = data['title']
-        description = data.get('description')
-        publish_time = int_or_none(data.get('publishTime'))
-        thumbnail = data.get('thumbnail')
 
         formats = []
         duration = None
         for entry in data.get('media'):
             if entry.get('id') == 'm3u8':
                 formats = self._extract_m3u8_formats(
-                    entry.get('url'), video_id, 'mp4', entry_protocol='m3u8_native', m3u8_id='hls')
+                    entry['url'], video_id, 'mp4',
+                    entry_protocol='m3u8_native', m3u8_id='hls')
                 duration = int_or_none(entry.get('duration'))
                 break
         self._sort_formats(formats)
+
+        def get_insight(kind):
+            return int_or_none(try_get(
+                data, lambda x: x['insight']['%ss' % kind]))
+
         return {
+            'extractor_key': EllenTubeIE.ie_key(),
             'id': video_id,
             'title': title,
-            'description': description,
+            'description': data.get('description'),
             'duration': duration,
-            'thumbnail': thumbnail,
-            'timestamp': publish_time,
+            'thumbnail': data.get('thumbnail'),
+            'timestamp': float_or_none(data.get('publishTime'), scale=1000),
+            'view_count': get_insight('view'),
+            'like_count': get_insight('like'),
             'formats': formats,
         }
 
 
-class EllenTubeVideoIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?ellentube\.com/video/(?P<id>.+)\.html'
-
-    _TEST = {
-        'url': 'https://www.ellentube.com/video/ellen-meets-las-vegas-survivors-jesus-campos-and-stephen-schuck.html',
+class EllenTubeIE(EllenTubeBaseIE):
+    _VALID_URL = r'''(?x)
+                        (?:
+                            ellentube:|
+                            https://api-prod\.ellentube\.com/ellenapi/api/item/
+                        )
+                        (?P<id>[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})
+                    '''
+    _TESTS = [{
+        'url': 'https://api-prod.ellentube.com/ellenapi/api/item/0822171c-3829-43bf-b99f-d77358ae75e3',
         'md5': '2fabc277131bddafdd120e0fc0f974c9',
         'info_dict': {
             'id': '0822171c-3829-43bf-b99f-d77358ae75e3',
             'ext': 'mp4',
             'title': 'Ellen Meets Las Vegas Survivors Jesus Campos and Stephen Schuck',
             'description': 'md5:76e3355e2242a78ad9e3858e5616923f',
+            'thumbnail': r're:^https?://.+?',
             'duration': 514,
-            'timestamp': 1508505120000,
-            'thumbnail': 'https://warnerbros-h.assetsadobe.com/is/image/content/dam/ellen/videos/episodes/season15/32/video--2728751654987218111',
+            'timestamp': 1508505120,
+            'upload_date': '20171020',
+            'view_count': int,
+            'like_count': int,
         }
-    }
+    }, {
+        'url': 'ellentube:734a3353-f697-4e79-9ca9-bfc3002dc1e0',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
-        display_id = self._match_id(url)
-        webpage = self._download_webpage(url, display_id)
-        video_id = self._html_search_regex(
-            r'(?s)<!--\s*CONTENT\s*-->.*data-config.+([\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})',
-            webpage, 'video id')
-        return self.url_result('ellentube:%s' % video_id, 'EllenTube')
+        video_id = self._match_id(url)
+        data = self._download_json(
+            'https://api-prod.ellentube.com/ellenapi/api/item/%s' % video_id,
+            video_id)
+        return self._extract_video(data, video_id)
 
 
-class EllenTubePlaylistIE(InfoExtractor):
-    def _extract_videos_from_json(self, data, display_id):
-        return [self.url_result('ellentube:%s' % elem['id'], 'EllenTube')
-                for elem in data if elem.get('type') == 'VIDEO']
+class EllenTubeVideoIE(EllenTubeBaseIE):
+    _VALID_URL = r'https?://(?:www\.)?ellentube\.com/video/(?P<id>.+?)\.html'
+    _TEST = {
+        'url': 'https://www.ellentube.com/video/ellen-meets-las-vegas-survivors-jesus-campos-and-stephen-schuck.html',
+        'only_matching': True,
+    }
 
-    def _extract_playlist(self, url, display_id, extract_description=True):
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
         webpage = self._download_webpage(url, display_id)
-        playlist_data = self._html_search_regex(
-            r'<div\s+data-component\s*=\s*"Details"(.+)</div>', webpage, 'playlist data')
-        playlist_title = self._search_regex(
-            r'"title"\s*:\s*"(.+?)"', playlist_data, 'playlist title')
-        playlist_description = clean_html(self._search_regex(
-            r'"description"\s*:\s*"(.+?)"', playlist_data, 'playlist description',
-            fatal=False)) if extract_description else None
-        api_search = self._search_regex(
-            r'"filter"\s*:\s*"(.+?)"', playlist_data, 'playlist api request')
-        api_data = self._download_json(
-            'https://api-prod.ellentube.com/ellenapi/api/feed/?%s' % api_search,
-            display_id)
-        return self.playlist_result(
-            self._extract_videos_from_json(api_data, display_id),
-            display_id, playlist_title, playlist_description)
+        video_id = self._extract_data_config(webpage, display_id)['id']
+        return self.url_result(
+            'ellentube:%s' % video_id, ie=EllenTubeIE.ie_key(),
+            video_id=video_id)
 
 
-class EllenTubeEpisodeIE(EllenTubePlaylistIE):
-    _VALID_URL = r'https?://(?:www\.)?ellentube\.com/episode/(?P<id>.+)\.html'
-
-    _TEST = {
+class EllenTubePlaylistIE(EllenTubeBaseIE):
+    _VALID_URL = r'https?://(?:www\.)?ellentube\.com/(?:episode|studios)/(?P<id>.+?)\.html'
+    _TESTS = [{
         'url': 'https://www.ellentube.com/episode/dax-shepard-jordan-fisher-haim.html',
         'info_dict': {
             'id': 'dax-shepard-jordan-fisher-haim',
-            'title': 'Dax Shepard, \'DWTS\' Team Jordan Fisher & Lindsay Arnold, HAIM',
-            'description': 'md5:aed85d42892f6126e71ec5ed2aea2a0d'
+            'title': "Dax Shepard, 'DWTS' Team Jordan Fisher & Lindsay Arnold, HAIM",
+            'description': 'md5:bfc982194dabb3f4e325e43aa6b2e21c',
         },
         'playlist_count': 6,
-    }
-
-    def _real_extract(self, url):
-        display_id = self._match_id(url)
-        return self._extract_playlist(url, display_id)
-
-
-class EllenTubeStudioIE(EllenTubePlaylistIE):
-    _VALID_URL = r'https?://(?:www\.)?ellentube\.com/studios/(?P<id>.+)\.html'
-
-    _TEST = {
+    }, {
         'url': 'https://www.ellentube.com/studios/macey-goes-rving0.html',
-        'info_dict': {
-            'id': 'macey-goes-rving0',
-            'title': 'Macey Goes RVing',
-        },
-        'playlist_mincount': 3,
-    }
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         display_id = self._match_id(url)
-        return self._extract_playlist(url, display_id, False)
+        webpage = self._download_webpage(url, display_id)
+        data = self._extract_data_config(webpage, display_id)['data']
+        feed = self._download_json(
+            'https://api-prod.ellentube.com/ellenapi/api/feed/?%s'
+            % data['filter'], display_id)
+        entries = [
+            self._extract_video(elem, elem['id'])
+            for elem in feed if elem.get('type') == 'VIDEO' and elem.get('id')]
+        return self.playlist_result(
+            entries, display_id, data.get('title'),
+            clean_html(data.get('description')))
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 55d2dd1..0177a2c 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -313,9 +313,8 @@ from .einthusan import EinthusanIE
 from .eitb import EitbIE
 from .ellentube import (
     EllenTubeIE,
-    EllenTubeEpisodeIE,
-    EllenTubeStudioIE,
     EllenTubeVideoIE,
+    EllenTubePlaylistIE,
 )
 from .elpais import ElPaisIE
 from .embedly import EmbedlyIE
-- 
cgit v1.1


From f4cc03d60b5dd713fb8964cd9ecf8ca2b1a8a556 Mon Sep 17 00:00:00 2001
From: Andrew Bottom <abottom@wennsoft.com>
Date: Tue, 24 Oct 2017 11:50:02 -0500
Subject: [stretchinternet] Add extractor

---
 youtube_dl/extractor/extractors.py      |  1 +
 youtube_dl/extractor/stretchinternet.py | 28 ++++++++++++++++++++++++++++
 2 files changed, 29 insertions(+)
 create mode 100644 youtube_dl/extractor/stretchinternet.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 0177a2c..612fd35 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1001,6 +1001,7 @@ from .streamango import StreamangoIE
 from .streamcloud import StreamcloudIE
 from .streamcz import StreamCZIE
 from .streetvoice import StreetVoiceIE
+from .stretchinternet import StretchInternetIE
 from .sunporno import SunPornoIE
 from .svt import (
     SVTIE,
diff --git a/youtube_dl/extractor/stretchinternet.py b/youtube_dl/extractor/stretchinternet.py
new file mode 100644
index 0000000..9a0ec0e
--- /dev/null
+++ b/youtube_dl/extractor/stretchinternet.py
@@ -0,0 +1,28 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class StretchInternetIE(InfoExtractor):
+    IE_DESC = 'StretchInternet'
+    _VALID_URL = r'https?://.*?stretchinternet\.com/[^/_?].*(?<=eventId=)(?P<id>.*)(?=&).*'
+    _TEST = {
+        'url': 'https://portal.stretchinternet.com/umary/portal.htm?eventId=313900&streamType=video',
+        'info_dict': {
+            'id': '313900',
+            'ext': 'mp4',
+            'title': 'StretchInternet'
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        stream = self._download_json('https://neo-client.stretchinternet.com/streamservice/v1/media/stream/v%s' % video_id, video_id)
+        stream_url = stream.get('source')
+        return {
+            'ie_key': 'Generic',
+            'id': video_id,
+            'url': 'http://%s' % stream_url,
+            'title': 'StretchInternet'
+        }
-- 
cgit v1.1


From a3de5e6c0e0efef4e8ff0cd37961c594b13c7fb9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 9 Dec 2017 17:58:08 +0700
Subject: [stretchinternet] Fix issues and improve (closes #14576)

---
 youtube_dl/extractor/stretchinternet.py | 38 +++++++++++++++++++++++++--------
 1 file changed, 29 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/stretchinternet.py b/youtube_dl/extractor/stretchinternet.py
index 9a0ec0e..ae2ac1b 100644
--- a/youtube_dl/extractor/stretchinternet.py
+++ b/youtube_dl/extractor/stretchinternet.py
@@ -1,28 +1,48 @@
-# coding: utf-8
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..utils import int_or_none
 
 
 class StretchInternetIE(InfoExtractor):
-    IE_DESC = 'StretchInternet'
-    _VALID_URL = r'https?://.*?stretchinternet\.com/[^/_?].*(?<=eventId=)(?P<id>.*)(?=&).*'
+    _VALID_URL = r'https?://portal\.stretchinternet\.com/[^/]+/portal\.htm\?.*?\beventId=(?P<id>\d+)'
     _TEST = {
         'url': 'https://portal.stretchinternet.com/umary/portal.htm?eventId=313900&streamType=video',
         'info_dict': {
             'id': '313900',
             'ext': 'mp4',
-            'title': 'StretchInternet'
+            'title': 'Augustana (S.D.) Baseball vs University of Mary',
+            'description': 'md5:7578478614aae3bdd4a90f578f787438',
+            'timestamp': 1490468400,
+            'upload_date': '20170325',
         }
     }
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        stream = self._download_json('https://neo-client.stretchinternet.com/streamservice/v1/media/stream/v%s' % video_id, video_id)
-        stream_url = stream.get('source')
+
+        stream = self._download_json(
+            'https://neo-client.stretchinternet.com/streamservice/v1/media/stream/v%s'
+            % video_id, video_id)
+
+        video_url = 'https://%s' % stream['source']
+
+        event = self._download_json(
+            'https://neo-client.stretchinternet.com/portal-ws/getEvent.json',
+            video_id, query={
+                'clientID': 99997,
+                'eventID': video_id,
+                'token': 'asdf',
+            })['event']
+
+        title = event.get('title') or event['mobileTitle']
+        description = event.get('customText')
+        timestamp = int_or_none(event.get('longtime'))
+
         return {
-            'ie_key': 'Generic',
             'id': video_id,
-            'url': 'http://%s' % stream_url,
-            'title': 'StretchInternet'
+            'title': title,
+            'description': description,
+            'timestamp': timestamp,
+            'url': video_url,
         }
-- 
cgit v1.1


From 6f1ec339a0332041d6469887a91cf4e3ff557477 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 9 Dec 2017 19:52:31 +0700
Subject: [udemy] Improve course id extraction (closes #14938)

---
 youtube_dl/extractor/udemy.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/udemy.py b/youtube_dl/extractor/udemy.py
index c248ea7..b660339 100644
--- a/youtube_dl/extractor/udemy.py
+++ b/youtube_dl/extractor/udemy.py
@@ -62,11 +62,11 @@ class UdemyIE(InfoExtractor):
     def _extract_course_info(self, webpage, video_id):
         course = self._parse_json(
             unescapeHTML(self._search_regex(
-                r'ng-init=["\'].*\bcourse=({.+?});', webpage, 'course', default='{}')),
+                r'ng-init=["\'].*\bcourse=({.+?})[;"\']',
+                webpage, 'course', default='{}')),
             video_id, fatal=False) or {}
         course_id = course.get('id') or self._search_regex(
-            (r'&quot;id&quot;\s*:\s*(\d+)', r'data-course-id=["\'](\d+)'),
-            webpage, 'course id')
+            r'data-course-id=["\'](\d+)', webpage, 'course id')
         return course_id, course.get('title')
 
     def _enroll_course(self, base_url, webpage, course_id):
-- 
cgit v1.1


From 913b61eeee9436a3ddf5675ec935d00c3a05b7a4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 9 Dec 2017 20:02:19 +0700
Subject: [udemy] Extract more HLS formats

---
 youtube_dl/extractor/udemy.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/extractor/udemy.py b/youtube_dl/extractor/udemy.py
index b660339..195f5ce 100644
--- a/youtube_dl/extractor/udemy.py
+++ b/youtube_dl/extractor/udemy.py
@@ -257,6 +257,11 @@ class UdemyIE(InfoExtractor):
                 video_url = source.get('file') or source.get('src')
                 if not video_url or not isinstance(video_url, compat_str):
                     continue
+                if source.get('type') == 'application/x-mpegURL' or determine_ext(video_url) == 'm3u8':
+                    formats.extend(self._extract_m3u8_formats(
+                        video_url, video_id, 'mp4', entry_protocol='m3u8_native',
+                        m3u8_id='hls', fatal=False))
+                    continue
                 format_id = source.get('label')
                 f = {
                     'url': video_url,
-- 
cgit v1.1


From 51f2863357af55dfe026bfe47808bf104e7eebf9 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sun, 10 Dec 2017 14:10:52 +0100
Subject: [twitter] improve extraction(closes #14197)

---
 youtube_dl/extractor/twitter.py | 141 ++++++++++++++++++----------------------
 1 file changed, 64 insertions(+), 77 deletions(-)

diff --git a/youtube_dl/extractor/twitter.py b/youtube_dl/extractor/twitter.py
index 1b0b963..d7e4250 100644
--- a/youtube_dl/extractor/twitter.py
+++ b/youtube_dl/extractor/twitter.py
@@ -43,7 +43,7 @@ class TwitterBaseIE(InfoExtractor):
 
 class TwitterCardIE(TwitterBaseIE):
     IE_NAME = 'twitter:card'
-    _VALID_URL = r'https?://(?:www\.)?twitter\.com/i/(?:cards/tfw/v1|videos(?:/tweet)?)/(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?twitter\.com/i/(?P<path>cards/tfw/v1|videos(?:/tweet)?)/(?P<id>\d+)'
     _TESTS = [
         {
             'url': 'https://twitter.com/i/cards/tfw/v1/560070183650213889',
@@ -51,11 +51,10 @@ class TwitterCardIE(TwitterBaseIE):
             'info_dict': {
                 'id': '560070183650213889',
                 'ext': 'mp4',
-                'title': 'Twitter Card',
+                'title': 'Twitter web player',
                 'thumbnail': r're:^https?://.*\.jpg$',
                 'duration': 30.033,
             },
-            'skip': 'Video gone',
         },
         {
             'url': 'https://twitter.com/i/cards/tfw/v1/623160978427936768',
@@ -63,11 +62,9 @@ class TwitterCardIE(TwitterBaseIE):
             'info_dict': {
                 'id': '623160978427936768',
                 'ext': 'mp4',
-                'title': 'Twitter Card',
-                'thumbnail': r're:^https?://.*\.jpg',
-                'duration': 80.155,
+                'title': 'Twitter web player',
+                'thumbnail': r're:^https?://.*(?:\bformat=|\.)jpg',
             },
-            'skip': 'Video gone',
         },
         {
             'url': 'https://twitter.com/i/cards/tfw/v1/654001591733886977',
@@ -120,15 +117,15 @@ class TwitterCardIE(TwitterBaseIE):
             elif media_url.endswith('.mpd'):
                 formats.extend(self._extract_mpd_formats(media_url, video_id, mpd_id='dash'))
             else:
-                vbr = int_or_none(dict_get(media_variant, ('bitRate', 'bitrate')), scale=1000)
+                tbr = int_or_none(dict_get(media_variant, ('bitRate', 'bitrate')), scale=1000)
                 a_format = {
                     'url': media_url,
-                    'format_id': 'http-%d' % vbr if vbr else 'http',
-                    'vbr': vbr,
+                    'format_id': 'http-%d' % tbr if tbr else 'http',
+                    'tbr': tbr,
                 }
                 # Reported bitRate may be zero
-                if not a_format['vbr']:
-                    del a_format['vbr']
+                if not a_format['tbr']:
+                    del a_format['tbr']
 
                 self._search_dimensions_in_video_url(a_format, media_url)
 
@@ -150,79 +147,83 @@ class TwitterCardIE(TwitterBaseIE):
         bearer_token = self._search_regex(
             r'BEARER_TOKEN\s*:\s*"([^"]+)"',
             main_script, 'bearer token')
-        guest_token = self._search_regex(
-            r'document\.cookie\s*=\s*decodeURIComponent\("gt=(\d+)',
-            webpage, 'guest token')
+        # https://developer.twitter.com/en/docs/tweets/post-and-engage/api-reference/get-statuses-show-id
         api_data = self._download_json(
-            'https://api.twitter.com/2/timeline/conversation/%s.json' % video_id,
-            video_id, 'Downloading mobile API data',
+            'https://api.twitter.com/1.1/statuses/show/%s.json' % video_id,
+            video_id, 'Downloading API data',
             headers={
                 'Authorization': 'Bearer ' + bearer_token,
-                'x-guest-token': guest_token,
             })
-        media_info = try_get(api_data, lambda o: o['globalObjects']['tweets'][video_id]
-                                                  ['extended_entities']['media'][0]['video_info']) or {}
+        media_info = try_get(api_data, lambda o: o['extended_entities']['media'][0]['video_info']) or {}
         return self._parse_media_info(media_info, video_id)
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
+        path, video_id = re.search(self._VALID_URL, url).groups()
 
         config = None
         formats = []
         duration = None
 
-        webpage = self._download_webpage(url, video_id)
+        urls = [url]
+        if path.startswith('cards/'):
+            urls.append('https://twitter.com/i/videos/' + video_id)
 
-        iframe_url = self._html_search_regex(
-            r'<iframe[^>]+src="((?:https?:)?//(?:www\.youtube\.com/embed/[^"]+|(?:www\.)?vine\.co/v/\w+/card))"',
-            webpage, 'video iframe', default=None)
-        if iframe_url:
-            return self.url_result(iframe_url)
+        for u in urls:
+            webpage = self._download_webpage(u, video_id)
 
-        config = self._parse_json(self._html_search_regex(
-            r'data-(?:player-)?config="([^"]+)"', webpage,
-            'data player config', default='{}'),
-            video_id)
+            iframe_url = self._html_search_regex(
+                r'<iframe[^>]+src="((?:https?:)?//(?:www\.youtube\.com/embed/[^"]+|(?:www\.)?vine\.co/v/\w+/card))"',
+                webpage, 'video iframe', default=None)
+            if iframe_url:
+                return self.url_result(iframe_url)
 
-        if config.get('source_type') == 'vine':
-            return self.url_result(config['player_url'], 'Vine')
+            config = self._parse_json(self._html_search_regex(
+                r'data-(?:player-)?config="([^"]+)"', webpage,
+                'data player config', default='{}'),
+                video_id)
 
-        periscope_url = PeriscopeIE._extract_url(webpage)
-        if periscope_url:
-            return self.url_result(periscope_url, PeriscopeIE.ie_key())
+            if config.get('source_type') == 'vine':
+                return self.url_result(config['player_url'], 'Vine')
 
-        video_url = config.get('video_url') or config.get('playlist', [{}])[0].get('source')
+            periscope_url = PeriscopeIE._extract_url(webpage)
+            if periscope_url:
+                return self.url_result(periscope_url, PeriscopeIE.ie_key())
 
-        if video_url:
-            if determine_ext(video_url) == 'm3u8':
-                formats.extend(self._extract_m3u8_formats(video_url, video_id, ext='mp4', m3u8_id='hls'))
-            else:
-                f = {
-                    'url': video_url,
-                }
+            video_url = config.get('video_url') or config.get('playlist', [{}])[0].get('source')
+
+            if video_url:
+                if determine_ext(video_url) == 'm3u8':
+                    formats.extend(self._extract_m3u8_formats(video_url, video_id, ext='mp4', m3u8_id='hls'))
+                else:
+                    f = {
+                        'url': video_url,
+                    }
+
+                    self._search_dimensions_in_video_url(f, video_url)
 
-                self._search_dimensions_in_video_url(f, video_url)
+                    formats.append(f)
 
-                formats.append(f)
+            vmap_url = config.get('vmapUrl') or config.get('vmap_url')
+            if vmap_url:
+                formats.extend(
+                    self._extract_formats_from_vmap_url(vmap_url, video_id))
 
-        vmap_url = config.get('vmapUrl') or config.get('vmap_url')
-        if vmap_url:
-            formats.extend(
-                self._extract_formats_from_vmap_url(vmap_url, video_id))
+            media_info = None
 
-        media_info = None
+            for entity in config.get('status', {}).get('entities', []):
+                if 'mediaInfo' in entity:
+                    media_info = entity['mediaInfo']
 
-        for entity in config.get('status', {}).get('entities', []):
-            if 'mediaInfo' in entity:
-                media_info = entity['mediaInfo']
+            if media_info:
+                formats.extend(self._parse_media_info(media_info, video_id))
+                duration = float_or_none(media_info.get('duration', {}).get('nanos'), scale=1e9)
 
-        if media_info:
-            formats.extend(self._parse_media_info(media_info, video_id))
-            duration = float_or_none(media_info.get('duration', {}).get('nanos'), scale=1e9)
+            username = config.get('user', {}).get('screen_name')
+            if username:
+                formats.extend(self._extract_mobile_formats(username, video_id))
 
-        username = config.get('user', {}).get('screen_name')
-        if username:
-            formats.extend(self._extract_mobile_formats(username, video_id))
+            if formats:
+                break
 
         self._remove_duplicate_formats(formats)
         self._sort_formats(formats)
@@ -258,9 +259,6 @@ class TwitterIE(InfoExtractor):
             'uploader_id': 'freethenipple',
             'duration': 12.922,
         },
-        'params': {
-            'skip_download': True,  # requires ffmpeg
-        },
     }, {
         'url': 'https://twitter.com/giphz/status/657991469417025536/photo/1',
         'md5': 'f36dcd5fb92bf7057f155e7d927eeb42',
@@ -277,7 +275,6 @@ class TwitterIE(InfoExtractor):
         'skip': 'Account suspended',
     }, {
         'url': 'https://twitter.com/starwars/status/665052190608723968',
-        'md5': '39b7199856dee6cd4432e72c74bc69d4',
         'info_dict': {
             'id': '665052190608723968',
             'ext': 'mp4',
@@ -303,20 +300,16 @@ class TwitterIE(InfoExtractor):
         },
     }, {
         'url': 'https://twitter.com/jaydingeer/status/700207533655363584',
-        'md5': '',
         'info_dict': {
             'id': '700207533655363584',
             'ext': 'mp4',
-            'title': 'あかさ - BEAT PROD: @suhmeduh #Damndaniel',
-            'description': 'あかさ on Twitter: "BEAT PROD: @suhmeduh  https://t.co/HBrQ4AfpvZ #Damndaniel https://t.co/byBooq2ejZ"',
+            'title': 'JG - BEAT PROD: @suhmeduh #Damndaniel',
+            'description': 'JG on Twitter: "BEAT PROD: @suhmeduh  https://t.co/HBrQ4AfpvZ #Damndaniel https://t.co/byBooq2ejZ"',
             'thumbnail': r're:^https?://.*\.jpg',
-            'uploader': 'あかさ',
+            'uploader': 'JG',
             'uploader_id': 'jaydingeer',
             'duration': 30.0,
         },
-        'params': {
-            'skip_download': True,  # requires ffmpeg
-        },
     }, {
         'url': 'https://twitter.com/Filmdrunk/status/713801302971588609',
         'md5': '89a15ed345d13b86e9a5a5e051fa308a',
@@ -342,9 +335,6 @@ class TwitterIE(InfoExtractor):
             'uploader': 'Captain America',
             'duration': 3.17,
         },
-        'params': {
-            'skip_download': True,  # requires ffmpeg
-        },
     }, {
         'url': 'https://twitter.com/OPP_HSD/status/779210622571536384',
         'info_dict': {
@@ -370,9 +360,6 @@ class TwitterIE(InfoExtractor):
             'uploader_id': 'news_al3alm',
             'duration': 277.4,
         },
-        'params': {
-            'format': 'best[format_id^=http-]',
-        },
     }, {
         'url': 'https://twitter.com/i/web/status/910031516746514432',
         'info_dict': {
-- 
cgit v1.1


From c38970ca10fc8c8e3ba05019d5f73ff235de9fc8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 10 Dec 2017 22:46:21 +0700
Subject: [culturebox] Improve video id extraction (closes #14947)

---
 youtube_dl/extractor/francetv.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py
index 80c5970..095bb39 100644
--- a/youtube_dl/extractor/francetv.py
+++ b/youtube_dl/extractor/francetv.py
@@ -356,6 +356,7 @@ class CultureboxIE(FranceTVBaseInfoExtractor):
             raise ExtractorError('Video %s is not available' % name, expected=True)
 
         video_id, catalogue = self._search_regex(
-            r'"https?://videos\.francetv\.fr/video/([^@]+@[^"]+)"', webpage, 'video id').split('@')
+            r'["\'>]https?://videos\.francetv\.fr/video/([^@]+@.+?)["\'<]',
+            webpage, 'video id').split('@')
 
         return self._extract_video(video_id, catalogue)
-- 
cgit v1.1


From fa1dd6d2cdaa41b67ee2fc47dc5184a040004d41 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 10 Dec 2017 23:15:24 +0700
Subject: [ChangeLog] Actualize

---
 ChangeLog | 27 +++++++++++++++++++++++++++
 1 file changed, 27 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 63837d6..fd3f3eb 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,30 @@
+version <unreleased>
+
+Core
++ [utils] Add sami mimetype to mimetype2ext
+
+Extractors
+* [culturebox] Improve video id extraction (#14947)
+* [twitter] Improve extraction (#14197)
++ [udemy] Extract more HLS formats
+* [udemy] Improve course id extraction (#14938)
++ [stretchinternet] Add support for portal.stretchinternet.com (#14576)
+* [ellentube] Fix extraction (#14407, #14570)
++ [raiplay:playlist] Add support for playlists (#14563)
+* [sonyliv] Bypass geo restriction
+* [sonyliv] Extract higher quality formats (#14922)
+* [fox] Extract subtitles
++ [fox] Add support for Adobe Pass authentication (#14205, #14489)
+- [dailymotion:cloud] Remove extractor (#6794)
+* [xhamster] Fix thumbnail extraction (#14780)
++ [xhamster] Add support for mobile URLs (#14780)
+* [generic] Don't pass video id as mpd id while extracting DASH (#14902)
+* [ard] Skip invalid stream URLs (#14906)
+* [porncom] Fix metadata extraction (#14911)
+* [pluralsight] Detect agreement request (#14913)
+* [toutv] Fix login (#14614)
+
+
 version 2017.12.02
 
 Core
-- 
cgit v1.1


From 1fa0dce2c044826eec87b712f90964b90198aaa4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 10 Dec 2017 23:18:53 +0700
Subject: release 2017.12.10

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 docs/supportedsites.md    | 8 +++++---
 youtube_dl/version.py     | 2 +-
 4 files changed, 10 insertions(+), 8 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 513823b..736869b 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.12.02*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.12.02**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.12.10*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.12.10**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through the [README](https://github.com/rg3/youtube-dl/blob/master/README.md), **most notably** the [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ Add the `-v` flag to **your command line** you run youtube-dl with (`youtube-dl
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.12.02
+[debug] youtube-dl version 2017.12.10
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index fd3f3eb..a1fdcab 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.12.10
 
 Core
 + [utils] Add sami mimetype to mimetype2ext
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 0287a40..d5e3a8a 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -198,7 +198,6 @@
  - **dailymotion**
  - **dailymotion:playlist**
  - **dailymotion:user**
- - **DailymotionCloud**
  - **DaisukiMotto**
  - **DaisukiMottoPlaylist**
  - **daum.net**
@@ -243,8 +242,9 @@
  - **eHow**
  - **Einthusan**
  - **eitb.tv**
- - **EllenTV**
- - **EllenTV:clips**
+ - **EllenTube**
+ - **EllenTubePlaylist**
+ - **EllenTubeVideo**
  - **ElPais**: El País
  - **Embedly**
  - **EMPFlix**
@@ -662,6 +662,7 @@
  - **Rai**
  - **RaiPlay**
  - **RaiPlayLive**
+ - **RaiPlayPlaylist**
  - **RBMARadio**
  - **RDS**: RDS.ca
  - **RedBullTV**
@@ -781,6 +782,7 @@
  - **streamcloud.eu**
  - **StreamCZ**
  - **StreetVoice**
+ - **StretchInternet**
  - **SunPorno**
  - **SVT**
  - **SVTPlay**: SVT Play and Öppet arkiv
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 88bf1d6..d0c4383 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.12.02'
+__version__ = '2017.12.10'
-- 
cgit v1.1


From 29ac31afaf627363fbc1f757aa50078d343acf1f Mon Sep 17 00:00:00 2001
From: sprhawk <465558+sprhawk@users.noreply.github.com>
Date: Mon, 11 Dec 2017 12:25:13 +0800
Subject: simply get the correct webpage, but not parsed to extract information

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/weibo.py      | 97 ++++++++++++++++++++++++++++++++++++++
 2 files changed, 98 insertions(+)
 create mode 100644 youtube_dl/extractor/weibo.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 2cc3bc4..12dc2e7 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1286,6 +1286,7 @@ from .webofstories import (
     WebOfStoriesIE,
     WebOfStoriesPlaylistIE,
 )
+from .weibo import WeiboIE
 from .weiqitv import WeiqiTVIE
 from .wimp import WimpIE
 from .wistia import WistiaIE
diff --git a/youtube_dl/extractor/weibo.py b/youtube_dl/extractor/weibo.py
new file mode 100644
index 0000000..195508e
--- /dev/null
+++ b/youtube_dl/extractor/weibo.py
@@ -0,0 +1,97 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+from urllib.request import Request
+from urllib.parse import urlencode
+import json
+import random as rnd
+
+class WeiboIE(InfoExtractor):
+    _VALID_URL = r'https?://weibo\.com/[0-9]+/(?P<id>[a-zA-Z0-9]+)'
+    _TEST = {
+            'url': 'https://weibo.com/6275294458/Fp6RGfbff?from=page_1005056275294458_profile&wvr=6&mod=weibotime&type=comment',
+            'md5': 'TODO: md5 sum of the first 10241 bytes of the video file (use --test)',
+            'info_dict': {
+                'id': '42',
+                'ext': 'mp4',
+                'title': 'Video title goes here',
+                'thumbnail': r're:^https?://.*\.jpg$',
+                # TODO more properties, either as:
+                # * A value
+                # * MD5 checksum; start the string with md5:
+                # * A regular expression; start the string with re:
+                # * Any Python type (for example int or float)
+                }
+            }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        headers = {
+            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
+            'Accept-Encoding': 'gzip, deflate, br',
+            'Accept-Language': 'en,zh-CN;q=0.9,zh;q=0.8',
+            'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36',
+            'Upgrade-Insecure-Requests': '1',
+        }
+        # to get Referer url for genvisitor
+        webpage,urlh = self._download_webpage_handle(url, video_id, headers=headers, note="first visit the page")
+
+        visitor_url = urlh.geturl()
+
+        data = urlencode({
+            "cb": "gen_callback",
+            "fp": '{"os":"2","browser":"Gecko57,0,0,0","fonts":"undefined","screenInfo":"1440*900*24","plugins":""}',
+            }).encode()
+        headers = {
+                'Accept-Encoding': 'gzip, deflate, br',
+                'Accept': '*/*',
+                'Referer': visitor_url,
+                }
+
+        r_genvisitor = Request(
+            'https://passport.weibo.com/visitor/genvisitor',
+            data = data,
+            headers = headers,
+            method = 'POST'
+            )
+        webpage,urlh = self._download_webpage_handle(r_genvisitor, video_id, note="gen visitor")
+        print("webpage", webpage)
+
+        p = webpage.split("&&")[1] # split "gen_callback && gen_callback(...)"
+        i1 = p.find('{')
+        i2 = p.rfind('}')
+        j = p[i1:i2+1] # get JSON object
+        d = json.loads(j)
+        tid = d["data"]["tid"]
+        cnfd = "%03d" % d["data"]["confidence"]
+
+        param = urlencode({
+            'a': 'incarnate',
+            't': tid,
+            'w': 2,
+            'c': cnfd,
+            'cb': 'cross_domain',
+            'from': 'weibo',
+            '_rand': rnd.random()
+            })
+        gencallback_url = "https://passport.weibo.com/visitor/visitor?" + param
+        webpage,urlh = self._download_webpage_handle(gencallback_url, video_id, note="gen callback")
+        print("webpage", webpage)
+
+        webpage,urlh = self._download_webpage_handle(url, video_id, headers=headers, note="retry to visit the page")
+        print("webpage", webpage)
+
+        # TODO more code goes here, for example ...
+        title = self._html_search_regex(r'<title>(.+?)</title>', webpage, 'title')
+
+        video_sources = self._search_regex(r'video-sources=(.+?)', webpage, 'video_sources')
+        print("video_sources:", video_sources)
+        return {
+                'id': video_id,
+                'title': title,
+                'description': self._og_search_description(webpage),
+                'uploader': self._search_regex(r'<div[^>]+id="uploader"[^>]*>([^<]+)<', webpage, 'uploader', fatal=False),
+                # TODO more properties (see youtube_dl/extractor/common.py)
+                }
-- 
cgit v1.1


From 3281af3464a910cb88f22ef0ece4a8323c2a4d38 Mon Sep 17 00:00:00 2001
From: sprhawk <465558+sprhawk@users.noreply.github.com>
Date: Mon, 11 Dec 2017 15:56:54 +0800
Subject: a working version

---
 youtube_dl/extractor/weibo.py | 41 +++++++++++++++++++++++++----------------
 1 file changed, 25 insertions(+), 16 deletions(-)

diff --git a/youtube_dl/extractor/weibo.py b/youtube_dl/extractor/weibo.py
index 195508e..9b398e9 100644
--- a/youtube_dl/extractor/weibo.py
+++ b/youtube_dl/extractor/weibo.py
@@ -5,24 +5,19 @@ from .common import InfoExtractor
 
 from urllib.request import Request
 from urllib.parse import urlencode
+from urllib import parse
 import json
 import random as rnd
+from os import path
 
 class WeiboIE(InfoExtractor):
     _VALID_URL = r'https?://weibo\.com/[0-9]+/(?P<id>[a-zA-Z0-9]+)'
     _TEST = {
-            'url': 'https://weibo.com/6275294458/Fp6RGfbff?from=page_1005056275294458_profile&wvr=6&mod=weibotime&type=comment',
-            'md5': 'TODO: md5 sum of the first 10241 bytes of the video file (use --test)',
+            'url': 'https://weibo.com/6275294458/Fp6RGfbff?type=comment',
             'info_dict': {
-                'id': '42',
+                'id': 'Fp6RGfbff',
                 'ext': 'mp4',
-                'title': 'Video title goes here',
-                'thumbnail': r're:^https?://.*\.jpg$',
-                # TODO more properties, either as:
-                # * A value
-                # * MD5 checksum; start the string with md5:
-                # * A regular expression; start the string with re:
-                # * Any Python type (for example int or float)
+                'title': 'You should have servants to massage you,... 来自Hosico_猫 - 微博',
                 }
             }
 
@@ -78,20 +73,34 @@ class WeiboIE(InfoExtractor):
             })
         gencallback_url = "https://passport.weibo.com/visitor/visitor?" + param
         webpage,urlh = self._download_webpage_handle(gencallback_url, video_id, note="gen callback")
-        print("webpage", webpage)
 
         webpage,urlh = self._download_webpage_handle(url, video_id, headers=headers, note="retry to visit the page")
-        print("webpage", webpage)
 
         # TODO more code goes here, for example ...
         title = self._html_search_regex(r'<title>(.+?)</title>', webpage, 'title')
 
-        video_sources = self._search_regex(r'video-sources=(.+?)', webpage, 'video_sources')
-        print("video_sources:", video_sources)
+        video_sources_text = self._search_regex("video-sources=\\\\\"(.+?)\"", webpage, 'video_sources')
+        
+        video_formats = parse.parse_qs(video_sources_text)
+
+        formats = []
+        supported_resolutions = ['720', '480']
+        for res in supported_resolutions:
+            f = video_formats.get(res)
+            if isinstance(f, list):
+                if len(f) > 0:
+                    vid_url = f[0]
+                    print("%s:%s" % (res, vid_url))
+                    formats.append({
+                        'url': vid_url
+                        })
+        self._sort_formats(formats)
+        uploader = self._og_search_property('nick-name', webpage, 'uploader', default = None)
+        print(title, uploader)
         return {
                 'id': video_id,
                 'title': title,
-                'description': self._og_search_description(webpage),
-                'uploader': self._search_regex(r'<div[^>]+id="uploader"[^>]*>([^<]+)<', webpage, 'uploader', fatal=False),
+                'uploader': uploader,
+                'formats': formats
                 # TODO more properties (see youtube_dl/extractor/common.py)
                 }
-- 
cgit v1.1


From 0c69958844a446bc3373f45f8f750cbc3202d14e Mon Sep 17 00:00:00 2001
From: sprhawk <465558+sprhawk@users.noreply.github.com>
Date: Mon, 11 Dec 2017 16:02:14 +0800
Subject: add other properties; remove print verbose

---
 youtube_dl/extractor/weibo.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/weibo.py b/youtube_dl/extractor/weibo.py
index 9b398e9..b835f89 100644
--- a/youtube_dl/extractor/weibo.py
+++ b/youtube_dl/extractor/weibo.py
@@ -52,7 +52,6 @@ class WeiboIE(InfoExtractor):
             method = 'POST'
             )
         webpage,urlh = self._download_webpage_handle(r_genvisitor, video_id, note="gen visitor")
-        print("webpage", webpage)
 
         p = webpage.split("&&")[1] # split "gen_callback && gen_callback(...)"
         i1 = p.find('{')
@@ -90,13 +89,13 @@ class WeiboIE(InfoExtractor):
             if isinstance(f, list):
                 if len(f) > 0:
                     vid_url = f[0]
-                    print("%s:%s" % (res, vid_url))
                     formats.append({
-                        'url': vid_url
+                        'url': vid_url,
+                        'format': 'mp4',
+                        'height': int(res),
                         })
         self._sort_formats(formats)
         uploader = self._og_search_property('nick-name', webpage, 'uploader', default = None)
-        print(title, uploader)
         return {
                 'id': video_id,
                 'title': title,
-- 
cgit v1.1


From b6f78d76c14a1787606aad23c2df0d5158d1fde1 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 11 Dec 2017 13:38:55 +0100
Subject: [tbs] fix extraction(fixes #13658)

---
 youtube_dl/extractor/tbs.py    | 134 ++++++++++++++++++++++++++++-------------
 youtube_dl/extractor/turner.py |  48 ++++++++-------
 2 files changed, 120 insertions(+), 62 deletions(-)

diff --git a/youtube_dl/extractor/tbs.py b/youtube_dl/extractor/tbs.py
index e947453..460bc5d 100644
--- a/youtube_dl/extractor/tbs.py
+++ b/youtube_dl/extractor/tbs.py
@@ -4,58 +4,110 @@ from __future__ import unicode_literals
 import re
 
 from .turner import TurnerBaseIE
-from ..utils import extract_attributes
+from ..utils import (
+    float_or_none,
+    int_or_none,
+    strip_or_none,
+)
 
 
 class TBSIE(TurnerBaseIE):
-    # https://github.com/rg3/youtube-dl/issues/13658
-    _WORKING = False
-
-    _VALID_URL = r'https?://(?:www\.)?(?P<site>tbs|tntdrama)\.com/videos/(?:[^/]+/)+(?P<id>[^/?#]+)\.html'
+    _VALID_URL = r'https?://(?:www\.)?(?P<site>tbs|tntdrama)\.com/(?:movies|shows/[^/]+/(?:clips|season-\d+/episode-\d+))/(?P<id>[^/?#]+)'
     _TESTS = [{
-        'url': 'http://www.tbs.com/videos/people-of-earth/season-1/extras/2007318/theatrical-trailer.html',
-        'md5': '9e61d680e2285066ade7199e6408b2ee',
+        'url': 'http://www.tntdrama.com/shows/the-alienist/clips/monster',
         'info_dict': {
-            'id': '2007318',
+            'id': '8d384cde33b89f3a43ce5329de42903ed5099887',
             'ext': 'mp4',
-            'title': 'Theatrical Trailer',
-            'description': 'Catch the latest comedy from TBS, People of Earth, premiering Halloween night--Monday, October 31, at 9/8c.',
+            'title': 'Monster',
+            'description': 'Get a first look at the theatrical trailer for TNT’s highly anticipated new psychological thriller The Alienist, which premieres January 22 on TNT.',
+            'timestamp': 1508175329,
+            'upload_date': '20171016',
         },
-        'skip': 'TBS videos are deleted after a while',
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        }
     }, {
-        'url': 'http://www.tntdrama.com/videos/good-behavior/season-1/extras/1538823/you-better-run.html',
-        'md5': 'ce53c6ead5e9f3280b4ad2031a6fab56',
-        'info_dict': {
-            'id': '1538823',
-            'ext': 'mp4',
-            'title': 'You Better Run',
-            'description': 'Letty Raines must figure out what she\'s running toward while running away from her past. Good Behavior premieres November 15 at 9/8c.',
-        },
-        'skip': 'TBS videos are deleted after a while',
+        'url': 'http://www.tbs.com/shows/search-party/season-1/episode-1/explicit-the-mysterious-disappearance-of-the-girl-no-one-knew',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.tntdrama.com/movies/star-wars-a-new-hope',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
         domain, display_id = re.match(self._VALID_URL, url).groups()
         site = domain[:3]
         webpage = self._download_webpage(url, display_id)
-        video_params = extract_attributes(self._search_regex(r'(<[^>]+id="page-video"[^>]*>)', webpage, 'video params'))
-        query = None
-        clip_id = video_params.get('clipid')
-        if clip_id:
-            query = 'id=' + clip_id
-        else:
-            query = 'titleId=' + video_params['titleid']
-        return self._extract_cvp_info(
-            'http://www.%s.com/service/cvpXml?%s' % (domain, query), display_id, {
-                'default': {
-                    'media_src': 'http://ht.cdn.turner.com/%s/big' % site,
-                },
-                'secure': {
-                    'media_src': 'http://androidhls-secure.cdn.turner.com/%s/big' % site,
-                    'tokenizer_src': 'http://www.%s.com/video/processors/services/token_ipadAdobe.do' % domain,
-                },
-            }, {
-                'url': url,
-                'site_name': site.upper(),
-                'auth_required': video_params.get('isAuthRequired') != 'false',
-            })
+        video_data = self._parse_json(self._search_regex(
+            r'<script[^>]+?data-drupal-selector="drupal-settings-json"[^>]*?>({.+?})</script>',
+            webpage, 'drupal setting'), display_id)['turner_playlist'][0]
+
+        media_id = video_data['mediaID']
+        title = video_data['title']
+
+        streams_data = self._download_json(
+            'http://medium.ngtv.io/media/%s/tv' % media_id,
+            media_id)['media']['tv']
+        duration = None
+        chapters = []
+        formats = []
+        for supported_type in ('unprotected', 'bulkaes'):
+            stream_data = streams_data.get(supported_type, {})
+            m3u8_url = stream_data.get('secureUrl') or stream_data.get('url')
+            if not m3u8_url:
+                continue
+            if stream_data.get('playlistProtection') == 'spe':
+                m3u8_url = self._add_akamai_spe_token(
+                    'http://www.%s.com/service/token_spe' % site,
+                    m3u8_url, media_id, {
+                        'url': url,
+                        'site_name': site.upper(),
+                        'auth_required': video_data.get('authRequired') == '1',
+                    })
+            formats.extend(self._extract_m3u8_formats(
+                m3u8_url, media_id, 'mp4', m3u8_id='hls', fatal=False))
+
+            duration = float_or_none(stream_data.get('totalRuntime') or video_data.get('duration'))
+
+            if not chapters:
+                for chapter in stream_data.get('contentSegments', []):
+                    start_time = float_or_none(chapter.get('start'))
+                    duration = float_or_none(chapter.get('duration'))
+                    if start_time is None or duration is None:
+                        continue
+                    chapters.append({
+                        'start_time': start_time,
+                        'end_time': start_time + duration,
+                    })
+        self._sort_formats(formats)
+
+        thumbnails = []
+        for image_id, image in video_data.get('images', {}).items():
+            image_url = image.get('url')
+            if not image_url or image.get('type') != 'video':
+                continue
+            i = {
+                'id': image_id,
+                'url': image_url,
+            }
+            mobj = re.search(r'(\d+)x(\d+)', image_url)
+            if mobj:
+                i.update({
+                    'width': int(mobj.group(1)),
+                    'height': int(mobj.group(2)),
+                })
+            thumbnails.append(i)
+
+        return {
+            'id': media_id,
+            'title': title,
+            'description': strip_or_none(video_data.get('descriptionNoTags') or video_data.get('shortDescriptionNoTags')),
+            'duration': duration,
+            'timestamp': int_or_none(video_data.get('created')),
+            'season_number': int_or_none(video_data.get('season')),
+            'episode_number': int_or_none(video_data.get('episode')),
+            'cahpters': chapters,
+            'thumbnails': thumbnails,
+            'formats': formats,
+        }
diff --git a/youtube_dl/extractor/turner.py b/youtube_dl/extractor/turner.py
index efeb677..e73b64a 100644
--- a/youtube_dl/extractor/turner.py
+++ b/youtube_dl/extractor/turner.py
@@ -18,9 +18,32 @@ from ..utils import (
 
 
 class TurnerBaseIE(AdobePassIE):
+    _AKAMAI_SPE_TOKEN_CACHE = {}
+
     def _extract_timestamp(self, video_data):
         return int_or_none(xpath_attr(video_data, 'dateCreated', 'uts'))
 
+    def _add_akamai_spe_token(self, tokenizer_src, video_url, content_id, ap_data):
+        secure_path = self._search_regex(r'https?://[^/]+(.+/)', video_url, 'secure path') + '*'
+        token = self._AKAMAI_SPE_TOKEN_CACHE.get(secure_path)
+        if not token:
+            query = {
+                'path': secure_path,
+                'videoId': content_id,
+            }
+            if ap_data.get('auth_required'):
+                query['accessToken'] = self._extract_mvpd_auth(ap_data['url'], content_id, ap_data['site_name'], ap_data['site_name'])
+            auth = self._download_xml(
+                tokenizer_src, content_id, query=query)
+            error_msg = xpath_text(auth, 'error/msg')
+            if error_msg:
+                raise ExtractorError(error_msg, expected=True)
+            token = xpath_text(auth, 'token')
+            if not token:
+                return video_url
+            self._AKAMAI_SPE_TOKEN_CACHE[secure_path] = token
+        return video_url + '?hdnea=' + token
+
     def _extract_cvp_info(self, data_src, video_id, path_data={}, ap_data={}):
         video_data = self._download_xml(data_src, video_id)
         video_id = video_data.attrib['id']
@@ -33,7 +56,6 @@ class TurnerBaseIE(AdobePassIE):
         #         rtmp_src = splited_rtmp_src[1]
         # aifp = xpath_text(video_data, 'akamai/aifp', default='')
 
-        tokens = {}
         urls = []
         formats = []
         rex = re.compile(
@@ -67,26 +89,10 @@ class TurnerBaseIE(AdobePassIE):
                 secure_path_data = path_data.get('secure')
                 if not secure_path_data:
                     continue
-                video_url = secure_path_data['media_src'] + video_url
-                secure_path = self._search_regex(r'https?://[^/]+(.+/)', video_url, 'secure path') + '*'
-                token = tokens.get(secure_path)
-                if not token:
-                    query = {
-                        'path': secure_path,
-                        'videoId': content_id,
-                    }
-                    if ap_data.get('auth_required'):
-                        query['accessToken'] = self._extract_mvpd_auth(ap_data['url'], video_id, ap_data['site_name'], ap_data['site_name'])
-                    auth = self._download_xml(
-                        secure_path_data['tokenizer_src'], video_id, query=query)
-                    error_msg = xpath_text(auth, 'error/msg')
-                    if error_msg:
-                        raise ExtractorError(error_msg, expected=True)
-                    token = xpath_text(auth, 'token')
-                    if not token:
-                        continue
-                    tokens[secure_path] = token
-                video_url = video_url + '?hdnea=' + token
+                video_url = self._add_akamai_spe_token(
+                    secure_path_data['tokenizer_src'],
+                    secure_path_data['media_src'] + video_url,
+                    content_id, ap_data)
             elif not re.match('https?://', video_url):
                 base_path_data = path_data.get(ext, path_data.get('default', {}))
                 media_src = base_path_data.get('media_src')
-- 
cgit v1.1


From 5868079e9921bfb03c64439d2188b765fc30dcfc Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 11 Dec 2017 14:37:03 +0100
Subject: [nick.com] improve extraction(fixes #14876)

---
 youtube_dl/extractor/nick.py | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/nick.py b/youtube_dl/extractor/nick.py
index 310eea2..cae8faf 100644
--- a/youtube_dl/extractor/nick.py
+++ b/youtube_dl/extractor/nick.py
@@ -10,7 +10,7 @@ from ..utils import update_url_query
 class NickIE(MTVServicesInfoExtractor):
     # None of videos on the website are still alive?
     IE_NAME = 'nick.com'
-    _VALID_URL = r'https?://(?:(?:www|beta)\.)?nick(?:jr)?\.com/(?:[^/]+/)?(?:videos/clip|[^/]+/videos)/(?P<id>[^/?#.]+)'
+    _VALID_URL = r'https?://(?P<domain>(?:(?:www|beta)\.)?nick(?:jr)?\.com)/(?:[^/]+/)?(?:videos/clip|[^/]+/videos)/(?P<id>[^/?#.]+)'
     _FEED_URL = 'http://udat.mtvnservices.com/service1/dispatch.htm'
     _GEO_COUNTRIES = ['US']
     _TESTS = [{
@@ -69,8 +69,14 @@ class NickIE(MTVServicesInfoExtractor):
             'mgid': uri,
         }
 
-    def _extract_mgid(self, webpage):
-        return self._search_regex(r'data-contenturi="([^"]+)', webpage, 'mgid')
+    def _real_extract(self, url):
+        domain, display_id = re.match(self._VALID_URL, url).groups()
+        video_data = self._download_json(
+            'http://%s/data/video.endLevel.json' % domain,
+            display_id, query={
+                'urlKey': display_id,
+            })
+        return self._get_videos_info(video_data['player'] + video_data['id'])
 
 
 class NickDeIE(MTVServicesInfoExtractor):
-- 
cgit v1.1


From 08d77a95c98a1d1dcaa5df99b709f119e7a64f37 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 11 Dec 2017 17:18:05 +0100
Subject: [nickelodeon:br] add support for Nickelodeon(Brazil) websites(closes
 #14893)

---
 youtube_dl/extractor/nick.py | 45 ++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 45 insertions(+)

diff --git a/youtube_dl/extractor/nick.py b/youtube_dl/extractor/nick.py
index cae8faf..f38c2c7 100644
--- a/youtube_dl/extractor/nick.py
+++ b/youtube_dl/extractor/nick.py
@@ -79,6 +79,51 @@ class NickIE(MTVServicesInfoExtractor):
         return self._get_videos_info(video_data['player'] + video_data['id'])
 
 
+class NickBeIE(MTVServicesInfoExtractor):
+    IE_NAME = 'nickelodeon:br'
+    _VALID_URL = r'https?://(?P<domain>(?:www\.)?nickjr|mundonick\.uol)\.com\.br/(?:programas/)?[^/]+/videos/(?:episodios/)?(?P<id>[^/?#.]+)'
+    _TESTS = [{
+        'url': 'http://www.nickjr.com.br/patrulha-canina/videos/210-labirinto-de-pipoca/',
+        'only_matching': True,
+    }, {
+        'url': 'http://mundonick.uol.com.br/programas/the-loud-house/videos/muitas-irmas/7ljo9j',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        domain, display_id = re.match(self._VALID_URL, url).groups()
+        webpage = self._download_webpage(url, display_id)
+        uri = self._search_regex(
+            r'data-(?:contenturi|mgid)="([^"]+)', webpage, 'mgid')
+        video_id = self._id_from_uri(uri)
+        config = self._download_json(
+            'http://media.mtvnservices.com/pmt/e1/access/index.html',
+            video_id, query={
+                'uri': uri,
+                'configtype': 'edge',
+            }, headers={
+                'Referer': url,
+            })
+        info_url = self._remove_template_parameter(config['feedWithQueryParams'])
+        if info_url == 'None':
+            if domain.startswith('www.'):
+                domain = domain[4:]
+            content_domain = {
+                'mundonick.uol': 'mundonick.com.br',
+                'nickjr': 'br.nickelodeonjunior.tv',
+            }[domain]
+            query = {
+                'mgid': uri,
+                'imageEp': content_domain,
+                'arcEp': content_domain,
+            }
+            if domain == 'nickjr.com.br':
+                query['ep'] = 'c4b16088'
+            info_url = update_url_query(
+                'http://feeds.mtvnservices.com/od/feed/intl-mrss-player-feed', query)
+        return self._get_videos_info_from_url(info_url, video_id)
+
+
 class NickDeIE(MTVServicesInfoExtractor):
     IE_NAME = 'nick.de'
     _VALID_URL = r'https?://(?:www\.)?(?P<host>nick\.(?:de|com\.pl|ch)|nickelodeon\.(?:nl|be|at|dk|no|se))/[^/]+/(?:[^/]+/)*(?P<id>[^/?#&]+)'
-- 
cgit v1.1


From e4f201bc1b4ffbf6423278e8d744727aa7d6f415 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 11 Dec 2017 17:19:16 +0100
Subject: [extractors] add import for NickBrIE

---
 youtube_dl/extractor/extractors.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 612fd35..25c4ce0 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -689,6 +689,7 @@ from .nhl import (
 )
 from .nick import (
     NickIE,
+    NickBeIE,
     NickDeIE,
     NickNightIE,
     NickRuIE,
-- 
cgit v1.1


From 127e98d31dd96b135f0f511c4ca171a300e089cc Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 11 Dec 2017 17:21:31 +0100
Subject: [nickelodeon:br] correct extractor name

---
 youtube_dl/extractor/extractors.py | 2 +-
 youtube_dl/extractor/nick.py       | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 25c4ce0..99887ee 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -689,7 +689,7 @@ from .nhl import (
 )
 from .nick import (
     NickIE,
-    NickBeIE,
+    NickBrIE,
     NickDeIE,
     NickNightIE,
     NickRuIE,
diff --git a/youtube_dl/extractor/nick.py b/youtube_dl/extractor/nick.py
index f38c2c7..7edd684 100644
--- a/youtube_dl/extractor/nick.py
+++ b/youtube_dl/extractor/nick.py
@@ -79,7 +79,7 @@ class NickIE(MTVServicesInfoExtractor):
         return self._get_videos_info(video_data['player'] + video_data['id'])
 
 
-class NickBeIE(MTVServicesInfoExtractor):
+class NickBrIE(MTVServicesInfoExtractor):
     IE_NAME = 'nickelodeon:br'
     _VALID_URL = r'https?://(?P<domain>(?:www\.)?nickjr|mundonick\.uol)\.com\.br/(?:programas/)?[^/]+/videos/(?:episodios/)?(?P<id>[^/?#.]+)'
     _TESTS = [{
-- 
cgit v1.1


From 23b6e23002d8bd3af3b2ccb2e1b833c1dd1b3cbf Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 11 Dec 2017 21:36:12 +0100
Subject: [tvnow] fix extraction(closes #7831)

---
 youtube_dl/extractor/extractors.py |   8 +-
 youtube_dl/extractor/nowtv.py      | 261 -------------------------------------
 youtube_dl/extractor/tvnow.py      | 175 +++++++++++++++++++++++++
 3 files changed, 179 insertions(+), 265 deletions(-)
 delete mode 100644 youtube_dl/extractor/nowtv.py
 create mode 100644 youtube_dl/extractor/tvnow.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 99887ee..fb9f365 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -722,10 +722,6 @@ from .nowness import (
     NownessPlaylistIE,
     NownessSeriesIE,
 )
-from .nowtv import (
-    NowTVIE,
-    NowTVListIE,
-)
 from .noz import NozIE
 from .npo import (
     AndereTijdenIE,
@@ -1105,6 +1101,10 @@ from .tvigle import TvigleIE
 from .tvland import TVLandIE
 from .tvn24 import TVN24IE
 from .tvnoe import TVNoeIE
+from .tvnow import (
+    TVNowIE,
+    TVNowListIE,
+)
 from .tvp import (
     TVPEmbedIE,
     TVPIE,
diff --git a/youtube_dl/extractor/nowtv.py b/youtube_dl/extractor/nowtv.py
deleted file mode 100644
index e43b371..0000000
--- a/youtube_dl/extractor/nowtv.py
+++ /dev/null
@@ -1,261 +0,0 @@
-# coding: utf-8
-from __future__ import unicode_literals
-
-import re
-
-from .common import InfoExtractor
-from ..compat import compat_str
-from ..utils import (
-    ExtractorError,
-    determine_ext,
-    int_or_none,
-    parse_iso8601,
-    parse_duration,
-    remove_start,
-)
-
-
-class NowTVBaseIE(InfoExtractor):
-    _VIDEO_FIELDS = (
-        'id', 'title', 'free', 'geoblocked', 'articleLong', 'articleShort',
-        'broadcastStartDate', 'seoUrl', 'duration', 'files',
-        'format.defaultImage169Format', 'format.defaultImage169Logo')
-
-    def _extract_video(self, info, display_id=None):
-        video_id = compat_str(info['id'])
-
-        files = info['files']
-        if not files:
-            if info.get('geoblocked', False):
-                raise ExtractorError(
-                    'Video %s is not available from your location due to geo restriction' % video_id,
-                    expected=True)
-            if not info.get('free', True):
-                raise ExtractorError(
-                    'Video %s is not available for free' % video_id, expected=True)
-
-        formats = []
-        for item in files['items']:
-            if determine_ext(item['path']) != 'f4v':
-                continue
-            app, play_path = remove_start(item['path'], '/').split('/', 1)
-            formats.append({
-                'url': 'rtmpe://fms.rtl.de',
-                'app': app,
-                'play_path': 'mp4:%s' % play_path,
-                'ext': 'flv',
-                'page_url': 'http://rtlnow.rtl.de',
-                'player_url': 'http://cdn.static-fra.de/now/vodplayer.swf',
-                'tbr': int_or_none(item.get('bitrate')),
-            })
-        self._sort_formats(formats)
-
-        title = info['title']
-        description = info.get('articleLong') or info.get('articleShort')
-        timestamp = parse_iso8601(info.get('broadcastStartDate'), ' ')
-        duration = parse_duration(info.get('duration'))
-
-        f = info.get('format', {})
-        thumbnail = f.get('defaultImage169Format') or f.get('defaultImage169Logo')
-
-        return {
-            'id': video_id,
-            'display_id': display_id or info.get('seoUrl'),
-            'title': title,
-            'description': description,
-            'thumbnail': thumbnail,
-            'timestamp': timestamp,
-            'duration': duration,
-            'formats': formats,
-        }
-
-
-class NowTVIE(NowTVBaseIE):
-    _WORKING = False
-    _VALID_URL = r'https?://(?:www\.)?nowtv\.(?:de|at|ch)/(?:rtl|rtl2|rtlnitro|superrtl|ntv|vox)/(?P<show_id>[^/]+)/(?:(?:list/[^/]+|jahr/\d{4}/\d{1,2})/)?(?P<id>[^/]+)/(?:player|preview)'
-
-    _TESTS = [{
-        # rtl
-        'url': 'http://www.nowtv.de/rtl/bauer-sucht-frau/die-neuen-bauern-und-eine-hochzeit/player',
-        'info_dict': {
-            'id': '203519',
-            'display_id': 'bauer-sucht-frau/die-neuen-bauern-und-eine-hochzeit',
-            'ext': 'flv',
-            'title': 'Inka Bause stellt die neuen Bauern vor',
-            'description': 'md5:e234e1ed6d63cf06be5c070442612e7e',
-            'thumbnail': r're:^https?://.*\.jpg$',
-            'timestamp': 1432580700,
-            'upload_date': '20150525',
-            'duration': 2786,
-        },
-        'params': {
-            # rtmp download
-            'skip_download': True,
-        },
-    }, {
-        # rtl2
-        'url': 'http://www.nowtv.de/rtl2/berlin-tag-nacht/berlin-tag-nacht-folge-934/player',
-        'info_dict': {
-            'id': '203481',
-            'display_id': 'berlin-tag-nacht/berlin-tag-nacht-folge-934',
-            'ext': 'flv',
-            'title': 'Berlin - Tag & Nacht (Folge 934)',
-            'description': 'md5:c85e88c2e36c552dfe63433bc9506dd0',
-            'thumbnail': r're:^https?://.*\.jpg$',
-            'timestamp': 1432666800,
-            'upload_date': '20150526',
-            'duration': 2641,
-        },
-        'params': {
-            # rtmp download
-            'skip_download': True,
-        },
-    }, {
-        # rtlnitro
-        'url': 'http://www.nowtv.de/rtlnitro/alarm-fuer-cobra-11-die-autobahnpolizei/hals-und-beinbruch-2014-08-23-21-10-00/player',
-        'info_dict': {
-            'id': '165780',
-            'display_id': 'alarm-fuer-cobra-11-die-autobahnpolizei/hals-und-beinbruch-2014-08-23-21-10-00',
-            'ext': 'flv',
-            'title': 'Hals- und Beinbruch',
-            'description': 'md5:b50d248efffe244e6f56737f0911ca57',
-            'thumbnail': r're:^https?://.*\.jpg$',
-            'timestamp': 1432415400,
-            'upload_date': '20150523',
-            'duration': 2742,
-        },
-        'params': {
-            # rtmp download
-            'skip_download': True,
-        },
-    }, {
-        # superrtl
-        'url': 'http://www.nowtv.de/superrtl/medicopter-117/angst/player',
-        'info_dict': {
-            'id': '99205',
-            'display_id': 'medicopter-117/angst',
-            'ext': 'flv',
-            'title': 'Angst!',
-            'description': 'md5:30cbc4c0b73ec98bcd73c9f2a8c17c4e',
-            'thumbnail': r're:^https?://.*\.jpg$',
-            'timestamp': 1222632900,
-            'upload_date': '20080928',
-            'duration': 3025,
-        },
-        'params': {
-            # rtmp download
-            'skip_download': True,
-        },
-    }, {
-        # ntv
-        'url': 'http://www.nowtv.de/ntv/ratgeber-geld/thema-ua-der-erste-blick-die-apple-watch/player',
-        'info_dict': {
-            'id': '203521',
-            'display_id': 'ratgeber-geld/thema-ua-der-erste-blick-die-apple-watch',
-            'ext': 'flv',
-            'title': 'Thema u.a.: Der erste Blick: Die Apple Watch',
-            'description': 'md5:4312b6c9d839ffe7d8caf03865a531af',
-            'thumbnail': r're:^https?://.*\.jpg$',
-            'timestamp': 1432751700,
-            'upload_date': '20150527',
-            'duration': 1083,
-        },
-        'params': {
-            # rtmp download
-            'skip_download': True,
-        },
-    }, {
-        # vox
-        'url': 'http://www.nowtv.de/vox/der-hundeprofi/buero-fall-chihuahua-joel/player',
-        'info_dict': {
-            'id': '128953',
-            'display_id': 'der-hundeprofi/buero-fall-chihuahua-joel',
-            'ext': 'flv',
-            'title': "Büro-Fall / Chihuahua 'Joel'",
-            'description': 'md5:e62cb6bf7c3cc669179d4f1eb279ad8d',
-            'thumbnail': r're:^https?://.*\.jpg$',
-            'timestamp': 1432408200,
-            'upload_date': '20150523',
-            'duration': 3092,
-        },
-        'params': {
-            # rtmp download
-            'skip_download': True,
-        },
-    }, {
-        'url': 'http://www.nowtv.de/rtl/bauer-sucht-frau/die-neuen-bauern-und-eine-hochzeit/preview',
-        'only_matching': True,
-    }, {
-        'url': 'http://www.nowtv.at/rtl/bauer-sucht-frau/die-neuen-bauern-und-eine-hochzeit/preview?return=/rtl/bauer-sucht-frau/die-neuen-bauern-und-eine-hochzeit',
-        'only_matching': True,
-    }, {
-        'url': 'http://www.nowtv.de/rtl2/echtzeit/list/aktuell/schnelles-geld-am-ende-der-welt/player',
-        'only_matching': True,
-    }, {
-        'url': 'http://www.nowtv.de/rtl2/zuhause-im-glueck/jahr/2015/11/eine-erschuetternde-diagnose/player',
-        'only_matching': True,
-    }]
-
-    def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        display_id = '%s/%s' % (mobj.group('show_id'), mobj.group('id'))
-
-        info = self._download_json(
-            'https://api.nowtv.de/v3/movies/%s?fields=%s'
-            % (display_id, ','.join(self._VIDEO_FIELDS)), display_id)
-
-        return self._extract_video(info, display_id)
-
-
-class NowTVListIE(NowTVBaseIE):
-    _VALID_URL = r'https?://(?:www\.)?nowtv\.(?:de|at|ch)/(?:rtl|rtl2|rtlnitro|superrtl|ntv|vox)/(?P<show_id>[^/]+)/list/(?P<id>[^?/#&]+)$'
-
-    _SHOW_FIELDS = ('title', )
-    _SEASON_FIELDS = ('id', 'headline', 'seoheadline', )
-
-    _TESTS = [{
-        'url': 'http://www.nowtv.at/rtl/stern-tv/list/aktuell',
-        'info_dict': {
-            'id': '17006',
-            'title': 'stern TV - Aktuell',
-        },
-        'playlist_count': 1,
-    }, {
-        'url': 'http://www.nowtv.at/rtl/das-supertalent/list/free-staffel-8',
-        'info_dict': {
-            'id': '20716',
-            'title': 'Das Supertalent - FREE Staffel 8',
-        },
-        'playlist_count': 14,
-    }]
-
-    def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        show_id = mobj.group('show_id')
-        season_id = mobj.group('id')
-
-        fields = []
-        fields.extend(self._SHOW_FIELDS)
-        fields.extend('formatTabs.%s' % field for field in self._SEASON_FIELDS)
-        fields.extend(
-            'formatTabs.formatTabPages.container.movies.%s' % field
-            for field in self._VIDEO_FIELDS)
-
-        list_info = self._download_json(
-            'https://api.nowtv.de/v3/formats/seo?fields=%s&name=%s.php'
-            % (','.join(fields), show_id),
-            season_id)
-
-        season = next(
-            season for season in list_info['formatTabs']['items']
-            if season.get('seoheadline') == season_id)
-
-        title = '%s - %s' % (list_info['title'], season['headline'])
-
-        entries = []
-        for container in season['formatTabPages']['items']:
-            for info in ((container.get('container') or {}).get('movies') or {}).get('items') or []:
-                entries.append(self._extract_video(info))
-
-        return self.playlist_result(
-            entries, compat_str(season.get('id') or season_id), title)
diff --git a/youtube_dl/extractor/tvnow.py b/youtube_dl/extractor/tvnow.py
new file mode 100644
index 0000000..e2169f2
--- /dev/null
+++ b/youtube_dl/extractor/tvnow.py
@@ -0,0 +1,175 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..compat import compat_str
+from ..utils import (
+    ExtractorError,
+    parse_iso8601,
+    parse_duration,
+    update_url_query,
+)
+
+
+class TVNowBaseIE(InfoExtractor):
+    _VIDEO_FIELDS = (
+        'id', 'title', 'free', 'geoblocked', 'articleLong', 'articleShort',
+        'broadcastStartDate', 'isDrm', 'duration', 'manifest.dashclear',
+        'format.defaultImage169Format', 'format.defaultImage169Logo')
+
+    def _call_api(self, path, video_id, query):
+        return self._download_json(
+            'https://api.tvnow.de/v3/' + path,
+            video_id, query=query)
+
+    def _extract_video(self, info, display_id):
+        video_id = compat_str(info['id'])
+        title = info['title']
+
+        mpd_url = info['manifest']['dashclear']
+        if not mpd_url:
+            if info.get('isDrm'):
+                raise ExtractorError(
+                    'Video %s is DRM protected' % video_id, expected=True)
+            if info.get('geoblocked'):
+                raise ExtractorError(
+                    'Video %s is not available from your location due to geo restriction' % video_id,
+                    expected=True)
+            if not info.get('free', True):
+                raise ExtractorError(
+                    'Video %s is not available for free' % video_id, expected=True)
+
+        mpd_url = update_url_query(mpd_url, {'filter': ''})
+        formats = self._extract_mpd_formats(mpd_url, video_id, mpd_id='dash', fatal=False)
+        formats.extend(self._extract_ism_formats(
+            mpd_url.replace('dash.', 'hss.').replace('/.mpd', '/Manifest'),
+            video_id, ism_id='mss', fatal=False))
+        formats.extend(self._extract_m3u8_formats(
+            mpd_url.replace('dash.', 'hls.').replace('/.mpd', '/.m3u8'),
+            video_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False))
+        self._sort_formats(formats)
+
+        description = info.get('articleLong') or info.get('articleShort')
+        timestamp = parse_iso8601(info.get('broadcastStartDate'), ' ')
+        duration = parse_duration(info.get('duration'))
+
+        f = info.get('format', {})
+        thumbnail = f.get('defaultImage169Format') or f.get('defaultImage169Logo')
+
+        return {
+            'id': video_id,
+            'display_id': display_id,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'timestamp': timestamp,
+            'duration': duration,
+            'formats': formats,
+        }
+
+
+class TVNowIE(TVNowBaseIE):
+    _VALID_URL = r'https?://(?:www\.)?tvnow\.(?:de|at|ch)/(?:rtl(?:2|plus)?|nitro|superrtl|ntv|vox)/(?P<show_id>[^/]+)/(?:(?:list/[^/]+|jahr/\d{4}/\d{1,2})/)?(?P<id>[^/]+)/(?:player|preview)'
+
+    _TESTS = [{
+        # rtl
+        'url': 'https://www.tvnow.de/rtl/alarm-fuer-cobra-11/freier-fall/player?return=/rtl',
+        'info_dict': {
+            'id': '385314',
+            'display_id': 'alarm-fuer-cobra-11/freier-fall',
+            'ext': 'mp4',
+            'title': 'Freier Fall',
+            'description': 'md5:8c2d8f727261adf7e0dc18366124ca02',
+            'thumbnail': r're:^https?://.*\.jpg$',
+            'timestamp': 1512677700,
+            'upload_date': '20171207',
+            'duration': 2862.0,
+        },
+    }, {
+        # rtl2
+        'url': 'https://www.tvnow.de/rtl2/armes-deutschland/episode-0008/player',
+        'only_matching': 'True',
+    }, {
+        # rtlnitro
+        'url': 'https://www.tvnow.de/nitro/alarm-fuer-cobra-11-die-autobahnpolizei/auf-eigene-faust-pilot/player',
+        'only_matching': 'True',
+    }, {
+        # superrtl
+        'url': 'https://www.tvnow.de/superrtl/die-lustigsten-schlamassel-der-welt/u-a-ketchup-effekt/player',
+        'only_matching': 'True',
+    }, {
+        # ntv
+        'url': 'https://www.tvnow.de/ntv/startup-news/goetter-in-weiss/player',
+        'only_matching': 'True',
+    }, {
+        # vox
+        'url': 'https://www.tvnow.de/vox/auto-mobil/neues-vom-automobilmarkt-2017-11-19-17-00-00/player',
+        'only_matching': 'True',
+    }, {
+        # rtlplus
+        'url': 'https://www.tvnow.de/rtlplus/op-ruft-dr-bruckner/die-vernaehte-frau/player',
+        'only_matching': 'True',
+    }]
+
+    def _real_extract(self, url):
+        display_id = '%s/%s' % re.match(self._VALID_URL, url).groups()
+
+        info = self._call_api(
+            'movies/' + display_id, display_id, query={
+                'fields': ','.join(self._VIDEO_FIELDS),
+            })
+
+        return self._extract_video(info, display_id)
+
+
+class TVNowListIE(TVNowBaseIE):
+    _VALID_URL = r'(?P<base_url>https?://(?:www\.)?tvnow\.(?:de|at|ch)/(?:rtl(?:2|plus)?|nitro|superrtl|ntv|vox)/(?P<show_id>[^/]+)/)list/(?P<id>[^?/#&]+)$'
+
+    _SHOW_FIELDS = ('title', )
+    _SEASON_FIELDS = ('id', 'headline', 'seoheadline', )
+    _VIDEO_FIELDS = ('id', 'headline', 'seoUrl', )
+
+    _TESTS = [{
+        'url': 'https://www.tvnow.de/rtl/30-minuten-deutschland/list/aktuell',
+        'info_dict': {
+            'id': '28296',
+            'title': '30 Minuten Deutschland - Aktuell',
+        },
+        'playlist_mincount': 1,
+    }]
+
+    def _real_extract(self, url):
+        base_url, show_id, season_id = re.match(self._VALID_URL, url).groups()
+
+        fields = []
+        fields.extend(self._SHOW_FIELDS)
+        fields.extend('formatTabs.%s' % field for field in self._SEASON_FIELDS)
+        fields.extend(
+            'formatTabs.formatTabPages.container.movies.%s' % field
+            for field in self._VIDEO_FIELDS)
+
+        list_info = self._call_api(
+            'formats/seo', season_id, query={
+                'fields': ','.join(fields),
+                'name': show_id + '.php'
+            })
+
+        season = next(
+            season for season in list_info['formatTabs']['items']
+            if season.get('seoheadline') == season_id)
+
+        title = '%s - %s' % (list_info['title'], season['headline'])
+
+        entries = []
+        for container in season['formatTabPages']['items']:
+            for info in ((container.get('container') or {}).get('movies') or {}).get('items') or []:
+                seo_url = info.get('seoUrl')
+                if not seo_url:
+                    continue
+                entries.append(self.url_result(
+                    base_url + seo_url + '/player', 'TVNow', info.get('id')))
+
+        return self.playlist_result(
+            entries, compat_str(season.get('id') or season_id), title)
-- 
cgit v1.1


From cb0c2310fbf232e09ae41013be3400034171d6d2 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 12 Dec 2017 10:33:03 +0100
Subject: [discovery] fix free videos extraction(#14157)(#14954)

---
 youtube_dl/extractor/discovery.py   | 122 +++++++++++++++++-------------------
 youtube_dl/extractor/discoverygo.py |  72 +++++++++++----------
 2 files changed, 94 insertions(+), 100 deletions(-)

diff --git a/youtube_dl/extractor/discovery.py b/youtube_dl/extractor/discovery.py
index 55853f7..f9cec1d 100644
--- a/youtube_dl/extractor/discovery.py
+++ b/youtube_dl/extractor/discovery.py
@@ -1,14 +1,18 @@
 from __future__ import unicode_literals
 
-from .common import InfoExtractor
+import random
+import re
+import string
+
+from .discoverygo import DiscoveryGoBaseIE
 from ..utils import (
-    parse_duration,
-    parse_iso8601,
+    ExtractorError,
+    update_url_query,
 )
-from ..compat import compat_str
+from ..compat import compat_HTTPError
 
 
-class DiscoveryIE(InfoExtractor):
+class DiscoveryIE(DiscoveryGoBaseIE):
     _VALID_URL = r'''(?x)https?://(?:www\.)?(?:
             discovery|
             investigationdiscovery|
@@ -19,79 +23,65 @@ class DiscoveryIE(InfoExtractor):
             sciencechannel|
             tlc|
             velocity
-        )\.com/(?:[^/]+/)*(?P<id>[^./?#]+)'''
+        )\.com(?P<path>/tv-shows/[^/]+/(?:video|full-episode)s/(?P<id>[^./?#]+))'''
     _TESTS = [{
-        'url': 'http://www.discovery.com/tv-shows/mythbusters/videos/mission-impossible-outtakes.htm',
+        'url': 'https://www.discovery.com/tv-shows/cash-cab/videos/dave-foley',
         'info_dict': {
-            'id': '20769',
+            'id': '5a2d9b4d6b66d17a5026e1fd',
             'ext': 'mp4',
-            'title': 'Mission Impossible Outtakes',
-            'description': ('Watch Jamie Hyneman and Adam Savage practice being'
-                            ' each other -- to the point of confusing Jamie\'s dog -- and '
-                            'don\'t miss Adam moon-walking as Jamie ... behind Jamie\'s'
-                            ' back.'),
-            'duration': 156,
-            'timestamp': 1302032462,
-            'upload_date': '20110405',
-            'uploader_id': '103207',
+            'title': 'Dave Foley',
+            'description': 'md5:4b39bcafccf9167ca42810eb5f28b01f',
+            'duration': 608,
         },
         'params': {
             'skip_download': True,  # requires ffmpeg
         }
     }, {
-        'url': 'http://www.discovery.com/tv-shows/mythbusters/videos/mythbusters-the-simpsons',
-        'info_dict': {
-            'id': 'mythbusters-the-simpsons',
-            'title': 'MythBusters: The Simpsons',
-        },
-        'playlist_mincount': 10,
-    }, {
-        'url': 'http://www.animalplanet.com/longfin-eels-maneaters/',
-        'info_dict': {
-            'id': '78326',
-            'ext': 'mp4',
-            'title': 'Longfin Eels: Maneaters?',
-            'description': 'Jeremy Wade tests whether or not New Zealand\'s longfin eels are man-eaters by covering himself in fish guts and getting in the water with them.',
-            'upload_date': '20140725',
-            'timestamp': 1406246400,
-            'duration': 116,
-            'uploader_id': '103207',
-        },
-        'params': {
-            'skip_download': True,  # requires ffmpeg
-        }
+        'url': 'https://www.investigationdiscovery.com/tv-shows/final-vision/full-episodes/final-vision',
+        'only_matching': True,
     }]
+    _GEO_COUNTRIES = ['US']
+    _GEO_BYPASS = False
 
     def _real_extract(self, url):
-        display_id = self._match_id(url)
-        info = self._download_json(url + '?flat=1', display_id)
-
-        video_title = info.get('playlist_title') or info.get('video_title')
+        path, display_id = re.match(self._VALID_URL, url).groups()
+        webpage = self._download_webpage(url, display_id)
 
-        entries = []
+        react_data = self._parse_json(self._search_regex(
+            r'window\.__reactTransmitPacket\s*=\s*({.+?});',
+            webpage, 'react data'), display_id)
+        content_blocks = react_data['layout'][path]['contentBlocks']
+        video = next(cb for cb in content_blocks if cb.get('type') == 'video')['content']['items'][0]
+        video_id = video['id']
 
-        for idx, video_info in enumerate(info['playlist']):
-            subtitles = {}
-            caption_url = video_info.get('captionsUrl')
-            if caption_url:
-                subtitles = {
-                    'en': [{
-                        'url': caption_url,
-                    }]
-                }
+        access_token = self._download_json(
+            'https://www.discovery.com/anonymous', display_id, query={
+                'authLink': update_url_query(
+                    'https://login.discovery.com/v1/oauth2/authorize', {
+                        'client_id': react_data['application']['apiClientId'],
+                        'redirect_uri': 'https://fusion.ddmcdn.com/app/mercury-sdk/180/redirectHandler.html',
+                        'response_type': 'anonymous',
+                        'state': 'nonce,' + ''.join([random.choice(string.ascii_letters) for _ in range(32)]),
+                    })
+            })['access_token']
 
-            entries.append({
-                '_type': 'url_transparent',
-                'url': 'http://players.brightcove.net/103207/default_default/index.html?videoId=ref:%s' % video_info['referenceId'],
-                'id': compat_str(video_info['id']),
-                'title': video_info['title'],
-                'description': video_info.get('description'),
-                'duration': parse_duration(video_info.get('video_length')),
-                'webpage_url': video_info.get('href') or video_info.get('url'),
-                'thumbnail': video_info.get('thumbnailURL'),
-                'alt_title': video_info.get('secondary_title'),
-                'timestamp': parse_iso8601(video_info.get('publishedDate')),
-                'subtitles': subtitles,
-            })
+        try:
+            stream = self._download_json(
+                'https://api.discovery.com/v1/streaming/video/' + video_id,
+                display_id, headers={
+                    'Authorization': 'Bearer ' + access_token,
+                })
+        except ExtractorError as e:
+            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
+                e_description = self._parse_json(
+                    e.cause.read().decode(), display_id)['description']
+                if 'resource not available for country' in e_description:
+                    self.raise_geo_restricted(countries=self._GEO_COUNTRIES)
+                if 'Authorized Networks' in e_description:
+                    raise ExtractorError(
+                        'This video is only available via cable service provider subscription that'
+                        ' is not currently supported. You may want to use --cookies.', expected=True)
+                raise ExtractorError(e_description)
+            raise
 
-        return self.playlist_result(entries, display_id, video_title)
+        return self._extract_video_info(video, stream, display_id)
diff --git a/youtube_dl/extractor/discoverygo.py b/youtube_dl/extractor/discoverygo.py
index 7cd5d42..9937645 100644
--- a/youtube_dl/extractor/discoverygo.py
+++ b/youtube_dl/extractor/discoverygo.py
@@ -27,42 +27,9 @@ class DiscoveryGoBaseIE(InfoExtractor):
             velocitychannel
         )go\.com/%s(?P<id>[^/?#&]+)'''
 
-
-class DiscoveryGoIE(DiscoveryGoBaseIE):
-    _VALID_URL = DiscoveryGoBaseIE._VALID_URL_TEMPLATE % r'(?:[^/]+/)+'
-    _GEO_COUNTRIES = ['US']
-    _TEST = {
-        'url': 'https://www.discoverygo.com/bering-sea-gold/reaper-madness/',
-        'info_dict': {
-            'id': '58c167d86b66d12f2addeb01',
-            'ext': 'mp4',
-            'title': 'Reaper Madness',
-            'description': 'md5:09f2c625c99afb8946ed4fb7865f6e78',
-            'duration': 2519,
-            'series': 'Bering Sea Gold',
-            'season_number': 8,
-            'episode_number': 6,
-            'age_limit': 14,
-        },
-    }
-
-    def _real_extract(self, url):
-        display_id = self._match_id(url)
-
-        webpage = self._download_webpage(url, display_id)
-
-        container = extract_attributes(
-            self._search_regex(
-                r'(<div[^>]+class=["\']video-player-container[^>]+>)',
-                webpage, 'video container'))
-
-        video = self._parse_json(
-            container.get('data-video') or container.get('data-json'),
-            display_id)
-
+    def _extract_video_info(self, video, stream, display_id):
         title = video['name']
 
-        stream = video.get('stream')
         if not stream:
             if video.get('authenticated') is True:
                 raise ExtractorError(
@@ -124,6 +91,43 @@ class DiscoveryGoIE(DiscoveryGoBaseIE):
         }
 
 
+class DiscoveryGoIE(DiscoveryGoBaseIE):
+    _VALID_URL = DiscoveryGoBaseIE._VALID_URL_TEMPLATE % r'(?:[^/]+/)+'
+    _GEO_COUNTRIES = ['US']
+    _TEST = {
+        'url': 'https://www.discoverygo.com/bering-sea-gold/reaper-madness/',
+        'info_dict': {
+            'id': '58c167d86b66d12f2addeb01',
+            'ext': 'mp4',
+            'title': 'Reaper Madness',
+            'description': 'md5:09f2c625c99afb8946ed4fb7865f6e78',
+            'duration': 2519,
+            'series': 'Bering Sea Gold',
+            'season_number': 8,
+            'episode_number': 6,
+            'age_limit': 14,
+        },
+    }
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, display_id)
+
+        container = extract_attributes(
+            self._search_regex(
+                r'(<div[^>]+class=["\']video-player-container[^>]+>)',
+                webpage, 'video container'))
+
+        video = self._parse_json(
+            container.get('data-video') or container.get('data-json'),
+            display_id)
+
+        stream = video.get('stream')
+
+        return self._extract_video_info(video, stream, display_id)
+
+
 class DiscoveryGoPlaylistIE(DiscoveryGoBaseIE):
     _VALID_URL = DiscoveryGoBaseIE._VALID_URL_TEMPLATE % ''
     _TEST = {
-- 
cgit v1.1


From e6b8803d599df4de3c115e4de4ef192a1e10c749 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 12 Dec 2017 11:11:44 +0100
Subject: [toutv] add support special video urls(closes #14179)

---
 youtube_dl/extractor/toutv.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/toutv.py b/youtube_dl/extractor/toutv.py
index 17c0adc..2e7876c 100644
--- a/youtube_dl/extractor/toutv.py
+++ b/youtube_dl/extractor/toutv.py
@@ -16,7 +16,7 @@ from ..utils import (
 class TouTvIE(InfoExtractor):
     _NETRC_MACHINE = 'toutv'
     IE_NAME = 'tou.tv'
-    _VALID_URL = r'https?://ici\.tou\.tv/(?P<id>[a-zA-Z0-9_-]+(?:/S[0-9]+E[0-9]+)?)'
+    _VALID_URL = r'https?://ici\.tou\.tv/(?P<id>[a-zA-Z0-9_-]+(?:/S[0-9]+[EC][0-9]+)?)'
     _access_token = None
     _claims = None
 
@@ -37,6 +37,9 @@ class TouTvIE(InfoExtractor):
     }, {
         'url': 'http://ici.tou.tv/hackers',
         'only_matching': True,
+    }, {
+        'url': 'https://ici.tou.tv/l-age-adulte/S01C501',
+        'only_matching': True,
     }]
 
     def _real_initialize(self):
-- 
cgit v1.1


From 6b2d8c91823c8f4c5dcda42cf174ab5468c44661 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 12 Dec 2017 18:04:05 +0100
Subject: [bbc.co.uk] fix extraction for 320k m3u8 streams broken since
 197224b7a4e37a6581bf1a0da18d0f67ea61a476

---
 youtube_dl/extractor/bbc.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 5525f7c..8b20c03 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -386,7 +386,7 @@ class BBCCoUkIE(InfoExtractor):
                             m3u8_id=format_id, fatal=False))
                         if re.search(self._USP_RE, href):
                             usp_formats = self._extract_m3u8_formats(
-                                re.sub(self._USP_RE, r'/\1\.ism/\1\.m3u8', href),
+                                re.sub(self._USP_RE, r'/\1.ism/\1.m3u8', href),
                                 programme_id, ext='mp4', entry_protocol='m3u8_native',
                                 m3u8_id=format_id, fatal=False)
                             for f in usp_formats:
-- 
cgit v1.1


From 15960255fe3caaae58132116d772d1c1bcf415fa Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 12 Dec 2017 18:16:45 +0100
Subject: [tbs] fix typo

---
 youtube_dl/extractor/tbs.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/tbs.py b/youtube_dl/extractor/tbs.py
index 460bc5d..eab22c3 100644
--- a/youtube_dl/extractor/tbs.py
+++ b/youtube_dl/extractor/tbs.py
@@ -36,8 +36,7 @@ class TBSIE(TurnerBaseIE):
     }]
 
     def _real_extract(self, url):
-        domain, display_id = re.match(self._VALID_URL, url).groups()
-        site = domain[:3]
+        site, display_id = re.match(self._VALID_URL, url).groups()
         webpage = self._download_webpage(url, display_id)
         video_data = self._parse_json(self._search_regex(
             r'<script[^>]+?data-drupal-selector="drupal-settings-json"[^>]*?>({.+?})</script>',
@@ -62,7 +61,7 @@ class TBSIE(TurnerBaseIE):
                     'http://www.%s.com/service/token_spe' % site,
                     m3u8_url, media_id, {
                         'url': url,
-                        'site_name': site.upper(),
+                        'site_name': site[:3].upper(),
                         'auth_required': video_data.get('authRequired') == '1',
                     })
             formats.extend(self._extract_m3u8_formats(
-- 
cgit v1.1


From c8be7d5f7417f5779cda2c02653f38401dd7a6b7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 13 Dec 2017 23:14:30 +0700
Subject: [byutv] Fix extraction (closes #14966, closes #14967)

---
 youtube_dl/extractor/byutv.py      | 71 ++++++++++----------------------------
 youtube_dl/extractor/extractors.py |  5 +--
 2 files changed, 19 insertions(+), 57 deletions(-)

diff --git a/youtube_dl/extractor/byutv.py b/youtube_dl/extractor/byutv.py
index 8ef0896..dd7d2f8 100644
--- a/youtube_dl/extractor/byutv.py
+++ b/youtube_dl/extractor/byutv.py
@@ -3,20 +3,19 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import ExtractorError
 
 
 class BYUtvIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?byutv\.org/watch/(?!event/)(?P<id>[0-9a-f-]+)(?:/(?P<display_id>[^/?#&]+))?'
+    _VALID_URL = r'https?://(?:www\.)?byutv\.org/(?:watch|player)/(?!event/)(?P<id>[0-9a-f-]+)(?:/(?P<display_id>[^/?#&]+))?'
     _TESTS = [{
         'url': 'http://www.byutv.org/watch/6587b9a3-89d2-42a6-a7f7-fd2f81840a7d/studio-c-season-5-episode-5',
         'info_dict': {
-            'id': '6587b9a3-89d2-42a6-a7f7-fd2f81840a7d',
+            'id': 'ZvanRocTpW-G5_yZFeltTAMv6jxOU9KH',
             'display_id': 'studio-c-season-5-episode-5',
             'ext': 'mp4',
             'title': 'Season 5 Episode 5',
-            'description': 'md5:e07269172baff037f8e8bf9956bc9747',
-            'thumbnail': r're:^https?://.*\.jpg$',
+            'description': 'md5:1d31dc18ef4f075b28f6a65937d22c65',
+            'thumbnail': r're:^https?://.*',
             'duration': 1486.486,
         },
         'params': {
@@ -26,6 +25,9 @@ class BYUtvIE(InfoExtractor):
     }, {
         'url': 'http://www.byutv.org/watch/6587b9a3-89d2-42a6-a7f7-fd2f81840a7d',
         'only_matching': True,
+    }, {
+        'url': 'https://www.byutv.org/player/27741493-dc83-40b0-8420-e7ae38a2ae98/byu-football-toledo-vs-byu-93016?listid=4fe0fee5-0d3c-4a29-b725-e4948627f472&listindex=0&q=toledo',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
@@ -33,16 +35,16 @@ class BYUtvIE(InfoExtractor):
         video_id = mobj.group('id')
         display_id = mobj.group('display_id') or video_id
 
-        webpage = self._download_webpage(url, display_id)
-        episode_code = self._search_regex(
-            r'(?s)episode:(.*?\}),\s*\n', webpage, 'episode information')
-
-        ep = self._parse_json(
-            episode_code, display_id, transform_source=lambda s:
-            re.sub(r'(\n\s+)([a-zA-Z]+):\s+\'(.*?)\'', r'\1"\2": "\3"', s))
-
-        if ep['providerType'] != 'Ooyala':
-            raise ExtractorError('Unsupported provider %s' % ep['provider'])
+        ep = self._download_json(
+            'https://api.byutv.org/api3/catalog/getvideosforcontent', video_id,
+            query={
+                'contentid': video_id,
+                'channel': 'byutv_global',
+                'x-byutv-context': 'web$Global',
+            }, headers={
+                'x-byutv-context': 'web$Global',
+                'x-byutv-platformkey': 'xsaaw9c7y5',
+            })['ooyalaVOD']
 
         return {
             '_type': 'url_transparent',
@@ -50,44 +52,7 @@ class BYUtvIE(InfoExtractor):
             'url': 'ooyala:%s' % ep['providerId'],
             'id': video_id,
             'display_id': display_id,
-            'title': ep['title'],
+            'title': ep.get('title'),
             'description': ep.get('description'),
             'thumbnail': ep.get('imageThumbnail'),
         }
-
-
-class BYUtvEventIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?byutv\.org/watch/event/(?P<id>[0-9a-f-]+)'
-    _TEST = {
-        'url': 'http://www.byutv.org/watch/event/29941b9b-8bf6-48d2-aebf-7a87add9e34b',
-        'info_dict': {
-            'id': '29941b9b-8bf6-48d2-aebf-7a87add9e34b',
-            'ext': 'mp4',
-            'title': 'Toledo vs. BYU (9/30/16)',
-        },
-        'params': {
-            'skip_download': True,
-        },
-        'add_ie': ['Ooyala'],
-    }
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-
-        webpage = self._download_webpage(url, video_id)
-
-        ooyala_id = self._search_regex(
-            r'providerId\s*:\s*(["\'])(?P<id>(?:(?!\1).)+)\1',
-            webpage, 'ooyala id', group='id')
-
-        title = self._search_regex(
-            r'class=["\']description["\'][^>]*>\s*<h1>([^<]+)</h1>', webpage,
-            'title').strip()
-
-        return {
-            '_type': 'url_transparent',
-            'ie_key': 'Ooyala',
-            'url': 'ooyala:%s' % ooyala_id,
-            'id': video_id,
-            'title': title,
-        }
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index fb9f365..4072455 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -138,10 +138,7 @@ from .brightcove import (
     BrightcoveNewIE,
 )
 from .buzzfeed import BuzzFeedIE
-from .byutv import (
-    BYUtvIE,
-    BYUtvEventIE,
-)
+from .byutv import BYUtvIE
 from .c56 import C56IE
 from .camdemy import (
     CamdemyIE,
-- 
cgit v1.1


From bec49996c68fe73b4e17e7418290072a4c4b7cdb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 13 Dec 2017 23:49:05 +0700
Subject: [downloader/http] Return actual download result (closes #14971)

---
 youtube_dl/downloader/http.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/youtube_dl/downloader/http.py b/youtube_dl/downloader/http.py
index 8a6638c..3ff26ff 100644
--- a/youtube_dl/downloader/http.py
+++ b/youtube_dl/downloader/http.py
@@ -284,8 +284,7 @@ class HttpFD(FileDownloader):
         while count <= retries:
             try:
                 establish_connection()
-                download()
-                return True
+                return download()
             except RetryDownload as e:
                 count += 1
                 if count <= retries:
-- 
cgit v1.1


From 6bf9c28b0ac5d44381008cebe4741f8f1d78c53d Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 13 Dec 2017 17:51:24 +0100
Subject: [byutv] add support for geo restricted videos

---
 youtube_dl/extractor/byutv.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/byutv.py b/youtube_dl/extractor/byutv.py
index dd7d2f8..4bf4efe 100644
--- a/youtube_dl/extractor/byutv.py
+++ b/youtube_dl/extractor/byutv.py
@@ -39,10 +39,10 @@ class BYUtvIE(InfoExtractor):
             'https://api.byutv.org/api3/catalog/getvideosforcontent', video_id,
             query={
                 'contentid': video_id,
-                'channel': 'byutv_global',
-                'x-byutv-context': 'web$Global',
+                'channel': 'byutv',
+                'x-byutv-context': 'web$US',
             }, headers={
-                'x-byutv-context': 'web$Global',
+                'x-byutv-context': 'web$US',
                 'x-byutv-platformkey': 'xsaaw9c7y5',
             })['ooyalaVOD']
 
-- 
cgit v1.1


From 7974e289a10f87ae52fdc32b5f5cc7e14a0fafda Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 14 Dec 2017 01:00:14 +0700
Subject: [postprocessor/xattr] Clarify NO_SPACE message (#14970)

---
 youtube_dl/postprocessor/xattrpp.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/postprocessor/xattrpp.py b/youtube_dl/postprocessor/xattrpp.py
index fbdfa02..b0aed9c 100644
--- a/youtube_dl/postprocessor/xattrpp.py
+++ b/youtube_dl/postprocessor/xattrpp.py
@@ -42,6 +42,7 @@ class XAttrMetadataPP(PostProcessor):
                 'user.dublincore.format': 'format',
             }
 
+            num_written = 0
             for xattrname, infoname in xattr_mapping.items():
 
                 value = info.get(infoname)
@@ -52,6 +53,7 @@ class XAttrMetadataPP(PostProcessor):
 
                     byte_value = value.encode('utf-8')
                     write_xattr(filename, xattrname, byte_value)
+                    num_written += 1
 
             return [], info
 
@@ -62,8 +64,8 @@ class XAttrMetadataPP(PostProcessor):
         except XAttrMetadataError as e:
             if e.reason == 'NO_SPACE':
                 self._downloader.report_warning(
-                    'There\'s no disk space left or disk quota exceeded. ' +
-                    'Extended attributes are not written.')
+                    'There\'s no disk space left, disk quota exceeded or filesystem xattr limit exceeded. ' +
+                    (('Some ' if num_written else '') + 'extended attributes are not written.').capitalize())
             elif e.reason == 'VALUE_TOO_LONG':
                 self._downloader.report_warning(
                     'Unable to write extended attributes due to too long values.')
-- 
cgit v1.1


From 3fae11ac006567bfbfe9368bc11f56da090e267a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 14 Dec 2017 04:49:07 +0700
Subject: [itv] Improve extraction, extract more subtitles and duration (closes
 #14944)

---
 youtube_dl/extractor/itv.py | 125 +++++++++++++++++++++++++++-----------------
 1 file changed, 77 insertions(+), 48 deletions(-)

diff --git a/youtube_dl/extractor/itv.py b/youtube_dl/extractor/itv.py
index 413a219..18a7d7f 100644
--- a/youtube_dl/extractor/itv.py
+++ b/youtube_dl/extractor/itv.py
@@ -26,7 +26,7 @@ from ..utils import (
 class ITVIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?itv\.com/hub/[^/]+/(?P<id>[0-9a-zA-Z]+)'
     _GEO_COUNTRIES = ['GB']
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.itv.com/hub/mr-bean-animated-series/2a2936a0053',
         'info_dict': {
             'id': '2a2936a0053',
@@ -37,7 +37,11 @@ class ITVIE(InfoExtractor):
             # rtmp download
             'skip_download': True,
         },
-    }
+    }, {
+        # unavailable via data-playlist-url
+        'url': 'https://www.itv.com/hub/through-the-keyhole/2a2271a0033',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
@@ -101,6 +105,18 @@ class ITVIE(InfoExtractor):
             'Content-Type': 'text/xml; charset=utf-8',
             'SOAPAction': 'http://tempuri.org/PlaylistService/GetPlaylist',
         })
+
+        info = self._search_json_ld(webpage, video_id, default={})
+        formats = []
+        subtitles = {}
+
+        def extract_subtitle(sub_url):
+            ext = determine_ext(sub_url, 'ttml')
+            subtitles.setdefault('en', []).append({
+                'url': sub_url,
+                'ext': 'ttml' if ext == 'xml' else ext,
+            })
+
         resp_env = self._download_xml(
             params['data-playlist-url'], video_id,
             headers=headers, data=etree.tostring(req_env))
@@ -111,37 +127,55 @@ class ITVIE(InfoExtractor):
             if fault_code == 'InvalidGeoRegion':
                 self.raise_geo_restricted(
                     msg=fault_string, countries=self._GEO_COUNTRIES)
-            raise ExtractorError('%s said: %s' % (self.IE_NAME, fault_string))
-        title = xpath_text(playlist, 'EpisodeTitle', fatal=True)
-        video_element = xpath_element(playlist, 'VideoEntries/Video', fatal=True)
-        media_files = xpath_element(video_element, 'MediaFiles', fatal=True)
-        rtmp_url = media_files.attrib['base']
+            elif fault_code != 'InvalidEntity':
+                raise ExtractorError(
+                    '%s said: %s' % (self.IE_NAME, fault_string), expected=True)
+            info.update({
+                'title': self._og_search_title(webpage),
+                'episode_title': params.get('data-video-episode'),
+                'series': params.get('data-video-title'),
+            })
+        else:
+            title = xpath_text(playlist, 'EpisodeTitle', default=None)
+            info.update({
+                'title': title,
+                'episode_title': title,
+                'episode_number': int_or_none(xpath_text(playlist, 'EpisodeNumber')),
+                'series': xpath_text(playlist, 'ProgrammeTitle'),
+                'duration': parse_duration(xpath_text(playlist, 'Duration')),
+            })
+            video_element = xpath_element(playlist, 'VideoEntries/Video', fatal=True)
+            media_files = xpath_element(video_element, 'MediaFiles', fatal=True)
+            rtmp_url = media_files.attrib['base']
 
-        formats = []
-        for media_file in media_files.findall('MediaFile'):
-            play_path = xpath_text(media_file, 'URL')
-            if not play_path:
-                continue
-            tbr = int_or_none(media_file.get('bitrate'), 1000)
-            f = {
-                'format_id': 'rtmp' + ('-%d' % tbr if tbr else ''),
-                'play_path': play_path,
-                # Providing this swfVfy allows to avoid truncated downloads
-                'player_url': 'http://www.itv.com/mercury/Mercury_VideoPlayer.swf',
-                'page_url': url,
-                'tbr': tbr,
-                'ext': 'flv',
-            }
-            app = self._search_regex(
-                'rtmpe?://[^/]+/(.+)$', rtmp_url, 'app', default=None)
-            if app:
-                f.update({
-                    'url': rtmp_url.split('?', 1)[0],
-                    'app': app,
-                })
-            else:
-                f['url'] = rtmp_url
-            formats.append(f)
+            for media_file in media_files.findall('MediaFile'):
+                play_path = xpath_text(media_file, 'URL')
+                if not play_path:
+                    continue
+                tbr = int_or_none(media_file.get('bitrate'), 1000)
+                f = {
+                    'format_id': 'rtmp' + ('-%d' % tbr if tbr else ''),
+                    'play_path': play_path,
+                    # Providing this swfVfy allows to avoid truncated downloads
+                    'player_url': 'http://www.itv.com/mercury/Mercury_VideoPlayer.swf',
+                    'page_url': url,
+                    'tbr': tbr,
+                    'ext': 'flv',
+                }
+                app = self._search_regex(
+                    'rtmpe?://[^/]+/(.+)$', rtmp_url, 'app', default=None)
+                if app:
+                    f.update({
+                        'url': rtmp_url.split('?', 1)[0],
+                        'app': app,
+                    })
+                else:
+                    f['url'] = rtmp_url
+                formats.append(f)
+
+            for caption_url in video_element.findall('ClosedCaptioningURIs/URL'):
+                if caption_url.text:
+                    extract_subtitle(caption_url.text)
 
         ios_playlist_url = params.get('data-video-playlist') or params.get('data-video-id')
         hmac = params.get('data-video-hmac')
@@ -198,27 +232,22 @@ class ITVIE(InfoExtractor):
                         formats.append({
                             'url': href,
                         })
-        self._sort_formats(formats)
+                subs = video_data.get('Subtitles')
+                if isinstance(subs, list):
+                    for sub in subs:
+                        if not isinstance(sub, dict):
+                            continue
+                        href = sub.get('Href')
+                        if isinstance(href, compat_str):
+                            extract_subtitle(href)
+                if not info.get('duration'):
+                    info['duration'] = parse_duration(video_data.get('Duration'))
 
-        subtitles = {}
-        for caption_url in video_element.findall('ClosedCaptioningURIs/URL'):
-            if not caption_url.text:
-                continue
-            ext = determine_ext(caption_url.text, 'ttml')
-            subtitles.setdefault('en', []).append({
-                'url': caption_url.text,
-                'ext': 'ttml' if ext == 'xml' else ext,
-            })
+        self._sort_formats(formats)
 
-        info = self._search_json_ld(webpage, video_id, default={})
         info.update({
             'id': video_id,
-            'title': title,
             'formats': formats,
             'subtitles': subtitles,
-            'episode_title': title,
-            'episode_number': int_or_none(xpath_text(playlist, 'EpisodeNumber')),
-            'series': xpath_text(playlist, 'ProgrammeTitle'),
-            'duartion': parse_duration(xpath_text(playlist, 'Duration')),
         })
         return info
-- 
cgit v1.1


From c6a5a811a197e1c6520764df1f5f93473692dd18 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 14 Dec 2017 05:15:51 +0700
Subject: [ChangeLog] Actualize

---
 ChangeLog | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index a1fdcab..254264c 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,23 @@
+version <unreleased>
+
+Core
+* [postprocessor/xattr] Clarify NO_SPACE message (#14970)
+* [downloader/http] Return actual download result from real_download (#14971)
+
+Extractors
++ [itv] Extract more subtitles and duration
+* [itv] Improve extraction (#14944)
++ [byutv] Add support for geo restricted videos
+* [byutv] Fix extraction (#14966, #14967)
++ [bbccouk] Fix extraction for 320k HLS streams
++ [toutv] Add support for special video URLs (#14179)
+* [discovery] Fix free videos extraction (#14157, #14954)
+* [tvnow] Fix extraction (#7831)
++ [nickelodeon:br] Add support for nickelodeon brazil websites (#14893)
+* [nick] Improve extraction (#14876)
+* [tbs] Fix extraction (#13658)
+
+
 version 2017.12.10
 
 Core
-- 
cgit v1.1


From 8ff2b16435cdf53c3e684532afe699dfdf34403c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 14 Dec 2017 05:19:21 +0700
Subject: release 2017.12.14

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 docs/supportedsites.md    | 8 ++++----
 youtube_dl/version.py     | 2 +-
 4 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 736869b..f37d8aa 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.12.10*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.12.10**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.12.14*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.12.14**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through the [README](https://github.com/rg3/youtube-dl/blob/master/README.md), **most notably** the [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ Add the `-v` flag to **your command line** you run youtube-dl with (`youtube-dl
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.12.10
+[debug] youtube-dl version 2017.12.14
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index 254264c..03d2def 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.12.14
 
 Core
 * [postprocessor/xattr] Clarify NO_SPACE message (#14970)
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index d5e3a8a..ebddd5b 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -122,7 +122,6 @@
  - **bt:vestlendingen**: Bergens Tidende - Vestlendingen
  - **BuzzFeed**
  - **BYUtv**
- - **BYUtvEvent**
  - **Camdemy**
  - **CamdemyFolder**
  - **CamWithHer**
@@ -538,6 +537,7 @@
  - **nhl.com:videocenter:category**: NHL videocenter category
  - **nick.com**
  - **nick.de**
+ - **nickelodeon:br**
  - **nickelodeonru**
  - **nicknight**
  - **niconico**: ニコニコ動画
@@ -556,8 +556,6 @@
  - **nowness**
  - **nowness:playlist**
  - **nowness:series**
- - **NowTV** (Currently broken)
- - **NowTVList**
  - **nowvideo**: NowVideo
  - **Noz**
  - **npo**: npo.nl, ntr.nl, omroepwnl.nl, zapp.nl and npo3.nl
@@ -794,7 +792,7 @@
  - **tagesschau:player**
  - **Tass**
  - **TastyTrade**
- - **TBS** (Currently broken)
+ - **TBS**
  - **TDSLifeway**
  - **teachertube**: teachertube.com videos
  - **teachertube:user:collection**: teachertube.com user and collection videos
@@ -865,6 +863,8 @@
  - **tvland.com**
  - **TVN24**
  - **TVNoe**
+ - **TVNow**
+ - **TVNowList**
  - **tvp**: Telewizja Polska
  - **tvp:embed**: Telewizja Polska
  - **tvp:series**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index d0c4383..2b5a634 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.12.10'
+__version__ = '2017.12.14'
-- 
cgit v1.1


From 1c4804ef9b03e92ef6b1d6720f9a93bb54d98324 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 14 Dec 2017 23:05:07 +0100
Subject: [voot] fix format extraction(closes #14758)

---
 youtube_dl/extractor/voot.py | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/voot.py b/youtube_dl/extractor/voot.py
index 5de3deb..5fd1c3d 100644
--- a/youtube_dl/extractor/voot.py
+++ b/youtube_dl/extractor/voot.py
@@ -2,7 +2,6 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from .kaltura import KalturaIE
 from ..utils import (
     ExtractorError,
     int_or_none,
@@ -17,11 +16,10 @@ class VootIE(InfoExtractor):
     _TESTS = [{
         'url': 'https://www.voot.com/shows/ishq-ka-rang-safed/1/360558/is-this-the-end-of-kamini-/441353',
         'info_dict': {
-            'id': '0_8ledb18o',
+            'id': '441353',
             'ext': 'mp4',
             'title': 'Ishq Ka Rang Safed - Season 01 - Episode 340',
             'description': 'md5:06291fbbbc4dcbe21235c40c262507c1',
-            'uploader_id': 'batchUser',
             'timestamp': 1472162937,
             'upload_date': '20160825',
             'duration': 1146,
@@ -61,8 +59,10 @@ class VootIE(InfoExtractor):
 
         media = media_info['assets']
 
-        entry_id = media['EntryId']
         title = media['MediaName']
+        formats = self._extract_m3u8_formats(
+            'https://cdnapisec.kaltura.com/p/1982551/playManifest/pt/https/f/applehttp/t/web/e/' + media['EntryId'],
+            video_id, 'mp4', m3u8_id='hls', fatal=False)
 
         description, series, season_number, episode, episode_number = [None] * 5
 
@@ -82,9 +82,7 @@ class VootIE(InfoExtractor):
                 episode_number = int_or_none(value)
 
         return {
-            '_type': 'url_transparent',
-            'url': 'kaltura:1982551:%s' % entry_id,
-            'ie_key': KalturaIE.ie_key(),
+            'id': video_id,
             'title': title,
             'description': description,
             'series': series,
@@ -95,4 +93,5 @@ class VootIE(InfoExtractor):
             'duration': int_or_none(media.get('Duration')),
             'view_count': int_or_none(media.get('ViewCounter')),
             'like_count': int_or_none(media.get('like_counter')),
+            'formats': formats,
         }
-- 
cgit v1.1


From 23f511f5c74d85222e69259996666b7ef97b9421 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Fri, 15 Dec 2017 09:05:59 +0100
Subject: [voot] sort formats

---
 youtube_dl/extractor/voot.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/voot.py b/youtube_dl/extractor/voot.py
index 5fd1c3d..4267544 100644
--- a/youtube_dl/extractor/voot.py
+++ b/youtube_dl/extractor/voot.py
@@ -62,7 +62,8 @@ class VootIE(InfoExtractor):
         title = media['MediaName']
         formats = self._extract_m3u8_formats(
             'https://cdnapisec.kaltura.com/p/1982551/playManifest/pt/https/f/applehttp/t/web/e/' + media['EntryId'],
-            video_id, 'mp4', m3u8_id='hls', fatal=False)
+            video_id, 'mp4', m3u8_id='hls')
+        self._sort_formats(formats)
 
         description, series, season_number, episode, episode_number = [None] * 5
 
-- 
cgit v1.1


From d05ba4b89e195ed18e8d51561649712f945769b9 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Fri, 15 Dec 2017 09:27:56 +0100
Subject: [disney] skip Apple FairPlay formats(#14982)

---
 youtube_dl/extractor/disney.py | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/disney.py b/youtube_dl/extractor/disney.py
index 968c4c7..0eee82f 100644
--- a/youtube_dl/extractor/disney.py
+++ b/youtube_dl/extractor/disney.py
@@ -10,6 +10,7 @@ from ..utils import (
     compat_str,
     determine_ext,
     ExtractorError,
+    update_url_query,
 )
 
 
@@ -108,9 +109,16 @@ class DisneyIE(InfoExtractor):
                 continue
             tbr = int_or_none(flavor.get('bitrate'))
             if tbr == 99999:
-                formats.extend(self._extract_m3u8_formats(
+                # wrong ks(Kaltura Signature) causes 404 Error
+                flavor_url = update_url_query(flavor_url, {'ks': ''})
+                m3u8_formats = self._extract_m3u8_formats(
                     flavor_url, video_id, 'mp4',
-                    m3u8_id=flavor_format, fatal=False))
+                    m3u8_id=flavor_format, fatal=False)
+                for f in m3u8_formats:
+                    # Apple FairPlay
+                    if '/fpshls/' in f['url']:
+                        continue
+                    formats.append(f)
                 continue
             format_id = []
             if flavor_format:
-- 
cgit v1.1


From 498a8a4ca56e66adb84f8f1488c0239d048c7adc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 15 Dec 2017 22:53:56 +0700
Subject: [vk] Make view count optional (closes #14979)

---
 youtube_dl/extractor/vk.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index 0d83765..d4838b3 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -414,7 +414,7 @@ class VKIE(VKBaseIE):
 
         view_count = str_to_int(self._search_regex(
             r'class=["\']mv_views_count[^>]+>\s*([\d,.]+)',
-            info_page, 'view count', fatal=False))
+            info_page, 'view count', default=None))
 
         formats = []
         for format_id, format_url in data.items():
-- 
cgit v1.1


From c402e7f3a03aceeb80dd032831ceafb6d0aaa935 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sat, 16 Dec 2017 12:55:20 +0100
Subject: [discoverygo] correct ttml subtitle extension

---
 youtube_dl/extractor/discoverygo.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/discoverygo.py b/youtube_dl/extractor/discoverygo.py
index 9937645..3368c4c 100644
--- a/youtube_dl/extractor/discoverygo.py
+++ b/youtube_dl/extractor/discoverygo.py
@@ -5,6 +5,7 @@ import re
 from .common import InfoExtractor
 from ..compat import compat_str
 from ..utils import (
+    determine_ext,
     extract_attributes,
     ExtractorError,
     int_or_none,
@@ -73,7 +74,11 @@ class DiscoveryGoBaseIE(InfoExtractor):
                         not subtitle_url.startswith('http')):
                     continue
                 lang = caption.get('fileLang', 'en')
-                subtitles.setdefault(lang, []).append({'url': subtitle_url})
+                ext = determine_ext(subtitle_url)
+                subtitles.setdefault(lang, []).append({
+                    'url': subtitle_url,
+                    'ext': 'ttml' if ext == 'xml' else ext,
+                })
 
         return {
             'id': video_id,
-- 
cgit v1.1


From b555ae9bf146ce9e2bf81327746847f0bc4d63e9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 16 Dec 2017 21:56:16 +0700
Subject: [utils] Add another date format pattern (#14999)

---
 test/test_utils.py  | 1 +
 youtube_dl/utils.py | 2 ++
 2 files changed, 3 insertions(+)

diff --git a/test/test_utils.py b/test/test_utils.py
index cc13f79..0857c0f 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -343,6 +343,7 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(unified_timestamp('Feb 7, 2016 at 6:35 pm'), 1454870100)
         self.assertEqual(unified_timestamp('2017-03-30T17:52:41Q'), 1490896361)
         self.assertEqual(unified_timestamp('Sep 11, 2013 | 5:49 AM'), 1378878540)
+        self.assertEqual(unified_timestamp('December 15, 2017 at 7:49 am'), 1513324140)
 
     def test_determine_ext(self):
         self.assertEqual(determine_ext('http://example.com/foo/bar.mp4/?download'), 'mp4')
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index eccbc0b..2843a3d 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -159,6 +159,8 @@ DATE_FORMATS = (
     '%Y-%m-%dT%H:%M',
     '%b %d %Y at %H:%M',
     '%b %d %Y at %H:%M:%S',
+    '%B %d %Y at %H:%M',
+    '%B %d %Y at %H:%M:%S',
 )
 
 DATE_FORMATS_DAY_FIRST = list(DATE_FORMATS)
-- 
cgit v1.1


From 06dbcd7be41f71346e75f78f3ad77d9eca0219ca Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 16 Dec 2017 21:57:30 +0700
Subject: [cbslocal] Fix timestamp extraction (closes #14999, closes #15000)

---
 youtube_dl/extractor/cbslocal.py | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/cbslocal.py b/youtube_dl/extractor/cbslocal.py
index 7d78e3a..90852a9 100644
--- a/youtube_dl/extractor/cbslocal.py
+++ b/youtube_dl/extractor/cbslocal.py
@@ -91,12 +91,10 @@ class CBSLocalIE(AnvatoIE):
 
         info_dict = self._extract_anvato_videos(webpage, display_id)
 
-        time_str = self._html_search_regex(
-            r'class="entry-date">([^<]+)<', webpage, 'released date', default=None)
-        if time_str:
-            timestamp = unified_timestamp(time_str)
-        else:
-            timestamp = parse_iso8601(self._html_search_meta('uploadDate', webpage))
+        timestamp = unified_timestamp(self._html_search_regex(
+            r'class="(?:entry|post)-date"[^>]*>([^<]+)', webpage,
+            'released date', default=None)) or parse_iso8601(
+            self._html_search_meta('uploadDate', webpage))
 
         info_dict.update({
             'display_id': display_id,
-- 
cgit v1.1


From 3dfa9ec21362f8781d0a2d2eef1c9474b525af79 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sun, 17 Dec 2017 09:14:52 +0100
Subject: [crunchyroll] Future-proof XML element checks(closes #15013)

---
 youtube_dl/extractor/crunchyroll.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index b53f2d7..b92f254 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -392,7 +392,7 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
                 'Downloading subtitles for ' + sub_name, data={
                     'subtitle_script_id': sub_id,
                 })
-            if not sub_doc:
+            if sub_doc is None:
                 continue
             sid = sub_doc.get('id')
             iv = xpath_text(sub_doc, 'iv', 'subtitle iv')
@@ -479,9 +479,9 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
                     'video_quality': stream_quality,
                     'current_page': url,
                 })
-            if streamdata:
+            if streamdata is not None:
                 stream_info = streamdata.find('./{default}preload/stream_info')
-                if stream_info:
+                if stream_info is not None:
                     stream_infos.append(stream_info)
             stream_info = self._call_rpc_api(
                 'VideoEncode_GetStreamInfo', video_id,
@@ -490,7 +490,7 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
                     'video_format': stream_format,
                     'video_encode_quality': stream_quality,
                 })
-            if stream_info:
+            if stream_info is not None:
                 stream_infos.append(stream_info)
             for stream_info in stream_infos:
                 video_encode_id = xpath_text(stream_info, './video_encode_id')
-- 
cgit v1.1


From 25475dfab356e625ee480e1631601b86403afb5a Mon Sep 17 00:00:00 2001
From: Hongjie Dong <hongjieuf@gmail.com>
Date: Mon, 4 Dec 2017 00:26:09 -0800
Subject: [mailru] Add support for embed URLs

---
 youtube_dl/extractor/mailru.py | 38 +++++++++++++++++++++++---------------
 1 file changed, 23 insertions(+), 15 deletions(-)

diff --git a/youtube_dl/extractor/mailru.py b/youtube_dl/extractor/mailru.py
index f7cc3c8..f1865fd 100644
--- a/youtube_dl/extractor/mailru.py
+++ b/youtube_dl/extractor/mailru.py
@@ -13,8 +13,7 @@ from ..utils import (
 class MailRuIE(InfoExtractor):
     IE_NAME = 'mailru'
     IE_DESC = 'Видео@Mail.Ru'
-    _VALID_URL = r'https?://(?:(?:www|m)\.)?my\.mail\.ru/(?:video/.*#video=/?(?P<idv1>(?:[^/]+/){3}\d+)|(?:(?P<idv2prefix>(?:[^/]+/){2})video/(?P<idv2suffix>[^/]+/\d+))\.html)'
-
+    _VALID_URL = r'https?://(?:(?:www|m)\.)?my\.mail\.ru/(?:video/.*#video=/?(?P<idv1>(?:[^/]+/){3}\d+)|(?:(?P<idv2prefix>(?:[^/]+/){2})video/(?P<idv2suffix>[^/]+/\d+))\.html|video/embed/(?P<meta_id>\d+))'
     _TESTS = [
         {
             'url': 'http://my.mail.ru/video/top#video=/mail/sonypicturesrus/75/76',
@@ -65,25 +64,34 @@ class MailRuIE(InfoExtractor):
         {
             'url': 'http://m.my.mail.ru/mail/3sktvtr/video/_myvideo/138.html',
             'only_matching': True,
+        },
+        {
+            'url': 'https://my.mail.ru/video/embed/7949340477499637815',
+            'only_matching': True,
         }
     ]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('idv1')
-
-        if not video_id:
-            video_id = mobj.group('idv2prefix') + mobj.group('idv2suffix')
-
-        webpage = self._download_webpage(url, video_id)
-
+        meta_id = mobj.group('meta_id')
+        page_config = None
         video_data = None
-
-        page_config = self._parse_json(self._search_regex(
-            r'(?s)<script[^>]+class="sp-video__page-config"[^>]*>(.+?)</script>',
-            webpage, 'page config', default='{}'), video_id, fatal=False)
-        if page_config:
-            meta_url = page_config.get('metaUrl') or page_config.get('video', {}).get('metaUrl')
+        video_id = None
+        if not meta_id:
+            video_id = mobj.group('idv1')
+            if not video_id:
+                video_id = mobj.group('idv2prefix') + mobj.group('idv2suffix')
+
+            webpage = self._download_webpage(url, video_id)
+
+            page_config = self._parse_json(self._search_regex(
+                r'(?s)<script[^>]+class="sp-video__page-config"[^>]*>(.+?)</script>',
+                webpage, 'page config', default='{}'), video_id, fatal=False)
+        if page_config or meta_id:
+            if page_config:
+                meta_url = page_config.get('metaUrl') or page_config.get('video', {}).get('metaUrl')
+            elif meta_id:
+                meta_url = 'https://my.mail.ru/+/video/meta/' + meta_id
             if meta_url:
                 video_data = self._download_json(
                     meta_url, video_id, 'Downloading video meta JSON', fatal=False)
-- 
cgit v1.1


From 549bb416f5a9d15c03749a98abd582d0e40418ac Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 17 Dec 2017 18:36:19 +0700
Subject: [mailru] Fix issues and improve (closes #14904)

---
 youtube_dl/extractor/mailru.py | 43 +++++++++++++++++++++++++++---------------
 1 file changed, 28 insertions(+), 15 deletions(-)

diff --git a/youtube_dl/extractor/mailru.py b/youtube_dl/extractor/mailru.py
index f1865fd..6b7c5e3 100644
--- a/youtube_dl/extractor/mailru.py
+++ b/youtube_dl/extractor/mailru.py
@@ -13,7 +13,15 @@ from ..utils import (
 class MailRuIE(InfoExtractor):
     IE_NAME = 'mailru'
     IE_DESC = 'Видео@Mail.Ru'
-    _VALID_URL = r'https?://(?:(?:www|m)\.)?my\.mail\.ru/(?:video/.*#video=/?(?P<idv1>(?:[^/]+/){3}\d+)|(?:(?P<idv2prefix>(?:[^/]+/){2})video/(?P<idv2suffix>[^/]+/\d+))\.html|video/embed/(?P<meta_id>\d+))'
+    _VALID_URL = r'''(?x)
+                    https?://
+                        (?:(?:www|m)\.)?my\.mail\.ru/
+                        (?:
+                            video/.*\#video=/?(?P<idv1>(?:[^/]+/){3}\d+)|
+                            (?:(?P<idv2prefix>(?:[^/]+/){2})video/(?P<idv2suffix>[^/]+/\d+))\.html|
+                            (?:video/embed|\+/video/meta)/(?P<metaid>\d+)
+                        )
+                    '''
     _TESTS = [
         {
             'url': 'http://my.mail.ru/video/top#video=/mail/sonypicturesrus/75/76',
@@ -22,7 +30,7 @@ class MailRuIE(InfoExtractor):
                 'id': '46301138_76',
                 'ext': 'mp4',
                 'title': 'Новый Человек-Паук. Высокое напряжение. Восстание Электро',
-                'timestamp': 1393232740,
+                'timestamp': 1393235077,
                 'upload_date': '20140224',
                 'uploader': 'sonypicturesrus',
                 'uploader_id': 'sonypicturesrus@mail.ru',
@@ -39,7 +47,7 @@ class MailRuIE(InfoExtractor):
                 'title': 'Samsung Galaxy S5 Hammer Smash Fail Battery Explosion',
                 'timestamp': 1397039888,
                 'upload_date': '20140409',
-                'uploader': 'hitech@corp.mail.ru',
+                'uploader': 'hitech',
                 'uploader_id': 'hitech@corp.mail.ru',
                 'duration': 245,
             },
@@ -68,33 +76,38 @@ class MailRuIE(InfoExtractor):
         {
             'url': 'https://my.mail.ru/video/embed/7949340477499637815',
             'only_matching': True,
+        },
+        {
+            'url': 'http://my.mail.ru/+/video/meta/7949340477499637815',
+            'only_matching': True,
         }
     ]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
-        meta_id = mobj.group('meta_id')
-        page_config = None
-        video_data = None
+        meta_id = mobj.group('metaid')
+
         video_id = None
-        if not meta_id:
+        if meta_id:
+            meta_url = 'https://my.mail.ru/+/video/meta/%s' % meta_id
+        else:
             video_id = mobj.group('idv1')
             if not video_id:
                 video_id = mobj.group('idv2prefix') + mobj.group('idv2suffix')
-
             webpage = self._download_webpage(url, video_id)
-
             page_config = self._parse_json(self._search_regex(
                 r'(?s)<script[^>]+class="sp-video__page-config"[^>]*>(.+?)</script>',
                 webpage, 'page config', default='{}'), video_id, fatal=False)
-        if page_config or meta_id:
             if page_config:
                 meta_url = page_config.get('metaUrl') or page_config.get('video', {}).get('metaUrl')
-            elif meta_id:
-                meta_url = 'https://my.mail.ru/+/video/meta/' + meta_id
-            if meta_url:
-                video_data = self._download_json(
-                    meta_url, video_id, 'Downloading video meta JSON', fatal=False)
+            else:
+                meta_url = None
+
+        video_data = None
+        if meta_url:
+            video_data = self._download_json(
+                meta_url, video_id or meta_id, 'Downloading video meta JSON',
+                fatal=not video_id)
 
         # Fallback old approach
         if not video_data:
-- 
cgit v1.1


From 7e810109877bb59d213574b115e46d9621db9a18 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sun, 17 Dec 2017 19:13:25 +0100
Subject: [cspan] add support for audio only pages and catch page errors(closes
 #14995)

---
 youtube_dl/extractor/cspan.py | 23 ++++++++++++++++++-----
 1 file changed, 18 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/cspan.py b/youtube_dl/extractor/cspan.py
index 171820e..67d6df4 100644
--- a/youtube_dl/extractor/cspan.py
+++ b/youtube_dl/extractor/cspan.py
@@ -4,13 +4,14 @@ import re
 
 from .common import InfoExtractor
 from ..utils import (
-    int_or_none,
-    unescapeHTML,
-    find_xpath_attr,
-    smuggle_url,
     determine_ext,
     ExtractorError,
     extract_attributes,
+    find_xpath_attr,
+    get_element_by_class,
+    int_or_none,
+    smuggle_url,
+    unescapeHTML,
 )
 from .senateisvp import SenateISVPIE
 from .ustream import UstreamIE
@@ -68,6 +69,10 @@ class CSpanIE(InfoExtractor):
             'uploader': 'HouseCommittee',
             'uploader_id': '12987475',
         },
+    }, {
+        # Audio Only
+        'url': 'https://www.c-span.org/video/?437336-1/judiciary-antitrust-competition-policy-consumer-rights',
+        'only_matching': True,
     }]
     BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/%s/%s_%s/index.html?videoId=%s'
 
@@ -111,7 +116,15 @@ class CSpanIE(InfoExtractor):
                     title = self._og_search_title(webpage)
                     surl = smuggle_url(senate_isvp_url, {'force_title': title})
                     return self.url_result(surl, 'SenateISVP', video_id, title)
+                video_id = self._search_regex(
+                    r'jwsetup\.clipprog\s*=\s*(\d+);',
+                    webpage, 'jwsetup program id', default=None)
+                if video_id:
+                    video_type = 'program'
         if video_type is None or video_id is None:
+            error_message = get_element_by_class('VLplayer-error-message', webpage)
+            if error_message:
+                raise ExtractorError(error_message)
             raise ExtractorError('unable to find video id and type')
 
         def get_text_attr(d, attr):
@@ -138,7 +151,7 @@ class CSpanIE(InfoExtractor):
         entries = []
         for partnum, f in enumerate(files):
             formats = []
-            for quality in f['qualities']:
+            for quality in f.get('qualities', []):
                 formats.append({
                     'format_id': '%s-%sp' % (get_text_attr(quality, 'bitrate'), get_text_attr(quality, 'height')),
                     'url': unescapeHTML(get_text_attr(quality, 'file')),
-- 
cgit v1.1


From 99081da90c9b8ce4ee7fe9452787507fed4251a0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 18 Dec 2017 03:31:53 +0700
Subject: [downloader/fragment] Encode filename of fragment being removed
 (closes #15020)

---
 youtube_dl/downloader/fragment.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/downloader/fragment.py b/youtube_dl/downloader/fragment.py
index 7bb61a5..ea5e3a4 100644
--- a/youtube_dl/downloader/fragment.py
+++ b/youtube_dl/downloader/fragment.py
@@ -112,7 +112,7 @@ class FragmentFD(FileDownloader):
             if self.__do_ytdl_file(ctx):
                 self._write_ytdl_file(ctx)
             if not self.params.get('keep_fragments', False):
-                os.remove(ctx['fragment_filename_sanitized'])
+                os.remove(encodeFilename(ctx['fragment_filename_sanitized']))
             del ctx['fragment_filename_sanitized']
 
     def _prepare_frag_download(self, ctx):
-- 
cgit v1.1


From 4a109f81bc39ed09931fd6f9d21d20f7abdc9742 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 19 Dec 2017 00:38:39 +0700
Subject: [afreecatv] Improve format extraction (closes #15019)

---
 youtube_dl/extractor/afreecatv.py | 17 +++++++++++++----
 1 file changed, 13 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/afreecatv.py b/youtube_dl/extractor/afreecatv.py
index e6513c7..513dd81 100644
--- a/youtube_dl/extractor/afreecatv.py
+++ b/youtube_dl/extractor/afreecatv.py
@@ -228,10 +228,19 @@ class AfreecaTVIE(InfoExtractor):
                     r'^(\d{8})_', key, 'upload date', default=None)
                 file_duration = int_or_none(file_element.get('duration'))
                 format_id = key if key else '%s_%s' % (video_id, file_num)
-                formats = self._extract_m3u8_formats(
-                    file_url, video_id, 'mp4', entry_protocol='m3u8_native',
-                    m3u8_id='hls',
-                    note='Downloading part %d m3u8 information' % file_num)
+                if determine_ext(file_url) == 'm3u8':
+                    formats = self._extract_m3u8_formats(
+                        file_url, video_id, 'mp4', entry_protocol='m3u8_native',
+                        m3u8_id='hls',
+                        note='Downloading part %d m3u8 information' % file_num)
+                else:
+                    formats = [{
+                        'url': file_url,
+                        'format_id': 'http',
+                    }]
+                if not formats:
+                    continue
+                self._sort_formats(formats)
                 file_info = common_entry.copy()
                 file_info.update({
                     'id': format_id,
-- 
cgit v1.1


From c10c93238e6db0df8746fc185ca316b9d8ccece5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 19 Dec 2017 03:51:03 +0700
Subject: [extractor/common] Introduce uploader, uploader_id and uploader_url
 meta fields for playlists (#11427, #15018)

---
 youtube_dl/extractor/common.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 80a9c98..e5ef5e4 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -301,8 +301,9 @@ class InfoExtractor(object):
     There must be a key "entries", which is a list, an iterable, or a PagedList
     object, each element of which is a valid dictionary by this specification.
 
-    Additionally, playlists can have "title", "description" and "id" attributes
-    with the same semantics as videos (see above).
+    Additionally, playlists can have "id", "title", "description", "uploader",
+    "uploader_id", "uploader_url" attributes with the same semantics as videos
+    (see above).
 
 
     _type "multi_video" indicates that there are multiple videos that
-- 
cgit v1.1


From 07aeced68e5aa24f3e2562aa7cb9ddd2f11b59ca Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 19 Dec 2017 03:51:28 +0700
Subject: [youtube] Extract uploader, uploader_id and uploader_url for
 playlists (#11427, #15018)

---
 youtube_dl/extractor/youtube.py | 22 +++++++++++++++++++++-
 1 file changed, 21 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 9943ddd..0919bef 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -2270,6 +2270,19 @@ class YoutubePlaylistIE(YoutubePlaylistBaseInfoExtractor):
             r'(?s)<h1 class="pl-header-title[^"]*"[^>]*>\s*(.*?)\s*</h1>',
             page, 'title', default=None)
 
+        _UPLOADER_BASE = r'class=["\']pl-header-details[^>]+>\s*<li>\s*<a[^>]+\bhref='
+        uploader = self._search_regex(
+            r'%s["\']/(?:user|channel)/[^>]+>([^<]+)' % _UPLOADER_BASE,
+            page, 'uploader', default=None)
+        mobj = re.search(
+            r'%s(["\'])(?P<path>/(?:user|channel)/(?P<uploader_id>.+?))\1' % _UPLOADER_BASE,
+            page)
+        if mobj:
+            uploader_id = mobj.group('uploader_id')
+            uploader_url = compat_urlparse.urljoin(url, mobj.group('path'))
+        else:
+            uploader_id = uploader_url = None
+
         has_videos = True
 
         if not playlist_title:
@@ -2280,8 +2293,15 @@ class YoutubePlaylistIE(YoutubePlaylistBaseInfoExtractor):
             except StopIteration:
                 has_videos = False
 
-        return has_videos, self.playlist_result(
+        playlist = self.playlist_result(
             self._entries(page, playlist_id), playlist_id, playlist_title)
+        playlist.update({
+            'uploader': uploader,
+            'uploader_id': uploader_id,
+            'uploader_url': uploader_url,
+        })
+
+        return has_videos, playlist
 
     def _check_download_just_video(self, url, playlist_id):
         # Check if it's a video-specific URL
-- 
cgit v1.1


From 3961c6cb9d3a1c30fe31db774b0809095952f1bd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 19 Dec 2017 03:53:44 +0700
Subject: [YoutubeDL] Add support for playlist_uploader and
 playlist_uploader_id in output template (closes #11427, #15018)

---
 README.md               | 2 ++
 youtube_dl/YoutubeDL.py | 2 ++
 2 files changed, 4 insertions(+)

diff --git a/README.md b/README.md
index cd30d14..47b0640 100644
--- a/README.md
+++ b/README.md
@@ -539,6 +539,8 @@ The basic usage is not to set any template arguments when downloading a single f
  - `playlist_index` (numeric): Index of the video in the playlist padded with leading zeros according to the total length of the playlist
  - `playlist_id` (string): Playlist identifier
  - `playlist_title` (string): Playlist title
+ - `playlist_uploader` (string): Full name of the playlist uploader
+ - `playlist_uploader_id` (string): Nickname or id of the playlist uploader
 
 Available for the video that belongs to some logical chapter or section:
 
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 68721e9..ace80f1 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -975,6 +975,8 @@ class YoutubeDL(object):
                     'playlist': playlist,
                     'playlist_id': ie_result.get('id'),
                     'playlist_title': ie_result.get('title'),
+                    'playlist_uploader': ie_result.get('uploader'),
+                    'playlist_uploader_id': ie_result.get('uploader_id'),
                     'playlist_index': i + playliststart,
                     'extractor': ie_result['extractor'],
                     'webpage_url': ie_result['webpage_url'],
-- 
cgit v1.1


From 78466fcab519d1b92fd9846bc8073885308a7e22 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 19 Dec 2017 02:00:13 +0100
Subject: [shahid] add support for show pages(closes #7401)

---
 youtube_dl/extractor/aws.py             |  78 +++++++++++++++
 youtube_dl/extractor/extractors.py      |   5 +-
 youtube_dl/extractor/scrippsnetworks.py | 103 ++++----------------
 youtube_dl/extractor/shahid.py          | 164 +++++++++++++++++++++++---------
 4 files changed, 219 insertions(+), 131 deletions(-)
 create mode 100644 youtube_dl/extractor/aws.py

diff --git a/youtube_dl/extractor/aws.py b/youtube_dl/extractor/aws.py
new file mode 100644
index 0000000..670abce
--- /dev/null
+++ b/youtube_dl/extractor/aws.py
@@ -0,0 +1,78 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import datetime
+import hashlib
+import hmac
+
+from .common import InfoExtractor
+from ..compat import compat_urllib_parse_urlencode
+
+
+class AWSIE(InfoExtractor):
+    _AWS_ALGORITHM = 'AWS4-HMAC-SHA256'
+    _AWS_REGION = 'us-east-1'
+
+    def _aws_execute_api(self, aws_dict, video_id, query=None):
+        query = query or {}
+        amz_date = datetime.datetime.utcnow().strftime('%Y%m%dT%H%M%SZ')
+        date = amz_date[:8]
+        headers = {
+            'Accept': 'application/json',
+            'Host': self._AWS_PROXY_HOST,
+            'X-Amz-Date': amz_date,
+        }
+        session_token = aws_dict.get('session_token')
+        if session_token:
+            headers['X-Amz-Security-Token'] = session_token
+        headers['X-Api-Key'] = self._AWS_API_KEY
+
+        def aws_hash(s):
+            return hashlib.sha256(s.encode('utf-8')).hexdigest()
+
+        # Task 1: http://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html
+        canonical_querystring = compat_urllib_parse_urlencode(query)
+        canonical_headers = ''
+        for header_name, header_value in headers.items():
+            canonical_headers += '%s:%s\n' % (header_name.lower(), header_value)
+        signed_headers = ';'.join([header.lower() for header in headers.keys()])
+        canonical_request = '\n'.join([
+            'GET',
+            aws_dict['uri'],
+            canonical_querystring,
+            canonical_headers,
+            signed_headers,
+            aws_hash('')
+        ])
+
+        # Task 2: http://docs.aws.amazon.com/general/latest/gr/sigv4-create-string-to-sign.html
+        credential_scope_list = [date, self._AWS_REGION, 'execute-api', 'aws4_request']
+        credential_scope = '/'.join(credential_scope_list)
+        string_to_sign = '\n'.join([self._AWS_ALGORITHM, amz_date, credential_scope, aws_hash(canonical_request)])
+
+        # Task 3: http://docs.aws.amazon.com/general/latest/gr/sigv4-calculate-signature.html
+        def aws_hmac(key, msg):
+            return hmac.new(key, msg.encode('utf-8'), hashlib.sha256)
+
+        def aws_hmac_digest(key, msg):
+            return aws_hmac(key, msg).digest()
+
+        def aws_hmac_hexdigest(key, msg):
+            return aws_hmac(key, msg).hexdigest()
+
+        k_signing = ('AWS4' + aws_dict['secret_key']).encode('utf-8')
+        for value in credential_scope_list:
+            k_signing = aws_hmac_digest(k_signing, value)
+
+        signature = aws_hmac_hexdigest(k_signing, string_to_sign)
+
+        # Task 4: http://docs.aws.amazon.com/general/latest/gr/sigv4-add-signature-to-request.html
+        headers['Authorization'] = ', '.join([
+            '%s Credential=%s/%s' % (self._AWS_ALGORITHM, aws_dict['access_key'], credential_scope),
+            'SignedHeaders=%s' % signed_headers,
+            'Signature=%s' % signature,
+        ])
+
+        return self._download_json(
+            'https://%s%s%s' % (self._AWS_PROXY_HOST, aws_dict['uri'], '?' + canonical_querystring if canonical_querystring else ''),
+            video_id, headers=headers)
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 4072455..5130748 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -927,7 +927,10 @@ from .sendtonews import SendtoNewsIE
 from .servingsys import ServingSysIE
 from .servus import ServusIE
 from .sexu import SexuIE
-from .shahid import ShahidIE
+from .shahid import (
+    ShahidIE,
+    ShahidShowIE,
+)
 from .shared import (
     SharedIE,
     VivoIE,
diff --git a/youtube_dl/extractor/scrippsnetworks.py b/youtube_dl/extractor/scrippsnetworks.py
index b446a02..4023aee 100644
--- a/youtube_dl/extractor/scrippsnetworks.py
+++ b/youtube_dl/extractor/scrippsnetworks.py
@@ -1,13 +1,11 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import datetime
 import json
 import hashlib
-import hmac
 import re
 
-from .common import InfoExtractor
+from .aws import AWSIE
 from .anvato import AnvatoIE
 from ..utils import (
     smuggle_url,
@@ -16,7 +14,7 @@ from ..utils import (
 )
 
 
-class ScrippsNetworksWatchIE(InfoExtractor):
+class ScrippsNetworksWatchIE(AWSIE):
     IE_NAME = 'scrippsnetworks:watch'
     _VALID_URL = r'''(?x)
                     https?://
@@ -64,44 +62,27 @@ class ScrippsNetworksWatchIE(InfoExtractor):
         'travelchannel': 'trav',
         'geniuskitchen': 'genius',
     }
-    _SNI_HOST = 'web.api.video.snidigital.com'
 
-    _AWS_REGION = 'us-east-1'
-    _AWS_IDENTITY_ID_JSON = json.dumps({
-        'IdentityId': '%s:7655847c-0ae7-4d9b-80d6-56c062927eb3' % _AWS_REGION
-    })
-    _AWS_USER_AGENT = 'aws-sdk-js/2.80.0 callback'
     _AWS_API_KEY = 'E7wSQmq0qK6xPrF13WmzKiHo4BQ7tip4pQcSXVl1'
-    _AWS_SERVICE = 'execute-api'
-    _AWS_REQUEST = 'aws4_request'
-    _AWS_SIGNED_HEADERS = ';'.join([
-        'host', 'x-amz-date', 'x-amz-security-token', 'x-api-key'])
-    _AWS_CANONICAL_REQUEST_TEMPLATE = '''GET
-%(uri)s
-
-host:%(host)s
-x-amz-date:%(date)s
-x-amz-security-token:%(token)s
-x-api-key:%(key)s
+    _AWS_PROXY_HOST = 'web.api.video.snidigital.com'
 
-%(signed_headers)s
-%(payload_hash)s'''
+    _AWS_USER_AGENT = 'aws-sdk-js/2.80.0 callback'
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         site_id, video_id = mobj.group('site', 'id')
 
-        def aws_hash(s):
-            return hashlib.sha256(s.encode('utf-8')).hexdigest()
-
+        aws_identity_id_json = json.dumps({
+            'IdentityId': '%s:7655847c-0ae7-4d9b-80d6-56c062927eb3' % self._AWS_REGION
+        }).encode('utf-8')
         token = self._download_json(
-            'https://cognito-identity.us-east-1.amazonaws.com/', video_id,
-            data=self._AWS_IDENTITY_ID_JSON.encode('utf-8'),
+            'https://cognito-identity.%s.amazonaws.com/' % self._AWS_REGION, video_id,
+            data=aws_identity_id_json,
             headers={
                 'Accept': '*/*',
                 'Content-Type': 'application/x-amz-json-1.1',
                 'Referer': url,
-                'X-Amz-Content-Sha256': aws_hash(self._AWS_IDENTITY_ID_JSON),
+                'X-Amz-Content-Sha256': hashlib.sha256(aws_identity_id_json).hexdigest(),
                 'X-Amz-Target': 'AWSCognitoIdentityService.GetOpenIdToken',
                 'X-Amz-User-Agent': self._AWS_USER_AGENT,
             })['Token']
@@ -124,64 +105,12 @@ x-api-key:%(key)s
                 sts, './/{https://sts.amazonaws.com/doc/2011-06-15/}%s' % key,
                 fatal=True)
 
-        access_key_id = get('AccessKeyId')
-        secret_access_key = get('SecretAccessKey')
-        session_token = get('SessionToken')
-
-        # Task 1: http://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html
-        uri = '/1/web/brands/%s/episodes/scrid/%s' % (self._SNI_TABLE[site_id], video_id)
-        datetime_now = datetime.datetime.utcnow().strftime('%Y%m%dT%H%M%SZ')
-        date = datetime_now[:8]
-        canonical_string = self._AWS_CANONICAL_REQUEST_TEMPLATE % {
-            'uri': uri,
-            'host': self._SNI_HOST,
-            'date': datetime_now,
-            'token': session_token,
-            'key': self._AWS_API_KEY,
-            'signed_headers': self._AWS_SIGNED_HEADERS,
-            'payload_hash': aws_hash(''),
-        }
-
-        # Task 2: http://docs.aws.amazon.com/general/latest/gr/sigv4-create-string-to-sign.html
-        credential_string = '/'.join([date, self._AWS_REGION, self._AWS_SERVICE, self._AWS_REQUEST])
-        string_to_sign = '\n'.join([
-            'AWS4-HMAC-SHA256', datetime_now, credential_string,
-            aws_hash(canonical_string)])
-
-        # Task 3: http://docs.aws.amazon.com/general/latest/gr/sigv4-calculate-signature.html
-        def aws_hmac(key, msg):
-            return hmac.new(key, msg.encode('utf-8'), hashlib.sha256)
-
-        def aws_hmac_digest(key, msg):
-            return aws_hmac(key, msg).digest()
-
-        def aws_hmac_hexdigest(key, msg):
-            return aws_hmac(key, msg).hexdigest()
-
-        k_secret = 'AWS4' + secret_access_key
-        k_date = aws_hmac_digest(k_secret.encode('utf-8'), date)
-        k_region = aws_hmac_digest(k_date, self._AWS_REGION)
-        k_service = aws_hmac_digest(k_region, self._AWS_SERVICE)
-        k_signing = aws_hmac_digest(k_service, self._AWS_REQUEST)
-
-        signature = aws_hmac_hexdigest(k_signing, string_to_sign)
-
-        auth_header = ', '.join([
-            'AWS4-HMAC-SHA256 Credential=%s' % '/'.join(
-                [access_key_id, date, self._AWS_REGION, self._AWS_SERVICE, self._AWS_REQUEST]),
-            'SignedHeaders=%s' % self._AWS_SIGNED_HEADERS,
-            'Signature=%s' % signature,
-        ])
-
-        mcp_id = self._download_json(
-            'https://%s%s' % (self._SNI_HOST, uri), video_id, headers={
-                'Accept': '*/*',
-                'Referer': url,
-                'Authorization': auth_header,
-                'X-Amz-Date': datetime_now,
-                'X-Amz-Security-Token': session_token,
-                'X-Api-Key': self._AWS_API_KEY,
-            })['results'][0]['mcpId']
+        mcp_id = self._aws_execute_api({
+            'uri': '/1/web/brands/%s/episodes/scrid/%s' % (self._SNI_TABLE[site_id], video_id),
+            'access_key': get('AccessKeyId'),
+            'secret_key': get('SecretAccessKey'),
+            'session_token': get('SessionToken'),
+        }, video_id)['results'][0]['mcpId']
 
         return self.url_result(
             smuggle_url(
diff --git a/youtube_dl/extractor/shahid.py b/youtube_dl/extractor/shahid.py
index 374f7fa..5c2a620 100644
--- a/youtube_dl/extractor/shahid.py
+++ b/youtube_dl/extractor/shahid.py
@@ -1,22 +1,53 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import re
 import json
+import math
+import re
 
-from .common import InfoExtractor
+from .aws import AWSIE
 from ..compat import compat_HTTPError
 from ..utils import (
+    clean_html,
     ExtractorError,
+    InAdvancePagedList,
     int_or_none,
     parse_iso8601,
     str_or_none,
     urlencode_postdata,
-    clean_html,
 )
 
 
-class ShahidIE(InfoExtractor):
+class ShahidBaseIE(AWSIE):
+    _AWS_PROXY_HOST = 'api2.shahid.net'
+    _AWS_API_KEY = '2RRtuMHx95aNI1Kvtn2rChEuwsCogUd4samGPjLh'
+
+    def _handle_error(self, e):
+        fail_data = self._parse_json(
+            e.cause.read().decode('utf-8'), None, fatal=False)
+        if fail_data:
+            faults = fail_data.get('faults', [])
+            faults_message = ', '.join([clean_html(fault['userMessage']) for fault in faults if fault.get('userMessage')])
+            if faults_message:
+                raise ExtractorError(faults_message, expected=True)
+
+    def _call_api(self, path, video_id, request=None):
+        query = {}
+        if request:
+            query['request'] = json.dumps(request)
+        try:
+            return self._aws_execute_api({
+                'uri': '/proxy/v2/' + path,
+                'access_key': 'AKIAI6X4TYCIXM2B7MUQ',
+                'secret_key': '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',
+            }, video_id, query)
+        except ExtractorError as e:
+            if isinstance(e.cause, compat_HTTPError):
+                self._handle_error(e)
+            raise
+
+
+class ShahidIE(ShahidBaseIE):
     _NETRC_MACHINE = 'shahid'
     _VALID_URL = r'https?://shahid\.mbc\.net/ar/(?:serie|show|movie)s/[^/]+/(?P<type>episode|clip|movie)-(?P<id>\d+)'
     _TESTS = [{
@@ -41,34 +72,25 @@ class ShahidIE(InfoExtractor):
         'only_matching': True
     }]
 
-    def _api2_request(self, *args, **kwargs):
-        try:
-            return self._download_json(*args, **kwargs)
-        except ExtractorError as e:
-            if isinstance(e.cause, compat_HTTPError):
-                fail_data = self._parse_json(
-                    e.cause.read().decode('utf-8'), None, fatal=False)
-                if fail_data:
-                    faults = fail_data.get('faults', [])
-                    faults_message = ', '.join([clean_html(fault['userMessage']) for fault in faults if fault.get('userMessage')])
-                    if faults_message:
-                        raise ExtractorError(faults_message, expected=True)
-            raise
-
     def _real_initialize(self):
         email, password = self._get_login_info()
         if email is None:
             return
 
-        user_data = self._api2_request(
-            'https://shahid.mbc.net/wd/service/users/login',
-            None, 'Logging in', data=json.dumps({
-                'email': email,
-                'password': password,
-                'basic': 'false',
-            }).encode('utf-8'), headers={
-                'Content-Type': 'application/json; charset=UTF-8',
-            })['user']
+        try:
+            user_data = self._download_json(
+                'https://shahid.mbc.net/wd/service/users/login',
+                None, 'Logging in', data=json.dumps({
+                    'email': email,
+                    'password': password,
+                    'basic': 'false',
+                }).encode('utf-8'), headers={
+                    'Content-Type': 'application/json; charset=UTF-8',
+                })['user']
+        except ExtractorError as e:
+            if isinstance(e.cause, compat_HTTPError):
+                self._handle_error(e)
+            raise
 
         self._download_webpage(
             'https://shahid.mbc.net/populateContext',
@@ -81,25 +103,13 @@ class ShahidIE(InfoExtractor):
                 'sessionId': user_data['sessionId'],
             }))
 
-    def _get_api_data(self, response):
-        data = response.get('data', {})
-
-        error = data.get('error')
-        if error:
-            raise ExtractorError(
-                '%s returned error: %s' % (self.IE_NAME, '\n'.join(error.values())),
-                expected=True)
-
-        return data
-
     def _real_extract(self, url):
         page_type, video_id = re.match(self._VALID_URL, url).groups()
         if page_type == 'clip':
             page_type = 'episode'
 
-        playout = self._api2_request(
-            'https://api2.shahid.net/proxy/v2/playout/url/' + video_id,
-            video_id, 'Downloading player JSON')['playout']
+        playout = self._call_api(
+            'playout/url/' + video_id, video_id)['playout']
 
         if playout.get('drm'):
             raise ExtractorError('This video is DRM protected.', expected=True)
@@ -107,13 +117,27 @@ class ShahidIE(InfoExtractor):
         formats = self._extract_m3u8_formats(playout['url'], video_id, 'mp4')
         self._sort_formats(formats)
 
-        video = self._get_api_data(self._download_json(
+        # video = self._call_api(
+        #     'product/id', video_id, {
+        #         'id': video_id,
+        #         'productType': 'ASSET',
+        #         'productSubType': page_type.upper()
+        #     })['productModel']
+
+        response = self._download_json(
             'http://api.shahid.net/api/v1_1/%s/%s' % (page_type, video_id),
             video_id, 'Downloading video JSON', query={
                 'apiKey': 'sh@hid0nlin3',
                 'hash': 'b2wMCTHpSmyxGqQjJFOycRmLSex+BpTK/ooxy6vHaqs=',
-            }))[page_type]
+            })
+        data = response.get('data', {})
+        error = data.get('error')
+        if error:
+            raise ExtractorError(
+                '%s returned error: %s' % (self.IE_NAME, '\n'.join(error.values())),
+                expected=True)
 
+        video = data[page_type]
         title = video['title']
         categories = [
             category['name']
@@ -135,3 +159,57 @@ class ShahidIE(InfoExtractor):
             'episode_id': video_id,
             'formats': formats,
         }
+
+
+class ShahidShowIE(ShahidBaseIE):
+    _VALID_URL = r'https?://shahid\.mbc\.net/ar/(?:show|serie)s/[^/]+/(?:show|series)-(?P<id>\d+)'
+    _TESTS = [{
+        'url': 'https://shahid.mbc.net/ar/shows/%D8%B1%D8%A7%D9%85%D8%B2-%D9%82%D8%B1%D8%B4-%D8%A7%D9%84%D8%A8%D8%AD%D8%B1/show-79187',
+        'info_dict': {
+            'id': '79187',
+            'title': 'رامز قرش البحر',
+            'description': 'md5:c88fa7e0f02b0abd39d417aee0d046ff',
+        },
+        'playlist_mincount': 32,
+    }, {
+        'url': 'https://shahid.mbc.net/ar/series/How-to-live-Longer-(The-Big-Think)/series-291861',
+        'only_matching': True
+    }]
+    _PAGE_SIZE = 30
+
+    def _real_extract(self, url):
+        show_id = self._match_id(url)
+
+        product = self._call_api(
+            'playableAsset', show_id, {'showId': show_id})['productModel']
+        playlist = product['playlist']
+        playlist_id = playlist['id']
+        show = product.get('show', {})
+
+        def page_func(page_num):
+            playlist = self._call_api(
+                'product/playlist', show_id, {
+                    'playListId': playlist_id,
+                    'pageNumber': page_num,
+                    'pageSize': 30,
+                    'sorts': [{
+                        'order': 'DESC',
+                        'type': 'SORTDATE'
+                    }],
+                })
+            for product in playlist.get('productList', {}).get('products', []):
+                product_url = product.get('productUrl', []).get('url')
+                if not product_url:
+                    continue
+                yield self.url_result(
+                    product_url, 'Shahid',
+                    str_or_none(product.get('id')),
+                    product.get('title'))
+
+        entries = InAdvancePagedList(
+            page_func,
+            math.ceil(playlist['count'] / self._PAGE_SIZE),
+            self._PAGE_SIZE)
+
+        return self.playlist_result(
+            entries, show_id, show.get('title'), show.get('description'))
-- 
cgit v1.1


From 17c3aced5d0d2cf7df41e9978500260756ee8ad9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 19 Dec 2017 22:53:04 +0700
Subject: [animeondemand] Relax login error regex

---
 youtube_dl/extractor/animeondemand.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/animeondemand.py b/youtube_dl/extractor/animeondemand.py
index 34c2b36..be032d5 100644
--- a/youtube_dl/extractor/animeondemand.py
+++ b/youtube_dl/extractor/animeondemand.py
@@ -85,8 +85,8 @@ class AnimeOnDemandIE(InfoExtractor):
 
         if all(p not in response for p in ('>Logout<', 'href="/users/sign_out"')):
             error = self._search_regex(
-                r'<p class="alert alert-danger">(.+?)</p>',
-                response, 'error', default=None)
+                r'<p[^>]+\bclass=(["\'])(?:(?!\1).)*\balert\s(?:(?!\1).)*\1[^>]*>(?P<error>.+?)</p>',
+                response, 'error', default=None, group='error')
             if error:
                 raise ExtractorError('Unable to login: %s' % error, expected=True)
             raise ExtractorError('Unable to log in')
-- 
cgit v1.1


From d2d766bc6d6f976c28fad8b69a1de060b55f5b17 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 20 Dec 2017 23:17:36 +0700
Subject: [animeondemand] Fix typo

---
 youtube_dl/extractor/animeondemand.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/animeondemand.py b/youtube_dl/extractor/animeondemand.py
index be032d5..e4fa72f 100644
--- a/youtube_dl/extractor/animeondemand.py
+++ b/youtube_dl/extractor/animeondemand.py
@@ -85,7 +85,7 @@ class AnimeOnDemandIE(InfoExtractor):
 
         if all(p not in response for p in ('>Logout<', 'href="/users/sign_out"')):
             error = self._search_regex(
-                r'<p[^>]+\bclass=(["\'])(?:(?!\1).)*\balert\s(?:(?!\1).)*\1[^>]*>(?P<error>.+?)</p>',
+                r'<p[^>]+\bclass=(["\'])(?:(?!\1).)*\balert\b(?:(?!\1).)*\1[^>]*>(?P<error>.+?)</p>',
                 response, 'error', default=None, group='error')
             if error:
                 raise ExtractorError('Unable to login: %s' % error, expected=True)
-- 
cgit v1.1


From 963d237d26c7e6da7b6f514c1d240a7046501b05 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 22 Dec 2017 23:38:16 +0700
Subject: Add LICENSE, AUTHORS and ChangeLog to PyPI package (closes #15054)

---
 MANIFEST.in | 3 +++
 setup.py    | 1 +
 2 files changed, 4 insertions(+)

diff --git a/MANIFEST.in b/MANIFEST.in
index 5743f60..af7518e 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -1,4 +1,7 @@
 include README.md
+include LICENSE
+include AUTHORS
+include ChangeLog
 include test/*.py
 include test/*.json
 include youtube-dl.bash-completion
diff --git a/setup.py b/setup.py
index 67d6633..7dbb580 100644
--- a/setup.py
+++ b/setup.py
@@ -109,6 +109,7 @@ setup(
     author_email='ytdl@yt-dl.org',
     maintainer='Sergey M.',
     maintainer_email='dstftw@gmail.com',
+    license='Unlicense',
     packages=[
         'youtube_dl',
         'youtube_dl.extractor', 'youtube_dl.downloader',
-- 
cgit v1.1


From 3e191da6d9d1cbe62d8f638ed68a93a46348b38b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 22 Dec 2017 23:46:08 +0700
Subject: [Makefile] Add AUTHORS to youtube-dl.tar.gz

---
 Makefile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Makefile b/Makefile
index 1c760be..fe24781 100644
--- a/Makefile
+++ b/Makefile
@@ -110,7 +110,7 @@ _EXTRACTOR_FILES = $(shell find youtube_dl/extractor -iname '*.py' -and -not -in
 youtube_dl/extractor/lazy_extractors.py: devscripts/make_lazy_extractors.py devscripts/lazy_load_template.py $(_EXTRACTOR_FILES)
 	$(PYTHON) devscripts/make_lazy_extractors.py $@
 
-youtube-dl.tar.gz: youtube-dl README.md README.txt youtube-dl.1 youtube-dl.bash-completion youtube-dl.zsh youtube-dl.fish ChangeLog
+youtube-dl.tar.gz: youtube-dl README.md README.txt youtube-dl.1 youtube-dl.bash-completion youtube-dl.zsh youtube-dl.fish ChangeLog AUTHORS
 	@tar -czf youtube-dl.tar.gz --transform "s|^|youtube-dl/|" --owner 0 --group 0 \
 		--exclude '*.DS_Store' \
 		--exclude '*.kate-swp' \
@@ -122,7 +122,7 @@ youtube-dl.tar.gz: youtube-dl README.md README.txt youtube-dl.1 youtube-dl.bash-
 		--exclude 'docs/_build' \
 		-- \
 		bin devscripts test youtube_dl docs \
-		ChangeLog LICENSE README.md README.txt \
+		ChangeLog AUTHORS LICENSE README.md README.txt \
 		Makefile MANIFEST.in youtube-dl.1 youtube-dl.bash-completion \
 		youtube-dl.zsh youtube-dl.fish setup.py setup.cfg \
 		youtube-dl
-- 
cgit v1.1


From 9e3682d555d431514d9583170ae8be1b6fc12839 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 22 Dec 2017 23:53:27 +0700
Subject: [MANIFEST.in] Include all test data in PyPI package

---
 MANIFEST.in | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/MANIFEST.in b/MANIFEST.in
index af7518e..4e43e99 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -2,9 +2,8 @@ include README.md
 include LICENSE
 include AUTHORS
 include ChangeLog
-include test/*.py
-include test/*.json
 include youtube-dl.bash-completion
 include youtube-dl.fish
 include youtube-dl.1
 recursive-include docs Makefile conf.py *.rst
+recursive-include test *
-- 
cgit v1.1


From 4b7dd1705a7c16c1426ed7ed39e51e275124b4f3 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sat, 23 Dec 2017 13:21:33 +0100
Subject: [7plus] Add new extractor(closes #15043)

---
 youtube_dl/extractor/brightcove.py | 116 +++++++++++++++++++------------------
 youtube_dl/extractor/extractors.py |   1 +
 youtube_dl/extractor/sevenplus.py  |  67 +++++++++++++++++++++
 3 files changed, 128 insertions(+), 56 deletions(-)
 create mode 100644 youtube_dl/extractor/sevenplus.py

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index 0ed59bc..f045050 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -464,7 +464,7 @@ class BrightcoveNewIE(AdobePassIE):
             'timestamp': 1441391203,
             'upload_date': '20150904',
             'uploader_id': '929656772001',
-            'formats': 'mincount:22',
+            'formats': 'mincount:20',
         },
     }, {
         # with rtmp streams
@@ -478,7 +478,7 @@ class BrightcoveNewIE(AdobePassIE):
             'timestamp': 1433556729,
             'upload_date': '20150606',
             'uploader_id': '4036320279001',
-            'formats': 'mincount:41',
+            'formats': 'mincount:39',
         },
         'params': {
             # m3u8 download
@@ -564,59 +564,7 @@ class BrightcoveNewIE(AdobePassIE):
 
         return entries
 
-    def _real_extract(self, url):
-        url, smuggled_data = unsmuggle_url(url, {})
-        self._initialize_geo_bypass(smuggled_data.get('geo_countries'))
-
-        account_id, player_id, embed, video_id = re.match(self._VALID_URL, url).groups()
-
-        webpage = self._download_webpage(
-            'http://players.brightcove.net/%s/%s_%s/index.min.js'
-            % (account_id, player_id, embed), video_id)
-
-        policy_key = None
-
-        catalog = self._search_regex(
-            r'catalog\(({.+?})\);', webpage, 'catalog', default=None)
-        if catalog:
-            catalog = self._parse_json(
-                js_to_json(catalog), video_id, fatal=False)
-            if catalog:
-                policy_key = catalog.get('policyKey')
-
-        if not policy_key:
-            policy_key = self._search_regex(
-                r'policyKey\s*:\s*(["\'])(?P<pk>.+?)\1',
-                webpage, 'policy key', group='pk')
-
-        api_url = 'https://edge.api.brightcove.com/playback/v1/accounts/%s/videos/%s' % (account_id, video_id)
-        try:
-            json_data = self._download_json(api_url, video_id, headers={
-                'Accept': 'application/json;pk=%s' % policy_key
-            })
-        except ExtractorError as e:
-            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
-                json_data = self._parse_json(e.cause.read().decode(), video_id)[0]
-                message = json_data.get('message') or json_data['error_code']
-                if json_data.get('error_subcode') == 'CLIENT_GEO':
-                    self.raise_geo_restricted(msg=message)
-                raise ExtractorError(message, expected=True)
-            raise
-
-        errors = json_data.get('errors')
-        if errors and errors[0].get('error_subcode') == 'TVE_AUTH':
-            custom_fields = json_data['custom_fields']
-            tve_token = self._extract_mvpd_auth(
-                smuggled_data['source_url'], video_id,
-                custom_fields['bcadobepassrequestorid'],
-                custom_fields['bcadobepassresourceid'])
-            json_data = self._download_json(
-                api_url, video_id, headers={
-                    'Accept': 'application/json;pk=%s' % policy_key
-                }, query={
-                    'tveToken': tve_token,
-                })
-
+    def _parse_brightcove_metadata(self, json_data, video_id):
         title = json_data['name'].strip()
 
         formats = []
@@ -682,6 +630,7 @@ class BrightcoveNewIE(AdobePassIE):
                     })
                 formats.append(f)
 
+        errors = json_data.get('errors')
         if not formats and errors:
             error = errors[0]
             raise ExtractorError(
@@ -708,9 +657,64 @@ class BrightcoveNewIE(AdobePassIE):
             'thumbnail': json_data.get('thumbnail') or json_data.get('poster'),
             'duration': duration,
             'timestamp': parse_iso8601(json_data.get('published_at')),
-            'uploader_id': account_id,
+            'uploader_id': json_data.get('account_id'),
             'formats': formats,
             'subtitles': subtitles,
             'tags': json_data.get('tags', []),
             'is_live': is_live,
         }
+
+    def _real_extract(self, url):
+        url, smuggled_data = unsmuggle_url(url, {})
+        self._initialize_geo_bypass(smuggled_data.get('geo_countries'))
+
+        account_id, player_id, embed, video_id = re.match(self._VALID_URL, url).groups()
+
+        webpage = self._download_webpage(
+            'http://players.brightcove.net/%s/%s_%s/index.min.js'
+            % (account_id, player_id, embed), video_id)
+
+        policy_key = None
+
+        catalog = self._search_regex(
+            r'catalog\(({.+?})\);', webpage, 'catalog', default=None)
+        if catalog:
+            catalog = self._parse_json(
+                js_to_json(catalog), video_id, fatal=False)
+            if catalog:
+                policy_key = catalog.get('policyKey')
+
+        if not policy_key:
+            policy_key = self._search_regex(
+                r'policyKey\s*:\s*(["\'])(?P<pk>.+?)\1',
+                webpage, 'policy key', group='pk')
+
+        api_url = 'https://edge.api.brightcove.com/playback/v1/accounts/%s/videos/%s' % (account_id, video_id)
+        try:
+            json_data = self._download_json(api_url, video_id, headers={
+                'Accept': 'application/json;pk=%s' % policy_key
+            })
+        except ExtractorError as e:
+            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
+                json_data = self._parse_json(e.cause.read().decode(), video_id)[0]
+                message = json_data.get('message') or json_data['error_code']
+                if json_data.get('error_subcode') == 'CLIENT_GEO':
+                    self.raise_geo_restricted(msg=message)
+                raise ExtractorError(message, expected=True)
+            raise
+
+        errors = json_data.get('errors')
+        if errors and errors[0].get('error_subcode') == 'TVE_AUTH':
+            custom_fields = json_data['custom_fields']
+            tve_token = self._extract_mvpd_auth(
+                smuggled_data['source_url'], video_id,
+                custom_fields['bcadobepassrequestorid'],
+                custom_fields['bcadobepassresourceid'])
+            json_data = self._download_json(
+                api_url, video_id, headers={
+                    'Accept': 'application/json;pk=%s' % policy_key
+                }, query={
+                    'tveToken': tve_token,
+                })
+
+        return self._parse_brightcove_metadata(json_data, video_id)
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 5130748..9ba1be2 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -926,6 +926,7 @@ from .senateisvp import SenateISVPIE
 from .sendtonews import SendtoNewsIE
 from .servingsys import ServingSysIE
 from .servus import ServusIE
+from .sevenplus import SevenPlusIE
 from .sexu import SexuIE
 from .shahid import (
     ShahidIE,
diff --git a/youtube_dl/extractor/sevenplus.py b/youtube_dl/extractor/sevenplus.py
new file mode 100644
index 0000000..9792f82
--- /dev/null
+++ b/youtube_dl/extractor/sevenplus.py
@@ -0,0 +1,67 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .brightcove import BrightcoveNewIE
+from ..utils import update_url_query
+
+
+class SevenPlusIE(BrightcoveNewIE):
+    IE_NAME = '7plus'
+    _VALID_URL = r'https?://(?:www\.)?7plus\.com\.au/(?P<path>[^?]+\?.*?\bepisode-id=(?P<id>[^&#]+))'
+    _TESTS = [{
+        'url': 'https://7plus.com.au/BEAT?episode-id=BEAT-001',
+        'info_dict': {
+            'id': 'BEAT-001',
+            'ext': 'mp4',
+            'title': 'S1 E1 - Help / Lucy In The Sky With Diamonds',
+            'description': 'md5:37718bea20a8eedaca7f7361af566131',
+            'uploader_id': '5303576322001',
+            'upload_date': '20171031',
+            'timestamp': 1509440068,
+        },
+        'params': {
+            'format': 'bestvideo',
+            'skip_download': True,
+        }
+    }, {
+        'url': 'https://7plus.com.au/UUUU?episode-id=AUMS43-001',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        path, episode_id = re.match(self._VALID_URL, url).groups()
+
+        media = self._download_json(
+            'https://videoservice.swm.digital/playback', episode_id, query={
+                'appId': '7plus',
+                'deviceType': 'web',
+                'platformType': 'web',
+                'accountId': 5303576322001,
+                'referenceId': 'ref:' + episode_id,
+                'deliveryId': 'csai',
+                'videoType': 'vod',
+            })['media']
+
+        for source in media.get('sources', {}):
+            src = source.get('src')
+            if not src:
+                continue
+            source['src'] = update_url_query(src, {'rule': ''})
+
+        info = self._parse_brightcove_metadata(media, episode_id)
+
+        content = self._download_json(
+            'https://component-cdn.swm.digital/content/' + path,
+            episode_id, headers={
+                'market-id': 4,
+            }, fatal=False) or {}
+        for item in content.get('items', {}):
+            if item.get('componentData', {}).get('componentType') == 'infoPanel':
+                for src_key, dst_key in [('title', 'title'), ('shortSynopsis', 'description')]:
+                    value = item.get(src_key)
+                    if value:
+                        info[dst_key] = value
+
+        return info
-- 
cgit v1.1


From 2132edaa03857085821b6a1214ce1410e0c2e463 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 23 Dec 2017 20:57:35 +0700
Subject: [extractor/common] Move X-Forwarded-For setup code into
 _request_webpage

---
 youtube_dl/extractor/common.py | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index e5ef5e4..3b79b8c 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -495,6 +495,16 @@ class InfoExtractor(object):
                 self.to_screen('%s' % (note,))
             else:
                 self.to_screen('%s: %s' % (video_id, note))
+
+        # Some sites check X-Forwarded-For HTTP header in order to figure out
+        # the origin of the client behind proxy. This allows bypassing geo
+        # restriction by faking this header's value to IP that belongs to some
+        # geo unrestricted country. We will do so once we encounter any
+        # geo restriction error.
+        if self._x_forwarded_for_ip:
+            if 'X-Forwarded-For' not in headers:
+                headers['X-Forwarded-For'] = self._x_forwarded_for_ip
+
         if isinstance(url_or_request, compat_urllib_request.Request):
             url_or_request = update_Request(
                 url_or_request, data=data, headers=headers, query=query)
@@ -524,15 +534,6 @@ class InfoExtractor(object):
         if isinstance(url_or_request, (compat_str, str)):
             url_or_request = url_or_request.partition('#')[0]
 
-        # Some sites check X-Forwarded-For HTTP header in order to figure out
-        # the origin of the client behind proxy. This allows bypassing geo
-        # restriction by faking this header's value to IP that belongs to some
-        # geo unrestricted country. We will do so once we encounter any
-        # geo restriction error.
-        if self._x_forwarded_for_ip:
-            if 'X-Forwarded-For' not in headers:
-                headers['X-Forwarded-For'] = self._x_forwarded_for_ip
-
         urlh = self._request_webpage(url_or_request, video_id, note, errnote, fatal, data=data, headers=headers, query=query)
         if urlh is False:
             assert not fatal
-- 
cgit v1.1


From 5c5e60cff894e5372f89e6ba45d7ab6575c0a0b6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 23 Dec 2017 20:59:14 +0700
Subject: [voot] Fix video identification

---
 youtube_dl/extractor/voot.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/voot.py b/youtube_dl/extractor/voot.py
index 4267544..751b21e 100644
--- a/youtube_dl/extractor/voot.py
+++ b/youtube_dl/extractor/voot.py
@@ -16,7 +16,7 @@ class VootIE(InfoExtractor):
     _TESTS = [{
         'url': 'https://www.voot.com/shows/ishq-ka-rang-safed/1/360558/is-this-the-end-of-kamini-/441353',
         'info_dict': {
-            'id': '441353',
+            'id': '0_8ledb18o',
             'ext': 'mp4',
             'title': 'Ishq Ka Rang Safed - Season 01 - Episode 340',
             'description': 'md5:06291fbbbc4dcbe21235c40c262507c1',
@@ -59,9 +59,10 @@ class VootIE(InfoExtractor):
 
         media = media_info['assets']
 
+        entry_id = media['EntryId']
         title = media['MediaName']
         formats = self._extract_m3u8_formats(
-            'https://cdnapisec.kaltura.com/p/1982551/playManifest/pt/https/f/applehttp/t/web/e/' + media['EntryId'],
+            'https://cdnapisec.kaltura.com/p/1982551/playManifest/pt/https/f/applehttp/t/web/e/' + entry_id,
             video_id, 'mp4', m3u8_id='hls')
         self._sort_formats(formats)
 
@@ -83,7 +84,8 @@ class VootIE(InfoExtractor):
                 episode_number = int_or_none(value)
 
         return {
-            'id': video_id,
+            'extractor_key': 'Kaltura',
+            'id': entry_id,
             'title': title,
             'description': description,
             'series': series,
-- 
cgit v1.1


From 69d69da98aa093c05776371beac1f2ffb4f5eea6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 23 Dec 2017 21:16:32 +0700
Subject: [kaltura] Add another embed pattern for entry_id For cases when
 player configuration map is setup via indexing operator, e.g.
 kalturaPlayerConfiguration_1_lre6rg3i_10[entry_id] = 1_lre6rg3i (see
 https://www.heise.de/video/artikel/odcast-c-t-uplink-20-1-Apple-CarPlay-vs-Android-Auto-Galileo-3D-Sound-erklaert-3919694.html)

---
 youtube_dl/extractor/kaltura.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/kaltura.py b/youtube_dl/extractor/kaltura.py
index bdac2df..e369959 100644
--- a/youtube_dl/extractor/kaltura.py
+++ b/youtube_dl/extractor/kaltura.py
@@ -125,9 +125,12 @@ class KalturaIE(InfoExtractor):
                         (?:https?:)?//cdnapi(?:sec)?\.kaltura\.com(?::\d+)?/(?:(?!(?P=q1)).)*\b(?:p|partner_id)/(?P<partner_id>\d+)(?:(?!(?P=q1)).)*
                     (?P=q1).*?
                     (?:
-                        entry_?[Ii]d|
-                        (?P<q2>["'])entry_?[Ii]d(?P=q2)
-                    )\s*:\s*
+                        (?:
+                            entry_?[Ii]d|
+                            (?P<q2>["'])entry_?[Ii]d(?P=q2)
+                        )\s*:\s*|
+                        \[\s*(?P<q2_1>["'])entry_?[Ii]d(?P=q2_1)\s*\]?\s*=\s*
+                    )
                     (?P<q3>["'])(?P<id>(?:(?!(?P=q3)).)+)(?P=q3)
                 ''', webpage) or
             re.search(
-- 
cgit v1.1


From f5a6321107db17ec8efaccaa2a4febc64b5aa5ea Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 23 Dec 2017 21:17:28 +0700
Subject: [ChangeLog] Actualize

---
 ChangeLog | 33 +++++++++++++++++++++++++++++++--
 1 file changed, 31 insertions(+), 2 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 03d2def..2d62f2f 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,32 @@
+version <unreleased>
+
+Core
+* [extractor/common] Move X-Forwarded-For setup code into _request_webpage
++ [YoutubeDL] Add support for playlist_uploader and playlist_uploader_id in
+  output template (#11427, #15018)
++ [extractor/common] Introduce uploader, uploader_id and uploader_url
+  meta fields for playlists (#11427, #15018)
+* [downloader/fragment] Encode filename of fragment being removed (#15020)
++ [utils] Add another date format pattern (#14999)
+
+Extractors
++ [kaltura] Add another embed pattern for entry_id
++ [7plus] Add support for 7plus.com.au (#15043)
+* [animeondemand] Relax login error regular expression
++ [shahid] Add support for show pages (#7401)
++ [youtube] Extract uploader, uploader_id and uploader_url for playlists
+  (#11427, #15018)
+* [afreecatv] Improve format extraction (#15019)
++ [cspan] Add support for audio only pages and catch page errors (#14995)
++ [mailru] Add support for embed URLs (#14904)
+* [crunchyroll] Future-proof XML element checks (#15013)
+* [cbslocal] Fix timestamp extraction (#14999, #15000)
+* [discoverygo] Correct TTML subtitle extension
+* [vk] Make view count optional (#14979)
+* [disney] Skip Apple FairPlay formats (#14982)
+* [voot] Fix format extraction (#14758)
+
+
 version 2017.12.14
 
 Core
@@ -148,8 +177,8 @@ Extractors
 + [fxnetworks] Extract series metadata (#14603)
 + [younow] Add support for younow.com (#9255, #9432, #12436)
 * [dctptv] Fix extraction (#14599)
-* [youtube] Restrict embed regex (#14600)
-* [vimeo] Restrict iframe embed regex (#14600)
+* [youtube] Restrict embed regular expression (#14600)
+* [vimeo] Restrict iframe embed regular expression (#14600)
 * [soundgasm] Improve extraction (#14588)
 - [myvideo] Remove extractor (#8557)
 + [nbc] Add support for classic-tv videos (#14575)
-- 
cgit v1.1


From c2f2f8b120628f7e0e4b0a6f7184884fa976d9c0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 23 Dec 2017 21:22:41 +0700
Subject: [kaltura] Fix typo

---
 youtube_dl/extractor/kaltura.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/kaltura.py b/youtube_dl/extractor/kaltura.py
index e369959..562e25f 100644
--- a/youtube_dl/extractor/kaltura.py
+++ b/youtube_dl/extractor/kaltura.py
@@ -129,7 +129,7 @@ class KalturaIE(InfoExtractor):
                             entry_?[Ii]d|
                             (?P<q2>["'])entry_?[Ii]d(?P=q2)
                         )\s*:\s*|
-                        \[\s*(?P<q2_1>["'])entry_?[Ii]d(?P=q2_1)\s*\]?\s*=\s*
+                        \[\s*(?P<q2_1>["'])entry_?[Ii]d(?P=q2_1)\s*\]\s*=\s*
                     )
                     (?P<q3>["'])(?P<id>(?:(?!(?P=q3)).)+)(?P=q3)
                 ''', webpage) or
-- 
cgit v1.1


From 307a7588b0a9205688e8ebc2539c1b0e19f68a6f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 23 Dec 2017 21:24:18 +0700
Subject: release 2017.12.23

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 docs/supportedsites.md    | 2 ++
 youtube_dl/version.py     | 2 +-
 4 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index f37d8aa..d7a9123 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.12.14*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.12.14**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.12.23*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.12.23**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through the [README](https://github.com/rg3/youtube-dl/blob/master/README.md), **most notably** the [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ Add the `-v` flag to **your command line** you run youtube-dl with (`youtube-dl
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.12.14
+[debug] youtube-dl version 2017.12.23
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index 2d62f2f..ba64f3e 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.12.23
 
 Core
 * [extractor/common] Move X-Forwarded-For setup code into _request_webpage
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index ebddd5b..eac35e3 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -10,6 +10,7 @@
  - **56.com**
  - **5min**
  - **6play**
+ - **7plus**
  - **8tracks**
  - **91porn**
  - **9c9media**
@@ -728,6 +729,7 @@
  - **Servus**
  - **Sexu**
  - **Shahid**
+ - **ShahidShow**
  - **Shared**: shared.sx
  - **ShowRoomLive**
  - **Sina**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 2b5a634..f999584 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.12.14'
+__version__ = '2017.12.23'
-- 
cgit v1.1


From 0e25a1a27875369a4fdf11b6a7fcfc969b1f482d Mon Sep 17 00:00:00 2001
From: JianxinLi <justcoxier@gmail.com>
Date: Mon, 4 Dec 2017 10:33:56 +0800
Subject: [youku] Update ccode

Change-Id: Id397e814e81ff560506d68563b7409eebbe5943d
---
 youtube_dl/extractor/youku.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index f0ba011..9d0caee 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -154,7 +154,7 @@ class YoukuIE(InfoExtractor):
         # request basic data
         basic_data_params = {
             'vid': video_id,
-            'ccode': '0501',
+            'ccode': '0507',
             'client_ip': '192.168.1.1',
             'utid': cna,
             'client_ts': time.time() / 1000,
-- 
cgit v1.1


From 116561697d605f22f749e3d092e8e4795ca0573d Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 23 Dec 2017 23:41:24 +0800
Subject: [ChangeLog] Update after #14903

---
 ChangeLog | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index ba64f3e..658c00c 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,9 @@
+version <unrequested>
+
+Extractors
+* [youku] Update ccode
+
+
 version 2017.12.23
 
 Core
-- 
cgit v1.1


From b954e72c8731e65b9b0548a537cd0e3275b54e4d Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 23 Dec 2017 23:42:02 +0800
Subject: [ChangeLog] typo

---
 ChangeLog | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ChangeLog b/ChangeLog
index 658c00c..cb750e2 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unrequested>
+version <unreleased>
 
 Extractors
 * [youku] Update ccode
-- 
cgit v1.1


From 273c23d960cbd2da18fadaef002473db41b5f56b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 24 Dec 2017 13:53:27 +0700
Subject: [openload] Add support for oload.stream (closes #15070)

---
 youtube_dl/extractor/openload.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/openload.py b/youtube_dl/extractor/openload.py
index a99af12..aed579f 100644
--- a/youtube_dl/extractor/openload.py
+++ b/youtube_dl/extractor/openload.py
@@ -242,7 +242,7 @@ class PhantomJSwrapper(object):
 
 
 class OpenloadIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?(?:openload\.(?:co|io|link)|oload\.tv)/(?:f|embed)/(?P<id>[a-zA-Z0-9-_]+)'
+    _VALID_URL = r'https?://(?:www\.)?(?:openload\.(?:co|io|link)|oload\.(?:tv|stream))/(?:f|embed)/(?P<id>[a-zA-Z0-9-_]+)'
 
     _TESTS = [{
         'url': 'https://openload.co/f/kUEfGclsU9o',
@@ -289,6 +289,9 @@ class OpenloadIE(InfoExtractor):
     }, {
         'url': 'http://www.openload.link/f/KnG-kKZdcfY',
         'only_matching': True,
+    }, {
+        'url': 'https://oload.stream/f/KnG-kKZdcfY',
+        'only_matching': True,
     }]
 
     _USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36'
-- 
cgit v1.1


From a75419586bb900df711de49adf5047afa9f083ef Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 24 Dec 2017 20:47:42 +0800
Subject: [openload] Remove a confusing exception

If phantomjs is not installed, there's an error besides the missing
phantomjs exception:

Exception ignored in: <bound method PhantomJSwrapper.__del__ of <youtube_dl.extractor.openload.PhantomJSwrapper object at 0x7f8ad5e78278>>
Traceback (most recent call last):
  File "/home/yen/Projects/youtube-dl/youtube_dl/extractor/openload.py", line 142, in __del__
    os.remove(self._TMP_FILES[name].name)
AttributeError: 'PhantomJSwrapper' object has no attribute '_TMP_FILES'
---
 youtube_dl/extractor/openload.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/openload.py b/youtube_dl/extractor/openload.py
index aed579f..d1eb3be 100644
--- a/youtube_dl/extractor/openload.py
+++ b/youtube_dl/extractor/openload.py
@@ -112,6 +112,8 @@ class PhantomJSwrapper(object):
         return get_exe_version('phantomjs', version_re=r'([0-9.]+)')
 
     def __init__(self, extractor, required_version=None, timeout=10000):
+        self._TMP_FILES = {}
+
         self.exe = check_executable('phantomjs', ['-v'])
         if not self.exe:
             raise ExtractorError('PhantomJS executable not found in PATH, '
@@ -130,7 +132,6 @@ class PhantomJSwrapper(object):
         self.options = {
             'timeout': timeout,
         }
-        self._TMP_FILES = {}
         for name in self._TMP_FILE_NAMES:
             tmp = tempfile.NamedTemporaryFile(delete=False)
             tmp.close()
@@ -140,7 +141,7 @@ class PhantomJSwrapper(object):
         for name in self._TMP_FILE_NAMES:
             try:
                 os.remove(self._TMP_FILES[name].name)
-            except (IOError, OSError):
+            except (IOError, OSError, KeyError):
                 pass
 
     def _save_cookies(self, url):
-- 
cgit v1.1


From d99a1000c7522cb37910afe772d7317687521eb0 Mon Sep 17 00:00:00 2001
From: JianxinLi <justcoxier@gmail.com>
Date: Sun, 24 Dec 2017 00:30:27 +0800
Subject: [youku] Fix list extraction.(close #15065)

Change-Id: I578fdc5b69509bdcd8d3191e3917afe47c234ff6
---
 youtube_dl/extractor/youku.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index 9d0caee..3e64cce 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -276,9 +276,9 @@ class YoukuShowIE(InfoExtractor):
             r'<div[^>]+id="(reload_\d+)', first_page, 'first page reload id')
         # The first reload_id has the same items as first_page
         reload_ids = re.findall('<li[^>]+data-id="([^"]+)">', first_page)
+        entries.extend(initial_entries)
         for idx, reload_id in enumerate(reload_ids):
             if reload_id == first_page_reload_id:
-                entries.extend(initial_entries)
                 continue
             _, new_entries = self._extract_entries(
                 'http://list.youku.com/show/episode', show_id,
-- 
cgit v1.1


From d3ca28323545a36819d9c32797907bc190095b5c Mon Sep 17 00:00:00 2001
From: JianxinLi <justcoxier@gmail.com>
Date: Mon, 25 Dec 2017 21:39:10 +0800
Subject: [youku] Add test case.

Some playlist has no data-id value.

Change-Id: I97455f2907f08bda03b538cdc13ec827e2f8ce26
---
 youtube_dl/extractor/youku.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index 3e64cce..c7947d4 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -241,6 +241,10 @@ class YoukuShowIE(InfoExtractor):
         # Ongoing playlist. The initial page is the last one
         'url': 'http://list.youku.com/show/id_za7c275ecd7b411e1a19e.html',
         'only_matching': True,
+    }, {
+        #  No data-id value.
+        'url': 'http://list.youku.com/show/id_zefbfbd61237fefbfbdef.html',
+        'only_matching': True,
     }]
 
     def _extract_entries(self, playlist_data_url, show_id, note, query):
-- 
cgit v1.1


From 173558ce9620bf1b22ba2d4c67288e2a45c715fc Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 25 Dec 2017 22:06:18 +0800
Subject: [ChangeLog] Update after #15065

---
 ChangeLog | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/ChangeLog b/ChangeLog
index cb750e2..420a1bd 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,7 +1,8 @@
 version <unreleased>
 
 Extractors
-* [youku] Update ccode
+* [youku] Update ccode (#14880)
+* [youku] Fix list extraction (#15065)
 
 
 version 2017.12.23
-- 
cgit v1.1


From 0f897e0929b2a3ebcae616f8b1bbdac8cd9c6f75 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 25 Dec 2017 23:28:51 +0100
Subject: [espn] add support for espnfc and extract more formats(closes #8053)

---
 youtube_dl/extractor/espn.py | 72 +++++++++++++++++++++++++++++++-------------
 1 file changed, 51 insertions(+), 21 deletions(-)

diff --git a/youtube_dl/extractor/espn.py b/youtube_dl/extractor/espn.py
index 7a74360..0e135b8 100644
--- a/youtube_dl/extractor/espn.py
+++ b/youtube_dl/extractor/espn.py
@@ -1,6 +1,9 @@
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
+from .once import OnceIE
 from ..compat import compat_str
 from ..utils import (
     determine_ext,
@@ -9,22 +12,27 @@ from ..utils import (
 )
 
 
-class ESPNIE(InfoExtractor):
+class ESPNIE(OnceIE):
     _VALID_URL = r'''(?x)
                     https?://
                         (?:
-                            (?:(?:\w+\.)+)?espn\.go|
-                            (?:www\.)?espn
-                        )\.com/
-                        (?:
                             (?:
-                                video/clip|
-                                watch/player
-                            )
-                            (?:
-                                \?.*?\bid=|
-                                /_/id/
-                            )
+                                (?:
+                                    (?:(?:\w+\.)+)?espn\.go|
+                                    (?:www\.)?espn
+                                )\.com/
+                                (?:
+                                    (?:
+                                        video/(?:clip|iframe/twitter)|
+                                        watch/player
+                                    )
+                                    (?:
+                                        .*?\?.*?\bid=|
+                                        /_/id/
+                                    )
+                                )
+                            )|
+                            (?:www\.)espnfc\.(?:com|us)/(?:video/)?[^/]+/\d+/video/
                         )
                         (?P<id>\d+)
                     '''
@@ -77,6 +85,15 @@ class ESPNIE(InfoExtractor):
     }, {
         'url': 'http://www.espn.com/video/clip/_/id/17989860',
         'only_matching': True,
+    }, {
+        'url': 'https://espn.go.com/video/iframe/twitter/?cms=espn&id=10365079',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.espnfc.us/video/espn-fc-tv/86/video/3319154/nashville-unveiled-as-the-newest-club-in-mls',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.espnfc.com/english-premier-league/23/video/3324163/premier-league-in-90-seconds-golden-tweets',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
@@ -93,7 +110,9 @@ class ESPNIE(InfoExtractor):
 
         def traverse_source(source, base_source_id=None):
             for source_id, source in source.items():
-                if isinstance(source, compat_str):
+                if source_id == 'alert':
+                    continue
+                elif isinstance(source, compat_str):
                     extract_source(source, base_source_id)
                 elif isinstance(source, dict):
                     traverse_source(
@@ -106,7 +125,9 @@ class ESPNIE(InfoExtractor):
                 return
             format_urls.add(source_url)
             ext = determine_ext(source_url)
-            if ext == 'smil':
+            if OnceIE.suitable(source_url):
+                formats.extend(self._extract_once_formats(source_url))
+            elif ext == 'smil':
                 formats.extend(self._extract_smil_formats(
                     source_url, video_id, fatal=False))
             elif ext == 'f4m':
@@ -117,12 +138,24 @@ class ESPNIE(InfoExtractor):
                     source_url, video_id, 'mp4', entry_protocol='m3u8_native',
                     m3u8_id=source_id, fatal=False))
             else:
-                formats.append({
+                f = {
                     'url': source_url,
                     'format_id': source_id,
-                })
-
-        traverse_source(clip['links']['source'])
+                }
+                mobj = re.search(r'(\d+)p(\d+)_(\d+)k\.', source_url)
+                if mobj:
+                    f.update({
+                        'height': int(mobj.group(1)),
+                        'fps': int(mobj.group(2)),
+                        'tbr': int(mobj.group(3)),
+                    })
+                if source_id == 'mezzanine':
+                    f['preference'] = 1
+                formats.append(f)
+
+        links = clip.get('links', {})
+        traverse_source(links.get('source', {}))
+        traverse_source(links.get('mobile', {}))
         self._sort_formats(formats)
 
         description = clip.get('caption') or clip.get('description')
@@ -144,9 +177,6 @@ class ESPNIE(InfoExtractor):
 class ESPNArticleIE(InfoExtractor):
     _VALID_URL = r'https?://(?:espn\.go|(?:www\.)?espn)\.com/(?:[^/]+/)*(?P<id>[^/]+)'
     _TESTS = [{
-        'url': 'https://espn.go.com/video/iframe/twitter/?cms=espn&id=10365079',
-        'only_matching': True,
-    }, {
         'url': 'http://espn.go.com/nba/recap?gameId=400793786',
         'only_matching': True,
     }, {
-- 
cgit v1.1


From 447a5a710dcd05741ea8cefa2fe98b333534e07d Mon Sep 17 00:00:00 2001
From: sprhawk <465558+sprhawk@users.noreply.github.com>
Date: Tue, 26 Dec 2017 16:18:35 +0800
Subject: added weibo mobile site support

---
 youtube_dl/extractor/extractors.py |  5 ++++-
 youtube_dl/extractor/weibo.py      | 46 ++++++++++++++++++++++++++++++++++++++
 2 files changed, 50 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 12dc2e7..f1ea735 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1286,7 +1286,10 @@ from .webofstories import (
     WebOfStoriesIE,
     WebOfStoriesPlaylistIE,
 )
-from .weibo import WeiboIE
+from .weibo import (
+    WeiboIE, 
+    WeiboMobileIE
+)
 from .weiqitv import WeiqiTVIE
 from .wimp import WimpIE
 from .wistia import WistiaIE
diff --git a/youtube_dl/extractor/weibo.py b/youtube_dl/extractor/weibo.py
index b835f89..eda0fa6 100644
--- a/youtube_dl/extractor/weibo.py
+++ b/youtube_dl/extractor/weibo.py
@@ -9,6 +9,11 @@ from urllib import parse
 import json
 import random as rnd
 from os import path
+import re
+
+from ..utils import (
+    js_to_json,
+)
 
 class WeiboIE(InfoExtractor):
     _VALID_URL = r'https?://weibo\.com/[0-9]+/(?P<id>[a-zA-Z0-9]+)'
@@ -103,3 +108,44 @@ class WeiboIE(InfoExtractor):
                 'formats': formats
                 # TODO more properties (see youtube_dl/extractor/common.py)
                 }
+
+class WeiboMobileIE(InfoExtractor):
+    _VALID_URL = r'https?://m.weibo.cn/status/(?P<id>[0-9]+)(\?.+)?'
+    _TEST = {
+            'url': 'https://m.weibo.cn/status/4189191225395228?wm=3333_2001&sourcetype=weixin&featurecode=newtitle&from=singlemessage&isappinstalled=0',
+            'info_dict': {
+                'id': '4189191225395228',
+                'ext': 'mp4',
+                'title': '午睡当然是要甜甜蜜蜜的啦',
+                'uploader': '柴犬柴犬'
+                }
+            }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        headers = {
+            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
+            'Accept-Encoding': 'gzip, deflate, br',
+            'Accept-Language': 'en,zh-CN;q=0.9,zh;q=0.8',
+            'Upgrade-Insecure-Requests': '1',
+        }
+        # to get Referer url for genvisitor
+        webpage,urlh = self._download_webpage_handle(url, video_id, headers=headers, note="visit the page")
+        js_code = self._search_regex(r'var\s+\$render_data\s*=\s*\[({.*})\]\[0\] \|\| {};', webpage, 'js_code', flags = re.DOTALL)
+        weibo_info = self._parse_json(js_code, video_id, transform_source=js_to_json)
+        page_info = weibo_info['status']['page_info']
+        title = weibo_info['status']['status_title']
+        format = {
+            'url': page_info['media_info']['stream_url'],
+            'format': 'mp4', 
+                  }
+        formats = [format]
+        uploader = weibo_info['status']['user']['screen_name']
+
+        return {
+                'id': video_id,
+                'title': title,
+                'uploader': uploader,
+                'formats': formats
+                # TODO more properties (see youtube_dl/extractor/common.py)
+                }
-- 
cgit v1.1


From d2be5bb5af7a1d7108b272315265e103a4358b28 Mon Sep 17 00:00:00 2001
From: sprhawk <465558+sprhawk@users.noreply.github.com>
Date: Tue, 26 Dec 2017 16:28:47 +0800
Subject: change to use compat urllib

---
 youtube_dl/extractor/weibo.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/weibo.py b/youtube_dl/extractor/weibo.py
index eda0fa6..6a4e0a4 100644
--- a/youtube_dl/extractor/weibo.py
+++ b/youtube_dl/extractor/weibo.py
@@ -3,14 +3,16 @@ from __future__ import unicode_literals
 
 from .common import InfoExtractor
 
-from urllib.request import Request
-from urllib.parse import urlencode
-from urllib import parse
 import json
 import random as rnd
 from os import path
 import re
 
+from ..compat import (
+    compat_urllib_parse_urlencode as urlencode,
+    compat_urllib_request as Request,
+    compat_urlparse as parse,
+)
 from ..utils import (
     js_to_json,
 )
-- 
cgit v1.1


From 951043724f91b3cfce60cf62cc3228a91a04ae81 Mon Sep 17 00:00:00 2001
From: sprhawk <465558+sprhawk@users.noreply.github.com>
Date: Tue, 26 Dec 2017 16:38:51 +0800
Subject: re-format code to pass flake8

---
 youtube_dl/extractor/weibo.py | 103 +++++++++++++++++++++---------------------
 1 file changed, 52 insertions(+), 51 deletions(-)

diff --git a/youtube_dl/extractor/weibo.py b/youtube_dl/extractor/weibo.py
index 6a4e0a4..b4ac7b9 100644
--- a/youtube_dl/extractor/weibo.py
+++ b/youtube_dl/extractor/weibo.py
@@ -5,7 +5,6 @@ from .common import InfoExtractor
 
 import json
 import random as rnd
-from os import path
 import re
 
 from ..compat import (
@@ -17,16 +16,17 @@ from ..utils import (
     js_to_json,
 )
 
+
 class WeiboIE(InfoExtractor):
     _VALID_URL = r'https?://weibo\.com/[0-9]+/(?P<id>[a-zA-Z0-9]+)'
     _TEST = {
-            'url': 'https://weibo.com/6275294458/Fp6RGfbff?type=comment',
-            'info_dict': {
-                'id': 'Fp6RGfbff',
-                'ext': 'mp4',
-                'title': 'You should have servants to massage you,... 来自Hosico_猫 - 微博',
-                }
-            }
+        'url': 'https://weibo.com/6275294458/Fp6RGfbff?type=comment',
+        'info_dict': {
+            'id': 'Fp6RGfbff',
+            'ext': 'mp4',
+            'title': 'You should have servants to massage you,... 来自Hosico_猫 - 微博',
+        }
+    }
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
@@ -38,32 +38,32 @@ class WeiboIE(InfoExtractor):
             'Upgrade-Insecure-Requests': '1',
         }
         # to get Referer url for genvisitor
-        webpage,urlh = self._download_webpage_handle(url, video_id, headers=headers, note="first visit the page")
+        webpage, urlh = self._download_webpage_handle(url, video_id, headers=headers, note="first visit the page")
 
         visitor_url = urlh.geturl()
 
         data = urlencode({
             "cb": "gen_callback",
             "fp": '{"os":"2","browser":"Gecko57,0,0,0","fonts":"undefined","screenInfo":"1440*900*24","plugins":""}',
-            }).encode()
+        }).encode()
         headers = {
-                'Accept-Encoding': 'gzip, deflate, br',
-                'Accept': '*/*',
-                'Referer': visitor_url,
-                }
+            'Accept-Encoding': 'gzip, deflate, br',
+            'Accept': '*/*',
+            'Referer': visitor_url,
+        }
 
         r_genvisitor = Request(
             'https://passport.weibo.com/visitor/genvisitor',
-            data = data,
-            headers = headers,
-            method = 'POST'
-            )
-        webpage,urlh = self._download_webpage_handle(r_genvisitor, video_id, note="gen visitor")
+            data=data,
+            headers=headers,
+            method='POST'
+        )
+        webpage, urlh = self._download_webpage_handle(r_genvisitor, video_id, note="gen visitor")
 
-        p = webpage.split("&&")[1] # split "gen_callback && gen_callback(...)"
+        p = webpage.split("&&")[1]  # split "gen_callback && gen_callback(...)"
         i1 = p.find('{')
         i2 = p.rfind('}')
-        j = p[i1:i2+1] # get JSON object
+        j = p[i1:i2 + 1]  # get JSON object
         d = json.loads(j)
         tid = d["data"]["tid"]
         cnfd = "%03d" % d["data"]["confidence"]
@@ -76,17 +76,17 @@ class WeiboIE(InfoExtractor):
             'cb': 'cross_domain',
             'from': 'weibo',
             '_rand': rnd.random()
-            })
+        })
         gencallback_url = "https://passport.weibo.com/visitor/visitor?" + param
-        webpage,urlh = self._download_webpage_handle(gencallback_url, video_id, note="gen callback")
+        webpage, urlh = self._download_webpage_handle(gencallback_url, video_id, note="gen callback")
 
-        webpage,urlh = self._download_webpage_handle(url, video_id, headers=headers, note="retry to visit the page")
+        webpage, urlh = self._download_webpage_handle(url, video_id, headers=headers, note="retry to visit the page")
 
         # TODO more code goes here, for example ...
         title = self._html_search_regex(r'<title>(.+?)</title>', webpage, 'title')
 
         video_sources_text = self._search_regex("video-sources=\\\\\"(.+?)\"", webpage, 'video_sources')
-        
+
         video_formats = parse.parse_qs(video_sources_text)
 
         formats = []
@@ -100,28 +100,29 @@ class WeiboIE(InfoExtractor):
                         'url': vid_url,
                         'format': 'mp4',
                         'height': int(res),
-                        })
+                    })
         self._sort_formats(formats)
-        uploader = self._og_search_property('nick-name', webpage, 'uploader', default = None)
+        uploader = self._og_search_property('nick-name', webpage, 'uploader', default=None)
         return {
-                'id': video_id,
-                'title': title,
-                'uploader': uploader,
-                'formats': formats
-                # TODO more properties (see youtube_dl/extractor/common.py)
-                }
+            'id': video_id,
+            'title': title,
+            'uploader': uploader,
+            'formats': formats
+            # TODO more properties (see youtube_dl/extractor/common.py)
+        }
+
 
 class WeiboMobileIE(InfoExtractor):
     _VALID_URL = r'https?://m.weibo.cn/status/(?P<id>[0-9]+)(\?.+)?'
     _TEST = {
-            'url': 'https://m.weibo.cn/status/4189191225395228?wm=3333_2001&sourcetype=weixin&featurecode=newtitle&from=singlemessage&isappinstalled=0',
-            'info_dict': {
-                'id': '4189191225395228',
-                'ext': 'mp4',
-                'title': '午睡当然是要甜甜蜜蜜的啦',
-                'uploader': '柴犬柴犬'
-                }
-            }
+        'url': 'https://m.weibo.cn/status/4189191225395228?wm=3333_2001&sourcetype=weixin&featurecode=newtitle&from=singlemessage&isappinstalled=0',
+        'info_dict': {
+            'id': '4189191225395228',
+            'ext': 'mp4',
+            'title': '午睡当然是要甜甜蜜蜜的啦',
+            'uploader': '柴犬柴犬'
+        }
+    }
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
@@ -132,22 +133,22 @@ class WeiboMobileIE(InfoExtractor):
             'Upgrade-Insecure-Requests': '1',
         }
         # to get Referer url for genvisitor
-        webpage,urlh = self._download_webpage_handle(url, video_id, headers=headers, note="visit the page")
-        js_code = self._search_regex(r'var\s+\$render_data\s*=\s*\[({.*})\]\[0\] \|\| {};', webpage, 'js_code', flags = re.DOTALL)
+        webpage, urlh = self._download_webpage_handle(url, video_id, headers=headers, note="visit the page")
+        js_code = self._search_regex(r'var\s+\$render_data\s*=\s*\[({.*})\]\[0\] \|\| {};', webpage, 'js_code', flags=re.DOTALL)
         weibo_info = self._parse_json(js_code, video_id, transform_source=js_to_json)
         page_info = weibo_info['status']['page_info']
         title = weibo_info['status']['status_title']
         format = {
             'url': page_info['media_info']['stream_url'],
-            'format': 'mp4', 
-                  }
+            'format': 'mp4',
+        }
         formats = [format]
         uploader = weibo_info['status']['user']['screen_name']
 
         return {
-                'id': video_id,
-                'title': title,
-                'uploader': uploader,
-                'formats': formats
-                # TODO more properties (see youtube_dl/extractor/common.py)
-                }
+            'id': video_id,
+            'title': title,
+            'uploader': uploader,
+            'formats': formats
+            # TODO more properties (see youtube_dl/extractor/common.py)
+        }
-- 
cgit v1.1


From 25936512245fc571ab716d59e2d73c50d8cad6ce Mon Sep 17 00:00:00 2001
From: sprhawk <465558+sprhawk@users.noreply.github.com>
Date: Tue, 26 Dec 2017 16:46:01 +0800
Subject: fix compat_urllib_request for python2.7

---
 youtube_dl/extractor/weibo.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/weibo.py b/youtube_dl/extractor/weibo.py
index b4ac7b9..f8a5ee7 100644
--- a/youtube_dl/extractor/weibo.py
+++ b/youtube_dl/extractor/weibo.py
@@ -9,7 +9,7 @@ import re
 
 from ..compat import (
     compat_urllib_parse_urlencode as urlencode,
-    compat_urllib_request as Request,
+    compat_urllib_request as request,
     compat_urlparse as parse,
 )
 from ..utils import (
@@ -52,11 +52,10 @@ class WeiboIE(InfoExtractor):
             'Referer': visitor_url,
         }
 
-        r_genvisitor = Request(
+        r_genvisitor = request.Request(
             'https://passport.weibo.com/visitor/genvisitor',
             data=data,
             headers=headers,
-            method='POST'
         )
         webpage, urlh = self._download_webpage_handle(r_genvisitor, video_id, note="gen visitor")
 
@@ -85,7 +84,7 @@ class WeiboIE(InfoExtractor):
         # TODO more code goes here, for example ...
         title = self._html_search_regex(r'<title>(.+?)</title>', webpage, 'title')
 
-        video_sources_text = self._search_regex("video-sources=\\\\\"(.+?)\"", webpage, 'video_sources')
+        video_sources_text = self._search_regex(r'video-sources=\\\"(.+?)\"', webpage, 'video_sources')
 
         video_formats = parse.parse_qs(video_sources_text)
 
-- 
cgit v1.1


From 45d20488f188b680daa39c5b9fa88d0bba102ab5 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 26 Dec 2017 12:32:04 +0100
Subject: [umg:de] Add new extractor(closes #11582)(closes #11584)

---
 youtube_dl/extractor/extractors.py |   1 +
 youtube_dl/extractor/umg.py        | 103 +++++++++++++++++++++++++++++++++++++
 2 files changed, 104 insertions(+)
 create mode 100644 youtube_dl/extractor/umg.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 9ba1be2..3269ed7 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1143,6 +1143,7 @@ from .udemy import (
 from .udn import UDNEmbedIE
 from .uktvplay import UKTVPlayIE
 from .digiteka import DigitekaIE
+from .umg import UMGDeIE
 from .unistra import UnistraIE
 from .unity import UnityIE
 from .uol import UOLIE
diff --git a/youtube_dl/extractor/umg.py b/youtube_dl/extractor/umg.py
new file mode 100644
index 0000000..d815cd9
--- /dev/null
+++ b/youtube_dl/extractor/umg.py
@@ -0,0 +1,103 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    parse_filesize,
+    parse_iso8601,
+)
+
+
+class UMGDeIE(InfoExtractor):
+    IE_NAME = 'umg:de'
+    IE_DESC = 'Universal Music Deutschland'
+    _VALID_URL = r'https?://(?:www\.)?universal-music\.de/[^/]+/videos/[^/?#]+-(?P<id>\d+)'
+    _TEST = {
+        'url': 'https://www.universal-music.de/sido/videos/jedes-wort-ist-gold-wert-457803',
+        'md5': 'ebd90f48c80dcc82f77251eb1902634f',
+        'info_dict': {
+            'id': '457803',
+            'ext': 'mp4',
+            'title': 'Jedes Wort ist Gold wert',
+            'timestamp': 1513591800,
+            'upload_date': '20171218',
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        video_data = self._download_json(
+            'https://api.universal-music.de/graphql',
+            video_id, query={
+                'query': '''{
+  universalMusic(channel:16) {
+    video(id:%s) {
+      headline
+      formats {
+        formatId
+        url
+        type
+        width
+        height
+        mimeType
+        fileSize
+      }
+      duration
+      createdDate
+    }
+  }
+}''' % video_id})['data']['universalMusic']['video']
+
+        title = video_data['headline']
+        hls_url_template = 'http://mediadelivery.universal-music-services.de/vod/mp4:autofill/storage/' + '/'.join(list(video_id)) + '/content/%s/file/playlist.m3u8'
+
+        thumbnails = []
+        formats = []
+
+        def add_m3u8_format(format_id):
+            m3u8_formats = self._extract_m3u8_formats(
+                hls_url_template % format_id, video_id, 'mp4',
+                'm3u8_native', m3u8_id='hls', fatal='False')
+            if m3u8_formats and m3u8_formats[0].get('height'):
+                formats.extend(m3u8_formats)
+
+        for f in video_data.get('formats', []):
+            f_url = f.get('url')
+            mime_type = f.get('mimeType')
+            if not f_url or mime_type == 'application/mxf':
+                continue
+            fmt = {
+                'url': f_url,
+                'width': int_or_none(f.get('width')),
+                'height': int_or_none(f.get('height')),
+                'filesize': parse_filesize(f.get('fileSize')),
+            }
+            f_type = f.get('type')
+            if f_type == 'Image':
+                thumbnails.append(fmt)
+            elif f_type == 'Video':
+                format_id = f.get('formatId')
+                if format_id:
+                    fmt['format_id'] = format_id
+                    if mime_type == 'video/mp4':
+                        add_m3u8_format(format_id)
+                urlh = self._request_webpage(f_url, video_id, fatal=False)
+                if urlh:
+                    first_byte = urlh.read(1)
+                    if first_byte not in (b'F', b'\x00'):
+                        continue
+                    formats.append(fmt)
+        if not formats:
+            for format_id in (867, 836, 940):
+                add_m3u8_format(format_id)
+        self._sort_formats(formats, ('width', 'height', 'filesize', 'tbr'))
+
+        return {
+            'id': video_id,
+            'title': title,
+            'duration': int_or_none(video_data.get('duration')),
+            'timestamp': parse_iso8601(video_data.get('createdDate'), ' '),
+            'thumbnails': thumbnails,
+            'formats': formats,
+        }
-- 
cgit v1.1


From db145ee54a57f5ccc89639de8c589eb111a91b19 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 26 Dec 2017 14:20:21 +0100
Subject: [espn] Add new extractor for http://fivethirtyeight.com(closes #6864)

---
 youtube_dl/extractor/espn.py       | 31 +++++++++++++++++++++++++++++++
 youtube_dl/extractor/extractors.py |  1 +
 2 files changed, 32 insertions(+)

diff --git a/youtube_dl/extractor/espn.py b/youtube_dl/extractor/espn.py
index 0e135b8..127c69b 100644
--- a/youtube_dl/extractor/espn.py
+++ b/youtube_dl/extractor/espn.py
@@ -205,3 +205,34 @@ class ESPNArticleIE(InfoExtractor):
 
         return self.url_result(
             'http://espn.go.com/video/clip?id=%s' % video_id, ESPNIE.ie_key())
+
+
+class FiveThirtyEightIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?fivethirtyeight\.com/features/(?P<id>[^/?#]+)'
+    _TEST = {
+        'url': 'http://fivethirtyeight.com/features/how-the-6-8-raiders-can-still-make-the-playoffs/',
+        'info_dict': {
+            'id': '21846851',
+            'ext': 'mp4',
+            'title': 'FiveThirtyEight: The Raiders can still make the playoffs',
+            'description': 'Neil Paine breaks down the simplest scenario that will put the Raiders into the playoffs at 8-8.',
+            'timestamp': 1513960621,
+            'upload_date': '20171222',
+        },
+        'params': {
+            'skip_download': True,
+        },
+        'expected_warnings': ['Unable to download f4m manifest'],
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        video_id = self._search_regex(
+            r'data-video-id=["\'](?P<id>\d+)',
+            webpage, 'video id', group='id')
+
+        return self.url_result(
+            'http://espn.go.com/video/clip?id=%s' % video_id, ESPNIE.ie_key())
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 3269ed7..91bd328 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -322,6 +322,7 @@ from .escapist import EscapistIE
 from .espn import (
     ESPNIE,
     ESPNArticleIE,
+    FiveThirtyEightIE,
 )
 from .esri import EsriVideoIE
 from .etonline import ETOnlineIE
-- 
cgit v1.1


From a14001a5a13b1639dc98b75b0775d251487aad1d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?D=C3=A9stin=20Reed?= <trox1972@users.noreply.github.com>
Date: Wed, 17 Aug 2016 16:02:59 +0200
Subject: [Filmweb] Add extractor

---
 youtube_dl/extractor/extractors.py       |  2 ++
 youtube_dl/extractor/filmweb.py          | 45 ++++++++++++++++++++++++++++++++
 youtube_dl/extractor/twentythreevideo.py | 45 ++++++++++++++++++++++++++++++++
 3 files changed, 92 insertions(+)
 create mode 100644 youtube_dl/extractor/filmweb.py
 create mode 100644 youtube_dl/extractor/twentythreevideo.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 91bd328..04125d9 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -345,6 +345,7 @@ from .filmon import (
     FilmOnIE,
     FilmOnChannelIE,
 )
+from .filmweb import FilmwebIE
 from .firsttv import FirstTVIE
 from .fivemin import FiveMinIE
 from .fivetv import FiveTVIE
@@ -1120,6 +1121,7 @@ from .tvplayer import TVPlayerIE
 from .tweakers import TweakersIE
 from .twentyfourvideo import TwentyFourVideoIE
 from .twentymin import TwentyMinutenIE
+from .twentythreevideo import TwentyThreeVideoIE
 from .twitch import (
     TwitchVideoIE,
     TwitchChapterIE,
diff --git a/youtube_dl/extractor/filmweb.py b/youtube_dl/extractor/filmweb.py
new file mode 100644
index 0000000..a3d9f87
--- /dev/null
+++ b/youtube_dl/extractor/filmweb.py
@@ -0,0 +1,45 @@
+from __future__ import unicode_literals
+
+from .twentythreevideo import TwentyThreeVideoIE
+
+
+class FilmwebIE(TwentyThreeVideoIE):
+    IE_NAME = 'Filmweb'
+    _VALID_URL = r'https?://(?:www\.)?filmweb\.no/trailere/article(?P<id>\d+).ece'
+    _TEST = {
+        'url': 'http://www.filmweb.no/trailere/article1264921.ece',
+        'md5': 'e353f47df98e557d67edaceda9dece89',
+        'info_dict': {
+            'id': '1264921',
+            'title': 'Det som en gang var',
+            'ext': 'mp4',
+            'description': 'Trailer: Scener fra et vennskap',
+        }
+    }
+
+    _CLIENT_NAME = 'filmweb'
+    _CLIENT_ID = '12732917'
+    _EMBED_BASE_URL = 'http://www.filmweb.no/template/ajax/json_trailerEmbed.jsp?articleId=%s&autoplay=true'
+
+    def _real_extract(self, url):
+        article_id = self._match_id(url)
+        webpage = self._download_webpage(url, article_id)
+
+        title = self._search_regex(r'var\s+jsTitle\s*=\s*escape\("([^"]+)"\);',
+            webpage, 'title', fatal=True)
+
+        format_url = self._proto_relative_url(
+            self._html_search_regex(r'"(//filmweb\.23video\.com/[^"]+)"',
+                self._download_json(self._EMBED_BASE_URL % article_id,
+                    article_id)['embedCode'], 'format url'))
+
+        formats = self._extract_formats(format_url, self._CLIENT_ID)
+        self._sort_formats(formats)
+
+        return {
+            'id': article_id,
+            'title': title,
+            'alt_title': self._og_search_title(webpage),
+            'formats': formats,
+            'description': self._og_search_description(webpage),
+        }
diff --git a/youtube_dl/extractor/twentythreevideo.py b/youtube_dl/extractor/twentythreevideo.py
new file mode 100644
index 0000000..2bad2db
--- /dev/null
+++ b/youtube_dl/extractor/twentythreevideo.py
@@ -0,0 +1,45 @@
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class TwentyThreeVideoIE(InfoExtractor):
+    IE_NAME = '23video'
+    _VALID_URL = r'https?://(?:www\.)?(?P<client>[\w-]+)\.23video\.com/v.ihtml/player.html.*photo_id=(?P<id>\d+)'
+    _TEST = {}
+
+    _URL_TEMPLATE = 'https://%s.23video.com/%s/%s/%s/%s/download-video.mp4'
+    _FORMATS = {
+        'video_hd': {
+            'width': 1280,
+            'height': 720,
+        },
+        'video_medium': {
+            'width': 640,
+            'height': 360,
+        },
+        'video_mobile_high': {
+            'width': 320,
+            'height': 180,
+        }
+    }
+
+    def _extract_formats(self, url, client_id):
+        client_name = self._search_regex(r'([a-z]+)\.23video\.com', url, 'client name')
+        video_id = self._search_regex(r'photo%5fid=([^?&]+)', url, 'video id')
+        token = self._search_regex(r'token=([^?&]+)', url, 'token')
+
+        formats = []
+        for format_key in self._FORMATS.keys():
+            formats.append({
+                'url': self._URL_TEMPLATE % (client_name, client_id, video_id,
+                    token, format_key),
+                'width': self._FORMATS.get(format_key, {}).get('width'),
+                'height': self._FORMATS.get(format_key, {}).get('height'),
+            })
+
+        return formats
+
+    def _real_extract(self, url):
+        # TODO: Find out how to extract client_id
+        raise NotImplementedError('Not able to extract the `client_id`')
-- 
cgit v1.1


From 42a1012c7767306626c5358a18ad3e86417bd7b7 Mon Sep 17 00:00:00 2001
From: sprhawk <465558+sprhawk@users.noreply.github.com>
Date: Tue, 26 Dec 2017 22:20:43 +0800
Subject: fix according to
 "https://github.com/rg3/youtube-dl/pull/15079#discussion_r158688607"

---
 youtube_dl/extractor/weibo.py | 83 ++++++++++++++++---------------------------
 1 file changed, 31 insertions(+), 52 deletions(-)

diff --git a/youtube_dl/extractor/weibo.py b/youtube_dl/extractor/weibo.py
index f8a5ee7..2be31fe 100644
--- a/youtube_dl/extractor/weibo.py
+++ b/youtube_dl/extractor/weibo.py
@@ -4,13 +4,12 @@ from __future__ import unicode_literals
 from .common import InfoExtractor
 
 import json
-import random as rnd
+import random
 import re
 
 from ..compat import (
-    compat_urllib_parse_urlencode as urlencode,
-    compat_urllib_request as request,
-    compat_urlparse as parse,
+    compat_urllib_parse_urlencode,
+    compat_urlparse,
 )
 from ..utils import (
     js_to_json,
@@ -30,34 +29,28 @@ class WeiboIE(InfoExtractor):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        headers = {
-            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
-            'Accept-Encoding': 'gzip, deflate, br',
-            'Accept-Language': 'en,zh-CN;q=0.9,zh;q=0.8',
-            'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36',
-            'Upgrade-Insecure-Requests': '1',
-        }
         # to get Referer url for genvisitor
-        webpage, urlh = self._download_webpage_handle(url, video_id, headers=headers, note="first visit the page")
+        webpage, urlh = self._download_webpage_handle(url, video_id, note="first visit the page")
 
         visitor_url = urlh.geturl()
+        headers = {
+            'Referer': visitor_url
+        }
 
-        data = urlencode({
+        fp = {
+            "os": "2",
+            "browser": "Gecko57,0,0,0",
+            "fonts": "undefined",
+            "screenInfo": "1440*900*24",
+            "plugins": ""
+        }
+        data = compat_urllib_parse_urlencode({
             "cb": "gen_callback",
-            "fp": '{"os":"2","browser":"Gecko57,0,0,0","fonts":"undefined","screenInfo":"1440*900*24","plugins":""}',
+            "fp": json.dumps(fp),
         }).encode()
-        headers = {
-            'Accept-Encoding': 'gzip, deflate, br',
-            'Accept': '*/*',
-            'Referer': visitor_url,
-        }
 
-        r_genvisitor = request.Request(
-            'https://passport.weibo.com/visitor/genvisitor',
-            data=data,
-            headers=headers,
-        )
-        webpage, urlh = self._download_webpage_handle(r_genvisitor, video_id, note="gen visitor")
+        genvisitor_url = 'https://passport.weibo.com/visitor/genvisitor'
+        webpage, _ = self._download_webpage_handle(genvisitor_url, video_id, data=data, headers=headers, note="gen visitor")
 
         p = webpage.split("&&")[1]  # split "gen_callback && gen_callback(...)"
         i1 = p.find('{')
@@ -67,29 +60,28 @@ class WeiboIE(InfoExtractor):
         tid = d["data"]["tid"]
         cnfd = "%03d" % d["data"]["confidence"]
 
-        param = urlencode({
+        query = {
             'a': 'incarnate',
             't': tid,
             'w': 2,
             'c': cnfd,
             'cb': 'cross_domain',
             'from': 'weibo',
-            '_rand': rnd.random()
-        })
-        gencallback_url = "https://passport.weibo.com/visitor/visitor?" + param
-        webpage, urlh = self._download_webpage_handle(gencallback_url, video_id, note="gen callback")
+            '_rand': random.random()
+        }
+        gencallback_url = "https://passport.weibo.com/visitor/visitor"
+        self._download_webpage_handle(gencallback_url, video_id, note="gen callback", query=query)
 
-        webpage, urlh = self._download_webpage_handle(url, video_id, headers=headers, note="retry to visit the page")
+        webpage, _ = self._download_webpage_handle(url, video_id, note="retry to visit the page")
 
-        # TODO more code goes here, for example ...
         title = self._html_search_regex(r'<title>(.+?)</title>', webpage, 'title')
 
         video_sources_text = self._search_regex(r'video-sources=\\\"(.+?)\"', webpage, 'video_sources')
 
-        video_formats = parse.parse_qs(video_sources_text)
+        video_formats = compat_urlparse.parse_qs(video_sources_text)
 
         formats = []
-        supported_resolutions = ['720', '480']
+        supported_resolutions = ('720', '480')
         for res in supported_resolutions:
             f = video_formats.get(res)
             if isinstance(f, list):
@@ -107,12 +99,11 @@ class WeiboIE(InfoExtractor):
             'title': title,
             'uploader': uploader,
             'formats': formats
-            # TODO more properties (see youtube_dl/extractor/common.py)
         }
 
 
 class WeiboMobileIE(InfoExtractor):
-    _VALID_URL = r'https?://m.weibo.cn/status/(?P<id>[0-9]+)(\?.+)?'
+    _VALID_URL = r'https?://m\.weibo\.cn/status/(?P<id>[0-9]+)(\?.+)?'
     _TEST = {
         'url': 'https://m.weibo.cn/status/4189191225395228?wm=3333_2001&sourcetype=weixin&featurecode=newtitle&from=singlemessage&isappinstalled=0',
         'info_dict': {
@@ -125,29 +116,17 @@ class WeiboMobileIE(InfoExtractor):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        headers = {
-            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
-            'Accept-Encoding': 'gzip, deflate, br',
-            'Accept-Language': 'en,zh-CN;q=0.9,zh;q=0.8',
-            'Upgrade-Insecure-Requests': '1',
-        }
         # to get Referer url for genvisitor
-        webpage, urlh = self._download_webpage_handle(url, video_id, headers=headers, note="visit the page")
+        webpage, _ = self._download_webpage_handle(url, video_id, note="visit the page")
         js_code = self._search_regex(r'var\s+\$render_data\s*=\s*\[({.*})\]\[0\] \|\| {};', webpage, 'js_code', flags=re.DOTALL)
         weibo_info = self._parse_json(js_code, video_id, transform_source=js_to_json)
         page_info = weibo_info['status']['page_info']
-        title = weibo_info['status']['status_title']
-        format = {
-            'url': page_info['media_info']['stream_url'],
-            'format': 'mp4',
-        }
-        formats = [format]
-        uploader = weibo_info['status']['user']['screen_name']
+        title = weibo_info.get('status').get('status_title')
+        uploader = weibo_info.get('status').get('user').get('screen_name')
 
         return {
             'id': video_id,
             'title': title,
             'uploader': uploader,
-            'formats': formats
-            # TODO more properties (see youtube_dl/extractor/common.py)
+            'url': page_info['media_info']['stream_url']
         }
-- 
cgit v1.1


From be069839b4acb645799f7b216d14c046fb4a3400 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 26 Dec 2017 19:41:08 +0100
Subject: [filmweb] improve extraction

---
 youtube_dl/extractor/filmweb.py          | 53 +++++++++----------
 youtube_dl/extractor/twentythreevideo.py | 90 ++++++++++++++++++++++----------
 2 files changed, 86 insertions(+), 57 deletions(-)

diff --git a/youtube_dl/extractor/filmweb.py b/youtube_dl/extractor/filmweb.py
index a3d9f87..56000bc 100644
--- a/youtube_dl/extractor/filmweb.py
+++ b/youtube_dl/extractor/filmweb.py
@@ -1,45 +1,42 @@
 from __future__ import unicode_literals
 
-from .twentythreevideo import TwentyThreeVideoIE
+import re
 
+from .common import InfoExtractor
 
-class FilmwebIE(TwentyThreeVideoIE):
-    IE_NAME = 'Filmweb'
-    _VALID_URL = r'https?://(?:www\.)?filmweb\.no/trailere/article(?P<id>\d+).ece'
+
+class FilmwebIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?filmweb\.no/(?P<type>trailere|filmnytt)/article(?P<id>\d+)\.ece'
     _TEST = {
         'url': 'http://www.filmweb.no/trailere/article1264921.ece',
         'md5': 'e353f47df98e557d67edaceda9dece89',
         'info_dict': {
-            'id': '1264921',
-            'title': 'Det som en gang var',
+            'id': '13033574',
             'ext': 'mp4',
-            'description': 'Trailer: Scener fra et vennskap',
+            'title': 'Det som en gang var',
+            'upload_date': '20160316',
+            'timestamp': 1458140101,
+            'uploader_id': '12639966',
+            'uploader': 'Live Roaldset',
         }
     }
 
-    _CLIENT_NAME = 'filmweb'
-    _CLIENT_ID = '12732917'
-    _EMBED_BASE_URL = 'http://www.filmweb.no/template/ajax/json_trailerEmbed.jsp?articleId=%s&autoplay=true'
-
     def _real_extract(self, url):
-        article_id = self._match_id(url)
-        webpage = self._download_webpage(url, article_id)
-
-        title = self._search_regex(r'var\s+jsTitle\s*=\s*escape\("([^"]+)"\);',
-            webpage, 'title', fatal=True)
-
-        format_url = self._proto_relative_url(
-            self._html_search_regex(r'"(//filmweb\.23video\.com/[^"]+)"',
-                self._download_json(self._EMBED_BASE_URL % article_id,
-                    article_id)['embedCode'], 'format url'))
-
-        formats = self._extract_formats(format_url, self._CLIENT_ID)
-        self._sort_formats(formats)
+        article_type, article_id = re.match(self._VALID_URL, url).groups()
+        if article_type == 'filmnytt':
+            webpage = self._download_webpage(url, article_id)
+            article_id = self._search_regex(r'data-videoid="(\d+)"', webpage, 'article id')
+        embed_code = self._download_json(
+            'https://www.filmweb.no/template_v2/ajax/json_trailerEmbed.jsp',
+            article_id, query={
+                'articleId': article_id,
+            })['embedCode']
+        iframe_url = self._proto_relative_url(self._search_regex(
+            r'<iframe[^>]+src="([^"]+)', embed_code, 'iframe url'))
 
         return {
+            '_type': 'url_transparent',
             'id': article_id,
-            'title': title,
-            'alt_title': self._og_search_title(webpage),
-            'formats': formats,
-            'description': self._og_search_description(webpage),
+            'url': iframe_url,
+            'ie_key': 'TwentyThreeVideo',
         }
diff --git a/youtube_dl/extractor/twentythreevideo.py b/youtube_dl/extractor/twentythreevideo.py
index 2bad2db..aa0c6e9 100644
--- a/youtube_dl/extractor/twentythreevideo.py
+++ b/youtube_dl/extractor/twentythreevideo.py
@@ -1,45 +1,77 @@
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
+from ..utils import int_or_none
 
 
 class TwentyThreeVideoIE(InfoExtractor):
     IE_NAME = '23video'
-    _VALID_URL = r'https?://(?:www\.)?(?P<client>[\w-]+)\.23video\.com/v.ihtml/player.html.*photo_id=(?P<id>\d+)'
-    _TEST = {}
-
-    _URL_TEMPLATE = 'https://%s.23video.com/%s/%s/%s/%s/download-video.mp4'
-    _FORMATS = {
-        'video_hd': {
-            'width': 1280,
-            'height': 720,
-        },
-        'video_medium': {
-            'width': 640,
-            'height': 360,
-        },
-        'video_mobile_high': {
-            'width': 320,
-            'height': 180,
+    _VALID_URL = r'https?://video\.(?P<domain>twentythree\.net|23video\.com|filmweb\.no)/v\.ihtml/player\.html\?(?P<query>.*?\bphoto(?:_|%5f)id=(?P<id>\d+).*)'
+    _TEST = {
+        'url': 'https://video.twentythree.net/v.ihtml/player.html?showDescriptions=0&source=site&photo%5fid=20448876&autoPlay=1',
+        'md5': '75fcf216303eb1dae9920d651f85ced4',
+        'info_dict': {
+            'id': '20448876',
+            'ext': 'mp4',
+            'title': 'Video Marketing Minute: Personalized Video',
+            'timestamp': 1513855354,
+            'upload_date': '20171221',
+            'uploader_id': '12258964',
+            'uploader': 'Rasmus Bysted',
         }
     }
 
-    def _extract_formats(self, url, client_id):
-        client_name = self._search_regex(r'([a-z]+)\.23video\.com', url, 'client name')
-        video_id = self._search_regex(r'photo%5fid=([^?&]+)', url, 'video id')
-        token = self._search_regex(r'token=([^?&]+)', url, 'token')
+    def _real_extract(self, url):
+        domain, query, photo_id = re.match(self._VALID_URL, url).groups()
+        base_url = 'https://video.%s' % domain
+        photo_data = self._download_json(
+            base_url + '/api/photo/list?' + query, photo_id, query={
+                'format': 'json',
+            }, transform_source=lambda s: self._search_regex(r'(?s)({.+})', s, 'photo data'))['photo']
+        title = photo_data['title']
 
         formats = []
-        for format_key in self._FORMATS.keys():
+
+        audio_path = photo_data.get('audio_download')
+        if audio_path:
             formats.append({
-                'url': self._URL_TEMPLATE % (client_name, client_id, video_id,
-                    token, format_key),
-                'width': self._FORMATS.get(format_key, {}).get('width'),
-                'height': self._FORMATS.get(format_key, {}).get('height'),
+                'format_id': 'audio',
+                'url': base_url + audio_path,
+                'filesize': int_or_none(photo_data.get('audio_size')),
+                'vcodec': 'none',
+            })
+
+        def add_common_info_to_list(l, template, id_field, id_value):
+            f_base = template % id_value
+            f_path = photo_data.get(f_base + 'download')
+            if not f_path:
+                return
+            l.append({
+                id_field: id_value,
+                'url': base_url + f_path,
+                'width': int_or_none(photo_data.get(f_base + 'width')),
+                'height': int_or_none(photo_data.get(f_base + 'height')),
+                'filesize': int_or_none(photo_data.get(f_base + 'size')),
             })
 
-        return formats
+        for f in ('mobile_high', 'medium', 'hd', '1080p', '4k'):
+            add_common_info_to_list(formats, 'video_%s_', 'format_id', f)
 
-    def _real_extract(self, url):
-        # TODO: Find out how to extract client_id
-        raise NotImplementedError('Not able to extract the `client_id`')
+        thumbnails = []
+        for t in ('quad16', 'quad50', 'quad75', 'quad100', 'small', 'portrait', 'standard', 'medium', 'large', 'original'):
+            add_common_info_to_list(thumbnails, '%s_', 'id', t)
+
+        return {
+            'id': photo_id,
+            'title': title,
+            'timestamp': int_or_none(photo_data.get('creation_date_epoch')),
+            'duration': int_or_none(photo_data.get('video_length')),
+            'view_count': int_or_none(photo_data.get('view_count')),
+            'comment_count': int_or_none(photo_data.get('number_of_comments')),
+            'uploader_id': photo_data.get('user_id'),
+            'uploader': photo_data.get('display_name'),
+            'thumbnails': thumbnails,
+            'formats': formats,
+        }
-- 
cgit v1.1


From 616bb95b280e74b1a5048e95128259e2a85ca962 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ond=C5=99ej=20Caletka?= <ondrej@caletka.cz>
Date: Wed, 27 Dec 2017 16:57:26 +0100
Subject: [playtvak] Relax video regex and make description optional

---
 youtube_dl/extractor/playtvak.py | 20 +++++++++++++++-----
 1 file changed, 15 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/playtvak.py b/youtube_dl/extractor/playtvak.py
index 391e1bd..4c5f579 100644
--- a/youtube_dl/extractor/playtvak.py
+++ b/youtube_dl/extractor/playtvak.py
@@ -24,7 +24,7 @@ class PlaytvakIE(InfoExtractor):
             'id': 'A150730_150323_hodinovy-manzel_kuko',
             'ext': 'mp4',
             'title': 'Vyžeňte vosy a sršně ze zahrady',
-            'description': 'md5:f93d398691044d303bc4a3de62f3e976',
+            'description': 'md5:4436e61b7df227a093778efb7e373571',
             'thumbnail': r're:(?i)^https?://.*\.(?:jpg|png)$',
             'duration': 279,
             'timestamp': 1438732860,
@@ -36,9 +36,19 @@ class PlaytvakIE(InfoExtractor):
         'info_dict': {
             'id': 'A150624_164934_planespotting_cat',
             'ext': 'flv',
-            'title': 're:^Přímý přenos iDNES.cz [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
+            'title': 're:^Planespotting [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
             'description': 'Sledujte provoz na ranveji Letiště Václava Havla v Praze',
-            'thumbnail': r're:(?i)^https?://.*\.(?:jpg|png)$',
+            'is_live': True,
+        },
+        'params': {
+            'skip_download': True,  # requires rtmpdump
+        },
+    }, {  # another live stream, this one without Misc.videoFLV
+        'url': 'https://slowtv.playtvak.cz/zive-sledujte-vlaky-v-primem-prenosu-dwi-/hlavni-nadrazi.aspx?c=A151218_145728_hlavni-nadrazi_plap',
+        'info_dict': {
+            'id': 'A151218_145728_hlavni-nadrazi_plap',
+            'ext': 'flv',
+            'title': 're:^Hlavní nádraží [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
             'is_live': True,
         },
         'params': {
@@ -95,7 +105,7 @@ class PlaytvakIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
 
         info_url = self._html_search_regex(
-            r'Misc\.videoFLV\(\s*{\s*data\s*:\s*"([^"]+)"', webpage, 'info url')
+            r'Misc\.video(?:FLV)?\(\s*{\s*data\s*:\s*"([^"]+)"', webpage, 'info url')
 
         parsed_url = compat_urlparse.urlparse(info_url)
 
@@ -160,7 +170,7 @@ class PlaytvakIE(InfoExtractor):
         if is_live:
             title = self._live_title(title)
         description = self._og_search_description(webpage, default=None) or self._html_search_meta(
-            'description', webpage, 'description')
+            'description', webpage, 'description', default=None)
         timestamp = None
         duration = None
         if not is_live:
-- 
cgit v1.1


From 1ae0f0a21dec0cfce85b6f8285cc64cd702846b5 Mon Sep 17 00:00:00 2001
From: Leonardo Taccari <iamleot@gmail.com>
Date: Wed, 13 Dec 2017 12:24:07 +0100
Subject: [internazionale] Add extractor

---
 youtube_dl/extractor/extractors.py     |  1 +
 youtube_dl/extractor/internazionale.py | 46 ++++++++++++++++++++++++++++++++++
 2 files changed, 47 insertions(+)
 create mode 100644 youtube_dl/extractor/internazionale.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 04125d9..e7b93a6 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -466,6 +466,7 @@ from .indavideo import (
 )
 from .infoq import InfoQIE
 from .instagram import InstagramIE, InstagramUserIE
+from .internazionale import InternazionaleIE
 from .internetvideoarchive import InternetVideoArchiveIE
 from .iprima import IPrimaIE
 from .iqiyi import IqiyiIE
diff --git a/youtube_dl/extractor/internazionale.py b/youtube_dl/extractor/internazionale.py
new file mode 100644
index 0000000..2ebf05d
--- /dev/null
+++ b/youtube_dl/extractor/internazionale.py
@@ -0,0 +1,46 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class InternazionaleIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?internazionale\.it/video/(?P<id>.*)'
+    _TESTS = [{
+        'url': 'https://www.internazionale.it/video/2015/02/19/richard-linklater-racconta-una-scena-di-boyhood',
+        'md5': '11b54a3d3333e455c00684e50a65c58e',
+        'info_dict': {
+            'id': '265968',
+            'ext': 'mp4',
+            'description': 'md5:efb7e5bbfb1a54ae2ed5a4a015f0e665',
+            'title': 'Richard Linklater racconta una scena di Boyhood',
+            'thumbnail': r're:^https?://.*\.jpg$',
+        }
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        data_job_id = self._html_search_regex(r'data-job-id="([^"]+)"', webpage, 'data-job-id')
+        data_video_path = self._html_search_regex(r'data-video-path="([^"]+)"', webpage, 'data-video-path')
+
+        formats = []
+
+        formats.extend(self._extract_m3u8_formats(
+            'https://video.internazionale.it/%s/%s.m3u8' % (data_video_path, data_job_id),
+            video_id))
+
+        formats.extend(self._extract_mpd_formats(
+            'https://video.internazionale.it/%s/%s.mpd' % (data_video_path, data_job_id),
+            video_id))
+
+        self._sort_formats(formats)
+
+        return {
+            'id': data_job_id,
+            'title': self._og_search_title(webpage),
+            'thumbnail': self._og_search_thumbnail(webpage),
+            'description': self._og_search_description(webpage),
+            'formats': formats,
+        }
-- 
cgit v1.1


From 640788f6f476f4344c13ed7c369c73575d3e705c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 27 Dec 2017 23:26:48 +0700
Subject: [internazionale] Improve extraction (closes #14973)

---
 youtube_dl/extractor/internazionale.py | 56 ++++++++++++++++++++++------------
 1 file changed, 37 insertions(+), 19 deletions(-)

diff --git a/youtube_dl/extractor/internazionale.py b/youtube_dl/extractor/internazionale.py
index 2ebf05d..10ba1f6 100644
--- a/youtube_dl/extractor/internazionale.py
+++ b/youtube_dl/extractor/internazionale.py
@@ -2,45 +2,63 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..utils import unified_timestamp
 
 
 class InternazionaleIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?internazionale\.it/video/(?P<id>.*)'
-    _TESTS = [{
+    _VALID_URL = r'https?://(?:www\.)?internazionale\.it/video/(?:[^/]+/)*(?P<id>[^/?#&]+)'
+    _TEST = {
         'url': 'https://www.internazionale.it/video/2015/02/19/richard-linklater-racconta-una-scena-di-boyhood',
-        'md5': '11b54a3d3333e455c00684e50a65c58e',
+        'md5': '3e39d32b66882c1218e305acbf8348ca',
         'info_dict': {
             'id': '265968',
+            'display_id': 'richard-linklater-racconta-una-scena-di-boyhood',
             'ext': 'mp4',
-            'description': 'md5:efb7e5bbfb1a54ae2ed5a4a015f0e665',
             'title': 'Richard Linklater racconta una scena di Boyhood',
+            'description': 'md5:efb7e5bbfb1a54ae2ed5a4a015f0e665',
+            'timestamp': 1424354635,
+            'upload_date': '20150219',
             'thumbnail': r're:^https?://.*\.jpg$',
-        }
-    }]
+        },
+        'params': {
+            'format': 'bestvideo',
+        },
+    }
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
+        display_id = self._match_id(url)
 
-        data_job_id = self._html_search_regex(r'data-job-id="([^"]+)"', webpage, 'data-job-id')
-        data_video_path = self._html_search_regex(r'data-video-path="([^"]+)"', webpage, 'data-video-path')
+        webpage = self._download_webpage(url, display_id)
 
-        formats = []
+        DATA_RE = r'data-%s=(["\'])(?P<value>(?:(?!\1).)+)\1'
 
-        formats.extend(self._extract_m3u8_formats(
-            'https://video.internazionale.it/%s/%s.m3u8' % (data_video_path, data_job_id),
-            video_id))
+        title = self._search_regex(
+            DATA_RE % 'video-title', webpage, 'title', default=None,
+            group='value') or self._og_search_title(webpage)
 
-        formats.extend(self._extract_mpd_formats(
-            'https://video.internazionale.it/%s/%s.mpd' % (data_video_path, data_job_id),
-            video_id))
+        video_id = self._search_regex(
+            DATA_RE % 'job-id', webpage, 'video id', group='value')
+        video_path = self._search_regex(
+            DATA_RE % 'video-path', webpage, 'video path', group='value')
 
+        video_base = 'https://video.internazionale.it/%s/%s.' % (video_path, video_id)
+
+        formats = self._extract_m3u8_formats(
+            video_base + 'm3u8', display_id, 'mp4',
+            entry_protocol='m3u8_native', m3u8_id='hls', fatal=False)
+        formats.extend(self._extract_mpd_formats(
+            video_base + 'mpd', display_id, mpd_id='dash', fatal=False))
         self._sort_formats(formats)
 
+        timestamp = unified_timestamp(self._html_search_meta(
+            'article:published_time', webpage, 'timestamp'))
+
         return {
-            'id': data_job_id,
-            'title': self._og_search_title(webpage),
+            'id': video_id,
+            'display_id': display_id,
+            'title': title,
             'thumbnail': self._og_search_thumbnail(webpage),
             'description': self._og_search_description(webpage),
+            'timestamp': timestamp,
             'formats': formats,
         }
-- 
cgit v1.1


From 99277daaac5b2cb6990f1d592b78ca8839635c72 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 28 Dec 2017 23:10:42 +0700
Subject: [ChangeLog] Actualize

---
 ChangeLog | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 420a1bd..794b002 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,7 +1,16 @@
 version <unreleased>
 
 Extractors
++ [internazionale] Add support for internazionale.it (#14973)
+* [playtvak] Relax video regular expression and make description optional
+  (#15037)
++ [filmweb] Add support for filmweb.no (#8773, #10368)
++ [23video] Add support for 23video.com
++ [espn] Add support for fivethirtyeight.com (#6864)
++ [umg:de] Add support for universal-music.de (#11582, #11584)
++ [espn] Add support for espnfc and extract more formats (#8053)
 * [youku] Update ccode (#14880)
++ [openload] Add support for oload.stream (#15070)
 * [youku] Fix list extraction (#15065)
 
 
-- 
cgit v1.1


From a491fd0c6f5abd68bae128cacbcc9926a017cee3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 28 Dec 2017 23:12:56 +0700
Subject: release 2017.12.28

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 docs/supportedsites.md    | 5 +++++
 youtube_dl/version.py     | 2 +-
 4 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index d7a9123..f40927d 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.12.23*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.12.23**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.12.28*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.12.28**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through the [README](https://github.com/rg3/youtube-dl/blob/master/README.md), **most notably** the [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ Add the `-v` flag to **your command line** you run youtube-dl with (`youtube-dl
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.12.23
+[debug] youtube-dl version 2017.12.28
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index 794b002..4323cfc 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.12.28
 
 Extractors
 + [internazionale] Add support for internazionale.it (#14973)
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index eac35e3..6b935e5 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -3,6 +3,7 @@
  - **1up.com**
  - **20min**
  - **220.ro**
+ - **23video**
  - **24video**
  - **3qsdn**: 3Q SDN
  - **3sat**
@@ -269,6 +270,8 @@
  - **Fczenit**
  - **filmon**
  - **filmon:channel**
+ - **Filmweb**
+ - **FiveThirtyEight**
  - **FiveTV**
  - **Flickr**
  - **Flipagram**
@@ -359,6 +362,7 @@
  - **InfoQ**
  - **Instagram**
  - **instagram:user**: Instagram user profile
+ - **Internazionale**
  - **InternetVideoArchive**
  - **IPrima**
  - **iqiyi**: 爱奇艺
@@ -889,6 +893,7 @@
  - **udemy:course**
  - **UDNEmbed**: 聯合影音
  - **UKTVPlay**
+ - **umg:de**: Universal Music Deutschland
  - **Unistra**
  - **Unity**
  - **uol.com.br**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index f999584..f4a1b67 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.12.23'
+__version__ = '2017.12.28'
-- 
cgit v1.1


From 84f085d4bdb66ee025fb337bcd571eab7469da97 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Fri, 29 Dec 2017 00:13:23 +0100
Subject: [aws] fix canonical/signed headers generation in python 2(closes
 #15102)

---
 youtube_dl/extractor/aws.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/aws.py b/youtube_dl/extractor/aws.py
index 670abce..dccfeaf 100644
--- a/youtube_dl/extractor/aws.py
+++ b/youtube_dl/extractor/aws.py
@@ -21,11 +21,11 @@ class AWSIE(InfoExtractor):
             'Accept': 'application/json',
             'Host': self._AWS_PROXY_HOST,
             'X-Amz-Date': amz_date,
+            'X-Api-Key': self._AWS_API_KEY
         }
         session_token = aws_dict.get('session_token')
         if session_token:
             headers['X-Amz-Security-Token'] = session_token
-        headers['X-Api-Key'] = self._AWS_API_KEY
 
         def aws_hash(s):
             return hashlib.sha256(s.encode('utf-8')).hexdigest()
@@ -33,9 +33,9 @@ class AWSIE(InfoExtractor):
         # Task 1: http://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html
         canonical_querystring = compat_urllib_parse_urlencode(query)
         canonical_headers = ''
-        for header_name, header_value in headers.items():
+        for header_name, header_value in sorted(headers.items()):
             canonical_headers += '%s:%s\n' % (header_name.lower(), header_value)
-        signed_headers = ';'.join([header.lower() for header in headers.keys()])
+        signed_headers = ';'.join([header.lower() for header in sorted(headers.keys())])
         canonical_request = '\n'.join([
             'GET',
             aws_dict['uri'],
-- 
cgit v1.1


From 9d6ac71c27b1dfb662c795ef598dbfd0286682da Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 29 Dec 2017 23:14:15 +0700
Subject: [extractor/common] Fix extraction of DASH formats with the same
 representation id (closes #15111)

---
 test/test_InfoExtractor.py     | 11 +++++++++++
 youtube_dl/extractor/common.py | 18 ++++++++----------
 2 files changed, 19 insertions(+), 10 deletions(-)

diff --git a/test/test_InfoExtractor.py b/test/test_InfoExtractor.py
index 8a372d2..7b31d51 100644
--- a/test/test_InfoExtractor.py
+++ b/test/test_InfoExtractor.py
@@ -493,10 +493,21 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
         _TEST_CASES = [
             (
                 # https://github.com/rg3/youtube-dl/issues/13919
+                # Also tests duplicate representation ids, see
+                # https://github.com/rg3/youtube-dl/issues/15111
                 'float_duration',
                 'http://unknown/manifest.mpd',
                 [{
                     'manifest_url': 'http://unknown/manifest.mpd',
+                    'ext': 'm4a',
+                    'format_id': '318597',
+                    'format_note': 'DASH audio',
+                    'protocol': 'http_dash_segments',
+                    'acodec': 'mp4a.40.2',
+                    'vcodec': 'none',
+                    'tbr': 61.587,
+                }, {
+                    'manifest_url': 'http://unknown/manifest.mpd',
                     'ext': 'mp4',
                     'format_id': '318597',
                     'format_note': 'DASH video',
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 3b79b8c..35d427e 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -2007,16 +2007,14 @@ class InfoExtractor(object):
                                     f['url'] = initialization_url
                                 f['fragments'].append({location_key(initialization_url): initialization_url})
                             f['fragments'].extend(representation_ms_info['fragments'])
-                        try:
-                            existing_format = next(
-                                fo for fo in formats
-                                if fo['format_id'] == representation_id)
-                        except StopIteration:
-                            full_info = formats_dict.get(representation_id, {}).copy()
-                            full_info.update(f)
-                            formats.append(full_info)
-                        else:
-                            existing_format.update(f)
+                        # According to [1, 5.3.5.2, Table 7, page 35] @id of Representation
+                        # is not necessarily unique within a Period thus formats with
+                        # the same `format_id` are quite possible. There are numerous examples
+                        # of such manifests (see https://github.com/rg3/youtube-dl/issues/15111,
+                        # https://github.com/rg3/youtube-dl/issues/13919)
+                        full_info = formats_dict.get(representation_id, {}).copy()
+                        full_info.update(f)
+                        formats.append(full_info)
                     else:
                         self.report_warning('Unknown MIME type %s in DASH manifest' % mime_type)
         return formats
-- 
cgit v1.1


From 5c97ec5ff5fd77a7975e1e946d53a76ccd5ef0de Mon Sep 17 00:00:00 2001
From: sprhawk <465558+sprhawk@users.noreply.github.com>
Date: Sat, 30 Dec 2017 01:08:56 +0800
Subject: replace urlencode.encode with urlencode_postdata

---
 youtube_dl/extractor/weibo.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/weibo.py b/youtube_dl/extractor/weibo.py
index 2be31fe..0b28952 100644
--- a/youtube_dl/extractor/weibo.py
+++ b/youtube_dl/extractor/weibo.py
@@ -8,11 +8,11 @@ import random
 import re
 
 from ..compat import (
-    compat_urllib_parse_urlencode,
     compat_urlparse,
 )
 from ..utils import (
     js_to_json,
+    urlencode_postdata,
 )
 
 
@@ -44,10 +44,10 @@ class WeiboIE(InfoExtractor):
             "screenInfo": "1440*900*24",
             "plugins": ""
         }
-        data = compat_urllib_parse_urlencode({
+        data = urlencode_postdata({
             "cb": "gen_callback",
             "fp": json.dumps(fp),
-        }).encode()
+        })
 
         genvisitor_url = 'https://passport.weibo.com/visitor/genvisitor'
         webpage, _ = self._download_webpage_handle(genvisitor_url, video_id, data=data, headers=headers, note="gen visitor")
-- 
cgit v1.1


From 6a41a12d2960efb7b32d3b6ef74cf6237766b569 Mon Sep 17 00:00:00 2001
From: sprhawk <465558+sprhawk@users.noreply.github.com>
Date: Sat, 30 Dec 2017 01:11:30 +0800
Subject: replace split with strip_jsonp

---
 youtube_dl/extractor/weibo.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/weibo.py b/youtube_dl/extractor/weibo.py
index 0b28952..71e7123 100644
--- a/youtube_dl/extractor/weibo.py
+++ b/youtube_dl/extractor/weibo.py
@@ -12,6 +12,7 @@ from ..compat import (
 )
 from ..utils import (
     js_to_json,
+    strip_jsonp,
     urlencode_postdata,
 )
 
@@ -52,7 +53,7 @@ class WeiboIE(InfoExtractor):
         genvisitor_url = 'https://passport.weibo.com/visitor/genvisitor'
         webpage, _ = self._download_webpage_handle(genvisitor_url, video_id, data=data, headers=headers, note="gen visitor")
 
-        p = webpage.split("&&")[1]  # split "gen_callback && gen_callback(...)"
+        p = strip_jsonp(webpage)
         i1 = p.find('{')
         i2 = p.rfind('}')
         j = p[i1:i2 + 1]  # get JSON object
-- 
cgit v1.1


From 48058d82dc3b448a72fd5ac1e7fa5492cd11f640 Mon Sep 17 00:00:00 2001
From: sprhawk <465558+sprhawk@users.noreply.github.com>
Date: Sat, 30 Dec 2017 01:14:21 +0800
Subject: replace unused _download_webpage_handle with _download_webpage

---
 youtube_dl/extractor/weibo.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/weibo.py b/youtube_dl/extractor/weibo.py
index 71e7123..34809bd 100644
--- a/youtube_dl/extractor/weibo.py
+++ b/youtube_dl/extractor/weibo.py
@@ -51,7 +51,7 @@ class WeiboIE(InfoExtractor):
         })
 
         genvisitor_url = 'https://passport.weibo.com/visitor/genvisitor'
-        webpage, _ = self._download_webpage_handle(genvisitor_url, video_id, data=data, headers=headers, note="gen visitor")
+        webpage = self._download_webpage(genvisitor_url, video_id, data=data, headers=headers, note="gen visitor")
 
         p = strip_jsonp(webpage)
         i1 = p.find('{')
@@ -71,9 +71,9 @@ class WeiboIE(InfoExtractor):
             '_rand': random.random()
         }
         gencallback_url = "https://passport.weibo.com/visitor/visitor"
-        self._download_webpage_handle(gencallback_url, video_id, note="gen callback", query=query)
+        self._download_webpage(gencallback_url, video_id, note="gen callback", query=query)
 
-        webpage, _ = self._download_webpage_handle(url, video_id, note="retry to visit the page")
+        webpage = self._download_webpage(url, video_id, note="retry to visit the page")
 
         title = self._html_search_regex(r'<title>(.+?)</title>', webpage, 'title')
 
@@ -118,7 +118,7 @@ class WeiboMobileIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
         # to get Referer url for genvisitor
-        webpage, _ = self._download_webpage_handle(url, video_id, note="visit the page")
+        webpage = self._download_webpage(url, video_id, note="visit the page")
         js_code = self._search_regex(r'var\s+\$render_data\s*=\s*\[({.*})\]\[0\] \|\| {};', webpage, 'js_code', flags=re.DOTALL)
         weibo_info = self._parse_json(js_code, video_id, transform_source=js_to_json)
         page_info = weibo_info['status']['page_info']
-- 
cgit v1.1


From 580f3c79d52ae3f880b4ca64e22665bf094ae65f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 30 Dec 2017 03:54:14 +0700
Subject: [vimeo] Improve password protected videos extraction (closes #15114)

---
 youtube_dl/extractor/vimeo.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index cedb548..6af7056 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -468,11 +468,12 @@ class VimeoIE(VimeoBaseInfoExtractor):
         request = sanitized_Request(url, headers=headers)
         try:
             webpage, urlh = self._download_webpage_handle(request, video_id)
+            redirect_url = compat_str(urlh.geturl())
             # Some URLs redirect to ondemand can't be extracted with
             # this extractor right away thus should be passed through
             # ondemand extractor (e.g. https://vimeo.com/73445910)
-            if VimeoOndemandIE.suitable(urlh.geturl()):
-                return self.url_result(urlh.geturl(), VimeoOndemandIE.ie_key())
+            if VimeoOndemandIE.suitable(redirect_url):
+                return self.url_result(redirect_url, VimeoOndemandIE.ie_key())
         except ExtractorError as ee:
             if isinstance(ee.cause, compat_HTTPError) and ee.cause.code == 403:
                 errmsg = ee.cause.read()
@@ -541,15 +542,15 @@ class VimeoIE(VimeoBaseInfoExtractor):
             if re.search(r'<form[^>]+?id="pw_form"', webpage) is not None:
                 if '_video_password_verified' in data:
                     raise ExtractorError('video password verification failed!')
-                self._verify_video_password(url, video_id, webpage)
+                self._verify_video_password(redirect_url, video_id, webpage)
                 return self._real_extract(
-                    smuggle_url(url, {'_video_password_verified': 'verified'}))
+                    smuggle_url(redirect_url, {'_video_password_verified': 'verified'}))
             else:
                 raise ExtractorError('Unable to extract info section',
                                      cause=e)
         else:
             if config.get('view') == 4:
-                config = self._verify_player_video_password(url, video_id)
+                config = self._verify_player_video_password(redirect_url, video_id)
 
         def is_rented():
             if '>You rented this title.<' in webpage:
-- 
cgit v1.1


From d2c5b5a951868ae974bc3af6659ab39b8abd2157 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 30 Dec 2017 05:52:35 +0700
Subject: [openload] Fallback on f-page extraction (closes #14665, closes
 #14879)

---
 youtube_dl/extractor/openload.py | 28 +++++++++++++++++++++-------
 1 file changed, 21 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/openload.py b/youtube_dl/extractor/openload.py
index d1eb3be..81c1317 100644
--- a/youtube_dl/extractor/openload.py
+++ b/youtube_dl/extractor/openload.py
@@ -285,6 +285,11 @@ class OpenloadIE(InfoExtractor):
         'url': 'https://openload.co/embed/Sxz5sADo82g/',
         'only_matching': True,
     }, {
+        # unavailable via https://openload.co/embed/e-Ixz9ZR5L0/ but available
+        # via https://openload.co/f/e-Ixz9ZR5L0/
+        'url': 'https://openload.co/f/e-Ixz9ZR5L0/',
+        'only_matching': True,
+    }, {
         'url': 'https://oload.tv/embed/KnG-kKZdcfY/',
         'only_matching': True,
     }, {
@@ -305,18 +310,27 @@ class OpenloadIE(InfoExtractor):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        url = 'https://openload.co/embed/%s/' % video_id
+        url_pattern = 'https://openload.co/%%s/%s/' % video_id
         headers = {
             'User-Agent': self._USER_AGENT,
         }
 
-        webpage = self._download_webpage(url, video_id, headers=headers)
-
-        if 'File not found' in webpage or 'deleted by the owner' in webpage:
-            raise ExtractorError('File not found', expected=True, video_id=video_id)
+        for path in ('embed', 'f'):
+            page_url = url_pattern % path
+            last = path == 'f'
+            webpage = self._download_webpage(
+                page_url, video_id, 'Downloading %s webpage' % path,
+                headers=headers, fatal=last)
+            if not webpage:
+                continue
+            if 'File not found' in webpage or 'deleted by the owner' in webpage:
+                if not last:
+                    continue
+                raise ExtractorError('File not found', expected=True, video_id=video_id)
+            break
 
         phantom = PhantomJSwrapper(self, required_version='2.0')
-        webpage, _ = phantom.get(url, html=webpage, video_id=video_id, headers=headers)
+        webpage, _ = phantom.get(page_url, html=webpage, video_id=video_id, headers=headers)
 
         decoded_id = get_element_by_id('streamurl', webpage)
 
@@ -327,7 +341,7 @@ class OpenloadIE(InfoExtractor):
             'title', default=None) or self._html_search_meta(
             'description', webpage, 'title', fatal=True)
 
-        entries = self._parse_html5_media_entries(url, webpage, video_id)
+        entries = self._parse_html5_media_entries(page_url, webpage, video_id)
         entry = entries[0] if entries else {}
         subtitles = entry.get('subtitles')
 
-- 
cgit v1.1


From 2c8e11b4af253fec9096a800ccd26b4daef521a1 Mon Sep 17 00:00:00 2001
From: 50csent <32077344+50csent@users.noreply.github.com>
Date: Fri, 29 Dec 2017 22:59:49 +0200
Subject: [pluralsight] Fix missing first line of subtitles (closes #11118)

---
 youtube_dl/extractor/pluralsight.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/pluralsight.py b/youtube_dl/extractor/pluralsight.py
index 597b112..aacc5d4 100644
--- a/youtube_dl/extractor/pluralsight.py
+++ b/youtube_dl/extractor/pluralsight.py
@@ -171,12 +171,12 @@ class PluralsightIE(PluralsightBaseIE):
         for num, current in enumerate(subs):
             current = subs[num]
             start, text = (
-                float_or_none(dict_get(current, TIME_OFFSET_KEYS)),
+                float_or_none(dict_get(current, TIME_OFFSET_KEYS, skip_false_values=False)),
                 dict_get(current, TEXT_KEYS))
             if start is None or text is None:
                 continue
             end = duration if num == len(subs) - 1 else float_or_none(
-                dict_get(subs[num + 1], TIME_OFFSET_KEYS))
+                dict_get(subs[num + 1], TIME_OFFSET_KEYS, skip_false_values=False))
             if end is None:
                 continue
             srt += os.linesep.join(
-- 
cgit v1.1


From d97cb84b313f94da76bff0520392334e753e2238 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sat, 30 Dec 2017 00:30:15 +0100
Subject: [ufctv] Add new extractor(closes #14520)

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/ufctv.py      | 55 ++++++++++++++++++++++++++++++++++++++
 2 files changed, 56 insertions(+)
 create mode 100644 youtube_dl/extractor/ufctv.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index e7b93a6..5ed0318 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1145,6 +1145,7 @@ from .udemy import (
     UdemyCourseIE
 )
 from .udn import UDNEmbedIE
+from .ufctv import UFCTVIE
 from .uktvplay import UKTVPlayIE
 from .digiteka import DigitekaIE
 from .umg import UMGDeIE
diff --git a/youtube_dl/extractor/ufctv.py b/youtube_dl/extractor/ufctv.py
new file mode 100644
index 0000000..ab82381
--- /dev/null
+++ b/youtube_dl/extractor/ufctv.py
@@ -0,0 +1,55 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    parse_duration,
+    parse_iso8601,
+)
+
+
+class UFCTVIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?ufc\.tv/video/(?P<id>[^/]+)'
+    _TEST = {
+        'url': 'https://www.ufc.tv/video/ufc-219-countdown-full-episode',
+        'info_dict': {
+            'id': '34167',
+            'ext': 'mp4',
+            'title': 'UFC 219 Countdown: Full Episode',
+            'description': 'md5:26d4e8bf4665ae5878842d7050c3c646',
+            'timestamp': 1513962360,
+            'upload_date': '20171222',
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        }
+    }
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        video_data = self._download_json(url, display_id, query={
+            'format': 'json',
+        })
+        video_id = str(video_data['id'])
+        title = video_data['name']
+        m3u8_url = self._download_json(
+            'https://www.ufc.tv/service/publishpoint', video_id, query={
+                'type': 'video',
+                'format': 'json',
+                'id': video_id,
+            }, headers={
+                'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 11_0_1 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A402 Safari/604.1',
+            })['path']
+        m3u8_url = m3u8_url.replace('_iphone.', '.')
+        formats = self._extract_m3u8_formats(m3u8_url, video_id, 'mp4')
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': video_data.get('description'),
+            'duration': parse_duration(video_data.get('runtime')),
+            'timestamp': parse_iso8601(video_data.get('releaseDate')),
+            'formats': formats,
+        }
-- 
cgit v1.1


From 2501d41ef4b9ed0349cf4f9838e12873350e60d5 Mon Sep 17 00:00:00 2001
From: felix <felix.von.s@posteo.de>
Date: Sat, 12 Nov 2016 22:15:51 +0100
Subject: [common] use AACL as the default fourcc when AudioTag is 255

---
 youtube_dl/extractor/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 35d427e..5e7e7a3 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -2054,7 +2054,7 @@ class InfoExtractor(object):
             stream_timescale = int_or_none(stream.get('TimeScale')) or timescale
             stream_name = stream.get('Name')
             for track in stream.findall('QualityLevel'):
-                fourcc = track.get('FourCC')
+                fourcc = track.get('FourCC', 'AACL' if track.get('AudioTag') == '255' else None)
                 # TODO: add support for WVC1 and WMAP
                 if fourcc not in ('H264', 'AVC1', 'AACL'):
                     self.report_warning('%s is not a supported codec' % fourcc)
-- 
cgit v1.1


From 8056c8542d008ac2f103b4fcb862dc1ad78dc8fc Mon Sep 17 00:00:00 2001
From: felix <felix.von.s@posteo.de>
Date: Sat, 12 Nov 2016 21:52:02 +0100
Subject: [mediasite] Add extractor, subsume sandia and collegerama extractors

---
 youtube_dl/extractor/collegerama.py |  93 ------------------
 youtube_dl/extractor/extractors.py  |   3 +-
 youtube_dl/extractor/generic.py     |  10 ++
 youtube_dl/extractor/mediasite.py   | 184 ++++++++++++++++++++++++++++++++++++
 youtube_dl/extractor/sandia.py      |  65 -------------
 5 files changed, 195 insertions(+), 160 deletions(-)
 delete mode 100644 youtube_dl/extractor/collegerama.py
 create mode 100644 youtube_dl/extractor/mediasite.py
 delete mode 100644 youtube_dl/extractor/sandia.py

diff --git a/youtube_dl/extractor/collegerama.py b/youtube_dl/extractor/collegerama.py
deleted file mode 100644
index 6a41db8..0000000
--- a/youtube_dl/extractor/collegerama.py
+++ /dev/null
@@ -1,93 +0,0 @@
-from __future__ import unicode_literals
-
-import json
-
-from .common import InfoExtractor
-from ..utils import (
-    float_or_none,
-    int_or_none,
-    sanitized_Request,
-)
-
-
-class CollegeRamaIE(InfoExtractor):
-    _VALID_URL = r'https?://collegerama\.tudelft\.nl/Mediasite/Play/(?P<id>[\da-f]+)'
-    _TESTS = [
-        {
-            'url': 'https://collegerama.tudelft.nl/Mediasite/Play/585a43626e544bdd97aeb71a0ec907a01d',
-            'md5': '481fda1c11f67588c0d9d8fbdced4e39',
-            'info_dict': {
-                'id': '585a43626e544bdd97aeb71a0ec907a01d',
-                'ext': 'mp4',
-                'title': 'Een nieuwe wereld: waarden, bewustzijn en techniek van de mensheid 2.0.',
-                'description': '',
-                'thumbnail': r're:^https?://.*\.jpg(?:\?.*?)?$',
-                'duration': 7713.088,
-                'timestamp': 1413309600,
-                'upload_date': '20141014',
-            },
-        },
-        {
-            'url': 'https://collegerama.tudelft.nl/Mediasite/Play/86a9ea9f53e149079fbdb4202b521ed21d?catalog=fd32fd35-6c99-466c-89d4-cd3c431bc8a4',
-            'md5': 'ef1fdded95bdf19b12c5999949419c92',
-            'info_dict': {
-                'id': '86a9ea9f53e149079fbdb4202b521ed21d',
-                'ext': 'wmv',
-                'title': '64ste Vakantiecursus: Afvalwater',
-                'description': 'md5:7fd774865cc69d972f542b157c328305',
-                'thumbnail': r're:^https?://.*\.jpg(?:\?.*?)?$',
-                'duration': 10853,
-                'timestamp': 1326446400,
-                'upload_date': '20120113',
-            },
-        },
-    ]
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-
-        player_options_request = {
-            'getPlayerOptionsRequest': {
-                'ResourceId': video_id,
-                'QueryString': '',
-            }
-        }
-
-        request = sanitized_Request(
-            'http://collegerama.tudelft.nl/Mediasite/PlayerService/PlayerService.svc/json/GetPlayerOptions',
-            json.dumps(player_options_request))
-        request.add_header('Content-Type', 'application/json')
-
-        player_options = self._download_json(request, video_id)
-
-        presentation = player_options['d']['Presentation']
-        title = presentation['Title']
-        description = presentation.get('Description')
-        thumbnail = None
-        duration = float_or_none(presentation.get('Duration'), 1000)
-        timestamp = int_or_none(presentation.get('UnixTime'), 1000)
-
-        formats = []
-        for stream in presentation['Streams']:
-            for video in stream['VideoUrls']:
-                thumbnail_url = stream.get('ThumbnailUrl')
-                if thumbnail_url:
-                    thumbnail = 'http://collegerama.tudelft.nl' + thumbnail_url
-                format_id = video['MediaType']
-                if format_id == 'SS':
-                    continue
-                formats.append({
-                    'url': video['Location'],
-                    'format_id': format_id,
-                })
-        self._sort_formats(formats)
-
-        return {
-            'id': video_id,
-            'title': title,
-            'description': description,
-            'thumbnail': thumbnail,
-            'duration': duration,
-            'timestamp': timestamp,
-            'formats': formats,
-        }
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 5ed0318..e64defe 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -205,7 +205,6 @@ from .cnn import (
     CNNArticleIE,
 )
 from .coub import CoubIE
-from .collegerama import CollegeRamaIE
 from .comedycentral import (
     ComedyCentralFullEpisodesIE,
     ComedyCentralIE,
@@ -576,6 +575,7 @@ from .massengeschmacktv import MassengeschmackTVIE
 from .matchtv import MatchTVIE
 from .mdr import MDRIE
 from .mediaset import MediasetIE
+from .mediasite import MediasiteIE
 from .medici import MediciIE
 from .megaphone import MegaphoneIE
 from .meipai import MeipaiIE
@@ -912,7 +912,6 @@ from .rutube import (
 from .rutv import RUTVIE
 from .ruutu import RuutuIE
 from .ruv import RuvIE
-from .sandia import SandiaIE
 from .safari import (
     SafariIE,
     SafariApiIE,
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index c7b6092..d5622c8 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -2883,6 +2883,16 @@ class GenericIE(InfoExtractor):
             return self.playlist_from_matches(
                 vshare_urls, video_id, video_title, ie=VShareIE.ie_key())
 
+        # Look for Mediasite embeds
+        mobj = re.search(r'''(?xi)
+                <iframe[^>]+src="((?:https?://[a-z0-9\-\.:\[\]]+)?
+                    /Mediasite/Play/[0-9a-f]{32,34}(?:\?.*?)?)"
+            ''', webpage)
+        if mobj is not None:
+            return self.url_result(smuggle_url(
+                compat_urlparse.urljoin(url, unescapeHTML(mobj.group(1))),
+                { 'UrlReferrer': url }), 'Livestream')
+
         def merge_dicts(dict1, dict2):
             merged = {}
             for k, v in dict1.items():
diff --git a/youtube_dl/extractor/mediasite.py b/youtube_dl/extractor/mediasite.py
new file mode 100644
index 0000000..5d28168
--- /dev/null
+++ b/youtube_dl/extractor/mediasite.py
@@ -0,0 +1,184 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+import json
+
+from .common import InfoExtractor
+from ..compat import compat_urlparse
+from ..utils import (
+    ExtractorError,
+    unsmuggle_url,
+    mimetype2ext,
+    float_or_none,
+)
+
+
+class MediasiteIE(InfoExtractor):
+    _VALID_URL = r'''(?xi)
+        https?://[a-z0-9\-\.:\[\]]+/Mediasite/Play/
+        (?P<id>[0-9a-f]{32,34})
+        (?P<QueryString>\?[^#]+|)
+    '''
+    _TESTS = [
+        {
+            'url': 'https://hitsmediaweb.h-its.org/mediasite/Play/2db6c271681e4f199af3c60d1f82869b1d',
+            'info_dict': {
+                'id': '2db6c271681e4f199af3c60d1f82869b1d',
+                'ext': 'mp4',
+                'title': 'Lecture: Tuesday, September 20, 2016 - Sir Andrew Wiles',
+                'description': 'Sir Andrew Wiles: “Equations in arithmetic”\\n\\nI will describe some of the interactions between modern number theory and the problem of solving equations in rational numbers or integers\\u0027.',
+                'timestamp': 1474268400.0,
+                'upload_date': '20160919',
+            },
+        },
+        {
+            'url': 'http://mediasite.uib.no/Mediasite/Play/90bb363295d945d6b548c867d01181361d?catalog=a452b7df-9ae1-46b7-a3ba-aceeb285f3eb',
+            'info_dict': {
+                'id': '90bb363295d945d6b548c867d01181361d',
+                'ext': 'mp4',
+                'upload_date': '20150429',
+                'title': '5) IT-forum 2015-Dag 1  - Dungbeetle -  How and why Rain created a tiny bug tracker for Unity',
+                'timestamp': 1430311380.0,
+            },
+        },
+        {
+            'url': 'https://collegerama.tudelft.nl/Mediasite/Play/585a43626e544bdd97aeb71a0ec907a01d',
+            'md5': '481fda1c11f67588c0d9d8fbdced4e39',
+            'info_dict': {
+                'id': '585a43626e544bdd97aeb71a0ec907a01d',
+                'ext': 'mp4',
+                'title': 'Een nieuwe wereld: waarden, bewustzijn en techniek van de mensheid 2.0.',
+                'description': '',
+                'thumbnail': r're:^https?://.*\.jpg(?:\?.*)?$',
+                'duration': 7713.088,
+                'timestamp': 1413309600,
+                'upload_date': '20141014',
+            },
+        },
+        {
+            'url': 'https://collegerama.tudelft.nl/Mediasite/Play/86a9ea9f53e149079fbdb4202b521ed21d?catalog=fd32fd35-6c99-466c-89d4-cd3c431bc8a4',
+            'md5': 'ef1fdded95bdf19b12c5999949419c92',
+            'info_dict': {
+                'id': '86a9ea9f53e149079fbdb4202b521ed21d',
+                'ext': 'wmv',
+                'title': '64ste Vakantiecursus: Afvalwater',
+                'description': 'md5:7fd774865cc69d972f542b157c328305',
+                'thumbnail': r're:^https?://.*\.jpg(?:\?.*?)?$',
+                'duration': 10853,
+                'timestamp': 1326446400,
+                'upload_date': '20120113',
+            },
+        },
+        {
+            'url': 'http://digitalops.sandia.gov/Mediasite/Play/24aace4429fc450fb5b38cdbf424a66e1d',
+            'md5': '9422edc9b9a60151727e4b6d8bef393d',
+            'info_dict': {
+                'id': '24aace4429fc450fb5b38cdbf424a66e1d',
+                'ext': 'mp4',
+                'title': 'Xyce Software Training - Section 1',
+                'description': r're:(?s)SAND Number: SAND 2013-7800.{200,}',
+                'upload_date': '20120409',
+                'timestamp': 1333983600,
+                'duration': 7794,
+            }
+        }
+    ]
+
+    # look in Mediasite.Core.js (Mediasite.ContentStreamType[*])
+    _STREAM_TYPES = {
+        0: 'video1', # the main video
+        2: 'slide',
+        3: 'presentation',
+        4: 'video2', # screencast?
+        5: 'video3',
+    }
+
+    def _real_extract(self, url):
+        url, data = unsmuggle_url(url, {})
+        mobj = re.match(self._VALID_URL, url)
+        ResourceId = mobj.group('id')
+        QueryString = mobj.group('QueryString')
+
+        webpage = self._download_webpage(url, ResourceId) # XXX: add UrlReferrer?
+
+        # XXX: might have also extracted UrlReferrer and QueryString from the html
+        ServicePath = compat_urlparse.urljoin(url, self._html_search_regex(
+            r'<div id="ServicePath">(.+?)</div>', webpage, ResourceId,
+            default='/Mediasite/PlayerService/PlayerService.svc/json'))
+
+        PlayerOptions = self._download_json(
+            '%s/GetPlayerOptions' % (ServicePath), ResourceId,
+            headers={
+                'Content-type': 'application/json; charset=utf-8',
+                'X-Requested-With': 'XMLHttpRequest',
+            },
+            data=json.dumps({
+                'getPlayerOptionsRequest': {
+                    'ResourceId': ResourceId,
+                    'QueryString': QueryString,
+                    'UrlReferrer': data.get('UrlReferrer', ''),
+                    'UseScreenReader': False,
+                }
+            }).encode('utf-8'))
+        Presentation = PlayerOptions['d']['Presentation']
+        if Presentation is None:
+            raise ExtractorError('Mediasite says: %s' %
+                (PlayerOptions['d']['PlayerPresentationStatusMessage'],),
+                expected=True)
+
+        thumbnails = []
+        formats = []
+        for snum, Stream in enumerate(Presentation['Streams']):
+            stream_type = self._STREAM_TYPES.get(
+                Stream['StreamType'], 'type%u' % Stream['StreamType'])
+
+            stream_formats = []
+            for unum, VideoUrl in enumerate(Stream['VideoUrls']):
+                url = VideoUrl['Location']
+                # XXX: if Stream.get('CanChangeScheme', False), switch scheme to HTTP/HTTPS
+
+                if VideoUrl['MediaType'] == 'SS':
+                    stream_formats.extend(self._extract_ism_formats(
+                        url, ResourceId, ism_id='%s-%u.%u' % (stream_type, snum, unum)))
+                    continue
+
+                stream_formats.append({
+                    'format_id': '%s-%u.%u' % (stream_type, snum, unum),
+                    'url': url,
+                    'ext': mimetype2ext(VideoUrl['MimeType']),
+                })
+
+            # TODO: if Stream['HasSlideContent']:
+            # synthesise an MJPEG video stream '%s-%u.slides' % (stream_type, snum)
+            # from Stream['Slides']
+            # this will require writing a custom downloader...
+
+            # disprefer 'secondary' streams
+            if Stream['StreamType'] != 0:
+                for fmt in stream_formats:
+                    fmt['preference'] = -1
+
+            ThumbnailUrl = Stream.get('ThumbnailUrl')
+            if ThumbnailUrl:
+                thumbnails.append({
+                    'id': '%s-%u' % (stream_type, snum),
+                    'url': compat_urlparse.urljoin(url, ThumbnailUrl),
+                    'preference': -1 if Stream['StreamType'] != 0 else 0,
+                })
+            formats.extend(stream_formats)
+
+        self._sort_formats(formats)
+
+        # XXX: Presentation['Presenters']
+        # XXX: Presentation['Transcript']
+
+        return {
+            'id': ResourceId,
+            'title': Presentation['Title'],
+            'description': Presentation.get('Description'),
+            'duration': float_or_none(Presentation.get('Duration'), 1000),
+            'timestamp': float_or_none(Presentation.get('UnixTime'), 1000),
+            'formats': formats,
+            'thumbnails': thumbnails,
+        }
diff --git a/youtube_dl/extractor/sandia.py b/youtube_dl/extractor/sandia.py
deleted file mode 100644
index 96e43af..0000000
--- a/youtube_dl/extractor/sandia.py
+++ /dev/null
@@ -1,65 +0,0 @@
-# coding: utf-8
-from __future__ import unicode_literals
-
-import json
-
-from .common import InfoExtractor
-from ..utils import (
-    int_or_none,
-    mimetype2ext,
-)
-
-
-class SandiaIE(InfoExtractor):
-    IE_DESC = 'Sandia National Laboratories'
-    _VALID_URL = r'https?://digitalops\.sandia\.gov/Mediasite/Play/(?P<id>[0-9a-f]+)'
-    _TEST = {
-        'url': 'http://digitalops.sandia.gov/Mediasite/Play/24aace4429fc450fb5b38cdbf424a66e1d',
-        'md5': '9422edc9b9a60151727e4b6d8bef393d',
-        'info_dict': {
-            'id': '24aace4429fc450fb5b38cdbf424a66e1d',
-            'ext': 'mp4',
-            'title': 'Xyce Software Training - Section 1',
-            'description': 're:(?s)SAND Number: SAND 2013-7800.{200,}',
-            'upload_date': '20120409',
-            'timestamp': 1333983600,
-            'duration': 7794,
-        }
-    }
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-
-        presentation_data = self._download_json(
-            'http://digitalops.sandia.gov/Mediasite/PlayerService/PlayerService.svc/json/GetPlayerOptions',
-            video_id, data=json.dumps({
-                'getPlayerOptionsRequest': {
-                    'ResourceId': video_id,
-                    'QueryString': '',
-                }
-            }), headers={
-                'Content-Type': 'application/json; charset=utf-8',
-            })['d']['Presentation']
-
-        title = presentation_data['Title']
-
-        formats = []
-        for stream in presentation_data.get('Streams', []):
-            for fd in stream.get('VideoUrls', []):
-                formats.append({
-                    'format_id': fd['MediaType'],
-                    'format_note': fd['MimeType'].partition('/')[2],
-                    'ext': mimetype2ext(fd['MimeType']),
-                    'url': fd['Location'],
-                    'protocol': 'f4m' if fd['MimeType'] == 'video/x-mp4-fragmented' else None,
-                })
-        self._sort_formats(formats)
-
-        return {
-            'id': video_id,
-            'title': title,
-            'description': presentation_data.get('Description'),
-            'formats': formats,
-            'timestamp': int_or_none(presentation_data.get('UnixTime'), 1000),
-            'duration': int_or_none(presentation_data.get('Duration'), 1000),
-        }
-- 
cgit v1.1


From 2ca7ed41fed73cf37581b07d0c67d3bad8a6acc3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 30 Dec 2017 07:28:18 +0700
Subject: [mediasite] Improve extraction and code style, add support for DASH
 (closes #11185, closes #14343, refs #5428)

---
 youtube_dl/extractor/generic.py   |  29 ++++++---
 youtube_dl/extractor/mediasite.py | 128 +++++++++++++++++++++++---------------
 2 files changed, 100 insertions(+), 57 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index d5622c8..cc4c90b 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -100,6 +100,7 @@ from .megaphone import MegaphoneIE
 from .vzaar import VzaarIE
 from .channel9 import Channel9IE
 from .vshare import VShareIE
+from .mediasite import MediasiteIE
 
 
 class GenericIE(InfoExtractor):
@@ -1925,6 +1926,18 @@ class GenericIE(InfoExtractor):
                 'title': 'vl14062007715967',
                 'ext': 'mp4',
             }
+        },
+        {
+            'url': 'http://www.heidelberg-laureate-forum.org/blog/video/lecture-friday-september-23-2016-sir-c-antony-r-hoare/',
+            'md5': 'aecd089f55b1cb5a59032cb049d3a356',
+            'info_dict': {
+                'id': '90227f51a80c4d8f86c345a7fa62bd9a1d',
+                'ext': 'mp4',
+                'title': 'Lecture: Friday, September 23, 2016 - Sir Tony Hoare',
+                'description': 'md5:5a51db84a62def7b7054df2ade403c6c',
+                'timestamp': 1474354800,
+                'upload_date': '20160920',
+            }
         }
         # {
         #     # TODO: find another test
@@ -2884,14 +2897,14 @@ class GenericIE(InfoExtractor):
                 vshare_urls, video_id, video_title, ie=VShareIE.ie_key())
 
         # Look for Mediasite embeds
-        mobj = re.search(r'''(?xi)
-                <iframe[^>]+src="((?:https?://[a-z0-9\-\.:\[\]]+)?
-                    /Mediasite/Play/[0-9a-f]{32,34}(?:\?.*?)?)"
-            ''', webpage)
-        if mobj is not None:
-            return self.url_result(smuggle_url(
-                compat_urlparse.urljoin(url, unescapeHTML(mobj.group(1))),
-                { 'UrlReferrer': url }), 'Livestream')
+        mediasite_urls = MediasiteIE._extract_urls(webpage)
+        if mediasite_urls:
+            entries = [
+                self.url_result(smuggle_url(
+                    compat_urlparse.urljoin(url, mediasite_url),
+                    {'UrlReferrer': url}), ie=MediasiteIE.ie_key())
+                for mediasite_url in mediasite_urls]
+            return self.playlist_result(entries, video_id, video_title)
 
         def merge_dicts(dict1, dict2):
             merged = {}
diff --git a/youtube_dl/extractor/mediasite.py b/youtube_dl/extractor/mediasite.py
index 5d28168..0e2645c 100644
--- a/youtube_dl/extractor/mediasite.py
+++ b/youtube_dl/extractor/mediasite.py
@@ -5,21 +5,22 @@ import re
 import json
 
 from .common import InfoExtractor
-from ..compat import compat_urlparse
+from ..compat import (
+    compat_str,
+    compat_urlparse,
+)
 from ..utils import (
     ExtractorError,
-    unsmuggle_url,
-    mimetype2ext,
     float_or_none,
+    mimetype2ext,
+    unescapeHTML,
+    unsmuggle_url,
+    urljoin,
 )
 
 
 class MediasiteIE(InfoExtractor):
-    _VALID_URL = r'''(?xi)
-        https?://[a-z0-9\-\.:\[\]]+/Mediasite/Play/
-        (?P<id>[0-9a-f]{32,34})
-        (?P<QueryString>\?[^#]+|)
-    '''
+    _VALID_URL = r'(?xi)https?://[^/]+/Mediasite/Play/(?P<id>[0-9a-f]{32,34})(?P<query>\?[^#]+|)'
     _TESTS = [
         {
             'url': 'https://hitsmediaweb.h-its.org/mediasite/Play/2db6c271681e4f199af3c60d1f82869b1d',
@@ -87,67 +88,96 @@ class MediasiteIE(InfoExtractor):
 
     # look in Mediasite.Core.js (Mediasite.ContentStreamType[*])
     _STREAM_TYPES = {
-        0: 'video1', # the main video
+        0: 'video1',  # the main video
         2: 'slide',
         3: 'presentation',
-        4: 'video2', # screencast?
+        4: 'video2',  # screencast?
         5: 'video3',
     }
 
+    @staticmethod
+    def _extract_urls(webpage):
+        return [
+            unescapeHTML(mobj.group('url'))
+            for mobj in re.finditer(
+                r'(?xi)<iframe\b[^>]+\bsrc=(["\'])(?P<url>(?:(?:https?:)?//[^/]+)?/Mediasite/Play/[0-9a-f]{32,34}(?:\?.*?)?)\1',
+                webpage)]
+
     def _real_extract(self, url):
         url, data = unsmuggle_url(url, {})
         mobj = re.match(self._VALID_URL, url)
-        ResourceId = mobj.group('id')
-        QueryString = mobj.group('QueryString')
+        resource_id = mobj.group('id')
+        query = mobj.group('query')
 
-        webpage = self._download_webpage(url, ResourceId) # XXX: add UrlReferrer?
+        webpage, urlh = self._download_webpage_handle(url, resource_id)  # XXX: add UrlReferrer?
+        redirect_url = compat_str(urlh.geturl())
 
         # XXX: might have also extracted UrlReferrer and QueryString from the html
-        ServicePath = compat_urlparse.urljoin(url, self._html_search_regex(
-            r'<div id="ServicePath">(.+?)</div>', webpage, ResourceId,
+        service_path = compat_urlparse.urljoin(redirect_url, self._html_search_regex(
+            r'<div[^>]+\bid=["\']ServicePath[^>]+>(.+?)</div>', webpage, resource_id,
             default='/Mediasite/PlayerService/PlayerService.svc/json'))
 
-        PlayerOptions = self._download_json(
-            '%s/GetPlayerOptions' % (ServicePath), ResourceId,
+        player_options = self._download_json(
+            '%s/GetPlayerOptions' % service_path, resource_id,
             headers={
                 'Content-type': 'application/json; charset=utf-8',
                 'X-Requested-With': 'XMLHttpRequest',
             },
             data=json.dumps({
                 'getPlayerOptionsRequest': {
-                    'ResourceId': ResourceId,
-                    'QueryString': QueryString,
+                    'ResourceId': resource_id,
+                    'QueryString': query,
                     'UrlReferrer': data.get('UrlReferrer', ''),
                     'UseScreenReader': False,
                 }
-            }).encode('utf-8'))
-        Presentation = PlayerOptions['d']['Presentation']
-        if Presentation is None:
-            raise ExtractorError('Mediasite says: %s' %
-                (PlayerOptions['d']['PlayerPresentationStatusMessage'],),
+            }).encode('utf-8'))['d']
+
+        presentation = player_options['Presentation']
+        title = presentation['Title']
+
+        if presentation is None:
+            raise ExtractorError(
+                'Mediasite says: %s' % player_options['PlayerPresentationStatusMessage'],
                 expected=True)
 
         thumbnails = []
         formats = []
-        for snum, Stream in enumerate(Presentation['Streams']):
-            stream_type = self._STREAM_TYPES.get(
-                Stream['StreamType'], 'type%u' % Stream['StreamType'])
+        for snum, Stream in enumerate(presentation['Streams']):
+            stream_type = Stream.get('StreamType')
+            if stream_type is None:
+                continue
+
+            video_urls = Stream.get('VideoUrls')
+            if not isinstance(video_urls, list):
+                video_urls = []
+
+            stream_id = self._STREAM_TYPES.get(
+                stream_type, 'type%u' % stream_type)
 
             stream_formats = []
-            for unum, VideoUrl in enumerate(Stream['VideoUrls']):
-                url = VideoUrl['Location']
+            for unum, VideoUrl in enumerate(video_urls):
+                video_url = VideoUrl.get('Location')
+                if not video_url or not isinstance(video_url, compat_str):
+                    continue
                 # XXX: if Stream.get('CanChangeScheme', False), switch scheme to HTTP/HTTPS
 
-                if VideoUrl['MediaType'] == 'SS':
+                media_type = VideoUrl.get('MediaType')
+                if media_type == 'SS':
                     stream_formats.extend(self._extract_ism_formats(
-                        url, ResourceId, ism_id='%s-%u.%u' % (stream_type, snum, unum)))
-                    continue
-
-                stream_formats.append({
-                    'format_id': '%s-%u.%u' % (stream_type, snum, unum),
-                    'url': url,
-                    'ext': mimetype2ext(VideoUrl['MimeType']),
-                })
+                        video_url, resource_id,
+                        ism_id='%s-%u.%u' % (stream_id, snum, unum),
+                        fatal=False))
+                elif media_type == 'Dash':
+                    stream_formats.extend(self._extract_mpd_formats(
+                        video_url, resource_id,
+                        mpd_id='%s-%u.%u' % (stream_id, snum, unum),
+                        fatal=False))
+                else:
+                    stream_formats.append({
+                        'format_id': '%s-%u.%u' % (stream_id, snum, unum),
+                        'url': video_url,
+                        'ext': mimetype2ext(VideoUrl.get('MimeType')),
+                    })
 
             # TODO: if Stream['HasSlideContent']:
             # synthesise an MJPEG video stream '%s-%u.slides' % (stream_type, snum)
@@ -155,16 +185,16 @@ class MediasiteIE(InfoExtractor):
             # this will require writing a custom downloader...
 
             # disprefer 'secondary' streams
-            if Stream['StreamType'] != 0:
+            if stream_type != 0:
                 for fmt in stream_formats:
                     fmt['preference'] = -1
 
-            ThumbnailUrl = Stream.get('ThumbnailUrl')
-            if ThumbnailUrl:
+            thumbnail_url = Stream.get('ThumbnailUrl')
+            if thumbnail_url:
                 thumbnails.append({
-                    'id': '%s-%u' % (stream_type, snum),
-                    'url': compat_urlparse.urljoin(url, ThumbnailUrl),
-                    'preference': -1 if Stream['StreamType'] != 0 else 0,
+                    'id': '%s-%u' % (stream_id, snum),
+                    'url': urljoin(redirect_url, thumbnail_url),
+                    'preference': -1 if stream_type != 0 else 0,
                 })
             formats.extend(stream_formats)
 
@@ -174,11 +204,11 @@ class MediasiteIE(InfoExtractor):
         # XXX: Presentation['Transcript']
 
         return {
-            'id': ResourceId,
-            'title': Presentation['Title'],
-            'description': Presentation.get('Description'),
-            'duration': float_or_none(Presentation.get('Duration'), 1000),
-            'timestamp': float_or_none(Presentation.get('UnixTime'), 1000),
+            'id': resource_id,
+            'title': title,
+            'description': presentation.get('Description'),
+            'duration': float_or_none(presentation.get('Duration'), 1000),
+            'timestamp': float_or_none(presentation.get('UnixTime'), 1000),
             'formats': formats,
             'thumbnails': thumbnails,
         }
-- 
cgit v1.1


From 620ee8712e264c71b444748c86a51b258818b4a1 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 30 Dec 2017 15:03:13 +0800
Subject: [openload] Fix extraction (closes #15118)

---
 ChangeLog                        | 6 ++++++
 youtube_dl/extractor/openload.py | 3 ++-
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/ChangeLog b/ChangeLog
index 4323cfc..08aed1b 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,9 @@
+version <unreleased>
+
+Extractors
+* [openload] Fix extraction (#15118)
+
+
 version 2017.12.28
 
 Extractors
diff --git a/youtube_dl/extractor/openload.py b/youtube_dl/extractor/openload.py
index 81c1317..b282bcf 100644
--- a/youtube_dl/extractor/openload.py
+++ b/youtube_dl/extractor/openload.py
@@ -332,7 +332,8 @@ class OpenloadIE(InfoExtractor):
         phantom = PhantomJSwrapper(self, required_version='2.0')
         webpage, _ = phantom.get(page_url, html=webpage, video_id=video_id, headers=headers)
 
-        decoded_id = get_element_by_id('streamurl', webpage)
+        decoded_id = (get_element_by_id('streamurl', webpage) or
+                      get_element_by_id('streamuri', webpage))
 
         video_url = 'https://openload.co/stream/%s?mime=true' % decoded_id
 
-- 
cgit v1.1


From 538d4f8681535507838d19942d2581478ebdc28b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 31 Dec 2017 01:15:35 +0700
Subject: [downloader/hls] Use HTTP headers for key request

---
 youtube_dl/downloader/hls.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/downloader/hls.py b/youtube_dl/downloader/hls.py
index 1a6e226..4dc3ab4 100644
--- a/youtube_dl/downloader/hls.py
+++ b/youtube_dl/downloader/hls.py
@@ -163,7 +163,8 @@ class HlsFD(FragmentFD):
                         return False
                     if decrypt_info['METHOD'] == 'AES-128':
                         iv = decrypt_info.get('IV') or compat_struct_pack('>8xq', media_sequence)
-                        decrypt_info['KEY'] = decrypt_info.get('KEY') or self.ydl.urlopen(decrypt_info['URI']).read()
+                        decrypt_info['KEY'] = decrypt_info.get('KEY') or self.ydl.urlopen(
+                            self._prepare_url(info_dict, decrypt_info['URI'])).read()
                         frag_content = AES.new(
                             decrypt_info['KEY'], AES.MODE_CBC, iv).decrypt(frag_content)
                     self._append_fragment(ctx, frag_content)
-- 
cgit v1.1


From 2e65e7db9e39f22a8d20ea950906ee39fbf2618c Mon Sep 17 00:00:00 2001
From: d2au <d2au.dev@gmail.com>
Date: Sat, 18 Nov 2017 05:28:58 +1030
Subject: [abc:iview] Fix extraction (closes #14711)

ABC dropped unmetering, so change to metered hls urls which
require auth.
---
 youtube_dl/extractor/abc.py | 28 +++++++++++++++++++++++-----
 1 file changed, 23 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/abc.py b/youtube_dl/extractor/abc.py
index 60f753b..9610d0c 100644
--- a/youtube_dl/extractor/abc.py
+++ b/youtube_dl/extractor/abc.py
@@ -1,6 +1,9 @@
 from __future__ import unicode_literals
 
+import hashlib
+import hmac
 import re
+import time
 
 from .common import InfoExtractor
 from ..compat import compat_str
@@ -126,20 +129,35 @@ class ABCIViewIE(InfoExtractor):
         title = video_params.get('title') or video_params['seriesTitle']
         stream = next(s for s in video_params['playlist'] if s.get('type') == 'program')
 
-        format_urls = [
-            try_get(stream, lambda x: x['hds-unmetered'], compat_str)]
+        key = 'android.content.res.Resources'.encode('utf-8')
+        time_str = str(int(time.time()))
+        house_number = video_params.get('episodeHouseNumber')
+        path = '/auth/hls/sign?ts={0}&hn={1}&d=android-mobile'.format(
+            time_str, house_number)
+        sig = hmac.new(key, path.encode('utf-8'), hashlib.sha256).hexdigest()
+        auth_url = 'http://iview.abc.net.au{0}&sig={1}'.format(path, sig)
+        token = self._download_webpage(auth_url, video_id)
+
+        format_urls = []
+
+        def tokenize_url(url, token):
+            return ''.join([url, '?hdnea=', token])
 
         # May have higher quality video
         sd_url = try_get(
-            stream, lambda x: x['streams']['hds']['sd'], compat_str)
+            stream, lambda x: x['streams']['hls']['sd'], compat_str)
         if sd_url:
-            format_urls.append(sd_url.replace('metered', 'um'))
+            format_urls.append(tokenize_url(sd_url, token))
+        else:
+            sd_low_url = try_get(
+                stream, lambda x: x['streams']['hls']['sd-low'], compat_str)
+            format_urls.append(tokenize_url(sd_low_url, token))
 
         formats = []
         for format_url in format_urls:
             if format_url:
                 formats.extend(
-                    self._extract_akamai_formats(format_url, video_id))
+                    self._extract_m3u8_formats(format_url, video_id, 'mp4'))
         self._sort_formats(formats)
 
         subtitles = {}
-- 
cgit v1.1


From 77341dae1418fa1442bdc7689e428450ff83e5d9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 31 Dec 2017 01:22:15 +0700
Subject: [abc:iview] Improve extraction and bypass geo restriction (closes
 #14782)

---
 youtube_dl/extractor/abc.py | 65 ++++++++++++++++++++++-----------------------
 1 file changed, 32 insertions(+), 33 deletions(-)

diff --git a/youtube_dl/extractor/abc.py b/youtube_dl/extractor/abc.py
index 9610d0c..87017ed 100644
--- a/youtube_dl/extractor/abc.py
+++ b/youtube_dl/extractor/abc.py
@@ -13,6 +13,7 @@ from ..utils import (
     int_or_none,
     parse_iso8601,
     try_get,
+    update_url_query,
 )
 
 
@@ -104,21 +105,24 @@ class ABCIE(InfoExtractor):
 class ABCIViewIE(InfoExtractor):
     IE_NAME = 'abc.net.au:iview'
     _VALID_URL = r'https?://iview\.abc\.net\.au/programs/[^/]+/(?P<id>[^/?#]+)'
+    _GEO_COUNTRIES = ['AU']
 
     # ABC iview programs are normally available for 14 days only.
     _TESTS = [{
-        'url': 'http://iview.abc.net.au/programs/diaries-of-a-broken-mind/ZX9735A001S00',
+        'url': 'http://iview.abc.net.au/programs/call-the-midwife/ZW0898A003S00',
         'md5': 'cde42d728b3b7c2b32b1b94b4a548afc',
         'info_dict': {
-            'id': 'ZX9735A001S00',
+            'id': 'ZW0898A003S00',
             'ext': 'mp4',
-            'title': 'Diaries Of A Broken Mind',
-            'description': 'md5:7de3903874b7a1be279fe6b68718fc9e',
-            'upload_date': '20161010',
-            'uploader_id': 'abc2',
-            'timestamp': 1476064920,
+            'title': 'Series 5 Ep 3',
+            'description': 'md5:e0ef7d4f92055b86c4f33611f180ed79',
+            'upload_date': '20171228',
+            'uploader_id': 'abc1',
+            'timestamp': 1514499187,
+        },
+        'params': {
+            'skip_download': True,
         },
-        'skip': 'Video gone',
     }]
 
     def _real_extract(self, url):
@@ -129,35 +133,30 @@ class ABCIViewIE(InfoExtractor):
         title = video_params.get('title') or video_params['seriesTitle']
         stream = next(s for s in video_params['playlist'] if s.get('type') == 'program')
 
-        key = 'android.content.res.Resources'.encode('utf-8')
-        time_str = str(int(time.time()))
         house_number = video_params.get('episodeHouseNumber')
         path = '/auth/hls/sign?ts={0}&hn={1}&d=android-mobile'.format(
-            time_str, house_number)
-        sig = hmac.new(key, path.encode('utf-8'), hashlib.sha256).hexdigest()
-        auth_url = 'http://iview.abc.net.au{0}&sig={1}'.format(path, sig)
-        token = self._download_webpage(auth_url, video_id)
-
-        format_urls = []
+            int(time.time()), house_number)
+        sig = hmac.new(
+            'android.content.res.Resources'.encode('utf-8'),
+            path.encode('utf-8'), hashlib.sha256).hexdigest()
+        token = self._download_webpage(
+            'http://iview.abc.net.au{0}&sig={1}'.format(path, sig), video_id)
 
         def tokenize_url(url, token):
-            return ''.join([url, '?hdnea=', token])
-
-        # May have higher quality video
-        sd_url = try_get(
-            stream, lambda x: x['streams']['hls']['sd'], compat_str)
-        if sd_url:
-            format_urls.append(tokenize_url(sd_url, token))
-        else:
-            sd_low_url = try_get(
-                stream, lambda x: x['streams']['hls']['sd-low'], compat_str)
-            format_urls.append(tokenize_url(sd_low_url, token))
-
-        formats = []
-        for format_url in format_urls:
-            if format_url:
-                formats.extend(
-                    self._extract_m3u8_formats(format_url, video_id, 'mp4'))
+            return update_url_query(url, {
+                'hdnea': token,
+            })
+
+        for sd in ('sd', 'sd-low'):
+            sd_url = try_get(
+                stream, lambda x: x['streams']['hls'][sd], compat_str)
+            if not sd_url:
+                continue
+            formats = self._extract_m3u8_formats(
+                tokenize_url(sd_url, token), video_id, 'mp4',
+                entry_protocol='m3u8_native', m3u8_id='hls', fatal=False)
+            if formats:
+                break
         self._sort_formats(formats)
 
         subtitles = {}
-- 
cgit v1.1


From 4f5cf319772b693c7781de47dbc5451c3c18a81c Mon Sep 17 00:00:00 2001
From: Windom <windom@users.noreply.github.com>
Date: Sat, 30 Dec 2017 20:41:07 +0200
Subject: [slutload] Add support for mobile URLs

---
 youtube_dl/extractor/slutload.py | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/slutload.py b/youtube_dl/extractor/slutload.py
index 7145d28..6fc2ff6 100644
--- a/youtube_dl/extractor/slutload.py
+++ b/youtube_dl/extractor/slutload.py
@@ -1,11 +1,13 @@
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 
 
 class SlutloadIE(InfoExtractor):
     _VALID_URL = r'^https?://(?:\w+\.)?slutload\.com/video/[^/]+/(?P<id>[^/]+)/?$'
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.slutload.com/video/virginie-baisee-en-cam/TD73btpBqSxc/',
         'md5': '868309628ba00fd488cf516a113fd717',
         'info_dict': {
@@ -15,11 +17,17 @@ class SlutloadIE(InfoExtractor):
             'age_limit': 18,
             'thumbnail': r're:https?://.*?\.jpg'
         }
-    }
+    }, {
+        # mobile site
+        'url': 'http://mobile.slutload.com/video/masturbation-solo/fviFLmc6kzJ/',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
+
+        desktop_url = re.sub(r'^(https?://)mobile\.', r'\1', url)
+        webpage = self._download_webpage(desktop_url, video_id)
 
         video_title = self._html_search_regex(r'<h1><strong>([^<]+)</strong>',
                                               webpage, 'title').strip()
-- 
cgit v1.1


From 126f225bcffbdc55725fe2272daf22489abacf54 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ond=C5=99ej=20Caletka?= <ondrej@caletka.cz>
Date: Sat, 30 Dec 2017 22:02:46 +0100
Subject: [extractor/common] Add container meta field for formats extracted in
 _parse_mpd_formats

---
 youtube_dl/extractor/common.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 5e7e7a3..5b6a09c 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1880,6 +1880,7 @@ class InfoExtractor(object):
                             'language': lang if lang not in ('mul', 'und', 'zxx', 'mis') else None,
                             'format_note': 'DASH %s' % content_type,
                             'filesize': filesize,
+                            'container': mimetype2ext(mime_type) + '_dash',
                         }
                         f.update(parse_codecs(representation_attrib.get('codecs')))
                         representation_ms_info = extract_multisegment_info(representation, adaption_set_ms_info)
-- 
cgit v1.1


From c95c08a856af2499729e2141f5b683b266324fd0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 31 Dec 2017 04:28:01 +0700
Subject: [ChangeLog] Actualize

---
 ChangeLog | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 08aed1b..eff1fe8 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,7 +1,28 @@
 version <unreleased>
 
+Core
++ [extractor/common] Add container meta field for formats extracted
+  in _parse_mpd_formats (#13616)
++ [downloader/hls] Use HTTP headers for key request
+* [common] Use AACL as the default fourcc when AudioTag is 255
+* [extractor/common] Fix extraction of DASH formats with the same
+  representation id (#15111)
+
 Extractors
++ [slutload] Add support for mobile URLs (#14806)
+* [abc:iview] Bypass geo restriction
+* [abc:iview] Fix extraction (#14711, #14782, #14838, #14917, #14963, #14985,
+  #15035, #15057, #15061, #15071, #15095, #15106)
 * [openload] Fix extraction (#15118)
+- [sandia] Remove extractor
+- [collegerama] Remove extractor
++ [mediasite] Add support for sites based on Mediasite Video Platform (#5428,
+  #11185, #14343)
++ [ufctv] Add support for ufc.tv (#14520)
+* [pluralsight] Fix missing first line of subtitles (#11118)
+* [openload] Fallback on f-page extraction (#14665, #14879)
+* [vimeo] Improve password protected videos extraction (#15114)
+* [aws] Fix canonical/signed headers generation on python 2 (#15102)
 
 
 version 2017.12.28
-- 
cgit v1.1


From 04cf1a191a6b47bac93364ed8eb237a25058ee5e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 31 Dec 2017 04:30:49 +0700
Subject: release 2017.12.31

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 docs/supportedsites.md    | 4 ++--
 youtube_dl/version.py     | 2 +-
 4 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index f40927d..3f89849 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.12.28*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.12.28**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.12.31*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.12.31**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through the [README](https://github.com/rg3/youtube-dl/blob/master/README.md), **most notably** the [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ Add the `-v` flag to **your command line** you run youtube-dl with (`youtube-dl
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.12.28
+[debug] youtube-dl version 2017.12.31
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index eff1fe8..bfffb1f 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2017.12.31
 
 Core
 + [extractor/common] Add container meta field for formats extracted
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 6b935e5..75bd5c9 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -171,7 +171,6 @@
  - **CNN**
  - **CNNArticle**
  - **CNNBlogs**
- - **CollegeRama**
  - **ComCarCoff**
  - **ComedyCentral**
  - **ComedyCentralFullEpisodes**
@@ -449,6 +448,7 @@
  - **media.ccc.de**
  - **Medialaan**
  - **Mediaset**
+ - **Mediasite**
  - **Medici**
  - **megaphone.fm**: megaphone.fm embedded players
  - **Meipai**: 美拍
@@ -717,7 +717,6 @@
  - **safari**: safaribooksonline.com online video
  - **safari:api**
  - **safari:course**: safaribooksonline.com online courses
- - **Sandia**: Sandia National Laboratories
  - **Sapo**: SAPO Vídeos
  - **savefrom.net**
  - **SBS**: sbs.com.au
@@ -892,6 +891,7 @@
  - **udemy**
  - **udemy:course**
  - **UDNEmbed**: 聯合影音
+ - **UFCTV**
  - **UKTVPlay**
  - **umg:de**: Universal Music Deutschland
  - **Unistra**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index f4a1b67..a3f84b9 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.12.28'
+__version__ = '2017.12.31'
-- 
cgit v1.1


From 6648fd8ad6e581354f46c840465cff4c92d2c6f3 Mon Sep 17 00:00:00 2001
From: sprhawk <465558+sprhawk@users.noreply.github.com>
Date: Mon, 1 Jan 2018 18:33:14 +0800
Subject: changed to use .get to get field from json object

---
 youtube_dl/extractor/weibo.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/weibo.py b/youtube_dl/extractor/weibo.py
index 34809bd..cbe0c32 100644
--- a/youtube_dl/extractor/weibo.py
+++ b/youtube_dl/extractor/weibo.py
@@ -121,7 +121,7 @@ class WeiboMobileIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id, note="visit the page")
         js_code = self._search_regex(r'var\s+\$render_data\s*=\s*\[({.*})\]\[0\] \|\| {};', webpage, 'js_code', flags=re.DOTALL)
         weibo_info = self._parse_json(js_code, video_id, transform_source=js_to_json)
-        page_info = weibo_info['status']['page_info']
+        page_info = weibo_info.get('status').get('page_info')
         title = weibo_info.get('status').get('status_title')
         uploader = weibo_info.get('status').get('user').get('screen_name')
 
-- 
cgit v1.1


From b300cda4769a987c501db298f3f107326521d45c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 1 Jan 2018 21:52:24 +0700
Subject: [YoutubeDL] Output python implementation in debug header

---
 youtube_dl/YoutubeDL.py | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index ace80f1..97bd9c5 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -2233,8 +2233,16 @@ class YoutubeDL(object):
                 sys.exc_clear()
             except Exception:
                 pass
-        self._write_string('[debug] Python version %s - %s\n' % (
-            platform.python_version(), platform_name()))
+
+        def python_implementation():
+            impl_name = platform.python_implementation()
+            if impl_name == 'PyPy' and hasattr(sys, 'pypy_version_info'):
+                return impl_name + ' version %d.%d.%d' % sys.pypy_version_info[:3]
+            return impl_name
+
+        self._write_string('[debug] Python version %s (%s) - %s\n' % (
+            platform.python_version(), python_implementation(),
+            platform_name()))
 
         exe_versions = FFmpegPostProcessor.get_versions(self)
         exe_versions['rtmpdump'] = rtmpdump_version()
-- 
cgit v1.1


From 54009c246e8eed38cfa8dc3eecb5619c1c81a1f2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 1 Jan 2018 21:54:28 +0700
Subject: [travis] Add PyPy builds

---
 .travis.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.travis.yml b/.travis.yml
index 5f4f392..7b175e9 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -7,6 +7,8 @@ python:
   - "3.4"
   - "3.5"
   - "3.6"
+  - "pypy"
+  - "pypy3"
 sudo: false
 env:
   - YTDL_TEST_SET=core
-- 
cgit v1.1


From d7cd9a9e847fab3ac3f0fb5b4ad2e4788aeea775 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 12 Dec 2014 04:01:08 +0100
Subject: [utils] Fix youtube-dl under PyPy3 on Windows

---
 youtube_dl/compat.py | 21 +++++++++++++++++++++
 youtube_dl/utils.py  | 15 ++++++++-------
 2 files changed, 29 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 2a62248..41ca9ad 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -3,12 +3,14 @@ from __future__ import unicode_literals
 
 import binascii
 import collections
+import ctypes
 import email
 import getpass
 import io
 import itertools
 import optparse
 import os
+import platform
 import re
 import shlex
 import shutil
@@ -2906,6 +2908,24 @@ except ImportError:  # not 2.6+ or is 3.x
     except ImportError:
         compat_zip = zip
 
+if platform.python_implementation() == 'PyPy' and sys.pypy_version_info < (5, 4, 0):
+    # PyPy2 prior to version 5.4.0 expects byte strings as Windows function
+    # names, see the original PyPy issue [1] and the youtube-dl one [2].
+    # 1. https://bitbucket.org/pypy/pypy/issues/2360/windows-ctypescdll-typeerror-function-name
+    # 2. https://github.com/rg3/youtube-dl/pull/4392
+    def compat_ctypes_WINFUNCTYPE(*args, **kwargs):
+        real = ctypes.WINFUNCTYPE(*args, **kwargs)
+
+        def resf(tpl, *args, **kwargs):
+            funcname, dll = tpl
+            return real((str(funcname), dll), *args, **kwargs)
+
+        return resf
+else:
+    def compat_ctypes_WINFUNCTYPE(*args, **kwargs):
+        return ctypes.WINFUNCTYPE(*args, **kwargs)
+
+
 __all__ = [
     'compat_HTMLParseError',
     'compat_HTMLParser',
@@ -2914,6 +2934,7 @@ __all__ = [
     'compat_chr',
     'compat_cookiejar',
     'compat_cookies',
+    'compat_ctypes_WINFUNCTYPE',
     'compat_etree_fromstring',
     'compat_etree_register_namespace',
     'compat_expanduser',
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 2843a3d..386897a 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -39,6 +39,7 @@ from .compat import (
     compat_HTMLParser,
     compat_basestring,
     compat_chr,
+    compat_ctypes_WINFUNCTYPE,
     compat_etree_fromstring,
     compat_expanduser,
     compat_html_entities,
@@ -1330,24 +1331,24 @@ def _windows_write_string(s, out):
     if fileno not in WIN_OUTPUT_IDS:
         return False
 
-    GetStdHandle = ctypes.WINFUNCTYPE(
+    GetStdHandle = compat_ctypes_WINFUNCTYPE(
         ctypes.wintypes.HANDLE, ctypes.wintypes.DWORD)(
-        (b'GetStdHandle', ctypes.windll.kernel32))
+        ('GetStdHandle', ctypes.windll.kernel32))
     h = GetStdHandle(WIN_OUTPUT_IDS[fileno])
 
-    WriteConsoleW = ctypes.WINFUNCTYPE(
+    WriteConsoleW = compat_ctypes_WINFUNCTYPE(
         ctypes.wintypes.BOOL, ctypes.wintypes.HANDLE, ctypes.wintypes.LPWSTR,
         ctypes.wintypes.DWORD, ctypes.POINTER(ctypes.wintypes.DWORD),
-        ctypes.wintypes.LPVOID)((b'WriteConsoleW', ctypes.windll.kernel32))
+        ctypes.wintypes.LPVOID)(('WriteConsoleW', ctypes.windll.kernel32))
     written = ctypes.wintypes.DWORD(0)
 
-    GetFileType = ctypes.WINFUNCTYPE(ctypes.wintypes.DWORD, ctypes.wintypes.DWORD)((b'GetFileType', ctypes.windll.kernel32))
+    GetFileType = compat_ctypes_WINFUNCTYPE(ctypes.wintypes.DWORD, ctypes.wintypes.DWORD)(('GetFileType', ctypes.windll.kernel32))
     FILE_TYPE_CHAR = 0x0002
     FILE_TYPE_REMOTE = 0x8000
-    GetConsoleMode = ctypes.WINFUNCTYPE(
+    GetConsoleMode = compat_ctypes_WINFUNCTYPE(
         ctypes.wintypes.BOOL, ctypes.wintypes.HANDLE,
         ctypes.POINTER(ctypes.wintypes.DWORD))(
-        (b'GetConsoleMode', ctypes.windll.kernel32))
+        ('GetConsoleMode', ctypes.windll.kernel32))
     INVALID_HANDLE_VALUE = ctypes.wintypes.DWORD(-1).value
 
     def not_a_console(handle):
-- 
cgit v1.1


From 7a6c204fcb6ba5a1a5149ea7a3c186eab87fc7e4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 1 Jan 2018 23:21:39 +0700
Subject: [travis] Add Jython build

---
 .travis.yml                  | 11 +++++++----
 devscripts/install_jython.sh |  5 +++++
 2 files changed, 12 insertions(+), 4 deletions(-)
 create mode 100755 devscripts/install_jython.sh

diff --git a/.travis.yml b/.travis.yml
index 7b175e9..92f3268 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -14,11 +14,14 @@ env:
   - YTDL_TEST_SET=core
   - YTDL_TEST_SET=download
 matrix:
+  include:
+    - env: JYTHON=true; YTDL_TEST_SET=core
+    - env: JYTHON=true; YTDL_TEST_SET=download
   fast_finish: true
   allow_failures:
     - env: YTDL_TEST_SET=download
+    - env: JYTHON=true; YTDL_TEST_SET=core
+    - env: JYTHON=true; YTDL_TEST_SET=download
+before_install:
+  - if [ "$JYTHON" == "true" ]; then ./devscripts/install_jython.sh; export PATH="$HOME/jython/bin:$PATH"; fi
 script: ./devscripts/run_tests.sh
-notifications:
-  email:
-    - filippo.valsorda@gmail.com
-    - yasoob.khld@gmail.com
diff --git a/devscripts/install_jython.sh b/devscripts/install_jython.sh
new file mode 100755
index 0000000..bafca4d
--- /dev/null
+++ b/devscripts/install_jython.sh
@@ -0,0 +1,5 @@
+#!/bin/bash
+
+wget http://central.maven.org/maven2/org/python/jython-installer/2.7.1/jython-installer-2.7.1.jar
+java -jar jython-installer-2.7.1.jar -s -d "$HOME/jython"
+$HOME/jython/bin/jython -m pip install nose
-- 
cgit v1.1


From b5e531f31a57d2cdc9f0edd77e0cbef426e016b9 Mon Sep 17 00:00:00 2001
From: Mattias Wadman <mattias.wadman@gmail.com>
Date: Tue, 2 Jan 2018 17:32:17 +0100
Subject: [acast] Fix extraction

---
 youtube_dl/extractor/acast.py | 23 ++++++++++++-----------
 1 file changed, 12 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/acast.py b/youtube_dl/extractor/acast.py
index 6dace30..5871e72 100644
--- a/youtube_dl/extractor/acast.py
+++ b/youtube_dl/extractor/acast.py
@@ -8,7 +8,7 @@ from .common import InfoExtractor
 from ..compat import compat_str
 from ..utils import (
     int_or_none,
-    parse_iso8601,
+    unified_timestamp,
     OnDemandPagedList,
 )
 
@@ -32,7 +32,7 @@ class ACastIE(InfoExtractor):
     }, {
         # test with multiple blings
         'url': 'https://www.acast.com/sparpodcast/2.raggarmordet-rosterurdetforflutna',
-        'md5': '55c0097badd7095f494c99a172f86501',
+        'md5': 'e87d5b8516cd04c0d81b6ee1caca28d0',
         'info_dict': {
             'id': '2a92b283-1a75-4ad8-8396-499c641de0d9',
             'ext': 'mp3',
@@ -40,23 +40,24 @@ class ACastIE(InfoExtractor):
             'timestamp': 1477346700,
             'upload_date': '20161024',
             'description': 'md5:4f81f6d8cf2e12ee21a321d8bca32db4',
-            'duration': 2797,
+            'duration': 2766,
         }
     }]
 
     def _real_extract(self, url):
         channel, display_id = re.match(self._VALID_URL, url).groups()
         cast_data = self._download_json(
-            'https://embed.acast.com/api/acasts/%s/%s' % (channel, display_id), display_id)
+            'https://play-api.acast.com/splash/%s/%s' % (channel, display_id), display_id)
+        e = cast_data['result']['episode']
         return {
-            'id': compat_str(cast_data['id']),
+            'id': compat_str(e['id']),
             'display_id': display_id,
-            'url': [b['audio'] for b in cast_data['blings'] if b['type'] == 'BlingAudio'][0],
-            'title': cast_data['name'],
-            'description': cast_data.get('description'),
-            'thumbnail': cast_data.get('image'),
-            'timestamp': parse_iso8601(cast_data.get('publishingDate')),
-            'duration': int_or_none(cast_data.get('duration')),
+            'url': e['mediaUrl'],
+            'title': e['name'],
+            'description': e.get('description'),
+            'thumbnail': e.get('image'),
+            'timestamp': unified_timestamp(e.get('publishingDate')),
+            'duration': int_or_none(e.get('duration')),
         }
 
 
-- 
cgit v1.1


From 9650c3e91d96fc054e37485d2bfd86a6a12417e5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@fastmail.net>
Date: Tue, 2 Jan 2018 21:12:39 +0100
Subject: [rtve.es:alacarta] Fix extraction of some new URLs

---
 ChangeLog                    | 6 ++++++
 youtube_dl/extractor/rtve.py | 3 +++
 2 files changed, 9 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index bfffb1f..3e6afca 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,9 @@
+version <unreleased>
+
+Extractors
+* [rtve.es:alacarta] Fix extraction of some new URLs
+
+
 version 2017.12.31
 
 Core
diff --git a/youtube_dl/extractor/rtve.py b/youtube_dl/extractor/rtve.py
index d9edf9d..fa60ffd 100644
--- a/youtube_dl/extractor/rtve.py
+++ b/youtube_dl/extractor/rtve.py
@@ -31,6 +31,9 @@ def _decrypt_url(png):
     hash_index = data.index('#')
     alphabet_data = data[:hash_index]
     url_data = data[hash_index + 1:]
+    if url_data[0] == 'H' and url_data[3] == '%':
+        # remove useless HQ%% at the start
+        url_data = url_data[4:]
 
     alphabet = []
     e = 0
-- 
cgit v1.1


From f0c6c2bce29281d37d2bbd589143b35323e38e3d Mon Sep 17 00:00:00 2001
From: Luc Ritchie <luc.ritchie@gmail.com>
Date: Wed, 3 Jan 2018 04:22:55 -0500
Subject: [twitch] Pass video id to url_result when extracting playlist

---
 youtube_dl/extractor/twitch.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index bf57eac..f9164af 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -358,9 +358,16 @@ class TwitchPlaylistBaseIE(TwitchBaseIE):
                 break
             offset += limit
         return self.playlist_result(
-            [self.url_result(entry) for entry in orderedSet(entries)],
+            [self._make_url_result(entry) for entry in orderedSet(entries)],
             channel_id, channel_name)
 
+    def _make_url_result(self, url):
+        try:
+            video_id = 'v%s' % TwitchVodIE._match_id(url)
+            return self.url_result(url, TwitchVodIE.ie_key(), video_id=video_id)
+        except AssertionError:
+            return self.url_result(url)
+
     def _extract_playlist_page(self, response):
         videos = response.get('videos')
         return [video['url'] for video in videos] if videos else []
-- 
cgit v1.1


From 75ba0efb52c898601ef9febcba3e7a84ff2f311e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 3 Jan 2018 16:41:28 +0700
Subject: [lynda] Skip invalid subtitles (closes #15159)

---
 youtube_dl/extractor/lynda.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/lynda.py b/youtube_dl/extractor/lynda.py
index 1b6f509..0d6026a 100644
--- a/youtube_dl/extractor/lynda.py
+++ b/youtube_dl/extractor/lynda.py
@@ -244,8 +244,9 @@ class LyndaIE(LyndaBaseIE):
     def _get_subtitles(self, video_id):
         url = 'https://www.lynda.com/ajax/player?videoId=%s&type=transcript' % video_id
         subs = self._download_json(url, None, False)
-        if subs:
-            return {'en': [{'ext': 'srt', 'data': self._fix_subtitles(subs)}]}
+        fixed_subs = self._fix_subtitles(subs)
+        if fixed_subs:
+            return {'en': [{'ext': 'srt', 'data': fixed_subs}]}
         else:
             return {}
 
-- 
cgit v1.1


From de329f64abd920e148701436d6a20bfc5b2d3ef3 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 4 Jan 2018 13:26:08 +0800
Subject: [openload] Fix extraction (closes #15166)

---
 ChangeLog                        | 1 +
 youtube_dl/extractor/openload.py | 6 +++++-
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/ChangeLog b/ChangeLog
index 3e6afca..94b27b6 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,6 +1,7 @@
 version <unreleased>
 
 Extractors
+* [openload] Fix extraction (#15166)
 * [rtve.es:alacarta] Fix extraction of some new URLs
 
 
diff --git a/youtube_dl/extractor/openload.py b/youtube_dl/extractor/openload.py
index b282bcf..eaaaf8a 100644
--- a/youtube_dl/extractor/openload.py
+++ b/youtube_dl/extractor/openload.py
@@ -333,7 +333,11 @@ class OpenloadIE(InfoExtractor):
         webpage, _ = phantom.get(page_url, html=webpage, video_id=video_id, headers=headers)
 
         decoded_id = (get_element_by_id('streamurl', webpage) or
-                      get_element_by_id('streamuri', webpage))
+                      get_element_by_id('streamuri', webpage) or
+                      get_element_by_id('streamurj', webpage))
+
+        if not decoded_id:
+            raise ExtractorError('Can\'t find stream URL', video_id=video_id)
 
         video_url = 'https://openload.co/stream/%s?mime=true' % decoded_id
 
-- 
cgit v1.1


From da35331c6c23f90fc29ca3315f7e00481aff1d5c Mon Sep 17 00:00:00 2001
From: JianxinLi <justcoxier@gmail.com>
Date: Thu, 4 Jan 2018 22:25:28 +0800
Subject: [youku] Fix list extraction.(close #15135) (#15137)

* [youku] Fix list extraction.(close #15135)

Change-Id: I2e9c920143f4f16012252625943a8f18b8ff40eb

* [youku] Remove KeyError try-except

Change-Id: Ic46327905cbef1356b7b12d5eb3db5d9746ca338
---
 youtube_dl/extractor/youku.py | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index c7947d4..5b0b248 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -245,13 +245,19 @@ class YoukuShowIE(InfoExtractor):
         #  No data-id value.
         'url': 'http://list.youku.com/show/id_zefbfbd61237fefbfbdef.html',
         'only_matching': True,
+    }, {
+        #  Wrong number of reload_id.
+        'url': 'http://list.youku.com/show/id_z20eb4acaf5c211e3b2ad.html',
+        'only_matching': True,
     }]
 
     def _extract_entries(self, playlist_data_url, show_id, note, query):
         query['callback'] = 'cb'
         playlist_data = self._download_json(
             playlist_data_url, show_id, query=query, note=note,
-            transform_source=lambda s: js_to_json(strip_jsonp(s)))['html']
+            transform_source=lambda s: js_to_json(strip_jsonp(s))).get('html')
+        if playlist_data is None:
+            return [None, None]
         drama_list = (get_element_by_class('p-drama-grid', playlist_data) or
                       get_element_by_class('p-drama-half-row', playlist_data))
         if drama_list is None:
@@ -291,8 +297,8 @@ class YoukuShowIE(InfoExtractor):
                     'id': page_config['showid'],
                     'stage': reload_id,
                 })
-            entries.extend(new_entries)
-
+            if new_entries is not None:
+                entries.extend(new_entries)
         desc = self._html_search_meta('description', webpage, fatal=False)
         playlist_title = desc.split(',')[0] if desc else None
         detail_li = get_element_by_class('p-intro', webpage)
-- 
cgit v1.1


From c2f18e1c4915d43b37c12337dc4fbae68d43a304 Mon Sep 17 00:00:00 2001
From: Chih-Hsuan Yen <yan12125@gmail.com>
Date: Thu, 4 Jan 2018 22:28:00 +0800
Subject: [ChangeLog] Update after #15137

[skip ci]
---
 ChangeLog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ChangeLog b/ChangeLog
index 94b27b6..96bc471 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,6 +1,7 @@
 version <unreleased>
 
 Extractors
+* [youku] Fix list extraction (#15135)
 * [openload] Fix extraction (#15166)
 * [rtve.es:alacarta] Fix extraction of some new URLs
 
-- 
cgit v1.1


From 0b0870f9d0dd3e72be3ff6be6bfa9fa43b693b50 Mon Sep 17 00:00:00 2001
From: Parmjit Virk <pvirk@mts.net>
Date: Thu, 4 Jan 2018 19:25:42 -0600
Subject: [soundcloud] Fallback to avatar picture for thumbnail (closes #12878)

---
 youtube_dl/extractor/soundcloud.py | 21 ++++++++++++++++++++-
 1 file changed, 20 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index 8894f4b..6c9816e 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -136,6 +136,25 @@ class SoundcloudIE(InfoExtractor):
                 'license': 'all-rights-reserved',
             },
         },
+        # no album art, use avatar pic for thumbnail
+        {
+            'url': 'https://soundcloud.com/garyvee/sideways-prod-mad-real',
+            'md5': '59c7872bc44e5d99b7211891664760c2',
+            'info_dict': {
+                'id': '309699954',
+                'ext': 'mp3',
+                'title': 'Sideways (Prod. Mad Real)',
+                'description': 'md5:d41d8cd98f00b204e9800998ecf8427e',
+                'uploader': 'garyvee',
+                'upload_date': '20170226',
+                'duration': 207,
+                'thumbnail': r're:https?://.*\.jpg',
+                'license': 'all-rights-reserved',
+            },
+            'params': {
+                'skip_download': True,
+            },
+        },
     ]
 
     _CLIENT_ID = 'c6CU49JDMapyrQo06UxU9xouB9ZVzqCn'
@@ -160,7 +179,7 @@ class SoundcloudIE(InfoExtractor):
         name = full_title or track_id
         if quiet:
             self.report_extraction(name)
-        thumbnail = info.get('artwork_url')
+        thumbnail = info.get('artwork_url') or info.get('user', {}).get('avatar_url')
         if isinstance(thumbnail, compat_str):
             thumbnail = thumbnail.replace('-large', '-t500x500')
         ext = 'mp3'
-- 
cgit v1.1


From b7c74c04036c07f8a81d3048b482afd6ef384b40 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 6 Jan 2018 23:12:30 +0700
Subject: [lynda] Relax _VALID_URL (closes #15185)

---
 youtube_dl/extractor/lynda.py | 23 +++++++++++++++++++++--
 1 file changed, 21 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/lynda.py b/youtube_dl/extractor/lynda.py
index 0d6026a..f5c7abc 100644
--- a/youtube_dl/extractor/lynda.py
+++ b/youtube_dl/extractor/lynda.py
@@ -94,7 +94,15 @@ class LyndaBaseIE(InfoExtractor):
 class LyndaIE(LyndaBaseIE):
     IE_NAME = 'lynda'
     IE_DESC = 'lynda.com videos'
-    _VALID_URL = r'https?://(?:www\.)?(?:lynda\.com|educourse\.ga)/(?:[^/]+/[^/]+/(?P<course_id>\d+)|player/embed)/(?P<id>\d+)'
+    _VALID_URL = r'''(?x)
+                    https?://
+                        (?:www\.)?(?:lynda\.com|educourse\.ga)/
+                        (?:
+                            (?:[^/]+/){2,3}(?P<course_id>\d+)|
+                            player/embed
+                        )/
+                        (?P<id>\d+)
+                    '''
 
     _TIMECODE_REGEX = r'\[(?P<timecode>\d+:\d+:\d+[\.,]\d+)\]'
 
@@ -113,6 +121,9 @@ class LyndaIE(LyndaBaseIE):
     }, {
         'url': 'https://educourse.ga/Bootstrap-tutorials/Using-exercise-files/110885/114408-4.html',
         'only_matching': True,
+    }, {
+        'url': 'https://www.lynda.com/de/Graphic-Design-tutorials/Willkommen-Grundlagen-guten-Gestaltung/393570/393572-4.html',
+        'only_matching': True,
     }]
 
     def _raise_unavailable(self, video_id):
@@ -257,7 +268,15 @@ class LyndaCourseIE(LyndaBaseIE):
 
     # Course link equals to welcome/introduction video link of same course
     # We will recognize it as course link
-    _VALID_URL = r'https?://(?:www|m)\.(?:lynda\.com|educourse\.ga)/(?P<coursepath>[^/]+/[^/]+/(?P<courseid>\d+))-\d\.html'
+    _VALID_URL = r'https?://(?:www|m)\.(?:lynda\.com|educourse\.ga)/(?P<coursepath>(?:[^/]+/){2,3}(?P<courseid>\d+))-2\.html'
+
+    _TESTS = [{
+        'url': 'https://www.lynda.com/Graphic-Design-tutorials/Grundlagen-guten-Gestaltung/393570-2.html',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.lynda.com/de/Graphic-Design-tutorials/Grundlagen-guten-Gestaltung/393570-2.html',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
-- 
cgit v1.1


From 45283afdec81af21ba50ff3aca3d86fb6d2584b0 Mon Sep 17 00:00:00 2001
From: Martin Weinelt <mweinelt@users.noreply.github.com>
Date: Sat, 6 Jan 2018 17:33:40 +0100
Subject: [motherless] Add support for groups

---
 youtube_dl/extractor/extractors.py |  5 ++-
 youtube_dl/extractor/motherless.py | 73 ++++++++++++++++++++++++++++++++++++++
 2 files changed, 77 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index e64defe..fb0997d 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -609,7 +609,10 @@ from .mofosex import MofosexIE
 from .mojvideo import MojvideoIE
 from .moniker import MonikerIE
 from .morningstar import MorningstarIE
-from .motherless import MotherlessIE
+from .motherless import (
+    MotherlessIE,
+    MotherlessGroupIE
+)
 from .motorsport import MotorsportIE
 from .movieclips import MovieClipsIE
 from .moviezine import MoviezineIE
diff --git a/youtube_dl/extractor/motherless.py b/youtube_dl/extractor/motherless.py
index 6fe3b60..90ed91b 100644
--- a/youtube_dl/extractor/motherless.py
+++ b/youtube_dl/extractor/motherless.py
@@ -4,8 +4,11 @@ import datetime
 import re
 
 from .common import InfoExtractor
+from ..compat import compat_urlparse
 from ..utils import (
     ExtractorError,
+    InAdvancePagedList,
+    orderedSet,
     str_to_int,
     unified_strdate,
 )
@@ -114,3 +117,73 @@ class MotherlessIE(InfoExtractor):
             'age_limit': age_limit,
             'url': video_url,
         }
+
+
+class MotherlessGroupIE(InfoExtractor):
+    _VALID_URL = 'https?://(?:www\.)?motherless\.com/gv?/(?P<id>[a-z0-9_]+)'
+    _TESTS = [{
+        'url': 'http://motherless.com/g/movie_scenes',
+        'info_dict': {
+            'id': 'movie_scenes',
+            'title': 'Movie Scenes',
+            'description': 'Hot and sexy scenes from "regular" movies... '
+                           'Beautiful actresses fully nude... A looot of '
+                           'skin! :)Enjoy!',
+        },
+        'playlist_mincount': 662,
+    }, {
+        'url': 'http://motherless.com/gv/sex_must_be_funny',
+        'info_dict': {
+            'id': 'sex_must_be_funny',
+            'title': 'Sex must be funny',
+            'description': 'Sex can be funny. Wide smiles,laugh, games, fun of '
+                           'any kind!'
+        },
+        'playlist_mincount': 9,
+    }]
+
+    @classmethod
+    def suitable(cls, url):
+        return (False if MotherlessIE.suitable(url)
+                else super(MotherlessGroupIE, cls).suitable(url))
+
+    def _extract_entries(self, webpage, base):
+        return [
+            self.url_result(
+                compat_urlparse.urljoin(base, video_path),
+                MotherlessIE.ie_key(), video_title=title)
+            for video_path, title in orderedSet(re.findall(
+                r'href="/([^"]+)"[^>]+>\s+<img[^>]+alt="[^-]+-\s([^"]+)"',
+                webpage))
+        ]
+
+    def _real_extract(self, url):
+        group_id = self._match_id(url)
+        page_url = compat_urlparse.urljoin(url, '/gv/%s' % group_id)
+        webpage = self._download_webpage(page_url, group_id)
+        title = self._search_regex(
+            r'<title>([\w\s]+\w)\s+-', webpage, 'title', fatal=False)
+        description = self._html_search_meta(
+            'description', webpage, fatal=False)
+        page_count = self._int(self._search_regex(
+            r'(\d+)</(?:a|span)><(?:a|span)[^>]+>\s*NEXT',
+            webpage, 'page_count'), 'page_count')
+        PAGE_SIZE = 80
+
+        def _get_page(idx):
+            webpage = self._download_webpage(
+                page_url, group_id, query={'page': idx + 1},
+                note='Downloading page %d/%d' % (idx + 1, page_count)
+            )
+            for entry in self._extract_entries(webpage, url):
+                yield entry
+
+        playlist = InAdvancePagedList(_get_page, page_count, PAGE_SIZE)
+
+        return {
+            '_type': 'playlist',
+            'id': group_id,
+            'title': title,
+            'description': description,
+            'entries': playlist
+        }
-- 
cgit v1.1


From f12628f934ff50cc8e6441c4e64fe61019ebae5c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 6 Jan 2018 23:58:00 +0700
Subject: [mitele] Fix extraction (closes #15186)

---
 youtube_dl/extractor/mitele.py | 30 ++++++++++++++++--------------
 1 file changed, 16 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/mitele.py b/youtube_dl/extractor/mitele.py
index 964dc54..42759ea 100644
--- a/youtube_dl/extractor/mitele.py
+++ b/youtube_dl/extractor/mitele.py
@@ -1,13 +1,13 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import json
 import uuid
 
 from .common import InfoExtractor
 from .ooyala import OoyalaIE
 from ..compat import (
     compat_str,
-    compat_urllib_parse_urlencode,
     compat_urlparse,
 )
 from ..utils import (
@@ -42,31 +42,33 @@ class MiTeleBaseIE(InfoExtractor):
                 duration = int_or_none(mmc.get('duration'))
             for location in mmc['locations']:
                 gat = self._proto_relative_url(location.get('gat'), 'http:')
-                bas = location.get('bas')
-                loc = location.get('loc')
+                gcp = location.get('gcp')
                 ogn = location.get('ogn')
-                if None in (gat, bas, loc, ogn):
+                if None in (gat, gcp, ogn):
                     continue
                 token_data = {
-                    'bas': bas,
-                    'icd': loc,
+                    'gcp': gcp,
                     'ogn': ogn,
-                    'sta': '0',
+                    'sta': 0,
                 }
                 media = self._download_json(
-                    '%s/?%s' % (gat, compat_urllib_parse_urlencode(token_data)),
-                    video_id, 'Downloading %s JSON' % location['loc'])
-                file_ = media.get('file')
-                if not file_:
+                    gat, video_id, data=json.dumps(token_data).encode('utf-8'),
+                    headers={
+                        'Content-Type': 'application/json;charset=utf-8',
+                        'Referer': url,
+                    })
+                stream = media.get('stream') or media.get('file')
+                if not stream:
                     continue
-                ext = determine_ext(file_)
+                ext = determine_ext(stream)
                 if ext == 'f4m':
                     formats.extend(self._extract_f4m_formats(
-                        file_ + '&hdcore=3.2.0&plugin=aasp-3.2.0.77.18',
+                        stream + '&hdcore=3.2.0&plugin=aasp-3.2.0.77.18',
                         video_id, f4m_id='hds', fatal=False))
                 elif ext == 'm3u8':
                     formats.extend(self._extract_m3u8_formats(
-                        file_, video_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False))
+                        stream, video_id, 'mp4', 'm3u8_native',
+                        m3u8_id='hls', fatal=False))
         self._sort_formats(formats)
 
         return {
-- 
cgit v1.1


From a133eb7764594b830cb975e3925972214e932704 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 7 Jan 2018 00:02:41 +0700
Subject: [motherless:group] Capture leading slash of video path

---
 youtube_dl/extractor/motherless.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/motherless.py b/youtube_dl/extractor/motherless.py
index 90ed91b..4adac69 100644
--- a/youtube_dl/extractor/motherless.py
+++ b/youtube_dl/extractor/motherless.py
@@ -153,7 +153,7 @@ class MotherlessGroupIE(InfoExtractor):
                 compat_urlparse.urljoin(base, video_path),
                 MotherlessIE.ie_key(), video_title=title)
             for video_path, title in orderedSet(re.findall(
-                r'href="/([^"]+)"[^>]+>\s+<img[^>]+alt="[^-]+-\s([^"]+)"',
+                r'href="(/[^"]+)"[^>]+>\s+<img[^>]+alt="[^-]+-\s([^"]+)"',
                 webpage))
         ]
 
-- 
cgit v1.1


From 0a5b1295b7c1aa6395b65ee137087c540b37b32b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 7 Jan 2018 00:31:53 +0700
Subject: [motherless:group] Relax entry extraction and add a fallback scenario

---
 youtube_dl/extractor/motherless.py | 29 +++++++++++++++++++++--------
 1 file changed, 21 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/motherless.py b/youtube_dl/extractor/motherless.py
index 4adac69..e24396e 100644
--- a/youtube_dl/extractor/motherless.py
+++ b/youtube_dl/extractor/motherless.py
@@ -148,14 +148,27 @@ class MotherlessGroupIE(InfoExtractor):
                 else super(MotherlessGroupIE, cls).suitable(url))
 
     def _extract_entries(self, webpage, base):
-        return [
-            self.url_result(
-                compat_urlparse.urljoin(base, video_path),
-                MotherlessIE.ie_key(), video_title=title)
-            for video_path, title in orderedSet(re.findall(
-                r'href="(/[^"]+)"[^>]+>\s+<img[^>]+alt="[^-]+-\s([^"]+)"',
-                webpage))
-        ]
+        entries = []
+        for mobj in re.finditer(
+                r'href="(?P<href>/[^"]+)"[^>]*>(?:\s*<img[^>]+alt="[^-]+-\s(?P<title>[^"]+)")?',
+                webpage):
+            video_url = compat_urlparse.urljoin(base, mobj.group('href'))
+            if not MotherlessIE.suitable(video_url):
+                continue
+            video_id = MotherlessIE._match_id(video_url)
+            title = mobj.group('title')
+            entries.append(self.url_result(
+                video_url, ie=MotherlessIE.ie_key(), video_id=video_id,
+                video_title=title))
+        # Alternative fallback
+        if not entries:
+            entries = [
+                self.url_result(
+                    compat_urlparse.urljoin(base, '/' + video_id),
+                    ie=MotherlessIE.ie_key(), video_id=video_id)
+                for video_id in orderedSet(re.findall(
+                    r'data-codename=["\']([A-Z0-9]+)', webpage))]
+        return entries
 
     def _real_extract(self, url):
         group_id = self._match_id(url)
-- 
cgit v1.1


From b0ead0e09aae6de6026a018cda7019eb7eade919 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 7 Jan 2018 21:49:23 +0700
Subject: [jwplatform] Add support for multiple embeds (closes #15192)

---
 youtube_dl/extractor/generic.py    |  6 +++---
 youtube_dl/extractor/jwplatform.py | 11 +++++++----
 2 files changed, 10 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index cc4c90b..9b0cd00 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -2708,9 +2708,9 @@ class GenericIE(InfoExtractor):
             return self.url_result(viewlift_url)
 
         # Look for JWPlatform embeds
-        jwplatform_url = JWPlatformIE._extract_url(webpage)
-        if jwplatform_url:
-            return self.url_result(jwplatform_url, 'JWPlatform')
+        jwplatform_urls = JWPlatformIE._extract_urls(webpage)
+        if jwplatform_urls:
+            return self.playlist_from_matches(jwplatform_urls, video_id, video_title, ie=JWPlatformIE.ie_key())
 
         # Look for Digiteka embeds
         digiteka_url = DigitekaIE._extract_url(webpage)
diff --git a/youtube_dl/extractor/jwplatform.py b/youtube_dl/extractor/jwplatform.py
index c9bcbb0..63d0dc9 100644
--- a/youtube_dl/extractor/jwplatform.py
+++ b/youtube_dl/extractor/jwplatform.py
@@ -23,11 +23,14 @@ class JWPlatformIE(InfoExtractor):
 
     @staticmethod
     def _extract_url(webpage):
-        mobj = re.search(
-            r'<(?:script|iframe)[^>]+?src=["\'](?P<url>(?:https?:)?//content.jwplatform.com/players/[a-zA-Z0-9]{8})',
+        urls = JWPlatformIE._extract_urls(webpage)
+        return urls[0] if urls else None
+
+    @staticmethod
+    def _extract_urls(webpage):
+        return re.findall(
+            r'<(?:script|iframe)[^>]+?src=["\']((?:https?:)?//content\.jwplatform\.com/players/[a-zA-Z0-9]{8})',
             webpage)
-        if mobj:
-            return mobj.group('url')
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-- 
cgit v1.1


From 8faa9576bb4599dc3e77b8d3339122aa4f1230b4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 7 Jan 2018 23:48:56 +0700
Subject: [ChangeLog] Actualize

---
 ChangeLog | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 96bc471..67de653 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,9 +1,21 @@
 version <unreleased>
 
+Core
+* [utils] Fix youtube-dl under PyPy3 on Windows
+* [YoutubeDL] Output python implementation in debug header
+
 Extractors
++ [jwplatform] Add support for multiple embeds (#15192)
+* [mitele] Fix extraction (#15186)
++ [motherless] Add support for groups (#15124)
+* [lynda] Relax URL regular expression (#15185)
+* [soundcloud] Fallback to avatar picture for thumbnail (#12878)
 * [youku] Fix list extraction (#15135)
 * [openload] Fix extraction (#15166)
+* [lynda] Skip invalid subtitles (#15159)
+* [twitch] Pass video id to url_result when extracting playlist (#15139)
 * [rtve.es:alacarta] Fix extraction of some new URLs
+* [acast] Fix extraction (#15147)
 
 
 version 2017.12.31
-- 
cgit v1.1


From 950b5f296986ed0a2dd9feeb69dbb950592b6047 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 7 Jan 2018 23:52:16 +0700
Subject: release 2018.01.07

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 docs/supportedsites.md    | 1 +
 youtube_dl/version.py     | 2 +-
 4 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 3f89849..ad52c89 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.12.31*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.12.31**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2018.01.07*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2018.01.07**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through the [README](https://github.com/rg3/youtube-dl/blob/master/README.md), **most notably** the [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ Add the `-v` flag to **your command line** you run youtube-dl with (`youtube-dl
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.12.31
+[debug] youtube-dl version 2018.01.07
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index 67de653..9c45ae0 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2018.01.07
 
 Core
 * [utils] Fix youtube-dl under PyPy3 on Windows
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 75bd5c9..79b3430 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -478,6 +478,7 @@
  - **Moniker**: allmyvideos.net and vidspot.net
  - **Morningstar**: morningstar.com
  - **Motherless**
+ - **MotherlessGroup**
  - **Motorsport**: motorsport.com
  - **MovieClips**
  - **MovieFap**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index a3f84b9..9030e24 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2017.12.31'
+__version__ = '2018.01.07'
-- 
cgit v1.1


From 3a513f29adc42fc46fd8b754806d38444bcee151 Mon Sep 17 00:00:00 2001
From: Luca Steeb <contact@luca-steeb.com>
Date: Sat, 6 Jan 2018 20:27:26 +0100
Subject: fix bilibili extraction (closes #15171)

---
 youtube_dl/extractor/bilibili.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bilibili.py b/youtube_dl/extractor/bilibili.py
index 1e57310..beffcec 100644
--- a/youtube_dl/extractor/bilibili.py
+++ b/youtube_dl/extractor/bilibili.py
@@ -102,6 +102,7 @@ class BiliBiliIE(InfoExtractor):
                     video_id, anime_id, compat_urlparse.urljoin(url, '//bangumi.bilibili.com/anime/%s' % anime_id)))
             headers = {
                 'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
+                'Referer': url
             }
             headers.update(self.geo_verification_headers())
 
@@ -116,10 +117,15 @@ class BiliBiliIE(InfoExtractor):
         payload = 'appkey=%s&cid=%s&otype=json&quality=2&type=mp4' % (self._APP_KEY, cid)
         sign = hashlib.md5((payload + self._BILIBILI_KEY).encode('utf-8')).hexdigest()
 
+        headers = {
+            'Referer': url
+        }
+        headers.update(self.geo_verification_headers())
+
         video_info = self._download_json(
             'http://interface.bilibili.com/playurl?%s&sign=%s' % (payload, sign),
             video_id, note='Downloading video info page',
-            headers=self.geo_verification_headers())
+            headers=headers)
 
         if 'durl' not in video_info:
             self._report_error(video_info)
-- 
cgit v1.1


From 7643916a3794f52169d16df093bd4a2b3abbb323 Mon Sep 17 00:00:00 2001
From: Chih-Hsuan Yen <yan12125@gmail.com>
Date: Mon, 8 Jan 2018 01:32:13 +0800
Subject: [ChangeLog] update after #15188

[ci skip]
---
 ChangeLog | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 9c45ae0..9d37cdc 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,9 @@
+version <unreleased>
+
+Extractors
+* [bilibili] fix extraction (#15188) 
+
+
 version 2018.01.07
 
 Core
-- 
cgit v1.1


From a39e15c516865259735bd8f4f5629de5b0e77847 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sun, 7 Jan 2018 22:15:44 +0100
Subject: [canalplus] fix extraction(closes #15072)

---
 youtube_dl/extractor/canalplus.py | 99 +++++----------------------------------
 1 file changed, 12 insertions(+), 87 deletions(-)

diff --git a/youtube_dl/extractor/canalplus.py b/youtube_dl/extractor/canalplus.py
index d8bf073..51c11cb 100644
--- a/youtube_dl/extractor/canalplus.py
+++ b/youtube_dl/extractor/canalplus.py
@@ -4,59 +4,36 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_parse_urlparse
 from ..utils import (
-    dict_get,
     # ExtractorError,
     # HEADRequest,
     int_or_none,
     qualities,
-    remove_end,
     unified_strdate,
 )
 
 
 class CanalplusIE(InfoExtractor):
-    IE_DESC = 'canalplus.fr, piwiplus.fr and d8.tv'
-    _VALID_URL = r'''(?x)
-                        https?://
-                            (?:
-                                (?:
-                                    (?:(?:www|m)\.)?canalplus\.fr|
-                                    (?:www\.)?piwiplus\.fr|
-                                    (?:www\.)?d8\.tv|
-                                    (?:www\.)?c8\.fr|
-                                    (?:www\.)?d17\.tv|
-                                    (?:(?:football|www)\.)?cstar\.fr|
-                                    (?:www\.)?itele\.fr
-                                )/(?:(?:[^/]+/)*(?P<display_id>[^/?#&]+))?(?:\?.*\bvid=(?P<vid>\d+))?|
-                                player\.canalplus\.fr/#/(?P<id>\d+)
-                            )
-
-                    '''
+    IE_DESC = 'mycanal.fr and piwiplus.fr'
+    _VALID_URL = r'https?://(?:www\.)?(?P<site>mycanal|piwiplus)\.fr/(?:[^/]+/)*(?P<display_id>[^?/]+)(?:\.html\?.*\bvid=|/p/)(?P<id>\d+)'
     _VIDEO_INFO_TEMPLATE = 'http://service.canal-plus.com/video/rest/getVideosLiees/%s/%s?format=json'
     _SITE_ID_MAP = {
-        'canalplus': 'cplus',
+        'mycanal': 'cplus',
         'piwiplus': 'teletoon',
-        'd8': 'd8',
-        'c8': 'd8',
-        'd17': 'd17',
-        'cstar': 'd17',
-        'itele': 'itele',
     }
 
     # Only works for direct mp4 URLs
     _GEO_COUNTRIES = ['FR']
 
     _TESTS = [{
-        'url': 'http://www.canalplus.fr/c-emissions/pid1830-c-zapping.html?vid=1192814',
+        'url': 'https://www.mycanal.fr/d17-emissions/lolywood/p/1397061',
         'info_dict': {
-            'id': '1405510',
-            'display_id': 'pid1830-c-zapping',
+            'id': '1397061',
+            'display_id': 'lolywood',
             'ext': 'mp4',
-            'title': 'Zapping - 02/07/2016',
-            'description': 'Le meilleur de toutes les chaînes, tous les jours',
-            'upload_date': '20160702',
+            'title': 'Euro 2016 : Je préfère te prévenir - Lolywood - Episode 34',
+            'description': 'md5:7d97039d455cb29cdba0d652a0efaa5e',
+            'upload_date': '20160602',
         },
     }, {
         # geo restricted, bypassed
@@ -70,64 +47,12 @@ class CanalplusIE(InfoExtractor):
             'upload_date': '20140724',
         },
         'expected_warnings': ['HTTP Error 403: Forbidden'],
-    }, {
-        # geo restricted, bypassed
-        'url': 'http://www.c8.fr/c8-divertissement/ms-touche-pas-a-mon-poste/pid6318-videos-integrales.html?vid=1443684',
-        'md5': 'bb6f9f343296ab7ebd88c97b660ecf8d',
-        'info_dict': {
-            'id': '1443684',
-            'display_id': 'pid6318-videos-integrales',
-            'ext': 'mp4',
-            'title': 'Guess my iep ! - TPMP - 07/04/2017',
-            'description': 'md5:6f005933f6e06760a9236d9b3b5f17fa',
-            'upload_date': '20170407',
-        },
-        'expected_warnings': ['HTTP Error 403: Forbidden'],
-    }, {
-        'url': 'http://www.itele.fr/chroniques/invite-michael-darmon/rachida-dati-nicolas-sarkozy-est-le-plus-en-phase-avec-les-inquietudes-des-francais-171510',
-        'info_dict': {
-            'id': '1420176',
-            'display_id': 'rachida-dati-nicolas-sarkozy-est-le-plus-en-phase-avec-les-inquietudes-des-francais-171510',
-            'ext': 'mp4',
-            'title': 'L\'invité de Michaël Darmon du 14/10/2016 - ',
-            'description': 'Chaque matin du lundi au vendredi, Michaël Darmon reçoit un invité politique à 8h25.',
-            'upload_date': '20161014',
-        },
-    }, {
-        'url': 'http://football.cstar.fr/cstar-minisite-foot/pid7566-feminines-videos.html?vid=1416769',
-        'info_dict': {
-            'id': '1416769',
-            'display_id': 'pid7566-feminines-videos',
-            'ext': 'mp4',
-            'title': 'France - Albanie : les temps forts de la soirée - 20/09/2016',
-            'description': 'md5:c3f30f2aaac294c1c969b3294de6904e',
-            'upload_date': '20160921',
-        },
-        'params': {
-            'skip_download': True,
-        },
-    }, {
-        'url': 'http://m.canalplus.fr/?vid=1398231',
-        'only_matching': True,
-    }, {
-        'url': 'http://www.d17.tv/emissions/pid8303-lolywood.html?vid=1397061',
-        'only_matching': True,
     }]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-
-        site_id = self._SITE_ID_MAP[compat_urllib_parse_urlparse(url).netloc.rsplit('.', 2)[-2]]
-
-        # Beware, some subclasses do not define an id group
-        display_id = remove_end(dict_get(mobj.groupdict(), ('display_id', 'id', 'vid')), '.html')
+        site, display_id, video_id = re.match(self._VALID_URL, url).groups()
 
-        webpage = self._download_webpage(url, display_id)
-        video_id = self._search_regex(
-            [r'<canal:player[^>]+?videoId=(["\'])(?P<id>\d+)',
-             r'id=["\']canal_video_player(?P<id>\d+)',
-             r'data-video=["\'](?P<id>\d+)'],
-            webpage, 'video id', default=mobj.group('vid'), group='id')
+        site_id = self._SITE_ID_MAP[site]
 
         info_url = self._VIDEO_INFO_TEMPLATE % (site_id, video_id)
         video_data = self._download_json(info_url, video_id, 'Downloading video JSON')
@@ -161,7 +86,7 @@ class CanalplusIE(InfoExtractor):
                     format_url + '?hdcore=2.11.3', video_id, f4m_id=format_id, fatal=False))
             else:
                 formats.append({
-                    # the secret extracted ya function in http://player.canalplus.fr/common/js/canalPlayer.js
+                    # the secret extracted from ya function in http://player.canalplus.fr/common/js/canalPlayer.js
                     'url': format_url + '?secret=pqzerjlsmdkjfoiuerhsdlfknaes',
                     'format_id': format_id,
                     'preference': preference(format_id),
-- 
cgit v1.1


From 8005dc68cbdfc15b6353a071ef87d7e57d69ff59 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 8 Jan 2018 21:53:03 +0700
Subject: [ok] Add support for live streams

---
 youtube_dl/extractor/odnoklassniki.py | 26 +++++++++++++++++++++++---
 1 file changed, 23 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/odnoklassniki.py b/youtube_dl/extractor/odnoklassniki.py
index 8e13bcf..5c8b37e 100644
--- a/youtube_dl/extractor/odnoklassniki.py
+++ b/youtube_dl/extractor/odnoklassniki.py
@@ -19,11 +19,11 @@ from ..utils import (
 
 
 class OdnoklassnikiIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:(?:www|m|mobile)\.)?(?:odnoklassniki|ok)\.ru/(?:video(?:embed)?|web-api/video/moviePlayer)/(?P<id>[\d-]+)'
+    _VALID_URL = r'https?://(?:(?:www|m|mobile)\.)?(?:odnoklassniki|ok)\.ru/(?:video(?:embed)?|web-api/video/moviePlayer|live)/(?P<id>[\d-]+)'
     _TESTS = [{
         # metadata in JSON
         'url': 'http://ok.ru/video/20079905452',
-        'md5': '6ba728d85d60aa2e6dd37c9e70fdc6bc',
+        'md5': '0b62089b479e06681abaaca9d204f152',
         'info_dict': {
             'id': '20079905452',
             'ext': 'mp4',
@@ -35,7 +35,6 @@ class OdnoklassnikiIE(InfoExtractor):
             'like_count': int,
             'age_limit': 0,
         },
-        'skip': 'Video has been blocked',
     }, {
         # metadataUrl
         'url': 'http://ok.ru/video/63567059965189-0?fromTime=5',
@@ -99,6 +98,9 @@ class OdnoklassnikiIE(InfoExtractor):
     }, {
         'url': 'http://mobile.ok.ru/video/20079905452',
         'only_matching': True,
+    }, {
+        'url': 'https://www.ok.ru/live/484531969818',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
@@ -184,6 +186,10 @@ class OdnoklassnikiIE(InfoExtractor):
             })
             return info
 
+        assert title
+        if provider == 'LIVE_TV_APP':
+            info['title'] = self._live_title(title)
+
         quality = qualities(('4', '0', '1', '2', '3', '5'))
 
         formats = [{
@@ -210,6 +216,20 @@ class OdnoklassnikiIE(InfoExtractor):
             if fmt_type:
                 fmt['quality'] = quality(fmt_type)
 
+        # Live formats
+        m3u8_url = metadata.get('hlsMasterPlaylistUrl')
+        if m3u8_url:
+            formats.extend(self._extract_m3u8_formats(
+                m3u8_url, video_id, 'mp4', entry_protocol='m3u8',
+                m3u8_id='hls', fatal=False))
+        rtmp_url = metadata.get('rtmpUrl')
+        if rtmp_url:
+            formats.append({
+                'url': rtmp_url,
+                'format_id': 'rtmp',
+                'ext': 'flv',
+            })
+
         self._sort_formats(formats)
 
         info['formats'] = formats
-- 
cgit v1.1


From 5eca00a2e33a6ca26a7f52589e5d77bab7e5edf4 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 9 Jan 2018 18:12:55 +0800
Subject: [weibo] Misc improvements

---
 youtube_dl/extractor/weibo.py | 133 ++++++++++++++++++++++--------------------
 1 file changed, 70 insertions(+), 63 deletions(-)

diff --git a/youtube_dl/extractor/weibo.py b/youtube_dl/extractor/weibo.py
index cbe0c32..3cb4d71 100644
--- a/youtube_dl/extractor/weibo.py
+++ b/youtube_dl/extractor/weibo.py
@@ -8,7 +8,8 @@ import random
 import re
 
 from ..compat import (
-    compat_urlparse,
+    compat_parse_qs,
+    compat_str,
 )
 from ..utils import (
     js_to_json,
@@ -31,70 +32,71 @@ class WeiboIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
         # to get Referer url for genvisitor
-        webpage, urlh = self._download_webpage_handle(url, video_id, note="first visit the page")
+        webpage, urlh = self._download_webpage_handle(url, video_id)
 
         visitor_url = urlh.geturl()
-        headers = {
-            'Referer': visitor_url
-        }
 
-        fp = {
-            "os": "2",
-            "browser": "Gecko57,0,0,0",
-            "fonts": "undefined",
-            "screenInfo": "1440*900*24",
-            "plugins": ""
-        }
-        data = urlencode_postdata({
-            "cb": "gen_callback",
-            "fp": json.dumps(fp),
-        })
-
-        genvisitor_url = 'https://passport.weibo.com/visitor/genvisitor'
-        webpage = self._download_webpage(genvisitor_url, video_id, data=data, headers=headers, note="gen visitor")
-
-        p = strip_jsonp(webpage)
-        i1 = p.find('{')
-        i2 = p.rfind('}')
-        j = p[i1:i2 + 1]  # get JSON object
-        d = json.loads(j)
-        tid = d["data"]["tid"]
-        cnfd = "%03d" % d["data"]["confidence"]
-
-        query = {
-            'a': 'incarnate',
-            't': tid,
-            'w': 2,
-            'c': cnfd,
-            'cb': 'cross_domain',
-            'from': 'weibo',
-            '_rand': random.random()
-        }
-        gencallback_url = "https://passport.weibo.com/visitor/visitor"
-        self._download_webpage(gencallback_url, video_id, note="gen callback", query=query)
+        if 'passport.weibo.com' in visitor_url:
+            # first visit
+            visitor_data = self._download_json(
+                'https://passport.weibo.com/visitor/genvisitor', video_id,
+                note='Generating first-visit data',
+                transform_source=strip_jsonp,
+                headers={'Referer': visitor_url},
+                data=urlencode_postdata({
+                    'cb': 'gen_callback',
+                    'fp': json.dumps({
+                        'os': '2',
+                        'browser': 'Gecko57,0,0,0',
+                        'fonts': 'undefined',
+                        'screenInfo': '1440*900*24',
+                        'plugins': '',
+                    }),
+                }))
+
+            tid = visitor_data['data']['tid']
+            cnfd = '%03d' % visitor_data['data']['confidence']
+
+            self._download_webpage(
+                'https://passport.weibo.com/visitor/visitor', video_id,
+                note='Running first-visit callback',
+                query={
+                    'a': 'incarnate',
+                    't': tid,
+                    'w': 2,
+                    'c': cnfd,
+                    'cb': 'cross_domain',
+                    'from': 'weibo',
+                    '_rand': random.random(),
+                })
+
+            webpage = self._download_webpage(
+                url, video_id, note='Revisiting webpage')
+
+        title = self._html_search_regex(
+            r'<title>(.+?)</title>', webpage, 'title')
+
+        video_formats = compat_parse_qs(self._search_regex(
+            r'video-sources=\\\"(.+?)\"', webpage, 'video_sources'))
 
-        webpage = self._download_webpage(url, video_id, note="retry to visit the page")
+        formats = []
+        supported_resolutions = (480, 720)
+        for res in supported_resolutions:
+            vid_urls = video_formats.get(compat_str(res))
+            if not vid_urls or not isinstance(vid_urls, list):
+                continue
 
-        title = self._html_search_regex(r'<title>(.+?)</title>', webpage, 'title')
+            vid_url = vid_urls[0]
+            formats.append({
+                'url': vid_url,
+                'height': res,
+            })
 
-        video_sources_text = self._search_regex(r'video-sources=\\\"(.+?)\"', webpage, 'video_sources')
+        self._sort_formats(formats)
 
-        video_formats = compat_urlparse.parse_qs(video_sources_text)
+        uploader = self._og_search_property(
+            'nick-name', webpage, 'uploader', default=None)
 
-        formats = []
-        supported_resolutions = ('720', '480')
-        for res in supported_resolutions:
-            f = video_formats.get(res)
-            if isinstance(f, list):
-                if len(f) > 0:
-                    vid_url = f[0]
-                    formats.append({
-                        'url': vid_url,
-                        'format': 'mp4',
-                        'height': int(res),
-                    })
-        self._sort_formats(formats)
-        uploader = self._og_search_property('nick-name', webpage, 'uploader', default=None)
         return {
             'id': video_id,
             'title': title,
@@ -118,12 +120,17 @@ class WeiboMobileIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
         # to get Referer url for genvisitor
-        webpage = self._download_webpage(url, video_id, note="visit the page")
-        js_code = self._search_regex(r'var\s+\$render_data\s*=\s*\[({.*})\]\[0\] \|\| {};', webpage, 'js_code', flags=re.DOTALL)
-        weibo_info = self._parse_json(js_code, video_id, transform_source=js_to_json)
-        page_info = weibo_info.get('status').get('page_info')
-        title = weibo_info.get('status').get('status_title')
-        uploader = weibo_info.get('status').get('user').get('screen_name')
+        webpage = self._download_webpage(url, video_id, note='visit the page')
+
+        weibo_info = self._parse_json(self._search_regex(
+            r'var\s+\$render_data\s*=\s*\[({.*})\]\[0\]\s*\|\|\s*{};',
+            webpage, 'js_code', flags=re.DOTALL),
+            video_id, transform_source=js_to_json)
+
+        status_data = weibo_info.get('status', {})
+        page_info = status_data.get('page_info')
+        title = status_data['status_title']
+        uploader = status_data.get('user', {}).get('screen_name')
 
         return {
             'id': video_id,
-- 
cgit v1.1


From 0f71de076144f59fae0b3b7e9a5251f44449cd9b Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 9 Jan 2018 18:13:49 +0800
Subject: [ChangeLog] Update after #15079

---
 ChangeLog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ChangeLog b/ChangeLog
index 9d37cdc..51825cc 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,6 +1,7 @@
 version <unreleased>
 
 Extractors
++ [weibo] Add extractor (#15079)
 * [bilibili] fix extraction (#15188) 
 
 
-- 
cgit v1.1


From 5b23845125ba20b83ab3a41fb8ff4b34e460a5dd Mon Sep 17 00:00:00 2001
From: Chih-Hsuan Yen <yan12125@gmail.com>
Date: Tue, 9 Jan 2018 19:35:39 +0800
Subject: Credit @sprhawk for the Weibo extractor (#15079)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 7e01224..5a090a3 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -231,3 +231,4 @@ John Dong
 Tatsuyuki Ishi
 Daniel Weber
 Kay Bouché
+Yang Hongbo
-- 
cgit v1.1


From 310ea4661ddaea002c86d0ebbf4663b6c943b8f4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 9 Jan 2018 22:04:50 +0700
Subject: [ndr:embed:base] Make separate formats extraction non fatal (closes
 #15203)

---
 youtube_dl/extractor/ndr.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/ndr.py b/youtube_dl/extractor/ndr.py
index 07528d1..aec2ea1 100644
--- a/youtube_dl/extractor/ndr.py
+++ b/youtube_dl/extractor/ndr.py
@@ -190,10 +190,12 @@ class NDREmbedBaseIE(InfoExtractor):
             ext = determine_ext(src, None)
             if ext == 'f4m':
                 formats.extend(self._extract_f4m_formats(
-                    src + '?hdcore=3.7.0&plugin=aasp-3.7.0.39.44', video_id, f4m_id='hds'))
+                    src + '?hdcore=3.7.0&plugin=aasp-3.7.0.39.44', video_id,
+                    f4m_id='hds', fatal=False))
             elif ext == 'm3u8':
                 formats.extend(self._extract_m3u8_formats(
-                    src, video_id, 'mp4', m3u8_id='hls', entry_protocol='m3u8_native'))
+                    src, video_id, 'mp4', m3u8_id='hls',
+                    entry_protocol='m3u8_native', fatal=False))
             else:
                 quality = f.get('quality')
                 ff = {
-- 
cgit v1.1


From 2b4e1ace4ac422acbe63be2f8cc23429de6812b8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 10 Jan 2018 05:36:03 +0700
Subject: [limelight] Tolerate empty pc formats (closes #15150, closes #15151,
 closes #15207)

---
 youtube_dl/extractor/limelight.py | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/limelight.py b/youtube_dl/extractor/limelight.py
index ad65b27..2803d7e 100644
--- a/youtube_dl/extractor/limelight.py
+++ b/youtube_dl/extractor/limelight.py
@@ -10,6 +10,7 @@ from ..utils import (
     float_or_none,
     int_or_none,
     smuggle_url,
+    try_get,
     unsmuggle_url,
     ExtractorError,
 )
@@ -220,6 +221,12 @@ class LimelightBaseIE(InfoExtractor):
             'subtitles': subtitles,
         }
 
+    def _extract_info_helper(self, pc, mobile, i, metadata):
+        return self._extract_info(
+            try_get(pc, lambda x: x['playlistItems'][i]['streams'], list) or [],
+            try_get(mobile, lambda x: x['mediaList'][i]['mobileUrls'], list) or [],
+            metadata)
+
 
 class LimelightMediaIE(LimelightBaseIE):
     IE_NAME = 'limelight'
@@ -282,10 +289,7 @@ class LimelightMediaIE(LimelightBaseIE):
             'getMobilePlaylistByMediaId', 'properties',
             smuggled_data.get('source_url'))
 
-        return self._extract_info(
-            pc['playlistItems'][0].get('streams', []),
-            mobile['mediaList'][0].get('mobileUrls', []) if mobile else [],
-            metadata)
+        return self._extract_info_helper(pc, mobile, 0, metadata)
 
 
 class LimelightChannelIE(LimelightBaseIE):
@@ -326,10 +330,7 @@ class LimelightChannelIE(LimelightBaseIE):
             'media', smuggled_data.get('source_url'))
 
         entries = [
-            self._extract_info(
-                pc['playlistItems'][i].get('streams', []),
-                mobile['mediaList'][i].get('mobileUrls', []) if mobile else [],
-                medias['media_list'][i])
+            self._extract_info_helper(pc, mobile, i, medias['media_list'][i])
             for i in range(len(medias['media_list']))]
 
         return self.playlist_result(entries, channel_id, pc['title'])
-- 
cgit v1.1


From e654829b4c4b8ebd4efb4554dd02cc1418c6fc23 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 10 Jan 2018 21:24:22 +0700
Subject: [digg] Add extractor (closes #15214)

---
 youtube_dl/extractor/digg.py       | 41 ++++++++++++++++++++++++++++++++++++++
 youtube_dl/extractor/extractors.py |  1 +
 2 files changed, 42 insertions(+)
 create mode 100644 youtube_dl/extractor/digg.py

diff --git a/youtube_dl/extractor/digg.py b/youtube_dl/extractor/digg.py
new file mode 100644
index 0000000..611134a
--- /dev/null
+++ b/youtube_dl/extractor/digg.py
@@ -0,0 +1,41 @@
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class DiggIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?digg\.com/video/(?P<id>[^/?#&]+)'
+    _TEST = {
+        'url': 'http://digg.com/video/sci-fi-short-jonah-daniel-kaluuya-get-out',
+        'info_dict': {
+            'id': 'LcqvmS0b',
+            'ext': 'mp4',
+            'title': "'Get Out' Star Daniel Kaluuya Goes On 'Moby Dick'-Like Journey In Sci-Fi Short 'Jonah'",
+            'description': 'md5:541bb847648b6ee3d6514bc84b82efda',
+            'upload_date': '20180109',
+            'timestamp': 1515530551,
+        },
+        'params': {
+            'skip_download': True,
+        },
+    }
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, display_id)
+
+        jwplatform_id = self._search_regex(
+            r'video_id\s*:\s*["\']([a-zA-Z0-9]{8})', webpage, 'jwplatform id',
+            default=None)
+
+        if not jwplatform_id:
+            return self.url_result(url, 'Generic')
+
+        return {
+            '_type': 'url_transparent',
+            'ie_key': 'JWPlatform',
+            'url': 'jwplatform:%s' % jwplatform_id,
+            'id': jwplatform_id,
+            'display_id': display_id,
+        }
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index a3ad4df..3bfd1b7 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -259,6 +259,7 @@ from .deezer import DeezerPlaylistIE
 from .democracynow import DemocracynowIE
 from .dfb import DFBIE
 from .dhm import DHMIE
+from .digg import DiggIE
 from .dotsub import DotsubIE
 from .douyutv import (
     DouyuShowIE,
-- 
cgit v1.1


From 1b79daffd965fcb3776e8304bd393db6573b50ea Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 10 Jan 2018 22:19:51 +0700
Subject: [digg] Improve extraction

---
 youtube_dl/extractor/digg.py | 47 +++++++++++++++++++++++++++++---------------
 1 file changed, 31 insertions(+), 16 deletions(-)

diff --git a/youtube_dl/extractor/digg.py b/youtube_dl/extractor/digg.py
index 611134a..913c175 100644
--- a/youtube_dl/extractor/digg.py
+++ b/youtube_dl/extractor/digg.py
@@ -1,11 +1,13 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..utils import js_to_json
 
 
 class DiggIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?digg\.com/video/(?P<id>[^/?#&]+)'
-    _TEST = {
+    _TESTS = [{
+        # JWPlatform via provider
         'url': 'http://digg.com/video/sci-fi-short-jonah-daniel-kaluuya-get-out',
         'info_dict': {
             'id': 'LcqvmS0b',
@@ -18,24 +20,37 @@ class DiggIE(InfoExtractor):
         'params': {
             'skip_download': True,
         },
-    }
+    }, {
+        # Youtube via provider
+        'url': 'http://digg.com/video/dog-boat-seal-play',
+        'only_matching': True,
+    }, {
+        # vimeo as regular embed
+        'url': 'http://digg.com/video/dream-girl-short-film',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         display_id = self._match_id(url)
 
         webpage = self._download_webpage(url, display_id)
 
-        jwplatform_id = self._search_regex(
-            r'video_id\s*:\s*["\']([a-zA-Z0-9]{8})', webpage, 'jwplatform id',
-            default=None)
-
-        if not jwplatform_id:
-            return self.url_result(url, 'Generic')
-
-        return {
-            '_type': 'url_transparent',
-            'ie_key': 'JWPlatform',
-            'url': 'jwplatform:%s' % jwplatform_id,
-            'id': jwplatform_id,
-            'display_id': display_id,
-        }
+        info = self._parse_json(
+            self._search_regex(
+                r'(?s)video_info\s*=\s*({.+?});\n', webpage, 'video info',
+                default='{}'), display_id, transform_source=js_to_json,
+            fatal=False)
+
+        video_id = info.get('video_id')
+
+        if video_id:
+            provider = info.get('provider_name')
+            if provider == 'youtube':
+                return self.url_result(
+                    video_id, ie='Youtube', video_id=video_id)
+            elif provider == 'jwplayer':
+                return self.url_result(
+                    'jwplatform:%s' % video_id, ie='JWPlatform',
+                    video_id=video_id)
+
+        return self.url_result(url, 'Generic')
-- 
cgit v1.1


From a90641fe87d62936b717b9c2fdbe453578b441ed Mon Sep 17 00:00:00 2001
From: scil <scil.zc@gmail.com>
Date: Thu, 11 Jan 2018 20:35:09 +0800
Subject: [ximalaya_extractor] Add new extractor ximalaya  (#14687)

* [ximalaya_extractor] Add new extractor

* format change according by flake8

* changes accoring to review by @yan12125 at github pull #14687

* change %d to %s in a temp str

* seond changes accoring to review by @yan12125 at github pull #1468

* improve TESTS about contains

* changes accoring to third review by @yan12125 at github pull #1468

* forth changes accoring to forth review by @yan12125 at github pull #1468
---
 youtube_dl/extractor/extractors.py |   4 +
 youtube_dl/extractor/ximalaya.py   | 233 +++++++++++++++++++++++++++++++++++++
 2 files changed, 237 insertions(+)
 create mode 100644 youtube_dl/extractor/ximalaya.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 3bfd1b7..37624d3 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1328,6 +1328,10 @@ from .xiami import (
     XiamiArtistIE,
     XiamiCollectionIE
 )
+from .ximalaya import (
+    XimalayaIE,
+    XimalayaAlbumIE
+)
 from .xminus import XMinusIE
 from .xnxx import XNXXIE
 from .xstream import XstreamIE
diff --git a/youtube_dl/extractor/ximalaya.py b/youtube_dl/extractor/ximalaya.py
new file mode 100644
index 0000000..a912e54
--- /dev/null
+++ b/youtube_dl/extractor/ximalaya.py
@@ -0,0 +1,233 @@
+# coding: utf-8
+
+from __future__ import unicode_literals
+
+import itertools
+import re
+
+from .common import InfoExtractor
+
+
+class XimalayaBaseIE(InfoExtractor):
+    _GEO_COUNTRIES = ['CN']
+
+
+class XimalayaIE(XimalayaBaseIE):
+    IE_NAME = 'ximalaya'
+    IE_DESC = '喜马拉雅FM'
+    _VALID_URL = r'https?://(?:www\.|m\.)?ximalaya\.com/(?P<uid>[0-9]+)/sound/(?P<id>[0-9]+)'
+    _USER_URL_FORMAT = '%s://www.ximalaya.com/zhubo/%i/'
+    _TESTS = [
+        {
+            'url': 'http://www.ximalaya.com/61425525/sound/47740352/',
+            'info_dict': {
+                'id': '47740352',
+                'ext': 'm4a',
+                'uploader': '小彬彬爱听书',
+                'uploader_id': 61425525,
+                'uploader_url': 'http://www.ximalaya.com/zhubo/61425525/',
+                'title': '261.唐诗三百首.卷八.送孟浩然之广陵.李白',
+                'description': "contains:《送孟浩然之广陵》\n作者：李白\n故人西辞黄鹤楼，烟花三月下扬州。\n孤帆远影碧空尽，惟见长江天际流。",
+                'thumbnails': [
+                    {
+                        'name': 'cover_url',
+                        'url': r're:^https?://.*\.jpg$',
+                    },
+                    {
+                        'name': 'cover_url_142',
+                        'url': r're:^https?://.*\.jpg$',
+                        'width': 180,
+                        'height': 180
+                    }
+                ],
+                'categories': ['renwen', '人文'],
+                'duration': 93,
+                'view_count': int,
+                'like_count': int,
+            }
+        },
+        {
+            'url': 'http://m.ximalaya.com/61425525/sound/47740352/',
+            'info_dict': {
+                'id': '47740352',
+                'ext': 'm4a',
+                'uploader': '小彬彬爱听书',
+                'uploader_id': 61425525,
+                'uploader_url': 'http://www.ximalaya.com/zhubo/61425525/',
+                'title': '261.唐诗三百首.卷八.送孟浩然之广陵.李白',
+                'description': "contains:《送孟浩然之广陵》\n作者：李白\n故人西辞黄鹤楼，烟花三月下扬州。\n孤帆远影碧空尽，惟见长江天际流。",
+                'thumbnails': [
+                    {
+                        'name': 'cover_url',
+                        'url': r're:^https?://.*\.jpg$',
+                    },
+                    {
+                        'name': 'cover_url_142',
+                        'url': r're:^https?://.*\.jpg$',
+                        'width': 180,
+                        'height': 180
+                    }
+                ],
+                'categories': ['renwen', '人文'],
+                'duration': 93,
+                'view_count': int,
+                'like_count': int,
+            }
+        },
+        {
+            'url': 'https://www.ximalaya.com/11045267/sound/15705996/',
+            'info_dict': {
+                'id': '15705996',
+                'ext': 'm4a',
+                'uploader': '李延隆老师',
+                'uploader_id': 11045267,
+                'uploader_url': 'https://www.ximalaya.com/zhubo/11045267/',
+                'title': 'Lesson 1 Excuse me!',
+                'description': "contains:Listen to the tape then answer\xa0this question. Whose handbag is it?\n"
+                               "听录音，然后回答问题，这是谁的手袋？",
+                'thumbnails': [
+                    {
+                        'name': 'cover_url',
+                        'url': r're:^https?://.*\.jpg$',
+                    },
+                    {
+                        'name': 'cover_url_142',
+                        'url': r're:^https?://.*\.jpg$',
+                        'width': 180,
+                        'height': 180
+                    }
+                ],
+                'categories': ['train', '外语'],
+                'duration': 40,
+                'view_count': int,
+                'like_count': int,
+            }
+        },
+    ]
+
+    def _real_extract(self, url):
+
+        is_m = 'm.ximalaya' in url
+        scheme = 'https' if url.startswith('https') else 'http'
+
+        audio_id = self._match_id(url)
+        webpage = self._download_webpage(url, audio_id,
+                                         note='Download sound page for %s' % audio_id,
+                                         errnote='Unable to get sound page')
+
+        audio_info_file = '%s://m.ximalaya.com/tracks/%s.json' % (scheme, audio_id)
+        audio_info = self._download_json(audio_info_file, audio_id,
+                                         'Downloading info json %s' % audio_info_file,
+                                         'Unable to download info file')
+
+        formats = []
+        for bps, k in (('24k', 'play_path_32'), ('64k', 'play_path_64')):
+            if audio_info.get(k):
+                formats.append({
+                    'format_id': bps,
+                    'url': audio_info[k],
+                })
+
+        thumbnails = []
+        for k in audio_info.keys():
+            # cover pics kyes like: cover_url', 'cover_url_142'
+            if k.startswith('cover_url'):
+                thumbnail = {'name': k, 'url': audio_info[k]}
+                if k == 'cover_url_142':
+                    thumbnail['width'] = 180
+                    thumbnail['height'] = 180
+                thumbnails.append(thumbnail)
+
+        audio_uploader_id = audio_info.get('uid')
+
+        if is_m:
+            audio_description = self._html_search_regex(r'(?s)<section\s+class=["\']content[^>]+>(.+?)</section>',
+                                                        webpage, 'audio_description', fatal=False)
+        else:
+            audio_description = self._html_search_regex(r'(?s)<div\s+class=["\']rich_intro[^>]*>(.+?</article>)',
+                                                        webpage, 'audio_description', fatal=False)
+
+        if not audio_description:
+            audio_description_file = '%s://www.ximalaya.com/sounds/%s/rich_intro' % (scheme, audio_id)
+            audio_description = self._download_webpage(audio_description_file, audio_id,
+                                                       note='Downloading description file %s' % audio_description_file,
+                                                       errnote='Unable to download descrip file',
+                                                       fatal=False)
+            audio_description = audio_description.strip() if audio_description else None
+
+        return {
+            'id': audio_id,
+            'uploader': audio_info.get('nickname'),
+            'uploader_id': audio_uploader_id,
+            'uploader_url': self._USER_URL_FORMAT % (scheme, audio_uploader_id) if audio_uploader_id else None,
+            'title': audio_info['title'],
+            'thumbnails': thumbnails,
+            'description': audio_description,
+            'categories': list(filter(None, (audio_info.get('category_name'), audio_info.get('category_title')))),
+            'duration': audio_info.get('duration'),
+            'view_count': audio_info.get('play_count'),
+            'like_count': audio_info.get('favorites_count'),
+            'formats': formats,
+        }
+
+
+class XimalayaAlbumIE(XimalayaBaseIE):
+    IE_NAME = 'ximalaya:album'
+    IE_DESC = '喜马拉雅FM 专辑'
+    _VALID_URL = r'https?://(?:www\.|m\.)?ximalaya\.com/(?P<uid>[0-9]+)/album/(?P<id>[0-9]+)'
+    _TEMPLATE_URL = '%s://www.ximalaya.com/%s/album/%s/'
+    _BASE_URL_TEMPL = '%s://www.ximalaya.com%s'
+    _LIST_VIDEO_RE = r'<a[^>]+?href="(?P<url>/%s/sound/(?P<id>\d+)/?)"[^>]+?title="(?P<title>[^>]+)">'
+    _TESTS = [{
+        'url': 'http://www.ximalaya.com/61425525/album/5534601/',
+        'info_dict': {
+            'title': '唐诗三百首（含赏析）',
+            'id': '5534601',
+        },
+        'playlist_count': 312,
+    }, {
+        'url': 'http://m.ximalaya.com/61425525/album/5534601',
+        'info_dict': {
+            'title': '唐诗三百首（含赏析）',
+            'id': '5534601',
+        },
+        'playlist_count': 312,
+    },
+    ]
+
+    def _real_extract(self, url):
+        self.scheme = scheme = 'https' if url.startswith('https') else 'http'
+
+        mobj = re.match(self._VALID_URL, url)
+        uid, playlist_id = mobj.group('uid'), mobj.group('id')
+
+        webpage = self._download_webpage(self._TEMPLATE_URL % (scheme, uid, playlist_id), playlist_id,
+                                         note='Download album page for %s' % playlist_id,
+                                         errnote='Unable to get album info')
+
+        title = self._html_search_regex(r'detailContent_title[^>]*><h1(?:[^>]+)?>([^<]+)</h1>',
+                                        webpage, 'title', fatal=False)
+
+        return self.playlist_result(self._entries(webpage, playlist_id, uid), playlist_id, title)
+
+    def _entries(self, page, playlist_id, uid):
+        html = page
+        for page_num in itertools.count(1):
+            for entry in self._process_page(html, uid):
+                yield entry
+
+            next_url = self._search_regex(r'<a\s+href=(["\'])(?P<more>[\S]+)\1[^>]+rel=(["\'])next\3',
+                                          html, 'list_next_url', default=None, group='more')
+            if not next_url:
+                break
+
+            next_full_url = self._BASE_URL_TEMPL % (self.scheme, next_url)
+            html = self._download_webpage(next_full_url, playlist_id)
+
+    def _process_page(self, html, uid):
+        find_from = html.index('album_soundlist')
+        for mobj in re.finditer(self._LIST_VIDEO_RE % uid, html[find_from:]):
+            yield self.url_result(self._BASE_URL_TEMPL % (self.scheme, mobj.group('url')),
+                                  XimalayaIE.ie_key(),
+                                  mobj.group('id'),
+                                  mobj.group('title'))
-- 
cgit v1.1


From 37941fe204ef855590af64584b75d7fc95997fc6 Mon Sep 17 00:00:00 2001
From: Chih-Hsuan Yen <yan12125@gmail.com>
Date: Thu, 11 Jan 2018 20:36:06 +0800
Subject: [ChangeLog] Update after #14687

[ci skip]
---
 ChangeLog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ChangeLog b/ChangeLog
index 51825cc..ea1be93 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,6 +1,7 @@
 version <unreleased>
 
 Extractors
++ [ximalaya] Add extractor (#14687)
 + [weibo] Add extractor (#15079)
 * [bilibili] fix extraction (#15188) 
 
-- 
cgit v1.1


From 64287560e4a7af9401e84318d9d04783c1b289af Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 11 Jan 2018 23:06:56 +0700
Subject: [pandoratv] Add support for new URL format (closes #15131)

---
 youtube_dl/extractor/pandoratv.py | 28 ++++++++++++++++++++++------
 1 file changed, 22 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/pandoratv.py b/youtube_dl/extractor/pandoratv.py
index fc7bd34..0c27a61 100644
--- a/youtube_dl/extractor/pandoratv.py
+++ b/youtube_dl/extractor/pandoratv.py
@@ -1,6 +1,8 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 from ..compat import (
     compat_str,
@@ -18,7 +20,13 @@ from ..utils import (
 class PandoraTVIE(InfoExtractor):
     IE_NAME = 'pandora.tv'
     IE_DESC = '판도라TV'
-    _VALID_URL = r'https?://(?:.+?\.)?channel\.pandora\.tv/channel/video\.ptv\?'
+    _VALID_URL = r'''(?x)
+                        https?://
+                            (?:
+                                (?:www\.)?pandora\.tv/view/(?P<user_id>[^/]+)/(?P<id>\d+)|  # new format
+                                (?:.+?\.)?channel\.pandora\.tv/channel/video\.ptv\?         # old format
+                            )
+                    '''
     _TESTS = [{
         'url': 'http://jp.channel.pandora.tv/channel/video.ptv?c1=&prgid=53294230&ch_userid=mikakim&ref=main&lot=cate_01_2',
         'info_dict': {
@@ -53,14 +61,22 @@ class PandoraTVIE(InfoExtractor):
             # Test metadata only
             'skip_download': True,
         },
+    }, {
+        'url': 'http://www.pandora.tv/view/mikakim/53294230#36797454_new',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
-        qs = compat_urlparse.parse_qs(compat_urlparse.urlparse(url).query)
-        video_id = qs.get('prgid', [None])[0]
-        user_id = qs.get('ch_userid', [None])[0]
-        if any(not f for f in (video_id, user_id,)):
-            raise ExtractorError('Invalid URL', expected=True)
+        mobj = re.match(self._VALID_URL, url)
+        user_id = mobj.group('user_id')
+        video_id = mobj.group('id')
+
+        if not user_id or not video_id:
+            qs = compat_urlparse.parse_qs(compat_urlparse.urlparse(url).query)
+            video_id = qs.get('prgid', [None])[0]
+            user_id = qs.get('ch_userid', [None])[0]
+            if any(not f for f in (video_id, user_id,)):
+                raise ExtractorError('Invalid URL', expected=True)
 
         data = self._download_json(
             'http://m.pandora.tv/?c=view&m=viewJsonApi&ch_userid=%s&prgid=%s'
-- 
cgit v1.1


From 609850acfb6c03dcdfa4d9cdba77df3b0a259968 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 11 Jan 2018 23:10:18 +0700
Subject: [pandoratv] Add support for mobile URLs (closes #12441)

---
 youtube_dl/extractor/pandoratv.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/pandoratv.py b/youtube_dl/extractor/pandoratv.py
index 0c27a61..538738c 100644
--- a/youtube_dl/extractor/pandoratv.py
+++ b/youtube_dl/extractor/pandoratv.py
@@ -24,7 +24,8 @@ class PandoraTVIE(InfoExtractor):
                         https?://
                             (?:
                                 (?:www\.)?pandora\.tv/view/(?P<user_id>[^/]+)/(?P<id>\d+)|  # new format
-                                (?:.+?\.)?channel\.pandora\.tv/channel/video\.ptv\?         # old format
+                                (?:.+?\.)?channel\.pandora\.tv/channel/video\.ptv\?|        # old format
+                                m\.pandora\.tv/?\?                                          # mobile
                             )
                     '''
     _TESTS = [{
@@ -64,6 +65,9 @@ class PandoraTVIE(InfoExtractor):
     }, {
         'url': 'http://www.pandora.tv/view/mikakim/53294230#36797454_new',
         'only_matching': True,
+    }, {
+        'url': 'http://m.pandora.tv/?c=view&ch_userid=mikakim&prgid=54600346',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
-- 
cgit v1.1


From e565a6386e61f7741a5386520f1b36efe2cb3310 Mon Sep 17 00:00:00 2001
From: Chih-Hsuan Yen <yan12125@gmail.com>
Date: Fri, 12 Jan 2018 15:36:01 +0800
Subject: Credit @scil for ximalaya extractor (#14687)

[ci skip]
---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 5a090a3..40215a5 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -232,3 +232,4 @@ Tatsuyuki Ishi
 Daniel Weber
 Kay Bouché
 Yang Hongbo
+Lei Wang
-- 
cgit v1.1


From 47e2a9bc53c1f4a10dda62e473ec553108f7ee89 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 13 Jan 2018 18:47:47 +0700
Subject: [viafree] Skip rtmp formats (closes #15232)

---
 youtube_dl/extractor/tvplay.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/tvplay.py b/youtube_dl/extractor/tvplay.py
index 46132ed..84597b5 100644
--- a/youtube_dl/extractor/tvplay.py
+++ b/youtube_dl/extractor/tvplay.py
@@ -273,6 +273,8 @@ class TVPlayIE(InfoExtractor):
                     'ext': ext,
                 }
                 if video_url.startswith('rtmp'):
+                    if smuggled_data.get('skip_rtmp'):
+                        continue
                     m = re.search(
                         r'^(?P<url>rtmp://[^/]+/(?P<app>[^/]+))/(?P<playpath>.+)$', video_url)
                     if not m:
@@ -434,6 +436,10 @@ class ViafreeIE(InfoExtractor):
         return self.url_result(
             smuggle_url(
                 'mtg:%s' % video_id,
-                {'geo_countries': [
-                    compat_urlparse.urlparse(url).netloc.rsplit('.', 1)[-1]]}),
+                {
+                    'geo_countries': [
+                        compat_urlparse.urlparse(url).netloc.rsplit('.', 1)[-1]],
+                    # rtmp host mtgfs.fplive.net for viafree is unresolvable
+                    'skip_rtmp': True,
+                }),
             ie=TVPlayIE.ie_key(), video_id=video_id)
-- 
cgit v1.1


From d4aedca3bd82288c802d4d766d5542bfcec4a91a Mon Sep 17 00:00:00 2001
From: "Hendrik v. Raven" <hendrik@consetetur.de>
Date: Sat, 6 Jan 2018 15:09:53 +0100
Subject: [gamestar] Add support for gamepro.de (closes #3384)

---
 youtube_dl/extractor/gamestar.py | 43 +++++++++++++++++++++++-----------------
 1 file changed, 25 insertions(+), 18 deletions(-)

diff --git a/youtube_dl/extractor/gamestar.py b/youtube_dl/extractor/gamestar.py
index e607d6a..7ce2f15 100644
--- a/youtube_dl/extractor/gamestar.py
+++ b/youtube_dl/extractor/gamestar.py
@@ -9,21 +9,27 @@ from ..utils import (
 
 
 class GameStarIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?gamestar\.de/videos/.*,(?P<id>[0-9]+)\.html'
-    _TEST = {
-        'url': 'http://www.gamestar.de/videos/trailer,3/hobbit-3-die-schlacht-der-fuenf-heere,76110.html',
-        'md5': '96974ecbb7fd8d0d20fca5a00810cea7',
-        'info_dict': {
-            'id': '76110',
-            'ext': 'mp4',
-            'title': 'Hobbit 3: Die Schlacht der Fünf Heere - Teaser-Trailer zum dritten Teil',
-            'description': 'Der Teaser-Trailer zu Hobbit 3: Die Schlacht der Fünf Heere zeigt einige Szenen aus dem dritten Teil der Saga und kündigt den...',
-            'thumbnail': r're:^https?://.*\.jpg$',
-            'timestamp': 1406542020,
-            'upload_date': '20140728',
-            'duration': 17
-        }
-    }
+    _VALID_URL = r'https?://(?:www\.)?game(?:pro|star)\.de/videos/.*,(?P<id>[0-9]+)\.html'
+    _TESTS = [
+        {
+            'url': 'http://www.gamestar.de/videos/trailer,3/hobbit-3-die-schlacht-der-fuenf-heere,76110.html',
+            'md5': 'ee782f1f8050448c95c5cacd63bc851c',
+            'info_dict': {
+                'id': '76110',
+                'ext': 'mp4',
+                'title': 'Hobbit 3: Die Schlacht der Fünf Heere - Teaser-Trailer zum dritten Teil',
+                'description': 'Der Teaser-Trailer zu Hobbit 3: Die Schlacht der Fünf Heere zeigt einige Szenen aus dem dritten Teil der Saga und kündigt den...',
+                'thumbnail': r're:^https?://.*\.jpg$',
+                'timestamp': 1406542380,
+                'upload_date': '20140728',
+                'duration': 17,
+            }
+        },
+        {
+            'url': 'http://www.gamepro.de/videos/top-10-indie-spiele-fuer-nintendo-switch-video-tolle-nindies-games-zum-download,95316.html',
+            'only_matching': True,
+        },
+    ]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
@@ -38,11 +44,12 @@ class GameStarIE(InfoExtractor):
             webpage, 'JSON-LD', group='json_ld'), video_id)
         info_dict = self._json_ld(json_ld, video_id)
         info_dict['title'] = remove_end(info_dict['title'], ' - GameStar')
+        info_dict['title'] = remove_end(info_dict['title'], ' - GamePro')
 
-        view_count = json_ld.get('interactionCount')
+        view_count = int_or_none(json_ld.get('interactionCount'))
         comment_count = int_or_none(self._html_search_regex(
-            r'([0-9]+) Kommentare</span>', webpage, 'comment_count',
-            fatal=False))
+            r'<span>Kommentare</span><span class="count">\(([0-9]+)\)</span>',
+            webpage, 'comment_count', fatal=False))
 
         info_dict.update({
             'id': video_id,
-- 
cgit v1.1


From df16e645f60def2b5e1cf88d74164d6ced0d5651 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 13 Jan 2018 19:36:26 +0700
Subject: [gamestar] Fix issues (closes #15179)

---
 youtube_dl/extractor/gamestar.py | 61 +++++++++++++++++++++-------------------
 1 file changed, 32 insertions(+), 29 deletions(-)

diff --git a/youtube_dl/extractor/gamestar.py b/youtube_dl/extractor/gamestar.py
index 7ce2f15..f00dab2 100644
--- a/youtube_dl/extractor/gamestar.py
+++ b/youtube_dl/extractor/gamestar.py
@@ -1,6 +1,8 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 from ..utils import (
     int_or_none,
@@ -9,33 +11,34 @@ from ..utils import (
 
 
 class GameStarIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?game(?:pro|star)\.de/videos/.*,(?P<id>[0-9]+)\.html'
-    _TESTS = [
-        {
-            'url': 'http://www.gamestar.de/videos/trailer,3/hobbit-3-die-schlacht-der-fuenf-heere,76110.html',
-            'md5': 'ee782f1f8050448c95c5cacd63bc851c',
-            'info_dict': {
-                'id': '76110',
-                'ext': 'mp4',
-                'title': 'Hobbit 3: Die Schlacht der Fünf Heere - Teaser-Trailer zum dritten Teil',
-                'description': 'Der Teaser-Trailer zu Hobbit 3: Die Schlacht der Fünf Heere zeigt einige Szenen aus dem dritten Teil der Saga und kündigt den...',
-                'thumbnail': r're:^https?://.*\.jpg$',
-                'timestamp': 1406542380,
-                'upload_date': '20140728',
-                'duration': 17,
-            }
-        },
-        {
-            'url': 'http://www.gamepro.de/videos/top-10-indie-spiele-fuer-nintendo-switch-video-tolle-nindies-games-zum-download,95316.html',
-            'only_matching': True,
-        },
-    ]
+    _VALID_URL = r'https?://(?:www\.)?game(?P<site>pro|star)\.de/videos/.*,(?P<id>[0-9]+)\.html'
+    _TESTS = [{
+        'url': 'http://www.gamestar.de/videos/trailer,3/hobbit-3-die-schlacht-der-fuenf-heere,76110.html',
+        'md5': 'ee782f1f8050448c95c5cacd63bc851c',
+        'info_dict': {
+            'id': '76110',
+            'ext': 'mp4',
+            'title': 'Hobbit 3: Die Schlacht der Fünf Heere - Teaser-Trailer zum dritten Teil',
+            'description': 'Der Teaser-Trailer zu Hobbit 3: Die Schlacht der Fünf Heere zeigt einige Szenen aus dem dritten Teil der Saga und kündigt den...',
+            'thumbnail': r're:^https?://.*\.jpg$',
+            'timestamp': 1406542380,
+            'upload_date': '20140728',
+            'duration': 17,
+        }
+    }, {
+        'url': 'http://www.gamepro.de/videos/top-10-indie-spiele-fuer-nintendo-switch-video-tolle-nindies-games-zum-download,95316.html',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.gamestar.de/videos/top-10-indie-spiele-fuer-nintendo-switch-video-tolle-nindies-games-zum-download,95316.html',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
+        mobj = re.match(self._VALID_URL, url)
+        site = mobj.group('site')
+        video_id = mobj.group('id')
 
-        url = 'http://gamestar.de/_misc/videos/portal/getVideoUrl.cfm?premium=0&videoId=' + video_id
+        webpage = self._download_webpage(url, video_id)
 
         # TODO: there are multiple ld+json objects in the webpage,
         # while _search_json_ld finds only the first one
@@ -43,17 +46,17 @@ class GameStarIE(InfoExtractor):
             r'(?s)<script[^>]+type=(["\'])application/ld\+json\1[^>]*>(?P<json_ld>[^<]+VideoObject[^<]+)</script>',
             webpage, 'JSON-LD', group='json_ld'), video_id)
         info_dict = self._json_ld(json_ld, video_id)
-        info_dict['title'] = remove_end(info_dict['title'], ' - GameStar')
-        info_dict['title'] = remove_end(info_dict['title'], ' - GamePro')
+        info_dict['title'] = remove_end(
+            info_dict['title'], ' - Game%s' % site.title())
 
         view_count = int_or_none(json_ld.get('interactionCount'))
         comment_count = int_or_none(self._html_search_regex(
-            r'<span>Kommentare</span><span class="count">\(([0-9]+)\)</span>',
-            webpage, 'comment_count', fatal=False))
+            r'<span>Kommentare</span>\s*<span[^>]+class=["\']count[^>]+>\s*\(\s*([0-9]+)',
+            webpage, 'comment count', fatal=False))
 
         info_dict.update({
             'id': video_id,
-            'url': url,
+            'url': 'http://gamestar.de/_misc/videos/portal/getVideoUrl.cfm?premium=0&videoId=' + video_id,
             'ext': 'mp4',
             'view_count': view_count,
             'comment_count': comment_count
-- 
cgit v1.1


From 2d8bb80c60289868d479e36a90cf1b73e9221893 Mon Sep 17 00:00:00 2001
From: Sebastian Leske <sebastian.leske@sleske.name>
Date: Wed, 25 Oct 2017 14:59:57 +0200
Subject: [wdr:elefant] Add extractor

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/wdr.py        | 54 +++++++++++++++++++++++++++++++++++---
 2 files changed, 52 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 37624d3..255df75 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1289,6 +1289,7 @@ from .watchbox import WatchBoxIE
 from .watchindianporn import WatchIndianPornIE
 from .wdr import (
     WDRIE,
+    WDRElefantIE,
     WDRMobileIE,
 )
 from .webcaster import (
diff --git a/youtube_dl/extractor/wdr.py b/youtube_dl/extractor/wdr.py
index 621de1e..4871ae9 100644
--- a/youtube_dl/extractor/wdr.py
+++ b/youtube_dl/extractor/wdr.py
@@ -16,7 +16,7 @@ from ..utils import (
 
 
 class WDRBaseIE(InfoExtractor):
-    def _extract_wdr_video(self, webpage, display_id):
+    def _extract_jsonp_url(self, webpage, display_id):
         # for wdr.de the data-extension is in a tag with the class "mediaLink"
         # for wdr.de radio players, in a tag with the class "wdrrPlayerPlayBtn"
         # for wdrmaus, in a tag with the class "videoButton" (previously a link
@@ -35,8 +35,9 @@ class WDRBaseIE(InfoExtractor):
 
         media_link_obj = self._parse_json(json_metadata, display_id,
                                           transform_source=js_to_json)
-        jsonp_url = media_link_obj['mediaObj']['url']
+        return media_link_obj['mediaObj']['url']
 
+    def _extract_wdr_video(self, jsonp_url, display_id):
         metadata = self._download_json(
             jsonp_url, display_id, transform_source=strip_jsonp)
 
@@ -206,7 +207,8 @@ class WDRIE(WDRBaseIE):
         display_id = mobj.group('display_id')
         webpage = self._download_webpage(url, display_id)
 
-        info_dict = self._extract_wdr_video(webpage, display_id)
+        jsonp_url = self._extract_jsonp_url(webpage, display_id)
+        info_dict = self._extract_wdr_video(jsonp_url, display_id)
 
         if not info_dict:
             entries = [
@@ -239,6 +241,52 @@ class WDRIE(WDRBaseIE):
         return info_dict
 
 
+class WDRElefantIE(WDRBaseIE):
+    _VALID_URL = r'https?://(?:www\.)wdrmaus.de/elefantenseite/#(?P<display_id>.+)'
+    IE_NAME = 'wdr:elefant'
+
+    _TESTS = [
+        {
+            'url': 'http://www.wdrmaus.de/elefantenseite/#folge_ostern_2015',
+            'info_dict': {
+                'title': 'Folge Oster-Spezial 2015',
+                'id': 'mdb-1088195',
+                'ext': 'mp4',
+                'age_limit': None,
+                'upload_date': '20150406'
+            },
+            'params': {
+                'skip_download' : True,
+            },
+        },
+    ]
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        display_id = mobj.group('display_id')
+
+        # Table of Contents seems to always be at this address, so fetch it directly.
+        # The website fetches configurationJS.php5, which links to tableOfContentsJS.php5.
+        table_of_contents = self._download_json(
+            'https://www.wdrmaus.de/elefantenseite/data/tableOfContentsJS.php5', display_id)
+        if display_id not in table_of_contents:
+            raise ExtractorError(
+                'No entry in site\'s table of contents for this URL. '
+                'Is the fragment part of the URL (after the #) correct?',
+                expected=True)
+        xml_metadata_path = table_of_contents[display_id]['xmlPath']
+        xml_metadata = self._download_xml(
+            'https://www.wdrmaus.de/elefantenseite/' + xml_metadata_path, display_id)
+        zmdb_url_element = xml_metadata.find('./movie/zmdb_url')
+        if zmdb_url_element is None:
+            raise ExtractorError(
+                'The URL looks valid, but no video was found. Note that download only works '
+                'on pages showing a single video, not on video selection pages.',
+                expected=True)
+        info_dict = self._extract_wdr_video(zmdb_url_element.text, display_id)
+        return info_dict
+
+
 class WDRMobileIE(InfoExtractor):
     _VALID_URL = r'''(?x)
         https?://mobile-ondemand\.wdr\.de/
-- 
cgit v1.1


From 54e8f62e01b54eeccd8313349f86ae541082704f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 13 Jan 2018 23:28:08 +0700
Subject: [wdr] Rework extractors (closes #14598)

---
 youtube_dl/extractor/extractors.py |   2 +-
 youtube_dl/extractor/sportschau.py |  38 ------
 youtube_dl/extractor/wdr.py        | 232 ++++++++++++++++++++-----------------
 3 files changed, 124 insertions(+), 148 deletions(-)
 delete mode 100644 youtube_dl/extractor/sportschau.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 255df75..c82614b 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -991,7 +991,6 @@ from .stitcher import StitcherIE
 from .sport5 import Sport5IE
 from .sportbox import SportBoxEmbedIE
 from .sportdeutschland import SportDeutschlandIE
-from .sportschau import SportschauIE
 from .sprout import SproutIE
 from .srgssr import (
     SRGSSRIE,
@@ -1289,6 +1288,7 @@ from .watchbox import WatchBoxIE
 from .watchindianporn import WatchIndianPornIE
 from .wdr import (
     WDRIE,
+    WDRPageIE,
     WDRElefantIE,
     WDRMobileIE,
 )
diff --git a/youtube_dl/extractor/sportschau.py b/youtube_dl/extractor/sportschau.py
deleted file mode 100644
index 0d7925a..0000000
--- a/youtube_dl/extractor/sportschau.py
+++ /dev/null
@@ -1,38 +0,0 @@
-# coding: utf-8
-from __future__ import unicode_literals
-
-from .wdr import WDRBaseIE
-from ..utils import get_element_by_attribute
-
-
-class SportschauIE(WDRBaseIE):
-    IE_NAME = 'Sportschau'
-    _VALID_URL = r'https?://(?:www\.)?sportschau\.de/(?:[^/]+/)+video-?(?P<id>[^/#?]+)\.html'
-    _TEST = {
-        'url': 'http://www.sportschau.de/uefaeuro2016/videos/video-dfb-team-geht-gut-gelaunt-ins-spiel-gegen-polen-100.html',
-        'info_dict': {
-            'id': 'mdb-1140188',
-            'display_id': 'dfb-team-geht-gut-gelaunt-ins-spiel-gegen-polen-100',
-            'ext': 'mp4',
-            'title': 'DFB-Team geht gut gelaunt ins Spiel gegen Polen',
-            'description': 'Vor dem zweiten Gruppenspiel gegen Polen herrscht gute Stimmung im deutschen Team. Insbesondere Bastian Schweinsteiger strotzt vor Optimismus nach seinem Tor gegen die Ukraine.',
-            'upload_date': '20160615',
-        },
-        'skip': 'Geo-restricted to Germany',
-    }
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-
-        webpage = self._download_webpage(url, video_id)
-        title = get_element_by_attribute('class', 'headline', webpage)
-        description = self._html_search_meta('description', webpage, 'description')
-
-        info = self._extract_wdr_video(webpage, video_id)
-
-        info.update({
-            'title': title,
-            'description': description,
-        })
-
-        return info
diff --git a/youtube_dl/extractor/wdr.py b/youtube_dl/extractor/wdr.py
index 4871ae9..6bf5aea 100644
--- a/youtube_dl/extractor/wdr.py
+++ b/youtube_dl/extractor/wdr.py
@@ -4,50 +4,52 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..compat import (
+    compat_str,
+    compat_urlparse,
+)
 from ..utils import (
     determine_ext,
     ExtractorError,
     js_to_json,
     strip_jsonp,
+    try_get,
     unified_strdate,
     update_url_query,
     urlhandle_detect_ext,
 )
 
 
-class WDRBaseIE(InfoExtractor):
-    def _extract_jsonp_url(self, webpage, display_id):
-        # for wdr.de the data-extension is in a tag with the class "mediaLink"
-        # for wdr.de radio players, in a tag with the class "wdrrPlayerPlayBtn"
-        # for wdrmaus, in a tag with the class "videoButton" (previously a link
-        # to the page in a multiline "videoLink"-tag)
-        json_metadata = self._html_search_regex(
-            r'''(?sx)class=
-                    (?:
-                        (["\'])(?:mediaLink|wdrrPlayerPlayBtn|videoButton)\b.*?\1[^>]+|
-                        (["\'])videoLink\b.*?\2[\s]*>\n[^\n]*
-                    )data-extension=(["\'])(?P<data>(?:(?!\3).)+)\3
-            ''',
-            webpage, 'media link', default=None, group='data')
-
-        if not json_metadata:
-            return
+class WDRIE(InfoExtractor):
+    _VALID_URL = r'https?://deviceids-medp\.wdr\.de/ondemand/\d+/(?P<id>\d+)\.js'
+    _TEST = {
+        'url': 'http://deviceids-medp.wdr.de/ondemand/155/1557833.js',
+        'info_dict': {
+            'id': 'mdb-1140188',
+            'display_id': 'dfb-team-geht-gut-gelaunt-ins-spiel-gegen-polen-100',
+            'ext': 'mp4',
+            'title': 'DFB-Team geht gut gelaunt ins Spiel gegen Polen',
+            'description': 'Vor dem zweiten Gruppenspiel gegen Polen herrscht gute Stimmung im deutschen Team. Insbesondere Bastian Schweinsteiger strotzt vor Optimismus nach seinem Tor gegen die Ukraine.',
+            'upload_date': '20160615',
+        },
+        'skip': 'Geo-restricted to Germany',
+    }
 
-        media_link_obj = self._parse_json(json_metadata, display_id,
-                                          transform_source=js_to_json)
-        return media_link_obj['mediaObj']['url']
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
 
-    def _extract_wdr_video(self, jsonp_url, display_id):
         metadata = self._download_json(
-            jsonp_url, display_id, transform_source=strip_jsonp)
+            url, video_id, transform_source=strip_jsonp)
 
-        metadata_tracker_data = metadata['trackerData']
-        metadata_media_resource = metadata['mediaResource']
+        is_live = metadata.get('mediaType') == 'live'
+
+        tracker_data = metadata['trackerData']
+        media_resource = metadata['mediaResource']
 
         formats = []
 
         # check if the metadata contains a direct URL to a file
-        for kind, media_resource in metadata_media_resource.items():
+        for kind, media_resource in media_resource.items():
             if kind not in ('dflt', 'alt'):
                 continue
 
@@ -58,13 +60,13 @@ class WDRBaseIE(InfoExtractor):
                 ext = determine_ext(medium_url)
                 if ext == 'm3u8':
                     formats.extend(self._extract_m3u8_formats(
-                        medium_url, display_id, 'mp4', 'm3u8_native',
+                        medium_url, video_id, 'mp4', 'm3u8_native',
                         m3u8_id='hls'))
                 elif ext == 'f4m':
                     manifest_url = update_url_query(
                         medium_url, {'hdcore': '3.2.0', 'plugin': 'aasp-3.2.0.77.18'})
                     formats.extend(self._extract_f4m_formats(
-                        manifest_url, display_id, f4m_id='hds', fatal=False))
+                        manifest_url, video_id, f4m_id='hds', fatal=False))
                 elif ext == 'smil':
                     formats.extend(self._extract_smil_formats(
                         medium_url, 'stream', fatal=False))
@@ -74,7 +76,7 @@ class WDRBaseIE(InfoExtractor):
                     }
                     if ext == 'unknown_video':
                         urlh = self._request_webpage(
-                            medium_url, display_id, note='Determining extension')
+                            medium_url, video_id, note='Determining extension')
                         ext = urlhandle_detect_ext(urlh)
                         a_format['ext'] = ext
                     formats.append(a_format)
@@ -82,30 +84,30 @@ class WDRBaseIE(InfoExtractor):
         self._sort_formats(formats)
 
         subtitles = {}
-        caption_url = metadata_media_resource.get('captionURL')
+        caption_url = media_resource.get('captionURL')
         if caption_url:
             subtitles['de'] = [{
                 'url': caption_url,
                 'ext': 'ttml',
             }]
 
-        title = metadata_tracker_data['trackerClipTitle']
+        title = tracker_data['trackerClipTitle']
 
         return {
-            'id': metadata_tracker_data.get('trackerClipId', display_id),
-            'display_id': display_id,
-            'title': title,
-            'alt_title': metadata_tracker_data.get('trackerClipSubcategory'),
+            'id': tracker_data.get('trackerClipId', video_id),
+            'title': self._live_title(title) if is_live else title,
+            'alt_title': tracker_data.get('trackerClipSubcategory'),
             'formats': formats,
             'subtitles': subtitles,
-            'upload_date': unified_strdate(metadata_tracker_data.get('trackerClipAirTime')),
+            'upload_date': unified_strdate(tracker_data.get('trackerClipAirTime')),
+            'is_live': is_live,
         }
 
 
-class WDRIE(WDRBaseIE):
+class WDRPageIE(InfoExtractor):
     _CURRENT_MAUS_URL = r'https?://(?:www\.)wdrmaus.de/(?:[^/]+/){1,2}[^/?#]+\.php5'
-    _PAGE_REGEX = r'/(?:mediathek/)?[^/]+/(?P<type>[^/]+)/(?P<display_id>.+)\.html'
-    _VALID_URL = r'(?P<page_url>https?://(?:www\d\.)?wdr\d?\.de)' + _PAGE_REGEX + '|' + _CURRENT_MAUS_URL
+    _PAGE_REGEX = r'/(?:mediathek/)?(?:[^/]+/)*(?P<display_id>[^/]+)\.html'
+    _VALID_URL = r'https?://(?:www\d?\.)?(?:wdr\d?|sportschau)\.de' + _PAGE_REGEX + '|' + _CURRENT_MAUS_URL
 
     _TESTS = [
         {
@@ -125,6 +127,7 @@ class WDRIE(WDRBaseIE):
                     'ext': 'ttml',
                 }]},
             },
+            'skip': 'HTTP Error 404: Not Found',
         },
         {
             'url': 'http://www1.wdr.de/mediathek/audio/wdr3/wdr3-gespraech-am-samstag/audio-schriftstellerin-juli-zeh-100.html',
@@ -140,19 +143,17 @@ class WDRIE(WDRBaseIE):
                 'is_live': False,
                 'subtitles': {}
             },
+            'skip': 'HTTP Error 404: Not Found',
         },
         {
             'url': 'http://www1.wdr.de/mediathek/video/live/index.html',
             'info_dict': {
-                'id': 'mdb-103364',
+                'id': 'mdb-1406149',
                 'ext': 'mp4',
-                'display_id': 'index',
-                'title': r're:^WDR Fernsehen im Livestream [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
+                'title': r're:^WDR Fernsehen im Livestream \(nur in Deutschland erreichbar\) [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
                 'alt_title': 'WDR Fernsehen Live',
-                'upload_date': None,
-                'description': 'md5:ae2ff888510623bf8d4b115f95a9b7c9',
+                'upload_date': '20150101',
                 'is_live': True,
-                'subtitles': {}
             },
             'params': {
                 'skip_download': True,  # m3u8 download
@@ -160,19 +161,18 @@ class WDRIE(WDRBaseIE):
         },
         {
             'url': 'http://www1.wdr.de/mediathek/video/sendungen/aktuelle-stunde/aktuelle-stunde-120.html',
-            'playlist_mincount': 8,
+            'playlist_mincount': 7,
             'info_dict': {
-                'id': 'aktuelle-stunde/aktuelle-stunde-120',
+                'id': 'aktuelle-stunde-120',
             },
         },
         {
             'url': 'http://www.wdrmaus.de/aktuelle-sendung/index.php5',
             'info_dict': {
-                'id': 'mdb-1323501',
+                'id': 'mdb-1552552',
                 'ext': 'mp4',
                 'upload_date': 're:^[0-9]{8}$',
                 'title': 're:^Die Sendung mit der Maus vom [0-9.]{10}$',
-                'description': 'Die Seite mit der Maus -',
             },
             'skip': 'The id changes from week to week because of the new episode'
         },
@@ -184,7 +184,6 @@ class WDRIE(WDRBaseIE):
                 'ext': 'mp4',
                 'upload_date': '20130919',
                 'title': 'Sachgeschichte - Achterbahn ',
-                'description': 'Die Seite mit der Maus -',
             },
         },
         {
@@ -192,83 +191,100 @@ class WDRIE(WDRBaseIE):
             # Live stream, MD5 unstable
             'info_dict': {
                 'id': 'mdb-869971',
-                'ext': 'flv',
-                'title': 'COSMO Livestream',
-                'description': 'md5:2309992a6716c347891c045be50992e4',
+                'ext': 'mp4',
+                'title': r're:^COSMO Livestream [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
                 'upload_date': '20160101',
             },
+            'params': {
+                'skip_download': True,  # m3u8 download
+            }
+        },
+        {
+            'url': 'http://www.sportschau.de/handballem2018/handball-nationalmannschaft-em-stolperstein-vorrunde-100.html',
+            'info_dict': {
+                'id': 'mdb-1556012',
+                'ext': 'mp4',
+                'title': 'DHB-Vizepräsident Bob Hanning - "Die Weltspitze ist extrem breit"',
+                'upload_date': '20180111',
+            },
+            'params': {
+                'skip_download': True,
+            },
+        },
+        {
+            'url': 'http://www.sportschau.de/handballem2018/audio-vorschau---die-handball-em-startet-mit-grossem-favoritenfeld-100.html',
+            'only_matching': True,
         }
     ]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
-        url_type = mobj.group('type')
-        page_url = mobj.group('page_url')
         display_id = mobj.group('display_id')
         webpage = self._download_webpage(url, display_id)
 
-        jsonp_url = self._extract_jsonp_url(webpage, display_id)
-        info_dict = self._extract_wdr_video(jsonp_url, display_id)
-
-        if not info_dict:
-            entries = [
-                self.url_result(page_url + href[0], 'WDR')
-                for href in re.findall(
-                    r'<a href="(%s)"[^>]+data-extension=' % self._PAGE_REGEX,
-                    webpage)
-            ]
-
-            if entries:  # Playlist page
-                return self.playlist_result(entries, playlist_id=display_id)
-
-            raise ExtractorError('No downloadable streams found', expected=True)
-
-        is_live = url_type == 'live'
+        entries = []
 
-        if is_live:
-            info_dict.update({
-                'title': self._live_title(info_dict['title']),
-                'upload_date': None,
-            })
-        elif 'upload_date' not in info_dict:
-            info_dict['upload_date'] = unified_strdate(self._html_search_meta('DC.Date', webpage, 'upload date'))
+        # Article with several videos
 
-        info_dict.update({
-            'description': self._html_search_meta('Description', webpage),
-            'is_live': is_live,
-        })
+        # for wdr.de the data-extension is in a tag with the class "mediaLink"
+        # for wdr.de radio players, in a tag with the class "wdrrPlayerPlayBtn"
+        # for wdrmaus, in a tag with the class "videoButton" (previously a link
+        # to the page in a multiline "videoLink"-tag)
+        for mobj in re.finditer(
+            r'''(?sx)class=
+                    (?:
+                        (["\'])(?:mediaLink|wdrrPlayerPlayBtn|videoButton)\b.*?\1[^>]+|
+                        (["\'])videoLink\b.*?\2[\s]*>\n[^\n]*
+                    )data-extension=(["\'])(?P<data>(?:(?!\3).)+)\3
+                    ''', webpage):
+            media_link_obj = self._parse_json(
+                mobj.group('data'), display_id, transform_source=js_to_json,
+                fatal=False)
+            if not media_link_obj:
+                continue
+            jsonp_url = try_get(
+                media_link_obj, lambda x: x['mediaObj']['url'], compat_str)
+            if jsonp_url:
+                entries.append(self.url_result(jsonp_url, ie=WDRIE.ie_key()))
 
-        return info_dict
+        # Playlist (e.g. https://www1.wdr.de/mediathek/video/sendungen/aktuelle-stunde/aktuelle-stunde-120.html)
+        if not entries:
+            entries = [
+                self.url_result(
+                    compat_urlparse.urljoin(url, mobj.group('href')),
+                    ie=WDRPageIE.ie_key())
+                for mobj in re.finditer(
+                    r'<a[^>]+\bhref=(["\'])(?P<href>(?:(?!\1).)+)\1[^>]+\bdata-extension=',
+                    webpage) if re.match(self._PAGE_REGEX, mobj.group('href'))
+            ]
 
+        return self.playlist_result(entries, playlist_id=display_id)
 
-class WDRElefantIE(WDRBaseIE):
-    _VALID_URL = r'https?://(?:www\.)wdrmaus.de/elefantenseite/#(?P<display_id>.+)'
-    IE_NAME = 'wdr:elefant'
 
-    _TESTS = [
-        {
-            'url': 'http://www.wdrmaus.de/elefantenseite/#folge_ostern_2015',
-            'info_dict': {
-                'title': 'Folge Oster-Spezial 2015',
-                'id': 'mdb-1088195',
-                'ext': 'mp4',
-                'age_limit': None,
-                'upload_date': '20150406'
-            },
-            'params': {
-                'skip_download' : True,
-            },
+class WDRElefantIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)wdrmaus\.de/elefantenseite/#(?P<id>.+)'
+    _TEST = {
+        'url': 'http://www.wdrmaus.de/elefantenseite/#folge_ostern_2015',
+        'info_dict': {
+            'title': 'Folge Oster-Spezial 2015',
+            'id': 'mdb-1088195',
+            'ext': 'mp4',
+            'age_limit': None,
+            'upload_date': '20150406'
         },
-    ]
+        'params': {
+            'skip_download': True,
+        },
+    }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        display_id = mobj.group('display_id')
+        display_id = self._match_id(url)
 
         # Table of Contents seems to always be at this address, so fetch it directly.
         # The website fetches configurationJS.php5, which links to tableOfContentsJS.php5.
         table_of_contents = self._download_json(
-            'https://www.wdrmaus.de/elefantenseite/data/tableOfContentsJS.php5', display_id)
+            'https://www.wdrmaus.de/elefantenseite/data/tableOfContentsJS.php5',
+            display_id)
         if display_id not in table_of_contents:
             raise ExtractorError(
                 'No entry in site\'s table of contents for this URL. '
@@ -276,15 +292,13 @@ class WDRElefantIE(WDRBaseIE):
                 expected=True)
         xml_metadata_path = table_of_contents[display_id]['xmlPath']
         xml_metadata = self._download_xml(
-            'https://www.wdrmaus.de/elefantenseite/' + xml_metadata_path, display_id)
+            'https://www.wdrmaus.de/elefantenseite/' + xml_metadata_path,
+            display_id)
         zmdb_url_element = xml_metadata.find('./movie/zmdb_url')
         if zmdb_url_element is None:
             raise ExtractorError(
-                'The URL looks valid, but no video was found. Note that download only works '
-                'on pages showing a single video, not on video selection pages.',
-                expected=True)
-        info_dict = self._extract_wdr_video(zmdb_url_element.text, display_id)
-        return info_dict
+                '%s is not a video' % display_id, expected=True)
+        return self.url_result(zmdb_url_element.text, ie=WDRIE.ie_key())
 
 
 class WDRMobileIE(InfoExtractor):
-- 
cgit v1.1


From 1915662d4fe09120d3f28db55c7be90b4d12a9f1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 13 Jan 2018 23:30:56 +0700
Subject: [wdr] Bypass geo restriction

---
 youtube_dl/extractor/wdr.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/wdr.py b/youtube_dl/extractor/wdr.py
index 6bf5aea..d6ba254 100644
--- a/youtube_dl/extractor/wdr.py
+++ b/youtube_dl/extractor/wdr.py
@@ -22,6 +22,7 @@ from ..utils import (
 
 class WDRIE(InfoExtractor):
     _VALID_URL = r'https?://deviceids-medp\.wdr\.de/ondemand/\d+/(?P<id>\d+)\.js'
+    _GEO_COUNTRIES = ['DE']
     _TEST = {
         'url': 'http://deviceids-medp.wdr.de/ondemand/155/1557833.js',
         'info_dict': {
-- 
cgit v1.1


From 0ce39bc542813462c2d95dc21f1b363c4ae7a1e0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 13 Jan 2018 23:33:52 +0700
Subject: [wdr] Fix test

---
 youtube_dl/extractor/wdr.py | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/wdr.py b/youtube_dl/extractor/wdr.py
index d6ba254..cf6f7c7 100644
--- a/youtube_dl/extractor/wdr.py
+++ b/youtube_dl/extractor/wdr.py
@@ -26,14 +26,11 @@ class WDRIE(InfoExtractor):
     _TEST = {
         'url': 'http://deviceids-medp.wdr.de/ondemand/155/1557833.js',
         'info_dict': {
-            'id': 'mdb-1140188',
-            'display_id': 'dfb-team-geht-gut-gelaunt-ins-spiel-gegen-polen-100',
+            'id': 'mdb-1557833',
             'ext': 'mp4',
-            'title': 'DFB-Team geht gut gelaunt ins Spiel gegen Polen',
-            'description': 'Vor dem zweiten Gruppenspiel gegen Polen herrscht gute Stimmung im deutschen Team. Insbesondere Bastian Schweinsteiger strotzt vor Optimismus nach seinem Tor gegen die Ukraine.',
-            'upload_date': '20160615',
+            'title': 'Biathlon-Staffel verpasst Podest bei Olympia-Generalprobe',
+            'upload_date': '20180112',
         },
-        'skip': 'Geo-restricted to Germany',
     }
 
     def _real_extract(self, url):
-- 
cgit v1.1


From 391dd6f0946e2dd499147c5f3c6bb13642314515 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 14 Jan 2018 00:03:22 +0700
Subject: [youtube] Fix live streams extraction (closes #15202)

---
 youtube_dl/extractor/youtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 0919bef..a01ec14 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1810,7 +1810,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'url': video_info['conn'][0],
                 'player_url': player_url,
             }]
-        elif len(video_info.get('url_encoded_fmt_stream_map', [''])[0]) >= 1 or len(video_info.get('adaptive_fmts', [''])[0]) >= 1:
+        elif not is_live and (len(video_info.get('url_encoded_fmt_stream_map', [''])[0]) >= 1 or len(video_info.get('adaptive_fmts', [''])[0]) >= 1):
             encoded_url_map = video_info.get('url_encoded_fmt_stream_map', [''])[0] + ',' + video_info.get('adaptive_fmts', [''])[0]
             if 'rtmpe%3Dyes' in encoded_url_map:
                 raise ExtractorError('rtmpe downloads are not supported, see https://github.com/rg3/youtube-dl/issues/343 for more information.', expected=True)
-- 
cgit v1.1


From dd896a6a07939a343776570770b5c8f69e8c0988 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 14 Jan 2018 00:10:04 +0700
Subject: [ChangeLog] Actualize

---
 ChangeLog | 17 +++++++++++++++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index ea1be93..7c63d4b 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,9 +1,22 @@
 version <unreleased>
 
 Extractors
-+ [ximalaya] Add extractor (#14687)
+* [youtube] Fix live streams extraction (#15202)
+* [wdr] Bypass geo restriction
+* [wdr] Rework extractors (#14598)
++ [wdr] Add support for wdrmaus.de/elefantenseite (#14598)
++ [gamestar] Add support for gamepro.de (#3384)
+* [viafree] Skip rtmp formats (#15232)
++ [pandoratv] Add support for mobile URLs (#12441)
++ [pandoratv] Add support for new URL format (#15131)
++ [ximalaya] Add support for ximalaya.com (#14687)
++ [digg] Add support for digg.com (#15214)
+* [limelight] Tolerate empty pc formats (#15150, #15151, #15207)
+* [ndr:embed:base] Make separate formats extraction non fatal (#15203)
 + [weibo] Add extractor (#15079)
-* [bilibili] fix extraction (#15188) 
++ [ok] Add support for live streams
+* [canalplus] Fix extraction (#15072)
+* [bilibili] Fix extraction (#15188)
 
 
 version 2018.01.07
-- 
cgit v1.1


From e11ccd76c6dc8438c01d30445627eab5203cb1dc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 14 Jan 2018 00:13:56 +0700
Subject: release 2018.01.14

---
 .github/ISSUE_TEMPLATE.md |  6 +++---
 ChangeLog                 |  2 +-
 docs/supportedsites.md    | 10 ++++++++--
 youtube_dl/version.py     |  2 +-
 4 files changed, 13 insertions(+), 7 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index ad52c89..6bc7d03 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2018.01.07*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2018.01.07**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2018.01.14*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2018.01.14**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through the [README](https://github.com/rg3/youtube-dl/blob/master/README.md), **most notably** the [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ Add the `-v` flag to **your command line** you run youtube-dl with (`youtube-dl
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2018.01.07
+[debug] youtube-dl version 2018.01.14
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index 7c63d4b..bfafaca 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2018.01.14
 
 Extractors
 * [youtube] Fix live streams extraction (#15202)
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 79b3430..c04a75b 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -128,7 +128,7 @@
  - **CamdemyFolder**
  - **CamWithHer**
  - **canalc2.tv**
- - **Canalplus**: canalplus.fr, piwiplus.fr and d8.tv
+ - **Canalplus**: mycanal.fr and piwiplus.fr
  - **Canvas**
  - **CanvasEen**: canvas.be and een.be
  - **CarambaTV**
@@ -210,6 +210,7 @@
  - **defense.gouv.fr**
  - **democracynow**
  - **DHM**: Filmarchiv - Deutsches Historisches Museum
+ - **Digg**
  - **DigitallySpeaking**
  - **Digiteka**
  - **Discovery**
@@ -773,7 +774,6 @@
  - **Sport5**
  - **SportBoxEmbed**
  - **SportDeutschland**
- - **Sportschau**
  - **Sprout**
  - **sr:mediathek**: Saarländischer Rundfunk
  - **SRGSSR**
@@ -1002,10 +1002,14 @@
  - **WatchIndianPorn**: Watch Indian Porn
  - **WDR**
  - **wdr:mobile**
+ - **WDRElefant**
+ - **WDRPage**
  - **Webcaster**
  - **WebcasterFeed**
  - **WebOfStories**
  - **WebOfStoriesPlaylist**
+ - **Weibo**
+ - **WeiboMobile**
  - **WeiqiTV**: WQTV
  - **wholecloud**: WholeCloud
  - **Wimp**
@@ -1025,6 +1029,8 @@
  - **xiami:artist**: 虾米音乐 - 歌手
  - **xiami:collection**: 虾米音乐 - 精选集
  - **xiami:song**: 虾米音乐
+ - **ximalaya**: 喜马拉雅FM
+ - **ximalaya:album**: 喜马拉雅FM 专辑
  - **XMinus**
  - **XNXX**
  - **Xstream**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 9030e24..4981491 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2018.01.07'
+__version__ = '2018.01.14'
-- 
cgit v1.1


From a86922c4702e2c8538337124c5bf02a4b5f9aa4a Mon Sep 17 00:00:00 2001
From: Reto Kromer <retokromer@users.noreply.github.com>
Date: Sat, 13 Jan 2018 18:58:38 +0100
Subject: [README.md] Clarify macOS name

---
 README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 47b0640..eb05f84 100644
--- a/README.md
+++ b/README.md
@@ -46,7 +46,7 @@ Or with [MacPorts](https://www.macports.org/):
 Alternatively, refer to the [developer instructions](#developer-instructions) for how to check out and work with the git repository. For further options, including PGP signatures, see the [youtube-dl Download Page](https://rg3.github.io/youtube-dl/download.html).
 
 # DESCRIPTION
-**youtube-dl** is a command-line program to download videos from YouTube.com and a few more sites. It requires the Python interpreter, version 2.6, 2.7, or 3.2+, and it is not platform specific. It should work on your Unix box, on Windows or on Mac OS X. It is released to the public domain, which means you can modify it, redistribute it or use it however you like.
+**youtube-dl** is a command-line program to download videos from YouTube.com and a few more sites. It requires the Python interpreter, version 2.6, 2.7, or 3.2+, and it is not platform specific. It should work on your Unix box, on Windows or on macOS. It is released to the public domain, which means you can modify it, redistribute it or use it however you like.
 
     youtube-dl [OPTIONS] URL [URL...]
 
@@ -863,7 +863,7 @@ Use the `--cookies` option, for example `--cookies /path/to/cookies/file.txt`.
 
 In order to extract cookies from browser use any conforming browser extension for exporting cookies. For example, [cookies.txt](https://chrome.google.com/webstore/detail/cookiestxt/njabckikapfpffapmjgojcnbfjonfjfg) (for Chrome) or [Export Cookies](https://addons.mozilla.org/en-US/firefox/addon/export-cookies/) (for Firefox).
 
-Note that the cookies file must be in Mozilla/Netscape format and the first line of the cookies file must be either `# HTTP Cookie File` or `# Netscape HTTP Cookie File`. Make sure you have correct [newline format](https://en.wikipedia.org/wiki/Newline) in the cookies file and convert newlines if necessary to correspond with your OS, namely `CRLF` (`\r\n`) for Windows and `LF` (`\n`) for Unix and Unix-like systems (Linux, Mac OS, etc.). `HTTP Error 400: Bad Request` when using `--cookies` is a good sign of invalid newline format.
+Note that the cookies file must be in Mozilla/Netscape format and the first line of the cookies file must be either `# HTTP Cookie File` or `# Netscape HTTP Cookie File`. Make sure you have correct [newline format](https://en.wikipedia.org/wiki/Newline) in the cookies file and convert newlines if necessary to correspond with your OS, namely `CRLF` (`\r\n`) for Windows and `LF` (`\n`) for Unix and Unix-like systems (Linux, macOS, etc.). `HTTP Error 400: Bad Request` when using `--cookies` is a good sign of invalid newline format.
 
 Passing cookies to youtube-dl is a good way to workaround login when a particular extractor does not implement it explicitly. Another use case is working around [CAPTCHA](https://en.wikipedia.org/wiki/CAPTCHA) some websites require you to solve in particular cases in order to get access (e.g. YouTube, CloudFlare).
 
-- 
cgit v1.1