From e33dfb445c547f210a7060e8b7abd592dbe42808 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 1 May 2021 17:53:27 +0700
Subject: [PATCH 01/43] [tv2dk] Fix extraction (closes #28888)

---
 youtube_dl/extractor/tv2dk.py | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/tv2dk.py b/youtube_dl/extractor/tv2dk.py
index 8bda9348d..8bd5fd640 100644
--- a/youtube_dl/extractor/tv2dk.py
+++ b/youtube_dl/extractor/tv2dk.py
@@ -74,6 +74,12 @@ class TV2DKIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
 
         entries = []
+
+        def add_entry(partner_id, kaltura_id):
+            entries.append(self.url_result(
+                'kaltura:%s:%s' % (partner_id, kaltura_id), 'Kaltura',
+                video_id=kaltura_id))
+
         for video_el in re.findall(r'(?s)<[^>]+\bdata-entryid\s*=[^>]*>', webpage):
             video = extract_attributes(video_el)
             kaltura_id = video.get('data-entryid')
@@ -82,9 +88,14 @@ class TV2DKIE(InfoExtractor):
             partner_id = video.get('data-partnerid')
             if not partner_id:
                 continue
-            entries.append(self.url_result(
-                'kaltura:%s:%s' % (partner_id, kaltura_id), 'Kaltura',
-                video_id=kaltura_id))
+            add_entry(partner_id, kaltura_id)
+        if not entries:
+            kaltura_id = self._search_regex(
+                r'entry_id\s*:\s*["\']([0-9a-z_]+)', webpage, 'kaltura id')
+            partner_id = self._search_regex(
+                (r'\\u002Fp\\u002F(\d+)\\u002F', r'/p/(\d+)/'), webpage,
+                'partner id')
+            add_entry(partner_id, kaltura_id)
         return self.playlist_result(entries)
 
 

From d2f72c40db0d1fe1102c98c017682b283579ad97 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 1 May 2021 18:09:32 +0700
Subject: [PATCH 02/43] [svtplay] Improve extraction (closes #28507, closes
 #28876)

---
 youtube_dl/extractor/svt.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/svt.py b/youtube_dl/extractor/svt.py
index aba9bb447..a5bb6daa7 100644
--- a/youtube_dl/extractor/svt.py
+++ b/youtube_dl/extractor/svt.py
@@ -146,7 +146,7 @@ class SVTPlayIE(SVTPlayBaseIE):
                         )
                         (?P<svt_id>[^/?#&]+)|
                         https?://(?:www\.)?(?:svtplay|oppetarkiv)\.se/(?:video|klipp|kanaler)/(?P<id>[^/?#&]+)
-                        (?:.*?modalId=(?P<modal_id>[\da-zA-Z-]+))?
+                        (?:.*?(?:modalId|id)=(?P<modal_id>[\da-zA-Z-]+))?
                     )
                     '''
     _TESTS = [{
@@ -177,6 +177,9 @@ class SVTPlayIE(SVTPlayBaseIE):
     }, {
         'url': 'https://www.svtplay.se/video/30479064/husdrommar/husdrommar-sasong-8-designdrommar-i-stenungsund?modalId=8zVbDPA',
         'only_matching': True,
+    }, {
+        'url': 'https://www.svtplay.se/video/30684086/rapport/rapport-24-apr-18-00-7?id=e72gVpa',
+        'only_matching': True,
     }, {
         # geo restricted to Sweden
         'url': 'http://www.oppetarkiv.se/video/5219710/trollflojten',
@@ -259,7 +262,7 @@ class SVTPlayIE(SVTPlayBaseIE):
         if not svt_id:
             svt_id = self._search_regex(
                 (r'<video[^>]+data-video-id=["\']([\da-zA-Z-]+)',
-                 r'<[^>]+\bdata-rt=["\']top-area-play-button["\'][^>]+\bhref=["\'][^"\']*video/%s/[^"\']*\bmodalId=([\da-zA-Z-]+)' % re.escape(video_id),
+                 r'<[^>]+\bdata-rt=["\']top-area-play-button["\'][^>]+\bhref=["\'][^"\']*video/%s/[^"\']*\b(?:modalId|id)=([\da-zA-Z-]+)' % re.escape(video_id),
                  r'["\']videoSvtId["\']\s*:\s*["\']([\da-zA-Z-]+)',
                  r'["\']videoSvtId\\?["\']\s*:\s*\\?["\']([\da-zA-Z-]+)',
                  r'"content"\s*:\s*{.*?"id"\s*:\s*"([\da-zA-Z-]+)"',

From ff04d43c469e4cf8c14ba3e2e79da0d35ef3c7db Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 1 May 2021 18:33:05 +0700
Subject: [PATCH 03/43] [xtube] Fix formats extraction (closes #28870)

---
 youtube_dl/extractor/xtube.py | 51 ++++++++++++++++++++++++++++-------
 1 file changed, 42 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/xtube.py b/youtube_dl/extractor/xtube.py
index 18969058f..7246409e3 100644
--- a/youtube_dl/extractor/xtube.py
+++ b/youtube_dl/extractor/xtube.py
@@ -11,6 +11,7 @@ from ..utils import (
     parse_duration,
     sanitized_Request,
     str_to_int,
+    url_or_none,
 )
 
 
@@ -87,10 +88,10 @@ class XTubeIE(InfoExtractor):
                 'Cookie': 'age_verified=1; cookiesAccepted=1',
             })
 
-        title, thumbnail, duration = [None] * 3
+        title, thumbnail, duration, sources, media_definition = [None] * 5
 
         config = self._parse_json(self._search_regex(
-            r'playerConf\s*=\s*({.+?})\s*,\s*(?:\n|loaderConf)', webpage, 'config',
+            r'playerConf\s*=\s*({.+?})\s*,\s*(?:\n|loaderConf|playerWrapper)', webpage, 'config',
             default='{}'), video_id, transform_source=js_to_json, fatal=False)
         if config:
             config = config.get('mainRoll')
@@ -99,20 +100,52 @@ class XTubeIE(InfoExtractor):
                 thumbnail = config.get('poster')
                 duration = int_or_none(config.get('duration'))
                 sources = config.get('sources') or config.get('format')
+                media_definition = config.get('mediaDefinition')
 
-        if not isinstance(sources, dict):
+        if not isinstance(sources, dict) and not media_definition:
             sources = self._parse_json(self._search_regex(
                 r'(["\'])?sources\1?\s*:\s*(?P<sources>{.+?}),',
                 webpage, 'sources', group='sources'), video_id,
                 transform_source=js_to_json)
 
         formats = []
-        for format_id, format_url in sources.items():
-            formats.append({
-                'url': format_url,
-                'format_id': format_id,
-                'height': int_or_none(format_id),
-            })
+        format_urls = set()
+
+        if isinstance(sources, dict):
+            for format_id, format_url in sources.items():
+                format_url = url_or_none(format_url)
+                if not format_url:
+                    continue
+                if format_url in format_urls:
+                    continue
+                format_urls.add(format_url)
+                formats.append({
+                    'url': format_url,
+                    'format_id': format_id,
+                    'height': int_or_none(format_id),
+                })
+
+        if isinstance(media_definition, list):
+            for media in media_definition:
+                video_url = url_or_none(media.get('videoUrl'))
+                if not video_url:
+                    continue
+                if video_url in format_urls:
+                    continue
+                format_urls.add(video_url)
+                format_id = media.get('format')
+                if format_id == 'hls':
+                    formats.extend(self._extract_m3u8_formats(
+                        video_url, video_id, 'mp4', entry_protocol='m3u8_native',
+                        m3u8_id='hls', fatal=False))
+                elif format_id == 'mp4':
+                    height = int_or_none(media.get('quality'))
+                    formats.append({
+                        'url': video_url,
+                        'format_id': '%s-%d' % (format_id, height) if height else format_id,
+                        'height': height,
+                    })
+
         self._remove_duplicate_formats(formats)
         self._sort_formats(formats)
 

From d1b9a5e2eff1c075b38815a3d2b25eb8b3f626bb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 1 May 2021 19:00:39 +0700
Subject: [PATCH 04/43] [twitter] Improve formats extraction from vmap URL
 (closes #28909)

---
 youtube_dl/extractor/twitter.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/twitter.py b/youtube_dl/extractor/twitter.py
index ed495f297..cfa7a7326 100644
--- a/youtube_dl/extractor/twitter.py
+++ b/youtube_dl/extractor/twitter.py
@@ -19,6 +19,7 @@ from ..utils import (
     strip_or_none,
     unified_timestamp,
     update_url_query,
+    url_or_none,
     xpath_text,
 )
 
@@ -52,6 +53,9 @@ class TwitterBaseIE(InfoExtractor):
             return [f]
 
     def _extract_formats_from_vmap_url(self, vmap_url, video_id):
+        vmap_url = url_or_none(vmap_url)
+        if not vmap_url:
+            return []
         vmap_data = self._download_xml(vmap_url, video_id)
         formats = []
         urls = []

From a0df8a06178e530a1097f177a1faf1d2c609ac99 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 1 May 2021 22:53:30 +0700
Subject: [PATCH 05/43] [cda] Improve extraction (closes #28709, closes #28937)

---
 youtube_dl/extractor/cda.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/cda.py b/youtube_dl/extractor/cda.py
index 1b4362144..e1b391937 100644
--- a/youtube_dl/extractor/cda.py
+++ b/youtube_dl/extractor/cda.py
@@ -133,6 +133,8 @@ class CDAIE(InfoExtractor):
             'age_limit': 18 if need_confirm_age else 0,
         }
 
+        info = self._search_json_ld(webpage, video_id, default={})
+
         # Source: https://www.cda.pl/js/player.js?t=1606154898
         def decrypt_file(a):
             for p in ('_XDDD', '_CDA', '_ADC', '_CXD', '_QWE', '_Q5', '_IKSDE'):
@@ -197,7 +199,7 @@ class CDAIE(InfoExtractor):
                 handler = self._download_webpage
 
             webpage = handler(
-                self._BASE_URL + href, video_id,
+                urljoin(self._BASE_URL, href), video_id,
                 'Downloading %s version information' % resolution, fatal=False)
             if not webpage:
                 # Manually report warning because empty page is returned when
@@ -209,6 +211,4 @@ class CDAIE(InfoExtractor):
 
         self._sort_formats(formats)
 
-        info = self._search_json_ld(webpage, video_id, default={})
-
         return merge_dicts(info_dict, info)

From 0204838163bd4068fe23b40414573d1307d817ab Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 5 May 2021 01:57:02 +0700
Subject: [PATCH 06/43] [kaltura] Make embed code alternatives actually work

---
 youtube_dl/extractor/kaltura.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/kaltura.py b/youtube_dl/extractor/kaltura.py
index 49d13460d..5d0ff0418 100644
--- a/youtube_dl/extractor/kaltura.py
+++ b/youtube_dl/extractor/kaltura.py
@@ -120,7 +120,7 @@ class KalturaIE(InfoExtractor):
     def _extract_urls(webpage):
         # Embed codes: https://knowledge.kaltura.com/embedding-kaltura-media-players-your-site
         finditer = (
-            re.finditer(
+            list(re.finditer(
                 r"""(?xs)
                     kWidget\.(?:thumb)?[Ee]mbed\(
                     \{.*?
@@ -128,8 +128,8 @@ class KalturaIE(InfoExtractor):
                         (?P<q2>['"])_?(?P<partner_id>(?:(?!(?P=q2)).)+)(?P=q2),.*?
                         (?P<q3>['"])entry_?[Ii]d(?P=q3)\s*:\s*
                         (?P<q4>['"])(?P<id>(?:(?!(?P=q4)).)+)(?P=q4)(?:,|\s*\})
-                """, webpage)
-            or re.finditer(
+                """, webpage))
+            or list(re.finditer(
                 r'''(?xs)
                     (?P<q1>["'])
                         (?:https?:)?//cdnapi(?:sec)?\.kaltura\.com(?::\d+)?/(?:(?!(?P=q1)).)*\b(?:p|partner_id)/(?P<partner_id>\d+)(?:(?!(?P=q1)).)*
@@ -142,8 +142,8 @@ class KalturaIE(InfoExtractor):
                         \[\s*(?P<q2_1>["'])entry_?[Ii]d(?P=q2_1)\s*\]\s*=\s*
                     )
                     (?P<q3>["'])(?P<id>(?:(?!(?P=q3)).)+)(?P=q3)
-                ''', webpage)
-            or re.finditer(
+                ''', webpage))
+            or list(re.finditer(
                 r'''(?xs)
                     <(?:iframe[^>]+src|meta[^>]+\bcontent)=(?P<q1>["'])
                       (?:https?:)?//(?:(?:www|cdnapi(?:sec)?)\.)?kaltura\.com/(?:(?!(?P=q1)).)*\b(?:p|partner_id)/(?P<partner_id>\d+)
@@ -151,7 +151,7 @@ class KalturaIE(InfoExtractor):
                       [?&;]entry_id=(?P<id>(?:(?!(?P=q1))[^&])+)
                       (?:(?!(?P=q1)).)*
                     (?P=q1)
-                ''', webpage)
+                ''', webpage))
         )
         urls = []
         for mobj in finditer:

From fe05191b8c59538a48b6cbc95f4fe54fc7e6a0ac Mon Sep 17 00:00:00 2001
From: Ben Rog-Wilhelm <zorba-github@pavlovian.net>
Date: Tue, 4 May 2021 14:14:35 -0500
Subject: [PATCH 07/43] [kaltura] Improve iframe extraction (#28969)

Co-authored-by: Sergey M. <dstftw@gmail.com>
---
 youtube_dl/extractor/gdcvault.py | 15 +++++++++++++++
 youtube_dl/extractor/kaltura.py  |  2 +-
 2 files changed, 16 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/gdcvault.py b/youtube_dl/extractor/gdcvault.py
index 2f555c1d4..5ad40ee23 100644
--- a/youtube_dl/extractor/gdcvault.py
+++ b/youtube_dl/extractor/gdcvault.py
@@ -102,6 +102,21 @@ class GDCVaultIE(InfoExtractor):
                 'format': 'mp4-408',
             },
         },
+        {
+            # Kaltura embed, whitespace between quote and embedded URL in iframe's src
+            'url': 'https://www.gdcvault.com/play/1025699',
+            'info_dict': {
+                'id': '0_zagynv0a',
+                'ext': 'mp4',
+                'title': 'Tech Toolbox',
+                'upload_date': '20190408',
+                'uploader_id': 'joe@blazestreaming.com',
+                'timestamp': 1554764629,
+            },
+            'params': {
+                'skip_download': True,
+            },
+        },
     ]
 
     def _login(self, webpage_url, display_id):
diff --git a/youtube_dl/extractor/kaltura.py b/youtube_dl/extractor/kaltura.py
index 5d0ff0418..c731612c4 100644
--- a/youtube_dl/extractor/kaltura.py
+++ b/youtube_dl/extractor/kaltura.py
@@ -145,7 +145,7 @@ class KalturaIE(InfoExtractor):
                 ''', webpage))
             or list(re.finditer(
                 r'''(?xs)
-                    <(?:iframe[^>]+src|meta[^>]+\bcontent)=(?P<q1>["'])
+                    <(?:iframe[^>]+src|meta[^>]+\bcontent)=(?P<q1>["'])\s*
                       (?:https?:)?//(?:(?:www|cdnapi(?:sec)?)\.)?kaltura\.com/(?:(?!(?P=q1)).)*\b(?:p|partner_id)/(?P<partner_id>\d+)
                       (?:(?!(?P=q1)).)*
                       [?&;]entry_id=(?P<id>(?:(?!(?P=q1))[^&])+)

From b8645c1f5885522ec8bb77649f49ce842e947c25 Mon Sep 17 00:00:00 2001
From: Ben Rog-Wilhelm <zorba-github@pavlovian.net>
Date: Sat, 17 Apr 2021 23:15:10 -0500
Subject: [PATCH 08/43] [dispeak] Improve FLV extraction (closes #13513)

---
 youtube_dl/extractor/dispeak.py | 50 ++++++++++++++++++++-------------
 1 file changed, 30 insertions(+), 20 deletions(-)

diff --git a/youtube_dl/extractor/dispeak.py b/youtube_dl/extractor/dispeak.py
index c345e0274..e776ac00c 100644
--- a/youtube_dl/extractor/dispeak.py
+++ b/youtube_dl/extractor/dispeak.py
@@ -32,6 +32,14 @@ class DigitallySpeakingIE(InfoExtractor):
         # From http://www.gdcvault.com/play/1013700/Advanced-Material
         'url': 'http://sevt.dispeak.com/ubm/gdc/eur10/xml/11256_1282118587281VNIT.xml',
         'only_matching': True,
+    }, {
+        # From https://gdcvault.com/play/1016624
+        'url': 'https://sevt.dispeak.com/ubm/gdc/online12/xml/201210-822101_1349794556671DDDD.xml',
+        'info_dict': {
+            'id': '201210-822101_1349794556671DDDD',
+            'ext': 'flv',
+            'title': 'Pre-launch - Preparing to Take the Plunge',
+        },
     }]
 
     def _parse_mp4(self, metadata):
@@ -84,26 +92,28 @@ class DigitallySpeakingIE(InfoExtractor):
                 'vcodec': 'none',
                 'format_id': audio.get('code'),
             })
-        slide_video_path = xpath_text(metadata, './slideVideo', fatal=True)
-        formats.append({
-            'url': 'rtmp://%s/ondemand?ovpfv=1.1' % akamai_url,
-            'play_path': remove_end(slide_video_path, '.flv'),
-            'ext': 'flv',
-            'format_note': 'slide deck video',
-            'quality': -2,
-            'preference': -2,
-            'format_id': 'slides',
-        })
-        speaker_video_path = xpath_text(metadata, './speakerVideo', fatal=True)
-        formats.append({
-            'url': 'rtmp://%s/ondemand?ovpfv=1.1' % akamai_url,
-            'play_path': remove_end(speaker_video_path, '.flv'),
-            'ext': 'flv',
-            'format_note': 'speaker video',
-            'quality': -1,
-            'preference': -1,
-            'format_id': 'speaker',
-        })
+        slide_video_path = xpath_text(metadata, './slideVideo')
+        if slide_video_path:
+            formats.append({
+                'url': 'rtmp://%s/ondemand?ovpfv=1.1' % akamai_url,
+                'play_path': remove_end(slide_video_path, '.flv'),
+                'ext': 'flv',
+                'format_note': 'slide deck video',
+                'quality': -2,
+                'preference': -2,
+                'format_id': 'slides',
+            })
+        speaker_video_path = xpath_text(metadata, './speakerVideo')
+        if speaker_video_path:
+            formats.append({
+                'url': 'rtmp://%s/ondemand?ovpfv=1.1' % akamai_url,
+                'play_path': remove_end(speaker_video_path, '.flv'),
+                'ext': 'flv',
+                'format_note': 'speaker video',
+                'quality': -1,
+                'preference': -1,
+                'format_id': 'speaker',
+            })
         return formats
 
     def _real_extract(self, url):

From 1786cd3fe4e555b83bdd3eea77ade3477293330d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 5 May 2021 02:30:42 +0700
Subject: [PATCH 09/43] [dispeak] DRY and update tests (closes #28970)

---
 youtube_dl/extractor/dispeak.py | 34 +++++++++++++++------------------
 1 file changed, 15 insertions(+), 19 deletions(-)

diff --git a/youtube_dl/extractor/dispeak.py b/youtube_dl/extractor/dispeak.py
index e776ac00c..276fd4b09 100644
--- a/youtube_dl/extractor/dispeak.py
+++ b/youtube_dl/extractor/dispeak.py
@@ -33,13 +33,17 @@ class DigitallySpeakingIE(InfoExtractor):
         'url': 'http://sevt.dispeak.com/ubm/gdc/eur10/xml/11256_1282118587281VNIT.xml',
         'only_matching': True,
     }, {
-        # From https://gdcvault.com/play/1016624
+        # From https://gdcvault.com/play/1016624, empty speakerVideo
         'url': 'https://sevt.dispeak.com/ubm/gdc/online12/xml/201210-822101_1349794556671DDDD.xml',
         'info_dict': {
             'id': '201210-822101_1349794556671DDDD',
             'ext': 'flv',
             'title': 'Pre-launch - Preparing to Take the Plunge',
         },
+    }, {
+        # From http://www.gdcvault.com/play/1014846/Conference-Keynote-Shigeru, empty slideVideo
+        'url': 'http://events.digitallyspeaking.com/gdc/project25/xml/p25-miyamoto1999_1282467389849HSVB.xml',
+        'only_matching': True,
     }]
 
     def _parse_mp4(self, metadata):
@@ -92,27 +96,19 @@ class DigitallySpeakingIE(InfoExtractor):
                 'vcodec': 'none',
                 'format_id': audio.get('code'),
             })
-        slide_video_path = xpath_text(metadata, './slideVideo')
-        if slide_video_path:
+        for video_key, format_id, preference in (
+                ('slide', 'slides', -2), ('speaker', 'speaker', -1)):
+            video_path = xpath_text(metadata, './%sVideo' % video_key)
+            if not video_path:
+                continue
             formats.append({
                 'url': 'rtmp://%s/ondemand?ovpfv=1.1' % akamai_url,
-                'play_path': remove_end(slide_video_path, '.flv'),
+                'play_path': remove_end(video_path, '.flv'),
                 'ext': 'flv',
-                'format_note': 'slide deck video',
-                'quality': -2,
-                'preference': -2,
-                'format_id': 'slides',
-            })
-        speaker_video_path = xpath_text(metadata, './speakerVideo')
-        if speaker_video_path:
-            formats.append({
-                'url': 'rtmp://%s/ondemand?ovpfv=1.1' % akamai_url,
-                'play_path': remove_end(speaker_video_path, '.flv'),
-                'ext': 'flv',
-                'format_note': 'speaker video',
-                'quality': -1,
-                'preference': -1,
-                'format_id': 'speaker',
+                'format_note': '%s video' % video_key,
+                'quality': preference,
+                'preference': preference,
+                'format_id': format_id,
             })
         return formats
 

From 504e4d804df0ee666d80ba6796017cf97e026c0b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 5 May 2021 02:44:29 +0700
Subject: [PATCH 10/43] [gdcvault] Add support for HTML5 videos

---
 youtube_dl/extractor/gdcvault.py | 19 ++++++++++++++++++-
 1 file changed, 18 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/gdcvault.py b/youtube_dl/extractor/gdcvault.py
index 5ad40ee23..acc6478b8 100644
--- a/youtube_dl/extractor/gdcvault.py
+++ b/youtube_dl/extractor/gdcvault.py
@@ -6,6 +6,7 @@ from .common import InfoExtractor
 from .kaltura import KalturaIE
 from ..utils import (
     HEADRequest,
+    remove_start,
     sanitized_Request,
     smuggle_url,
     urlencode_postdata,
@@ -117,6 +118,11 @@ class GDCVaultIE(InfoExtractor):
                 'skip_download': True,
             },
         },
+        {
+            # HTML5 video
+            'url': 'http://www.gdcvault.com/play/1014846/Conference-Keynote-Shigeru',
+            'only_matching': True,
+        },
     ]
 
     def _login(self, webpage_url, display_id):
@@ -190,7 +196,18 @@ class GDCVaultIE(InfoExtractor):
 
             xml_name = self._html_search_regex(
                 r'<iframe src=".*?\?xml(?:=|URL=xml/)(.+?\.xml).*?".*?</iframe>',
-                start_page, 'xml filename')
+                start_page, 'xml filename', default=None)
+            if not xml_name:
+                info = self._parse_html5_media_entries(url, start_page, video_id)[0]
+                info.update({
+                    'title': remove_start(self._search_regex(
+                        r'>Session Name:\s*<.*?>\s*<td>(.+?)</td>', start_page,
+                        'title', default=None) or self._og_search_title(
+                        start_page, default=None), 'GDC Vault - '),
+                    'id': video_id,
+                    'display_id': display_id,
+                })
+                return info
             embed_url = '%s/xml/%s' % (xml_root, xml_name)
             ie_key = 'DigitallySpeaking'
 

From 04be55307a3a13f7091a9a2970a1cc20863769e3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 5 May 2021 02:54:12 +0700
Subject: [PATCH 11/43] [funimation] Add support for optional lang code in URLs
 (closes #28950)

---
 youtube_dl/extractor/funimation.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/funimation.py b/youtube_dl/extractor/funimation.py
index 8bbedca26..d8f1e169a 100644
--- a/youtube_dl/extractor/funimation.py
+++ b/youtube_dl/extractor/funimation.py
@@ -16,7 +16,7 @@ from ..utils import (
 
 
 class FunimationIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?funimation(?:\.com|now\.uk)/shows/[^/]+/(?P<id>[^/?#&]+)'
+    _VALID_URL = r'https?://(?:www\.)?funimation(?:\.com|now\.uk)/(?:[^/]+/)?shows/[^/]+/(?P<id>[^/?#&]+)'
 
     _NETRC_MACHINE = 'funimation'
     _TOKEN = None
@@ -51,6 +51,10 @@ class FunimationIE(InfoExtractor):
     }, {
         'url': 'https://www.funimationnow.uk/shows/puzzle-dragons-x/drop-impact/simulcast/',
         'only_matching': True,
+    }, {
+        # with lang code
+        'url': 'https://www.funimation.com/en/shows/hacksign/role-play/',
+        'only_matching': True,
     }]
 
     def _login(self):

From b797c1cc750b1f617678281731303611e21c70d0 Mon Sep 17 00:00:00 2001
From: Jacob Chapman <7908073+chapmanjacobd@users.noreply.github.com>
Date: Wed, 5 May 2021 03:31:24 +0700
Subject: [PATCH 12/43] [YoutubeDL] Improve extract_info doc (#28946)

Co-authored-by: Sergey M. <dstftw@gmail.com>
---
 youtube_dl/YoutubeDL.py | 19 ++++++++++++++-----
 1 file changed, 14 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 8f65c6499..fe30758ef 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -773,11 +773,20 @@ class YoutubeDL(object):
 
     def extract_info(self, url, download=True, ie_key=None, extra_info={},
                      process=True, force_generic_extractor=False):
-        '''
-        Returns a list with a dictionary for each video we find.
-        If 'download', also downloads the videos.
-        extra_info is a dict containing the extra values to add to each result
-        '''
+        """
+        Return a list with a dictionary for each video extracted.
+
+        Arguments:
+        url -- URL to extract
+
+        Keyword arguments:
+        download -- whether to download videos during extraction
+        ie_key -- extractor key hint
+        extra_info -- dictionary containing the extra values to add to each result
+        process -- whether to resolve all unresolved references (URLs, playlist items),
+            must be True for download to work.
+        force_generic_extractor -- force using the generic extractor
+        """
 
         if not ie_key and force_generic_extractor:
             ie_key = 'Generic'

From 03afef753878c2528260dc8084092556428f4343 Mon Sep 17 00:00:00 2001
From: catboy <79282513+catboy-oss@users.noreply.github.com>
Date: Tue, 4 May 2021 20:44:07 +0000
Subject: [PATCH 13/43] [medaltv] Relax _VALID_URL (#28884)

Co-authored-by: Sergey M. <dstftw@gmail.com>
---
 youtube_dl/extractor/medaltv.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/medaltv.py b/youtube_dl/extractor/medaltv.py
index ef2283dea..67bb4debb 100644
--- a/youtube_dl/extractor/medaltv.py
+++ b/youtube_dl/extractor/medaltv.py
@@ -15,7 +15,7 @@ from ..utils import (
 
 
 class MedalTVIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?medal\.tv/clips/(?P<id>[a-zA-Z0-9]+)'
+    _VALID_URL = r'https?://(?:www\.)?medal\.tv/clips/(?P<id>[^/?#&]+)'
     _TESTS = [{
         'url': 'https://medal.tv/clips/2mA60jWAGQCBH',
         'md5': '7b07b064331b1cf9e8e5c52a06ae68fa',
@@ -42,6 +42,12 @@ class MedalTVIE(InfoExtractor):
             'upload_date': '20201117',
             'uploader_id': '5156321',
         }
+    }, {
+        'url': 'https://medal.tv/clips/37rMeFpryCC-9',
+        'only_matching': True,
+    }, {
+        'url': 'https://medal.tv/clips/2WRj40tpY_EU9',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From a7260099873acc6dc7d76cafad2f6b139087afd0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 5 May 2021 04:12:35 +0700
Subject: [PATCH 14/43] [blinkx] Remove extractor (closes #28941)

No longer exists.
---
 youtube_dl/extractor/blinkx.py     | 86 ------------------------------
 youtube_dl/extractor/extractors.py |  1 -
 2 files changed, 87 deletions(-)
 delete mode 100644 youtube_dl/extractor/blinkx.py

diff --git a/youtube_dl/extractor/blinkx.py b/youtube_dl/extractor/blinkx.py
deleted file mode 100644
index db5e12b21..000000000
--- a/youtube_dl/extractor/blinkx.py
+++ /dev/null
@@ -1,86 +0,0 @@
-from __future__ import unicode_literals
-
-import json
-
-from .common import InfoExtractor
-from ..utils import (
-    remove_start,
-    int_or_none,
-)
-
-
-class BlinkxIE(InfoExtractor):
-    _VALID_URL = r'(?:https?://(?:www\.)blinkx\.com/#?ce/|blinkx:)(?P<id>[^?]+)'
-    IE_NAME = 'blinkx'
-
-    _TEST = {
-        'url': 'http://www.blinkx.com/ce/Da0Gw3xc5ucpNduzLuDDlv4WC9PuI4fDi1-t6Y3LyfdY2SZS5Urbvn-UPJvrvbo8LTKTc67Wu2rPKSQDJyZeeORCR8bYkhs8lI7eqddznH2ofh5WEEdjYXnoRtj7ByQwt7atMErmXIeYKPsSDuMAAqJDlQZ-3Ff4HJVeH_s3Gh8oQ',
-        'md5': '337cf7a344663ec79bf93a526a2e06c7',
-        'info_dict': {
-            'id': 'Da0Gw3xc',
-            'ext': 'mp4',
-            'title': 'No Daily Show for John Oliver; HBO Show Renewed - IGN News',
-            'uploader': 'IGN News',
-            'upload_date': '20150217',
-            'timestamp': 1424215740,
-            'description': 'HBO has renewed Last Week Tonight With John Oliver for two more seasons.',
-            'duration': 47.743333,
-        },
-    }
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        display_id = video_id[:8]
-
-        api_url = ('https://apib4.blinkx.com/api.php?action=play_video&'
-                   + 'video=%s' % video_id)
-        data_json = self._download_webpage(api_url, display_id)
-        data = json.loads(data_json)['api']['results'][0]
-        duration = None
-        thumbnails = []
-        formats = []
-        for m in data['media']:
-            if m['type'] == 'jpg':
-                thumbnails.append({
-                    'url': m['link'],
-                    'width': int(m['w']),
-                    'height': int(m['h']),
-                })
-            elif m['type'] == 'original':
-                duration = float(m['d'])
-            elif m['type'] == 'youtube':
-                yt_id = m['link']
-                self.to_screen('Youtube video detected: %s' % yt_id)
-                return self.url_result(yt_id, 'Youtube', video_id=yt_id)
-            elif m['type'] in ('flv', 'mp4'):
-                vcodec = remove_start(m['vcodec'], 'ff')
-                acodec = remove_start(m['acodec'], 'ff')
-                vbr = int_or_none(m.get('vbr') or m.get('vbitrate'), 1000)
-                abr = int_or_none(m.get('abr') or m.get('abitrate'), 1000)
-                tbr = vbr + abr if vbr and abr else None
-                format_id = '%s-%sk-%s' % (vcodec, tbr, m['w'])
-                formats.append({
-                    'format_id': format_id,
-                    'url': m['link'],
-                    'vcodec': vcodec,
-                    'acodec': acodec,
-                    'abr': abr,
-                    'vbr': vbr,
-                    'tbr': tbr,
-                    'width': int_or_none(m.get('w')),
-                    'height': int_or_none(m.get('h')),
-                })
-
-        self._sort_formats(formats)
-
-        return {
-            'id': display_id,
-            'fullid': video_id,
-            'title': data['title'],
-            'formats': formats,
-            'uploader': data['channel_name'],
-            'timestamp': data['pubdate_epoch'],
-            'description': data.get('description'),
-            'thumbnails': thumbnails,
-            'duration': duration,
-        }
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index ac33cd996..71584b1e6 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -132,7 +132,6 @@ from .bleacherreport import (
     BleacherReportIE,
     BleacherReportCMSIE,
 )
-from .blinkx import BlinkxIE
 from .bloomberg import BloombergIE
 from .bokecc import BokeCCIE
 from .bongacams import BongaCamsIE

From 2202cef0e4551293913e0be06b72b4cffccae0aa Mon Sep 17 00:00:00 2001
From: Lukas Anzinger <lukas@lukasanzinger.at>
Date: Sun, 16 May 2021 14:54:15 +0200
Subject: [PATCH 15/43] [orf:radio] Switch download URLs to HTTPS (closes
 #29012) (#29046)

---
 youtube_dl/extractor/orf.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/orf.py b/youtube_dl/extractor/orf.py
index 700ce448c..3fadbcbea 100644
--- a/youtube_dl/extractor/orf.py
+++ b/youtube_dl/extractor/orf.py
@@ -182,7 +182,7 @@ class ORFRadioIE(InfoExtractor):
             duration = end - start if end and start else None
             entries.append({
                 'id': loop_stream_id.replace('.mp3', ''),
-                'url': 'http://loopstream01.apa.at/?channel=%s&id=%s' % (self._LOOP_STATION, loop_stream_id),
+                'url': 'https://loopstream01.apa.at/?channel=%s&id=%s' % (self._LOOP_STATION, loop_stream_id),
                 'title': title,
                 'description': clean_html(data.get('subtitle')),
                 'duration': duration,

From 552b1399110360232bf1bf2ba5ed50cb8b30e818 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 16 May 2021 20:28:32 +0700
Subject: [PATCH 16/43] [generic] Add Referer header for direct videojs
 download URLs (closes #2879, closes #20217, closes #29053)

---
 youtube_dl/extractor/generic.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index f99d887ca..648a58c77 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -3400,6 +3400,9 @@ class GenericIE(InfoExtractor):
                         'url': src,
                         'ext': (mimetype2ext(src_type)
                                 or ext if ext in KNOWN_EXTENSIONS else 'mp4'),
+                        'http_headers': {
+                            'Referer': full_response.geturl(),
+                        },
                     })
             if formats:
                 self._sort_formats(formats)

From 8536dcafd82380c006a25409bdc2c3dc0d14195c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 16 May 2021 20:48:24 +0700
Subject: [PATCH 17/43] [vk] Add support for sibnet embeds (closes #9500)

---
 youtube_dl/extractor/vk.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index 00ec006c4..6b3513ee0 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -300,6 +300,13 @@ class VKIE(VKBaseIE):
             'only_matching': True,
         }]
 
+    @staticmethod
+    def _extract_sibnet_urls(webpage):
+        # https://help.sibnet.ru/?sibnet_video_embed
+        return [unescapeHTML(mobj.group('url')) for mobj in re.finditer(
+            r'<iframe\b[^>]+\bsrc=(["\'])(?P<url>(?:https?:)?//video\.sibnet\.ru/shell\.php\?.*?\bvideoid=\d+.*?)\1',
+            webpage)]
+
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('videoid')
@@ -408,6 +415,10 @@ class VKIE(VKBaseIE):
         if odnoklassniki_url:
             return self.url_result(odnoklassniki_url, OdnoklassnikiIE.ie_key())
 
+        sibnet_urls = self._extract_sibnet_urls(info_page)
+        if sibnet_urls:
+            return self.url_result(sibnet_urls[0])
+
         m_opts = re.search(r'(?s)var\s+opts\s*=\s*({.+?});', info_page)
         if m_opts:
             m_opts_url = re.search(r"url\s*:\s*'((?!/\b)[^']+)", m_opts.group(1))

From 286e01ce30b4d4d7a631512c3d1f983b30d9059c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 16 May 2021 20:50:32 +0700
Subject: [PATCH 18/43] [generic] Add support for sibnet embeds

---
 youtube_dl/extractor/generic.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 648a58c77..7b6f07318 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -126,6 +126,7 @@ from .viqeo import ViqeoIE
 from .expressen import ExpressenIE
 from .zype import ZypeIE
 from .odnoklassniki import OdnoklassnikiIE
+from .vk import VKIE
 from .kinja import KinjaEmbedIE
 from .arcpublishing import ArcPublishingIE
 from .medialaan import MedialaanIE
@@ -2248,6 +2249,11 @@ class GenericIE(InfoExtractor):
             },
             'playlist_mincount': 52,
         },
+        {
+            # Sibnet embed (https://help.sibnet.ru/?sibnet_video_embed)
+            'url': 'https://phpbb3.x-tk.ru/bbcode-video-sibnet-t24.html',
+            'only_matching': True,
+        },
     ]
 
     def report_following_redirect(self, new_url):
@@ -2777,6 +2783,11 @@ class GenericIE(InfoExtractor):
         if odnoklassniki_url:
             return self.url_result(odnoklassniki_url, OdnoklassnikiIE.ie_key())
 
+        # Look for sibnet embedded player
+        sibnet_urls = VKIE._extract_sibnet_urls(webpage)
+        if sibnet_urls:
+            return self.playlist_from_matches(sibnet_urls, video_id, video_title)
+
         # Look for embedded ivi player
         mobj = re.search(r'<embed[^>]+?src=(["\'])(?P<url>https?://(?:www\.)?ivi\.ru/video/player.+?)\1', webpage)
         if mobj is not None:

From eb5080286a8882eedbb77d1a8cd72f1c85b75737 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 16 May 2021 21:21:14 +0700
Subject: [PATCH 19/43] [phoenix] Fix extraction (closes #29057)

---
 youtube_dl/extractor/phoenix.py | 51 ++++++++++++++++++---------------
 1 file changed, 28 insertions(+), 23 deletions(-)

diff --git a/youtube_dl/extractor/phoenix.py b/youtube_dl/extractor/phoenix.py
index dbbfce983..e3ea01443 100644
--- a/youtube_dl/extractor/phoenix.py
+++ b/youtube_dl/extractor/phoenix.py
@@ -9,8 +9,9 @@ from ..compat import compat_str
 from ..utils import (
     int_or_none,
     merge_dicts,
+    try_get,
     unified_timestamp,
-    xpath_text,
+    urljoin,
 )
 
 
@@ -27,10 +28,11 @@ class PhoenixIE(ZDFBaseIE):
             'title': 'Wohin führt der Protest in der Pandemie?',
             'description': 'md5:7d643fe7f565e53a24aac036b2122fbd',
             'duration': 1691,
-            'timestamp': 1613906100,
+            'timestamp': 1613902500,
             'upload_date': '20210221',
             'uploader': 'Phoenix',
-            'channel': 'corona nachgehakt',
+            'series': 'corona nachgehakt',
+            'episode': 'Wohin führt der Protest in der Pandemie?',
         },
     }, {
         # Youtube embed
@@ -79,50 +81,53 @@ class PhoenixIE(ZDFBaseIE):
 
         video_id = compat_str(video.get('basename') or video.get('content'))
 
-        details = self._download_xml(
+        details = self._download_json(
             'https://www.phoenix.de/php/mediaplayer/data/beitrags_details.php',
-            video_id, 'Downloading details XML', query={
+            video_id, 'Downloading details JSON', query={
                 'ak': 'web',
                 'ptmd': 'true',
                 'id': video_id,
                 'profile': 'player2',
             })
 
-        title = title or xpath_text(
-            details, './/information/title', 'title', fatal=True)
-        content_id = xpath_text(
-            details, './/video/details/basename', 'content id', fatal=True)
+        title = title or details['title']
+        content_id = details['tracking']['nielsen']['content']['assetid']
 
         info = self._extract_ptmd(
             'https://tmd.phoenix.de/tmd/2/ngplayer_2_3/vod/ptmd/phoenix/%s' % content_id,
             content_id, None, url)
 
-        timestamp = unified_timestamp(xpath_text(details, './/details/airtime'))
+        duration = int_or_none(try_get(
+            details, lambda x: x['tracking']['nielsen']['content']['length']))
+        timestamp = unified_timestamp(details.get('editorialDate'))
+        series = try_get(
+            details, lambda x: x['tracking']['nielsen']['content']['program'],
+            compat_str)
+        episode = title if details.get('contentType') == 'episode' else None
 
         thumbnails = []
-        for node in details.findall('.//teaserimages/teaserimage'):
-            thumbnail_url = node.text
+        teaser_images = try_get(details, lambda x: x['teaserImageRef']['layouts'], dict) or {}
+        for thumbnail_key, thumbnail_url in teaser_images.items():
+            thumbnail_url = urljoin(url, thumbnail_url)
             if not thumbnail_url:
                 continue
             thumbnail = {
                 'url': thumbnail_url,
             }
-            thumbnail_key = node.get('key')
-            if thumbnail_key:
-                m = re.match('^([0-9]+)x([0-9]+)$', thumbnail_key)
-                if m:
-                    thumbnail['width'] = int(m.group(1))
-                    thumbnail['height'] = int(m.group(2))
+            m = re.match('^([0-9]+)x([0-9]+)$', thumbnail_key)
+            if m:
+                thumbnail['width'] = int(m.group(1))
+                thumbnail['height'] = int(m.group(2))
             thumbnails.append(thumbnail)
 
         return merge_dicts(info, {
             'id': content_id,
             'title': title,
-            'description': xpath_text(details, './/information/detail'),
-            'duration': int_or_none(xpath_text(details, './/details/lengthSec')),
+            'description': details.get('leadParagraph'),
+            'duration': duration,
             'thumbnails': thumbnails,
             'timestamp': timestamp,
-            'uploader': xpath_text(details, './/details/channel'),
-            'uploader_id': xpath_text(details, './/details/originChannelId'),
-            'channel': xpath_text(details, './/details/originChannelTitle'),
+            'uploader': details.get('tvService'),
+            'series': series,
+            'episode': episode,
         })

From 6423d7054eb2a73a1557c8531f631330e6d7e084 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 16 May 2021 21:34:10 +0700
Subject: [PATCH 20/43] [options] Fix thumbnail option group name (closes
 #29042)

---
 youtube_dl/options.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 241cf110f..0a0641bd4 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -768,7 +768,7 @@ def parseOpts(overrideArguments=None):
         action='store_true', dest='rm_cachedir',
         help='Delete all filesystem cache files')
 
-    thumbnail = optparse.OptionGroup(parser, 'Thumbnail images')
+    thumbnail = optparse.OptionGroup(parser, 'Thumbnail Options')
     thumbnail.add_option(
         '--write-thumbnail',
         action='store_true', dest='writethumbnail', default=False,

From 1e8aaa1d155d2f6b04ba1caa040876372bb0bb44 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 16 May 2021 21:42:38 +0700
Subject: [PATCH 21/43] [generic] Add support for og:audio (closes #28311,
 closes #29015)

---
 youtube_dl/extractor/generic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 7b6f07318..87594534f 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -3482,7 +3482,7 @@ class GenericIE(InfoExtractor):
             m_video_type = re.findall(r'<meta.*?property="og:video:type".*?content="video/(.*?)"', webpage)
             # We only look in og:video if the MIME type is a video, don't try if it's a Flash player:
             if m_video_type is not None:
-                found = filter_video(re.findall(r'<meta.*?property="og:video".*?content="(.*?)"', webpage))
+                found = filter_video(re.findall(r'<meta.*?property="og:(?:video|audio)".*?content="(.*?)"', webpage))
         if not found:
             REDIRECT_REGEX = r'[0-9]{,2};\s*(?:URL|url)=\'?([^\'"]+)'
             found = re.search(

From ef03721f470606deaddb4a57e51550045a32fb04 Mon Sep 17 00:00:00 2001
From: kr4ssi <44404263+kr4ssi@users.noreply.github.com>
Date: Sun, 16 May 2021 16:46:32 +0200
Subject: [PATCH 22/43] [vivo] Add support for vivo.st (#29009)

Co-authored-by: Sergey M. <dstftw@gmail.com>
---
 youtube_dl/extractor/shared.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/shared.py b/youtube_dl/extractor/shared.py
index 02295d1a4..93ab2a167 100644
--- a/youtube_dl/extractor/shared.py
+++ b/youtube_dl/extractor/shared.py
@@ -86,10 +86,10 @@ class SharedIE(SharedBaseIE):
 
 class VivoIE(SharedBaseIE):
     IE_DESC = 'vivo.sx'
-    _VALID_URL = r'https?://vivo\.sx/(?P<id>[\da-z]{10})'
+    _VALID_URL = r'https?://vivo\.s[xt]/(?P<id>[\da-z]{10})'
     _FILE_NOT_FOUND = '>The file you have requested does not exists or has been removed'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://vivo.sx/d7ddda0e78',
         'md5': '15b3af41be0b4fe01f4df075c2678b2c',
         'info_dict': {
@@ -98,7 +98,10 @@ class VivoIE(SharedBaseIE):
             'title': 'Chicken',
             'filesize': 515659,
         },
-    }
+    }, {
+        'url': 'http://vivo.st/d7ddda0e78',
+        'only_matching': True,
+    }]
 
     def _extract_title(self, webpage):
         title = self._html_search_regex(

From 503a3744ad620a2fe21c82cb2c595dc939310a1e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 16 May 2021 21:56:30 +0700
Subject: [PATCH 23/43] [eroprofile] Fix extraction (closes #23200, closes
 #23626, closes #29008)

---
 youtube_dl/extractor/eroprofile.py | 18 +++++++-----------
 1 file changed, 7 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/eroprofile.py b/youtube_dl/extractor/eroprofile.py
index c08643a17..9ee549d77 100644
--- a/youtube_dl/extractor/eroprofile.py
+++ b/youtube_dl/extractor/eroprofile.py
@@ -6,7 +6,7 @@ from .common import InfoExtractor
 from ..compat import compat_urllib_parse_urlencode
 from ..utils import (
     ExtractorError,
-    unescapeHTML
+    merge_dicts,
 )
 
 
@@ -77,19 +77,15 @@ class EroProfileIE(InfoExtractor):
             [r"glbUpdViews\s*\('\d*','(\d+)'", r'p/report/video/(\d+)'],
             webpage, 'video id', default=None)
 
-        video_url = unescapeHTML(self._search_regex(
-            r'<source src="([^"]+)', webpage, 'video url'))
         title = self._html_search_regex(
-            r'Title:</th><td>([^<]+)</td>', webpage, 'title')
-        thumbnail = self._search_regex(
-            r'onclick="showVideoPlayer\(\)"><img src="([^"]+)',
-            webpage, 'thumbnail', fatal=False)
+            (r'Title:</th><td>([^<]+)</td>', r'<h1[^>]*>(.+?)</h1>'),
+            webpage, 'title')
 
-        return {
+        info = self._parse_html5_media_entries(url, webpage, video_id)[0]
+
+        return merge_dicts(info, {
             'id': video_id,
             'display_id': display_id,
-            'url': video_url,
             'title': title,
-            'thumbnail': thumbnail,
             'age_limit': 18,
-        }
+        })

From 199c645bee2052e43ec33cc8d0b0fa0c18853da8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 16 May 2021 22:01:51 +0700
Subject: [PATCH 24/43] [eroprofile] Skip test

---
 youtube_dl/extractor/eroprofile.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/eroprofile.py b/youtube_dl/extractor/eroprofile.py
index 9ee549d77..c460dc7f9 100644
--- a/youtube_dl/extractor/eroprofile.py
+++ b/youtube_dl/extractor/eroprofile.py
@@ -24,7 +24,8 @@ class EroProfileIE(InfoExtractor):
             'title': 'sexy babe softcore',
             'thumbnail': r're:https?://.*\.jpg',
             'age_limit': 18,
-        }
+        },
+        'skip': 'Video not found',
     }, {
         'url': 'http://www.eroprofile.com/m/videos/view/Try-It-On-Pee_cut_2-wmv-4shared-com-file-sharing-download-movie-file',
         'md5': '1baa9602ede46ce904c431f5418d8916',

From e90a890f01ad253b611d8edd365f41b0c4553b67 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 16 May 2021 22:31:37 +0700
Subject: [PATCH 25/43] [playstuff] Add extractor (closes #28901, closes
 #28931)

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/playstuff.py  | 65 ++++++++++++++++++++++++++++++
 2 files changed, 66 insertions(+)
 create mode 100644 youtube_dl/extractor/playstuff.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 71584b1e6..402e542ae 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -925,6 +925,7 @@ from .platzi import (
 from .playfm import PlayFMIE
 from .playplustv import PlayPlusTVIE
 from .plays import PlaysTVIE
+from .playstuff import PlayStuffIE
 from .playtvak import PlaytvakIE
 from .playvid import PlayvidIE
 from .playwire import PlaywireIE
diff --git a/youtube_dl/extractor/playstuff.py b/youtube_dl/extractor/playstuff.py
new file mode 100644
index 000000000..5a329957f
--- /dev/null
+++ b/youtube_dl/extractor/playstuff.py
@@ -0,0 +1,65 @@
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..compat import compat_str
+from ..utils import (
+    smuggle_url,
+    try_get,
+)
+
+
+class PlayStuffIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?play\.stuff\.co\.nz/details/(?P<id>[^/?#&]+)'
+    _TESTS = [{
+        'url': 'https://play.stuff.co.nz/details/608778ac1de1c4001a3fa09a',
+        'md5': 'c82d3669e5247c64bc382577843e5bd0',
+        'info_dict': {
+            'id': '6250584958001',
+            'ext': 'mp4',
+            'title': 'Episode 1: Rotorua/Mt Maunganui/Tauranga',
+            'description': 'md5:c154bafb9f0dd02d01fd4100fb1c1913',
+            'uploader_id': '6005208634001',
+            'timestamp': 1619491027,
+            'upload_date': '20210427',
+        },
+        'add_ie': ['BrightcoveNew'],
+    }, {
+        # geo restricted, bypassable
+        'url': 'https://play.stuff.co.nz/details/_6155660351001',
+        'only_matching': True,
+    }]
+    BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/%s/%s_default/index.html?videoId=%s'
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        state = self._parse_json(
+            self._search_regex(
+                r'__INITIAL_STATE__\s*=\s*({.+?})\s*;', webpage, 'state'),
+            video_id)
+
+        account_id = try_get(
+            state, lambda x: x['configurations']['accountId'],
+            compat_str) or '6005208634001'
+        player_id = try_get(
+            state, lambda x: x['configurations']['playerId'],
+            compat_str) or 'default'
+
+        entries = []
+        for item_id, video in state['items'].items():
+            if not isinstance(video, dict):
+                continue
+            asset_id = try_get(
+                video, lambda x: x['content']['attributes']['assetId'],
+                compat_str)
+            if not asset_id:
+                continue
+            entries.append(self.url_result(
+                smuggle_url(
+                    self.BRIGHTCOVE_URL_TEMPLATE % (account_id, player_id, asset_id),
+                    {'geo_countries': ['NZ']}),
+                'BrightcoveNew', video_id))
+
+        return self.playlist_result(entries, video_id)

From efeb9e0fbf1e33043c19dbfda9d12984bf0c6e34 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 16 May 2021 22:40:39 +0700
Subject: [PATCH 26/43] [ChangeLog] Actualize [ci skip]

---
 ChangeLog | 31 +++++++++++++++++++++++++++++++
 1 file changed, 31 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index f15c84225..7c327ab9d 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,34 @@
+version <unreleased>
+
+Core
+* [options] Fix thumbnail option group name (#29042)
+* [YoutubeDL] Improve extract_info doc (#28946)
+
+Extractors
++ [playstuff] Add support for play.stuff.co.nz (#28901, #28931)
+* [eroprofile] Fix extraction (#23200, #23626, #29008)
++ [vivo] Add support for vivo.st (#29009)
++ [generic] Add support for og:audio (#28311, #29015)
+* [phoenix] Fix extraction (#29057)
++ [generic] Add support for sibnet embeds
++ [vk] Add support for sibnet embeds (#9500)
++ [generic] Add Referer header for direct videojs download URLs (#2879,
+  #20217, #29053)
+* [orf:radio] Switch download URLs to HTTPS (#29012, #29046)
+- [blinkx] Remove extractor (#28941)
+* [medaltv] Relax URL regular expression (#28884)
++ [funimation] Add support for optional lang code in URLs (#28950)
++ [gdcvault] Add support for HTML5 videos
+* [dispeak] Improve FLV extraction (#13513, #28970)
+* [kaltura] Improve iframe extraction (#28969)
+* [kaltura] Make embed code alternatives actually work
+* [cda] Improve extraction (#28709, #28937)
+* [twitter] Improve formats extraction from vmap URL (#28909)
+* [xtube] Fix formats extraction (#28870)
+* [svtplay] Improve extraction (#28507, #28876)
+* [tv2dk] Fix extraction (#28888)
+
+
 version 2021.04.26
 
 Extractors

From f47627a1c9a790fac29d5c166bc0f7944fcb1a98 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 16 May 2021 22:55:05 +0700
Subject: [PATCH 27/43] release 2021.05.16

---
 .github/ISSUE_TEMPLATE/1_broken_site.md          | 6 +++---
 .github/ISSUE_TEMPLATE/2_site_support_request.md | 4 ++--
 .github/ISSUE_TEMPLATE/3_site_feature_request.md | 4 ++--
 .github/ISSUE_TEMPLATE/4_bug_report.md           | 6 +++---
 .github/ISSUE_TEMPLATE/5_feature_request.md      | 4 ++--
 ChangeLog                                        | 2 +-
 README.md                                        | 2 +-
 docs/supportedsites.md                           | 2 +-
 youtube_dl/version.py                            | 2 +-
 9 files changed, 16 insertions(+), 16 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE/1_broken_site.md b/.github/ISSUE_TEMPLATE/1_broken_site.md
index 6ece3e031..d67bb482c 100644
--- a/.github/ISSUE_TEMPLATE/1_broken_site.md
+++ b/.github/ISSUE_TEMPLATE/1_broken_site.md
@@ -18,7 +18,7 @@ title: ''
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.04.26. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.05.16. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Make sure that all provided video/audio/playlist URLs (if any) are alive and playable in a browser.
 - Make sure that all URLs and arguments with special characters are properly quoted or escaped as explained in http://yt-dl.org/escape.
 - Search the bugtracker for similar issues: http://yt-dl.org/search-issues. DO NOT post duplicates.
@@ -26,7 +26,7 @@ Carefully read and work through this check list in order to prevent the most com
 -->
 
 - [ ] I'm reporting a broken site support
-- [ ] I've verified that I'm running youtube-dl version **2021.04.26**
+- [ ] I've verified that I'm running youtube-dl version **2021.05.16**
 - [ ] I've checked that all provided URLs are alive and playable in a browser
 - [ ] I've checked that all URLs and arguments with special characters are properly quoted or escaped
 - [ ] I've searched the bugtracker for similar issues including closed ones
@@ -41,7 +41,7 @@ Add the `-v` flag to your command line you run youtube-dl with (`youtube-dl -v <
  [debug] User config: []
  [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
  [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
- [debug] youtube-dl version 2021.04.26
+ [debug] youtube-dl version 2021.05.16
  [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
  [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
  [debug] Proxy map: {}
diff --git a/.github/ISSUE_TEMPLATE/2_site_support_request.md b/.github/ISSUE_TEMPLATE/2_site_support_request.md
index f923b2d5f..efe9fef8c 100644
--- a/.github/ISSUE_TEMPLATE/2_site_support_request.md
+++ b/.github/ISSUE_TEMPLATE/2_site_support_request.md
@@ -19,7 +19,7 @@ labels: 'site-support-request'
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.04.26. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.05.16. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Make sure that all provided video/audio/playlist URLs (if any) are alive and playable in a browser.
 - Make sure that site you are requesting is not dedicated to copyright infringement, see https://yt-dl.org/copyright-infringement. youtube-dl does not support such sites. In order for site support request to be accepted all provided example URLs should not violate any copyrights.
 - Search the bugtracker for similar site support requests: http://yt-dl.org/search-issues. DO NOT post duplicates.
@@ -27,7 +27,7 @@ Carefully read and work through this check list in order to prevent the most com
 -->
 
 - [ ] I'm reporting a new site support request
-- [ ] I've verified that I'm running youtube-dl version **2021.04.26**
+- [ ] I've verified that I'm running youtube-dl version **2021.05.16**
 - [ ] I've checked that all provided URLs are alive and playable in a browser
 - [ ] I've checked that none of provided URLs violate any copyrights
 - [ ] I've searched the bugtracker for similar site support requests including closed ones
diff --git a/.github/ISSUE_TEMPLATE/3_site_feature_request.md b/.github/ISSUE_TEMPLATE/3_site_feature_request.md
index 97d605653..e213fc1a9 100644
--- a/.github/ISSUE_TEMPLATE/3_site_feature_request.md
+++ b/.github/ISSUE_TEMPLATE/3_site_feature_request.md
@@ -18,13 +18,13 @@ title: ''
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.04.26. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.05.16. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Search the bugtracker for similar site feature requests: http://yt-dl.org/search-issues. DO NOT post duplicates.
 - Finally, put x into all relevant boxes (like this [x])
 -->
 
 - [ ] I'm reporting a site feature request
-- [ ] I've verified that I'm running youtube-dl version **2021.04.26**
+- [ ] I've verified that I'm running youtube-dl version **2021.05.16**
 - [ ] I've searched the bugtracker for similar site feature requests including closed ones
 
 
diff --git a/.github/ISSUE_TEMPLATE/4_bug_report.md b/.github/ISSUE_TEMPLATE/4_bug_report.md
index 73a806833..1645087ad 100644
--- a/.github/ISSUE_TEMPLATE/4_bug_report.md
+++ b/.github/ISSUE_TEMPLATE/4_bug_report.md
@@ -18,7 +18,7 @@ title: ''
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.04.26. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.05.16. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Make sure that all provided video/audio/playlist URLs (if any) are alive and playable in a browser.
 - Make sure that all URLs and arguments with special characters are properly quoted or escaped as explained in http://yt-dl.org/escape.
 - Search the bugtracker for similar issues: http://yt-dl.org/search-issues. DO NOT post duplicates.
@@ -27,7 +27,7 @@ Carefully read and work through this check list in order to prevent the most com
 -->
 
 - [ ] I'm reporting a broken site support issue
-- [ ] I've verified that I'm running youtube-dl version **2021.04.26**
+- [ ] I've verified that I'm running youtube-dl version **2021.05.16**
 - [ ] I've checked that all provided URLs are alive and playable in a browser
 - [ ] I've checked that all URLs and arguments with special characters are properly quoted or escaped
 - [ ] I've searched the bugtracker for similar bug reports including closed ones
@@ -43,7 +43,7 @@ Add the `-v` flag to your command line you run youtube-dl with (`youtube-dl -v <
  [debug] User config: []
  [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
  [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
- [debug] youtube-dl version 2021.04.26
+ [debug] youtube-dl version 2021.05.16
  [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
  [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
  [debug] Proxy map: {}
diff --git a/.github/ISSUE_TEMPLATE/5_feature_request.md b/.github/ISSUE_TEMPLATE/5_feature_request.md
index ee19a75f5..e6e569af6 100644
--- a/.github/ISSUE_TEMPLATE/5_feature_request.md
+++ b/.github/ISSUE_TEMPLATE/5_feature_request.md
@@ -19,13 +19,13 @@ labels: 'request'
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.04.26. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.05.16. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Search the bugtracker for similar feature requests: http://yt-dl.org/search-issues. DO NOT post duplicates.
 - Finally, put x into all relevant boxes (like this [x])
 -->
 
 - [ ] I'm reporting a feature request
-- [ ] I've verified that I'm running youtube-dl version **2021.04.26**
+- [ ] I've verified that I'm running youtube-dl version **2021.05.16**
 - [ ] I've searched the bugtracker for similar feature requests including closed ones
 
 
diff --git a/ChangeLog b/ChangeLog
index 7c327ab9d..5ea1d3150 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2021.05.16
 
 Core
 * [options] Fix thumbnail option group name (#29042)
diff --git a/README.md b/README.md
index 94c34d89a..059141611 100644
--- a/README.md
+++ b/README.md
@@ -287,7 +287,7 @@ Alternatively, refer to the [developer instructions](#developer-instructions) fo
     --no-cache-dir                       Disable filesystem caching
     --rm-cache-dir                       Delete all filesystem cache files
 
-## Thumbnail images:
+## Thumbnail Options:
     --write-thumbnail                    Write thumbnail image to disk
     --write-all-thumbnails               Write all thumbnail image formats to
                                          disk
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 88d474de4..ed0d5e9d9 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -119,7 +119,6 @@
  - **BitChuteChannel**
  - **BleacherReport**
  - **BleacherReportCMS**
- - **blinkx**
  - **Bloomberg**
  - **BokeCC**
  - **BongaCams**
@@ -713,6 +712,7 @@
  - **play.fm**
  - **player.sky.it**
  - **PlayPlusTV**
+ - **PlayStuff**
  - **PlaysTV**
  - **Playtvak**: Playtvak.cz, iDNES.cz and Lidovky.cz
  - **Playvid**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 576f721db..b16a84100 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2021.04.26'
+__version__ = '2021.05.16'

From e1a9d0ef780b7a0cdcdc706909ad2de9bd06138c Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 17 May 2021 12:37:39 +0100
Subject: [PATCH 28/43] [shahid] relax _VALID_URL(closes  #28772, closes
 #28930)

---
 youtube_dl/extractor/shahid.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/shahid.py b/youtube_dl/extractor/shahid.py
index b5e093bd2..88b938e05 100644
--- a/youtube_dl/extractor/shahid.py
+++ b/youtube_dl/extractor/shahid.py
@@ -21,6 +21,7 @@ from ..utils import (
 class ShahidBaseIE(AWSIE):
     _AWS_PROXY_HOST = 'api2.shahid.net'
     _AWS_API_KEY = '2RRtuMHx95aNI1Kvtn2rChEuwsCogUd4samGPjLh'
+    _VALID_URL_BASE = r'https?://shahid\.mbc\.net/[a-z]{2}/'
 
     def _handle_error(self, e):
         fail_data = self._parse_json(
@@ -49,7 +50,7 @@ class ShahidBaseIE(AWSIE):
 
 class ShahidIE(ShahidBaseIE):
     _NETRC_MACHINE = 'shahid'
-    _VALID_URL = r'https?://shahid\.mbc\.net/ar/(?:serie|show|movie)s/[^/]+/(?P<type>episode|clip|movie)-(?P<id>\d+)'
+    _VALID_URL = ShahidBaseIE._VALID_URL_BASE + r'(?:serie|show|movie)s/[^/]+/(?P<type>episode|clip|movie)-(?P<id>\d+)'
     _TESTS = [{
         'url': 'https://shahid.mbc.net/ar/shows/%D9%85%D8%AA%D8%AD%D9%81-%D8%A7%D9%84%D8%AF%D8%AD%D9%8A%D8%AD-%D8%A7%D9%84%D9%85%D9%88%D8%B3%D9%85-1-%D9%83%D9%84%D9%8A%D8%A8-1/clip-816924',
         'info_dict': {
@@ -73,6 +74,9 @@ class ShahidIE(ShahidBaseIE):
         # shahid plus subscriber only
         'url': 'https://shahid.mbc.net/ar/series/%D9%85%D8%B1%D8%A7%D9%8A%D8%A7-2011-%D8%A7%D9%84%D9%85%D9%88%D8%B3%D9%85-1-%D8%A7%D9%84%D8%AD%D9%84%D9%82%D8%A9-1/episode-90511',
         'only_matching': True
+    }, {
+        'url': 'https://shahid.mbc.net/en/shows/Ramez-Fi-Al-Shallal-season-1-episode-1/episode-359319',
+        'only_matching': True
     }]
 
     def _real_initialize(self):
@@ -168,7 +172,7 @@ class ShahidIE(ShahidBaseIE):
 
 
 class ShahidShowIE(ShahidBaseIE):
-    _VALID_URL = r'https?://shahid\.mbc\.net/ar/(?:show|serie)s/[^/]+/(?:show|series)-(?P<id>\d+)'
+    _VALID_URL = ShahidBaseIE._VALID_URL_BASE + r'(?:show|serie)s/[^/]+/(?:show|series)-(?P<id>\d+)'
     _TESTS = [{
         'url': 'https://shahid.mbc.net/ar/shows/%D8%B1%D8%A7%D9%85%D8%B2-%D9%82%D8%B1%D8%B4-%D8%A7%D9%84%D8%A8%D8%AD%D8%B1/show-79187',
         'info_dict': {

From dfbbe2902fc67f0f93ee47a8077c148055c67a9b Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 17 May 2021 12:56:49 +0100
Subject: [PATCH 29/43] [redbulltv] fix embed data extraction(closes #28770)

---
 youtube_dl/extractor/redbulltv.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/redbulltv.py b/youtube_dl/extractor/redbulltv.py
index 3aae79f5d..6d000b372 100644
--- a/youtube_dl/extractor/redbulltv.py
+++ b/youtube_dl/extractor/redbulltv.py
@@ -133,8 +133,10 @@ class RedBullEmbedIE(RedBullTVIE):
         rrn_id = self._match_id(url)
         asset_id = self._download_json(
             'https://edge-graphql.crepo-production.redbullaws.com/v1/graphql',
-            rrn_id, headers={'API-KEY': 'e90a1ff11335423998b100c929ecc866'},
-            query={
+            rrn_id, headers={
+                'Accept': 'application/json',
+                'API-KEY': 'e90a1ff11335423998b100c929ecc866',
+            }, query={
                 'query': '''{
   resource(id: "%s", enforceGeoBlocking: false) {
     %s

From 1980ff4550a3f040fbc1e054d6b91013e9d8cb96 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 26 May 2021 11:04:39 +0100
Subject: [PATCH 30/43] [vimeo] fix vimeo pro embed extraction(closes #29126)

---
 youtube_dl/extractor/vimeo.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 102687b82..0b386f450 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -647,7 +647,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
                         expected=True)
             raise
 
-        if '://player.vimeo.com/video/' in url:
+        if '//player.vimeo.com/video/' in url:
             config = self._parse_json(self._search_regex(
                 r'\bconfig\s*=\s*({.+?})\s*;', webpage, 'info section'), video_id)
             if config.get('view') == 4:

From 24297a42efc52862cb9510d32b28efd7faf49af6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 30 May 2021 00:36:26 +0700
Subject: [PATCH 31/43] [youtube] Fix get_video_info request (closes #29086,
 closes #29165)

---
 youtube_dl/extractor/youtube.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 0c52e5a8b..bf858c39d 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1499,6 +1499,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                     'unable to download video info webpage', query={
                         'video_id': video_id,
                         'eurl': 'https://youtube.googleapis.com/v/' + video_id,
+                        'html5': 1,
                     }, fatal=False)),
                 lambda x: x['player_response'][0],
                 compat_str) or '{}', video_id)

From e13a01061d149f4fac7db1a50124c4745a11c16e Mon Sep 17 00:00:00 2001
From: phlip <git@mrphlip.com>
Date: Fri, 28 May 2021 11:01:59 +1000
Subject: [PATCH 32/43] [twitch:clips] Add access token query to download URLs
 (closes #29136)

---
 youtube_dl/extractor/twitch.py | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index a7867f4d3..7f9738d43 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -49,6 +49,7 @@ class TwitchBaseIE(InfoExtractor):
         'ChannelCollectionsContent': '07e3691a1bad77a36aba590c351180439a40baefc1c275356f40fc7082419a84',
         'StreamMetadata': '1c719a40e481453e5c48d9bb585d971b8b372f8ebb105b17076722264dfa5b3e',
         'ComscoreStreamingQuery': 'e1edae8122517d013405f237ffcc124515dc6ded82480a88daef69c83b53ac01',
+        'VideoAccessToken_Clip': '36b89d2507fce29e5ca551df756d27c1cfe079e2609642b4390aa4c35796eb11',
         'VideoPreviewOverlay': '3006e77e51b128d838fa4e835723ca4dc9a05c5efd4466c1085215c6e437e65c',
         'VideoMetadata': '226edb3e692509f727fd56821f5653c05740242c82b0388883e0c0e75dcbf687',
     }
@@ -924,6 +925,17 @@ class TwitchClipsIE(TwitchBaseIE):
             raise ExtractorError(
                 'This clip is no longer available', expected=True)
 
+        access_token = self._download_gql(
+            video_id, [{
+                'operationName': 'VideoAccessToken_Clip',
+                'variables': {
+                    'slug': video_id,
+                },
+            }],
+            'Downloading access token GraphQL')
+        access_token = try_get(
+            access_token, lambda x: x[0]['data']['clip']['playbackAccessToken'])
+
         formats = []
         for option in clip.get('videoQualities', []):
             if not isinstance(option, dict):
@@ -931,6 +943,14 @@ class TwitchClipsIE(TwitchBaseIE):
             source = url_or_none(option.get('sourceURL'))
             if not source:
                 continue
+            if access_token:
+                source = "%s%s%s" % (
+                    source,
+                    "&" if "?" in source else "?",
+                    compat_urllib_parse_urlencode({
+                        "sig": access_token.get('signature'),
+                        "token": access_token.get('value'),
+                    }))
             formats.append({
                 'url': source,
                 'format_id': option.get('quality'),

From f3cd1d9cec91943a459a0662cbcffe3b2e1f6675 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 30 May 2021 01:46:49 +0700
Subject: [PATCH 33/43] [twitch:clips] Improve extraction (closes #29149)

---
 youtube_dl/extractor/twitch.py | 48 ++++++++++++++++------------------
 1 file changed, 23 insertions(+), 25 deletions(-)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index 7f9738d43..a378bd6dc 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -894,7 +894,25 @@ class TwitchClipsIE(TwitchBaseIE):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        clip = self._download_base_gql(
+        clip = self._download_gql(
+            video_id, [{
+                'operationName': 'VideoAccessToken_Clip',
+                'variables': {
+                    'slug': video_id,
+                },
+            }],
+            'Downloading clip access token GraphQL')[0]['data']['clip']
+
+        if not clip:
+            raise ExtractorError(
+                'This clip is no longer available', expected=True)
+
+        access_query = {
+            'sig': clip['playbackAccessToken']['signature'],
+            'token': clip['playbackAccessToken']['value'],
+        }
+
+        data = self._download_base_gql(
             video_id, {
                 'query': '''{
   clip(slug: "%s") {
@@ -919,22 +937,10 @@ class TwitchClipsIE(TwitchBaseIE):
     }
     viewCount
   }
-}''' % video_id}, 'Downloading clip GraphQL')['data']['clip']
+}''' % video_id}, 'Downloading clip GraphQL', fatal=False)
 
-        if not clip:
-            raise ExtractorError(
-                'This clip is no longer available', expected=True)
-
-        access_token = self._download_gql(
-            video_id, [{
-                'operationName': 'VideoAccessToken_Clip',
-                'variables': {
-                    'slug': video_id,
-                },
-            }],
-            'Downloading access token GraphQL')
-        access_token = try_get(
-            access_token, lambda x: x[0]['data']['clip']['playbackAccessToken'])
+        if data:
+            clip = try_get(data, lambda x: x['data']['clip'], dict) or clip
 
         formats = []
         for option in clip.get('videoQualities', []):
@@ -943,16 +949,8 @@ class TwitchClipsIE(TwitchBaseIE):
             source = url_or_none(option.get('sourceURL'))
             if not source:
                 continue
-            if access_token:
-                source = "%s%s%s" % (
-                    source,
-                    "&" if "?" in source else "?",
-                    compat_urllib_parse_urlencode({
-                        "sig": access_token.get('signature'),
-                        "token": access_token.get('value'),
-                    }))
             formats.append({
-                'url': source,
+                'url': update_url_query(source, access_query),
                 'format_id': option.get('quality'),
                 'height': int_or_none(option.get('quality')),
                 'fps': int_or_none(option.get('frameRate')),

From 6511b8e8d7db78d4ba3706df5122a74e1c9b9b57 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 30 May 2021 03:05:22 +0700
Subject: [PATCH 34/43] [ted] Prefer own formats over external sources (closes
 #29142)

---
 youtube_dl/extractor/ted.py | 24 ++++++++++++++----------
 1 file changed, 14 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/ted.py b/youtube_dl/extractor/ted.py
index 63e2455b2..f09f1a3f9 100644
--- a/youtube_dl/extractor/ted.py
+++ b/youtube_dl/extractor/ted.py
@@ -123,6 +123,10 @@ class TEDIE(InfoExtractor):
         'params': {
             'skip_download': True,
         },
+    }, {
+        # with own formats and private Youtube external
+        'url': 'https://www.ted.com/talks/spencer_wells_a_family_tree_for_humanity',
+        'only_matching': True,
     }]
 
     _NATIVE_FORMATS = {
@@ -210,16 +214,6 @@ class TEDIE(InfoExtractor):
 
         player_talk = talk_info['player_talks'][0]
 
-        external = player_talk.get('external')
-        if isinstance(external, dict):
-            service = external.get('service')
-            if isinstance(service, compat_str):
-                ext_url = None
-                if service.lower() == 'youtube':
-                    ext_url = external.get('code')
-
-                return self.url_result(ext_url or external['uri'])
-
         resources_ = player_talk.get('resources') or talk_info.get('resources')
 
         http_url = None
@@ -294,6 +288,16 @@ class TEDIE(InfoExtractor):
                 'vcodec': 'none',
             })
 
+        if not formats:
+            external = player_talk.get('external')
+            if isinstance(external, dict):
+                service = external.get('service')
+                if isinstance(service, compat_str):
+                    ext_url = None
+                    if service.lower() == 'youtube':
+                        ext_url = external.get('code')
+                    return self.url_result(ext_url or external['uri'])
+
         self._sort_formats(formats)
 
         video_id = compat_str(talk_info['id'])

From 2ee6c7f11074917c08253af4c47f9258aa1e0dad Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 30 May 2021 03:43:59 +0700
Subject: [PATCH 35/43] [ustream] Detect https embeds (closes #29133)

---
 youtube_dl/extractor/ustream.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ustream.py b/youtube_dl/extractor/ustream.py
index 9e860aeb7..1e29cbe22 100644
--- a/youtube_dl/extractor/ustream.py
+++ b/youtube_dl/extractor/ustream.py
@@ -75,7 +75,7 @@ class UstreamIE(InfoExtractor):
     @staticmethod
     def _extract_url(webpage):
         mobj = re.search(
-            r'<iframe[^>]+?src=(["\'])(?P<url>http://(?:www\.)?(?:ustream\.tv|video\.ibm\.com)/embed/.+?)\1', webpage)
+            r'<iframe[^>]+?src=(["\'])(?P<url>https?://(?:www\.)?(?:ustream\.tv|video\.ibm\.com)/embed/.+?)\1', webpage)
         if mobj is not None:
             return mobj.group('url')
 

From d495292852b6c2f1bd58bc2141ff2b0265c952cf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 30 May 2021 06:14:59 +0700
Subject: [PATCH 36/43] [ard] Relax _VALID_URL and fix video ids (closes
 #22724, closes #29091)

---
 youtube_dl/extractor/ard.py | 21 +++++++++++++++------
 1 file changed, 15 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/ard.py b/youtube_dl/extractor/ard.py
index d57c5ba0f..d45a9fe52 100644
--- a/youtube_dl/extractor/ard.py
+++ b/youtube_dl/extractor/ard.py
@@ -249,14 +249,14 @@ class ARDMediathekIE(ARDMediathekBaseIE):
 
 
 class ARDIE(InfoExtractor):
-    _VALID_URL = r'(?P<mainurl>https?://(?:www\.)?daserste\.de/[^?#]+/videos(?:extern)?/(?P<display_id>[^/?#]+)-(?:video-?)?(?P<id>[0-9]+))\.html'
+    _VALID_URL = r'(?P<mainurl>https?://(?:www\.)?daserste\.de/(?:[^/?#&]+/)+(?P<id>[^/?#&]+))\.html'
     _TESTS = [{
         # available till 7.01.2022
         'url': 'https://www.daserste.de/information/talk/maischberger/videos/maischberger-die-woche-video100.html',
         'md5': '867d8aa39eeaf6d76407c5ad1bb0d4c1',
         'info_dict': {
-            'display_id': 'maischberger-die-woche',
-            'id': '100',
+            'id': 'maischberger-die-woche-video100',
+            'display_id': 'maischberger-die-woche-video100',
             'ext': 'mp4',
             'duration': 3687.0,
             'title': 'maischberger. die woche vom 7. Januar 2021',
@@ -264,16 +264,25 @@ class ARDIE(InfoExtractor):
             'thumbnail': r're:^https?://.*\.jpg$',
         },
     }, {
-        'url': 'https://www.daserste.de/information/reportage-dokumentation/erlebnis-erde/videosextern/woelfe-und-herdenschutzhunde-ungleiche-brueder-102.html',
+        'url': 'https://www.daserste.de/information/politik-weltgeschehen/morgenmagazin/videosextern/dominik-kahun-aus-der-nhl-direkt-zur-weltmeisterschaft-100.html',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.daserste.de/information/nachrichten-wetter/tagesthemen/videosextern/tagesthemen-17736.html',
         'only_matching': True,
     }, {
         'url': 'http://www.daserste.de/information/reportage-dokumentation/dokus/videos/die-story-im-ersten-mission-unter-falscher-flagge-100.html',
         'only_matching': True,
+    }, {
+        'url': 'https://www.daserste.de/unterhaltung/serie/in-aller-freundschaft-die-jungen-aerzte/Drehpause-100.html',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.daserste.de/unterhaltung/film/filmmittwoch-im-ersten/videos/making-ofwendezeit-video-100.html',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
-        display_id = mobj.group('display_id')
+        display_id = mobj.group('id')
 
         player_url = mobj.group('mainurl') + '~playerXml.xml'
         doc = self._download_xml(player_url, display_id)
@@ -324,7 +333,7 @@ class ARDIE(InfoExtractor):
         self._sort_formats(formats)
 
         return {
-            'id': mobj.group('id'),
+            'id': xpath_text(video_node, './videoId', default=display_id),
             'formats': formats,
             'display_id': display_id,
             'title': video_node.find('./title').text,

From 82f3993ba3f4d435d3bc9e37426ab225f5549510 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Fri, 4 Jun 2021 17:51:44 +0100
Subject: [PATCH 37/43] [formula1] fix extraction(closes #29206)

---
 youtube_dl/extractor/formula1.py | 32 +++++++++++++-------------------
 1 file changed, 13 insertions(+), 19 deletions(-)

diff --git a/youtube_dl/extractor/formula1.py b/youtube_dl/extractor/formula1.py
index fecfc28ae..67662e6de 100644
--- a/youtube_dl/extractor/formula1.py
+++ b/youtube_dl/extractor/formula1.py
@@ -5,29 +5,23 @@ from .common import InfoExtractor
 
 
 class Formula1IE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?formula1\.com/(?:content/fom-website/)?en/video/\d{4}/\d{1,2}/(?P<id>.+?)\.html'
-    _TESTS = [{
-        'url': 'http://www.formula1.com/content/fom-website/en/video/2016/5/Race_highlights_-_Spain_2016.html',
-        'md5': '8c79e54be72078b26b89e0e111c0502b',
+    _VALID_URL = r'https?://(?:www\.)?formula1\.com/en/latest/video\.[^.]+\.(?P<id>\d+)\.html'
+    _TEST = {
+        'url': 'https://www.formula1.com/en/latest/video.race-highlights-spain-2016.6060988138001.html',
+        'md5': 'be7d3a8c2f804eb2ab2aa5d941c359f8',
         'info_dict': {
-            'id': 'JvYXJpMzE6pArfHWm5ARp5AiUmD-gibV',
+            'id': '6060988138001',
             'ext': 'mp4',
             'title': 'Race highlights - Spain 2016',
+            'timestamp': 1463332814,
+            'upload_date': '20160515',
+            'uploader_id': '6057949432001',
         },
-        'params': {
-            # m3u8 download
-            'skip_download': True,
-        },
-        'add_ie': ['Ooyala'],
-    }, {
-        'url': 'http://www.formula1.com/en/video/2016/5/Race_highlights_-_Spain_2016.html',
-        'only_matching': True,
-    }]
+        'add_ie': ['BrightcoveNew'],
+    }
+    BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/6057949432001/S1WMrhjlh_default/index.html?videoId=%s'
 
     def _real_extract(self, url):
-        display_id = self._match_id(url)
-        webpage = self._download_webpage(url, display_id)
-        ooyala_embed_code = self._search_regex(
-            r'data-videoid="([^"]+)"', webpage, 'ooyala embed code')
+        bc_id = self._match_id(url)
         return self.url_result(
-            'ooyala:%s' % ooyala_embed_code, 'Ooyala', ooyala_embed_code)
+            self.BRIGHTCOVE_URL_TEMPLATE % bc_id, 'BrightcoveNew', bc_id)

From 943070af4a9e13ef2b81c5e484d9c799f1845aab Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 5 Jun 2021 23:42:25 +0700
Subject: [PATCH 38/43] [orf:tvthek] Fix thumbnails extraction (closes #29217)

---
 youtube_dl/extractor/orf.py | 21 ++++++++++++++++++++-
 1 file changed, 20 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/orf.py b/youtube_dl/extractor/orf.py
index 3fadbcbea..ed8a9a841 100644
--- a/youtube_dl/extractor/orf.py
+++ b/youtube_dl/extractor/orf.py
@@ -140,6 +140,25 @@ class ORFTVthekIE(InfoExtractor):
                 })
 
             upload_date = unified_strdate(sd.get('created_date'))
+
+            thumbnails = []
+            preview = sd.get('preview_image_url')
+            if preview:
+                thumbnails.append({
+                    'id': 'preview',
+                    'url': preview,
+                    'preference': 0,
+                })
+            image = sd.get('image_full_url')
+            if not image and len(data_jsb) == 1:
+                image = self._og_search_thumbnail(webpage)
+            if image:
+                thumbnails.append({
+                    'id': 'full',
+                    'url': image,
+                    'preference': 1,
+                })
+
             entries.append({
                 '_type': 'video',
                 'id': video_id,
@@ -149,7 +168,7 @@ class ORFTVthekIE(InfoExtractor):
                 'description': sd.get('description'),
                 'duration': int_or_none(sd.get('duration_in_seconds')),
                 'upload_date': upload_date,
-                'thumbnail': sd.get('image_full_url'),
+                'thumbnails': thumbnails,
             })
 
         return {

From fdf91c52a8b58b3b7c12a393629fc962d6ab7618 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 6 Jun 2021 00:11:09 +0700
Subject: [PATCH 39/43] [youporn] Fix formats and view count extraction (closes
 #29216)

---
 youtube_dl/extractor/youporn.py | 111 ++++++++++++--------------------
 1 file changed, 42 insertions(+), 69 deletions(-)

diff --git a/youtube_dl/extractor/youporn.py b/youtube_dl/extractor/youporn.py
index 33114363d..7084d3d12 100644
--- a/youtube_dl/extractor/youporn.py
+++ b/youtube_dl/extractor/youporn.py
@@ -4,13 +4,12 @@ import re
 
 from .common import InfoExtractor
 from ..utils import (
+    extract_attributes,
     int_or_none,
     str_to_int,
-    unescapeHTML,
     unified_strdate,
     url_or_none,
 )
-from ..aes import aes_decrypt_text
 
 
 class YouPornIE(InfoExtractor):
@@ -34,6 +33,7 @@ class YouPornIE(InfoExtractor):
             'tags': list,
             'age_limit': 18,
         },
+        'skip': 'This video has been disabled',
     }, {
         # Unknown uploader
         'url': 'http://www.youporn.com/watch/561726/big-tits-awesome-brunette-on-amazing-webcam-show/?from=related3&al=2&from_id=561726&pos=4',
@@ -78,6 +78,40 @@ class YouPornIE(InfoExtractor):
         video_id = mobj.group('id')
         display_id = mobj.group('display_id') or video_id
 
+        definitions = self._download_json(
+            'https://www.youporn.com/api/video/media_definitions/%s/' % video_id,
+            display_id)
+
+        formats = []
+        for definition in definitions:
+            if not isinstance(definition, dict):
+                continue
+            video_url = url_or_none(definition.get('videoUrl'))
+            if not video_url:
+                continue
+            f = {
+                'url': video_url,
+                'filesize': int_or_none(definition.get('videoSize')),
+            }
+            height = int_or_none(definition.get('quality'))
+            # Video URL's path looks like this:
+            #  /201012/17/505835/720p_1500k_505835/YouPorn%20-%20Sex%20Ed%20Is%20It%20Safe%20To%20Masturbate%20Daily.mp4
+            #  /201012/17/505835/vl_240p_240k_505835/YouPorn%20-%20Sex%20Ed%20Is%20It%20Safe%20To%20Masturbate%20Daily.mp4
+            #  /videos/201703/11/109285532/1080P_4000K_109285532.mp4
+            # We will benefit from it by extracting some metadata
+            mobj = re.search(r'(?P<height>\d{3,4})[pP]_(?P<bitrate>\d+)[kK]_\d+', video_url)
+            if mobj:
+                if not height:
+                    height = int(mobj.group('height'))
+                bitrate = int(mobj.group('bitrate'))
+                f.update({
+                    'format_id': '%dp-%dk' % (height, bitrate),
+                    'tbr': bitrate,
+                })
+            f['height'] = height
+            formats.append(f)
+        self._sort_formats(formats)
+
         webpage = self._download_webpage(
             'http://www.youporn.com/watch/%s' % video_id, display_id,
             headers={'Cookie': 'age_verified=1'})
@@ -88,65 +122,6 @@ class YouPornIE(InfoExtractor):
             webpage, default=None) or self._html_search_meta(
             'title', webpage, fatal=True)
 
-        links = []
-
-        # Main source
-        definitions = self._parse_json(
-            self._search_regex(
-                r'mediaDefinition\s*[=:]\s*(\[.+?\])\s*[;,]', webpage,
-                'media definitions', default='[]'),
-            video_id, fatal=False)
-        if definitions:
-            for definition in definitions:
-                if not isinstance(definition, dict):
-                    continue
-                video_url = url_or_none(definition.get('videoUrl'))
-                if video_url:
-                    links.append(video_url)
-
-        # Fallback #1, this also contains extra low quality 180p format
-        for _, link in re.findall(r'<a[^>]+href=(["\'])(http(?:(?!\1).)+\.mp4(?:(?!\1).)*)\1[^>]+title=["\']Download [Vv]ideo', webpage):
-            links.append(link)
-
-        # Fallback #2 (unavailable as at 22.06.2017)
-        sources = self._search_regex(
-            r'(?s)sources\s*:\s*({.+?})', webpage, 'sources', default=None)
-        if sources:
-            for _, link in re.findall(r'[^:]+\s*:\s*(["\'])(http.+?)\1', sources):
-                links.append(link)
-
-        # Fallback #3 (unavailable as at 22.06.2017)
-        for _, link in re.findall(
-                r'(?:videoSrc|videoIpadUrl|html5PlayerSrc)\s*[:=]\s*(["\'])(http.+?)\1', webpage):
-            links.append(link)
-
-        # Fallback #4, encrypted links (unavailable as at 22.06.2017)
-        for _, encrypted_link in re.findall(
-                r'encryptedQuality\d{3,4}URL\s*=\s*(["\'])([\da-zA-Z+/=]+)\1', webpage):
-            links.append(aes_decrypt_text(encrypted_link, title, 32).decode('utf-8'))
-
-        formats = []
-        for video_url in set(unescapeHTML(link) for link in links):
-            f = {
-                'url': video_url,
-            }
-            # Video URL's path looks like this:
-            #  /201012/17/505835/720p_1500k_505835/YouPorn%20-%20Sex%20Ed%20Is%20It%20Safe%20To%20Masturbate%20Daily.mp4
-            #  /201012/17/505835/vl_240p_240k_505835/YouPorn%20-%20Sex%20Ed%20Is%20It%20Safe%20To%20Masturbate%20Daily.mp4
-            #  /videos/201703/11/109285532/1080P_4000K_109285532.mp4
-            # We will benefit from it by extracting some metadata
-            mobj = re.search(r'(?P<height>\d{3,4})[pP]_(?P<bitrate>\d+)[kK]_\d+', video_url)
-            if mobj:
-                height = int(mobj.group('height'))
-                bitrate = int(mobj.group('bitrate'))
-                f.update({
-                    'format_id': '%dp-%dk' % (height, bitrate),
-                    'height': height,
-                    'tbr': bitrate,
-                })
-            formats.append(f)
-        self._sort_formats(formats)
-
         description = self._html_search_regex(
             r'(?s)<div[^>]+\bid=["\']description["\'][^>]*>(.+?)</div>',
             webpage, 'description',
@@ -169,13 +144,12 @@ class YouPornIE(InfoExtractor):
 
         age_limit = self._rta_search(webpage)
 
-        average_rating = int_or_none(self._search_regex(
-            r'<div[^>]+class=["\']videoRatingPercentage["\'][^>]*>(\d+)%</div>',
-            webpage, 'average rating', fatal=False))
-
-        view_count = str_to_int(self._search_regex(
-            r'(?s)<div[^>]+class=(["\']).*?\bvideoInfoViews\b.*?\1[^>]*>.*?(?P<count>[\d,.]+)<',
-            webpage, 'view count', fatal=False, group='count'))
+        view_count = None
+        views = self._search_regex(
+            r'(<div[^>]+\bclass=["\']js_videoInfoViews["\']>)', webpage,
+            'views', default=None)
+        if views:
+            view_count = str_to_int(extract_attributes(views).get('data-value'))
         comment_count = str_to_int(self._search_regex(
             r'>All [Cc]omments? \(([\d,.]+)\)',
             webpage, 'comment count', default=None))
@@ -201,7 +175,6 @@ class YouPornIE(InfoExtractor):
             'duration': duration,
             'uploader': uploader,
             'upload_date': upload_date,
-            'average_rating': average_rating,
             'view_count': view_count,
             'comment_count': comment_count,
             'categories': categories,

From bb7ac1ed669d67d79fa1a3b9e5c70271892ecbcc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 6 Jun 2021 01:16:43 +0700
Subject: [PATCH 40/43] [facebook] Improve login required detection

---
 youtube_dl/extractor/facebook.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py
index cb34c59f5..04650af39 100644
--- a/youtube_dl/extractor/facebook.py
+++ b/youtube_dl/extractor/facebook.py
@@ -521,7 +521,10 @@ class FacebookIE(InfoExtractor):
                 raise ExtractorError(
                     'The video is not available, Facebook said: "%s"' % m_msg.group(1),
                     expected=True)
-            elif '>You must log in to continue' in webpage:
+            elif any(p in webpage for p in (
+                    '>You must log in to continue',
+                    'id="login_form"',
+                    'id="loginbutton"')):
                 self.raise_login_required()
 
         if not video_data and '/watchparty/' in url:

From 5f85eb820cb7eb89dcb567f9cbfefb5d9038b9c0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 6 Jun 2021 01:32:15 +0700
Subject: [PATCH 41/43] [ChangeLog] Actualize [ci skip]

---
 ChangeLog | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 5ea1d3150..06efe32ab 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,21 @@
+version <unreleased>
+
+Extractors
+* [facebook] Improve login required detection
+* [youporn] Fix formats and view count extraction (#29216)
+* [orf:tvthek] Fix thumbnails extraction (#29217)
+* [formula1] Fix extraction (#29206)
+* [ard] Relax URL regular expression and fix video ids (#22724, #29091)
++ [ustream] Detect https embeds (#29133)
+* [ted] Prefer own formats over external sources (#29142)
+* [twitch:clips] Improve extraction (#29149)
++ [twitch:clips] Add access token query to download URLs (#29136)
+* [youtube] Fix get_video_info request (#29086, #29165)
+* [vimeo] Fix vimeo pro embed extraction (#29126)
+* [redbulltv] Fix embed data extraction (#28770)
+* [shahid] Relax URL regular expression (#28772, #28930)
+
+
 version 2021.05.16
 
 Core

From b224cf39d53bd16bcfda2ac493712c3ff449ecb8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 6 Jun 2021 01:38:22 +0700
Subject: [PATCH 42/43] release 2021.06.06

---
 .github/ISSUE_TEMPLATE/1_broken_site.md          | 6 +++---
 .github/ISSUE_TEMPLATE/2_site_support_request.md | 4 ++--
 .github/ISSUE_TEMPLATE/3_site_feature_request.md | 4 ++--
 .github/ISSUE_TEMPLATE/4_bug_report.md           | 6 +++---
 .github/ISSUE_TEMPLATE/5_feature_request.md      | 4 ++--
 ChangeLog                                        | 2 +-
 youtube_dl/version.py                            | 2 +-
 7 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE/1_broken_site.md b/.github/ISSUE_TEMPLATE/1_broken_site.md
index d67bb482c..4eb505231 100644
--- a/.github/ISSUE_TEMPLATE/1_broken_site.md
+++ b/.github/ISSUE_TEMPLATE/1_broken_site.md
@@ -18,7 +18,7 @@ title: ''
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.05.16. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.06.06. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Make sure that all provided video/audio/playlist URLs (if any) are alive and playable in a browser.
 - Make sure that all URLs and arguments with special characters are properly quoted or escaped as explained in http://yt-dl.org/escape.
 - Search the bugtracker for similar issues: http://yt-dl.org/search-issues. DO NOT post duplicates.
@@ -26,7 +26,7 @@ Carefully read and work through this check list in order to prevent the most com
 -->
 
 - [ ] I'm reporting a broken site support
-- [ ] I've verified that I'm running youtube-dl version **2021.05.16**
+- [ ] I've verified that I'm running youtube-dl version **2021.06.06**
 - [ ] I've checked that all provided URLs are alive and playable in a browser
 - [ ] I've checked that all URLs and arguments with special characters are properly quoted or escaped
 - [ ] I've searched the bugtracker for similar issues including closed ones
@@ -41,7 +41,7 @@ Add the `-v` flag to your command line you run youtube-dl with (`youtube-dl -v <
  [debug] User config: []
  [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
  [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
- [debug] youtube-dl version 2021.05.16
+ [debug] youtube-dl version 2021.06.06
  [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
  [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
  [debug] Proxy map: {}
diff --git a/.github/ISSUE_TEMPLATE/2_site_support_request.md b/.github/ISSUE_TEMPLATE/2_site_support_request.md
index efe9fef8c..9fed0b489 100644
--- a/.github/ISSUE_TEMPLATE/2_site_support_request.md
+++ b/.github/ISSUE_TEMPLATE/2_site_support_request.md
@@ -19,7 +19,7 @@ labels: 'site-support-request'
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.05.16. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.06.06. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Make sure that all provided video/audio/playlist URLs (if any) are alive and playable in a browser.
 - Make sure that site you are requesting is not dedicated to copyright infringement, see https://yt-dl.org/copyright-infringement. youtube-dl does not support such sites. In order for site support request to be accepted all provided example URLs should not violate any copyrights.
 - Search the bugtracker for similar site support requests: http://yt-dl.org/search-issues. DO NOT post duplicates.
@@ -27,7 +27,7 @@ Carefully read and work through this check list in order to prevent the most com
 -->
 
 - [ ] I'm reporting a new site support request
-- [ ] I've verified that I'm running youtube-dl version **2021.05.16**
+- [ ] I've verified that I'm running youtube-dl version **2021.06.06**
 - [ ] I've checked that all provided URLs are alive and playable in a browser
 - [ ] I've checked that none of provided URLs violate any copyrights
 - [ ] I've searched the bugtracker for similar site support requests including closed ones
diff --git a/.github/ISSUE_TEMPLATE/3_site_feature_request.md b/.github/ISSUE_TEMPLATE/3_site_feature_request.md
index e213fc1a9..573e8ded0 100644
--- a/.github/ISSUE_TEMPLATE/3_site_feature_request.md
+++ b/.github/ISSUE_TEMPLATE/3_site_feature_request.md
@@ -18,13 +18,13 @@ title: ''
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.05.16. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.06.06. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Search the bugtracker for similar site feature requests: http://yt-dl.org/search-issues. DO NOT post duplicates.
 - Finally, put x into all relevant boxes (like this [x])
 -->
 
 - [ ] I'm reporting a site feature request
-- [ ] I've verified that I'm running youtube-dl version **2021.05.16**
+- [ ] I've verified that I'm running youtube-dl version **2021.06.06**
 - [ ] I've searched the bugtracker for similar site feature requests including closed ones
 
 
diff --git a/.github/ISSUE_TEMPLATE/4_bug_report.md b/.github/ISSUE_TEMPLATE/4_bug_report.md
index 1645087ad..c0031bf7a 100644
--- a/.github/ISSUE_TEMPLATE/4_bug_report.md
+++ b/.github/ISSUE_TEMPLATE/4_bug_report.md
@@ -18,7 +18,7 @@ title: ''
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.05.16. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.06.06. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Make sure that all provided video/audio/playlist URLs (if any) are alive and playable in a browser.
 - Make sure that all URLs and arguments with special characters are properly quoted or escaped as explained in http://yt-dl.org/escape.
 - Search the bugtracker for similar issues: http://yt-dl.org/search-issues. DO NOT post duplicates.
@@ -27,7 +27,7 @@ Carefully read and work through this check list in order to prevent the most com
 -->
 
 - [ ] I'm reporting a broken site support issue
-- [ ] I've verified that I'm running youtube-dl version **2021.05.16**
+- [ ] I've verified that I'm running youtube-dl version **2021.06.06**
 - [ ] I've checked that all provided URLs are alive and playable in a browser
 - [ ] I've checked that all URLs and arguments with special characters are properly quoted or escaped
 - [ ] I've searched the bugtracker for similar bug reports including closed ones
@@ -43,7 +43,7 @@ Add the `-v` flag to your command line you run youtube-dl with (`youtube-dl -v <
  [debug] User config: []
  [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
  [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
- [debug] youtube-dl version 2021.05.16
+ [debug] youtube-dl version 2021.06.06
  [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
  [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
  [debug] Proxy map: {}
diff --git a/.github/ISSUE_TEMPLATE/5_feature_request.md b/.github/ISSUE_TEMPLATE/5_feature_request.md
index e6e569af6..1138ab2ca 100644
--- a/.github/ISSUE_TEMPLATE/5_feature_request.md
+++ b/.github/ISSUE_TEMPLATE/5_feature_request.md
@@ -19,13 +19,13 @@ labels: 'request'
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.05.16. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.06.06. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Search the bugtracker for similar feature requests: http://yt-dl.org/search-issues. DO NOT post duplicates.
 - Finally, put x into all relevant boxes (like this [x])
 -->
 
 - [ ] I'm reporting a feature request
-- [ ] I've verified that I'm running youtube-dl version **2021.05.16**
+- [ ] I've verified that I'm running youtube-dl version **2021.06.06**
 - [ ] I've searched the bugtracker for similar feature requests including closed ones
 
 
diff --git a/ChangeLog b/ChangeLog
index 06efe32ab..680fffdf8 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2021.06.06
 
 Extractors
 * [facebook] Improve login required detection
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index b16a84100..461dd87ca 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2021.05.16'
+__version__ = '2021.06.06'

From c2350cac243ba1ec1586fe85b0d62d1b700047a2 Mon Sep 17 00:00:00 2001
From: Sergey M <dstftw@gmail.com>
Date: Sun, 6 Jun 2021 05:32:27 +0700
Subject: [PATCH 43/43] [README.md] Update MSVC 2010 redist URL (closes #29222)

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 059141611..2841ed68f 100644
--- a/README.md
+++ b/README.md
@@ -893,7 +893,7 @@ Since June 2012 ([#342](https://github.com/ytdl-org/youtube-dl/issues/342)) yout
 
 ### The exe throws an error due to missing `MSVCR100.dll`
 
-To run the exe you need to install first the [Microsoft Visual C++ 2010 Redistributable Package (x86)](https://www.microsoft.com/en-US/download/details.aspx?id=5555).
+To run the exe you need to install first the [Microsoft Visual C++ 2010 Service Pack 1 Redistributable Package (x86)](https://download.microsoft.com/download/1/6/5/165255E7-1014-4D0A-B094-B6A430A6BFFC/vcredist_x86.exe).
 
 ### On Windows, how should I set up ffmpeg and youtube-dl? Where should I put the exe files?