From 105748fc7c448ca482668b9844ba95b7b5924cb0 Mon Sep 17 00:00:00 2001 From: Mattia Date: Fri, 11 Sep 2020 15:45:50 +0200 Subject: [PATCH 1/8] Fix ClosedCaption test --- test/closed_caption_test.dart | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/test/closed_caption_test.dart b/test/closed_caption_test.dart index a356460..1d6e148 100644 --- a/test/closed_caption_test.dart +++ b/test/closed_caption_test.dart @@ -13,11 +13,11 @@ void main() { }); test('GetClosedCaptionTracksOfAnyVideo', () async { - var manifest = await yt.videos.closedCaptions.getManifest('_QdPW8JrYzQ'); + var manifest = await yt.videos.closedCaptions.getManifest('WOxr2dmLHLo'); expect(manifest.tracks, isNotEmpty); }); test('GetClosedCaptionTrackOfAnyVideoSpecific', () async { - var manifest = await yt.videos.closedCaptions.getManifest('_QdPW8JrYzQ'); + var manifest = await yt.videos.closedCaptions.getManifest('WOxr2dmLHLo'); var trackInfo = manifest.tracks.first; var track = await yt.videos.closedCaptions.get(trackInfo); @@ -25,18 +25,18 @@ void main() { }); test('GetClosedCaptionTrackAtSpecificTime', () async { var manifest = await yt.videos.closedCaptions - .getManifest('https://www.youtube.com/watch?v=YltHGKX80Y8'); + .getManifest('https://www.youtube.com/watch?v=ppJy5uGZLi4'); var trackInfo = manifest.getByLanguage('en'); var track = await yt.videos.closedCaptions.get(trackInfo); var caption = - track.getByTime(const Duration(hours: 0, minutes: 10, seconds: 41)); + track.getByTime(const Duration(hours: 0, minutes: 13, seconds: 22)); var captionPart = - caption.getPartByTime(const Duration(milliseconds: 650)); + caption.getPartByTime(const Duration(milliseconds: 200)); expect(caption, isNotNull); expect(captionPart, isNotNull); - expect(caption.text, 'know I worked really hard on not doing'); - expect(captionPart.text, ' hard'); + expect(caption.text, 'how about this black there are some'); + expect(captionPart.text, ' about'); }); }); } From c962c1999eabd559f2286bac75577cc1aaef6d24 Mon Sep 17 00:00:00 2001 From: Mattia Date: Fri, 11 Sep 2020 18:24:19 +0200 Subject: [PATCH 2/8] Fix SearchVideo See https://github.com/Tyrrrz/YoutubeExplode/issues/438 --- CHANGELOG.md | 3 +++ .../reverse_engineering/responses/playlist_response.dart | 7 ++++++- pubspec.yaml | 2 +- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f0d10fa..655121c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,9 @@ - Only throw custom exceptions from the library. - `getUploadsFromPage` no longer throws. +## 1.5.1 +- Fix Video Search: https://github.com/Tyrrrz/YoutubeExplode/issues/438 + ## 1.5.0 - BREAKING CHANGE: Renamed `Container` class to `StreamContainer` to avoid conflicting with Flutter `Container`. See #66 diff --git a/lib/src/reverse_engineering/responses/playlist_response.dart b/lib/src/reverse_engineering/responses/playlist_response.dart index 7a41ac6..f205e6f 100644 --- a/lib/src/reverse_engineering/responses/playlist_response.dart +++ b/lib/src/reverse_engineering/responses/playlist_response.dart @@ -67,7 +67,12 @@ class PlaylistResponse { var url = 'https://youtube.com/search_ajax?style=json&search_query=' '${Uri.encodeQueryComponent(query)}&page=$page&hl=en'; return retry(() async { - var raw = await httpClient.getString(url, validate: false); + var raw = await httpClient.getString(url, + validate: false, + headers: const { + 'x-youtube-client-name': '56', + 'x-youtube-client-version': '20200911' + }); return PlaylistResponse.parse(raw); }); } diff --git a/pubspec.yaml b/pubspec.yaml index 7b66e9c..6fcb695 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -1,6 +1,6 @@ name: youtube_explode_dart description: A port in dart of the youtube explode library. Supports several API functions without the need of Youtube API Key. -version: 1.5.0 +version: 1.5.1 homepage: https://github.com/Hexer10/youtube_explode_dart environment: From 2981e041c93536067766aff1294a44d01ce08e1c Mon Sep 17 00:00:00 2001 From: Mattia Date: Fri, 11 Sep 2020 20:13:30 +0200 Subject: [PATCH 3/8] Create FUNDING.yml --- .github/FUNDING.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 .github/FUNDING.yml diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 0000000..315c1b9 --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1,12 @@ +# These are supported funding model platforms + +github: # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2] +patreon: # Replace with a single Patreon username +open_collective: # Replace with a single Open Collective username +ko_fi: hexah +tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel +community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry +liberapay: # Replace with a single Liberapay username +issuehunt: # Replace with a single IssueHunt username +otechie: # Replace with a single Otechie username +custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2'] From 5f5da821a9f139eba572d00cc762db65423d7a85 Mon Sep 17 00:00:00 2001 From: "Michael J. Miller" Date: Fri, 18 Sep 2020 02:12:41 -0600 Subject: [PATCH 4/8] channel description --- lib/src/channels/channel.dart | 5 +- lib/src/channels/channel_about.dart | 13 ++ lib/src/channels/channel_client.dart | 10 +- .../responses/channel_about_page.dart | 113 ++++++++++++++++++ 4 files changed, 138 insertions(+), 3 deletions(-) create mode 100644 lib/src/channels/channel_about.dart create mode 100644 lib/src/reverse_engineering/responses/channel_about_page.dart diff --git a/lib/src/channels/channel.dart b/lib/src/channels/channel.dart index 8c47ba1..dbaaf4c 100644 --- a/lib/src/channels/channel.dart +++ b/lib/src/channels/channel.dart @@ -13,11 +13,14 @@ class Channel with EquatableMixin { /// Channel title. final String title; + /// Channel description + final String description; + /// URL of the channel's logo image. final String logoUrl; /// Initializes an instance of [Channel] - Channel(this.id, this.title, this.logoUrl); + Channel(this.id, this.title, this.description, this.logoUrl); @override String toString() => 'Channel ($title)'; diff --git a/lib/src/channels/channel_about.dart b/lib/src/channels/channel_about.dart new file mode 100644 index 0000000..f27c6f6 --- /dev/null +++ b/lib/src/channels/channel_about.dart @@ -0,0 +1,13 @@ +import 'package:equatable/equatable.dart'; + +/// YouTube channel about metadata. +class ChannelAbout with EquatableMixin { + /// Channel description. + final String description; + + /// Initializes an instance of [ChannelAbout] + ChannelAbout(this.description); + + @override + List get props => [description]; +} diff --git a/lib/src/channels/channel_client.dart b/lib/src/channels/channel_client.dart index 8ad79eb..b47d0f0 100644 --- a/lib/src/channels/channel_client.dart +++ b/lib/src/channels/channel_client.dart @@ -1,5 +1,6 @@ import '../extensions/helpers_extension.dart'; import '../playlists/playlists.dart'; +import '../reverse_engineering/responses/channel_about_page.dart'; import '../reverse_engineering/responses/channel_upload_page.dart'; import '../reverse_engineering/responses/responses.dart'; import '../reverse_engineering/youtube_http_client.dart'; @@ -24,8 +25,10 @@ class ChannelClient { Future get(dynamic id) async { id = ChannelId.fromString(id); var channelPage = await ChannelPage.get(_httpClient, id.value); + var channelAboutPage = await ChannelAboutPage.get(_httpClient, id.value); - return Channel(id, channelPage.channelTitle, channelPage.channelLogoUrl); + return Channel(id, channelPage.channelTitle, channelAboutPage.description, + channelPage.channelLogoUrl); } /// Gets the metadata associated with the channel of the specified user. @@ -36,8 +39,11 @@ class ChannelClient { var channelPage = await ChannelPage.getByUsername(_httpClient, username.value); + var channelAboutPage = + await ChannelAboutPage.getByUsername(_httpClient, username.value); + return Channel(ChannelId(channelPage.channelId), channelPage.channelTitle, - channelPage.channelLogoUrl); + channelAboutPage.description, channelPage.channelLogoUrl); } /// Gets the metadata associated with the channel diff --git a/lib/src/reverse_engineering/responses/channel_about_page.dart b/lib/src/reverse_engineering/responses/channel_about_page.dart new file mode 100644 index 0000000..9278d9a --- /dev/null +++ b/lib/src/reverse_engineering/responses/channel_about_page.dart @@ -0,0 +1,113 @@ +import 'dart:convert'; + +import 'package:html/dom.dart'; +import 'package:html/parser.dart' as parser; + +import '../../exceptions/exceptions.dart'; +import '../../extensions/helpers_extension.dart'; +import '../../retry.dart'; +import '../youtube_http_client.dart'; + +/// +class ChannelAboutPage { + final Document _root; + + _InitialData _initialData; + + /// + _InitialData get initialData => + _initialData ??= _InitialData(json.decode(_matchJson(_extractJson( + _root + .querySelectorAll('script') + .map((e) => e.text) + .toList() + .firstWhere((e) => e.contains('window["ytInitialData"] =')), + 'window["ytInitialData"] =')))); + + /// + bool get isOk => initialData != null; + + /// + String get description => initialData.description; + + String _extractJson(String html, String separator) { + return _matchJson( + html.substring(html.indexOf(separator) + separator.length)); + } + + String _matchJson(String str) { + var bracketCount = 0; + int lastI; + for (var i = 0; i < str.length; i++) { + lastI = i; + if (str[i] == '{') { + bracketCount++; + } else if (str[i] == '}') { + bracketCount--; + } else if (str[i] == ';') { + if (bracketCount == 0) { + return str.substring(0, i); + } + } + } + return str.substring(0, lastI + 1); + } + + /// + ChannelAboutPage(this._root); + + /// + ChannelAboutPage.parse(String raw) : _root = parser.parse(raw); + + /// + static Future get(YoutubeHttpClient httpClient, String id) { + var url = 'https://www.youtube.com/channel/$id/about?hl=en'; + + return retry(() async { + var raw = await httpClient.getString(url); + var result = ChannelAboutPage.parse(raw); + + if (!result.isOk) { + throw TransientFailureException('Channel about page is broken'); + } + return result; + }); + } + + /// + static Future getByUsername( + YoutubeHttpClient httpClient, String username) { + var url = 'https://www.youtube.com/user/$username/about?hl=en'; + + return retry(() async { + var raw = await httpClient.getString(url); + var result = ChannelAboutPage.parse(raw); + + if (!result.isOk) { + throw TransientFailureException('Channel about page is broken'); + } + return result; + }); + } +} + +class _InitialData { + // Json parsed map + final Map root; + + _InitialData(this.root); + + /* Cache results */ + + String _description; + + Map getDescriptionContext(Map root) { + if (root['metadata'] != null) { + return root['metadata']['channelMetadataRenderer']; + } + return null; + } + + String get description => _description ??= + getDescriptionContext(root)?.getValue('description') ?? ''; +} From 08c51163baf472e9c2a98780ba9f453c86d35915 Mon Sep 17 00:00:00 2001 From: "Michael J. Miller" Date: Fri, 18 Sep 2020 02:13:56 -0600 Subject: [PATCH 5/8] cleanup --- lib/src/channels/channel_about.dart | 13 ------------- 1 file changed, 13 deletions(-) delete mode 100644 lib/src/channels/channel_about.dart diff --git a/lib/src/channels/channel_about.dart b/lib/src/channels/channel_about.dart deleted file mode 100644 index f27c6f6..0000000 --- a/lib/src/channels/channel_about.dart +++ /dev/null @@ -1,13 +0,0 @@ -import 'package:equatable/equatable.dart'; - -/// YouTube channel about metadata. -class ChannelAbout with EquatableMixin { - /// Channel description. - final String description; - - /// Initializes an instance of [ChannelAbout] - ChannelAbout(this.description); - - @override - List get props => [description]; -} From 00bada77c6fd237e3640eff987521f8fc2b63972 Mon Sep 17 00:00:00 2001 From: Mattia Date: Thu, 1 Oct 2020 16:58:11 +0200 Subject: [PATCH 6/8] Revert "Channel description" --- lib/src/channels/channel.dart | 5 +- lib/src/channels/channel_client.dart | 10 +- .../responses/channel_about_page.dart | 113 ------------------ 3 files changed, 3 insertions(+), 125 deletions(-) delete mode 100644 lib/src/reverse_engineering/responses/channel_about_page.dart diff --git a/lib/src/channels/channel.dart b/lib/src/channels/channel.dart index dbaaf4c..8c47ba1 100644 --- a/lib/src/channels/channel.dart +++ b/lib/src/channels/channel.dart @@ -13,14 +13,11 @@ class Channel with EquatableMixin { /// Channel title. final String title; - /// Channel description - final String description; - /// URL of the channel's logo image. final String logoUrl; /// Initializes an instance of [Channel] - Channel(this.id, this.title, this.description, this.logoUrl); + Channel(this.id, this.title, this.logoUrl); @override String toString() => 'Channel ($title)'; diff --git a/lib/src/channels/channel_client.dart b/lib/src/channels/channel_client.dart index b47d0f0..8ad79eb 100644 --- a/lib/src/channels/channel_client.dart +++ b/lib/src/channels/channel_client.dart @@ -1,6 +1,5 @@ import '../extensions/helpers_extension.dart'; import '../playlists/playlists.dart'; -import '../reverse_engineering/responses/channel_about_page.dart'; import '../reverse_engineering/responses/channel_upload_page.dart'; import '../reverse_engineering/responses/responses.dart'; import '../reverse_engineering/youtube_http_client.dart'; @@ -25,10 +24,8 @@ class ChannelClient { Future get(dynamic id) async { id = ChannelId.fromString(id); var channelPage = await ChannelPage.get(_httpClient, id.value); - var channelAboutPage = await ChannelAboutPage.get(_httpClient, id.value); - return Channel(id, channelPage.channelTitle, channelAboutPage.description, - channelPage.channelLogoUrl); + return Channel(id, channelPage.channelTitle, channelPage.channelLogoUrl); } /// Gets the metadata associated with the channel of the specified user. @@ -39,11 +36,8 @@ class ChannelClient { var channelPage = await ChannelPage.getByUsername(_httpClient, username.value); - var channelAboutPage = - await ChannelAboutPage.getByUsername(_httpClient, username.value); - return Channel(ChannelId(channelPage.channelId), channelPage.channelTitle, - channelAboutPage.description, channelPage.channelLogoUrl); + channelPage.channelLogoUrl); } /// Gets the metadata associated with the channel diff --git a/lib/src/reverse_engineering/responses/channel_about_page.dart b/lib/src/reverse_engineering/responses/channel_about_page.dart deleted file mode 100644 index 9278d9a..0000000 --- a/lib/src/reverse_engineering/responses/channel_about_page.dart +++ /dev/null @@ -1,113 +0,0 @@ -import 'dart:convert'; - -import 'package:html/dom.dart'; -import 'package:html/parser.dart' as parser; - -import '../../exceptions/exceptions.dart'; -import '../../extensions/helpers_extension.dart'; -import '../../retry.dart'; -import '../youtube_http_client.dart'; - -/// -class ChannelAboutPage { - final Document _root; - - _InitialData _initialData; - - /// - _InitialData get initialData => - _initialData ??= _InitialData(json.decode(_matchJson(_extractJson( - _root - .querySelectorAll('script') - .map((e) => e.text) - .toList() - .firstWhere((e) => e.contains('window["ytInitialData"] =')), - 'window["ytInitialData"] =')))); - - /// - bool get isOk => initialData != null; - - /// - String get description => initialData.description; - - String _extractJson(String html, String separator) { - return _matchJson( - html.substring(html.indexOf(separator) + separator.length)); - } - - String _matchJson(String str) { - var bracketCount = 0; - int lastI; - for (var i = 0; i < str.length; i++) { - lastI = i; - if (str[i] == '{') { - bracketCount++; - } else if (str[i] == '}') { - bracketCount--; - } else if (str[i] == ';') { - if (bracketCount == 0) { - return str.substring(0, i); - } - } - } - return str.substring(0, lastI + 1); - } - - /// - ChannelAboutPage(this._root); - - /// - ChannelAboutPage.parse(String raw) : _root = parser.parse(raw); - - /// - static Future get(YoutubeHttpClient httpClient, String id) { - var url = 'https://www.youtube.com/channel/$id/about?hl=en'; - - return retry(() async { - var raw = await httpClient.getString(url); - var result = ChannelAboutPage.parse(raw); - - if (!result.isOk) { - throw TransientFailureException('Channel about page is broken'); - } - return result; - }); - } - - /// - static Future getByUsername( - YoutubeHttpClient httpClient, String username) { - var url = 'https://www.youtube.com/user/$username/about?hl=en'; - - return retry(() async { - var raw = await httpClient.getString(url); - var result = ChannelAboutPage.parse(raw); - - if (!result.isOk) { - throw TransientFailureException('Channel about page is broken'); - } - return result; - }); - } -} - -class _InitialData { - // Json parsed map - final Map root; - - _InitialData(this.root); - - /* Cache results */ - - String _description; - - Map getDescriptionContext(Map root) { - if (root['metadata'] != null) { - return root['metadata']['channelMetadataRenderer']; - } - return null; - } - - String get description => _description ??= - getDescriptionContext(root)?.getValue('description') ?? ''; -} From 0186ad3d7a3ee94df0214cafeca323bd9b67d31c Mon Sep 17 00:00:00 2001 From: Mattia Date: Thu, 1 Oct 2020 18:04:56 +0200 Subject: [PATCH 7/8] New version 1.15.2 Fix extraction for same videos. This closes #76 --- CHANGELOG.md | 3 +++ lib/src/reverse_engineering/responses/embed_page.dart | 2 +- lib/src/reverse_engineering/responses/player_source.dart | 2 +- lib/src/reverse_engineering/responses/watch_page.dart | 8 +++++--- lib/src/videos/streams/streams_client.dart | 1 + pubspec.yaml | 2 +- test/streams_test.dart | 2 +- test/video_test.dart | 2 +- 8 files changed, 14 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 655121c..465bc74 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,9 @@ - Only throw custom exceptions from the library. - `getUploadsFromPage` no longer throws. +## 1.15.2 +- Fix extraction for same videos (#76) + ## 1.5.1 - Fix Video Search: https://github.com/Tyrrrz/YoutubeExplode/issues/438 diff --git a/lib/src/reverse_engineering/responses/embed_page.dart b/lib/src/reverse_engineering/responses/embed_page.dart index f404247..e4e79d7 100644 --- a/lib/src/reverse_engineering/responses/embed_page.dart +++ b/lib/src/reverse_engineering/responses/embed_page.dart @@ -10,7 +10,7 @@ import '../youtube_http_client.dart'; /// class EmbedPage { static final _playerConfigExp = - RegExp(r"yt\.setConfig\({'PLAYER_CONFIG':(.*)}\);"); + RegExp(r"'PLAYER_CONFIG':\s*(\{.*\})\}"); final Document _root; _PlayerConfig _playerConfig; diff --git a/lib/src/reverse_engineering/responses/player_source.dart b/lib/src/reverse_engineering/responses/player_source.dart index 68ee871..81df2f2 100644 --- a/lib/src/reverse_engineering/responses/player_source.dart +++ b/lib/src/reverse_engineering/responses/player_source.dart @@ -30,7 +30,7 @@ class PlayerSource { var val = RegExp(r'(?<=invalid namespace.*?;[\w\s]+=)\d+') .stringMatch(_root) ?.nullIfWhitespace ?? - RegExp(r'(?<=this\.signatureTimestamp=)\d+"') + RegExp(r'(?<=this\.signatureTimestamp=)\d+') .stringMatch(_root) ?.nullIfWhitespace; if (val == null) { diff --git a/lib/src/reverse_engineering/responses/watch_page.dart b/lib/src/reverse_engineering/responses/watch_page.dart index ea8f56a..7d53aa7 100644 --- a/lib/src/reverse_engineering/responses/watch_page.dart +++ b/lib/src/reverse_engineering/responses/watch_page.dart @@ -86,11 +86,13 @@ class WatchPage { ?.nullIfWhitespace ?? '0'); + static final _playerConfigExp = RegExp(r'ytplayer\.config\s*=\s*(\{.*\}\});'); + /// _PlayerConfig get playerConfig => - _playerConfig ??= _PlayerConfig(json.decode(_matchJson(_extractJson( - _root.getElementsByTagName('html').first.text, - 'ytplayer.config = ')))); + _playerConfig ??= _PlayerConfig(json.decode(_playerConfigExp + .firstMatch(_root.getElementsByTagName('html').first.text) + ?.group(1))); String _extractJson(String html, String separator) { return _matchJson( diff --git a/lib/src/videos/streams/streams_client.dart b/lib/src/videos/streams/streams_client.dart index aae8935..d55df7c 100644 --- a/lib/src/videos/streams/streams_client.dart +++ b/lib/src/videos/streams/streams_client.dart @@ -218,6 +218,7 @@ class StreamsClient { // We can try to extract the manifest from two sources: // get_video_info and the video watch page. // In some cases one works, in some cases another does. + try { var context = await _getStreamContextFromVideoInfo(videoId); return _getManifest(context); diff --git a/pubspec.yaml b/pubspec.yaml index 6fcb695..64e3610 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -1,6 +1,6 @@ name: youtube_explode_dart description: A port in dart of the youtube explode library. Supports several API functions without the need of Youtube API Key. -version: 1.5.1 +version: 1.5.2 homepage: https://github.com/Hexer10/youtube_explode_dart environment: diff --git a/test/streams_test.dart b/test/streams_test.dart index f3f4944..0dda6b6 100644 --- a/test/streams_test.dart +++ b/test/streams_test.dart @@ -14,7 +14,7 @@ void main() { var data = { '9bZkp7q19f0', -// 'SkRSXFQerZs', age restricted videos are not supported anymore. + 'SkRSXFQerZs', 'hySoCSoH-g8', '_kmeFXjjGfk', 'MeJVWBSsPAY', diff --git a/test/video_test.dart b/test/video_test.dart index f238677..46e3786 100644 --- a/test/video_test.dart +++ b/test/video_test.dart @@ -25,7 +25,7 @@ void main() { expect(video.uploadDate.millisecondsSinceEpoch, inInclusiveRange(rangeMs - 86400000, rangeMs + 86400000)); expect(video.description, contains('246pp')); - expect(video.duration, const Duration(minutes: 1, seconds: 49)); + expect(video.duration, const Duration(minutes: 1, seconds: 48)); expect(video.thumbnails.lowResUrl, isNotEmpty); expect(video.thumbnails.mediumResUrl, isNotEmpty); expect(video.thumbnails.highResUrl, isNotEmpty); From 25d5901e91e937e529f989d5c7443ffc52c05ff1 Mon Sep 17 00:00:00 2001 From: Mattia Date: Thu, 1 Oct 2020 18:11:42 +0200 Subject: [PATCH 8/8] Changelog typo --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 465bc74..d33f8d5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,7 @@ - Only throw custom exceptions from the library. - `getUploadsFromPage` no longer throws. -## 1.15.2 +## 1.5.2 - Fix extraction for same videos (#76) ## 1.5.1