From 9c8e9630abad932dbe0d1c0ed0e030db2481ded8 Mon Sep 17 00:00:00 2001 From: Mattia Date: Tue, 27 Oct 2020 14:44:11 +0100 Subject: [PATCH] Fix #80 Version 1.6.2 --- CHANGELOG.md | 3 +++ lib/src/extensions/helpers_extension.dart | 26 +++++++++++++++++++ .../responses/embed_page.dart | 16 ++++++++++-- .../responses/player_source.dart | 2 +- .../responses/watch_page.dart | 17 +++++++++--- .../youtube_http_client.dart | 2 +- lib/src/videos/streams/streams_client.dart | 8 +++--- test/channel_about_test.dart | 4 +-- test/channel_test.dart | 4 +-- test/closed_caption_test.dart | 4 +-- test/comments_client_test.dart | 4 +-- test/playlist_test.dart | 4 +-- test/search_test.dart | 10 +++---- test/streams_test.dart | 6 ++--- test/video_test.dart | 4 +-- 15 files changed, 82 insertions(+), 32 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7f0c34e..0cd8dbb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,9 @@ - Only throw custom exceptions from the library. - `getUploadsFromPage` no longer throws. +## 1.6.2 +- Bug fixes: #80 + ## 1.6.1 - Add thumbnail to `SearchVideo` thanks to @shinyford ! diff --git a/lib/src/extensions/helpers_extension.dart b/lib/src/extensions/helpers_extension.dart index 120c9ad..90b3ea0 100644 --- a/lib/src/extensions/helpers_extension.dart +++ b/lib/src/extensions/helpers_extension.dart @@ -29,6 +29,32 @@ extension StringUtility on String { /// Strips out all non digit characters. String stripNonDigits() => replaceAll(_exp, ''); + + /// + String extractJson() { + var buffer = StringBuffer(); + var depth = 0; + + for (var i = 0; i < length; i++) { + var ch = this[i]; + var chPrv = i > 0 ? this[i - 1] : ''; + + buffer.write(ch); + + if (ch == '{' && chPrv != '\\') { + depth++; + } else if (ch == '}' && chPrv != '\\') { + depth--; + } + + if (depth == 0) { + break; + } + } + return buffer.toString(); + } + + } /// List decipher utility. diff --git a/lib/src/reverse_engineering/responses/embed_page.dart b/lib/src/reverse_engineering/responses/embed_page.dart index 8f94350..2a3d7a6 100644 --- a/lib/src/reverse_engineering/responses/embed_page.dart +++ b/lib/src/reverse_engineering/responses/embed_page.dart @@ -7,14 +7,25 @@ import '../../extensions/helpers_extension.dart'; import '../../retry.dart'; import '../youtube_http_client.dart'; + /// class EmbedPage { - static final _playerConfigExp = RegExp(r"'PLAYER_CONFIG':\s*(\{.*\})\}"); + static final _playerConfigExp = + RegExp('[\'"]PLAYER_CONFIG[\'"]\\s*:\\s*(\\{.*\\})'); final Document _root; _PlayerConfig _playerConfig; String __playerConfigJson; + /// + String get sourceUrl { + var url = _root.querySelector('*[name="player_ias/base"]').attributes['src']; + if (url == null) { + return null; + } + return 'https://youtube.com$url'; + } + /// _PlayerConfig get playerconfig { if (_playerConfig != null) { @@ -24,7 +35,8 @@ class EmbedPage { if (playerConfigJson == null) { return null; } - return _playerConfig = _PlayerConfig(json.decode(playerConfigJson)); + return _playerConfig = + _PlayerConfig(json.decode(playerConfigJson.extractJson())); } String get _playerConfigJson => __playerConfigJson ??= _root diff --git a/lib/src/reverse_engineering/responses/player_source.dart b/lib/src/reverse_engineering/responses/player_source.dart index 81df2f2..af37bc8 100644 --- a/lib/src/reverse_engineering/responses/player_source.dart +++ b/lib/src/reverse_engineering/responses/player_source.dart @@ -30,7 +30,7 @@ class PlayerSource { var val = RegExp(r'(?<=invalid namespace.*?;[\w\s]+=)\d+') .stringMatch(_root) ?.nullIfWhitespace ?? - RegExp(r'(?<=this\.signatureTimestamp=)\d+') + RegExp(r'(?<=signatureTimestamp[=\:])\d+') .stringMatch(_root) ?.nullIfWhitespace; if (val == null) { diff --git a/lib/src/reverse_engineering/responses/watch_page.dart b/lib/src/reverse_engineering/responses/watch_page.dart index f6f826f..d4d706a 100644 --- a/lib/src/reverse_engineering/responses/watch_page.dart +++ b/lib/src/reverse_engineering/responses/watch_page.dart @@ -1,5 +1,3 @@ -import 'dart:convert'; - import 'package:html/dom.dart'; import 'package:html/parser.dart' as parser; @@ -35,6 +33,16 @@ class WatchPage { String _xsfrToken; _PlayerConfig _playerConfig; + /// + String get sourceUrl { + var url = + _root.querySelector('*[name="player_ias/base"]').attributes['src']; + if (url == null) { + return null; + } + return 'https://youtube.com$url'; + } + /// _InitialData get initialData => _initialData ??= _InitialData(WatchPageId.fromRawJson(_extractJson( @@ -86,13 +94,14 @@ class WatchPage { ?.nullIfWhitespace ?? '0'); - static final _playerConfigExp = RegExp(r'ytplayer\.config\s*=\s*(\{.*\}\});'); + static final _playerConfigExp = RegExp(r'ytplayer\.config\s*=\s*(\{.*\})'); /// _PlayerConfig get playerConfig => _playerConfig ??= _PlayerConfig( PlayerConfigJson.fromRawJson(_playerConfigExp .firstMatch(_root.getElementsByTagName('html').first.text) - ?.group(1))); + ?.group(1) + ?.extractJson())); String _extractJson(String html, String separator) { return _matchJson( diff --git a/lib/src/reverse_engineering/youtube_http_client.dart b/lib/src/reverse_engineering/youtube_http_client.dart index 4dadd2f..677eab8 100644 --- a/lib/src/reverse_engineering/youtube_http_client.dart +++ b/lib/src/reverse_engineering/youtube_http_client.dart @@ -11,7 +11,7 @@ class YoutubeHttpClient extends http.BaseClient { final Map _defaultHeaders = const { 'user-agent': - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36', + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.111 Safari/537.36', 'accept-language': 'en-US,en;q=1.0', 'x-youtube-client-name': '1', 'x-youtube-client-version': '2.20200609.04.02', diff --git a/lib/src/videos/streams/streams_client.dart b/lib/src/videos/streams/streams_client.dart index 5f86033..0035cdd 100644 --- a/lib/src/videos/streams/streams_client.dart +++ b/lib/src/videos/streams/streams_client.dart @@ -39,8 +39,8 @@ class StreamsClient { throw VideoUnplayableException.unplayable(videoId); } - var playerSource = - await PlayerSource.get(_httpClient, playerConfig.sourceUrl); + var playerSource = await PlayerSource.get( + _httpClient, embedPage.sourceUrl ?? playerConfig.sourceUrl); var cipherOperations = playerSource.getCiperOperations(); var videoInfoResponse = await VideoInfoResponse.get( @@ -91,8 +91,8 @@ class StreamsClient { videoId, VideoId(previewVideoId)); } - var playerSource = - await PlayerSource.get(_httpClient, playerConfig.sourceUrl); + var playerSource = await PlayerSource.get( + _httpClient, watchPage.sourceUrl ?? playerConfig.sourceUrl); var cipherOperations = playerSource.getCiperOperations(); if (!playerResponse.isVideoPlayable) { diff --git a/test/channel_about_test.dart b/test/channel_about_test.dart index 7e315e3..ea12daf 100644 --- a/test/channel_about_test.dart +++ b/test/channel_about_test.dart @@ -3,11 +3,11 @@ import 'package:youtube_explode_dart/youtube_explode_dart.dart'; void main() { YoutubeExplode yt; - setUp(() { + setUpAll(() { yt = YoutubeExplode(); }); - tearDown(() { + tearDownAll(() { yt.close(); }); diff --git a/test/channel_test.dart b/test/channel_test.dart index 516a77b..4135888 100644 --- a/test/channel_test.dart +++ b/test/channel_test.dart @@ -3,11 +3,11 @@ import 'package:youtube_explode_dart/youtube_explode_dart.dart'; void main() { YoutubeExplode yt; - setUp(() { + setUpAll(() { yt = YoutubeExplode(); }); - tearDown(() { + tearDownAll(() { yt.close(); }); diff --git a/test/closed_caption_test.dart b/test/closed_caption_test.dart index d490664..7350954 100644 --- a/test/closed_caption_test.dart +++ b/test/closed_caption_test.dart @@ -3,11 +3,11 @@ import 'package:youtube_explode_dart/youtube_explode_dart.dart'; void main() { YoutubeExplode yt; - setUp(() { + setUpAll(() { yt = YoutubeExplode(); }); - tearDown(() { + tearDownAll(() { yt.close(); }); diff --git a/test/comments_client_test.dart b/test/comments_client_test.dart index 548aee3..1a115ca 100644 --- a/test/comments_client_test.dart +++ b/test/comments_client_test.dart @@ -3,11 +3,11 @@ import 'package:youtube_explode_dart/youtube_explode_dart.dart'; void main() { YoutubeExplode yt; - setUp(() { + setUpAll(() { yt = YoutubeExplode(); }); - tearDown(() { + tearDownAll(() { yt.close(); }); diff --git a/test/playlist_test.dart b/test/playlist_test.dart index c4f93aa..3da2380 100644 --- a/test/playlist_test.dart +++ b/test/playlist_test.dart @@ -3,11 +3,11 @@ import 'package:youtube_explode_dart/youtube_explode_dart.dart'; void main() { YoutubeExplode yt; - setUp(() { + setUpAll(() { yt = YoutubeExplode(); }); - tearDown(() { + tearDownAll(() { yt.close(); }); diff --git a/test/search_test.dart b/test/search_test.dart index 2070bc8..c3497bb 100644 --- a/test/search_test.dart +++ b/test/search_test.dart @@ -35,12 +35,12 @@ void main() { }); test('Search youtube videos have thumbnails', () async { - var searchQuery = await yt.search.queryFromPage('hello'); - expect(searchQuery.content.first, isA()); + var searchQuery = await yt.search.queryFromPage('hello'); + expect(searchQuery.content.first, isA()); - var video = searchQuery.content.first as SearchVideo; - expect(video.videoThumbnails, isNotEmpty); - }); + var video = searchQuery.content.first as SearchVideo; + expect(video.videoThumbnails, isNotEmpty); + }); test('Search youtube videos from search page (stream)', () async { var query = await yt.search.getVideosFromPage('hello').take(30).toList(); diff --git a/test/streams_test.dart b/test/streams_test.dart index 3e51020..ab6240d 100644 --- a/test/streams_test.dart +++ b/test/streams_test.dart @@ -3,11 +3,11 @@ import 'package:youtube_explode_dart/youtube_explode_dart.dart'; void main() { YoutubeExplode yt; - setUp(() { + setUpAll(() { yt = YoutubeExplode(); }); - tearDown(() { + tearDownAll(() { yt.close(); }); @@ -53,5 +53,5 @@ void main() { } }); } - }, skip: 'Occasionally may fail with certain videos'); + }); } diff --git a/test/video_test.dart b/test/video_test.dart index e26a9d0..e80207e 100644 --- a/test/video_test.dart +++ b/test/video_test.dart @@ -3,11 +3,11 @@ import 'package:youtube_explode_dart/youtube_explode_dart.dart'; void main() { YoutubeExplode yt; - setUp(() { + setUpAll(() { yt = YoutubeExplode(); }); - tearDown(() { + tearDownAll(() { yt.close(); });