diff --git a/CHANGELOG.md b/CHANGELOG.md index ca35d13..7573d0f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +## 1.7.4 +- Fix slow download ( #92 ) +- Fix stream retrieving on some videos ( #90 ) +- Updates tests + ## 1.7.3 - Fix exceptions on some videos. - Closes #89, #88 diff --git a/lib/src/reverse_engineering/responses/channel_about_page.dart b/lib/src/reverse_engineering/responses/channel_about_page.dart index f431b46..f02c362 100644 --- a/lib/src/reverse_engineering/responses/channel_about_page.dart +++ b/lib/src/reverse_engineering/responses/channel_about_page.dart @@ -15,14 +15,35 @@ class ChannelAboutPage { _InitialData _initialData; /// - _InitialData get initialData => - _initialData ??= _InitialData(ChannelAboutPageId.fromRawJson(_extractJson( - _root - .querySelectorAll('script') - .map((e) => e.text) - .toList() - .firstWhere((e) => e.contains('window["ytInitialData"] =')), - 'window["ytInitialData"] ='))); + _InitialData get initialData { + if (_initialData != null) { + return _initialData; + } + + final scriptText = _root + .querySelectorAll('script') + .map((e) => e.text) + .toList(growable: false); + + var initialDataText = scriptText.firstWhere( + (e) => e.contains('window["ytInitialData"] ='), + orElse: () => null); + if (initialDataText != null) { + return _initialData = _InitialData(ChannelAboutPageId.fromRawJson( + _extractJson(initialDataText, 'window["ytInitialData"] ='))); + } + + initialDataText = scriptText.firstWhere( + (e) => e.contains('var ytInitialData = '), + orElse: () => null); + if (initialDataText != null) { + return _initialData = _InitialData(ChannelAboutPageId.fromRawJson( + _extractJson(initialDataText, 'var ytInitialData = '))); + } + + throw TransientFailureException( + 'Failed to retrieve initial data from the channel about page, please report this to the project GitHub page.'); // ignore: lines_longer_than_80_chars + } /// bool get isOk => initialData != null; diff --git a/lib/src/reverse_engineering/responses/channel_upload_page.dart b/lib/src/reverse_engineering/responses/channel_upload_page.dart index 9af9351..694fff8 100644 --- a/lib/src/reverse_engineering/responses/channel_upload_page.dart +++ b/lib/src/reverse_engineering/responses/channel_upload_page.dart @@ -19,14 +19,35 @@ class ChannelUploadPage { _InitialData _initialData; /// - _InitialData get initialData => _initialData ??= _InitialData( - ChannelUploadPageId.fromJson(json.decode(_extractJson( - _root - .querySelectorAll('script') - .map((e) => e.text) - .toList() - .firstWhere((e) => e.contains('window["ytInitialData"] =')), - 'window["ytInitialData"] =')))); + _InitialData get initialData { + if (_initialData != null) { + return _initialData; + } + + final scriptText = _root + .querySelectorAll('script') + .map((e) => e.text) + .toList(growable: false); + + var initialDataText = scriptText.firstWhere( + (e) => e.contains('window["ytInitialData"] ='), + orElse: () => null); + if (initialDataText != null) { + return _initialData = _InitialData(ChannelUploadPageId.fromRawJson( + _extractJson(initialDataText, 'window["ytInitialData"] ='))); + } + + initialDataText = scriptText.firstWhere( + (e) => e.contains('var ytInitialData = '), + orElse: () => null); + if (initialDataText != null) { + return _initialData = _InitialData(ChannelUploadPageId.fromRawJson( + _extractJson(initialDataText, 'var ytInitialData = '))); + } + + throw TransientFailureException( + 'Failed to retrieve initial data from the channel upload page, please report this to the project GitHub page.'); // ignore: lines_longer_than_80_chars + } String _extractJson(String html, String separator) { return _matchJson( diff --git a/lib/src/reverse_engineering/responses/search_page.dart b/lib/src/reverse_engineering/responses/search_page.dart index 3915542..b019e19 100644 --- a/lib/src/reverse_engineering/responses/search_page.dart +++ b/lib/src/reverse_engineering/responses/search_page.dart @@ -38,17 +38,30 @@ class SearchPage { if (_initialData != null) { return _initialData; } - var scriptTag = _extractJson( - _root.querySelectorAll('script').map((e) => e.text).toList().firstWhere( - (e) => e.contains('window["ytInitialData"] ='), - orElse: () => null), - 'window["ytInitialData"] ='); - scriptTag ??= _extractJson( - _root.querySelectorAll('script').map((e) => e.text).toList().firstWhere( - (e) => e.contains('var ytInitialData ='), - orElse: () => '{}'), - 'var ytInitialData ='); - return _initialData ??= _InitialData(SearchPageId.fromRawJson(scriptTag)); + + final scriptText = _root + .querySelectorAll('script') + .map((e) => e.text) + .toList(growable: false); + + var initialDataText = scriptText.firstWhere( + (e) => e.contains('window["ytInitialData"] ='), + orElse: () => null); + if (initialDataText != null) { + return _initialData = _InitialData(SearchPageId.fromRawJson( + _extractJson(initialDataText, 'window["ytInitialData"] ='))); + } + + initialDataText = scriptText.firstWhere( + (e) => e.contains('var ytInitialData = '), + orElse: () => null); + if (initialDataText != null) { + return _initialData = _InitialData(SearchPageId.fromRawJson( + _extractJson(initialDataText, 'var ytInitialData = '))); + } + + throw TransientFailureException( + 'Failed to retrieve initial data from the search page, please report this to the project GitHub page.'); // ignore: lines_longer_than_80_chars } String _extractJson(String html, String separator) { diff --git a/lib/src/reverse_engineering/responses/watch_page.dart b/lib/src/reverse_engineering/responses/watch_page.dart index 5e02172..a69196d 100644 --- a/lib/src/reverse_engineering/responses/watch_page.dart +++ b/lib/src/reverse_engineering/responses/watch_page.dart @@ -51,14 +51,35 @@ class WatchPage { } /// - _InitialData get initialData => - _initialData ??= _InitialData(WatchPageId.fromRawJson(_extractJson( - _root - .querySelectorAll('script') - .map((e) => e.text) - .toList() - .firstWhere((e) => e.contains('window["ytInitialData"] =')), - 'window["ytInitialData"] ='))); + _InitialData get initialData { + if (_initialData != null) { + return _initialData; + } + + final scriptText = _root + .querySelectorAll('script') + .map((e) => e.text) + .toList(growable: false); + + var initialDataText = scriptText.firstWhere( + (e) => e.contains('window["ytInitialData"] ='), + orElse: () => null); + if (initialDataText != null) { + return _initialData = _InitialData(WatchPageId.fromRawJson( + _extractJson(initialDataText, 'window["ytInitialData"] ='))); + } + + initialDataText = scriptText.firstWhere( + (e) => e.contains('var ytInitialData = '), + orElse: () => null); + if (initialDataText != null) { + return _initialData = _InitialData(WatchPageId.fromRawJson( + _extractJson(initialDataText, 'var ytInitialData = '))); + } + + throw TransientFailureException( + 'Failed to retrieve initial data from the watch page, please report this to the project GitHub page.'); // ignore: lines_longer_than_80_chars + } /// String get xsfrToken => _xsfrToken ??= _xsfrTokenExp @@ -110,10 +131,10 @@ class WatchPage { ?.group(1) ?.extractJson())); + /// PlayerResponse get playerResponse => PlayerResponse.parse(_root .querySelectorAll('script') .map((e) => e.text) - .map((e) => null) .map((e) => _playerResponseExp.firstMatch(e)?.group(1)) .firstWhere((e) => !e.isNullOrWhiteSpace) .extractJson()); diff --git a/lib/src/reverse_engineering/youtube_http_client.dart b/lib/src/reverse_engineering/youtube_http_client.dart index a90eebe..b73e836 100644 --- a/lib/src/reverse_engineering/youtube_http_client.dart +++ b/lib/src/reverse_engineering/youtube_http_client.dart @@ -84,25 +84,41 @@ class YoutubeHttpClient extends http.BaseClient { return response.body; } - /// Stream> getStream(StreamInfo streamInfo, {Map headers, bool validate = true, int start = 0, int errorCount = 0}) async* { var url = streamInfo.url; - - var query = Map.from(url.queryParameters); - query['ratebypass'] = 'yes'; - url = url.replace(queryParameters: query); - - var request = http.Request('get', url); - request.headers.addAll(_defaultHeaders); - var response = await request.send(); - if (validate) { - _validateResponse(response, response.statusCode); + var bytesCount = start; + for (var i = start; i < streamInfo.size.totalBytes; i += 9898989) { + try { + final request = http.Request('get', url); + request.headers['range'] = 'bytes=$i-${i + 9898989 - 1}'; + final response = await send(request); + if (validate) { + _validateResponse(response, response.statusCode); + } + final stream = StreamController>(); + response.stream.listen((data) { + bytesCount += data.length; + stream.add(data); + }, onError: (_) => null, onDone: stream.close, cancelOnError: false); + errorCount = 0; + yield* stream.stream; + } on Exception { + if (errorCount == 5) { + rethrow; + } + await Future.delayed(const Duration(milliseconds: 500)); + yield* getStream(streamInfo, + headers: headers, + validate: validate, + start: bytesCount, + errorCount: errorCount + 1); + break; + } } - yield* response.stream; } /// diff --git a/lib/src/videos/streams/streams_client.dart b/lib/src/videos/streams/streams_client.dart index 68a8adb..6aee3aa 100644 --- a/lib/src/videos/streams/streams_client.dart +++ b/lib/src/videos/streams/streams_client.dart @@ -78,7 +78,13 @@ class StreamsClient { Future _getStreamContextFromWatchPage(VideoId videoId) async { var watchPage = await WatchPage.get(_httpClient, videoId.toString()); - var playerConfig = watchPage.playerConfig; + + dynamic /* _PlayerConfig */ playerConfig; + try { + playerConfig = watchPage.playerConfig; + } on FormatException { + playerConfig = null; + } var playerResponse = playerConfig?.playerResponse ?? watchPage.playerResponse; if (playerResponse == null) { @@ -86,12 +92,13 @@ class StreamsClient { } var previewVideoId = playerResponse.previewVideoId; - if (!previewVideoId.isNullOrWhiteSpace) { + if (!((previewVideoId as String)?.isNullOrWhiteSpace ?? true)) { throw VideoRequiresPurchaseException.preview( videoId, VideoId(previewVideoId)); } - var playerSourceUrl = watchPage.sourceUrl ?? playerConfig?.sourceUrl; + var playerSourceUrl = + watchPage.sourceUrl ?? playerConfig?.sourceUrl as String; var playerSource = !playerSourceUrl.isNullOrWhiteSpace ? await PlayerSource.get(_httpClient, playerSourceUrl) : null; @@ -112,7 +119,7 @@ class StreamsClient { ]; var dashManifestUrl = playerResponse.dashManifestUrl; - if (!dashManifestUrl.isNullOrWhiteSpace) { + if (!(dashManifestUrl?.isNullOrWhiteSpace ?? true)) { var dashManifest = await _getDashManifest(Uri.parse(dashManifestUrl), cipherOperations); streamInfoProviders.addAll(dashManifest.streams); diff --git a/pubspec.yaml b/pubspec.yaml index 9b62067..d6a83a5 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -1,6 +1,6 @@ name: youtube_explode_dart description: A port in dart of the youtube explode library. Supports several API functions without the need of Youtube API Key. -version: 1.7.3 +version: 1.7.4 homepage: https://github.com/Hexer10/youtube_explode_dart environment: diff --git a/test/streams_test.dart b/test/streams_test.dart index ab6240d..4f85318 100644 --- a/test/streams_test.dart +++ b/test/streams_test.dart @@ -11,12 +11,18 @@ void main() { yt.close(); }); - group('Get streams of any video', () { + group('Get streams manifest of any video', () { for (var val in { - VideoId('5VGm0dczmHc'), // rating is not allowed + VideoId('9bZkp7q19f0'), // very popular + VideoId('SkRSXFQerZs'), // age restricted (embed allowed) + VideoId('hySoCSoH-g8'), // age restricted (embed not allowed) + VideoId('_kmeFXjjGfk'), // embed not allowed (type 1) + VideoId('MeJVWBSsPAY'), // embed not allowed (type 2) + VideoId('5VGm0dczmHc'), // rating not allowed VideoId('ZGdLIwrGHG8'), // unlisted - VideoId('rsAAeyAr-9Y'), - VideoId('AI7ULzgf8RU') + VideoId('rsAAeyAr-9Y'), // recording of a live stream + VideoId('AI7ULzgf8RU'), // has DASH manifest + VideoId('-xNN-bJQ4vI'), // 360° video }) { test('VideoId - ${val.value}', () async { var manifest = await yt.videos.streamsClient.getManifest(val); @@ -39,12 +45,18 @@ void main() { } }); - group('Get stream of any playable video', () { + group('Get specific stream of any playable video', () { for (var val in { - VideoId('5VGm0dczmHc'), // rating is not allowed + VideoId('9bZkp7q19f0'), // very popular + VideoId('SkRSXFQerZs'), // age restricted (embed allowed) + VideoId('hySoCSoH-g8'), // age restricted (embed not allowed) + VideoId('_kmeFXjjGfk'), // embed not allowed (type 1) + VideoId('MeJVWBSsPAY'), // embed not allowed (type 2) + VideoId('5VGm0dczmHc'), // rating not allowed VideoId('ZGdLIwrGHG8'), // unlisted - VideoId('rsAAeyAr-9Y'), - VideoId('AI7ULzgf8RU') + VideoId('rsAAeyAr-9Y'), // recording of a live stream + VideoId('AI7ULzgf8RU'), // has DASH manifest + VideoId('-xNN-bJQ4vI'), // 360° video }) { test('VideoId - ${val.value}', () async { var manifest = await yt.videos.streamsClient.getManifest(val); diff --git a/test/video_test.dart b/test/video_test.dart index 5b71ffb..bc7652c 100644 --- a/test/video_test.dart +++ b/test/video_test.dart @@ -32,7 +32,12 @@ void main() { expect(video.thumbnails.highResUrl, isNotEmpty); expect(video.thumbnails.standardResUrl, isNotEmpty); expect(video.thumbnails.maxResUrl, isNotEmpty); - expect(video.keywords, containsAll(['osu', 'mouse' /*, 'rhythm game'*/])); + expect( + video.keywords, + containsAll([ + 'osu', + 'mouse' /*, 'rhythm game'*/ + ])); expect(video.engagement.viewCount, greaterThanOrEqualTo(134)); expect(video.engagement.likeCount, greaterThanOrEqualTo(5)); expect(video.engagement.dislikeCount, greaterThanOrEqualTo(0));