From f75767ea2a353ed60ea88c9a8038b20197df9f21 Mon Sep 17 00:00:00 2001 From: vi-k Date: Fri, 3 Sep 2021 16:12:30 +1000 Subject: [PATCH 1/3] Fix the error of incomplete data loading on the Android emulator. --- lib/src/reverse_engineering/youtube_http_client.dart | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/src/reverse_engineering/youtube_http_client.dart b/lib/src/reverse_engineering/youtube_http_client.dart index 242baec..e63883f 100644 --- a/lib/src/reverse_engineering/youtube_http_client.dart +++ b/lib/src/reverse_engineering/youtube_http_client.dart @@ -117,21 +117,21 @@ class YoutubeHttpClient extends http.BaseClient { int errorCount = 0}) async* { var url = streamInfo.url; var bytesCount = start; - for (var i = start; i < streamInfo.size.totalBytes; i += 9898989) { + while (bytesCount != streamInfo.size.totalBytes) { try { final response = await retry(() { final request = http.Request('get', url); - request.headers['range'] = 'bytes=$i-${i + 9898989 - 1}'; + request.headers['range'] = 'bytes=$bytesCount-${bytesCount + 9898989 - 1}'; return send(request); }); if (validate) { _validateResponse(response, response.statusCode); } final stream = StreamController>(); - response.stream.listen((data) { - bytesCount += data.length; - stream.add(data); - }, onError: (_) => null, onDone: stream.close, cancelOnError: false); + response.stream.listen((data) { + bytesCount += data.length; + stream.add(data); + }, onError: (_) => null, onDone: stream.close, cancelOnError: false); errorCount = 0; yield* stream.stream; } on Exception { From 1df6a2e1845f40f271ca551ce9ef6993e39fe1b4 Mon Sep 17 00:00:00 2001 From: vi-k Date: Fri, 3 Sep 2021 16:26:51 +1000 Subject: [PATCH 2/3] Correct the code. --- lib/src/reverse_engineering/youtube_http_client.dart | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/src/reverse_engineering/youtube_http_client.dart b/lib/src/reverse_engineering/youtube_http_client.dart index e63883f..e0be453 100644 --- a/lib/src/reverse_engineering/youtube_http_client.dart +++ b/lib/src/reverse_engineering/youtube_http_client.dart @@ -128,10 +128,10 @@ class YoutubeHttpClient extends http.BaseClient { _validateResponse(response, response.statusCode); } final stream = StreamController>(); - response.stream.listen((data) { - bytesCount += data.length; - stream.add(data); - }, onError: (_) => null, onDone: stream.close, cancelOnError: false); + response.stream.listen((data) { + bytesCount += data.length; + stream.add(data); + }, onError: (_) => null, onDone: stream.close, cancelOnError: false); errorCount = 0; yield* stream.stream; } on Exception { From 09b28a7bebe34d164e826ef2a9cec05f36c4930d Mon Sep 17 00:00:00 2001 From: vi-k Date: Fri, 10 Sep 2021 19:44:47 +1000 Subject: [PATCH 3/3] Fix error when the http-client is closed and the request is still running. --- CHANGELOG.md | 9 +++-- lib/src/exceptions/exceptions.dart | 1 + lib/src/exceptions/http_client_closed.dart | 9 +++++ lib/src/retry.dart | 7 +++- .../clients/closed_caption_client.dart | 2 +- .../clients/comments_client.dart | 2 +- .../clients/embedded_player_client.dart | 2 +- .../reverse_engineering/dash_manifest.dart | 2 +- .../pages/channel_about_page.dart | 4 +- .../pages/channel_page.dart | 4 +- .../pages/channel_upload_page.dart | 2 +- .../reverse_engineering/pages/embed_page.dart | 2 +- .../pages/playlist_page.dart | 2 +- .../pages/search_page.dart | 2 +- .../reverse_engineering/pages/watch_page.dart | 2 +- .../player/player_source.dart | 2 +- .../youtube_http_client.dart | 37 +++++++++++++++---- lib/src/search/search_client.dart | 2 +- 18 files changed, 67 insertions(+), 26 deletions(-) create mode 100644 lib/src/exceptions/http_client_closed.dart diff --git a/CHANGELOG.md b/CHANGELOG.md index af97996..e104d71 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +## 1.10.7 +- Fix the error of incomplete data loading on the Android emulator. +- Fix error when the http-client is closed and the request is still running. ## 1.10.6 - Implement `Playlist.videoCount`. @@ -20,11 +23,11 @@ ## 1.10.1 - Fix issue #146: Closed Captions couldn't be extracted anymore. - Code cleanup. - + ## 1.10.0 - Fix issue #144: get_video_info was removed from yt. -- Min sdk version now is 2.13.0 +- Min sdk version now is 2.13.0 - BREAKING CHANGE: New comments API implementation. ## 1.9.10 @@ -45,7 +48,7 @@ ## 1.9.6 - Fix comment client. - Fix issue #130 (ClosedCaptions) - + ## 1.9.5 - Temporary for issue #130 diff --git a/lib/src/exceptions/exceptions.dart b/lib/src/exceptions/exceptions.dart index 74e468c..766eb89 100644 --- a/lib/src/exceptions/exceptions.dart +++ b/lib/src/exceptions/exceptions.dart @@ -1,6 +1,7 @@ library youtube_explode.exceptions; export 'fatal_failure_exception.dart'; +export 'http_client_closed.dart'; export 'request_limit_exceeded_exception.dart'; export 'search_item_section_exception.dart'; export 'transient_failure_exception.dart'; diff --git a/lib/src/exceptions/http_client_closed.dart b/lib/src/exceptions/http_client_closed.dart new file mode 100644 index 0000000..b674eee --- /dev/null +++ b/lib/src/exceptions/http_client_closed.dart @@ -0,0 +1,9 @@ +import 'youtube_explode_exception.dart'; + +/// An exception is thrown when the http-client is closed +/// and the request is still running. +class HttpClientClosedException extends YoutubeExplodeException { + HttpClientClosedException() + : super('The request could not be completed because ' + "the YoutubeExplode's http-client was closed."); +} diff --git a/lib/src/retry.dart b/lib/src/retry.dart index fbbb4f4..54b7166 100644 --- a/lib/src/retry.dart +++ b/lib/src/retry.dart @@ -4,11 +4,12 @@ import 'dart:async'; import 'package:http/http.dart'; +import '../youtube_explode_dart.dart'; import 'exceptions/exceptions.dart'; /// Run the [function] each time an exception is thrown until the retryCount /// is 0. -Future retry(FutureOr Function() function) async { +Future retry(YoutubeHttpClient? client, FutureOr Function() function) async { var retryCount = 5; // ignore: literal_only_boolean_expressions @@ -17,6 +18,10 @@ Future retry(FutureOr Function() function) async { return await function(); // ignore: avoid_catches_without_on_clauses } on Exception catch (e) { + if (client != null && client.closed) { + throw HttpClientClosedException(); + } + retryCount -= getExceptionCost(e); if (retryCount <= 0) { rethrow; diff --git a/lib/src/reverse_engineering/clients/closed_caption_client.dart b/lib/src/reverse_engineering/clients/closed_caption_client.dart index bb924b5..c61049d 100644 --- a/lib/src/reverse_engineering/clients/closed_caption_client.dart +++ b/lib/src/reverse_engineering/clients/closed_caption_client.dart @@ -23,7 +23,7 @@ class ClosedCaptionClient { static Future get( YoutubeHttpClient httpClient, Uri url) { final formatUrl = url.replaceQueryParameters({'fmt': 'srv3'}); - return retry(() async { + return retry(httpClient, () async { var raw = await httpClient.getString(formatUrl); return ClosedCaptionClient.parse(raw); }); diff --git a/lib/src/reverse_engineering/clients/comments_client.dart b/lib/src/reverse_engineering/clients/comments_client.dart index adfbd56..8ad079f 100644 --- a/lib/src/reverse_engineering/clients/comments_client.dart +++ b/lib/src/reverse_engineering/clients/comments_client.dart @@ -22,7 +22,7 @@ class CommentsClient { static Future get( YoutubeHttpClient httpClient, Video video) async { final watchPage = video.watchPage ?? - await retry( + await retry(httpClient, () async => WatchPage.get(httpClient, video.id.value)); final continuation = watchPage.commentsContinuation; diff --git a/lib/src/reverse_engineering/clients/embedded_player_client.dart b/lib/src/reverse_engineering/clients/embedded_player_client.dart index 9a4c264..1490920 100644 --- a/lib/src/reverse_engineering/clients/embedded_player_client.dart +++ b/lib/src/reverse_engineering/clients/embedded_player_client.dart @@ -63,7 +63,7 @@ class EmbeddedPlayerClient { final url = Uri.parse('https://www.youtube.com/youtubei/v1/player'); - return retry(() async { + return retry(httpClient, () async { final raw = await httpClient.post(url, body: json.encode(body), headers: { diff --git a/lib/src/reverse_engineering/dash_manifest.dart b/lib/src/reverse_engineering/dash_manifest.dart index 2a2dd44..6bd093a 100644 --- a/lib/src/reverse_engineering/dash_manifest.dart +++ b/lib/src/reverse_engineering/dash_manifest.dart @@ -29,7 +29,7 @@ class DashManifest { /// static Future get(YoutubeHttpClient httpClient, dynamic url) { - return retry(() async { + return retry(httpClient, () async { var raw = await httpClient.getString(url); return DashManifest.parse(raw); }); diff --git a/lib/src/reverse_engineering/pages/channel_about_page.dart b/lib/src/reverse_engineering/pages/channel_about_page.dart index c398b72..7fc3f7c 100644 --- a/lib/src/reverse_engineering/pages/channel_about_page.dart +++ b/lib/src/reverse_engineering/pages/channel_about_page.dart @@ -39,7 +39,7 @@ class ChannelAboutPage extends YoutubePage<_InitialData> { static Future get(YoutubeHttpClient httpClient, String id) { var url = 'https://www.youtube.com/channel/$id/about?hl=en'; - return retry(() async { + return retry(httpClient, () async { var raw = await httpClient.getString(url); var result = ChannelAboutPage.parse(raw); @@ -52,7 +52,7 @@ class ChannelAboutPage extends YoutubePage<_InitialData> { YoutubeHttpClient httpClient, String username) { var url = 'https://www.youtube.com/user/$username/about?hl=en'; - return retry(() async { + return retry(httpClient, () async { var raw = await httpClient.getString(url); var result = ChannelAboutPage.parse(raw); diff --git a/lib/src/reverse_engineering/pages/channel_page.dart b/lib/src/reverse_engineering/pages/channel_page.dart index dda8d42..6e08943 100644 --- a/lib/src/reverse_engineering/pages/channel_page.dart +++ b/lib/src/reverse_engineering/pages/channel_page.dart @@ -40,7 +40,7 @@ class ChannelPage extends YoutubePage<_InitialData> { static Future get(YoutubeHttpClient httpClient, String id) { var url = 'https://www.youtube.com/channel/$id?hl=en'; - return retry(() async { + return retry(httpClient, () async { var raw = await httpClient.getString(url); var result = ChannelPage.parse(raw); @@ -56,7 +56,7 @@ class ChannelPage extends YoutubePage<_InitialData> { YoutubeHttpClient httpClient, String username) { var url = 'https://www.youtube.com/user/$username?hl=en'; - return retry(() async { + return retry(httpClient, () async { var raw = await httpClient.getString(url); var result = ChannelPage.parse(raw); diff --git a/lib/src/reverse_engineering/pages/channel_upload_page.dart b/lib/src/reverse_engineering/pages/channel_upload_page.dart index 40e3e9f..664b57c 100644 --- a/lib/src/reverse_engineering/pages/channel_upload_page.dart +++ b/lib/src/reverse_engineering/pages/channel_upload_page.dart @@ -36,7 +36,7 @@ class ChannelUploadPage extends YoutubePage<_InitialData> { YoutubeHttpClient httpClient, String channelId, String sorting) { var url = 'https://www.youtube.com/channel/$channelId/videos?view=0&sort=$sorting&flow=grid'; - return retry(() async { + return retry(httpClient, () async { var raw = await httpClient.getString(url); return ChannelUploadPage.parse(raw, channelId); }); diff --git a/lib/src/reverse_engineering/pages/embed_page.dart b/lib/src/reverse_engineering/pages/embed_page.dart index eb7a853..0699ba1 100644 --- a/lib/src/reverse_engineering/pages/embed_page.dart +++ b/lib/src/reverse_engineering/pages/embed_page.dart @@ -71,7 +71,7 @@ class EmbedPage { static Future get(YoutubeHttpClient httpClient, String videoId) { var url = 'https://youtube.com/embed/$videoId?hl=en'; // final url = 'http://localhost:8080/embed/$videoId?hl=en'; - return retry(() async { + return retry(httpClient, () async { var raw = await httpClient.getString(url); return EmbedPage.parse(raw); }); diff --git a/lib/src/reverse_engineering/pages/playlist_page.dart b/lib/src/reverse_engineering/pages/playlist_page.dart index 5238a0b..23f6249 100644 --- a/lib/src/reverse_engineering/pages/playlist_page.dart +++ b/lib/src/reverse_engineering/pages/playlist_page.dart @@ -47,7 +47,7 @@ class PlaylistPage extends YoutubePage<_InitialData> { String id, ) async { var url = 'https://www.youtube.com/playlist?list=$id&hl=en&persist_hl=1'; - return retry(() async { + return retry(httpClient, () async { var raw = await httpClient.getString(url); return PlaylistPage.parse(raw, id); }); diff --git a/lib/src/reverse_engineering/pages/search_page.dart b/lib/src/reverse_engineering/pages/search_page.dart index 43fe0fb..6e2bc61 100644 --- a/lib/src/reverse_engineering/pages/search_page.dart +++ b/lib/src/reverse_engineering/pages/search_page.dart @@ -45,7 +45,7 @@ class SearchPage extends YoutubePage<_InitialData> { {SearchFilter filter = const SearchFilter('')}) { var url = 'https://www.youtube.com/results?search_query=${Uri.encodeQueryComponent(queryString)}&sp=${filter.value}'; - return retry(() async { + return retry(httpClient, () async { var raw = await httpClient.getString(url); return SearchPage.parse(raw, queryString); }); diff --git a/lib/src/reverse_engineering/pages/watch_page.dart b/lib/src/reverse_engineering/pages/watch_page.dart index 395cb92..e5a183d 100644 --- a/lib/src/reverse_engineering/pages/watch_page.dart +++ b/lib/src/reverse_engineering/pages/watch_page.dart @@ -119,7 +119,7 @@ class WatchPage extends YoutubePage<_InitialData> { /// static Future get(YoutubeHttpClient httpClient, String videoId) { final url = 'https://youtube.com/watch?v=$videoId&bpctr=9999999999&hl=en'; - return retry(() async { + return retry(httpClient, () async { var req = await httpClient.get(url, validate: true); var cookies = req.headers['set-cookie']!; diff --git a/lib/src/reverse_engineering/player/player_source.dart b/lib/src/reverse_engineering/player/player_source.dart index 3221a9c..9f0bf1d 100644 --- a/lib/src/reverse_engineering/player/player_source.dart +++ b/lib/src/reverse_engineering/player/player_source.dart @@ -108,7 +108,7 @@ class PlayerSource { static Future get( YoutubeHttpClient httpClient, String url) async { if (_cache[url]?.expired ?? true) { - var val = await retry(() async { + var val = await retry(httpClient, () async { var raw = await httpClient.getString(url); return PlayerSource.parse(raw); }); diff --git a/lib/src/reverse_engineering/youtube_http_client.dart b/lib/src/reverse_engineering/youtube_http_client.dart index e0be453..07d08fe 100644 --- a/lib/src/reverse_engineering/youtube_http_client.dart +++ b/lib/src/reverse_engineering/youtube_http_client.dart @@ -2,17 +2,21 @@ import 'dart:async'; import 'dart:convert'; import 'package:http/http.dart' as http; -import 'package:youtube_explode_dart/src/retry.dart'; import '../exceptions/exceptions.dart'; import '../extensions/helpers_extension.dart'; +import '../retry.dart'; import '../videos/streams/streams.dart'; /// HttpClient wrapper for YouTube class YoutubeHttpClient extends http.BaseClient { final http.Client _httpClient; - final Map _defaultHeaders = const { + // Flag to interrupt receiving stream. + bool _closed = false; + bool get closed => _closed; + + static const Map _defaultHeaders = { 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.111 Safari/537.36', 'cookie': 'CONSENT=YES+cb', @@ -33,7 +37,10 @@ class YoutubeHttpClient extends http.BaseClient { /// Throws if something is wrong with the response. void _validateResponse(http.BaseResponse response, int statusCode) { + if (_closed) return; + var request = response.request!; + if (request.url.host.endsWith('.google.com') && request.url.path.startsWith('/sorry/')) { throw RequestLimitExceededException.httpRequest(response); @@ -56,6 +63,7 @@ class YoutubeHttpClient extends http.BaseClient { Future getString(dynamic url, {Map headers = const {}, bool validate = true}) async { var response = await get(url, headers: headers); + if (_closed) throw HttpClientClosedException(); if (validate) { _validateResponse(response, response.statusCode); @@ -72,6 +80,8 @@ class YoutubeHttpClient extends http.BaseClient { url = Uri.parse(url); } var response = await super.get(url, headers: headers); + if (_closed) throw HttpClientClosedException(); + if (validate) { _validateResponse(response, response.statusCode); } @@ -86,6 +96,8 @@ class YoutubeHttpClient extends http.BaseClient { bool validate = false}) async { final response = await super.post(url, headers: headers, body: body, encoding: encoding); + if (_closed) throw HttpClientClosedException(); + if (validate) { _validateResponse(response, response.statusCode); } @@ -102,6 +114,7 @@ class YoutubeHttpClient extends http.BaseClient { url = Uri.parse(url); } var response = await post(url, headers: headers, body: body); + if (_closed) throw HttpClientClosedException(); if (validate) { _validateResponse(response, response.statusCode); @@ -117,9 +130,9 @@ class YoutubeHttpClient extends http.BaseClient { int errorCount = 0}) async* { var url = streamInfo.url; var bytesCount = start; - while (bytesCount != streamInfo.size.totalBytes) { + while (!_closed && bytesCount != streamInfo.size.totalBytes) { try { - final response = await retry(() { + final response = await retry(this, () { final request = http.Request('get', url); request.headers['range'] = 'bytes=$bytesCount-${bytesCount + 9898989 - 1}'; return send(request); @@ -134,6 +147,8 @@ class YoutubeHttpClient extends http.BaseClient { }, onError: (_) => null, onDone: stream.close, cancelOnError: false); errorCount = 0; yield* stream.stream; + } on HttpClientClosedException { + break; } on Exception { if (errorCount == 5) { rethrow; @@ -153,6 +168,7 @@ class YoutubeHttpClient extends http.BaseClient { Future getContentLength(dynamic url, {Map headers = const {}, bool validate = true}) async { var response = await head(url, headers: headers); + if (_closed) throw HttpClientClosedException(); if (validate) { _validateResponse(response, response.statusCode); @@ -179,24 +195,31 @@ class YoutubeHttpClient extends http.BaseClient { final url = Uri.parse( 'https://www.youtube.com/youtubei/v1/$action?key=AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8'); - return retry(() async { + return retry(this, () async { final raw = await post(url, body: json.encode(body)); + if (_closed) throw HttpClientClosedException(); + return json.decode(raw.body); }); } @override - void close() => _httpClient.close(); + void close() { + _closed = true; + _httpClient.close(); + } @override Future send(http.BaseRequest request) { + if (_closed) throw HttpClientClosedException(); + _defaultHeaders.forEach((key, value) { if (request.headers[key] == null) { request.headers[key] = _defaultHeaders[key]!; } }); // print('Request: $request'); -// print('Stack:\n${StackTrace.current}'); + // print('Stack:\n${StackTrace.current}'); return _httpClient.send(request); } } diff --git a/lib/src/search/search_client.dart b/lib/src/search/search_client.dart index 0bb72e7..5ec9852 100644 --- a/lib/src/search/search_client.dart +++ b/lib/src/search/search_client.dart @@ -58,7 +58,7 @@ class SearchClient { // ignore: literal_only_boolean_expressions for (;;) { if (page == null) { - page = await retry(() async => + page = await retry(_httpClient, () async => SearchPage.get(_httpClient, searchQuery, filter: filter)); } else { page = await page.nextPage(_httpClient);