From 95a77244a0bbc8822fe4fc8affb052b3f9f13d1b Mon Sep 17 00:00:00 2001 From: Mattia Date: Thu, 18 Mar 2021 22:22:55 +0100 Subject: [PATCH] dartfmt --- lib/src/extensions/helpers_extension.dart | 17 +- .../responses/channel_about_page.dart | 46 +++- .../responses/channel_upload_page.dart | 210 +++++++++--------- .../responses/embed_page.dart | 9 +- .../responses/playlist_page.dart | 88 ++++++-- .../responses/search_page.dart | 89 ++++++-- .../responses/watch_page.dart | 71 ++++-- lib/src/videos/streams/streams_client.dart | 96 +++++--- test/search_test.dart | 1 - 9 files changed, 416 insertions(+), 211 deletions(-) diff --git a/lib/src/extensions/helpers_extension.dart b/lib/src/extensions/helpers_extension.dart index e355ce2..f4c9b75 100644 --- a/lib/src/extensions/helpers_extension.dart +++ b/lib/src/extensions/helpers_extension.dart @@ -14,7 +14,8 @@ extension StringUtility on String { String substringUntil(String separator) => substring(0, indexOf(separator)); /// - String substringAfter(String separator) => substring(indexOf(separator) + separator.length); + String substringAfter(String separator) => + substring(indexOf(separator) + separator.length); static final _exp = RegExp(r'\D'); @@ -35,7 +36,8 @@ extension StringUtility on String { while (true) { try { - return json.decode(str.substring(startIdx, endIdx + 1)) as Map; + return json.decode(str.substring(startIdx, endIdx + 1)) + as Map; } on FormatException { endIdx = str.lastIndexOf(str.substring(0, endIdx)); if (endIdx == 0) { @@ -171,9 +173,14 @@ extension RunsParser on List { extension GenericExtract on List { /// Used to extract initial data that start with `var ytInitialData = ` or 'window["ytInitialData"] ='. - T extractGenericData(T Function(Map) builder, Exception Function() orThrow) { - var initialData = firstWhereOrNull((e) => e.contains('var ytInitialData = '))?.extractJson('var ytInitialData = '); - initialData ??= firstWhereOrNull((e) => e.contains('window["ytInitialData"] ='))?.extractJson('window["ytInitialData"] ='); + T extractGenericData( + T Function(Map) builder, Exception Function() orThrow) { + var initialData = + firstWhereOrNull((e) => e.contains('var ytInitialData = ')) + ?.extractJson('var ytInitialData = '); + initialData ??= + firstWhereOrNull((e) => e.contains('window["ytInitialData"] =')) + ?.extractJson('window["ytInitialData"] ='); if (initialData != null) { return builder(initialData); diff --git a/lib/src/reverse_engineering/responses/channel_about_page.dart b/lib/src/reverse_engineering/responses/channel_about_page.dart index 57a43c4..222d405 100644 --- a/lib/src/reverse_engineering/responses/channel_about_page.dart +++ b/lib/src/reverse_engineering/responses/channel_about_page.dart @@ -16,7 +16,10 @@ class ChannelAboutPage { late final _InitialData initialData = _getInitialData(); _InitialData _getInitialData() { - final scriptText = _root.querySelectorAll('script').map((e) => e.text).toList(growable: false); + final scriptText = _root + .querySelectorAll('script') + .map((e) => e.text) + .toList(growable: false); return scriptText.extractGenericData( (obj) => _InitialData(obj), () => TransientFailureException( @@ -45,7 +48,8 @@ class ChannelAboutPage { } /// - static Future getByUsername(YoutubeHttpClient httpClient, String username) { + static Future getByUsername( + YoutubeHttpClient httpClient, String username) { var url = 'https://www.youtube.com/user/$username/about?hl=en'; return retry(() async { @@ -84,29 +88,49 @@ class _InitialData { .get('channelAboutFullMetadataRenderer')!; } - late final String description = content.get('description')!.getT('simpleText')!; + late final String description = + content.get('description')!.getT('simpleText')!; late final List channelLinks = content .getList('primaryLinks')! .map((e) => ChannelLink( e.get('title')?.getT('simpleText') ?? '', - extractUrl(e.get('navigationEndpoint')?.get('commandMetadata')?.get('webCommandMetadata')?.getT('url') ?? - e.get('navigationEndpoint')?.get('urlEndpoint')?.getT('url') ?? + extractUrl(e + .get('navigationEndpoint') + ?.get('commandMetadata') + ?.get('webCommandMetadata') + ?.getT('url') ?? + e + .get('navigationEndpoint') + ?.get('urlEndpoint') + ?.getT('url') ?? ''), - Uri.parse(e.get('icon')?.getList('thumbnails')?.firstOrNull?.getT('url') ?? ''))) + Uri.parse(e + .get('icon') + ?.getList('thumbnails') + ?.firstOrNull + ?.getT('url') ?? + ''))) .toList(); - late final int viewCount = int.parse(content.get('viewCountText')!.getT('simpleText')!.stripNonDigits()); + late final int viewCount = int.parse(content + .get('viewCountText')! + .getT('simpleText')! + .stripNonDigits()); - late final String joinDate = content.get('joinedDateText')!.getList('runs')![1].getT('text')!; + late final String joinDate = + content.get('joinedDateText')!.getList('runs')![1].getT('text')!; late final String title = content.get('title')!.getT('simpleText')!; - late final List> avatar = content.get('avatar')!.getList('thumbnails')!; + late final List> avatar = + content.get('avatar')!.getList('thumbnails')!; String get country => content.get('country')!.getT('simpleText')!; - String parseRuns(List? runs) => runs?.map((e) => e.text).join() ?? ''; + String parseRuns(List? runs) => + runs?.map((e) => e.text).join() ?? ''; - Uri extractUrl(String text) => Uri.parse(Uri.decodeFull(_urlExp.firstMatch(text)?.group(1) ?? '')); + Uri extractUrl(String text) => + Uri.parse(Uri.decodeFull(_urlExp.firstMatch(text)?.group(1) ?? '')); } diff --git a/lib/src/reverse_engineering/responses/channel_upload_page.dart b/lib/src/reverse_engineering/responses/channel_upload_page.dart index 25fa9d5..f2a7667 100644 --- a/lib/src/reverse_engineering/responses/channel_upload_page.dart +++ b/lib/src/reverse_engineering/responses/channel_upload_page.dart @@ -30,140 +30,142 @@ class ChannelUploadPage { .map((e) => e.text) .toList(growable: false); - return scriptText.extractGenericData((obj) => _InitialData(obj), () => - TransientFailureException( + return scriptText.extractGenericData( + (obj) => _InitialData(obj), + () => TransientFailureException( 'Failed to retrieve initial data from the channel upload page, please report this to the project GitHub page.')); } - /// - ChannelUploadPage(this._root, this.channelId, [_InitialData ? initialData]): _initialData = initialData; + /// + ChannelUploadPage(this._root, this.channelId, [_InitialData? initialData]) + : _initialData = initialData; - /// - Future nextPage(YoutubeHttpClient httpClient) { + /// + Future nextPage(YoutubeHttpClient httpClient) { if (initialData.continuation.isEmpty) { - return Future.value(null); + return Future.value(null); } var url = - 'https://www.youtube.com/browse_ajax?ctoken=${initialData.continuation}&continuation=${initialData.continuation}&itct=${initialData.clickTrackingParams}'; + 'https://www.youtube.com/browse_ajax?ctoken=${initialData.continuation}&continuation=${initialData.continuation}&itct=${initialData.clickTrackingParams}'; return retry(() async { - var raw = await httpClient.getString(url); - return ChannelUploadPage( - null, channelId, _InitialData(json.decode(raw)[1])); + var raw = await httpClient.getString(url); + return ChannelUploadPage( + null, channelId, _InitialData(json.decode(raw)[1])); }); - } - - /// - static Future get( - YoutubeHttpClient httpClient, String channelId, String sorting) { - var url = - 'https://www.youtube.com/channel/$channelId/videos?view=0&sort=$sorting&flow=grid'; - return retry(() async { - var raw = await httpClient.getString(url); - return ChannelUploadPage.parse(raw, channelId); - }); - } - - /// - ChannelUploadPage.parse(String raw, this.channelId) - : _root = parser.parse(raw); } - class _InitialData { + /// + static Future get( + YoutubeHttpClient httpClient, String channelId, String sorting) { + var url = + 'https://www.youtube.com/channel/$channelId/videos?view=0&sort=$sorting&flow=grid'; + return retry(() async { + var raw = await httpClient.getString(url); + return ChannelUploadPage.parse(raw, channelId); + }); + } + + /// + ChannelUploadPage.parse(String raw, this.channelId) + : _root = parser.parse(raw); +} + +class _InitialData { // Json parsed map final Map root; _InitialData(this.root); late final Map? continuationContext = - getContinuationContext(); + getContinuationContext(); late final String clickTrackingParams = - continuationContext?.getT('continuationContext') ?? ''; + continuationContext?.getT('continuationContext') ?? ''; late final List uploads = - getContentContext().map(_parseContent).whereNotNull().toList(); + getContentContext().map(_parseContent).whereNotNull().toList(); late final String continuation = - continuationContext?.getT('continuation') ?? ''; + continuationContext?.getT('continuation') ?? ''; List> getContentContext() { - List>? context; - if (root.containsKey('contents')) { - context = root - .get('contents') - ?.get('twoColumnBrowseResultsRenderer') - ?.getList('tabs') - ?.map((e) => e['tabRenderer']) - .cast>() - .firstWhereOrNull((e) => e['selected'] as bool) - ?.get('content') - ?.get('sectionListRenderer') - ?.getList('contents') - ?.firstOrNull - ?.get('itemSectionRenderer') - ?.getList('contents') - ?.firstOrNull - ?.get('gridRenderer') - ?.getList('items') - ?.cast>(); - } - if (context == null && root.containsKey('response')) { - context = root - .get('response') - ?.get('continuationContents') - ?.get('gridContinuation') - ?.getList('items') - ?.cast>(); - } - if (context == null) { - throw FatalFailureException('Failed to get initial data context.'); - } - return context; + List>? context; + if (root.containsKey('contents')) { + context = root + .get('contents') + ?.get('twoColumnBrowseResultsRenderer') + ?.getList('tabs') + ?.map((e) => e['tabRenderer']) + .cast>() + .firstWhereOrNull((e) => e['selected'] as bool) + ?.get('content') + ?.get('sectionListRenderer') + ?.getList('contents') + ?.firstOrNull + ?.get('itemSectionRenderer') + ?.getList('contents') + ?.firstOrNull + ?.get('gridRenderer') + ?.getList('items') + ?.cast>(); + } + if (context == null && root.containsKey('response')) { + context = root + .get('response') + ?.get('continuationContents') + ?.get('gridContinuation') + ?.getList('items') + ?.cast>(); + } + if (context == null) { + throw FatalFailureException('Failed to get initial data context.'); + } + return context; } Map? getContinuationContext() { - if (root.containsKey('contents')) { - return root - .get('contents') - ?.get('twoColumnBrowseResultsRenderer') - ?.getList('tabs') - ?.map((e) => e['tabRenderer']) - .cast>() - .firstWhereOrNull((e) => e['selected'] as bool) - ?.get('content') - ?.get('sectionListRenderer') - ?.getList('contents') - ?.firstOrNull - ?.get('itemSectionRenderer') - ?.getList('contents') - ?.firstOrNull - ?.get('gridRenderer') - ?.getList('continuations') - ?.firstOrNull - ?.get('nextContinuationData'); - } - if (root.containsKey('response')) { - return root - .get('response') - ?.get('continuationContents') - ?.get('gridContinuation') - ?.getList('continuations') - ?.firstOrNull - ?.get('nextContinuationData'); - } - return null; + if (root.containsKey('contents')) { + return root + .get('contents') + ?.get('twoColumnBrowseResultsRenderer') + ?.getList('tabs') + ?.map((e) => e['tabRenderer']) + .cast>() + .firstWhereOrNull((e) => e['selected'] as bool) + ?.get('content') + ?.get('sectionListRenderer') + ?.getList('contents') + ?.firstOrNull + ?.get('itemSectionRenderer') + ?.getList('contents') + ?.firstOrNull + ?.get('gridRenderer') + ?.getList('continuations') + ?.firstOrNull + ?.get('nextContinuationData'); + } + if (root.containsKey('response')) { + return root + .get('response') + ?.get('continuationContents') + ?.get('gridContinuation') + ?.getList('continuations') + ?.firstOrNull + ?.get('nextContinuationData'); + } + return null; } ChannelVideo? _parseContent(Map? content) { - if (content == null || !content.containsKey('gridVideoRenderer')) { - return null; - } + if (content == null || !content.containsKey('gridVideoRenderer')) { + return null; + } - var video = content.get('gridVideoRenderer')!; - return ChannelVideo( - VideoId(video.getT('videoId')!), - video.get('title')?.getT('simpleText') ?? - video.get('title')?.getList('runs')?.map((e) => e['text']).join() ?? - ''); - } + var video = content.get('gridVideoRenderer')!; + return ChannelVideo( + VideoId(video.getT('videoId')!), + video.get('title')?.getT('simpleText') ?? + video.get('title')?.getList('runs')?.map((e) => e['text']).join() ?? + ''); } +} diff --git a/lib/src/reverse_engineering/responses/embed_page.dart b/lib/src/reverse_engineering/responses/embed_page.dart index d7fec61..636499a 100644 --- a/lib/src/reverse_engineering/responses/embed_page.dart +++ b/lib/src/reverse_engineering/responses/embed_page.dart @@ -9,7 +9,8 @@ import 'player_config_base.dart'; /// class EmbedPage { - static final _playerConfigExp = RegExp('[\'""]PLAYER_CONFIG[\'""]\\s*:\\s*(\\{.*\\})'); + static final _playerConfigExp = + RegExp('[\'""]PLAYER_CONFIG[\'""]\\s*:\\s*(\\{.*\\})'); static final _playerConfigExp2 = RegExp(r'yt.setConfig\((\{.*\})'); final Document root; @@ -21,7 +22,8 @@ class EmbedPage { .querySelectorAll('*[name="player_ias/base"]') .map((e) => e.attributes['src']) .where((e) => !e.isNullOrWhiteSpace) - .firstWhere((e) => e!.contains('player_ias') && e.endsWith('.js'), orElse: () => null); + .firstWhere((e) => e!.contains('player_ias') && e.endsWith('.js'), + orElse: () => null); // _root.querySelector('*[name="player_ias/base"]').attributes['src']; if (url == null) { return null; @@ -31,7 +33,8 @@ class EmbedPage { /// EmbedPlayerConfig? getPlayerConfig() { - var playerConfigJson = (_playerConfigJson ?? _playerConfigJson2)?.extractJson(); + var playerConfigJson = + (_playerConfigJson ?? _playerConfigJson2)?.extractJson(); if (playerConfigJson == null) { return null; } diff --git a/lib/src/reverse_engineering/responses/playlist_page.dart b/lib/src/reverse_engineering/responses/playlist_page.dart index 47fb58a..34ae8ee 100644 --- a/lib/src/reverse_engineering/responses/playlist_page.dart +++ b/lib/src/reverse_engineering/responses/playlist_page.dart @@ -24,7 +24,10 @@ class PlaylistPage { return _initialData!; } - final scriptText = root!.querySelectorAll('script').map((e) => e.text).toList(growable: false); + final scriptText = root! + .querySelectorAll('script') + .map((e) => e.text) + .toList(growable: false); return scriptText.extractGenericData( (obj) => _InitialData(obj), @@ -33,7 +36,8 @@ class PlaylistPage { } /// - PlaylistPage(this.root, this.playlistId, [_InitialData? initialData]) : _initialData = initialData; + PlaylistPage(this.root, this.playlistId, [_InitialData? initialData]) + : _initialData = initialData; /// Future nextPage(YoutubeHttpClient httpClient) async { @@ -44,19 +48,26 @@ class PlaylistPage { } /// - static Future get(YoutubeHttpClient httpClient, String id, {String? token}) { + static Future get(YoutubeHttpClient httpClient, String id, + {String? token}) { if (token != null && token.isNotEmpty) { - var url = 'https://www.youtube.com/youtubei/v1/guide?key=AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8'; + var url = + 'https://www.youtube.com/youtubei/v1/guide?key=AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8'; return retry(() async { var body = { 'context': const { - 'client': {'hl': 'en', 'clientName': 'WEB', 'clientVersion': '2.20200911.04.00'} + 'client': { + 'hl': 'en', + 'clientName': 'WEB', + 'clientVersion': '2.20200911.04.00' + } }, 'continuation': token }; - var raw = await httpClient.post(Uri.parse(url), body: json.encode(body)); + var raw = + await httpClient.post(Uri.parse(url), body: json.encode(body)); return PlaylistPage(null, id, _InitialData(json.decode(raw.body))); }); // Ask for next page, @@ -80,7 +91,10 @@ class _InitialData { _InitialData(this.root); - late final String? title = root.get('metadata')?.get('playlistMetadataRenderer')?.getT('title'); + late final String? title = root + .get('metadata') + ?.get('playlistMetadataRenderer') + ?.getT('title'); late final String? author = root .get('sidebar') @@ -94,7 +108,10 @@ class _InitialData { ?.getT>('runs') ?.parseRuns(); - late final String? description = root.get('metadata')?.get('playlistMetadataRenderer')?.getT('description'); + late final String? description = root + .get('metadata') + ?.get('playlistMetadataRenderer') + ?.getT('description'); late final int? viewCount = root .get('sidebar') @@ -107,12 +124,13 @@ class _InitialData { ?.getT('simpleText') ?.parseInt(); - late final String? continuationToken = (videosContent ?? playlistVideosContent) - ?.firstWhereOrNull((e) => e['continuationItemRenderer'] != null) - ?.get('continuationItemRenderer') - ?.get('continuationEndpoint') - ?.get('continuationCommand') - ?.getT('token'); + late final String? continuationToken = + (videosContent ?? playlistVideosContent) + ?.firstWhereOrNull((e) => e['continuationItemRenderer'] != null) + ?.get('continuationItemRenderer') + ?.get('continuationEndpoint') + ?.get('continuationCommand') + ?.getT('token'); List>? get playlistVideosContent => root @@ -130,7 +148,11 @@ class _InitialData { ?.firstOrNull ?.get('playlistVideoListRenderer') ?.getList('contents') ?? - root.getList('onResponseReceivedActions')?.firstOrNull?.get('appendContinuationItemsAction')?.getList('continuationItems'); + root + .getList('onResponseReceivedActions') + ?.firstOrNull + ?.get('appendContinuationItemsAction') + ?.getList('continuationItems'); late final List>? videosContent = root .get('contents') @@ -138,10 +160,17 @@ class _InitialData { ?.get('primaryContents') ?.get('sectionListRenderer') ?.getList('contents') ?? - root.getList('onResponseReceivedCommands')?.firstOrNull?.get('appendContinuationItemsAction')?.getList('continuationItems'); + root + .getList('onResponseReceivedCommands') + ?.firstOrNull + ?.get('appendContinuationItemsAction') + ?.getList('continuationItems'); List<_Video> get playlistVideos => - playlistVideosContent?.where((e) => e['playlistVideoRenderer'] != null).map((e) => _Video(e['playlistVideoRenderer'])).toList() ?? + playlistVideosContent + ?.where((e) => e['playlistVideoRenderer'] != null) + .map((e) => _Video(e['playlistVideoRenderer'])) + .toList() ?? const []; List<_Video> get videos => @@ -168,7 +197,13 @@ class _Video { ''; String get channelId => - root.get('ownerText')?.getList('runs')?.firstOrNull?.get('navigationEndpoint')?.get('browseEndpoint')?.getT('browseId') ?? + root + .get('ownerText') + ?.getList('runs') + ?.firstOrNull + ?.get('navigationEndpoint') + ?.get('browseEndpoint') + ?.getT('browseId') ?? root .get('shortBylineText') ?.getList('runs') @@ -180,11 +215,14 @@ class _Video { String get title => root.get('title')?.getList('runs')?.parseRuns() ?? ''; - String get description => root.getList('descriptionSnippet')?.parseRuns() ?? ''; + String get description => + root.getList('descriptionSnippet')?.parseRuns() ?? ''; - Duration? get duration => _stringToDuration(root.get('lengthText')?.getT('simpleText')); + Duration? get duration => + _stringToDuration(root.get('lengthText')?.getT('simpleText')); - int get viewCount => root.get('viewCountText')?.getT('simpleText')?.parseInt() ?? 0; + int get viewCount => + root.get('viewCountText')?.getT('simpleText')?.parseInt() ?? 0; /// Format: HH:MM:SS static Duration? _stringToDuration(String? string) { @@ -199,10 +237,14 @@ class _Video { return Duration(seconds: int.parse(parts.first)); } if (parts.length == 2) { - return Duration(minutes: int.parse(parts.first), seconds: int.parse(parts[1])); + return Duration( + minutes: int.parse(parts.first), seconds: int.parse(parts[1])); } if (parts.length == 3) { - return Duration(hours: int.parse(parts[0]), minutes: int.parse(parts[1]), seconds: int.parse(parts[2])); + return Duration( + hours: int.parse(parts[0]), + minutes: int.parse(parts[1]), + seconds: int.parse(parts[2])); } throw Error(); } diff --git a/lib/src/reverse_engineering/responses/search_page.dart b/lib/src/reverse_engineering/responses/search_page.dart index 08fc39c..229f4f7 100644 --- a/lib/src/reverse_engineering/responses/search_page.dart +++ b/lib/src/reverse_engineering/responses/search_page.dart @@ -28,7 +28,10 @@ class SearchPage { return _initialData!; } - final scriptText = root!.querySelectorAll('script').map((e) => e.text).toList(growable: false); + final scriptText = root! + .querySelectorAll('script') + .map((e) => e.text) + .toList(growable: false); return scriptText.extractGenericData( (obj) => _InitialData(obj), () => TransientFailureException( @@ -36,35 +39,47 @@ class SearchPage { } /// - SearchPage(this.root, this.queryString, [_InitialData? initialData]) : _initialData = initialData; + SearchPage(this.root, this.queryString, [_InitialData? initialData]) + : _initialData = initialData; Future nextPage(YoutubeHttpClient httpClient) async { - if (initialData.continuationToken == '' || initialData.estimatedResults == 0) { + if (initialData.continuationToken == '' || + initialData.estimatedResults == 0) { return null; } return get(httpClient, queryString, token: initialData.continuationToken); } /// - static Future get(YoutubeHttpClient httpClient, String queryString, {String? token}) { + static Future get( + YoutubeHttpClient httpClient, String queryString, + {String? token}) { if (token != null) { - var url = 'https://www.youtube.com/youtubei/v1/search?key=AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8'; + var url = + 'https://www.youtube.com/youtubei/v1/search?key=AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8'; return retry(() async { var body = { 'context': const { - 'client': {'hl': 'en', 'clientName': 'WEB', 'clientVersion': '2.20200911.04.00'} + 'client': { + 'hl': 'en', + 'clientName': 'WEB', + 'clientVersion': '2.20200911.04.00' + } }, 'continuation': token }; - var raw = await httpClient.post(Uri.parse(url), body: json.encode(body)); - return SearchPage(null, queryString, _InitialData(json.decode(raw.body))); + var raw = + await httpClient.post(Uri.parse(url), body: json.encode(body)); + return SearchPage( + null, queryString, _InitialData(json.decode(raw.body))); }); // Ask for next page, } - var url = 'https://www.youtube.com/results?search_query=${Uri.encodeQueryComponent(queryString)}'; + var url = + 'https://www.youtube.com/results?search_query=${Uri.encodeQueryComponent(queryString)}'; return retry(() async { var raw = await httpClient.getString(url); return SearchPage.parse(raw, queryString); @@ -142,7 +157,9 @@ class _InitialData { } // Contains only [SearchVideo] or [SearchPlaylist] - late final List searchContent = getContentContext()?.map(_parseContent).whereNotNull().toList() ?? const []; + late final List searchContent = + getContentContext()?.map(_parseContent).whereNotNull().toList() ?? + const []; List get relatedQueries => getContentContext() @@ -150,8 +167,10 @@ class _InitialData { .map((e) => e.get('horizontalCardListRenderer')?.getList('cards')) .firstOrNull ?.map((e) => e['searchRefinementCardRenderer']) - .map((e) => - RelatedQuery(e.searchEndpoint.searchEndpoint.query, VideoId(Uri.parse(e.thumbnail.thumbnails.first.url).pathSegments[1]))) + .map((e) => RelatedQuery( + e.searchEndpoint.searchEndpoint.query, + VideoId( + Uri.parse(e.thumbnail.thumbnails.first.url).pathSegments[1]))) .toList() .cast() ?? const []; @@ -159,7 +178,11 @@ class _InitialData { List get relatedVideos => getContentContext() ?.where((e) => e['shelfRenderer'] != null) - .map((e) => e.get('shelfRenderer')?.get('content')?.get('verticalListRenderer')?.getList('items')) + .map((e) => e + .get('shelfRenderer') + ?.get('content') + ?.get('verticalListRenderer') + ?.getList('items')) .firstOrNull ?.map(_parseContent) .whereNotNull() @@ -168,7 +191,8 @@ class _InitialData { late final String? continuationToken = _getContinuationToken(); - late final int estimatedResults = int.parse(root.getT('estimatedResults') ?? '0'); + late final int estimatedResults = + int.parse(root.getT('estimatedResults') ?? '0'); BaseSearchContent? _parseContent(Map? content) { if (content == null) { @@ -183,24 +207,47 @@ class _InitialData { _parseRuns(renderer.get('ownerText')?.getList('runs')), _parseRuns(renderer.get('descriptionSnippet')?.getList('runs')), renderer.get('lengthText')?.getT('simpleText') ?? '', - int.parse(renderer.get('viewCountText')?.getT('simpleText')?.stripNonDigits().nullIfWhitespace ?? - renderer.get('viewCountText')?.getList('runs')?.firstOrNull?.getT('text')?.stripNonDigits().nullIfWhitespace ?? + int.parse(renderer + .get('viewCountText') + ?.getT('simpleText') + ?.stripNonDigits() + .nullIfWhitespace ?? + renderer + .get('viewCountText') + ?.getList('runs') + ?.firstOrNull + ?.getT('text') + ?.stripNonDigits() + .nullIfWhitespace ?? '0'), (renderer.get('thumbnail')?.getList('thumbnails') ?? const []) - .map((e) => Thumbnail(Uri.parse(e['url']), e['height'], e['width'])) + .map((e) => + Thumbnail(Uri.parse(e['url']), e['height'], e['width'])) .toList(), renderer.get('publishedTimeText')?.getT('simpleText'), - renderer.get('viewCountText')?.getList('runs')?.elementAtSafe(1)?.getT('text')?.trim() == 'watching'); + renderer + .get('viewCountText') + ?.getList('runs') + ?.elementAtSafe(1) + ?.getT('text') + ?.trim() == + 'watching'); } if (content['radioRenderer'] != null) { var renderer = content.get('radioRenderer')!; - return SearchPlaylist(PlaylistId(renderer.getT('playlistId')!), renderer.get('title')!.getT('simpleText')!, - int.parse(_parseRuns(renderer.get('videoCountText')?.getList('runs')).stripNonDigits().nullIfWhitespace ?? '0')); + return SearchPlaylist( + PlaylistId(renderer.getT('playlistId')!), + renderer.get('title')!.getT('simpleText')!, + int.parse(_parseRuns(renderer.get('videoCountText')?.getList('runs')) + .stripNonDigits() + .nullIfWhitespace ?? + '0')); } // Here ignore 'horizontalCardListRenderer' & 'shelfRenderer' return null; } - String _parseRuns(List? runs) => runs?.map((e) => e['text']).join() ?? ''; + String _parseRuns(List? runs) => + runs?.map((e) => e['text']).join() ?? ''; } diff --git a/lib/src/reverse_engineering/responses/watch_page.dart b/lib/src/reverse_engineering/responses/watch_page.dart index 3cbf28a..49a2ed2 100644 --- a/lib/src/reverse_engineering/responses/watch_page.dart +++ b/lib/src/reverse_engineering/responses/watch_page.dart @@ -12,11 +12,15 @@ import 'player_response.dart'; /// class WatchPage { - static final RegExp _videoLikeExp = RegExp(r'"label"\s*:\s*"([\d,\.]+) likes"'); - static final RegExp _videoDislikeExp = RegExp(r'"label"\s*:\s*"([\d,\.]+) dislikes"'); - static final RegExp _visitorInfoLiveExp = RegExp('VISITOR_INFO1_LIVE=([^;]+)'); + static final RegExp _videoLikeExp = + RegExp(r'"label"\s*:\s*"([\d,\.]+) likes"'); + static final RegExp _videoDislikeExp = + RegExp(r'"label"\s*:\s*"([\d,\.]+) dislikes"'); + static final RegExp _visitorInfoLiveExp = + RegExp('VISITOR_INFO1_LIVE=([^;]+)'); static final RegExp _yscExp = RegExp('YSC=([^;]+)'); - static final RegExp _playerResponseExp = RegExp(r'var\s+ytInitialPlayerResponse\s*=\s*(\{.*\})'); + static final RegExp _playerResponseExp = + RegExp(r'var\s+ytInitialPlayerResponse\s*=\s*(\{.*\})'); static final _xsfrTokenExp = RegExp(r'"XSRF_TOKEN"\s*:\s*"(.+?)"'); @@ -51,7 +55,10 @@ class WatchPage { return _initialData!; } - final scriptText = root.querySelectorAll('script').map((e) => e.text).toList(growable: false); + final scriptText = root + .querySelectorAll('script') + .map((e) => e.text) + .toList(growable: false); return scriptText.extractGenericData( (obj) => _InitialData(obj), () => TransientFailureException( @@ -62,23 +69,45 @@ class WatchPage { /// String? getXsfrToken() { - return _xsfrTokenExp.firstMatch(root.querySelectorAll('script').firstWhere((e) => _xsfrTokenExp.hasMatch(e.text)).text)?.group(1); + return _xsfrTokenExp + .firstMatch(root + .querySelectorAll('script') + .firstWhere((e) => _xsfrTokenExp.hasMatch(e.text)) + .text) + ?.group(1); } /// bool get isOk => root.body?.querySelector('#player') != null; /// - bool get isVideoAvailable => root.querySelector('meta[property="og:url"]') != null; + bool get isVideoAvailable => + root.querySelector('meta[property="og:url"]') != null; /// - int get videoLikeCount => int.parse(_videoLikeExp.firstMatch(root.outerHtml)?.group(1)?.stripNonDigits().nullIfWhitespace ?? - root.querySelector('.like-button-renderer-like-button')?.text.stripNonDigits().nullIfWhitespace ?? + int get videoLikeCount => int.parse(_videoLikeExp + .firstMatch(root.outerHtml) + ?.group(1) + ?.stripNonDigits() + .nullIfWhitespace ?? + root + .querySelector('.like-button-renderer-like-button') + ?.text + .stripNonDigits() + .nullIfWhitespace ?? '0'); /// - int get videoDislikeCount => int.parse(_videoDislikeExp.firstMatch(root.outerHtml)?.group(1)?.stripNonDigits().nullIfWhitespace ?? - root.querySelector('.like-button-renderer-dislike-button')?.text.stripNonDigits().nullIfWhitespace ?? + int get videoDislikeCount => int.parse(_videoDislikeExp + .firstMatch(root.outerHtml) + ?.group(1) + ?.stripNonDigits() + .nullIfWhitespace ?? + root + .querySelector('.like-button-renderer-dislike-button') + ?.text + .stripNonDigits() + .nullIfWhitespace ?? '0'); static final _playerConfigExp = RegExp(r'ytplayer\.config\s*=\s*(\{.*\})'); @@ -89,7 +118,10 @@ class WatchPage { /// WatchPlayerConfig? getPlayerConfig() { - final jsonMap = _playerConfigExp.firstMatch(root.getElementsByTagName('html').first.text)?.group(1)?.extractJson(); + final jsonMap = _playerConfigExp + .firstMatch(root.getElementsByTagName('html').first.text) + ?.group(1) + ?.extractJson(); if (jsonMap == null) { return null; } @@ -114,7 +146,8 @@ class WatchPage { WatchPage(this.root, this.visitorInfoLive, this.ysc); /// - WatchPage.parse(String raw, this.visitorInfoLive, this.ysc) : root = parser.parse(raw); + WatchPage.parse(String raw, this.visitorInfoLive, this.ysc) + : root = parser.parse(raw); /// static Future get(YoutubeHttpClient httpClient, String videoId) { @@ -148,10 +181,12 @@ class WatchPlayerConfig implements PlayerConfigBase> { WatchPlayerConfig(this.root); @override - late final String sourceUrl = 'https://youtube.com${root.get('assets')!.getT('js')}'; + late final String sourceUrl = + 'https://youtube.com${root.get('assets')!.getT('js')}'; /// - late final PlayerResponse playerResponse = PlayerResponse.parse(root.get('args')!.getT('playerResponse')!); + late final PlayerResponse playerResponse = + PlayerResponse.parse(root.get('args')!.getT('playerResponse')!); } class _InitialData { @@ -177,7 +212,9 @@ class _InitialData { return null; } - late final String continuation = getContinuationContext()?.getT('continuation') ?? ''; + late final String continuation = + getContinuationContext()?.getT('continuation') ?? ''; - late final String clickTrackingParams = getContinuationContext()?.getT('clickTrackingParams') ?? ''; + late final String clickTrackingParams = + getContinuationContext()?.getT('clickTrackingParams') ?? ''; } diff --git a/lib/src/videos/streams/streams_client.dart b/lib/src/videos/streams/streams_client.dart index 1fd92b9..3549a9c 100644 --- a/lib/src/videos/streams/streams_client.dart +++ b/lib/src/videos/streams/streams_client.dart @@ -21,8 +21,10 @@ class StreamsClient { /// Initializes an instance of [StreamsClient] StreamsClient(this._httpClient); - Future _getDashManifest(Uri dashManifestUrl, Iterable cipherOperations) { - var signature = DashManifest.getSignatureFromUrl(dashManifestUrl.toString()); + Future _getDashManifest( + Uri dashManifestUrl, Iterable cipherOperations) { + var signature = + DashManifest.getSignatureFromUrl(dashManifestUrl.toString()); if (!signature.isNullOrWhiteSpace) { signature = cipherOperations.decipher(signature!); dashManifestUrl = dashManifestUrl.setQueryParam('signature', signature); @@ -37,30 +39,38 @@ class StreamsClient { throw VideoUnplayableException.unplayable(videoId); } - var playerSource = await PlayerSource.get(_httpClient, embedPage.sourceUrl ?? playerConfig.sourceUrl); + var playerSource = await PlayerSource.get( + _httpClient, embedPage.sourceUrl ?? playerConfig.sourceUrl); var cipherOperations = playerSource.getCipherOperations(); - var videoInfoResponse = await VideoInfoResponse.get(_httpClient, videoId.toString(), playerSource.sts); + var videoInfoResponse = await VideoInfoResponse.get( + _httpClient, videoId.toString(), playerSource.sts); var playerResponse = videoInfoResponse.playerResponse; var previewVideoId = playerResponse.previewVideoId; if (!previewVideoId.isNullOrWhiteSpace) { - throw VideoRequiresPurchaseException.preview(videoId, VideoId(previewVideoId!)); + throw VideoRequiresPurchaseException.preview( + videoId, VideoId(previewVideoId!)); } if (!playerResponse.isVideoPlayable) { - throw VideoUnplayableException.unplayable(videoId, reason: playerResponse.videoPlayabilityError ?? ''); + throw VideoUnplayableException.unplayable(videoId, + reason: playerResponse.videoPlayabilityError ?? ''); } if (playerResponse.isLive) { throw VideoUnplayableException.liveStream(videoId); } - var streamInfoProviders = [...videoInfoResponse.streams, ...playerResponse.streams]; + var streamInfoProviders = [ + ...videoInfoResponse.streams, + ...playerResponse.streams + ]; var dashManifestUrl = playerResponse.dashManifestUrl; if (!dashManifestUrl.isNullOrWhiteSpace) { - var dashManifest = await _getDashManifest(Uri.parse(dashManifestUrl!), cipherOperations); + var dashManifest = + await _getDashManifest(Uri.parse(dashManifestUrl!), cipherOperations); streamInfoProviders.addAll(dashManifest.streams); } return StreamContext(streamInfoProviders, cipherOperations); @@ -71,22 +81,28 @@ class StreamsClient { final playerConfig = watchPage.playerConfig; - var playerResponse = playerConfig?.playerResponse ?? watchPage.playerResponse; + var playerResponse = + playerConfig?.playerResponse ?? watchPage.playerResponse; if (playerResponse == null) { throw VideoUnplayableException.unplayable(videoId); } var previewVideoId = playerResponse.previewVideoId; if (!previewVideoId.isNullOrWhiteSpace) { - throw VideoRequiresPurchaseException.preview(videoId, VideoId(previewVideoId!)); + throw VideoRequiresPurchaseException.preview( + videoId, VideoId(previewVideoId!)); } var playerSourceUrl = watchPage.sourceUrl ?? playerConfig?.sourceUrl; - var playerSource = !playerSourceUrl.isNullOrWhiteSpace ? await PlayerSource.get(_httpClient, playerSourceUrl!) : null; - var cipherOperations = playerSource?.getCipherOperations() ?? const []; + var playerSource = !playerSourceUrl.isNullOrWhiteSpace + ? await PlayerSource.get(_httpClient, playerSourceUrl!) + : null; + var cipherOperations = + playerSource?.getCipherOperations() ?? const []; if (!playerResponse.isVideoPlayable) { - throw VideoUnplayableException.unplayable(videoId, reason: playerResponse.videoPlayabilityError ?? ''); + throw VideoUnplayableException.unplayable(videoId, + reason: playerResponse.videoPlayabilityError ?? ''); } if (playerResponse.isLive) { @@ -99,7 +115,8 @@ class StreamsClient { var dashManifestUrl = playerResponse.dashManifestUrl; if (!(dashManifestUrl?.isNullOrWhiteSpace ?? true)) { - var dashManifest = await _getDashManifest(Uri.parse(dashManifestUrl!), cipherOperations); + var dashManifest = + await _getDashManifest(Uri.parse(dashManifestUrl!), cipherOperations); streamInfoProviders.addAll(dashManifest.streams); } return StreamContext(streamInfoProviders, cipherOperations); @@ -123,7 +140,9 @@ class StreamsClient { } // Content length - var contentLength = streamInfo.contentLength ?? await _httpClient.getContentLength(url, validate: false) ?? 0; + var contentLength = streamInfo.contentLength ?? + await _httpClient.getContentLength(url, validate: false) ?? + 0; if (contentLength <= 0) { continue; @@ -140,31 +159,53 @@ class StreamsClient { // Muxed or Video-only if (!videoCodec.isNullOrWhiteSpace) { var framerate = Framerate(streamInfo.framerate ?? 24); - var videoQualityLabel = - streamInfo.videoQualityLabel ?? VideoQualityUtil.getLabelFromTagWithFramerate(tag, framerate.framesPerSecond.toDouble()); + var videoQualityLabel = streamInfo.videoQualityLabel ?? + VideoQualityUtil.getLabelFromTagWithFramerate( + tag, framerate.framesPerSecond.toDouble()); var videoQuality = VideoQualityUtil.fromLabel(videoQualityLabel); var videoWidth = streamInfo.videoWidth; var videoHeight = streamInfo.videoHeight; - var videoResolution = - videoWidth != -1 && videoHeight != -1 ? VideoResolution(videoWidth ?? 0, videoHeight ?? 0) : videoQuality.toVideoResolution(); + var videoResolution = videoWidth != -1 && videoHeight != -1 + ? VideoResolution(videoWidth ?? 0, videoHeight ?? 0) + : videoQuality.toVideoResolution(); // Muxed if (!audioCodec.isNullOrWhiteSpace) { - streams[tag] = MuxedStreamInfo(tag, url, container, fileSize, bitrate, audioCodec!, videoCodec!, videoQualityLabel, videoQuality, - videoResolution, framerate); + streams[tag] = MuxedStreamInfo( + tag, + url, + container, + fileSize, + bitrate, + audioCodec!, + videoCodec!, + videoQualityLabel, + videoQuality, + videoResolution, + framerate); continue; } // Video only streams[tag] = VideoOnlyStreamInfo( - tag, url, container, fileSize, bitrate, videoCodec!, videoQualityLabel, videoQuality, videoResolution, framerate); + tag, + url, + container, + fileSize, + bitrate, + videoCodec!, + videoQualityLabel, + videoQuality, + videoResolution, + framerate); continue; } // Audio-only if (!audioCodec.isNullOrWhiteSpace) { - streams[tag] = AudioOnlyStreamInfo(tag, url, container, fileSize, bitrate, audioCodec!); + streams[tag] = AudioOnlyStreamInfo( + tag, url, container, fileSize, bitrate, audioCodec!); } // #if DEBUG @@ -194,10 +235,12 @@ class StreamsClient { /// Gets the HTTP Live Stream (HLS) manifest URL /// for the specified video (if it's a live video stream). Future getHttpLiveStreamUrl(VideoId videoId) async { - var videoInfoResponse = await VideoInfoResponse.get(_httpClient, videoId.toString()); + var videoInfoResponse = + await VideoInfoResponse.get(_httpClient, videoId.toString()); var playerResponse = videoInfoResponse.playerResponse; if (!playerResponse.isVideoPlayable) { - throw VideoUnplayableException.unplayable(videoId, reason: playerResponse.videoPlayabilityError ?? ''); + throw VideoUnplayableException.unplayable(videoId, + reason: playerResponse.videoPlayabilityError ?? ''); } var hlsManifest = playerResponse.hlsManifestUrl; @@ -208,5 +251,6 @@ class StreamsClient { } /// Gets the actual stream which is identified by the specified metadata. - Stream> get(StreamInfo streamInfo) => _httpClient.getStream(streamInfo); + Stream> get(StreamInfo streamInfo) => + _httpClient.getStream(streamInfo); } diff --git a/test/search_test.dart b/test/search_test.dart index 8e1e886..dbba1eb 100644 --- a/test/search_test.dart +++ b/test/search_test.dart @@ -17,7 +17,6 @@ void main() { }); test('Search a youtube video from the search page-2', () async { - var videos = await yt!.search .getVideosFromPage('hello') .where((e) => e is SearchVideo) // Take only the videos.