diff --git a/CHANGELOG.md b/CHANGELOG.md index 8bd65b2..e8bb978 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ - Now playlists with more than 100 videos return all the videos. Thanks to @ATiltedTree. - Implemented `ChannelAboutPage`, check the tests their usage. - Implement filters for `search.getVideos`. See `filter` getter. +- Now video's from search queries return the channel id. +- Implemented publishDate for videos. Thanks to @mymikemiller , PR: #115. ## 1.8.0 - Fixed playlist client. diff --git a/lib/src/extensions/helpers_extension.dart b/lib/src/extensions/helpers_extension.dart index 665f6bf..6ccb95a 100644 --- a/lib/src/extensions/helpers_extension.dart +++ b/lib/src/extensions/helpers_extension.dart @@ -14,7 +14,8 @@ extension StringUtility on String { String substringUntil(String separator) => substring(0, indexOf(separator)); /// - String substringAfter(String separator) => substring(indexOf(separator) + separator.length); + String substringAfter(String separator) => + substring(indexOf(separator) + separator.length); static final _exp = RegExp(r'\D'); @@ -35,7 +36,8 @@ extension StringUtility on String { while (true) { try { - return json.decode(str.substring(startIdx, endIdx + 1)) as Map; + return json.decode(str.substring(startIdx, endIdx + 1)) + as Map; } on FormatException { endIdx = str.lastIndexOf(str.substring(0, endIdx)); if (endIdx == 0) { @@ -58,10 +60,14 @@ extension StringUtility on String { return Duration(seconds: int.parse(parts.first)); } if (parts.length == 2) { - return Duration(minutes: int.parse(parts[0]), seconds: int.parse(parts[1])); + return Duration( + minutes: int.parse(parts[0]), seconds: int.parse(parts[1])); } if (parts.length == 3) { - return Duration(hours: int.parse(parts[0]), minutes: int.parse(parts[1]), seconds: int.parse(parts[2])); + return Duration( + hours: int.parse(parts[0]), + minutes: int.parse(parts[1]), + seconds: int.parse(parts[2])); } // Shouldn't reach here. throw Error(); @@ -233,9 +239,14 @@ extension RunsParser on List { extension GenericExtract on List { /// Used to extract initial data that start with `var ytInitialData = ` or 'window["ytInitialData"] ='. - T extractGenericData(T Function(Map) builder, Exception Function() orThrow) { - var initialData = firstWhereOrNull((e) => e.contains('var ytInitialData = '))?.extractJson('var ytInitialData = '); - initialData ??= firstWhereOrNull((e) => e.contains('window["ytInitialData"] ='))?.extractJson('window["ytInitialData"] ='); + T extractGenericData( + T Function(Map) builder, Exception Function() orThrow) { + var initialData = + firstWhereOrNull((e) => e.contains('var ytInitialData = ')) + ?.extractJson('var ytInitialData = '); + initialData ??= + firstWhereOrNull((e) => e.contains('window["ytInitialData"] =')) + ?.extractJson('window["ytInitialData"] ='); if (initialData != null) { return builder(initialData); diff --git a/lib/src/playlists/playlist_client.dart b/lib/src/playlists/playlist_client.dart index 8279964..e26f864 100644 --- a/lib/src/playlists/playlist_client.dart +++ b/lib/src/playlists/playlist_client.dart @@ -64,7 +64,7 @@ class PlaylistClient { ThumbnailSet(videoId), null, Engagement(video.viewCount, null, null), - null); + false); } continuationToken = response.initialData.continuationToken; if (response.initialData.continuationToken?.isEmpty ?? true) { diff --git a/lib/src/reverse_engineering/responses/search_page.dart b/lib/src/reverse_engineering/responses/search_page.dart index e5b5781..7928455 100644 --- a/lib/src/reverse_engineering/responses/search_page.dart +++ b/lib/src/reverse_engineering/responses/search_page.dart @@ -30,7 +30,10 @@ class SearchPage { return _initialData!; } - final scriptText = root!.querySelectorAll('script').map((e) => e.text).toList(growable: false); + final scriptText = root! + .querySelectorAll('script') + .map((e) => e.text) + .toList(growable: false); return scriptText.extractGenericData( (obj) => _InitialData(obj), () => TransientFailureException( @@ -38,36 +41,47 @@ class SearchPage { } /// - SearchPage(this.root, this.queryString, [_InitialData? initialData]) : _initialData = initialData; + SearchPage(this.root, this.queryString, [_InitialData? initialData]) + : _initialData = initialData; Future nextPage(YoutubeHttpClient httpClient) async { - if (initialData.continuationToken == '' || initialData.estimatedResults == 0) { + if (initialData.continuationToken == '' || + initialData.estimatedResults == 0) { return null; } return get(httpClient, queryString, token: initialData.continuationToken); } /// - static Future get(YoutubeHttpClient httpClient, String queryString, + static Future get( + YoutubeHttpClient httpClient, String queryString, {String? token, SearchFilter filter = const SearchFilter('')}) { if (token != null) { - var url = 'https://www.youtube.com/youtubei/v1/search?key=AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8'; + var url = + 'https://www.youtube.com/youtubei/v1/search?key=AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8'; return retry(() async { var body = { 'context': const { - 'client': {'hl': 'en', 'clientName': 'WEB', 'clientVersion': '2.20200911.04.00'} + 'client': { + 'hl': 'en', + 'clientName': 'WEB', + 'clientVersion': '2.20200911.04.00' + } }, 'continuation': token }; - var raw = await httpClient.post(Uri.parse(url), body: json.encode(body)); - return SearchPage(null, queryString, _InitialData(json.decode(raw.body))); + var raw = + await httpClient.post(Uri.parse(url), body: json.encode(body)); + return SearchPage( + null, queryString, _InitialData(json.decode(raw.body))); }); // Ask for next page, } - var url = 'https://www.youtube.com/results?search_query=${Uri.encodeQueryComponent(queryString)}&sp=${filter.value}'; + var url = + 'https://www.youtube.com/results?search_query=${Uri.encodeQueryComponent(queryString)}&sp=${filter.value}'; return retry(() async { var raw = await httpClient.getString(url); return SearchPage.parse(raw, queryString); @@ -145,7 +159,9 @@ class _InitialData { } // Contains only [SearchVideo] or [SearchPlaylist] - late final List searchContent = getContentContext()?.map(_parseContent).whereNotNull().toList() ?? const []; + late final List searchContent = + getContentContext()?.map(_parseContent).whereNotNull().toList() ?? + const []; List get relatedQueries => getContentContext() @@ -153,8 +169,10 @@ class _InitialData { .map((e) => e.get('horizontalCardListRenderer')?.getList('cards')) .firstOrNull ?.map((e) => e['searchRefinementCardRenderer']) - .map((e) => - RelatedQuery(e.searchEndpoint.searchEndpoint.query, VideoId(Uri.parse(e.thumbnail.thumbnails.first.url).pathSegments[1]))) + .map((e) => RelatedQuery( + e.searchEndpoint.searchEndpoint.query, + VideoId( + Uri.parse(e.thumbnail.thumbnails.first.url).pathSegments[1]))) .toList() .cast() ?? const []; @@ -162,7 +180,11 @@ class _InitialData { List get relatedVideos => getContentContext() ?.where((e) => e['shelfRenderer'] != null) - .map((e) => e.get('shelfRenderer')?.get('content')?.get('verticalListRenderer')?.getList('items')) + .map((e) => e + .get('shelfRenderer') + ?.get('content') + ?.get('verticalListRenderer') + ?.getList('items')) .firstOrNull ?.map(_parseContent) .whereNotNull() @@ -171,7 +193,8 @@ class _InitialData { late final String? continuationToken = _getContinuationToken(); - late final int estimatedResults = int.parse(root.getT('estimatedResults') ?? '0'); + late final int estimatedResults = + int.parse(root.getT('estimatedResults') ?? '0'); BaseSearchContent? _parseContent(Map? content) { if (content == null) { @@ -186,32 +209,63 @@ class _InitialData { _parseRuns(renderer.get('ownerText')?.getList('runs')), _parseRuns(renderer.get('descriptionSnippet')?.getList('runs')), renderer.get('lengthText')?.getT('simpleText') ?? '', - int.parse(renderer.get('viewCountText')?.getT('simpleText')?.stripNonDigits().nullIfWhitespace ?? - renderer.get('viewCountText')?.getList('runs')?.firstOrNull?.getT('text')?.stripNonDigits().nullIfWhitespace ?? + int.parse(renderer + .get('viewCountText') + ?.getT('simpleText') + ?.stripNonDigits() + .nullIfWhitespace ?? + renderer + .get('viewCountText') + ?.getList('runs') + ?.firstOrNull + ?.getT('text') + ?.stripNonDigits() + .nullIfWhitespace ?? '0'), (renderer.get('thumbnail')?.getList('thumbnails') ?? const []) - .map((e) => Thumbnail(Uri.parse(e['url']), e['height'], e['width'])) + .map((e) => + Thumbnail(Uri.parse(e['url']), e['height'], e['width'])) .toList(), renderer.get('publishedTimeText')?.getT('simpleText'), - renderer.get('viewCountText')?.getList('runs')?.elementAtSafe(1)?.getT('text')?.trim() == 'watching'); + renderer + .get('viewCountText') + ?.getList('runs') + ?.elementAtSafe(1) + ?.getT('text') + ?.trim() == + 'watching', + renderer['ownerText']['runs'][0]['navigationEndpoint'] + ['browseEndpoint']['browseId']); } if (content['radioRenderer'] != null) { var renderer = content.get('radioRenderer')!; - return SearchPlaylist(PlaylistId(renderer.getT('playlistId')!), renderer.get('title')!.getT('simpleText')!, - int.parse(_parseRuns(renderer.get('videoCountText')?.getList('runs')).stripNonDigits().nullIfWhitespace ?? '0')); + return SearchPlaylist( + PlaylistId(renderer.getT('playlistId')!), + renderer.get('title')!.getT('simpleText')!, + int.parse(_parseRuns(renderer.get('videoCountText')?.getList('runs')) + .stripNonDigits() + .nullIfWhitespace ?? + '0')); } if (content['channelRenderer'] != null) { var renderer = content.get('channelRenderer')!; return SearchChannel( ChannelId(renderer.getT('channelId')!), renderer.get('title')!.getT('simpleText')!, - renderer.get('descriptionSnippet')?.getList('runs')?.parseRuns() ?? '', - renderer.get('videoCountText')!.getList('runs')!.first.getT('text')!.parseInt()!); + renderer.get('descriptionSnippet')?.getList('runs')?.parseRuns() ?? + '', + renderer + .get('videoCountText')! + .getList('runs')! + .first + .getT('text')! + .parseInt()!); } // Here ignore 'horizontalCardListRenderer' & 'shelfRenderer' return null; } - String _parseRuns(List? runs) => runs?.map((e) => e['text']).join() ?? ''; + String _parseRuns(List? runs) => + runs?.map((e) => e['text']).join() ?? ''; } diff --git a/lib/src/reverse_engineering/youtube_http_client.dart b/lib/src/reverse_engineering/youtube_http_client.dart index ffbf632..14ce2eb 100644 --- a/lib/src/reverse_engineering/youtube_http_client.dart +++ b/lib/src/reverse_engineering/youtube_http_client.dart @@ -147,7 +147,7 @@ class YoutubeHttpClient extends http.BaseClient { request.headers[key] = _defaultHeaders[key]!; } }); - print('Request: $request'); + // print('Request: $request'); // print('Stack:\n${StackTrace.current}'); return _httpClient.send(request); } diff --git a/lib/src/search/search_client.dart b/lib/src/search/search_client.dart index bb55354..c9dd4b9 100644 --- a/lib/src/search/search_client.dart +++ b/lib/src/search/search_client.dart @@ -21,14 +21,26 @@ class SearchClient { /// (from the video search page). /// The videos are sent in batch of 20 videos. /// You [SearchList.nextPage] to get the next batch of videos. - Future getVideos(String searchQuery, {SearchFilter filter = const SearchFilter('')}) async { + Future getVideos(String searchQuery, + {SearchFilter filter = const SearchFilter('')}) async { final page = await SearchPage.get(_httpClient, searchQuery, filter: filter); return SearchList( page.initialData.searchContent .whereType() - .map((e) => Video(e.id, e.title, e.author, null, e.uploadDate?.toDateTime(), null, e.description, e.duration.toDuration(), - ThumbnailSet(e.id.value), null, Engagement(e.viewCount, null, null), e.isLive)) + .map((e) => Video( + e.id, + e.title, + e.author, + ChannelId(e.channelId), + e.uploadDate?.toDateTime(), + null, + e.description, + e.duration.toDuration(), + ThumbnailSet(e.id.value), + null, + Engagement(e.viewCount, null, null), + e.isLive)) .toList(), page, _httpClient); @@ -37,13 +49,17 @@ class SearchClient { /// Enumerates videos returned by the specified search query /// (from the video search page). /// Contains only instances of [SearchVideo] or [SearchPlaylist] + @Deprecated( + 'Since version 1.9.0 this is the same as [SearchClient.getVideos].') Stream getVideosFromPage(String searchQuery, - {bool onlyVideos = true, SearchFilter filter = const SearchFilter('')}) async* { + {bool onlyVideos = true, + SearchFilter filter = const SearchFilter('')}) async* { SearchPage? page; // ignore: literal_only_boolean_expressions for (;;) { if (page == null) { - page = await retry(() async => SearchPage.get(_httpClient, searchQuery, filter: filter)); + page = await retry(() async => + SearchPage.get(_httpClient, searchQuery, filter: filter)); } else { page = await page.nextPage(_httpClient); if (page == null) { @@ -52,7 +68,8 @@ class SearchClient { } if (onlyVideos) { - yield* Stream.fromIterable(page!.initialData.searchContent.whereType()); + yield* Stream.fromIterable( + page!.initialData.searchContent.whereType()); } else { yield* Stream.fromIterable(page!.initialData.searchContent); } @@ -74,7 +91,8 @@ class SearchClient { /// Queries to YouTube to get the results. @Deprecated('Use getVideosFromPage instead - ' 'Should be used only to get related videos') - Future queryFromPage(String searchQuery) => SearchQuery.search(_httpClient, searchQuery); + Future queryFromPage(String searchQuery) => + SearchQuery.search(_httpClient, searchQuery); } /* diff --git a/lib/src/search/search_list.dart b/lib/src/search/search_list.dart index b3dd5b8..d1e104c 100644 --- a/lib/src/search/search_list.dart +++ b/lib/src/search/search_list.dart @@ -27,7 +27,7 @@ class SearchList extends DelegatingList