import 'dart:convert'; import 'package:html/dom.dart'; import 'package:html/parser.dart' as parser; import '../../../youtube_explode_dart.dart'; import '../../extensions/helpers_extension.dart'; import '../../playlists/playlist_id.dart'; import '../../retry.dart'; import '../../search/related_query.dart'; import '../../search/search_playlist.dart'; import '../../search/search_video.dart'; import '../../videos/videos.dart'; import '../youtube_http_client.dart'; /// class SearchPage { static final _xsfrTokenExp = RegExp('"XSRF_TOKEN":"(.+?)"'); /// final String queryString; final Document _root; _InitialData _initialData; String _xsrfToken; /// _InitialData get initialData => _initialData ??= _InitialData(json.decode(_matchJson(_extractJson( _root .querySelectorAll('script') .map((e) => e.text) .toList() .firstWhere((e) => e.contains('window["ytInitialData"] =')), 'window["ytInitialData"] =')))); /// String get xsfrToken => _xsrfToken ??= _xsfrTokenExp .firstMatch(_root .querySelectorAll('script') .firstWhere((e) => _xsfrTokenExp.hasMatch(e.text)) .text) .group(1); String _extractJson(String html, String separator) { return _matchJson( html.substring(html.indexOf(separator) + separator.length)); } String _matchJson(String str) { var bracketCount = 0; int lastI; for (var i = 0; i < str.length; i++) { lastI = i; if (str[i] == '{') { bracketCount++; } else if (str[i] == '}') { bracketCount--; } else if (str[i] == ';') { if (bracketCount == 0) { return str.substring(0, i); } } } return str.substring(0, lastI + 1); } /// SearchPage(this._root, this.queryString, [_InitialData initalData, String xsfrToken]) : _initialData = initalData, _xsrfToken = xsfrToken; /// // TODO: Replace this in favour of async* when quering; Future nextPage(YoutubeHttpClient httpClient) async { if (initialData.continuation == '') { return null; } return get(httpClient, queryString, ctoken: initialData.continuation, itct: initialData.clickTrackingParams, xsrfToken: xsfrToken); } /// static Future get( YoutubeHttpClient httpClient, String queryString, {String ctoken, String itct, String xsrfToken}) { var url = 'https://www.youtube.com/results?search_query=${Uri.encodeQueryComponent(queryString)}'; if (ctoken != null) { assert(itct != null, 'If ctoken is not null itct cannot be null'); url += '&pbj=1'; url += '&ctoken=${Uri.encodeQueryComponent(ctoken)}'; url += '&continuation=${Uri.encodeQueryComponent(ctoken)}'; url += '&itct=${Uri.encodeQueryComponent(itct)}'; } return retry(() async { Map body; if (xsrfToken != null) { body = {'session_token': xsrfToken}; } var raw = await httpClient.postString(url, body: body); if (ctoken != null) { return SearchPage( null, queryString, _InitialData(json.decode(raw)[1]), xsrfToken); } return SearchPage.parse(raw, queryString); }); } /// SearchPage.parse(String raw, this.queryString) : _root = parser.parse(raw); } class _InitialData { // Json parsed map final Map _root; _InitialData(this._root); /* Cache results */ List _searchContent; List _relatedVideos; List _relatedQueries; String _continuation; String _clickTrackingParams; List> getContentContext(Map root) { if (root['contents'] != null) { return _root['contents']['twoColumnSearchResultsRenderer'] ['primaryContents']['sectionListRenderer']['contents'] .first['itemSectionRenderer']['contents'] .cast>(); } if (root['response'] != null) { return _root['response']['continuationContents'] ['itemSectionContinuation']['contents'] .cast>(); } throw FatalFailureException('Failed to get initial data context.'); } Map getContinuationContext(Map root) { if (_root['contents'] != null) { return (_root['contents']['twoColumnSearchResultsRenderer'] ['primaryContents']['sectionListRenderer']['contents'] ?.first['itemSectionRenderer']['continuations'] ?.first as Map) ?.getValue('nextContinuationData') ?.cast(); } if (_root['response'] != null) { return _root['response']['continuationContents'] ['itemSectionContinuation']['continuations'] ?.first['nextContinuationData'] ?.cast(); } return null; } // Contains only [SearchVideo] or [SearchPlaylist] List get searchContent => _searchContent ??= getContentContext(_root) .map(_parseContent) .where((e) => e != null) .toList(); List get relatedQueries => (_relatedQueries ??= getContentContext(_root) ?.where((e) => e.containsKey('horizontalCardListRenderer')) ?.map((e) => e['horizontalCardListRenderer']['cards']) ?.firstOrNull ?.map((e) => e['searchRefinementCardRenderer']) ?.map((e) => RelatedQuery( e['searchEndpoint']['searchEndpoint']['query'], VideoId(Uri.parse(e['thumbnail']['thumbnails'].first['url']) .pathSegments[1]))) ?.toList() ?.cast()) ?? const []; List get relatedVideos => (_relatedVideos ??= getContentContext(_root) ?.where((e) => e.containsKey('shelfRenderer')) ?.map((e) => e['shelfRenderer']['content']['verticalListRenderer']['items']) ?.firstOrNull ?.map(_parseContent) ?.toList()) ?? const []; String get continuation => _continuation ??= getContinuationContext(_root)?.getValue('continuation') ?? ''; String get clickTrackingParams => _clickTrackingParams ??= getContinuationContext(_root)?.getValue('clickTrackingParams') ?? ''; int get estimatedResults => int.parse(_root['estimatedResults'] ?? 0); dynamic _parseContent(dynamic content) { if (content == null) { return null; } if (content.containsKey('videoRenderer')) { Map renderer = content['videoRenderer']; //TODO: Add if it's a live return SearchVideo( VideoId(renderer['videoId']), _parseRuns(renderer['title']), _parseRuns(renderer['ownerText']), _parseRuns(renderer['descriptionSnippet']), renderer.get('lengthText')?.getValue('simpleText') ?? '', int.parse(renderer['viewCountText']['simpleText'] .toString() .stripNonDigits() .nullIfWhitespace ?? '0')); } if (content.containsKey('radioRenderer')) { var renderer = content['radioRenderer']; return SearchPlaylist( PlaylistId(renderer['playlistId']), renderer['title']['simpleText'], int.parse(_parseRuns(renderer['videoCountText']) .stripNonDigits() .nullIfWhitespace ?? 0)); } // Here ignore 'horizontalCardListRenderer' & 'shelfRenderer' return null; } String _parseRuns(Map runs) => runs?.getValue('runs')?.map((e) => e['text'])?.join() ?? ''; } // ['contents']['twoColumnSearchResultsRenderer']['primaryContents'] // ['sectionListRenderer']['contents'].first['itemSectionRenderer'] // // // ['contents'] -> @See ContentsList // ['continuations'] -> Data to see more //ContentsList: // Key -> 'videoRenderer' // videoId --> VideoId // title['runs'].loop -> ['text'] -> concatenate --> "Video Title" // descriptionSnippet['runs'].loop -> ['text'] -> concatenate // --> "Video Description snippet" // ownerText['runs'].first -> ['text'] --> "Video Author" // lengthText['simpleText'] -> Parse format H:M:S -> "Video Duration" // viewCountText['simpleText'] -> Strip non digit -> int.parse // --> "Video View Count" // // Key -> 'radioRenderer' // playlistId -> PlaylistId // title['simpleText'] --> "Playlist Title" // // Key -> 'horizontalCardListRenderer' // Queries related to this search // cards --> List of Maps -> loop -> ['searchRefinementCardRenderer'].first // thumbnail -> ['thumbnails'].first -> ['url'] // --> "Thumbnail url" -> Find video id from id. // searchEndpoint -> ['searchEndpoint'] // -> ['query'] -> "Related query string" // // Key -> 'shelfRenderer' // Videos related to this search // contents -> ['verticalListRenderer']['items'] -> loop -> parseContent