Fully Implement #20

lib/src/extensions/helpers_extension.dart

@@ -44,7 +44,7 @@ extension ListDecipher on Iterable<CipherOperation> {
 }
 
 /// List Utility.
-extension ListFirst<E> on List<E> {
+extension ListFirst<E> on Iterable<E> {
   /// Returns the first element of a list or null if empty.
   E get firstOrNull {
     if (length == 0) {

lib/src/reverse_engineering/responses/search_page.dart

@@ -3,13 +3,17 @@ import 'dart:convert';
 import 'package:html/dom.dart';
 import 'package:html/parser.dart' as parser;
 
-import '../../playlists/playlists.dart';
+import '../../extensions/helpers_extension.dart';
+import '../../playlists/playlist_id.dart';
 import '../../retry.dart';
 import '../../search/related_query.dart';
+import '../../search/search_playlist.dart';
+import '../../search/search_video.dart';
 import '../../videos/videos.dart';
 import '../youtube_http_client.dart';
 
 class SearchPage {
+  final String queryString;
   final Document _root;
 
   _InitialData _initialData;
@@ -23,6 +27,17 @@ class SearchPage {
               .firstWhere((e) => e.contains('window["ytInitialData"] =')),
           'window["ytInitialData"] ='))));
 
+  String _xsrfToken;
+
+  static final _xsfrTokenExp = RegExp('"XSRF_TOKEN":"(.+?)"');
+
+  String get xsfrToken => _xsrfToken ??= _xsfrTokenExp
+      .firstMatch(_root
+          .querySelectorAll('script')
+          .firstWhere((e) => _xsfrTokenExp.hasMatch(e.text))
+          .text)
+      .group(1);
+
   String _extractJson(String html, String separator) {
     return _matchJson(
         html.substring(html.indexOf(separator) + separator.length));
@@ -46,22 +61,53 @@ class SearchPage {
     return str.substring(0, lastI + 1);
   }
 
-  SearchPage(this._root);
+  SearchPage(this._root, this.queryString,
+      [_InitialData initalData, String xsfrToken])
+      : _initialData = initalData,
+        _xsrfToken = xsfrToken;
+
+  Future<SearchPage> nextPage(YoutubeHttpClient httpClient) {
+    if (initialData.continuation == '') {
+      return null;
+    }
+    return get(httpClient, queryString,
+          ctoken: initialData.continuation,
+          itct: initialData.clickTrackingParams,
+          xsrfToken: xsfrToken);
+  }
 
   static Future<SearchPage> get(
-      YoutubeHttpClient httpClient, String queryString) {
-    final url =
+      YoutubeHttpClient httpClient, String queryString,
+      {String ctoken, String itct, String xsrfToken}) {
+    var url =
         'https://www.youtube.com/results?search_query=${Uri.encodeQueryComponent(queryString)}';
+    if (ctoken != null) {
+      assert(itct != null, 'If ctoken is not null itct cannot be null');
+      url += '&pbj=1';
+      url += '&ctoken=${Uri.encodeQueryComponent(ctoken)}';
+      url += '&continuation=${Uri.encodeQueryComponent(ctoken)}';
+      url += '&itct=${Uri.encodeQueryComponent(itct)}';
+    }
     return retry(() async {
+      Map<String, String> body;
+      if (xsrfToken != null) {
+        body = {'session_token': xsrfToken};
+      }
       var raw = await httpClient.postString(url);
-      return SearchPage.parse(raw);
+      if (ctoken != null) {
+        return SearchPage(
+            null, queryString, _InitialData(json.decode(raw)[1]), xsrfToken);
+      }
+      return SearchPage.parse(raw, queryString);
     });
   }
 
-  SearchPage.parse(String raw) : _root = parser.parse(raw);
+  SearchPage.parse(String raw, this.queryString) : _root = parser.parse(raw);
 }
 
 class _InitialData {
+  //TODO: Add total result
+
   // Json parsed map
   final Map<String, dynamic> _root;
 
@@ -72,57 +118,114 @@ class _InitialData {
   List<dynamic> _searchContent;
   List<dynamic> _relatedVideos;
   List<RelatedQuery> _relatedQueries;
+  String _continuation;
+  String _clickTrackingParams;
 
-  // Contains only [VideoId] or [PlaylistId]
-  List<dynamic> get searchContent =>
-      _searchContent ??= _root['contents']['twoColumnSearchResultsRenderer']
+  List<Map<String, dynamic>> getContentContext(Map<String, dynamic> root) {
+    if (root['contents'] != null) {
+      return _root['contents']['twoColumnSearchResultsRenderer']
               ['primaryContents']['sectionListRenderer']['contents']
           .first['itemSectionRenderer']['contents']
-          .map(_parseContent)
-          .where((e) => e != null)
-          .toList();
+          .cast<Map<String, dynamic>>();
+    }
+    if (root['response'] != null) {
+      return _root['response']['continuationContents']
+              ['itemSectionContinuation']['contents']
+          .cast<Map<String, dynamic>>();
+    }
+    throw Exception('Couldn\'t find the content data');
+  }
+
+  Map<String, dynamic> getContinuationContext(Map<String, dynamic> root) {
+    if (_root['contents'] != null) {
+      return _root['contents']['twoColumnSearchResultsRenderer']
+              ['primaryContents']['sectionListRenderer']['contents']
+          ?.first['itemSectionRenderer']['continuations']
+          ?.first
+          ?.getValue('nextContinuationData')
+          ?.cast<String, dynamic>();
+    }
+    if (_root['response'] != null) {
+      return _root['response']['continuationContents']
+              ['itemSectionContinuation']['continuations']
+          ?.first['nextContinuationData']
+          ?.cast<String, dynamic>();
+    }
+    return null;
+  }
+
+  // Contains only [SearchVideo] or [SearchPlaylist]
+  List<dynamic> get searchContent => _searchContent ??= getContentContext(_root)
+      .map(_parseContent)
+      .where((e) => e != null)
+      .toList();
 
   List<RelatedQuery> get relatedQueries =>
-      _relatedQueries ??= _root['contents']['twoColumnSearchResultsRenderer']
-              ['primaryContents']['sectionListRenderer']['contents']
-          .first['itemSectionRenderer']['contents']
-          .where((e) => e.containsKey('horizontalCardListRenderer') as bool)
-          .map((e) => e['horizontalCardListRenderer']['cards'])
-          .first
-          .map((e) => e['searchRefinementCardRenderer'])
-          .map((e) => RelatedQuery(
+      (_relatedQueries ??= getContentContext(_root)
+          ?.where((e) => e.containsKey('horizontalCardListRenderer'))
+          ?.map((e) => e['horizontalCardListRenderer']['cards'])
+          ?.firstOrNull
+          ?.map((e) => e['searchRefinementCardRenderer'])
+          ?.map((e) => RelatedQuery(
               e['searchEndpoint']['searchEndpoint']['query'],
               VideoId(Uri.parse(e['thumbnail']['thumbnails'].first['url'])
                   .pathSegments[1])))
-          .toList()
-          .cast<RelatedQuery>();
+          ?.toList()
+          ?.cast<RelatedQuery>()) ??
+      const [];
 
-  List<dynamic> get relatedVideos => _relatedVideos ??= _root['contents']
-              ['twoColumnSearchResultsRenderer']['primaryContents']
-          ['sectionListRenderer']['contents']
-      .first['itemSectionRenderer']['contents']
-      .where((e) => e.containsKey('shelfRenderer') as bool)
-      .map(
-          (e) => e['shelfRenderer']['content']['verticalListRenderer']['items'])
-      .first
-      .map(_parseContent)
-      .toList();
+  List<dynamic> get relatedVideos =>
+      (_relatedVideos ??= getContentContext(_root)
+          ?.where((e) => e.containsKey('shelfRenderer'))
+          ?.map((e) =>
+              e['shelfRenderer']['content']['verticalListRenderer']['items'])
+          ?.firstOrNull
+          ?.map(_parseContent)
+          ?.toList()) ??
+      const [];
+
+  String get continuation => _continuation ??=
+      getContinuationContext(_root)?.getValue('continuation') ?? '';
+
+  String get clickTrackingParams => _clickTrackingParams ??=
+      getContinuationContext(_root)?.getValue('clickTrackingParams') ?? '';
 
   dynamic _parseContent(dynamic content) {
-    // If is a video
-    print(content);
     if (content == null) {
       return null;
     }
     if (content.containsKey('videoRenderer')) {
-      return VideoId(content['videoRenderer']['videoId']);
+      Map<String, dynamic> renderer = content['videoRenderer'];
+      //TODO: Add it's a live
+      return SearchVideo(
+          VideoId(renderer['videoId']),
+          _parseRuns(renderer['title']),
+          _parseRuns(renderer['ownerText']),
+          _parseRuns(renderer['descriptionSnippet']),
+          renderer.get('lengthText')?.getValue('simpleText') ?? '',
+          int.parse(renderer['viewCountText']['simpleText']
+                  .toString()
+                  .stripNonDigits()
+                  .nullIfWhitespace ??
+              '0'));
     }
     if (content.containsKey('radioRenderer')) {
-      return PlaylistId(content['radioRenderer']['playlistId']);
+      var renderer = content['radioRenderer'];
+
+      return SearchPlaylist(
+          PlaylistId(renderer['playlistId']),
+          renderer['title']['simpleText'],
+          int.parse(_parseRuns(renderer['videoCountText'])
+                  .stripNonDigits()
+                  .nullIfWhitespace ??
+              0));
     }
     // Here ignore 'horizontalCardListRenderer' & 'shelfRenderer'
     return null;
   }
+
+  String _parseRuns(Map<dynamic, dynamic> runs) =>
+      runs?.getValue('runs')?.map((e) => e['text'])?.join() ?? '';
 }
 
 // ['contents']['twoColumnSearchResultsRenderer']['primaryContents']['sectionListRenderer']['contents'].first['itemSectionRenderer']

lib/src/reverse_engineering/youtube_http_client.dart

@@ -9,7 +9,9 @@ class YoutubeHttpClient {
   final Map<String, String> _defaultHeaders = const {
     'user-agent':
         'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36',
-    'accept-language': 'en-US,en;q=1.0'
+    'accept-language': 'en-US,en;q=1.0',
+    'x-youtube-client-name': '1',
+    'x-youtube-client-version': '2.20200609.04.02',
   };
 
   /// Throws if something is wrong with the response.
@@ -58,9 +60,11 @@ class YoutubeHttpClient {
   }
 
   Future<String> postString(dynamic url,
-      {Map<String, String> headers, bool validate = true}) async {
-    var response =
-        await _httpClient.post(url, headers: {...?headers, ..._defaultHeaders});
+      {Map<String, String> body,
+      Map<String, String> headers,
+      bool validate = true}) async {
+    var response = await _httpClient.post(url,
+        headers: {...?headers, ..._defaultHeaders}, body: body);
 
     if (validate) {
       _validateResponse(response, response.statusCode);

lib/src/search/related_query.dart

@@ -5,7 +5,7 @@ class RelatedQuery {
   /// Query related to a search query.
   final String query;
 
-  /// Video related to a seach query.
+  /// Video related to a search query.
   final VideoId videoId;
 
   /// Initialize a [RelatedQuery] instance.

lib/src/search/search.dart

@@ -0,0 +1,7 @@
+library youtube_explode.search;
+
+export 'related_query.dart';
+export 'search_client.dart';
+export 'search_playlist.dart';
+export 'search_query.dart';
+export 'search_video.dart';

lib/src/search/search_channel.dart

@@ -0,0 +1,18 @@
+import '../channels/channel_id.dart';
+
+import '../playlists/playlist_id.dart';
+
+/// Metadata related to a search query result (channel)
+class SearchChannel {
+  /// ChannelId.
+  final ChannelId channelId;
+
+  /// Channel name.
+  final String channelName;
+
+  /// Initialize a [SearchChannel] instance.
+  SearchChannel(this.channelId, this.channelName);
+
+  @override
+  String toString() => '(Channel) $channelName ($channelId)';
+}

lib/src/search/search_client.dart

@@ -1,3 +1,6 @@
+import 'package:youtube_explode_dart/src/search/search_query.dart'
+    show SearchQuery;
+
 import '../common/common.dart';
 import '../reverse_engineering/responses/playerlist_response.dart';
 import '../reverse_engineering/youtube_http_client.dart';
@@ -46,4 +49,8 @@ class SearchClient {
       }
     }
   }
+
+  /// Queries to YouTube to get the results.
+  Future<SearchQuery> queryFromPage(String searchQuery) =>
+      SearchQuery.search(_httpClient, searchQuery);
 }

lib/src/search/search_playlist.dart

@@ -0,0 +1,18 @@
+import '../playlists/playlist_id.dart';
+
+/// Metadata related to a search query result (playlist)
+class SearchPlaylist {
+  /// PlaylistId.
+  final PlaylistId playlistId;
+
+  /// Playlist title.
+  final String playlistTitle;
+
+  /// Playlist video count, cannot be greater than 50.
+  final int playlistVideoCount;
+
+  SearchPlaylist(this.playlistId, this.playlistTitle, this.playlistVideoCount);
+
+  @override
+  String toString() => '(Playlist) $playlistTitle ($playlistId)';
+}

lib/src/search/search_query.dart

@@ -0,0 +1,44 @@
+import '../reverse_engineering/responses/search_page.dart';
+import '../reverse_engineering/youtube_http_client.dart';
+import 'related_query.dart';
+
+///
+class SearchQuery {
+  final YoutubeHttpClient _httpClient;
+
+  /// Search query
+  final String searchQuery;
+
+  final SearchPage _page;
+
+  /// Initializes a SearchQuery
+  SearchQuery(this._httpClient, this.searchQuery, this._page);
+
+  /// Search a video.
+  static Future<SearchQuery> search(
+      YoutubeHttpClient httpClient, String searchQuery) async {
+    var page = await SearchPage.get(httpClient, searchQuery);
+    return SearchQuery(httpClient, searchQuery, page);
+  }
+
+  /// Get the data of the next page.
+  Future<SearchQuery> nextPage() async {
+    var page = await _page.nextPage(_httpClient);
+    if (page == null) {
+      // TODO: Throw custom exception
+      throw Exception('Page limit reached!');
+    }
+    return SearchQuery(_httpClient, searchQuery, page);
+  }
+
+  /// Content of this search.
+  /// Contains either [SearchVideo] or [SearchPlaylist]
+  List<dynamic> get content => _page.initialData.searchContent;
+
+  /// Videos related to this search.
+  /// Contains either [SearchVideo] or [SearchPlaylist]
+  List<dynamic> get relatedVideos => _page.initialData.relatedVideos;
+
+  /// Returns the queries related to this search.
+  List<RelatedQuery> get relatedQueries => _page.initialData.relatedQueries;
+}

lib/src/search/search_video.dart

@@ -0,0 +1,29 @@
+import '../videos/video_id.dart';
+
+/// Metadata related to a search query result (video).
+class SearchVideo {
+  /// VideoId.
+  final VideoId videoId;
+
+  /// Video title.
+  final String videoTitle;
+
+  /// Video author.
+  final String videoAuthor;
+
+  /// Video description snippet. (Part of the full description if too long)
+  final String videoDescriptionSnippet;
+
+  /// Video duration as String, HH:MM:SS
+  final String videoDuration;
+
+  /// Video View Count
+  final int videoViewCount;
+
+  /// Initialize a [RelatedQuery] instance.
+  SearchVideo(this.videoId, this.videoTitle, this.videoAuthor,
+      this.videoDescriptionSnippet, this.videoDuration, this.videoViewCount);
+
+  @override
+  String toString() => '(Video) $videoTitle ($videoId)';
+}

lib/youtube_explode_dart.dart

@@ -4,6 +4,6 @@ export 'src/channels/channels.dart';
 export 'src/common/common.dart';
 export 'src/exceptions/exceptions.dart';
 export 'src/playlists/playlists.dart';
-export 'src/search/search_client.dart';
+export 'src/search/search.dart';
 export 'src/videos/videos.dart';
 export 'src/youtube_explode_base.dart';

test/search_test.dart

@@ -12,11 +12,18 @@ void main() {
       yt.close();
     });
 
-    test('SearchYouTubeVideos', () async {
+    test('SearchYouTubeVideosFromApi', () async {
       var videos = await yt.search
           .getVideosAsync('undead corporation megalomania')
           .toList();
       expect(videos, isNotEmpty);
     });
+    test('SearchYouTubeVideosFromPage', () async {
+      var searchQuery = await yt.search
+          .queryFromPage('hello');
+      expect(searchQuery.content, isNotEmpty);
+      expect(searchQuery.relatedVideos, isNotEmpty);
+      expect(searchQuery.relatedQueries, isNotEmpty);
+    });
   });
 }