More Types

CHANGELOG.md

@@ -3,10 +3,16 @@
 - Only throw custom exceptions from the library.
 - `getUploadsFromPage` no longer throws.
 
+## 1.5.1
+- BREAKING CHANGE: Renamed `getVideosAsync` to `getVideos`.
+- Implemented `getVideosFromPage` which supersedes `queryFromPage`.
+- Implemented JSON Classes for reverse engineer.
+- Added `forceWatchPage` to the video client to assure the fetching of the video page. (ATM useful only if using the comments api)
+- Remove adaptive streams. These are not used anymore. 
+ 
 ## 1.5.0
 - BREAKING CHANGE: Renamed `Container` class to `StreamContainer` to avoid conflicting with Flutter `Container`. See #66
 
-
 ## 1.4.4
 - Expose HttpClient in APIs
 - Fix #55: Typo in README.md

lib/src/reverse_engineering/responses/generated/player_response_json.g.dart

@@ -0,0 +1,237 @@
+// To parse this JSON data, do
+//
+//     final playerConfigJson = playerConfigJsonFromJson(jsonString);
+
+import 'dart:convert';
+
+class PlayerConfigJson {
+  PlayerConfigJson({
+    this.assets,
+    this.attrs,
+    this.args,
+  });
+
+  final Assets assets;
+  final Attrs attrs;
+  final Args args;
+
+  factory PlayerConfigJson.fromRawJson(String str) =>
+      PlayerConfigJson.fromJson(json.decode(str));
+
+  String toRawJson() => json.encode(toJson());
+
+  factory PlayerConfigJson.fromJson(Map<String, dynamic> json) =>
+      PlayerConfigJson(
+        assets: json["assets"] == null ? null : Assets.fromJson(json["assets"]),
+        attrs: json["attrs"] == null ? null : Attrs.fromJson(json["attrs"]),
+        args: json["args"] == null ? null : Args.fromJson(json["args"]),
+      );
+
+  Map<String, dynamic> toJson() => {
+        "assets": assets == null ? null : assets.toJson(),
+        "attrs": attrs == null ? null : attrs.toJson(),
+        "args": args == null ? null : args.toJson(),
+      };
+}
+
+class Args {
+  Args({
+    this.innertubeApiKey,
+    this.showMiniplayerButton,
+    this.useMiniplayerUi,
+    this.gapiHintParams,
+    this.playerResponse,
+    this.cbrver,
+    this.cbr,
+    this.innertubeApiVersion,
+    this.innertubeContextClientVersion,
+    this.vssHost,
+    this.hostLanguage,
+    this.cr,
+    this.externalFullscreen,
+    this.useFastSizingOnWatchDefault,
+    this.c,
+    this.ps,
+    this.csiPageType,
+    this.cos,
+    this.enablecsi,
+    this.watermark,
+    this.cver,
+    this.transparentBackground,
+    this.hl,
+    this.enablejsapi,
+    this.cosver,
+    this.loaderUrl,
+  });
+
+  final String innertubeApiKey;
+  final String showMiniplayerButton;
+  final String useMiniplayerUi;
+  final String gapiHintParams;
+  final String playerResponse;
+  final String cbrver;
+  final String cbr;
+  final String innertubeApiVersion;
+  final String innertubeContextClientVersion;
+  final String vssHost;
+  final String hostLanguage;
+  final String cr;
+  final bool externalFullscreen;
+  final bool useFastSizingOnWatchDefault;
+  final String c;
+  final String ps;
+  final String csiPageType;
+  final String cos;
+  final String enablecsi;
+  final String watermark;
+  final String cver;
+  final String transparentBackground;
+  final String hl;
+  final String enablejsapi;
+  final String cosver;
+  final String loaderUrl;
+
+  factory Args.fromRawJson(String str) => Args.fromJson(json.decode(str));
+
+  String toRawJson() => json.encode(toJson());
+
+  factory Args.fromJson(Map<String, dynamic> json) => Args(
+        innertubeApiKey: json["innertube_api_key"] == null
+            ? null
+            : json["innertube_api_key"],
+        showMiniplayerButton: json["show_miniplayer_button"] == null
+            ? null
+            : json["show_miniplayer_button"],
+        useMiniplayerUi: json["use_miniplayer_ui"] == null
+            ? null
+            : json["use_miniplayer_ui"],
+        gapiHintParams:
+            json["gapi_hint_params"] == null ? null : json["gapi_hint_params"],
+        playerResponse:
+            json["player_response"] == null ? null : json["player_response"],
+        cbrver: json["cbrver"] == null ? null : json["cbrver"],
+        cbr: json["cbr"] == null ? null : json["cbr"],
+        innertubeApiVersion: json["innertube_api_version"] == null
+            ? null
+            : json["innertube_api_version"],
+        innertubeContextClientVersion:
+            json["innertube_context_client_version"] == null
+                ? null
+                : json["innertube_context_client_version"],
+        vssHost: json["vss_host"] == null ? null : json["vss_host"],
+        hostLanguage:
+            json["host_language"] == null ? null : json["host_language"],
+        cr: json["cr"] == null ? null : json["cr"],
+        externalFullscreen: json["external_fullscreen"] == null
+            ? null
+            : json["external_fullscreen"],
+        useFastSizingOnWatchDefault:
+            json["use_fast_sizing_on_watch_default"] == null
+                ? null
+                : json["use_fast_sizing_on_watch_default"],
+        c: json["c"] == null ? null : json["c"],
+        ps: json["ps"] == null ? null : json["ps"],
+        csiPageType:
+            json["csi_page_type"] == null ? null : json["csi_page_type"],
+        cos: json["cos"] == null ? null : json["cos"],
+        enablecsi: json["enablecsi"] == null ? null : json["enablecsi"],
+        watermark: json["watermark"] == null ? null : json["watermark"],
+        cver: json["cver"] == null ? null : json["cver"],
+        transparentBackground: json["transparent_background"] == null
+            ? null
+            : json["transparent_background"],
+        hl: json["hl"] == null ? null : json["hl"],
+        enablejsapi: json["enablejsapi"] == null ? null : json["enablejsapi"],
+        cosver: json["cosver"] == null ? null : json["cosver"],
+        loaderUrl: json["loaderUrl"] == null ? null : json["loaderUrl"],
+      );
+
+  Map<String, dynamic> toJson() => {
+        "innertube_api_key": innertubeApiKey == null ? null : innertubeApiKey,
+        "show_miniplayer_button":
+            showMiniplayerButton == null ? null : showMiniplayerButton,
+        "use_miniplayer_ui": useMiniplayerUi == null ? null : useMiniplayerUi,
+        "gapi_hint_params": gapiHintParams == null ? null : gapiHintParams,
+        "player_response": playerResponse == null ? null : playerResponse,
+        "cbrver": cbrver == null ? null : cbrver,
+        "cbr": cbr == null ? null : cbr,
+        "innertube_api_version":
+            innertubeApiVersion == null ? null : innertubeApiVersion,
+        "innertube_context_client_version":
+            innertubeContextClientVersion == null
+                ? null
+                : innertubeContextClientVersion,
+        "vss_host": vssHost == null ? null : vssHost,
+        "host_language": hostLanguage == null ? null : hostLanguage,
+        "cr": cr == null ? null : cr,
+        "external_fullscreen":
+            externalFullscreen == null ? null : externalFullscreen,
+        "use_fast_sizing_on_watch_default": useFastSizingOnWatchDefault == null
+            ? null
+            : useFastSizingOnWatchDefault,
+        "c": c == null ? null : c,
+        "ps": ps == null ? null : ps,
+        "csi_page_type": csiPageType == null ? null : csiPageType,
+        "cos": cos == null ? null : cos,
+        "enablecsi": enablecsi == null ? null : enablecsi,
+        "watermark": watermark == null ? null : watermark,
+        "cver": cver == null ? null : cver,
+        "transparent_background":
+            transparentBackground == null ? null : transparentBackground,
+        "hl": hl == null ? null : hl,
+        "enablejsapi": enablejsapi == null ? null : enablejsapi,
+        "cosver": cosver == null ? null : cosver,
+        "loaderUrl": loaderUrl == null ? null : loaderUrl,
+      };
+}
+
+class Assets {
+  Assets({
+    this.playerCanaryState,
+    this.js,
+    this.css,
+  });
+
+  final String playerCanaryState;
+  final String js;
+  final String css;
+
+  factory Assets.fromRawJson(String str) => Assets.fromJson(json.decode(str));
+
+  String toRawJson() => json.encode(toJson());
+
+  factory Assets.fromJson(Map<String, dynamic> json) => Assets(
+        playerCanaryState: json["player_canary_state"] == null
+            ? null
+            : json["player_canary_state"],
+        js: json["js"] == null ? null : json["js"],
+        css: json["css"] == null ? null : json["css"],
+      );
+
+  Map<String, dynamic> toJson() => {
+        "player_canary_state":
+            playerCanaryState == null ? null : playerCanaryState,
+        "js": js == null ? null : js,
+        "css": css == null ? null : css,
+      };
+}
+
+class Attrs {
+  Attrs({
+    this.id,
+  });
+
+  final String id;
+
+  factory Attrs.fromRawJson(String str) => Attrs.fromJson(json.decode(str));
+
+  String toRawJson() => json.encode(toJson());
+
+  factory Attrs.fromJson(Map<String, dynamic> json) => Attrs(
+        id: json["id"] == null ? null : json["id"],
+      );
+
+  Map<String, dynamic> toJson() => {
+        "id": id == null ? null : id,
+      };
+}

lib/src/reverse_engineering/responses/generated/playlist_response.g.dart

@@ -85,7 +85,7 @@ class Video {
   final int timeCreated;
   final bool ccLicense;
   final String title;
-  final int rating;
+  final num rating;
   final bool isHd;
   final Privacy privacy;
   final int lengthSeconds;

lib/src/reverse_engineering/responses/playlist_response.dart

@@ -37,12 +37,13 @@ class PlaylistResponse {
   int get dislikeCount => _root.dislikes;
 
   ///
-  List<_Video> get videos => _videos ??= _root.video.map((e) => _Video(e));
+  List<_Video> get videos =>
+      _videos ??= _root.video.map((e) => _Video(e)).toList();
 
   ///
   PlaylistResponse.parse(String raw) {
     final t = json.tryDecode(raw);
-    if (_root == null) {
+    if (t == null) {
       throw TransientFailureException('Playerlist response is broken.');
     }
     _root = PlaylistResponseJson.fromJson(t);

lib/src/reverse_engineering/responses/search_page.dart

@@ -2,31 +2,40 @@ import 'dart:convert';
 
 import 'package:html/dom.dart';
 import 'package:html/parser.dart' as parser;
+import 'package:youtube_explode_dart/src/search/base_search_content.dart';
 
 import '../../../youtube_explode_dart.dart';
 import '../../extensions/helpers_extension.dart';
-import '../../playlists/playlist_id.dart';
 import '../../retry.dart';
 import '../../search/related_query.dart';
-import '../../search/search_playlist.dart';
 import '../../search/search_video.dart';
 import '../../videos/videos.dart';
 import '../youtube_http_client.dart';
+import 'generated/search_page_id.g.dart' hide PlaylistId;
 
 ///
 class SearchPage {
-  static final _xsfrTokenExp = RegExp('"XSRF_TOKEN":"(.+?)"');
+  final _apiKeyExp = RegExp(r'"INNERTUBE_API_KEY":"(\w+?)"');
 
   ///
   final String queryString;
   final Document _root;
 
+  String _apiKey;
+
+  ///
+  String get apiKey => _apiKey ??= _apiKeyExp
+      .firstMatch(_root
+          .querySelectorAll('script')
+          .firstWhere((e) => e.text.contains('INNERTUBE_API_KEY'))
+          .text)
+      .group(1);
+
   _InitialData _initialData;
-  String _xsrfToken;
 
   ///
   _InitialData get initialData =>
-      _initialData ??= _InitialData(json.decode(_extractJson(
+      _initialData ??= _InitialData(SearchPageId.fromRawJson(_extractJson(
           _root
               .querySelectorAll('script')
               .map((e) => e.text)
@@ -34,14 +43,6 @@ class SearchPage {
               .firstWhere((e) => e.contains('window["ytInitialData"] =')),
           'window["ytInitialData"] =')));
 
-  ///
-  String get xsfrToken => _xsrfToken ??= _xsfrTokenExp
-      .firstMatch(_root
-          .querySelectorAll('script')
-          .firstWhere((e) => _xsfrTokenExp.hasMatch(e.text))
-          .text)
-      .group(1);
-
   String _extractJson(String html, String separator) {
     return _matchJson(
         html.substring(html.indexOf(separator) + separator.length));
@@ -67,47 +68,54 @@ class SearchPage {
 
   ///
   SearchPage(this._root, this.queryString,
-      [_InitialData initalData, String xsfrToken])
-      : _initialData = initalData,
-        _xsrfToken = xsfrToken;
+      [_InitialData initialData, this._apiKey])
+      : _initialData = initialData;
 
   ///
   // TODO: Replace this in favour of async* when quering;
   Future<SearchPage> nextPage(YoutubeHttpClient httpClient) async {
-    if (initialData.continuation == '') {
+    if (initialData.continuationToken == '' ||
+        initialData.estimatedResults == 0) {
       return null;
     }
     return get(httpClient, queryString,
-        ctoken: initialData.continuation,
-        itct: initialData.clickTrackingParams,
-        xsrfToken: xsfrToken);
+        token: initialData.continuationToken, key: apiKey);
   }
 
   ///
   static Future<SearchPage> get(
       YoutubeHttpClient httpClient, String queryString,
-      {String ctoken, String itct, String xsrfToken}) {
+      {String token, String key}) {
+    if (token != null) {
+      assert(key != null, 'A key must be supplied along with a token');
+      var url = 'https://www.youtube.com/youtubei/v1/search?key=$key';
+
+      return retry(() async {
+        var body = {
+          'context': const {
+            'client': {
+              'hl': 'en',
+              'clientName': 'WEB',
+              'clientVersion': '2.20200911.04.00'
+            }
+          },
+          'continuation': token
+        };
+
+        var raw = await httpClient.post(url, body: json.encode(body));
+        return SearchPage(null, queryString,
+            _InitialData(SearchPageId.fromJson(json.decode(raw.body))), key);
+      });
+      // Ask for next page,
+
+    }
     var url =
         'https://www.youtube.com/results?search_query=${Uri.encodeQueryComponent(queryString)}';
-    if (ctoken != null) {
-      assert(itct != null, 'If ctoken is not null itct cannot be null');
-      url += '&pbj=1';
-      url += '&ctoken=${Uri.encodeQueryComponent(ctoken)}';
-      url += '&continuation=${Uri.encodeQueryComponent(ctoken)}';
-      url += '&itct=${Uri.encodeQueryComponent(itct)}';
-    }
     return retry(() async {
-      Map<String, String> body;
-      if (xsrfToken != null) {
-        body = {'session_token': xsrfToken};
-      }
-      var raw = await httpClient.postString(url, body: body);
-      if (ctoken != null) {
-        return SearchPage(
-            null, queryString, _InitialData(json.decode(raw)[1]), xsrfToken);
-      }
+      var raw = await httpClient.getString(url);
       return SearchPage.parse(raw, queryString);
     });
+    // ask for next page
   }
 
   ///
@@ -116,115 +124,113 @@ class SearchPage {
 
 class _InitialData {
   // Json parsed map
-  final Map<String, dynamic> _root;
+  final SearchPageId root;
 
-  _InitialData(this._root);
+  _InitialData(this.root);
 
   /* Cache results */
 
   List<dynamic> _searchContent;
   List<dynamic> _relatedVideos;
   List<RelatedQuery> _relatedQueries;
-  String _continuation;
-  String _clickTrackingParams;
 
-  List<Map<String, dynamic>> getContentContext(Map<String, dynamic> root) {
-    if (root['contents'] != null) {
-      return _root['contents']['twoColumnSearchResultsRenderer']
-              ['primaryContents']['sectionListRenderer']['contents']
-          .first['itemSectionRenderer']['contents']
-          .cast<Map<String, dynamic>>();
+  List<PurpleContent> getContentContext() {
+    if (root.contents != null) {
+      return root.contents.twoColumnSearchResultsRenderer.primaryContents
+          .sectionListRenderer.contents.first.itemSectionRenderer.contents;
     }
-    if (root['response'] != null) {
-      return _root['response']['continuationContents']
-              ['itemSectionContinuation']['contents']
-          .cast<Map<String, dynamic>>();
+    if (root.onResponseReceivedCommands != null) {
+      return root.onResponseReceivedCommands.first.appendContinuationItemsAction
+          .continuationItems[0].itemSectionRenderer.contents;
     }
     throw FatalFailureException('Failed to get initial data context.');
   }
 
-  Map<String, dynamic> getContinuationContext(Map<String, dynamic> root) {
-    if (_root['contents'] != null) {
-      return (_root['contents']['twoColumnSearchResultsRenderer']
-                  ['primaryContents']['sectionListRenderer']['contents']
-              ?.first['itemSectionRenderer']['continuations']
-              ?.first as Map)
-          ?.getValue('nextContinuationData')
-          ?.cast<String, dynamic>();
+  String _getContinuationToken() {
+    if (root.contents != null) {
+      var contents = root.contents.twoColumnSearchResultsRenderer
+          .primaryContents.sectionListRenderer.contents;
+
+      if (contents.length <= 1) {
+        return null;
+      }
+      return contents[1]
+          .continuationItemRenderer
+          .continuationEndpoint
+          .continuationCommand
+          .token;
     }
-    if (_root['response'] != null) {
-      return _root['response']['continuationContents']
-              ['itemSectionContinuation']['continuations']
-          ?.first['nextContinuationData']
-          ?.cast<String, dynamic>();
+    if (root.onResponseReceivedCommands != null) {
+      return root
+              .onResponseReceivedCommands
+              .first
+              .appendContinuationItemsAction
+              .continuationItems[1]
+              ?.continuationItemRenderer
+              ?.continuationEndpoint
+              ?.continuationCommand
+              ?.token ??
+          ' ';
     }
     return null;
   }
 
   // Contains only [SearchVideo] or [SearchPlaylist]
-  List<dynamic> get searchContent => _searchContent ??= getContentContext(_root)
-      .map(_parseContent)
-      .where((e) => e != null)
-      .toList();
+  List<BaseSearchContent> get searchContent => _searchContent ??=
+      getContentContext().map(_parseContent).where((e) => e != null).toList();
 
   List<RelatedQuery> get relatedQueries =>
-      (_relatedQueries ??= getContentContext(_root)
-          ?.where((e) => e.containsKey('horizontalCardListRenderer'))
-          ?.map((e) => e['horizontalCardListRenderer']['cards'])
+      (_relatedQueries ??= getContentContext()
+          ?.where((e) => e.horizontalCardListRenderer != null)
+          ?.map((e) => e.horizontalCardListRenderer.cards)
           ?.firstOrNull
-          ?.map((e) => e['searchRefinementCardRenderer'])
+          ?.map((e) => e.searchRefinementCardRenderer)
           ?.map((e) => RelatedQuery(
-              e['searchEndpoint']['searchEndpoint']['query'],
-              VideoId(Uri.parse(e['thumbnail']['thumbnails'].first['url'])
-                  .pathSegments[1])))
+              e.searchEndpoint.searchEndpoint.query,
+              VideoId(
+                  Uri.parse(e.thumbnail.thumbnails.first.url).pathSegments[1])))
           ?.toList()
           ?.cast<RelatedQuery>()) ??
       const [];
 
   List<dynamic> get relatedVideos =>
-      (_relatedVideos ??= getContentContext(_root)
-          ?.where((e) => e.containsKey('shelfRenderer'))
-          ?.map((e) =>
-              e['shelfRenderer']['content']['verticalListRenderer']['items'])
+      (_relatedVideos ??= getContentContext()
+          ?.where((e) => e.shelfRenderer != null)
+          ?.map((e) => e.shelfRenderer.content.verticalListRenderer.items)
           ?.firstOrNull
           ?.map(_parseContent)
           ?.toList()) ??
       const [];
 
-  String get continuation => _continuation ??=
-      getContinuationContext(_root)?.getValue('continuation') ?? '';
+  String get continuationToken => _getContinuationToken();
 
-  String get clickTrackingParams => _clickTrackingParams ??=
-      getContinuationContext(_root)?.getValue('clickTrackingParams') ?? '';
+  int get estimatedResults => int.parse(root.estimatedResults ?? 0);
 
-  int get estimatedResults => int.parse(_root['estimatedResults'] ?? 0);
-
-  dynamic _parseContent(dynamic content) {
+  BaseSearchContent _parseContent(PurpleContent content) {
     if (content == null) {
       return null;
     }
-    if (content.containsKey('videoRenderer')) {
-      Map<String, dynamic> renderer = content['videoRenderer'];
+    if (content.videoRenderer != null) {
+      var renderer = content.videoRenderer;
       //TODO: Add if it's a live
       return SearchVideo(
-          VideoId(renderer['videoId']),
-          _parseRuns(renderer['title']),
-          _parseRuns(renderer['ownerText']),
-          _parseRuns(renderer['descriptionSnippet']),
-          renderer.get('lengthText')?.getValue('simpleText') ?? '',
-          int.parse(renderer['viewCountText']['simpleText']
-                  .toString()
-                  .stripNonDigits()
-                  .nullIfWhitespace ??
+          VideoId(renderer.videoId),
+          _parseRuns(renderer.title.runs),
+          _parseRuns(renderer.ownerText.runs),
+          _parseRuns(renderer.descriptionSnippet?.runs),
+          renderer.lengthText?.simpleText ?? '',
+          int.parse(renderer.viewCountText?.simpleText
+                  ?.stripNonDigits()
+                  ?.nullIfWhitespace ??
               '0'));
     }
-    if (content.containsKey('radioRenderer')) {
-      var renderer = content['radioRenderer'];
+    if (content.radioRenderer != null) {
+      var renderer = content.radioRenderer;
 
       return SearchPlaylist(
-          PlaylistId(renderer['playlistId']),
-          renderer['title']['simpleText'],
-          int.parse(_parseRuns(renderer['videoCountText'])
+          PlaylistId(renderer.playlistId),
+          renderer.title.simpleText,
+          int.parse(_parseRuns(renderer.videoCountText.runs)
                   .stripNonDigits()
                   .nullIfWhitespace ??
               0));
@@ -233,38 +239,6 @@ class _InitialData {
     return null;
   }
 
-  String _parseRuns(Map<dynamic, dynamic> runs) =>
-      runs?.getValue('runs')?.map((e) => e['text'])?.join() ?? '';
+  String _parseRuns(List<dynamic> runs) =>
+      runs?.map((e) => e.text)?.join() ?? '';
 }
-
-// ['contents']['twoColumnSearchResultsRenderer']['primaryContents']
-// ['sectionListRenderer']['contents'].first['itemSectionRenderer']
-//
-//
-// ['contents'] -> @See ContentsList
-// ['continuations'] -> Data to see more
-
-//ContentsList:
-// Key -> 'videoRenderer'
-//    videoId --> VideoId
-//    title['runs'].loop -> ['text'] -> concatenate --> "Video Title"
-//    descriptionSnippet['runs'].loop -> ['text'] -> concatenate
-//      --> "Video Description snippet"
-//    ownerText['runs'].first -> ['text'] --> "Video Author"
-//    lengthText['simpleText'] -> Parse format H:M:S -> "Video Duration"
-//    viewCountText['simpleText'] -> Strip non digit -> int.parse
-//      --> "Video View Count"
-//
-// Key -> 'radioRenderer'
-//    playlistId -> PlaylistId
-//    title['simpleText'] --> "Playlist Title"
-//
-// Key -> 'horizontalCardListRenderer' // Queries related to this search
-//    cards --> List of Maps -> loop -> ['searchRefinementCardRenderer'].first
-//      thumbnail -> ['thumbnails'].first -> ['url']
-//        --> "Thumbnail url" -> Find video id from id.
-//      searchEndpoint -> ['searchEndpoint']
-//        -> ['query'] -> "Related query string"
-//
-// Key -> 'shelfRenderer' // Videos related to this search
-//  contents -> ['verticalListRenderer']['items'] -> loop -> parseContent

lib/src/reverse_engineering/responses/watch_page.dart

@@ -1,16 +1,14 @@
-import 'dart:convert';
-
 import 'package:html/dom.dart';
 import 'package:html/parser.dart' as parser;
-import 'package:http_parser/http_parser.dart';
 
 import '../../../youtube_explode_dart.dart';
 import '../../extensions/helpers_extension.dart';
 import '../../retry.dart';
 import '../../videos/video_id.dart';
 import '../youtube_http_client.dart';
+import 'generated/player_response_json.g.dart';
+import 'generated/watch_page_id.g.dart';
 import 'player_response.dart';
-import 'stream_info_provider.dart';
 
 ///
 class WatchPage {
@@ -37,13 +35,13 @@ class WatchPage {
 
   ///
   _InitialData get initialData =>
-      _initialData ??= _InitialData(json.decode(_matchJson(_extractJson(
+      _initialData ??= _InitialData(WatchPageId.fromRawJson(_extractJson(
           _root
               .querySelectorAll('script')
               .map((e) => e.text)
               .toList()
               .firstWhere((e) => e.contains('window["ytInitialData"] =')),
-          'window["ytInitialData"] ='))));
+          'window["ytInitialData"] =')));
 
   ///
   String get xsfrToken => _xsfrToken ??= _xsfrTokenExp
@@ -88,9 +86,9 @@ class WatchPage {
 
   ///
   _PlayerConfig get playerConfig =>
-      _playerConfig ??= _PlayerConfig(json.decode(_matchJson(_extractJson(
+      _playerConfig ??= _PlayerConfig(PlayerConfigJson.fromRawJson(_extractJson(
           _root.getElementsByTagName('html').first.text,
-          'ytplayer.config = '))));
+          'ytplayer.config = ')));
 
   String _extractJson(String html, String separator) {
     return _matchJson(
@@ -145,99 +143,21 @@ class WatchPage {
   }
 }
 
-class _StreamInfo extends StreamInfoProvider {
-  final Map<String, String> _root;
-
-  _StreamInfo(this._root);
-
-  @override
-  int get bitrate => int.parse(_root['bitrate']);
-
-  @override
-  int get tag => int.parse(_root['itag']);
-
-  @override
-  String get url => _root['url'];
-
-  @override
-  String get signature => _root['s'];
-
-  @override
-  String get signatureParameter => _root['sp'];
-
-  @override
-  int get contentLength => int.tryParse(_root['clen'] ??
-      StreamInfoProvider.contentLenExp
-          .firstMatch(url)
-          .group(1)
-          .nullIfWhitespace ??
-      '');
-
-  MediaType get mimeType => MediaType.parse(_root['mimeType']);
-
-  @override
-  String get container => mimeType.subtype;
-
-  bool get isAudioOnly => mimeType.type == 'audio';
-
-  @override
-  String get audioCodec => codecs.last;
-
-  @override
-  String get videoCodec => isAudioOnly ? null : codecs.first;
-
-  List<String> get codecs =>
-      mimeType.parameters['codecs'].split(',').map((e) => e.trim());
-
-  @override
-  String get videoQualityLabel => _root['quality_label'];
-
-  List<int> get _size =>
-      _root['size'].split(',').map((e) => int.tryParse(e ?? ''));
-
-  @override
-  int get videoWidth => _size.first;
-
-  @override
-  int get videoHeight => _size.last;
-
-  @override
-  int get framerate => int.tryParse(_root['fps'] ?? '');
-}
-
 class _PlayerConfig {
   // Json parsed map
-  final Map<String, dynamic> _root;
+  final PlayerConfigJson root;
 
-  _PlayerConfig(this._root);
+  _PlayerConfig(this.root);
 
-  String get sourceUrl => 'https://youtube.com${_root['assets']['js']}';
+  String get sourceUrl => 'https://youtube.com${root.assets.js}';
 
   PlayerResponse get playerResponse =>
-      PlayerResponse.parse(_root['args']['player_response']);
-
-  List<_StreamInfo> get muxedStreams =>
-      _root
-          .get('args')
-          ?.getValue('url_encoded_fmt_stream_map')
-          ?.split(',')
-          ?.map((e) => _StreamInfo(Uri.splitQueryString(e))) ??
-      const [];
-
-  List<_StreamInfo> get adaptiveStreams =>
-      _root
-          .get('args')
-          ?.getValue('adaptive_fmts')
-          ?.split(',')
-          ?.map((e) => _StreamInfo(Uri.splitQueryString(e))) ??
-      const [];
-
-  List<_StreamInfo> get streams => [...muxedStreams, ...adaptiveStreams];
+      PlayerResponse.parse(root.args.playerResponse);
 }
 
 class _InitialData {
   // Json parsed map
-  final Map<String, dynamic> root;
+  final WatchPageId root;
 
   _InitialData(this.root);
 
@@ -246,26 +166,21 @@ class _InitialData {
   String _continuation;
   String _clickTrackingParams;
 
-  Map<String, dynamic> getContinuationContext(Map<String, dynamic> root) {
-    if (root['contents'] != null) {
-      return (root['contents']['twoColumnWatchNextResults']['results']
-              ['results']['contents'] as List<dynamic>)
-          ?.firstWhere((e) => e.containsKey('itemSectionRenderer'))[
-              'itemSectionRenderer']['continuations']
-          ?.first['nextContinuationData']
-          ?.cast<String, dynamic>();
-    }
-    if (root['response'] != null) {
-      return root['response']['itemSectionContinuation']['continuations']
-          ?.first['nextContinuationData']
-          ?.cast<String, dynamic>();
+  NextContinuationData getContinuationContext() {
+    if (root.contents != null) {
+      return root.contents.twoColumnWatchNextResults.results.results.contents
+          .firstWhere((e) => e.itemSectionRenderer != null)
+          .itemSectionRenderer
+          .continuations
+          .first
+          .nextContinuationData;
     }
     return null;
   }
 
-  String get continuation => _continuation ??=
-      getContinuationContext(root)?.getValue('continuation') ?? '';
+  String get continuation =>
+      _continuation ??= getContinuationContext()?.continuation ?? '';
 
   String get clickTrackingParams => _clickTrackingParams ??=
-      getContinuationContext(root)?.getValue('clickTrackingParams') ?? '';
+      getContinuationContext()?.clickTrackingParams ?? '';
 }

lib/src/search/base_search_content.dart

@@ -0,0 +1,5 @@
+///
+abstract class BaseSearchContent {
+  ///
+  const BaseSearchContent();
+}

lib/src/search/related_query.dart

@@ -1,7 +1,9 @@
+import 'package:equatable/equatable.dart';
+
 import '../videos/video_id.dart';
 
 ///
-class RelatedQuery {
+class RelatedQuery with EquatableMixin {
   /// Query related to a search query.
   final String query;
 
@@ -10,4 +12,10 @@ class RelatedQuery {
 
   /// Initialize a [RelatedQuery] instance.
   RelatedQuery(this.query, this.videoId);
+
+  @override
+  String toString() => 'RelatedQuery($videoId): $query';
+
+  @override
+  List<Object> get props => [query, videoId];
 }

lib/src/search/search_client.dart

@@ -1,8 +1,11 @@
+import 'package:youtube_explode_dart/src/reverse_engineering/responses/search_page.dart';
+
 import '../common/common.dart';
 import '../reverse_engineering/responses/playlist_response.dart';
 import '../reverse_engineering/youtube_http_client.dart';
 import '../videos/video.dart';
 import '../videos/video_id.dart';
+import 'base_search_content.dart';
 import 'search_query.dart';
 
 /// YouTube search queries.
@@ -13,7 +16,8 @@ class SearchClient {
   SearchClient(this._httpClient);
 
   /// Enumerates videos returned by the specified search query.
-  Stream<Video> getVideosAsync(String searchQuery) async* {
+  /// (from the YouTube Embedded API)
+  Stream<Video> getVideos(String searchQuery) async* {
     var encounteredVideoIds = <String>{};
 
     for (var page = 0; page < double.maxFinite; page++) {
@@ -49,7 +53,41 @@ class SearchClient {
     }
   }
 
+  /// Enumerates videos returned by the specified search query
+  /// (from the video search page).
+  /// Contains only instances of [SearchVideo] or [SearchPlaylist]
+  Stream<BaseSearchContent> getVideosFromPage(String searchQuery) async* {
+    var page = await SearchPage.get(_httpClient, searchQuery);
+    yield* Stream.fromIterable(page.initialData.searchContent);
+
+    // ignore: literal_only_boolean_expressions
+    while (true) {
+      page = await page.nextPage(_httpClient);
+      if (page == null) {
+        return;
+      }
+      yield* Stream.fromIterable(page.initialData.searchContent);
+    }
+  }
+
   /// Queries to YouTube to get the results.
+  @Deprecated('Use getVideosFromPage instead')
   Future<SearchQuery> queryFromPage(String searchQuery) =>
       SearchQuery.search(_httpClient, searchQuery);
 }
+
+/*
+    channelId = ChannelId.fromString(channelId);
+    var page = await ChannelUploadPage.get(
+        _httpClient, channelId.value, videoSorting.code);
+    yield* Stream.fromIterable(page.initialData.uploads);
+
+    // ignore: literal_only_boolean_expressions
+    while (true) {
+      page = await page.nextPage(_httpClient);
+      if (page == null) {
+        return;
+      }
+      yield* Stream.fromIterable(page.initialData.uploads);
+    }
+ */

lib/src/search/search_playlist.dart

@@ -1,9 +1,10 @@
 import 'package:equatable/equatable.dart';
 
 import '../playlists/playlist_id.dart';
+import 'base_search_content.dart';
 
 /// Metadata related to a search query result (playlist)
-class SearchPlaylist with EquatableMixin {
+class SearchPlaylist extends BaseSearchContent with EquatableMixin {
   /// PlaylistId.
   final PlaylistId playlistId;
 
@@ -17,7 +18,7 @@ class SearchPlaylist with EquatableMixin {
   SearchPlaylist(this.playlistId, this.playlistTitle, this.playlistVideoCount);
 
   @override
-  String toString() => '(Playlist) $playlistTitle ($playlistId)';
+  String toString() => '[Playlist] $playlistTitle ($playlistId)';
 
   @override
   List<Object> get props => [playlistId];

lib/src/search/search_video.dart

@@ -1,7 +1,8 @@
 import '../videos/video_id.dart';
+import 'base_search_content.dart';
 
 /// Metadata related to a search query result (video).
-class SearchVideo {
+class SearchVideo extends BaseSearchContent {
   /// VideoId.
   final VideoId videoId;
 

lib/src/videos/comments/comments_client.dart

@@ -45,6 +45,8 @@ class CommentsClient {
   /// the results.
   ///
   /// The streams doesn't emit any data if [Video.hasWatchPage] is false.
+  /// Use `videos.get(videoId, forceWatchPage: true)` to assure that the
+  /// WatchPage is fetched.
   Stream<Comment> getComments(Video video) async* {
     if (video.watchPage == null) {
       return;

lib/src/videos/streams/streams_client.dart

@@ -104,10 +104,7 @@ class StreamsClient {
       throw VideoUnplayableException.liveStream(videoId);
     }
 
-    var streamInfoProviders = <StreamInfoProvider>[
-      ...playerConfig.streams,
-      ...playerResponse.streams
-    ];
+    var streamInfoProviders = <StreamInfoProvider>[...playerResponse.streams];
 
     var dashManifestUrl = playerResponse.dashManifestUrl;
     if (!dashManifestUrl.isNullOrWhiteSpace) {

lib/src/videos/video_client.dart

@@ -71,9 +71,13 @@ class VideoClient {
   }
 
   /// Get a [Video] instance from a [videoId]
-  Future<Video> get(dynamic videoId) async {
+  Future<Video> get(dynamic videoId, {forceWatchPage = false}) async {
     videoId = VideoId.fromString(videoId);
 
+    if (forceWatchPage) {
+      return _getVideoFromWatchPage(videoId);
+    }
+
     try {
       return await _getVideoFromFixPlaylist(videoId);
     } on YoutubeExplodeException {

test/closed_caption_test.dart

@@ -13,11 +13,11 @@ void main() {
     });
 
     test('GetClosedCaptionTracksOfAnyVideo', () async {
-      var manifest = await yt.videos.closedCaptions.getManifest('_QdPW8JrYzQ');
+      var manifest = await yt.videos.closedCaptions.getManifest('WOxr2dmLHLo');
       expect(manifest.tracks, isNotEmpty);
     });
     test('GetClosedCaptionTrackOfAnyVideoSpecific', () async {
-      var manifest = await yt.videos.closedCaptions.getManifest('_QdPW8JrYzQ');
+      var manifest = await yt.videos.closedCaptions.getManifest('WOxr2dmLHLo');
       var trackInfo = manifest.tracks.first;
       var track = await yt.videos.closedCaptions.get(trackInfo);
 
@@ -25,7 +25,7 @@ void main() {
     });
     test('GetClosedCaptionTrackAtSpecificTime', () async {
       var manifest = await yt.videos.closedCaptions
-          .getManifest('https://www.youtube.com/watch?v=YltHGKX80Y8');
+          .getManifest('https://www.youtube.com/watch?v=ppJy5uGZLi4');
       var trackInfo = manifest.getByLanguage('en');
       var track = await yt.videos.closedCaptions.get(trackInfo);
       var caption =
@@ -35,8 +35,8 @@ void main() {
 
       expect(caption, isNotNull);
       expect(captionPart, isNotNull);
-      expect(caption.text, 'know I worked really hard on not doing');
-      expect(captionPart.text, ' hard');
+      expect(caption.text, 'how about this black there are some');
+      expect(captionPart.text, ' about');
     });
   });
 }

test/comments_client_test.dart

@@ -14,7 +14,7 @@ void main() {
 
     test('GetCommentOfVideo', () async {
       var videoUrl = 'https://www.youtube.com/watch?v=AI7ULzgf8RU';
-      var video = await yt.videos.get(VideoId(videoUrl));
+      var video = await yt.videos.get(VideoId(videoUrl), forceWatchPage: true);
       var comments = await yt.videos.commentsClient.getComments(video).toList();
       expect(comments.length, greaterThanOrEqualTo(1));
     }, skip: 'This may fail on some environments');

test/search_test.dart

@@ -13,9 +13,8 @@ void main() {
     });
 
     test('SearchYouTubeVideosFromApi', () async {
-      var videos = await yt.search
-          .getVideosAsync('undead corporation megalomania')
-          .toList();
+      var videos =
+          await yt.search.getVideos('undead corporation megalomania').toList();
       expect(videos, isNotEmpty);
     }, skip: 'Endpoint removed from YouTube');
 
@@ -36,5 +35,10 @@ void main() {
       var nextPage = await query.nextPage();
       expect(nextPage, isNull);
     });
+
+    test('SearchYouTubeVideosFromPageStream', () async {
+      var query = await yt.search.getVideosFromPage('hello').take(30).toList();
+      expect(query, hasLength(30));
+    });
   });
 }