Version 1.8.0-beta.3

#100

CHANGELOG.md

@@ -1,3 +1,6 @@
+## 1.8.0-beta.3
+- Fixed playlists
+
 ## 1.8.0-beta.2
 - `search.getVideos` now returns a `Video` instance.
 

lib/src/extensions/helpers_extension.dart

@@ -4,6 +4,10 @@ import '../reverse_engineering/cipher/cipher_operations.dart';
 
 /// Utility for Strings.
 extension StringUtility on String {
+  /// Parses this value as int stripping the non digit characters,
+  /// returns null if this fails.
+  int parseInt() => int.tryParse(this?.stripNonDigits());
+
   /// Returns null if this string is whitespace.
   String get nullIfWhitespace => trim().isEmpty ? null : this;
 
@@ -68,7 +72,7 @@ extension ListDecipher on Iterable<CipherOperation> {
 }
 
 /// List Utility.
-extension ListFirst<E> on Iterable<E> {
+extension ListUtil<E> on Iterable<E> {
   /// Returns the first element of a list or null if empty.
   E get firstOrNull {
     if (length == 0) {
@@ -76,6 +80,15 @@ extension ListFirst<E> on Iterable<E> {
     }
     return first;
   }
+
+  /// Same as [elementAt] but if the index is higher than the length returns
+  /// null
+  E elementAtSafe(int index) {
+    if (index >= length) {
+      return null;
+    }
+    return elementAt(index);
+  }
 }
 
 /// Uri utility
@@ -112,6 +125,32 @@ extension GetOrNullMap on Map {
     }
     return v;
   }
+
+  /// Get a value inside a map.
+  /// If it is null this returns null, if of another type this throws.
+  T getT<T>(String key) {
+    var v = this[key];
+    if (v == null) {
+      return null;
+    }
+    if (v is! T) {
+      throw Exception('Invalid type: ${v.runtimeType} should be $T');
+    }
+    return v;
+  }
+
+  /// Get a List<Map<String, dynamic>>> from a map.
+  List<Map<String, dynamic>> getList(String key) {
+    var v = this[key];
+    if (v == null) {
+      return null;
+    }
+    if (v is! List<dynamic>) {
+      throw Exception('Invalid type: ${v.runtimeType} should be of type List');
+    }
+
+    return (v.toList() as List<dynamic>).cast<Map<String, dynamic>>();
+  }
 }
 
 ///
@@ -124,3 +163,8 @@ extension UriUtils on Uri {
     return replace(queryParameters: query);
   }
 }
+
+/// Parse properties with `runs` method.
+extension RunsParser on List<dynamic> {
+  String parseRuns() => this?.map((e) => e['text'])?.join() ?? '';
+}

lib/src/playlists/playlist_client.dart

@@ -1,5 +1,7 @@
+import 'package:youtube_explode_dart/src/channels/channel_id.dart';
+import 'package:youtube_explode_dart/src/reverse_engineering/responses/playlist_page.dart';
+
 import '../common/common.dart';
-import '../reverse_engineering/responses/responses.dart';
 import '../reverse_engineering/youtube_http_client.dart';
 import '../videos/video.dart';
 import '../videos/video_id.dart';
@@ -17,28 +19,28 @@ class PlaylistClient {
   Future<Playlist> get(dynamic id) async {
     id = PlaylistId.fromString(id);
 
-    var response = await PlaylistResponse.get(_httpClient, id.value);
+    var response = await PlaylistPage.get(_httpClient, id.value);
     return Playlist(
         id,
-        response.title,
-        response.author,
-        response.description ?? '',
-        response.thumbnails,
-        Engagement(response.viewCount ?? 0, response.likeCount ?? 0,
-            response.dislikeCount ?? 0));
+        response.initialData.title,
+        response.initialData.author,
+        response.initialData.description,
+        ThumbnailSet(id.value),
+        Engagement(response.initialData.viewCount ?? 0, null, null));
   }
 
   /// Enumerates videos included in the specified playlist.
   Stream<Video> getVideos(dynamic id) async* {
     id = PlaylistId.fromString(id);
     var encounteredVideoIds = <String>{};
-    var index = 0;
+    var continuationToken = '';
+
     // ignore: literal_only_boolean_expressions
     while (true) {
-      var response =
-          await PlaylistResponse.get(_httpClient, id.value, index: index);
-      var countDelta = 0;
-      for (var video in response.videos) {
+      var response = await PlaylistPage.get(_httpClient, id.value,
+          token: continuationToken);
+
+      for (var video in response.initialData.playlistVideos) {
         var videoId = video.id;
 
         // Already added
@@ -46,26 +48,27 @@ class PlaylistClient {
           continue;
         }
 
+        if (video.channelId.isEmpty) {
+          continue;
+        }
+
         yield Video(
             VideoId(videoId),
             video.title,
             video.author,
-            video.channelId,
-            video.uploadDate,
+            ChannelId(video.channelId),
+            null,
             video.description,
             video.duration,
             ThumbnailSet(videoId),
-            video.keywords,
-            Engagement(video.viewCount, video.likes, video.dislikes),
+            null,
+            Engagement(video.viewCount, null, null),
             null);
-        countDelta++;
       }
-
-      // Videos loop around, so break when we stop seeing new videos
-      if (countDelta <= 0) {
+      continuationToken = response.initialData.continuationToken;
+      if (response.initialData.continuationToken?.isEmpty ?? true) {
         break;
       }
-      index += countDelta;
     }
   }
 }

lib/src/reverse_engineering/responses/channel_upload_page.dart

@@ -142,24 +142,24 @@ class _InitialData {
 
   NextContinuationData getContinuationContext() {
     if (root.contents != null) {
-      return root.contents.twoColumnBrowseResultsRenderer.tabs
-          .map((e) => e.tabRenderer)
-          .firstWhere((e) => e.selected)
-          .content
-          .sectionListRenderer
-          .contents
-          .first
-          .itemSectionRenderer
-          .contents
-          .first
-          .gridRenderer
-          .continuations
-          .first
-          .nextContinuationData;
+      return root.contents?.twoColumnBrowseResultsRenderer?.tabs
+          ?.map((e) => e.tabRenderer)
+          ?.firstWhere((e) => e.selected)
+          ?.content
+          ?.sectionListRenderer
+          ?.contents
+          ?.first
+          ?.itemSectionRenderer
+          ?.contents
+          ?.first
+          ?.gridRenderer
+          ?.continuations
+          ?.first
+          ?.nextContinuationData;
     }
     if (root.response != null) {
-      return root.response.continuationContents.gridContinuation.continuations
-          .first.nextContinuationData;
+      return root?.response?.continuationContents?.gridContinuation
+          ?.continuations?.first?.nextContinuationData;
     }
     return null;
   }
@@ -170,7 +170,7 @@ class _InitialData {
       ?.toList();
 
   String get continuation =>
-      _continuation ??= getContinuationContext().continuation ?? '';
+      _continuation ??= getContinuationContext()?.continuation ?? '';
 
   String get clickTrackingParams => _clickTrackingParams ??=
       getContinuationContext()?.clickTrackingParams ?? '';

lib/src/reverse_engineering/responses/playlist_page.dart

@@ -0,0 +1,305 @@
+import 'dart:convert';
+
+import 'package:html/dom.dart';
+import 'package:html/parser.dart' as parser;
+
+import '../../../youtube_explode_dart.dart';
+import '../../extensions/helpers_extension.dart';
+import '../../retry.dart';
+import '../youtube_http_client.dart';
+
+///
+class PlaylistPage {
+  final _apiKeyExp = RegExp(r'"INNERTUBE_API_KEY":"(\w+?)"');
+
+  ///
+  final String playlistId;
+  final Document _root;
+
+  String _apiKey;
+
+  ///
+  String get apiKey => _apiKey ??= _apiKeyExp
+      .firstMatch(_root
+          .querySelectorAll('script')
+          .firstWhere((e) => e.text.contains('INNERTUBE_API_KEY'))
+          .text)
+      .group(1);
+
+  _InitialData _initialData;
+
+  ///
+  _InitialData get initialData {
+    if (_initialData != null) {
+      return _initialData;
+    }
+
+    final scriptText = _root
+        .querySelectorAll('script')
+        .map((e) => e.text)
+        .toList(growable: false);
+
+    var initialDataText = scriptText.firstWhere(
+        (e) => e.contains('window["ytInitialData"] ='),
+        orElse: () => null);
+    if (initialDataText != null) {
+      return _initialData = _InitialData(json
+          .decode(_extractJson(initialDataText, 'window["ytInitialData"] =')));
+    }
+
+    initialDataText = scriptText.firstWhere(
+        (e) => e.contains('var ytInitialData = '),
+        orElse: () => null);
+    if (initialDataText != null) {
+      return _initialData = _InitialData(
+          json.decode(_extractJson(initialDataText, 'var ytInitialData = ')));
+    }
+
+    throw TransientFailureException(
+        'Failed to retrieve initial data from the search page, please report this to the project GitHub page.'); // ignore: lines_longer_than_80_chars
+  }
+
+  String _extractJson(String html, String separator) {
+    if (html == null || separator == null) {
+      return null;
+    }
+    var index = html.indexOf(separator) + separator.length;
+    if (index > html.length) {
+      return null;
+    }
+    return _matchJson(html.substring(index));
+  }
+
+  String _matchJson(String str) {
+    var bracketCount = 0;
+    int lastI;
+    for (var i = 0; i < str.length; i++) {
+      lastI = i;
+      if (str[i] == '{') {
+        bracketCount++;
+      } else if (str[i] == '}') {
+        bracketCount--;
+      } else if (str[i] == ';') {
+        if (bracketCount == 0) {
+          return str.substring(0, i);
+        }
+      }
+    }
+    return str.substring(0, lastI + 1);
+  }
+
+  ///
+  PlaylistPage(this._root, this.playlistId,
+      [_InitialData initialData, this._apiKey])
+      : _initialData = initialData;
+
+  ///
+  Future<PlaylistPage> nextPage(YoutubeHttpClient httpClient) async {
+    if (initialData.continuationToken == null) {
+      return null;
+    }
+    return get(httpClient, playlistId, token: initialData.continuationToken);
+  }
+
+  ///
+  static Future<PlaylistPage> get(YoutubeHttpClient httpClient, String id,
+      {String token}) {
+    if (token != null && token.isNotEmpty) {
+      var url =
+          'https://www.youtube.com/youtubei/v1/search?key=AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8';
+
+      return retry(() async {
+        var body = {
+          'context': const {
+            'client': {
+              'hl': 'en',
+              'clientName': 'WEB',
+              'clientVersion': '2.20200911.04.00'
+            }
+          },
+          'continuation': token
+        };
+
+        var raw = await httpClient.post(url, body: json.encode(body));
+        return PlaylistPage(null, id, _InitialData(json.decode(raw.body)));
+      });
+      // Ask for next page,
+
+    }
+    var url = 'https://www.youtube.com/playlist?list=$id&hl=en&persist_hl=1';
+    return retry(() async {
+      var raw = await httpClient.getString(url);
+      return PlaylistPage.parse(raw, id);
+    });
+    // ask for next page
+  }
+
+  ///
+  PlaylistPage.parse(String raw, this.playlistId) : _root = parser.parse(raw);
+}
+
+class _InitialData {
+  // Json parsed map
+  final Map<String, dynamic> root;
+
+  _InitialData(this.root);
+
+  String get title => root
+      ?.get('metadata')
+      ?.get('playlistMetadataRenderer')
+      ?.getT<String>('title');
+
+  String get author => root
+      .get('sidebar')
+      ?.get('playlistSidebarRenderer')
+      ?.getList('items')
+      ?.elementAtSafe(1)
+      ?.get('playlistSidebarSecondaryInfoRenderer')
+      ?.get('videoOwner')
+      ?.get('videoOwnerRenderer')
+      ?.get('title')
+      ?.getT<List<dynamic>>('runs')
+      ?.parseRuns();
+
+  String get description => root
+      ?.get('metadata')
+      ?.get('playlistMetadataRenderer')
+      ?.getT<String>('description');
+
+  int get viewCount => root
+      ?.get('sidebar')
+      ?.get('playlistSidebarRenderer')
+      ?.getList('items')
+      ?.firstOrNull
+      ?.get('playlistSidebarPrimaryInfoRenderer')
+      ?.getList('stats')
+      ?.elementAtSafe(1)
+      ?.getT<String>('simpleText')
+      ?.parseInt();
+
+  String get continuationToken => (videosContent ?? playlistVideosContent)
+      ?.firstWhere((e) => e['continuationItemRenderer'] != null,
+          orElse: () => null)
+      ?.get('continuationItemRenderer')
+      ?.get('continuationEndpoint')
+      ?.get('continuationCommand')
+      ?.getT<String>('token');
+
+  List<Map<String, dynamic>> get playlistVideosContent =>
+      root
+          .get('contents')
+          ?.get('twoColumnBrowseResultsRenderer')
+          ?.getList('tabs')
+          ?.firstOrNull
+          ?.get('tabRenderer')
+          ?.get('content')
+          ?.get('sectionListRenderer')
+          ?.getList('contents')
+          ?.firstOrNull
+          ?.get('itemSectionRenderer')
+          ?.getList('contents')
+          ?.firstOrNull
+          ?.get('playlistVideoListRenderer')
+          ?.getList('contents') ??
+      root
+          .getList('onResponseReceivedActions')
+          ?.firstOrNull
+          ?.get('appendContinuationItemsAction')
+          ?.get('continuationItems');
+
+  List<Map<String, dynamic>> get videosContent =>
+      root
+          .get('contents')
+          ?.get('twoColumnSearchResultsRenderer')
+          ?.get('primaryContents')
+          ?.get('sectionListRenderer')
+          ?.getList('contents') ??
+      root
+          ?.getList('onResponseReceivedCommands')
+          ?.firstOrNull
+          ?.get('appendContinuationItemsAction')
+          ?.get('continuationItems');
+
+  List<_Video> get playlistVideos =>
+      playlistVideosContent
+          ?.where((e) => e['playlistVideoRenderer'] != null)
+          ?.map((e) => _Video(e['playlistVideoRenderer']))
+          ?.toList() ??
+      const [];
+
+  List<_Video> get videos =>
+      videosContent?.firstOrNull
+          ?.get('itemSectionRenderer')
+          ?.getList('contents')
+          ?.where((e) => e['videoRenderer'] != null)
+          ?.map((e) => _Video(e))
+          ?.toList() ??
+      const [];
+}
+
+class _Video {
+  // Json parsed map
+  final Map<String, dynamic> root;
+
+  _Video(this.root);
+
+  String get id => root?.getT<String>('videoId');
+
+  String get author =>
+      root?.get('ownerText')?.getT<List<dynamic>>('runs')?.parseRuns() ??
+      root?.get('shortBylineText')?.getT<List<dynamic>>('runs')?.parseRuns() ??
+      '';
+
+  String get channelId =>
+      root
+          .get('ownerText')
+          ?.getList('runs')
+          ?.firstOrNull
+          ?.get('navigationEndpoint')
+          ?.get('browseEndpoint')
+          ?.getT<String>('browseId') ??
+      root
+          .get('shortBylineText')
+          ?.getList('runs')
+          ?.firstOrNull
+          ?.get('navigationEndpoint')
+          ?.get('browseEndpoint')
+          ?.getT<String>('browseId') ??
+      '';
+
+  String get title => root.get('title')?.getList('runs')?.parseRuns() ?? '';
+
+  String get description =>
+      root.getList('descriptionSnippet')?.parseRuns() ?? '';
+
+  Duration get duration =>
+      _stringToDuration(root.get('lengthText')?.getT<String>('simpleText'));
+
+  int get viewCount =>
+      root.get('viewCountText')?.getT<String>('simpleText')?.parseInt() ?? 0;
+
+  /// Format: HH:MM:SS
+  static Duration _stringToDuration(String string) {
+    if (string == null || string.trim().isEmpty) {
+      return null;
+    }
+
+    var parts = string.split(':');
+    assert(parts.length <= 3);
+
+    if (parts.length == 1) {
+      return Duration(seconds: int.parse(parts.first));
+    }
+    if (parts.length == 2) {
+      return Duration(
+          minutes: int.parse(parts.first), seconds: int.parse(parts[1]));
+    }
+    if (parts.length == 3) {
+      return Duration(
+          hours: int.parse(parts[0]),
+          minutes: int.parse(parts[1]),
+          seconds: int.parse(parts[2]));
+    }
+    throw Error();
+  }
+}

lib/src/reverse_engineering/responses/playlist_response.dart

@@ -1,122 +0,0 @@
-import 'dart:convert';
-
-import '../../channels/channel_id.dart';
-import '../../common/common.dart';
-import '../../exceptions/exceptions.dart';
-import '../../extensions/helpers_extension.dart';
-import '../../retry.dart';
-import '../youtube_http_client.dart';
-import 'generated/playlist_response.g.dart';
-
-///
-class PlaylistResponse {
-  List<_Video> _videos;
-
-  // Json parsed map
-  PlaylistResponseJson _root;
-
-  ///
-  String get title => _root.title;
-
-  ///
-  String get author => _root.author;
-
-  ///
-  String get description => _root.description;
-
-  ///
-  ThumbnailSet get thumbnails => ThumbnailSet(videos.firstOrNull.id);
-
-  ///
-  int get viewCount => _root.views;
-
-  ///
-  int get likeCount => _root.likes;
-
-  ///
-  int get dislikeCount => _root.dislikes;
-
-  ///
-  List<_Video> get videos =>
-      _videos ??= _root.video.map((e) => _Video(e)).toList();
-
-  ///
-  PlaylistResponse.parse(String raw) {
-    final t = json.tryDecode(raw);
-    if (t == null) {
-      throw TransientFailureException('Playerlist response is broken.');
-    }
-    _root = PlaylistResponseJson.fromJson(t);
-  }
-
-  ///
-  static Future<PlaylistResponse> get(YoutubeHttpClient httpClient, String id,
-      {int index = 0}) {
-    var url =
-        'https://youtube.com/list_ajax?style=json&action_get_list=1&list=$id&index=$index&hl=en';
-    return retry(() async {
-      var raw = await httpClient.getString(url);
-      return PlaylistResponse.parse(raw);
-    });
-  }
-
-  ///
-  static Future<PlaylistResponse> searchResults(
-      YoutubeHttpClient httpClient, String query,
-      {int page = 0}) {
-    var url = 'https://youtube.com/search_ajax?style=json&search_query='
-        '${Uri.encodeQueryComponent(query)}&page=$page&hl=en';
-    return retry(() async {
-      var raw = await httpClient.getString(url,
-          validate: false,
-          headers: const {
-            'x-youtube-client-name': '56',
-            'x-youtube-client-version': '20200911'
-          });
-      return PlaylistResponse.parse(raw);
-    });
-  }
-}
-
-class _Video {
-  // Json parsed map
-  final Video root;
-
-  _Video(this.root);
-
-  String get id => root.encryptedId;
-
-  String get author => root.author;
-
-  ChannelId get channelId => ChannelId('UC${root.userId}');
-
-  DateTime get uploadDate =>
-      DateTime.fromMillisecondsSinceEpoch(root.timeCreated * 1000);
-
-  String get title => root.title;
-
-  String get description => root.description;
-
-  Duration get duration => Duration(seconds: root.lengthSeconds);
-
-  int get viewCount => int.parse(root.views.stripNonDigits());
-
-  int get likes => root.likes;
-
-  int get dislikes => root.dislikes;
-
-  Iterable<String> get keywords => RegExp(r'"[^\"]+"|\S+')
-      .allMatches(root.keywords)
-      .map((e) => e.group(0))
-      .toList(growable: false);
-}
-
-extension on JsonCodec {
-  dynamic tryDecode(String source) {
-    try {
-      return json.decode(source);
-    } on FormatException {
-      return null;
-    }
-  }
-}

lib/src/reverse_engineering/responses/responses.dart

@@ -6,7 +6,7 @@ export 'dash_manifest.dart';
 export 'embed_page.dart';
 export 'player_response.dart';
 export 'player_source.dart';
-export 'playlist_response.dart';
+export 'playlist_page.dart';
 export 'stream_info_provider.dart';
 export 'video_info_response.dart';
 export 'watch_page.dart';

lib/src/reverse_engineering/responses/search_page.dart

@@ -105,17 +105,16 @@ class SearchPage {
         initialData.estimatedResults == 0) {
       return null;
     }
-    return get(httpClient, queryString,
-        token: initialData.continuationToken, key: apiKey);
+    return get(httpClient, queryString, token: initialData.continuationToken);
   }
 
   ///
   static Future<SearchPage> get(
       YoutubeHttpClient httpClient, String queryString,
-      {String token, String key}) {
+      {String token}) {
     if (token != null) {
-      assert(key != null, 'A key must be supplied along with a token');
-      var url = 'https://www.youtube.com/youtubei/v1/search?key=$key';
+      var url =
+          'https://www.youtube.com/youtubei/v1/search?key=AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8';
 
       return retry(() async {
         var body = {
@@ -131,7 +130,7 @@ class SearchPage {
 
         var raw = await httpClient.post(url, body: json.encode(body));
         return SearchPage(null, queryString,
-            _InitialData(SearchPageId.fromJson(json.decode(raw.body))), key);
+            _InitialData(SearchPageId.fromJson(json.decode(raw.body))));
       });
       // Ask for next page,
 

lib/src/search/search_list.dart

@@ -78,7 +78,7 @@ class SearchList extends DelegatingList<Video> {
     return DateTime.now().subtract(time);
   }
 
-  /// Format: HH:MM:SS (5 years ago)
+  /// Format: HH:MM:SS
   static Duration _stringToDuration(String string) {
     if (string == null || string.trim().isEmpty) {
       return null;
@@ -100,7 +100,7 @@ class SearchList extends DelegatingList<Video> {
           minutes: int.parse(parts[1]),
           seconds: int.parse(parts[0]));
     }
-    // Should reach here.
+    // Shouldn't reach here.
     throw Error();
   }
 }

pubspec.yaml

@@ -1,6 +1,6 @@
 name: youtube_explode_dart
 description: A port in dart of the youtube explode library. Supports several API functions without the need of Youtube API Key.
-version: 1.8.0-beta.2
+version: 1.8.0-beta.3
 homepage: https://github.com/Hexer10/youtube_explode_dart
 
 environment:

test/playlist_test.dart

@@ -21,8 +21,8 @@ void main() {
     expect(playlist.author, 'Tyrrrz');
     expect(playlist.description, 'My best osu! plays');
     expect(playlist.engagement.viewCount, greaterThanOrEqualTo(133));
-    expect(playlist.engagement.likeCount, greaterThanOrEqualTo(0));
-    expect(playlist.engagement.dislikeCount, greaterThanOrEqualTo(0));
+    expect(playlist.engagement.likeCount, isNull);
+    expect(playlist.engagement.dislikeCount, isNull);
     expect(playlist.thumbnails.lowResUrl, isNotEmpty);
     expect(playlist.thumbnails.mediumResUrl, isNotEmpty);
     expect(playlist.thumbnails.highResUrl, isNotEmpty);
@@ -65,16 +65,13 @@ void main() {
 
   group('Get videos in any playlist', () {
     for (var val in {
-      PlaylistId('PL601B2E69B03FAB9D'),
       PlaylistId('PLI5YfMzCfRtZ8eV576YoY3vIYrHjyVm_e'),
       PlaylistId('PLWwAypAcFRgKFlxtLbn_u14zddtDJj3mk'),
       PlaylistId('OLAK5uy_mtOdjCW76nDvf5yOzgcAVMYpJ5gcW5uKU'),
-      PlaylistId('RD1hu8-y6fKg0'),
-      PlaylistId('RDMMU-ty-2B02VY'),
       PlaylistId('RDCLAK5uy_lf8okgl2ygD075nhnJVjlfhwp8NsUgEbs'),
-      PlaylistId('ULl6WWX-BgIiE'),
       PlaylistId('UUTMt7iMWa7jy0fNXIktwyLA'),
-      PlaylistId('FLEnBXANsKmyj2r9xVyKoDiQ')
+      PlaylistId('OLAK5uy_lLeonUugocG5J0EUAEDmbskX4emejKwcM'),
+      PlaylistId('PL601B2E69B03FAB9D'),
     }) {
       test('PlaylistID - ${val.value}', () async {
         expect(yt.playlists.getVideos(val), emits(isNotNull));