Version 1.10.2

Better comments api.
Fix #142

CHANGELOG.md

@@ -1,7 +1,10 @@
+## 1.10.2
+- Better comments API: Implemented API to fetch more comments & replies.
+
 ## 1.10.1
 - Fix issue #146: Closed Captions couldn't be extracted anymore.
 - Code cleanup.
-- 
+ 
 
 ## 1.10.0
 - Fix issue #144: get_video_info was removed from yt.

lib/src/common/engagement.dart

@@ -5,7 +5,6 @@ part 'engagement.freezed.dart';
 /// User activity statistics.
 @freezed
 class Engagement with _$Engagement {
-
   const factory Engagement(
     /// View count.
     int viewCount,

lib/src/exceptions/youtube_explode_exception.dart

@@ -1,3 +1,5 @@
+import 'package:meta/meta.dart';
+
 /// Parent class for domain exceptions thrown by [YoutubeExplode]
 abstract class YoutubeExplodeException implements Exception {
   /// Generic message.
@@ -13,6 +15,7 @@ abstract class YoutubeExplodeException implements Exception {
   YoutubeExplodeException(this.message);
 
   @override
+  @nonVirtual
   String toString() {
     if (_others.isEmpty) {
       return '$runtimeType: $message';

lib/src/extensions/helpers_extension.dart

@@ -80,7 +80,7 @@ extension StringUtility on String {
 
 /// Utility for Strings.
 extension StringUtility2 on String? {
-  static final RegExp _unitSplit = RegExp(r'^(\d+(?:\.\d)?)(\w)');
+  static final RegExp _unitSplit = RegExp(r'^(\d+(?:\.\d+)?)(\w)?');
 
   /// Parses this value as int stripping the non digit characters,
   /// returns null if this fails.
@@ -104,9 +104,6 @@ extension StringUtility2 on String? {
     }
 
     final multiplierText = match.group(2);
-    if (multiplierText == null) {
-      return null;
-    }
 
     var multiplier = 1;
     if (multiplierText == 'K') {
@@ -238,10 +235,17 @@ extension GetOrNullMap on Map {
   }
 
   /// Get a List<Map<String, dynamic>>> from a map.
-  List<Map<String, dynamic>>? getList(String key) {
+  List<Map<String, dynamic>>? getList(String key, [String? orKey]) {
     var v = this[key];
     if (v == null) {
-      return null;
+      if (orKey != null) {
+        v = this[orKey];
+        if (v == null) {
+          return null;
+        }
+      } else {
+        return null;
+      }
     }
     if (v is! List<dynamic>) {
       throw Exception('Invalid type: ${v.runtimeType} should be of type List');

lib/src/reverse_engineering/pages/watch_page.dart

@@ -22,7 +22,6 @@ class WatchPage extends YoutubePage<_InitialData> {
       RegExp('VISITOR_INFO1_LIVE=([^;]+)');
   static final RegExp _yscExp = RegExp('YSC=([^;]+)');
 
-
   @override
   // Overridden to be non-nullable.
   // ignore: overridden_fields

lib/src/reverse_engineering/player/player_response.dart

@@ -148,8 +148,7 @@ class ClosedCaptionTrack {
   String get languageCode => root.getT<String>('languageCode')!;
 
   ///
-  String? get languageName =>
+  String? get languageName => root.get('name')!.getT<String>('simpleText');
-      root.get('name')!.getT<String>('simpleText');
 
   ///
   bool get autoGenerated =>

lib/src/reverse_engineering/responses/comments_client.dart

@@ -14,6 +14,8 @@ class CommentsClient {
   late final List<_Comment> comments =
       _commentRenderers.map((e) => _Comment(e)).toList(growable: false);
 
+  late final String? _continuationToken = _getContinuationToken();
+
   CommentsClient(this.root);
 
   ///
@@ -32,21 +34,82 @@ class CommentsClient {
     return CommentsClient(data);
   }
 
+  ///
+  static Future<CommentsClient?> getReplies(
+      YoutubeHttpClient httpClient, String token) async {
+    final data = await httpClient.sendPost('next', token);
+    return CommentsClient(data);
+  }
+
   List<JsonMap> _getCommentRenderers() {
     return root
-        .getList('onResponseReceivedEndpoints')![1]
+            .getList('onResponseReceivedEndpoints')!
-        .get('reloadContinuationItemsCommand')!
+            .last
-        .getList('continuationItems')!
+            .get('appendContinuationItemsAction')
-        .where((e) => e['commentThreadRenderer'] != null)
+            ?.getList('continuationItems')
-        .map((e) => e.get('commentThreadRenderer')!)
+            ?.where((e) => e['commentRenderer'] != null)
-        .toList(growable: false);
+            .toList(growable: false) /* Used for the replies */ ??
+        root
+            .getList('onResponseReceivedEndpoints')!
+            .last
+            .get('reloadContinuationItemsCommand')!
+            .getList('continuationItems', 'appendContinuationItemsAction')!
+            .where((e) => e['commentThreadRenderer'] != null)
+            .map((e) => e.get('commentThreadRenderer')!)
+            .toList(growable: false);
+  }
+
+  String? _getContinuationToken() {
+    return root
+            .getList('onResponseReceivedEndpoints')!
+            .last
+            .get('appendContinuationItemsAction')
+            ?.getList('continuationItems')
+            ?.firstWhereOrNull((e) => e['continuationItemRenderer'] != null)
+            ?.get('continuationItemRenderer')
+            ?.get('button')
+            ?.get('buttonRenderer')
+            ?.get('command')
+            ?.get('continuationCommand')
+            ?.getT<String>('token') /* Used for the replies */ ??
+        root
+            .getList('onResponseReceivedEndpoints')!
+            .last
+            .get('reloadContinuationItemsCommand')!
+            .getList('continuationItems', 'appendContinuationItemsAction')!
+            .firstWhereOrNull((e) => e['continuationItemRenderer'] != null)
+            ?.get('continuationItemRenderer')
+            ?.get('continuationEndpoint')
+            ?.get('continuationCommand')
+            ?.getT<String>('token');
+  }
+
+  int getCommentsCount() => root
+      .getList('onResponseReceivedEndpoints')![1]
+      .get('reloadContinuationItemsCommand')!
+      .getList('continuationItems')!
+      .first
+      .get('commentsHeaderRenderer')!
+      .get('commentsCount')!
+      .getList('runs')!
+      .first
+      .getT<String>('text')
+      .parseIntWithUnits()!;
+
+  Future<CommentsClient?> nextPage(YoutubeHttpClient httpClient) async {
+    if (_continuationToken == null) {
+      return null;
+    }
+
+    final data = await httpClient.sendPost('next', _continuationToken!);
+    return CommentsClient(data);
   }
 }
 
 class _Comment {
   final JsonMap root;
 
-  late final JsonMap _commentRenderer =
+  late final JsonMap _commentRenderer = root.get('commentRenderer') ??
       root.get('comment')!.get('commentRenderer')!;
 
   late final JsonMap? _commentRepliesRenderer =
@@ -61,14 +124,7 @@ class _Comment {
       ?.get('continuationCommand')
       ?.getT<String>('token');
 
-  late final int? repliesCount = _commentRepliesRenderer
+  late final int? repliesCount = _commentRenderer.getT<int>('replyCount');
-      ?.get('viewReplies')
-      ?.get('buttonRenderer')
-      ?.get('text')
-      ?.getList('runs')
-      ?.elementAtSafe(2)
-      ?.getT<String>('text')
-      ?.parseIntWithUnits();
 
   late final String author =
       _commentRenderer.get('authorText')!.getT<String>('simpleText')!;
@@ -95,18 +151,8 @@ class _Comment {
       .first
       .getT<String>('text')!;
 
-  /// Needs to be parsed as an int current is like: 1.2K
   late final int? likeCount = _commentRenderer
-      .get('actionButtons')
+      .get('voteCount')
-      ?.get('commentActionButtonsRenderer')
-      ?.get('likeButton')
-      ?.get('toggleButtonRenderer')
-      ?.get('defaultServiceEndpoint')
-      ?.get('performCommentActionEndpoint')
-      ?.getList('clientActions')
-      ?.first
-      .get('updateCommentVoteAction')
-      ?.get('voteCount')
       ?.getT<String>('simpleText')
       ?.parseIntWithUnits();
 

lib/src/reverse_engineering/youtube_http_client.dart

@@ -105,9 +105,11 @@ class YoutubeHttpClient extends http.BaseClient {
     var bytesCount = start;
     for (var i = start; i < streamInfo.size.totalBytes; i += 9898989) {
       try {
-        final request = http.Request('get', url);
+        final response = await retry(() {
-        request.headers['range'] = 'bytes=$i-${i + 9898989 - 1}';
+          final request = http.Request('get', url);
-        final response = await retry(() => send(request));
+          request.headers['range'] = 'bytes=$i-${i + 9898989 - 1}';
+          return send(request);
+        });
         if (validate) {
           _validateResponse(response, response.statusCode);
         }

lib/src/search/search_list.dart

@@ -7,7 +7,7 @@ import '../../youtube_explode_dart.dart';
 import '../extensions/helpers_extension.dart';
 
 /// This list contains search videos.
-/// /// This behaves like a [List] but has the [SearchList.nextPage] to get the next batch of videos.
+///This behaves like a [List] but has the [SearchList.nextPage] to get the next batch of videos.
 class SearchList extends DelegatingList<Video> {
   final SearchPage _page;
   final YoutubeHttpClient _httpClient;

lib/src/videos/closed_captions/closed_caption_client.dart

@@ -35,8 +35,7 @@ class ClosedCaptionClient {
       ]}) async {
     videoId = VideoId.fromString(videoId);
     var tracks = <ClosedCaptionTrackInfo>{};
-    var watchPage =
+    var watchPage = await WatchPage.get(_httpClient, videoId.value);
-        await WatchPage.get(_httpClient, videoId.value);
     var playerResponse = watchPage.playerResponse!;
 
     for (final track in playerResponse.closedCaptionTrack) {

lib/src/videos/comments/comments_client.dart

@@ -5,31 +5,30 @@ import '../../reverse_engineering/responses/comments_client.dart' as re;
 import '../../reverse_engineering/youtube_http_client.dart';
 import '../videos.dart';
 import 'comment.dart';
+import 'comments_list.dart';
 
 /// Queries related to comments of YouTube videos.
-@experimental
 class CommentsClient {
   final YoutubeHttpClient _httpClient;
 
   /// Initializes an instance of [CommentsClient]
   CommentsClient(this._httpClient);
 
-  /// Returns a stream emitting all the [video]'s comment.
+  /// Returns a [List<Comment>] containing the first batch of comments.
-  /// A request is page for every comment page,
+  /// You can use [CommentsList.nextPage()] to get the next batch of comments.
-  /// a page contains at most 20 comments, use .take if you want to limit
+  Future<CommentsList?> getComments(Video video) async {
-  /// the results.
-  ///
-  /// The streams doesn't emit any data if [Video.hasWatchPage] is false.
-  /// Use `videos.get(videoId, forceWatchPage: true)` to assure that the
-  /// WatchPage is fetched.
-  Future<List<Comment>> getComments(Video video) async {
     if (video.watchPage == null) {
-      return const [];
+      return null;
     }
 
     final page = await re.CommentsClient.get(_httpClient, video);
 
-    return page?.comments
+    if (page == null) {
+      return null;
+    }
+
+    return CommentsList(
+        page.comments
             .map((e) => Comment(
                 e.author,
                 ChannelId(e.channelId),
@@ -38,7 +37,35 @@ class CommentsClient {
                 e.publishTime,
                 e.repliesCount ?? 0,
                 e.continuation))
-            .toList(growable: false) ??
+            .toList(growable: false),
-        const [];
+        page,
+        _httpClient);
+  }
+
+  Future<CommentsList?> getReplies(Comment comment) async {
+    if (comment.continuation == null) {
+      return null;
+    }
+
+    final page =
+        await re.CommentsClient.getReplies(_httpClient, comment.continuation!);
+
+    if (page == null) {
+      return null;
+    }
+
+    return CommentsList(
+        page.comments
+            .map((e) => Comment(
+                e.author,
+                ChannelId(e.channelId),
+                e.text,
+                e.likeCount ?? 0,
+                e.publishTime,
+                e.repliesCount ?? 0,
+                e.continuation))
+            .toList(growable: false),
+        page,
+        _httpClient);
   }
 }

lib/src/videos/comments/comments_list.dart

@@ -0,0 +1,42 @@
+import 'dart:async';
+
+import 'package:collection/collection.dart';
+import 'package:youtube_explode_dart/src/reverse_engineering/responses/comments_client.dart'
+    as re;
+
+import '../../../youtube_explode_dart.dart';
+
+/// This list contains search videos.
+///This behaves like a [List] but has the [SearchList.nextPage] to get the next batch of videos.
+class CommentsList extends DelegatingList<Comment> {
+  final re.CommentsClient _client;
+  final YoutubeHttpClient _httpClient;
+
+  /// Construct an instance of [SearchList]
+  /// See [SearchList]
+  CommentsList(List<Comment> base, this._client, this._httpClient)
+      : super(base);
+
+  /// Fetches the next batch of videos or returns null if there are no more
+  /// results.
+  Future<CommentsList?> nextPage() async {
+    final page = await _client.nextPage(_httpClient);
+    if (page == null) {
+      return null;
+    }
+
+    return CommentsList(
+        page.comments
+            .map((e) => Comment(
+                e.author,
+                ChannelId(e.channelId),
+                e.text,
+                e.likeCount ?? 0,
+                e.publishTime,
+                e.repliesCount ?? 0,
+                e.continuation))
+            .toList(growable: false),
+        page,
+        _httpClient);
+  }
+}

test/comments_client_test.dart

@@ -15,6 +15,6 @@ void main() {
     var videoUrl = 'https://www.youtube.com/watch?v=AI7ULzgf8RU';
     var video = await yt!.videos.get(VideoId(videoUrl));
     var comments = await yt!.videos.commentsClient.getComments(video);
-    expect(comments.length, greaterThanOrEqualTo(1));
+    expect(comments!.length, greaterThanOrEqualTo(1));
   });
 }

test/streams_test.dart

@@ -23,7 +23,6 @@ void main() {
       VideoId('YltHGKX80Y8'), //ContainsClosedCaptions
       VideoId('_kmeFXjjGfk'), //EmbedRestrictedByYouTube
       VideoId('MeJVWBSsPAY'), //EmbedRestrictedByAuthor
-      VideoId('SkRSXFQerZs'), //AgeRestricted
       VideoId('hySoCSoH-g8'), //AgeRestrictedEmbedRestricted
       VideoId('5VGm0dczmHc'), //RatingDisabled
       VideoId('-xNN-bJQ4vI'), // 360° video
@@ -31,7 +30,7 @@ void main() {
       test('VideoId - ${val.value}', () async {
         var manifest = await yt!.videos.streamsClient.getManifest(val);
         expect(manifest.streams, isNotEmpty);
-      });
+      }, timeout: const Timeout(Duration(seconds: 90)));
     }
   });
 
@@ -40,6 +39,10 @@ void main() {
         throwsA(const TypeMatcher<VideoRequiresPurchaseException>()));
   });
 
+  test('Stream of age-limited video throws VideoUnplayableException', () {
+    expect(yt!.videos.streamsClient.getManifest(VideoId('SkRSXFQerZs')),
+        throwsA(const TypeMatcher<VideoUnplayableException>()));
+  });
   test('Get the hls manifest of a live stream', () async {
     expect(
         await yt!.videos.streamsClient
@@ -68,7 +71,6 @@ void main() {
       VideoId('YltHGKX80Y8'), //ContainsClosedCaptions
       VideoId('_kmeFXjjGfk'), //EmbedRestrictedByYouTube
       VideoId('MeJVWBSsPAY'), //EmbedRestrictedByAuthor
-      VideoId('SkRSXFQerZs'), //AgeRestricted
       VideoId('hySoCSoH-g8'), //AgeRestrictedEmbedRestricted
       VideoId('5VGm0dczmHc'), //RatingDisabled
     }) {