Merge pull request #43 from Hexer10/comments

Implement comment api

lib/src/retry.dart

@@ -26,7 +26,8 @@ Future<T> retry<T>(FutureOr<T> function()) async {
 
 /// Get "retry" cost of each YoutubeExplode exception.
 int getExceptionCost(Exception e) {
-  if (e is TransientFailureException) {
+  if (e is TransientFailureException || e is FormatException) {
+    print('Ripperoni!');
     return 1;
   }
   if (e is RequestLimitExceededException) {

lib/src/reverse_engineering/responses/watch_page.dart

@@ -13,13 +13,36 @@ import 'player_response.dart';
 import 'stream_info_provider.dart';
 
 class WatchPage {
-  final RegExp _videoLikeExp = RegExp(r'"label"\s*:\s*"([\d,\.]+) likes"');
-  final RegExp _videoDislikeExp =
+  static final RegExp _videoLikeExp =
+      RegExp(r'"label"\s*:\s*"([\d,\.]+) likes"');
+  static final RegExp _videoDislikeExp =
       RegExp(r'"label"\s*:\s*"([\d,\.]+) dislikes"');
+  static final RegExp _visitorInfoLiveExp =
+      RegExp('VISITOR_INFO1_LIVE=([^;]+)');
+  static final RegExp _yscExp = RegExp('YSC=([^;]+)');
+  static final _xsfrTokenExp = RegExp(r'"XSRF_TOKEN"\s*:\s*"(.+?)"');
 
   final Document _root;
+  final String visitorInfoLive;
+  final String ysc;
 
-  WatchPage(this._root);
+  WatchPage(this._root, this.visitorInfoLive, this.ysc);
+
+  _InitialData get initialData =>
+      _InitialData(json.decode(_matchJson(_extractJson(
+          _root
+              .querySelectorAll('script')
+              .map((e) => e.text)
+              .toList()
+              .firstWhere((e) => e.contains('window["ytInitialData"] =')),
+          'window["ytInitialData"] ='))));
+
+  String get xsfrToken => _xsfrTokenExp
+      .firstMatch(_root
+          .querySelectorAll('script')
+          .firstWhere((e) => _xsfrTokenExp.hasMatch(e.text))
+          .text)
+      .group(1);
 
   bool get isOk => _root.body.querySelector('#player') != null;
 
@@ -78,14 +101,18 @@ class WatchPage {
     return str.substring(0, lastI + 1);
   }
 
-  WatchPage.parse(String raw) : _root = parser.parse(raw);
+  WatchPage.parse(String raw, this.visitorInfoLive, this.ysc)
+      : _root = parser.parse(raw);
 
   static Future<WatchPage> get(YoutubeHttpClient httpClient, String videoId) {
     final url = 'https://youtube.com/watch?v=$videoId&bpctr=9999999999&hl=en';
     return retry(() async {
-      var raw = await httpClient.getString(url);
+      var req = await httpClient.get(url, validate: true);
 
-      var result = WatchPage.parse(raw);
+      var cookies = req.headers['set-cookie'];
+      var visitorInfoLive = _visitorInfoLiveExp.firstMatch(cookies).group(1);
+      var ysc = _yscExp.firstMatch(cookies).group(1);
+      var result = WatchPage.parse(req.body, visitorInfoLive, ysc);
 
       if (!result.isOk) {
         throw TransientFailureException("Video watch page is broken.");
@@ -188,3 +215,38 @@ class _PlayerConfig {
 
   List<_StreamInfo> get streams => [...muxedStreams, ...adaptiveStreams];
 }
+
+class _InitialData {
+  // Json parsed map
+  final Map<String, dynamic> _root;
+
+  _InitialData(this._root);
+
+  /* Cache results */
+
+  String _continuation;
+  String _clickTrackingParams;
+
+  Map<String, dynamic> getContinuationContext(Map<String, dynamic> root) {
+    if (_root['contents'] != null) {
+      return (_root['contents']['twoColumnWatchNextResults']['results']
+              ['results']['contents'] as List<dynamic>)
+          ?.firstWhere((e) => e.containsKey('itemSectionRenderer'))[
+              'itemSectionRenderer']['continuations']
+          ?.first['nextContinuationData']
+          ?.cast<String, dynamic>();
+    }
+    if (_root['response'] != null) {
+      return _root['response']['itemSectionContinuation']['continuations']
+          ?.first['nextContinuationData']
+          ?.cast<String, dynamic>();
+    }
+    return null;
+  }
+
+  String get continuation => _continuation ??=
+      getContinuationContext(_root)?.getValue('continuation') ?? '';
+
+  String get clickTrackingParams => _clickTrackingParams ??=
+      getContinuationContext(_root)?.getValue('clickTrackingParams') ?? '';
+}

lib/src/reverse_engineering/youtube_http_client.dart

@@ -12,6 +12,12 @@ class YoutubeHttpClient extends http.BaseClient {
     'accept-language': 'en-US,en;q=1.0',
     'x-youtube-client-name': '1',
     'x-youtube-client-version': '2.20200609.04.02',
+    'x-spf-previous': 'https://www.youtube.com/',
+    'x-spf-referer': 'https://www.youtube.com/',
+    'x-youtube-device':
+        'cbr=Chrome&cbrver=81.0.4044.138&ceng=WebKit&cengver=537.36'
+            '&cos=Windows&cosver=10.0',
+    'x-youtube-page-label': 'youtube.ytfe.desktop_20200617_1_RC1'
   };
 
   /// Throws if something is wrong with the response.
@@ -37,8 +43,8 @@ class YoutubeHttpClient extends http.BaseClient {
 
   Future<String> getString(dynamic url,
       {Map<String, String> headers, bool validate = true}) async {
-    var response = await get(url, headers: {...?headers, ..._defaultHeaders});
-
+    var response = await get(url, headers: headers);
+    
     if (validate) {
       _validateResponse(response, response.statusCode);
     }
@@ -46,6 +52,16 @@ class YoutubeHttpClient extends http.BaseClient {
     return response.body;
   }
 
+  @override
+  Future<http.Response> get(dynamic url,
+      {Map<String, String> headers, bool validate = false}) async {
+    var response = await super.get(url, headers: headers);
+    if (validate) {
+      _validateResponse(response, response.statusCode);
+    }
+    return response;
+  }
+
   Future<String> postString(dynamic url,
       {Map<String, String> body,
       Map<String, String> headers,
@@ -63,8 +79,8 @@ class YoutubeHttpClient extends http.BaseClient {
   Stream<List<int>> getStream(StreamInfo streamInfo,
       {Map<String, String> headers, bool validate = true}) async* {
     var url = streamInfo.url;
-//    if (streamInfo.isRateLimited()) {
-//      var request = Request('get', url);
+//    if (!streamInfo.isRateLimited()) {
+//      var request = http.Request('get', url);
 //      request.headers.addAll(_defaultHeaders);
 //      var response = await request.send();
 //      if (validate) {
@@ -100,7 +116,11 @@ class YoutubeHttpClient extends http.BaseClient {
 
   @override
   Future<http.StreamedResponse> send(http.BaseRequest request) {
-    request.headers.addAll(_defaultHeaders);
+    _defaultHeaders.forEach((key, value) {
+      if (request.headers[key] == null) {
+        request.headers[key] = _defaultHeaders[key];
+      }
+    });
     return _httpClient.send(request);
   }
 }

lib/src/videos/comments/comment.dart

@@ -0,0 +1,54 @@
+import 'package:equatable/equatable.dart';
+import 'package:meta/meta.dart';
+
+import '../../channels/channel_id.dart';
+
+/// YouTube comment metadata.
+class Comment with EquatableMixin {
+  /// Comment id.
+  final String commentId;
+
+  /// Comment author name.
+  final String author;
+
+  /// Comment author channel id.
+  final ChannelId channelId;
+
+  /// Comment text.
+  final String text;
+
+  /// Comment likes count.
+  final int likeCount;
+
+  /// Published time as string. (For example: "2 years ago")
+  final String publishedTime;
+
+  /// Comment reply count.
+  final int replyCount;
+
+  /// Used internally.
+  @protected
+  final String continuation;
+
+  /// Used internally.
+  @protected
+  final String clicktrackingParams;
+
+  /// Initializes an instance of [Comment]
+  Comment(
+      this.commentId,
+      this.author,
+      this.channelId,
+      this.text,
+      this.likeCount,
+      this.publishedTime,
+      this.replyCount,
+      this.continuation,
+      this.clicktrackingParams);
+
+  @override
+  String toString() => 'Comment($author): $text';
+
+  @override
+  List<Object> get props => [commentId];
+}

lib/src/videos/comments/comments.dart

@@ -0,0 +1 @@
+export 'comment.dart';

lib/src/videos/comments/comments_client.dart

@@ -0,0 +1,138 @@
+import 'dart:convert';
+
+import '../../channels/channel_id.dart';
+import '../../extensions/helpers_extension.dart';
+import '../../retry.dart';
+import '../../reverse_engineering/youtube_http_client.dart';
+import '../videos.dart';
+import 'comment.dart';
+
+/// Queries related to comments of YouTube videos.
+class CommentsClient {
+  final YoutubeHttpClient _httpClient;
+
+  /// Initializes an instance of [CommentsClient]
+  CommentsClient(this._httpClient);
+
+  /// Returns the json parsed comments map.
+  Future<Map<String, dynamic>> _getCommentJson(
+      String service,
+      String continuation,
+      String clickTrackingParams,
+      String xsfrToken,
+      String visitorInfoLive,
+      String ysc) async {
+    var url = 'https://www.youtube.com/comment_service_ajax?'
+        '$service=1&'
+        'pbj=1&'
+        'ctoken=$continuation&'
+        'continuation=$continuation&'
+        'itct=$clickTrackingParams';
+    return retry(() async {
+      var raw = await _httpClient.postString(url, headers: {
+        'cookie': 'YSC=$ysc; GPS=1; VISITOR_INFO1_LIVE=$visitorInfoLive;'
+            ' CONSENT=WP.288163; PREF=f4=4000000',
+      }, body: {
+        'session_token': xsfrToken
+      });
+      return json.decode(raw);
+    });
+  }
+
+  /// Returns a stream emitting all the [video]'s comment.
+  /// A request is page for every comment page,
+  /// a page contains at most 20 comments, use .take if you want to limit
+  /// the results.
+  ///
+  /// Throws an exception if the given video has not a watch page available.
+  /// this happens for the videos from playlist or search queries.
+  Stream<Comment> getComments(Video video) async* {
+    if (video.watchPage == null) {
+      //TODO: Implement custom exception.
+      throw Exception('Watch page not available for this video');
+    }
+    yield* _getComments(
+        video.watchPage.initialData.continuation,
+        video.watchPage.initialData.clickTrackingParams,
+        video.watchPage.xsfrToken,
+        video.watchPage.visitorInfoLive,
+        video.watchPage.ysc);
+  }
+
+  Stream<Comment> _getComments(String continuation, String clickTrackingParams,
+      String xsfrToken, String visitorInfoLive, String ysc) async* {
+    var data = await _getCommentJson('action_get_comments', continuation,
+        clickTrackingParams, xsfrToken, visitorInfoLive, ysc);
+    var contentRoot = data['response']['continuationContents']
+            ['itemSectionContinuation']['contents']
+        ?.map((e) => e['commentThreadRenderer'])
+        ?.toList()
+        ?.cast<Map<String, dynamic>>() as List<Map<String, dynamic>>;
+    if (contentRoot == null) {
+      return;
+    }
+    for (var content in contentRoot) {
+      var commentRaw = content['comment']['commentRenderer'];
+      String continuation;
+      String clickTrackingParams;
+      if (content['replies'] != null) {
+        continuation = content['replies']['commentRepliesRenderer']
+                ['continuations']
+            .first['nextContinuationData']['continuation'];
+        clickTrackingParams = content['replies']['commentRepliesRenderer']
+                ['continuations']
+            .first['nextContinuationData']['clickTrackingParams'];
+      }
+      var comment = Comment(
+          commentRaw['commentId'],
+          commentRaw['authorText']['simpleText'],
+          ChannelId(commentRaw['authorEndpoint']['browseEndpoint']['browseId']),
+          _parseRuns(commentRaw['contentText']),
+          commentRaw['likeCount'] ?? 0,
+          _parseRuns(commentRaw['publishedTimeText']),
+          commentRaw['replyCount'],
+          continuation,
+          clickTrackingParams);
+      yield comment;
+    }
+    var continuationRoot = (data
+            ?.get('response')
+            ?.get('continuationContents')
+            ?.get('itemSectionContinuation')
+            ?.getValue('continuations')
+            ?.first as Map<String, dynamic>)
+        ?.get('nextContinuationData');
+    if (continuationRoot != null) {
+      yield* _getComments(
+          continuationRoot['continuation'],
+          continuationRoot['clickTrackingParams'],
+          xsfrToken,
+          visitorInfoLive,
+          ysc);
+    }
+  }
+
+  String _parseRuns(Map<dynamic, dynamic> runs) =>
+      runs?.getValue('runs')?.map((e) => e['text'])?.join() ?? '';
+
+//TODO: Implement replies
+/*  Stream<Comment> getReplies(Video video, Comment comment) async* {
+    if (video.watchPage == null || comment.continuation == null
+    || comment.clicktrackingParams == null) {
+      return;
+    }
+    yield* _getReplies(
+        video.watchPage.initialData.continuation,
+        video.watchPage.initialData.clickTrackingParams,
+        video.watchPage.xsfrToken,
+        video.watchPage.visitorInfoLive,
+        video.watchPage.ysc);
+  }
+
+  Stream<Comment> _getReplies(String continuation, String clickTrackingParams,
+      String xsfrToken, String visitorInfoLive, String ysc) async* {
+    var data = await _getCommentJson('action_get_comment_replies', continuation,
+        clickTrackingParams, xsfrToken, visitorInfoLive, ysc);
+    print(data);
+  }*/
+}

lib/src/videos/video.dart

@@ -1,8 +1,10 @@
 import 'dart:collection';
 
 import 'package:equatable/equatable.dart';
+import 'package:meta/meta.dart';
 
 import '../common/common.dart';
+import '../reverse_engineering/responses/responses.dart';
 import 'video_id.dart';
 
 /// YouTube video metadata.
@@ -37,6 +39,10 @@ class Video with EquatableMixin {
   /// Engagement statistics for this video.
   final Engagement engagement;
 
+  /// Used internally.
+  @protected
+  final WatchPage watchPage;
+
   /// Initializes an instance of [Video]
   Video(
       this.id,
@@ -47,7 +53,8 @@ class Video with EquatableMixin {
       this.duration,
       this.thumbnails,
       Iterable<String> keywords,
-      this.engagement)
+      this.engagement,
+      [this.watchPage])
       : keywords = UnmodifiableListView(keywords);
 
   @override

lib/src/videos/video_client.dart

@@ -2,6 +2,7 @@ import '../common/common.dart';
 import '../reverse_engineering/responses/responses.dart';
 import '../reverse_engineering/youtube_http_client.dart';
 import 'closed_captions/closed_caption_client.dart';
+import 'comments/comments_client.dart';
 import 'videos.dart';
 
 /// Queries related to YouTube videos.
@@ -14,10 +15,14 @@ class VideoClient {
   /// Queries related to closed captions of YouTube videos.
   final ClosedCaptionClient closedCaptions;
 
+  /// Queries related to a YouTube video.
+  final CommentsClient commentsClient;
+
   /// Initializes an instance of [VideoClient].
   VideoClient(this._httpClient)
       : streamsClient = StreamsClient(_httpClient),
-        closedCaptions = ClosedCaptionClient(_httpClient);
+        closedCaptions = ClosedCaptionClient(_httpClient),
+        commentsClient = CommentsClient(_httpClient);
 
   /// Gets the metadata associated with the specified video.
   Future<Video> get(dynamic videoId) async {
@@ -37,6 +42,7 @@ class VideoClient {
         ThumbnailSet(videoId.value),
         playerResponse.videoKeywords,
         Engagement(playerResponse.videoViewCount ?? 0, watchPage.videoLikeCount,
-            watchPage.videoDislikeCount));
+            watchPage.videoDislikeCount),
+        watchPage);
   }
 }

lib/src/videos/videos.dart

@@ -1,3 +1,6 @@
+library youtube_explode.videos;
+
+export 'comments/comments.dart';
 export 'streams/streams.dart';
 export 'video.dart';
 export 'video_client.dart';

test/comments_client_test.dart

@@ -0,0 +1,22 @@
+import 'package:test/test.dart';
+import 'package:youtube_explode_dart/youtube_explode_dart.dart';
+
+void main() {
+  group('Comments', () {
+    YoutubeExplode yt;
+    setUp(() {
+      yt = YoutubeExplode();
+    });
+
+    tearDown(() {
+      yt.close();
+    });
+
+    test('GetCommentOfVideo', () async {
+      var videoUrl = 'https://www.youtube.com/watch?v=AI7ULzgf8RU';
+      var video = await yt.videos.get(VideoId(videoUrl));
+      var comments = await yt.videos.commentsClient.getComments(video).toList();
+      expect(comments.length, greaterThanOrEqualTo(1));
+    });
+  });
+}