From 73504b2a4405d0aee3967368b9a94bd0eaa22619 Mon Sep 17 00:00:00 2001 From: Mattia Date: Thu, 22 Jul 2021 15:03:07 +0200 Subject: [PATCH] Version 1.10.0 Code refactoring Fix #144 Bumped min sdk version to 2.13.0 Changed comments api interface (still experimental) --- CHANGELOG.md | 5 + lib/src/channels/channel.dart | 7 +- lib/src/channels/channel_client.dart | 11 +- lib/src/extensions/helpers_extension.dart | 53 +++++-- .../models/youtube_page.dart | 1 + .../reverse_engineering/pages/watch_page.dart | 31 ++-- ...sponse.dart => closed_caption_client.dart} | 12 +- .../responses/comment_request_response.dart | 0 .../responses/comments_client.dart | 117 +++++++++++++++ ...o_response.dart => video_info_client.dart} | 12 +- .../closed_caption_client.dart | 11 +- lib/src/videos/comments/comment.dart | 7 - lib/src/videos/comments/comment.freezed.dart | 104 ++------------ lib/src/videos/comments/comments_client.dart | 136 +++--------------- lib/src/videos/streams/streams_client.dart | 24 +--- lib/src/videos/video_client.dart | 9 +- pubspec.yaml | 2 +- test/comments_client_test.dart | 2 +- 18 files changed, 250 insertions(+), 294 deletions(-) rename lib/src/reverse_engineering/responses/{closed_caption_track_response.dart => closed_caption_client.dart} (80%) delete mode 100644 lib/src/reverse_engineering/responses/comment_request_response.dart create mode 100644 lib/src/reverse_engineering/responses/comments_client.dart rename lib/src/reverse_engineering/responses/{video_info_response.dart => video_info_client.dart} (93%) diff --git a/CHANGELOG.md b/CHANGELOG.md index ea74ef4..2a1ad56 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +## 1.10.0 +- Fix issue #144: get_video_info was removed from yt. +- Min sdk version now is 2.13.0 +- BREAKING CHANGE: New comments API implementation. + ## 1.9.10 - Close #139: Implement Channel.subscribersCount. diff --git a/lib/src/channels/channel.dart b/lib/src/channels/channel.dart index 2d301a9..610757c 100644 --- a/lib/src/channels/channel.dart +++ b/lib/src/channels/channel.dart @@ -5,10 +5,9 @@ import 'channel_id.dart'; part 'channel.freezed.dart'; /// YouTube channel metadata. -@Freezed() +@freezed class Channel with _$Channel { - const Channel._(); - + /// const factory Channel( /// Channel ID. ChannelId id, @@ -25,4 +24,6 @@ class Channel with _$Channel { /// Channel URL. String get url => 'https://www.youtube.com/channel/$id'; + + const Channel._(); } diff --git a/lib/src/channels/channel_client.dart b/lib/src/channels/channel_client.dart index 1b2cbc7..9e2b845 100644 --- a/lib/src/channels/channel_client.dart +++ b/lib/src/channels/channel_client.dart @@ -1,6 +1,6 @@ import 'package:youtube_explode_dart/src/channels/channel_uploads_list.dart'; import 'package:youtube_explode_dart/src/reverse_engineering/pages/channel_page.dart'; -import 'package:youtube_explode_dart/src/reverse_engineering/responses/video_info_response.dart'; +import 'package:youtube_explode_dart/src/reverse_engineering/pages/watch_page.dart'; import '../common/common.dart'; import '../extensions/helpers_extension.dart'; @@ -54,7 +54,7 @@ class ChannelClient { channelId = ChannelId.fromString(channelId); final aboutPage = await ChannelAboutPage.get(_httpClient, channelId.value); - final id = aboutPage.initialData; + return ChannelAbout( aboutPage.description, aboutPage.viewCount, @@ -76,6 +76,8 @@ class ChannelClient { var channelAboutPage = await ChannelAboutPage.getByUsername(_httpClient, username.value); + + // TODO: Expose metadata from the [ChannelAboutPage] class. var id = channelAboutPage.initialData; return ChannelAbout( id.description, @@ -94,9 +96,8 @@ class ChannelClient { /// that uploaded the specified video. Future getByVideo(dynamic videoId) async { videoId = VideoId.fromString(videoId); - var videoInfoResponse = - await VideoInfoResponse.get(_httpClient, videoId.value); - var playerResponse = videoInfoResponse.playerResponse; + var videoInfoResponse = await WatchPage.get(_httpClient, videoId.value); + var playerResponse = videoInfoResponse.playerResponse!; var channelId = playerResponse.videoChannelId; return get(ChannelId(channelId)); diff --git a/lib/src/extensions/helpers_extension.dart b/lib/src/extensions/helpers_extension.dart index 83c02cf..2a6631e 100644 --- a/lib/src/extensions/helpers_extension.dart +++ b/lib/src/extensions/helpers_extension.dart @@ -80,10 +80,44 @@ extension StringUtility on String { /// Utility for Strings. extension StringUtility2 on String? { + static final RegExp _unitSplit = RegExp(r'^(\d+(?:\.\d)?)(\w)'); + /// Parses this value as int stripping the non digit characters, /// returns null if this fails. int? parseInt() => int.tryParse(this?.stripNonDigits() ?? ''); + int? parseIntWithUnits() { + if (this == null) { + return null; + } + final match = _unitSplit.firstMatch(this!.trim()); + if (match == null) { + return null; + } + if (match.groupCount != 2) { + return null; + } + + final count = double.tryParse(match.group(1) ?? ''); + if (count == null) { + return null; + } + + final multiplierText = match.group(2); + if (multiplierText == null) { + return null; + } + + var multiplier = 1; + if (multiplierText == 'K') { + multiplier = 1000; + } else if (multiplierText == 'M') { + multiplier = 1000000; + } + + return (count * multiplier).toInt(); + } + /// Returns true if the string is null or empty. bool get isNullOrWhiteSpace { if (this == null) { @@ -235,19 +269,18 @@ extension RunsParser on List { } extension GenericExtract on List { - /// Used to extract initial data that start with `var ytInitialData = ` or 'window["ytInitialData"] ='. - T extractGenericData( + /// Used to extract initial data. + T extractGenericData(List match, T Function(Map) builder, Exception Function() orThrow) { - var initialData = - firstWhereOrNull((e) => e.contains('var ytInitialData = ')) - ?.extractJson('var ytInitialData = '); - initialData ??= - firstWhereOrNull((e) => e.contains('window["ytInitialData"] =')) - ?.extractJson('window["ytInitialData"] ='); + JsonMap? initialData; - if (initialData != null) { - return builder(initialData); + for (final m in match) { + initialData = firstWhereOrNull((e) => e.contains(m))?.extractJson(m); + if (initialData != null) { + return builder(initialData); + } } + throw orThrow(); } } diff --git a/lib/src/reverse_engineering/models/youtube_page.dart b/lib/src/reverse_engineering/models/youtube_page.dart index 3591479..8aaeb35 100644 --- a/lib/src/reverse_engineering/models/youtube_page.dart +++ b/lib/src/reverse_engineering/models/youtube_page.dart @@ -23,6 +23,7 @@ abstract class YoutubePage { .map((e) => e.text) .toList(growable: false); return scriptText.extractGenericData( + ['var ytInitialData = ', 'window["ytInitialData"] ='], initialDataBuilder!, () => TransientFailureException( 'Failed to retrieve initial data from $runtimeType, please report this to the project GitHub page.')); diff --git a/lib/src/reverse_engineering/pages/watch_page.dart b/lib/src/reverse_engineering/pages/watch_page.dart index 9713a73..da83bb4 100644 --- a/lib/src/reverse_engineering/pages/watch_page.dart +++ b/lib/src/reverse_engineering/pages/watch_page.dart @@ -93,6 +93,8 @@ class WatchPage extends YoutubePage<_InitialData> { .nullIfWhitespace ?? '0'); + String? get commentsContinuation => initialData.commentsContinuation; + static final _playerConfigExp = RegExp(r'ytplayer\.config\s*=\s*(\{.*\})'); late final WatchPlayerConfig? playerConfig = getPlayerConfig(); @@ -111,18 +113,16 @@ class WatchPage extends YoutubePage<_InitialData> { return WatchPlayerConfig(jsonMap); } - /// PlayerResponse? getPlayerResponse() { - final val = root + final scriptText = root .querySelectorAll('script') .map((e) => e.text) - .map((e) => _playerResponseExp.firstMatch(e)?.group(1)) - .firstWhereOrNull((e) => !e.isNullOrWhiteSpace) - ?.extractJson(); - if (val == null) { - return null; - } - return PlayerResponse(val); + .toList(growable: false); + return scriptText.extractGenericData( + ['var ytInitialPlayerResponse = '], + (root) => PlayerResponse(root), + () => TransientFailureException( + 'Failed to retrieve initial player response, please report this to the project GitHub page.')); } /// @@ -183,16 +183,15 @@ class _InitialData extends InitialData { ?.getList('contents') ?.firstWhere((e) => e['itemSectionRenderer'] != null) .get('itemSectionRenderer') - ?.getList('continuations') + ?.getList('contents') ?.firstOrNull - ?.get('nextContinuationData'); + ?.get('continuationItemRenderer') + ?.get('continuationEndpoint') + ?.get('continuationCommand'); } return null; } - late final String continuation = - getContinuationContext()?.getT('continuation') ?? ''; - - late final String clickTrackingParams = - getContinuationContext()?.getT('clickTrackingParams') ?? ''; + late final String commentsContinuation = + getContinuationContext()?.getT('token') ?? ''; } diff --git a/lib/src/reverse_engineering/responses/closed_caption_track_response.dart b/lib/src/reverse_engineering/responses/closed_caption_client.dart similarity index 80% rename from lib/src/reverse_engineering/responses/closed_caption_track_response.dart rename to lib/src/reverse_engineering/responses/closed_caption_client.dart index 31e2b4a..bb924b5 100644 --- a/lib/src/reverse_engineering/responses/closed_caption_track_response.dart +++ b/lib/src/reverse_engineering/responses/closed_caption_client.dart @@ -5,7 +5,7 @@ import '../../retry.dart'; import '../youtube_http_client.dart'; /// -class ClosedCaptionTrackResponse { +class ClosedCaptionClient { final xml.XmlDocument root; /// @@ -13,19 +13,19 @@ class ClosedCaptionTrackResponse { root.findAllElements('p').map((e) => ClosedCaption._(e)); /// - ClosedCaptionTrackResponse(this.root); + ClosedCaptionClient(this.root); /// // ignore: deprecated_member_use - ClosedCaptionTrackResponse.parse(String raw) : root = xml.parse(raw); + ClosedCaptionClient.parse(String raw) : root = xml.parse(raw); /// - static Future get( + static Future get( YoutubeHttpClient httpClient, Uri url) { - var formatUrl = url.replaceQueryParameters({'fmt': 'srv3'}); + final formatUrl = url.replaceQueryParameters({'fmt': 'srv3'}); return retry(() async { var raw = await httpClient.getString(formatUrl); - return ClosedCaptionTrackResponse.parse(raw); + return ClosedCaptionClient.parse(raw); }); } } diff --git a/lib/src/reverse_engineering/responses/comment_request_response.dart b/lib/src/reverse_engineering/responses/comment_request_response.dart deleted file mode 100644 index e69de29..0000000 diff --git a/lib/src/reverse_engineering/responses/comments_client.dart b/lib/src/reverse_engineering/responses/comments_client.dart new file mode 100644 index 0000000..67d3a97 --- /dev/null +++ b/lib/src/reverse_engineering/responses/comments_client.dart @@ -0,0 +1,117 @@ +import 'package:collection/collection.dart'; +import 'package:youtube_explode_dart/src/reverse_engineering/pages/watch_page.dart'; + +import '../../../youtube_explode_dart.dart'; +import '../../extensions/helpers_extension.dart'; +import '../../retry.dart'; +import '../youtube_http_client.dart'; + +class CommentsClient { + final JsonMap root; + + late final List _commentRenderers = _getCommentRenderers(); + + late final List<_Comment> comments = + _commentRenderers.map((e) => _Comment(e)).toList(growable: false); + + CommentsClient(this.root); + + /// + static Future get( + YoutubeHttpClient httpClient, Video video) async { + final watchPage = video.watchPage ?? + await retry( + () async => WatchPage.get(httpClient, video.id.value)); + + final continuation = watchPage.commentsContinuation; + if (continuation == null) { + return null; + } + + final data = await httpClient.sendPost('next', continuation); + return CommentsClient(data); + } + + List _getCommentRenderers() { + return root + .getList('onResponseReceivedEndpoints')![1] + .get('reloadContinuationItemsCommand')! + .getList('continuationItems')! + .where((e) => e['commentThreadRenderer'] != null) + .map((e) => e.get('commentThreadRenderer')!) + .toList(growable: false); + } +} + +class _Comment { + final JsonMap root; + + late final JsonMap _commentRenderer = + root.get('comment')!.get('commentRenderer')!; + + late final JsonMap? _commentRepliesRenderer = + root.get('replies')?.get('commentRepliesRenderer'); + + /// Used to get replies + late final String? continuation = _commentRepliesRenderer + ?.getList('contents') + ?.firstOrNull + ?.get('continuationItemRenderer') + ?.get('continuationEndpoint') + ?.get('continuationCommand') + ?.getT('token'); + + late final int? repliesCount = _commentRepliesRenderer + ?.get('viewReplies') + ?.get('buttonRenderer') + ?.get('text') + ?.getList('runs') + ?.elementAtSafe(2) + ?.getT('text') + ?.parseIntWithUnits(); + + late final String author = + _commentRenderer.get('authorText')!.getT('simpleText')!; + + late final String channelThumbnail = _commentRenderer + .get('authorThumbnail')! + .getList('thumbnails')! + .last + .getT('url')!; + + late final String channelId = _commentRenderer + .get('authorEndpoint')! + .get('browseEndpoint')! + .getT('browseId')!; + + late final String text = _commentRenderer + .get('contentText')! + .getT>('runs')! + .parseRuns(); + + late final String publishTime = _commentRenderer + .get('publishedTimeText')! + .getList('runs')! + .first + .getT('text')!; + + /// Needs to be parsed as an int current is like: 1.2K + late final int? likeCount = _commentRenderer + .get('actionButtons') + ?.get('commentActionButtonsRenderer') + ?.get('likeButton') + ?.get('toggleButtonRenderer') + ?.get('defaultServiceEndpoint') + ?.get('performCommentActionEndpoint') + ?.getList('clientActions') + ?.first + .get('updateCommentVoteAction') + ?.get('voteCount') + ?.getT('simpleText') + ?.parseIntWithUnits(); + + _Comment(this.root); + + @override + String toString() => '$author: $text'; +} diff --git a/lib/src/reverse_engineering/responses/video_info_response.dart b/lib/src/reverse_engineering/responses/video_info_client.dart similarity index 93% rename from lib/src/reverse_engineering/responses/video_info_response.dart rename to lib/src/reverse_engineering/responses/video_info_client.dart index 6f7efd9..279b6f7 100644 --- a/lib/src/reverse_engineering/responses/video_info_response.dart +++ b/lib/src/reverse_engineering/responses/video_info_client.dart @@ -8,7 +8,9 @@ import '../player/player_response.dart'; import '../models/stream_info_provider.dart'; /// -class VideoInfoResponse { +/// +@deprecated +class VideoInfoClient { final Map root; /// @@ -43,13 +45,13 @@ class VideoInfoResponse { ]; /// - VideoInfoResponse(this.root); + VideoInfoClient(this.root); /// - VideoInfoResponse.parse(String raw) : root = Uri.splitQueryString(raw); + VideoInfoClient.parse(String raw) : root = Uri.splitQueryString(raw); /// - static Future get( + static Future get( YoutubeHttpClient httpClient, String videoId, [String? sts]) { var eurl = Uri.encodeFull('https://youtube.googleapis.com/v/$videoId'); @@ -71,7 +73,7 @@ class VideoInfoResponse { return retry(() async { var raw = await httpClient.getString(url); - var result = VideoInfoResponse.parse(raw); + var result = VideoInfoClient.parse(raw); if (!result.isVideoAvailable || !result.playerResponse.isVideoAvailable) { throw VideoUnplayableException(videoId); diff --git a/lib/src/videos/closed_captions/closed_caption_client.dart b/lib/src/videos/closed_captions/closed_caption_client.dart index 4ec1be9..fa38be4 100644 --- a/lib/src/videos/closed_captions/closed_caption_client.dart +++ b/lib/src/videos/closed_captions/closed_caption_client.dart @@ -1,7 +1,7 @@ import '../../extensions/helpers_extension.dart'; -import '../../reverse_engineering/responses/closed_caption_track_response.dart' - show ClosedCaptionTrackResponse; -import '../../reverse_engineering/responses/video_info_response.dart'; +import '../../reverse_engineering/responses/closed_caption_client.dart' as re + show ClosedCaptionClient; +import '../../reverse_engineering/responses/video_info_client.dart'; import '../../reverse_engineering/youtube_http_client.dart'; import '../videos.dart'; import 'closed_caption.dart'; @@ -35,7 +35,7 @@ class ClosedCaptionClient { videoId = VideoId.fromString(videoId); var tracks = {}; var videoInfoResponse = - await VideoInfoResponse.get(_httpClient, videoId.value); + await VideoInfoClient.get(_httpClient, videoId.value); var playerResponse = videoInfoResponse.playerResponse; for (final track in playerResponse.closedCaptionTrack) { @@ -54,8 +54,7 @@ class ClosedCaptionClient { /// Gets the actual closed caption track which is /// identified by the specified metadata. Future get(ClosedCaptionTrackInfo trackInfo) async { - var response = - await ClosedCaptionTrackResponse.get(_httpClient, trackInfo.url); + var response = await re.ClosedCaptionClient.get(_httpClient, trackInfo.url); var captions = response.closedCaptions .where((e) => !e.text.isNullOrWhiteSpace) diff --git a/lib/src/videos/comments/comment.dart b/lib/src/videos/comments/comment.dart index 27bbefa..558b124 100644 --- a/lib/src/videos/comments/comment.dart +++ b/lib/src/videos/comments/comment.dart @@ -9,9 +9,6 @@ part 'comment.freezed.dart'; class Comment with _$Comment { /// Initializes an instance of [Comment] const factory Comment( - /// Comment id. - String commentId, - /// Comment author name. String author, @@ -33,9 +30,5 @@ class Comment with _$Comment { /// Used internally. /// Shouldn't be used in the code. @internal String? continuation, - - /// Used internally. - /// Shouldn't be used in the code. - @internal String? clicktrackingParams, ) = _Comment; } diff --git a/lib/src/videos/comments/comment.freezed.dart b/lib/src/videos/comments/comment.freezed.dart index 732a1f5..e4d37cc 100644 --- a/lib/src/videos/comments/comment.freezed.dart +++ b/lib/src/videos/comments/comment.freezed.dart @@ -16,18 +16,9 @@ final _privateConstructorUsedError = UnsupportedError( class _$CommentTearOff { const _$CommentTearOff(); - _Comment call( - String commentId, - String author, - ChannelId channelId, - String text, - int likeCount, - String publishedTime, - int replyCount, - @internal String? continuation, - @internal String? clicktrackingParams) { + _Comment call(String author, ChannelId channelId, String text, int likeCount, + String publishedTime, int replyCount, @internal String? continuation) { return _Comment( - commentId, author, channelId, text, @@ -35,7 +26,6 @@ class _$CommentTearOff { publishedTime, replyCount, continuation, - clicktrackingParams, ); } } @@ -45,9 +35,6 @@ const $Comment = _$CommentTearOff(); /// @nodoc mixin _$Comment { - /// Comment id. - String get commentId => throw _privateConstructorUsedError; - /// Comment author name. String get author => throw _privateConstructorUsedError; @@ -71,11 +58,6 @@ mixin _$Comment { @internal String? get continuation => throw _privateConstructorUsedError; - /// Used internally. - /// Shouldn't be used in the code. - @internal - String? get clicktrackingParams => throw _privateConstructorUsedError; - @JsonKey(ignore: true) $CommentCopyWith get copyWith => throw _privateConstructorUsedError; } @@ -85,15 +67,13 @@ abstract class $CommentCopyWith<$Res> { factory $CommentCopyWith(Comment value, $Res Function(Comment) then) = _$CommentCopyWithImpl<$Res>; $Res call( - {String commentId, - String author, + {String author, ChannelId channelId, String text, int likeCount, String publishedTime, int replyCount, - @internal String? continuation, - @internal String? clicktrackingParams}); + @internal String? continuation}); $ChannelIdCopyWith<$Res> get channelId; } @@ -108,7 +88,6 @@ class _$CommentCopyWithImpl<$Res> implements $CommentCopyWith<$Res> { @override $Res call({ - Object? commentId = freezed, Object? author = freezed, Object? channelId = freezed, Object? text = freezed, @@ -116,13 +95,8 @@ class _$CommentCopyWithImpl<$Res> implements $CommentCopyWith<$Res> { Object? publishedTime = freezed, Object? replyCount = freezed, Object? continuation = freezed, - Object? clicktrackingParams = freezed, }) { return _then(_value.copyWith( - commentId: commentId == freezed - ? _value.commentId - : commentId // ignore: cast_nullable_to_non_nullable - as String, author: author == freezed ? _value.author : author // ignore: cast_nullable_to_non_nullable @@ -151,10 +125,6 @@ class _$CommentCopyWithImpl<$Res> implements $CommentCopyWith<$Res> { ? _value.continuation : continuation // ignore: cast_nullable_to_non_nullable as String?, - clicktrackingParams: clicktrackingParams == freezed - ? _value.clicktrackingParams - : clicktrackingParams // ignore: cast_nullable_to_non_nullable - as String?, )); } @@ -172,15 +142,13 @@ abstract class _$CommentCopyWith<$Res> implements $CommentCopyWith<$Res> { __$CommentCopyWithImpl<$Res>; @override $Res call( - {String commentId, - String author, + {String author, ChannelId channelId, String text, int likeCount, String publishedTime, int replyCount, - @internal String? continuation, - @internal String? clicktrackingParams}); + @internal String? continuation}); @override $ChannelIdCopyWith<$Res> get channelId; @@ -197,7 +165,6 @@ class __$CommentCopyWithImpl<$Res> extends _$CommentCopyWithImpl<$Res> @override $Res call({ - Object? commentId = freezed, Object? author = freezed, Object? channelId = freezed, Object? text = freezed, @@ -205,13 +172,8 @@ class __$CommentCopyWithImpl<$Res> extends _$CommentCopyWithImpl<$Res> Object? publishedTime = freezed, Object? replyCount = freezed, Object? continuation = freezed, - Object? clicktrackingParams = freezed, }) { return _then(_Comment( - commentId == freezed - ? _value.commentId - : commentId // ignore: cast_nullable_to_non_nullable - as String, author == freezed ? _value.author : author // ignore: cast_nullable_to_non_nullable @@ -240,10 +202,6 @@ class __$CommentCopyWithImpl<$Res> extends _$CommentCopyWithImpl<$Res> ? _value.continuation : continuation // ignore: cast_nullable_to_non_nullable as String?, - clicktrackingParams == freezed - ? _value.clicktrackingParams - : clicktrackingParams // ignore: cast_nullable_to_non_nullable - as String?, )); } } @@ -251,23 +209,11 @@ class __$CommentCopyWithImpl<$Res> extends _$CommentCopyWithImpl<$Res> /// @nodoc class _$_Comment implements _Comment { - const _$_Comment( - this.commentId, - this.author, - this.channelId, - this.text, - this.likeCount, - this.publishedTime, - this.replyCount, - @internal this.continuation, - @internal this.clicktrackingParams); + const _$_Comment(this.author, this.channelId, this.text, this.likeCount, + this.publishedTime, this.replyCount, @internal this.continuation); @override - /// Comment id. - final String commentId; - @override - /// Comment author name. final String author; @override @@ -296,25 +242,16 @@ class _$_Comment implements _Comment { /// Shouldn't be used in the code. @internal final String? continuation; - @override - - /// Used internally. - /// Shouldn't be used in the code. - @internal - final String? clicktrackingParams; @override String toString() { - return 'Comment(commentId: $commentId, author: $author, channelId: $channelId, text: $text, likeCount: $likeCount, publishedTime: $publishedTime, replyCount: $replyCount, continuation: $continuation, clicktrackingParams: $clicktrackingParams)'; + return 'Comment(author: $author, channelId: $channelId, text: $text, likeCount: $likeCount, publishedTime: $publishedTime, replyCount: $replyCount, continuation: $continuation)'; } @override bool operator ==(dynamic other) { return identical(this, other) || (other is _Comment && - (identical(other.commentId, commentId) || - const DeepCollectionEquality() - .equals(other.commentId, commentId)) && (identical(other.author, author) || const DeepCollectionEquality().equals(other.author, author)) && (identical(other.channelId, channelId) || @@ -333,24 +270,19 @@ class _$_Comment implements _Comment { .equals(other.replyCount, replyCount)) && (identical(other.continuation, continuation) || const DeepCollectionEquality() - .equals(other.continuation, continuation)) && - (identical(other.clicktrackingParams, clicktrackingParams) || - const DeepCollectionEquality() - .equals(other.clicktrackingParams, clicktrackingParams))); + .equals(other.continuation, continuation))); } @override int get hashCode => runtimeType.hashCode ^ - const DeepCollectionEquality().hash(commentId) ^ const DeepCollectionEquality().hash(author) ^ const DeepCollectionEquality().hash(channelId) ^ const DeepCollectionEquality().hash(text) ^ const DeepCollectionEquality().hash(likeCount) ^ const DeepCollectionEquality().hash(publishedTime) ^ const DeepCollectionEquality().hash(replyCount) ^ - const DeepCollectionEquality().hash(continuation) ^ - const DeepCollectionEquality().hash(clicktrackingParams); + const DeepCollectionEquality().hash(continuation); @JsonKey(ignore: true) @override @@ -360,22 +292,16 @@ class _$_Comment implements _Comment { abstract class _Comment implements Comment { const factory _Comment( - String commentId, String author, ChannelId channelId, String text, int likeCount, String publishedTime, int replyCount, - @internal String? continuation, - @internal String? clicktrackingParams) = _$_Comment; + @internal String? continuation) = _$_Comment; @override - /// Comment id. - String get commentId => throw _privateConstructorUsedError; - @override - /// Comment author name. String get author => throw _privateConstructorUsedError; @override @@ -405,12 +331,6 @@ abstract class _Comment implements Comment { @internal String? get continuation => throw _privateConstructorUsedError; @override - - /// Used internally. - /// Shouldn't be used in the code. - @internal - String? get clicktrackingParams => throw _privateConstructorUsedError; - @override @JsonKey(ignore: true) _$CommentCopyWith<_Comment> get copyWith => throw _privateConstructorUsedError; diff --git a/lib/src/videos/comments/comments_client.dart b/lib/src/videos/comments/comments_client.dart index 670241d..7f99693 100644 --- a/lib/src/videos/comments/comments_client.dart +++ b/lib/src/videos/comments/comments_client.dart @@ -1,47 +1,20 @@ -import 'dart:convert'; +import 'package:freezed_annotation/freezed_annotation.dart'; import '../../channels/channel_id.dart'; import '../../extensions/helpers_extension.dart'; -import '../../retry.dart'; +import '../../reverse_engineering/responses/comments_client.dart' as re; import '../../reverse_engineering/youtube_http_client.dart'; import '../videos.dart'; import 'comment.dart'; /// Queries related to comments of YouTube videos. +@experimental class CommentsClient { final YoutubeHttpClient _httpClient; /// Initializes an instance of [CommentsClient] CommentsClient(this._httpClient); - /// Returns the json parsed comments map. - Future> _getCommentJson( - String continuation, - String clickTrackingParams, - String xsfrToken, - String visitorInfoLive, - String ysc) async { - final url = Uri( - scheme: 'https', - host: 'www.youtube.com', - path: '/next', - queryParameters: { - 'key': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8', - }); - - return retry(() async { - var raw = await _httpClient.postString(url, headers: { - 'x-youtube-client-name': '1', - 'x-youtube-client-version': '2.20210622.10.00', - 'cookie': - 'YSC=$ysc; CONSENT=YES+cb; GPS=1; VISITOR_INFO1_LIVE=$visitorInfoLive', - }, body: { - 'session_token': xsfrToken - }); - return json.decode(raw); - }); - } - /// Returns a stream emitting all the [video]'s comment. /// A request is page for every comment page, /// a page contains at most 20 comments, use .take if you want to limit @@ -50,98 +23,23 @@ class CommentsClient { /// The streams doesn't emit any data if [Video.hasWatchPage] is false. /// Use `videos.get(videoId, forceWatchPage: true)` to assure that the /// WatchPage is fetched. - Stream getComments(Video video) async* { + Future> getComments(Video video) async { if (video.watchPage == null) { - return; + return const []; } - yield* _getComments( - video.watchPage!.initialData.continuation, - video.watchPage!.initialData.clickTrackingParams, - video.watchPage!.xsfrToken, - video.watchPage!.visitorInfoLive, - video.watchPage!.ysc); - } - Stream _getComments(String continuation, String clickTrackingParams, - String xsfrToken, String visitorInfoLive, String ysc) async* { - // contents.twoColumnWatchNextResults.results.results.contents[2](firstWhere itemSectionRenderer != null).itemSectionRenderer.contents[0].continuationItemRenderer - var data = await _getCommentJson( - continuation, clickTrackingParams, xsfrToken, visitorInfoLive, ysc); - var contentRoot = data - .get('response') - ?.get('continuationContents') - ?.get('itemSectionContinuation') - ?.getT>('contents') - ?.map((e) => e['commentThreadRenderer']) - .toList() - .cast>(); - if (contentRoot == null) { - return; - } - for (final content in contentRoot) { - var commentRaw = content.get('comment')!.get('commentRenderer')!; - String? continuation; - String? clickTrackingParams; - final replies = content.get('replies'); - if (replies != null) { - final continuationData = replies - .get('commentRepliesRenderer')! - .getList('continuations')! - .first - .get('nextContinuationData')!; + final page = await re.CommentsClient.get(_httpClient, video); - continuation = continuationData.getT('continuation'); - clickTrackingParams = - continuationData.getT('clickTrackingParams'); - } - yield Comment( - commentRaw.getT('commentId')!, - commentRaw.get('authorText')!.getT('simpleText')!, - ChannelId(commentRaw - .get('authorEndpoint')! - .get('browseEndpoint')! - .getT('browseId')!), - commentRaw - .get('contentText')! - .getT>('runs')! - .parseRuns(), - commentRaw.get('voteCount')?.getT('simpleText')?.parseInt() ?? - commentRaw - .get('voteCount') - ?.getT>('runs') - ?.parseRuns() - .parseInt() ?? - 0, - commentRaw - .get('publishedTimeText')! - .getT>('runs')! - .parseRuns(), - commentRaw.getT('replyCount') ?? 0, - continuation, - clickTrackingParams); - } - var continuationRoot = (data - .get('response') - ?.get('continuationContents') - ?.get('itemSectionContinuation') - ?.getT>('continuations') - ?.first) - ?.get('nextContinuationData'); - if (continuationRoot != null) { - yield* _getComments( - continuationRoot['continuation'], - continuationRoot['clickTrackingParams'], - xsfrToken, - visitorInfoLive, - ysc); - } - } - - Stream getReplies(Video video, Comment comment) async* { - if (video.watchPage == null || - comment.continuation == null || - comment.clicktrackingParams == null) { - return; - } + return page?.comments + .map((e) => Comment( + e.author, + ChannelId(e.channelId), + e.text, + e.likeCount ?? 0, + e.publishTime, + e.repliesCount ?? 0, + e.continuation)) + .toList(growable: false) ?? + const []; } } diff --git a/lib/src/videos/streams/streams_client.dart b/lib/src/videos/streams/streams_client.dart index 3b23f36..eff79b9 100644 --- a/lib/src/videos/streams/streams_client.dart +++ b/lib/src/videos/streams/streams_client.dart @@ -1,13 +1,13 @@ import '../../exceptions/exceptions.dart'; import '../../extensions/helpers_extension.dart'; import '../../reverse_engineering/cipher/cipher_operations.dart'; +import '../../reverse_engineering/dash_manifest.dart'; import '../../reverse_engineering/heuristics.dart'; +import '../../reverse_engineering/models/stream_info_provider.dart'; import '../../reverse_engineering/pages/embed_page.dart'; import '../../reverse_engineering/pages/watch_page.dart'; -import '../../reverse_engineering/dash_manifest.dart'; import '../../reverse_engineering/player/player_source.dart'; -import '../../reverse_engineering/models/stream_info_provider.dart'; -import '../../reverse_engineering/responses/video_info_response.dart'; +import '../../reverse_engineering/responses/video_info_client.dart'; import '../../reverse_engineering/youtube_http_client.dart'; import '../video_id.dart'; import 'bitrate.dart'; @@ -48,7 +48,7 @@ class StreamsClient { _httpClient, embedPage.sourceUrl ?? playerConfig.sourceUrl); var cipherOperations = playerSource.getCipherOperations(); - var videoInfoResponse = await VideoInfoResponse.get( + var videoInfoResponse = await VideoInfoClient.get( _httpClient, videoId.toString(), playerSource.sts); var playerResponse = videoInfoResponse.playerResponse; @@ -224,21 +224,9 @@ class StreamsClient { /// about available streams in the specified video. Future getManifest(dynamic videoId) async { videoId = VideoId.fromString(videoId); - // We can try to extract the manifest from two sources: - // get_video_info and the video watch page. - // In some cases one works, in some cases another does. - try { - var context = await _getStreamContextFromVideoInfo(videoId); - return _getManifest(context); - } on YoutubeExplodeException catch (e) { - try { - var context = await _getStreamContextFromWatchPage(videoId); - return _getManifest(context); - } on YoutubeExplodeException catch (e1) { - throw e..combine(e1); - } - } + var context = await _getStreamContextFromWatchPage(videoId); + return _getManifest(context); } /// Gets the HTTP Live Stream (HLS) manifest URL diff --git a/lib/src/videos/video_client.dart b/lib/src/videos/video_client.dart index f65cb75..e662a2b 100644 --- a/lib/src/videos/video_client.dart +++ b/lib/src/videos/video_client.dart @@ -1,8 +1,9 @@ +import 'package:youtube_explode_dart/src/reverse_engineering/player/player_response.dart'; + import '../channels/channel_id.dart'; import '../common/common.dart'; import '../extensions/helpers_extension.dart'; import '../reverse_engineering/pages/watch_page.dart'; -import '../reverse_engineering/responses/video_info_response.dart'; import '../reverse_engineering/youtube_http_client.dart'; import 'closed_captions/closed_caption_client.dart'; import 'comments/comments_client.dart'; @@ -29,11 +30,9 @@ class VideoClient { /// Gets the metadata associated with the specified video. Future