poka
/
youtube_explode
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

dartfmt

This commit is contained in:
Mattia 2021-03-18 22:22:55 +01:00
parent 36f3efac70
commit 95a77244a0
9 changed files with 416 additions and 211 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
lib/src/extensions/helpers_extension.dart
View File

@ -14,7 +14,8 @@ extension StringUtility on String {
String substringUntil(String separator) => substring(0, indexOf(separator));
///
String substringAfter(String separator) => substring(indexOf(separator) + separator.length);
String substringAfter(String separator) =>
substring(indexOf(separator) + separator.length);
static final _exp = RegExp(r'\D');
@ -35,7 +36,8 @@ extension StringUtility on String {
while (true) {
try {
return json.decode(str.substring(startIdx, endIdx + 1)) as Map<String, dynamic>;
return json.decode(str.substring(startIdx, endIdx + 1))
as Map<String, dynamic>;
} on FormatException {
endIdx = str.lastIndexOf(str.substring(0, endIdx));
if (endIdx == 0) {
@ -171,9 +173,14 @@ extension RunsParser on List<dynamic> {
extension GenericExtract on List<String> {
/// Used to extract initial data that start with `var ytInitialData = ` or 'window["ytInitialData"] ='.
T extractGenericData<T>(T Function(Map<String, dynamic>) builder, Exception Function() orThrow) {
var initialData = firstWhereOrNull((e) => e.contains('var ytInitialData = '))?.extractJson('var ytInitialData = ');
initialData ??= firstWhereOrNull((e) => e.contains('window["ytInitialData"] ='))?.extractJson('window["ytInitialData"] =');
T extractGenericData<T>(
T Function(Map<String, dynamic>) builder, Exception Function() orThrow) {
var initialData =
firstWhereOrNull((e) => e.contains('var ytInitialData = '))
?.extractJson('var ytInitialData = ');
initialData ??=
firstWhereOrNull((e) => e.contains('window["ytInitialData"] ='))
?.extractJson('window["ytInitialData"] =');
if (initialData != null) {
return builder(initialData);

46
lib/src/reverse_engineering/responses/channel_about_page.dart
View File

@ -16,7 +16,10 @@ class ChannelAboutPage {
late final _InitialData initialData = _getInitialData();
_InitialData _getInitialData() {
final scriptText = _root.querySelectorAll('script').map((e) => e.text).toList(growable: false);
final scriptText = _root
.querySelectorAll('script')
.map((e) => e.text)
.toList(growable: false);
return scriptText.extractGenericData(
(obj) => _InitialData(obj),
() => TransientFailureException(
@ -45,7 +48,8 @@ class ChannelAboutPage {
}
///
static Future<ChannelAboutPage> getByUsername(YoutubeHttpClient httpClient, String username) {
static Future<ChannelAboutPage> getByUsername(
YoutubeHttpClient httpClient, String username) {
var url = 'https://www.youtube.com/user/$username/about?hl=en';
return retry(() async {
@ -84,29 +88,49 @@ class _InitialData {
.get('channelAboutFullMetadataRenderer')!;
}
late final String description = content.get('description')!.getT<String>('simpleText')!;
late final String description =
content.get('description')!.getT<String>('simpleText')!;
late final List<ChannelLink> channelLinks = content
.getList('primaryLinks')!
.map((e) => ChannelLink(
e.get('title')?.getT<String>('simpleText') ?? '',
extractUrl(e.get('navigationEndpoint')?.get('commandMetadata')?.get('webCommandMetadata')?.getT<String>('url') ??
e.get('navigationEndpoint')?.get('urlEndpoint')?.getT<String>('url') ??
extractUrl(e
.get('navigationEndpoint')
?.get('commandMetadata')
?.get('webCommandMetadata')
?.getT<String>('url') ??
e
.get('navigationEndpoint')
?.get('urlEndpoint')
?.getT<String>('url') ??
''),
Uri.parse(e.get('icon')?.getList('thumbnails')?.firstOrNull?.getT<String>('url') ?? '')))
Uri.parse(e
.get('icon')
?.getList('thumbnails')
?.firstOrNull
?.getT<String>('url') ??
'')))
.toList();
late final int viewCount = int.parse(content.get('viewCountText')!.getT<String>('simpleText')!.stripNonDigits());
late final int viewCount = int.parse(content
.get('viewCountText')!
.getT<String>('simpleText')!
.stripNonDigits());
late final String joinDate = content.get('joinedDateText')!.getList('runs')![1].getT<String>('text')!;
late final String joinDate =
content.get('joinedDateText')!.getList('runs')![1].getT<String>('text')!;
late final String title = content.get('title')!.getT<String>('simpleText')!;
late final List<Map<String, dynamic>> avatar = content.get('avatar')!.getList('thumbnails')!;
late final List<Map<String, dynamic>> avatar =
content.get('avatar')!.getList('thumbnails')!;
String get country => content.get('country')!.getT<String>('simpleText')!;
String parseRuns(List<dynamic>? runs) => runs?.map((e) => e.text).join() ?? '';
String parseRuns(List<dynamic>? runs) =>
runs?.map((e) => e.text).join() ?? '';
Uri extractUrl(String text) => Uri.parse(Uri.decodeFull(_urlExp.firstMatch(text)?.group(1) ?? ''));
Uri extractUrl(String text) =>
Uri.parse(Uri.decodeFull(_urlExp.firstMatch(text)?.group(1) ?? ''));
}

210
lib/src/reverse_engineering/responses/channel_upload_page.dart
View File

@ -30,140 +30,142 @@ class ChannelUploadPage {
.map((e) => e.text)
.toList(growable: false);
return scriptText.extractGenericData((obj) => _InitialData(obj), () =>
TransientFailureException(
return scriptText.extractGenericData(
(obj) => _InitialData(obj),
() => TransientFailureException(
'Failed to retrieve initial data from the channel upload page, please report this to the project GitHub page.'));
}
///
ChannelUploadPage(this._root, this.channelId, [_InitialData ? initialData]): _initialData = initialData;
///
ChannelUploadPage(this._root, this.channelId, [_InitialData? initialData])
: _initialData = initialData;
///
Future<ChannelUploadPage?> nextPage(YoutubeHttpClient httpClient) {
///
Future<ChannelUploadPage?> nextPage(YoutubeHttpClient httpClient) {
if (initialData.continuation.isEmpty) {
return Future.value(null);
return Future.value(null);
}
var url =
'https://www.youtube.com/browse_ajax?ctoken=${initialData.continuation}&continuation=${initialData.continuation}&itct=${initialData.clickTrackingParams}';
'https://www.youtube.com/browse_ajax?ctoken=${initialData.continuation}&continuation=${initialData.continuation}&itct=${initialData.clickTrackingParams}';
return retry(() async {
var raw = await httpClient.getString(url);
return ChannelUploadPage(
null, channelId, _InitialData(json.decode(raw)[1]));
var raw = await httpClient.getString(url);
return ChannelUploadPage(
null, channelId, _InitialData(json.decode(raw)[1]));
});
}
///
static Future<ChannelUploadPage> get(
YoutubeHttpClient httpClient, String channelId, String sorting) {
var url =
'https://www.youtube.com/channel/$channelId/videos?view=0&sort=$sorting&flow=grid';
return retry(() async {
var raw = await httpClient.getString(url);
return ChannelUploadPage.parse(raw, channelId);
});
}
///
ChannelUploadPage.parse(String raw, this.channelId)
: _root = parser.parse(raw);
}
class _InitialData {
///
static Future<ChannelUploadPage> get(
YoutubeHttpClient httpClient, String channelId, String sorting) {
var url =
'https://www.youtube.com/channel/$channelId/videos?view=0&sort=$sorting&flow=grid';
return retry(() async {
var raw = await httpClient.getString(url);
return ChannelUploadPage.parse(raw, channelId);
});
}
///
ChannelUploadPage.parse(String raw, this.channelId)
: _root = parser.parse(raw);
}
class _InitialData {
// Json parsed map
final Map<String, dynamic> root;
_InitialData(this.root);
late final Map<String, dynamic>? continuationContext =
getContinuationContext();
getContinuationContext();
late final String clickTrackingParams =
continuationContext?.getT<String>('continuationContext') ?? '';
continuationContext?.getT<String>('continuationContext') ?? '';
late final List<ChannelVideo> uploads =
getContentContext().map(_parseContent).whereNotNull().toList();
getContentContext().map(_parseContent).whereNotNull().toList();
late final String continuation =
continuationContext?.getT<String>('continuation') ?? '';
continuationContext?.getT<String>('continuation') ?? '';
List<Map<String, dynamic>> getContentContext() {
List<Map<String, dynamic>>? context;
if (root.containsKey('contents')) {
context = root
.get('contents')
?.get('twoColumnBrowseResultsRenderer')
?.getList('tabs')
?.map((e) => e['tabRenderer'])
.cast<Map<String, dynamic>>()
.firstWhereOrNull((e) => e['selected'] as bool)
?.get('content')
?.get('sectionListRenderer')
?.getList('contents')
?.firstOrNull
?.get('itemSectionRenderer')
?.getList('contents')
?.firstOrNull
?.get('gridRenderer')
?.getList('items')
?.cast<Map<String, dynamic>>();
}
if (context == null && root.containsKey('response')) {
context = root
.get('response')
?.get('continuationContents')
?.get('gridContinuation')
?.getList('items')
?.cast<Map<String, dynamic>>();
}
if (context == null) {
throw FatalFailureException('Failed to get initial data context.');
}
return context;
List<Map<String, dynamic>>? context;
if (root.containsKey('contents')) {
context = root
.get('contents')
?.get('twoColumnBrowseResultsRenderer')
?.getList('tabs')
?.map((e) => e['tabRenderer'])
.cast<Map<String, dynamic>>()
.firstWhereOrNull((e) => e['selected'] as bool)
?.get('content')
?.get('sectionListRenderer')
?.getList('contents')
?.firstOrNull
?.get('itemSectionRenderer')
?.getList('contents')
?.firstOrNull
?.get('gridRenderer')
?.getList('items')
?.cast<Map<String, dynamic>>();
}
if (context == null && root.containsKey('response')) {
context = root
.get('response')
?.get('continuationContents')
?.get('gridContinuation')
?.getList('items')
?.cast<Map<String, dynamic>>();
}
if (context == null) {
throw FatalFailureException('Failed to get initial data context.');
}
return context;
}
Map<String, dynamic>? getContinuationContext() {
if (root.containsKey('contents')) {
return root
.get('contents')
?.get('twoColumnBrowseResultsRenderer')
?.getList('tabs')
?.map((e) => e['tabRenderer'])
.cast<Map<String, dynamic>>()
.firstWhereOrNull((e) => e['selected'] as bool)
?.get('content')
?.get('sectionListRenderer')
?.getList('contents')
?.firstOrNull
?.get('itemSectionRenderer')
?.getList('contents')
?.firstOrNull
?.get('gridRenderer')
?.getList('continuations')
?.firstOrNull
?.get('nextContinuationData');
}
if (root.containsKey('response')) {
return root
.get('response')
?.get('continuationContents')
?.get('gridContinuation')
?.getList('continuations')
?.firstOrNull
?.get('nextContinuationData');
}
return null;
if (root.containsKey('contents')) {
return root
.get('contents')
?.get('twoColumnBrowseResultsRenderer')
?.getList('tabs')
?.map((e) => e['tabRenderer'])
.cast<Map<String, dynamic>>()
.firstWhereOrNull((e) => e['selected'] as bool)
?.get('content')
?.get('sectionListRenderer')
?.getList('contents')
?.firstOrNull
?.get('itemSectionRenderer')
?.getList('contents')
?.firstOrNull
?.get('gridRenderer')
?.getList('continuations')
?.firstOrNull
?.get('nextContinuationData');
}
if (root.containsKey('response')) {
return root
.get('response')
?.get('continuationContents')
?.get('gridContinuation')
?.getList('continuations')
?.firstOrNull
?.get('nextContinuationData');
}
return null;
}
ChannelVideo? _parseContent(Map<String, dynamic>? content) {
if (content == null || !content.containsKey('gridVideoRenderer')) {
return null;
}
if (content == null || !content.containsKey('gridVideoRenderer')) {
return null;
}
var video = content.get('gridVideoRenderer')!;
return ChannelVideo(
VideoId(video.getT<String>('videoId')!),
video.get('title')?.getT<String>('simpleText') ??
video.get('title')?.getList('runs')?.map((e) => e['text']).join() ??
'');
}
var video = content.get('gridVideoRenderer')!;
return ChannelVideo(
VideoId(video.getT<String>('videoId')!),
video.get('title')?.getT<String>('simpleText') ??
video.get('title')?.getList('runs')?.map((e) => e['text']).join() ??
'');
}
}

9
lib/src/reverse_engineering/responses/embed_page.dart
View File

@ -9,7 +9,8 @@ import 'player_config_base.dart';
///
class EmbedPage {
static final _playerConfigExp = RegExp('[\'""]PLAYER_CONFIG[\'""]\\s*:\\s*(\\{.*\\})');
static final _playerConfigExp =
RegExp('[\'""]PLAYER_CONFIG[\'""]\\s*:\\s*(\\{.*\\})');
static final _playerConfigExp2 = RegExp(r'yt.setConfig\((\{.*\})');
final Document root;
@ -21,7 +22,8 @@ class EmbedPage {
.querySelectorAll('*[name="player_ias/base"]')
.map((e) => e.attributes['src'])
.where((e) => !e.isNullOrWhiteSpace)
.firstWhere((e) => e!.contains('player_ias') && e.endsWith('.js'), orElse: () => null);
.firstWhere((e) => e!.contains('player_ias') && e.endsWith('.js'),
orElse: () => null);
// _root.querySelector('*[name="player_ias/base"]').attributes['src'];
if (url == null) {
return null;
@ -31,7 +33,8 @@ class EmbedPage {
///
EmbedPlayerConfig? getPlayerConfig() {
var playerConfigJson = (_playerConfigJson ?? _playerConfigJson2)?.extractJson();
var playerConfigJson =
(_playerConfigJson ?? _playerConfigJson2)?.extractJson();
if (playerConfigJson == null) {
return null;
}

88
lib/src/reverse_engineering/responses/playlist_page.dart
View File

@ -24,7 +24,10 @@ class PlaylistPage {
return _initialData!;
}
final scriptText = root!.querySelectorAll('script').map((e) => e.text).toList(growable: false);
final scriptText = root!
.querySelectorAll('script')
.map((e) => e.text)
.toList(growable: false);
return scriptText.extractGenericData(
(obj) => _InitialData(obj),
@ -33,7 +36,8 @@ class PlaylistPage {
}
///
PlaylistPage(this.root, this.playlistId, [_InitialData? initialData]) : _initialData = initialData;
PlaylistPage(this.root, this.playlistId, [_InitialData? initialData])
: _initialData = initialData;
///
Future<PlaylistPage?> nextPage(YoutubeHttpClient httpClient) async {
@ -44,19 +48,26 @@ class PlaylistPage {
}
///
static Future<PlaylistPage> get(YoutubeHttpClient httpClient, String id, {String? token}) {
static Future<PlaylistPage> get(YoutubeHttpClient httpClient, String id,
{String? token}) {
if (token != null && token.isNotEmpty) {
var url = 'https://www.youtube.com/youtubei/v1/guide?key=AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8';
var url =
'https://www.youtube.com/youtubei/v1/guide?key=AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8';
return retry(() async {
var body = {
'context': const {
'client': {'hl': 'en', 'clientName': 'WEB', 'clientVersion': '2.20200911.04.00'}
'client': {
'hl': 'en',
'clientName': 'WEB',
'clientVersion': '2.20200911.04.00'
}
},
'continuation': token
};
var raw = await httpClient.post(Uri.parse(url), body: json.encode(body));
var raw =
await httpClient.post(Uri.parse(url), body: json.encode(body));
return PlaylistPage(null, id, _InitialData(json.decode(raw.body)));
});
// Ask for next page,
@ -80,7 +91,10 @@ class _InitialData {
_InitialData(this.root);
late final String? title = root.get('metadata')?.get('playlistMetadataRenderer')?.getT<String>('title');
late final String? title = root
.get('metadata')
?.get('playlistMetadataRenderer')
?.getT<String>('title');
late final String? author = root
.get('sidebar')
@ -94,7 +108,10 @@ class _InitialData {
?.getT<List<dynamic>>('runs')
?.parseRuns();
late final String? description = root.get('metadata')?.get('playlistMetadataRenderer')?.getT<String>('description');
late final String? description = root
.get('metadata')
?.get('playlistMetadataRenderer')
?.getT<String>('description');
late final int? viewCount = root
.get('sidebar')
@ -107,12 +124,13 @@ class _InitialData {
?.getT<String>('simpleText')
?.parseInt();
late final String? continuationToken = (videosContent ?? playlistVideosContent)
?.firstWhereOrNull((e) => e['continuationItemRenderer'] != null)
?.get('continuationItemRenderer')
?.get('continuationEndpoint')
?.get('continuationCommand')
?.getT<String>('token');
late final String? continuationToken =
(videosContent ?? playlistVideosContent)
?.firstWhereOrNull((e) => e['continuationItemRenderer'] != null)
?.get('continuationItemRenderer')
?.get('continuationEndpoint')
?.get('continuationCommand')
?.getT<String>('token');
List<Map<String, dynamic>>? get playlistVideosContent =>
root
@ -130,7 +148,11 @@ class _InitialData {
?.firstOrNull
?.get('playlistVideoListRenderer')
?.getList('contents') ??
root.getList('onResponseReceivedActions')?.firstOrNull?.get('appendContinuationItemsAction')?.getList('continuationItems');
root
.getList('onResponseReceivedActions')
?.firstOrNull
?.get('appendContinuationItemsAction')
?.getList('continuationItems');
late final List<Map<String, dynamic>>? videosContent = root
.get('contents')
@ -138,10 +160,17 @@ class _InitialData {
?.get('primaryContents')
?.get('sectionListRenderer')
?.getList('contents') ??
root.getList('onResponseReceivedCommands')?.firstOrNull?.get('appendContinuationItemsAction')?.getList('continuationItems');
root
.getList('onResponseReceivedCommands')
?.firstOrNull
?.get('appendContinuationItemsAction')
?.getList('continuationItems');
List<_Video> get playlistVideos =>
playlistVideosContent?.where((e) => e['playlistVideoRenderer'] != null).map((e) => _Video(e['playlistVideoRenderer'])).toList() ??
playlistVideosContent
?.where((e) => e['playlistVideoRenderer'] != null)
.map((e) => _Video(e['playlistVideoRenderer']))
.toList() ??
const [];
List<_Video> get videos =>
@ -168,7 +197,13 @@ class _Video {
'';
String get channelId =>
root.get('ownerText')?.getList('runs')?.firstOrNull?.get('navigationEndpoint')?.get('browseEndpoint')?.getT<String>('browseId') ??
root
.get('ownerText')
?.getList('runs')
?.firstOrNull
?.get('navigationEndpoint')
?.get('browseEndpoint')
?.getT<String>('browseId') ??
root
.get('shortBylineText')
?.getList('runs')
@ -180,11 +215,14 @@ class _Video {
String get title => root.get('title')?.getList('runs')?.parseRuns() ?? '';
String get description => root.getList('descriptionSnippet')?.parseRuns() ?? '';
String get description =>
root.getList('descriptionSnippet')?.parseRuns() ?? '';
Duration? get duration => _stringToDuration(root.get('lengthText')?.getT<String>('simpleText'));
Duration? get duration =>
_stringToDuration(root.get('lengthText')?.getT<String>('simpleText'));
int get viewCount => root.get('viewCountText')?.getT<String>('simpleText')?.parseInt() ?? 0;
int get viewCount =>
root.get('viewCountText')?.getT<String>('simpleText')?.parseInt() ?? 0;
/// Format: HH:MM:SS
static Duration? _stringToDuration(String? string) {
@ -199,10 +237,14 @@ class _Video {
return Duration(seconds: int.parse(parts.first));
}
if (parts.length == 2) {
return Duration(minutes: int.parse(parts.first), seconds: int.parse(parts[1]));
return Duration(
minutes: int.parse(parts.first), seconds: int.parse(parts[1]));
}
if (parts.length == 3) {
return Duration(hours: int.parse(parts[0]), minutes: int.parse(parts[1]), seconds: int.parse(parts[2]));
return Duration(
hours: int.parse(parts[0]),
minutes: int.parse(parts[1]),
seconds: int.parse(parts[2]));
}
throw Error();
}

89
lib/src/reverse_engineering/responses/search_page.dart
View File

@ -28,7 +28,10 @@ class SearchPage {
return _initialData!;
}
final scriptText = root!.querySelectorAll('script').map((e) => e.text).toList(growable: false);
final scriptText = root!
.querySelectorAll('script')
.map((e) => e.text)
.toList(growable: false);
return scriptText.extractGenericData(
(obj) => _InitialData(obj),
() => TransientFailureException(
@ -36,35 +39,47 @@ class SearchPage {
}
///
SearchPage(this.root, this.queryString, [_InitialData? initialData]) : _initialData = initialData;
SearchPage(this.root, this.queryString, [_InitialData? initialData])
: _initialData = initialData;
Future<SearchPage?> nextPage(YoutubeHttpClient httpClient) async {
if (initialData.continuationToken == '' || initialData.estimatedResults == 0) {
if (initialData.continuationToken == '' ||
initialData.estimatedResults == 0) {
return null;
}
return get(httpClient, queryString, token: initialData.continuationToken);
}
///
static Future<SearchPage> get(YoutubeHttpClient httpClient, String queryString, {String? token}) {
static Future<SearchPage> get(
YoutubeHttpClient httpClient, String queryString,
{String? token}) {
if (token != null) {
var url = 'https://www.youtube.com/youtubei/v1/search?key=AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8';
var url =
'https://www.youtube.com/youtubei/v1/search?key=AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8';
return retry(() async {
var body = {
'context': const {
'client': {'hl': 'en', 'clientName': 'WEB', 'clientVersion': '2.20200911.04.00'}
'client': {
'hl': 'en',
'clientName': 'WEB',
'clientVersion': '2.20200911.04.00'
}
},
'continuation': token
};
var raw = await httpClient.post(Uri.parse(url), body: json.encode(body));
return SearchPage(null, queryString, _InitialData(json.decode(raw.body)));
var raw =
await httpClient.post(Uri.parse(url), body: json.encode(body));
return SearchPage(
null, queryString, _InitialData(json.decode(raw.body)));
});
// Ask for next page,
}
var url = 'https://www.youtube.com/results?search_query=${Uri.encodeQueryComponent(queryString)}';
var url =
'https://www.youtube.com/results?search_query=${Uri.encodeQueryComponent(queryString)}';
return retry(() async {
var raw = await httpClient.getString(url);
return SearchPage.parse(raw, queryString);
@ -142,7 +157,9 @@ class _InitialData {
}
// Contains only [SearchVideo] or [SearchPlaylist]
late final List<BaseSearchContent> searchContent = getContentContext()?.map(_parseContent).whereNotNull().toList() ?? const [];
late final List<BaseSearchContent> searchContent =
getContentContext()?.map(_parseContent).whereNotNull().toList() ??
const [];
List<RelatedQuery> get relatedQueries =>
getContentContext()
@ -150,8 +167,10 @@ class _InitialData {
.map((e) => e.get('horizontalCardListRenderer')?.getList('cards'))
.firstOrNull
?.map((e) => e['searchRefinementCardRenderer'])
.map((e) =>
RelatedQuery(e.searchEndpoint.searchEndpoint.query, VideoId(Uri.parse(e.thumbnail.thumbnails.first.url).pathSegments[1])))
.map((e) => RelatedQuery(
e.searchEndpoint.searchEndpoint.query,
VideoId(
Uri.parse(e.thumbnail.thumbnails.first.url).pathSegments[1])))
.toList()
.cast<RelatedQuery>() ??
const [];
@ -159,7 +178,11 @@ class _InitialData {
List<dynamic> get relatedVideos =>
getContentContext()
?.where((e) => e['shelfRenderer'] != null)
.map((e) => e.get('shelfRenderer')?.get('content')?.get('verticalListRenderer')?.getList('items'))
.map((e) => e
.get('shelfRenderer')
?.get('content')
?.get('verticalListRenderer')
?.getList('items'))
.firstOrNull
?.map(_parseContent)
.whereNotNull()
@ -168,7 +191,8 @@ class _InitialData {
late final String? continuationToken = _getContinuationToken();
late final int estimatedResults = int.parse(root.getT<String>('estimatedResults') ?? '0');
late final int estimatedResults =
int.parse(root.getT<String>('estimatedResults') ?? '0');
BaseSearchContent? _parseContent(Map<String, dynamic>? content) {
if (content == null) {
@ -183,24 +207,47 @@ class _InitialData {
_parseRuns(renderer.get('ownerText')?.getList('runs')),
_parseRuns(renderer.get('descriptionSnippet')?.getList('runs')),
renderer.get('lengthText')?.getT<String>('simpleText') ?? '',
int.parse(renderer.get('viewCountText')?.getT<String>('simpleText')?.stripNonDigits().nullIfWhitespace ??
renderer.get('viewCountText')?.getList('runs')?.firstOrNull?.getT<String>('text')?.stripNonDigits().nullIfWhitespace ??
int.parse(renderer
.get('viewCountText')
?.getT<String>('simpleText')
?.stripNonDigits()
.nullIfWhitespace ??
renderer
.get('viewCountText')
?.getList('runs')
?.firstOrNull
?.getT<String>('text')
?.stripNonDigits()
.nullIfWhitespace ??
'0'),
(renderer.get('thumbnail')?.getList('thumbnails') ?? const [])
.map((e) => Thumbnail(Uri.parse(e['url']), e['height'], e['width']))
.map((e) =>
Thumbnail(Uri.parse(e['url']), e['height'], e['width']))
.toList(),
renderer.get('publishedTimeText')?.getT<String>('simpleText'),
renderer.get('viewCountText')?.getList('runs')?.elementAtSafe(1)?.getT<String>('text')?.trim() == 'watching');
renderer
.get('viewCountText')
?.getList('runs')
?.elementAtSafe(1)
?.getT<String>('text')
?.trim() ==
'watching');
}
if (content['radioRenderer'] != null) {
var renderer = content.get('radioRenderer')!;
return SearchPlaylist(PlaylistId(renderer.getT<String>('playlistId')!), renderer.get('title')!.getT<String>('simpleText')!,
int.parse(_parseRuns(renderer.get('videoCountText')?.getList('runs')).stripNonDigits().nullIfWhitespace ?? '0'));
return SearchPlaylist(
PlaylistId(renderer.getT<String>('playlistId')!),
renderer.get('title')!.getT<String>('simpleText')!,
int.parse(_parseRuns(renderer.get('videoCountText')?.getList('runs'))
.stripNonDigits()
.nullIfWhitespace ??
'0'));
}
// Here ignore 'horizontalCardListRenderer' & 'shelfRenderer'
return null;
}
String _parseRuns(List<dynamic>? runs) => runs?.map((e) => e['text']).join() ?? '';
String _parseRuns(List<dynamic>? runs) =>
runs?.map((e) => e['text']).join() ?? '';
}

71
lib/src/reverse_engineering/responses/watch_page.dart
View File

@ -12,11 +12,15 @@ import 'player_response.dart';
///
class WatchPage {
static final RegExp _videoLikeExp = RegExp(r'"label"\s*:\s*"([\d,\.]+) likes"');
static final RegExp _videoDislikeExp = RegExp(r'"label"\s*:\s*"([\d,\.]+) dislikes"');
static final RegExp _visitorInfoLiveExp = RegExp('VISITOR_INFO1_LIVE=([^;]+)');
static final RegExp _videoLikeExp =
RegExp(r'"label"\s*:\s*"([\d,\.]+) likes"');
static final RegExp _videoDislikeExp =
RegExp(r'"label"\s*:\s*"([\d,\.]+) dislikes"');
static final RegExp _visitorInfoLiveExp =
RegExp('VISITOR_INFO1_LIVE=([^;]+)');
static final RegExp _yscExp = RegExp('YSC=([^;]+)');
static final RegExp _playerResponseExp = RegExp(r'var\s+ytInitialPlayerResponse\s*=\s*(\{.*\})');
static final RegExp _playerResponseExp =
RegExp(r'var\s+ytInitialPlayerResponse\s*=\s*(\{.*\})');
static final _xsfrTokenExp = RegExp(r'"XSRF_TOKEN"\s*:\s*"(.+?)"');
@ -51,7 +55,10 @@ class WatchPage {
return _initialData!;
}
final scriptText = root.querySelectorAll('script').map((e) => e.text).toList(growable: false);
final scriptText = root
.querySelectorAll('script')
.map((e) => e.text)
.toList(growable: false);
return scriptText.extractGenericData(
(obj) => _InitialData(obj),
() => TransientFailureException(
@ -62,23 +69,45 @@ class WatchPage {
///
String? getXsfrToken() {
return _xsfrTokenExp.firstMatch(root.querySelectorAll('script').firstWhere((e) => _xsfrTokenExp.hasMatch(e.text)).text)?.group(1);
return _xsfrTokenExp
.firstMatch(root
.querySelectorAll('script')
.firstWhere((e) => _xsfrTokenExp.hasMatch(e.text))
.text)
?.group(1);
}
///
bool get isOk => root.body?.querySelector('#player') != null;
///
bool get isVideoAvailable => root.querySelector('meta[property="og:url"]') != null;
bool get isVideoAvailable =>
root.querySelector('meta[property="og:url"]') != null;
///
int get videoLikeCount => int.parse(_videoLikeExp.firstMatch(root.outerHtml)?.group(1)?.stripNonDigits().nullIfWhitespace ??
root.querySelector('.like-button-renderer-like-button')?.text.stripNonDigits().nullIfWhitespace ??
int get videoLikeCount => int.parse(_videoLikeExp
.firstMatch(root.outerHtml)
?.group(1)
?.stripNonDigits()
.nullIfWhitespace ??
root
.querySelector('.like-button-renderer-like-button')
?.text
.stripNonDigits()
.nullIfWhitespace ??
'0');
///
int get videoDislikeCount => int.parse(_videoDislikeExp.firstMatch(root.outerHtml)?.group(1)?.stripNonDigits().nullIfWhitespace ??
root.querySelector('.like-button-renderer-dislike-button')?.text.stripNonDigits().nullIfWhitespace ??
int get videoDislikeCount => int.parse(_videoDislikeExp
.firstMatch(root.outerHtml)
?.group(1)
?.stripNonDigits()
.nullIfWhitespace ??
root
.querySelector('.like-button-renderer-dislike-button')
?.text
.stripNonDigits()
.nullIfWhitespace ??
'0');
static final _playerConfigExp = RegExp(r'ytplayer\.config\s*=\s*(\{.*\})');
@ -89,7 +118,10 @@ class WatchPage {
///
WatchPlayerConfig? getPlayerConfig() {
final jsonMap = _playerConfigExp.firstMatch(root.getElementsByTagName('html').first.text)?.group(1)?.extractJson();
final jsonMap = _playerConfigExp
.firstMatch(root.getElementsByTagName('html').first.text)
?.group(1)
?.extractJson();
if (jsonMap == null) {
return null;
}
@ -114,7 +146,8 @@ class WatchPage {
WatchPage(this.root, this.visitorInfoLive, this.ysc);
///
WatchPage.parse(String raw, this.visitorInfoLive, this.ysc) : root = parser.parse(raw);
WatchPage.parse(String raw, this.visitorInfoLive, this.ysc)
: root = parser.parse(raw);
///
static Future<WatchPage> get(YoutubeHttpClient httpClient, String videoId) {
@ -148,10 +181,12 @@ class WatchPlayerConfig implements PlayerConfigBase<Map<String, dynamic>> {
WatchPlayerConfig(this.root);
@override
late final String sourceUrl = 'https://youtube.com${root.get('assets')!.getT<String>('js')}';
late final String sourceUrl =
'https://youtube.com${root.get('assets')!.getT<String>('js')}';
///
late final PlayerResponse playerResponse = PlayerResponse.parse(root.get('args')!.getT<String>('playerResponse')!);
late final PlayerResponse playerResponse =
PlayerResponse.parse(root.get('args')!.getT<String>('playerResponse')!);
}
class _InitialData {
@ -177,7 +212,9 @@ class _InitialData {
return null;
}
late final String continuation = getContinuationContext()?.getT<String>('continuation') ?? '';
late final String continuation =
getContinuationContext()?.getT<String>('continuation') ?? '';
late final String clickTrackingParams = getContinuationContext()?.getT<String>('clickTrackingParams') ?? '';
late final String clickTrackingParams =
getContinuationContext()?.getT<String>('clickTrackingParams') ?? '';
}

96
lib/src/videos/streams/streams_client.dart
View File

@ -21,8 +21,10 @@ class StreamsClient {
/// Initializes an instance of [StreamsClient]
StreamsClient(this._httpClient);
Future<DashManifest> _getDashManifest(Uri dashManifestUrl, Iterable<CipherOperation> cipherOperations) {
var signature = DashManifest.getSignatureFromUrl(dashManifestUrl.toString());
Future<DashManifest> _getDashManifest(
Uri dashManifestUrl, Iterable<CipherOperation> cipherOperations) {
var signature =
DashManifest.getSignatureFromUrl(dashManifestUrl.toString());
if (!signature.isNullOrWhiteSpace) {
signature = cipherOperations.decipher(signature!);
dashManifestUrl = dashManifestUrl.setQueryParam('signature', signature);
@ -37,30 +39,38 @@ class StreamsClient {
throw VideoUnplayableException.unplayable(videoId);
}
var playerSource = await PlayerSource.get(_httpClient, embedPage.sourceUrl ?? playerConfig.sourceUrl);
var playerSource = await PlayerSource.get(
_httpClient, embedPage.sourceUrl ?? playerConfig.sourceUrl);
var cipherOperations = playerSource.getCipherOperations();
var videoInfoResponse = await VideoInfoResponse.get(_httpClient, videoId.toString(), playerSource.sts);
var videoInfoResponse = await VideoInfoResponse.get(
_httpClient, videoId.toString(), playerSource.sts);
var playerResponse = videoInfoResponse.playerResponse;
var previewVideoId = playerResponse.previewVideoId;
if (!previewVideoId.isNullOrWhiteSpace) {
throw VideoRequiresPurchaseException.preview(videoId, VideoId(previewVideoId!));
throw VideoRequiresPurchaseException.preview(
videoId, VideoId(previewVideoId!));
}
if (!playerResponse.isVideoPlayable) {
throw VideoUnplayableException.unplayable(videoId, reason: playerResponse.videoPlayabilityError ?? '');
throw VideoUnplayableException.unplayable(videoId,
reason: playerResponse.videoPlayabilityError ?? '');
}
if (playerResponse.isLive) {
throw VideoUnplayableException.liveStream(videoId);
}
var streamInfoProviders = <StreamInfoProvider>[...videoInfoResponse.streams, ...playerResponse.streams];
var streamInfoProviders = <StreamInfoProvider>[
...videoInfoResponse.streams,
...playerResponse.streams
];
var dashManifestUrl = playerResponse.dashManifestUrl;
if (!dashManifestUrl.isNullOrWhiteSpace) {
var dashManifest = await _getDashManifest(Uri.parse(dashManifestUrl!), cipherOperations);
var dashManifest =
await _getDashManifest(Uri.parse(dashManifestUrl!), cipherOperations);
streamInfoProviders.addAll(dashManifest.streams);
}
return StreamContext(streamInfoProviders, cipherOperations);
@ -71,22 +81,28 @@ class StreamsClient {
final playerConfig = watchPage.playerConfig;
var playerResponse = playerConfig?.playerResponse ?? watchPage.playerResponse;
var playerResponse =
playerConfig?.playerResponse ?? watchPage.playerResponse;
if (playerResponse == null) {
throw VideoUnplayableException.unplayable(videoId);
}
var previewVideoId = playerResponse.previewVideoId;
if (!previewVideoId.isNullOrWhiteSpace) {
throw VideoRequiresPurchaseException.preview(videoId, VideoId(previewVideoId!));
throw VideoRequiresPurchaseException.preview(
videoId, VideoId(previewVideoId!));
}
var playerSourceUrl = watchPage.sourceUrl ?? playerConfig?.sourceUrl;
var playerSource = !playerSourceUrl.isNullOrWhiteSpace ? await PlayerSource.get(_httpClient, playerSourceUrl!) : null;
var cipherOperations = playerSource?.getCipherOperations() ?? const <CipherOperation>[];
var playerSource = !playerSourceUrl.isNullOrWhiteSpace
? await PlayerSource.get(_httpClient, playerSourceUrl!)
: null;
var cipherOperations =
playerSource?.getCipherOperations() ?? const <CipherOperation>[];
if (!playerResponse.isVideoPlayable) {
throw VideoUnplayableException.unplayable(videoId, reason: playerResponse.videoPlayabilityError ?? '');
throw VideoUnplayableException.unplayable(videoId,
reason: playerResponse.videoPlayabilityError ?? '');
}
if (playerResponse.isLive) {
@ -99,7 +115,8 @@ class StreamsClient {
var dashManifestUrl = playerResponse.dashManifestUrl;
if (!(dashManifestUrl?.isNullOrWhiteSpace ?? true)) {
var dashManifest = await _getDashManifest(Uri.parse(dashManifestUrl!), cipherOperations);
var dashManifest =
await _getDashManifest(Uri.parse(dashManifestUrl!), cipherOperations);
streamInfoProviders.addAll(dashManifest.streams);
}
return StreamContext(streamInfoProviders, cipherOperations);
@ -123,7 +140,9 @@ class StreamsClient {
}
// Content length
var contentLength = streamInfo.contentLength ?? await _httpClient.getContentLength(url, validate: false) ?? 0;
var contentLength = streamInfo.contentLength ??
await _httpClient.getContentLength(url, validate: false) ??
0;
if (contentLength <= 0) {
continue;
@ -140,31 +159,53 @@ class StreamsClient {
// Muxed or Video-only
if (!videoCodec.isNullOrWhiteSpace) {
var framerate = Framerate(streamInfo.framerate ?? 24);
var videoQualityLabel =
streamInfo.videoQualityLabel ?? VideoQualityUtil.getLabelFromTagWithFramerate(tag, framerate.framesPerSecond.toDouble());
var videoQualityLabel = streamInfo.videoQualityLabel ??
VideoQualityUtil.getLabelFromTagWithFramerate(
tag, framerate.framesPerSecond.toDouble());
var videoQuality = VideoQualityUtil.fromLabel(videoQualityLabel);
var videoWidth = streamInfo.videoWidth;
var videoHeight = streamInfo.videoHeight;
var videoResolution =
videoWidth != -1 && videoHeight != -1 ? VideoResolution(videoWidth ?? 0, videoHeight ?? 0) : videoQuality.toVideoResolution();
var videoResolution = videoWidth != -1 && videoHeight != -1
? VideoResolution(videoWidth ?? 0, videoHeight ?? 0)
: videoQuality.toVideoResolution();
// Muxed
if (!audioCodec.isNullOrWhiteSpace) {
streams[tag] = MuxedStreamInfo(tag, url, container, fileSize, bitrate, audioCodec!, videoCodec!, videoQualityLabel, videoQuality,
videoResolution, framerate);
streams[tag] = MuxedStreamInfo(
tag,
url,
container,
fileSize,
bitrate,
audioCodec!,
videoCodec!,
videoQualityLabel,
videoQuality,
videoResolution,
framerate);
continue;
}
// Video only
streams[tag] = VideoOnlyStreamInfo(
tag, url, container, fileSize, bitrate, videoCodec!, videoQualityLabel, videoQuality, videoResolution, framerate);
tag,
url,
container,
fileSize,
bitrate,
videoCodec!,
videoQualityLabel,
videoQuality,
videoResolution,
framerate);
continue;
}
// Audio-only
if (!audioCodec.isNullOrWhiteSpace) {
streams[tag] = AudioOnlyStreamInfo(tag, url, container, fileSize, bitrate, audioCodec!);
streams[tag] = AudioOnlyStreamInfo(
tag, url, container, fileSize, bitrate, audioCodec!);
}
// #if DEBUG
@ -194,10 +235,12 @@ class StreamsClient {
/// Gets the HTTP Live Stream (HLS) manifest URL
/// for the specified video (if it's a live video stream).
Future<String> getHttpLiveStreamUrl(VideoId videoId) async {
var videoInfoResponse = await VideoInfoResponse.get(_httpClient, videoId.toString());
var videoInfoResponse =
await VideoInfoResponse.get(_httpClient, videoId.toString());
var playerResponse = videoInfoResponse.playerResponse;
if (!playerResponse.isVideoPlayable) {
throw VideoUnplayableException.unplayable(videoId, reason: playerResponse.videoPlayabilityError ?? '');
throw VideoUnplayableException.unplayable(videoId,
reason: playerResponse.videoPlayabilityError ?? '');
}
var hlsManifest = playerResponse.hlsManifestUrl;
@ -208,5 +251,6 @@ class StreamsClient {
}
/// Gets the actual stream which is identified by the specified metadata.
Stream<List<int>> get(StreamInfo streamInfo) => _httpClient.getStream(streamInfo);
Stream<List<int>> get(StreamInfo streamInfo) =>
_httpClient.getStream(streamInfo);
}

1
test/search_test.dart
View File

@ -17,7 +17,6 @@ void main() {
});
test('Search a youtube video from the search page-2', () async {
var videos = await yt!.search
.getVideosFromPage('hello')
.where((e) => e is SearchVideo) // Take only the videos.