poka
/
youtube_explode
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Version 1.9.0-nullsafety.7 

See #115
Add channel id to video search.
This commit is contained in:
Mattia 2021-03-24 10:40:22 +01:00
parent 1668f5e77b
commit d90658c4ac
13 changed files with 169 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
CHANGELOG.md
View File

@ -4,6 +4,8 @@
- Now playlists with more than 100 videos return all the videos. Thanks to @ATiltedTree.
- Implemented `ChannelAboutPage`, check the tests their usage.
- Implement filters for `search.getVideos`. See `filter` getter.
- Now video's from search queries return the channel id.
- Implemented publishDate for videos. Thanks to @mymikemiller , PR: #115.
## 1.8.0
- Fixed playlist client.

25
lib/src/extensions/helpers_extension.dart
View File

@ -14,7 +14,8 @@ extension StringUtility on String {
String substringUntil(String separator) => substring(0, indexOf(separator));
///
String substringAfter(String separator) => substring(indexOf(separator) + separator.length);
String substringAfter(String separator) =>
substring(indexOf(separator) + separator.length);
static final _exp = RegExp(r'\D');
@ -35,7 +36,8 @@ extension StringUtility on String {
while (true) {
try {
return json.decode(str.substring(startIdx, endIdx + 1)) as Map<String, dynamic>;
return json.decode(str.substring(startIdx, endIdx + 1))
as Map<String, dynamic>;
} on FormatException {
endIdx = str.lastIndexOf(str.substring(0, endIdx));
if (endIdx == 0) {
@ -58,10 +60,14 @@ extension StringUtility on String {
return Duration(seconds: int.parse(parts.first));
}
if (parts.length == 2) {
return Duration(minutes: int.parse(parts[0]), seconds: int.parse(parts[1]));
return Duration(
minutes: int.parse(parts[0]), seconds: int.parse(parts[1]));
}
if (parts.length == 3) {
return Duration(hours: int.parse(parts[0]), minutes: int.parse(parts[1]), seconds: int.parse(parts[2]));
return Duration(
hours: int.parse(parts[0]),
minutes: int.parse(parts[1]),
seconds: int.parse(parts[2]));
}
// Shouldn't reach here.
throw Error();
@ -233,9 +239,14 @@ extension RunsParser on List<dynamic> {
extension GenericExtract on List<String> {
/// Used to extract initial data that start with `var ytInitialData = ` or 'window["ytInitialData"] ='.
T extractGenericData<T>(T Function(Map<String, dynamic>) builder, Exception Function() orThrow) {
var initialData = firstWhereOrNull((e) => e.contains('var ytInitialData = '))?.extractJson('var ytInitialData = ');
initialData ??= firstWhereOrNull((e) => e.contains('window["ytInitialData"] ='))?.extractJson('window["ytInitialData"] =');
T extractGenericData<T>(
T Function(Map<String, dynamic>) builder, Exception Function() orThrow) {
var initialData =
firstWhereOrNull((e) => e.contains('var ytInitialData = '))
?.extractJson('var ytInitialData = ');
initialData ??=
firstWhereOrNull((e) => e.contains('window["ytInitialData"] ='))
?.extractJson('window["ytInitialData"] =');
if (initialData != null) {
return builder(initialData);

2
lib/src/playlists/playlist_client.dart
View File

@ -64,7 +64,7 @@ class PlaylistClient {
ThumbnailSet(videoId),
null,
Engagement(video.viewCount, null, null),
null);
false);
}
continuationToken = response.initialData.continuationToken;
if (response.initialData.continuationToken?.isEmpty ?? true) {

100
lib/src/reverse_engineering/responses/search_page.dart
View File

@ -30,7 +30,10 @@ class SearchPage {
return _initialData!;
}
final scriptText = root!.querySelectorAll('script').map((e) => e.text).toList(growable: false);
final scriptText = root!
.querySelectorAll('script')
.map((e) => e.text)
.toList(growable: false);
return scriptText.extractGenericData(
(obj) => _InitialData(obj),
() => TransientFailureException(
@ -38,36 +41,47 @@ class SearchPage {
}
///
SearchPage(this.root, this.queryString, [_InitialData? initialData]) : _initialData = initialData;
SearchPage(this.root, this.queryString, [_InitialData? initialData])
: _initialData = initialData;
Future<SearchPage?> nextPage(YoutubeHttpClient httpClient) async {
if (initialData.continuationToken == '' || initialData.estimatedResults == 0) {
if (initialData.continuationToken == '' ||
initialData.estimatedResults == 0) {
return null;
}
return get(httpClient, queryString, token: initialData.continuationToken);
}
///
static Future<SearchPage> get(YoutubeHttpClient httpClient, String queryString,
static Future<SearchPage> get(
YoutubeHttpClient httpClient, String queryString,
{String? token, SearchFilter filter = const SearchFilter('')}) {
if (token != null) {
var url = 'https://www.youtube.com/youtubei/v1/search?key=AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8';
var url =
'https://www.youtube.com/youtubei/v1/search?key=AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8';
return retry(() async {
var body = {
'context': const {
'client': {'hl': 'en', 'clientName': 'WEB', 'clientVersion': '2.20200911.04.00'}
'client': {
'hl': 'en',
'clientName': 'WEB',
'clientVersion': '2.20200911.04.00'
}
},
'continuation': token
};
var raw = await httpClient.post(Uri.parse(url), body: json.encode(body));
return SearchPage(null, queryString, _InitialData(json.decode(raw.body)));
var raw =
await httpClient.post(Uri.parse(url), body: json.encode(body));
return SearchPage(
null, queryString, _InitialData(json.decode(raw.body)));
});
// Ask for next page,
}
var url = 'https://www.youtube.com/results?search_query=${Uri.encodeQueryComponent(queryString)}&sp=${filter.value}';
var url =
'https://www.youtube.com/results?search_query=${Uri.encodeQueryComponent(queryString)}&sp=${filter.value}';
return retry(() async {
var raw = await httpClient.getString(url);
return SearchPage.parse(raw, queryString);
@ -145,7 +159,9 @@ class _InitialData {
}
// Contains only [SearchVideo] or [SearchPlaylist]
late final List<BaseSearchContent> searchContent = getContentContext()?.map(_parseContent).whereNotNull().toList() ?? const [];
late final List<BaseSearchContent> searchContent =
getContentContext()?.map(_parseContent).whereNotNull().toList() ??
const [];
List<RelatedQuery> get relatedQueries =>
getContentContext()
@ -153,8 +169,10 @@ class _InitialData {
.map((e) => e.get('horizontalCardListRenderer')?.getList('cards'))
.firstOrNull
?.map((e) => e['searchRefinementCardRenderer'])
.map((e) =>
RelatedQuery(e.searchEndpoint.searchEndpoint.query, VideoId(Uri.parse(e.thumbnail.thumbnails.first.url).pathSegments[1])))
.map((e) => RelatedQuery(
e.searchEndpoint.searchEndpoint.query,
VideoId(
Uri.parse(e.thumbnail.thumbnails.first.url).pathSegments[1])))
.toList()
.cast<RelatedQuery>() ??
const [];
@ -162,7 +180,11 @@ class _InitialData {
List<dynamic> get relatedVideos =>
getContentContext()
?.where((e) => e['shelfRenderer'] != null)
.map((e) => e.get('shelfRenderer')?.get('content')?.get('verticalListRenderer')?.getList('items'))
.map((e) => e
.get('shelfRenderer')
?.get('content')
?.get('verticalListRenderer')
?.getList('items'))
.firstOrNull
?.map(_parseContent)
.whereNotNull()
@ -171,7 +193,8 @@ class _InitialData {
late final String? continuationToken = _getContinuationToken();
late final int estimatedResults = int.parse(root.getT<String>('estimatedResults') ?? '0');
late final int estimatedResults =
int.parse(root.getT<String>('estimatedResults') ?? '0');
BaseSearchContent? _parseContent(Map<String, dynamic>? content) {
if (content == null) {
@ -186,32 +209,63 @@ class _InitialData {
_parseRuns(renderer.get('ownerText')?.getList('runs')),
_parseRuns(renderer.get('descriptionSnippet')?.getList('runs')),
renderer.get('lengthText')?.getT<String>('simpleText') ?? '',
int.parse(renderer.get('viewCountText')?.getT<String>('simpleText')?.stripNonDigits().nullIfWhitespace ??
renderer.get('viewCountText')?.getList('runs')?.firstOrNull?.getT<String>('text')?.stripNonDigits().nullIfWhitespace ??
int.parse(renderer
.get('viewCountText')
?.getT<String>('simpleText')
?.stripNonDigits()
.nullIfWhitespace ??
renderer
.get('viewCountText')
?.getList('runs')
?.firstOrNull
?.getT<String>('text')
?.stripNonDigits()
.nullIfWhitespace ??
'0'),
(renderer.get('thumbnail')?.getList('thumbnails') ?? const [])
.map((e) => Thumbnail(Uri.parse(e['url']), e['height'], e['width']))
.map((e) =>
Thumbnail(Uri.parse(e['url']), e['height'], e['width']))
.toList(),
renderer.get('publishedTimeText')?.getT<String>('simpleText'),
renderer.get('viewCountText')?.getList('runs')?.elementAtSafe(1)?.getT<String>('text')?.trim() == 'watching');
renderer
.get('viewCountText')
?.getList('runs')
?.elementAtSafe(1)
?.getT<String>('text')
?.trim() ==
'watching',
renderer['ownerText']['runs'][0]['navigationEndpoint']
['browseEndpoint']['browseId']);
}
if (content['radioRenderer'] != null) {
var renderer = content.get('radioRenderer')!;
return SearchPlaylist(PlaylistId(renderer.getT<String>('playlistId')!), renderer.get('title')!.getT<String>('simpleText')!,
int.parse(_parseRuns(renderer.get('videoCountText')?.getList('runs')).stripNonDigits().nullIfWhitespace ?? '0'));
return SearchPlaylist(
PlaylistId(renderer.getT<String>('playlistId')!),
renderer.get('title')!.getT<String>('simpleText')!,
int.parse(_parseRuns(renderer.get('videoCountText')?.getList('runs'))
.stripNonDigits()
.nullIfWhitespace ??
'0'));
}
if (content['channelRenderer'] != null) {
var renderer = content.get('channelRenderer')!;
return SearchChannel(
ChannelId(renderer.getT<String>('channelId')!),
renderer.get('title')!.getT<String>('simpleText')!,
renderer.get('descriptionSnippet')?.getList('runs')?.parseRuns() ?? '',
renderer.get('videoCountText')!.getList('runs')!.first.getT<String>('text')!.parseInt()!);
renderer.get('descriptionSnippet')?.getList('runs')?.parseRuns() ??
'',
renderer
.get('videoCountText')!
.getList('runs')!
.first
.getT<String>('text')!
.parseInt()!);
}
// Here ignore 'horizontalCardListRenderer' & 'shelfRenderer'
return null;
}
String _parseRuns(List<dynamic>? runs) => runs?.map((e) => e['text']).join() ?? '';
String _parseRuns(List<dynamic>? runs) =>
runs?.map((e) => e['text']).join() ?? '';
}

2
lib/src/reverse_engineering/youtube_http_client.dart
View File

@ -147,7 +147,7 @@ class YoutubeHttpClient extends http.BaseClient {
request.headers[key] = _defaultHeaders[key]!;
}
});
print('Request: $request');
// print('Request: $request');
// print('Stack:\n${StackTrace.current}');
return _httpClient.send(request);
}

32
lib/src/search/search_client.dart
View File

@ -21,14 +21,26 @@ class SearchClient {
/// (from the video search page).
/// The videos are sent in batch of 20 videos.
/// You [SearchList.nextPage] to get the next batch of videos.
Future<SearchList> getVideos(String searchQuery, {SearchFilter filter = const SearchFilter('')}) async {
Future<SearchList> getVideos(String searchQuery,
{SearchFilter filter = const SearchFilter('')}) async {
final page = await SearchPage.get(_httpClient, searchQuery, filter: filter);
return SearchList(
page.initialData.searchContent
.whereType<SearchVideo>()
.map((e) => Video(e.id, e.title, e.author, null, e.uploadDate?.toDateTime(), null, e.description, e.duration.toDuration(),
ThumbnailSet(e.id.value), null, Engagement(e.viewCount, null, null), e.isLive))
.map((e) => Video(
e.id,
e.title,
e.author,
ChannelId(e.channelId),
e.uploadDate?.toDateTime(),
null,
e.description,
e.duration.toDuration(),
ThumbnailSet(e.id.value),
null,
Engagement(e.viewCount, null, null),
e.isLive))
.toList(),
page,
_httpClient);
@ -37,13 +49,17 @@ class SearchClient {
/// Enumerates videos returned by the specified search query
/// (from the video search page).
/// Contains only instances of [SearchVideo] or [SearchPlaylist]
@Deprecated(
'Since version 1.9.0 this is the same as [SearchClient.getVideos].')
Stream<BaseSearchContent> getVideosFromPage(String searchQuery,
{bool onlyVideos = true, SearchFilter filter = const SearchFilter('')}) async* {
{bool onlyVideos = true,
SearchFilter filter = const SearchFilter('')}) async* {
SearchPage? page;
// ignore: literal_only_boolean_expressions
for (;;) {
if (page == null) {
page = await retry(() async => SearchPage.get(_httpClient, searchQuery, filter: filter));
page = await retry(() async =>
SearchPage.get(_httpClient, searchQuery, filter: filter));
} else {
page = await page.nextPage(_httpClient);
if (page == null) {
@ -52,7 +68,8 @@ class SearchClient {
}
if (onlyVideos) {
yield* Stream.fromIterable(page!.initialData.searchContent.whereType<SearchVideo>());
yield* Stream.fromIterable(
page!.initialData.searchContent.whereType<SearchVideo>());
} else {
yield* Stream.fromIterable(page!.initialData.searchContent);
}
@ -74,7 +91,8 @@ class SearchClient {
/// Queries to YouTube to get the results.
@Deprecated('Use getVideosFromPage instead - '
'Should be used only to get related videos')
Future<SearchQuery> queryFromPage(String searchQuery) => SearchQuery.search(_httpClient, searchQuery);
Future<SearchQuery> queryFromPage(String searchQuery) =>
SearchQuery.search(_httpClient, searchQuery);
}
/*

2
lib/src/search/search_list.dart
View File

@ -27,7 +27,7 @@ class SearchList extends DelegatingList<Video> {
e.id,
e.title,
e.author,
null,
ChannelId(e.channelId),
e.uploadDate.toDateTime(),
null,
e.description,

7
lib/src/search/search_video.dart
View File

@ -31,6 +31,9 @@ class SearchVideo extends BaseSearchContent {
/// True if this video is a live stream.
final bool isLive;
/// Channel id
final String channelId;
/// Initialize a [SearchVideo] instance.
const SearchVideo(
this.id,
@ -41,8 +44,8 @@ class SearchVideo extends BaseSearchContent {
this.viewCount,
this.thumbnails,
this.uploadDate,
this.isLive // ignore: avoid_positional_boolean_parameters
);
this.isLive, // ignore: avoid_positional_boolean_parameters
this.channelId);
@override
String toString() => '(Video) $title ($id)';

5
lib/src/videos/video.dart
View File

@ -22,8 +22,7 @@ class Video with EquatableMixin {
final String author;
/// Video author Id.
/// Note: null if the video is from a search query.
final ChannelId? channelId;
final ChannelId channelId;
/// Video upload date.
/// Note: For search queries it is calculated with:
@ -49,7 +48,7 @@ class Video with EquatableMixin {
final Engagement engagement;
/// Returns true if this is a live stream.
final bool? isLive;
final bool isLive;
/// Used internally.
/// Shouldn't be used in the code.

8
pubspec.yaml
View File

@ -1,6 +1,6 @@
name: youtube_explode_dart
description: A port in dart of the youtube explode library. Supports several API functions without the need of Youtube API Key.
version: 1.9.0-nullsafety.6
version: 1.9.0-nullsafety.7
homepage: https://github.com/Hexer10/youtube_explode_dart
@ -19,10 +19,10 @@ dependencies:
dev_dependencies:
#TODO: Add build_runner when is nnbd
# build_runner: ^1.11.5
build_runner: ^1.12.2
console: ^4.0.0
grinder: ^0.9.0-nullsafety.0
json_serializable: ^4.0.2
json_serializable: ^4.1.0
lint: ^1.5.3
pedantic: ^1.11.0
test: ^1.16.7
test: ^1.16.8

29
test/all.dart Normal file
View File

@ -0,0 +1,29 @@
import 'channel_about_test.dart' as i0;
import 'channel_id_test.dart' as i1;
import 'channel_test.dart' as i2;
import 'closed_caption_test.dart' as i3;
import 'comments_client_test.dart' as i4;
import 'playlist_id_test.dart' as i5;
import 'playlist_test.dart' as i6;
import 'search_test.dart' as i7;
import 'streams_test.dart' as i8;
import 'user_name_test.dart' as i9;
import 'user_name_test.dart' as i10;
import 'video_id_test.dart' as i11;
import 'video_test.dart' as i12;
void main() {
i0.main();
i1.main();
i2.main();
i3.main();
i4.main();
i5.main();
i6.main();
i7.main();
i8.main();
i9.main();
i10.main();
i11.main();
i12.main();
}

14
test/search_test.dart
View File

@ -18,6 +18,7 @@ void main() {
test('Search a youtube video from the search page-2', () async {
var videos = await yt!.search
// ignore: deprecated_member_use_from_same_package
.getVideosFromPage('hello')
.where((e) => e is SearchVideo) // Take only the videos.
.cast<SearchVideo>()
@ -35,14 +36,6 @@ void main() {
expect(video.thumbnails, isNotEmpty);
});
test('Search a youtube videos from the search page - old', () async {
// ignore: deprecated_member_use_from_same_package
var searchQuery = await yt!.search.queryFromPage('hello');
expect(searchQuery.content, isNotEmpty);
expect(searchQuery.relatedVideos, isNotEmpty);
expect(searchQuery.relatedQueries, isNotEmpty);
}, skip: 'Not supported anymore');
test('Search with no results - old', () async {
var query =
// ignore: deprecated_member_use_from_same_package
@ -62,9 +55,4 @@ void main() {
var video = searchQuery.content.first as SearchVideo;
expect(video.thumbnails, isNotEmpty);
});
test('Search youtube videos from search page (stream) - old', () async {
var query = await yt!.search.getVideosFromPage('hello').take(30).toList();
expect(query, hasLength(30));
});
}

4
test/video_test.dart
View File

@ -17,12 +17,14 @@ void main() {
expect(video.id.value, 'AI7ULzgf8RU');
expect(video.url, videoUrl);
expect(video.title, 'Aka no Ha [Another] +HDHR');
expect(video.channelId!.value, 'UCEnBXANsKmyj2r9xVyKoDiQ');
expect(video.channelId.value, 'UCEnBXANsKmyj2r9xVyKoDiQ');
expect(video.author, 'Tyrrrz');
var rangeMs = DateTime(2017, 09, 30, 17, 15, 26).millisecondsSinceEpoch;
// 1day margin since the uploadDate could differ from timezones
expect(video.uploadDate!.millisecondsSinceEpoch,
inInclusiveRange(rangeMs - 86400000, rangeMs + 86400000));
expect(video.publishDate!.millisecondsSinceEpoch,
inInclusiveRange(rangeMs - 86400000, rangeMs + 86400000));
expect(video.description, contains('246pp'));
// Should be 1:38 but sometimes it differs
// so we're using a 10 seconds range from it.