Version 1.9.0-nullsafety.6

Fix #112
This commit is contained in:
Mattia 2021-03-20 18:31:53 +01:00
parent a07921a8ae
commit 4dc635678e
11 changed files with 295 additions and 203 deletions

View File

@ -3,6 +3,7 @@
- New api: `getQuerySuggestions`: Returns the suggestions youtube provides while making a video search.
- Now playlists with more than 100 videos return all the videos. Thanks to @ATiltedTree.
- Implemented `ChannelAboutPage`, check the tests their usage.
- Implement filters for `search.getVideos`. See `filter` getter.
## 1.8.0
- Fixed playlist client.

View File

@ -14,8 +14,7 @@ extension StringUtility on String {
String substringUntil(String separator) => substring(0, indexOf(separator));
///
String substringAfter(String separator) =>
substring(indexOf(separator) + separator.length);
String substringAfter(String separator) => substring(indexOf(separator) + separator.length);
static final _exp = RegExp(r'\D');
@ -36,8 +35,7 @@ extension StringUtility on String {
while (true) {
try {
return json.decode(str.substring(startIdx, endIdx + 1))
as Map<String, dynamic>;
return json.decode(str.substring(startIdx, endIdx + 1)) as Map<String, dynamic>;
} on FormatException {
endIdx = str.lastIndexOf(str.substring(0, endIdx));
if (endIdx == 0) {
@ -47,6 +45,28 @@ extension StringUtility on String {
}
}
/// Format: HH:MM:SS
Duration? toDuration() {
if (/*string == null ||*/ trim().isEmpty) {
return null;
}
var parts = split(':');
assert(parts.length <= 3);
if (parts.length == 1) {
return Duration(seconds: int.parse(parts.first));
}
if (parts.length == 2) {
return Duration(minutes: int.parse(parts[0]), seconds: int.parse(parts[1]));
}
if (parts.length == 3) {
return Duration(hours: int.parse(parts[0]), minutes: int.parse(parts[1]), seconds: int.parse(parts[2]));
}
// Shouldn't reach here.
throw Error();
}
DateTime parseDateTime() => DateTime.parse(this);
}
@ -66,6 +86,46 @@ extension StringUtility2 on String? {
}
return false;
}
/// Format: <quantity> <unit> ago (5 years ago)
DateTime? toDateTime() {
if (this == null) {
return null;
}
var parts = this!.split(' ');
if (parts.length == 4) {
// Streamed x y ago
parts = parts.skip(1).toList();
}
assert(parts.length == 3);
var qty = int.parse(parts.first);
// Try to get the unit
var unit = parts[1];
Duration time;
if (unit.startsWith('second')) {
time = Duration(seconds: qty);
} else if (unit.startsWith('minute')) {
time = Duration(minutes: qty);
} else if (unit.startsWith('hour')) {
time = Duration(hours: qty);
} else if (unit.startsWith('day')) {
time = Duration(days: qty);
} else if (unit.startsWith('week')) {
time = Duration(days: qty * 7);
} else if (unit.startsWith('month')) {
time = Duration(days: qty * 30);
} else if (unit.startsWith('year')) {
time = Duration(days: qty * 365);
} else {
throw StateError('Couldn\'t parse $unit unit of time. '
'Please report this to the project page!');
}
return DateTime.now().subtract(time);
}
}
/// List decipher utility.
@ -173,14 +233,9 @@ extension RunsParser on List<dynamic> {
extension GenericExtract on List<String> {
/// Used to extract initial data that start with `var ytInitialData = ` or 'window["ytInitialData"] ='.
T extractGenericData<T>(
T Function(Map<String, dynamic>) builder, Exception Function() orThrow) {
var initialData =
firstWhereOrNull((e) => e.contains('var ytInitialData = '))
?.extractJson('var ytInitialData = ');
initialData ??=
firstWhereOrNull((e) => e.contains('window["ytInitialData"] ='))
?.extractJson('window["ytInitialData"] =');
T extractGenericData<T>(T Function(Map<String, dynamic>) builder, Exception Function() orThrow) {
var initialData = firstWhereOrNull((e) => e.contains('var ytInitialData = '))?.extractJson('var ytInitialData = ');
initialData ??= firstWhereOrNull((e) => e.contains('window["ytInitialData"] ='))?.extractJson('window["ytInitialData"] =');
if (initialData != null) {
return builder(initialData);

View File

@ -3,12 +3,14 @@ import 'dart:convert';
import 'package:collection/collection.dart';
import 'package:html/dom.dart';
import 'package:html/parser.dart' as parser;
import 'package:youtube_explode_dart/src/search/search_channel.dart';
import '../../../youtube_explode_dart.dart';
import '../../extensions/helpers_extension.dart';
import '../../retry.dart';
import '../../search/base_search_content.dart';
import '../../search/related_query.dart';
import '../../search/search_filter.dart';
import '../../search/search_video.dart';
import '../../videos/videos.dart';
import '../youtube_http_client.dart';
@ -28,10 +30,7 @@ class SearchPage {
return _initialData!;
}
final scriptText = root!
.querySelectorAll('script')
.map((e) => e.text)
.toList(growable: false);
final scriptText = root!.querySelectorAll('script').map((e) => e.text).toList(growable: false);
return scriptText.extractGenericData(
(obj) => _InitialData(obj),
() => TransientFailureException(
@ -39,47 +38,36 @@ class SearchPage {
}
///
SearchPage(this.root, this.queryString, [_InitialData? initialData])
: _initialData = initialData;
SearchPage(this.root, this.queryString, [_InitialData? initialData]) : _initialData = initialData;
Future<SearchPage?> nextPage(YoutubeHttpClient httpClient) async {
if (initialData.continuationToken == '' ||
initialData.estimatedResults == 0) {
if (initialData.continuationToken == '' || initialData.estimatedResults == 0) {
return null;
}
return get(httpClient, queryString, token: initialData.continuationToken);
}
///
static Future<SearchPage> get(
YoutubeHttpClient httpClient, String queryString,
{String? token}) {
static Future<SearchPage> get(YoutubeHttpClient httpClient, String queryString,
{String? token, SearchFilter filter = const SearchFilter('')}) {
if (token != null) {
var url =
'https://www.youtube.com/youtubei/v1/search?key=AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8';
var url = 'https://www.youtube.com/youtubei/v1/search?key=AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8';
return retry(() async {
var body = {
'context': const {
'client': {
'hl': 'en',
'clientName': 'WEB',
'clientVersion': '2.20200911.04.00'
}
'client': {'hl': 'en', 'clientName': 'WEB', 'clientVersion': '2.20200911.04.00'}
},
'continuation': token
};
var raw =
await httpClient.post(Uri.parse(url), body: json.encode(body));
return SearchPage(
null, queryString, _InitialData(json.decode(raw.body)));
var raw = await httpClient.post(Uri.parse(url), body: json.encode(body));
return SearchPage(null, queryString, _InitialData(json.decode(raw.body)));
});
// Ask for next page,
}
var url =
'https://www.youtube.com/results?search_query=${Uri.encodeQueryComponent(queryString)}';
var url = 'https://www.youtube.com/results?search_query=${Uri.encodeQueryComponent(queryString)}&sp=${filter.value}';
return retry(() async {
var raw = await httpClient.getString(url);
return SearchPage.parse(raw, queryString);
@ -157,9 +145,7 @@ class _InitialData {
}
// Contains only [SearchVideo] or [SearchPlaylist]
late final List<BaseSearchContent> searchContent =
getContentContext()?.map(_parseContent).whereNotNull().toList() ??
const [];
late final List<BaseSearchContent> searchContent = getContentContext()?.map(_parseContent).whereNotNull().toList() ?? const [];
List<RelatedQuery> get relatedQueries =>
getContentContext()
@ -167,10 +153,8 @@ class _InitialData {
.map((e) => e.get('horizontalCardListRenderer')?.getList('cards'))
.firstOrNull
?.map((e) => e['searchRefinementCardRenderer'])
.map((e) => RelatedQuery(
e.searchEndpoint.searchEndpoint.query,
VideoId(
Uri.parse(e.thumbnail.thumbnails.first.url).pathSegments[1])))
.map((e) =>
RelatedQuery(e.searchEndpoint.searchEndpoint.query, VideoId(Uri.parse(e.thumbnail.thumbnails.first.url).pathSegments[1])))
.toList()
.cast<RelatedQuery>() ??
const [];
@ -178,11 +162,7 @@ class _InitialData {
List<dynamic> get relatedVideos =>
getContentContext()
?.where((e) => e['shelfRenderer'] != null)
.map((e) => e
.get('shelfRenderer')
?.get('content')
?.get('verticalListRenderer')
?.getList('items'))
.map((e) => e.get('shelfRenderer')?.get('content')?.get('verticalListRenderer')?.getList('items'))
.firstOrNull
?.map(_parseContent)
.whereNotNull()
@ -191,8 +171,7 @@ class _InitialData {
late final String? continuationToken = _getContinuationToken();
late final int estimatedResults =
int.parse(root.getT<String>('estimatedResults') ?? '0');
late final int estimatedResults = int.parse(root.getT<String>('estimatedResults') ?? '0');
BaseSearchContent? _parseContent(Map<String, dynamic>? content) {
if (content == null) {
@ -207,47 +186,32 @@ class _InitialData {
_parseRuns(renderer.get('ownerText')?.getList('runs')),
_parseRuns(renderer.get('descriptionSnippet')?.getList('runs')),
renderer.get('lengthText')?.getT<String>('simpleText') ?? '',
int.parse(renderer
.get('viewCountText')
?.getT<String>('simpleText')
?.stripNonDigits()
.nullIfWhitespace ??
renderer
.get('viewCountText')
?.getList('runs')
?.firstOrNull
?.getT<String>('text')
?.stripNonDigits()
.nullIfWhitespace ??
int.parse(renderer.get('viewCountText')?.getT<String>('simpleText')?.stripNonDigits().nullIfWhitespace ??
renderer.get('viewCountText')?.getList('runs')?.firstOrNull?.getT<String>('text')?.stripNonDigits().nullIfWhitespace ??
'0'),
(renderer.get('thumbnail')?.getList('thumbnails') ?? const [])
.map((e) =>
Thumbnail(Uri.parse(e['url']), e['height'], e['width']))
.map((e) => Thumbnail(Uri.parse(e['url']), e['height'], e['width']))
.toList(),
renderer.get('publishedTimeText')?.getT<String>('simpleText'),
renderer
.get('viewCountText')
?.getList('runs')
?.elementAtSafe(1)
?.getT<String>('text')
?.trim() ==
'watching');
renderer.get('viewCountText')?.getList('runs')?.elementAtSafe(1)?.getT<String>('text')?.trim() == 'watching');
}
if (content['radioRenderer'] != null) {
var renderer = content.get('radioRenderer')!;
return SearchPlaylist(
PlaylistId(renderer.getT<String>('playlistId')!),
return SearchPlaylist(PlaylistId(renderer.getT<String>('playlistId')!), renderer.get('title')!.getT<String>('simpleText')!,
int.parse(_parseRuns(renderer.get('videoCountText')?.getList('runs')).stripNonDigits().nullIfWhitespace ?? '0'));
}
if (content['channelRenderer'] != null) {
var renderer = content.get('channelRenderer')!;
return SearchChannel(
ChannelId(renderer.getT<String>('channelId')!),
renderer.get('title')!.getT<String>('simpleText')!,
int.parse(_parseRuns(renderer.get('videoCountText')?.getList('runs'))
.stripNonDigits()
.nullIfWhitespace ??
'0'));
renderer.get('descriptionSnippet')?.getList('runs')?.parseRuns() ?? '',
renderer.get('videoCountText')!.getList('runs')!.first.getT<String>('text')!.parseInt()!);
}
// Here ignore 'horizontalCardListRenderer' & 'shelfRenderer'
return null;
}
String _parseRuns(List<dynamic>? runs) =>
runs?.map((e) => e['text']).join() ?? '';
String _parseRuns(List<dynamic>? runs) => runs?.map((e) => e['text']).join() ?? '';
}

View File

@ -156,7 +156,7 @@ class WatchPage {
var req = await httpClient.get(url, validate: true);
var cookies = req.headers['set-cookie']!;
var visitorInfoLive = _visitorInfoLiveExp.firstMatch(cookies)?.group(1)!;
var visitorInfoLive = _visitorInfoLiveExp.firstMatch(cookies)?.group(1);
var ysc = _yscExp.firstMatch(cookies)!.group(1)!;
var result = WatchPage.parse(req.body, visitorInfoLive ?? '', ysc);

View File

@ -147,7 +147,7 @@ class YoutubeHttpClient extends http.BaseClient {
request.headers[key] = _defaultHeaders[key]!;
}
});
// print('Request: $request');
print('Request: $request');
// print('Stack:\n${StackTrace.current}');
return _httpClient.send(request);
}

View File

@ -5,6 +5,7 @@ library youtube_explode.search;
export 'related_query.dart';
export 'search_client.dart';
export 'search_filter.dart';
export 'search_list.dart';
export 'search_playlist.dart';
export 'search_query.dart';

View File

@ -1,16 +1,29 @@
import 'package:equatable/equatable.dart';
import '../channels/channel_id.dart';
import 'base_search_content.dart';
/// Metadata related to a search query result (channel)
class SearchChannel {
/// ChannelId.
final ChannelId channelId;
class SearchChannel extends BaseSearchContent with EquatableMixin {
/// Channel id.
final ChannelId id;
/// Channel name.
final String channelName;
final String name;
/// Description snippet.
/// Can be empty.
final String description;
/// Channel uploaded videos.
final int videoCount;
/// Initialize a [SearchChannel] instance.
SearchChannel(this.channelId, this.channelName);
SearchChannel(this.id, this.name, this.description, this.videoCount);
@override
String toString() => '(Channel) $channelName ($channelId)';
String toString() => '(Channel) $name ($id)';
@override
List<Object?> get props => [id, name, description, videoCount];
}

View File

@ -1,10 +1,12 @@
import 'dart:convert';
import '../../youtube_explode_dart.dart';
import '../extensions/helpers_extension.dart';
import '../retry.dart';
import '../reverse_engineering/responses/search_page.dart';
import '../reverse_engineering/youtube_http_client.dart';
import 'base_search_content.dart';
import 'search_filter.dart';
import 'search_list.dart';
import 'search_query.dart';
@ -19,37 +21,40 @@ class SearchClient {
/// (from the video search page).
/// The videos are sent in batch of 20 videos.
/// You [SearchList.nextPage] to get the next batch of videos.
Future<SearchList> getVideos(String searchQuery) {
var stream = getVideosFromPage(searchQuery).cast<SearchVideo>();
return SearchList.create(stream);
Future<SearchList> getVideos(String searchQuery, {SearchFilter filter = const SearchFilter('')}) async {
final page = await SearchPage.get(_httpClient, searchQuery, filter: filter);
return SearchList(
page.initialData.searchContent
.whereType<SearchVideo>()
.map((e) => Video(e.id, e.title, e.author, null, e.uploadDate?.toDateTime(), e.description, e.duration.toDuration(),
ThumbnailSet(e.id.value), null, Engagement(e.viewCount, null, null), e.isLive))
.toList(),
page,
_httpClient);
}
/// Enumerates videos returned by the specified search query
/// (from the video search page).
/// Contains only instances of [SearchVideo] or [SearchPlaylist]
Stream<BaseSearchContent> getVideosFromPage(String searchQuery,
{bool onlyVideos = true}) async* {
SearchPage? page =
await retry(() async => SearchPage.get(_httpClient, searchQuery));
if (onlyVideos) {
yield* Stream.fromIterable(
page!.initialData.searchContent.whereType<SearchVideo>());
} else {
yield* Stream.fromIterable(page!.initialData.searchContent);
}
{bool onlyVideos = true, SearchFilter filter = const SearchFilter('')}) async* {
SearchPage? page;
// ignore: literal_only_boolean_expressions
while (true) {
page = await page!.nextPage(_httpClient);
for (;;) {
if (page == null) {
return;
page = await retry(() async => SearchPage.get(_httpClient, searchQuery, filter: filter));
} else {
page = await page.nextPage(_httpClient);
if (page == null) {
return;
}
}
if (onlyVideos) {
yield* Stream.fromIterable(
page.initialData.searchContent.whereType<SearchVideo>());
yield* Stream.fromIterable(page!.initialData.searchContent.whereType<SearchVideo>());
} else {
yield* Stream.fromIterable(page.initialData.searchContent);
yield* Stream.fromIterable(page!.initialData.searchContent);
}
}
}
@ -69,8 +74,7 @@ class SearchClient {
/// Queries to YouTube to get the results.
@Deprecated('Use getVideosFromPage instead - '
'Should be used only to get related videos')
Future<SearchQuery> queryFromPage(String searchQuery) =>
SearchQuery.search(_httpClient, searchQuery);
Future<SearchQuery> queryFromPage(String searchQuery) => SearchQuery.search(_httpClient, searchQuery);
}
/*

View File

@ -0,0 +1,132 @@
import 'search_client.dart';
class SearchFilter {
/// The value fo the 'sp' argument.
final String value;
const SearchFilter(this.value);
}
/// Video filters to be used with [SearchClient.getVideos]
class Filters {
const Filters._();
/// Features filters.
FeatureFilters get features => const FeatureFilters._();
/// Upload date filters.
UploadDateFilter get uploadDate => const UploadDateFilter._();
/// Types filters.
TypeFilters get types => const TypeFilters._();
/// Duration filters.
DurationFilters get duration => const DurationFilters._();
/// Videos sorting.
SortFilters get sort => const SortFilters._();
}
/// Video filters to be used with [SearchClient.getVideos]
const filters = Filters._();
class FeatureFilters {
const FeatureFilters._();
/// Live video.
SearchFilter get live => const SearchFilter('EgJAAQ%253D%253D');
/// 4K video.
SearchFilter get v4k => const SearchFilter('EgJwAQ%253D%253D');
/// HD video.
SearchFilter get hd => const SearchFilter('EgIgAQ%253D%253D');
/// Subtitled video.
SearchFilter get subTitles => const SearchFilter('EgIoAQ%253D%253D');
/// Creative comments video.
SearchFilter get creativeCommons => const SearchFilter('EgIwAQ%253D%253D');
/// 360° video.
SearchFilter get v360 => const SearchFilter('EgJ4AQ%253D%253D');
/// VR 180° video.
SearchFilter get vr180 => const SearchFilter('EgPQAQE%253D');
/// 3D video.
SearchFilter get v3D => const SearchFilter('EgI4AQ%253D%253D');
/// HDR video.
SearchFilter get hdr => const SearchFilter('EgPIAQE%253D');
/// Video with location.
SearchFilter get location => const SearchFilter('EgO4AQE%253D');
/// Purchased video.
SearchFilter get purchased => const SearchFilter('EgJIAQ%253D%253D');
}
class UploadDateFilter {
const UploadDateFilter._();
/// Videos uploaded in the last hour.
SearchFilter get lastHour => const SearchFilter('EgIIAQ%253D%253D');
/// Videos uploaded today.
SearchFilter get today => const SearchFilter('EgIIAg%253D%253D');
/// Videos uploaded in the last week.
SearchFilter get lastWeek => const SearchFilter('EgIIAw%253D%253D');
/// Videos uploaded in the last month.
SearchFilter get lastMonth => const SearchFilter('EgIIBA%253D%253D');
/// Videos uploaded in the last year.
SearchFilter get lastYear => const SearchFilter('EgIIBQ%253D%253D');
}
class TypeFilters {
const TypeFilters._();
/// Videos.
SearchFilter get video => const SearchFilter('EgIQAQ%253D%253D');
/// Channels.
SearchFilter get channel => const SearchFilter('EgIQAg%253D%253D');
/// Playlists.
SearchFilter get playlist => const SearchFilter('EgIQAw%253D%253D');
/// Movies.
SearchFilter get movie => const SearchFilter('EgIQBA%253D%253D');
/// Shows.
SearchFilter get show => const SearchFilter('EgIQBQ%253D%253D');
}
class DurationFilters {
const DurationFilters._();
/// Short videos, < 4 minutes.
SearchFilter get short => const SearchFilter('EgIYAQ%253D%253D');
/// Long videos, > 20 minutes.
SearchFilter get long => const SearchFilter('EgIYAg%253D%253D');
}
class SortFilters {
const SortFilters._();
/// Sort by relevance (default).
SearchFilter get relevance => const SearchFilter('CAASAhAB');
/// Sort by upload date (default).
SearchFilter get uploadDate => const SearchFilter('CAI%253D');
/// Sort by view count (default).
SearchFilter get viewCount => const SearchFilter('CAM%253D');
/// Sort by rating (default).
SearchFilter get rating => const SearchFilter('CAE%253D');
}

View File

@ -1,110 +1,32 @@
import 'dart:async';
import 'package:collection/collection.dart';
import 'package:youtube_explode_dart/src/reverse_engineering/responses/search_page.dart';
import '../../youtube_explode_dart.dart';
import '../extensions/helpers_extension.dart';
/// This list contains search videos.
class SearchList extends DelegatingList<Video> {
final Stream<Video> _stream;
final SearchPage _page;
final YoutubeHttpClient _httpClient;
///
SearchList._(List<Video> base, this._stream) : super(base);
SearchList(List<Video> base, this._page, this._httpClient) : super(base);
///
static Future<SearchList> create(Stream<SearchVideo> stream) async {
Stream<Video> broadcast;
broadcast = stream
.map((e) => Video(
e.id,
e.title,
e.author,
null,
_stringToDateTime(e.uploadDate),
e.description,
_stringToDuration(e.duration),
ThumbnailSet(e.id.value),
null,
Engagement(e.viewCount, null, null),
e.isLive))
.asBroadcastStream(onCancel: (subscription) {
subscription.pause();
}, onListen: (subscription) {
subscription.resume();
});
final base = await broadcast.take(20).toList();
return SearchList._(base, broadcast);
}
///
Future<SearchList> nextPage() async {
final base = await _stream.take(20).toList();
return SearchList._(base, _stream);
}
/// Format: <quantity> <unit> ago (5 years ago)
static DateTime? _stringToDateTime(String? string) {
if (string == null) {
Future<SearchList?> nextPage() async {
final page = await _page.nextPage(_httpClient);
if (page == null) {
return null;
}
var parts = string.split(' ');
if (parts.length == 4) {
// Streamed x y ago
parts = parts.skip(1).toList();
}
assert(parts.length == 3);
var qty = int.parse(parts.first);
// Try to get the unit
var unit = parts[1];
Duration time;
if (unit.startsWith('second')) {
time = Duration(seconds: qty);
} else if (unit.startsWith('minute')) {
time = Duration(minutes: qty);
} else if (unit.startsWith('hour')) {
time = Duration(hours: qty);
} else if (unit.startsWith('day')) {
time = Duration(days: qty);
} else if (unit.startsWith('week')) {
time = Duration(days: qty * 7);
} else if (unit.startsWith('month')) {
time = Duration(days: qty * 30);
} else if (unit.startsWith('year')) {
time = Duration(days: qty * 365);
} else {
throw StateError('Couldn\'t parse $unit unit of time. '
'Please report this to the project page!');
}
return DateTime.now().subtract(time);
}
/// Format: HH:MM:SS
static Duration? _stringToDuration(String string) {
if (/*string == null ||*/ string.trim().isEmpty) {
return null;
}
var parts = string.split(':');
assert(parts.length <= 3);
if (parts.length == 1) {
return Duration(seconds: int.parse(parts.first));
}
if (parts.length == 2) {
return Duration(
minutes: int.parse(parts[0]), seconds: int.parse(parts[1]));
}
if (parts.length == 3) {
return Duration(
hours: int.parse(parts[0]),
minutes: int.parse(parts[1]),
seconds: int.parse(parts[2]));
}
// Shouldn't reach here.
throw Error();
return SearchList(
page.initialData.searchContent
.whereType<SearchVideo>()
.map((e) => Video(e.id, e.title, e.author, null, e.uploadDate.toDateTime(), e.description, e.duration.toDuration(),
ThumbnailSet(e.id.value), null, Engagement(e.viewCount, null, null), e.isLive))
.toList(),
page,
_httpClient);
}
}

View File

@ -1,6 +1,6 @@
name: youtube_explode_dart
description: A port in dart of the youtube explode library. Supports several API functions without the need of Youtube API Key.
version: 1.9.0-nullsafety.5
version: 1.9.0-nullsafety.6
homepage: https://github.com/Hexer10/youtube_explode_dart