poka
/
youtube_explode
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

New Version 1.7.4 

Closes #92 #90
This commit is contained in:
Mattia 2020-12-25 23:29:01 +01:00
parent 10312329d3
commit 8372726a65
10 changed files with 183 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
CHANGELOG.md
View File

@ -1,3 +1,8 @@
## 1.7.4
- Fix slow download ( #92 )
- Fix stream retrieving on some videos ( #90 )
- Updates tests
## 1.7.3
- Fix exceptions on some videos.
- Closes #89, #88

37
lib/src/reverse_engineering/responses/channel_about_page.dart
View File

@ -15,14 +15,35 @@ class ChannelAboutPage {
_InitialData _initialData;
///
_InitialData get initialData =>
_initialData ??= _InitialData(ChannelAboutPageId.fromRawJson(_extractJson(
_root
.querySelectorAll('script')
.map((e) => e.text)
.toList()
.firstWhere((e) => e.contains('window["ytInitialData"] =')),
'window["ytInitialData"] =')));
_InitialData get initialData {
if (_initialData != null) {
return _initialData;
}
final scriptText = _root
.querySelectorAll('script')
.map((e) => e.text)
.toList(growable: false);
var initialDataText = scriptText.firstWhere(
(e) => e.contains('window["ytInitialData"] ='),
orElse: () => null);
if (initialDataText != null) {
return _initialData = _InitialData(ChannelAboutPageId.fromRawJson(
_extractJson(initialDataText, 'window["ytInitialData"] =')));
}
initialDataText = scriptText.firstWhere(
(e) => e.contains('var ytInitialData = '),
orElse: () => null);
if (initialDataText != null) {
return _initialData = _InitialData(ChannelAboutPageId.fromRawJson(
_extractJson(initialDataText, 'var ytInitialData = ')));
}
throw TransientFailureException(
'Failed to retrieve initial data from the channel about page, please report this to the project GitHub page.'); // ignore: lines_longer_than_80_chars
}
///
bool get isOk => initialData != null;

37
lib/src/reverse_engineering/responses/channel_upload_page.dart
View File

@ -19,14 +19,35 @@ class ChannelUploadPage {
_InitialData _initialData;
///
_InitialData get initialData => _initialData ??= _InitialData(
ChannelUploadPageId.fromJson(json.decode(_extractJson(
_root
.querySelectorAll('script')
.map((e) => e.text)
.toList()
.firstWhere((e) => e.contains('window["ytInitialData"] =')),
'window["ytInitialData"] ='))));
_InitialData get initialData {
if (_initialData != null) {
return _initialData;
}
final scriptText = _root
.querySelectorAll('script')
.map((e) => e.text)
.toList(growable: false);
var initialDataText = scriptText.firstWhere(
(e) => e.contains('window["ytInitialData"] ='),
orElse: () => null);
if (initialDataText != null) {
return _initialData = _InitialData(ChannelUploadPageId.fromRawJson(
_extractJson(initialDataText, 'window["ytInitialData"] =')));
}
initialDataText = scriptText.firstWhere(
(e) => e.contains('var ytInitialData = '),
orElse: () => null);
if (initialDataText != null) {
return _initialData = _InitialData(ChannelUploadPageId.fromRawJson(
_extractJson(initialDataText, 'var ytInitialData = ')));
}
throw TransientFailureException(
'Failed to retrieve initial data from the channel upload page, please report this to the project GitHub page.'); // ignore: lines_longer_than_80_chars
}
String _extractJson(String html, String separator) {
return _matchJson(

35
lib/src/reverse_engineering/responses/search_page.dart
View File

@ -38,17 +38,30 @@ class SearchPage {
if (_initialData != null) {
return _initialData;
}
var scriptTag = _extractJson(
_root.querySelectorAll('script').map((e) => e.text).toList().firstWhere(
(e) => e.contains('window["ytInitialData"] ='),
orElse: () => null),
'window["ytInitialData"] =');
scriptTag ??= _extractJson(
_root.querySelectorAll('script').map((e) => e.text).toList().firstWhere(
(e) => e.contains('var ytInitialData ='),
orElse: () => '{}'),
'var ytInitialData =');
return _initialData ??= _InitialData(SearchPageId.fromRawJson(scriptTag));
final scriptText = _root
.querySelectorAll('script')
.map((e) => e.text)
.toList(growable: false);
var initialDataText = scriptText.firstWhere(
(e) => e.contains('window["ytInitialData"] ='),
orElse: () => null);
if (initialDataText != null) {
return _initialData = _InitialData(SearchPageId.fromRawJson(
_extractJson(initialDataText, 'window["ytInitialData"] =')));
}
initialDataText = scriptText.firstWhere(
(e) => e.contains('var ytInitialData = '),
orElse: () => null);
if (initialDataText != null) {
return _initialData = _InitialData(SearchPageId.fromRawJson(
_extractJson(initialDataText, 'var ytInitialData = ')));
}
throw TransientFailureException(
'Failed to retrieve initial data from the search page, please report this to the project GitHub page.'); // ignore: lines_longer_than_80_chars
}
String _extractJson(String html, String separator) {

39
lib/src/reverse_engineering/responses/watch_page.dart
View File

@ -51,14 +51,35 @@ class WatchPage {
}
///
_InitialData get initialData =>
_initialData ??= _InitialData(WatchPageId.fromRawJson(_extractJson(
_root
.querySelectorAll('script')
.map((e) => e.text)
.toList()
.firstWhere((e) => e.contains('window["ytInitialData"] =')),
'window["ytInitialData"] =')));
_InitialData get initialData {
if (_initialData != null) {
return _initialData;
}
final scriptText = _root
.querySelectorAll('script')
.map((e) => e.text)
.toList(growable: false);
var initialDataText = scriptText.firstWhere(
(e) => e.contains('window["ytInitialData"] ='),
orElse: () => null);
if (initialDataText != null) {
return _initialData = _InitialData(WatchPageId.fromRawJson(
_extractJson(initialDataText, 'window["ytInitialData"] =')));
}
initialDataText = scriptText.firstWhere(
(e) => e.contains('var ytInitialData = '),
orElse: () => null);
if (initialDataText != null) {
return _initialData = _InitialData(WatchPageId.fromRawJson(
_extractJson(initialDataText, 'var ytInitialData = ')));
}
throw TransientFailureException(
'Failed to retrieve initial data from the watch page, please report this to the project GitHub page.'); // ignore: lines_longer_than_80_chars
}
///
String get xsfrToken => _xsfrToken ??= _xsfrTokenExp
@ -110,10 +131,10 @@ class WatchPage {
?.group(1)
?.extractJson()));
///
PlayerResponse get playerResponse => PlayerResponse.parse(_root
.querySelectorAll('script')
.map((e) => e.text)
.map((e) => null)
.map((e) => _playerResponseExp.firstMatch(e)?.group(1))
.firstWhere((e) => !e.isNullOrWhiteSpace)
.extractJson());

40
lib/src/reverse_engineering/youtube_http_client.dart
View File

@ -84,25 +84,41 @@ class YoutubeHttpClient extends http.BaseClient {
return response.body;
}
///
Stream<List<int>> getStream(StreamInfo streamInfo,
{Map<String, String> headers,
bool validate = true,
int start = 0,
int errorCount = 0}) async* {
var url = streamInfo.url;
var query = Map<String, String>.from(url.queryParameters);
query['ratebypass'] = 'yes';
url = url.replace(queryParameters: query);
var request = http.Request('get', url);
request.headers.addAll(_defaultHeaders);
var response = await request.send();
if (validate) {
_validateResponse(response, response.statusCode);
var bytesCount = start;
for (var i = start; i < streamInfo.size.totalBytes; i += 9898989) {
try {
final request = http.Request('get', url);
request.headers['range'] = 'bytes=$i-${i + 9898989 - 1}';
final response = await send(request);
if (validate) {
_validateResponse(response, response.statusCode);
}
final stream = StreamController<List<int>>();
response.stream.listen((data) {
bytesCount += data.length;
stream.add(data);
}, onError: (_) => null, onDone: stream.close, cancelOnError: false);
errorCount = 0;
yield* stream.stream;
} on Exception {
if (errorCount == 5) {
rethrow;
}
await Future.delayed(const Duration(milliseconds: 500));
yield* getStream(streamInfo,
headers: headers,
validate: validate,
start: bytesCount,
errorCount: errorCount + 1);
break;
}
}
yield* response.stream;
}
///

15
lib/src/videos/streams/streams_client.dart
View File

@ -78,7 +78,13 @@ class StreamsClient {
Future<StreamContext> _getStreamContextFromWatchPage(VideoId videoId) async {
var watchPage = await WatchPage.get(_httpClient, videoId.toString());
var playerConfig = watchPage.playerConfig;
dynamic /* _PlayerConfig */ playerConfig;
try {
playerConfig = watchPage.playerConfig;
} on FormatException {
playerConfig = null;
}
var playerResponse =
playerConfig?.playerResponse ?? watchPage.playerResponse;
if (playerResponse == null) {
@ -86,12 +92,13 @@ class StreamsClient {
}
var previewVideoId = playerResponse.previewVideoId;
if (!previewVideoId.isNullOrWhiteSpace) {
if (!((previewVideoId as String)?.isNullOrWhiteSpace ?? true)) {
throw VideoRequiresPurchaseException.preview(
videoId, VideoId(previewVideoId));
}
var playerSourceUrl = watchPage.sourceUrl ?? playerConfig?.sourceUrl;
var playerSourceUrl =
watchPage.sourceUrl ?? playerConfig?.sourceUrl as String;
var playerSource = !playerSourceUrl.isNullOrWhiteSpace
? await PlayerSource.get(_httpClient, playerSourceUrl)
: null;
@ -112,7 +119,7 @@ class StreamsClient {
];
var dashManifestUrl = playerResponse.dashManifestUrl;
if (!dashManifestUrl.isNullOrWhiteSpace) {
if (!(dashManifestUrl?.isNullOrWhiteSpace ?? true)) {
var dashManifest =
await _getDashManifest(Uri.parse(dashManifestUrl), cipherOperations);
streamInfoProviders.addAll(dashManifest.streams);

2
pubspec.yaml
View File

@ -1,6 +1,6 @@
name: youtube_explode_dart
description: A port in dart of the youtube explode library. Supports several API functions without the need of Youtube API Key.
version: 1.7.3
version: 1.7.4
homepage: https://github.com/Hexer10/youtube_explode_dart
environment:

28
test/streams_test.dart
View File

@ -11,12 +11,18 @@ void main() {
yt.close();
});
group('Get streams of any video', () {
group('Get streams manifest of any video', () {
for (var val in {
VideoId('5VGm0dczmHc'), // rating is not allowed
VideoId('9bZkp7q19f0'), // very popular
VideoId('SkRSXFQerZs'), // age restricted (embed allowed)
VideoId('hySoCSoH-g8'), // age restricted (embed not allowed)
VideoId('_kmeFXjjGfk'), // embed not allowed (type 1)
VideoId('MeJVWBSsPAY'), // embed not allowed (type 2)
VideoId('5VGm0dczmHc'), // rating not allowed
VideoId('ZGdLIwrGHG8'), // unlisted
VideoId('rsAAeyAr-9Y'),
VideoId('AI7ULzgf8RU')
VideoId('rsAAeyAr-9Y'), // recording of a live stream
VideoId('AI7ULzgf8RU'), // has DASH manifest
VideoId('-xNN-bJQ4vI'), // 360° video
}) {
test('VideoId - ${val.value}', () async {
var manifest = await yt.videos.streamsClient.getManifest(val);
@ -39,12 +45,18 @@ void main() {
}
});
group('Get stream of any playable video', () {
group('Get specific stream of any playable video', () {
for (var val in {
VideoId('5VGm0dczmHc'), // rating is not allowed
VideoId('9bZkp7q19f0'), // very popular
VideoId('SkRSXFQerZs'), // age restricted (embed allowed)
VideoId('hySoCSoH-g8'), // age restricted (embed not allowed)
VideoId('_kmeFXjjGfk'), // embed not allowed (type 1)
VideoId('MeJVWBSsPAY'), // embed not allowed (type 2)
VideoId('5VGm0dczmHc'), // rating not allowed
VideoId('ZGdLIwrGHG8'), // unlisted
VideoId('rsAAeyAr-9Y'),
VideoId('AI7ULzgf8RU')
VideoId('rsAAeyAr-9Y'), // recording of a live stream
VideoId('AI7ULzgf8RU'), // has DASH manifest
VideoId('-xNN-bJQ4vI'), // 360° video
}) {
test('VideoId - ${val.value}', () async {
var manifest = await yt.videos.streamsClient.getManifest(val);

7
test/video_test.dart
View File

@ -32,7 +32,12 @@ void main() {
expect(video.thumbnails.highResUrl, isNotEmpty);
expect(video.thumbnails.standardResUrl, isNotEmpty);
expect(video.thumbnails.maxResUrl, isNotEmpty);
expect(video.keywords, containsAll(['osu', 'mouse' /*, 'rhythm game'*/]));
expect(
video.keywords,
containsAll([
'osu',
'mouse' /*, 'rhythm game'*/
]));
expect(video.engagement.viewCount, greaterThanOrEqualTo(134));
expect(video.engagement.likeCount, greaterThanOrEqualTo(5));
expect(video.engagement.dislikeCount, greaterThanOrEqualTo(0));