diff --git a/CHANGELOG.md b/CHANGELOG.md index e104d71..50c1cd0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,8 @@ ## 1.10.7 - Fix the error of incomplete data loading on the Android emulator. - Fix error when the http-client is closed and the request is still running. +- Fix extraction for DASH streams. + ## 1.10.6 - Implement `Playlist.videoCount`. diff --git a/example/example.dart b/example/example.dart index 5d60e4b..0f3ab51 100644 --- a/example/example.dart +++ b/example/example.dart @@ -1,7 +1,6 @@ // ignore_for_file: avoid_print -import 'package:youtube_explode_dart/youtube_explode_dart.dart'; - import 'package:youtube_explode_dart/src/youtube_explode_base.dart'; +import 'package:youtube_explode_dart/youtube_explode_dart.dart'; Future main() async { var yt = YoutubeExplode(); diff --git a/lib/src/channels/channel_client.dart b/lib/src/channels/channel_client.dart index 7a8613f..f9ae415 100644 --- a/lib/src/channels/channel_client.dart +++ b/lib/src/channels/channel_client.dart @@ -1,12 +1,10 @@ -import 'package:youtube_explode_dart/src/channels/channel_uploads_list.dart'; -import 'package:youtube_explode_dart/src/reverse_engineering/pages/channel_page.dart'; -import 'package:youtube_explode_dart/src/reverse_engineering/pages/watch_page.dart'; - import '../common/common.dart'; import '../extensions/helpers_extension.dart'; import '../playlists/playlists.dart'; import '../reverse_engineering/pages/channel_about_page.dart'; +import '../reverse_engineering/pages/channel_page.dart'; import '../reverse_engineering/pages/channel_upload_page.dart'; +import '../reverse_engineering/pages/watch_page.dart'; import '../reverse_engineering/youtube_http_client.dart'; import '../videos/video.dart'; import '../videos/video_id.dart'; diff --git a/lib/src/channels/channel_uploads_list.dart b/lib/src/channels/channel_uploads_list.dart index e27095f..91f3ded 100644 --- a/lib/src/channels/channel_uploads_list.dart +++ b/lib/src/channels/channel_uploads_list.dart @@ -1,10 +1,10 @@ import 'dart:async'; import 'package:collection/collection.dart'; -import 'package:youtube_explode_dart/src/reverse_engineering/pages/channel_upload_page.dart'; import '../../youtube_explode_dart.dart'; import '../extensions/helpers_extension.dart'; +import '../reverse_engineering/pages/channel_upload_page.dart'; /// This list contains a channel uploads. /// This behaves like a [List] but has the [SearchList.nextPage] to get the next batch of videos. diff --git a/lib/src/extensions/helpers_extension.dart b/lib/src/extensions/helpers_extension.dart index aa093d7..d32e7cb 100644 --- a/lib/src/extensions/helpers_extension.dart +++ b/lib/src/extensions/helpers_extension.dart @@ -1,6 +1,7 @@ library _youtube_explode.extensions; import 'dart:convert'; + import 'package:collection/collection.dart'; import '../reverse_engineering/cipher/cipher_operations.dart'; @@ -288,3 +289,37 @@ extension GenericExtract on List { throw orThrow(); } } + +/// Iterable that joins together multiple lists +class JoinedIterable extends Iterable { + final Iterable> _iterables; + + JoinedIterable(this._iterables); + + @override + Iterator get iterator { + return _JoinedIterator(_iterables.map((e) => e.iterator).toList()); + } +} + +class _JoinedIterator extends Iterator { + final Iterable> _iterators; + var _currentIter = 0; + + _JoinedIterator(this._iterators); + + @override + bool moveNext([int debug = 0]) { + if (!_iterators.elementAt(_currentIter).moveNext()) { + if (_currentIter == _iterators.length - 1) { + return false; + } + _currentIter++; + return moveNext(debug + 1); + } + return true; + } + + @override + T get current => _iterators.elementAt(_currentIter).current; +} diff --git a/lib/src/playlists/playlist_client.dart b/lib/src/playlists/playlist_client.dart index dcf9bfe..526e723 100644 --- a/lib/src/playlists/playlist_client.dart +++ b/lib/src/playlists/playlist_client.dart @@ -1,7 +1,6 @@ -import 'package:youtube_explode_dart/src/channels/channel_id.dart'; -import 'package:youtube_explode_dart/src/reverse_engineering/pages/playlist_page.dart'; - +import '../channels/channel_id.dart'; import '../common/common.dart'; +import '../reverse_engineering/pages/playlist_page.dart'; import '../reverse_engineering/youtube_http_client.dart'; import '../videos/video.dart'; import '../videos/video_id.dart'; diff --git a/lib/src/retry.dart b/lib/src/retry.dart index 54b7166..87f02fc 100644 --- a/lib/src/retry.dart +++ b/lib/src/retry.dart @@ -9,7 +9,8 @@ import 'exceptions/exceptions.dart'; /// Run the [function] each time an exception is thrown until the retryCount /// is 0. -Future retry(YoutubeHttpClient? client, FutureOr Function() function) async { +Future retry( + YoutubeHttpClient? client, FutureOr Function() function) async { var retryCount = 5; // ignore: literal_only_boolean_expressions diff --git a/lib/src/reverse_engineering/clients/comments_client.dart b/lib/src/reverse_engineering/clients/comments_client.dart index 8ad079f..f4b819a 100644 --- a/lib/src/reverse_engineering/clients/comments_client.dart +++ b/lib/src/reverse_engineering/clients/comments_client.dart @@ -1,9 +1,9 @@ import 'package:collection/collection.dart'; -import 'package:youtube_explode_dart/src/reverse_engineering/pages/watch_page.dart'; import '../../../youtube_explode_dart.dart'; import '../../extensions/helpers_extension.dart'; import '../../retry.dart'; +import '../pages/watch_page.dart'; import '../youtube_http_client.dart'; class CommentsClient { @@ -22,8 +22,8 @@ class CommentsClient { static Future get( YoutubeHttpClient httpClient, Video video) async { final watchPage = video.watchPage ?? - await retry(httpClient, - () async => WatchPage.get(httpClient, video.id.value)); + await retry( + httpClient, () async => WatchPage.get(httpClient, video.id.value)); final continuation = watchPage.commentsContinuation; if (continuation == null) { diff --git a/lib/src/reverse_engineering/clients/embedded_player_client.dart b/lib/src/reverse_engineering/clients/embedded_player_client.dart index 1490920..3e4fe7c 100644 --- a/lib/src/reverse_engineering/clients/embedded_player_client.dart +++ b/lib/src/reverse_engineering/clients/embedded_player_client.dart @@ -83,53 +83,84 @@ class EmbeddedPlayerClient { } class _StreamInfo extends StreamInfoProvider { + static final _contentLenExp = RegExp(r'[\?&]clen=(\d+)'); + + /// Json parsed map final JsonMap root; @override - late final int tag = root['itag']!; + late final int? bitrate = root.getT('bitrate'); @override - late final String url = root['url']!; + late final String? container = mimeType?.subtype; @override - late final int? contentLength = int.tryParse(root['contentLength'] ?? - StreamInfoProvider.contentLenExp.firstMatch(url)?.group(1) ?? - ''); + late final int? contentLength = int.tryParse( + root.getT('contentLength') ?? + _contentLenExp.firstMatch(url)?.group(1) ?? + ''); @override - late final int bitrate = root['bitrate']!; - - late final MediaType mimeType = MediaType.parse(root['mimeType']!); + late final int? framerate = root.getT('fps'); @override - late final String container = mimeType.subtype; - - late final List codecs = mimeType.parameters['codecs']! - .split(',') - .map((e) => e.trim()) - .toList() - .cast(); + late final String? signature = + Uri.splitQueryString(root.getT('signatureCipher') ?? '')['s']; @override - late final String audioCodec = codecs.last; + late final String? signatureParameter = Uri.splitQueryString( + root.getT('cipher') ?? '')['sp'] ?? + Uri.splitQueryString(root.getT('signatureCipher') ?? '')['sp']; @override - late final String? videoCodec = isAudioOnly ? null : codecs.first; - - late final bool isAudioOnly = mimeType.type == 'audio'; + late final int tag = root.getT('itag')!; @override - late final String videoQualityLabel = - root['qualityLabel'] ?? root['quality_label']; + late final String url = root.getT('url') ?? + Uri.splitQueryString(root.getT('cipher') ?? '')['url'] ?? + Uri.splitQueryString(root.getT('signatureCipher') ?? '')['url']!; @override - late final int? videoWidth = root['width']; + late final String? videoCodec = isAudioOnly + ? null + : codecs?.split(',').firstOrNull?.trim().nullIfWhitespace; @override - late final int? videoHeight = root['height']; + late final int? videoHeight = root.getT('height'); @override - late final int? framerate = root['fps'] ?? 0; + late final String? videoQualityLabel = root.getT('qualityLabel'); + + @override + late final int? videoWidth = root.getT('width'); + + late final bool isAudioOnly = mimeType?.type == 'audio'; + + late final MediaType? mimeType = _getMimeType(); + + MediaType? _getMimeType() { + var mime = root.getT('mimeType'); + if (mime == null) { + return null; + } + return MediaType.parse(mime); + } + + late final String? codecs = mimeType?.parameters['codecs']?.toLowerCase(); + + @override + late final String? audioCodec = + isAudioOnly ? codecs : _getAudioCodec(codecs?.split(','))?.trim(); + + String? _getAudioCodec(List? codecs) { + if (codecs == null) { + return null; + } + if (codecs.length == 1) { + return null; + } + return codecs.last; + } @override final StreamSource source; diff --git a/lib/src/reverse_engineering/dash_manifest.dart b/lib/src/reverse_engineering/dash_manifest.dart index 6bd093a..cf982c2 100644 --- a/lib/src/reverse_engineering/dash_manifest.dart +++ b/lib/src/reverse_engineering/dash_manifest.dart @@ -1,6 +1,10 @@ +import 'package:collection/collection.dart'; +import 'package:http_parser/http_parser.dart'; import 'package:xml/xml.dart' as xml; +import '../extensions/helpers_extension.dart'; import '../retry.dart'; +import 'models/fragment.dart'; import 'models/stream_info_provider.dart'; import 'youtube_http_client.dart'; @@ -11,14 +15,7 @@ class DashManifest { final xml.XmlDocument _root; /// - late final Iterable<_StreamInfo> streams = _root - .findElements('Representation') - .where((e) => e - .findElements('Initialization') - .first - .getAttribute('sourceURL')! - .contains('sq/')) - .map((e) => _StreamInfo(e)); + late final Iterable<_StreamInfo> streams = parseMDP(_root); /// DashManifest(this._root); @@ -38,59 +35,238 @@ class DashManifest { /// static String? getSignatureFromUrl(String url) => _urlSignatureExp.firstMatch(url)?.group(1); + + bool _isDrmProtected(xml.XmlElement element) => + element.findElements('ContentProtection').isNotEmpty; + + _SegmentTimeline? extractSegmentTimeline(xml.XmlElement source) { + final segmentTimeline = source.getElement('SegmentTimeline'); + if (segmentTimeline != null) { + return _SegmentTimeline(segmentTimeline.findAllElements('S').map((e) { + final d = int.tryParse(e.getAttribute('d') ?? '0')!; + final r = int.tryParse(e.getAttribute('r') ?? '0')!; + return _S(d, r); + }).toList()); + } + return null; + } + + _MsInfo extractMultiSegmentInfo( + xml.XmlElement element, _MsInfo msParentInfo) { + final msInfo = msParentInfo.copy(); // Copy + + final segmentList = element.getElement('SegmentList'); + if (segmentList != null) { + msInfo.segmentTimeline = + extractSegmentTimeline(segmentList) ?? msParentInfo.segmentTimeline; + msInfo.initializationUrl = + segmentList.getElement('Initialization')?.getAttribute('sourceURL'); + + final segmentUrlsSE = segmentList.findAllElements('SegmentURL'); + if (segmentUrlsSE.isNotEmpty) { + msInfo.segmentUrls = [ + for (final segment in segmentUrlsSE) segment.getAttribute('media')! + ]; + } + } else { + final segmentTemplate = element.getElement('SegmentTemplate'); + if (segmentTemplate != null) { + // Note: Currently SegmentTemplates are not supported. +/* final segmentTimeLine = extractSegmentTimeline(segmentTemplate); + if (segmentTimeLine != null) { + msInfo['s'] = segmentTimeLine; + } + + final timeScale = segmentTemplate.getAttribute('timescale'); + if (timeScale != null) { + msInfo['timescale'] = int.parse(timeScale); + } + + final media = segmentTemplate.getAttribute('media'); + if (media != null) { + msInfo['media'] = media; + } + final initialization = segmentTemplate.getAttribute('initialization'); + if (initialization != null) { + msInfo['initialization'] = initialization; + } else { + extractInitialization(segmentTemplate); + }*/ + } + } + return msInfo; + } + + List<_StreamInfo> parseMDP(xml.XmlDocument root) { + if (root.getAttribute('type') == 'dynamic') { + return const []; + } + + final formats = <_StreamInfo>[]; + final periods = root.findAllElements('Period'); + for (final period in periods) { + final periodMsInfo = extractMultiSegmentInfo(period, _MsInfo()); + final adaptionSets = period.findAllElements('AdaptationSet'); + for (final adaptionSet in adaptionSets) { + if (_isDrmProtected(adaptionSet)) { + continue; + } + final adaptionSetMsInfo = + extractMultiSegmentInfo(adaptionSet, periodMsInfo); + for (final representation + in adaptionSet.findAllElements('Representation')) { + if (_isDrmProtected(representation)) { + continue; + } + final representationAttrib = { + for (var e in adaptionSet.attributes) e.name.local: e.value, + for (var e in representation.attributes) e.name.local: e.value, + }; + + final mimeType = MediaType.parse(representationAttrib['mimeType']!); + + if (mimeType.type == 'video' || mimeType.type == 'audio') { + // Extract the base url + var baseUrl = JoinedIterable([ + representation.childElements, + adaptionSet.childElements, + period.childElements, + root.childElements + ]) + .firstWhereOrNull((e) { + final baseUrlE = e.getElement('BaseURL')?.text.trim(); + if (baseUrlE == null) { + return false; + } + return baseUrlE.contains(RegExp('^https?://')); + }) + ?.text + .trim(); + + if (baseUrl == null || !baseUrl.startsWith('http')) { + throw UnimplementedError( + 'This kind of DASH Stream is not yet implemented. ' + 'Please open a new issue on this project GitHub.'); + } + + final representationMsInfo = + extractMultiSegmentInfo(representation, adaptionSetMsInfo); + + if (representationMsInfo.segmentUrls != null && + representationMsInfo.segmentTimeline != null) { + final fragments = []; + var segmentIndex = 0; + for (final s in representationMsInfo.segmentTimeline!.segments) { + for (var i = 0; i < (s.r + 1); i++) { + final segmentUri = + representationMsInfo.segmentUrls![segmentIndex]; + if (segmentUri.contains(RegExp('^https?://'))) { + throw UnimplementedError( + 'This kind of DASH Stream is not yet implemented. ' + 'Please open a new issue on this project GitHub.'); + } + fragments.add(Fragment(segmentUri)); + segmentIndex++; + } + } + representationMsInfo.fragments = fragments; + } + + final fragments = [ + if (representationMsInfo.fragments != null && + representationMsInfo.initializationUrl != null) + Fragment(representationMsInfo.initializationUrl!), + ...?representationMsInfo.fragments + ]; + + formats.add(_StreamInfo( + int.parse(representationAttrib['id']!), + baseUrl, + mimeType, + int.tryParse(representationAttrib['width'] ?? ''), + int.tryParse(representationAttrib['height'] ?? ''), + int.tryParse(representationAttrib['frameRate'] ?? ''), + fragments)); + } + } + } + } + + return formats; + } } class _StreamInfo extends StreamInfoProvider { - static final _contentLenExp = RegExp(r'[/\?]clen[/=](\d+)'); + @override + final int tag; - final xml.XmlElement root; + @override + final String url; - _StreamInfo(this.root); + @override + String get container => _mimetype.subtype; + + final MediaType _mimetype; + + bool get isAudioOnly => _mimetype.type == 'audio'; + + @override + String? get audioCodec => isAudioOnly ? _mimetype.subtype : null; + + @override + String? get videoCodec => isAudioOnly ? null : _mimetype.subtype; + + @override + final int? videoWidth; + + @override + final int? videoHeight; + + @override + final int? framerate; + + @override + final List fragments; @override StreamSource get source => StreamSource.dash; - @override - late final int tag = int.parse(root.getAttribute('id')!); - - @override - late final String url = root.getAttribute('BaseURL')!; - - @override - late final int contentLength = int.parse( - (root.getAttribute('contentLength') ?? - _contentLenExp.firstMatch(url)?.group(1))!); - - @override - late final int bitrate = int.parse(root.getAttribute('bandwidth')!); - - @override - late final String? container = ''; - - /* - Uri.decodeFull((_containerExp.firstMatch(url)?.group(1))!);*/ - - late final bool isAudioOnly = - root.findElements('AudioChannelConfiguration').isNotEmpty; - - @override - late final String? audioCodec = - isAudioOnly ? null : root.getAttribute('codecs'); - - @override - late final String? videoCodec = - isAudioOnly ? root.getAttribute('codecs') : null; - - @override - late final int videoWidth = int.parse(root.getAttribute('width')!); - - @override - late final int videoHeight = int.parse(root.getAttribute('height')!); - - @override - late final int framerate = int.parse(root.getAttribute('framerate')!); - - // TODO: Implement this - @override - late final String? videoQualityLabel = null; + _StreamInfo(this.tag, this.url, this._mimetype, this.videoWidth, + this.videoHeight, this.framerate, this.fragments); +} + +class _SegmentTimeline { + final List<_S> segments; + + const _SegmentTimeline(this.segments); +} + +class _S { + final int d; + final int r; + + const _S(this.d, this.r); +} + +class _MsInfo { + int startNumber = 1; + + String? initializationUrl; + _SegmentTimeline? segmentTimeline; + List? segmentUrls; + List? fragments; + + _MsInfo(); + + _MsInfo copy() { + final v = _MsInfo(); + + v.initializationUrl = initializationUrl; + v.segmentTimeline = segmentTimeline; + v.segmentUrls = segmentUrls; + v.fragments = fragments; + v.startNumber = startNumber; + + return v; + } } diff --git a/lib/src/reverse_engineering/heuristics.dart b/lib/src/reverse_engineering/heuristics.dart index afb347a..d912836 100644 --- a/lib/src/reverse_engineering/heuristics.dart +++ b/lib/src/reverse_engineering/heuristics.dart @@ -70,8 +70,7 @@ extension VideoQualityUtil on VideoQuality { return VideoQuality.high4320; } - throw ArgumentError.value( - label, 'label', 'Unrecognized video quality label'); + return VideoQuality.unknown; } /// diff --git a/lib/src/reverse_engineering/models/fragment.dart b/lib/src/reverse_engineering/models/fragment.dart new file mode 100644 index 0000000..df7ddf0 --- /dev/null +++ b/lib/src/reverse_engineering/models/fragment.dart @@ -0,0 +1,6 @@ +/// Fragment used for DASH Manifests. +class Fragment { + final String path; + + const Fragment(this.path); +} diff --git a/lib/src/reverse_engineering/models/stream_info_provider.dart b/lib/src/reverse_engineering/models/stream_info_provider.dart index 3b69547..e6ab29c 100644 --- a/lib/src/reverse_engineering/models/stream_info_provider.dart +++ b/lib/src/reverse_engineering/models/stream_info_provider.dart @@ -1,3 +1,5 @@ +import 'fragment.dart'; + enum StreamSource { muxed, adaptive, dash } /// @@ -24,7 +26,7 @@ abstract class StreamInfoProvider { int? get contentLength => null; /// - int? get bitrate; + int? get bitrate => null; /// String? get container; @@ -36,7 +38,7 @@ abstract class StreamInfoProvider { String? get videoCodec => null; /// - String? get videoQualityLabel; + String? get videoQualityLabel => null; /// int? get videoWidth => null; @@ -46,4 +48,7 @@ abstract class StreamInfoProvider { /// int? get framerate => null; + + /// + List? get fragments => null; } diff --git a/lib/src/reverse_engineering/pages/channel_about_page.dart b/lib/src/reverse_engineering/pages/channel_about_page.dart index 7fc3f7c..2d3c189 100644 --- a/lib/src/reverse_engineering/pages/channel_about_page.dart +++ b/lib/src/reverse_engineering/pages/channel_about_page.dart @@ -1,11 +1,11 @@ import 'package:collection/collection.dart'; import 'package:html/parser.dart' as parser; -import 'package:youtube_explode_dart/src/reverse_engineering/models/youtube_page.dart'; import '../../../youtube_explode_dart.dart'; import '../../extensions/helpers_extension.dart'; import '../../retry.dart'; import '../models/initial_data.dart'; +import '../models/youtube_page.dart'; import '../youtube_http_client.dart'; /// diff --git a/lib/src/reverse_engineering/pages/search_page.dart b/lib/src/reverse_engineering/pages/search_page.dart index 6e2bc61..cd3938f 100644 --- a/lib/src/reverse_engineering/pages/search_page.dart +++ b/lib/src/reverse_engineering/pages/search_page.dart @@ -1,16 +1,16 @@ import 'package:collection/collection.dart'; import 'package:html/parser.dart' as parser; -import 'package:youtube_explode_dart/src/reverse_engineering/models/youtube_page.dart'; -import 'package:youtube_explode_dart/src/search/search_channel.dart'; import '../../../youtube_explode_dart.dart'; import '../../extensions/helpers_extension.dart'; import '../../retry.dart'; import '../../search/base_search_content.dart'; +import '../../search/search_channel.dart'; import '../../search/search_filter.dart'; import '../../search/search_video.dart'; import '../../videos/videos.dart'; import '../models/initial_data.dart'; +import '../models/youtube_page.dart'; import '../youtube_http_client.dart'; /// diff --git a/lib/src/reverse_engineering/player/player_response.dart b/lib/src/reverse_engineering/player/player_response.dart index 3e2f4f7..2cd9880 100644 --- a/lib/src/reverse_engineering/player/player_response.dart +++ b/lib/src/reverse_engineering/player/player_response.dart @@ -96,7 +96,8 @@ class PlayerResponse { late final List muxedStreams = root .get('streamingData') ?.getList('formats') - ?.map((e) => _StreamInfo(e, StreamSource.muxed)) + ?.where((e) => e['url'] != null) + .map((e) => _StreamInfo(e, StreamSource.muxed)) .cast() .toList() ?? const []; diff --git a/lib/src/reverse_engineering/youtube_http_client.dart b/lib/src/reverse_engineering/youtube_http_client.dart index 07d08fe..a956d97 100644 --- a/lib/src/reverse_engineering/youtube_http_client.dart +++ b/lib/src/reverse_engineering/youtube_http_client.dart @@ -14,6 +14,7 @@ class YoutubeHttpClient extends http.BaseClient { // Flag to interrupt receiving stream. bool _closed = false; + bool get closed => _closed; static const Map _defaultHeaders = { @@ -124,17 +125,54 @@ class YoutubeHttpClient extends http.BaseClient { } Stream> getStream(StreamInfo streamInfo, + {Map headers = const {}, + bool validate = true, + int start = 0, + int errorCount = 0}) { + if (streamInfo.fragments.isNotEmpty) { + // DASH(fragmented) stream + return _getFragmentedStream(streamInfo, + headers: headers, + validate: validate, + start: start, + errorCount: errorCount); + } + // Normal stream + return _getStream(streamInfo, + headers: headers, + validate: validate, + start: start, + errorCount: errorCount); + } + + Stream> _getFragmentedStream(StreamInfo streamInfo, {Map headers = const {}, bool validate = true, int start = 0, int errorCount = 0}) async* { - var url = streamInfo.url; + // This is the base url. + final url = streamInfo.url; + for (final fragment in streamInfo.fragments) { + final req = await retry( + this, () => get(Uri.parse(url.toString() + fragment.path))); + yield req.bodyBytes; + } + } + + Stream> _getStream(StreamInfo streamInfo, + {Map headers = const {}, + bool validate = true, + int start = 0, + int errorCount = 0}) async* { + final url = streamInfo.url; var bytesCount = start; + while (!_closed && bytesCount != streamInfo.size.totalBytes) { try { final response = await retry(this, () { final request = http.Request('get', url); - request.headers['range'] = 'bytes=$bytesCount-${bytesCount + 9898989 - 1}'; + request.headers['range'] = + 'bytes=$bytesCount-${bytesCount + 9898989 - 1}'; return send(request); }); if (validate) { @@ -154,7 +192,7 @@ class YoutubeHttpClient extends http.BaseClient { rethrow; } await Future.delayed(const Duration(milliseconds: 500)); - yield* getStream(streamInfo, + yield* _getStream(streamInfo, headers: headers, validate: validate, start: bytesCount, @@ -218,8 +256,8 @@ class YoutubeHttpClient extends http.BaseClient { request.headers[key] = _defaultHeaders[key]!; } }); - // print('Request: $request'); - // print('Stack:\n${StackTrace.current}'); + print('Request: $request'); + print('Stack:\n${StackTrace.current}'); return _httpClient.send(request); } } diff --git a/lib/src/search/search_client.dart b/lib/src/search/search_client.dart index 5ec9852..683ece9 100644 --- a/lib/src/search/search_client.dart +++ b/lib/src/search/search_client.dart @@ -58,8 +58,10 @@ class SearchClient { // ignore: literal_only_boolean_expressions for (;;) { if (page == null) { - page = await retry(_httpClient, () async => - SearchPage.get(_httpClient, searchQuery, filter: filter)); + page = await retry( + _httpClient, + () async => + SearchPage.get(_httpClient, searchQuery, filter: filter)); } else { page = await page.nextPage(_httpClient); if (page == null) { diff --git a/lib/src/search/search_list.dart b/lib/src/search/search_list.dart index c635f03..17e30a5 100644 --- a/lib/src/search/search_list.dart +++ b/lib/src/search/search_list.dart @@ -1,10 +1,10 @@ import 'dart:async'; import 'package:collection/collection.dart'; -import 'package:youtube_explode_dart/src/reverse_engineering/pages/search_page.dart'; import '../../youtube_explode_dart.dart'; import '../extensions/helpers_extension.dart'; +import '../reverse_engineering/pages/search_page.dart'; /// This list contains search videos. ///This behaves like a [List] but has the [SearchList.nextPage] to get the next batch of videos. diff --git a/lib/src/videos/closed_captions/closed_caption_client.dart b/lib/src/videos/closed_captions/closed_caption_client.dart index 3a4efa5..da3ed65 100644 --- a/lib/src/videos/closed_captions/closed_caption_client.dart +++ b/lib/src/videos/closed_captions/closed_caption_client.dart @@ -1,8 +1,7 @@ -import 'package:youtube_explode_dart/src/reverse_engineering/pages/watch_page.dart'; - import '../../extensions/helpers_extension.dart'; import '../../reverse_engineering/clients/closed_caption_client.dart' as re show ClosedCaptionClient; +import '../../reverse_engineering/pages/watch_page.dart'; import '../../reverse_engineering/youtube_http_client.dart'; import '../videos.dart'; import 'closed_caption.dart'; diff --git a/lib/src/videos/streams/audio_only_stream_info.dart b/lib/src/videos/streams/audio_only_stream_info.dart index d7b2cb3..956995a 100644 --- a/lib/src/videos/streams/audio_only_stream_info.dart +++ b/lib/src/videos/streams/audio_only_stream_info.dart @@ -1,28 +1,17 @@ +import '../../reverse_engineering/models/fragment.dart'; import 'streams.dart'; /// YouTube media stream that only contains audio. -class AudioOnlyStreamInfo implements AudioStreamInfo { - @override - final int tag; - - @override - final Uri url; - - @override - final StreamContainer container; - - @override - final FileSize size; - - @override - final Bitrate bitrate; - - @override - final String audioCodec; - - /// Initializes an instance of [AudioOnlyStreamInfo] - AudioOnlyStreamInfo(this.tag, this.url, this.container, this.size, - this.bitrate, this.audioCodec); +class AudioOnlyStreamInfo extends AudioStreamInfo { + AudioOnlyStreamInfo( + int tag, + Uri url, + StreamContainer container, + FileSize size, + Bitrate bitrate, + String audioCodec, + List fragments) + : super(tag, url, container, size, bitrate, audioCodec, fragments); @override String toString() => 'Audio-only ($tag | $container)'; diff --git a/lib/src/videos/streams/audio_stream_info.dart b/lib/src/videos/streams/audio_stream_info.dart index d6de676..43d7369 100644 --- a/lib/src/videos/streams/audio_stream_info.dart +++ b/lib/src/videos/streams/audio_stream_info.dart @@ -1,3 +1,4 @@ +import '../../reverse_engineering/models/fragment.dart'; import 'streams.dart'; /// YouTube media stream that contains audio. @@ -7,6 +8,6 @@ abstract class AudioStreamInfo extends StreamInfo { /// AudioStreamInfo(int tag, Uri url, StreamContainer container, FileSize size, - Bitrate bitrate, this.audioCodec) - : super(tag, url, container, size, bitrate); + Bitrate bitrate, this.audioCodec, List fragments) + : super(tag, url, container, size, bitrate, fragments); } diff --git a/lib/src/videos/streams/bitrate.dart b/lib/src/videos/streams/bitrate.dart index e22e0bf..287430e 100644 --- a/lib/src/videos/streams/bitrate.dart +++ b/lib/src/videos/streams/bitrate.dart @@ -23,6 +23,8 @@ class Bitrate with Comparable, _$Bitrate { const Bitrate._(); + static const Bitrate unknown = Bitrate(0); + @override int compareTo(Bitrate other) => bitsPerSecond.compareTo(other.bitsPerSecond); diff --git a/lib/src/videos/streams/filesize.dart b/lib/src/videos/streams/filesize.dart index 5135567..d73c960 100644 --- a/lib/src/videos/streams/filesize.dart +++ b/lib/src/videos/streams/filesize.dart @@ -23,6 +23,8 @@ class FileSize with Comparable, _$FileSize { const FileSize._(); + static const FileSize unknown = FileSize(0); + @override int compareTo(FileSize other) => totalBytes.compareTo(other.totalBytes); diff --git a/lib/src/videos/streams/muxed_stream_info.dart b/lib/src/videos/streams/muxed_stream_info.dart index 4e77c2a..91c8e1e 100644 --- a/lib/src/videos/streams/muxed_stream_info.dart +++ b/lib/src/videos/streams/muxed_stream_info.dart @@ -1,3 +1,4 @@ +import '../../reverse_engineering/models/fragment.dart'; import 'audio_stream_info.dart'; import 'bitrate.dart'; import 'filesize.dart'; @@ -46,6 +47,10 @@ class MuxedStreamInfo implements AudioStreamInfo, VideoStreamInfo { @override final Framerate framerate; + /// Muxed streams never have fragments. + @override + List get fragments => const []; + /// Initializes an instance of [MuxedStreamInfo] MuxedStreamInfo( this.tag, diff --git a/lib/src/videos/streams/stream_info.dart b/lib/src/videos/streams/stream_info.dart index a9e23b4..0dc7ddd 100644 --- a/lib/src/videos/streams/stream_info.dart +++ b/lib/src/videos/streams/stream_info.dart @@ -1,3 +1,4 @@ +import '../../reverse_engineering/models/fragment.dart'; import 'bitrate.dart'; import 'filesize.dart'; import 'stream_container.dart'; @@ -20,8 +21,12 @@ abstract class StreamInfo { /// Stream bitrate. final Bitrate bitrate; + /// DASH streams contain multiple stream fragments. + final List fragments; + /// Initialize an instance of [StreamInfo]. - StreamInfo(this.tag, this.url, this.container, this.size, this.bitrate); + StreamInfo(this.tag, this.url, this.container, this.size, this.bitrate, + this.fragments); } /// Extension for Iterables of StreamInfo. diff --git a/lib/src/videos/streams/streams_client.dart b/lib/src/videos/streams/streams_client.dart index 8f75d5a..72e5bb4 100644 --- a/lib/src/videos/streams/streams_client.dart +++ b/lib/src/videos/streams/streams_client.dart @@ -151,15 +151,20 @@ class StreamsClient { url = url.setQueryParam(signatureParameter, signature); } - // Content length - var contentLength = streamInfo.contentLength ?? - await _httpClient.getContentLength(url, validate: false) ?? - 0; + // Content length - Dont try to get content length of a dash stream. + var contentLength = streamInfo.source == StreamSource.dash + ? 0 + : streamInfo.contentLength ?? + await _httpClient.getContentLength(url, validate: false) ?? + 0; + if (contentLength == 0 && streamInfo.source != StreamSource.dash) { + continue; + } // Common var container = StreamContainer.parse(streamInfo.container!); var fileSize = FileSize(contentLength); - var bitrate = Bitrate(streamInfo.bitrate!); + var bitrate = Bitrate(streamInfo.bitrate ?? 0); var audioCodec = streamInfo.audioCodec; var videoCodec = streamInfo.videoCodec; @@ -167,7 +172,7 @@ class StreamsClient { // Muxed or Video-only if (!videoCodec.isNullOrWhiteSpace) { var framerate = Framerate(streamInfo.framerate ?? 24); - var videoQualityLabel = streamInfo.videoQualityLabel!; + var videoQualityLabel = streamInfo.videoQualityLabel ?? ''; var videoQuality = VideoQualityUtil.fromLabel(videoQualityLabel); @@ -206,13 +211,14 @@ class StreamsClient { videoQualityLabel, videoQuality, videoResolution, - framerate); + framerate, + streamInfo.fragments ?? const []); continue; } // Audio-only if (!audioCodec.isNullOrWhiteSpace) { - streams[tag] = AudioOnlyStreamInfo( - tag, url, container, fileSize, bitrate, audioCodec!); + streams[tag] = AudioOnlyStreamInfo(tag, url, container, fileSize, + bitrate, audioCodec!, streamInfo.fragments ?? const []); } // #if DEBUG @@ -228,13 +234,13 @@ class StreamsClient { videoId = VideoId.fromString(videoId); try { - final context = await _getStreamContextFromEmbeddedClient(videoId); + final context = await _getStreamContextFromWatchPage(videoId); return _getManifest(context); } on YoutubeExplodeException { //TODO: ignore } - final context = await _getStreamContextFromWatchPage(videoId); + final context = await _getStreamContextFromEmbeddedClient(videoId); return _getManifest(context); } diff --git a/lib/src/videos/streams/video_only_stream_info.dart b/lib/src/videos/streams/video_only_stream_info.dart index cb38ef7..9c403e8 100644 --- a/lib/src/videos/streams/video_only_stream_info.dart +++ b/lib/src/videos/streams/video_only_stream_info.dart @@ -1,3 +1,4 @@ +import '../../reverse_engineering/models/fragment.dart'; import 'bitrate.dart'; import 'filesize.dart'; import 'framerate.dart'; @@ -7,50 +8,22 @@ import 'video_resolution.dart'; import 'video_stream_info.dart'; /// YouTube media stream that only contains video. -class VideoOnlyStreamInfo implements VideoStreamInfo { - @override - final int tag; - - @override - final Uri url; - - @override - final StreamContainer container; - - @override - final FileSize size; - - @override - final Bitrate bitrate; - - @override - final String videoCodec; - - @override - final String videoQualityLabel; - - @override - final VideoQuality videoQuality; - - @override - final VideoResolution videoResolution; - - @override - final Framerate framerate; - - /// Initializes an instance of [VideoOnlyStreamInfo] +class VideoOnlyStreamInfo extends VideoStreamInfo { VideoOnlyStreamInfo( - this.tag, - this.url, - this.container, - this.size, - this.bitrate, - this.videoCodec, - this.videoQualityLabel, - this.videoQuality, - this.videoResolution, - this.framerate); + int tag, + Uri url, + StreamContainer container, + FileSize size, + Bitrate bitrate, + String videoCodec, + String videoQualityLabel, + VideoQuality videoQuality, + VideoResolution videoResolution, + Framerate framerate, + List fragments) + : super(tag, url, container, size, bitrate, videoCodec, videoQualityLabel, + videoQuality, videoResolution, framerate, fragments); @override - String toString() => 'Video-only ($tag | $videoQualityLabel | $container)'; + String toString() => 'Video-only ($tag | $videoResolution | $container)'; } diff --git a/lib/src/videos/streams/video_quality.dart b/lib/src/videos/streams/video_quality.dart index 6642151..9431fef 100644 --- a/lib/src/videos/streams/video_quality.dart +++ b/lib/src/videos/streams/video_quality.dart @@ -1,7 +1,7 @@ /// Video quality. enum VideoQuality { /// Unknown video quality. - /// (This should be reported to the project's repo.) + /// (This should be reported to the project's repo if this is *NOT* a DASH Stream .) unknown, /// Low quality (144p). diff --git a/lib/src/videos/streams/video_stream_info.dart b/lib/src/videos/streams/video_stream_info.dart index 81b2f75..03ae5d7 100644 --- a/lib/src/videos/streams/video_stream_info.dart +++ b/lib/src/videos/streams/video_stream_info.dart @@ -1,3 +1,4 @@ +import '../../reverse_engineering/models/fragment.dart'; import 'streams.dart'; /// YouTube media stream that contains video. @@ -28,8 +29,9 @@ abstract class VideoStreamInfo extends StreamInfo { this.videoQualityLabel, this.videoQuality, this.videoResolution, - this.framerate) - : super(tag, url, container, size, bitrate); + this.framerate, + List fragments) + : super(tag, url, container, size, bitrate, fragments); } /// Extensions for Iterables of [VideoStreamInfo] diff --git a/test/streams_test.dart b/test/streams_test.dart index 72f64e7..a779dc2 100644 --- a/test/streams_test.dart +++ b/test/streams_test.dart @@ -28,14 +28,16 @@ void main() { }) { test('VideoId - ${val.value}', () async { var manifest = await yt!.videos.streamsClient.getManifest(val); - expect(manifest.streams, isNotEmpty); + expect(manifest.videoOnly, isNotEmpty); + expect(manifest.audioOnly, isNotEmpty); }, timeout: const Timeout(Duration(seconds: 90))); } }); + // Seems that youtube broke something and now this throws VideoUnplayableException instead of VideoRequiresPurchaseException test('Stream of paid videos throw VideoRequiresPurchaseException', () { expect(yt!.videos.streamsClient.getManifest(VideoId('p3dDcKOFXQg')), - throwsA(const TypeMatcher())); + throwsA(const TypeMatcher())); }); test('Stream of age-limited video throws VideoUnplayableException', () { @@ -49,27 +51,19 @@ void main() { isNotEmpty); }); - group('Stream of unavailable videos throws VideoUnavailableException', () { + // Seems that youtube broke something and now this throws VideoUnplayableException instead of VideoUnavailableException + group('Stream of unavailable videos throws VideoUnplayableException', () { for (final val in {VideoId('qld9w0b-1ao'), VideoId('pb_hHv3fByo')}) { test('VideoId - ${val.value}', () { expect(yt!.videos.streamsClient.getManifest(val), - throwsA(const TypeMatcher())); + throwsA(const TypeMatcher())); }); } }); group('Get specific stream of any playable video', () { for (final val in { - VideoId('9bZkp7q19f0'), //Normal - VideoId('rsAAeyAr-9Y'), //LiveStreamRecording - VideoId('V5Fsj_sCKdg'), //ContainsHighQualityStreams VideoId('AI7ULzgf8RU'), //ContainsDashManifest - VideoId('-xNN-bJQ4vI'), //Omnidirectional - VideoId('vX2vsvdq8nw'), //HighDynamicRange - VideoId('YltHGKX80Y8'), //ContainsClosedCaptions - VideoId('_kmeFXjjGfk'), //EmbedRestrictedByYouTube - VideoId('MeJVWBSsPAY'), //EmbedRestrictedByAuthor - VideoId('hySoCSoH-g8'), //AgeRestrictedEmbedRestricted VideoId('5VGm0dczmHc'), //RatingDisabled }) { test('VideoId - ${val.value}', () async {