From 911712cfa19b5d555c363426971ea33040f54e0c Mon Sep 17 00:00:00 2001 From: Hexah Date: Sun, 31 May 2020 23:36:23 +0200 Subject: [PATCH] More on v5 --- analysis_options.yaml | 1 + lib/src/exceptions/exceptions.dart | 6 +- .../exceptions/fatal_failure_exception.dart | 6 +- .../request_limit_exceeded_exception.dart | 8 +- .../transient_failure_exception.dart | 25 +++ .../unrecognized_structure_exception.dart | 18 -- .../video_requires_purchase_exception.dart | 26 ++- .../video_stream_unavailable_exception.dart | 20 -- .../video_unavailable_exception.dart | 22 ++- .../video_unplayable_exception.dart | 40 ++-- lib/src/extensions/caption_extension.dart | 2 +- lib/src/models/models.dart | 1 + lib/src/models/video_id.dart | 68 +++++++ lib/src/retry.dart | 36 ++++ .../responses/channel_page.dart | 59 +++--- .../closed_caption_track_response.dart | 63 +++++++ .../responses/dash_manifest.dart | 68 +++++++ .../responses/embed_page.dart | 43 +++++ .../responses/player_response.dart | 174 ++++++++++++++++++ .../responses/player_source.dart | 116 ++++++++++++ .../responses/playerlist_response.dart | 103 +++++++++++ .../responses/stream_info_provider.dart | 40 ++++ .../responses/video_info_response.dart | 108 +++++++++++ .../responses/watch_page.dart | 64 +++++++ tools/test.dart | 33 ++++ 25 files changed, 1048 insertions(+), 102 deletions(-) create mode 100644 lib/src/exceptions/transient_failure_exception.dart delete mode 100644 lib/src/exceptions/unrecognized_structure_exception.dart delete mode 100644 lib/src/exceptions/video_stream_unavailable_exception.dart create mode 100644 lib/src/models/video_id.dart create mode 100644 lib/src/retry.dart create mode 100644 lib/src/reverse_engineering/responses/closed_caption_track_response.dart create mode 100644 lib/src/reverse_engineering/responses/dash_manifest.dart create mode 100644 lib/src/reverse_engineering/responses/embed_page.dart create mode 100644 lib/src/reverse_engineering/responses/player_response.dart create mode 100644 lib/src/reverse_engineering/responses/player_source.dart create mode 100644 lib/src/reverse_engineering/responses/playerlist_response.dart create mode 100644 lib/src/reverse_engineering/responses/stream_info_provider.dart create mode 100644 lib/src/reverse_engineering/responses/video_info_response.dart create mode 100644 lib/src/reverse_engineering/responses/watch_page.dart create mode 100644 tools/test.dart diff --git a/analysis_options.yaml b/analysis_options.yaml index 3f512d8..e54b130 100644 --- a/analysis_options.yaml +++ b/analysis_options.yaml @@ -13,6 +13,7 @@ linter: - prefer_const_literals_to_create_immutables - prefer_constructors_over_static_methods - prefer_contains + - annotate_overrides analyzer: exclude: diff --git a/lib/src/exceptions/exceptions.dart b/lib/src/exceptions/exceptions.dart index c9e5c3d..885350b 100644 --- a/lib/src/exceptions/exceptions.dart +++ b/lib/src/exceptions/exceptions.dart @@ -1,7 +1,9 @@ library youtube_explode.exceptions; -export 'unrecognized_structure_exception.dart'; +export 'fatal_failure_exception.dart'; +export 'request_limit_exceeded_exception.dart'; +export 'transient_failure_exception.dart'; export 'video_requires_purchase_exception.dart'; -export 'video_stream_unavailable_exception.dart'; export 'video_unavailable_exception.dart'; export 'video_unplayable_exception.dart'; +export 'youtube_explode_exception.dart'; diff --git a/lib/src/exceptions/fatal_failure_exception.dart b/lib/src/exceptions/fatal_failure_exception.dart index ce6c6b4..e9d519d 100644 --- a/lib/src/exceptions/fatal_failure_exception.dart +++ b/lib/src/exceptions/fatal_failure_exception.dart @@ -1,17 +1,17 @@ -import 'dart:io'; - import 'package:http/http.dart'; import 'youtube_explode_exception.dart'; /// Exception thrown when a fatal failure occurs. class FatalFailureException implements YoutubeExplodeException { + + /// Description message final String message; /// Initializes an instance of [FatalFailureException] FatalFailureException(this.message); - /// Initializes an instance of [FatalFailureException] with an [HttpRequest] + /// Initializes an instance of [FatalFailureException] with a [Response] FatalFailureException.httpRequest(Response response) : message = ''' Failed to perform an HTTP request to YouTube due to a fatal failure. diff --git a/lib/src/exceptions/request_limit_exceeded_exception.dart b/lib/src/exceptions/request_limit_exceeded_exception.dart index edc90f1..779e1a4 100644 --- a/lib/src/exceptions/request_limit_exceeded_exception.dart +++ b/lib/src/exceptions/request_limit_exceeded_exception.dart @@ -4,12 +4,14 @@ import 'youtube_explode_exception.dart'; /// Exception thrown when a fatal failure occurs. class RequestLimitExceeded implements YoutubeExplodeException { + + /// Description message final String message; - /// Initializes an instance of [FatalFailureException] + /// Initializes an instance of [RequestLimitExceeded] RequestLimitExceeded(this.message); - /// Initializes an instance of [FatalFailureException] with an [HttpRequest] + /// Initializes an instance of [RequestLimitExceeded] with a [Response] RequestLimitExceeded.httpRequest(Response response) : message = ''' Failed to perform an HTTP request to YouTube because of rate limiting. @@ -21,5 +23,5 @@ Response: $response '''; @override - String toString() => 'FatalFailureException: $message'; + String toString() => 'RequestLimitExceeded: $message'; } diff --git a/lib/src/exceptions/transient_failure_exception.dart b/lib/src/exceptions/transient_failure_exception.dart new file mode 100644 index 0000000..269640a --- /dev/null +++ b/lib/src/exceptions/transient_failure_exception.dart @@ -0,0 +1,25 @@ +import 'package:http/http.dart'; + +import 'youtube_explode_exception.dart'; + +/// Exception thrown when a fatal failure occurs. +class TransientFailureException implements YoutubeExplodeException { + final String message; + + /// Initializes an instance of [TransientFailureException] + TransientFailureException(this.message); + + /// Initializes an instance of [TransientFailureException] with a [Response] + TransientFailureException.httpRequest(Response response) + : message = ''' +Failed to perform an HTTP request to YouTube due to a transient failure. +In most cases, this error indicates that the problem is on YouTube's side and this is not a bug in the library. +To resolve this error, please wait some time and try again. +If this issue persists, please report it on the project's GitHub page. +Request: ${response.request} +Response: $response +'''; + + @override + String toString() => 'TransientFailureException: $message'; +} diff --git a/lib/src/exceptions/unrecognized_structure_exception.dart b/lib/src/exceptions/unrecognized_structure_exception.dart deleted file mode 100644 index 097ca21..0000000 --- a/lib/src/exceptions/unrecognized_structure_exception.dart +++ /dev/null @@ -1,18 +0,0 @@ -/// Thrown when YoutubeExplode fails to extract required information. -/// This usually happens when YouTube makes changes that break YoutubeExplode. -class UnrecognizedStructureException implements FormatException { - ///A message describing the format error. - @override - final String message; - - /// The actual source input which caused the error. - @override - final String source; - - /// Initializes an instance of [UnrecognizedStructureException] - const UnrecognizedStructureException([this.message, this.source]); - - /// Unimplemented - @override - int get offset => throw UnsupportedError('Offset not supported'); -} diff --git a/lib/src/exceptions/video_requires_purchase_exception.dart b/lib/src/exceptions/video_requires_purchase_exception.dart index 6e85ca8..4afa685 100644 --- a/lib/src/exceptions/video_requires_purchase_exception.dart +++ b/lib/src/exceptions/video_requires_purchase_exception.dart @@ -1,16 +1,24 @@ -import 'exceptions.dart'; +import '../models/models.dart'; -/// Thrown when a video is not playable because it requires purchase. +import 'video_unplayable_exception.dart'; + +/// Exception thrown when the requested video requires purchase. class VideoRequiresPurchaseException implements VideoUnplayableException { - /// ID of the video. - final String videoId; + /// Description message + final String message; - /// ID of the preview video. - final String previewVideoId; + /// VideoId instance + final VideoId previewVideoId; /// Initializes an instance of [VideoRequiresPurchaseException] - const VideoRequiresPurchaseException(this.videoId, this.previewVideoId); + VideoRequiresPurchaseException(this.message, this.previewVideoId); - @override - String get reason => 'Requires purchase'; + /// Initializes an instance of [VideoUnplayableException] with a [VideoId] + VideoRequiresPurchaseException.unavailable(this.previewVideoId) + : message = 'Video \'$previewVideoId\' is unavailable.\n' + 'In most cases, this error indicates that the video doesn\'t exist, ' // ignore: lines_longer_than_80_chars + 'is private, or has been taken down.\n' + 'If you can however open this video in your browser in incognito mode, ' // ignore: lines_longer_than_80_chars + 'it most likely means that YouTube changed something, which broke this library.\n' // ignore: lines_longer_than_80_chars + 'Please report this issue on GitHub in that case.'; } diff --git a/lib/src/exceptions/video_stream_unavailable_exception.dart b/lib/src/exceptions/video_stream_unavailable_exception.dart deleted file mode 100644 index a387f13..0000000 --- a/lib/src/exceptions/video_stream_unavailable_exception.dart +++ /dev/null @@ -1,20 +0,0 @@ - - -/// Thrown when a video stream is not available -/// and returns a status code not equal to 200 OK. -class VideoStreamUnavailableException implements Exception { - - /// The returned status code. - final int statusCode; - - /// Url - final Uri url; - - /// Initializes an instance of [VideoStreamUnavailableException] - VideoStreamUnavailableException(this.statusCode, this.url); - - @override - String toString() => 'VideoStreamUnavailableException: ' - 'The video stream in not availble (status code: $statusCode).\n' - 'Url: $url'; -} \ No newline at end of file diff --git a/lib/src/exceptions/video_unavailable_exception.dart b/lib/src/exceptions/video_unavailable_exception.dart index 0cf57e7..7ad9856 100644 --- a/lib/src/exceptions/video_unavailable_exception.dart +++ b/lib/src/exceptions/video_unavailable_exception.dart @@ -1,14 +1,22 @@ +import 'exceptions.dart'; +import '../models/models.dart'; + /// Thrown when a video is not available and cannot be processed. /// This can happen because the video does not exist, is deleted, /// is private, or due to other reasons. -class VideoUnavailableException implements Exception { - /// ID of the video. - final String videoId; +class VideoUnavailableException implements VideoUnplayableException { + /// Description message + final String message; /// Initializes an instance of [VideoUnavailableException] - const VideoUnavailableException(this.videoId); + VideoUnavailableException(this.message); - @override - String toString() => - 'VideoUnavailableException: Video $videoId is unavailable.'; + /// Initializes an instance of [VideoUnplayableException] with a [VideoId] + VideoUnavailableException.unavailable(VideoId videoId) + : message = 'Video \'$videoId\' is unavailable\n' + 'In most cases, this error indicates that the video doesn\'t exist, ' // ignore: lines_longer_than_80_chars + 'is private, or has been taken down.\n' + 'If you can however open this video in your browser in incognito mode, ' // ignore: lines_longer_than_80_chars + 'it most likely means that YouTube changed something, which broke this library.\n' // ignore: lines_longer_than_80_chars + 'Please report this issue on GitHub in that case.'; } diff --git a/lib/src/exceptions/video_unplayable_exception.dart b/lib/src/exceptions/video_unplayable_exception.dart index 3beaf45..e4e688c 100644 --- a/lib/src/exceptions/video_unplayable_exception.dart +++ b/lib/src/exceptions/video_unplayable_exception.dart @@ -1,17 +1,33 @@ -/// Thrown when a video is not playable and its streams cannot be resolved. -/// This can happen because the video requires purchase, -/// is blocked in your country, is controversial, or due to other reasons. -class VideoUnplayableException { - /// ID of the video. - final String videoId; +import '../models/models.dart'; +import 'youtube_explode_exception.dart'; - /// Reason why the video can't be played. - final String reason; +/// Exception thrown when the requested video is unplayable. +class VideoUnplayableException implements YoutubeExplodeException { + /// Description message + final String message; /// Initializes an instance of [VideoUnplayableException] - const VideoUnplayableException(this.videoId, [this.reason]); + VideoUnplayableException(this.message); - String toString() => - 'VideoUnplayableException: Video $videoId couldn\'t be played.' - '${reason == null ? '' : 'Reason: $reason'}'; + /// Initializes an instance of [VideoUnplayableException] with a [VideoId] + VideoUnplayableException.unplayable(VideoId videoId, {String reason = ''}) + : message = 'Video \'$videoId\' is unplayable.\n' + 'Streams are not available for this video.\n' + 'In most cases, this error indicates that there are \n' + 'some restrictions in place that prevent watching this video.\n' + 'Reason: $reason'; + + /// Initializes an instance of [VideoUnplayableException] with a [VideoId] + VideoUnplayableException.liveStream(VideoId videoId) + : message = 'Video \'$videoId\' is an ongoing live stream.\n' + 'Streams are not available for this video.\n' + 'Please wait until the live stream finishes and try again.'; + + /// Initializes an instance of [VideoUnplayableException] with a [VideoId] + VideoUnplayableException.notLiveStream(VideoId videoId) + : message = 'Video \'$videoId\' is not an ongoing live stream.\n' + 'Live stream manifest is not available for this video'; + + @override + String toString() => 'VideoUnplayableException: $message'; } diff --git a/lib/src/extensions/caption_extension.dart b/lib/src/extensions/caption_extension.dart index ed92b2e..c198053 100644 --- a/lib/src/extensions/caption_extension.dart +++ b/lib/src/extensions/caption_extension.dart @@ -67,7 +67,7 @@ extension CaptionExtension on YoutubeExplode { var trackXml = await _getClosedCaptionTrackXml(info.url); var captions = []; - for (var captionXml in trackXml.findAllElements('p')) { + for (var captionXml in trackXml.findElements('p')) { var text = captionXml.text; if (text.isNullOrWhiteSpace) { continue; diff --git a/lib/src/models/models.dart b/lib/src/models/models.dart index 4356d43..f0d0b7f 100644 --- a/lib/src/models/models.dart +++ b/lib/src/models/models.dart @@ -21,3 +21,4 @@ export 'playlist_type.dart'; export 'statistics.dart'; export 'thumbnail_set.dart'; export 'video.dart'; +export 'video_id.dart'; diff --git a/lib/src/models/video_id.dart b/lib/src/models/video_id.dart new file mode 100644 index 0000000..ce68303 --- /dev/null +++ b/lib/src/models/video_id.dart @@ -0,0 +1,68 @@ +import 'package:equatable/equatable.dart'; + +import '../extensions/extensions.dart'; + +class VideoId extends Equatable { + static final _regMatchExp = RegExp(r'youtube\..+?/watch.*?v=(.*?)(?:&|/|$)'); + static final _shortMatchExp = RegExp(r'youtu\.be/(.*?)(?:\?|&|/|$)'); + static final _embedMatchExp = RegExp(r'youtube\..+?/embed/(.*?)(?:\?|&|/|$)'); + + /// ID as string. + final String value; + + /// Initializes an instance of [VideoId] with a url or video id. + VideoId(String url) + : value = parseVideoId(url) ?? + ArgumentError('Invalid YouTube video ID or URL: $url.'); + + String toString() => value; + + + @override + List get props => [value]; + + /// Returns true if the given [videoId] is valid. + static bool validateVideoId(String videoId) { + if (videoId.isNullOrWhiteSpace) { + return false; + } + + if (videoId.length != 11) { + return false; + } + + return !RegExp(r'[^0-9a-zA-Z_\-]').hasMatch(videoId); + } + + /// Parses a video id from url or if given a valid id as url returns itself. + /// Returns null if the id couldn't be extracted. + static String parseVideoId(String url) { + if (url.isNullOrWhiteSpace) { + return null; + } + + if (validateVideoId(url)) { + return url; + } + + // https://www.youtube.com/watch?v=yIVRs6YSbOM + var regMatch = _regMatchExp.firstMatch(url)?.group(1); + if (!regMatch.isNullOrWhiteSpace && validateVideoId(regMatch)) { + return regMatch; + } + + // https://youtu.be/yIVRs6YSbOM + var shortMatch = _shortMatchExp.firstMatch(url)?.group(1); + if (!shortMatch.isNullOrWhiteSpace && validateVideoId(shortMatch)) { + return shortMatch; + } + + // https://www.youtube.com/embed/yIVRs6YSbOM + var embedMatch = _embedMatchExp.firstMatch(url)?.group(1); + if (!embedMatch.isNullOrWhiteSpace && validateVideoId(embedMatch)) { + return embedMatch; + } + return null; + } + +} diff --git a/lib/src/retry.dart b/lib/src/retry.dart new file mode 100644 index 0000000..4f7fdb7 --- /dev/null +++ b/lib/src/retry.dart @@ -0,0 +1,36 @@ +import 'dart:async'; + +import 'exceptions/exceptions.dart'; + +/// Run the [function] each time an exception is thrown until the retryCount +/// is 0. +Future retry(FutureOr function()) async { + var retryCount = 5; + + while (true) { + try { + return await function(); + // ignore: avoid_catches_without_on_clauses + } on Exception catch (e) { + retryCount -= getExceptionCost(e); + if (retryCount <= 0) { + rethrow; + } + await Future.delayed(const Duration(milliseconds: 500)); + } + } +} + +/// Get "retry" cost of each YoutubeExplode exception. +int getExceptionCost(Exception e) { + if (e is TransientFailureException) { + return 1; + } + if (e is RequestLimitExceeded) { + return 2; + } + if (e is FatalFailureException) { + return 3; + } + return 100; +} diff --git a/lib/src/reverse_engineering/responses/channel_page.dart b/lib/src/reverse_engineering/responses/channel_page.dart index a424f9d..a828912 100644 --- a/lib/src/reverse_engineering/responses/channel_page.dart +++ b/lib/src/reverse_engineering/responses/channel_page.dart @@ -1,50 +1,55 @@ import 'package:html/dom.dart'; import 'package:html/parser.dart' as parser; +import 'package:youtube_explode_dart/src/exceptions/exceptions.dart'; +import 'package:youtube_explode_dart/src/retry.dart'; +import 'package:youtube_explode_dart/src/reverse_engineering/reverse_engineering.dart'; class ChannelPage { final Document _root; - bool get isOk => _root.querySelector('meta[property="og:url"]') != null; - String get channelUrl => _root - .querySelectorThrow('meta[property="og:url"]') - .getAttributeThrow('content'); + String get channelUrl => + _root.querySelector('meta[property="og:url"]')?.attributes['content']; String get channelId => channelId.substringAfter('channel/'); - String get channelTitle => _root - .querySelectorThrow('meta[property="og:title"]') - .getAttributeThrow('content'); + String get channelTitle => + _root.querySelector('meta[property="og:title"]')?.attributes['content']; - String get channelLogoUrl => _root - .querySelectorThrow('meta[property="og:image"]') - .getAttributeThrow('content'); + String get channelLogoUrl => + _root.querySelector('meta[property="og:image"]')?.attributes['content']; ChannelPage(this._root); ChannelPage.parse(String raw) : _root = parser.parse(raw); - static Future hello() {} -} + static Future get(YoutubeHttpClient httpClient, String id) { + var url = 'https://www.youtube.com/channel/$id?hl=en'; -extension on Document { - Element querySelectorThrow(String selectors) { - var element = querySelector(selectors); - if (element == null) { - //TODO: throw - } - return element; + return retry(() async { + var raw = await httpClient.getString(url); + var result = ChannelPage.parse(raw); + + if (!result.isOk) { + throw TransientFailureException('Channel page is broken'); + } + return result; + }); } -} -extension on Element { - String getAttributeThrow(String name) { - var attribute = attributes[name]; - if (attribute == null) { - //TODO: throw - } - return attribute; + static Future getByUsername(YoutubeHttpClient httpClient, String username) { + var url = 'https://www.youtube.com/user/$username?hl=en'; + + return retry(() async { + var raw = await httpClient.getString(url); + var result = ChannelPage.parse(raw); + + if (!result.isOk) { + throw TransientFailureException('Channel page is broken'); + } + return result; + }); } } diff --git a/lib/src/reverse_engineering/responses/closed_caption_track_response.dart b/lib/src/reverse_engineering/responses/closed_caption_track_response.dart new file mode 100644 index 0000000..2f807c8 --- /dev/null +++ b/lib/src/reverse_engineering/responses/closed_caption_track_response.dart @@ -0,0 +1,63 @@ +import 'package:xml/xml.dart' as xml; +import 'package:youtube_explode_dart/src/retry.dart'; +import 'package:youtube_explode_dart/src/reverse_engineering/reverse_engineering.dart'; + +class ClosedCaptionTrackResponse { + final xml.XmlDocument _root; + + ClosedCaptionTrackResponse(this._root); + + Iterable get closedCaptions => + _root.findElements('p').map((e) => ClosedCaption._(e)); + + ClosedCaptionTrackResponse.parse(String raw) : _root = xml.parse(raw); + + static Future get( + YoutubeHttpClient httpClient, String url) { + var formatUrl = _setQueryParameters(url, {'format': '3'}); + return retry(() async { + var raw = await httpClient.getString(formatUrl); + return ClosedCaptionTrackResponse.parse(raw); + }); + } + + static Uri _setQueryParameters(String url, Map parameters) { + var uri = Uri.parse(url); + + var query = Map.from(uri.queryParameters); + query.addAll(parameters); + + return uri.replace(queryParameters: query); + } +} + +class ClosedCaption { + final xml.XmlElement _root; + + ClosedCaption._(this._root); + + String get text => _root.toXmlString(); + + Duration get offset => + Duration(milliseconds: int.parse(_root.getAttribute('t') ?? 0)); + + Duration get duration => + Duration(milliseconds: int.parse(_root.getAttribute('d') ?? 0)); + + Duration get end => offset + duration; + + void getParts() { + _root.findElements('s').map((e) => ClosedCaptionPart._(e)); + } +} + +class ClosedCaptionPart { + final xml.XmlElement _root; + + ClosedCaptionPart._(this._root); + + String get text => _root.toXmlString(); + + Duration get offset => + Duration(milliseconds: int.parse(_root.getAttribute('t') ?? 0)); +} diff --git a/lib/src/reverse_engineering/responses/dash_manifest.dart b/lib/src/reverse_engineering/responses/dash_manifest.dart new file mode 100644 index 0000000..c013c76 --- /dev/null +++ b/lib/src/reverse_engineering/responses/dash_manifest.dart @@ -0,0 +1,68 @@ +import 'package:http_parser/http_parser.dart'; +import 'package:xml/xml.dart' as xml; +import 'package:youtube_explode_dart/src/retry.dart'; +import 'package:youtube_explode_dart/src/reverse_engineering/responses/stream_info_provider.dart'; +import 'package:youtube_explode_dart/src/reverse_engineering/reverse_engineering.dart'; + +class DashManifest { + static final _urlSignatureExp = RegExp(r'/s/(.*?)(?:/|$)'); + + final xml.XmlDocument _root; + + DashManifest(this._root); + + Iterable<_StreamInfo> get streams => _root + .findElements('Representation') + .where((e) => e + .findElements('Initialization') + .first + .getAttribute('sourceURL') + .contains('sq/')) + .map((e) => _StreamInfo(e)); + + DashManifest.parse(String raw) : _root = xml.parse(raw); + + Future get(YoutubeHttpClient httpClient, String url) { + retry(() async { + var raw = await httpClient.getString(url); + return DashManifest.parse(raw); + }); + } + + String getSignatureFromUrl(String url) => + _urlSignatureExp.firstMatch(url).group(1); +} + +class _StreamInfo extends StreamInfoProvider { + static final _contentLenExp = RegExp(r'clen[/=](\d+)'); + static final _containerExp = RegExp(r'mime[/=]\w*%2F([\w\d]*)'); + + final xml.XmlElement _root; + + _StreamInfo(this._root); + + int get tag => int.parse(_root.getAttribute('id')); + + String get url => _root.getAttribute('BaseURL'); + + int get contentLength => int.parse(_root.getAttribute('contentLength') ?? + _contentLenExp.firstMatch(url).group(1)); + + int get bitrate => int.parse(_root.getAttribute('bandwidth')); + + String get container => + Uri.decodeFull(_containerExp.firstMatch(url).group(1)); + + bool get isAudioOnly => + _root.findElements('AudioChannelConfiguration').isNotEmpty; + + String get audioCodec => isAudioOnly ? null : _root.getAttribute('codecs'); + + String get videoCodec => isAudioOnly ? _root.getAttribute('codecs') : null; + + int get videoWidth => int.parse(_root.getAttribute('width')); + + int get videoHeight => int.parse(_root.getAttribute('height')); + + int get framerate => int.parse(_root.getAttribute('framerate')); +} diff --git a/lib/src/reverse_engineering/responses/embed_page.dart b/lib/src/reverse_engineering/responses/embed_page.dart new file mode 100644 index 0000000..2061130 --- /dev/null +++ b/lib/src/reverse_engineering/responses/embed_page.dart @@ -0,0 +1,43 @@ +import 'dart:convert'; + +import 'package:html/dom.dart'; +import 'package:html/parser.dart' as parser; +import 'package:youtube_explode_dart/src/retry.dart'; +import 'package:youtube_explode_dart/src/reverse_engineering/reverse_engineering.dart'; +import '../../extensions/extensions.dart'; + +class EmbedPage { + static final _playerConfigExp = + RegExp(r"yt\.setConfig\({'PLAYER_CONFIG':(.*)}\);"); + + final Document _root; + + EmbedPage(this._root); + + _PlayerConfig get playerconfig => _PlayerConfig(json.decode(_playerConfigJson)); + + String get _playerConfigJson => _root + .getElementsByTagName('script') + .map((e) => e.text) + .map((e) => _playerConfigExp.firstMatch(e).group(1)) + .firstWhere((e) => !e.isNullOrWhiteSpace); + + EmbedPage.parse(String raw) : _root = parser.parse(raw); + + Future get(YoutubeHttpClient httpClient, String videoId) { + var url = 'https://youtube.com/embed/$videoId?hl=en'; + return retry(() async { + var raw = await httpClient.getString(url); + return EmbedPage.parse(raw); + }); + } +} + +class _PlayerConfig { + // Json parsed map. + final Map _root; + + _PlayerConfig(this._root); + + String get sourceUrl => 'https://youtube.com ${_root['assets']['js']}'; +} diff --git a/lib/src/reverse_engineering/responses/player_response.dart b/lib/src/reverse_engineering/responses/player_response.dart new file mode 100644 index 0000000..123450e --- /dev/null +++ b/lib/src/reverse_engineering/responses/player_response.dart @@ -0,0 +1,174 @@ +import 'dart:convert'; + +import 'package:http_parser/http_parser.dart'; +import 'package:youtube_explode_dart/src/reverse_engineering/responses/stream_info_provider.dart'; + +class PlayerResponse { + // Json parsed map + final Map _root; + + PlayerResponse(this._root); + + String get playabilityStatus => _root['playabilityStatus']['status']; + + // Can be null + String get getVideoPlayabilityError => + _root.get('playabilityStatus')?.get('reason'); + + bool get isVideoAvailable => playabilityStatus != 'error'; + + bool get isVideoPlayable => playabilityStatus == 'ok'; + + String get videoTitle => _root['videoDetails']['title']; + + String get videoAuthor => _root['videoDetails']['author']; + + //TODO: Check how this is formatted. + + String /* DateTime */ get videoUploadDate => + _root['microformat']['playerMicroformatRenderer']['uploadDate']; + + String get videoChannelId => _root['videoDetails']['channelId']; + + Duration get videoDuration => + Duration(seconds: _root['videoDetails']['lengthSeconds']); + + Iterable get videoKeywords => + _root['videoDetails']['keywords'] ?? const []; + + String get videoDescription => _root['videoDetails']['shortDescription']; + + int get videoViewCount => int.parse(_root['videoDetails']['viewCount']); + + // Can be null + String get previewVideoId => + _root + .get('playabilityStatus') + ?.get('errorScreen') + ?.get('playerLegacyDesktopYpcTrailerRenderer') + ?.get('trailerVideoId') ?? + Uri.splitQueryString(_root + .get('playabilityStatus') + ?.get('errorScreen') + ?.get('') + ?.get('ypcTrailerRenderer') + ?.get('playerVars') ?? + '')['video_id']; + + bool get isLive => _root['videoDetails'].get('isLive') ?? false; + + // Can be null + String get dashManifestUrl => + _root.get('streamingData')?.get('dashManifestUrl'); + + Iterable get muxedStreams => + _root?.get('streamingData')?.get('formats')?.map((e) => _StreamInfo(e)) ?? + const []; + + Iterable get adaptiveStreams => + _root + ?.get('streamingData') + ?.get('adaptiveFormats') + ?.map((e) => _StreamInfo(e)) ?? + const []; + + Iterable get streams => + [...muxedStreams, ...adaptiveStreams]; + + Iterable get closedCaptionTrack => + _root + .get('captions') + ?.get('playerCaptionsTracklistRenderer') + ?.get('captionTracks') + ?.map((e) => ClosedCaptionTrack(e)) ?? + const []; + + PlayerResponse.parse(String raw) : _root = json.decode(raw); +} + +class ClosedCaptionTrack { + // Json parsed map + final Map _root; + + ClosedCaptionTrack(this._root); + + String get url => _root['baseUrl']; + + String get language => _root['name']['simpleText']; + + bool get autoGenerated => _root['vssId'].startsWith("a."); +} + +class _StreamInfo extends StreamInfoProvider { + // Json parsed map + final Map _root; + + _StreamInfo(this._root); + + @override + int get bitrate => _root['bitrate']; + + @override + String get container => mimeType.subtype; + + @override + int get contentLength => + _root['contentLength'] ?? + StreamInfoProvider.contentLenExp.firstMatch(url).group(1); + + @override + int get framerate => int.tryParse(_root['fps'] ?? ''); + + @override + String get signature => Uri.splitQueryString(_root.get('cipher') ?? '')['s']; + + @override + String get signatureParameter => + Uri.splitQueryString(_root['cipher'] ?? '')['sp']; + + @override + int get tag => int.parse(_root['itag']); + + @override + String get url => + _root?.get('url') ?? + Uri.splitQueryString(_root?.get('cipher') ?? '')['s']; + + @override + // TODO: implement videoCodec, gotta debug how the mimeType is formatted + String get videoCodec => throw UnimplementedError(); + + @override + // TODO: implement videoHeight, gotta debug how the mimeType is formatted + int get videoHeight => _root['height']; + + @override + // TODO: implement videoQualityLabel + String get videoQualityLabel => _root['qualityLabel']; + + @override + // TODO: implement videoWidth + int get videoWidth => _root['width']; + + // TODO: implement audioOnly, gotta debug how the mimeType is formatted + bool get audioOnly => throw UnimplementedError(); + + MediaType get mimeType => MediaType.parse(_root['mimeType']); + + String get codecs => mimeType?.parameters['codecs']?.toLowerCase(); + + @override + // TODO: Finish implementing this, gotta debug how the mimeType is formatted + String get audioCodec => audioOnly ? codecs : throw UnimplementedError(); +} + +/// +extension GetOrNull on Map { + V get(K key) { + var v = this[key]; + if (v == null) { + return null; + } + return v; + } +} diff --git a/lib/src/reverse_engineering/responses/player_source.dart b/lib/src/reverse_engineering/responses/player_source.dart new file mode 100644 index 0000000..1e3c2cc --- /dev/null +++ b/lib/src/reverse_engineering/responses/player_source.dart @@ -0,0 +1,116 @@ +import 'package:youtube_explode_dart/src/exceptions/exceptions.dart'; +import 'package:youtube_explode_dart/src/retry.dart'; +import 'package:youtube_explode_dart/src/reverse_engineering/cipher/cipher_operations.dart'; +import 'package:youtube_explode_dart/src/reverse_engineering/reverse_engineering.dart'; + +class PlayerSource { + final RegExp _statIndexExp = RegExp(r'\(\w+,(\d+)\)'); + + final RegExp _funcBodyExp = RegExp( + r'(\w+)=function\(\w+\){(\w+)=\2\.split\(\x22{2}\);.*?return\s+\2\.join\(\x22{2}\)}'); + + final RegExp _funcNameExp = RegExp(r'(\w+).\w+\(\w+,\d+\);'); + + final RegExp _calledFuncNameExp = + RegExp(r'\w+(?:.|\[)(\""?\w+(?:\"")?)\]?\("'); + + final String _root; + + PlayerSource(this._root); + + String get sts { + var val = RegExp(r'(?<=invalid namespace.*?;var \w\s*=)\d+') + .stringMatch(_root) + .nullIfWhitespace; + if (val == null) { + throw FatalFailureException('Could not find sts in player source.'); + } + } + + Iterable getCiperOperations() sync* { + var funcBody = _getDeciphererFuncBody(); + + if (funcBody == null) { + throw FatalFailureException( + 'Could not find signature decipherer function body.'); + } + + var definitionBody = _getDeciphererDefinitionBody(funcBody); + if (definitionBody == null) { + throw FatalFailureException( + 'Could not find signature decipherer definition body.'); + } + + for (var statement in funcBody.split(';')) { + var calledFuncName = _calledFuncNameExp.firstMatch(statement).group(1); + if (calledFuncName.isNullOrWhiteSpace) { + continue; + } + + var escapedFuncName = RegExp.escape(calledFuncName); + // Slice + var exp = RegExp('$escapedFuncName' + r':\bfunction\b\(\w+\,\w\).\bvar\b.\bc=a\b'); + + if (exp.hasMatch(calledFuncName)) { + var index = int.parse(_statIndexExp.firstMatch(statement).group(1)); + yield SliceCipherOperation(index); + } + + // Swap + exp = RegExp( + '$escapedFuncName' r':\bfunction\b\(\w+\,\w\).\bvar\b.\bc=a\b'); + if (exp.hasMatch(calledFuncName)) { + var index = int.parse(_statIndexExp.firstMatch(statement).group(1)); + yield SwapCipherOperation(index); + } + + // Reverse + exp = RegExp('$escapedFuncName' r':\bfunction\b\(\w+\)'); + if (exp.hasMatch(calledFuncName)) { + yield const ReverseCipherOperation(); + } + } + } + + String _getDeciphererFuncBody() { + var funcName = _funcBodyExp.firstMatch(_root).group(1); + + var exp = RegExp( + r'(?!h\.)' '${RegExp.escape(funcName)}' r'=function\(\w+\)\{{(.*?)\}}'); + return exp.firstMatch(_root).group(1).nullIfWhitespace; + } + + String _getDeciphererDefinitionBody(String deciphererFuncBody) { + var funcName = _funcNameExp.firstMatch(deciphererFuncBody).group(1); + + var exp = RegExp(r'var\s+' + '${RegExp.escape(funcName)}' + r'=\{{(\w+:function\(\w+(,\w+)?\)\{{(.*?)\}}),?\}};'); + return exp.firstMatch(_root).group(0).nullIfWhitespace; + } + + // Same as default constructor + PlayerSource.parse(this._root); + + Future get(YoutubeHttpClient httpClient, String url) { + return retry(() async { + var raw = await httpClient.getString(url); + return PlayerSource.parse(raw); + }); + } +} + +extension on String { + String get nullIfWhitespace => trim().isEmpty ? null : this; + + bool get isNullOrWhiteSpace { + if (this == null) { + return true; + } + if (trim().isEmpty) { + return true; + } + return false; + } +} diff --git a/lib/src/reverse_engineering/responses/playerlist_response.dart b/lib/src/reverse_engineering/responses/playerlist_response.dart new file mode 100644 index 0000000..cb8296c --- /dev/null +++ b/lib/src/reverse_engineering/responses/playerlist_response.dart @@ -0,0 +1,103 @@ +import 'dart:convert'; + +import 'package:youtube_explode_dart/src/exceptions/exceptions.dart'; +import 'package:youtube_explode_dart/src/retry.dart'; +import 'package:youtube_explode_dart/src/reverse_engineering/reverse_engineering.dart'; + +class PlaylistResponse { + // Json parsed map + final Map _root; + + PlaylistResponse(this._root); + + String get title => _root['title']; + + String get author => _root['author']; + + String get description => _root['description']; + + int get viewCount => int.tryParse(_root['views'] ?? ''); + + int get likeCount => int.tryParse(_root['likes']); + + int get dislikeCount => int.tryParse(_root['dislikes']); + + Iterable