Better closed captions.

#81, #82

analysis_options.yaml

@@ -1,5 +1,15 @@
 include: package:effective_dart/analysis_options.yaml
 
+analyzer:
+  exclude: #most likely not all of these are needed, but as it is now it works.
+    - "**/*.g.dart"
+    - /**/*.g.dart
+    - \**\*.g.dart
+    - "*.g.dart"
+    - "**.g.dart"
+    - example\**
+    - lib\src\reverse_engineering\responses\generated\**
+
 linter:
   rules:
     - valid_regexps
@@ -57,13 +67,3 @@ linter:
     - prefer_single_quotes
     - use_function_type_syntax_for_parameters
 
-analyzer:
-  exclude: #most likely not all of these are needed, but as it is now it works.
-    - "**/*.g.dart"
-    - "**\*.g.dart"
-    - /**/*.g.dart
-    - \**\*.g.dart
-    - "*.g.dart"
-    - "**.g.dart"
-    - example\**
-    - lib\src\reverse_engineering\responses\generated\**

example/example.dart

@@ -2,11 +2,14 @@ import 'package:youtube_explode_dart/youtube_explode_dart.dart';
 
 Future<void> main() async {
   var yt = YoutubeExplode();
-  var video =
-      await yt.videos.get('https://www.youtube.com/watch?v=AI7ULzgf8RU');
 
-  print('Title: ${video.title}');
+  var manifest = await yt.videos.closedCaptions
+      .getManifest('Pxgvgh9IFqA', autoGenerated: true);
+  print(manifest.tracks);
+  print('\n\n---------------------\n\n');
 
-  // Close the YoutubeExplode's http client.
+  manifest = await yt.videos.closedCaptions
+      .getManifest('Pxgvgh9IFqA', autoGenerated: false);
+  print(manifest.tracks);
   yt.close();
 }

lib/src/channels/channel_about.dart

@@ -1,7 +1,7 @@
 import 'package:equatable/equatable.dart';
 
-import 'channel_link.dart';
 import '../common/thumbnail.dart';
+import 'channel_link.dart';
 
 /// YouTube channel's about page metadata.
 class ChannelAbout with EquatableMixin {

lib/src/extensions/helpers_extension.dart

@@ -113,3 +113,14 @@ extension GetOrNullMap on Map {
     return v;
   }
 }
+
+///
+extension UriUtils on Uri {
+  ///
+  Uri replaceQueryParameters(Map<String, String> parameters) {
+    var query = Map<String, String>.from(queryParameters);
+    query.addAll(parameters);
+
+    return replace(queryParameters: query);
+  }
+}

lib/src/reverse_engineering/responses/channel_about_page.dart

@@ -1,12 +1,12 @@
 import 'package:html/dom.dart';
 import 'package:html/parser.dart' as parser;
-import 'package:youtube_explode_dart/youtube_explode_dart.dart';
 
+import '../../../youtube_explode_dart.dart';
 import '../../exceptions/exceptions.dart';
+import '../../extensions/helpers_extension.dart';
 import '../../retry.dart';
 import '../youtube_http_client.dart';
 import 'generated/channel_about_page_id.g.dart';
-import '../../extensions/helpers_extension.dart';
 
 ///
 class ChannelAboutPage {

lib/src/reverse_engineering/responses/closed_caption_track_response.dart

@@ -1,5 +1,6 @@
 import 'package:xml/xml.dart' as xml;
 
+import '../../extensions/helpers_extension.dart';
 import '../../retry.dart';
 import '../youtube_http_client.dart';
 
@@ -23,21 +24,12 @@ class ClosedCaptionTrackResponse {
   ///
   static Future<ClosedCaptionTrackResponse> get(
       YoutubeHttpClient httpClient, String url) {
-    var formatUrl = _setQueryParameters(url, {'format': '3'});
+    var formatUrl = Uri.parse(url).replaceQueryParameters({'fmt': 'srv3'});
     return retry(() async {
       var raw = await httpClient.getString(formatUrl);
       return ClosedCaptionTrackResponse.parse(raw);
     });
   }
-
-  static Uri _setQueryParameters(String url, Map<String, String> parameters) {
-    var uri = Uri.parse(url);
-
-    var query = Map<String, String>.from(uri.queryParameters);
-    query.addAll(parameters);
-
-    return uri.replace(queryParameters: query);
-  }
 }
 
 ///
@@ -47,7 +39,7 @@ class ClosedCaption {
   Duration _offset;
   Duration _duration;
   Duration _end;
-  Iterable<ClosedCaptionPart> _parts;
+  List<ClosedCaptionPart> _parts;
 
   ///
   String get text => _root.text;
@@ -64,8 +56,8 @@ class ClosedCaption {
   Duration get end => _end ??= offset + duration;
 
   ///
-  Iterable<ClosedCaptionPart> getParts() =>
-      _parts ??= _root.findAllElements('s').map((e) => ClosedCaptionPart._(e));
+  List<ClosedCaptionPart> getParts() => _parts ??=
+      _root.findAllElements('s').map((e) => ClosedCaptionPart._(e)).toList();
 
   ClosedCaption._(this._root);
 }

lib/src/reverse_engineering/responses/player_response.dart

@@ -1,9 +1,9 @@
 import 'dart:convert';
 
 import 'package:http_parser/http_parser.dart';
-import 'package:youtube_explode_dart/src/reverse_engineering/responses/generated/player_response.g.dart';
 
 import '../../extensions/helpers_extension.dart';
+import 'generated/player_response.g.dart';
 import 'stream_info_provider.dart';
 
 ///

lib/src/reverse_engineering/responses/search_page.dart

@@ -2,11 +2,11 @@ import 'dart:convert';
 
 import 'package:html/dom.dart';
 import 'package:html/parser.dart' as parser;
-import 'package:youtube_explode_dart/src/search/base_search_content.dart';
 
 import '../../../youtube_explode_dart.dart';
 import '../../extensions/helpers_extension.dart';
 import '../../retry.dart';
+import '../../search/base_search_content.dart';
 import '../../search/related_query.dart';
 import '../../search/search_video.dart';
 import '../../videos/videos.dart';

lib/src/reverse_engineering/youtube_http_client.dart

@@ -85,53 +85,24 @@ class YoutubeHttpClient extends http.BaseClient {
   }
 
   ///
-  // TODO: Check why isRateLimited is not working.
   Stream<List<int>> getStream(StreamInfo streamInfo,
       {Map<String, String> headers,
       bool validate = true,
       int start = 0,
       int errorCount = 0}) async* {
     var url = streamInfo.url;
-//    if (!streamInfo.isRateLimited()) {
-//      var request = http.Request('get', url);
-//      request.headers.addAll(_defaultHeaders);
-//      var response = await request.send();
-//      if (validate) {
-//        _validateResponse(response, response.statusCode);
-//      }
-//      yield* response.stream;
-//    } else {
 
-    var bytesCount = start;
-    for (var i = start; i < streamInfo.size.totalBytes; i += 9898989) {
-      try {
-        final request = http.Request('get', url);
-        request.headers['range'] = 'bytes=$i-${i + 9898989 - 1}';
-        final response = await send(request);
-        if (validate) {
-          _validateResponse(response, response.statusCode);
-        }
-        final stream = StreamController<List<int>>();
-        response.stream.listen((data) {
-          bytesCount += data.length;
-          stream.add(data);
-        }, onError: (_) => null, onDone: stream.close, cancelOnError: false);
-        errorCount = 0;
-        yield* stream.stream;
-      } on Exception {
-        if (errorCount == 5) {
-          rethrow;
-        }
-        await Future.delayed(const Duration(milliseconds: 500));
-        yield* getStream(streamInfo,
-            headers: headers,
-            validate: validate,
-            start: bytesCount,
-            errorCount: errorCount + 1);
-        break;
-      }
+    var query = Map.from(url.queryParameters);
+    query['ratebypass'] = 'yes';
+    url = url.replace(queryParameters: query);
+
+    var request = http.Request('get', url);
+    request.headers.addAll(_defaultHeaders);
+    var response = await request.send();
+    if (validate) {
+      _validateResponse(response, response.statusCode);
     }
-//    }
+    yield* response.stream;
   }
 
   ///

lib/src/search/search_client.dart

@@ -1,7 +1,6 @@
-import 'package:youtube_explode_dart/src/reverse_engineering/responses/search_page.dart';
-
 import '../common/common.dart';
 import '../reverse_engineering/responses/playlist_response.dart';
+import '../reverse_engineering/responses/search_page.dart';
 import '../reverse_engineering/youtube_http_client.dart';
 import '../videos/video.dart';
 import '../videos/video_id.dart';

lib/src/videos/closed_captions/closed_caption.dart

@@ -32,4 +32,7 @@ class ClosedCaption {
   /// Note that some captions may not have any parts at all.
   ClosedCaptionPart getPartByTime(Duration offset) =>
       parts.firstWhere((e) => e.offset >= offset, orElse: () => null);
+
+  @override
+  String toString() => 'Text: $text';
 }

lib/src/videos/closed_captions/closed_caption_client.dart

@@ -1,7 +1,8 @@
+import 'package:xml/xml.dart' as xml;
+
 import '../../extensions/helpers_extension.dart';
-import '../../reverse_engineering/responses/closed_caption_track_response.dart'
-    hide ClosedCaption, ClosedCaptionPart;
-import '../../reverse_engineering/responses/video_info_response.dart';
+import '../../reverse_engineering/responses/responses.dart'
+    hide ClosedCaption, ClosedCaptionPart, ClosedCaptionTrack;
 import '../../reverse_engineering/youtube_http_client.dart';
 import '../videos.dart';
 import 'closed_caption.dart';
@@ -20,16 +21,57 @@ class ClosedCaptionClient {
 
   /// Gets the manifest that contains information
   /// about available closed caption tracks in the specified video.
-  Future<ClosedCaptionManifest> getManifest(dynamic videoId) async {
+  Future<ClosedCaptionManifest> getManifest(dynamic videoId,
+      {bool autoGenerated = false}) async {
     videoId = VideoId.fromString(videoId);
-    var videoInfoResponse =
-        await VideoInfoResponse.get(_httpClient, videoId.value);
-    var playerResponse = videoInfoResponse.playerResponse;
+    var tracks = <ClosedCaptionTrackInfo>[];
+    if (!autoGenerated) {
+      var subList = await _httpClient.get(
+          'https://video.google.com/timedtext?hl=en&type=list&v=${videoId.value}',
+          validate: true);
+      // ignore: deprecated_member_use
+      var content = xml.parse(subList.body);
 
-    var tracks = playerResponse.closedCaptionTrack.map((track) =>
-        ClosedCaptionTrackInfo(Uri.parse(track.url),
-            Language(track.languageCode, track.languageName),
-            isAutoGenerated: track.autoGenerated));
+      var langList = <String>[];
+      for (var track in content.findAllElements('track')) {
+        var lang = track.getAttribute('lang_code');
+        if (langList.contains(lang)) {
+          continue;
+        }
+        langList.add(lang);
+        for (var ext in ClosedCaptionFormat.values) {
+          tracks.add(ClosedCaptionTrackInfo(
+              Uri.parse('https://www.youtube.com/api/timedtext')
+                  .replaceQueryParameters({
+                'lang': lang,
+                'v': videoId.value,
+                'fmt': ext.formatCode,
+                'name': track.getAttribute('name'),
+              }),
+              Language(lang, track.getAttribute('lang_translated')),
+              format: ext));
+        }
+      }
+      if (langList.isEmpty) {
+        return ClosedCaptionManifest([]);
+      }
+      return ClosedCaptionManifest(tracks);
+    } else {
+      var videoInfoResponse =
+          await VideoInfoResponse.get(_httpClient, videoId.value);
+      var playerResponse = videoInfoResponse.playerResponse;
+
+      for (var track in playerResponse.closedCaptionTrack) {
+        for (var ext in ClosedCaptionFormat.values) {
+          tracks.add(ClosedCaptionTrackInfo(
+              Uri.parse(track.url)
+                  .replaceQueryParameters({'fmt': ext.formatCode}),
+              Language(track.languageCode, track.languageName),
+              isAutoGenerated: track.autoGenerated,
+              format: ext));
+        }
+      }
+    }
     return ClosedCaptionManifest(tracks);
   }
 
@@ -46,62 +88,17 @@ class ClosedCaptionClient {
     return ClosedCaptionTrack(captions);
   }
 
-  ///
-  Future<String> getSrt(ClosedCaptionTrackInfo trackInfo) async {
-    var track = await get(trackInfo);
-
-    var buffer = StringBuffer();
-    for (var i = 0; i < track.captions.length; i++) {
-      var caption = track.captions[i];
-
-      // Line number
-      buffer.writeln('${i + 1}');
-
-      // Time start --> time end
-      buffer.write(caption.offset.toSrtFormat());
-      buffer.write(' --> ');
-      buffer.write(caption.end.toSrtFormat());
-      buffer.writeln();
-
-      // Actual text
-      buffer.writeln(caption.text);
-      buffer.writeln();
-    }
-    return buffer.toString();
-  }
-}
-
-extension on Duration {
-  String toSrtFormat() {
-    String threeDigits(int n) {
-      if (n >= 1000) {
-        return n.toString().substring(0, 3);
-      }
-      if (n >= 100) {
-        return '$n';
-      }
-      if (n >= 10) {
-        return '0$n';
-      }
-      return '00$n';
-    }
-
-    String twoDigits(int n) {
-      if (n >= 10) {
-        return '$n';
-      }
-      return '0$n';
-    }
-
-    if (inMicroseconds < 0) {
-      return '-${-this}';
-    }
-    var twoDigitHours = twoDigits(inHours);
-    var twoDigitMinutes =
-        twoDigits(inMinutes.remainder(Duration.minutesPerHour));
-    var twoDigitSeconds =
-        twoDigits(inSeconds.remainder(Duration.secondsPerMinute));
-    var fourDigitsUs = threeDigits(inMilliseconds.remainder(1000));
-    return '$twoDigitHours:$twoDigitMinutes:$twoDigitSeconds,$fourDigitsUs';
+  /// Auto translated a closed caption track.
+  ClosedCaptionTrackInfo autoTranslate(
+      ClosedCaptionTrackInfo trackInfo, String lang) {
+    return ClosedCaptionTrackInfo(
+        trackInfo.url.replaceQueryParameters({'tlang': lang}),
+        Language(lang, ''),
+        isAutoGenerated: trackInfo.isAutoGenerated,
+        format: trackInfo.format);
   }
+
+  /// Returns the subtitles as a string.
+  Future<String> getSubTitles(ClosedCaptionTrackInfo trackInfo) =>
+      _httpClient.getString(trackInfo.url);
 }

lib/src/videos/closed_captions/closed_caption_track_info.dart

@@ -13,8 +13,13 @@ class ClosedCaptionTrackInfo extends Equatable {
   /// Whether the associated track was automatically generated.
   final bool isAutoGenerated;
 
+  /// Track format
+  final ClosedCaptionFormat format;
+
   /// Initializes an instance of [ClosedCaptionTrackInfo]
-  const ClosedCaptionTrackInfo(this.url, this.language, {this.isAutoGenerated});
+  const ClosedCaptionTrackInfo(this.url, this.language,
+      {this.isAutoGenerated = false, this.format})
+      : assert(format != null);
 
   @override
   String toString() => 'CC Track ($language)';
@@ -22,3 +27,29 @@ class ClosedCaptionTrackInfo extends Equatable {
   @override
   List<Object> get props => [url, language, isAutoGenerated];
 }
+
+/// SubTiles format.
+class ClosedCaptionFormat {
+  /// .srv format(1).
+  static const ClosedCaptionFormat srv1 = ClosedCaptionFormat._('srv1');
+
+  /// .srv format(2).
+  static const ClosedCaptionFormat srv2 = ClosedCaptionFormat._('srv2');
+
+  /// .srv format(3).
+  static const ClosedCaptionFormat srv3 = ClosedCaptionFormat._('srv3');
+
+  /// .ttml format.
+  static const ClosedCaptionFormat ttml = ClosedCaptionFormat._('ttml');
+
+  /// .vtt format.
+  static const ClosedCaptionFormat vtt = ClosedCaptionFormat._('vtt');
+
+  /// List of all sub titles format.
+  static const List<ClosedCaptionFormat> values = [srv1, srv2, srv3, ttml, vtt];
+
+  /// Format code as string.
+  final String formatCode;
+
+  const ClosedCaptionFormat._(this.formatCode);
+}

lib/src/videos/streams/stream_info.dart

@@ -24,14 +24,6 @@ abstract class StreamInfo {
   StreamInfo(this.tag, this.url, this.container, this.size, this.bitrate);
 }
 
-/// Extensions for [StreamInfo].
-extension StreamInfoExt on StreamInfo {
-  static final _exp = RegExp('ratebypass[=/]yes');
-
-  /// Returns true if this video is rate limited.
-  bool isRateLimited() => _exp.hasMatch(url.toString());
-}
-
 /// Extension for Iterables of StreamInfo.
 extension StreamInfoIterableExt<T extends StreamInfo> on Iterable<T> {
   /// Gets the stream with highest bitrate.

lib/src/videos/video_client.dart

@@ -71,7 +71,7 @@ class VideoClient {
   }
 
   /// Get a [Video] instance from a [videoId]
-  Future<Video> get(dynamic videoId, {forceWatchPage = false}) async {
+  Future<Video> get(dynamic videoId, {bool forceWatchPage = false}) async {
     videoId = VideoId.fromString(videoId);
 
     if (forceWatchPage) {

test/search_test.dart

@@ -19,17 +19,16 @@ void main() {
 
   test('Search a youtube videos from the search page', () async {
     // ignore: deprecated_member_use_from_same_package
-    var searchQuery = await yt.search.queryFromPage(
-        'hello');
+    var searchQuery = await yt.search.queryFromPage('hello');
     expect(searchQuery.content, isNotEmpty);
     expect(searchQuery.relatedVideos, isNotEmpty);
     expect(searchQuery.relatedQueries, isNotEmpty);
   });
 
   test('Search with no results', () async {
-    // ignore: deprecated_member_use_from_same_package
-    var query = await yt.search.queryFromPage(
-        'g;jghEOGHJeguEPOUIhjegoUEHGOGHPSASG');
+    var query =
+        // ignore: deprecated_member_use_from_same_package
+        await yt.search.queryFromPage('g;jghEOGHJeguEPOUIhjegoUEHGOGHPSASG');
     expect(query.content, isEmpty);
     expect(query.relatedQueries, isEmpty);
     expect(query.relatedVideos, isEmpty);
@@ -39,8 +38,7 @@ void main() {
 
   test('Search youtube videos have thumbnails', () async {
     // ignore: deprecated_member_use_from_same_package
-    var searchQuery = await yt.search.queryFromPage(
-        'hello');
+    var searchQuery = await yt.search.queryFromPage('hello');
     expect(searchQuery.content.first, isA<SearchVideo>());
 
     var video = searchQuery.content.first as SearchVideo;