From 13249aed18510ae576d5f0b9ea1207ec11fe3e47 Mon Sep 17 00:00:00 2001
From: Mattia <hexer504@gmail.com>
Date: Fri, 26 Feb 2021 16:08:48 +0100
Subject: [PATCH] Fix search issues. Fix `NoSuchMethodError` exception.

Closes #102
---
 lib/src/exceptions/exceptions.dart            |  1 +
 .../exceptions/fatal_failure_exception.dart   |  3 +-
 .../search_item_section_exception.dart        | 10 +++
 .../exceptions/youtube_explode_exception.dart |  2 +-
 lib/src/retry.dart                            |  4 +-
 .../responses/embed_page.dart                 |  2 +-
 .../responses/player_config_base.dart         |  3 +-
 .../responses/search_page.dart                | 28 ++++----
 .../responses/watch_page.dart                 |  2 +-
 .../youtube_http_client.dart                  |  2 +-
 lib/src/search/search_client.dart             | 72 ++++++++-----------
 lib/src/search/search_list.dart               | 33 +++++++++
 lib/src/videos/streams/streams_client.dart    |  3 +-
 lib/src/videos/video.dart                     |  1 +
 pubspec.yaml                                  |  1 +
 test/search_test.dart                         |  2 +-
 16 files changed, 100 insertions(+), 69 deletions(-)
 create mode 100644 lib/src/exceptions/search_item_section_exception.dart
 create mode 100644 lib/src/search/search_list.dart

diff --git a/lib/src/exceptions/exceptions.dart b/lib/src/exceptions/exceptions.dart
index 885350b..74e468c 100644
--- a/lib/src/exceptions/exceptions.dart
+++ b/lib/src/exceptions/exceptions.dart
@@ -2,6 +2,7 @@ library youtube_explode.exceptions;
 
 export 'fatal_failure_exception.dart';
 export 'request_limit_exceeded_exception.dart';
+export 'search_item_section_exception.dart';
 export 'transient_failure_exception.dart';
 export 'video_requires_purchase_exception.dart';
 export 'video_unavailable_exception.dart';
diff --git a/lib/src/exceptions/fatal_failure_exception.dart b/lib/src/exceptions/fatal_failure_exception.dart
index d4a2fd3..273b1e1 100644
--- a/lib/src/exceptions/fatal_failure_exception.dart
+++ b/lib/src/exceptions/fatal_failure_exception.dart
@@ -3,8 +3,7 @@ import 'package:http/http.dart';
 import 'youtube_explode_exception.dart';
 
 /// Exception thrown when a fatal failure occurs.
-class FatalFailureException
-    implements YoutubeExplodeException {
+class FatalFailureException implements YoutubeExplodeException {
   /// Description message
   @override
   final String message;
diff --git a/lib/src/exceptions/search_item_section_exception.dart b/lib/src/exceptions/search_item_section_exception.dart
new file mode 100644
index 0000000..0ffeb28
--- /dev/null
+++ b/lib/src/exceptions/search_item_section_exception.dart
@@ -0,0 +1,10 @@
+//
+
+import '../../youtube_explode_dart.dart';
+
+/// Exception thrown when the Item Section is missing from a search request.
+class SearchItemSectionException implements YoutubeExplodeException {
+  @override
+  // TODO: implement message
+  String get message => 'Failed to find the item section.';
+}
diff --git a/lib/src/exceptions/youtube_explode_exception.dart b/lib/src/exceptions/youtube_explode_exception.dart
index 44a75c8..618040f 100644
--- a/lib/src/exceptions/youtube_explode_exception.dart
+++ b/lib/src/exceptions/youtube_explode_exception.dart
@@ -5,4 +5,4 @@ abstract class YoutubeExplodeException implements Exception {
 
   ///
   YoutubeExplodeException(this.message);
-}
\ No newline at end of file
+}
diff --git a/lib/src/retry.dart b/lib/src/retry.dart
index 562ea47..4df2cde 100644
--- a/lib/src/retry.dart
+++ b/lib/src/retry.dart
@@ -26,7 +26,9 @@ Future<T> retry<T>(FutureOr<T> Function() function) async {
 
 /// Get "retry" cost of each YoutubeExplode exception.
 int getExceptionCost(Exception e) {
-  if (e is TransientFailureException || e is FormatException) {
+  if (e is TransientFailureException ||
+      e is FormatException ||
+      e is SearchItemSectionException) {
     return 1;
   }
   if (e is RequestLimitExceededException) {
diff --git a/lib/src/reverse_engineering/responses/embed_page.dart b/lib/src/reverse_engineering/responses/embed_page.dart
index 24a597d..ac608ea 100644
--- a/lib/src/reverse_engineering/responses/embed_page.dart
+++ b/lib/src/reverse_engineering/responses/embed_page.dart
@@ -2,11 +2,11 @@ import 'dart:convert';
 
 import 'package:html/dom.dart';
 import 'package:html/parser.dart' as parser;
-import 'package:youtube_explode_dart/src/reverse_engineering/responses/player_config_base.dart';
 
 import '../../extensions/helpers_extension.dart';
 import '../../retry.dart';
 import '../youtube_http_client.dart';
+import 'player_config_base.dart';
 
 ///
 class EmbedPage {
diff --git a/lib/src/reverse_engineering/responses/player_config_base.dart b/lib/src/reverse_engineering/responses/player_config_base.dart
index a91ac70..5761b74 100644
--- a/lib/src/reverse_engineering/responses/player_config_base.dart
+++ b/lib/src/reverse_engineering/responses/player_config_base.dart
@@ -1,6 +1,5 @@
 /// Base class for PlayerConfig.
 abstract class PlayerConfigBase<T> {
-
   /// Root node.
   final T root;
 
@@ -9,4 +8,4 @@ abstract class PlayerConfigBase<T> {
 
   /// Player source url.
   String get sourceUrl;
-}
\ No newline at end of file
+}
diff --git a/lib/src/reverse_engineering/responses/search_page.dart b/lib/src/reverse_engineering/responses/search_page.dart
index b019e19..aa31a1a 100644
--- a/lib/src/reverse_engineering/responses/search_page.dart
+++ b/lib/src/reverse_engineering/responses/search_page.dart
@@ -155,20 +155,22 @@ class _InitialData {
 
   _InitialData(this.root);
 
-  /* Cache results */
-
-  List<dynamic> _searchContent;
-  List<dynamic> _relatedVideos;
-  List<RelatedQuery> _relatedQueries;
-
   List<PurpleContent> getContentContext() {
     if (root.contents != null) {
       return root.contents.twoColumnSearchResultsRenderer.primaryContents
           .sectionListRenderer.contents.first.itemSectionRenderer.contents;
     }
     if (root.onResponseReceivedCommands != null) {
-      return root.onResponseReceivedCommands.first.appendContinuationItemsAction
-          .continuationItems[0].itemSectionRenderer.contents;
+      final itemSection = root
+          .onResponseReceivedCommands
+          .first
+          .appendContinuationItemsAction
+          .continuationItems[0]
+          .itemSectionRenderer;
+      if (itemSection == null) {
+        throw SearchItemSectionException();
+      }
+      return itemSection.contents;
     }
     return null;
   }
@@ -203,11 +205,11 @@ class _InitialData {
   }
 
   // Contains only [SearchVideo] or [SearchPlaylist]
-  List<BaseSearchContent> get searchContent => _searchContent ??=
+  List<BaseSearchContent> get searchContent =>
       getContentContext().map(_parseContent).where((e) => e != null).toList();
 
   List<RelatedQuery> get relatedQueries =>
-      (_relatedQueries ??= getContentContext()
+      getContentContext()
           ?.where((e) => e.horizontalCardListRenderer != null)
           ?.map((e) => e.horizontalCardListRenderer.cards)
           ?.firstOrNull
@@ -217,16 +219,16 @@ class _InitialData {
               VideoId(
                   Uri.parse(e.thumbnail.thumbnails.first.url).pathSegments[1])))
           ?.toList()
-          ?.cast<RelatedQuery>()) ??
+          ?.cast<RelatedQuery>() ??
       const [];
 
   List<dynamic> get relatedVideos =>
-      (_relatedVideos ??= getContentContext()
+      getContentContext()
           ?.where((e) => e.shelfRenderer != null)
           ?.map((e) => e.shelfRenderer.content.verticalListRenderer.items)
           ?.firstOrNull
           ?.map(_parseContent)
-          ?.toList()) ??
+          ?.toList() ??
       const [];
 
   String get continuationToken => _getContinuationToken();
diff --git a/lib/src/reverse_engineering/responses/watch_page.dart b/lib/src/reverse_engineering/responses/watch_page.dart
index 4f5bdf5..5b180c7 100644
--- a/lib/src/reverse_engineering/responses/watch_page.dart
+++ b/lib/src/reverse_engineering/responses/watch_page.dart
@@ -1,6 +1,5 @@
 import 'package:html/dom.dart';
 import 'package:html/parser.dart' as parser;
-import 'package:youtube_explode_dart/src/reverse_engineering/responses/player_config_base.dart';
 
 import '../../../youtube_explode_dart.dart';
 import '../../extensions/helpers_extension.dart';
@@ -9,6 +8,7 @@ import '../../videos/video_id.dart';
 import '../youtube_http_client.dart';
 import 'generated/player_response_json.g.dart';
 import 'generated/watch_page_id.g.dart';
+import 'player_config_base.dart';
 import 'player_response.dart';
 
 ///
diff --git a/lib/src/reverse_engineering/youtube_http_client.dart b/lib/src/reverse_engineering/youtube_http_client.dart
index aa1b169..4e5d854 100644
--- a/lib/src/reverse_engineering/youtube_http_client.dart
+++ b/lib/src/reverse_engineering/youtube_http_client.dart
@@ -143,7 +143,7 @@ class YoutubeHttpClient extends http.BaseClient {
         request.headers[key] = _defaultHeaders[key];
       }
     });
-   // print('Request: $request');
+    print('Request: $request');
 //    print('Stack:\n${StackTrace.current}');
     return _httpClient.send(request);
   }
diff --git a/lib/src/search/search_client.dart b/lib/src/search/search_client.dart
index cb471b1..e44bdf8 100644
--- a/lib/src/search/search_client.dart
+++ b/lib/src/search/search_client.dart
@@ -1,11 +1,10 @@
-import '../common/common.dart';
-import '../reverse_engineering/responses/playlist_response.dart';
+import '../../youtube_explode_dart.dart';
+import '../retry.dart';
 import '../reverse_engineering/responses/search_page.dart';
 import '../reverse_engineering/youtube_http_client.dart';
-import '../videos/video.dart';
-import '../videos/video_id.dart';
 import 'base_search_content.dart';
 import 'search_query.dart';
+import 'search_list.dart';
 
 /// YouTube search queries.
 class SearchClient {
@@ -14,50 +13,29 @@ class SearchClient {
   /// Initializes an instance of [SearchClient]
   SearchClient(this._httpClient);
 
-  /// Enumerates videos returned by the specified search query.
-  /// (from the YouTube Embedded API)
-  Stream<Video> getVideos(String searchQuery) async* {
-    var encounteredVideoIds = <String>{};
-
-    for (var page = 0; page < double.maxFinite; page++) {
-      var response =
-          await PlaylistResponse.searchResults(_httpClient, searchQuery);
-
-      var countDelta = 0;
-      for (var video in response.videos) {
-        var videoId = video.id;
-
-        if (!encounteredVideoIds.add(videoId)) {
-          continue;
-        }
-
-        yield Video(
-            VideoId(videoId),
-            video.title,
-            video.author,
-            video.channelId,
-            video.uploadDate,
-            video.description,
-            video.duration,
-            ThumbnailSet(videoId),
-            video.keywords,
-            Engagement(video.viewCount, video.likes, video.dislikes));
-        countDelta++;
-      }
-
-      // Videos loop around, so break when we stop seeing new videos
-      if (countDelta <= 0) {
-        break;
-      }
-    }
+  /// Enumerates videos returned by the specified search query
+  /// (from the video search page).
+  /// The videos are sent in batch of 20 videos.
+  /// You [SearchList.nextPage] to get the next batch of videos.
+  Future<SearchList> getVideos(String searchQuery) {
+    var stream =
+        getVideosFromPage(searchQuery, onlyVideos: true).cast<SearchVideo>();
+    return SearchList.create(stream);
   }
 
   /// Enumerates videos returned by the specified search query
   /// (from the video search page).
   /// Contains only instances of [SearchVideo] or [SearchPlaylist]
-  Stream<BaseSearchContent> getVideosFromPage(String searchQuery) async* {
-    var page = await SearchPage.get(_httpClient, searchQuery);
-    yield* Stream.fromIterable(page.initialData.searchContent);
+  Stream<BaseSearchContent> getVideosFromPage(String searchQuery,
+      {bool onlyVideos = true}) async* {
+    var page =
+        await retry(() async => SearchPage.get(_httpClient, searchQuery));
+    if (onlyVideos) {
+      yield* Stream.fromIterable(
+          page.initialData.searchContent.whereType<SearchVideo>());
+    } else {
+      yield* Stream.fromIterable(page.initialData.searchContent);
+    }
 
     // ignore: literal_only_boolean_expressions
     while (true) {
@@ -65,7 +43,13 @@ class SearchClient {
       if (page == null) {
         return;
       }
-      yield* Stream.fromIterable(page.initialData.searchContent);
+
+      if (onlyVideos) {
+        yield* Stream.fromIterable(
+            page.initialData.searchContent.whereType<SearchVideo>());
+      } else {
+        yield* Stream.fromIterable(page.initialData.searchContent);
+      }
     }
   }
 
diff --git a/lib/src/search/search_list.dart b/lib/src/search/search_list.dart
new file mode 100644
index 0000000..88c466a
--- /dev/null
+++ b/lib/src/search/search_list.dart
@@ -0,0 +1,33 @@
+import 'dart:async';
+
+import 'package:collection/collection.dart';
+
+import '../../youtube_explode_dart.dart';
+
+///
+class SearchList extends DelegatingList<SearchVideo> {
+  final Stream<SearchVideo> _stream;
+
+  ///
+  SearchList._(List<SearchVideo> base, this._stream) : super(base);
+
+  ///
+  static Future<SearchList> create(Stream<SearchVideo> stream) async {
+    Stream<SearchVideo> broadcast;
+    broadcast = stream.asBroadcastStream(onCancel: (subscription) {
+      print('Pause');
+      subscription.pause();
+    }, onListen: (subscription) {
+      print('Resume');
+      subscription.resume();
+    });
+    final base = await broadcast.take(20).toList();
+    return SearchList._(base, broadcast);
+  }
+
+  ///
+  Future<SearchList> nextPage() async {
+    final base = await _stream.take(20).toList();
+    return SearchList._(base, _stream);
+  }
+}
diff --git a/lib/src/videos/streams/streams_client.dart b/lib/src/videos/streams/streams_client.dart
index 31e08f2..3bc78c6 100644
--- a/lib/src/videos/streams/streams_client.dart
+++ b/lib/src/videos/streams/streams_client.dart
@@ -97,8 +97,7 @@ class StreamsClient {
           videoId, VideoId(previewVideoId));
     }
 
-    var playerSourceUrl =
-        watchPage.sourceUrl ?? playerConfig?.sourceUrl;
+    var playerSourceUrl = watchPage.sourceUrl ?? playerConfig?.sourceUrl;
     var playerSource = !playerSourceUrl.isNullOrWhiteSpace
         ? await PlayerSource.get(_httpClient, playerSourceUrl)
         : null;
diff --git a/lib/src/videos/video.dart b/lib/src/videos/video.dart
index 27eba3a..9c69479 100644
--- a/lib/src/videos/video.dart
+++ b/lib/src/videos/video.dart
@@ -7,6 +7,7 @@ import '../common/common.dart';
 import '../reverse_engineering/responses/responses.dart';
 import 'video_id.dart';
 
+@Deprecated('This class is not used anymore - Since youtube changes in 02/2021')
 /// YouTube video metadata.
 class Video with EquatableMixin {
   /// Video ID.
diff --git a/pubspec.yaml b/pubspec.yaml
index f229f52..793df32 100644
--- a/pubspec.yaml
+++ b/pubspec.yaml
@@ -14,6 +14,7 @@ dependencies:
   equatable: ^1.1.0
   meta: ^1.1.8
   json_annotation: ^3.1.0
+  collection: ^1.14.13
 
 dev_dependencies:
   effective_dart: ^1.2.4
diff --git a/test/search_test.dart b/test/search_test.dart
index 6c86422..df9cd46 100644
--- a/test/search_test.dart
+++ b/test/search_test.dart
@@ -13,7 +13,7 @@ void main() {
 
   test('Search a youtube video from the api', () async {
     var videos =
-        await yt.search.getVideos('undead corporation megalomania').toList();
+        await yt.search.getVideos('undead corporation megalomania');
     expect(videos, isNotEmpty);
   });