2020-07-10 22:28:19 +02:00
|
|
|
import 'dart:convert';
|
|
|
|
|
|
|
|
import 'package:html/dom.dart';
|
|
|
|
import 'package:html/parser.dart' as parser;
|
|
|
|
|
|
|
|
import '../../channels/channel_video.dart';
|
2020-07-16 20:02:54 +02:00
|
|
|
import '../../exceptions/exceptions.dart';
|
2020-07-10 22:28:19 +02:00
|
|
|
import '../../retry.dart';
|
|
|
|
import '../../videos/videos.dart';
|
|
|
|
import '../youtube_http_client.dart';
|
2020-09-12 11:20:22 +02:00
|
|
|
import 'generated/channel_upload_page_id.g.dart';
|
2020-07-10 22:28:19 +02:00
|
|
|
|
2020-07-16 20:02:54 +02:00
|
|
|
///
|
2020-07-12 18:24:22 +02:00
|
|
|
class ChannelUploadPage {
|
2020-07-16 20:02:54 +02:00
|
|
|
///
|
2020-07-10 22:28:19 +02:00
|
|
|
final String channelId;
|
|
|
|
final Document _root;
|
|
|
|
|
|
|
|
_InitialData _initialData;
|
|
|
|
|
2020-07-16 20:02:54 +02:00
|
|
|
///
|
2020-12-25 23:29:01 +01:00
|
|
|
_InitialData get initialData {
|
|
|
|
if (_initialData != null) {
|
|
|
|
return _initialData;
|
|
|
|
}
|
|
|
|
|
|
|
|
final scriptText = _root
|
|
|
|
.querySelectorAll('script')
|
|
|
|
.map((e) => e.text)
|
|
|
|
.toList(growable: false);
|
|
|
|
|
|
|
|
var initialDataText = scriptText.firstWhere(
|
|
|
|
(e) => e.contains('window["ytInitialData"] ='),
|
|
|
|
orElse: () => null);
|
|
|
|
if (initialDataText != null) {
|
|
|
|
return _initialData = _InitialData(ChannelUploadPageId.fromRawJson(
|
|
|
|
_extractJson(initialDataText, 'window["ytInitialData"] =')));
|
|
|
|
}
|
|
|
|
|
|
|
|
initialDataText = scriptText.firstWhere(
|
|
|
|
(e) => e.contains('var ytInitialData = '),
|
|
|
|
orElse: () => null);
|
|
|
|
if (initialDataText != null) {
|
|
|
|
return _initialData = _InitialData(ChannelUploadPageId.fromRawJson(
|
|
|
|
_extractJson(initialDataText, 'var ytInitialData = ')));
|
|
|
|
}
|
|
|
|
|
|
|
|
throw TransientFailureException(
|
|
|
|
'Failed to retrieve initial data from the channel upload page, please report this to the project GitHub page.'); // ignore: lines_longer_than_80_chars
|
|
|
|
}
|
2020-07-10 22:28:19 +02:00
|
|
|
|
|
|
|
String _extractJson(String html, String separator) {
|
|
|
|
return _matchJson(
|
|
|
|
html.substring(html.indexOf(separator) + separator.length));
|
|
|
|
}
|
|
|
|
|
|
|
|
String _matchJson(String str) {
|
|
|
|
var bracketCount = 0;
|
|
|
|
int lastI;
|
|
|
|
for (var i = 0; i < str.length; i++) {
|
|
|
|
lastI = i;
|
|
|
|
if (str[i] == '{') {
|
|
|
|
bracketCount++;
|
|
|
|
} else if (str[i] == '}') {
|
|
|
|
bracketCount--;
|
|
|
|
} else if (str[i] == ';') {
|
|
|
|
if (bracketCount == 0) {
|
|
|
|
return str.substring(0, i);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return str.substring(0, lastI + 1);
|
|
|
|
}
|
|
|
|
|
2020-07-16 20:02:54 +02:00
|
|
|
///
|
2020-07-12 18:24:22 +02:00
|
|
|
ChannelUploadPage(this._root, this.channelId, [_InitialData initialData])
|
|
|
|
: _initialData = initialData;
|
2020-07-10 22:28:19 +02:00
|
|
|
|
2020-07-16 20:02:54 +02:00
|
|
|
///
|
2020-07-12 18:24:22 +02:00
|
|
|
Future<ChannelUploadPage> nextPage(YoutubeHttpClient httpClient) {
|
|
|
|
if (initialData.continuation.isEmpty) {
|
|
|
|
return Future.value(null);
|
|
|
|
}
|
|
|
|
var url =
|
|
|
|
'https://www.youtube.com/browse_ajax?ctoken=${initialData.continuation}&continuation=${initialData.continuation}&itct=${initialData.clickTrackingParams}';
|
|
|
|
return retry(() async {
|
|
|
|
var raw = await httpClient.getString(url);
|
2020-09-12 11:20:22 +02:00
|
|
|
return ChannelUploadPage(null, channelId,
|
|
|
|
_InitialData(ChannelUploadPageId.fromJson(json.decode(raw)[1])));
|
2020-07-12 18:24:22 +02:00
|
|
|
});
|
|
|
|
}
|
2020-07-10 22:28:19 +02:00
|
|
|
|
2020-07-16 20:02:54 +02:00
|
|
|
///
|
2020-07-12 18:24:22 +02:00
|
|
|
static Future<ChannelUploadPage> get(
|
|
|
|
YoutubeHttpClient httpClient, String channelId, String sorting) {
|
|
|
|
assert(sorting != null);
|
2020-07-10 22:28:19 +02:00
|
|
|
var url =
|
2020-07-12 18:24:22 +02:00
|
|
|
'https://www.youtube.com/channel/$channelId/videos?view=0&sort=$sorting&flow=grid';
|
2020-07-10 22:28:19 +02:00
|
|
|
return retry(() async {
|
|
|
|
var raw = await httpClient.getString(url);
|
2020-07-12 18:24:22 +02:00
|
|
|
return ChannelUploadPage.parse(raw, channelId);
|
2020-07-10 22:28:19 +02:00
|
|
|
});
|
|
|
|
}
|
|
|
|
|
2020-07-16 20:02:54 +02:00
|
|
|
///
|
2020-07-12 18:24:22 +02:00
|
|
|
ChannelUploadPage.parse(String raw, this.channelId)
|
2020-07-10 22:28:19 +02:00
|
|
|
: _root = parser.parse(raw);
|
|
|
|
}
|
|
|
|
|
|
|
|
class _InitialData {
|
|
|
|
// Json parsed map
|
2020-09-12 11:20:22 +02:00
|
|
|
final ChannelUploadPageId root;
|
2020-07-10 22:28:19 +02:00
|
|
|
|
2020-09-12 11:20:22 +02:00
|
|
|
_InitialData(this.root);
|
2020-07-10 22:28:19 +02:00
|
|
|
|
|
|
|
/* Cache results */
|
|
|
|
|
|
|
|
List<ChannelVideo> _uploads;
|
|
|
|
String _continuation;
|
|
|
|
String _clickTrackingParams;
|
|
|
|
|
2020-09-13 11:19:23 +02:00
|
|
|
List<GridRendererItem> getContentContext() {
|
2020-09-12 11:20:22 +02:00
|
|
|
if (root.contents != null) {
|
|
|
|
return root.contents.twoColumnBrowseResultsRenderer.tabs
|
|
|
|
.map((e) => e.tabRenderer)
|
|
|
|
.firstWhere((e) => e.selected)
|
|
|
|
.content
|
|
|
|
.sectionListRenderer
|
|
|
|
.contents
|
|
|
|
.first
|
|
|
|
.itemSectionRenderer
|
|
|
|
.contents
|
|
|
|
.first
|
|
|
|
.gridRenderer
|
|
|
|
.items;
|
2020-07-10 22:28:19 +02:00
|
|
|
}
|
2020-09-12 11:20:22 +02:00
|
|
|
if (root.response != null) {
|
|
|
|
return root.response.continuationContents.gridContinuation.items;
|
2020-07-10 22:28:19 +02:00
|
|
|
}
|
2020-07-16 19:28:49 +02:00
|
|
|
throw FatalFailureException('Failed to get initial data context.');
|
2020-07-10 22:28:19 +02:00
|
|
|
}
|
|
|
|
|
2020-09-12 11:20:22 +02:00
|
|
|
NextContinuationData getContinuationContext() {
|
|
|
|
if (root.contents != null) {
|
2021-03-04 10:46:37 +01:00
|
|
|
return root.contents?.twoColumnBrowseResultsRenderer?.tabs
|
|
|
|
?.map((e) => e.tabRenderer)
|
|
|
|
?.firstWhere((e) => e.selected)
|
|
|
|
?.content
|
|
|
|
?.sectionListRenderer
|
|
|
|
?.contents
|
|
|
|
?.first
|
|
|
|
?.itemSectionRenderer
|
|
|
|
?.contents
|
|
|
|
?.first
|
|
|
|
?.gridRenderer
|
|
|
|
?.continuations
|
|
|
|
?.first
|
|
|
|
?.nextContinuationData;
|
2020-07-10 22:28:19 +02:00
|
|
|
}
|
2020-09-12 11:20:22 +02:00
|
|
|
if (root.response != null) {
|
2021-03-04 10:46:37 +01:00
|
|
|
return root?.response?.continuationContents?.gridContinuation
|
|
|
|
?.continuations?.first?.nextContinuationData;
|
2020-07-10 22:28:19 +02:00
|
|
|
}
|
|
|
|
return null;
|
|
|
|
}
|
|
|
|
|
2020-09-12 11:20:22 +02:00
|
|
|
List<ChannelVideo> get uploads => _uploads ??= getContentContext()
|
2020-07-10 22:28:19 +02:00
|
|
|
?.map(_parseContent)
|
|
|
|
?.where((e) => e != null)
|
2020-09-13 11:19:23 +02:00
|
|
|
?.toList();
|
2020-07-10 22:28:19 +02:00
|
|
|
|
2020-09-12 11:20:22 +02:00
|
|
|
String get continuation =>
|
2021-03-04 10:46:37 +01:00
|
|
|
_continuation ??= getContinuationContext()?.continuation ?? '';
|
2020-07-10 22:28:19 +02:00
|
|
|
|
|
|
|
String get clickTrackingParams => _clickTrackingParams ??=
|
2020-09-12 11:20:22 +02:00
|
|
|
getContinuationContext()?.clickTrackingParams ?? '';
|
2020-07-10 22:28:19 +02:00
|
|
|
|
2020-09-13 11:19:23 +02:00
|
|
|
ChannelVideo _parseContent(GridRendererItem content) {
|
2020-09-12 11:20:22 +02:00
|
|
|
if (content == null || content.gridVideoRenderer == null) {
|
2020-07-10 22:28:19 +02:00
|
|
|
return null;
|
|
|
|
}
|
2020-09-12 11:20:22 +02:00
|
|
|
var video = content.gridVideoRenderer;
|
2020-09-13 11:19:23 +02:00
|
|
|
return ChannelVideo(
|
|
|
|
VideoId(video.videoId),
|
|
|
|
video.title?.simpleText ??
|
|
|
|
video.title?.runs?.map((e) => e.text)?.join() ??
|
|
|
|
'');
|
2020-07-10 22:28:19 +02:00
|
|
|
}
|
|
|
|
}
|