Compare commits

...

11 Commits

17 changed files with 445 additions and 127 deletions

1
.gitignore vendored
View File

@ -3,4 +3,3 @@ __pycache__/
yggcrawl/__pycache__/
yggcrawl/__init__.pyc
login.py
yggcrawl/gecko/geckodriver.log

View File

@ -16,30 +16,49 @@ import requests
import json
import sys
import os
import shutil
import subprocess
import login
import time
import re
from termcolor import colored
from optparse import OptionParser
from urllib.parse import unquote
# Load options
parser = OptionParser()
parser.add_option("-s", "--seed", action="store_false", dest="rmTracker", default=True,
help="Keep the tracker for this torrent. So ratio is supported.")
parser.add_option("-q", "--quiet",
action="store_false", dest="verbose", default=True,
help="don't print status messages to stdout")
(options, args) = parser.parse_args()
# Load scraper
from yggcrawl import YggTorrentScraper
scraper = YggTorrentScraper(requests.session())
from yggcrawl import set_yggtorrent_tld
set_yggtorrent_tld("se")
from yggtorrentscraper import set_yggtorrent_tld
set_yggtorrent_tld("si")
name = ' '.join(sys.argv[1:])
# Search torrent name
research = os.popen('./lib/scrabash.sh search --best=true ' + name).read()
if ("https://" not in name):
name = re.sub(r'\w*-\w*', '', name)
research = os.popen('./lib/scrabash.sh search --best=true ' + name).read()
try:
research.index("No torrent found")
except ValueError:
True
try:
research.index("No torrent found")
except ValueError:
True
else:
print(colored('No torrent named "' + name + '" on YggTorrent', 'blue'))
sys.exit(1)
else:
sys.exit(1)
name = re.sub(r'\w*--seed\w*', '', name)
research = unquote(name, errors='strict')
# Rollong Files
# Allow only one torrent downling in same time, and remove oldest torrent if disk size is full.
def rollingFiles():
def isDL():
downloading = os.popen('./trans-ctl.sh downloading').read()
@ -102,15 +121,11 @@ def rollingFiles():
# Download Torrent
def downloadTorrent():
# Download torrent file
if(scraper.login(login.user, login.passwd)):
print(colored("Login success", 'green'))
subprocess.Popen('[[ $(ls data/tmp/torrents/) ]] && rm data/tmp/torrents/*', executable='/bin/bash', stdout=subprocess.PIPE, stderr=subprocess.PIPE)
scraper.download_from_torrent_url(research)
# os.popen(f'cd data/tmp/torrents/ && mv *.torrent {idTorrent.strip()}.torrent && mv {idTorrent.strip()}.torrent ../../torrents/').read()
os.popen('cd data/tmp/torrents/ && mv *.torrent ../../torrents/')
else:
print(colored("Login failed", 'red'))
sys.exit(1)
if len(os.listdir('data/tmp/torrents') ) != 0:
shutil.rmtree('data/tmp/torrents', ignore_errors=True)
os.mkdir("data/tmp/torrents")
scraper.download_from_torrent_url(research)
os.popen(f'cd data/tmp/torrents/ && mv *.torrent ../../torrents/{idTorrent.strip()}.torrent')
# Remove tracker
def removeTracker():
@ -121,11 +136,18 @@ def removeTracker():
time.sleep(tkdelay)
os.popen('./trans-ctl.sh rmtracker ' + name)
os.popen('./trans-ctl.sh rmtracker ' + higherid)
# print(tkresult)
os.replace(f'data/torrents/{idTorrent.strip()}.torrent.added', f'data/meta/{idTorrent.strip()}/{idTorrent.strip()}.torrent')
if(scraper.login(login.user, login.passwd)): #Check if user can login
print(colored("Login success", 'green'))
rollingFiles()
downloadTorrent()
removeTracker() if options.rmTracker else time.sleep(2); os.replace(f'data/torrents/{idTorrent.strip()}.torrent.added', f'data/meta/{idTorrent.strip()}/{idTorrent.strip()}.torrent')
else:
print(colored("Login failed", 'red'))
sys.exit(1)
rollingFiles()
downloadTorrent()
removeTracker()
# End
print(colored("Done", 'green'))

View File

@ -31,12 +31,6 @@ sbotc() {
transmission() {
echo -e "${c_yellow}Installing Transmision...$c_"
sudo apt install transmission-daemon --install-suggests
sudo apt install transmission-cli
# stop
# Copy login.py info to /etc/transmission/settings.json
# start
}
# Install pip tools
@ -74,8 +68,8 @@ pip3() {
iptubes() {
[[ -z $(which pip3) ]] && pip3
/usr/bin/pip3 install $(curl -s https://raw.githubusercontent.com/Harkame/YggTorrentScraper/master/requirements.txt)
sudo chgrp -R debian-transmission data/
sudo chmod -R g+w data/
chgrp -R debian-transmission data/
chmod -R g+w data/
sudo service transmission-daemon restart
cp login.py.template login.py
cd lib/py/
@ -86,6 +80,16 @@ iptubes() {
nano login.py
}
nordvpn() {
wget -qnc https://repo.nordvpn.com/deb/nordvpn/debian/pool/main/nordvpn-release_1.0.0_all.deb -O /tmp/nordvpn.deb
sudo dpkg -i /tmp/nordvpn.dev
rm /tmp/nordvpn.dev
sudo apt update
sudo apt install nordvpn
}
# Check installs
[[ -z $(which ipfs) ]] && ipfs
[[ -z $(which sbotc) ]] && sbotc

View File

@ -16,7 +16,7 @@ try:
except NameError:
from yggcrawl import YggTorrentScraper
scraper = YggTorrentScraper(requests.session())
from yggcrawl import set_yggtorrent_tld
from yggtorrentscraper import set_yggtorrent_tld
set_yggtorrent_tld("se")
cmd = sys.argv[1]

22
lib/ratiomaster/README.md Normal file
View File

@ -0,0 +1,22 @@
# Ratio.py
Ratio.py is a small command line RatioMaster.Net like in Python3. It fakes upload stats of a torrent.
Current emulators available are:
* Transmission 2.92
## Requirements:
1. Python 3.x
2. pip install -r requirements.txt
## Usage:
```console
foo@bar:~/ratio.py$ python ratio.py -c configuration.json
```
## Configuration example
```js
{
"torrent": "<Torrent file path>",
"upload": "<Upload speed (kB/s)>"
}
```

View File

View File

@ -0,0 +1,90 @@
import re
import logging
import binascii
class bencoding():
def __init__(self):
self.decimal_match = re.compile('\d')
self.data = b''
self.dict = {}
def get_dict(self, key):
if key not in self.dict:
return ''
start = self.dict[key][0]
end = self.dict[key][1]
return self.data[start:end]
def get_item(self, chunks):
item = chunks[self.i]
self.i += 1
if not type(item) == str:
item = bytes([item])
try:
item = item.decode('utf-8')
except:
item = '\\x{}'.format(binascii.hexlify(item))
return item
def decoding_byte_string(self, chunks, item):
# logging.debug('decoding string')
num = ''
while self.decimal_match.search(item):
num += item
item = self.get_item(chunks)
line = ''
for i in range(int(num)):
line += self.get_item(chunks)
return line
def decoding_integer(self, chunks):
# logging.debug('decoding integer')
item = self.get_item(chunks)
num = ''
while item != 'e':
num += item
item = self.get_item(chunks)
return int(num)
def decoding_list(self, chunks):
# logging.debug('decoding list')
item = self.get_item(chunks)
list = []
while item != 'e':
self.i -= 1
list.append(self._dechunk(chunks))
item = self.get_item(chunks)
return list
def decoding_dictionnary(self, chunks):
# logging.debug('decoding dictionnary')
item = self.get_item(chunks)
hash = {}
while item != 'e':
self.i -= 1
key = self._dechunk(chunks)
start = self.i
hash[key] = self._dechunk(chunks)
end = self.i
self.dict[key] = (start, end)
item = self.get_item(chunks)
return hash
def _dechunk(self, chunks):
item = self.get_item(chunks)
if item == 'd':
return self.decoding_dictionnary(chunks)
elif item == 'l':
return self.decoding_list(chunks)
elif item == 'i':
return self.decoding_integer(chunks)
elif self.decimal_match.search(item):
return self.decoding_byte_string(chunks, item)
raise "Invalid input!"
def bdecode(self, data):
self.data = data
chunks = list(self.data)
self.i = 0
root = self._dechunk(chunks)
return root

View File

@ -0,0 +1,28 @@
import requests
from pprint import pformat
def get_headers(headers):
res = ''
for k, v in headers.items():
res += '{}: {}\n'.format(k, v)
return res
def pretty_GET(url, headers, params):
req = requests.Request('GET', url, headers=headers, params=params)
s = requests.Session()
prepared = s.prepare_request(req)
p = '-----START-----\n'
p +=('{} {}\n{}'.format(prepared.method, prepared.url,
get_headers(prepared.headers),
)
)
if prepared.body:
pi += prepared.body
p += '------END------'
return p
def pretty_data(data):
return pformat(data)

View File

@ -0,0 +1,118 @@
from code.decoding_bencoded import bencoding
from code.torrentclientfactory import Transmission292
from code.pretty import pretty_data, pretty_GET
from hashlib import sha1
from urllib.parse import quote_plus
import requests
import logging
import random
from tqdm import tqdm
from time import sleep
from struct import unpack
logging.basicConfig(level=logging.DEBUG)
class process_torrent():
def __init__(self, configuration):
self.configuration = configuration
self.open_torrent()
self.torrentclient = Transmission292(self.tracker_info_hash())
def open_torrent(self):
torrent_file = self.configuration['torrent']
with open(torrent_file, 'rb') as tf:
data = tf.read()
self.b_enc = bencoding()
self.metainfo = self.b_enc.bdecode(data)
self.info = self.metainfo['info']
if 'length' not in self.info:
self.info['length'] = 0
for file in self.info['files']:
self.info['length'] += file['length']
print(pretty_data(self.info['files']))
def tracker_info_hash(self):
raw_info = self.b_enc.get_dict('info')
hash_factory = sha1()
hash_factory.update(raw_info)
hashed = hash_factory.hexdigest()
sha = bytearray.fromhex(hashed)
return str(quote_plus(sha))
def send_request(self, params, headers):
url = self.metainfo['announce']
print(pretty_GET(url, headers, params))
while True:
try:
r = requests.get(url, params=params, headers=headers)
except requests.exceptions.ConnectionError as e:
sleep(1)
continue
break
return r.content
def tracker_start_request(self):
tc = self.torrentclient
headers = tc.get_headers()
params = tc.get_query(uploaded=0,
downloaded=0,
event='started')
print('----------- First Command to Tracker --------')
content = self.send_request(params, headers)
self.tracker_response_parser(content)
def tracker_response_parser(self, tr_response):
b_enc = bencoding()
response = b_enc.bdecode(tr_response)
print('----------- Received Tracker Response --------')
print(pretty_data(response))
raw_peers = b_enc.get_dict('peers')
i = 0
peers = []
while i<len(raw_peers)-6:
peer = raw_peers[i:i+6]
i+=6
unpacked_ip = unpack('BBBB', peer[0:4])
ip = ".".join(str(i) for i in unpacked_ip)
unpacked_port = unpack('!H', peer[4:6])
port = unpacked_port[0]
peers.append((ip, port))
self.interval = response['interval']
def wait(self):
pbar = tqdm(total=self.interval)
print('sleep: {}'.format(self.interval))
t = 0
while t < self.interval:
t += 1
pbar.update(1)
sleep(1)
pbar.close()
def tracker_process(self):
while True:
self.tracker_start_request()
print('----------- Sending Command to Tracker --------')
# get upload
min_up = self.interval-(self.interval*0.1)
max_up = self.interval
randomize_upload = random.randint(min_up, max_up)
uploaded = int(self.configuration['upload'])*1000*randomize_upload
# get download
downloaded = 0
tc = self.torrentclient
headers = tc.get_headers()
params = tc.get_query(uploaded=uploaded,
downloaded=downloaded,
event='stopped')
content = self.send_request(params, headers)
self.tracker_response_parser(content)
self.wait()

View File

@ -0,0 +1,60 @@
import random
import string
class Transmission292():
def __init__(self, info_hash):
self.name = "Transmission 2.92 (14714)"
parameters = {}
# urlencoded 20-byte SHA1 hash of the value of the info key from the Metainfo file
parameters['info_hash'] = info_hash
# urlencoded 20-byte string used as a unique ID for the client
parameters["peer_id"] = self.generate_peer_id()
# The port number that the client is listening on
parameters["port"] = random.randint(1025, 65535)
# Number of peers that the client would like to receive from the tracker
parameters["numwant"] = 80
# An additional identification that is not shared with any other peers
parameters["key"] = self.generate_key()
# Setting this to 1 indicates that the client accepts a compact response
parameters["compact"] = 0
# Setting this to 1 indicates that the client accepts crypto
parameters["supportcrypto"] = 1
self.parameters = parameters
def get_headers(self):
headers = {}
headers['User-Agent'] = 'Transmission/2.92'
headers['Accept'] = '*/*'
headers['Accept-Encoding'] = 'Accept-Encoding: gzip;q=1.0, deflate, identity'
return headers
def get_query(self, uploaded, downloaded, left=0, event=None):
# The total amount uploaded (since the client sent the 'started' event)
self.parameters["uploaded"] = uploaded
# The total amount downloaded (since the client sent the 'started' event)
self.parameters["downloaded"] = downloaded
# The number of bytes this client still has to download
self.parameters["left"] = left
# If specified, must be one of started, completed, stopped
if event:
self.parameters["event"] = event
params = '&'.join('{}={}'.format(k, v)
for k, v in self.parameters.items())
return params
def id_generator(self, chars, size):
id = ''
for _ in range(size):
id += random.choice(chars)
return id
def generate_peer_id(self):
chars = string.ascii_lowercase + string.digits
rand_id = self.id_generator(chars, 12)
peer_id = "-TR2920-" + rand_id
return peer_id
def generate_key(self):
chars = 'ABCDEF' + string.digits
key = self.id_generator(chars, 8)
return key

34
lib/ratiomaster/ratio.py Normal file
View File

@ -0,0 +1,34 @@
from code.process_torrent import process_torrent
import argparse
import json
import sys
def parse_args():
"""Create the arguments"""
parser = argparse.ArgumentParser('\nratio.py -c <configuration-file.json>')
parser.add_argument("-c", "--configuration", help="Configuration file")
return parser.parse_args()
def load_configuration(configuration_file):
with open(configuration_file) as f:
configuration = json.load(f)
if 'torrent' not in configuration:
return None
return configuration
if __name__ == "__main__":
args = parse_args()
if args.configuration:
configuration = load_configuration(args.configuration)
else:
sys.exit()
if not configuration:
sys.exit()
to = process_torrent(configuration)
to.tracker_process()

View File

@ -0,0 +1,2 @@
requests
tqdm

View File

@ -51,20 +51,19 @@ get_details() {
# Get image
[[ ! -d img ]] && mkdir img && cd img
url=$(wget -qO- -np -nd $name | awk -v RS=' ' '/.jpg/' | awk -F '"' '{ print $2 }' | head -n1)
curl -s -O $url
[[ $url ]] && curl -s -O $url
fi
}
vpn() {
[[ ! $(which nordvpn) ]] && echo "Installaling NordVPN client... && ./install.sh nordvpn"
vpn_citie=$(shuf -n1 .vpn/countries)
echo "Warning: trying to connect to random cities in the world via NordVPN. If you are connected to this machine via SSH, you will lost the connection..."
echo "VPN connection in 5 seconds, press CTRL+C to cancel..."
sleep 5
nordvpn c $vpn_citie
[[ ! $(which nordvpn) ]] && echo "Installaling NordVPN client... && ./install.sh nordvpn"
vpn_citie=$(shuf -n1 .vpn/countries)
echo "Warning: trying to connect to random cities in the world via NordVPN. If you are connected to this machine via SSH, you will lost the connection..."
echo "VPN connection in 5 seconds, press CTRL+C to cancel..."
sleep 5
nordvpn c $vpn_citie
}
$1
$cmd
[[ $err == 1 ]] && exit 1 || exit 0

17
tata.sh
View File

@ -1,17 +0,0 @@
#!/usr/bin/env bash
readWords() {
declare -i int="$1"
(( int == 0 )) && {
printf "%s\n" "$int is 0, cant find 0 words"
return 1
}
while read getWords;do
if [[ ${#getWords} -eq $int ]];then
printf "%s\n" "$getWords"
fi
done < /usr/share/dict/words
}
readWords 20

View File

@ -26,9 +26,12 @@ getid() {
# Get ID
else
j=0
for i in "$name"; do
[[ $j == 0 ]] && result=$($transcmd --list | grep -vE 'Sum:|ID Done' | grep -i "$i")
result=$(echo "$result" | grep -vE 'Sum:|ID Done' | grep -iw "$i")
for i in $name; do
if [[ $j == 0 ]];then
result=$($transcmd --list | grep -vE 'Sum:|ID Done' | grep -iw "$i")
else
result=$(echo "$result" | grep -iw "$i")
fi
((j++))
done
fi
@ -36,7 +39,7 @@ getid() {
echo "$result" | awk '{ print $1 }'
else
echo "No torrent found"
fi
fi
}
getlowerid() {
@ -86,13 +89,16 @@ case "$1" in
remove)
idt=$(getid | tr -d '*')
if [[ $idt =~ ^[+-]?[0-9]+([.][0-9]+)?$ ]]; then
for i in "$($transcmd --list | grep -vE 'Sum:|ID Done' )"; do
torrentList=$($transcmd --list | grep -vE 'Sum:|ID Done' )
IFS=$'\n'
for i in $torrentList; do
if [[ $(echo "$i" | awk '{ print $1 }') == $idt ]]; then
fileName=$(echo "$i" | awk '{ print $NF }')
break
fi
done
IFS=$' '
[[ ! $fileName ]] && echo "Can't find torrent to remove." && exit 1
cd data/meta
torrentId=$(grep -r $fileName | head -n1 | awk -F '/' '{ print $1 }')
rm -rf $torrentId

View File

@ -1,44 +0,0 @@
#!/usr/bin/python3
# Early exemple of how to use selenium with gecko to bypass cloudflare bots detections
# The only way to block this should be using of captcha in front of every yggtorrent pages by sessions...
import sys
import time
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
# Exit if no arguments
if len(sys.argv)==1: sys.exit("Please choose a film ou serie name")
else: arg1 = sys.argv[1]
search_url = f"https://www2.yggtorrent.se/engine/search?name={arg1}&description=&file=&uploader=&category=all&sub_category=&do=search&order=desc&sort=seed"
# Load webdriver with Gecko
options = webdriver.FirefoxOptions()
options.add_argument('-headless')
driver = webdriver.Firefox(options=options, executable_path=r'/usr/local/bin/geckodriver')
driver.get(search_url)
# Wait to bypass cloudflare
print("Page atteinte, attente de redirection anti-crawling...")
wait = WebDriverWait(driver, 10)
wait.until(lambda driver: driver.current_url != search_url)
# Wait 2 seconds to load page
print("Anti-crawling passé, affichage dans 2 secondes ...")
time.sleep(2)
# Filter torrent urls
elems = driver.find_elements_by_css_selector(".results [href]")
links = [elem.get_attribute('href') for elem in elems]
links = [k for k in links if '/torrent/' in k]
# Print torrents urls
print("\n".join(links))
driver.quit()

View File

@ -9,7 +9,7 @@ from bs4 import BeautifulSoup
from .torrent import Torrent, TorrentComment, TorrentFile
from .categories import categories
YGGTORRENT_TLD = "se"
YGGTORRENT_TLD = "si"
YGGTORRENT_BASE_URL = f"https://www2.yggtorrent.{YGGTORRENT_TLD}"
@ -44,7 +44,7 @@ YGGTORRENT_SEARCH_URL_DO = "&do="
YGGTORRENT_SEARCH_URL_PAGE = "&page="
YGGTORRENT_GET_FILES = f"{YGGTORRENT_BASE_URL}/engine/get_files?torrent="
YGGTORRENT_GET_INFO = f"https://www2.yggtorrent.se/engine/get_nfo?torrent="
YGGTORRENT_GET_INFO = f"{YGGTORRENT_BASE_URL}/engine/get_nfo?torrent="
YGGTORRENT_MOST_COMPLETED_URL = f"{YGGTORRENT_BASE_URL}/engine/mostcompleted"
@ -52,7 +52,6 @@ TORRENT_PER_PAGE = 50
YGGTORRENT_FILES_URL = f"{YGGTORRENT_BASE_URL}/engine/get_files?torrent="
headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36'}
def set_yggtorrent_tld(yggtorrent_tld=None):
"""
@ -78,7 +77,7 @@ def set_yggtorrent_tld(yggtorrent_tld=None):
YGGTORRENT_SEARCH_URL = f"{YGGTORRENT_BASE_URL}/engine/search?name="
YGGTORRENT_DOMAIN = ".yggtorrent.se"
YGGTORRENT_DOMAIN = ".yggtorrent.gg"
YGGTORRENT_GET_FILES = f"{YGGTORRENT_BASE_URL}/engine/get_files?torrent="
YGGTORRENT_GET_INFO = f"https://www2.yggtorrentchg/engine/get_nfo?torrent="
@ -109,7 +108,7 @@ class YggTorrentScraper:
"User-Agent": "PostmanRuntime/7.17.1",
"Accept": "*/*",
"Cache-Control": "no-cache",
"Host": f"www2.yggtorrent.{YGGTORRENT_TLD}",
"Host": f"www.yggtorrent.{YGGTORRENT_TLD}",
"Accept-Encoding": "gzip, deflate",
"Connection": "keep-alive",
}
@ -146,7 +145,7 @@ class YggTorrentScraper:
"""
Logout request
"""
response = self.session.get(YGGTORRENT_LOGOUT_URL, headers=headers)
response = self.session.get(YGGTORRENT_LOGOUT_URL)
self.session.cookies.clear()
@ -161,18 +160,12 @@ class YggTorrentScraper:
return False
#kopa
def search_old(self, parameters):
def search(self, parameters):
search_url = create_search_url(parameters)
torrents_url = self.get_torrents_url(search_url, parameters)
return torrents_url
def search(self, parameters):
# torrents_url = os.popen('gecko/torrent_search.py didier')
torrents_url = exec(open('/home/iptubes/astroport-iptubes/yggcrawl/gecko/torrent_search.py').read())
return torrents_url
def extract_details(self, torrent_url):
"""
Extract informations from torrent's url
@ -181,7 +174,7 @@ class YggTorrentScraper:
torrents = []
response = self.session.get(torrent_url, headers=headers)
response = self.session.get(torrent_url)
torrent_page = BeautifulSoup(response.content, features="lxml")
@ -244,7 +237,7 @@ class YggTorrentScraper:
"input", {"type": "hidden", "name": "target"}
)["value"]
response = self.session.get(YGGTORRENT_GET_FILES + torrent_id, headers=headers)
response = self.session.get(YGGTORRENT_GET_FILES + torrent_id)
files_page = BeautifulSoup(response.content, features="lxml")
@ -299,12 +292,12 @@ class YggTorrentScraper:
return torrents_url
#kopaa
def get_torrents_url(self, search_url, parameters):
"""
Return
"""
response = self.session.get(search_url, headers=headers)
response = self.session.get(search_url)
search_page = BeautifulSoup(response.content, features="lxml")
@ -324,7 +317,7 @@ class YggTorrentScraper:
search_url = create_search_url(parameters)
response = self.session.get(search_url, headers=headers)
response = self.session.get(search_url)
search_page = BeautifulSoup(response.content, features="lxml")
@ -335,6 +328,7 @@ class YggTorrentScraper:
return torrents
#kopa
def download_from_torrent_url(self, torrent_url=None, destination_path="./data/tmp/torrents/"):
if torrent_url is not None:
torrent = self.extract_details(torrent_url)
@ -355,7 +349,7 @@ class YggTorrentScraper:
if torrent_url is None:
raise Exception("Invalid torrent_url, make sure you are logged")
response = self.session.get(YGGTORRENT_BASE_URL + torrent_url, headers=headers)
response = self.session.get(YGGTORRENT_BASE_URL + torrent_url)
temp_file_name = response.headers.get("content-disposition")
@ -374,6 +368,7 @@ class YggTorrentScraper:
return file_full_path
def create_search_url(parameters):
"""
Return a formated URL for torrent's search