astroport-iptubes/crawl.py

#!/usr/bin/python3

################################################################################
# Author: Poka (poka@p2p.legal)
# Version: 0.1.0
# License: AGPL-3.0 (https://choosealicense.com/licenses/agpl-3.0/)
# Git: https://git.p2p.legal/axiom-team/astroport
################################################################################

# Config
diskLimit = 15
tkdelay = 3
displayDetails = "true"

import requests
import json
import sys
import os
import shutil
import subprocess
import login
import time
import re
from termcolor import colored
from optparse import OptionParser
from urllib.parse import unquote

# Load options
parser = OptionParser()
parser.add_option("-s", "--seed", action="store_false", dest="rmTracker", default=True,
                  help="Keep the tracker for this torrent. So ratio is supported.")
parser.add_option("-q", "--quiet",
                  action="store_false", dest="verbose", default=True,
                  help="don't print status messages to stdout")
(options, args) = parser.parse_args()


# Load scraper
from yggcrawl import YggTorrentScraper
scraper = YggTorrentScraper(requests.session())
from yggtorrentscraper import set_yggtorrent_tld
set_yggtorrent_tld("si")
name = ' '.join(sys.argv[1:])

# Search torrent name
if ("https://" not in name):
    name = re.sub(r'\w*-\w*', '', name)
    research = os.popen('./lib/scrabash.sh search --best=true ' + name).read()

    try:
        research.index("No torrent found")
    except ValueError:
        True
    else:
        print(colored('No torrent named "' + name + '" on YggTorrent', 'blue'))
        sys.exit(1)
else:
    name = re.sub(r'\w*--seed\w*', '', name)
    research = unquote(name, errors='strict')

# Allow only one torrent downling in same time, and remove oldest torrent if disk size is full.
def rollingFiles():
        def isDL():
            downloading = os.popen('./trans-ctl.sh downloading').read()
            if "There is a torrent downloading" in downloading:
                print("A torrent is downloading, waiting 1 minute")
                time.sleep(60)
                isDL()

        isDL()
        global idTorrent
        idTorrent = os.popen('./lib/scrabash.sh get_details ' + research).read()
        dbPath = "./data/meta/{}/db.py".format(idTorrent.rstrip())
        torrentDB = os.popen('cat ' + dbPath).read()
        print(colored(torrentDB, 'blue'))
        print("---")

        torrentDBFile = open(dbPath, "r")

        for line in torrentDBFile:
            if re.search("Size", line):
                fields = line.strip().split()
                if "M" in fields[2]:
                    torrentSize = 1
                else:
                    torrentSize = float(fields[2]) #kopa

        # Check space on disk
        def checkDisk():
            global diskSize
            diskSize = os.popen('./trans-ctl.sh checkdisk').read()
            if re.search("You have reached", diskSize):
                fields = diskSize.strip().split()
                if "M" in diskSize:
                    diskSize = float(fields[-1])

            diskEnd = float(diskSize) + torrentSize
            if torrentSize >= diskLimit:
                print(colored('This torrent is larger as your disk limit:', 'red'))
                print("Disk usage: " + str(diskSize.rstrip()))
                print("Limit: " + str(diskLimit))
                print("Size of this torrent: " + str(colored(torrentSize, 'red')))
                sys.exit(colored("Torrent too large", 'red'))

            if diskEnd >= diskLimit:
                print(colored("You have filled the entire space allocated to torrents:", 'red'))
                print("Disk usage: " + str(diskSize.rstrip()))
                print("Limit: " + str(diskLimit))
                print("With this torrent: " + str(colored(diskEnd, 'red')))
                time.sleep(1)

                print(colored('Remove oldest torrent...', 'yellow'))
                lowerid = os.popen('./trans-ctl.sh getlowerid').read()
                os.popen('./trans-ctl.sh remove ' + lowerid)
                time.sleep(3)
                checkDisk()
                # sys.exit(colored("Disk full", 'red'))

        checkDisk()

# Download Torrent
def downloadTorrent():
        # Download torrent file
        if len(os.listdir('data/tmp/torrents') ) != 0:
            shutil.rmtree('data/tmp/torrents', ignore_errors=True)
            os.mkdir("data/tmp/torrents")
        scraper.download_from_torrent_url(research)
        os.popen(f'cd data/tmp/torrents/ && mv *.torrent ../../torrents/{idTorrent.strip()}.torrent')

# Remove tracker
def removeTracker():
        print(colored("Start to remove tracker ...", 'yellow'))
        higherid = os.popen('./trans-ctl.sh gethigherid').read()
        time.sleep(1)
        print("Wait " + str(tkdelay) + " seconds to get peers before remove tracker...")
        time.sleep(tkdelay)
        os.popen('./trans-ctl.sh rmtracker ' + name)
        os.popen('./trans-ctl.sh rmtracker ' + higherid)
        os.replace(f'data/torrents/{idTorrent.strip()}.torrent.added', f'data/meta/{idTorrent.strip()}/{idTorrent.strip()}.torrent')


if(scraper.login(login.user, login.passwd)): #Check if user can login
    print(colored("Login success", 'green'))
    rollingFiles()
    downloadTorrent()
    removeTracker() if options.rmTracker else time.sleep(2); os.replace(f'data/torrents/{idTorrent.strip()}.torrent.added', f'data/meta/{idTorrent.strip()}/{idTorrent.strip()}.torrent')
else:
    print(colored("Login failed", 'red'))
    sys.exit(1)


# End
print(colored("Done", 'green'))
sys.exit(0)