Comming back from older commit, scraper is alive.
This commit is contained in:
parent
06e385d439
commit
b2ddcf2f95
8
crawl.py
8
crawl.py
|
@ -16,6 +16,7 @@ import requests
|
||||||
import json
|
import json
|
||||||
import sys
|
import sys
|
||||||
import os
|
import os
|
||||||
|
import shutil
|
||||||
import subprocess
|
import subprocess
|
||||||
import login
|
import login
|
||||||
import time
|
import time
|
||||||
|
@ -39,7 +40,7 @@ except ValueError:
|
||||||
else:
|
else:
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
|
|
||||||
# Rollong Files
|
# Allow only one torrent downling in same time, and remove oldest torrent if disk size is full.
|
||||||
def rollingFiles():
|
def rollingFiles():
|
||||||
def isDL():
|
def isDL():
|
||||||
downloading = os.popen('./trans-ctl.sh downloading').read()
|
downloading = os.popen('./trans-ctl.sh downloading').read()
|
||||||
|
@ -104,7 +105,9 @@ def downloadTorrent():
|
||||||
# Download torrent file
|
# Download torrent file
|
||||||
if(scraper.login(login.user, login.passwd)):
|
if(scraper.login(login.user, login.passwd)):
|
||||||
print(colored("Login success", 'green'))
|
print(colored("Login success", 'green'))
|
||||||
subprocess.Popen('[[ $(ls data/tmp/torrents/) ]] && rm data/tmp/torrents/*', executable='/bin/bash')
|
if len(os.listdir('data/tmp/torrents') ) != 0:
|
||||||
|
shutil.rmtree('data/tmp/torrents', ignore_errors=True)
|
||||||
|
os.mkdir("data/tmp/torrents")
|
||||||
scraper.download_from_torrent_url(research)
|
scraper.download_from_torrent_url(research)
|
||||||
# os.popen(f'cd data/tmp/torrents/ && mv *.torrent {idTorrent}.torrent && mv *.torrent ../../torrents/')
|
# os.popen(f'cd data/tmp/torrents/ && mv *.torrent {idTorrent}.torrent && mv *.torrent ../../torrents/')
|
||||||
os.popen(f'cd data/tmp/torrents/ && mv *.torrent {idTorrent.strip()}.torrent && mv {idTorrent.strip()}.torrent ../../torrents/')
|
os.popen(f'cd data/tmp/torrents/ && mv *.torrent {idTorrent.strip()}.torrent && mv {idTorrent.strip()}.torrent ../../torrents/')
|
||||||
|
@ -122,7 +125,6 @@ def removeTracker():
|
||||||
time.sleep(tkdelay)
|
time.sleep(tkdelay)
|
||||||
os.popen('./trans-ctl.sh rmtracker ' + name)
|
os.popen('./trans-ctl.sh rmtracker ' + name)
|
||||||
os.popen('./trans-ctl.sh rmtracker ' + higherid)
|
os.popen('./trans-ctl.sh rmtracker ' + higherid)
|
||||||
# print(tkresult)
|
|
||||||
|
|
||||||
rollingFiles()
|
rollingFiles()
|
||||||
downloadTorrent()
|
downloadTorrent()
|
||||||
|
|
|
@ -28,7 +28,7 @@ sbotc() {
|
||||||
}
|
}
|
||||||
|
|
||||||
# Install Transmission
|
# Install Transmission
|
||||||
transmisison() {
|
transmission() {
|
||||||
echo -e "${c_yellow}Installing Transmision...$c_"
|
echo -e "${c_yellow}Installing Transmision...$c_"
|
||||||
sudo apt install transmission-daemon --install-suggests
|
sudo apt install transmission-daemon --install-suggests
|
||||||
}
|
}
|
||||||
|
|
7
tata.py
7
tata.py
|
@ -1,7 +0,0 @@
|
||||||
#!/usr/bin/python3
|
|
||||||
|
|
||||||
import os
|
|
||||||
|
|
||||||
tata = "187364"
|
|
||||||
|
|
||||||
os.popen(f'cd data/tmp/torrents/ && mv caca.torrent {tata}.torrent')
|
|
18
trans-ctl.sh
18
trans-ctl.sh
|
@ -26,9 +26,12 @@ getid() {
|
||||||
# Get ID
|
# Get ID
|
||||||
else
|
else
|
||||||
j=0
|
j=0
|
||||||
for i in "$name"; do
|
for i in $name; do
|
||||||
[[ $j == 0 ]] && result=$($transcmd --list | grep -vE 'Sum:|ID Done' | grep -i "$i")
|
if [[ $j == 0 ]];then
|
||||||
result=$(echo "$result" | grep -vE 'Sum:|ID Done' | grep -iw "$i")
|
result=$($transcmd --list | grep -vE 'Sum:|ID Done' | grep -iw "$i")
|
||||||
|
else
|
||||||
|
result=$(echo "$result" | grep -iw "$i")
|
||||||
|
fi
|
||||||
((j++))
|
((j++))
|
||||||
done
|
done
|
||||||
fi
|
fi
|
||||||
|
@ -36,7 +39,7 @@ getid() {
|
||||||
echo "$result" | awk '{ print $1 }'
|
echo "$result" | awk '{ print $1 }'
|
||||||
else
|
else
|
||||||
echo "No torrent found"
|
echo "No torrent found"
|
||||||
fi
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
getlowerid() {
|
getlowerid() {
|
||||||
|
@ -86,13 +89,16 @@ case "$1" in
|
||||||
remove)
|
remove)
|
||||||
idt=$(getid | tr -d '*')
|
idt=$(getid | tr -d '*')
|
||||||
if [[ $idt =~ ^[+-]?[0-9]+([.][0-9]+)?$ ]]; then
|
if [[ $idt =~ ^[+-]?[0-9]+([.][0-9]+)?$ ]]; then
|
||||||
for i in "$($transcmd --list | grep -vE 'Sum:|ID Done' )"; do
|
torrentList=$($transcmd --list | grep -vE 'Sum:|ID Done' )
|
||||||
|
IFS=$'\n'
|
||||||
|
for i in $torrentList; do
|
||||||
if [[ $(echo "$i" | awk '{ print $1 }') == $idt ]]; then
|
if [[ $(echo "$i" | awk '{ print $1 }') == $idt ]]; then
|
||||||
fileName=$(echo "$i" | awk '{ print $NF }')
|
fileName=$(echo "$i" | awk '{ print $NF }')
|
||||||
break
|
break
|
||||||
fi
|
fi
|
||||||
done
|
done
|
||||||
|
IFS=$' '
|
||||||
|
[[ ! $fileName ]] && echo "Can't find torrent to remove." && exit 1
|
||||||
cd data/meta
|
cd data/meta
|
||||||
torrentId=$(grep -r $fileName | head -n1 | awk -F '/' '{ print $1 }')
|
torrentId=$(grep -r $fileName | head -n1 | awk -F '/' '{ print $1 }')
|
||||||
rm -rf $torrentId
|
rm -rf $torrentId
|
||||||
|
|
|
@ -9,7 +9,7 @@ from bs4 import BeautifulSoup
|
||||||
from .torrent import Torrent, TorrentComment, TorrentFile
|
from .torrent import Torrent, TorrentComment, TorrentFile
|
||||||
from .categories import categories
|
from .categories import categories
|
||||||
|
|
||||||
YGGTORRENT_TLD = "se"
|
YGGTORRENT_TLD = "si"
|
||||||
|
|
||||||
YGGTORRENT_BASE_URL = f"https://www2.yggtorrent.{YGGTORRENT_TLD}"
|
YGGTORRENT_BASE_URL = f"https://www2.yggtorrent.{YGGTORRENT_TLD}"
|
||||||
|
|
||||||
|
@ -44,7 +44,7 @@ YGGTORRENT_SEARCH_URL_DO = "&do="
|
||||||
YGGTORRENT_SEARCH_URL_PAGE = "&page="
|
YGGTORRENT_SEARCH_URL_PAGE = "&page="
|
||||||
|
|
||||||
YGGTORRENT_GET_FILES = f"{YGGTORRENT_BASE_URL}/engine/get_files?torrent="
|
YGGTORRENT_GET_FILES = f"{YGGTORRENT_BASE_URL}/engine/get_files?torrent="
|
||||||
YGGTORRENT_GET_INFO = f"https://www2.yggtorrentchg/engine/get_nfo?torrent="
|
YGGTORRENT_GET_INFO = f"{YGGTORRENT_BASE_URL}/engine/get_nfo?torrent="
|
||||||
|
|
||||||
YGGTORRENT_MOST_COMPLETED_URL = f"{YGGTORRENT_BASE_URL}/engine/mostcompleted"
|
YGGTORRENT_MOST_COMPLETED_URL = f"{YGGTORRENT_BASE_URL}/engine/mostcompleted"
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue