Global big change

.gitignore

@@ -1,5 +1,7 @@
 data/files/*
 data/torrents/*
+data/nfo/*
 __pycache__/
 yggcrawl/__pycache__/
+yggcrawl/__init__.pyc
 login.py

crawl.py

@@ -1,3 +1,5 @@
+#!/usr/bin/python3
+
 ################################################################################
 # Author: Poka (poka@p2p.legal)
 # Version: 0.0.2
@@ -5,12 +7,18 @@
 # Git: https://git.p2p.legal/axiom-team/astroport
 ################################################################################
 
+# Config
+tkdelay = 3
+displayDetails = "true"
+
 import requests
 import json
 import sys
+import os
 import subprocess
 import login
 import time
+import re
 
 # Load scraper
 from yggcrawl import YggTorrentScraper
@@ -19,33 +27,32 @@ from yggtorrentscraper import set_yggtorrent_tld
 set_yggtorrent_tld("se")
 name = ' '.join(sys.argv[1:])
 
-# Give paramaters to scraper
+#research = os.system("./lib/scrabash.sh search --best=true " + name)
-parameters = {
+research = os.popen('./lib/scrabash.sh search --best=true ' + name).read()
-    "name": name,
+#research = run(["./lib/scrabash.sh", "search", "--best-true"])
-    "category": "films_&_videos",
-    "subcategory": "film",
-    "sort": "seed",
-    "order": "desc"
-}
 
-# Execute scrapping
+print(research)
-research = scraper.search(parameters)
 
-# Check if not found
+try:
-if research == []:
+    research.index("No torrent found")
-    print("Aucun torrent trouvé ayant pour nom \"" + name + "\"")
+except ValueError:
-    sys.exit()
+    True
+else:
+    sys.exit(1)
+if displayDetails == "true":
+    detail = subprocess.call(["lib/scrabash.sh", "get_details", research])
 
-# Options
+#torrentSize = 
-#most_completed = scraper.most_completed()
+
-#print(json.dumps(research, sort_keys=True, indent=4))
+diskSize = subprocess.call(["./trans-ctl.sh", "checkdisk"])
-#print(json.dumps(most_completed, sort_keys=True, indent=4))
+if diskSize >= 15:
+    print("You have filled the entire space allocated to torrents: " + diskSize)
+    sys.exit("Disk full")
 
 # Download torrent file
 if(scraper.login(login.user, login.passwd)):
     print("Login success")
-    first_torrent = research[0]
+    scraper.download_from_torrent_url(research)
-    scraper.download_from_torrent_url(first_torrent)
 else:
     print("Login failed")
 
@@ -53,8 +60,8 @@ else:
 
 print("Start to remove tracker ...")
 time.sleep(1)
-print("Wait 10 seconds to get peers before remove tracker...")
+print("Wait " + str(tkdelay) + " seconds to get peers before remove tracker...")
-time.sleep(10)
+time.sleep(tkdelay)
 subprocess.call(["./trans-ctl.sh", "rmtracker", name])
 
 print("Done")

install.sh

@@ -33,6 +33,13 @@ transmisison() {
 	sudo apt install transmission-daemon --install-suggests
 }
 
+# Install pip tools
+piptools() {
+	sudo apt install python3-pip
+	pip3 install --user pyped
+	export PATH=$PATH:/home/iptubes/.local/bin
+}
+
 torrengo() {
         ## Install GO
         wget https://dl.google.com/go/go1.13.4.linux-amd64.tar.gz -P /tmp/

lib/py/scrapactions.py

@@ -0,0 +1,83 @@
+#!/usr/bin/python3
+# coding: utf-8
+
+import requests
+import json
+import sys
+import subprocess
+import login
+import time
+import re
+#import pyped
+
+# Load scraper
+try:
+    scraper
+except NameError:
+    from yggcrawl import YggTorrentScraper
+    scraper = YggTorrentScraper(requests.session())
+    from yggtorrentscraper import set_yggtorrent_tld
+    set_yggtorrent_tld("se")
+
+cmd = sys.argv[1]
+name = ' '.join(sys.argv[2:])
+subcat = ""
+
+def args():
+    parameter_dict = {}
+    for user_input in sys.argv[1:]:
+        if "=" not in user_input:
+            continue
+        else:
+            global name
+            name = name.replace(user_input, '')
+
+        varname = user_input.split("=")[0]
+        varvalue = user_input.split("=")[1]
+        parameter_dict[varname] = varvalue
+
+    if "--subcat" in parameter_dict:
+        global subcat
+        subcat = parameter_dict["--subcat"]
+
+args()
+
+# Search a torrent
+def search(name=name):
+    # Give paramaters to scraper
+    parameters = {
+        "name": name,
+        "category": "films_&_videos",
+        "subcategory": subcat,
+        "sort": "seed",
+        "order": "desc"
+    }
+
+    # Execute scrapping
+    research = scraper.search(parameters)
+
+    # Check if not found
+    if research == []:
+        print("No torrent found with name: " + name)
+        sys.exit()
+    else:
+        print(json.dumps(research, indent=2))
+
+# Get details of torrent
+def get_details():
+    if(scraper.login(login.user, login.passwd)):
+        print("Login success")
+        # first_torrent = research[0]
+        first_torrent = name
+        details = scraper.extract_details(first_torrent)
+        print(details.__str__(files=True))
+#        re.findall('merci', details)
+        #    subprocess.call(['/bin/grep', 'merci', ])
+        #    print(details.__str__(files=True, comments=True))
+    else:
+        print("Login failed")
+
+if cmd == "search":
+    search()
+elif cmd == "get_details":
+    get_details()

lib/py/yggcrawl

@@ -0,0 +1 @@
+../../yggcrawl/

lib/scrabash.sh

@@ -0,0 +1,47 @@
+#!/bin/bash
+
+name="${@:2}"
+nameR=$(echo $name | sed -e 's/[^ ]*=[^ ]*//ig')
+
+## Get options
+args="${@:2}"
+[[ $args =~ "--best=true" ]] && best=true
+[[ $args =~ "--print=true" ]] && print=true
+
+search() {
+	result=$(./lib/py/scrapactions.py $1 $name | jq .[] 2>&1 | tr -d '"')
+	[[ $result =~ "parse error: Invalid " ]] && echo "$nameR: No torrent found" && err=1 && exit 1
+	[[ $best == true ]] && echo "$result" | head -n1 || echo "$result"
+}
+
+get_details() {
+	result=$(./lib/py/scrapactions.py $1 $name | grep -v "Login success")
+#	echo "$result"
+
+	idTorrent=$(echo "$name" | awk -F '/' '{print $NF}' | awk -F '-' '{ print $1 }')
+	if [[ $print == true ]]; then
+		[[ ! -d data/nfo/$idTorrent ]] && mkdir data/nfo/$idTorrent
+		echo "$result" > ./data/nfo/$idTorrent/info.txt
+
+		torrentName=$(echo "$result" | grep -w "Name      :" | awk -F ": " '{ print $2 }')
+		torrentUrl=$(echo "$result" | grep -w "Url       :" | awk -F ": " '{ print $2 }')
+		torrentSize=$(echo "$result" | grep -w "Size      :" | awk -F ": " '{ print $2 }')
+		if [[ $torrentSize =~ "Go" ]]; then
+			unity="Go"
+		else
+			unity="Mo"
+		fi
+		torrentSize=$(echo "$torrentSize" | tr -d 'Go')
+		torrentSeeders=$(echo "$result" | grep -w "Seeders   :" | awk -F ": " '{ print $2 }')
+
+		echo "Name = '$torrentName'
+Url = '$torrentUrl'
+Size = $torrentSize
+Unity = '$unity'
+Seeders = $torrentSeeders" > ./data/nfo/$idTorrent/db.py
+	fi
+}
+
+$1 $1
+
+[[ $err == 1 ]] && exit 1 || exit 0

trans-ctl.sh

@@ -1,8 +1,13 @@
 #!/bin/bash
+
+# Config
+diskLimit="15"
+
 [[ -z $transUser ]] && transUser=$(grep "transUser" login.py | awk -F '=' '{ print $2 }' | tr -d '"' | tr -d ' ')
 [[ -z $transPwd ]] && transPwd=$(grep "transPwd" login.py | awk -F '=' '{ print $2 }' | tr -d '"' | tr -d ' ')
 
 transcmd="transmission-remote --auth $transUser:$transPwd"
+args="$@"
 
 [[ -z $1 ]] && echo "Please choose an option:
 - list
@@ -19,8 +24,8 @@ getid() {
 		else
 			j=0
 			for i in "${@:2}"; do
-				[[ $j == 0 ]] && result+=$($transcmd --list | grep -vE 'Sum:|ID     Done' | grep -i "$i")
+				[[ $j == 0 ]] && result=$($transcmd --list | grep -vE 'Sum:|ID     Done' | grep -i "$i")
-				result=$(echo "$result" | grep -vE 'Sum:|ID     Done' | grep -i "$i")
+				result=$(echo "$result" | grep -vE 'Sum:|ID     Done' | grep -iw "$i")
 				((j++))
 			done
 		fi
@@ -31,6 +36,12 @@ getid() {
 		fi
 }
 
+checkdisk() {
+	diskSize=$(du -hs data/files/ | awk '{ print $1 }')
+	[[ $diskSize =~ "G" ]] && diskSize=$(echo $diskSize | tr -d 'G') || return 0
+	[[ "$diskSize" -ge "$diskLimit" ]] && echo "You have reached the maximum space allocated to torrents: $diskSize"
+}
+
 case "$1" in
 	list)
 		$transcmd --list
@@ -40,7 +51,7 @@ case "$1" in
 			echo "Please precise an torrentID or title of the torrent you want to anonymize"
 		else
 			if [[ ! "$2" =~ ^[+-]?[0-9]+([.][0-9]+)?$ ]]; then
-				idt=$(getid "$@")
+				idt=$(getid "$args" | tr -d '*')
 			else
 				idt=$2
 			fi
@@ -52,14 +63,21 @@ case "$1" in
 		fi
 		;;
 	getid)
-		getid "$@"
+		getid "$args"
 		;;
 	remove)
-		idt=$(getid $1)
+		idt=$(getid "$args" | tr -d '*')
-		$transcmd -t $idt --remove-and-delete > /dev/null && echo "Torrent $idt removed." || echo "Failed to remove $1"
+		if [[ $idt =~ ^[+-]?[0-9]+([.][0-9]+)?$ ]]; then
+			$transcmd -t $idt --remove-and-delete > /dev/null && echo "Torrent $idt removed." || echo "Failed to remove $1"
+		else
+			echo "$idt"
+		fi
+		;;
+	checkdisk)
+		checkdisk
 		;;
 	*)
-		cmd="$@"
+		cmd="$args"
 		$transcmd "$cmd"
 		;;
 esac