diff options
| -rw-r--r-- | modules-dl.py | 69 |
1 files changed, 69 insertions, 0 deletions
diff --git a/modules-dl.py b/modules-dl.py new file mode 100644 index 0000000..b6ce2b1 --- /dev/null +++ b/modules-dl.py @@ -0,0 +1,69 @@ +#!/bin/python + +from bs4 import BeautifulSoup +import requests +import lxml +import re + +from tqdm import tqdm +import urllib.request + + +import sys + +if len(sys.argv) < 2: + print("Usage:") + print(" {} <user_id>".format(sys.argv[0])) + exit(1) + +user = sys.argv[1] + +class DownloadProgressBar(tqdm): + def update_to(self, b=1, bsize=1, tsize=None): + if tsize is not None: + self.total = tsize + self.update(b * bsize - self.n) + +def download_url(url, out_path): + with DownloadProgressBar(unit='B', unit_scale=True, + miniters=1, desc=url.split('/')[-1]) as t: + urllib.request.urlretrieve(url, filename=out_path, reporthook=t.update_to) + +def get_mods(url): + + r = requests.get(url) + + soup = BeautifulSoup(r.text, "lxml") + + search_download = re.compile(".*download.*") + search_page = re.compile(".*pagination.*") + + is_number = re.compile(".*[0-9].*") + + for link in soup.find_all('a', href=re.compile("download")): + link_href = link.get("href") + link_name_arr = link.get("href").split("#") + if len(link_name_arr) > 1: + print("Downloading {} module...".format(link_name_arr[1])) + link_name = link_name_arr[1] + download_url(link_href, link_name) + else: + print("Can't get get file name... :/") + + page_selected = "0" + go_to = "1" + pages = [] + for link in soup.find_all('a', class_=re.compile(".*pagination.*")): + if is_number.match(link.text) and "pagination-selected" in link.get("class"): + page_selected = link.text + if is_number.match(link.text) and "pagination" in link.get("class"): + if link.text not in pages: + pages.append(link.text) + + go_to = str(int(page_selected)+1) + if go_to in pages: + get_mods("https://modarchive.org/index.php?request=view_artist_modules&query=" + user+"&page="+ go_to +"#mods") + + +url = "https://modarchive.org/index.php?request=view_artist_modules&query=" + user+"&page=1#mods" +get_mods(url) |