diff -r d7e256c9aec9 -r dc1b075c538a update-hib.py --- a/update-hib.py Wed Jan 22 19:37:59 2014 -0500 +++ b/update-hib.py Sat Mar 29 00:13:35 2014 -0400 @@ -56,7 +56,7 @@ self.attrs = set(chain.from_iterable(cleanup(attr) for attr in ids)) urls = button.a.attrs logging.debug("URLS are %r", urls) - self.torrent = urls["data-bt"] + self.torrent = urls["data-bt"] if "data-bt" in urls.keys() else None self.web = urls["data-web"] details = soup.find(class_="dldetails").find(class_="dlsize") size = details.find(class_="mbs") @@ -79,7 +79,7 @@ class Downloads: def __init__(self, soup): - self.id = [class_ for class_ in soup["class"] if class_ != "downloads"][0] + self.id = [class_ for class_ in soup["class"] if class_ not in ("downloads","js-platform")][0] self.elements = [] self.others = [] self.addchilds(soup) @@ -91,12 +91,18 @@ classes = child["class"] if "class" in child.attrs else [] if [True for attr in classes if attr in ("arc-toggle", "downloads")]: self.addchilds(child) - elif "download" in classes: - desc = child.find(class_="flexbtn").span.string - if desc == "Stream": - logging.info("Ignoring Stream URLs for %s", self.id) - else: - self.elements.append(Download(self.id, child)) + elif "download-buttons" in classes: + for subchild in child.children: + if type(subchild) is not bs4.element.Tag: + continue + btn = subchild.find(class_="flexbtn") + if not btn: + continue + desc = btn.span.string + if desc == "Stream": + logging.info("Ignoring Stream URLs for %s", self.id) + else: + self.elements.append(Download(self.id, subchild)) elif [True for attr in classes if attr in ("clearfix","label")]: pass else: @@ -227,6 +233,8 @@ selector = FileSelector() downloads = [] import sys + import os + import urllib.parse with open("torrents.log", "w") as l: for game in parseGamesFromFile(fn): logging.info("Parsing game %s (%d downloads)", game.title, len(game.downloads)) @@ -234,13 +242,14 @@ scores = list(selector(dls)) choosen = selectHighestScore(scores) for score, dl in scores: - print("[%s] %2d | %-30s | %-15s | %-30s | %-15s | %s " % ( + print("[%s] %2d | %-30s | %-15s | %-30s | %-15s | %s <%s>" % ( "*" if dl in choosen else " ", score, game.title, dls.id, dl.date, ", ".join(sorted(dl.attrs)), + os.path.basename(urllib.parse.urlsplit(dl.torrent).path), dl.torrent), file=l) if dl in choosen: @@ -250,8 +259,6 @@ print("-" * 80, file=l) import urllib.request - import urllib.parse - import os urlfile = open('http-download.sh','w') opener = urllib.request.build_opener() for dl in (dl for dl in downloads):