--- a/update-hib.py Wed Jan 22 19:37:59 2014 -0500
+++ b/update-hib.py Sat Mar 29 00:13:35 2014 -0400
@@ -56,7 +56,7 @@
self.attrs = set(chain.from_iterable(cleanup(attr) for attr in ids))
urls = button.a.attrs
logging.debug("URLS are %r", urls)
- self.torrent = urls["data-bt"]
+ self.torrent = urls["data-bt"] if "data-bt" in urls.keys() else None
self.web = urls["data-web"]
details = soup.find(class_="dldetails").find(class_="dlsize")
size = details.find(class_="mbs")
@@ -79,7 +79,7 @@
class Downloads:
def __init__(self, soup):
- self.id = [class_ for class_ in soup["class"] if class_ != "downloads"][0]
+ self.id = [class_ for class_ in soup["class"] if class_ not in ("downloads","js-platform")][0]
self.elements = []
self.others = []
self.addchilds(soup)
@@ -91,12 +91,18 @@
classes = child["class"] if "class" in child.attrs else []
if [True for attr in classes if attr in ("arc-toggle", "downloads")]:
self.addchilds(child)
- elif "download" in classes:
- desc = child.find(class_="flexbtn").span.string
- if desc == "Stream":
- logging.info("Ignoring Stream URLs for %s", self.id)
- else:
- self.elements.append(Download(self.id, child))
+ elif "download-buttons" in classes:
+ for subchild in child.children:
+ if type(subchild) is not bs4.element.Tag:
+ continue
+ btn = subchild.find(class_="flexbtn")
+ if not btn:
+ continue
+ desc = btn.span.string
+ if desc == "Stream":
+ logging.info("Ignoring Stream URLs for %s", self.id)
+ else:
+ self.elements.append(Download(self.id, subchild))
elif [True for attr in classes if attr in ("clearfix","label")]:
pass
else:
@@ -227,6 +233,8 @@
selector = FileSelector()
downloads = []
import sys
+ import os
+ import urllib.parse
with open("torrents.log", "w") as l:
for game in parseGamesFromFile(fn):
logging.info("Parsing game %s (%d downloads)", game.title, len(game.downloads))
@@ -234,13 +242,14 @@
scores = list(selector(dls))
choosen = selectHighestScore(scores)
for score, dl in scores:
- print("[%s] %2d | %-30s | %-15s | %-30s | %-15s | %s " % (
+ print("[%s] %2d | %-30s | %-15s | %-30s | %-15s | %s <%s>" % (
"*" if dl in choosen else " ",
score,
game.title,
dls.id,
dl.date,
", ".join(sorted(dl.attrs)),
+ os.path.basename(urllib.parse.urlsplit(dl.torrent).path),
dl.torrent),
file=l)
if dl in choosen:
@@ -250,8 +259,6 @@
print("-" * 80, file=l)
import urllib.request
- import urllib.parse
- import os
urlfile = open('http-download.sh','w')
opener = urllib.request.build_opener()
for dl in (dl for dl in downloads):