update-hib.py
changeset 11 dc1b075c538a
parent 10 d7e256c9aec9
child 12 9d5880ecdb82
--- a/update-hib.py	Wed Jan 22 19:37:59 2014 -0500
+++ b/update-hib.py	Sat Mar 29 00:13:35 2014 -0400
@@ -56,7 +56,7 @@
         self.attrs = set(chain.from_iterable(cleanup(attr) for attr in ids))
         urls = button.a.attrs
         logging.debug("URLS are %r", urls)
-        self.torrent = urls["data-bt"]
+        self.torrent = urls["data-bt"] if "data-bt" in urls.keys() else None
         self.web = urls["data-web"]
         details = soup.find(class_="dldetails").find(class_="dlsize")
         size = details.find(class_="mbs")
@@ -79,7 +79,7 @@
 
 class Downloads:
     def __init__(self, soup):
-        self.id = [class_ for class_ in soup["class"] if class_ != "downloads"][0]
+        self.id = [class_ for class_ in soup["class"] if class_ not in ("downloads","js-platform")][0]
         self.elements = []
         self.others = []
         self.addchilds(soup)
@@ -91,12 +91,18 @@
             classes = child["class"] if "class" in child.attrs else []
             if [True for attr in classes if attr in ("arc-toggle", "downloads")]:
                 self.addchilds(child)
-            elif "download" in classes:
-                desc = child.find(class_="flexbtn").span.string
-                if desc == "Stream":
-                    logging.info("Ignoring Stream URLs for %s", self.id)
-                else:
-                    self.elements.append(Download(self.id, child))
+            elif "download-buttons" in classes:
+                for subchild in child.children:
+                    if type(subchild) is not bs4.element.Tag:
+                        continue
+                    btn = subchild.find(class_="flexbtn")
+                    if not btn:
+                        continue
+                    desc = btn.span.string
+                    if desc == "Stream":
+                        logging.info("Ignoring Stream URLs for %s", self.id)
+                    else:
+                        self.elements.append(Download(self.id, subchild))
             elif [True for attr in classes if attr in ("clearfix","label")]:
                 pass
             else:
@@ -227,6 +233,8 @@
     selector = FileSelector()
     downloads = []
     import sys
+    import os
+    import urllib.parse
     with open("torrents.log", "w") as l:
         for game in parseGamesFromFile(fn):
             logging.info("Parsing game %s (%d downloads)", game.title, len(game.downloads))
@@ -234,13 +242,14 @@
                 scores = list(selector(dls))
                 choosen = selectHighestScore(scores)
                 for score, dl in scores:
-                    print("[%s] %2d | %-30s | %-15s | %-30s | %-15s | %s " % (
+                    print("[%s] %2d | %-30s | %-15s | %-30s | %-15s | %s <%s>" % (
                             "*" if dl in choosen else " ",
                             score,
                             game.title,
                             dls.id,
                             dl.date,
                             ", ".join(sorted(dl.attrs)),
+			    os.path.basename(urllib.parse.urlsplit(dl.torrent).path),
                             dl.torrent),
                           file=l)
                     if dl in choosen:
@@ -250,8 +259,6 @@
                 print("-" * 80, file=l)
 
     import urllib.request
-    import urllib.parse
-    import os
     urlfile = open('http-download.sh','w')
     opener = urllib.request.build_opener()
     for dl in (dl for dl in downloads):