update-hib.py
changeset 4 e102d2bb7a9e
parent 2 3675dd7daf59
child 5 b6a3b0987bfc
equal deleted inserted replaced
3:478647351440 4:e102d2bb7a9e
    18 
    18 
    19 
    19 
    20 from html.parser import HTMLParser
    20 from html.parser import HTMLParser
    21 from pprint import pprint
    21 from pprint import pprint
    22 import xml.dom
    22 import xml.dom
    23 from itertools import chain
    23 from itertools import chain, groupby
       
    24 import logging
       
    25 import operator
    24 
    26 
    25 class Node:
    27 class Node:
    26     def __init__(self, **args):
    28     def __init__(self, **args):
    27         self.childs = []
    29         self.childs = []
    28         self.attrs = {}
    30         self.attrs = {}
   237 
   239 
   238 class FileSelector:    
   240 class FileSelector:    
   239     def scoreDownload(self, dl):
   241     def scoreDownload(self, dl):
   240         if dl.dltype == "audio":
   242         if dl.dltype == "audio":
   241             if "FLAC" in dl.attrs:
   243             if "FLAC" in dl.attrs:
   242                 return 2
   244                 return 1
       
   245             if "OGG" in dl.attrs:
       
   246                 return 1 
   243             if "MP3" in dl.attrs:
   247             if "MP3" in dl.attrs:
   244                 return 1
   248                 return 1 
   245             if "website" in dl.attrs:
   249             if "website" in dl.attrs:
   246                 return -1
   250                 return -1
   247             raise Exception("Unknow audio type: %r" % (dl.attrs))
   251             raise Exception("Unknow audio type: %r" % (dl.attrs))
   248         if dl.dltype in ("mac","windows"):
   252         if dl.dltype in ("mac","windows"):
   249             return -1
   253             return -1
   250         if dl.dltype == "linux":
   254         if dl.dltype == "linux":
   251             score = 1
   255             score = 1
   252             if "x64" in dl.attrs:
   256             if "x64" in dl.attrs:
   253                 score += 1
   257                 score += 2
   254             if "deb" in dl.attrs:
   258             if "deb" in dl.attrs:
   255                 score += 1
   259                 score += 1
   256             return score
   260             return score
   257         if dl.dltype == "android":
   261         if dl.dltype == "android":
   258             return -1
   262             return -1
   262         return sorted(((self.scoreDownload(dl),dl) for dl in dls), key=lambda x: x[0], reverse=True)
   266         return sorted(((self.scoreDownload(dl),dl) for dl in dls), key=lambda x: x[0], reverse=True)
   263 
   267 
   264     def __call__(self, dls):
   268     def __call__(self, dls):
   265         return self.chooseDownloads(dls)
   269         return self.chooseDownloads(dls)
   266 
   270 
       
   271 def selectHighestScore(scores):
       
   272     if scores:
       
   273         get_first = operator.itemgetter(0)
       
   274         score, dls = next(groupby(sorted(scores, key = get_first, reverse=True), get_first))
       
   275         if score > 0:
       
   276             return list(dl for s, dl in dls)
       
   277         else:
       
   278             return []
       
   279     logging.debug("Empty scores list: %r", scores)
       
   280     return []
       
   281 
   267 def main(fn):
   282 def main(fn):
   268     selector = FileSelector()
   283     selector = FileSelector()
   269     downloads = []
   284     downloads = []
   270     for game in parseGamesFromFile(fn):
   285     for game in parseGamesFromFile(fn):
   271         for dls in game.downloads:
   286         for dls in game.downloads:
   272             scores = selector(dls)
   287             scores = list(selector(dls))
   273             choosen = list(dl for score, dl in scores if score >= 0)[:1]
   288             choosen = selectHighestScore(scores)
   274             for score, dl in scores:
   289             for score, dl in scores:
   275                 print("[%s] %2d | %-20s | %-10s | %-25s | %s " % (
   290                 print("[%s] %2d | %-20s | %-10s | %-25s | %s " % (
   276                         "*" if dl in choosen else " ",
   291                         "*" if dl in choosen else " ",
   277                         score,
   292                         score,
   278                         game.title, 
   293                         game.title, 
   287 
   302 
   288     import urllib.request
   303     import urllib.request
   289     import urllib.parse
   304     import urllib.parse
   290     import os
   305     import os
   291     opener = urllib.request.build_opener()
   306     opener = urllib.request.build_opener()
   292     for dl in downloads:
   307     for dl in (dl for dl in downloads):
   293         fn = os.path.basename(urllib.parse.urlsplit(dl.torrent).path)
   308         if dl.torrent:
   294         print("Saving",dl.torrent,"as",fn)
   309             try:
   295         with opener.open(dl.torrent) as u:
   310                 fn = os.path.basename(urllib.parse.urlsplit(dl.torrent).path)
   296             with open(fn,"wb") as f:
   311                 if os.path.exists(fn):
   297                 f.write(u.read())
   312                     logging.info("Skipping existing torrent %s", fn)
       
   313                 else:
       
   314                     logging.info("Saving %s as %s", dl.torrent, fn)
       
   315                     with opener.open(dl.torrent) as u:
       
   316                         with open(fn,"wb") as f:
       
   317                             f.write(u.read())
       
   318             except:
       
   319                 logging.exception("Error with download %r", dl)
       
   320         else:
       
   321             logging.info("No torrent, url is %s", dl.web)
   298 
   322 
   299 
   323 
   300 if __name__ == '__main__':
   324 if __name__ == '__main__':
   301     import sys
   325     import sys
       
   326     logging.getLogger().setLevel(logging.INFO)
   302     main(sys.argv[1])
   327     main(sys.argv[1])