diff --git a/movie_list/index.py b/movie_list/index.py index 002991d..1d26fe3 100755 --- a/movie_list/index.py +++ b/movie_list/index.py @@ -12,9 +12,9 @@ import sys import time import threading from pathlib import Path -from http.client import IncompleteRead import progressbar from imdb import IMDb +from imdb._exceptions import IMDbParserError class MovieList: @@ -59,27 +59,35 @@ class MovieList: # Scan IMDb for a given movie and append it to the html # This collects rating, genres, official name and a hyperlink imdb = IMDb() - save_stdout = sys.stdout - with open(os.devnull, 'wb') as sys.stdout: - while True: - try: - query = imdb.search_movie(f'{arg["title"]} {arg["year"]}') - break - except IncompleteRead: - pass - sys.stdout = save_stdout + while True: + try: + query = imdb.search_movie(f'{arg["title"]} {arg["year"]}') + break + except IMDbParserError as exc: + query = [] + #print(exc) + break + except Exception as exc: + #print(f'error: {exc.__class__.__name__}: {arg["title"]}') + time.sleep(10) movie = None for entry in query: - has_minimum_keys = True - for key in ['kind', 'year', 'title']: - if key not in entry.keys(): - has_minimum_keys = False - if not has_minimum_keys: + #print(entry) + imdb.update(entry) + # in case any of these keys is missing in the query, continue + if not all(key in entry.keys() for key in ['kind', 'year', 'title']): + #print(f'missing key {entry.keys()}') + continue + if arg['status'] == 'DONE' and 'rating' not in entry.keys(): continue # Try to eliminate episode results - if [i for i in entry.keys() if 'episode' in i.lower()] or \ - 'episode' in entry['title'].lower(): + # Must not have "episode" in the object keys + # Must not have "episode" in the query title key, + # unless "episode" is in the query search string + if [i for i in entry.keys() if 'episode' in i.lower()] or ( + 'episode' in entry['title'].lower() and \ + 'episode' not in arg['title'].lower()): continue if entry['kind'].lower() == arg['kind'].lower(): movie = entry @@ -91,7 +99,6 @@ class MovieList: 'year': arg['year'], 'dummy': None } - if 'genres' not in movie.keys(): movie['genres'] = ['N/A'] if 'rating' not in movie.keys(): @@ -101,7 +108,7 @@ class MovieList: f'{movie["title"]}' self.html_table[index] = ( f'\n{" "*8}{index + 1}' - f'{html_title_td}' + f'{html_title_td}' f'{movie["year"]}{movie["rating"]}' f'{", ".join(movie["genres"])}' f'{arg["status"]}' @@ -124,7 +131,7 @@ class MovieList: for raw_line in mlist_raw.splitlines(): self.movie_list.update({ len(self.movie_list): { - 'title': raw_line.split('(', 1)[0].strip(), + 'title': raw_line[0:next((i for i, ch in enumerate(raw_line) if ch in {'<', '('}), None) - 1], 'kind': raw_line[raw_line.find('<')+1:raw_line.rfind('>')+1].strip('<>') or 'movie', 'year': raw_line[raw_line.find('(')+1:raw_line.find(')')], 'status': raw_line[raw_line.find('[')+1:raw_line.find(']')], @@ -135,28 +142,46 @@ class MovieList: # Progress bar. Enough said pbar = progressbar.ProgressBar(max_value=len(self.movie_list)) for idx, movie in self.movie_list.items(): - match = [html_row for html_row in self.prev_html if movie['title'] in html_row] + match = [html_row for html_row in self.prev_html if movie['title'] in html_row and 'N/A' not in html_row] if match: # Update movies as DONE in case of change match = match[0].replace('*', movie['status']) # Directly insert the current HTML line from the older output self.html_table[idx] = \ f'\n{" "*8}{idx + 1}{match[match.find("") + 5:]}' - pbar.update(idx + 1) + pbar.increment() else: thread = threading.Thread(target=self._worker, args=(movie, idx)) self.threads.append(thread) - thread.start() - pbar.update(idx+1) - time.sleep(0.2) - if len(self.threads) % 16 == 0: - time.sleep(6) - for thread in self.threads: - thread.join() + max_threads = 10 + while self.threads: + threads_alive = self.get_alive_threads() + threads_to_be_started = [i for i in self.threads if i not in threads_alive] + for idx in range(max_threads if max_threads < len(threads_to_be_started) else len(threads_to_be_started)): + threads_to_be_started[idx].start() + pbar.increment() + time.sleep(1) + time.sleep(1) + self.delete_finished_threads() + self.html += ''.join(self.html_table) return True + def delete_finished_threads(self): + for idx, thread in enumerate(self.threads): + if not thread.is_alive() and thread._started.is_set(): + thread.join() + self.threads[idx] = None + self.threads = list(filter(lambda a: a is not None, self.threads)) + + def get_alive_threads(self): + threads = [] + for thread in self.threads: + if thread.is_alive() or thread._started.is_set(): + threads.append(thread) + return threads + def write(self, dst=None): """ Write the HTML list to index.html """ out_path = dst if dst else self.dst diff --git a/movie_list/movie_list b/movie_list/movie_list index 158324d..d139ae5 100644 --- a/movie_list/movie_list +++ b/movie_list/movie_list @@ -18,7 +18,7 @@ Mary and Max (2009) [DONE] Lucky Number Slevin (2006) [DONE] The Road (2009) [DONE] Magnolia (1999) [DONE] -Il Mostro (1994) [DONE] +The Monster (1994) [DONE] The Tiger and the Snow (2005) [DONE] Lucy (2014) [DONE] End of Watch (2012) [DONE] @@ -115,7 +115,7 @@ Star Wars: Episode III - Revenge of the Sith (2005) [DONE] Star Wars: Episode IV - A New Hope (1977) [DONE] Star Wars: Episode V - The Empire Strikes Back (1980) [DONE] Star Wars: Episode VI - Return of the Jedi (1983) [DONE] -Star Wars: Episode: The Force Awakens (2015) [DONE] +Star Wars: Episode VII: The Force Awakens (2015) [DONE] Rogue One: A Star Wars Story (2016) [DONE] Men in Black (1997) [DONE] Men in Black II (2002) [DONE] @@ -160,7 +160,7 @@ Unforgiven (1992) [DONE] Scarface (1983) [DONE] Die Hard (1988) [DONE] Die Hard 2 (1990) [DONE] -Die Hard: With a Vengeance (1995) [DONE] +Die Hard with a Vengeance (1995) [DONE] Live Free or Die Hard (2007) [DONE] A Good Day to Die Hard (2013) [DONE] Die Another Day (2002) [DONE] @@ -184,13 +184,13 @@ Pan (2015) [DONE] The Cobbler (2014) [DONE] The Conjuring (2013) [DONE] Starred Up (2013) [DONE] -Kraftidioten (2014) [DONE] +In Order of Disappearance (2014) [DONE] The Imitation Game (2014) [DONE] Begin Again (2013) [DONE] A Walk Among the Tombstones (2014) [DONE] Detachment (2011) [DONE] Identity (2003) [DONE] -Turist (Force Majeure) (2014) [DONE] +Force Majeure (2014) [DONE] Gone Girl (2014) [DONE] The Equalizer (2014) [DONE] The Equalizer 2 (2018) [DONE] @@ -351,7 +351,6 @@ Togo (2019) [*] Knives Out (2019) [DONE] Children of Men (2006) [DONE] Bogowie (2014) [DONE] -Spiral (2021) [*] Ip Man 4 (2019) [DONE] The Collini Case (2019) [DONE] The Traitor (2019) [DONE] @@ -397,7 +396,7 @@ Arrival (2016) [DONE] Duel (1971) [DONE] The Courier (2020) [DONE] The Bone Collector (1999) [DONE] -Oslo (2021) [DONE] +Oslo (2021) [DONE] Wrath of Man (2021) [DONE] Nobody (2021) [DONE] Голата истина за група Жигули (2021) [DONE] @@ -411,3 +410,5 @@ Boss Level (2020) [DONE] Boite Noire (2021) [DONE] Fresh (2022) [DONE] Death on the Nile (2022) [DONE] +All Quiet on the Western Front (2022) [DONE] +Triangle of Sadness (2022) [DONE]