diff --git a/movie_list/gen_movie_list.py b/movie_list/gen_movie_list.py deleted file mode 100755 index 966d721..0000000 --- a/movie_list/gen_movie_list.py +++ /dev/null @@ -1,571 +0,0 @@ -#!/usr/bin/env python - -""" -1. Import a movie_list txt file -2. Query IMDb for each entry, retrieving actual movie name, rating and genres -3. Generate an HTML table from the IMDb data -4. Store the HTML in index.html -""" - -import os -import sys -import time -import threading -from pathlib import Path -import progressbar -from imdb import IMDb -from imdb._exceptions import IMDbParserError, IMDbDataAccessError - - -class MovieList: - """ Class to generate a movie list HTML table """ - def __init__(self, src=None, dst=None): - self.prev_html = [] - self.html = """ - - - - - My Movie List - - - - - - - - -
-

🎬 My Movie Collection

- - - - - - - - - - - - """ - self.src = src - self.dst = Path(dst) if dst else Path(os.path.dirname(sys.argv[0])) / 'index.html' - self.movie_list = [] - self.threads = [] - self.read_prev_output() - self.html_table = None - - def _worker(self, arg, index): - # Scan IMDb for a given movie and append it to the html - # This collects rating, genres, official name and a hyperlink - imdb = IMDb() - first_run = True - while True: - if not first_run: - time.sleep(10) - else: - first_run = False - try: - query = imdb.search_movie(f'{arg["title"]} {arg["year"]}') - break - except IMDbDataAccessError as imdb_data_exc: - exc = str(imdb_data_exc) - if '503' in exc: - sys.stderr.write('503 - Service Unavailable, retrying...') - elif '403' in exc: - sys.stderr.write('403 - Forbidden, retrying...\n') - query = [] - time.sleep(10) - except IMDbParserError as imdb_parser_exc: - query = [] - break - except Exception as exc: - time.sleep(10) - - movie = None - for entry in query: - try: - imdb.update(entry) - except Exception as e: - sys.stderr.write('update err') - # in case any of these keys is missing in the query, continue - if not all(key in entry.keys() for key in ['kind', 'year', 'title']): - continue - if arg['status'] == 'DONE' and 'rating' not in entry.keys(): - continue - # Try to eliminate episode results - if [i for i in entry.keys() if 'episode' in i.lower()] or ( - 'episode' in entry['title'].lower() and \ - 'episode' not in arg['title'].lower()): - continue - if entry['kind'].lower() == arg['kind'].lower(): - movie = entry - break - if not movie: - movie = { - 'title': arg['title'], - 'kind': arg['kind'], - 'year': arg['year'], - 'dummy': None - } - if 'genres' not in movie.keys(): - movie['genres'] = ['N/A'] - if 'rating' not in movie.keys(): - movie['rating'] = 'N/A' - - html_title_td = movie['title'] if 'dummy' in movie.keys() else \ - f'{movie["title"]}' - - self.html_table[index] = ( - f'\n ' - f'' - f'' - f'' - f'' - f'' - f'' - f'' - ) - - def gen(self): - """ Generate an HTML list based on input, using a threaded worker """ - if not self.src: - self.src = Path(os.path.dirname(sys.argv[0])) / 'movie_list' - else: - self.src = Path(self.src) - if not self.src.exists(): - sys.stderr.write(f'error: input does not exist - {self.src}\n') - return False - - self.movie_list = {} - seen_titles = set() # Track unique titles - - # Open the movie list & split the columns - with open(self.src, 'r', encoding='utf-8') as fp_handle: - mlist_raw = fp_handle.read() - idx = 0 - for raw_line in mlist_raw.splitlines(): - # In case the line is empty - if not raw_line: - continue - - title = raw_line[0:next((i for i, ch in enumerate(raw_line) if ch in {'<', '('}), None) - 1] - - # Skip if we've already seen this title - if title in seen_titles: - continue - - seen_titles.add(title) - self.movie_list.update({ - idx: { - 'title': title, - 'kind': raw_line[raw_line.find('<')+1:raw_line.rfind('>')+1].strip('<>') or 'movie', - 'year': raw_line[raw_line.find('(')+1:raw_line.find(')')], - 'status': raw_line[raw_line.find('[')+1:raw_line.find(']')], - } - }) - idx += 1 - - self.html_table = [None] * len(self.movie_list) - - # Progress bar - pbar = progressbar.ProgressBar(max_value=len(self.movie_list)) - for idx, movie in self.movie_list.items(): - # More precise matching - look for the hidden

tag with exact title - match = [html_row for html_row in self.prev_html - if f'

' in html_row - and 'N/A' not in html_row] - if match: - # Update the index and status from the cached row - match_str = match[0] - # Replace the status (* -> DONE or vice versa) - match_str = match_str.replace('*', movie['status']).replace('DONE', movie['status']) - # Update the index number - if '', 1)[1] if '' in match_str else match_str - self.html_table[idx] = f'\n {after_index}' - else: - self.html_table[idx] = match_str - pbar.increment() - else: - thread = threading.Thread(target=self._worker, args=(movie, idx)) - self.threads.append(thread) - - max_threads = 10 - while self.threads: - threads_alive = self.get_alive_threads() - threads_to_be_started = [i for i in self.threads if i not in threads_alive] - for idx in range(max_threads if max_threads < len(threads_to_be_started) else len(threads_to_be_started)): - threads_to_be_started[idx].start() - pbar.increment() - time.sleep(2) - time.sleep(2) - self.delete_finished_threads() - - # Don't append to self.html here - we'll do it in write() - return True - - def delete_finished_threads(self): - for idx, thread in enumerate(self.threads): - if not thread.is_alive() and thread._started.is_set(): - thread.join() - self.threads[idx] = None - self.threads = list(filter(lambda a: a is not None, self.threads)) - - def get_alive_threads(self): - threads = [] - for thread in self.threads: - if thread.is_alive() or thread._started.is_set(): - threads.append(thread) - return threads - - def write(self, dst=None): - """ Write the HTML list to index.html """ - out_path = dst if dst else self.dst - timestamp = time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime()) - - # Build the final HTML - don't append, rebuild from scratch - final_html = self.html.split('')[0] + '' - final_html += ''.join([row for row in self.html_table if row is not None]) - final_html += f''' - -
#TitleYearRatingGenreStatus
{index + 1}{html_title_td}{movie["year"]}{movie["rating"]}{", ".join(movie["genres"])}{arg["status"]}
' in match_str: - # Extract everything after the index cell - after_index = match_str.split('
{idx + 1}
-
-
Generated {timestamp} UTC
- -''' - - with open(out_path, 'wb') as fp_handle: - fp_handle.write(final_html.encode('utf8')) - - def read_prev_output(self): - """ Import a previous HTML table """ - if self.dst.exists(): - with open(self.dst, 'rb') as fp_handle: - self.prev_html = fp_handle.read().decode('utf8').split('\n') - - def deduplicate_html(self): - """ Remove duplicate entries from html_table based on movie titles """ - seen_titles = set() - deduplicated = [] - - for idx, row in enumerate(self.html_table): - if row is None: - continue - - # Extract the hidden title from the row - if '' in row: - start = row.find('', start) - title = row[start:end] - - if title not in seen_titles: - seen_titles.add(title) - deduplicated.append(row) - else: - # Skip duplicate - continue - else: - # If we can't find the hidden title, keep the row anyway - deduplicated.append(row) - - # Update html_table with deduplicated content - self.html_table = deduplicated - return len(self.html_table) - - -def main(): - """ Default run """ - src = dst = None - if len(sys.argv) > 3: - sys.stderr.write(f'error: max 2 variables, {len(sys.argv)-1} given!\n') - sys.exit(1) - - if len(sys.argv) > 1: - src = sys.argv[1] - if len(sys.argv) == 3: - dst = sys.argv[2] - - mlist = MovieList(src=src, dst=dst) - if mlist.gen(): - mlist.write(dst=dst) - - -if __name__ == "__main__": - main() diff --git a/movie_list/movie_list b/movie_list/movie_list index b52d911..3d89b18 100644 --- a/movie_list/movie_list +++ b/movie_list/movie_list @@ -467,3 +467,10 @@ The French Dispatch (2021) [DONE] Sentimental Value (2025) [DONE] 28 Years Later: The Bone Temple (2026) [*] Youth (2015) [DONE] +Dune: Part One (2021) [DONE] +Dune: Part Two (2024) [DONE] +Dune: Part Three (2026) [*] +The Swedish Connection (2026) [DONE] +Atlas of the Universe (2026) [*] +I Swear (2025) [DONE] +The Odyssey (2026) [DONE] diff --git a/movie_list/movie_list_omdb.py b/movie_list/movie_list_omdb.py new file mode 100755 index 0000000..4f33872 --- /dev/null +++ b/movie_list/movie_list_omdb.py @@ -0,0 +1,645 @@ +#!/usr/bin/env python + +""" +1. Import a movie_list txt file +2. Query OMDb (the Open Movie Database, powered by real IMDb data) for each + entry, retrieving the actual movie title, year, IMDb rating and genres +3. Generate an HTML table from the OMDb data +4. Store the HTML in index.html + +Requires: pip install requests progressbar2 + +You need a free OMDb API key: + 1. Go to https://www.omdbapi.com/apikey.aspx + 2. Choose the free "1,000 daily limit" option and register with your email + 3. OMDb emails you a key (check spam folder if it doesn't show up quickly) + 4. Set it as an environment variable before running this script: + export OMDB_API_KEY=your_key_here # macOS/Linux + setx OMDB_API_KEY your_key_here # Windows (new shells) + +The free tier is limited to 1,000 requests/day. Because this script only +fetches entries that aren't already cached in the previous index.html (see +read_prev_output/gen), you'll generally only "spend" requests on movies that +are new to your list. If you do hit the daily cap mid-run, the script +detects OMDb's "Request limit reached!" response, stops making further +requests, fills in the remaining rows from cache (if available) or raw list +data, and finishes writing whatever it has. Just re-run the script the next +day to pick up where it left off. +""" + +import os +import sys +import time +import threading +from pathlib import Path + +import progressbar +import requests + + +# Map kind strings from the movie_list file (cinemagoer-style, e.g. 'tv series') +# to OMDb's `type` parameter, which only distinguishes movie / series / episode +# / game. OMDb doesn't have separate buckets for shorts, tv movies, etc., so +# several kinds collapse onto the closest OMDb type; if there's no good match +# we leave the type filter off entirely and let OMDb's own search decide. +OMDB_TYPE_MAP = { + 'movie': 'movie', + 'tv movie': 'movie', + 'video': 'movie', + 'short': 'movie', + 'tv series': 'series', + 'tv mini series': 'series', + 'tv special': 'series', + 'tv episode': 'episode', + 'tv short': 'episode', + 'video game': 'game', +} + +OMDB_BASE_URL = 'https://www.omdbapi.com/' + +# Set once any worker sees OMDb's daily quota error, so every other thread +# stops retrying (retrying against a blown daily cap would just spin forever). +DAILY_LIMIT_REACHED = threading.Event() + + +class MovieList: + """ Class to generate a movie list HTML table """ + def __init__(self, src=None, dst=None, api_key=None): + self.prev_html = [] + self.html = """ + + + + + My Movie List + + + + + + + + +
+

🎬 My Movie Collection

+ + + + + + + + + + + + """ + self.src = src + self.dst = Path(dst) if dst else Path(os.path.dirname(sys.argv[0])) / 'index.html' + self.movie_list = [] + self.threads = [] + self.api_key = api_key or os.environ.get('OMDB_API_KEY') + if not self.api_key: + sys.stderr.write( + 'error: no OMDb API key found.\n' + 'Get a free key at https://www.omdbapi.com/apikey.aspx and set it via:\n' + ' export OMDB_API_KEY=your_key_here\n' + ) + sys.exit(1) + self.read_prev_output() + self.html_table = None + + # ------------------------------------------------------------------ + # Internal helpers + # ------------------------------------------------------------------ + + @staticmethod + def _omdb_type(arg_kind: str): + """Return the OMDb `type` filter value for a movie_list kind tag, or + None if there's no good match (in which case OMDb's own fuzzy search + decides without a type constraint).""" + return OMDB_TYPE_MAP.get(arg_kind.lower()) + + def _omdb_request(self, params: dict): + """GET against the OMDb API with the API key attached. Returns the + parsed JSON dict. Raises requests.RequestException on network/HTTP + failure (the caller is responsible for retrying).""" + query = dict(params) + query['apikey'] = self.api_key + response = requests.get(OMDB_BASE_URL, params=query, timeout=15) + response.raise_for_status() + return response.json() + + @staticmethod + def _is_daily_limit_error(data: dict) -> bool: + return ( + data.get('Response') == 'False' + and 'request limit' in (data.get('Error') or '').lower() + ) + + def _fallback_row(self, arg: dict, index: int) -> str: + """Build a row from cached HTML (even if it has N/A fields) or, if + nothing cached exists, from the raw movie_list data. Used when the + daily OMDb quota has been exhausted so the run can still finish.""" + match = [ + row for row in self.prev_html + if f'' in row + ] + if match: + match_str = match[0].replace('*', arg['status']).replace('DONE', arg['status']) + if '', 1)[1] + return f'\n {after_index}' + return match_str + + return ( + f'\n ' + f'' + f'' + f'' + f'' + f'' + f'' + f'' + ) + + # ------------------------------------------------------------------ + # Worker thread + # ------------------------------------------------------------------ + + def _worker(self, arg, index): + """Fetch OMDb data for one movie and write it into self.html_table.""" + if DAILY_LIMIT_REACHED.is_set(): + self.html_table[index] = self._fallback_row(arg, index) + return + + omdb_type = self._omdb_type(arg['kind']) + movie_detail = None + first_run = True + + while True: + if DAILY_LIMIT_REACHED.is_set(): + break + if not first_run: + time.sleep(10) + else: + first_run = False + + try: + # ---- attempt 1: direct title+year lookup ---- + # OMDb's `t=` endpoint does fuzzy title matching server-side + # and returns full details (rating, genre, etc.) in one call. + params = {'t': arg['title'], 'plot': 'short'} + if arg['year']: + params['y'] = arg['year'] + if omdb_type: + params['type'] = omdb_type + data = self._omdb_request(params) + + if self._is_daily_limit_error(data): + DAILY_LIMIT_REACHED.set() + sys.stderr.write('OMDb daily request limit reached — stopping further lookups.\n') + break + + # ---- attempt 2: drop the year, in case it's slightly off ---- + if data.get('Response') == 'False' and arg['year']: + params.pop('y', None) + data = self._omdb_request(params) + if self._is_daily_limit_error(data): + DAILY_LIMIT_REACHED.set() + break + + # ---- attempt 3: fuzzy search, take the top hit, fetch by ID ---- + if data.get('Response') == 'False': + search_params = {'s': arg['title']} + if omdb_type: + search_params['type'] = omdb_type + search_data = self._omdb_request(search_params) + if self._is_daily_limit_error(search_data): + DAILY_LIMIT_REACHED.set() + break + results = search_data.get('Search') or [] + if results: + data = self._omdb_request({'i': results[0]['imdbID'], 'plot': 'short'}) + if self._is_daily_limit_error(data): + DAILY_LIMIT_REACHED.set() + break + + if data.get('Response') == 'True': + movie_detail = data + break + + except requests.RequestException as exc: + sys.stderr.write(f'network error ({arg["title"]}): {exc}, retrying…\n') + + # ---- build the table row ---- + if movie_detail: + title = movie_detail.get('Title') or arg['title'] + year = movie_detail.get('Year') or arg['year'] + rating = movie_detail.get('imdbRating') or 'N/A' + genres = movie_detail.get('Genre') or 'N/A' + imdb_id = movie_detail.get('imdbID') or '' + if imdb_id: + html_title_td = ( + f'' + f'{title}' + ) + else: + html_title_td = title + elif DAILY_LIMIT_REACHED.is_set(): + self.html_table[index] = self._fallback_row(arg, index) + return + else: + # Nothing found — show the raw data from the list file + title = arg['title'] + year = arg['year'] + rating = 'N/A' + genres = 'N/A' + html_title_td = title + + self.html_table[index] = ( + f'\n ' + f'' + f'' + f'' + f'' + f'' + f'' + f'' + ) + + # ------------------------------------------------------------------ + # Public interface + # ------------------------------------------------------------------ + + def gen(self): + """ Generate an HTML list based on input, using a threaded worker """ + if not self.src: + self.src = Path(os.path.dirname(sys.argv[0])) / 'movie_list' + else: + self.src = Path(self.src) + if not self.src.exists(): + sys.stderr.write(f'error: input does not exist - {self.src}\n') + return False + + self.movie_list = {} + seen_titles = set() + + with open(self.src, 'r', encoding='utf-8') as fp_handle: + mlist_raw = fp_handle.read() + idx = 0 + for raw_line in mlist_raw.splitlines(): + if not raw_line: + continue + + title = raw_line[0:next( + (i for i, ch in enumerate(raw_line) if ch in {'<', '('}), None + ) - 1] + + if title in seen_titles: + continue + seen_titles.add(title) + + self.movie_list[idx] = { + 'title': title, + 'kind': raw_line[raw_line.find('<')+1:raw_line.rfind('>')+1].strip('<>') or 'movie', + 'year': raw_line[raw_line.find('(')+1:raw_line.find(')')], + 'status': raw_line[raw_line.find('[')+1:raw_line.find(']')], + } + idx += 1 + + self.html_table = [None] * len(self.movie_list) + + pbar = progressbar.ProgressBar(max_value=len(self.movie_list)) + + for idx, movie in self.movie_list.items(): + # Re-use the cached row if available and complete (no N/A) + match = [ + row for row in self.prev_html + if f'' in row and 'N/A' not in row + ] + if match: + match_str = match[0] + # Sync status field + match_str = match_str.replace('*', movie['status']).replace('DONE', movie['status']) + # Sync the row index number + if '', 1)[1] if '' in match_str else match_str + self.html_table[idx] = ( + f'\n ' + f'{after_index}' + ) + else: + self.html_table[idx] = match_str + pbar.increment() + elif DAILY_LIMIT_REACHED.is_set(): + # Quota already blown earlier in this run — don't bother + # spawning a thread that will just no-op. + self.html_table[idx] = self._fallback_row(movie, idx) + pbar.increment() + else: + thread = threading.Thread(target=self._worker, args=(movie, idx)) + self.threads.append(thread) + + max_threads = 10 + while self.threads: + threads_alive = self.get_alive_threads() + threads_to_start = [t for t in self.threads if t not in threads_alive] + for i in range(min(max_threads, len(threads_to_start))): + threads_to_start[i].start() + pbar.increment() + time.sleep(2) # gentle rate-limiting between thread launches + time.sleep(2) + self.delete_finished_threads() + + return True + + def delete_finished_threads(self): + for idx, thread in enumerate(self.threads): + if not thread.is_alive() and thread._started.is_set(): + thread.join() + self.threads[idx] = None + self.threads = [t for t in self.threads if t is not None] + + def get_alive_threads(self): + return [t for t in self.threads if t.is_alive() or t._started.is_set()] + + def write(self, dst=None): + """ Write the HTML list to index.html """ + out_path = dst if dst else self.dst + timestamp = time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime()) + + final_html = self.html.split('')[0] + '' + final_html += ''.join(row for row in self.html_table if row is not None) + final_html += f''' + +
#TitleYearRatingGenreStatus
' in match_str: + after_index = match_str.split('
{index + 1}
{index + 1}{arg["title"]}{arg["year"]}N/AN/A{arg["status"]}
{index + 1}{html_title_td}{year}{rating}{genres}{arg["status"]}
' in match_str: + after_index = match_str.split('
{idx + 1}
+
+
Generated {timestamp} UTC
+ +''' + + with open(out_path, 'wb') as fp_handle: + fp_handle.write(final_html.encode('utf8')) + + if DAILY_LIMIT_REACHED.is_set(): + sys.stderr.write( + 'Note: OMDb\'s daily request limit was hit during this run. ' + 'Remaining entries were filled from cache/raw data where possible. ' + 'Re-run tomorrow to fetch anything still missing.\n' + ) + + def read_prev_output(self): + """ Import a previous HTML table for caching """ + if self.dst.exists(): + with open(self.dst, 'rb') as fp_handle: + self.prev_html = fp_handle.read().decode('utf8').split('\n') + + def deduplicate_html(self): + """ Remove duplicate entries from html_table based on movie titles """ + seen_titles = set() + deduplicated = [] + + for row in self.html_table: + if row is None: + continue + if '' in row: + start = row.find('', start) + title = row[start:end] + if title not in seen_titles: + seen_titles.add(title) + deduplicated.append(row) + else: + deduplicated.append(row) + + self.html_table = deduplicated + return len(self.html_table) + + +def main(): + """ Default run """ + src = dst = None + if len(sys.argv) > 3: + sys.stderr.write(f'error: max 2 variables, {len(sys.argv)-1} given!\n') + sys.exit(1) + if len(sys.argv) > 1: + src = sys.argv[1] + if len(sys.argv) == 3: + dst = sys.argv[2] + + mlist = MovieList(src=src, dst=dst) + if mlist.gen(): + mlist.write(dst=dst) + + +if __name__ == "__main__": + main()