movie_list: migrate from cinemagoer to omdb
Signed-off-by: Bogomil Vasilev <smirky@smirky.net>
This commit is contained in:
@@ -1,571 +0,0 @@
|
|||||||
#!/usr/bin/env python
|
|
||||||
|
|
||||||
"""
|
|
||||||
1. Import a movie_list txt file
|
|
||||||
2. Query IMDb for each entry, retrieving actual movie name, rating and genres
|
|
||||||
3. Generate an HTML table from the IMDb data
|
|
||||||
4. Store the HTML in index.html
|
|
||||||
"""
|
|
||||||
|
|
||||||
import os
|
|
||||||
import sys
|
|
||||||
import time
|
|
||||||
import threading
|
|
||||||
from pathlib import Path
|
|
||||||
import progressbar
|
|
||||||
from imdb import IMDb
|
|
||||||
from imdb._exceptions import IMDbParserError, IMDbDataAccessError
|
|
||||||
|
|
||||||
|
|
||||||
class MovieList:
|
|
||||||
""" Class to generate a movie list HTML table """
|
|
||||||
def __init__(self, src=None, dst=None):
|
|
||||||
self.prev_html = []
|
|
||||||
self.html = """<!DOCTYPE html>
|
|
||||||
<html lang="en">
|
|
||||||
<head>
|
|
||||||
<meta charset="UTF-8">
|
|
||||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
|
||||||
<title>My Movie List</title>
|
|
||||||
<link rel="stylesheet" type="text/css" href="https://cdn.datatables.net/1.10.24/css/jquery.dataTables.min.css">
|
|
||||||
<script src="../jquery-3.7.1.min.js"></script>
|
|
||||||
<script src="https://cdn.datatables.net/1.10.24/js/jquery.dataTables.min.js"></script>
|
|
||||||
<style>
|
|
||||||
* {
|
|
||||||
margin: 0;
|
|
||||||
padding: 0;
|
|
||||||
box-sizing: border-box;
|
|
||||||
}
|
|
||||||
|
|
||||||
body {
|
|
||||||
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Oxygen, Ubuntu, Cantarell, sans-serif;
|
|
||||||
background: transparent;
|
|
||||||
height: 100vh;
|
|
||||||
padding: 0.5rem;
|
|
||||||
color: #fff;
|
|
||||||
overflow: hidden;
|
|
||||||
display: flex;
|
|
||||||
flex-direction: column;
|
|
||||||
}
|
|
||||||
|
|
||||||
.container {
|
|
||||||
background: rgba(0, 0, 0, 0.6);
|
|
||||||
backdrop-filter: blur(10px);
|
|
||||||
border-radius: 8px;
|
|
||||||
border: 1px solid rgba(255, 255, 255, 0.1);
|
|
||||||
padding: 0.75rem;
|
|
||||||
height: 100%;
|
|
||||||
min-height: 0;
|
|
||||||
overflow: hidden;
|
|
||||||
display: flex;
|
|
||||||
flex-direction: column;
|
|
||||||
}
|
|
||||||
|
|
||||||
h1 {
|
|
||||||
color: #fff;
|
|
||||||
margin-bottom: 0.75rem;
|
|
||||||
font-size: 1.5rem;
|
|
||||||
font-weight: 600;
|
|
||||||
text-align: center;
|
|
||||||
}
|
|
||||||
|
|
||||||
.dataTables_wrapper {
|
|
||||||
flex: 1;
|
|
||||||
overflow: auto;
|
|
||||||
display: flex;
|
|
||||||
flex-direction: column;
|
|
||||||
min-height: 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
.dataTables_filter {
|
|
||||||
margin-bottom: 0.5rem;
|
|
||||||
text-align: right;
|
|
||||||
flex-shrink: 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
.dataTables_filter label {
|
|
||||||
display: flex;
|
|
||||||
align-items: center;
|
|
||||||
justify-content: flex-end;
|
|
||||||
gap: 0.5rem;
|
|
||||||
font-size: 0.9rem;
|
|
||||||
color: rgba(255, 255, 255, 0.9);
|
|
||||||
}
|
|
||||||
|
|
||||||
.dataTables_filter input {
|
|
||||||
padding: 0.4rem 0.75rem;
|
|
||||||
border: 1px solid rgba(255, 255, 255, 0.2);
|
|
||||||
border-radius: 6px;
|
|
||||||
background: rgba(0, 0, 0, 0.3);
|
|
||||||
color: #fff;
|
|
||||||
font-size: 0.9rem;
|
|
||||||
width: 250px;
|
|
||||||
}
|
|
||||||
|
|
||||||
.dataTables_filter input::placeholder {
|
|
||||||
color: rgba(255, 255, 255, 0.5);
|
|
||||||
}
|
|
||||||
|
|
||||||
.dataTables_filter input:focus {
|
|
||||||
outline: none;
|
|
||||||
border-color: rgba(255, 255, 255, 0.4);
|
|
||||||
background: rgba(0, 0, 0, 0.4);
|
|
||||||
}
|
|
||||||
|
|
||||||
.dataTables_info {
|
|
||||||
padding: 0.5rem 0;
|
|
||||||
color: rgba(255, 255, 255, 0.6);
|
|
||||||
font-size: 0.85rem;
|
|
||||||
flex-shrink: 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
.dataTables_scroll {
|
|
||||||
display: none;
|
|
||||||
}
|
|
||||||
|
|
||||||
.dataTables_scrollHead {
|
|
||||||
display: none;
|
|
||||||
}
|
|
||||||
|
|
||||||
.dataTables_scrollBody {
|
|
||||||
display: none;
|
|
||||||
}
|
|
||||||
|
|
||||||
.dataTables_scrollBody::-webkit-scrollbar {
|
|
||||||
width: 6px;
|
|
||||||
}
|
|
||||||
|
|
||||||
.dataTables_scrollBody::-webkit-scrollbar-track {
|
|
||||||
background: rgba(255, 255, 255, 0.05);
|
|
||||||
}
|
|
||||||
|
|
||||||
.dataTables_scrollBody::-webkit-scrollbar-thumb {
|
|
||||||
background: rgba(255, 255, 255, 0.2);
|
|
||||||
border-radius: 3px;
|
|
||||||
}
|
|
||||||
|
|
||||||
.dataTables_wrapper::-webkit-scrollbar {
|
|
||||||
width: 6px;
|
|
||||||
}
|
|
||||||
|
|
||||||
.dataTables_wrapper::-webkit-scrollbar-track {
|
|
||||||
background: rgba(255, 255, 255, 0.05);
|
|
||||||
}
|
|
||||||
|
|
||||||
.dataTables_wrapper::-webkit-scrollbar-thumb {
|
|
||||||
background: rgba(255, 255, 255, 0.2);
|
|
||||||
border-radius: 3px;
|
|
||||||
}
|
|
||||||
|
|
||||||
table.dataTable.stripe tbody tr.odd,
|
|
||||||
table.dataTable.display tbody tr.odd,
|
|
||||||
table.dataTable tbody tr {
|
|
||||||
background: transparent !important;
|
|
||||||
}
|
|
||||||
|
|
||||||
table.dataTable.hover tbody tr:hover,
|
|
||||||
table.dataTable.display tbody tr:hover {
|
|
||||||
background: rgba(255, 255, 255, 0.1) !important;
|
|
||||||
}
|
|
||||||
|
|
||||||
table.dataTable tbody td {
|
|
||||||
background: transparent !important;
|
|
||||||
}
|
|
||||||
|
|
||||||
#sortable {
|
|
||||||
width: 100%;
|
|
||||||
border-collapse: collapse;
|
|
||||||
background: transparent;
|
|
||||||
}
|
|
||||||
|
|
||||||
#sortable thead {
|
|
||||||
background: rgba(0, 0, 0, 0.4);
|
|
||||||
}
|
|
||||||
|
|
||||||
#sortable thead th {
|
|
||||||
padding: 0.5rem 0.75rem;
|
|
||||||
text-align: left;
|
|
||||||
font-weight: 600;
|
|
||||||
font-size: 0.85rem;
|
|
||||||
color: rgba(255, 255, 255, 0.9);
|
|
||||||
cursor: pointer;
|
|
||||||
border-bottom: 1px solid rgba(255, 255, 255, 0.1);
|
|
||||||
white-space: nowrap;
|
|
||||||
}
|
|
||||||
|
|
||||||
#sortable thead th:hover {
|
|
||||||
background: rgba(255, 255, 255, 0.05);
|
|
||||||
}
|
|
||||||
|
|
||||||
#sortable thead th.sorting,
|
|
||||||
#sortable thead th.sorting_asc,
|
|
||||||
#sortable thead th.sorting_desc {
|
|
||||||
padding-right: 1.5rem;
|
|
||||||
position: relative;
|
|
||||||
}
|
|
||||||
|
|
||||||
#sortable thead th.sorting:after {
|
|
||||||
content: '⇅';
|
|
||||||
position: absolute;
|
|
||||||
right: 0.5rem;
|
|
||||||
top: 50%;
|
|
||||||
transform: translateY(-50%);
|
|
||||||
font-size: 0.75rem;
|
|
||||||
color: rgba(255, 255, 255, 0.5);
|
|
||||||
}
|
|
||||||
|
|
||||||
#sortable thead th.sorting_asc:after {
|
|
||||||
content: '↑';
|
|
||||||
position: absolute;
|
|
||||||
right: 0.5rem;
|
|
||||||
top: 50%;
|
|
||||||
transform: translateY(-50%);
|
|
||||||
font-size: 0.75rem;
|
|
||||||
color: rgba(255, 255, 255, 0.9);
|
|
||||||
}
|
|
||||||
|
|
||||||
#sortable thead th.sorting_desc:after {
|
|
||||||
content: '↓';
|
|
||||||
position: absolute;
|
|
||||||
right: 0.5rem;
|
|
||||||
top: 50%;
|
|
||||||
transform: translateY(-50%);
|
|
||||||
font-size: 0.75rem;
|
|
||||||
color: rgba(255, 255, 255, 0.9);
|
|
||||||
}
|
|
||||||
|
|
||||||
#sortable tbody tr {
|
|
||||||
border-bottom: 1px solid rgba(255, 255, 255, 0.05);
|
|
||||||
transition: background 0.15s ease;
|
|
||||||
background: transparent !important;
|
|
||||||
}
|
|
||||||
|
|
||||||
#sortable tbody tr:hover {
|
|
||||||
background: rgba(255, 255, 255, 0.1) !important;
|
|
||||||
}
|
|
||||||
|
|
||||||
#sortable tbody td {
|
|
||||||
padding: 0.5rem 0.75rem;
|
|
||||||
font-size: 0.9rem;
|
|
||||||
color: rgba(255, 255, 255, 0.9);
|
|
||||||
background: transparent !important;
|
|
||||||
}
|
|
||||||
|
|
||||||
#sortable tbody td:first-child {
|
|
||||||
color: rgba(255, 255, 255, 0.5);
|
|
||||||
font-size: 0.85rem;
|
|
||||||
width: 50px;
|
|
||||||
}
|
|
||||||
|
|
||||||
#sortable tbody td a {
|
|
||||||
color: rgba(135, 206, 250, 0.9);
|
|
||||||
text-decoration: none;
|
|
||||||
}
|
|
||||||
|
|
||||||
#sortable tbody td a:hover {
|
|
||||||
color: rgba(135, 206, 250, 1);
|
|
||||||
text-decoration: underline;
|
|
||||||
}
|
|
||||||
|
|
||||||
#sortable tbody td p[hidden] {
|
|
||||||
display: none;
|
|
||||||
}
|
|
||||||
|
|
||||||
#sortable tbody td:nth-child(4) {
|
|
||||||
color: rgba(255, 193, 7, 0.9);
|
|
||||||
font-weight: 500;
|
|
||||||
}
|
|
||||||
|
|
||||||
.timestamp {
|
|
||||||
text-align: center;
|
|
||||||
margin-top: 0.5rem;
|
|
||||||
padding-top: 0.5rem;
|
|
||||||
border-top: 1px solid rgba(255, 255, 255, 0.1);
|
|
||||||
color: rgba(255, 255, 255, 0.5);
|
|
||||||
font-size: 0.8rem;
|
|
||||||
}
|
|
||||||
</style>
|
|
||||||
<script>
|
|
||||||
$(document).ready(function(){
|
|
||||||
try {
|
|
||||||
$('#sortable').DataTable({
|
|
||||||
"paging": false,
|
|
||||||
"info": true,
|
|
||||||
"searching": true,
|
|
||||||
"ordering": true,
|
|
||||||
"order": [[0, "desc"]],
|
|
||||||
"language": {
|
|
||||||
"search": "Search:",
|
|
||||||
"info": "Showing _TOTAL_ movies",
|
|
||||||
"infoEmpty": "No movies",
|
|
||||||
"infoFiltered": "(filtered from _MAX_)"
|
|
||||||
}
|
|
||||||
});
|
|
||||||
} catch(e) {
|
|
||||||
console.error("DataTables init error:", e);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
</script>
|
|
||||||
</head>
|
|
||||||
<body>
|
|
||||||
<base target="_parent" />
|
|
||||||
<div class="container">
|
|
||||||
<h1>🎬 My Movie Collection</h1>
|
|
||||||
<table id="sortable" class="sortable">
|
|
||||||
<thead>
|
|
||||||
<tr>
|
|
||||||
<th>#</th>
|
|
||||||
<th>Title</th>
|
|
||||||
<th>Year</th>
|
|
||||||
<th>Rating</th>
|
|
||||||
<th>Genre</th>
|
|
||||||
<th>Status</th>
|
|
||||||
</tr>
|
|
||||||
</thead>
|
|
||||||
<tbody>"""
|
|
||||||
self.src = src
|
|
||||||
self.dst = Path(dst) if dst else Path(os.path.dirname(sys.argv[0])) / 'index.html'
|
|
||||||
self.movie_list = []
|
|
||||||
self.threads = []
|
|
||||||
self.read_prev_output()
|
|
||||||
self.html_table = None
|
|
||||||
|
|
||||||
def _worker(self, arg, index):
|
|
||||||
# Scan IMDb for a given movie and append it to the html
|
|
||||||
# This collects rating, genres, official name and a hyperlink
|
|
||||||
imdb = IMDb()
|
|
||||||
first_run = True
|
|
||||||
while True:
|
|
||||||
if not first_run:
|
|
||||||
time.sleep(10)
|
|
||||||
else:
|
|
||||||
first_run = False
|
|
||||||
try:
|
|
||||||
query = imdb.search_movie(f'{arg["title"]} {arg["year"]}')
|
|
||||||
break
|
|
||||||
except IMDbDataAccessError as imdb_data_exc:
|
|
||||||
exc = str(imdb_data_exc)
|
|
||||||
if '503' in exc:
|
|
||||||
sys.stderr.write('503 - Service Unavailable, retrying...')
|
|
||||||
elif '403' in exc:
|
|
||||||
sys.stderr.write('403 - Forbidden, retrying...\n')
|
|
||||||
query = []
|
|
||||||
time.sleep(10)
|
|
||||||
except IMDbParserError as imdb_parser_exc:
|
|
||||||
query = []
|
|
||||||
break
|
|
||||||
except Exception as exc:
|
|
||||||
time.sleep(10)
|
|
||||||
|
|
||||||
movie = None
|
|
||||||
for entry in query:
|
|
||||||
try:
|
|
||||||
imdb.update(entry)
|
|
||||||
except Exception as e:
|
|
||||||
sys.stderr.write('update err')
|
|
||||||
# in case any of these keys is missing in the query, continue
|
|
||||||
if not all(key in entry.keys() for key in ['kind', 'year', 'title']):
|
|
||||||
continue
|
|
||||||
if arg['status'] == 'DONE' and 'rating' not in entry.keys():
|
|
||||||
continue
|
|
||||||
# Try to eliminate episode results
|
|
||||||
if [i for i in entry.keys() if 'episode' in i.lower()] or (
|
|
||||||
'episode' in entry['title'].lower() and \
|
|
||||||
'episode' not in arg['title'].lower()):
|
|
||||||
continue
|
|
||||||
if entry['kind'].lower() == arg['kind'].lower():
|
|
||||||
movie = entry
|
|
||||||
break
|
|
||||||
if not movie:
|
|
||||||
movie = {
|
|
||||||
'title': arg['title'],
|
|
||||||
'kind': arg['kind'],
|
|
||||||
'year': arg['year'],
|
|
||||||
'dummy': None
|
|
||||||
}
|
|
||||||
if 'genres' not in movie.keys():
|
|
||||||
movie['genres'] = ['N/A']
|
|
||||||
if 'rating' not in movie.keys():
|
|
||||||
movie['rating'] = 'N/A'
|
|
||||||
|
|
||||||
html_title_td = movie['title'] if 'dummy' in movie.keys() else \
|
|
||||||
f'<a href="https://www.imdb.com/title/tt{movie.movieID}" target="_blank">{movie["title"]}</a>'
|
|
||||||
|
|
||||||
self.html_table[index] = (
|
|
||||||
f'\n <tr>'
|
|
||||||
f'<td data-label="#">{index + 1}</td>'
|
|
||||||
f'<td data-label="Title"><p hidden>{arg["title"]}</p>{html_title_td}</td>'
|
|
||||||
f'<td data-label="Year">{movie["year"]}</td>'
|
|
||||||
f'<td data-label="Rating" align="center">{movie["rating"]}</td>'
|
|
||||||
f'<td data-label="Genre">{", ".join(movie["genres"])}</td>'
|
|
||||||
f'<td data-label="Status" align="center">{arg["status"]}</td>'
|
|
||||||
f'</tr>'
|
|
||||||
)
|
|
||||||
|
|
||||||
def gen(self):
|
|
||||||
""" Generate an HTML list based on input, using a threaded worker """
|
|
||||||
if not self.src:
|
|
||||||
self.src = Path(os.path.dirname(sys.argv[0])) / 'movie_list'
|
|
||||||
else:
|
|
||||||
self.src = Path(self.src)
|
|
||||||
if not self.src.exists():
|
|
||||||
sys.stderr.write(f'error: input does not exist - {self.src}\n')
|
|
||||||
return False
|
|
||||||
|
|
||||||
self.movie_list = {}
|
|
||||||
seen_titles = set() # Track unique titles
|
|
||||||
|
|
||||||
# Open the movie list & split the columns
|
|
||||||
with open(self.src, 'r', encoding='utf-8') as fp_handle:
|
|
||||||
mlist_raw = fp_handle.read()
|
|
||||||
idx = 0
|
|
||||||
for raw_line in mlist_raw.splitlines():
|
|
||||||
# In case the line is empty
|
|
||||||
if not raw_line:
|
|
||||||
continue
|
|
||||||
|
|
||||||
title = raw_line[0:next((i for i, ch in enumerate(raw_line) if ch in {'<', '('}), None) - 1]
|
|
||||||
|
|
||||||
# Skip if we've already seen this title
|
|
||||||
if title in seen_titles:
|
|
||||||
continue
|
|
||||||
|
|
||||||
seen_titles.add(title)
|
|
||||||
self.movie_list.update({
|
|
||||||
idx: {
|
|
||||||
'title': title,
|
|
||||||
'kind': raw_line[raw_line.find('<')+1:raw_line.rfind('>')+1].strip('<>') or 'movie',
|
|
||||||
'year': raw_line[raw_line.find('(')+1:raw_line.find(')')],
|
|
||||||
'status': raw_line[raw_line.find('[')+1:raw_line.find(']')],
|
|
||||||
}
|
|
||||||
})
|
|
||||||
idx += 1
|
|
||||||
|
|
||||||
self.html_table = [None] * len(self.movie_list)
|
|
||||||
|
|
||||||
# Progress bar
|
|
||||||
pbar = progressbar.ProgressBar(max_value=len(self.movie_list))
|
|
||||||
for idx, movie in self.movie_list.items():
|
|
||||||
# More precise matching - look for the hidden <p> tag with exact title
|
|
||||||
match = [html_row for html_row in self.prev_html
|
|
||||||
if f'<p hidden>{movie["title"]}</p>' in html_row
|
|
||||||
and 'N/A' not in html_row]
|
|
||||||
if match:
|
|
||||||
# Update the index and status from the cached row
|
|
||||||
match_str = match[0]
|
|
||||||
# Replace the status (* -> DONE or vice versa)
|
|
||||||
match_str = match_str.replace('*', movie['status']).replace('DONE', movie['status'])
|
|
||||||
# Update the index number
|
|
||||||
if '<td data-label="#">' in match_str:
|
|
||||||
# Extract everything after the index cell
|
|
||||||
after_index = match_str.split('</td>', 1)[1] if '</td>' in match_str else match_str
|
|
||||||
self.html_table[idx] = f'\n <tr><td data-label="#">{idx + 1}</td>{after_index}'
|
|
||||||
else:
|
|
||||||
self.html_table[idx] = match_str
|
|
||||||
pbar.increment()
|
|
||||||
else:
|
|
||||||
thread = threading.Thread(target=self._worker, args=(movie, idx))
|
|
||||||
self.threads.append(thread)
|
|
||||||
|
|
||||||
max_threads = 10
|
|
||||||
while self.threads:
|
|
||||||
threads_alive = self.get_alive_threads()
|
|
||||||
threads_to_be_started = [i for i in self.threads if i not in threads_alive]
|
|
||||||
for idx in range(max_threads if max_threads < len(threads_to_be_started) else len(threads_to_be_started)):
|
|
||||||
threads_to_be_started[idx].start()
|
|
||||||
pbar.increment()
|
|
||||||
time.sleep(2)
|
|
||||||
time.sleep(2)
|
|
||||||
self.delete_finished_threads()
|
|
||||||
|
|
||||||
# Don't append to self.html here - we'll do it in write()
|
|
||||||
return True
|
|
||||||
|
|
||||||
def delete_finished_threads(self):
|
|
||||||
for idx, thread in enumerate(self.threads):
|
|
||||||
if not thread.is_alive() and thread._started.is_set():
|
|
||||||
thread.join()
|
|
||||||
self.threads[idx] = None
|
|
||||||
self.threads = list(filter(lambda a: a is not None, self.threads))
|
|
||||||
|
|
||||||
def get_alive_threads(self):
|
|
||||||
threads = []
|
|
||||||
for thread in self.threads:
|
|
||||||
if thread.is_alive() or thread._started.is_set():
|
|
||||||
threads.append(thread)
|
|
||||||
return threads
|
|
||||||
|
|
||||||
def write(self, dst=None):
|
|
||||||
""" Write the HTML list to index.html """
|
|
||||||
out_path = dst if dst else self.dst
|
|
||||||
timestamp = time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime())
|
|
||||||
|
|
||||||
# Build the final HTML - don't append, rebuild from scratch
|
|
||||||
final_html = self.html.split('<tbody>')[0] + '<tbody>'
|
|
||||||
final_html += ''.join([row for row in self.html_table if row is not None])
|
|
||||||
final_html += f'''
|
|
||||||
</tbody>
|
|
||||||
</table>
|
|
||||||
</div>
|
|
||||||
<div class="timestamp">Generated {timestamp} UTC</div>
|
|
||||||
</body>
|
|
||||||
</html>'''
|
|
||||||
|
|
||||||
with open(out_path, 'wb') as fp_handle:
|
|
||||||
fp_handle.write(final_html.encode('utf8'))
|
|
||||||
|
|
||||||
def read_prev_output(self):
|
|
||||||
""" Import a previous HTML table """
|
|
||||||
if self.dst.exists():
|
|
||||||
with open(self.dst, 'rb') as fp_handle:
|
|
||||||
self.prev_html = fp_handle.read().decode('utf8').split('\n')
|
|
||||||
|
|
||||||
def deduplicate_html(self):
|
|
||||||
""" Remove duplicate entries from html_table based on movie titles """
|
|
||||||
seen_titles = set()
|
|
||||||
deduplicated = []
|
|
||||||
|
|
||||||
for idx, row in enumerate(self.html_table):
|
|
||||||
if row is None:
|
|
||||||
continue
|
|
||||||
|
|
||||||
# Extract the hidden title from the row
|
|
||||||
if '<p hidden>' in row and '</p>' in row:
|
|
||||||
start = row.find('<p hidden>') + 10
|
|
||||||
end = row.find('</p>', start)
|
|
||||||
title = row[start:end]
|
|
||||||
|
|
||||||
if title not in seen_titles:
|
|
||||||
seen_titles.add(title)
|
|
||||||
deduplicated.append(row)
|
|
||||||
else:
|
|
||||||
# Skip duplicate
|
|
||||||
continue
|
|
||||||
else:
|
|
||||||
# If we can't find the hidden title, keep the row anyway
|
|
||||||
deduplicated.append(row)
|
|
||||||
|
|
||||||
# Update html_table with deduplicated content
|
|
||||||
self.html_table = deduplicated
|
|
||||||
return len(self.html_table)
|
|
||||||
|
|
||||||
|
|
||||||
def main():
|
|
||||||
""" Default run """
|
|
||||||
src = dst = None
|
|
||||||
if len(sys.argv) > 3:
|
|
||||||
sys.stderr.write(f'error: max 2 variables, {len(sys.argv)-1} given!\n')
|
|
||||||
sys.exit(1)
|
|
||||||
|
|
||||||
if len(sys.argv) > 1:
|
|
||||||
src = sys.argv[1]
|
|
||||||
if len(sys.argv) == 3:
|
|
||||||
dst = sys.argv[2]
|
|
||||||
|
|
||||||
mlist = MovieList(src=src, dst=dst)
|
|
||||||
if mlist.gen():
|
|
||||||
mlist.write(dst=dst)
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
main()
|
|
||||||
@@ -467,3 +467,10 @@ The French Dispatch (2021) [DONE]
|
|||||||
Sentimental Value (2025) [DONE]
|
Sentimental Value (2025) [DONE]
|
||||||
28 Years Later: The Bone Temple (2026) [*]
|
28 Years Later: The Bone Temple (2026) [*]
|
||||||
Youth (2015) [DONE]
|
Youth (2015) [DONE]
|
||||||
|
Dune: Part One (2021) [DONE]
|
||||||
|
Dune: Part Two (2024) [DONE]
|
||||||
|
Dune: Part Three (2026) [*]
|
||||||
|
The Swedish Connection (2026) [DONE]
|
||||||
|
Atlas of the Universe (2026) [*]
|
||||||
|
I Swear (2025) [DONE]
|
||||||
|
The Odyssey (2026) [DONE]
|
||||||
|
|||||||
Executable
+645
@@ -0,0 +1,645 @@
|
|||||||
|
#!/usr/bin/env python
|
||||||
|
|
||||||
|
"""
|
||||||
|
1. Import a movie_list txt file
|
||||||
|
2. Query OMDb (the Open Movie Database, powered by real IMDb data) for each
|
||||||
|
entry, retrieving the actual movie title, year, IMDb rating and genres
|
||||||
|
3. Generate an HTML table from the OMDb data
|
||||||
|
4. Store the HTML in index.html
|
||||||
|
|
||||||
|
Requires: pip install requests progressbar2
|
||||||
|
|
||||||
|
You need a free OMDb API key:
|
||||||
|
1. Go to https://www.omdbapi.com/apikey.aspx
|
||||||
|
2. Choose the free "1,000 daily limit" option and register with your email
|
||||||
|
3. OMDb emails you a key (check spam folder if it doesn't show up quickly)
|
||||||
|
4. Set it as an environment variable before running this script:
|
||||||
|
export OMDB_API_KEY=your_key_here # macOS/Linux
|
||||||
|
setx OMDB_API_KEY your_key_here # Windows (new shells)
|
||||||
|
|
||||||
|
The free tier is limited to 1,000 requests/day. Because this script only
|
||||||
|
fetches entries that aren't already cached in the previous index.html (see
|
||||||
|
read_prev_output/gen), you'll generally only "spend" requests on movies that
|
||||||
|
are new to your list. If you do hit the daily cap mid-run, the script
|
||||||
|
detects OMDb's "Request limit reached!" response, stops making further
|
||||||
|
requests, fills in the remaining rows from cache (if available) or raw list
|
||||||
|
data, and finishes writing whatever it has. Just re-run the script the next
|
||||||
|
day to pick up where it left off.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
import threading
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import progressbar
|
||||||
|
import requests
|
||||||
|
|
||||||
|
|
||||||
|
# Map kind strings from the movie_list file (cinemagoer-style, e.g. 'tv series')
|
||||||
|
# to OMDb's `type` parameter, which only distinguishes movie / series / episode
|
||||||
|
# / game. OMDb doesn't have separate buckets for shorts, tv movies, etc., so
|
||||||
|
# several kinds collapse onto the closest OMDb type; if there's no good match
|
||||||
|
# we leave the type filter off entirely and let OMDb's own search decide.
|
||||||
|
OMDB_TYPE_MAP = {
|
||||||
|
'movie': 'movie',
|
||||||
|
'tv movie': 'movie',
|
||||||
|
'video': 'movie',
|
||||||
|
'short': 'movie',
|
||||||
|
'tv series': 'series',
|
||||||
|
'tv mini series': 'series',
|
||||||
|
'tv special': 'series',
|
||||||
|
'tv episode': 'episode',
|
||||||
|
'tv short': 'episode',
|
||||||
|
'video game': 'game',
|
||||||
|
}
|
||||||
|
|
||||||
|
OMDB_BASE_URL = 'https://www.omdbapi.com/'
|
||||||
|
|
||||||
|
# Set once any worker sees OMDb's daily quota error, so every other thread
|
||||||
|
# stops retrying (retrying against a blown daily cap would just spin forever).
|
||||||
|
DAILY_LIMIT_REACHED = threading.Event()
|
||||||
|
|
||||||
|
|
||||||
|
class MovieList:
|
||||||
|
""" Class to generate a movie list HTML table """
|
||||||
|
def __init__(self, src=None, dst=None, api_key=None):
|
||||||
|
self.prev_html = []
|
||||||
|
self.html = """<!DOCTYPE html>
|
||||||
|
<html lang="en">
|
||||||
|
<head>
|
||||||
|
<meta charset="UTF-8">
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||||
|
<title>My Movie List</title>
|
||||||
|
<link rel="stylesheet" type="text/css" href="https://cdn.datatables.net/1.10.24/css/jquery.dataTables.min.css">
|
||||||
|
<script src="../jquery-3.7.1.min.js"></script>
|
||||||
|
<script src="https://cdn.datatables.net/1.10.24/js/jquery.dataTables.min.js"></script>
|
||||||
|
<style>
|
||||||
|
* {
|
||||||
|
margin: 0;
|
||||||
|
padding: 0;
|
||||||
|
box-sizing: border-box;
|
||||||
|
}
|
||||||
|
|
||||||
|
body {
|
||||||
|
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Oxygen, Ubuntu, Cantarell, sans-serif;
|
||||||
|
background: transparent;
|
||||||
|
height: 100vh;
|
||||||
|
padding: 0.5rem;
|
||||||
|
color: #fff;
|
||||||
|
overflow: hidden;
|
||||||
|
display: flex;
|
||||||
|
flex-direction: column;
|
||||||
|
}
|
||||||
|
|
||||||
|
.container {
|
||||||
|
background: rgba(0, 0, 0, 0.6);
|
||||||
|
backdrop-filter: blur(10px);
|
||||||
|
border-radius: 8px;
|
||||||
|
border: 1px solid rgba(255, 255, 255, 0.1);
|
||||||
|
padding: 0.75rem;
|
||||||
|
height: 100%;
|
||||||
|
min-height: 0;
|
||||||
|
overflow: hidden;
|
||||||
|
display: flex;
|
||||||
|
flex-direction: column;
|
||||||
|
}
|
||||||
|
|
||||||
|
h1 {
|
||||||
|
color: #fff;
|
||||||
|
margin-bottom: 0.75rem;
|
||||||
|
font-size: 1.5rem;
|
||||||
|
font-weight: 600;
|
||||||
|
text-align: center;
|
||||||
|
}
|
||||||
|
|
||||||
|
.dataTables_wrapper {
|
||||||
|
flex: 1;
|
||||||
|
overflow: auto;
|
||||||
|
display: flex;
|
||||||
|
flex-direction: column;
|
||||||
|
min-height: 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
.dataTables_filter {
|
||||||
|
margin-bottom: 0.5rem;
|
||||||
|
text-align: right;
|
||||||
|
flex-shrink: 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
.dataTables_filter label {
|
||||||
|
display: flex;
|
||||||
|
align-items: center;
|
||||||
|
justify-content: flex-end;
|
||||||
|
gap: 0.5rem;
|
||||||
|
font-size: 0.9rem;
|
||||||
|
color: rgba(255, 255, 255, 0.9);
|
||||||
|
}
|
||||||
|
|
||||||
|
.dataTables_filter input {
|
||||||
|
padding: 0.4rem 0.75rem;
|
||||||
|
border: 1px solid rgba(255, 255, 255, 0.2);
|
||||||
|
border-radius: 6px;
|
||||||
|
background: rgba(0, 0, 0, 0.3);
|
||||||
|
color: #fff;
|
||||||
|
font-size: 0.9rem;
|
||||||
|
width: 250px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.dataTables_filter input::placeholder {
|
||||||
|
color: rgba(255, 255, 255, 0.5);
|
||||||
|
}
|
||||||
|
|
||||||
|
.dataTables_filter input:focus {
|
||||||
|
outline: none;
|
||||||
|
border-color: rgba(255, 255, 255, 0.4);
|
||||||
|
background: rgba(0, 0, 0, 0.4);
|
||||||
|
}
|
||||||
|
|
||||||
|
.dataTables_info {
|
||||||
|
padding: 0.5rem 0;
|
||||||
|
color: rgba(255, 255, 255, 0.6);
|
||||||
|
font-size: 0.85rem;
|
||||||
|
flex-shrink: 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
.dataTables_scroll { display: none; }
|
||||||
|
.dataTables_scrollHead { display: none; }
|
||||||
|
.dataTables_scrollBody { display: none; }
|
||||||
|
|
||||||
|
.dataTables_scrollBody::-webkit-scrollbar { width: 6px; }
|
||||||
|
.dataTables_scrollBody::-webkit-scrollbar-track { background: rgba(255, 255, 255, 0.05); }
|
||||||
|
.dataTables_scrollBody::-webkit-scrollbar-thumb { background: rgba(255, 255, 255, 0.2); border-radius: 3px; }
|
||||||
|
|
||||||
|
.dataTables_wrapper::-webkit-scrollbar { width: 6px; }
|
||||||
|
.dataTables_wrapper::-webkit-scrollbar-track { background: rgba(255, 255, 255, 0.05); }
|
||||||
|
.dataTables_wrapper::-webkit-scrollbar-thumb { background: rgba(255, 255, 255, 0.2); border-radius: 3px; }
|
||||||
|
|
||||||
|
table.dataTable.stripe tbody tr.odd,
|
||||||
|
table.dataTable.display tbody tr.odd,
|
||||||
|
table.dataTable tbody tr { background: transparent !important; }
|
||||||
|
|
||||||
|
table.dataTable.hover tbody tr:hover,
|
||||||
|
table.dataTable.display tbody tr:hover { background: rgba(255, 255, 255, 0.1) !important; }
|
||||||
|
|
||||||
|
table.dataTable tbody td { background: transparent !important; }
|
||||||
|
|
||||||
|
#sortable {
|
||||||
|
width: 100%;
|
||||||
|
border-collapse: collapse;
|
||||||
|
background: transparent;
|
||||||
|
}
|
||||||
|
|
||||||
|
#sortable thead { background: rgba(0, 0, 0, 0.4); }
|
||||||
|
|
||||||
|
#sortable thead th {
|
||||||
|
padding: 0.5rem 0.75rem;
|
||||||
|
text-align: left;
|
||||||
|
font-weight: 600;
|
||||||
|
font-size: 0.85rem;
|
||||||
|
color: rgba(255, 255, 255, 0.9);
|
||||||
|
cursor: pointer;
|
||||||
|
border-bottom: 1px solid rgba(255, 255, 255, 0.1);
|
||||||
|
white-space: nowrap;
|
||||||
|
}
|
||||||
|
|
||||||
|
#sortable thead th:hover { background: rgba(255, 255, 255, 0.05); }
|
||||||
|
|
||||||
|
#sortable thead th.sorting,
|
||||||
|
#sortable thead th.sorting_asc,
|
||||||
|
#sortable thead th.sorting_desc { padding-right: 1.5rem; position: relative; }
|
||||||
|
|
||||||
|
#sortable thead th.sorting:after {
|
||||||
|
content: '⇅'; position: absolute; right: 0.5rem; top: 50%;
|
||||||
|
transform: translateY(-50%); font-size: 0.75rem; color: rgba(255, 255, 255, 0.5);
|
||||||
|
}
|
||||||
|
#sortable thead th.sorting_asc:after {
|
||||||
|
content: '↑'; position: absolute; right: 0.5rem; top: 50%;
|
||||||
|
transform: translateY(-50%); font-size: 0.75rem; color: rgba(255, 255, 255, 0.9);
|
||||||
|
}
|
||||||
|
#sortable thead th.sorting_desc:after {
|
||||||
|
content: '↓'; position: absolute; right: 0.5rem; top: 50%;
|
||||||
|
transform: translateY(-50%); font-size: 0.75rem; color: rgba(255, 255, 255, 0.9);
|
||||||
|
}
|
||||||
|
|
||||||
|
#sortable tbody tr {
|
||||||
|
border-bottom: 1px solid rgba(255, 255, 255, 0.05);
|
||||||
|
transition: background 0.15s ease;
|
||||||
|
background: transparent !important;
|
||||||
|
}
|
||||||
|
#sortable tbody tr:hover { background: rgba(255, 255, 255, 0.1) !important; }
|
||||||
|
|
||||||
|
#sortable tbody td {
|
||||||
|
padding: 0.5rem 0.75rem;
|
||||||
|
font-size: 0.9rem;
|
||||||
|
color: rgba(255, 255, 255, 0.9);
|
||||||
|
background: transparent !important;
|
||||||
|
}
|
||||||
|
#sortable tbody td:first-child {
|
||||||
|
color: rgba(255, 255, 255, 0.5);
|
||||||
|
font-size: 0.85rem;
|
||||||
|
width: 50px;
|
||||||
|
}
|
||||||
|
#sortable tbody td a {
|
||||||
|
color: rgba(135, 206, 250, 0.9);
|
||||||
|
text-decoration: none;
|
||||||
|
}
|
||||||
|
#sortable tbody td a:hover {
|
||||||
|
color: rgba(135, 206, 250, 1);
|
||||||
|
text-decoration: underline;
|
||||||
|
}
|
||||||
|
#sortable tbody td p[hidden] { display: none; }
|
||||||
|
#sortable tbody td:nth-child(4) {
|
||||||
|
color: rgba(255, 193, 7, 0.9);
|
||||||
|
font-weight: 500;
|
||||||
|
}
|
||||||
|
|
||||||
|
.timestamp {
|
||||||
|
text-align: center;
|
||||||
|
margin-top: 0.5rem;
|
||||||
|
padding-top: 0.5rem;
|
||||||
|
border-top: 1px solid rgba(255, 255, 255, 0.1);
|
||||||
|
color: rgba(255, 255, 255, 0.5);
|
||||||
|
font-size: 0.8rem;
|
||||||
|
}
|
||||||
|
</style>
|
||||||
|
<script>
|
||||||
|
$(document).ready(function(){
|
||||||
|
try {
|
||||||
|
$('#sortable').DataTable({
|
||||||
|
"paging": false,
|
||||||
|
"info": true,
|
||||||
|
"searching": true,
|
||||||
|
"ordering": true,
|
||||||
|
"order": [[0, "desc"]],
|
||||||
|
"language": {
|
||||||
|
"search": "Search:",
|
||||||
|
"info": "Showing _TOTAL_ movies",
|
||||||
|
"infoEmpty": "No movies",
|
||||||
|
"infoFiltered": "(filtered from _MAX_)"
|
||||||
|
}
|
||||||
|
});
|
||||||
|
} catch(e) {
|
||||||
|
console.error("DataTables init error:", e);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
</script>
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
<base target="_parent" />
|
||||||
|
<div class="container">
|
||||||
|
<h1>🎬 My Movie Collection</h1>
|
||||||
|
<table id="sortable" class="sortable">
|
||||||
|
<thead>
|
||||||
|
<tr>
|
||||||
|
<th>#</th>
|
||||||
|
<th>Title</th>
|
||||||
|
<th>Year</th>
|
||||||
|
<th>Rating</th>
|
||||||
|
<th>Genre</th>
|
||||||
|
<th>Status</th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>"""
|
||||||
|
self.src = src
|
||||||
|
self.dst = Path(dst) if dst else Path(os.path.dirname(sys.argv[0])) / 'index.html'
|
||||||
|
self.movie_list = []
|
||||||
|
self.threads = []
|
||||||
|
self.api_key = api_key or os.environ.get('OMDB_API_KEY')
|
||||||
|
if not self.api_key:
|
||||||
|
sys.stderr.write(
|
||||||
|
'error: no OMDb API key found.\n'
|
||||||
|
'Get a free key at https://www.omdbapi.com/apikey.aspx and set it via:\n'
|
||||||
|
' export OMDB_API_KEY=your_key_here\n'
|
||||||
|
)
|
||||||
|
sys.exit(1)
|
||||||
|
self.read_prev_output()
|
||||||
|
self.html_table = None
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
# Internal helpers
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _omdb_type(arg_kind: str):
|
||||||
|
"""Return the OMDb `type` filter value for a movie_list kind tag, or
|
||||||
|
None if there's no good match (in which case OMDb's own fuzzy search
|
||||||
|
decides without a type constraint)."""
|
||||||
|
return OMDB_TYPE_MAP.get(arg_kind.lower())
|
||||||
|
|
||||||
|
def _omdb_request(self, params: dict):
|
||||||
|
"""GET against the OMDb API with the API key attached. Returns the
|
||||||
|
parsed JSON dict. Raises requests.RequestException on network/HTTP
|
||||||
|
failure (the caller is responsible for retrying)."""
|
||||||
|
query = dict(params)
|
||||||
|
query['apikey'] = self.api_key
|
||||||
|
response = requests.get(OMDB_BASE_URL, params=query, timeout=15)
|
||||||
|
response.raise_for_status()
|
||||||
|
return response.json()
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _is_daily_limit_error(data: dict) -> bool:
|
||||||
|
return (
|
||||||
|
data.get('Response') == 'False'
|
||||||
|
and 'request limit' in (data.get('Error') or '').lower()
|
||||||
|
)
|
||||||
|
|
||||||
|
def _fallback_row(self, arg: dict, index: int) -> str:
|
||||||
|
"""Build a row from cached HTML (even if it has N/A fields) or, if
|
||||||
|
nothing cached exists, from the raw movie_list data. Used when the
|
||||||
|
daily OMDb quota has been exhausted so the run can still finish."""
|
||||||
|
match = [
|
||||||
|
row for row in self.prev_html
|
||||||
|
if f'<p hidden>{arg["title"]}</p>' in row
|
||||||
|
]
|
||||||
|
if match:
|
||||||
|
match_str = match[0].replace('*', arg['status']).replace('DONE', arg['status'])
|
||||||
|
if '<td data-label="#">' in match_str:
|
||||||
|
after_index = match_str.split('</td>', 1)[1]
|
||||||
|
return f'\n <tr><td data-label="#">{index + 1}</td>{after_index}'
|
||||||
|
return match_str
|
||||||
|
|
||||||
|
return (
|
||||||
|
f'\n <tr>'
|
||||||
|
f'<td data-label="#">{index + 1}</td>'
|
||||||
|
f'<td data-label="Title"><p hidden>{arg["title"]}</p>{arg["title"]}</td>'
|
||||||
|
f'<td data-label="Year">{arg["year"]}</td>'
|
||||||
|
f'<td data-label="Rating" align="center">N/A</td>'
|
||||||
|
f'<td data-label="Genre">N/A</td>'
|
||||||
|
f'<td data-label="Status" align="center">{arg["status"]}</td>'
|
||||||
|
f'</tr>'
|
||||||
|
)
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
# Worker thread
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
|
||||||
|
def _worker(self, arg, index):
|
||||||
|
"""Fetch OMDb data for one movie and write it into self.html_table."""
|
||||||
|
if DAILY_LIMIT_REACHED.is_set():
|
||||||
|
self.html_table[index] = self._fallback_row(arg, index)
|
||||||
|
return
|
||||||
|
|
||||||
|
omdb_type = self._omdb_type(arg['kind'])
|
||||||
|
movie_detail = None
|
||||||
|
first_run = True
|
||||||
|
|
||||||
|
while True:
|
||||||
|
if DAILY_LIMIT_REACHED.is_set():
|
||||||
|
break
|
||||||
|
if not first_run:
|
||||||
|
time.sleep(10)
|
||||||
|
else:
|
||||||
|
first_run = False
|
||||||
|
|
||||||
|
try:
|
||||||
|
# ---- attempt 1: direct title+year lookup ----
|
||||||
|
# OMDb's `t=` endpoint does fuzzy title matching server-side
|
||||||
|
# and returns full details (rating, genre, etc.) in one call.
|
||||||
|
params = {'t': arg['title'], 'plot': 'short'}
|
||||||
|
if arg['year']:
|
||||||
|
params['y'] = arg['year']
|
||||||
|
if omdb_type:
|
||||||
|
params['type'] = omdb_type
|
||||||
|
data = self._omdb_request(params)
|
||||||
|
|
||||||
|
if self._is_daily_limit_error(data):
|
||||||
|
DAILY_LIMIT_REACHED.set()
|
||||||
|
sys.stderr.write('OMDb daily request limit reached — stopping further lookups.\n')
|
||||||
|
break
|
||||||
|
|
||||||
|
# ---- attempt 2: drop the year, in case it's slightly off ----
|
||||||
|
if data.get('Response') == 'False' and arg['year']:
|
||||||
|
params.pop('y', None)
|
||||||
|
data = self._omdb_request(params)
|
||||||
|
if self._is_daily_limit_error(data):
|
||||||
|
DAILY_LIMIT_REACHED.set()
|
||||||
|
break
|
||||||
|
|
||||||
|
# ---- attempt 3: fuzzy search, take the top hit, fetch by ID ----
|
||||||
|
if data.get('Response') == 'False':
|
||||||
|
search_params = {'s': arg['title']}
|
||||||
|
if omdb_type:
|
||||||
|
search_params['type'] = omdb_type
|
||||||
|
search_data = self._omdb_request(search_params)
|
||||||
|
if self._is_daily_limit_error(search_data):
|
||||||
|
DAILY_LIMIT_REACHED.set()
|
||||||
|
break
|
||||||
|
results = search_data.get('Search') or []
|
||||||
|
if results:
|
||||||
|
data = self._omdb_request({'i': results[0]['imdbID'], 'plot': 'short'})
|
||||||
|
if self._is_daily_limit_error(data):
|
||||||
|
DAILY_LIMIT_REACHED.set()
|
||||||
|
break
|
||||||
|
|
||||||
|
if data.get('Response') == 'True':
|
||||||
|
movie_detail = data
|
||||||
|
break
|
||||||
|
|
||||||
|
except requests.RequestException as exc:
|
||||||
|
sys.stderr.write(f'network error ({arg["title"]}): {exc}, retrying…\n')
|
||||||
|
|
||||||
|
# ---- build the table row ----
|
||||||
|
if movie_detail:
|
||||||
|
title = movie_detail.get('Title') or arg['title']
|
||||||
|
year = movie_detail.get('Year') or arg['year']
|
||||||
|
rating = movie_detail.get('imdbRating') or 'N/A'
|
||||||
|
genres = movie_detail.get('Genre') or 'N/A'
|
||||||
|
imdb_id = movie_detail.get('imdbID') or ''
|
||||||
|
if imdb_id:
|
||||||
|
html_title_td = (
|
||||||
|
f'<a href="https://www.imdb.com/title/{imdb_id}" target="_blank">'
|
||||||
|
f'{title}</a>'
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
html_title_td = title
|
||||||
|
elif DAILY_LIMIT_REACHED.is_set():
|
||||||
|
self.html_table[index] = self._fallback_row(arg, index)
|
||||||
|
return
|
||||||
|
else:
|
||||||
|
# Nothing found — show the raw data from the list file
|
||||||
|
title = arg['title']
|
||||||
|
year = arg['year']
|
||||||
|
rating = 'N/A'
|
||||||
|
genres = 'N/A'
|
||||||
|
html_title_td = title
|
||||||
|
|
||||||
|
self.html_table[index] = (
|
||||||
|
f'\n <tr>'
|
||||||
|
f'<td data-label="#">{index + 1}</td>'
|
||||||
|
f'<td data-label="Title"><p hidden>{arg["title"]}</p>{html_title_td}</td>'
|
||||||
|
f'<td data-label="Year">{year}</td>'
|
||||||
|
f'<td data-label="Rating" align="center">{rating}</td>'
|
||||||
|
f'<td data-label="Genre">{genres}</td>'
|
||||||
|
f'<td data-label="Status" align="center">{arg["status"]}</td>'
|
||||||
|
f'</tr>'
|
||||||
|
)
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
# Public interface
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
|
||||||
|
def gen(self):
|
||||||
|
""" Generate an HTML list based on input, using a threaded worker """
|
||||||
|
if not self.src:
|
||||||
|
self.src = Path(os.path.dirname(sys.argv[0])) / 'movie_list'
|
||||||
|
else:
|
||||||
|
self.src = Path(self.src)
|
||||||
|
if not self.src.exists():
|
||||||
|
sys.stderr.write(f'error: input does not exist - {self.src}\n')
|
||||||
|
return False
|
||||||
|
|
||||||
|
self.movie_list = {}
|
||||||
|
seen_titles = set()
|
||||||
|
|
||||||
|
with open(self.src, 'r', encoding='utf-8') as fp_handle:
|
||||||
|
mlist_raw = fp_handle.read()
|
||||||
|
idx = 0
|
||||||
|
for raw_line in mlist_raw.splitlines():
|
||||||
|
if not raw_line:
|
||||||
|
continue
|
||||||
|
|
||||||
|
title = raw_line[0:next(
|
||||||
|
(i for i, ch in enumerate(raw_line) if ch in {'<', '('}), None
|
||||||
|
) - 1]
|
||||||
|
|
||||||
|
if title in seen_titles:
|
||||||
|
continue
|
||||||
|
seen_titles.add(title)
|
||||||
|
|
||||||
|
self.movie_list[idx] = {
|
||||||
|
'title': title,
|
||||||
|
'kind': raw_line[raw_line.find('<')+1:raw_line.rfind('>')+1].strip('<>') or 'movie',
|
||||||
|
'year': raw_line[raw_line.find('(')+1:raw_line.find(')')],
|
||||||
|
'status': raw_line[raw_line.find('[')+1:raw_line.find(']')],
|
||||||
|
}
|
||||||
|
idx += 1
|
||||||
|
|
||||||
|
self.html_table = [None] * len(self.movie_list)
|
||||||
|
|
||||||
|
pbar = progressbar.ProgressBar(max_value=len(self.movie_list))
|
||||||
|
|
||||||
|
for idx, movie in self.movie_list.items():
|
||||||
|
# Re-use the cached row if available and complete (no N/A)
|
||||||
|
match = [
|
||||||
|
row for row in self.prev_html
|
||||||
|
if f'<p hidden>{movie["title"]}</p>' in row and 'N/A' not in row
|
||||||
|
]
|
||||||
|
if match:
|
||||||
|
match_str = match[0]
|
||||||
|
# Sync status field
|
||||||
|
match_str = match_str.replace('*', movie['status']).replace('DONE', movie['status'])
|
||||||
|
# Sync the row index number
|
||||||
|
if '<td data-label="#">' in match_str:
|
||||||
|
after_index = match_str.split('</td>', 1)[1] if '</td>' in match_str else match_str
|
||||||
|
self.html_table[idx] = (
|
||||||
|
f'\n <tr>'
|
||||||
|
f'<td data-label="#">{idx + 1}</td>{after_index}'
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
self.html_table[idx] = match_str
|
||||||
|
pbar.increment()
|
||||||
|
elif DAILY_LIMIT_REACHED.is_set():
|
||||||
|
# Quota already blown earlier in this run — don't bother
|
||||||
|
# spawning a thread that will just no-op.
|
||||||
|
self.html_table[idx] = self._fallback_row(movie, idx)
|
||||||
|
pbar.increment()
|
||||||
|
else:
|
||||||
|
thread = threading.Thread(target=self._worker, args=(movie, idx))
|
||||||
|
self.threads.append(thread)
|
||||||
|
|
||||||
|
max_threads = 10
|
||||||
|
while self.threads:
|
||||||
|
threads_alive = self.get_alive_threads()
|
||||||
|
threads_to_start = [t for t in self.threads if t not in threads_alive]
|
||||||
|
for i in range(min(max_threads, len(threads_to_start))):
|
||||||
|
threads_to_start[i].start()
|
||||||
|
pbar.increment()
|
||||||
|
time.sleep(2) # gentle rate-limiting between thread launches
|
||||||
|
time.sleep(2)
|
||||||
|
self.delete_finished_threads()
|
||||||
|
|
||||||
|
return True
|
||||||
|
|
||||||
|
def delete_finished_threads(self):
|
||||||
|
for idx, thread in enumerate(self.threads):
|
||||||
|
if not thread.is_alive() and thread._started.is_set():
|
||||||
|
thread.join()
|
||||||
|
self.threads[idx] = None
|
||||||
|
self.threads = [t for t in self.threads if t is not None]
|
||||||
|
|
||||||
|
def get_alive_threads(self):
|
||||||
|
return [t for t in self.threads if t.is_alive() or t._started.is_set()]
|
||||||
|
|
||||||
|
def write(self, dst=None):
|
||||||
|
""" Write the HTML list to index.html """
|
||||||
|
out_path = dst if dst else self.dst
|
||||||
|
timestamp = time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime())
|
||||||
|
|
||||||
|
final_html = self.html.split('<tbody>')[0] + '<tbody>'
|
||||||
|
final_html += ''.join(row for row in self.html_table if row is not None)
|
||||||
|
final_html += f'''
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
</div>
|
||||||
|
<div class="timestamp">Generated {timestamp} UTC</div>
|
||||||
|
</body>
|
||||||
|
</html>'''
|
||||||
|
|
||||||
|
with open(out_path, 'wb') as fp_handle:
|
||||||
|
fp_handle.write(final_html.encode('utf8'))
|
||||||
|
|
||||||
|
if DAILY_LIMIT_REACHED.is_set():
|
||||||
|
sys.stderr.write(
|
||||||
|
'Note: OMDb\'s daily request limit was hit during this run. '
|
||||||
|
'Remaining entries were filled from cache/raw data where possible. '
|
||||||
|
'Re-run tomorrow to fetch anything still missing.\n'
|
||||||
|
)
|
||||||
|
|
||||||
|
def read_prev_output(self):
|
||||||
|
""" Import a previous HTML table for caching """
|
||||||
|
if self.dst.exists():
|
||||||
|
with open(self.dst, 'rb') as fp_handle:
|
||||||
|
self.prev_html = fp_handle.read().decode('utf8').split('\n')
|
||||||
|
|
||||||
|
def deduplicate_html(self):
|
||||||
|
""" Remove duplicate entries from html_table based on movie titles """
|
||||||
|
seen_titles = set()
|
||||||
|
deduplicated = []
|
||||||
|
|
||||||
|
for row in self.html_table:
|
||||||
|
if row is None:
|
||||||
|
continue
|
||||||
|
if '<p hidden>' in row and '</p>' in row:
|
||||||
|
start = row.find('<p hidden>') + 10
|
||||||
|
end = row.find('</p>', start)
|
||||||
|
title = row[start:end]
|
||||||
|
if title not in seen_titles:
|
||||||
|
seen_titles.add(title)
|
||||||
|
deduplicated.append(row)
|
||||||
|
else:
|
||||||
|
deduplicated.append(row)
|
||||||
|
|
||||||
|
self.html_table = deduplicated
|
||||||
|
return len(self.html_table)
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
""" Default run """
|
||||||
|
src = dst = None
|
||||||
|
if len(sys.argv) > 3:
|
||||||
|
sys.stderr.write(f'error: max 2 variables, {len(sys.argv)-1} given!\n')
|
||||||
|
sys.exit(1)
|
||||||
|
if len(sys.argv) > 1:
|
||||||
|
src = sys.argv[1]
|
||||||
|
if len(sys.argv) == 3:
|
||||||
|
dst = sys.argv[2]
|
||||||
|
|
||||||
|
mlist = MovieList(src=src, dst=dst)
|
||||||
|
if mlist.gen():
|
||||||
|
mlist.write(dst=dst)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
Reference in New Issue
Block a user