Rumperuu/Cataloguer
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

process tv series'

Browse source
This commit is contained in:
Ben Goldsworthy 2024-01-14 16:11:01 +01:00
parent d3dd9a1051
commit 5b92efcc98
7 changed files with 1538 additions and 613 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.gitignore vendored
View file

@ -1,4 +1,5 @@
cgi-bin/
__pycache__/
.well-known/
resources/
node_modules/

12
data/films/log.json
View file

@ -1,4 +1,16 @@
[
{
"id": 596179,
"title": "Climate Change: The Facts",
"original_language": "en",
"original_title": "Climate Change: The Facts",
"overview": "After one of the hottest years on record, Sir David Attenborough looks at the science of climate change and potential solutions to this global threat. Interviews with some of the world\u2019s leading climate scientists explore recent extreme weather conditions such as unprecedented storms and catastrophic wildfires. They also reveal what dangerous levels of climate change could mean for both human populations and the natural world in the future.",
"poster_path": "/dLsPnpMoj6CtBw8OHIrCm3QSGsL.jpg",
"release_date": "2019-04-18",
"date_added": "2024-01-14",
"date_watched": "2019-09-11",
"imdb_id": "tt10095266"
},
{
"id": 18491,
"title": "Neon Genesis Evangelion: The End of Evangelion",

6
data/tv-episodes/log.json
View file

@ -9845,12 +9845,6 @@
"Date Watched": "2019-09-15",
"Episode Number": "1x01"
},
{
"Series Title": "Climate Change - The Facts",
"Episode Title": "Climate Change - The Facts",
"Date Watched": "2019-09-11",
"Episode Number": "1x01"
},
{
"Series Title": "Last Week Tonight With John Oliver",
"Episode Title": "Episode 171",

12
data/tv-series/current.json Normal file
View file

@ -0,0 +1,12 @@
[
{
"id": 242807,
"name": "Skibidi Toilet",
"overview": "Skibidi Toilet is a apocalyptic series where camera-mans fight with the skibidi toilets.",
"poster_path": "/4YtVG3wrFYwt4JjQKiasqWdweLV.jpg",
"first_air_date": "2023-02-07",
"date_added": "2024-01-14",
"date_started": "2024-01-12",
"added_by_id": "tt27814427"
}
]

1783
data/tv-series/log.json
View file

File diff suppressed because it is too large Load diff

192
scripts/add_item.py
View file

@ -9,6 +9,68 @@ import re
import requests
from urllib.request import urlopen
logging.basicConfig(filename='./logs/run.log', encoding='utf-8', level=logging.DEBUG)
load_dotenv()
TMDB_API_KEY = os.getenv('TMDB_API_KEY')
TVDB_API_KEY = os.getenv('TVDB_API_KEY')
OPENLIBRARY_API_KEY = os.getenv('OPENLIBRARY_API_KEY')
if "" == TMDB_API_KEY: logging.error("TMDB API key not found")
if "" == TVDB_API_KEY: logging.error("TVDB API key not found")
if "" == OPENLIBRARY_API_KEY: logging.error("OpenLibrary API key not found")
def add_item_to_log(item_id, media_type, log):
"""Add a film, book, TV series or TV episode to a log"""
logging.info(f"Processing {item_id}…")
item = import_by_id(item_id, media_type)
if log in ['log', 'current']:
if 'log' == log:
date_watched = ''
while re.search('[0-9]{4}-[0-9]{2}-[0-9]{2}', date_watched) is None:
date_watched = input("Enter date watched [YYYY-MM-DD, t for today]:")
if 't' == date_watched: date_watched = datetime.today().strftime('%Y-%m-%d')
item['date_watched'] = date_watched
elif 'current' == log:
date_started = ''
while re.search('[0-9]{4}-[0-9]{2}-[0-9]{2}', date_started) is None:
date_started = input("Enter date started [YYYY-MM-DD, t for today]:")
if 't' == date_started: date_started = datetime.today().strftime('%Y-%m-%d')
item['date_started'] = date_started
is_rewatch = ''
while is_rewatch not in ['y', 'n']:
is_rewatch = input("Is this a rewatch? [y/n]:")
if 'y' == is_rewatch: item['is_rewatch'] = True
item['added_by_id'] = item_id
comments = input("Enter comments (optional):")
if '' != comments: item['comments'] = comments
# Validation step
correct = ''
print(f"{media_type} data to add:\n")
print(json.dumps(item, indent=4))
if 'y' != input("\nDoes this look correct? [y]: "): return
# Save changes
logging.info(f"Adding {media_type} to {log}…")
with open(f"./data/{media_type}/{log}.json", "r") as log_file:
log_items = json.load(log_file)
log_items.insert(0, item)
with open(f"./data/{media_type}/{log}.json", "w") as log_file:
json.dump(log_items, log_file, indent=4)
logging.info(f"Added {media_type} {item_id} to {log}")
def import_by_id(import_id, media_type):
if media_type in ['films', 'tv-series']:
@ -44,8 +106,8 @@ def import_from_imdb_by_id(imdb_id, media_type):
logging.error(response.text)
if ('films' == media_type): results_key = 'movie_results'
elif ('tv-episodes' == media_type): results_key = 'tv_episode_results'
elif ('tv-series' == media_type): results_key = 'data'
elif ('tv-episodes' == media_type): results_key = 'TODO'
elif ('tv-series' == media_type): results_key = 'tv_results'
response_data = json.loads(response.text)[results_key]
@ -60,7 +122,7 @@ def import_from_imdb_by_id(imdb_id, media_type):
print(json.dumps(response_data, indent=4))
idx = input("\nEnter the index of the result to use: ")
try:
item = response_data[idx]
item = response_data[int(idx)]
except:
logging.error("Index invalid!")
print("Index invalid!")
@ -69,109 +131,67 @@ def import_from_imdb_by_id(imdb_id, media_type):
return cleanup_result(item)
def add_item_to_log(imdb_id, media_type, log):
"""Add a film or TV episode to a log"""
logging.info(f"Processing {imdb_id}…")
item = import_from_imdb_by_id(imdb_id, media_type)
if 'log' == log:
date_watched = ''
while re.search('[0-9]{4}-[0-9]{2}-[0-9]{2}', date_watched) is None:
date_watched = input("Enter date watched [YYYY-MM-DD, t for today]:")
if 't' == date_watched: date_watched = datetime.today().strftime('%Y-%m-%d')
item['date_watched'] = date_watched
is_rewatch = ''
while is_rewatch not in ['y', 'n']:
is_rewatch = input("Is this a rewatch? [y/n]:")
if 'y' == is_rewatch: item['is_rewatch'] = True
item['imdb_id'] = imdb_id
comments = input("Enter comments (optional):")
if '' != comments: item['comments'] = comments
# Validation step
correct = ''
print(f"{media_type} data to add:\n")
print(json.dumps(item, indent=4))
if 'y' != input("\nDoes this look correct? [y]:"): return
# Save changes
logging.info(f"Adding {media_type} to {log}…")
with open(f"./data/{media_type}/{log}.json", "r") as log_file:
log_items = json.load(log_file)
log_items.insert(0, item)
with open(f"./data/{media_type}/{log}.json", "w") as log_file:
json.dump(log_items, log_file, indent=4)
logging.info(f"Added {media_type} {film['title']} ({film['release_date']}) to {log}")
def cleanup_result(item):
"""Process a film or TV episode returned by the TMDB API by removing unnecessary fields and adding others"""
for field_name in ['adult', 'backdrop_path', 'episode_type', 'genre_ids', 'media_type', 'popularity', 'production_code', 'runtime', 'still_path', 'video', 'vote_average', 'vote_count']:
for field_name in ['adult', 'backdrop_path', 'episode_type', 'genre_ids', 'media_type', 'origin_country', 'popularity', 'production_code', 'runtime', 'still_path', 'video', 'vote_average', 'vote_count']:
if field_name in item: del item[field_name]
if 'original_title' in item and 'original_language' in item:
if item['original_title'] == item['title'] and item['original_language'] == 'en':
del item['original_title'], item['original_language']
# TODO - select automatically
title_key = 'name'
if f"original_{title_key}" in item and 'original_language' in item:
if item[f"original_{title_key}"] == item[title_key] and item['original_language'] == 'en':
del item[f"original_{title_key}"], item['original_language']
if 'date_added' not in item: item['date_added'] = datetime.today().strftime('%Y-%m-%d')
return item
logging.basicConfig(filename='./logs/run.log', encoding='utf-8', level=logging.DEBUG)
def main():
media_type = ''
while media_type not in ['films', 'tv-episodes', 'tv-series', 'books']:
media_type = input("Select media type [films|tv-episodes|tv-series|books]:")
load_dotenv()
if 'films' == media_type:
log = ''
while log not in ['log', 'wishlist']:
log = input ("Enter log to update [log|wishlist]:")
TMDB_API_KEY = os.getenv('TMDB_API_KEY')
TVDB_API_KEY = os.getenv('TVDB_API_KEY')
OPENLIBRARY_API_KEY = os.getenv('OPENLIBRARY_API_KEY')
imdb_id = ''
while re.search("tt[0-9]+", imdb_id) is None:
imdb_id = input("Enter IMDB ID:")
if "" == TMDB_API_KEY: logging.error("TMDB API key not found")
if "" == TVDB_API_KEY: logging.error("TVDB API key not found")
if "" == OPENLIBRARY_API_KEY: logging.error("OpenLibrary API key not found")
add_item_to_log(imdb_id, media_type, log)
media_type = ''
while media_type not in ['films', 'tv-episodes', 'tv-series', 'books']:
media_type = input("Select media type [films|tv-episodes|tv-series|books]:")
elif 'books' == media_type:
log = ''
while log not in ['log', 'current', 'wishlist']:
log = input ("Enter log to update [log|current|wishlist]:")
if 'films' == media_type:
log = ''
while log not in ['log', 'wishlist']:
log = input ("Enter log to update [log|wishlist]:")
isbn = ''
while re.search("[0-9]+", isbn) is None:
isbn = input("Enter ISBN:")
imdb_id = ''
while re.search("tt[0-9]+", imdb_id) is None:
imdb_id = input("Enter IMDB ID:")
elif 'tv-episodes' == media_type:
imdb_id = ''
while re.search("tt[0-9]+", imdb_id) is None:
imdb_id = input("Enter IMDB ID:")
add_item_to_log(imdb_id, media_type, log)
add_item_to_log(imdb_id, media_type, 'log')
elif 'books' == media_type:
log = ''
while log not in ['log', 'current', 'wishlist']:
log = input ("Enter log to update [log|current|wishlist]:")
elif 'tv-series' == media_type:
log = ''
while log not in ['log', 'current', 'wishlist']:
log = input ("Enter log to update [log|current|wishlist]:")
isbn = ''
while re.search("[0-9]+", isbn) is None:
isbn = input("Enter ISBN:")
imdb_id = ''
while re.search("tt[0-9]+", imdb_id) is None:
imdb_id = input("Enter IMDB ID:")
elif 'tv-episodes' == media_type:
imdb_id = ''
while re.search("tt[0-9]+", imdb_id) is None:
imdb_id = input("Enter IMDB ID:")
add_item_to_log(imdb_id, media_type, log)
add_item_to_log(imdb_id, media_type, 'log')
elif 'tv-series' == media_type:
log = ''
while log not in ['log', 'current', 'wishlist']:
log = input ("Enter log to update [log|current|wishlist]:")
add_item_to_log(media_type, log)
if __name__ == "__main__":
main()

143
scripts/process_logs.py
View file

@ -2,11 +2,23 @@ from dotenv import load_dotenv
import json
import logging
import os
import re
import requests
import time
from urllib.request import urlopen
from scripts.add_item import cleanup_result, import_by_id
from add_item import cleanup_result, import_by_id
logging.basicConfig(filename='./logs/run.log', encoding='utf-8', level=logging.DEBUG)
load_dotenv()
TMDB_API_KEY = os.getenv('TMDB_API_KEY')
TVDB_API_KEY = os.getenv('TVDB_API_KEY')
OPENLIBRARY_API_KEY = os.getenv('OPENLIBRARY_API_KEY')
if "" == TMDB_API_KEY: logging.error("TMDB API key not found")
if "" == TVDB_API_KEY: logging.error("TVDB API key not found")
if "" == OPENLIBRARY_API_KEY: logging.error("OpenLibrary API key not found")
def process_log(media_type, log):
logging.info(f"Processing {media_type}/{log}…")
@ -17,53 +29,59 @@ def process_log(media_type, log):
log_item_values = {}
for i, item in enumerate(log_items):
try:
if 'id' not in item:
if 'films' == media_type: item_title = item['Title']
elif 'tv-episodes' == media_type: item_title = item['Episode Title']
elif 'tv-series' == media_type: item_title = item['Show Title']
if 'id' not in item:
if 'films' == media_type: item_title = item['Title']
elif 'tv-episodes' == media_type: item_title = item['Episode Title']
elif 'tv-series' == media_type: item_title = item['Show Title']
logging.debug(f"Processing {item_title}…")
logging.debug(f"Processing {item_title}…")
# Rename pre-existing fields
if 'Date Added' in item:
log_item_values['date_added'] = item['Date Added']
del item['Date Added']
if 'Date Watched' in item:
log_item_values['date_watched'] = item['Date Watched']
del item['Date Watched']
if 'Rewatch' in item:
log_item_values['is_rewatch'] = item['Rewatch']
del item['Rewatch']
if 'Comments' in item:
log_item_values['comments'] = item['Comments']
del item['Comments']
if 'Series Title' in item:
log_item_values['series_title'] = item['Series Title']
del item['Series Title']
if 'Episode Title' in item:
log_item_values['name'] = item['Episode Title']
del item['Episode Title']
if 'Episode Number' in item:
split_num = log_item_values['episode_number'].split("E")
log_item_values['episode_number'] = split_num[1]
log_item_values['season_number'] = split_num[0] or None
del item['Episode Number']
# Rename pre-existing fields
if 'Date Added' in item:
log_item_values['date_added'] = item['Date Added']
del item['Date Added']
if 'Date Watched' in item:
log_item_values['date_watched'] = item['Date Watched']
del item['Date Watched']
if 'Rewatch' in item:
log_item_values['is_rewatch'] = item['Rewatch']
del item['Rewatch']
if 'Comments' in item:
log_item_values['comments'] = item['Comments']
del item['Comments']
if 'Series Title' in item:
log_item_values['series_title'] = item['Series Title']
del item['Series Title']
if 'Episode Title' in item:
log_item_values['name'] = item['Episode Title']
del item['Episode Title']
if 'Episode Number' in item:
split_num = log_item_values['episode_number'].split("E")
log_item_values['episode_number'] = split_num[1]
log_item_values['season_number'] = split_num[0] or None
del item['Episode Number']
if 'IMDB ID' in item:
log_items[i] = import_by_id(item['IMDB ID'], media_type)
else:
log_items[i] = import_by_details(item, item_title, media_type)
if log_items[i] is None:
item['imdb_id'] = input(f"Enter IMDB ID for {item_title}: ")
if re.search("tt[0-9]+", item['imdb_id']) is not None:
log_items[i] = import_by_id(item['imdb_id'], media_type)
if 'IMDB ID' in item:
log_items[i] = import_by_id(item['IMDB ID'], media_type)
else:
logging.warning(f"Skipped {item_title}")
log_items[i] = import_by_details(item, item_title, media_type)
log_items[i] |= log_item_values
if log_items[i] is None:
item['imdb_id'] = input(f"Enter IMDB ID for {item_title}: ")
if re.search("tt[0-9]+", item['imdb_id']) is not None:
log_items[i] = import_by_id(item['imdb_id'], media_type)
with open(f"./data/{media_type}/{log}.json", "w") as log_file:
json.dump(log_items, log_file, indent=4)
else:
logging.warning(f"Skipped {item_title}")
if log_items[i] is not None: log_items[i] |= log_item_values
except KeyError:
print(json.dumps(item, indent=4))
with open(f"./data/{media_type}/{log}.json", "w") as log_file:
json.dump(log_items, log_file, indent=4)
@ -83,15 +101,15 @@ def import_by_details(item, item_title, media_type):
def import_from_tmdb_by_details(item, item_title, media_type):
"""Retrieve a film or TV series from TMDB using its title"""
logging.info(f"Processing {item['Title']}…")
logging.info(f"Processing {item_title}…")
api_url = f"https://api.themoviedb.org/3/search/{'movie' if 'films' === media_type else 'tv'}"
api_url = f"https://api.themoviedb.org/3/search/{'movie' if 'films' == media_type else 'tv'}"
# Sending API request
response = requests.get(
api_url,
params={
'query': item_title¸
'query': item_title,
'include_adult': True,
'year': item['Release Year'] if 'Release Year' in item else None
},
@ -113,10 +131,13 @@ def import_from_tmdb_by_details(item, item_title, media_type):
return cleanup_result(response_data[0])
elif 0 == len(response_data):
logging.warning(f"Returned no {media_type} for {item['Title']} ({item['Release Year']})")
logging.warning(f"Returned no {media_type} for {item_title}")
elif 1 < len(response_data):
response_data = [film for film in response_data if film['title'] == item['Title']]
if 'films' == media_type: title_key = 'title'
elif 'tv-series' == media_type: title_key = 'name'
response_data = [result for result in response_data if result[title_key] == item_title]
if 1 == len(response_data):
return cleanup_result(response_data[0])
@ -128,35 +149,23 @@ def import_from_tmdb_by_details(item, item_title, media_type):
if "" != idx:
try:
return cleanup_result(response_data[idx])
return cleanup_result(response_data[int(idx)])
except:
logging.error("Index invalid!")
print("Index invalid!")
item['IMDB ID'] = input(f"Enter IMDB ID for {item['Title']} ({item['Release Year']}):")
item['IMDB ID'] = input(f"Enter IMDB ID for {item_title}: ")
if '' != item['IMDB ID']:
return import_by_id(item['IMDB_ID'], media_type)
return import_by_id(item['IMDB ID'], media_type)
else:
logging.warning(f"Skipped {item['Title']} ({item['Release Year']})")
logging.warning(f"Skipped {item_title}")
return item
logging.basicConfig(filename='./logs/run.log', encoding='utf-8', level=logging.DEBUG)
load_dotenv()
TMDB_API_KEY = os.getenv('TMDB_API_KEY')
TVDB_API_KEY = os.getenv('TVDB_API_KEY')
OPENLIBRARY_API_KEY = os.getenv('OPENLIBRARY_API_KEY')
if "" === TMDB_API_KEY: logging.error("TMDB API key not found")
if "" === TVDB_API_KEY: logging.error("TVDB API key not found")
if "" === OPENLIBRARY_API_KEY: logging.error("OpenLibrary API key not found")
media_type = ''
while media_type not in ['films', 'tv-episodes', 'tv-series', 'books']:
media_type = input("Select media type [films|tv-episodes|tv-series|books]:")
media_type = input("Select media type [films|tv-episodes|tv-series|books]: ")
if 'films' == media_type:
log = ''
@ -168,7 +177,7 @@ while media_type not in ['films', 'tv-episodes', 'tv-series', 'books']:
elif 'books' == media_type:
log = ''
while log not in ['log', 'current', 'wishlist']:
log = input ("Enter log to update [log|current|wishlist]:")
log = input ("Enter log to process [log|current|wishlist]:")
elif 'tv-episodes' == media_type:
process_log(media_type, 'log')
@ -176,7 +185,7 @@ while media_type not in ['films', 'tv-episodes', 'tv-series', 'books']:
elif 'tv-series' == media_type:
log = ''
while log not in ['log', 'current', 'wishlist']:
log = input ("Enter log to update [log|current|wishlist]:")
log = input ("Enter log to process [log|current|wishlist]:")
process_log(media_type, log)