lint scripts

This commit is contained in:
Ben Goldsworthy 2024-01-17 23:02:56 +01:00
parent 33e8270afe
commit 29592be6ce
3 changed files with 112 additions and 96 deletions

View file

@ -7,7 +7,7 @@
"deploy": "rsync -rP ./public/ ovhvps:~/catalogue/content", "deploy": "rsync -rP ./public/ ovhvps:~/catalogue/content",
"lint:json": "jsonlint ./**/*.json -s", "lint:json": "jsonlint ./**/*.json -s",
"lint:json:fix": "npm run lint:json -- -i", "lint:json:fix": "npm run lint:json -- -i",
"lint:py": "pylint ./scripts/*.py" "lint:py": "pylint --disable=broad-exception-raised --disable=logging-fstring-interpolation ./scripts/*.py"
}, },
"devDependencies": { "devDependencies": {
"jsonlint": "^1.6.3" "jsonlint": "^1.6.3"

View file

@ -1,19 +1,23 @@
# Script to add a new item to the log """
Add a new item to a media catalogue, using various APIs.
"""
from datetime import datetime
from dotenv import load_dotenv
import json import json
import logging import logging
import os import os
import re import re
import time
from datetime import datetime
import requests import requests
from urllib.request import urlopen from dotenv import load_dotenv
def setup_logger(name="add_item"): def setup_logger(name="add_item"):
"""Set up the logger for console and file"""
logging.root.setLevel(logging.NOTSET) logging.root.setLevel(logging.NOTSET)
logger = logging.getLogger(name) logr = logging.getLogger(name)
c_handler = logging.StreamHandler() c_handler = logging.StreamHandler()
f_handler = logging.FileHandler("./logs/run.log") f_handler = logging.FileHandler("./logs/run.log")
@ -27,10 +31,10 @@ def setup_logger(name="add_item"):
c_handler.setFormatter(c_format) c_handler.setFormatter(c_format)
f_handler.setFormatter(f_format) f_handler.setFormatter(f_format)
logger.addHandler(c_handler) logr.addHandler(c_handler)
logger.addHandler(f_handler) logr.addHandler(f_handler)
return logger return logr
logger = setup_logger() logger = setup_logger()
@ -46,12 +50,12 @@ if "" == TVDB_API_KEY:
logger.error("TVDB API key not found") logger.error("TVDB API key not found")
def add_item_to_log(item_id, media_type, log): def add_item_to_log(item_id, media_type, log) -> None:
"""Add a film, book, TV series or TV episode to a log""" """Add a film, book, TV series or TV episode to a log"""
logger.info(f"Processing {item_id}") logger.info(f"Processing {item_id}")
item = import_by_id(item_id, media_type) item: dict = import_by_id(item_id, media_type)
if log in ["log", "current"]: if log in ["log", "current"]:
# TODO - review this when moving from one log to another # TODO - review this when moving from one log to another
@ -74,7 +78,7 @@ def add_item_to_log(item_id, media_type, log):
# TODO - do this automatically # TODO - do this automatically
is_repeat = "" is_repeat = ""
while is_repeat not in ["y", "n"]: while is_repeat not in ["y", "n"]:
is_repeat = input(f"Is this a repeat entry? [y/n]: ") is_repeat = input("Is this a repeat entry? [y/n]: ")
if "y" == is_repeat: if "y" == is_repeat:
item["is_repeat"] = True item["is_repeat"] = True
item["added_by_id"] = item_id item["added_by_id"] = item_id
@ -84,7 +88,6 @@ def add_item_to_log(item_id, media_type, log):
item["comments"] = comments item["comments"] = comments
# Validation step # Validation step
correct = ""
print(f"{media_type} data to add:\n") print(f"{media_type} data to add:\n")
print(json.dumps(item, indent=4)) print(json.dumps(item, indent=4))
if "y" != input("\nDoes this look correct? [y]: "): if "y" != input("\nDoes this look correct? [y]: "):
@ -93,29 +96,31 @@ def add_item_to_log(item_id, media_type, log):
# Save changes # Save changes
logger.info(f"Adding {media_type} to {log}") logger.info(f"Adding {media_type} to {log}")
with open(f"./data/{media_type}/{log}.json", "r") as log_file: with open(f"./data/{media_type}/{log}.json", "r", encoding='utf-8') as log_file:
log_items = json.load(log_file) log_items = json.load(log_file)
log_items.insert(0, item) log_items.insert(0, item)
with open(f"./data/{media_type}/{log}.json", "w") as log_file: with open(f"./data/{media_type}/{log}.json", "w", encoding='utf-8') as log_file:
json.dump(log_items, log_file, indent=4) json.dump(log_items, log_file, indent=4)
logger.info(f"Added {media_type} {item_id} to {log}") logger.info(f"Added {media_type} {item_id} to {log}")
def import_by_id(import_id, media_type): def import_by_id(import_id, media_type) -> dict:
"""Import from the appropriate API by unique ID"""
if media_type in ["films", "tv-series"]: if media_type in ["films", "tv-series"]:
return import_from_imdb_by_id(import_id, media_type) return import_from_imdb_by_id(import_id, media_type)
elif media_type in ["tv-episodes"]: if media_type in ["tv-episodes"]:
return #import_from_tvdb_by_id(import_id, media_type) return #import_from_tvdb_by_id(import_id, media_type)
elif media_type in ["books"]: if media_type in ["books"]:
return import_from_openlibrary_by_id(import_id, media_type) return import_from_openlibrary_by_id(import_id, media_type)
def import_from_imdb_by_id(imdb_id, media_type): def import_from_imdb_by_id(imdb_id, media_type) -> dict:
"""Retrieve a film, TV show or TV episode from TMDB using an IMDB ID""" """Retrieve a film, TV show or TV episode from TMDB using an IMDB ID"""
api_url = f"https://api.themoviedb.org/3/find/{imdb_id}" api_url = f"https://api.themoviedb.org/3/find/{imdb_id}"
@ -125,6 +130,7 @@ def import_from_imdb_by_id(imdb_id, media_type):
api_url, api_url,
params={"external_source": "imdb_id"}, params={"external_source": "imdb_id"},
headers={"Authorization": f"Bearer {TMDB_API_KEY}"}, headers={"Authorization": f"Bearer {TMDB_API_KEY}"},
timeout=15
) )
# Process the response # Process the response
@ -133,8 +139,7 @@ def import_from_imdb_by_id(imdb_id, media_type):
elif 429 == response.status_code: elif 429 == response.status_code:
time.sleep(2) time.sleep(2)
import_from_imdb_by_id(imdb_id, media_type) return import_from_imdb_by_id(imdb_id, media_type)
return
else: else:
raise Exception(f"Error {response.status_code}: {response.text}") raise Exception(f"Error {response.status_code}: {response.text}")
@ -161,20 +166,20 @@ def import_from_imdb_by_id(imdb_id, media_type):
try: try:
item = response_data[int(idx)] item = response_data[int(idx)]
except: except Exception as exc:
raise Exception(f"Index {idx} is invalid") raise Exception(f"Index {idx} is invalid") from exc
# Modify the returned result to add additional data # Modify the returned result to add additional data
return cleanup_result(item, media_type) return cleanup_result(item, media_type)
def import_from_openlibrary_by_id(isbn, media_type): def import_from_openlibrary_by_id(isbn, media_type) -> dict:
"""Retrieve a film, TV show or TV episode from TMDB using an IMDB ID""" """Retrieve a film, TV show or TV episode from TMDB using an IMDB ID"""
api_url = f"https://openlibrary.org/isbn/{isbn}" api_url = f"https://openlibrary.org/isbn/{isbn}"
# Sending API request # Sending API request
response = requests.get(api_url, headers={"accept": "application/json"}) response = requests.get(api_url, headers={"accept": "application/json"}, timeout=15)
# Process the response # Process the response
if 200 == response.status_code: if 200 == response.status_code:
@ -182,8 +187,7 @@ def import_from_openlibrary_by_id(isbn, media_type):
elif 429 == response.status_code: elif 429 == response.status_code:
time.sleep(2) time.sleep(2)
import_from_openlibrary_by_id(isbn, media_type) return import_from_openlibrary_by_id(isbn, media_type)
return
else: else:
raise Exception(f"Error {response.status_code}: {response.text}") raise Exception(f"Error {response.status_code}: {response.text}")
@ -199,7 +203,6 @@ def import_from_openlibrary_by_id(isbn, media_type):
if len(item["works"]) > 1: if len(item["works"]) > 1:
raise Exception(f"Multiple works found for {isbn}") raise Exception(f"Multiple works found for {isbn}")
else:
item["work"] = item["works"][0] item["work"] = item["works"][0]
del item["works"] del item["works"]
@ -207,8 +210,8 @@ def import_from_openlibrary_by_id(isbn, media_type):
return cleanup_result(item, media_type) return cleanup_result(item, media_type)
def import_from_openlibrary_by_ol_key(key): def import_from_openlibrary_by_ol_key(key) -> dict:
"""Retrieves an item (author or work) from OpenLibrary using an OL key""" """Retrieves an item (author or work, NOT edition) from OpenLibrary using an OL key"""
_, mode, ol_id = key.split("/") _, mode, ol_id = key.split("/")
@ -216,7 +219,7 @@ def import_from_openlibrary_by_ol_key(key):
api_url = f"https://openlibrary.org{key}" api_url = f"https://openlibrary.org{key}"
# Sending API request # Sending API request
response = requests.get(api_url, headers={"accept": "application/json"}) response = requests.get(api_url, headers={"accept": "application/json"}, timeout=15)
# Process the response # Process the response
if 200 == response.status_code: if 200 == response.status_code:
@ -224,8 +227,7 @@ def import_from_openlibrary_by_ol_key(key):
elif 429 == response.status_code: elif 429 == response.status_code:
time.sleep(2) time.sleep(2)
import_from_openlibrary_by_ol_key(key) return import_from_openlibrary_by_ol_key(key)
return
else: else:
raise Exception(f"Error {response.status_code}: {response.text}") raise Exception(f"Error {response.status_code}: {response.text}")
@ -241,12 +243,12 @@ def import_from_openlibrary_by_ol_key(key):
return author return author
elif "works" == mode: if "works" == mode:
work = {"id": ol_id, "title": item["title"]} work = {"id": ol_id, "title": item["title"]}
for key in ["first_publish_date", "subjects"]: for result_key in ["first_publish_date", "subjects"]:
if key in item: if result_key in item:
work[key] = item[key] work[result_key] = item[result_key]
return work return work
@ -254,8 +256,9 @@ def import_from_openlibrary_by_ol_key(key):
raise Exception(f"Unknown OpenLibrary key '{mode}'") raise Exception(f"Unknown OpenLibrary key '{mode}'")
def cleanup_result(item, media_type): def cleanup_result(item, media_type) -> dict:
"""Process a film, TV series, TV episode or book returned by their respecitve APIs by removing unnecessary fields and adding others""" """Process a film, TV series, TV episode or book returned by their
respective APIs by removing unnecessary fields and adding others"""
for field_name in [ for field_name in [
"adult", # TMDB "adult", # TMDB
@ -307,14 +310,12 @@ def cleanup_result(item, media_type):
if len(item[key]) > 1: if len(item[key]) > 1:
raise Exception("Multiple ISBN results") raise Exception("Multiple ISBN results")
else:
item[key] = item[key][0] item[key] = item[key][0]
if "publish_places" in item: if "publish_places" in item:
if len(item["publish_places"]) > 1: if len(item["publish_places"]) > 1:
raise Exception("Multiple publish_places") raise Exception("Multiple publish_places")
else:
item["published_in"] = item["publish_places"][0] item["published_in"] = item["publish_places"][0]
del item["publish_places"] del item["publish_places"]
@ -328,14 +329,14 @@ def cleanup_result(item, media_type):
del item["translation_of"] del item["translation_of"]
else: else:
raise Exception( raise Exception(
f"translation_of '{item['translation_of']}' is different to work title '{item['work']['title']}'" f"translation_of '{item['translation_of']}' \
is different to work title '{item['work']['title']}'"
) )
if "translated_from" in item: if "translated_from" in item:
if len(item["translated_from"]) > 1: if len(item["translated_from"]) > 1:
raise Exception("Multiple translated_from results") raise Exception("Multiple translated_from results")
else:
item["work"]["original_language"] = item["translated_from"][0][ item["work"]["original_language"] = item["translated_from"][0][
"key" "key"
].split("/")[2] ].split("/")[2]
@ -347,7 +348,9 @@ def cleanup_result(item, media_type):
return item return item
def main(): def main() -> None:
"""Prompt user to select media type and log to process"""
media_type = "" media_type = ""
while media_type not in ["films", "tv-episodes", "tv-series", "books"]: while media_type not in ["films", "tv-episodes", "tv-series", "books"]:
media_type = input("Select media type [films|tv-episodes|tv-series|books]: ") media_type = input("Select media type [films|tv-episodes|tv-series|books]: ")
@ -393,9 +396,8 @@ def main():
add_item_to_log(imdb_id, media_type, log) add_item_to_log(imdb_id, media_type, log)
except Exception as error: except Exception:
logger.exception("Exception occurred") logger.exception("Exception occurred")
print(error)
if __name__ == "__main__": if __name__ == "__main__":

View file

@ -1,10 +1,13 @@
from dotenv import load_dotenv """
Process logs derived from social cataloguing site data exports, using various APIs.
"""
import json import json
import os import os
import re import re
import requests
import time import time
from urllib.request import urlopen import requests
from dotenv import load_dotenv
from add_item import cleanup_result, import_by_id, setup_logger from add_item import cleanup_result, import_by_id, setup_logger
logger = setup_logger("process_logs") logger = setup_logger("process_logs")
@ -20,12 +23,12 @@ if "" == TVDB_API_KEY:
logger.warning("TVDB API key not found") logger.warning("TVDB API key not found")
def process_log(media_type, log): def process_log(media_type, log) -> None:
"""Run through a log and call the appropriate API for each item found""" """Run through a log and call the appropriate API for each item found"""
logger.info(f"Processing {media_type}/{log}") logger.info(f"Processing {media_type}/{log}")
with open(f"./data/{media_type}/{log}.json", "r") as log_file: with open(f"./data/{media_type}/{log}.json", "r", encoding='utf-8') as log_file:
log_items = json.load(log_file) log_items = json.load(log_file)
log_item_values = {} log_item_values = {}
@ -105,7 +108,11 @@ def process_log(media_type, log):
if re.search("tt[0-9]+", item["imdb_id"]) is not None: if re.search("tt[0-9]+", item["imdb_id"]) is not None:
log_items[i] = import_by_id(item["imdb_id"], media_type) log_items[i] = import_by_id(item["imdb_id"], media_type)
with open(f"./data/{media_type}/{log}.json", "w") as log_file: with open(
f"./data/{media_type}/{log}.json",
"w",
encoding='utf-8'
) as log_file:
json.dump(log_items, log_file, indent=4) json.dump(log_items, log_file, indent=4)
else: else:
@ -115,7 +122,11 @@ def process_log(media_type, log):
log_items[i] = new_log_item log_items[i] = new_log_item
if i % 15 == 0: if i % 15 == 0:
with open(f"./data/{media_type}/{log}.json", "w") as log_file: with open(
f"./data/{media_type}/{log}.json",
"w",
encoding='utf-8'
) as log_file:
json.dump(log_items, log_file, indent=4) json.dump(log_items, log_file, indent=4)
if log_items[i] is not None: if log_items[i] is not None:
@ -124,29 +135,29 @@ def process_log(media_type, log):
except KeyError: except KeyError:
print(json.dumps(item, indent=4)) print(json.dumps(item, indent=4))
with open(f"./data/{media_type}/{log}.json", "w") as log_file: with open(f"./data/{media_type}/{log}.json", "w", encoding='utf-8') as log_file:
json.dump(log_items, log_file, indent=4) json.dump(log_items, log_file, indent=4)
logger.info(f"Finished processing {media_type}/{log}") logger.info(f"Finished processing {media_type}/{log}")
def import_by_details(item, item_title, media_type): def import_by_details(item, item_title, media_type) -> dict:
"""Import an item when lacking a unique identifier""" """Import an item when lacking a unique identifier"""
if media_type in ["films", "tv-series"]: if media_type in ["films", "tv-series"]:
return import_from_tmdb_by_details(item, item_title, media_type) return import_from_tmdb_by_details(item, item_title, media_type)
elif media_type in ["tv-episodes"]: if media_type in ["tv-episodes"]:
return # import_from_tvdb_by_details(item, item_title, media_type) return # import_from_tvdb_by_details(item, item_title, media_type)
elif media_type in ["books"]: if media_type in ["books"]:
return # import_from_openlibrary_by_details(item, item_title, media_type) return # import_from_openlibrary_by_details(item, item_title, media_type)
elif media_type in ["games"]: if media_type in ["games"]:
return # import_from_igdb_by_details(item, item_title, media_type) return # import_from_igdb_by_details(item, item_title, media_type)
def import_from_tmdb_by_details(item, item_title, media_type): def import_from_tmdb_by_details(item, item_title, media_type) -> dict:
"""Retrieve a film or TV series from TMDB using its title""" """Retrieve a film or TV series from TMDB using its title"""
logger.info(f"Processing {item_title}") logger.info(f"Processing {item_title}")
@ -162,6 +173,7 @@ def import_from_tmdb_by_details(item, item_title, media_type):
"year": item["Release Year"] if "Release Year" in item else None, "year": item["Release Year"] if "Release Year" in item else None,
}, },
headers={"Authorization": f"Bearer {TMDB_API_KEY}"}, headers={"Authorization": f"Bearer {TMDB_API_KEY}"},
timeout=15
) )
# Process the response # Process the response
@ -169,7 +181,7 @@ def import_from_tmdb_by_details(item, item_title, media_type):
logger.debug(response.status_code) logger.debug(response.status_code)
elif 429 == response.status_code: elif 429 == response.status_code:
time.sleep(2) time.sleep(2)
import_from_tmdb_by_details(item) return import_from_tmdb_by_details(item, item_title, media_type)
else: else:
logger.error(response.text) logger.error(response.text)
@ -178,7 +190,7 @@ def import_from_tmdb_by_details(item, item_title, media_type):
if 1 == len(response_data): if 1 == len(response_data):
return cleanup_result(response_data[0], media_type) return cleanup_result(response_data[0], media_type)
elif 0 == len(response_data): if 0 == len(response_data):
logger.warning(f"Returned no {media_type} for {item_title}") logger.warning(f"Returned no {media_type} for {item_title}")
elif 1 < len(response_data): elif 1 < len(response_data):
@ -190,11 +202,11 @@ def import_from_tmdb_by_details(item, item_title, media_type):
filtered_response_data = [ filtered_response_data = [
result for result in response_data if result[title_key] == item_title result for result in response_data if result[title_key] == item_title
] ]
frd_len = len(filtered_response_data)
if 1 == len(filtered_response_data): if 1 == frd_len:
return cleanup_result(response_data[0], media_type) return cleanup_result(response_data[0], media_type)
else:
logger.warning(f"Returned more than one {media_type} for '{item_title}':\n") logger.warning(f"Returned more than one {media_type} for '{item_title}':\n")
print( print(
json.dumps( json.dumps(
@ -204,28 +216,31 @@ def import_from_tmdb_by_details(item, item_title, media_type):
indent=4, indent=4,
) )
) )
last_index = len(filtered_response_data if frd_len > 0 else response_data) - 1
idx = input( idx = input(
f"\nEnter the index of the result to use [0-{len(filtered_response_data if len(filtered_response_data) > 0 else response_data) - 1}]: " f"\nEnter the index of the result to use [0-{last_index}]: "
) )
if "" != idx: if "" != idx:
try: try:
return cleanup_result(response_data[int(idx)], media_type) return cleanup_result(response_data[int(idx)], media_type)
except: except Exception as exc:
logger.error("Index invalid!") raise Exception("Index invalid") from exc
print("Index invalid!")
item["IMDB ID"] = input(f"Enter IMDB ID for {item_title}: ") item["IMDB ID"] = input(f"Enter IMDB ID for {item_title}: ")
if "" != item["IMDB ID"]: if "" != item["IMDB ID"]:
return import_by_id(item["IMDB ID"], media_type) return import_by_id(item["IMDB ID"], media_type)
else:
logger.warning(f"Skipped {item_title}") logger.warning(f"Skipped {item_title}")
return item return item
def main(): def main() -> None:
"""Prompt user to select media type and log to process"""
media_type = "" media_type = ""
while media_type not in ["films", "tv-episodes", "tv-series", "books"]: while media_type not in ["films", "tv-episodes", "tv-series", "books"]:
media_type = input("Select media type [films|tv-episodes|tv-series|books]: ") media_type = input("Select media type [films|tv-episodes|tv-series|books]: ")
@ -257,9 +272,8 @@ def main():
process_log(media_type, log) process_log(media_type, log)
except Exception as error: except Exception:
logger.exception("Exception occurred") logger.exception("Exception occurred")
print(error)
if __name__ == "__main__": if __name__ == "__main__":