From f5de09bd095fd326e499387df7b711aefb167b99 Mon Sep 17 00:00:00 2001 From: mungai-njoroge Date: Sat, 1 Jul 2023 01:39:39 +0300 Subject: [PATCH] add last fm similar artists to db table + add db methods for the above + try and discard last fm store --- app/api/album.py | 5 +- app/api/artist.py | 63 +++++++++++++--------- app/db/sqlite/lastfm/__init__.py | 0 app/db/sqlite/lastfm/similar_artists.py | 62 +++++++++++++++++++++ app/db/sqlite/queries.py | 11 ++-- app/lib/populate.py | 71 +++++++++++++++++++++---- app/models/album.py | 8 ++- app/models/lastfm.py | 13 +++++ app/requests/artists.py | 19 +++++-- app/setup/__init__.py | 5 +- app/utils/parsers.py | 1 + 11 files changed, 209 insertions(+), 49 deletions(-) create mode 100644 app/db/sqlite/lastfm/__init__.py create mode 100644 app/db/sqlite/lastfm/similar_artists.py create mode 100644 app/models/lastfm.py diff --git a/app/api/album.py b/app/api/album.py index e4e51100..11105287 100644 --- a/app/api/album.py +++ b/app/api/album.py @@ -100,7 +100,6 @@ def get_album_tracks(albumhash: str): t["_pos"] = int(f"{t['disc']}{track}") tracks = sorted(tracks, key=lambda t: t["_pos"]) - tracks = [track_serializer(t, _remove={"_pos"}) for t in tracks] return {"tracks": tracks} @@ -164,4 +163,8 @@ def get_album_versions(): and create_hash(og_album_title) != create_hash(a.og_title) ] + for a in albums: + tracks = TrackStore.get_tracks_by_albumhash(a.albumhash) + a.get_date_from_tracks(tracks) + return {"data": albums} diff --git a/app/api/artist.py b/app/api/artist.py index efe180c1..c1a589b4 100644 --- a/app/api/artist.py +++ b/app/api/artist.py @@ -2,10 +2,13 @@ Contains all the artist(s) routes. """ from collections import deque +import random from flask import Blueprint, request from app.db.sqlite.favorite import SQLiteFavoriteMethods as favdb +from app.db.sqlite.lastfm.similar_artists import SQLiteLastFMSimilarArtists as fmdb + from app.models import Album, FavType, Track from app.serializers.album import serialize_for_card_many from app.serializers.track import serialize_tracks @@ -141,6 +144,11 @@ class ArtistsCache: album_tracks = remove_duplicates(album_tracks) album.get_date_from_tracks(album_tracks) + + if album.date == 0: + AlbumStore.remove_album_by_hash(album.albumhash) + continue + album.check_is_single(album_tracks) entry.type_checked = True @@ -288,28 +296,33 @@ def get_all_artist_tracks(artisthash: str): return {"tracks": serialize_tracks(tracks)} -# -# @api.route("/artist//similar", methods=["GET"]) -# def get_similar_artists(artisthash: str): -# """ -# Returns similar artists. -# """ -# limit = request.args.get("limit") -# -# if limit is None: -# limit = 6 -# -# limit = int(limit) -# -# artist = ArtistStore.get_artist_by_hash(artisthash) -# -# if artist is None: -# return {"error": "Artist not found"}, 404 -# -# similar_hashes = fetch_similar_artists(artist.name) -# similar = ArtistStore.get_artists_by_hashes(similar_hashes) -# -# if len(similar) > limit: -# similar = random.sample(similar, limit) -# -# return {"similar": similar[:limit]} +@api.route("/artist//similar", methods=["GET"]) +def get_similar_artists(artisthash: str): + """ + Returns similar artists. + """ + limit = request.args.get("limit") + + if limit is None: + limit = 6 + + limit = int(limit) + + artist = ArtistStore.get_artist_by_hash(artisthash) + + if artist is None: + return {"error": "Artist not found"}, 404 + + # result = LastFMStore.get_similar_artists_for(artist.artisthash) + result = fmdb.get_similar_artists_for(artist.artisthash) + + if result is None: + return {"artists": []} + + similar = ArtistStore.get_artists_by_hashes(result.get_artist_hash_set()) + + # print(similar) + if len(similar) > limit: + similar = random.sample(similar, limit) + + return {"artists": similar[:limit]} diff --git a/app/db/sqlite/lastfm/__init__.py b/app/db/sqlite/lastfm/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/app/db/sqlite/lastfm/similar_artists.py b/app/db/sqlite/lastfm/similar_artists.py new file mode 100644 index 00000000..22a5825c --- /dev/null +++ b/app/db/sqlite/lastfm/similar_artists.py @@ -0,0 +1,62 @@ +from app.models.lastfm import SimilarArtist + +from ..utils import SQLiteManager + + +class SQLiteLastFMSimilarArtists: + """ + This class contains methods for interacting with the lastfm_similar_artists table. + """ + + @classmethod + def insert_one(cls, artist: SimilarArtist): + """ + Inserts a single artist into the database. + """ + sql = """INSERT OR REPLACE INTO lastfm_similar_artists(artisthash, similar_artists) VALUES(?,?)""" + + with SQLiteManager(userdata_db=True) as cur: + cur.execute(sql, (artist.artisthash, artist.similar_artist_hashes)) + cur.close() + + @classmethod + def get_similar_artists_for(cls, artisthash: str): + """ + Returns a list of similar artists. + """ + sql = """SELECT * FROM lastfm_similar_artists WHERE artisthash = ?""" + with SQLiteManager(userdata_db=True) as cur: + cur.execute(sql, (artisthash,)) + similar_artists = cur.fetchone() + cur.close() + + if similar_artists is None: + return None + + return SimilarArtist(artisthash, similar_artists[2]) + + @classmethod + def get_all(cls): + """ + Returns a list of all similar artists. + """ + sql = """SELECT * FROM lastfm_similar_artists""" + with SQLiteManager(userdata_db=True) as cur: + cur.execute(sql) + similar_artists = cur.fetchall() + cur.close() + + for a in similar_artists: + yield SimilarArtist(a[1], a[2]) + + @classmethod + def exists(cls, artisthash: str): + """ + Checks if an artist exists in the database by counting the number of rows + """ + sql = """SELECT COUNT(*) FROM lastfm_similar_artists WHERE artisthash = ?""" + with SQLiteManager(userdata_db=True) as cur: + cur.execute(sql, (artisthash,)) + count = cur.fetchone()[0] + cur.close() + return count > 0 diff --git a/app/db/sqlite/queries.py b/app/db/sqlite/queries.py index 7613cd54..9616c7f4 100644 --- a/app/db/sqlite/queries.py +++ b/app/db/sqlite/queries.py @@ -26,7 +26,14 @@ CREATE TABLE IF NOT EXISTS settings ( root_dirs text NOT NULL, exclude_dirs text, artist_separators text -) +); + +CREATE TABLE IF NOT EXISTS lastfm_similar_artists ( + id integer PRIMARY KEY, + artisthash text NOT NULL, + similar_artists text NOT NULL, + UNIQUE (artisthash) +); """ CREATE_APPDB_TABLES = """ @@ -58,8 +65,6 @@ CREATE TABLE IF NOT EXISTS albums ( UNIQUE (albumhash) ); - - CREATE TABLE IF NOT EXISTS artists ( id integer PRIMARY KEY, artisthash text NOT NULL, diff --git a/app/lib/populate.py b/app/lib/populate.py index 8ffb8b14..784eab16 100644 --- a/app/lib/populate.py +++ b/app/lib/populate.py @@ -1,26 +1,29 @@ -from collections import deque +import json import os +from collections import deque from typing import Generator -from tqdm import tqdm + from requests import ConnectionError as RequestConnectionError from requests import ReadTimeout +from tqdm import tqdm from app import settings -from app.db.sqlite.tracks import SQLiteTrackMethods -from app.db.sqlite.settings import SettingsSQLMethods as sdb from app.db.sqlite.favorite import SQLiteFavoriteMethods as favdb +from app.db.sqlite.lastfm.similar_artists import SQLiteLastFMSimilarArtists as lastfmdb +from app.db.sqlite.settings import SettingsSQLMethods as sdb +from app.db.sqlite.tracks import SQLiteTrackMethods from app.lib.artistlib import CheckArtistImages from app.lib.colorlib import ProcessAlbumColors, ProcessArtistColors - from app.lib.taglib import extract_thumb, get_tags from app.lib.trackslib import validate_tracks from app.logger import log from app.models import Album, Artist, Track -from app.utils.filesystem import run_fast_scandir - +from app.models.lastfm import SimilarArtist +from app.requests.artists import fetch_similar_artists from app.store.albums import AlbumStore -from app.store.tracks import TrackStore from app.store.artists import ArtistStore +from app.store.tracks import TrackStore +from app.utils.filesystem import run_fast_scandir from app.utils.network import Ping get_all_tracks = SQLiteTrackMethods.get_all_tracks @@ -100,6 +103,9 @@ class Populate: if tried_to_download_new_images: ProcessArtistColors() + if Ping()(): + FetchSimilarArtistsLastFM() + @staticmethod def remove_modified(tracks: Generator[Track, None, None]): """ @@ -111,9 +117,14 @@ class Populate: modified = set() for track in tracks: - if track.last_mod == os.path.getmtime(track.filepath): - unmodified.add(track.filepath) - continue + try: + if track.last_mod == os.path.getmtime(track.filepath): + unmodified.add(track.filepath) + continue + except FileNotFoundError: + print(f"File not found: {track.filepath}") + TrackStore.tracks.remove(track) + remove_tracks_by_filepaths(track.filepath) modified.add(track.filepath) @@ -214,3 +225,41 @@ class ProcessTrackThumbnails: ) list(results) + + +def save_similar_artists(artist: Artist): + """ + Downloads and saves similar artists to the database. + """ + + if lastfmdb.exists(artist.artisthash): + return + + artist_hashes = fetch_similar_artists(artist.name) + artist_ = SimilarArtist(artist.artisthash, "~".join(artist_hashes)) + + if len(artist_.similar_artist_hashes) == 0: + return + + print(artist.artisthash, artist.name) + lastfmdb.insert_one(artist_) + + +class FetchSimilarArtistsLastFM: + """ + Fetches similar artists from LastFM using a process pool. + """ + + def __init__(self) -> None: + artists = ArtistStore.artists + + with Pool(processes=cpu_count()) as pool: + results = list( + tqdm( + pool.imap_unordered(save_similar_artists, artists), + total=len(artists), + desc="Downloading similar artists", + ) + ) + + list(results) diff --git a/app/models/album.py b/app/models/album.py index 14c54d0c..17867473 100644 --- a/app/models/album.py +++ b/app/models/album.py @@ -43,6 +43,7 @@ class Album: self.og_title = self.title self.image = self.albumhash + ".webp" + # Fetch album artists from title if get_flag(ParserFlags.EXTRACT_FEAT): featured, self.title = parse_feat_from_title(self.title) @@ -56,6 +57,7 @@ class Album: TrackStore.append_track_artists(self.albumhash, featured, self.title) + # Handle album version data if get_flag(ParserFlags.CLEAN_ALBUM_TITLE): get_versions = not get_flag(ParserFlags.MERGE_ALBUM_VERSIONS) @@ -66,6 +68,8 @@ class Album: if "super_deluxe" in self.versions: self.versions.remove("deluxe") + + self.versions = [v.replace("_", " ") for v in self.versions] else: self.base_title = get_base_title_and_versions( self.title, get_versions=False @@ -180,10 +184,12 @@ class Album: Args: tracks (list[Track]): The tracks of the album. """ + if self.date: + return + dates = {t.date for t in tracks if t.date} if len(dates) == 0: self.date = 0 - return self.date = datetime.datetime.fromtimestamp(min(dates)).year diff --git a/app/models/lastfm.py b/app/models/lastfm.py new file mode 100644 index 00000000..ebd2333c --- /dev/null +++ b/app/models/lastfm.py @@ -0,0 +1,13 @@ +from dataclasses import dataclass + + +@dataclass +class SimilarArtist: + artisthash: str + similar_artist_hashes: str + + def get_artist_hash_set(self) -> set[str]: + """ + Returns a set of similar artists. + """ + return set(self.similar_artist_hashes.split("~")) diff --git a/app/requests/artists.py b/app/requests/artists.py index 9ffb8ce4..f9c1a99b 100644 --- a/app/requests/artists.py +++ b/app/requests/artists.py @@ -1,24 +1,33 @@ """ Requests related to artists """ +from pprint import pprint import requests from app import settings from app.utils.hashing import create_hash +from requests import ConnectionError +import urllib.parse def fetch_similar_artists(name: str): """ Fetches similar artists from Last.fm """ - url = f"https://ws.audioscrobbler.com/2.0/?method=artist.getsimilar&artist={name}&api_key=" \ - f"{settings.Keys.LASTFM_API}&format=json&limit=250" + url = f"https://ws.audioscrobbler.com/2.0/?method=artist.getsimilar&artist={urllib.parse.quote_plus(name, safe='')}&api_key={settings.Keys.LASTFM_API}&format=json&limit=250" - response = requests.get(url, timeout=10) - response.raise_for_status() + try: + response = requests.get(url, timeout=10) + response.raise_for_status() + except ConnectionError: + return [] data = response.json() - artists = data["similarartists"]["artist"] + + try: + artists = data["similarartists"]["artist"] + except KeyError: + return [] for artist in artists: yield create_hash(artist["name"]) diff --git a/app/setup/__init__.py b/app/setup/__init__.py index 4661897a..233edc45 100644 --- a/app/setup/__init__.py +++ b/app/setup/__init__.py @@ -2,11 +2,10 @@ Prepares the server for use. """ from app.setup.files import create_config_dir -from app.setup.sqlite import setup_sqlite, run_migrations - +from app.setup.sqlite import run_migrations, setup_sqlite from app.store.albums import AlbumStore -from app.store.tracks import TrackStore from app.store.artists import ArtistStore +from app.store.tracks import TrackStore def run_setup(): diff --git a/app/utils/parsers.py b/app/utils/parsers.py index 604a7571..a9349b96 100644 --- a/app/utils/parsers.py +++ b/app/utils/parsers.py @@ -132,6 +132,7 @@ class AlbumVersionEnum(Enum): DELUXE = ("deluxe",) SUPER_DELUXE = ("super deluxe",) + COMPLETE = ("complete",) LEGACY = ("legacy",) SPECIAL = ("special",)