add last fm similar artists to db table

+ add db methods for the above
+ try and discard last fm store

app/api/album.py

@@ -100,7 +100,6 @@ def get_album_tracks(albumhash: str):
         t["_pos"] = int(f"{t['disc']}{track}")
 
     tracks = sorted(tracks, key=lambda t: t["_pos"])
-    tracks = [track_serializer(t, _remove={"_pos"}) for t in tracks]
 
     return {"tracks": tracks}
 
@@ -164,4 +163,8 @@ def get_album_versions():
         and create_hash(og_album_title) != create_hash(a.og_title)
     ]
 
+    for a in albums:
+        tracks = TrackStore.get_tracks_by_albumhash(a.albumhash)
+        a.get_date_from_tracks(tracks)
+
     return {"data": albums}

app/api/artist.py

@@ -2,10 +2,13 @@
 Contains all the artist(s) routes.
 """
 from collections import deque
+import random
 
 from flask import Blueprint, request
 
 from app.db.sqlite.favorite import SQLiteFavoriteMethods as favdb
+from app.db.sqlite.lastfm.similar_artists import SQLiteLastFMSimilarArtists as fmdb
+
 from app.models import Album, FavType, Track
 from app.serializers.album import serialize_for_card_many
 from app.serializers.track import serialize_tracks
@@ -141,6 +144,11 @@ class ArtistsCache:
             album_tracks = remove_duplicates(album_tracks)
 
             album.get_date_from_tracks(album_tracks)
+
+            if album.date == 0:
+                AlbumStore.remove_album_by_hash(album.albumhash)
+                continue
+
             album.check_is_single(album_tracks)
 
         entry.type_checked = True
@@ -288,28 +296,33 @@ def get_all_artist_tracks(artisthash: str):
     return {"tracks": serialize_tracks(tracks)}
 
 
-#
+@api.route("/artist/<artisthash>/similar", methods=["GET"])
-# @api.route("/artist/<artisthash>/similar", methods=["GET"])
+def get_similar_artists(artisthash: str):
-# def get_similar_artists(artisthash: str):
+    """
-#     """
+    Returns similar artists.
-#     Returns similar artists.
+    """
-#     """
+    limit = request.args.get("limit")
-#     limit = request.args.get("limit")
+
-#
+    if limit is None:
-#     if limit is None:
+        limit = 6
-#         limit = 6
+
-#
+    limit = int(limit)
-#     limit = int(limit)
+
-#
+    artist = ArtistStore.get_artist_by_hash(artisthash)
-#     artist = ArtistStore.get_artist_by_hash(artisthash)
+
-#
+    if artist is None:
-#     if artist is None:
+        return {"error": "Artist not found"}, 404
-#         return {"error": "Artist not found"}, 404
+
-#
+    # result = LastFMStore.get_similar_artists_for(artist.artisthash)
-#     similar_hashes = fetch_similar_artists(artist.name)
+    result = fmdb.get_similar_artists_for(artist.artisthash)
-#     similar = ArtistStore.get_artists_by_hashes(similar_hashes)
+
-#
+    if result is None:
-#     if len(similar) > limit:
+        return {"artists": []}
-#         similar = random.sample(similar, limit)
+
-#
+    similar = ArtistStore.get_artists_by_hashes(result.get_artist_hash_set())
-#     return {"similar": similar[:limit]}
+
+    # print(similar)
+    if len(similar) > limit:
+        similar = random.sample(similar, limit)
+
+    return {"artists": similar[:limit]}

app/db/sqlite/lastfm/similar_artists.py

@@ -0,0 +1,62 @@
+from app.models.lastfm import SimilarArtist
+
+from ..utils import SQLiteManager
+
+
+class SQLiteLastFMSimilarArtists:
+    """
+    This class contains methods for interacting with the lastfm_similar_artists table.
+    """
+
+    @classmethod
+    def insert_one(cls, artist: SimilarArtist):
+        """
+        Inserts a single artist into the database.
+        """
+        sql = """INSERT OR REPLACE INTO lastfm_similar_artists(artisthash, similar_artists) VALUES(?,?)"""
+
+        with SQLiteManager(userdata_db=True) as cur:
+            cur.execute(sql, (artist.artisthash, artist.similar_artist_hashes))
+            cur.close()
+
+    @classmethod
+    def get_similar_artists_for(cls, artisthash: str):
+        """
+        Returns a list of similar artists.
+        """
+        sql = """SELECT * FROM lastfm_similar_artists WHERE artisthash = ?"""
+        with SQLiteManager(userdata_db=True) as cur:
+            cur.execute(sql, (artisthash,))
+            similar_artists = cur.fetchone()
+            cur.close()
+
+            if similar_artists is None:
+                return None
+
+            return SimilarArtist(artisthash, similar_artists[2])
+
+    @classmethod
+    def get_all(cls):
+        """
+        Returns a list of all similar artists.
+        """
+        sql = """SELECT * FROM lastfm_similar_artists"""
+        with SQLiteManager(userdata_db=True) as cur:
+            cur.execute(sql)
+            similar_artists = cur.fetchall()
+            cur.close()
+
+            for a in similar_artists:
+                yield SimilarArtist(a[1], a[2])
+
+    @classmethod
+    def exists(cls, artisthash: str):
+        """
+        Checks if an artist exists in the database by counting the number of rows
+        """
+        sql = """SELECT COUNT(*) FROM lastfm_similar_artists WHERE artisthash = ?"""
+        with SQLiteManager(userdata_db=True) as cur:
+            cur.execute(sql, (artisthash,))
+            count = cur.fetchone()[0]
+            cur.close()
+            return count > 0

app/db/sqlite/queries.py

@@ -26,7 +26,14 @@ CREATE TABLE IF NOT EXISTS settings (
     root_dirs text NOT NULL,
     exclude_dirs text,
     artist_separators text
-)
+);
+
+CREATE TABLE IF NOT EXISTS lastfm_similar_artists (
+    id integer PRIMARY KEY,
+    artisthash text NOT NULL,
+    similar_artists text NOT NULL,
+    UNIQUE (artisthash)
+);
 """
 
 CREATE_APPDB_TABLES = """
@@ -58,8 +65,6 @@ CREATE TABLE IF NOT EXISTS albums (
     UNIQUE (albumhash)
 );
 
-
-
 CREATE TABLE IF NOT EXISTS artists (
     id integer PRIMARY KEY,
     artisthash text NOT NULL,

app/lib/populate.py

@@ -1,26 +1,29 @@
-from collections import deque
+import json
 import os
+from collections import deque
 from typing import Generator
-from tqdm import tqdm
+
 from requests import ConnectionError as RequestConnectionError
 from requests import ReadTimeout
+from tqdm import tqdm
 
 from app import settings
-from app.db.sqlite.tracks import SQLiteTrackMethods
-from app.db.sqlite.settings import SettingsSQLMethods as sdb
 from app.db.sqlite.favorite import SQLiteFavoriteMethods as favdb
+from app.db.sqlite.lastfm.similar_artists import SQLiteLastFMSimilarArtists as lastfmdb
+from app.db.sqlite.settings import SettingsSQLMethods as sdb
+from app.db.sqlite.tracks import SQLiteTrackMethods
 from app.lib.artistlib import CheckArtistImages
 from app.lib.colorlib import ProcessAlbumColors, ProcessArtistColors
-
 from app.lib.taglib import extract_thumb, get_tags
 from app.lib.trackslib import validate_tracks
 from app.logger import log
 from app.models import Album, Artist, Track
-from app.utils.filesystem import run_fast_scandir
+from app.models.lastfm import SimilarArtist
-
+from app.requests.artists import fetch_similar_artists
 from app.store.albums import AlbumStore
-from app.store.tracks import TrackStore
 from app.store.artists import ArtistStore
+from app.store.tracks import TrackStore
+from app.utils.filesystem import run_fast_scandir
 from app.utils.network import Ping
 
 get_all_tracks = SQLiteTrackMethods.get_all_tracks
@@ -100,6 +103,9 @@ class Populate:
         if tried_to_download_new_images:
             ProcessArtistColors()
 
+        if Ping()():
+            FetchSimilarArtistsLastFM()
+
     @staticmethod
     def remove_modified(tracks: Generator[Track, None, None]):
         """
@@ -111,9 +117,14 @@ class Populate:
         modified = set()
 
         for track in tracks:
+            try:
                 if track.last_mod == os.path.getmtime(track.filepath):
                     unmodified.add(track.filepath)
                     continue
+            except FileNotFoundError:
+                print(f"File not found: {track.filepath}")
+                TrackStore.tracks.remove(track)
+                remove_tracks_by_filepaths(track.filepath)
 
             modified.add(track.filepath)
 
@@ -214,3 +225,41 @@ class ProcessTrackThumbnails:
             )
 
             list(results)
+
+
+def save_similar_artists(artist: Artist):
+    """
+    Downloads and saves similar artists to the database.
+    """
+
+    if lastfmdb.exists(artist.artisthash):
+        return
+
+    artist_hashes = fetch_similar_artists(artist.name)
+    artist_ = SimilarArtist(artist.artisthash, "~".join(artist_hashes))
+
+    if len(artist_.similar_artist_hashes) == 0:
+        return
+
+    print(artist.artisthash, artist.name)
+    lastfmdb.insert_one(artist_)
+
+
+class FetchSimilarArtistsLastFM:
+    """
+    Fetches similar artists from LastFM using a process pool.
+    """
+
+    def __init__(self) -> None:
+        artists = ArtistStore.artists
+
+        with Pool(processes=cpu_count()) as pool:
+            results = list(
+                tqdm(
+                    pool.imap_unordered(save_similar_artists, artists),
+                    total=len(artists),
+                    desc="Downloading similar artists",
+                )
+            )
+
+            list(results)

app/models/album.py

@@ -43,6 +43,7 @@ class Album:
         self.og_title = self.title
         self.image = self.albumhash + ".webp"
 
+        # Fetch album artists from title
         if get_flag(ParserFlags.EXTRACT_FEAT):
             featured, self.title = parse_feat_from_title(self.title)
 
@@ -56,6 +57,7 @@ class Album:
 
                 TrackStore.append_track_artists(self.albumhash, featured, self.title)
 
+        # Handle album version data
         if get_flag(ParserFlags.CLEAN_ALBUM_TITLE):
             get_versions = not get_flag(ParserFlags.MERGE_ALBUM_VERSIONS)
 
@@ -66,6 +68,8 @@ class Album:
 
             if "super_deluxe" in self.versions:
                 self.versions.remove("deluxe")
+
+            self.versions = [v.replace("_", " ") for v in self.versions]
         else:
             self.base_title = get_base_title_and_versions(
                 self.title, get_versions=False
@@ -180,10 +184,12 @@ class Album:
         Args:
             tracks (list[Track]): The tracks of the album.
         """
+        if self.date:
+            return
+
         dates = {t.date for t in tracks if t.date}
 
         if len(dates) == 0:
             self.date = 0
-            return
 
         self.date = datetime.datetime.fromtimestamp(min(dates)).year

app/models/lastfm.py

@@ -0,0 +1,13 @@
+from dataclasses import dataclass
+
+
+@dataclass
+class SimilarArtist:
+    artisthash: str
+    similar_artist_hashes: str
+
+    def get_artist_hash_set(self) -> set[str]:
+        """
+        Returns a set of similar artists.
+        """
+        return set(self.similar_artist_hashes.split("~"))

app/requests/artists.py

@@ -1,24 +1,33 @@
 """
 Requests related to artists
 """
+from pprint import pprint
 import requests
 
 from app import settings
 from app.utils.hashing import create_hash
+from requests import ConnectionError
+import urllib.parse
 
 
 def fetch_similar_artists(name: str):
     """
     Fetches similar artists from Last.fm
     """
-    url = f"https://ws.audioscrobbler.com/2.0/?method=artist.getsimilar&artist={name}&api_key=" \
+    url = f"https://ws.audioscrobbler.com/2.0/?method=artist.getsimilar&artist={urllib.parse.quote_plus(name, safe='')}&api_key={settings.Keys.LASTFM_API}&format=json&limit=250"
-          f"{settings.Keys.LASTFM_API}&format=json&limit=250"
 
+    try:
         response = requests.get(url, timeout=10)
         response.raise_for_status()
+    except ConnectionError:
+        return []
 
     data = response.json()
+
+    try:
         artists = data["similarartists"]["artist"]
+    except KeyError:
+        return []
 
     for artist in artists:
         yield create_hash(artist["name"])

app/setup/__init__.py

@@ -2,11 +2,10 @@
 Prepares the server for use.
 """
 from app.setup.files import create_config_dir
-from app.setup.sqlite import setup_sqlite, run_migrations
+from app.setup.sqlite import run_migrations, setup_sqlite
-
 from app.store.albums import AlbumStore
-from app.store.tracks import TrackStore
 from app.store.artists import ArtistStore
+from app.store.tracks import TrackStore
 
 
 def run_setup():

app/utils/parsers.py

@@ -132,6 +132,7 @@ class AlbumVersionEnum(Enum):
 
     DELUXE = ("deluxe",)
     SUPER_DELUXE = ("super deluxe",)
+    COMPLETE = ("complete",)
 
     LEGACY = ("legacy",)
     SPECIAL = ("special",)