diff --git a/TODO.md b/TODO.md index a7a80a75..ee8a7879 100644 --- a/TODO.md +++ b/TODO.md @@ -51,3 +51,7 @@ - New table to hold similar artist entries - Create 2 way relationships, such that if an artist A is similar to another B with a certain weight, then artist B is similar to A with the same weight, unless overwritten. + +# Bug fixes + +- Duplicates on search \ No newline at end of file diff --git a/app/api/album.py b/app/api/album.py index aa84e7eb..1eb3660b 100644 --- a/app/api/album.py +++ b/app/api/album.py @@ -3,7 +3,6 @@ Contains all the album routes. """ from dataclasses import asdict -from pprint import pprint import random from pydantic import BaseModel, Field @@ -39,7 +38,6 @@ def get_album_tracks_and_info(body: AlbumHashSchema): Returns album info and tracks for the given albumhash. """ albumhash = body.albumhash - # album = AlbumDb.get_album_by_albumhash(albumhash) albumentry = AlbumStore.albummap.get(albumhash) if albumentry is None: @@ -53,14 +51,9 @@ def get_album_tracks_and_info(body: AlbumHashSchema): tracks=tracks, singleTrackAsSingle=UserConfig().showAlbumsAsSingles ) - print("is_favorite", album.is_favorite) - track_total = sum({int(t.extra.get("track_total", 1) or 1) for t in tracks}) avg_bitrate = sum(t.bitrate for t in tracks) // (len(tracks) or 1) - album.fav_userids = [1] - pprint(album) - return { "info": { **asdict(album), @@ -128,10 +121,11 @@ def get_more_from_artist(body: GetMoreFromArtistsBody): a for a in albums # INFO: filter out albums added to other artists - if a.albumhash not in seen_hashes + if a.albumhash not in seen_hashes and artisthash in a.artisthashes # INFO: filter out albums with the same base title and create_hash(a.base_title) != create_hash(base_title) ] + all_albums[artisthash] = serialize_for_card_many( [a for a in albums if create_hash(a.base_title) != create_hash(base_title)][ :limit diff --git a/app/api/artist.py b/app/api/artist.py index 762ddca9..150796cd 100644 --- a/app/api/artist.py +++ b/app/api/artist.py @@ -21,7 +21,7 @@ from app.config import UserConfig from app.db.userdata import SimilarArtistTable from app.serializers.album import serialize_for_card_many -from app.serializers.artist import serialize_for_cards +from app.serializers.artist import serialize_for_cards, serialize_for_card from app.serializers.track import serialize_tracks from app.store.albums import AlbumStore @@ -69,7 +69,14 @@ def get_artist(path: ArtistHashSchema, query: TrackLimitSchema): artist.genres.insert(0, {"name": decade, "genrehash": decade}) return { - "artist": {**asdict(artist), "is_favorite": artist.is_favorite}, + "artist": { + **serialize_for_card(artist), + "duration": sum(t.duration for t in tracks) if tracks else 0, + "trackcount": tcount, + "albumcount": artist.albumcount, + "genres": artist.genres, + "is_favorite": artist.is_favorite, + }, "tracks": serialize_tracks(tracks[:limit]), } @@ -128,7 +135,10 @@ def get_artist_albums(path: ArtistHashSchema, query: GetArtistAlbumsQuery): res["singles_and_eps"].append(album) elif album.type == "compilation": res["compilations"].append(album) - elif album.albumhash in missing_albumhashes or artisthash not in album.artisthashes: + elif ( + album.albumhash in missing_albumhashes + or artisthash not in album.artisthashes + ): res["appearances"].append(album) else: res["albums"].append(album) diff --git a/app/api/favorites.py b/app/api/favorites.py index 481bbf66..7e8aaaae 100644 --- a/app/api/favorites.py +++ b/app/api/favorites.py @@ -1,13 +1,11 @@ from typing import List, TypeVar -from flask_jwt_extended import current_user from flask_openapi3 import Tag from flask_openapi3 import APIBlueprint from pydantic import BaseModel, Field from app.api.apischemas import GenericLimitSchema -from app.db.libdata import ArtistTable -from app.db.libdata import AlbumTable, TrackTable +from app.db.libdata import TrackTable from app.db.userdata import FavoritesTable from app.models import FavType from app.settings import Defaults @@ -45,6 +43,29 @@ class FavoritesAddBody(BaseModel): type: str = Field(description="The type of the item", example=FavType.album) +def toggle_fav(type: str, hash: str): + """ + Toggles a favorite item. + """ + if type == FavType.track: + entry = TrackStore.trackhashmap.get(hash) + if entry is not None: + entry.toggle_favorite_user() + + elif type == FavType.album: + entry = AlbumStore.albummap.get(hash) + + if entry is not None: + entry.toggle_favorite_user() + elif type == FavType.artist: + entry = ArtistStore.artistmap.get(hash) + + if entry is not None: + entry.toggle_favorite_user() + + return {"msg": "Added to favorites"} + + @api.post("/add") def toggle_favorite(body: FavoritesAddBody): """ @@ -56,21 +77,7 @@ def toggle_favorite(body: FavoritesAddBody): except: return {"msg": "Failed! An error occured"}, 500 - if body.type == FavType.track: - entry = TrackStore.trackhashmap.get(body.hash) - if entry is not None: - entry.toggle_favorite_user() - - elif body.type == FavType.album: - entry = AlbumStore.albummap.get(body.hash) - - if entry is not None: - entry.toggle_favorite_user() - elif body.type == FavType.artist: - entry = ArtistStore.artistmap.get(body.hash) - - if entry is not None: - entry.toggle_favorite_user() + toggle_fav(body.type, body.hash) return {"msg": "Added to favorites"} @@ -80,14 +87,12 @@ def remove_favorite(body: FavoritesAddBody): """ Removes a favorite from the database. """ - FavoritesTable.remove_item({"hash": body.hash, "type": body.type}) + try: + FavoritesTable.remove_item({"hash": body.hash, "type": body.type}) + except: + return {"msg": "Failed! An error occured"}, 500 - if body.type == FavType.track: - TrackTable.set_is_favorite(body.hash, False) - elif body.type == FavType.album: - AlbumTable.set_is_favorite(body.hash, False) - elif body.type == FavType.artist: - ArtistTable.set_is_favorite(body.hash, False) + toggle_fav(body.type, body.hash) return {"msg": "Removed from favorites"} diff --git a/app/db/userdata.py b/app/db/userdata.py index 57e87433..9f3bea2f 100644 --- a/app/db/userdata.py +++ b/app/db/userdata.py @@ -1,6 +1,4 @@ import datetime -import enum -from shlex import join from typing import Any from sqlalchemy import ( JSON, @@ -13,15 +11,12 @@ from sqlalchemy import ( insert, select, update, - join, ) from sqlalchemy.orm import Mapped, mapped_column from app.db.engine import DbEngine from app.db.utils import ( - albums_to_dataclasses, - artists_to_dataclasses, favorites_to_dataclass, playlist_to_dataclass, playlists_to_dataclasses, @@ -29,7 +24,6 @@ from app.db.utils import ( similar_artist_to_dataclass, similar_artists_to_dataclass, tracklog_to_dataclasses, - tracks_to_dataclasses, user_to_dataclass, user_to_dataclasses, ) @@ -377,11 +371,12 @@ class PlaylistTable(Base): ) -class ArtistData(Base): +class LibDataTable(Base): __tablename__ = "artistdata" id: Mapped[int] = mapped_column(primary_key=True) - artisthash: Mapped[str] = mapped_column(String(), index=True) + itemhash: Mapped[str] = mapped_column(String(), unique=True, index=True) + itemtype: Mapped[str] = mapped_column(String()) color: Mapped[str] = mapped_column(String(), nullable=True) bio: Mapped[str] = mapped_column(String(), nullable=True) info: Mapped[dict[str, Any]] = mapped_column(JSON(), nullable=True) @@ -390,11 +385,21 @@ class ArtistData(Base): ) @classmethod - def find_one(cls, artisthash: str): - result = cls.execute(select(cls).where(cls.artisthash == artisthash)) + def update_one(cls, hash: str, data: dict[str, Any]): + return cls.execute( + update(cls).where(cls.itemhash == hash).values(data), commit=True + ) + + @classmethod + def find_one(cls, hash: str, type: str): + result = cls.execute( + select(cls).where((cls.itemhash == hash) & (cls.itemtype == type)) + ) return result.fetchone() @classmethod - def get_all_colors(cls) -> dict[str, str]: - result = cls.execute(select(cls.artisthash, cls.color)) - return dict(result.fetchall()) + def get_all_colors(cls, type: str) -> list[dict[str, str]]: + result = cls.execute( + select(cls.itemhash, cls.color).where(cls.itemtype == type) + ) + return [{"itemhash": r[0], "color": r[1]} for r in result.fetchall()] diff --git a/app/lib/colorlib.py b/app/lib/colorlib.py index 8b73dfcf..5b56dbe0 100644 --- a/app/lib/colorlib.py +++ b/app/lib/colorlib.py @@ -2,19 +2,16 @@ Contains everything that deals with image color extraction. """ -import json from pathlib import Path import colorgram from app import settings -from app.db.sqlite.albumcolors import SQLiteAlbumMethods as aldb -from app.db.sqlite.artistcolors import SQLiteArtistMethods as adb -from app.db.sqlite.utils import SQLiteManager -from app.db.userdata import ArtistData +from app.db.userdata import LibDataTable from app.logger import log from app.lib.errors import PopulateCancelledError +from app.store.albums import AlbumStore from app.store.artists import ArtistStore from app.utils.progressbar import tqdm @@ -52,47 +49,45 @@ def process_color(item_hash: str, is_album=True): return get_image_colors(str(path)) -# class ProcessAlbumColors: -# """ -# Extracts the most dominant color from the album art and saves it to the database. -# """ +class ProcessAlbumColors: + """ + Extracts the most dominant color from the album art and saves it to the database. + """ -# def __init__(self, instance_key: str) -> None: -# global PROCESS_ALBUM_COLORS_KEY -# PROCESS_ALBUM_COLORS_KEY = instance_key + def __init__(self, instance_key: str) -> None: + global PROCESS_ALBUM_COLORS_KEY + PROCESS_ALBUM_COLORS_KEY = instance_key -# albums = [ -# a -# for a in AlbumStore.albums -# if a is not None and a.colors is not None and len(a.colors) == 0 -# ] + albums = [a for a in AlbumStore.get_flat_list() if not a.color] -# with SQLiteManager() as cur: -# try: -# for album in tqdm(albums, desc="Processing missing album colors"): -# if PROCESS_ALBUM_COLORS_KEY != instance_key: -# raise PopulateCancelledError( -# "A newer 'ProcessAlbumColors' instance is running. Stopping this one." -# ) + for album in tqdm(albums, desc="Processing missing album colors"): + albumhash = album.albumhash + if PROCESS_ALBUM_COLORS_KEY != instance_key: + raise PopulateCancelledError( + "A newer 'ProcessAlbumColors' instance is running. Stopping this one." + ) -# # TODO: Stop hitting the database for every album. -# # Instead, fetch all the data from the database and -# # check from memory. + albumrecord = LibDataTable.find_one(albumhash, type="album") + if albumrecord is not None and albumrecord.color is not None: + continue -# exists = aldb.exists(album.albumhash, cur=cur) -# if exists: -# continue + colors = process_color(albumhash) -# colors = process_color(album.albumhash) + if colors is None: + continue -# if colors is None: -# continue + album = AlbumStore.albummap.get(albumhash) -# album.set_colors(colors) -# color_str = json.dumps(colors) -# aldb.insert_one_album(cur, album.albumhash, color_str) -# finally: -# cur.close() + if album: + album.set_color(colors[0]) + + # INFO: Write to the database. + if albumrecord is None: + LibDataTable.insert_one( + {"itemhash": albumhash, "color": colors[0], "itemtype": "album"} + ) + else: + LibDataTable.update_one(albumhash, {"color": colors[0]}) class ProcessArtistColors: @@ -101,29 +96,37 @@ class ProcessArtistColors: """ def __init__(self, instance_key: str) -> None: - all_artists = ArtistStore.get_flat_list() + all_artists = [a for a in ArtistStore.get_flat_list() if not a.color] global PROCESS_ARTIST_COLORS_KEY PROCESS_ARTIST_COLORS_KEY = instance_key - try: - for artist in tqdm(all_artists, desc="Processing missing artist colors"): - if PROCESS_ARTIST_COLORS_KEY != instance_key: - raise PopulateCancelledError( - "A newer 'ProcessArtistColors' instance is running. Stopping this one." - ) + for artist in tqdm(all_artists, desc="Processing missing artist colors"): + artisthash = artist.artisthash + if PROCESS_ARTIST_COLORS_KEY != instance_key: + raise PopulateCancelledError( + "A newer 'ProcessArtistColors' instance is running. Stopping this one." + ) - # exists = adb.exists(artist.artisthash, cur=cur) - artist = ArtistData.find_one(artist.artisthash) - if artist and artist.color is not None: - continue + record = LibDataTable.find_one(artisthash, "artist") - colors = process_color(artist.artisthash, is_album=False) + if (record is not None) and (record.color is not None): + continue - if colors is None: - continue + colors = process_color(artisthash, is_album=False) - artist.set_colors(colors) - adb.insert_one_artist(cur, artist.artisthash, colors) - finally: - cur.close() + if colors is None: + continue + + artist = ArtistStore.artistmap.get(artisthash) + + if artist: + artist.set_color(colors[0]) + + # INFO: Write to the database. + if record is None: + LibDataTable.insert_one( + {"itemhash": artisthash, "color": colors[0], "itemtype": "artist"} + ) + else: + LibDataTable.update_one(artisthash, {"color": colors[0]}) diff --git a/app/lib/home/recentlyadded.py b/app/lib/home/recentlyadded.py index a3216830..4c2c98ab 100644 --- a/app/lib/home/recentlyadded.py +++ b/app/lib/home/recentlyadded.py @@ -96,10 +96,6 @@ def check_folder_type(group_: dict): key: str = group_["folder"] tracks: list[Track] = group_["tracks"] time: float = group_["time"] - - print(f"Checking folder: {key}") - print(f"Tracks: {len(tracks)}") - existing_artist_hashes: set[str] = set(ArtistStore.artistmap.keys()) existing_album_hashes: set[str] = set(AlbumStore.albummap.keys()) diff --git a/app/lib/index.py b/app/lib/index.py index 4749ba7e..604158f0 100644 --- a/app/lib/index.py +++ b/app/lib/index.py @@ -1,4 +1,4 @@ -from app.lib.mapstuff import map_favorites, map_scrobble_data +from app.lib.mapstuff import map_album_colors, map_artist_colors, map_favorites, map_scrobble_data from app.lib.populate import CordinateMedia from app.lib.tagger import IndexTracks from app.store.folder import FolderStore @@ -16,7 +16,9 @@ class IndexEverything: FolderStore.load_filepaths() map_scrobble_data() map_favorites() - # CordinateMedia(instance_key=str(time())) + map_artist_colors() + map_album_colors() + CordinateMedia(instance_key=str(time())) gc.collect() diff --git a/app/lib/mapstuff.py b/app/lib/mapstuff.py index 658e3b8a..7f356fe0 100644 --- a/app/lib/mapstuff.py +++ b/app/lib/mapstuff.py @@ -1,4 +1,4 @@ -from app.db.userdata import FavoritesTable, ScrobbleTable +from app.db.userdata import LibDataTable, FavoritesTable, ScrobbleTable from app.store.albums import AlbumStore from app.store.artists import ArtistStore from app.store.tracks import TrackStore @@ -66,3 +66,23 @@ def map_favorites(): track = TrackStore.trackhashmap.get(entry.hash) if track: track.toggle_favorite_user(entry.userid) + + +def map_artist_colors(): + colors = LibDataTable.get_all_colors(type="artist") + + for color in colors: + artist = ArtistStore.artistmap.get(color["itemhash"]) + + if artist: + artist.set_color(color["color"]) + + +def map_album_colors(): + colors = LibDataTable.get_all_colors(type="album") + + for color in colors: + album = AlbumStore.albummap.get(color["itemhash"]) + + if album: + album.set_color(color["color"]) diff --git a/app/lib/populate.py b/app/lib/populate.py index 5506741e..0b12565c 100644 --- a/app/lib/populate.py +++ b/app/lib/populate.py @@ -8,7 +8,7 @@ from requests import ReadTimeout from app import settings from app.db.sqlite.tracks import SQLiteTrackMethods from app.lib.artistlib import CheckArtistImages -from app.lib.colorlib import ProcessArtistColors +from app.lib.colorlib import ProcessAlbumColors, ProcessArtistColors from app.lib.errors import PopulateCancelledError from app.lib.taglib import extract_thumb from app.logger import log @@ -40,6 +40,7 @@ class CordinateMedia: try: ProcessTrackThumbnails(instance_key) + ProcessAlbumColors(instance_key) ProcessArtistColors(instance_key) except PopulateCancelledError as e: log.warn(e) diff --git a/app/lib/tagger.py b/app/lib/tagger.py index b7613125..fab4f718 100644 --- a/app/lib/tagger.py +++ b/app/lib/tagger.py @@ -3,7 +3,6 @@ from app import settings from app.config import UserConfig from app.db.libdata import TrackTable -# from app.lib.populate import CordinateMedia from app.lib.taglib import extract_thumb, get_tags from app.models.album import Album from app.models.artist import Artist @@ -29,7 +28,6 @@ class IndexTracks: global POPULATE_KEY POPULATE_KEY = instance_key - # dirs_to_scan = sdb.get_root_dirs() dirs_to_scan = UserConfig().rootDirs if len(dirs_to_scan) == 0: @@ -159,12 +157,12 @@ def create_albums(): "playcount": track.playcount, "playduration": track.playduration, "title": track.album, - "trackcount": 1, + "tracks": {track.trackhash}, "extra": {}, } else: album = albums[track.albumhash] - album["trackcount"] += 1 + album["tracks"].add(track.trackhash) album["playcount"] += track.playcount album["playduration"] += track.playduration album["lastplayed"] = max(album["lastplayed"], track.lastplayed) @@ -186,8 +184,12 @@ def create_albums(): album["base_title"], _ = get_base_album_title(album["og_title"]) del genres + trackhashes = album.pop("tracks") + album["trackcount"] = len(trackhashes) - return [Album(**album) for album in albums.values()] + albums[album["albumhash"]] = (Album(**album), trackhashes) + + return list(albums.values()) # class IndexArtists: @@ -225,7 +227,7 @@ def create_artists(): "extra": {}, } else: - artist = artists[thisartist["artisthash"]] + artist: dict = artists[thisartist["artisthash"]] artist["duration"] += track.duration artist["playcount"] += track.playcount artist["playduration"] += track.playduration @@ -235,6 +237,8 @@ def create_artists(): artist["created_date"] = min(artist["created_date"], track.last_mod) artist["names"].add(thisartist["name"]) + artist.setdefault("albums", set()) + if thisartist.get("in_track", True): artist["tracks"].add(track.trackhash) @@ -257,12 +261,13 @@ def create_artists(): # INFO: Delete temporary keys del artist["names"] - del artist["tracks"] - del artist["albums"] + + tracks = artist.pop("tracks") + albums = artist.pop("albums") # INFO: Delete local variables del genres - return [Artist(**artist) for artist in artists.values()] - + artists[artist["artisthash"]] = (Artist(**artist), tracks, albums) + return list(artists.values()) diff --git a/app/models/artist.py b/app/models/artist.py index 540098a7..59ae76f4 100644 --- a/app/models/artist.py +++ b/app/models/artist.py @@ -55,6 +55,7 @@ class Artist: id: int = -1 image: str = "" + color: str = "" fav_userids: list[int] = dataclasses.field(default_factory=list) @property diff --git a/app/serializers/album.py b/app/serializers/album.py index 4bec0b4e..35405b4b 100644 --- a/app/serializers/album.py +++ b/app/serializers/album.py @@ -34,6 +34,7 @@ def serialize_for_card(album: Album): "type", "playduration", "genrehashes", + "fav_userids", "extra", "id", "lastplayed", diff --git a/app/store/albums.py b/app/store/albums.py index 14cc0086..7cbe8b8f 100644 --- a/app/store/albums.py +++ b/app/store/albums.py @@ -4,7 +4,6 @@ from pprint import pprint import random from typing import Iterable -from app.db.sqlite.albumcolors import SQLiteAlbumMethods as aldb from app.lib.tagger import create_albums from app.models import Album, Track from app.store.artists import ArtistStore @@ -21,9 +20,9 @@ ALBUM_LOAD_KEY = "" class AlbumMapEntry: - def __init__(self, album: Album) -> None: + def __init__(self, album: Album, trackhashes: set[str]) -> None: self.album = album - self.trackhashes: set[str] = set() + self.trackhashes = trackhashes @property def basetitle(self): @@ -40,6 +39,9 @@ class AlbumMapEntry: self.album.toggle_favorite_user(userid) + def set_color(self, color: str): + self.album.color = color + class AlbumStore: albums: list[Album] = CustomList() @@ -67,26 +69,9 @@ class AlbumStore: print("Loading albums... ", end="") cls.albummap = { - album.albumhash: AlbumMapEntry(album=album) for album in create_albums() + album.albumhash: AlbumMapEntry(album=album, trackhashes=trackhashes) + for album, trackhashes in create_albums() } - tracks = remove_duplicates(TrackStore.get_flat_list()) - tracks = sorted(tracks, key=lambda t: t.albumhash) - grouped = groupby(tracks, lambda t: t.albumhash) - - for albumhash, tracks in grouped: - cls.albummap[albumhash].trackhashes = {t.trackhash for t in tracks} - - # db_albums: list[tuple] = aldb.get_all_albums() - - # for album in db_albums: - # albumhash = album[1] - # colors = json.loads(album[2]) - - # for _al in cls.albums: - # if _al.albumhash == albumhash: - # _al.set_colors(colors) - # break - print("Done!") @classmethod diff --git a/app/store/artists.py b/app/store/artists.py index b4d71ae2..f88dcb83 100644 --- a/app/store/artists.py +++ b/app/store/artists.py @@ -1,27 +1,22 @@ import json from typing import Iterable -from app.db.sqlite.artistcolors import SQLiteArtistMethods as ardb from app.lib.tagger import create_artists from app.models import Artist -from app.utils import flatten from app.utils.auth import get_current_userid -from app.utils.bisection import use_bisection from app.utils.customlist import CustomList -from app.utils.progressbar import tqdm -from .tracks import TrackStore - -# from .albums import AlbumStore from .tracks import TrackStore ARTIST_LOAD_KEY = "" class ArtistMapEntry: - def __init__(self, artist: Artist) -> None: + def __init__( + self, artist: Artist, albumhashes: set[str], trackhashes: set[str] + ) -> None: self.artist = artist - self.albumhashes: set[str] = set() - self.trackhashes: set[str] = set() + self.albumhashes: set[str] = albumhashes + self.trackhashes: set[str] = trackhashes def increment_playcount(self, duration: int, timestamp: int): self.artist.lastplayed = timestamp @@ -34,6 +29,9 @@ class ArtistMapEntry: self.artist.toggle_favorite_user(userid) + def set_color(self, color: str): + self.artist.color = color + class ArtistStore: artists: list[Artist] = CustomList() @@ -51,17 +49,19 @@ class ArtistStore: cls.artistmap.clear() cls.artistmap = { - artist.artisthash: ArtistMapEntry(artist=artist) - for artist in create_artists() + artist.artisthash: ArtistMapEntry( + artist=artist, albumhashes=albumhashes, trackhashes=trackhashes + ) + for artist, trackhashes, albumhashes in create_artists() } - for track in TrackStore.get_flat_list(): - if instance_key != ARTIST_LOAD_KEY: - return + # for track in TrackStore.get_flat_list(): + # if instance_key != ARTIST_LOAD_KEY: + # return - for hash in track.artisthashes: - cls.artistmap[hash].trackhashes.add(track.trackhash) - cls.artistmap[hash].albumhashes.add(track.albumhash) + # for hash in track.artisthashes: + # cls.artistmap[hash].trackhashes.add(track.trackhash) + # cls.artistmap[hash].albumhashes.add(track.albumhash) print("Done!") # for artist in ardb.get_all_artists(): diff --git a/manage.py b/manage.py index f2f11639..80faac61 100644 --- a/manage.py +++ b/manage.py @@ -21,9 +21,7 @@ import setproctitle from app.api import create_api from app.arg_handler import ProcessArgs -from app.lib.mapstuff import map_favorites, map_scrobble_data from app.lib.index import IndexEverything -from app.lib.watchdogg import Watcher as WatchDog from app.plugins.register import register_plugins from app.settings import FLASKVARS, TCOLOR, Info from app.setup import load_into_mem, run_setup